def checkpoint_setup(cp, manifest_server_obj): #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ """Sets up and defines all checkpoints. All checkpoints must have an entry (in chronological order) here. format: step_setup( message, name, [dataset list]) message: the message to print to the screen to indicate the step name: ascii name for the step. dataset list: list with the names of the datasets to snapshot """ finalizer_script_list = get_manifest_list(manifest_server_obj, FINALIZER_SCRIPT_NAME) # Set up a checkpoint for each finalizer script specified. for script in finalizer_script_list: checkpoint_message = get_manifest_value(manifest_server_obj, FINALIZER_SCRIPT_NAME_TO_CHECKPOINT_MESSAGE % script) checkpoint_name = get_manifest_value(manifest_server_obj, FINALIZER_SCRIPT_NAME_TO_CHECKPOINT_NAME % script) if checkpoint_name is None : dc_log = logging.getLogger(DC_LOGGER_NAME) dc_log.error("The checkpoint name to use for the " \ "finalizer script %s is missing." % script) dc_log.error("Please check your manifest file.") return -1 if checkpoint_message is None: checkpoint_message = "Executing " + script cp.step_setup(checkpoint_message, checkpoint_name, [cp.get_build_area_dataset() + BUILD_DATA]) return 0
def determine_chckpnt_avail(cp, manifest_server_obj): #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ """ Determine if checkpointing is available by reading the manifest file variable and checking to see if zfs is on they system. """ dc_log = logging.getLogger(DC_LOGGER_NAME) # The checkpointing available is set to true by default in the # checkpointing class __init__ method. Here we check to see if # checkpointing is disabled in the manifest file. If so, return if get_manifest_value(manifest_server_obj, CHECKPOINT_ENABLE).lower() == "false": cp.set_checkpointing_avail(False) return # Check to see if checkpointing is disabled because zfs is not on # the system. try: if os.access('/sbin/zfs', os.X_OK) == True: return except OSError: dc_log.error("Unable to determine if ZFS is available " "on the system. Checkpointing will be unavailable") # zfs not on the system so no checkpointing cp.set_checkpointing_avail(False) cp.set_zfs_found(False) return
" boot archive build area, media area, grub setup type.") # collect input arguments from what this script sees as a commandline. MFEST_SOCKET = sys.argv[1] # Manifest reader socket PKG_IMG_PATH = sys.argv[2] # package image area mountpoint GRUB_SETUP_TYPE = sys.argv[6] # Grub setup type # get the manifest reader object from the socket MANIFEST_READER_OBJ = ManifestRead(MFEST_SOCKET) # if a string is specified in the manifest to be used as the title of the grub # menu entries, that string will be used. Otherwise, use the first # line of /etc/release as the title # Get grub title from manifest, if any RELEASE = get_manifest_value(MANIFEST_READER_OBJ, GRUB_TITLE) if RELEASE is not None: # User specified a special grub menu, record that in .image_info IMG_INFO_FD = None try: IMG_INFO_PATH = PKG_IMG_PATH + "/" + IMAGE_INFO_FILE try: IMG_INFO_FD = open(IMG_INFO_PATH, "a+") IMG_INFO_FD.write(IMAGE_INFO_GRUB_TITLE_KEYWORD + RELEASE + "\n") except Exception, err: print >> sys.stderr, sys.argv[0] + \ "Unable to write to " + IMG_INFO_PATH raise err finally: if (IMG_INFO_FD != None): IMG_INFO_FD.close()
if __name__ == "__main__": if (len(sys.argv) != 6): # Don't forget sys.argv[0] is the script itself. raise Exception, (sys.argv[0] + ": Requires 5 args:\n" + " Reader socket, pkg_image area, temp dir,\n" + " boot archive build area, media area.") # collect input arguments from what this script sees as a commandline. mfest_socket = sys.argv[1] # Manifest reader socket pkg_img_path = sys.argv[2] # package image area mountpoint # get the manifest reader object from the socket manifest_reader_obj = ManifestRead(mfest_socket) # Get the password for the root user from the manifest root_passwd_text = get_manifest_value(manifest_reader_obj, ROOT_PASSWD) is_plaintext = get_manifest_boolean(manifest_reader_obj, ROOT_PASSWD_PLAINTEXT) print "root_passwd_text = " + root_passwd_text print "is_plaintext = " + str(is_plaintext) if is_plaintext: encrypted_root_passwd = encrypt_password(root_passwd_text, alt_root=pkg_img_path) else: encrypted_root_passwd = root_passwd_text print "Encrypted root password: " + encrypted_root_passwd try:
if (len(sys.argv) != 7): # Don't forget sys.argv[0] is the script itself. raise Exception, ( sys.argv[0] + ": Requires 6 args:\n" + " Reader socket, pkg_image area, temp dir,\n" + " boot archive build area, media area, loader setup type.") # collect input arguments from what this script sees as a commandline. MFEST_SOCKET = sys.argv[1] # Manifest reader socket PKG_IMG_PATH = sys.argv[2] # package image area mountpoint LOADER_SETUP_TYPE = sys.argv[6] # Grub setup type # get the manifest reader object from the socket MANIFEST_READER_OBJ = ManifestRead(MFEST_SOCKET) # Get default timeout from manifest. if it exists. TIMEOUT = get_manifest_value(MANIFEST_READER_OBJ, LOADER_DEFAULT_TIMEOUT) if TIMEOUT is not None and TIMEOUT != DEFAULT_TIMEOUT: # Open loader.conf file. try: LOADER_CONF_FILE = open(PKG_IMG_PATH + "/boot/loader.conf", "a") except IOError, err: print >> sys.stderr, "Error opening loader.conf for writing" raise LOADER_CONF_FILE.write("autoboot_delay=" + TIMEOUT + "\n") LOADER_CONF_FILE.close() CMD = CP + " " + LOADER_MENU_FILES + "/* " + PKG_IMG_PATH + "/boot" if (LOADER_SETUP_TYPE == "ai"): # The following entries are the standard "hardwired" entries for AI
def main_func(): #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ """Main processing function for the Distribution Constructor. Will set up the checkpoints, create the build area, start the socket server to read the manifest file, create the finalizer script queue and then start the execution of the finalizer scripts. """ dc_log = logging.getLogger(DC_LOGGER_NAME) # Must be root to run DC. Check for that. if os.getuid() != 0: dc_log.error("You must be root to run distro_const") return 1 # Sets the umask for the DC app os.umask(022) cp = dc_ckp.Checkpoints() try: # Create the object used to extract the data manifest_server_obj = get_manifest_server_obj(cp) except UsageError: raise except: return 1 try: # Start the socket server start_manifest_server(manifest_server_obj) # Set up to shut down socket server cleanly on exit atexit.register(manifest_server_obj.stop_socket_server) except ManifestServError: return 1 # create the build area and determine if # checkpointing is available. When creating the # build area we also create the areas for the # package image (pkg_image), output media (media), # logfiles (logs) and boot archive (boot_archive). if ti.create_build_area(cp, manifest_server_obj): return 1 # Set up the structures for all checkpoints if cp.get_checkpointing_avail() == True: if dc_ckp.checkpoint_setup(cp, manifest_server_obj): return 1 # Parse the command line so we know to resume (and where) or not ret = parse_command_line(cp, manifest_server_obj) if ret == 1: # Don't continue processing. Return but nothing is wrong. return 0 if ret == -1: # Error parsing the command line. Return indicating such. return 1 # The build is going to start, will start logging (simple_log_name, detail_log_name) = \ dcu.setup_dc_logfile_names(cp.get_build_area_mntpt() + LOGS) dc_log.info("Simple Log: " + simple_log_name) dc_log.info("Detail Log: " + detail_log_name) dcu.add_file_logging(simple_log_name, detail_log_name) dc_log.info("Build started " + time.asctime(time.localtime())) dc_log.info("Distribution name: " + (dcu.get_manifest_value(manifest_server_obj, DISTRO_NAME))) dataset = cp.get_build_area_dataset() if dataset is not None: dc_log.info("Build Area dataset: " + dataset) dc_log.info("Build Area mount point: " + cp.get_build_area_mntpt()) # If we're doing a build that is a -r or -R build, then # we don't need to cleanup. if cp.get_resume_step() == -1: # Cleanup the pkg_image area via either a remove of the files # if there is not checkpointing or rolling back to the @empty # snapshot if there is checkpointing. if cleanup_build_data_area(cp) != 0: dc_log.info("Build completed " + time.asctime(time.localtime())) dc_log.info("Build failed.") return 1 stop_on_err_bool = dcu.get_manifest_boolean(manifest_server_obj, STOP_ON_ERR) # register the scripts with the finalizer build_area = cp.get_build_area_mntpt() pkg_img_area = build_area + PKG_IMAGE tmp_area = build_area + TMP ba_build_area = build_area + BOOT_ARCHIVE media_dir = build_area + MEDIA finalizer_obj = DCFinalizer([manifest_server_obj.get_sockname(), pkg_img_area, tmp_area, ba_build_area, media_dir]) if (finalizer_obj.change_exec_params(stop_on_err=stop_on_err_bool, logger_name = DC_LOGGER_NAME) != 0): dc_log.error("Unable to set stop on error or logger name " "for finalizer") status = dc_ckp.add_finalizer_scripts(cp, manifest_server_obj, finalizer_obj) if (status != SUCCESS): dc_log.info("Build completed " + time.asctime(time.localtime())) dc_log.info("Build failed.") return (status) # Execute the finalizer scripts status = finalizer_obj.execute() dc_log.info("Build completed " + time.asctime(time.localtime())) if (status != 0): dc_log.info("Build failed.") else: dc_log.info("Build is successful.") return (status)
def parse_command_line(cp, manifest_server_obj): #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ """Parse the command line options. usage: dist_const build [-R] manifest-file dist_const build [-r <integer or string>] manifest-file dist_const build [-p <integer or string>] manifest-file dist_const build [-l] manifest-file -R will resume from the last executed step -r will resume from the specified step -p will pause at the specified step -l will list the valid steps to resume/pause at Also, verify that the pause step and the resume step are valid. The pause step must be one of the steps. The resume step must be less than or equal to the last step executed. This method will also remove the zfs snapshots and the .step file for steps equal to or that come after the step where execution will start. For a full build or a pause, this means all checkpoints will be removed. For a resume, the checkpoint to be resumed from and all later checkpoints are removed. Return 0 = success 1 = success but don't continue on with building. -1 = Error """ subcommand = sys.argv[1] # See if the user has specified a step to resume from in the manifest # file. If so, keep it around. If the user has also specified one on # the command line, that will overwrite the manifest value. stepname = dcu.get_manifest_value(manifest_server_obj, CHECKPOINT_RESUME) if stepname is not None: step = dc_ckp.step_from_name(cp, stepname) if step is None: return -1 stepno = step.get_step_num() if stepno: cp.set_resume_step(stepno) # Read the command line arguments and parse them. try: opts2 = getopt.getopt(sys.argv[2:], "r:p:hRl?")[0] except getopt.GetoptError: usage() if subcommand == "build": step_resume = False resume = False pause = False do_list = False dc_log = logging.getLogger(DC_LOGGER_NAME) for opt, arg in opts2: if (opt == "-h") or (opt == "-?"): usage() # Since all command line opts have to do with # checkpointing, check here to see # if checkpointing is available. if not cp.get_checkpointing_avail(): dc_log.error("Checkpointing is not available") dc_log.error("Rerun the build without " + opt) return -1 if opt == "-r": # Check to see if -r has already been specified if step_resume == True: usage() # resume from the specified step. step = dc_ckp.step_from_name(cp, arg) if step is None: return -1 stepno = step.get_step_num() cp.set_resume_step(stepno) step_resume = True elif opt == "-p": # Check to see if -p has already been specified if pause: usage() # pause at the specified step. step = dc_ckp.step_from_name(cp, arg) if step is None: return -1 stepno = step.get_step_num() cp.set_pause_step(stepno) pause = True elif opt == "-R": # resume from the last executed step. stepno = dc_ckp.determine_resume_step(cp) if stepno == -1: dc_log.error("There are no valid " \ "steps to resume from.") dc_log.error("Please rerun the build " \ "without the -r or -R options.") return -1 cp.set_resume_step(stepno) resume = True elif opt == "-l": do_list = True # If a resume step was specified via -r, -R or the manifest file, # check to see if it's valid. If not, abort the build. if cp.get_resume_step() != -1: err = dc_ckp.verify_resume_step(cp, cp.get_resume_step()) if err != 0 : return -1 # -R and -r not allowed on the same command line. if resume and step_resume: dc_log.error("-R and -r cannot be specified for the " "same build. ") usage() # -l and -R, -r, or -p combination not allowed. # -l must be the only option. if do_list and (pause or resume or step_resume): dc_log.error("-l and -R, -r, or -p cannot be " "specified for the same build") usage() # We've checked for the bad combos. # If a -l was specified, print out the info. if do_list: # query for valid resume/pause steps # All steps are valid to pause at. The # steps that are valid to resume from # will be marked "resumable" laststep = dc_ckp.determine_resume_step(cp) dc_log.error("\nStep Resumable Description") dc_log.error("-------------- --------- -------------") for step_obj in cp.step_list: if laststep != -1 \ and step_obj.get_step_num() <= laststep: r_flag = "X" else: r_flag = " " dc_log.error("%s%s%s" % \ (step_obj.get_step_name().ljust(15), r_flag.center(10), step_obj.get_step_message().ljust(10))) return 1 # If -r/-R and -p were both specified, # the pause step must be later than the resume. if cp.get_pause_step() <= cp.get_resume_step() : dc_log.error("The resume step must be earlier " "than the pause step.") usage() if cp.get_resume_step() != -1 : dc_ckp.verify_resume_state(cp) # destroy all zfs snapshots and .step files for # steps after the resume step for step_obj in cp.step_list[cp.get_resume_step() + 1:]: dc_ckp.delete_checkpoint(step_obj) else : # Delete all zfs snapshots and .step files for step_obj in cp.step_list: dc_ckp.delete_checkpoint(step_obj) else : # Currently we only support the build subcommand dc_log.error("Invalid or missing subcommand") usage() return 0
BA_ARCHFILE = PKG_IMG_MNT_PT + BA_FILENAME_AMD64 BA_BUILD = TMP_DIR + "/" + KERNEL_ARCH STRIP_ARCHIVE = True else: BA_ARCHFILE = PKG_IMG_MNT_PT + BA_FILENAME_ALL BA_BUILD = BA_MASTER STRIP_ARCHIVE = False # Location of the lofi file mountpoint, known only to this file. BA_LOFI_MNT_PT = TMP_DIR + "/ba_lofimnt" # get the manifest reader object from the socket MANIFEST_READER_OBJ = ManifestRead(MFEST_SOCKET) # Boot archive compression type and level, and padding amount. BA_COMPR_LEVEL = get_manifest_value(MANIFEST_READER_OBJ, BOOT_ARCHIVE_COMPRESSION_LEVEL) # compression level is not applicable to Sparc platform if not IS_SPARC and BA_COMPR_LEVEL is None: raise Exception, (sys.argv[0] + ": boot archive compression level missing from manifest") BA_COMPR_TYPE = get_manifest_value(MANIFEST_READER_OBJ, BOOT_ARCHIVE_COMPRESSION_TYPE) if BA_COMPR_TYPE is None: raise Exception, (sys.argv[0] + ": boot archive compression type missing from manifest") PADDING = -1 BA_PAD_SIZE_STR = get_manifest_value(MANIFEST_READER_OBJ, BOOT_ARCHIVE_SIZE_PAD)
def create_build_area(ckp, manifest_server_obj): # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ """Create the build area. This may be a normal directory or a zfs dataset. If it is specified and doesn't start with / a zfs dataset will attempt to be created. If it starts with a /, check to see if it's a zfs mountpoint. If not, a normal directory (mkdir) will be created. This is where all the sub working directories are created. This includes the pkg_image, logs, media and boot_archive directories that reside under the build area. Args: ckp - checkpointing object manifest_server_obj: manifest data extraction object Returns: -1 on failure 0 on success """ # Check manifest file and existence of zfs to see if # checkpointing is possible. dc_ckp.determine_chckpnt_avail(ckp, manifest_server_obj) dc_log = logging.getLogger(DC_LOGGER_NAME) # Read the build_area from the manifest file. This can be either # a zfs dataset or a mountpoint. build_area = dcu.get_manifest_value(manifest_server_obj, BUILD_AREA) if build_area is None: dc_log.error(BUILD_AREA + " in the manifest file is invalid " \ "or missing. Build aborted") return -1 # First check build_area to see # if there is a leading /. If there isn't, it has to be a zfs dataset. if build_area.startswith('/'): # Leading /. build_area can be either a zfs mountpoint or # a ufs mountpoint. if ckp.get_zfs_found(): # zfs is on the system so it's at least possible # that build_area is a zfs mountpoint. # zfs list -o mountpoint <build_area> will return # the mountpoint for the build_area specified. cmd = "/usr/sbin/zfs list -H -o \"mountpoint\" " \ + build_area try: mntpt = Popen(cmd, shell=True, universal_newlines=True, stdout=PIPE).communicate()[0].strip() except OSError: dc_log.error("Error determining if the build " \ "area exists") return -1 # zfs list -H -o name <build_area> will return # the zfs dataset associated with build_area cmd = "/usr/sbin/zfs list -H -o \"name\" " \ + build_area try: dataset = Popen(cmd, shell=True, universal_newlines=True, stdout=PIPE).communicate()[0].strip() except OSError: dc_log.error("Error finding the build area dataset") return -1 # If we have found a dataset, check to see if # the mountpoint and the build_area are the same. # If so, the build_area that was specified is # a zfs mountpoint that directly correlates to # a zfs dataset. If the mountpoint and the build_area # are not the same, there is not a direct match # and we can't use checkpointing. if dataset: if mntpt == build_area: # We have a zfs dataset and # mountpoint so we don't have # to create one. Just save # the dataset and mountpoint ofr # later use and create the subdirs. ckp.set_build_area_dataset(dataset) ckp.set_build_area_mntpt(mntpt) # And now create the subdirs # for pkg_image, media, logs, # tmp, and boot_archive ret = create_subdirs(ckp) if ret: return -1 return 0 # We have a build area that doesn't # have a direct matchup to a mountpoint. # Checkpointing must not be used. If # we checkpoint, we run the risk of # rollinging back more data than would # be wise. ex. build area is # /export/home/someone but the mntpt # is /export/home. ckp.set_checkpointing_avail(False) ckp.set_build_area_mntpt(build_area) # And now create the subdirs # for pkg_image, media, logs, # tmp and boot_archive ret = create_subdirs(ckp) if ret: return -1 return 0 # No zfs on the system or no zfs dataset that relates # to the build_area, create a ufs style dir. ckp.set_checkpointing_avail(False) ret = create_ufs_dir(build_area) if ret: dc_log.error("Unable to create the build area at " + build_area) return -1 else: ckp.set_build_area_mntpt(build_area) # And now create the subdirs # for pkg_image, media, logs, # tmp and boot_archive ret = create_subdirs(ckp) if ret: return -1 else: # build_area doesn't start with a /, thus it # has to be a zfs dataset. # Check to see if zfs is on the system. If # not we have an error since a zfs dataset # was specified. if not ckp.get_zfs_found(): dc_log.error("ZFS dataset was specified for the build area,") dc_log.error("but zfs is not installed on the system.") return -1 # create zfs fs ret = create_zfs_fs(build_area) if ret: return -1 # The zfs fs was created, get the associated mountpoint cmd = "/usr/sbin/zfs list -H -o \"mountpoint\" " + build_area try: mntpt = Popen(cmd, shell=True, universal_newlines=True, stdout=PIPE).communicate()[0].strip() except OSError: dc_log.error("Unable to get the mountpoint for the " "zfs dataset " + build_area) return -1 ckp.set_build_area_mntpt(mntpt) ckp.set_build_area_dataset(build_area) # And now create the subdirs # for pkg_image, media, logs, # tmp and boot_archive ret = create_subdirs(ckp) if ret: return -1 return 0
raise Exception(sys.argv[0] + ": Requires 5 args:\n" + " Reader socket, pkg_image area, tmp dir,\n" + " bootroot build area, media area") # Collect input arguments from what # this script sees as a commandline. MFEST_SOCKET = sys.argv[1] # Manifest reader socket PKG_IMG_MNT_PT = sys.argv[2] # package image area mountpoint TMP_DIR = sys.argv[3] # temp directory to contain boot archive UNSET_AUTH_LIST = [] UNSET_MIRROR_LIST = [] MANIFEST_SERVER_OBJ = ManifestRead(MFEST_SOCKET) PKG_URL = dcu.get_manifest_value(MANIFEST_SERVER_OBJ, DEFAULT_MAIN_URL) PKG_AUTH = dcu.get_manifest_value(MANIFEST_SERVER_OBJ, DEFAULT_MAIN_AUTHNAME) if PKG_AUTH is None: raise Exception(sys.argv[0] + ": pkg publisher is missing from manifest") if PKG_URL is None: raise Exception(sys.argv[0] + ": pkg url is missing from manifest") QUIT_ON_PKG_FAILURE = dcu.get_manifest_value(MANIFEST_SERVER_OBJ, STOP_ON_ERR).lower() # Initialize log instance # Once initialized, STDOUT messages will show up in the detailed # log and STDERR messages will show up in the detailed and simple