def find_likely_pkn(instance_name, pkn): """Sometimes an ambiguous package name is requested. This attempts to help a guy out.""" progress_file = get_progress_path(instance_name) try: status_yml = json.loads(open(progress_file).read()) except ValueError: Logger.error("Progress data in %s is not parsable." % progress_file) raise pkns = [] status_packages = status_yml['install-progress'] for name in status_packages: if status_packages[name]['INSTALLED'] in [ 'NA', 'BROKEN' ]: continue if pkn.lower() in name.lower(): pkns.append(name) if len(pkns) > 1: msg = 'Ambiguous package name: %s could be any of %s' % (pkn, str(pkns)) Logger.error( msg ) exit_with_return_code(FAIL) if len(pkns) == 0: Logger.error( 'Package not found: %s' %pkn ) exit_with_return_code(FAIL) else: pkn = '-'.join(pkns[0].split('-')[:-1]) Logger.info( 'Using %s' %pkn) return pkn
def _uninstall_pkgs(self, del_pkd, uninstall_order, dry_run=False): """ Given a dictionary of packages and an uninstall order, make some changes on this machine del_pkd -- A dictionary of package objects that are to be uninstalled uninstall_order -- the order with which to uninstall packages dry_run -- boolean flag indicating if we're serious about this or not """ status = OK remove_full_pkns = [] # Logger.info("UninstallOrder: %s" % uninstall_order) for name in uninstall_order: if del_pkd[name].full_name: name_str = del_pkd[name].full_name else: name_str = name remove_full_pkns.append(name_str) msg = "Packages to remove: %s" % remove_full_pkns Logger.info(msg) for pkn in uninstall_order: uninstall_status = del_pkd[pkn].uninstall(dry_run) if uninstall_status == FAIL: return FAIL return status
def reconcile_system(self, action): """ Figure out everything that needs to happen to get this machine into compliance with its stated configuration and do it action -- DRY_RUN or RECONCILE """ self.operation_status = OK self.operation_output = [] pkns = [] dry_run = False if action == DRY_RUN: msg = "Reconcile dry-run starting..." Logger.info(msg) dry_run = True else: msg = "Reconcile starting..." Logger.info(msg) add_pkd, del_pkd, uninstall_order = self._ck_inst_stat(pkns) # Logger.info("uninstall_order: %s" % uninstall_order) status = self._uninstall_pkgs(del_pkd, uninstall_order, dry_run) if status == FAIL: msg = "Uninstallation failed. Aborting reconcile." self.operation_output.append(msg) self.operation_status = FAIL return self._cleanup() add_pkd, del_pkd, uninstall_order = self._ck_inst_stat(pkns) status = self._install_pkgs(add_pkd, dry_run) if status != OK: self.operation_status = FAIL self.operation_output.append("Finished installing") return self._cleanup()
def unzip_type_4(self, pkg_path, full_name): ''' Perform untar/ungzip operations. (NOTE: NOT RELIABLE IN Python) pkg_path -- full path to package file, minus extension full_name -- name of the package, including version ''' Logger.info("Unzipping %s" % full_name) self.untargz(pkg_path + ".spkg") return OK
def fix_spkg(instance_name, pkn, action, package_factory): """A user is requesting to take a package that is currently flagged as 'broken' and set it to be 'installed'. Perhaps they have manually fixed the package on the system.""" progress_file = get_progress_path(instance_name) status_data = open(progress_file, 'r').read() try: status = json.loads(status_data) except ValueError: Logger.error("Data in %s is not parsable by json" % progress_file) return FAIL if status.get("install-progress") == None: status["install-progress"] = {} Logger.warning( "Status file is empty." ) now = time.asctime() if action == FIX: fix_name = [] base_names = re.compile("(\s+)\-\d+").findall(pkn) if base_names: base_name = base_names[0] else: base_name = pkn for possible_pkn in status["install-progress"]: if base_name in possible_pkn: fix_name.append(possible_pkn) if len(fix_name) > 1: msg = "Package name %s is ambigious. (possible %s)" msg = msg % (pkn, ' '.join(fix_name)) Logger.error(msg) return FAIL elif len(fix_name) == 1: pkn = fix_name[0] elif len(fix_name) == 0: new_package = package_factory.get_me_one(pkn) pkn = new_package.full_name Logger.info("Selecting previously UNINSTALLED package: %s" % pkn) status["install-progress"]["%s" % pkn] = {"INSTALLED": now, "UNINSTALLED": "NA", "VERIFIED": now} Logger.info("==OUTPUT==:%s has been set to INSTALLED." % pkn ) elif action == PURGE: if status["install-progress"].get(pkn): del status["install-progress"][pkn] msg = "==OUTPUT==:%s has been removed from %s status" msg = msg % (pkn, instance_name) Logger.info(msg) else: Logger.info("==OUTPUT==:%s is not in the status file" % pkn) pkns = status["install-progress"] possible_names = [x for x in pkns if pkn in x] msg = "==OUTPUT==:Maybe you want one of these: %s" msg = msg % str(possible_names) Logger.info(msg) return FAIL open(get_progress_path(instance_name), 'w').write(json.dumps(status)) return OK
def use_pkg(self, pkn, action, script_name, arguments): """ Main entry point to the class. Performs an action using a package pkn -- name of the package to use action -- STATUS, INSTALL, UNINSTALL, CONFIGURE, VERIFY, or EXEC script_name -- the name of a method to run within the package arguments -- arguments to the executable, typically a restore target """ try: pkg = self._get_new_pkg(pkn) if pkg.status == FAIL: self.operation_status = FAIL return self._cleanup() if action == INSTALL: pdat = self.progress.get_progress_data(False) installed_pkns, broken_pkns = Progress.get_installed(pdat) if pkn in [installed_pkns + broken_pkns]: Logger.error("Package %s cannot be installed." % pkn) self.operation_status = FAIL return FAIL add_pkd = {pkn: pkg} status = self._install_pkgs(add_pkd) if action == UNINSTALL: pdat = self.progress.get_progress_data(False) installed_pkns, broken_pkns = Progress.get_installed(pdat) bom_pkns = installed_pkns if pkn in bom_pkns: bom_pkns.remove(pkn) self.config.set_bom_pkgs(bom_pkns) add_pkd, del_pkd, uninstall_order = self._ck_inst_stat([]) status = self._uninstall_pkgs(del_pkd, uninstall_order) if action == VERIFY: status = pkg.verify() if action == CONFIGURE: self._check_configuration(pkg) status = pkg.configure() hash_path = make_path(pkg.get_path(), HASH_FILE) msg = "Writing configuration fingerprint to %s" % hash_path Logger.info(msg) self.config.save_hash(hash_path) if action in [EXECUTE, BACKUP, RESTORE]: status = pkg.execute_maint_script(script_name, arguments) if status == FAIL: self.operation_status = FAIL msg = "Finished %s for %s." msg = msg % (ACTION_REVERSE_LOOKUP[action], pkn) self.operation_output.append(msg) status = self._cleanup() return self._cleanup() except Exceptions.BadPackage, err: errmsg = "Cannot perform action %s on package %s: %s" errmsg = errmsg % (ACTION_REVERSE_LOOKUP[action], err.pkn, err.errmsg) Logger.warning(errmsg) Logger.info("RETURN FAIL 1") return FAIL
def execute_maint_script(self, script_name, arguments): ''' execute a user-defined function script_name -- name of the function to run arguments -- arguments to the script to be run ''' Package.execute_maint_script(self, script_name) self.status = self._find_cmd(script_name, arguments=arguments) msg = "%s result for %s : %s" Logger.info(msg % (script_name, self.full_name, self.status)) return self.status
def execute_maint_script(self, script_name): ''' execute a user-defined function script_name -- name of the function to run ''' self._download() # remove old history output_path = make_path(get_spkg_path(), self.instance_name, "output", "%s-output.yml" % script_name) if os.path.isfile(output_path): os.unlink(output_path) message = "Executing (%s) inside package (%s)" Logger.info(message % (script_name, self.full_name))
def verify(self, dry_run=False): ''' Perform a verification action on this package dry_run -- whether to actually perform the action ''' self._download() message = "Verifying package %s" % self.full_name Logger.info(message) self.status = self._find_cmd(VERIFY, dry_run=dry_run) if self.action != INSTALL: self._write_progress() Logger.info("Verify result for %s : %s" % (self.full_name, self.status)) return self.status
def _install(self, future_pkns, dry_run=False): ''' perform an installation operation future_pkns -- package names to be installed after this one dry_run -- whether or not to actuall perform the installation ''' dry_run_string = "" if dry_run: dry_run_string = " --DRY_RUN-- " self._download() message = "Beginning installation of (%s)%s" Logger.info(message % (self.full_name, dry_run_string)) self.status = self._find_cmd(INSTALL, future_pkns=future_pkns, dry_run=dry_run) msg = "Install result for %s%s : %s" Logger.info(msg % (self.full_name, dry_run_string, self.status)) return self.status
def uninstall(self, dry_run=False): ''' Perform an uninstall action on this package dry_run -- whether to actually perform the action ''' self.action = UNINSTALL self._download() if self.status != OK: return self.status dry_run_string = "" if dry_run: dry_run_string = " --DRY_RUN-- " msg = "Uninstalling package %s%s" % (self.name, dry_run_string) Logger.info(msg) self.status = self._find_cmd(UNINSTALL, dry_run=dry_run) if not dry_run: self._write_progress() msg = "Uninstall result for %s%s : %s" Logger.info(msg % (self.full_name, dry_run_string, self.status)) return self.status
def execute(self): 'starts a job and waits for it to finish' self.start_time = time.time() status = FAIL if self.is_running(): msg = "Refusing to run %s; it is already running" % self.cmd Logger.error(msg) return FAIL self.job_thread = JobThread(self.import_string, self.cmd, self.config) self.job_thread.start() Logger.info("Started job...") counter = 1 while self.is_running(): time.sleep(1) counter += 1 if not counter % 100: msg = "Waiting for completion (%s)..." Logger.info(msg % time.strftime(time.ctime())) counter = 1 status = self.job_thread.cmd_status return status
def _sort_uninstalled_pkgs(self, uninstall_order): """Make sure packages get uninstalled in the right order, paying attention to dependencies. uninstall_order -- a list of package names the order of which indicates which should be uninstalled first, second... """ dependency_dict = {} if len(uninstall_order) > 1: Logger.info("Determining uninstallation order...") for pkn in uninstall_order: dependency_dict[pkn] = [] for pkn in uninstall_order: pkg = self._get_new_pkg(pkn) for other_pkn in uninstall_order: if other_pkn in pkg.dependencies: dependency_dict[other_pkn].append(pkn) else: return uninstall_order new_proper_order = copy.deepcopy(uninstall_order) swapped = True while swapped: proper_order = copy.deepcopy(new_proper_order) swapped = False for pkn in proper_order: dependency_list = dependency_dict[pkn] for dependency in dependency_list: index1 = proper_order.index(dependency) index2 = proper_order.index(pkn) if index1 > index2: new_proper_order = swap(proper_order, index1, index2) swapped = True break if swapped: break Logger.info("Uninstallation Order: %s" % new_proper_order) return new_proper_order
def check_system(self): """ After a system has had all of its packages installed and it's in stable-state, this is a method we can use to verify that everything is still kosher """ Logger.info("System-check starting...") bom_pkns = self.config.get_bom_pkns() pdat = self.progress.get_progress_data() full_pdat = self.progress.get_progress_data(False) full_installed_pkns, _full_bpkns = Progress.get_installed(full_pdat) msg = "Packages that are installed: %s" Logger.info(msg % " ".join(full_installed_pkns)) installed_pkns, broken_pkns = Progress.get_installed(pdat) should_be_installed, shouldnt_be_installed = self._check_bom(bom_pkns) # check the configuration for each installed package pkg_info = { "Packages installed properly": installed_pkns, "Packages to be RECONFIGURED": {}, "Packages to be INSTALLED": should_be_installed, "Packages to be REMOVED": shouldnt_be_installed, "Packages that are BROKEN": broken_pkns, } for pkn in shouldnt_be_installed: pkg_info["Packages installed properly"].remove(pkn) for pkn in installed_pkns: differences = self._check_configuration_hash(pkn) if differences: if pkn in pkg_info["Packages installed properly"]: pkg_info["Packages to be RECONFIGURED"][pkn] = differences pkg_info["Packages installed properly"].remove(pkn) for key in pkg_info: Logger.info("==OUTPUT==:%s: %s" % (key, pkg_info[key])) return pkg_info
def get_type_4(self, full_name): ''' Get a type-4 package from the filesystem, and process it full_name -- name of package (with version) ''' pkg_dir = get_package_path(self.instance_name) os.system('bash -c "mkdir -p %s"' % pkg_dir) pkg_path = make_path(pkg_dir, full_name) if not os.path.isfile(pkg_path + ".spkg"): erstr = "No package file in %s." % (pkg_path + ".spkg") Logger.error(erstr) raise Exceptions.BadPackage(full_name, erstr) if sys.platform != 'win32': cmd = 'bash -c "cd %s && tar -mxzf %s.spkg"' % (pkg_dir, full_name) Logger.info("Untarring with command: %s" %cmd) if not os.system(cmd) == OK: raise Exceptions.BadPackage(full_name, "Could not unpack") return OK if self.unzip_type_5(pkg_path, full_name) == FAIL: raise Exceptions.BadPackage(full_name, "could not unzip") # tar = tarfile.open(pkg_path + ".tar", "r") # tar.errorlevel = 2 cwd = get_slash_cwd() os.chdir(pkg_dir) # for tarinfo in tar: # try: # tar.extract(tarinfo) # except tarfile.ExtractError, err: # Logger.warning("Error with package %s,%s: "\ # "%s" % (full_name, tarinfo.name, err)) # tar.close() if not os.path.isdir(make_path(pkg_path, full_name)): erstr = "Package %s is malformed." % (full_name) os.chdir(cwd) raise Exceptions.BadPackage(full_name, erstr) os.chdir(cwd) # os.unlink(pkg_path + ".tar") return OK
def get_type_5(self, full_name, injectors_info, libs_info): ''' Get type-5 package components from the filesystem, and process them full_name -- name of the package, including version info injectors_info -- dictionary describing injector libraries libs_info -- dictionary describing python code libraries ''' self.hunt_and_explode() pkg_path = make_path(get_spkg_path(), self.instance_name, "packages", full_name) injector_path = make_path(pkg_path, "injectors") lib_path = make_path(pkg_path, "libs") if not os.path.isdir(pkg_path): Logger.info("Making directory %s" % pkg_path) for path in [pkg_path, injector_path, lib_path]: cmd = 'bash -c "mkdir -p %s"' % path if os.system(cmd) != OK: msg = "Could not create directory structure (%s)" % path raise Exceptions.BadPackage(full_name, msg) info_dict = {"injectors": injectors_info, "libs": libs_info} self.make_symlinks(pkg_path, info_dict, full_name) return OK
def _get_uninst_pkg_dep(self, pkd, del_pkns, installed_pkns): """Add any packages that are installed already which are dependent upon those to the list as well pkd -- The dictionary of package objects del_pkns -- names of packages we want to remove installed_pkns -- names of packages that are installed already """ msg = "Checking dependencies of packages to be uninstalled %s..." msg = msg % del_pkns Logger.info(msg) v_pkgs = VirtualPackages(self.repository.pkg_data) uninstall_order = copy.deepcopy(del_pkns) while del_pkns: new_dependency_names = [] del_pkns = [] for pkn in installed_pkns: if pkn in pkd.keys(): Logger.debug("Package %s will already be deleted -- " "ignoring" % pkn) continue pkg = self._get_new_pkg(pkn) pkg.action = UNINSTALL for tombstoned_pkns in pkd.keys(): vpn = v_pkgs.get_vpn_from_pkn(tombstoned_pkns) if vpn in pkg.dependencies: erstr = "Adding to package removal list: %s" " (depends on %s)" uninstall_order.insert(0, pkn) Logger.info(erstr % (pkn, tombstoned_pkns)) pkd[tombstoned_pkns].depends_on_me.append(pkn) if pkn not in pkd.keys(): if pkn not in new_dependency_names: pkd[pkn] = pkg new_dependency_names.append(pkn) del_pkns = new_dependency_names proper_order = self._sort_uninstalled_pkgs(uninstall_order) return pkd, proper_order
def _get_object(self, future_pkns): ''' Import modules from the package in order to run an appropriate method on the class within. future_pkns -- future package names. Some packages want to know about the packages that will come after them ''' lib_path = self._get_lib_path() sys.path.insert(0, lib_path) Logger.info("Adding %s to our path..." % lib_path) obj = None try: class_name = '.'.join(self.class_name.split('.')[1:]) obj = Spkg.SpkgV5(self.config) os.chdir(self.working_dir) Logger.info("CHANGING TO: %s" % self.working_dir) letters = [ chr( x ) for x in range(65, 91) ] random.shuffle(letters) rand_string = ''.join(letters) exec("import %s as %s" % (self.class_name, rand_string)) self.config["__FUTURE_PACKAGES__"] = future_pkns self.config["__INSTANCE__"] = self.instance_name cmd_str = "obj = %s.%s(self.config)" exec(cmd_str % (rand_string, class_name)) except ImportError, ie: tb_str = StringIO.StringIO() traceback.print_exc(file=tb_str) tb_str.seek(0) data = tb_str.read() ermsg = '' for line in data.split('\n'): Logger.error(line) self.status = FAIL msg = "Class %s is not importable." % self.class_name raise BadPackage(self.name, msg)
def _install_pkgs(self, add_pkd, dry_run=False): """ Provided a dictionary of packages, we will install those that need to be installed. add_pkd -- a dictionary of packages that need to be added dry_run -- whether or not to just 'fake it' """ status = OK making_progress = True pkns_left = ["initialize"] while making_progress and pkns_left: making_progress = False install_order = self._find_install_order(add_pkd) pkns_left = list(install_order) for pkn in install_order: msg = "Packages remaining to install (in order):" Logger.info(msg) for tpkn in pkns_left: Logger.info(" + %s" % tpkn) pkns_left.remove(pkn) pkg = add_pkd[pkn] erstr = "Currently installing package priority %s [%s]" Logger.info(erstr % (pkg.get_priority(), pkn)) pkg_status = pkg.install_and_verify(pkns_left) if not dry_run: hash_path = make_path(pkg.get_path(), HASH_FILE) self.config.save_hash(hash_path) if pkg_status == FAIL: status = FAIL erstr = "Package installation failure -- re-calculating" " package installation order" Logger.error(erstr) break else: making_progress = True if status != OK: msg = "There are packages that are broken, and we have done all" " we can do. ; ;" Logger.error(msg) return status
def main(): Logger.add_std_err_logging() parser = optparse.OptionParser('\n'.join(USAGE)) parser.add_option("-s", "--status", dest="action", action="store_const", const=CHECK_STATUS, help="display the status of the system") parser.add_option("-c", "--configure", dest="action", action="store_const", const=CONFIGURE, help="configure a package") parser.add_option("-v", "--verify", dest="action", action="store_const", const=VERIFY, help="verify a package") parser.add_option("-i", "--install", dest="action", action="store_const", const=INSTALL, help="install a package") parser.add_option("-r", "--reconcile", dest="action", action="store_const", const=RECONCILE, help="reconcile the system") parser.add_option("-x", "--execute", dest="action", action="store_const", const=EXECUTE, help="Execute a maintenance script") parser.add_option("-u", "--uninstall", dest="action", action="store_const", const=UNINSTALL, help="uninstall a package") parser.add_option("-f", "--fix", dest="action", action="store_const", const=FIX, help="set a package status to INSTALLED without doing anything") parser.add_option("-p", "--purge", dest="action", action="store_const", const=PURGE, help="Remove a package from the status") parser.add_option("-d", "--dry-run", dest="action", action="store_const", const=DRY_RUN, help="Do a test reconcile") parser.add_option("-n", "--init", dest="action", action="store_const", const=INIT, help="Initialize the client after installation") parser.add_option("-B", "--backup", dest="action", action="store_const", const=BACKUP, help="Tell a package to back itself up") parser.add_option("-R", "--restore", dest="action", action="store_const", const=RESTORE, help="Tell a package to restore itself") (options, args) = parser.parse_args() Logger.info("options: (%s) / args: (%s)" % (options, args)) if options.action: Logger.info("Action: %s" % (ACTION_REVERSE_LOOKUP[options.action])) if len(args) < 1: print "CMD: %s" % ' '.join(sys.argv) print "This command requires an instance name." parser.print_help() exit_with_return_code(1) instance_name = args[0] env = BombardierEnvironment(instance_name) env.data_request() package_factory = PackageFactory(env) arguments = [] if options.action in [ RECONCILE, CHECK_STATUS, DRY_RUN, INIT ]: status = process_action(options.action, instance_name, '', '', '', package_factory) else: method_name = "" if len(args) < 2: print "CMD: %s" % ' '.join(sys.argv) print "This command requires a package name as an argument." Logger.error("This command requires a package name as an argument.") parser.print_help() exit_with_return_code( 1 ) package_name = args[1] if options.action == BACKUP: method_name = "backup" if options.action == RESTORE: method_name = "restore" arguments = args[2:] if options.action == EXECUTE: if len(args) < 3: print "CMD: %s" % ' '.join(sys.argv) print "This command requires a package name and a script name." Logger.error("This command requires a package name and a script name.") parser.print_help() exit_with_return_code( 1 ) package_name = args[1] method_name = args[2] if len(args) > 3: arguments = args[3:] status = process_action(options.action, instance_name, package_name, method_name, arguments, package_factory) if status != OK: exit_with_return_code(status) exit_with_return_code(status)
def _backup(self, obj, backup_data, future_pkns, dry_run): "Perform basic backup functions for a package" pre_backup_cmd = obj.pre_backup post_backup_cmd = obj.post_backup if pre_backup_cmd: status = self._find_cmd(pre_backup_cmd, future_pkns=future_pkns, dry_run=dry_run) if status != OK: erstr = "%s: backup FAILED because pre-backup command failed." Logger.error(erstr % self.full_name) return FAIL file_names = backup_data.get("file_names") if type(file_names) != type([]): errmsg = "Package %s did not define its backup data correctly."\ " '%s' should be a list." Logger.error(errmsg % (self.full_name, file_names)) return FAIL options = backup_data.get("options", [COMPRESS]) if type(options) != type([]): errmsg = "Package %s did not define its backup data correctly."\ " '%s' should be a list." Logger.error(errmsg % (self.full_name, options)) return FAIL backup_dir = tempfile.mkdtemp() Logger.info("Temporary backup dir: %s" % backup_dir) start_time = time.time() backup_data = {"__START_TIME__": start_time} for file_name in file_names: backup_data[file_name] = {} current_start = time.time() if file_name.startswith(os.path.sep): backup_destination = make_path(backup_dir, file_name[1:]) fpf = ForwardPluggableFileProcessor(file_name, backup_destination, options, Logger) backup_file_name, md5_dict = fpf.process_all() if not os.path.isfile(backup_file_name): Logger.error("Backup file not created.") return FAIL backup_data[file_name]["md5"] = md5_dict backup_data[file_name]["backup_file"] = rpartition(backup_file_name, os.path.sep)[-1] elapsed_time = time.time() - current_start backup_data[file_name]["elapsed_time"] = elapsed_time size = os.stat(file_name)[stat.ST_SIZE] backup_data[file_name]["size"] = size backup_data[file_name]["status"] = OK if post_backup_cmd: status = self._find_cmd(post_backup_cmd, future_pkns=future_pkns, dry_run=dry_run) if status != OK: erstr = "%s: backup FAILED because post-backup command failed." Logger.error(erstr % self.full_name) return FAIL backup_data["__BACKUP_DIR__"] = backup_dir dump_string = yaml_dump(backup_data) for line in dump_string.split('\n'): Logger.info("==REPORT==:%s" % line) return OK