def genWorkflowSystem(self, hosts): """ Generates a wfsystem object for upgrade which is combination of workflow and task file, describing which tasks to run on which servers with status as INITIAL Arguments: hosts - Hosts object that contains details of hosts Returns: wfsystem - WorkflowSystem object """ wfsystem = WorkflowSystem(self.name, self.config) self._genWorkflowPhase(self.display, wfsystem.display, hosts) self._genWorkflowPhase(self.precheck, wfsystem.precheck, hosts) self._genWorkflowPhase(self.execute, wfsystem.execute, hosts) self._genWorkflowPhase(self.postcheck, wfsystem.postcheck, hosts) return wfsystem
def dynamicUpdate(self, wf, wfsys, sysfilename, hosts, menuMsgs): """ updates the master status file - firstly as a tmp file, then reads in and validates the tmp file and if ok renames to master status file Arguments: wf: the original workflow (for validation) wfsys: the workflowsys sysfilename: name of the master status file hosts: the hosts object menuMsgs: list to hold messages for display during interaction Returns: True if master valid master status file was produced, False if errors were encountered """ # now update the workflow file as tmp file tmpsysfile = "%s_tmp" % sysfilename wfsys.write(tmpsysfile) log.debug("New tmp status file %s written" % tmpsysfile) # now attempt to reload the tmp file - which will validate against xsd loadtmp = WorkflowSystem("", self.config) if not loadtmp.load(tmpsysfile, hosts): menuMsgs.append("Failed to reload new master status file. " \ "Investigate the problem, then remove invalid " \ "temporary status file %s," \ " and rerun the dynamic alteration" % tmpsysfile) return False # now perform equivalency test against original workflow wfcheck = wf.genWorkflowSystem(hosts) if not wfcheck.isEquivalent(loadtmp): menuMsgs.append("New master status file does not match original" \ " workflow. " \ "Investigate the problem, then remove invalid " \ "temporary status file %s," \ " and rerun the dynamic alteration" % tmpsysfile) return False # if validated ok then rename the tmp file as the new master status # file # write backup file before updating the status file tstamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") backupdir = "dynamic_alteration_backups" if '/' in sysfilename: sysfilesplit = sysfilename.rsplit('/', 1) basedir = sysfilesplit[0] basename = sysfilesplit[1] else: basedir = "./" basename = sysfilename timestampedBackup = "%s/%s/%s.%s" % \ (basedir, backupdir, basename, tstamp) # retrieve full path of backup dir backupdir = timestampedBackup.rsplit('/', 1)[0] if not os.path.exists(backupdir): log.debug("creating backup directory %s" % backupdir) os.mkdir(backupdir) wfsys.write(timestampedBackup) log.debug("Timestamped backup %s created" % timestampedBackup) log.debug("Renaming temp status file %s as master status file %s" % \ (tmpsysfile, sysfilename)) try: os.rename(tmpsysfile, sysfilename) except: menuMsgs.append("Error renaming temporary file %s to master " "status file %s" % (tmpsysfile, sysfilename)) return False return True
def dynamicAlterations(self, o, hosts, ini): """ does the processing for a dynamic alterations run Arguments: o: the WorkflowOptions object hosts: hosts object ini: the ini file params object Returns: boolean - True if completed ok """ wf = Workflow(self.config) sysfilename = o.getSysStatusName() menuMsgs = [] if not wf.parse(o.wfile): log.info("Error encountered when parsing workflow file") return False try: wfsys = wf.genWorkflowSystem(hosts) except Exception as err: log.error("Failed to generate workflow : %s" % str(err)) log.debug("Exception: %s" % (traceback.format_exc())) return False dynamicMenu = DynamicMenuMgr(self.term_size, self.config, hosts, self.dynamicTaskValidator) while True: log.debug("Starting loop with menuMsgs length %d" % \ len(menuMsgs)) # Check if already got work file system if os.path.isfile(sysfilename): loadsys = WorkflowSystem("", self.config) loadsys.load(sysfilename, hosts) if wfsys.isEquivalent(loadsys): log.debug("Previous run is equivalent to current " "files - operating on status file") wfsys = loadsys else: log.error("Workflow and/or hosts file has " "changed since previous run, please investigate") return False else: log.debug("A new master status file will be created") dyntask = dynamicMenu.getDynamicAlteration(wfsys, menuMsgs) if dyntask == None: log.info("Quitting dynamic alterations") return True elif dyntask == {}: log.debug("Dynamic alteration empty") menuMsgs = [] else: log.debug("Dynamic task input: %s" % dyntask) menuMsgs = [] if not self.processDynamicAlteration( dyntask, wf, wfsys, sysfilename, hosts, menuMsgs): if len(menuMsgs) == 0: menuMsgs.append("Unspecified error during " \ "processing %s, dynamic alteration " \ "abandoned, please check logs" % \ dyntask[constants.DYN_ID]) log.debug("Finished processDynamicAlteration with " "menuMsg %s" % menuMsgs)
def start(self): # Read in command line options o = WorkflowOptions() o.parse() if o.version: if os.path.exists(constants.VERSION_FILE): with open(constants.VERSION_FILE, 'r') as fin: print fin.read() else: print "VERSION information was unavailable" return True FORMAT = '%(asctime)-15s %(message)s' # Find the filename bit of filename workfile = utils.get_file_less_ext(o.wfile) hostfile = utils.get_file_less_ext(o.hfile) starttime = datetime.datetime.now().strftime("%d%m%Y_%H%M%S") # Check if already running - note that this is protecting us # from having two processes updating the same # master status file if self.alreadyRunning(o.wfile, o.hfile, o.allow_multiple): if o.allow_multiple: print "Exiting as instance of wfeng already running " + \ "with same hostfile and workfile" else: print "Exiting as instance of wfeng already running" return False self.logfilename = "%s/%s_%s_%s.log" %\ (constants.LOG_DIR, workfile, hostfile, starttime) self.tracefilename = "%s/%s_%s_%s.trc" %\ (constants.LOG_DIR, workfile, hostfile, starttime) logging.addLevelName(constants.TRACE, "TRACE") logging.addLevelName(constants.DEBUGNOTIME, "DEBUGNOTIME") logging.basicConfig(filename=self.tracefilename, level=constants.TRACE, format=FORMAT) console = logging.StreamHandler() console.setLevel(logging.INFO) logging.getLogger('wfeng').addHandler(console) # Add debug handler debug = logging.FileHandler(self.logfilename) debug.setLevel(logging.DEBUG) formatter = WfengFormatter('%(asctime)s %(message)s') debug.setFormatter(formatter) logging.getLogger('wfeng').addHandler(debug) # Create queue for subprocesses self.logqueue = multiprocessing.Queue() self.config = WfmgrConfig() self.config.load() if not os.path.isfile(o.wfile): log.error("Workflow file %s does not exist" % o.wfile) return False if not os.path.isfile(o.hfile): log.error("Host file %s does not exist" % o.hfile) return False ini = WfengIni() # Load default if not ini.load(): return False if o.inifile is not None: if not ini.load(o.inifile): return False self.config.iniparams = ini.vars hosts = Hosts() if not hosts.parse(o.hfile): return False # if we are adding/removing dynamic pause/esc then # process interactively and exit if o.alter_pause_escape: try: dynAltOk = self.dynamicAlterations(o, hosts, ini) if not dynAltOk: log.info("Error in dynamic alterations menu interaction") return False return True except KeyboardInterrupt: log.info("\nUnexpected interrupt (CTRL/C) received - " \ "please check status file is as expected\n") return False if o.needMenu(): o.getMenuOptions(self.term_size, self.config) os.system('clear') log.info("WORKFLOW ENGINE".center(self.term_size)) log.info("----------------".center(self.term_size)) log.info("\nOptions chosen:") log.info(" workflow: %s" % o.wfile) log.info(" hosts: %s" % o.hfile) log.info(" ini: %s" % o.inifile) log.info(" timeout: %s" % o.timeout) log.info(" phase: %s" % o.phase) log.info(" server types: %s" % o.unparsed_servertypes) log.info(" server name: %s" % o.unparsed_servernames) log.info(" excluded servers: %s" % o.exclude) log.info(" task id: %s" % o.task) log.info(" tag id: %s" % o.tag) log.info(" output level: %d" % o.output_level) if o.force: log.info(" force: True") if o.yes: log.info(" yes: True") if o.automate: log.info(" automate: True") if o.list: log.info(" list: True") # for listing then if phase and task are not specified we will # run all phases if o.phase == None and o.task == None: log.info("All phases will be listed") o.phase = "postcheck" if o.fix: log.info(" fix: True") # check here that any server in the exclusion list is found in the # hosts file if o.exclude != None: excludes = [] # Now strip of spaces excludes = [x.strip() for x in o.exclude.split(',')] for excludedServer in excludes: if not hosts.host_exists(excludedServer): log.error( "Excluded server %s is not in the hosts file %s" % \ (excludedServer, o.hfile)) return False # validate servertype o.servertypes = [x.strip() for x in o.unparsed_servertypes.split(',')] if o.unparsed_servertypes != constants.ALL: for servertype in o.servertypes: if not servertype in hosts.hosts: log.error( "Server type %s selected is not in hosts file" % \ servertype) return False # validate servername o.servernames = [x.strip() for x in o.unparsed_servernames.split(',')] if o.unparsed_servernames != constants.ALL: for servername in o.servernames: if not hosts.host_exists(servername): log.error( "Server name %s selected is not in hosts file" % \ servername) return False wf = Workflow(self.config) if not wf.parse(o.wfile): return False try: wfsys = wf.genWorkflowSystem(hosts) except Exception as err: log.error("Failed to generate workflow list: %s" % str(err)) log.debug("Exception: %s" % (traceback.format_exc())) return False # validate task is valid if o.task != None: if not wfsys.taskInWorkflow(o.task): log.error("Task id %s selected is not in workflow file" % \ o.task) return False # validate tag is valid if o.tag != None: if not wfsys.tagInWorkflow(o.tag): log.error("Tag %s selected is not in workflow file" % \ o.tag) return False # Validate fix task is FabricTask if o.fix: taskobj = wfsys.getTask(o.task) if not isinstance(taskobj, FabricTask): log.error("Fix is not supported on task %s" % \ o.task) return False # Global status is in same directory as workfile, but appended with # hostname file and status.xml prefix = o.wfile.split('.xml') dirs = o.hfile.split("/") hostfile = dirs[len(dirs) - 1].split(".")[0] sysfilename = o.getSysStatusName() # flag for whether we are simply merging wflow and hosts and exiting # at start of a --list run listExitEarly = False # Check if already got work file system if os.path.isfile(sysfilename): log.info(" previous run: %s" % sysfilename) loadsys = WorkflowSystem("", self.config) loadsys.load(sysfilename, hosts) if wfsys.isEquivalent(loadsys): log.debug("Previous run is equivalent to current files") wfsys = loadsys else: log.error("Workflow and/or hosts file has changed since "\ "previous run, please investigate") return False else: # if --list and there is no pre-existing workflow file then we # generate a listing file and exit straight away if o.list: log.info("There is no pre-existing status file") listExitEarly = True else: log.info(" previous run: N/A") wfsys.write(sysfilename) # if --list then ascertain whether all eligible tasks are in a # complete state if o.list and wfsys.eligibleDisplayTasks(o.servertypes, o.servernames, o.exclude, o.force, o.exact_match): log.info("Display phase has not been completed and therefore " "later phases cannot be predicted") listExitEarly = True if listExitEarly: fnamesuffix = constants.LISTFILE_SUFFIX log.info("\nWorkflow file and hosts file have been merged to " \ "create list file %s%s, equivalent to a master status file" % \ (sysfilename, constants.LISTFILE_SUFFIX)) wfsys.write("%s%s" % (sysfilename, constants.LISTFILE_SUFFIX)) sys.exit(0) wfsys.logqueue = self.logqueue runner = CmdLineWorkflowRunner(wfsys, self.term_size, o) if o.list: log.info("\nListing has been run from the pre-existing master "\ "status file onwards, with predictive output written " "to file %s%s" % (sysfilename, constants.LISTFILE_SUFFIX)) return runner.run()
def dynamicAlterations(self, o, hosts, ini): """ does the processing for a dynamic alterations run Arguments: o: the WorkflowOptions object hosts: hosts object ini: the ini file params object Returns: boolean - True if completed ok """ wf = Workflow(self.config) sysfilename = o.getSysStatusName() menuMsgs = [] if not wf.parse(o.wfile): log.info("Error encountered when parsing workflow file") return False try: wfsys = wf.genWorkflowSystem(hosts) except Exception as err: log.error("Failed to generate workflow : %s" % str(err)) log.debug("Exception: %s" % (traceback.format_exc())) return False dynamicMenu = DynamicMenuMgr(self.term_size, self.config, hosts, self.dynamicTaskValidator) while True: log.debug("Starting loop with menuMsgs length %d" % \ len(menuMsgs)) # Check if already got work file system if os.path.isfile(sysfilename): loadsys = WorkflowSystem("", self.config) loadsys.load(sysfilename, hosts) if wfsys.isEquivalent(loadsys): log.debug("Previous run is equivalent to current " "files - operating on status file") wfsys = loadsys else: log.error("Workflow and/or hosts file has " "changed since previous run, please investigate") return False else: log.debug("A new master status file will be created") dyntask = dynamicMenu.getDynamicAlteration(wfsys, menuMsgs) if dyntask == None: log.info("Quitting dynamic alterations") return True elif dyntask == {}: log.debug("Dynamic alteration empty") menuMsgs = [] else: log.debug("Dynamic task input: %s" % dyntask) menuMsgs = [] if not self.processDynamicAlteration(dyntask, wf, wfsys, sysfilename, hosts, menuMsgs): if len(menuMsgs) == 0: menuMsgs.append("Unspecified error during " \ "processing %s, dynamic alteration " \ "abandoned, please check logs" % \ dyntask[constants.DYN_ID]) log.debug("Finished processDynamicAlteration with " "menuMsg %s" % menuMsgs)