def __init__(self): self.dynamicTaskValidator = DynamicTaskValidator() self.term_size = 80 args = os.popen('stty -a', 'r').read().split(';') for x in args: x = x.strip().lstrip() if x.startswith("columns"): # Either split by = or ' ' splitChar = ' ' if '=' in x: splitChar = '=' k, v = x.split(splitChar) self.term_size = int(v)
class WorkflowEngine: """The main class of the workflow manager.""" def __init__(self): self.dynamicTaskValidator = DynamicTaskValidator() self.term_size = 80 args = os.popen('stty -a', 'r').read().split(';') for x in args: x = x.strip().lstrip() if x.startswith("columns"): # Either split by = or ' ' splitChar = ' ' if '=' in x: splitChar = '=' k, v = x.split(splitChar) self.term_size = int(v) def deletePidFile(self): pid = str(os.getpid()) if not os.path.exists(constants.LOCK_DIR): log.error("Lock directory does not exist") return filename = os.path.join(constants.LOCK_DIR, pid) if os.path.exists(filename): os.remove(filename) def getRunningPid(self, workfile, hostfile, specific=False): """ Gets the pid of wfeng that is still running. If specific is true then returns instance running with specified workfile and hostfile, else returns pid of any running wfeng Arguments: workfile - ignored unless specific is true hostfile - ignored unless specific is true specific - if True then return pid of wfeng instance that is with this workfile/hostfile, else return pid of any running wfeng Returns: list of pids that are running """ pids = [] files = os.listdir(constants.LOCK_DIR) # Find out what locks we have, each filename is name of pid # and contents are workfile, hostfile for filename in files: full_filename = os.path.join(constants.LOCK_DIR, filename) f = open(full_filename, "r") for line in f: args = line.rstrip().split(",") pid = None if len(args) == 2: # Check well formatted line if specific: if workfile == args[0] and hostfile == args[1]: pid = filename else: pid = filename if pid != None: # This is pid of an instance of wfeng we care about try: os.kill(int(pid), 0) # Pid is still running pids.append(pid) # This might be a pid of except OSError: # Pid nolonger running pass f.close() return pids def alreadyRunning(self, workfile, hostfile, allowMultiple=False): """ Returns True if another instance is running""" absw = os.path.abspath(workfile) absh = os.path.abspath(hostfile) if len(self.getRunningPid(absw, absh, allowMultiple)) != 0: return True # Not currently running a clashing wfeng, create lockfile pid = str(os.getpid()) self.writePid(pid, absw, absh) # Wait and check pid not altered, to ensure another process wasn't # started at same time as us # We aren't expecting multiple to be submitted within seconds of each # other, so this should suffice sleep(1) # Has anyone altered the pid for our parameters, invalid whether or # not allowMultipe is False pids = self.getRunningPid(absw, absh, True) if len(pids) != 1 or pids[0] != pid: return True # If allowMultiple is False, then we need to know the only running # instanceis us if not allowMultiple: pids = self.getRunningPid(absw, absh, False) if len(pids) != 1 or pids[0] != pid: # Not just our pid returned return True return False def writePid(self, pid, workfile, hostfile): """ Writes our pid to pid file""" if not os.path.exists(constants.LOCK_DIR): log.error("Lock directory does not exist") return filename = os.path.join(constants.LOCK_DIR, pid) file = open(filename, 'w') file.write("%s,%s\n" % (workfile, hostfile)) file.close() def start(self): # Read in command line options o = WorkflowOptions() o.parse() if o.version: if os.path.exists(constants.VERSION_FILE): with open(constants.VERSION_FILE, 'r') as fin: print fin.read() else: print "VERSION information was unavailable" return True FORMAT = '%(asctime)-15s %(message)s' # Find the filename bit of filename workfile = utils.get_file_less_ext(o.wfile) hostfile = utils.get_file_less_ext(o.hfile) starttime = datetime.datetime.now().strftime("%d%m%Y_%H%M%S") # Check if already running - note that this is protecting us # from having two processes updating the same # master status file if self.alreadyRunning(o.wfile, o.hfile, o.allow_multiple): if o.allow_multiple: print "Exiting as instance of wfeng already running " + \ "with same hostfile and workfile" else: print "Exiting as instance of wfeng already running" return False self.logfilename = "%s/%s_%s_%s.log" %\ (constants.LOG_DIR, workfile, hostfile, starttime) self.tracefilename = "%s/%s_%s_%s.trc" %\ (constants.LOG_DIR, workfile, hostfile, starttime) logging.addLevelName(constants.TRACE, "TRACE") logging.addLevelName(constants.DEBUGNOTIME, "DEBUGNOTIME") logging.basicConfig(filename=self.tracefilename, level=constants.TRACE, format=FORMAT) console = logging.StreamHandler() console.setLevel(logging.INFO) logging.getLogger('wfeng').addHandler(console) # Add debug handler debug = logging.FileHandler(self.logfilename) debug.setLevel(logging.DEBUG) formatter = WfengFormatter('%(asctime)s %(message)s') debug.setFormatter(formatter) logging.getLogger('wfeng').addHandler(debug) # Create queue for subprocesses self.logqueue = multiprocessing.Queue() self.config = WfmgrConfig() self.config.load() if not os.path.isfile(o.wfile): log.error("Workflow file %s does not exist" % o.wfile) return False if not os.path.isfile(o.hfile): log.error("Host file %s does not exist" % o.hfile) return False ini = WfengIni() # Load default if not ini.load(): return False if o.inifile is not None: if not ini.load(o.inifile): return False self.config.iniparams = ini.vars hosts = Hosts() if not hosts.parse(o.hfile): return False # if we are adding/removing dynamic pause/esc then # process interactively and exit if o.alter_pause_escape: try: dynAltOk = self.dynamicAlterations(o, hosts, ini) if not dynAltOk: log.info("Error in dynamic alterations menu interaction") return False return True except KeyboardInterrupt: log.info("\nUnexpected interrupt (CTRL/C) received - " \ "please check status file is as expected\n") return False if o.needMenu(): o.getMenuOptions(self.term_size, self.config) os.system('clear') log.info("WORKFLOW ENGINE".center(self.term_size)) log.info("----------------".center(self.term_size)) log.info("\nOptions chosen:") log.info(" workflow: %s" % o.wfile) log.info(" hosts: %s" % o.hfile) log.info(" ini: %s" % o.inifile) log.info(" timeout: %s" % o.timeout) log.info(" phase: %s" % o.phase) log.info(" server types: %s" % o.unparsed_servertypes) log.info(" server name: %s" % o.unparsed_servernames) log.info(" excluded servers: %s" % o.exclude) log.info(" task id: %s" % o.task) log.info(" tag id: %s" % o.tag) log.info(" output level: %d" % o.output_level) if o.force: log.info(" force: True") if o.yes: log.info(" yes: True") if o.automate: log.info(" automate: True") if o.list: log.info(" list: True") # for listing then if phase and task are not specified we will # run all phases if o.phase == None and o.task == None: log.info("All phases will be listed") o.phase = "postcheck" if o.fix: log.info(" fix: True") # check here that any server in the exclusion list is found in the # hosts file if o.exclude != None: excludes = [] # Now strip of spaces excludes = [x.strip() for x in o.exclude.split(',')] for excludedServer in excludes: if not hosts.host_exists(excludedServer): log.error( "Excluded server %s is not in the hosts file %s" % \ (excludedServer, o.hfile)) return False # validate servertype o.servertypes = [x.strip() for x in o.unparsed_servertypes.split(',')] if o.unparsed_servertypes != constants.ALL: for servertype in o.servertypes: if not servertype in hosts.hosts: log.error( "Server type %s selected is not in hosts file" % \ servertype) return False # validate servername o.servernames = [x.strip() for x in o.unparsed_servernames.split(',')] if o.unparsed_servernames != constants.ALL: for servername in o.servernames: if not hosts.host_exists(servername): log.error( "Server name %s selected is not in hosts file" % \ servername) return False wf = Workflow(self.config) if not wf.parse(o.wfile): return False try: wfsys = wf.genWorkflowSystem(hosts) except Exception as err: log.error("Failed to generate workflow list: %s" % str(err)) log.debug("Exception: %s" % (traceback.format_exc())) return False # validate task is valid if o.task != None: if not wfsys.taskInWorkflow(o.task): log.error("Task id %s selected is not in workflow file" % \ o.task) return False # validate tag is valid if o.tag != None: if not wfsys.tagInWorkflow(o.tag): log.error("Tag %s selected is not in workflow file" % \ o.tag) return False # Validate fix task is FabricTask if o.fix: taskobj = wfsys.getTask(o.task) if not isinstance(taskobj, FabricTask): log.error("Fix is not supported on task %s" % \ o.task) return False # Global status is in same directory as workfile, but appended with # hostname file and status.xml prefix = o.wfile.split('.xml') dirs = o.hfile.split("/") hostfile = dirs[len(dirs) - 1].split(".")[0] sysfilename = o.getSysStatusName() # flag for whether we are simply merging wflow and hosts and exiting # at start of a --list run listExitEarly = False # Check if already got work file system if os.path.isfile(sysfilename): log.info(" previous run: %s" % sysfilename) loadsys = WorkflowSystem("", self.config) loadsys.load(sysfilename, hosts) if wfsys.isEquivalent(loadsys): log.debug("Previous run is equivalent to current files") wfsys = loadsys else: log.error("Workflow and/or hosts file has changed since "\ "previous run, please investigate") return False else: # if --list and there is no pre-existing workflow file then we # generate a listing file and exit straight away if o.list: log.info("There is no pre-existing status file") listExitEarly = True else: log.info(" previous run: N/A") wfsys.write(sysfilename) # if --list then ascertain whether all eligible tasks are in a # complete state if o.list and wfsys.eligibleDisplayTasks(o.servertypes, o.servernames, o.exclude, o.force, o.exact_match): log.info("Display phase has not been completed and therefore " "later phases cannot be predicted") listExitEarly = True if listExitEarly: fnamesuffix = constants.LISTFILE_SUFFIX log.info("\nWorkflow file and hosts file have been merged to " \ "create list file %s%s, equivalent to a master status file" % \ (sysfilename, constants.LISTFILE_SUFFIX)) wfsys.write("%s%s" % (sysfilename, constants.LISTFILE_SUFFIX)) sys.exit(0) wfsys.logqueue = self.logqueue runner = CmdLineWorkflowRunner(wfsys, self.term_size, o) if o.list: log.info("\nListing has been run from the pre-existing master "\ "status file onwards, with predictive output written " "to file %s%s" % (sysfilename, constants.LISTFILE_SUFFIX)) return runner.run() def dynamicAlterations(self, o, hosts, ini): """ does the processing for a dynamic alterations run Arguments: o: the WorkflowOptions object hosts: hosts object ini: the ini file params object Returns: boolean - True if completed ok """ wf = Workflow(self.config) sysfilename = o.getSysStatusName() menuMsgs = [] if not wf.parse(o.wfile): log.info("Error encountered when parsing workflow file") return False try: wfsys = wf.genWorkflowSystem(hosts) except Exception as err: log.error("Failed to generate workflow : %s" % str(err)) log.debug("Exception: %s" % (traceback.format_exc())) return False dynamicMenu = DynamicMenuMgr(self.term_size, self.config, hosts, self.dynamicTaskValidator) while True: log.debug("Starting loop with menuMsgs length %d" % \ len(menuMsgs)) # Check if already got work file system if os.path.isfile(sysfilename): loadsys = WorkflowSystem("", self.config) loadsys.load(sysfilename, hosts) if wfsys.isEquivalent(loadsys): log.debug("Previous run is equivalent to current " "files - operating on status file") wfsys = loadsys else: log.error("Workflow and/or hosts file has " "changed since previous run, please investigate") return False else: log.debug("A new master status file will be created") dyntask = dynamicMenu.getDynamicAlteration(wfsys, menuMsgs) if dyntask == None: log.info("Quitting dynamic alterations") return True elif dyntask == {}: log.debug("Dynamic alteration empty") menuMsgs = [] else: log.debug("Dynamic task input: %s" % dyntask) menuMsgs = [] if not self.processDynamicAlteration( dyntask, wf, wfsys, sysfilename, hosts, menuMsgs): if len(menuMsgs) == 0: menuMsgs.append("Unspecified error during " \ "processing %s, dynamic alteration " \ "abandoned, please check logs" % \ dyntask[constants.DYN_ID]) log.debug("Finished processDynamicAlteration with " "menuMsg %s" % menuMsgs) def processDynamicAlteration(self, dyntask, wf, wfsys, sysfilename, hosts, menuMsgs): """ processes a dynamic alteration (add/remove) for pause/esc Arguments: dyntask: a dictionary containing the details for the alteration wf: the original workflow (for validation in dynamicUpdate) wfsys: the workflowsys sysfilename: the name of the master status file hosts: host object containing details of the hosts in this run menuMsgs: list to hold messages for display during interaction Returns: boolean: True if alteration successful, False if alteration failed """ dyntaskid = dyntask[constants.DYN_ID] if dyntaskid == "" or dyntaskid == None: menuMsgs.append("Dynamic task id is empty") return False dyntype = dyntask[constants.DYN_TYPE] if not dyntype == constants.DYNAMIC_ESCAPE and \ not dyntype == constants.DYNAMIC_PAUSE: menuMsgs.append("Dynamic task type for dynamic task %s must be " "%s or %s - but found %s" % \ (dyntaskid, constants.DYNAMIC_ESCAPE, constants.DYNAMIC_PAUSE, dyntype)) return False if dyntask[constants.DYN_ACTION] == constants.DYNAMIC_ADD: if wfsys.taskInWorkflow(dyntaskid): menuMsgs.append( "Dynamic task id %s duplicates one already in the workflow" \ % dyntaskid) return False if not self.dynamicAlterationAdd(dyntask, wf, wfsys, sysfilename, hosts, menuMsgs): menuMsgs.append("Unable to add dynamic task %s" % \ dyntask[constants.DYN_ID]) return False elif dyntask[constants.DYN_ACTION] == constants.DYNAMIC_REMOVE: # look for dynamic task id in workflow execute phase if wfsys.execute.getTask(dyntaskid) is None: menuMsgs.append("Dynamic %s task id %s not found in workflow " "execute phase" % (dyntype, dyntaskid)) return False if not self.dynamicAlterationRemove(dyntask, wf, wfsys, sysfilename, hosts, menuMsgs): menuMsgs.append("Unable to remove dynamic task %s" % \ dyntask[constants.DYN_ID]) return False else: menuMsgs.append("Unable to determine action %s" % \ dyntask[constants.DYN_ACTION]) return False return True def dynamicAlterationAdd(self, dyntask, wf, wfsys, sysfilename, hosts, menuMsgs): """performs dynamic addition for pause/esc Arguments: dyntask: a dictionary containing the details for the alteration wf: the original workflow (for validation in dynamicUpdate) wfsys: the workflowsys sysfilename: the name of the master status file hosts: host object containing details of the hosts in this run menuMsgs: list to hold messages for display during interaction Returns: boolean: True if alteration successful, False if alteration failed """ # get the execute phase for the workflow executephase = wfsys.getExecutePhase() refid = dyntask[constants.DYN_REFID] taskdict = executephase.getTaskIndices(refid) indRefs = taskdict[constants.TASK_LIST] groupid = taskdict[constants.TASK_IN_GROUP] if len(indRefs) == 0: menuMsgs.append("Unable to find valid reference task %s " "in workflow execute phase" % refid) return False firstRefInd = indRefs[0] lastRefInd = indRefs[len(indRefs) - 1] pos = dyntask[constants.DYN_POS] dyntaskid = dyntask[constants.DYN_ID] dyntype = dyntask[constants.DYN_TYPE] if pos == constants.DYNAMIC_BEFORE: insertionPoint = firstRefInd elif pos == constants.DYNAMIC_AFTER: insertionPoint = lastRefInd + 1 else: menuMsgs.append("Position for dynamic task %s must be %s " "or %s - but found %s" % \ (dyntaskid, constants.DYNAMIC_BEFORE, constants.DYNAMIC_AFTER, pos)) return False log.debug("inserting task %s into execute phase at " "index %s" % (dyntaskid, insertionPoint)) checkparams = {} if dyntask[constants.DYN_CHECKPARAMS] is None: checkParams = None else: chks = dyntask[constants.DYN_CHECKPARAMS].split(',') for chk in chks: chckName, chckVar = chk.split('=') checkparams[chckName] = chckVar # note that for dynamic tasks dynamic and optional will always be true dynamic = True optional = True if dyntype == constants.DYNAMIC_ESCAPE: taskforinsert = EscapeTask( dyntaskid, self.config, dyntask[constants.DYN_MSG], dyntask[constants.DYN_HOSTS], dyntask[constants.DYN_SERVER], dyntask[constants.DYN_SWVER], dyntask[constants.DYN_OSVER], dyntask[constants.DYN_DEP], optional, utils.get_stringtoboolean(dyntask[constants.DYN_DEPSINGLE]), checkparams, groupid, dynamic) else: taskforinsert = PauseTask( dyntaskid, self.config, dyntask[constants.DYN_MSG], dyntask[constants.DYN_HOSTS], dyntask[constants.DYN_SERVER], dyntask[constants.DYN_SWVER], dyntask[constants.DYN_OSVER], dyntask[constants.DYN_DEP], optional, utils.get_stringtoboolean(dyntask[constants.DYN_DEPSINGLE]), checkparams, groupid, dynamic) if groupid == None: if not executephase.insertDynamic(taskforinsert, hosts, insertionPoint, menuMsgs): menuMsgs.append("Failed to insert dynamic task %s " "into execute phase" % \ taskforinsert.name) return False else: if not executephase.insertDynamicIntoGroup(taskforinsert, hosts, indRefs, pos, menuMsgs): menuMsgs.append("Failed to insert dynamic task %s " "into group %s execute phase" % \ (taskforinsert.name, groupid)) return False if not self.dynamicUpdate(wf, wfsys, sysfilename, hosts, menuMsgs): menuMsgs.append("Dynamic addition update for task %s " "failed - please check logs" % \ taskforinsert.name) return False menuMsgs.append("New master status file %s created with " "dynamic entry %s added" % (sysfilename, dyntaskid)) return True def dynamicAlterationRemove(self, dyntask, wf, wfsys, sysfilename, hosts, menuMsgs): """performs dynamic addition for pause/esc Arguments: dyntask: a dictionary containing the details for the alteration wf: the original workflow (for validation in dynamicUpdate) wfsys: the workflowsys sysfilename: the name of the master status file hosts: host object containing details of the hosts in this run menuMsgs: list to hold messages for display during interaction Returns: boolean: True if alteration successful, False if alteration failed """ # get the execute phase for the workflow executephase = wfsys.getExecutePhase() dyntaskid = dyntask[constants.DYN_ID] dyntype = dyntask[constants.DYN_TYPE] if self.dynamicTaskValidator.hasDependents(dyntaskid, wfsys, menuMsgs): menuMsgs.append("Dynamic removal failed validation") return False if not executephase.removeDynamic(dyntaskid, dyntype, menuMsgs): menuMsgs.append("Failed to remove dynamic task %s " "from execute phase, please check logs" % \ dyntaskid) return False if not self.dynamicUpdate(wf, wfsys, sysfilename, hosts, menuMsgs): menuMsgs.append("Dynamic removal update for task %s failed " "- please check logs" % \ dyntaskid) return False menuMsgs.append("New master status file %s created with dynamic " "entry %s removed" % (sysfilename, dyntaskid)) return True def dynamicUpdate(self, wf, wfsys, sysfilename, hosts, menuMsgs): """ updates the master status file - firstly as a tmp file, then reads in and validates the tmp file and if ok renames to master status file Arguments: wf: the original workflow (for validation) wfsys: the workflowsys sysfilename: name of the master status file hosts: the hosts object menuMsgs: list to hold messages for display during interaction Returns: True if master valid master status file was produced, False if errors were encountered """ # now update the workflow file as tmp file tmpsysfile = "%s_tmp" % sysfilename wfsys.write(tmpsysfile) log.debug("New tmp status file %s written" % tmpsysfile) # now attempt to reload the tmp file - which will validate against xsd loadtmp = WorkflowSystem("", self.config) if not loadtmp.load(tmpsysfile, hosts): menuMsgs.append("Failed to reload new master status file. " \ "Investigate the problem, then remove invalid " \ "temporary status file %s," \ " and rerun the dynamic alteration" % tmpsysfile) return False # now perform equivalency test against original workflow wfcheck = wf.genWorkflowSystem(hosts) if not wfcheck.isEquivalent(loadtmp): menuMsgs.append("New master status file does not match original" \ " workflow. " \ "Investigate the problem, then remove invalid " \ "temporary status file %s," \ " and rerun the dynamic alteration" % tmpsysfile) return False # if validated ok then rename the tmp file as the new master status # file # write backup file before updating the status file tstamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") backupdir = "dynamic_alteration_backups" if '/' in sysfilename: sysfilesplit = sysfilename.rsplit('/', 1) basedir = sysfilesplit[0] basename = sysfilesplit[1] else: basedir = "./" basename = sysfilename timestampedBackup = "%s/%s/%s.%s" % \ (basedir, backupdir, basename, tstamp) # retrieve full path of backup dir backupdir = timestampedBackup.rsplit('/', 1)[0] if not os.path.exists(backupdir): log.debug("creating backup directory %s" % backupdir) os.mkdir(backupdir) wfsys.write(timestampedBackup) log.debug("Timestamped backup %s created" % timestampedBackup) log.debug("Renaming temp status file %s as master status file %s" % \ (tmpsysfile, sysfilename)) try: os.rename(tmpsysfile, sysfilename) except: menuMsgs.append("Error renaming temporary file %s to master " "status file %s" % (tmpsysfile, sysfilename)) return False return True
class WorkflowEngine: """The main class of the workflow manager.""" def __init__(self): self.dynamicTaskValidator = DynamicTaskValidator() self.term_size = 80 args = os.popen('stty -a', 'r').read().split(';') for x in args: x = x.strip().lstrip() if x.startswith("columns"): # Either split by = or ' ' splitChar = ' ' if '=' in x: splitChar = '=' k, v = x.split(splitChar) self.term_size = int(v) def deletePidFile(self): pid = str(os.getpid()) if not os.path.exists(constants.LOCK_DIR): log.error("Lock directory does not exist") return filename = os.path.join(constants.LOCK_DIR, pid) if os.path.exists(filename): os.remove(filename) def getRunningPid(self, workfile, hostfile, specific=False): """ Gets the pid of wfeng that is still running. If specific is true then returns instance running with specified workfile and hostfile, else returns pid of any running wfeng Arguments: workfile - ignored unless specific is true hostfile - ignored unless specific is true specific - if True then return pid of wfeng instance that is with this workfile/hostfile, else return pid of any running wfeng Returns: list of pids that are running """ pids = [] files = os.listdir(constants.LOCK_DIR) # Find out what locks we have, each filename is name of pid # and contents are workfile, hostfile for filename in files: full_filename = os.path.join(constants.LOCK_DIR, filename) f = open(full_filename, "r") for line in f: args = line.rstrip().split(",") pid = None if len(args) == 2: # Check well formatted line if specific: if workfile == args[0] and hostfile == args[1]: pid = filename else: pid = filename if pid != None: # This is pid of an instance of wfeng we care about try: os.kill(int(pid), 0) # Pid is still running pids.append(pid) # This might be a pid of except OSError: # Pid nolonger running pass f.close() return pids def alreadyRunning(self, workfile, hostfile, allowMultiple=False): """ Returns True if another instance is running""" absw = os.path.abspath(workfile) absh = os.path.abspath(hostfile) if len(self.getRunningPid(absw, absh, allowMultiple)) != 0: return True # Not currently running a clashing wfeng, create lockfile pid = str(os.getpid()) self.writePid(pid, absw, absh) # Wait and check pid not altered, to ensure another process wasn't # started at same time as us # We aren't expecting multiple to be submitted within seconds of each # other, so this should suffice sleep(1) # Has anyone altered the pid for our parameters, invalid whether or # not allowMultipe is False pids = self.getRunningPid(absw, absh, True) if len(pids) != 1 or pids[0] != pid: return True # If allowMultiple is False, then we need to know the only running # instanceis us if not allowMultiple: pids = self.getRunningPid(absw, absh, False) if len(pids) != 1 or pids[0] != pid: # Not just our pid returned return True return False def writePid(self, pid, workfile, hostfile): """ Writes our pid to pid file""" if not os.path.exists(constants.LOCK_DIR): log.error("Lock directory does not exist") return filename = os.path.join(constants.LOCK_DIR, pid) file = open(filename, 'w') file.write("%s,%s\n" % (workfile, hostfile)) file.close() def start(self): # Read in command line options o = WorkflowOptions() o.parse() if o.version: if os.path.exists(constants.VERSION_FILE): with open(constants.VERSION_FILE, 'r') as fin: print fin.read() else: print "VERSION information was unavailable" return True FORMAT = '%(asctime)-15s %(message)s' # Find the filename bit of filename workfile = utils.get_file_less_ext(o.wfile) hostfile = utils.get_file_less_ext(o.hfile) starttime = datetime.datetime.now().strftime("%d%m%Y_%H%M%S") # Check if already running - note that this is protecting us # from having two processes updating the same # master status file if self.alreadyRunning(o.wfile, o.hfile, o.allow_multiple): if o.allow_multiple: print "Exiting as instance of wfeng already running " + \ "with same hostfile and workfile" else: print "Exiting as instance of wfeng already running" return False self.logfilename = "%s/%s_%s_%s.log" %\ (constants.LOG_DIR, workfile, hostfile, starttime) self.tracefilename = "%s/%s_%s_%s.trc" %\ (constants.LOG_DIR, workfile, hostfile, starttime) logging.addLevelName(constants.TRACE, "TRACE") logging.addLevelName(constants.DEBUGNOTIME, "DEBUGNOTIME") logging.basicConfig(filename=self.tracefilename, level=constants.TRACE, format=FORMAT) console = logging.StreamHandler() console.setLevel(logging.INFO) logging.getLogger('wfeng').addHandler(console) # Add debug handler debug = logging.FileHandler(self.logfilename) debug.setLevel(logging.DEBUG) formatter = WfengFormatter('%(asctime)s %(message)s') debug.setFormatter(formatter) logging.getLogger('wfeng').addHandler(debug) # Create queue for subprocesses self.logqueue = multiprocessing.Queue() self.config = WfmgrConfig() self.config.load() if not os.path.isfile(o.wfile): log.error("Workflow file %s does not exist" % o.wfile) return False if not os.path.isfile(o.hfile): log.error("Host file %s does not exist" % o.hfile) return False ini = WfengIni() # Load default if not ini.load(): return False if o.inifile is not None: if not ini.load(o.inifile): return False self.config.iniparams = ini.vars hosts = Hosts() if not hosts.parse(o.hfile): return False # if we are adding/removing dynamic pause/esc then # process interactively and exit if o.alter_pause_escape: try: dynAltOk = self.dynamicAlterations(o, hosts, ini) if not dynAltOk: log.info("Error in dynamic alterations menu interaction") return False return True except KeyboardInterrupt: log.info("\nUnexpected interrupt (CTRL/C) received - " \ "please check status file is as expected\n") return False if o.needMenu(): o.getMenuOptions(self.term_size, self.config) os.system('clear') log.info("WORKFLOW ENGINE".center(self.term_size)) log.info("----------------".center(self.term_size)) log.info("\nOptions chosen:") log.info(" workflow: %s" % o.wfile) log.info(" hosts: %s" % o.hfile) log.info(" ini: %s" % o.inifile) log.info(" timeout: %s" % o.timeout) log.info(" phase: %s" % o.phase) log.info(" server types: %s" % o.unparsed_servertypes) log.info(" server name: %s" % o.unparsed_servernames) log.info(" excluded servers: %s" % o.exclude) log.info(" task id: %s" % o.task) log.info(" tag id: %s" % o.tag) log.info(" output level: %d" % o.output_level) if o.force: log.info(" force: True") if o.yes: log.info(" yes: True") if o.automate: log.info(" automate: True") if o.list: log.info(" list: True") # for listing then if phase and task are not specified we will # run all phases if o.phase == None and o.task == None: log.info("All phases will be listed") o.phase = "postcheck" if o.fix: log.info(" fix: True") # check here that any server in the exclusion list is found in the # hosts file if o.exclude != None: excludes = [] # Now strip of spaces excludes = [x.strip() for x in o.exclude.split(',')] for excludedServer in excludes: if not hosts.host_exists(excludedServer): log.error( "Excluded server %s is not in the hosts file %s" % \ (excludedServer, o.hfile)) return False # validate servertype o.servertypes = [x.strip() for x in o.unparsed_servertypes.split(',')] if o.unparsed_servertypes != constants.ALL: for servertype in o.servertypes: if not servertype in hosts.hosts: log.error( "Server type %s selected is not in hosts file" % \ servertype) return False # validate servername o.servernames = [x.strip() for x in o.unparsed_servernames.split(',')] if o.unparsed_servernames != constants.ALL: for servername in o.servernames: if not hosts.host_exists(servername): log.error( "Server name %s selected is not in hosts file" % \ servername) return False wf = Workflow(self.config) if not wf.parse(o.wfile): return False try: wfsys = wf.genWorkflowSystem(hosts) except Exception as err: log.error("Failed to generate workflow list: %s" % str(err)) log.debug("Exception: %s" % (traceback.format_exc())) return False # validate task is valid if o.task != None: if not wfsys.taskInWorkflow(o.task): log.error("Task id %s selected is not in workflow file" % \ o.task) return False # validate tag is valid if o.tag != None: if not wfsys.tagInWorkflow(o.tag): log.error("Tag %s selected is not in workflow file" % \ o.tag) return False # Validate fix task is FabricTask if o.fix: taskobj = wfsys.getTask(o.task) if not isinstance(taskobj, FabricTask): log.error("Fix is not supported on task %s" % \ o.task) return False # Global status is in same directory as workfile, but appended with # hostname file and status.xml prefix = o.wfile.split('.xml') dirs = o.hfile.split("/") hostfile = dirs[len(dirs) - 1].split(".")[0] sysfilename = o.getSysStatusName() # flag for whether we are simply merging wflow and hosts and exiting # at start of a --list run listExitEarly = False # Check if already got work file system if os.path.isfile(sysfilename): log.info(" previous run: %s" % sysfilename) loadsys = WorkflowSystem("", self.config) loadsys.load(sysfilename, hosts) if wfsys.isEquivalent(loadsys): log.debug("Previous run is equivalent to current files") wfsys = loadsys else: log.error("Workflow and/or hosts file has changed since "\ "previous run, please investigate") return False else: # if --list and there is no pre-existing workflow file then we # generate a listing file and exit straight away if o.list: log.info("There is no pre-existing status file") listExitEarly = True else: log.info(" previous run: N/A") wfsys.write(sysfilename) # if --list then ascertain whether all eligible tasks are in a # complete state if o.list and wfsys.eligibleDisplayTasks(o.servertypes, o.servernames, o.exclude, o.force, o.exact_match): log.info("Display phase has not been completed and therefore " "later phases cannot be predicted") listExitEarly = True if listExitEarly: fnamesuffix = constants.LISTFILE_SUFFIX log.info("\nWorkflow file and hosts file have been merged to " \ "create list file %s%s, equivalent to a master status file" % \ (sysfilename, constants.LISTFILE_SUFFIX)) wfsys.write("%s%s" % (sysfilename, constants.LISTFILE_SUFFIX)) sys.exit(0) wfsys.logqueue = self.logqueue runner = CmdLineWorkflowRunner(wfsys, self.term_size, o) if o.list: log.info("\nListing has been run from the pre-existing master "\ "status file onwards, with predictive output written " "to file %s%s" % (sysfilename, constants.LISTFILE_SUFFIX)) return runner.run() def dynamicAlterations(self, o, hosts, ini): """ does the processing for a dynamic alterations run Arguments: o: the WorkflowOptions object hosts: hosts object ini: the ini file params object Returns: boolean - True if completed ok """ wf = Workflow(self.config) sysfilename = o.getSysStatusName() menuMsgs = [] if not wf.parse(o.wfile): log.info("Error encountered when parsing workflow file") return False try: wfsys = wf.genWorkflowSystem(hosts) except Exception as err: log.error("Failed to generate workflow : %s" % str(err)) log.debug("Exception: %s" % (traceback.format_exc())) return False dynamicMenu = DynamicMenuMgr(self.term_size, self.config, hosts, self.dynamicTaskValidator) while True: log.debug("Starting loop with menuMsgs length %d" % \ len(menuMsgs)) # Check if already got work file system if os.path.isfile(sysfilename): loadsys = WorkflowSystem("", self.config) loadsys.load(sysfilename, hosts) if wfsys.isEquivalent(loadsys): log.debug("Previous run is equivalent to current " "files - operating on status file") wfsys = loadsys else: log.error("Workflow and/or hosts file has " "changed since previous run, please investigate") return False else: log.debug("A new master status file will be created") dyntask = dynamicMenu.getDynamicAlteration(wfsys, menuMsgs) if dyntask == None: log.info("Quitting dynamic alterations") return True elif dyntask == {}: log.debug("Dynamic alteration empty") menuMsgs = [] else: log.debug("Dynamic task input: %s" % dyntask) menuMsgs = [] if not self.processDynamicAlteration(dyntask, wf, wfsys, sysfilename, hosts, menuMsgs): if len(menuMsgs) == 0: menuMsgs.append("Unspecified error during " \ "processing %s, dynamic alteration " \ "abandoned, please check logs" % \ dyntask[constants.DYN_ID]) log.debug("Finished processDynamicAlteration with " "menuMsg %s" % menuMsgs) def processDynamicAlteration(self, dyntask, wf, wfsys, sysfilename, hosts, menuMsgs): """ processes a dynamic alteration (add/remove) for pause/esc Arguments: dyntask: a dictionary containing the details for the alteration wf: the original workflow (for validation in dynamicUpdate) wfsys: the workflowsys sysfilename: the name of the master status file hosts: host object containing details of the hosts in this run menuMsgs: list to hold messages for display during interaction Returns: boolean: True if alteration successful, False if alteration failed """ dyntaskid = dyntask[constants.DYN_ID] if dyntaskid == "" or dyntaskid == None: menuMsgs.append("Dynamic task id is empty") return False dyntype = dyntask[constants.DYN_TYPE] if not dyntype == constants.DYNAMIC_ESCAPE and \ not dyntype == constants.DYNAMIC_PAUSE: menuMsgs.append("Dynamic task type for dynamic task %s must be " "%s or %s - but found %s" % \ (dyntaskid, constants.DYNAMIC_ESCAPE, constants.DYNAMIC_PAUSE, dyntype)) return False if dyntask[constants.DYN_ACTION] == constants.DYNAMIC_ADD: if wfsys.taskInWorkflow(dyntaskid): menuMsgs.append( "Dynamic task id %s duplicates one already in the workflow" \ % dyntaskid) return False if not self.dynamicAlterationAdd(dyntask, wf, wfsys, sysfilename, hosts, menuMsgs): menuMsgs.append("Unable to add dynamic task %s" % \ dyntask[constants.DYN_ID]) return False elif dyntask[constants.DYN_ACTION] == constants.DYNAMIC_REMOVE: # look for dynamic task id in workflow execute phase if wfsys.execute.getTask(dyntaskid) is None: menuMsgs.append("Dynamic %s task id %s not found in workflow " "execute phase" % (dyntype, dyntaskid)) return False if not self.dynamicAlterationRemove(dyntask, wf, wfsys, sysfilename, hosts, menuMsgs): menuMsgs.append("Unable to remove dynamic task %s" % \ dyntask[constants.DYN_ID]) return False else: menuMsgs.append("Unable to determine action %s" % \ dyntask[constants.DYN_ACTION]) return False return True def dynamicAlterationAdd(self, dyntask, wf, wfsys, sysfilename, hosts, menuMsgs): """performs dynamic addition for pause/esc Arguments: dyntask: a dictionary containing the details for the alteration wf: the original workflow (for validation in dynamicUpdate) wfsys: the workflowsys sysfilename: the name of the master status file hosts: host object containing details of the hosts in this run menuMsgs: list to hold messages for display during interaction Returns: boolean: True if alteration successful, False if alteration failed """ # get the execute phase for the workflow executephase = wfsys.getExecutePhase() refid = dyntask[constants.DYN_REFID] taskdict = executephase.getTaskIndices(refid) indRefs = taskdict[constants.TASK_LIST] groupid = taskdict[constants.TASK_IN_GROUP] if len(indRefs) == 0: menuMsgs.append("Unable to find valid reference task %s " "in workflow execute phase" % refid) return False firstRefInd = indRefs[0] lastRefInd = indRefs[len(indRefs) - 1] pos = dyntask[constants.DYN_POS] dyntaskid = dyntask[constants.DYN_ID] dyntype = dyntask[constants.DYN_TYPE] if pos == constants.DYNAMIC_BEFORE: insertionPoint = firstRefInd elif pos == constants.DYNAMIC_AFTER: insertionPoint = lastRefInd + 1 else: menuMsgs.append("Position for dynamic task %s must be %s " "or %s - but found %s" % \ (dyntaskid, constants.DYNAMIC_BEFORE, constants.DYNAMIC_AFTER, pos)) return False log.debug("inserting task %s into execute phase at " "index %s" % (dyntaskid, insertionPoint)) checkparams = {} if dyntask[constants.DYN_CHECKPARAMS] is None: checkParams = None else: chks = dyntask[constants.DYN_CHECKPARAMS].split(',') for chk in chks: chckName, chckVar = chk.split('=') checkparams[chckName] = chckVar # note that for dynamic tasks dynamic and optional will always be true dynamic = True optional = True if dyntype == constants.DYNAMIC_ESCAPE: taskforinsert = EscapeTask(dyntaskid, self.config, dyntask[constants.DYN_MSG], dyntask[constants.DYN_HOSTS], dyntask[constants.DYN_SERVER], dyntask[constants.DYN_SWVER], dyntask[constants.DYN_OSVER], dyntask[constants.DYN_DEP], optional, utils.get_stringtoboolean(dyntask[constants.DYN_DEPSINGLE]), checkparams, groupid, dynamic) else: taskforinsert = PauseTask(dyntaskid, self.config, dyntask[constants.DYN_MSG], dyntask[constants.DYN_HOSTS], dyntask[constants.DYN_SERVER], dyntask[constants.DYN_SWVER], dyntask[constants.DYN_OSVER], dyntask[constants.DYN_DEP], optional, utils.get_stringtoboolean(dyntask[constants.DYN_DEPSINGLE]), checkparams, groupid, dynamic) if groupid == None: if not executephase.insertDynamic(taskforinsert, hosts, insertionPoint, menuMsgs): menuMsgs.append("Failed to insert dynamic task %s " "into execute phase" % \ taskforinsert.name) return False else: if not executephase.insertDynamicIntoGroup(taskforinsert, hosts, indRefs, pos, menuMsgs): menuMsgs.append("Failed to insert dynamic task %s " "into group %s execute phase" % \ (taskforinsert.name, groupid)) return False if not self.dynamicUpdate(wf, wfsys, sysfilename, hosts, menuMsgs): menuMsgs.append("Dynamic addition update for task %s " "failed - please check logs" % \ taskforinsert.name) return False menuMsgs.append("New master status file %s created with " "dynamic entry %s added" % (sysfilename, dyntaskid)) return True def dynamicAlterationRemove(self, dyntask, wf, wfsys, sysfilename, hosts, menuMsgs): """performs dynamic addition for pause/esc Arguments: dyntask: a dictionary containing the details for the alteration wf: the original workflow (for validation in dynamicUpdate) wfsys: the workflowsys sysfilename: the name of the master status file hosts: host object containing details of the hosts in this run menuMsgs: list to hold messages for display during interaction Returns: boolean: True if alteration successful, False if alteration failed """ # get the execute phase for the workflow executephase = wfsys.getExecutePhase() dyntaskid = dyntask[constants.DYN_ID] dyntype = dyntask[constants.DYN_TYPE] if self.dynamicTaskValidator.hasDependents(dyntaskid, wfsys, menuMsgs): menuMsgs.append("Dynamic removal failed validation") return False if not executephase.removeDynamic(dyntaskid, dyntype, menuMsgs): menuMsgs.append("Failed to remove dynamic task %s " "from execute phase, please check logs" % \ dyntaskid) return False if not self.dynamicUpdate(wf, wfsys, sysfilename, hosts, menuMsgs): menuMsgs.append("Dynamic removal update for task %s failed " "- please check logs" % \ dyntaskid) return False menuMsgs.append("New master status file %s created with dynamic " "entry %s removed" % (sysfilename, dyntaskid)) return True def dynamicUpdate(self, wf, wfsys, sysfilename, hosts, menuMsgs): """ updates the master status file - firstly as a tmp file, then reads in and validates the tmp file and if ok renames to master status file Arguments: wf: the original workflow (for validation) wfsys: the workflowsys sysfilename: name of the master status file hosts: the hosts object menuMsgs: list to hold messages for display during interaction Returns: True if master valid master status file was produced, False if errors were encountered """ # now update the workflow file as tmp file tmpsysfile = "%s_tmp" % sysfilename wfsys.write(tmpsysfile) log.debug("New tmp status file %s written" % tmpsysfile) # now attempt to reload the tmp file - which will validate against xsd loadtmp = WorkflowSystem("", self.config) if not loadtmp.load(tmpsysfile, hosts): menuMsgs.append("Failed to reload new master status file. " \ "Investigate the problem, then remove invalid " \ "temporary status file %s," \ " and rerun the dynamic alteration" % tmpsysfile) return False # now perform equivalency test against original workflow wfcheck = wf.genWorkflowSystem(hosts) if not wfcheck.isEquivalent(loadtmp): menuMsgs.append("New master status file does not match original" \ " workflow. " \ "Investigate the problem, then remove invalid " \ "temporary status file %s," \ " and rerun the dynamic alteration" % tmpsysfile) return False # if validated ok then rename the tmp file as the new master status # file # write backup file before updating the status file tstamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") backupdir = "dynamic_alteration_backups" if '/' in sysfilename: sysfilesplit = sysfilename.rsplit('/', 1) basedir = sysfilesplit[0] basename = sysfilesplit[1] else: basedir = "./" basename = sysfilename timestampedBackup = "%s/%s/%s.%s" % \ (basedir, backupdir, basename, tstamp) # retrieve full path of backup dir backupdir = timestampedBackup.rsplit('/', 1)[0] if not os.path.exists(backupdir): log.debug("creating backup directory %s" % backupdir) os.mkdir(backupdir) wfsys.write(timestampedBackup) log.debug("Timestamped backup %s created" % timestampedBackup) log.debug("Renaming temp status file %s as master status file %s" % \ (tmpsysfile, sysfilename)) try: os.rename(tmpsysfile, sysfilename) except: menuMsgs.append("Error renaming temporary file %s to master " "status file %s" % (tmpsysfile, sysfilename)) return False return True