def issueJobs(self, jobCommands): """Issues parasol with job commands. """ issuedJobs = {} for jobCommand, memory, cpu, logFile in jobCommands: assert memory != None assert cpu != None assert logFile != None pattern = re.compile("your job ([0-9]+).*") command = "parasol -verbose -ram=%i -cpu=%i -results=%s add job '%s'" % (memory, cpu, self.parasolResultsFile, jobCommand) while True: #time.sleep(0.1) #Sleep to let parasol catch up #Apparently unnecessary popenParasolCommand(command, self.scratchFile) fileHandle = open(self.scratchFile, 'r') line = fileHandle.readline() fileHandle.close() match = pattern.match(line) if match != None: #This is because parasol add job will return success, even if the job was not properly issued! break else: logger.info("We failed to properly add the job, we will try again after a sleep") time.sleep(5) jobID = int(match.group(1)) logger.debug("Got the job id: %s from line: %s" % (jobID, line)) assert jobID not in issuedJobs.keys() issuedJobs[jobID] = jobCommand logger.debug("Issued the job command: %s with job id: %i " % (command, jobID)) return issuedJobs
def getUpdatedJobs(self): """We use the parasol results to update the status of jobs, adding them to the list of updated jobs. Results have the following structure.. (thanks Mark D!) int status; /* Job status - wait() return format. 0 is good. */ char *host; /* Machine job ran on. */ char *jobId; /* Job queuing system job ID */ char *exe; /* Job executable file (no path) */ int usrTicks; /* 'User' CPU time in ticks. */ int sysTicks; /* 'System' CPU time in ticks. */ unsigned submitTime; /* Job submission time in seconds since 1/1/1970 */ unsigned startTime; /* Job start time in seconds since 1/1/1970 */ unsigned endTime; /* Job end time in seconds since 1/1/1970 */ char *user; /* User who ran job */ char *errFile; /* Location of stderr file on host */ plus you finally have the command name.. """ line = self.parasolResultsFileHandle.readline() updatedJobs = {} while line != '': results = line.split() if line[-1] == '\n': line = line[:-1] logger.debug("Parasol completed a job, this is what we got: %s" % line) result = int(results[0]) jobID = int(results[2]) updatedJobs[jobID] = result line = self.parasolResultsFileHandle.readline() return updatedJobs
def addJob(self, command, sgeJobID, issuedJobs, index=None): jobID = self.nextJobID self.nextJobID += 1 self.jobIDs[(sgeJobID, index)] = jobID self.sgeJobIDs[jobID] = (sgeJobID, index) assert jobID not in issuedJobs.keys() issuedJobs[jobID] = command logger.debug("Issued the job command: %s with job id: %s " % (command, str(jobID))) self.currentjobs.add(jobID) self.newJobsQueue.put((sgeJobID, index))
def prepareQsub(cpu, mem): qsubline = list(["qsub","-b","y","-terse","-j" ,"y", "-cwd","-v", "LD_LIBRARY_PATH=%s" % os.environ["LD_LIBRARY_PATH"]]) reqline = list() if cpu is not None: reqline.append("p="+str(cpu)) if mem is not None: reqline.append("vf="+str(mem/ 1024)+"K") if len(reqline) > 0: qsubline.extend(["-hard","-l", ",".join(reqline)]) logger.debug("**"+" ".join(qsubline)) return qsubline
def issueJobs(self, commands): """Runs the jobs right away. """ issuedJobs = {} for command, memory, cpu, logFile in commands: #Add the commands to the queue assert memory != None assert cpu != None assert logFile != None logger.debug("Issuing the command: %s with memory: %i, cpu: %i" % (command, memory, cpu)) self.jobs[self.jobIndex] = command issuedJobs[self.jobIndex] = command self.inputQueue.put((command, logFile, self.jobIndex)) self.jobIndex += 1 return issuedJobs
def getUpdatedJobs(self): """Returns a map of the run jobs and the return value of their processes. """ runJobs = {} try: while True: command, exitValue, jobID = self.outputQueue.get_nowait() runJobs[jobID] = exitValue self.jobs.pop(jobID) logger.debug("Ran the command: %s with exit value: %i" % (command, exitValue)) self.outputQueue.task_done() except Empty: pass return runJobs
def issueJobs(jobs, jobIDsToJobsHash, batchSystem): """Issues jobs to the batch system. """ jobCommands = {} for job in jobs: jobCommand = os.path.join(workflowRootPath(), "bin", "jobTreeSlave") followOnJob = job.find("followOns").findall("followOn")[-1] jobCommands["%s -E %s %s --job %s" % (sys.executable, jobCommand, os.path.split(workflowRootPath())[0], job.attrib["file"])] = (job.attrib["file"], int(followOnJob.attrib["memory"]), int(followOnJob.attrib["cpu"]), job.attrib["slave_log_file"]) issuedJobs = batchSystem.issueJobs([ (key, jobCommands[key][1], jobCommands[key][2], jobCommands[key][3]) for key in jobCommands.keys() ]) assert len(issuedJobs.keys()) == len(jobCommands.keys()) for jobID in issuedJobs.keys(): command = issuedJobs[jobID] jobFile = jobCommands[command][0] assert jobID not in jobIDsToJobsHash jobIDsToJobsHash[jobID] = jobFile logger.debug("Issued the job: %s with job id: %i " % (jobFile, jobID))
def qsub(qsubline): logger.debug("**"+" ".join(qsubline)) process = subprocess.Popen(qsubline, stdout=subprocess.PIPE) result = int(process.stdout.readline().strip().split('.')[0]) logger.debug("Got the job id: %s" % (str(result))) return result
def mainLoop(config, batchSystem): """This is the main loop from which jobs are issued and processed. """ waitDuration = float(config.attrib["wait_duration"]) assert waitDuration >= 0 rescueJobsFrequency = float(config.attrib["rescue_jobs_frequency"]) maxJobDuration = float(config.attrib["max_job_duration"]) assert maxJobDuration >= 0 logger.info("Got parameters, wait duration %s, rescue jobs frequency: %s max job duration: %s" % \ (waitDuration, rescueJobsFrequency, maxJobDuration)) #Kill any jobs on the batch system queue from the last time. assert len(batchSystem.getIssuedJobIDs()) == 0 #Batch system must start with no active jobs! logger.info("Checked batch system has no running jobs and no updated jobs") jobFiles = config.attrib["job_file_dir"].listFiles() logger.info("Got a list of job files") #Repair the job tree using any .old files fixJobsList(config, jobFiles) logger.info("Fixed the job files using any .old files") #Get jobs that were running, or that had failed reset to 'white' status restartFailedJobs(config, jobFiles) logger.info("Reworked failed jobs") updatedJobFiles = set() #Jobs whose status needs updating, either because they have finished, or because they need to be started. for jobFile in jobFiles: job = ET.parse(jobFile).getroot() if job.attrib["colour"] not in ("grey", "blue"): updatedJobFiles.add(jobFile) logger.info("Got the active (non grey/blue) job files") totalJobFiles = len(jobFiles) #Total number of job files we have. jobIDsToJobsHash = {} #A hash of the currently running jobs ids, made by the batch system. idealJobTime = float(config.attrib["job_time"]) assert idealJobTime > 0.0 maxIssuedJobs = int(config.attrib["max_jobs"]) #The maximum number of jobs to issue to the batch system assert maxIssuedJobs >= 1 stats = config.attrib.has_key("stats") if stats: startTime = time.time() startClock = getTotalCpuTime() logger.info("Starting the main loop") timeSinceJobsLastRescued = time.time() - rescueJobsFrequency + 100 #We hack it so that we rescue jobs after the first 100 seconds to get around an apparent parasol bug while True: if len(updatedJobFiles) > 0: logger.debug("Built the jobs list, currently have %i job files, %i jobs to update and %i jobs currently issued" % (totalJobFiles, len(updatedJobFiles), len(jobIDsToJobsHash))) jobsToIssue = [] for jobFile in list(updatedJobFiles): job = ET.parse(jobFile).getroot() assert job.attrib["colour"] not in ("grey", "blue") if job.attrib["colour"] == "white": #Get ready to start the job if len(jobIDsToJobsHash) < maxIssuedJobs: logger.debug("Job: %s is being started" % job.attrib["file"]) updatedJobFiles.remove(job.attrib["file"]) #Reset the log files for the job. open(job.attrib["slave_log_file"], 'w').close() open(job.attrib["log_file"], 'w').close() job.attrib["colour"] = "grey" #writeJobs([ job ]) #Check point, do this before issuing job, so state is not read until issued #issueJobs([ job ], jobIDsToJobsHash, batchSystem) jobsToIssue.append(job) else: logger.debug("Job: %s is not being issued yet because we have %i jobs issued" % (job.attrib["file"], len(jobIDsToJobsHash))) elif job.attrib["colour"] == "black": #Job has finished okay logger.debug("Job: %s has finished okay" % job.attrib["file"]) #Deal with stats if stats: system("cat %s >> %s" % (job.attrib["stats"], config.attrib["stats"])) open(job.attrib["stats"], 'w').close() #Reset the stats file childCount = int(job.attrib["child_count"]) blackChildCount = int(job.attrib["black_child_count"]) assert childCount == blackChildCount #Has no currently running child jobs #Launch any unborn children unbornChildren = job.find("children") unbornChild = unbornChildren.find("child") if unbornChild != None: #We must give birth to the unborn children logger.debug("Job: %s has children to schedule" % job.attrib["file"]) newChildren = [] while unbornChild != None: cummulativeChildTime = float(unbornChild.attrib["time"]) newJob = createJob(unbornChild.attrib.copy(), job.attrib["file"], config) totalJobFiles += 1 updatedJobFiles.add(newJob.attrib["file"]) newChildren.append(newJob) unbornChildren.remove(unbornChild) unbornChild = unbornChildren.find("child") #This was code to aggregate groups of children into one job, but we don't do this now #while cummulativeChildTime < idealJobTime and unbornChild != None: #Cummulate a series of children into each job as a stack of jobs (to balance cost of parellelism with cost of running stuff in serially). # cummulativeChildTime += float(unbornChild.attrib["time"]) # ET.SubElement(newJob.find("followOns"), "followOn", unbornChild.attrib.copy()) # unbornChildren.remove(unbornChild) # unbornChild = unbornChildren.find("child") newJob.attrib["total_time"] = str(cummulativeChildTime) updatedJobFiles.remove(job.attrib["file"]) job.attrib["child_count"] = str(childCount + len(newChildren)) job.attrib["colour"] = "blue" #Blue - has children running. writeJobs([ job ] + newChildren ) #Check point elif len(job.find("followOns").findall("followOn")) != 0: #Has another job logger.debug("Job: %s has a new command that we can now issue" % job.attrib["file"]) ##Reset the job run info job.attrib["remaining_retry_count"] = config.attrib["retry_count"] job.attrib["colour"] = "white" ##End resetting the job writeJobs([ job ]) else: #Job has finished, so we can defer to any parent logger.debug("Job: %s is now dead" % job.attrib["file"]) job.attrib["colour"] = "dead" if job.attrib.has_key("parent"): parent = ET.parse(job.attrib["parent"]).getroot() assert parent.attrib["colour"] == "blue" assert int(parent.attrib["black_child_count"]) < int(parent.attrib["child_count"]) parent.attrib["black_child_count"] = str(int(parent.attrib["black_child_count"]) + 1) if int(parent.attrib["child_count"]) == int(parent.attrib["black_child_count"]): parent.attrib["colour"] = "black" assert parent.attrib["file"] not in updatedJobFiles updatedJobFiles.add(parent.attrib["file"]) writeJobs([ job, parent ]) #Check point else: writeJobs([ job ]) elif job.attrib["colour"] == "red": #Job failed logger.critical("Job: %s failed" % job.attrib["file"]) logger.critical("The log file of the failed job") logFile(job.attrib["log_file"], logger.critical) logger.critical("The log file of the slave for the failed job") logFile(job.attrib["slave_log_file"], logger.critical) #We log the job log file in the main loop #Checks assert len(job.find("children").findall("child")) == 0 assert int(job.attrib["child_count"]) == int(job.attrib["black_child_count"]) remainingRetyCount = int(job.attrib["remaining_retry_count"]) if remainingRetyCount > 0: #Give it another try, maybe there is a bad node somewhere job.attrib["remaining_retry_count"] = str(remainingRetyCount-1) job.attrib["colour"] = "white" logger.critical("Job: %s will be restarted, it has %s goes left" % (job.attrib["file"], job.attrib["remaining_retry_count"])) writeJobs([ job ]) #Check point else: assert remainingRetyCount == 0 updatedJobFiles.remove(job.attrib["file"]) logger.critical("Job: %s is completely failed" % job.attrib["file"]) else: logger.debug("Job: %s is already dead, we'll get rid of it" % job.attrib["file"]) assert job.attrib["colour"] == "dead" updatedJobFiles.remove(job.attrib["file"]) totalJobFiles -= 1 deleteJob(job, config) #This could be done earlier, but I like it this way. ###End of for loop writeJobs(jobsToIssue) #Check point, do this before issuing job, so state is not read until issued issueJobs(jobsToIssue, jobIDsToJobsHash, batchSystem) if len(jobIDsToJobsHash) == 0 and len(updatedJobFiles) == 0: logger.info("Only failed jobs and their dependents (%i total) are remaining, so exiting." % totalJobFiles) break if len(updatedJobFiles) > 0: updatedJobs = batchSystem.getUpdatedJobs() #Asks the batch system what jobs have been completed. else: updatedJobs = pauseForUpdatedJobs(batchSystem.getUpdatedJobs) #Asks the batch system what jobs have been completed. for jobID in updatedJobs.keys(): #Runs through a map of updated jobs and there status, result = updatedJobs[jobID] if jobIDsToJobsHash.has_key(jobID): if result == 0: logger.debug("Batch system is reporting that the job %s ended sucessfully" % jobIDsToJobsHash[jobID]) else: logger.critical("Batch system is reporting that the job %s failed with exit value %i" % (jobIDsToJobsHash[jobID], result)) processFinishedJob(jobID, result, updatedJobFiles, jobIDsToJobsHash) else: logger.info("A result seems to already have been processed: %i" % jobID) #T if time.time() - timeSinceJobsLastRescued >= rescueJobsFrequency: #We only rescue jobs every N seconds reissueOverLongJobs(updatedJobFiles, jobIDsToJobsHash, config, batchSystem) logger.info("Reissued any over long jobs") reissueMissingJobs(updatedJobFiles, jobIDsToJobsHash, batchSystem) logger.info("Rescued any (long) missing jobs") timeSinceJobsLastRescued = time.time() #Going to sleep to let the job system catch up. time.sleep(waitDuration) if stats: fileHandle = open(config.attrib["stats"], 'a') fileHandle.write("<total_time time='%s' clock='%s'/></stats>" % (str(time.time() - startTime), str(getTotalCpuTime() - startClock))) fileHandle.close() logger.info("Finished the main loop") return totalJobFiles #Returns number of failed jobs
def processFinishedJob(jobID, resultStatus, updatedJobFiles, jobIDsToJobsHash): """Function reads a processed job file and updates it state. """ assert jobID in jobIDsToJobsHash jobFile = jobIDsToJobsHash.pop(jobID) updatingFileIsPresent = os.path.isfile(jobFile + ".updating") newFileIsPresent = os.path.isfile(jobFile + ".new") if resultStatus == 0 and updatingFileIsPresent: logger.critical("Despite the batch system claiming success there is a .updating file present: %s", jobFile + ".updating") if resultStatus == 0 and newFileIsPresent: logger.critical("Despite the batch system claiming success there is a .new file present: %s", jobFile + ".new") if resultStatus != 0 or newFileIsPresent or updatingFileIsPresent: #Job not successful according to batchsystem, or according to the existance of a .new or .updating file if updatingFileIsPresent: #The job failed while attempting to write the job file. logger.critical("There was an .updating file for the crashed job: %s" % jobFile) if os.path.isfile(jobFile + ".new"): #The job failed while writing the updated job file. logger.critical("There was an .new file for the crashed job: %s" % jobFile) os.remove(jobFile + ".new") #The existance of the .updating file means it wasn't complete os.remove(jobFile + ".updating") #Delete second the updating file second to preserve a correct state assert os.path.isfile(jobFile) job = ET.parse(jobFile).getroot() #The original must still be there. assert job.find("children").find("child") == None #The original can not reflect the end state of the job. assert int(job.attrib["black_child_count"]) == int(job.attrib["child_count"]) job.attrib["colour"] = "red" #It failed, so we mark it so and continue. writeJobs([ job ]) logger.critical("We've reverted to the original job file and marked it as failed: %s" % jobFile) else: if newFileIsPresent: #The job was not properly updated before crashing logger.critical("There is a valid .new file %s" % jobFile) if os.path.isfile(jobFile): os.remove(jobFile) os.rename(jobFile + ".new", jobFile) job = ET.parse(jobFile).getroot() if job.attrib["colour"] == "grey": #The job failed while preparing to run another job on the slave assert job.find("children").find("child") == None #File job.attrib["colour"] = "red" writeJobs([ job ]) assert job.attrib["colour"] in ("black", "red") else: logger.critical("There was no valid .new file %s" % jobFile) assert os.path.isfile(jobFile) job = ET.parse(jobFile).getroot() #The job may have failed before or after creating this file, we check the state. if job.attrib["colour"] == "black": #The job completed okay, so we'll keep it logger.critical("Despite the batch system job failing, the job appears to have completed okay") else: assert job.attrib["colour"] in ("grey", "red") assert job.find("children").find("child") == None #File assert int(job.attrib["black_child_count"]) == int(job.attrib["child_count"]) if job.attrib["colour"] == "grey": job.attrib["colour"] = "red" writeJobs([ job ]) logger.critical("We've reverted to the original job file and marked it as failed: %s" % jobFile) else: job = ET.parse(jobFile).getroot() ##Now check the log files exist, because they must ultimately be cleaned up by their respective file trees. if not os.path.isfile(job.attrib["log_file"]): #We need to keep these files in existence. open(job.attrib["log_file"], 'w').close() logger.critical("The log file %s for job %s had disappeared" % (job.attrib["log_file"], jobFile)) if not os.path.isfile(job.attrib["slave_log_file"]): open(job.attrib["slave_log_file"], 'w').close() logger.critical("The slave log file %s for job %s had disappeared" % (job.attrib["slave_log_file"], jobFile)) assert jobFile not in updatedJobFiles updatedJobFiles.add(jobFile) #Now we know the job is done we can add it to the list of updated job files logger.debug("Added job: %s to active jobs" % jobFile)