def check(logfile, job_id, cluster_job_name): # todo: this needs to be updated for array jobs errors = [] joblist = logfile.readFromLogfile() jobIsRunning = query(logfile, job_id) if not joblist.get(job_id): if not jobIsRunning: errors.append("Job %d is not running but also has no entry in the logfile %s." % (job_id, logfile.getName())) else: errors.append("Job %d is running but has no entry in the logfile %s." % (job_id, logfile.getName())) else: cname = cluster_job_name dir = joblist[job_id]["Directory"] if not jobIsRunning: outputfile = os.path.join(dir, "%(cname)s.o%(job_id)d" % vars()) if os.path.exists(outputfile): F = open(outputfile, "r") contents = F.read() F.close() success = re.compile('''Done!\s*</output>\s*<enddate>(.*?)</enddate>\s*</make_fragments>\s*$''', re.DOTALL) match = success.search(contents) if match: colorprinter.message("Job %d finished successfully on %s. Results are in %s." % (job_id, match.groups(1)[0].strip(), dir)) else: errors.append("Job %d has finished running but was not successful. Results are in %s." % (job_id, dir)) errcode = ERRCODE_JOBFAILED else: errors.append("The output file %s associated with job %d could not be found. Searched in %s." % (outputfile, job_id, dir)) errcode = ERRCODE_NOOUTPUT else: colorprinter.warning("Job %d is still running. Results are being stored in %s." % (job_id, dir)) return errors
def query(logfile, jobID = None): """If jobID is an integer then return False if the job has finished and True if it is still running. Otherwise, returns a table of jobs run by the user.""" joblist = logfile.readFromLogfile() if jobID and type(jobID) == type(1): command = ['qstat', '-j', str(jobID)] else: command = ['qstat'] processoutput = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() output = processoutput[0] serror = processoutput[1] # Form command jobs = {} if type(jobID) == type(1): if serror.find("Following jobs do not exist") != -1: return False else: return True if not output.strip(): colorprinter.message("No jobs running at present.") output = output.strip().split("\n") if len(output) > 2: for line in output[2:]: # We assume that our script names contain no spaces for the parsing below to work tokens = line.split() jid = int(tokens[0]) jobstate = tokens[4] details = { "jobid" : jid, "prior" : tokens[1], "name" : tokens[2], "user" : tokens[3], "state" : jobstate, "submit/start at" : "%s %s" % (tokens[5], tokens[6]) } jataskID = 0 if jobstate == "r": details["queue"] = tokens[7] details["slots"] = tokens[8] elif jobstate == "qw": details["slots"] = tokens[7] if len(tokens) >= 9: jataskID = tokens[8] details["ja-task-ID"] = jataskID if len(tokens) > 9: jataskID = tokens[9] details["ja-task-ID"] = jataskID jobs[jid] = jobs.get(jid) or {} jobs[jid][jataskID] = details if joblist.get(jid): jobdir = joblist[jid]["Directory"] jobtime = joblist[jid]["TimeInSeconds"] colorprinter.message("Job %d submitted %d minutes ago. Status: '%s'. Destination directory: %s." % (jid, jobtime / 60, jobstate, jobdir)) else: colorprinter.message("Job %d submitted at %s %s. Status: '%s'. Destination directory unknown." % (jid, tokens[5], tokens[6], jobstate)) return True