def check(logfile, job_id, cluster_job_name):
# todo: this needs to be updated for array jobs
errors = []
joblist = logfile.readFromLogfile()
jobIsRunning = query(logfile, job_id)
if not joblist.get(job_id):
if not jobIsRunning:
errors.append("Job %d is not running but also has no entry in the logfile %s." % (job_id, logfile.getName()))
else:
errors.append("Job %d is running but has no entry in the logfile %s." % (job_id, logfile.getName()))
else:
cname = cluster_job_name
dir = joblist[job_id]["Directory"]
if not jobIsRunning:
outputfile = os.path.join(dir, "%(cname)s.o%(job_id)d" % vars())
if os.path.exists(outputfile):
F = open(outputfile, "r")
contents = F.read()
F.close()
success = re.compile('''Done!\s*</output>\s*<enddate>(.*?)</enddate>\s*</make_fragments>\s*$''', re.DOTALL)
match = success.search(contents)
if match:
colorprinter.message("Job %d finished successfully on %s. Results are in %s." % (job_id, match.groups(1)[0].strip(), dir))
else:
errors.append("Job %d has finished running but was not successful. Results are in %s." % (job_id, dir))
errcode = ERRCODE_JOBFAILED
else:
errors.append("The output file %s associated with job %d could not be found. Searched in %s." % (outputfile, job_id, dir))
errcode = ERRCODE_NOOUTPUT
else:
colorprinter.warning("Job %d is still running. Results are being stored in %s." % (job_id, dir))
return errors
def query(logfile, jobID = None):
"""If jobID is an integer then return False if the job has finished and True if it is still running.
Otherwise, returns a table of jobs run by the user."""
joblist = logfile.readFromLogfile()
if jobID and type(jobID) == type(1):
command = ['qstat', '-j', str(jobID)]
else:
command = ['qstat']
processoutput = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
output = processoutput[0]
serror = processoutput[1]
# Form command
jobs = {}
if type(jobID) == type(1):
if serror.find("Following jobs do not exist") != -1:
return False
else:
return True
if not output.strip():
colorprinter.message("No jobs running at present.")
output = output.strip().split("\n")
if len(output) > 2:
for line in output[2:]:
# We assume that our script names contain no spaces for the parsing below to work
tokens = line.split()
jid = int(tokens[0])
jobstate = tokens[4]
details = { "jobid" : jid,
"prior" : tokens[1],
"name" : tokens[2],
"user" : tokens[3],
"state" : jobstate,
"submit/start at" : "%s %s" % (tokens[5], tokens[6])
}
jataskID = 0
if jobstate == "r":
details["queue"] = tokens[7]
details["slots"] = tokens[8]
elif jobstate == "qw":
details["slots"] = tokens[7]
if len(tokens) >= 9:
jataskID = tokens[8]
details["ja-task-ID"] = jataskID
if len(tokens) > 9:
jataskID = tokens[9]
details["ja-task-ID"] = jataskID
jobs[jid] = jobs.get(jid) or {}
jobs[jid][jataskID] = details
if joblist.get(jid):
jobdir = joblist[jid]["Directory"]
jobtime = joblist[jid]["TimeInSeconds"]
colorprinter.message("Job %d submitted %d minutes ago. Status: '%s'. Destination directory: %s." % (jid, jobtime / 60, jobstate, jobdir))
else:
colorprinter.message("Job %d submitted at %s %s. Status: '%s'. Destination directory unknown." % (jid, tokens[5], tokens[6], jobstate))
return True
