def update_task_states_recursively(task):
task_start = 0
task_end = 0
for j in task.jobs:
if isjob(j):
start, end = update_job_status(j)
else:
start, end = update_task_states_recursively(j)
task_start = min(task_start, start) if task_start > 0 else start
task_end = max(task_end, end)
db.update_task(task.taskid, status=task.status, tm_start=task_start, tm_end=task_end)
return task_start, task_end
def launch_jobs(jobs, conf):
# Group jobs with identical config
sge_path = GLOBALS["sge_dir"]
conf2jobs = defaultdict(list)
for j, cmd in jobs:
job_config = conf["sge"].copy()
job_config["-pe smp"] = j.cores
for k,v in j.sge.iteritems():
job_config[k] = v
conf_key = tuple(sorted(job_config.items()))
conf2jobs[conf_key].append((j,cmd))
for job_config, commands in conf2jobs.iteritems():
job_config = dict(job_config)
job_file = "%s_%d_jobs" %(time.ctime().replace(" ", "_").replace(":","-"),
len(commands))
cmds_file = os.path.join(sge_path, job_file+".cmds")
qsub_file = os.path.join(sge_path, job_file+".qsub")
script = '''#!/bin/sh\n'''
for k,v in job_config.iteritems():
if not k.startswith("_"):
script += '#$ %s %s\n' %(k,v)
script += '#$ -f \n'
script += '#$ -o %s\n' % sge_path
script += '#$ -e %s\n' % sge_path
script += '#$ -N %s\n' % "NPR%djobs" %len(commands)
script += '#$ -t 1-%d\n' % len(commands)
script += 'SEEDFILE=%s\n' % cmds_file
script += 'sh -c "`cat $SEEDFILE | head -n $SGE_TASK_ID | tail -n 1`" \n'
open(cmds_file, "w").write('\n'.join([cmd for j,cmd in commands]))
open(qsub_file, "w").write(script)
log.log(28, "Launching %d SGE jobs." %len(commands))
log.debug(script)
answer = run("SGE_CELL=%s qsub %s" %(job_config["_cell"], qsub_file))
log.debug(answer)
match = re.search(OK_PATTERN, answer)
if match:
jobid = match.groups()[0]
for j, cmd in commands:
db.update_task(j.jobid, status = "Q", host="@sge", pid=jobid)
else:
raise SgeError(answer)
# store task data
task.store_data(db)
for j in task.jobs:
if isjob(j):
pass
else:
store_task_data_recursively(j)
def remove_task_dir_recursively(task):
# store task data
for j in task.jobs:
if isjob(j):
shutil.rmtree(j.jobdir)
else:
remove_task_dir_recursively(j)
shutil.rmtree(task.taskdir)
def update_job_status(j):
start = None
end = None
if j.status == "D":
try:
start, end = read_time_file(j.time_file)
except Exception, e:
log.warning("Execution time could not be loaded into DB: %s", j.jobid[:6])
log.warning(e)
db.update_task(j.jobid, status=j.status, tm_start=start, tm_end=end)
return start, end