def submitJob(input,
model,
dummy=False,
saveWorkDirs=False,
rerun=False,
skipTags=[".QUEUED"],
hideFinished=False,
eventTag=None,
program=None):
# Check for output files
if not (rerun or getSkip(
input, skipTags)): # this input has already been submitted
if not hideFinished:
print >> sys.stderr, "Processing", input
print >> sys.stderr, "input already processed"
return False
print >> sys.stderr, "Processing", input
# Check if the job is still running
queuedAndRunningJobNames = getJobs()
stemDir, finalDir = os.path.normpath(input).rsplit("/", 1)
if finalDir in queuedAndRunningJobNames:
print >> sys.stderr, "Job", finalDir, "is currently queued/running"
return False
# We can queue the job
if program != None:
args = ["python"] + program.split() + ["-i", input]
else:
args = ["python", "PredictTask.py", "-i", input]
if model != None:
args += ["-m", model]
if saveWorkDirs:
args += ["-w", input + ".workdir"]
if eventTag != None:
args += ["--eventTag", eventTag]
script = MakeJobScript.makeJob(finalDir, "48:00", stemDir,
args) # make job script
# script += "\nrm " + input + ".QUEUED" # remove submission marker at the end of the job
if not dummy:
print >> sys.stderr, "Submitting job"
if MakeJobScript.isVuori():
#f = open(input + ".jobscript", "wt")
#f.write(script)
#f.close
#subprocess.call("sbatch", shell=True, stdin=script) # submit job
p = Popen(['sbatch'], stdout=PIPE, stdin=PIPE, stderr=STDOUT)
pstdout = p.communicate(input=script)[0]
print(pstdout)
else: # assume Murska
#subprocess.call(script + " | bsub", shell=True) # submit job
p = Popen(['bsub'], stdout=PIPE, stdin=PIPE, stderr=STDOUT)
pstdout = p.communicate(input=script)[0]
print(pstdout)
open(input + ".QUEUED", "w").close() # mark job as submitted
else:
print >> sys.stderr, "Dummy mode"
#print >> sys.stderr, "Script", finalDir
#print >> sys.stderr, script
return True
def numJobs(username="******"):
"""
Get number of queued (pending or running) jobs
"""
if MakeJobScript.isVuori():
p = subprocess.Popen("squeue | grep " + username + " | wc -l", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
else:
p = subprocess.Popen("bjobs | grep " + username + " | wc -l", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
lines = p.stdout.readlines()
assert len(lines) == 1
return int(lines[0])
def submitJob(input, model, dummy=False, saveWorkDirs=False, rerun=False, skipTags=[".QUEUED"], hideFinished=False, eventTag=None, program=None):
# Check for output files
if not (rerun or getSkip(input, skipTags)): # this input has already been submitted
if not hideFinished:
print >> sys.stderr, "Processing", input
print >> sys.stderr, "input already processed"
return False
print >> sys.stderr, "Processing", input
# Check if the job is still running
queuedAndRunningJobNames = getJobs()
stemDir, finalDir = os.path.normpath(input).rsplit("/", 1)
if finalDir in queuedAndRunningJobNames:
print >> sys.stderr, "Job", finalDir, "is currently queued/running"
return False
# We can queue the job
if program != None:
args = ["python"] + program.split() + ["-i", input]
else:
args = ["python", "PredictTask.py", "-i", input]
if model != None:
args += ["-m", model]
if saveWorkDirs:
args += ["-w", input + ".workdir"]
if eventTag != None:
args += ["--eventTag", eventTag]
script = MakeJobScript.makeJob(finalDir, "48:00", stemDir, args) # make job script
# script += "\nrm " + input + ".QUEUED" # remove submission marker at the end of the job
if not dummy:
print >> sys.stderr, "Submitting job"
if MakeJobScript.isVuori():
#f = open(input + ".jobscript", "wt")
#f.write(script)
#f.close
#subprocess.call("sbatch", shell=True, stdin=script) # submit job
p = Popen(['sbatch'], stdout=PIPE, stdin=PIPE, stderr=STDOUT)
pstdout = p.communicate(input=script)[0]
print(pstdout)
else: # assume Murska
#subprocess.call(script + " | bsub", shell=True) # submit job
p = Popen(['bsub'], stdout=PIPE, stdin=PIPE, stderr=STDOUT)
pstdout = p.communicate(input=script)[0]
print(pstdout)
open(input + ".QUEUED", "w").close() # mark job as submitted
else:
print >> sys.stderr, "Dummy mode"
#print >> sys.stderr, "Script", finalDir
#print >> sys.stderr, script
return True
def numJobs(username="******"):
"""
Get number of queued (pending or running) jobs
"""
if MakeJobScript.isVuori():
p = subprocess.Popen("squeue | grep " + username + " | wc -l",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
p = subprocess.Popen("bjobs | grep " + username + " | wc -l",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
lines = p.stdout.readlines()
assert len(lines) == 1
return int(lines[0])