def main(opts, args):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
allJobs = []
for task in taskDirs:
if not os.path.exists(task):
print "%s: Task directory missing" % task
continue
jobs = multicrab.crabStatusToJobs(task)
if not "Created" in jobs:
print "%s: no 'Created' jobs to submit" % task
continue
allJobs.extend(filter(lambda j: isInRange(opts, j), jobs["Created"]))
maxJobs = len(allJobs)
if opts.maxJobs >= 0 and int(opts.maxJobs) < int(maxJobs):
maxJobs = opts.maxJobs
njobsSubmitted = 0
while njobsSubmitted < maxJobs:
njobsToSubmit = min(opts.jobs, maxJobs - njobsSubmitted, len(allJobs))
njobsSubmitted += njobsToSubmit
jobsToSubmit = {}
for n in xrange(0, njobsToSubmit):
job = allJobs.pop(0)
multicrab._addToDictList(jobsToSubmit, job.task, job.id)
for task, jobs in jobsToSubmit.iteritems():
pretty = multicrab.prettyJobnums(jobs)
command = ["crab", "-c", task, "-submit", pretty] + args
print "Submitting %d jobs from task %s" % (len(jobs), task)
print "Command", " ".join(command)
if not opts.test:
ret = subprocess.call(command)
if ret != 0:
message = "Command '%s' failed with exit code %d" % (
" ".join(command), ret)
if opts.allowFails:
print message
else:
raise Exception()
if njobsSubmitted < maxJobs:
print "Submitted, sleeping %f seconds" % opts.sleep
time.sleep(opts.sleep)
else:
print "Submitted"
return 0
def main(opts, args):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
allJobs = []
for task in taskDirs:
if not os.path.exists(task):
print "%s: Task directory missing" % task
continue
jobs = multicrab.crabStatusToJobs(task)
if not "Created" in jobs:
print "%s: no 'Created' jobs to submit" % task
continue
allJobs.extend(filter(lambda j: isInRange(opts, j), jobs["Created"]))
maxJobs = len(allJobs)
if opts.maxJobs >= 0 and int(opts.maxJobs) < int(maxJobs):
maxJobs = opts.maxJobs
njobsSubmitted = 0
while njobsSubmitted < maxJobs:
njobsToSubmit = min(opts.jobs, maxJobs-njobsSubmitted, len(allJobs))
njobsSubmitted += njobsToSubmit
jobsToSubmit = {}
for n in xrange(0, njobsToSubmit):
job = allJobs.pop(0)
multicrab._addToDictList(jobsToSubmit, job.task, job.id)
for task, jobs in jobsToSubmit.iteritems():
pretty = multicrab.prettyJobnums(jobs)
command = ["crab", "-c", task, "-submit", pretty] + args
print "Submitting %d jobs from task %s" % (len(jobs), task)
print "Command", " ".join(command)
if not opts.test:
ret = subprocess.call(command)
if ret != 0:
message = "Command '%s' failed with exit code %d" % (" ".join(command), ret)
if opts.allowFails:
print message
else:
raise Exception()
if njobsSubmitted < maxJobs:
print "Submitted, sleeping %f seconds" % opts.sleep
time.sleep(opts.sleep)
else:
print "Submitted"
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
resubmitJobs = {}
failedJobs = {}
stats = {}
allJobs = 0
for task in taskDirs:
if not os.path.exists(task):
if opts.showMissing:
print "%s: Task directory missing" % task
continue
jobs = multicrab.crabStatusToJobs(task)
jobSummaries = {}
njobs = 0
for key, item in jobs.iteritems():
hosts = {}
for job in item:
if job.host != None:
hosts[job.host] = 1
l = len(item)
jobSummaries[key] = JobSummary(item, hosts)
njobs += l
allJobs += l
if key in stats:
stats[key] += l
else:
stats[key] = l
# First the succesfully done
line = "%s (%d jobs):" % (task, njobs)
for s in order_done:
if s in jobSummaries:
line = formatSummaries(opts, line, s, jobSummaries[s])
del jobSummaries[s]
# Then the aborted-submitted to the end of the line
line_end = ""
for s in order_run:
if s in jobSummaries:
line_end = formatSummaries(opts, line_end, s, jobSummaries[s])
del jobSummaries[s]
# Then the failed ones to the middle
keys = jobSummaries.keys()
keys.sort()
for key in keys:
line = formatSummaries(opts, line, key, jobSummaries[key])
line += line_end
if line[-1] == ",":
line = line[0:-1]
print line
# Infer the jobs to be resubmitted
failed = []
for key, joblist in jobs.iteritems():
for job in joblist:
if job.failed(opts.resubmit):
failed.append( (job.id, job.jobExitCode) )
if len(failed) > 0:
failed.sort()
pretty = multicrab.prettyJobnums([x[0] for x in failed])
resubmitJobs[task] = pretty
for jobId, jobCode in failed:
multicrab._addToDictList(failedJobs, jobCode, "%s/res/CMSSW_%d.stdout" % (task, jobId))
print "----------------------------------------"
print "Summary for %d task(s), total %d job(s):" % (len(taskDirs), allJobs)
for s in order_done:
if s in stats:
print status_format % (s+":", stats[s])
del stats[s]
b = []
for s in order_run:
if s in stats:
b.append(status_format % (s+":", stats[s]))
del stats[s]
keys = stats.keys()
keys.sort()
for key in keys:
print status_format % (key+":", stats[key])
for line in b:
print line
print "----------------------------------------"
if len(resubmitJobs) == 0:
print "No failed/aborted jobs to resubmit"
else:
print "Following jobs failed/aborted, and can be resubmitted"
print
for task in taskDirs:
if task in resubmitJobs:
print "crab -c %s -resubmit %s" % (task, resubmitJobs[task])
print
if opts.failedLogs:
print "----------------------------------------"
print "Log files of failed jobs"
keys = failedJobs.keys()
keys.sort()
for code in keys:
print
print "Job exit code %d:" % code
print "\n".join(failedJobs[code])
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
resubmitJobs = {}
failedJobs = {}
stats = {}
allJobs = 0
for task in taskDirs:
if not os.path.exists(task):
if opts.showMissing:
print "%s: Task directory missing" % task
continue
jobs = multicrab.crabStatusToJobs(task)
jobSummaries = {}
njobs = 0
for key, item in jobs.iteritems():
hosts = {}
for job in item:
if job.host != None:
hosts[job.host] = 1
l = len(item)
jobSummaries[key] = JobSummary(item, hosts)
njobs += l
allJobs += l
if key in stats:
stats[key] += l
else:
stats[key] = l
# First the succesfully done
line = "%s (%d jobs):" % (task, njobs)
for s in order_done:
if s in jobSummaries:
line = formatSummaries(opts, line, s, jobSummaries[s])
del jobSummaries[s]
# Then the aborted-submitted to the end of the line
line_end = ""
for s in order_run:
if s in jobSummaries:
line_end = formatSummaries(opts, line_end, s, jobSummaries[s])
del jobSummaries[s]
# Then the failed ones to the middle
keys = jobSummaries.keys()
keys.sort()
for key in keys:
line = formatSummaries(opts, line, key, jobSummaries[key])
line += line_end
if line[-1] == ",":
line = line[0:-1]
print line
# Infer the jobs to be resubmitted
failed = []
for key, joblist in jobs.iteritems():
for job in joblist:
if job.failed(opts.resubmit):
failed.append((job.id, job.jobExitCode))
if len(failed) > 0:
failed.sort()
pretty = multicrab.prettyJobnums([x[0] for x in failed])
resubmitJobs[task] = pretty
for jobId, jobCode in failed:
multicrab._addToDictList(
failedJobs, jobCode,
"%s/res/CMSSW_%d.stdout" % (task, jobId))
print "----------------------------------------"
print "Summary for %d task(s), total %d job(s):" % (len(taskDirs), allJobs)
for s in order_done:
if s in stats:
print status_format % (s + ":", stats[s])
del stats[s]
b = []
for s in order_run:
if s in stats:
b.append(status_format % (s + ":", stats[s]))
del stats[s]
keys = stats.keys()
keys.sort()
for key in keys:
print status_format % (key + ":", stats[key])
for line in b:
print line
print "----------------------------------------"
if len(resubmitJobs) == 0:
print "No failed/aborted jobs to resubmit"
else:
print "Following jobs failed/aborted, and can be resubmitted"
print
for task in taskDirs:
if task in resubmitJobs:
print "crab -c %s -resubmit %s" % (task, resubmitJobs[task])
print
if opts.failedLogs:
print "----------------------------------------"
print "Log files of failed jobs"
keys = failedJobs.keys()
keys.sort()
for code in keys:
print
print "Job exit code %d:" % code
print "\n".join(failedJobs[code])
return 0