def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
analyses = []
watchdogAnalyses = []
if opts.time:
analyses.append(TimeAnalysis())
if opts.size:
analyses.append(SizeAnalysis(opts.sizeFile))
if opts.memory:
watchdogAnalyses.append(MemoryAnalysis())
if opts.host:
analyses.append(HostAnalysis())
if opts.watchdog:
analyses.append(WatchdogAnalysis())
if len(analyses)+len(watchdogAnalyses) == 0:
return 1
for task in taskDirs:
files = glob.glob(os.path.join(task, "res", "CMSSW_*.stdout"))
files = excludeInclude(files, opts.include, opts.exclude)
wfiles = []
if len(watchdogAnalyses) > 0:
wfiles = glob.glob(os.path.join(task, "res", "Watchdog_*.log.gz"))
wfiles = excludeInclude(wfiles, opts.include, opts.exclude)
if opts.byStatus:
try:
jobs = multicrab.crabStatusToJobs(task, opts.printCrab)
except Exception:
if not opts.allowFails:
raise
print "%s: crab -status failed" % task
continue
print "Task %s" % task
# Ignore running jobs
for status in multicrabStatus.order_run:
if status in jobs:
del jobs[status]
stats = jobs.keys()
stats.sort()
for status in stats:
ids = ",".join(["%d"%j.id for j in jobs[status]])
f = excludeInclude(files, ids)
wf = excludeInclude(wfiles, ids)
print " %s, %d jobs" % (status, len(files))
analyseTask(f, wf, analyses, watchdogAnalyses, prefix=" ")
else:
print "Task %s, %d jobs" % (task, len(files))
analyseTask(files, wfiles, analyses, watchdogAnalyses)
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
analyses = []
watchdogAnalyses = []
if opts.time:
analyses.append(TimeAnalysis())
if opts.size:
analyses.append(SizeAnalysis(opts.sizeFile))
if opts.memory:
watchdogAnalyses.append(MemoryAnalysis())
if opts.host:
analyses.append(HostAnalysis())
if opts.watchdog:
analyses.append(WatchdogAnalysis())
if len(analyses) + len(watchdogAnalyses) == 0:
return 1
for task in taskDirs:
files = glob.glob(os.path.join(task, "res", "CMSSW_*.stdout"))
files = excludeInclude(files, opts.include, opts.exclude)
wfiles = []
if len(watchdogAnalyses) > 0:
wfiles = glob.glob(os.path.join(task, "res", "Watchdog_*.log.gz"))
wfiles = excludeInclude(wfiles, opts.include, opts.exclude)
if opts.byStatus:
try:
jobs = multicrab.crabStatusToJobs(task, opts.printCrab)
except Exception:
if not opts.allowFails:
raise
print "%s: crab -status failed" % task
continue
print "Task %s" % task
# Ignore running jobs
for status in multicrabStatus.order_run:
if status in jobs:
del jobs[status]
stats = jobs.keys()
stats.sort()
for status in stats:
ids = ",".join(["%d" % j.id for j in jobs[status]])
f = excludeInclude(files, ids)
wf = excludeInclude(wfiles, ids)
print " %s, %d jobs" % (status, len(files))
analyseTask(f, wf, analyses, watchdogAnalyses, prefix=" ")
else:
print "Task %s, %d jobs" % (task, len(files))
analyseTask(files, wfiles, analyses, watchdogAnalyses)
return 0
def main(opts, args):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
allJobs = []
for task in taskDirs:
if not os.path.exists(task):
print "%s: Task directory missing" % task
continue
jobs = multicrab.crabStatusToJobs(task)
if not "Created" in jobs:
print "%s: no 'Created' jobs to submit" % task
continue
allJobs.extend(filter(lambda j: isInRange(opts, j), jobs["Created"]))
maxJobs = len(allJobs)
if opts.maxJobs >= 0 and int(opts.maxJobs) < int(maxJobs):
maxJobs = opts.maxJobs
njobsSubmitted = 0
while njobsSubmitted < maxJobs:
njobsToSubmit = min(opts.jobs, maxJobs - njobsSubmitted, len(allJobs))
njobsSubmitted += njobsToSubmit
jobsToSubmit = {}
for n in xrange(0, njobsToSubmit):
job = allJobs.pop(0)
multicrab._addToDictList(jobsToSubmit, job.task, job.id)
for task, jobs in jobsToSubmit.iteritems():
pretty = multicrab.prettyJobnums(jobs)
command = ["crab", "-c", task, "-submit", pretty] + args
print "Submitting %d jobs from task %s" % (len(jobs), task)
print "Command", " ".join(command)
if not opts.test:
ret = subprocess.call(command)
if ret != 0:
message = "Command '%s' failed with exit code %d" % (
" ".join(command), ret)
if opts.allowFails:
print message
else:
raise Exception()
if njobsSubmitted < maxJobs:
print "Submitted, sleeping %f seconds" % opts.sleep
time.sleep(opts.sleep)
else:
print "Submitted"
return 0
def main(opts, args):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
allJobs = []
for task in taskDirs:
if not os.path.exists(task):
print "%s: Task directory missing" % task
continue
jobs = multicrab.crabStatusToJobs(task)
if not "Created" in jobs:
print "%s: no 'Created' jobs to submit" % task
continue
allJobs.extend(filter(lambda j: isInRange(opts, j), jobs["Created"]))
maxJobs = len(allJobs)
if opts.maxJobs >= 0 and int(opts.maxJobs) < int(maxJobs):
maxJobs = opts.maxJobs
njobsSubmitted = 0
while njobsSubmitted < maxJobs:
njobsToSubmit = min(opts.jobs, maxJobs-njobsSubmitted, len(allJobs))
njobsSubmitted += njobsToSubmit
jobsToSubmit = {}
for n in xrange(0, njobsToSubmit):
job = allJobs.pop(0)
multicrab._addToDictList(jobsToSubmit, job.task, job.id)
for task, jobs in jobsToSubmit.iteritems():
pretty = multicrab.prettyJobnums(jobs)
command = ["crab", "-c", task, "-submit", pretty] + args
print "Submitting %d jobs from task %s" % (len(jobs), task)
print "Command", " ".join(command)
if not opts.test:
ret = subprocess.call(command)
if ret != 0:
message = "Command '%s' failed with exit code %d" % (" ".join(command), ret)
if opts.allowFails:
print message
else:
raise Exception()
if njobsSubmitted < maxJobs:
print "Submitted, sleeping %f seconds" % opts.sleep
time.sleep(opts.sleep)
else:
print "Submitted"
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
for task in taskDirs:
if not os.path.exists(task):
print "%s: Task directory missing" % task
continue
jobsList = []
if opts.getoutput is None:
jobs = multicrab.crabStatusToJobs(task, printCrab=False)
for key in jobs.keys():
if "Done" in key:
jobsList.extend([j.id for j in jobs[key]])
else:
jobsList.extend(multicrab.prettyToJobList(opts.getoutput))
if len(jobsList) == 0:
print "%s: no jobs to retrieve" % task
continue
# Getoutput loop
maxJobs = len(jobsList)
if opts.jobs > 0:
maxJobs = opts.jobs
for i in xrange(0, int(math.ceil(float(len(jobsList))/maxJobs))):
jobsToGet = jobsList[i*maxJobs:(i+1)*maxJobs]
jobsStr = ",".join([str(j) for j in jobsToGet])
command = ["crab", "-c", task, "-getoutput", jobsStr]
print "Getting %d jobs from task %s" % (len(jobsToGet), task)
print "Command", " ".join(command)
ret = subprocess.call(command)
if ret != 0:
print "Command '%s' failed with exit code %s" % (" ".join(command), ret)
if not opts.allowFails:
return 1
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
resubmitMode = (len(opts.resubmit) > 0)
resubmitIdListMode = (opts.resubmit not in ["failed", "aborted"])
if resubmitMode and resubmitIdListMode and len(taskDirs) != 1:
print "Option '--resubmit job_id_list' can be used with only one task, trying to use with %d tasks" % len(taskDirs)
return 1
if resubmitMode and resubmitIdListMode:
resubmitJobList = multicrab.prettyToJobList(opts.resubmit)
# Obtain all jobs to be (re)submitted
allJobs = []
seBlackLists = {}
for task in taskDirs:
if not os.path.exists(task):
print "%s: Task directory missing" % task
continue
cfgparser = ConfigParser.ConfigParser()
cfgparser.read(os.path.join(task, "share", "crab.cfg"))
if cfgparser.has_section("GRID"):
availableOptions = cfgparser.options("GRID")
blacklist = None
for ao in availableOptions:
if ao.lower() == "se_black_list":
blacklist = cfgparser.get("GRID", ao)
break
seBlackLists[task] = blacklist
jobs = multicrab.crabStatusToJobs(task, printCrab=False)
if not resubmitMode: # normal submission
if not "Created" in jobs:
print "%s: no 'Created' jobs to submit" % task
continue
allJobs.extend(filter(lambda j: isInRange(opts, j), jobs["Created"]))
elif not resubmitIdListMode: # resubmit all failed jobs
status = "all"
if opts.resubmit == "aborted":
status = "aborted"
for joblist in jobs.itervalues():
for job in joblist:
if job.failed(status):
allJobs.append(job)
else: # resubmit explicit list of jobs
for joblist in jobs.itervalues():
for job in joblist:
if job.id in resubmitJobList:
allJobs.append(job)
resubmitJobList.remove(job.id)
# Set the number of maximum jobs to submit
maxJobs = len(allJobs)
if opts.maxJobs >= 0 and int(opts.maxJobs) < int(maxJobs):
maxJobs = opts.maxJobs
submitCommand = "-submit"
if len(opts.resubmit) > 0:
submitCommand = "-resubmit"
sites = []
siteSubmitIndex = 0
if len(opts.toSites) > 0:
sites = opts.toSites.split(",")
# Submission loop
njobsSubmitted = 0
while njobsSubmitted < maxJobs:
# Construct list of jobs per task to submit
njobsToSubmit = min(opts.jobs, maxJobs-njobsSubmitted, len(allJobs))
njobsSubmitted += njobsToSubmit
jobsToSubmit = {}
for n in xrange(0, njobsToSubmit):
job = allJobs.pop(0)
aux.addToDictList(jobsToSubmit, job.task, job.id)
# If explicit list of sites to submit was given, get the site to submit this time
crabOptions = []
if len(sites) > 0:
site = sites[siteSubmitIndex]
siteSubmitIndex = (siteSubmitIndex+1) % len(sites)
crabOptions.append("-GRID.se_black_list= -GRID.se_white_list="+site)
# Actual submission
for task, jobs in jobsToSubmit.iteritems():
pretty = multicrab.prettyJobnums(jobs)
if len(jobs) == 1:
pretty += "," # CRAB thinks one number is number of jobs, the comma translates it to job ID
command = ["crab", "-c", task, submitCommand, pretty]
if opts.crabArgs != "":
command.extend(opts.crabArgs.split(" "))
if len(crabOptions) > 0:
command.extend(crabOptions)
if opts.addSeBlackList != "":
lst = seBlackLists[task]
if lst is None:
lst = opts.addSeBlackList
else:
lst += ","+opts.addSeBlackList
command.extend(["-GRID.se_black_list="+lst])
print "Submitting %d jobs from task %s" % (len(jobs), task)
print "Command", " ".join(command)
if not opts.test:
timesLeft = 1
if opts.tryAgainTimes > 0:
timesLeft = opts.tryAgainTimes
while timesLeft > 0:
ret = subprocess.call(command)
if ret == 0:
break
else:
timesLeft -= 1
message = "Command '%s' failed with exit code %d" % (" ".join(command), ret)
if opts.allowFails:
print message
if opts.tryAgainTimes > 0:
print message
if timesLeft > 0:
print "Trying again after %d seconds (%d trials left)" % (opts.tryAgainSeconds, timesLeft)
time.sleep(opts.tryAgainSeconds)
else:
print "No trials left, continuing with next job block"
else:
raise Exception()
# Sleep between submissions
if njobsSubmitted < maxJobs:
print "Submitted, sleeping %f seconds" % opts.sleep
time.sleep(opts.sleep)
else:
print "Submitted"
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
resubmitJobs = {}
failedJobs = {}
stats = {}
allJobs = 0
for task in taskDirs:
if not os.path.exists(task):
if opts.showMissing:
print "%s: Task directory missing" % task
continue
jobs = multicrab.crabStatusToJobs(task)
jobSummaries = {}
njobs = 0
for key, item in jobs.iteritems():
hosts = {}
for job in item:
if job.host != None:
hosts[job.host] = 1
l = len(item)
jobSummaries[key] = JobSummary(item, hosts)
njobs += l
allJobs += l
if key in stats:
stats[key] += l
else:
stats[key] = l
# First the succesfully done
line = "%s (%d jobs):" % (task, njobs)
for s in order_done:
if s in jobSummaries:
line = formatSummaries(opts, line, s, jobSummaries[s])
del jobSummaries[s]
# Then the aborted-submitted to the end of the line
line_end = ""
for s in order_run:
if s in jobSummaries:
line_end = formatSummaries(opts, line_end, s, jobSummaries[s])
del jobSummaries[s]
# Then the failed ones to the middle
keys = jobSummaries.keys()
keys.sort()
for key in keys:
line = formatSummaries(opts, line, key, jobSummaries[key])
line += line_end
if line[-1] == ",":
line = line[0:-1]
print line
# Infer the jobs to be resubmitted
failed = []
for key, joblist in jobs.iteritems():
for job in joblist:
if job.failed(opts.resubmit):
failed.append( (job.id, job.jobExitCode) )
if len(failed) > 0:
failed.sort()
pretty = multicrab.prettyJobnums([x[0] for x in failed])
resubmitJobs[task] = pretty
for jobId, jobCode in failed:
multicrab._addToDictList(failedJobs, jobCode, "%s/res/CMSSW_%d.stdout" % (task, jobId))
print "----------------------------------------"
print "Summary for %d task(s), total %d job(s):" % (len(taskDirs), allJobs)
for s in order_done:
if s in stats:
print status_format % (s+":", stats[s])
del stats[s]
b = []
for s in order_run:
if s in stats:
b.append(status_format % (s+":", stats[s]))
del stats[s]
keys = stats.keys()
keys.sort()
for key in keys:
print status_format % (key+":", stats[key])
for line in b:
print line
print "----------------------------------------"
if len(resubmitJobs) == 0:
print "No failed/aborted jobs to resubmit"
else:
print "Following jobs failed/aborted, and can be resubmitted"
print
for task in taskDirs:
if task in resubmitJobs:
print "crab -c %s -resubmit %s" % (task, resubmitJobs[task])
print
if opts.failedLogs:
print "----------------------------------------"
print "Log files of failed jobs"
keys = failedJobs.keys()
keys.sort()
for code in keys:
print
print "Job exit code %d:" % code
print "\n".join(failedJobs[code])
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
resubmitMode = (len(opts.resubmit) > 0)
resubmitIdListMode = (opts.resubmit not in ["failed", "aborted"])
if resubmitMode and resubmitIdListMode and len(taskDirs) != 1:
print "Option '--resubmit job_id_list' can be used with only one task, trying to use with %d tasks" % len(
taskDirs)
return 1
if resubmitMode and resubmitIdListMode:
resubmitJobList = multicrab.prettyToJobList(opts.resubmit)
# Obtain all jobs to be (re)submitted
allJobs = []
seBlackLists = {}
for task in taskDirs:
if not os.path.exists(task):
print "%s: Task directory missing" % task
continue
cfgparser = ConfigParser.ConfigParser()
cfgparser.read(os.path.join(task, "share", "crab.cfg"))
if cfgparser.has_section("GRID"):
availableOptions = cfgparser.options("GRID")
blacklist = None
for ao in availableOptions:
if ao.lower() == "se_black_list":
blacklist = cfgparser.get("GRID", ao)
break
seBlackLists[task] = blacklist
jobs = multicrab.crabStatusToJobs(task, printCrab=False)
if not resubmitMode: # normal submission
if not "Created" in jobs:
print "%s: no 'Created' jobs to submit" % task
continue
allJobs.extend(
filter(lambda j: isInRange(opts, j), jobs["Created"]))
elif not resubmitIdListMode: # resubmit all failed jobs
status = "all"
if opts.resubmit == "aborted":
status = "aborted"
for joblist in jobs.itervalues():
for job in joblist:
if job.failed(status):
allJobs.append(job)
else: # resubmit explicit list of jobs
for joblist in jobs.itervalues():
for job in joblist:
if job.id in resubmitJobList:
allJobs.append(job)
resubmitJobList.remove(job.id)
# Set the number of maximum jobs to submit
maxJobs = len(allJobs)
if opts.maxJobs >= 0 and int(opts.maxJobs) < int(maxJobs):
maxJobs = opts.maxJobs
submitCommand = "-submit"
if len(opts.resubmit) > 0:
submitCommand = "-resubmit"
sites = []
siteSubmitIndex = 0
if len(opts.toSites) > 0:
sites = opts.toSites.split(",")
# Submission loop
njobsSubmitted = 0
while njobsSubmitted < maxJobs:
# Construct list of jobs per task to submit
njobsToSubmit = min(opts.jobs, maxJobs - njobsSubmitted, len(allJobs))
njobsSubmitted += njobsToSubmit
jobsToSubmit = {}
for n in xrange(0, njobsToSubmit):
job = allJobs.pop(0)
aux.addToDictList(jobsToSubmit, job.task, job.id)
# If explicit list of sites to submit was given, get the site to submit this time
crabOptions = []
if len(sites) > 0:
site = sites[siteSubmitIndex]
siteSubmitIndex = (siteSubmitIndex + 1) % len(sites)
crabOptions.append("-GRID.se_black_list= -GRID.se_white_list=" +
site)
# Actual submission
for task, jobs in jobsToSubmit.iteritems():
pretty = multicrab.prettyJobnums(jobs)
if len(jobs) == 1:
pretty += "," # CRAB thinks one number is number of jobs, the comma translates it to job ID
command = ["crab", "-c", task, submitCommand, pretty]
if opts.crabArgs != "":
command.extend(opts.crabArgs.split(" "))
if len(crabOptions) > 0:
command.extend(crabOptions)
if opts.addSeBlackList != "":
lst = seBlackLists[task]
if lst is None:
lst = opts.addSeBlackList
else:
lst += "," + opts.addSeBlackList
command.extend(["-GRID.se_black_list=" + lst])
print "Submitting %d jobs from task %s" % (len(jobs), task)
print "Command", " ".join(command)
if not opts.test:
timesLeft = 1
if opts.tryAgainTimes > 0:
timesLeft = opts.tryAgainTimes
while timesLeft > 0:
ret = subprocess.call(command)
if ret == 0:
break
else:
timesLeft -= 1
message = "Command '%s' failed with exit code %d" % (
" ".join(command), ret)
if opts.allowFails:
print message
if opts.tryAgainTimes > 0:
print message
if timesLeft > 0:
print "Trying again after %d seconds (%d trials left)" % (
opts.tryAgainSeconds, timesLeft)
time.sleep(opts.tryAgainSeconds)
else:
print "No trials left, continuing with next job block"
else:
raise Exception()
# Sleep between submissions
if njobsSubmitted < maxJobs:
print "Submitted, sleeping %f seconds" % opts.sleep
time.sleep(opts.sleep)
else:
print "Submitted"
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
resubmitJobs = {}
failedJobs = {}
stats = {}
allJobs = 0
for task in taskDirs:
if not os.path.exists(task):
if opts.showMissing:
print "%s: Task directory missing" % task
continue
jobs = multicrab.crabStatusToJobs(task)
jobSummaries = {}
njobs = 0
for key, item in jobs.iteritems():
hosts = {}
for job in item:
if job.host != None:
hosts[job.host] = 1
l = len(item)
jobSummaries[key] = JobSummary(item, hosts)
njobs += l
allJobs += l
if key in stats:
stats[key] += l
else:
stats[key] = l
# First the succesfully done
line = "%s (%d jobs):" % (task, njobs)
for s in order_done:
if s in jobSummaries:
line = formatSummaries(opts, line, s, jobSummaries[s])
del jobSummaries[s]
# Then the aborted-submitted to the end of the line
line_end = ""
for s in order_run:
if s in jobSummaries:
line_end = formatSummaries(opts, line_end, s, jobSummaries[s])
del jobSummaries[s]
# Then the failed ones to the middle
keys = jobSummaries.keys()
keys.sort()
for key in keys:
line = formatSummaries(opts, line, key, jobSummaries[key])
line += line_end
if line[-1] == ",":
line = line[0:-1]
print line
# Infer the jobs to be resubmitted
failed = []
for key, joblist in jobs.iteritems():
for job in joblist:
if job.failed(opts.resubmit):
failed.append((job.id, job.jobExitCode))
if len(failed) > 0:
failed.sort()
pretty = multicrab.prettyJobnums([x[0] for x in failed])
resubmitJobs[task] = pretty
for jobId, jobCode in failed:
multicrab._addToDictList(
failedJobs, jobCode,
"%s/res/CMSSW_%d.stdout" % (task, jobId))
print "----------------------------------------"
print "Summary for %d task(s), total %d job(s):" % (len(taskDirs), allJobs)
for s in order_done:
if s in stats:
print status_format % (s + ":", stats[s])
del stats[s]
b = []
for s in order_run:
if s in stats:
b.append(status_format % (s + ":", stats[s]))
del stats[s]
keys = stats.keys()
keys.sort()
for key in keys:
print status_format % (key + ":", stats[key])
for line in b:
print line
print "----------------------------------------"
if len(resubmitJobs) == 0:
print "No failed/aborted jobs to resubmit"
else:
print "Following jobs failed/aborted, and can be resubmitted"
print
for task in taskDirs:
if task in resubmitJobs:
print "crab -c %s -resubmit %s" % (task, resubmitJobs[task])
print
if opts.failedLogs:
print "----------------------------------------"
print "Log files of failed jobs"
keys = failedJobs.keys()
keys.sort()
for code in keys:
print
print "Job exit code %d:" % code
print "\n".join(failedJobs[code])
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
resubmitJobs = {}
failedJobs = {}
stats = {}
allJobs = 0
if opts.byHost:
global status_format
status_format = status_format.replace("18s", "40s")
if opts.save:
out = open(opts.saveFile, "w")
for task in taskDirs:
if not os.path.exists(task):
if opts.showMissing:
print >> sys.stderr, "%s: Task directory missing" % task
continue
try:
jobs = multicrab.crabStatusToJobs(task, opts.printCrab)
except Exception:
if not opts.allowFails:
raise
print "%s: crab -status failed" % task
continue
jobSummaries = {}
njobs = 0
for key, item in jobs.iteritems():
hosts = {}
for job in item:
if job.host != None:
aux.addToDictList(hosts, job.host, job)
if opts.byHost:
for host, joblist in hosts.iteritems():
jobSummaries[key + " " + host] = JobSummary(
joblist, [host])
else:
jobSummaries[key] = JobSummary(item, hosts)
l = len(item)
njobs += l
allJobs += l
if key in stats:
stats[key] += l
else:
stats[key] = l
# First the succesfully done
line = "%s (%d jobs):" % (task, njobs)
for s in order_done:
if s in jobSummaries:
line = formatSummaries(opts, line, s, jobSummaries[s])
del jobSummaries[s]
# Then the aborted-submitted to the end of the line
line_end = ""
for s in order_run:
if s in jobSummaries:
line_end = formatSummaries(opts, line_end, s, jobSummaries[s])
del jobSummaries[s]
# Then the failed ones to the middle
keys = jobSummaries.keys()
keys.sort()
for key in keys:
line = formatSummaries(opts, line, key, jobSummaries[key])
line += line_end
if line[-1] == ",":
line = line[0:-1]
if opts.save:
out.write(line)
out.write("\n")
print line
# Infer the jobs to be resubmitted
failed = []
for key, joblist in jobs.iteritems():
for job in joblist:
if job.failed(opts.resubmit):
failed.append((job.id, job.jobExitCode))
if len(failed) > 0:
failed.sort()
pretty = multicrab.prettyJobnums([x[0] for x in failed])
resubmitJobs[task] = pretty
for jobId, jobCode in failed:
aux.addToDictList(failedJobs, jobCode,
"%s/res/CMSSW_%d.stdout" % (task, jobId))
summary = StringIO.StringIO()
summary.write("----------------------------------------\n")
print "Summary for %d task(s), total %d job(s):" % (len(taskDirs), allJobs)
for s in order_done:
if s in stats:
summary.write(status_format % (s + ":", stats[s]))
summary.write("\n")
del stats[s]
b = []
for s in order_run:
if s in stats:
b.append(status_format % (s + ":", stats[s]))
del stats[s]
keys = stats.keys()
keys.sort()
for key in keys:
summary.write(status_format % (key + ":", stats[key]))
summary.write("\n")
for line in b:
summary.write(line)
summary.write("\n")
summary.write("----------------------------------------\n")
if len(resubmitJobs) == 0:
summary.write("No failed/aborted jobs to resubmit\n")
else:
summary.write(
"Following jobs failed/aborted, and can be resubmitted\n\n")
for task in taskDirs:
if task in resubmitJobs:
summary.write("crab -c %s -resubmit %s\n" %
(task, resubmitJobs[task]))
summary.write("\n")
if opts.failedLogs:
summary.write("----------------------------------------\n")
summary.write("Log files of failed jobs\n")
keys = failedJobs.keys()
keys.sort()
for code in keys:
summary.write("\nJob exit code %d:\n" % code)
summary.write("\n".join(failedJobs[code]))
summary.write("\n")
if opts.save:
out.write(summary.getvalue())
out.close()
print summary.getvalue()
return 0