def main(opts, args):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
allJobs = []
for task in taskDirs:
if not os.path.exists(task):
print "%s: Task directory missing" % task
continue
jobs = multicrab.crabStatusToJobs(task)
if not "Created" in jobs:
print "%s: no 'Created' jobs to submit" % task
continue
allJobs.extend(filter(lambda j: isInRange(opts, j), jobs["Created"]))
maxJobs = len(allJobs)
if opts.maxJobs >= 0 and int(opts.maxJobs) < int(maxJobs):
maxJobs = opts.maxJobs
njobsSubmitted = 0
while njobsSubmitted < maxJobs:
njobsToSubmit = min(opts.jobs, maxJobs - njobsSubmitted, len(allJobs))
njobsSubmitted += njobsToSubmit
jobsToSubmit = {}
for n in xrange(0, njobsToSubmit):
job = allJobs.pop(0)
multicrab._addToDictList(jobsToSubmit, job.task, job.id)
for task, jobs in jobsToSubmit.iteritems():
pretty = multicrab.prettyJobnums(jobs)
command = ["crab", "-c", task, "-submit", pretty] + args
print "Submitting %d jobs from task %s" % (len(jobs), task)
print "Command", " ".join(command)
if not opts.test:
ret = subprocess.call(command)
if ret != 0:
message = "Command '%s' failed with exit code %d" % (
" ".join(command), ret)
if opts.allowFails:
print message
else:
raise Exception()
if njobsSubmitted < maxJobs:
print "Submitted, sleeping %f seconds" % opts.sleep
time.sleep(opts.sleep)
else:
print "Submitted"
return 0
def main(opts, args):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
allJobs = []
for task in taskDirs:
if not os.path.exists(task):
print "%s: Task directory missing" % task
continue
jobs = multicrab.crabStatusToJobs(task)
if not "Created" in jobs:
print "%s: no 'Created' jobs to submit" % task
continue
allJobs.extend(filter(lambda j: isInRange(opts, j), jobs["Created"]))
maxJobs = len(allJobs)
if opts.maxJobs >= 0 and int(opts.maxJobs) < int(maxJobs):
maxJobs = opts.maxJobs
njobsSubmitted = 0
while njobsSubmitted < maxJobs:
njobsToSubmit = min(opts.jobs, maxJobs-njobsSubmitted, len(allJobs))
njobsSubmitted += njobsToSubmit
jobsToSubmit = {}
for n in xrange(0, njobsToSubmit):
job = allJobs.pop(0)
multicrab._addToDictList(jobsToSubmit, job.task, job.id)
for task, jobs in jobsToSubmit.iteritems():
pretty = multicrab.prettyJobnums(jobs)
command = ["crab", "-c", task, "-submit", pretty] + args
print "Submitting %d jobs from task %s" % (len(jobs), task)
print "Command", " ".join(command)
if not opts.test:
ret = subprocess.call(command)
if ret != 0:
message = "Command '%s' failed with exit code %d" % (" ".join(command), ret)
if opts.allowFails:
print message
else:
raise Exception()
if njobsSubmitted < maxJobs:
print "Submitted, sleeping %f seconds" % opts.sleep
time.sleep(opts.sleep)
else:
print "Submitted"
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
for task in taskDirs:
if not os.path.exists(task):
print "%s: Task directory missing" % task
continue
jobsList = []
if opts.getoutput is None:
jobs = multicrab.crabStatusToJobs(task, printCrab=False)
for key in jobs.keys():
if "Done" in key:
jobsList.extend([j.id for j in jobs[key]])
else:
jobsList.extend(multicrab.prettyToJobList(opts.getoutput))
if len(jobsList) == 0:
print "%s: no jobs to retrieve" % task
continue
# Getoutput loop
maxJobs = len(jobsList)
if opts.jobs > 0:
maxJobs = opts.jobs
for i in xrange(0, int(math.ceil(float(len(jobsList))/maxJobs))):
jobsToGet = jobsList[i*maxJobs:(i+1)*maxJobs]
jobsStr = ",".join([str(j) for j in jobsToGet])
command = ["crab", "-c", task, "-getoutput", jobsStr]
print "Getting %d jobs from task %s" % (len(jobsToGet), task)
print "Command", " ".join(command)
ret = subprocess.call(command)
if ret != 0:
print "Command '%s' failed with exit code %s" % (" ".join(command), ret)
if not opts.allowFails:
return 1
return 0
def main(opts, args):
if opts.report:
multicrab.checkCrabInPath()
cell = "\|\s+(?P<%s>\S+)\s+"
lumi_re = re.compile((cell % "deliveredls")+
(cell % "delivered")+
(cell % "selectedls")+
(cell % "recorded")+"\|")
#lumi_re = re.compile("\|\s(?P<recorded>\S+)\s")
unit_re = re.compile("Recorded\(/(?P<unit>.*)\)")
if not opts.truncate and os.path.exists(opts.output):
f = open(opts.output, "r")
data = json.load(f)
f.close()
files = []
# only if no explicit files, or some directories explicitly given
if len(opts.files) == 0 or len(opts.dirs) > 0:
crabdirs = multicrab.getTaskDirectories(opts)
for d in crabdirs:
if isMCTask(d):
print " Ignoring task directory '%s', it looks like MC" % d
continue
if opts.report:
cmd = ["crab", "-report", "-c", d]
if opts.verbose:
print " ".join(cmd)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.communicate()[0]
ret = p.returncode
if ret != 0:
print "Call to 'crab -report -d %s' failed with return value %d" % (d, ret)
print output
return 1
if opts.verbose:
print output
files.append((d, os.path.join(d, "res", "lumiSummary.json")))
files.extend([(None, f) for f in opts.files])
data = {}
for task, jsonfile in files:
#print
#print "================================================================================"
#print "Dataset %s:" % d
cmd = ["lumiCalc2.py", "-i", jsonfile, "--nowarning", "overview", "-b", "stable"]
if opts.lumicalc1:
cmd = ["lumiCalc.py", "-i", jsonfile, "--with-correction", "--nowarning", "overview", "-b", "stable"]
#cmd = ["lumiCalc.py", "-c", "frontier://LumiCalc/CMS_LUMI_PROD", "-r", "132440", "--nowarning", "overview"]
#ret = subprocess.call(cmd)
if opts.verbose:
print " ".join(cmd)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.communicate()[0]
ret = p.returncode
if ret != 0:
print "Call to lumiCalc.py failed with return value %d with command" % ret
print " ".join(cmd)
print output
return 1
if opts.verbose:
print output
lines = output.split("\n")
lines.reverse()
lumi = -1.0
unit = None
for line in lines:
m = unit_re.search(line)
if m:
unit = m.group("unit")
break
m = lumi_re.search(line)
if m:
lumi = float(m.group("recorded")) # lumiCalc2.py returns pb^-1
# if opts.lumicalc1:
# lumi = lumi/1e6 # ub^-1 -> pb^-1, lumiCalc.py returns ub^-1
continue
if unit == None:
raise Exception("Didn't find unit information from lumiCalc output, command was %s" % " ".join(cmd))
lumi = convertLumi(lumi, unit)
if task == None:
print "File %s recorded luminosity %f pb^-1" % (jsonfile, lumi)
else:
print "Task %s recorded luminosity %f pb^-1" % (task, lumi)
data[task] = lumi
# Save the json file after each data task in case of future errors
if len(data) > 0:
f = open(opts.output, "wb")
json.dump(data, f, sort_keys=True, indent=2)
f.close()
if len(data) > 0:
f = open(opts.output, "wb")
json.dump(data, f, sort_keys=True, indent=2)
f.close()
return 0
def main(opts):
mc_conf_file = opts.cfgfile
crab_conf_file = None
py_conf_file = None
json_files = []
if not os.path.exists(mc_conf_file):
print "Multicrab configuration file %s doesn't exist!" % mc_conf_file
return 1
mc_parser = ConfigParser.ConfigParser()
mc_parser.read(mc_conf_file)
crab_conf_file = mc_parser.get("MULTICRAB", "cfg")
try:
py_conf_file = mc_parser.get("COMMON", "CMSSW.pset")
except ConfigParser.NoOptionError:
pass
for s in mc_parser.sections():
try:
json_files.append(mc_parser.get(s, "CMSSW.lumi_mask"))
except ConfigParser.NoOptionError:
pass
crab_parser = ConfigParser.ConfigParser()
crab_parser.read(crab_conf_file)
if py_conf_file == None:
py_conf_file = crab_parser.get("CMSSW", "pset")
try:
json_files.append(crab_parser.get("CMSSW", "lumi_mask"))
except ConfigParser.NoOptionError:
pass
# Unique list of json files
keys = {}
for f in json_files:
keys[f] = 1
json_files = keys.keys()
if crab_conf_file == None:
print "Did not find crab configuration file"
return 1
if py_conf_file == None:
print "Did not find CMSSW python configuration file"
return 1
# Check crab environment
multicrab.checkCrabInPath()
dirname = multicrab.createTaskDir(prefix=opts.prefix)
shutil.copy(mc_conf_file, os.path.join(dirname, "multicrab.cfg"))
flist = [crab_conf_file, py_conf_file]
if len(json_files) > 0:
flist.extend(json_files)
for f in flist:
shutil.copy(f, dirname)
print "Copied %s to %s" % (", ".join(flist), dirname)
print "Creating multicrab task"
print
print "############################################################"
print
os.chdir(dirname)
subprocess.call(["multicrab", "-create"])
print
print "############################################################"
print
print "Created multicrab task to subdirectory "+dirname
print
print "Jobs can be submitted by e.g. 'cd %s; multicrab -submit" % dirname
print
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
resubmitMode = (len(opts.resubmit) > 0)
resubmitIdListMode = (opts.resubmit not in ["failed", "aborted"])
if resubmitMode and resubmitIdListMode and len(taskDirs) != 1:
print "Option '--resubmit job_id_list' can be used with only one task, trying to use with %d tasks" % len(taskDirs)
return 1
if resubmitMode and resubmitIdListMode:
resubmitJobList = multicrab.prettyToJobList(opts.resubmit)
# Obtain all jobs to be (re)submitted
allJobs = []
seBlackLists = {}
for task in taskDirs:
if not os.path.exists(task):
print "%s: Task directory missing" % task
continue
cfgparser = ConfigParser.ConfigParser()
cfgparser.read(os.path.join(task, "share", "crab.cfg"))
if cfgparser.has_section("GRID"):
availableOptions = cfgparser.options("GRID")
blacklist = None
for ao in availableOptions:
if ao.lower() == "se_black_list":
blacklist = cfgparser.get("GRID", ao)
break
seBlackLists[task] = blacklist
jobs = multicrab.crabStatusToJobs(task, printCrab=False)
if not resubmitMode: # normal submission
if not "Created" in jobs:
print "%s: no 'Created' jobs to submit" % task
continue
allJobs.extend(filter(lambda j: isInRange(opts, j), jobs["Created"]))
elif not resubmitIdListMode: # resubmit all failed jobs
status = "all"
if opts.resubmit == "aborted":
status = "aborted"
for joblist in jobs.itervalues():
for job in joblist:
if job.failed(status):
allJobs.append(job)
else: # resubmit explicit list of jobs
for joblist in jobs.itervalues():
for job in joblist:
if job.id in resubmitJobList:
allJobs.append(job)
resubmitJobList.remove(job.id)
# Set the number of maximum jobs to submit
maxJobs = len(allJobs)
if opts.maxJobs >= 0 and int(opts.maxJobs) < int(maxJobs):
maxJobs = opts.maxJobs
submitCommand = "-submit"
if len(opts.resubmit) > 0:
submitCommand = "-resubmit"
sites = []
siteSubmitIndex = 0
if len(opts.toSites) > 0:
sites = opts.toSites.split(",")
# Submission loop
njobsSubmitted = 0
while njobsSubmitted < maxJobs:
# Construct list of jobs per task to submit
njobsToSubmit = min(opts.jobs, maxJobs-njobsSubmitted, len(allJobs))
njobsSubmitted += njobsToSubmit
jobsToSubmit = {}
for n in xrange(0, njobsToSubmit):
job = allJobs.pop(0)
aux.addToDictList(jobsToSubmit, job.task, job.id)
# If explicit list of sites to submit was given, get the site to submit this time
crabOptions = []
if len(sites) > 0:
site = sites[siteSubmitIndex]
siteSubmitIndex = (siteSubmitIndex+1) % len(sites)
crabOptions.append("-GRID.se_black_list= -GRID.se_white_list="+site)
# Actual submission
for task, jobs in jobsToSubmit.iteritems():
pretty = multicrab.prettyJobnums(jobs)
if len(jobs) == 1:
pretty += "," # CRAB thinks one number is number of jobs, the comma translates it to job ID
command = ["crab", "-c", task, submitCommand, pretty]
if opts.crabArgs != "":
command.extend(opts.crabArgs.split(" "))
if len(crabOptions) > 0:
command.extend(crabOptions)
if opts.addSeBlackList != "":
lst = seBlackLists[task]
if lst is None:
lst = opts.addSeBlackList
else:
lst += ","+opts.addSeBlackList
command.extend(["-GRID.se_black_list="+lst])
print "Submitting %d jobs from task %s" % (len(jobs), task)
print "Command", " ".join(command)
if not opts.test:
timesLeft = 1
if opts.tryAgainTimes > 0:
timesLeft = opts.tryAgainTimes
while timesLeft > 0:
ret = subprocess.call(command)
if ret == 0:
break
else:
timesLeft -= 1
message = "Command '%s' failed with exit code %d" % (" ".join(command), ret)
if opts.allowFails:
print message
if opts.tryAgainTimes > 0:
print message
if timesLeft > 0:
print "Trying again after %d seconds (%d trials left)" % (opts.tryAgainSeconds, timesLeft)
time.sleep(opts.tryAgainSeconds)
else:
print "No trials left, continuing with next job block"
else:
raise Exception()
# Sleep between submissions
if njobsSubmitted < maxJobs:
print "Submitted, sleeping %f seconds" % opts.sleep
time.sleep(opts.sleep)
else:
print "Submitted"
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
resubmitJobs = {}
failedJobs = {}
stats = {}
allJobs = 0
for task in taskDirs:
if not os.path.exists(task):
if opts.showMissing:
print "%s: Task directory missing" % task
continue
jobs = multicrab.crabStatusToJobs(task)
jobSummaries = {}
njobs = 0
for key, item in jobs.iteritems():
hosts = {}
for job in item:
if job.host != None:
hosts[job.host] = 1
l = len(item)
jobSummaries[key] = JobSummary(item, hosts)
njobs += l
allJobs += l
if key in stats:
stats[key] += l
else:
stats[key] = l
# First the succesfully done
line = "%s (%d jobs):" % (task, njobs)
for s in order_done:
if s in jobSummaries:
line = formatSummaries(opts, line, s, jobSummaries[s])
del jobSummaries[s]
# Then the aborted-submitted to the end of the line
line_end = ""
for s in order_run:
if s in jobSummaries:
line_end = formatSummaries(opts, line_end, s, jobSummaries[s])
del jobSummaries[s]
# Then the failed ones to the middle
keys = jobSummaries.keys()
keys.sort()
for key in keys:
line = formatSummaries(opts, line, key, jobSummaries[key])
line += line_end
if line[-1] == ",":
line = line[0:-1]
print line
# Infer the jobs to be resubmitted
failed = []
for key, joblist in jobs.iteritems():
for job in joblist:
if job.failed(opts.resubmit):
failed.append( (job.id, job.jobExitCode) )
if len(failed) > 0:
failed.sort()
pretty = multicrab.prettyJobnums([x[0] for x in failed])
resubmitJobs[task] = pretty
for jobId, jobCode in failed:
multicrab._addToDictList(failedJobs, jobCode, "%s/res/CMSSW_%d.stdout" % (task, jobId))
print "----------------------------------------"
print "Summary for %d task(s), total %d job(s):" % (len(taskDirs), allJobs)
for s in order_done:
if s in stats:
print status_format % (s+":", stats[s])
del stats[s]
b = []
for s in order_run:
if s in stats:
b.append(status_format % (s+":", stats[s]))
del stats[s]
keys = stats.keys()
keys.sort()
for key in keys:
print status_format % (key+":", stats[key])
for line in b:
print line
print "----------------------------------------"
if len(resubmitJobs) == 0:
print "No failed/aborted jobs to resubmit"
else:
print "Following jobs failed/aborted, and can be resubmitted"
print
for task in taskDirs:
if task in resubmitJobs:
print "crab -c %s -resubmit %s" % (task, resubmitJobs[task])
print
if opts.failedLogs:
print "----------------------------------------"
print "Log files of failed jobs"
keys = failedJobs.keys()
keys.sort()
for code in keys:
print
print "Job exit code %d:" % code
print "\n".join(failedJobs[code])
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
resubmitMode = (len(opts.resubmit) > 0)
resubmitIdListMode = (opts.resubmit not in ["failed", "aborted"])
if resubmitMode and resubmitIdListMode and len(taskDirs) != 1:
print "Option '--resubmit job_id_list' can be used with only one task, trying to use with %d tasks" % len(
taskDirs)
return 1
if resubmitMode and resubmitIdListMode:
resubmitJobList = multicrab.prettyToJobList(opts.resubmit)
# Obtain all jobs to be (re)submitted
allJobs = []
seBlackLists = {}
for task in taskDirs:
if not os.path.exists(task):
print "%s: Task directory missing" % task
continue
cfgparser = ConfigParser.ConfigParser()
cfgparser.read(os.path.join(task, "share", "crab.cfg"))
if cfgparser.has_section("GRID"):
availableOptions = cfgparser.options("GRID")
blacklist = None
for ao in availableOptions:
if ao.lower() == "se_black_list":
blacklist = cfgparser.get("GRID", ao)
break
seBlackLists[task] = blacklist
jobs = multicrab.crabStatusToJobs(task, printCrab=False)
if not resubmitMode: # normal submission
if not "Created" in jobs:
print "%s: no 'Created' jobs to submit" % task
continue
allJobs.extend(
filter(lambda j: isInRange(opts, j), jobs["Created"]))
elif not resubmitIdListMode: # resubmit all failed jobs
status = "all"
if opts.resubmit == "aborted":
status = "aborted"
for joblist in jobs.itervalues():
for job in joblist:
if job.failed(status):
allJobs.append(job)
else: # resubmit explicit list of jobs
for joblist in jobs.itervalues():
for job in joblist:
if job.id in resubmitJobList:
allJobs.append(job)
resubmitJobList.remove(job.id)
# Set the number of maximum jobs to submit
maxJobs = len(allJobs)
if opts.maxJobs >= 0 and int(opts.maxJobs) < int(maxJobs):
maxJobs = opts.maxJobs
submitCommand = "-submit"
if len(opts.resubmit) > 0:
submitCommand = "-resubmit"
sites = []
siteSubmitIndex = 0
if len(opts.toSites) > 0:
sites = opts.toSites.split(",")
# Submission loop
njobsSubmitted = 0
while njobsSubmitted < maxJobs:
# Construct list of jobs per task to submit
njobsToSubmit = min(opts.jobs, maxJobs - njobsSubmitted, len(allJobs))
njobsSubmitted += njobsToSubmit
jobsToSubmit = {}
for n in xrange(0, njobsToSubmit):
job = allJobs.pop(0)
aux.addToDictList(jobsToSubmit, job.task, job.id)
# If explicit list of sites to submit was given, get the site to submit this time
crabOptions = []
if len(sites) > 0:
site = sites[siteSubmitIndex]
siteSubmitIndex = (siteSubmitIndex + 1) % len(sites)
crabOptions.append("-GRID.se_black_list= -GRID.se_white_list=" +
site)
# Actual submission
for task, jobs in jobsToSubmit.iteritems():
pretty = multicrab.prettyJobnums(jobs)
if len(jobs) == 1:
pretty += "," # CRAB thinks one number is number of jobs, the comma translates it to job ID
command = ["crab", "-c", task, submitCommand, pretty]
if opts.crabArgs != "":
command.extend(opts.crabArgs.split(" "))
if len(crabOptions) > 0:
command.extend(crabOptions)
if opts.addSeBlackList != "":
lst = seBlackLists[task]
if lst is None:
lst = opts.addSeBlackList
else:
lst += "," + opts.addSeBlackList
command.extend(["-GRID.se_black_list=" + lst])
print "Submitting %d jobs from task %s" % (len(jobs), task)
print "Command", " ".join(command)
if not opts.test:
timesLeft = 1
if opts.tryAgainTimes > 0:
timesLeft = opts.tryAgainTimes
while timesLeft > 0:
ret = subprocess.call(command)
if ret == 0:
break
else:
timesLeft -= 1
message = "Command '%s' failed with exit code %d" % (
" ".join(command), ret)
if opts.allowFails:
print message
if opts.tryAgainTimes > 0:
print message
if timesLeft > 0:
print "Trying again after %d seconds (%d trials left)" % (
opts.tryAgainSeconds, timesLeft)
time.sleep(opts.tryAgainSeconds)
else:
print "No trials left, continuing with next job block"
else:
raise Exception()
# Sleep between submissions
if njobsSubmitted < maxJobs:
print "Submitted, sleeping %f seconds" % opts.sleep
time.sleep(opts.sleep)
else:
print "Submitted"
return 0
def main(opts, args):
workdir = os.getcwd()
crabdirs = multicrab.getTaskDirectories(opts)
for d in crabdirs:
# Run crab -report
if opts.report:
multicrab.checkCrabInPath()
cmd = ["crab", "-report", "-c", d]
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = p.communicate()[0]
ret = p.returncode
if ret != 0:
print "Call to 'crab -report -d %s' failed with return value %d" % (
d, ret)
print output
return 1
if os.path.exists(os.path.join(d, "task.tar.gz")):
print "Skipping %s, task.tar.gz already exists" % d
continue
# Go to task directory (in order to get the paths in the archive correctly easily)
os.chdir(d)
# Remove default.tgz
tmp = os.path.join("share", "default.tgz")
if os.path.exists(tmp):
os.remove(tmp)
# Create the tar archive
tar = tarfile.open("task.tar.gz", mode="w:gz")
files = []
for f in [
"CMSSW_*.std*", "crab_fjr_*.xml", "Submission_*",
"Watchdog_*.log*"
]:
files.extend(glob.glob(os.path.join("res", f)))
files.extend(["job", "log", "share"])
#print "\n".join(files)
for f in files:
if os.path.exists(f):
tar.add(f)
tar.close()
# Keep share/crab.cfg
files.remove("share")
sharefiles = glob.glob(os.path.join("share", "*"))
sharefiles = filter(lambda x: not "crab.cfg" in x, sharefiles)
files.extend(sharefiles)
# Delete the files just added to the archive
for f in files:
if os.path.isfile(f):
#print "rm "+f
try:
os.remove(f)
except OSError, e:
print "Warning: failed to remove %s: %s" % (f, str(e))
elif os.path.isdir(f):
#print "rm -fR "+f
try:
shutil.rmtree(f)
except OSError, e:
print "Warning: failed to remove %s: %s" % (f, str(e))
def main(opts, args):
if opts.report:
multicrab.checkCrabInPath()
cell = "\|\s+(?P<%s>\S+)\s+"
lumi_re = re.compile((cell % "deliveredls") + (cell % "delivered") +
(cell % "selectedls") + (cell % "recorded") + "\|")
#lumi_re = re.compile("\|\s(?P<recorded>\S+)\s")
unit_re = re.compile("Recorded\(/(?P<unit>.*)\)")
if not opts.truncate and os.path.exists(opts.output):
f = open(opts.output, "r")
data = json.load(f)
f.close()
files = []
# only if no explicit files, or some directories explicitly given
if len(opts.files) == 0 or len(opts.dirs) > 0:
crabdirs = multicrab.getTaskDirectories(opts)
for d in crabdirs:
if isMCTask(d):
print " Ignoring task directory '%s', it looks like MC" % d
continue
if opts.report:
cmd = ["crab", "-report", "-c", d]
if opts.verbose:
print " ".join(cmd)
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = p.communicate()[0]
ret = p.returncode
if ret != 0:
print "Call to 'crab -report -d %s' failed with return value %d" % (
d, ret)
print output
return 1
if opts.verbose:
print output
files.append((d, os.path.join(d, "res", "lumiSummary.json")))
files.extend([(None, f) for f in opts.files])
data = {}
for task, jsonfile in files:
#print
#print "================================================================================"
#print "Dataset %s:" % d
cmd = [
"lumiCalc2.py", "-i", jsonfile, "--nowarning", "overview", "-b",
"stable"
]
if opts.lumicalc1:
cmd = [
"lumiCalc.py", "-i", jsonfile, "--with-correction",
"--nowarning", "overview", "-b", "stable"
]
#cmd = ["lumiCalc.py", "-c", "frontier://LumiCalc/CMS_LUMI_PROD", "-r", "132440", "--nowarning", "overview"]
#ret = subprocess.call(cmd)
if opts.verbose:
print " ".join(cmd)
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = p.communicate()[0]
ret = p.returncode
if ret != 0:
print "Call to lumiCalc.py failed with return value %d with command" % ret
print " ".join(cmd)
print output
return 1
if opts.verbose:
print output
lines = output.split("\n")
lines.reverse()
lumi = -1.0
unit = None
for line in lines:
m = unit_re.search(line)
if m:
unit = m.group("unit")
break
m = lumi_re.search(line)
if m:
lumi = float(m.group("recorded")) # lumiCalc2.py returns pb^-1
# if opts.lumicalc1:
# lumi = lumi/1e6 # ub^-1 -> pb^-1, lumiCalc.py returns ub^-1
continue
if unit == None:
raise Exception(
"Didn't find unit information from lumiCalc output, command was %s"
% " ".join(cmd))
lumi = convertLumi(lumi, unit)
if task == None:
print "File %s recorded luminosity %f pb^-1" % (jsonfile, lumi)
else:
print "Task %s recorded luminosity %f pb^-1" % (task, lumi)
data[task] = lumi
# Save the json file after each data task in case of future errors
if len(data) > 0:
f = open(opts.output, "wb")
json.dump(data, f, sort_keys=True, indent=2)
f.close()
if len(data) > 0:
f = open(opts.output, "wb")
json.dump(data, f, sort_keys=True, indent=2)
f.close()
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
resubmitJobs = {}
failedJobs = {}
stats = {}
allJobs = 0
for task in taskDirs:
if not os.path.exists(task):
if opts.showMissing:
print "%s: Task directory missing" % task
continue
jobs = multicrab.crabStatusToJobs(task)
jobSummaries = {}
njobs = 0
for key, item in jobs.iteritems():
hosts = {}
for job in item:
if job.host != None:
hosts[job.host] = 1
l = len(item)
jobSummaries[key] = JobSummary(item, hosts)
njobs += l
allJobs += l
if key in stats:
stats[key] += l
else:
stats[key] = l
# First the succesfully done
line = "%s (%d jobs):" % (task, njobs)
for s in order_done:
if s in jobSummaries:
line = formatSummaries(opts, line, s, jobSummaries[s])
del jobSummaries[s]
# Then the aborted-submitted to the end of the line
line_end = ""
for s in order_run:
if s in jobSummaries:
line_end = formatSummaries(opts, line_end, s, jobSummaries[s])
del jobSummaries[s]
# Then the failed ones to the middle
keys = jobSummaries.keys()
keys.sort()
for key in keys:
line = formatSummaries(opts, line, key, jobSummaries[key])
line += line_end
if line[-1] == ",":
line = line[0:-1]
print line
# Infer the jobs to be resubmitted
failed = []
for key, joblist in jobs.iteritems():
for job in joblist:
if job.failed(opts.resubmit):
failed.append((job.id, job.jobExitCode))
if len(failed) > 0:
failed.sort()
pretty = multicrab.prettyJobnums([x[0] for x in failed])
resubmitJobs[task] = pretty
for jobId, jobCode in failed:
multicrab._addToDictList(
failedJobs, jobCode,
"%s/res/CMSSW_%d.stdout" % (task, jobId))
print "----------------------------------------"
print "Summary for %d task(s), total %d job(s):" % (len(taskDirs), allJobs)
for s in order_done:
if s in stats:
print status_format % (s + ":", stats[s])
del stats[s]
b = []
for s in order_run:
if s in stats:
b.append(status_format % (s + ":", stats[s]))
del stats[s]
keys = stats.keys()
keys.sort()
for key in keys:
print status_format % (key + ":", stats[key])
for line in b:
print line
print "----------------------------------------"
if len(resubmitJobs) == 0:
print "No failed/aborted jobs to resubmit"
else:
print "Following jobs failed/aborted, and can be resubmitted"
print
for task in taskDirs:
if task in resubmitJobs:
print "crab -c %s -resubmit %s" % (task, resubmitJobs[task])
print
if opts.failedLogs:
print "----------------------------------------"
print "Log files of failed jobs"
keys = failedJobs.keys()
keys.sort()
for code in keys:
print
print "Job exit code %d:" % code
print "\n".join(failedJobs[code])
return 0
def main(opts, args):
workdir = os.getcwd()
crabdirs = multicrab.getTaskDirectories(opts)
for d in crabdirs:
# Run crab -report
if opts.report:
multicrab.checkCrabInPath()
cmd = ["crab", "-report", "-c", d]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.communicate()[0]
ret = p.returncode
if ret != 0:
print "Call to 'crab -report -d %s' failed with return value %d" % (d, ret)
print output
return 1
if os.path.exists(os.path.join(d, "task.tar.gz")):
print "Skipping %s, task.tar.gz already exists" % d
continue
# Go to task directory (in order to get the paths in the archive correctly easily)
os.chdir(d)
# Remove default.tgz
tmp = os.path.join("share", "default.tgz")
if os.path.exists(tmp):
os.remove(tmp)
# Create the tar archive
tar = tarfile.open("task.tar.gz", mode="w:gz")
files = []
for f in ["CMSSW_*.std*", "crab_fjr_*.xml", "Submission_*", "Watchdog_*.log*"]:
files.extend(glob.glob(os.path.join("res", f)))
files.extend(["job", "log", "share"])
#print "\n".join(files)
for f in files:
if os.path.exists(f):
tar.add(f)
tar.close()
# Keep share/crab.cfg
files.remove("share")
sharefiles = glob.glob(os.path.join("share", "*"))
sharefiles = filter(lambda x: not "crab.cfg" in x, sharefiles)
files.extend(sharefiles)
# Delete the files just added to the archive
for f in files:
if os.path.isfile(f):
#print "rm "+f
try:
os.remove(f)
except OSError, e:
print "Warning: failed to remove %s: %s" % (f, str(e))
elif os.path.isdir(f):
#print "rm -fR "+f
try:
shutil.rmtree(f)
except OSError, e:
print "Warning: failed to remove %s: %s" % (f, str(e))
def main(opts):
mc_conf_file = opts.cfgfile
crab_conf_file = None
py_conf_file = None
json_files = []
if not os.path.exists(mc_conf_file):
print "Multicrab configuration file %s doesn't exist!" % mc_conf_file
return 1
mc_parser = ConfigParser.ConfigParser()
mc_parser.read(mc_conf_file)
crab_conf_file = mc_parser.get("MULTICRAB", "cfg")
try:
py_conf_file = mc_parser.get("COMMON", "CMSSW.pset")
except ConfigParser.NoOptionError:
pass
for s in mc_parser.sections():
try:
json_files.append(mc_parser.get(s, "CMSSW.lumi_mask"))
except ConfigParser.NoOptionError:
pass
crab_parser = ConfigParser.ConfigParser()
crab_parser.read(crab_conf_file)
if py_conf_file == None:
py_conf_file = crab_parser.get("CMSSW", "pset")
try:
json_files.append(crab_parser.get("CMSSW", "lumi_mask"))
except ConfigParser.NoOptionError:
pass
# Unique list of json files
keys = {}
for f in json_files:
keys[f] = 1
json_files = keys.keys()
if crab_conf_file == None:
print "Did not find crab configuration file"
return 1
if py_conf_file == None:
print "Did not find CMSSW python configuration file"
return 1
dirname = multicrab.createTaskDir(prefix=opts.prefix)
shutil.copy(mc_conf_file, os.path.join(dirname, "multicrab.cfg"))
flist = [crab_conf_file, py_conf_file]
if len(json_files) > 0:
flist.extend(json_files)
for f in flist:
shutil.copy(f, dirname)
if opts.codeVersion:
multicrab.writeGitVersion(dirname)
print "Copied %s to %s" % (", ".join(flist), dirname)
if not opts.create:
print
print "Now (since you explicitly asked with --noCreate), you have to create the crab tasks yourself, e.g."
print "cd %s; multicrab -create" % dirname
print "Then you can submit them e.g. by multicrab -submit"
return 0
# Check crab environment
multicrab.checkCrabInPath()
print "Creating multicrab task"
print
print "############################################################"
print
os.chdir(dirname)
subprocess.call(["multicrab", "-create"])
print
print "############################################################"
print
print "Created multicrab task to subdirectory "+dirname
print
print "Jobs can be submitted by e.g. 'cd %s; multicrab -submit" % dirname
print
return 0
def main(opts):
taskDirs = multicrab.getTaskDirectories(opts)
multicrab.checkCrabInPath()
resubmitJobs = {}
failedJobs = {}
stats = {}
allJobs = 0
if opts.byHost:
global status_format
status_format = status_format.replace("18s", "40s")
if opts.save:
out = open(opts.saveFile, "w")
for task in taskDirs:
if not os.path.exists(task):
if opts.showMissing:
print >> sys.stderr, "%s: Task directory missing" % task
continue
try:
jobs = multicrab.crabStatusToJobs(task, opts.printCrab)
except Exception:
if not opts.allowFails:
raise
print "%s: crab -status failed" % task
continue
jobSummaries = {}
njobs = 0
for key, item in jobs.iteritems():
hosts = {}
for job in item:
if job.host != None:
aux.addToDictList(hosts, job.host, job)
if opts.byHost:
for host, joblist in hosts.iteritems():
jobSummaries[key + " " + host] = JobSummary(
joblist, [host])
else:
jobSummaries[key] = JobSummary(item, hosts)
l = len(item)
njobs += l
allJobs += l
if key in stats:
stats[key] += l
else:
stats[key] = l
# First the succesfully done
line = "%s (%d jobs):" % (task, njobs)
for s in order_done:
if s in jobSummaries:
line = formatSummaries(opts, line, s, jobSummaries[s])
del jobSummaries[s]
# Then the aborted-submitted to the end of the line
line_end = ""
for s in order_run:
if s in jobSummaries:
line_end = formatSummaries(opts, line_end, s, jobSummaries[s])
del jobSummaries[s]
# Then the failed ones to the middle
keys = jobSummaries.keys()
keys.sort()
for key in keys:
line = formatSummaries(opts, line, key, jobSummaries[key])
line += line_end
if line[-1] == ",":
line = line[0:-1]
if opts.save:
out.write(line)
out.write("\n")
print line
# Infer the jobs to be resubmitted
failed = []
for key, joblist in jobs.iteritems():
for job in joblist:
if job.failed(opts.resubmit):
failed.append((job.id, job.jobExitCode))
if len(failed) > 0:
failed.sort()
pretty = multicrab.prettyJobnums([x[0] for x in failed])
resubmitJobs[task] = pretty
for jobId, jobCode in failed:
aux.addToDictList(failedJobs, jobCode,
"%s/res/CMSSW_%d.stdout" % (task, jobId))
summary = StringIO.StringIO()
summary.write("----------------------------------------\n")
print "Summary for %d task(s), total %d job(s):" % (len(taskDirs), allJobs)
for s in order_done:
if s in stats:
summary.write(status_format % (s + ":", stats[s]))
summary.write("\n")
del stats[s]
b = []
for s in order_run:
if s in stats:
b.append(status_format % (s + ":", stats[s]))
del stats[s]
keys = stats.keys()
keys.sort()
for key in keys:
summary.write(status_format % (key + ":", stats[key]))
summary.write("\n")
for line in b:
summary.write(line)
summary.write("\n")
summary.write("----------------------------------------\n")
if len(resubmitJobs) == 0:
summary.write("No failed/aborted jobs to resubmit\n")
else:
summary.write(
"Following jobs failed/aborted, and can be resubmitted\n\n")
for task in taskDirs:
if task in resubmitJobs:
summary.write("crab -c %s -resubmit %s\n" %
(task, resubmitJobs[task]))
summary.write("\n")
if opts.failedLogs:
summary.write("----------------------------------------\n")
summary.write("Log files of failed jobs\n")
keys = failedJobs.keys()
keys.sort()
for code in keys:
summary.write("\nJob exit code %d:\n" % code)
summary.write("\n".join(failedJobs[code]))
summary.write("\n")
if opts.save:
out.write(summary.getvalue())
out.close()
print summary.getvalue()
return 0
