def GetCrabDirectories(opts):
#Verbose("GetCrabDirectories()")
opts2 = None
crabDirs = multicrab.getTaskDirectories(opts2)
#crabDirs = GetIncludeExcludeDatasets(crabDirsTmp, opts)
crabDirs = filter(lambda x: "multicrab_" not in x, crabDirs) #remove "multicrab_" directory from list
return crabDirs
def GetCrabDirectories(opts):
#Verbose("GetCrabDirectories()")
opts2 = None
crabDirs = multicrab.getTaskDirectories(opts2)
#crabDirs = GetIncludeExcludeDatasets(crabDirsTmp, opts)
crabDirs = filter(lambda x: "multicrab_" not in x,
crabDirs) #remove "multicrab_" directory from list
return crabDirs
def GetCrabDirectories(opts):
'''
Return an alphabetically sorted list of datasets found under
the multicrab dir used (opts.dirName)
'''
Verbose("GetCrabDirectories()", True)
crabdirs = multicrab.getTaskDirectories(opts)
crabdirs = filter(lambda x: "multicrab_" not in x, crabdirs) # Remove "multicrab_" directory from list
crabdirs = GetIncludeExcludeDatasets(crabdirs, opts)
# Return list (alphabetically ordered)
return sorted(crabdirs)
def GetCrabDirectories(opts):
'''
Return an alphabetically sorted list of datasets found under
the multicrab dir used (opts.dirName)
'''
Verbose("GetCrabDirectories()", True)
crabdirs = multicrab.getTaskDirectories(opts)
crabdirs = filter(lambda x: "multicrab_" not in x, crabdirs) # Remove "multicrab_" directory from list
crabdirs = GetIncludeExcludeDatasets(crabdirs, opts)
# Return list (alphabetically ordered)
return sorted(crabdirs)
def main(opts, args):
cell = "\|\s+(?P<%s>\S+)\s+"
# lumi_re = re.compile((cell % "deliveredls")+
# (cell % "delivered")+
# (cell % "selectedls")+
# (cell % "recorded")+"\|")
#lumi_re = re.compile("\|\s(?P<recorded>\S+)\s")
lumi_re = re.compile("\|\s+(?P<recorded>\d+\.*\d*)\s+\|\s*$")
#Summary:
#+-------+------+------+------+-------------------+------------------+
#| nfill | nrun | nls | ncms | totdelivered(/pb) | totrecorded(/pb) |
#+-------+------+------+------+-------------------+------------------+
#| 1 | 1 | 1585 | 1585 | 25.515 | 25.028 |
#+-------+------+------+------+-------------------+------------------+
unit_re = re.compile("totrecorded\(/(?P<unit>.*)\)")
if not opts.truncate and os.path.exists(opts.output):
f = open(opts.output, "r")
data = json.load(f)
f.close()
files = []
# only if no explicit files, or some directories explicitly given
if len(opts.files) == 0 or len(opts.dirs) > 0:
crabdirs = multicrab.getTaskDirectories(opts)
for d in crabdirs:
if isMCTask(d):
print " Ignoring task directory '%s', it looks like MC" % d
continue
if isEmpty(d):
print " Ignoring task directory '%s', it looks empty" % d
continue
if opts.report:
multicrab.checkCrabInPath()
cmd = ["crab", "report", d]
if opts.verbose:
print " ".join(cmd)
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = p.communicate()[0]
ret = p.returncode
if ret != 0:
print "Call to 'crab -report -d %s' failed with return value %d" % (
d, ret)
print output
return 1
if opts.verbose:
print output
files.append((d, os.path.join(d, "results", "lumiSummary.json")))
files.extend([(None, f) for f in opts.files])
data = {}
for task, jsonfile in files:
lumicalc = opts.lumicalc
#print
#print "================================================================================"
#print "Dataset %s:" % d
# if lumicalc == "lumiCalc1":
# cmd = ["lumiCalc.py", "-i", jsonfile, "--with-correction", "--nowarning", "overview", "-b", "stable"]
# if lumicalc == "lumiCalc2":
# cmd = ["lumiCalc2.py", "-i", jsonfile, "--nowarning", "overview", "-b", "stable"]
# if lumicalc == "pixelLumiCalc":
# cmd = ["pixelLumiCalc.py", "-i", jsonfile, "--nowarning", "overview"]
#cmd = ["lumiCalc.py", "-c", "frontier://LumiCalc/CMS_LUMI_PROD", "-r", "132440", "--nowarning", "overview"]
#ret = subprocess.call(cmd)
# brilcalc lumi -u /pb -i JSON-file
home = os.environ['HOME']
path = os.path.join(home, ".local/bin")
exe = os.path.join(path, "brilcalc")
if not os.path.exists(exe):
print " brilcalc not found, have you installed it?"
print " http://cms-service-lumi.web.cern.ch/cms-service-lumi/brilwsdoc.html"
sys.exit()
cmd = [exe, "lumi", "--byls", "-u/pb", "-i", jsonfile]
if opts.verbose:
print " ".join(cmd)
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = p.communicate()[0]
ret = p.returncode
if ret != 0:
print "Call to", cmd[
0], "failed with return value %d with command" % ret
print " ".join(cmd)
print output
return 1
if opts.verbose:
print output
lines = output.split("\n")
lumi = -1.0
unit = None
for line in lines:
m = unit_re.search(line)
if m:
unit = m.group("unit")
# break
m = lumi_re.search(line)
if m:
lumi = float(m.group("recorded")) # lumiCalc2.py returns pb^-1
if unit == None:
raise Exception(
"Didn't find unit information from lumiCalc output, command was %s"
% " ".join(cmd))
lumi = convertLumi(lumi, unit)
# PileUp
fOUT = os.path.join(task, "results", "PileUp.root")
pucmd = [
"pileupCalc.py", "-i", jsonfile, "--inputLumiJSON", PileUpJSON,
"--calcMode", "true", "--minBiasXsec", "80000", "--maxPileupBin",
"50", "--numPileupBins", "50", fOUT
]
pu = subprocess.Popen(pucmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
puoutput = pu.communicate()[0]
puret = pu.returncode
if puret != 0:
print "Call to", pucmd[
0], "failed with return value %d with command" % puret
print " ".join(pucmd)
print puoutput
return puret
if task == None:
print "File %s recorded luminosity %f pb^-1" % (jsonfile, lumi)
else:
print "Task %s recorded luminosity %f pb^-1" % (task, lumi)
data[task] = lumi
# Save the json file after each data task in case of future errors
if len(data) > 0:
f = open(opts.output, "wb")
json.dump(data, f, sort_keys=True, indent=2)
f.close()
if len(data) > 0:
f = open(opts.output, "wb")
json.dump(data, f, sort_keys=True, indent=2)
f.close()
return 0
def main(opts, args):
crabdirs = multicrab.getTaskDirectories(opts)
global re_histos
re_histos.append(re.compile("^output files:.*?(?P<file>%s)" % opts.input))
re_histos.append(re.compile("^\s+file\s+=\s+(?P<file>%s)" % opts.input))
exit_re = re.compile("/results/cmsRun_(?P<exitcode>\d+)\.log\.tar\.gz")
mergedFiles = []
for d in crabdirs:
d = d.replace("/", "")
stdoutFiles = glob.glob(os.path.join(d, "results", "cmsRun_*.log.tar.gz"))
files = []
exitCodes = []
for f in stdoutFiles:
try:
if opts.filesInSE:
histoFile = getHistogramFileSE(f, opts)
if histoFile != None:
files.append(histoFile)
else:
print "Task %s, skipping job %s: input root file not found from stdout" % (d, f)
else:
histoFile = getHistogramFile(f, opts)
if histoFile != None:
path = os.path.join(os.path.dirname(f), histoFile)
if os.path.exists(path):
files.append(path)
else:
print "Task %s, skipping job %s: input root file found from stdout, but does not exist" % (d, f)
else:
print "Task %s, skipping job %s: input root file not found from stdout" % (d, f)
except multicrab.ExitCodeException, e:
print "Task %s, skipping job %s: %s" % (d, f, str(e))
exit_match = exit_re.search(f)
if exit_match:
exitCodes.append(int(exit_match.group("exitcode")))
if opts.test:
if len(exitCodes) > 0:
print " jobs with problems:",sorted(exitCodes)
continue
if len(files) == 0:
print "Task %s, skipping, no files to merge" % d
continue
for f in files:
if not os.path.isfile(f):
raise Exception("File %s is marked as output file in the CMSSW_N.stdout, but does not exist" % f)
filesSplit = splitFiles(files, opts.filesPerMerge)
if len(filesSplit) == 1:
print "Task %s, merging %d file(s)" % (d, len(files))
else:
print "Task %s, merging %d file(s) to %d files" % (d, len(files), len(filesSplit))
for index, inputFiles in filesSplit:
tmp = d
if len(filesSplit) > 1:
tmp += "-%d" % index
mergeName = os.path.join(d, "results", opts.output % tmp)
if os.path.exists(mergeName) and not opts.test:
if opts.verbose:
print "mv %s %s" % (mergeName, mergeName+".backup")
shutil.move(mergeName, mergeName+".backup")
# FIXME: add here reading of first xrootd file, finding all TTrees, and writing the TList to mergeName file
if opts.filesInSE:
raise Exception("--filesInSE feature is not fully implemented")
if len(inputFiles) == 1:
if opts.verbose:
print "cp %s %s" % (inputFiles[0], mergeName)
if not opts.test:
shutil.copy(inputFiles[0], mergeName)
else:
if opts.fast:
ret = hplusHadd(opts, mergeName, inputFiles)
if ret != 0:
return ret
else:
ret = hadd(opts, mergeName, inputFiles)
if ret != 0:
return ret
if len(filesSplit) > 1:
print " done %d" % index
mergedFiles.append((mergeName, inputFiles))
try:
sanityCheck(mergeName, inputFiles)
except SanityCheckException, e:
print "Task %s: %s; disabling input file deletion" % (d, str(e))
opts.deleteImmediately = False
opts.delete = False
if opts.deleteImmediately:
for srcFile in inputFiles:
if opts.verbose:
print "rm %s" % srcFile
if not opts.test:
os.remove(srcFile)
def main(opts, args):
cell = "\|\s+(?P<%s>\S+)\s+"
# lumi_re = re.compile((cell % "deliveredls")+
# (cell % "delivered")+
# (cell % "selectedls")+
# (cell % "recorded")+"\|")
#lumi_re = re.compile("\|\s(?P<recorded>\S+)\s")
lumi_re = re.compile("\|\s+(?P<recorded>\d+\.*\d*)\s+\|\s*$")
#Summary:
#+-------+------+------+------+-------------------+------------------+
#| nfill | nrun | nls | ncms | totdelivered(/pb) | totrecorded(/pb) |
#+-------+------+------+------+-------------------+------------------+
#| 1 | 1 | 1585 | 1585 | 25.515 | 25.028 |
#+-------+------+------+------+-------------------+------------------+
unit_re = re.compile("totrecorded\(/(?P<unit>.*)\)")
if not opts.truncate and os.path.exists(opts.output):
f = open(opts.output, "r")
data = json.load(f)
f.close()
files = []
# only if no explicit files, or some directories explicitly given
if len(opts.files) == 0 or len(opts.dirs) > 0:
crabdirs = multicrab.getTaskDirectories(opts)
for d in crabdirs:
if isMCTask(d):
print " Ignoring task directory '%s', it looks like MC" % d
continue
if isEmpty(d):
print " Ignoring task directory '%s', it looks empty" % d
continue
if opts.report:
multicrab.checkCrabInPath()
cmd = ["crab", "report", d]
if opts.verbose:
print " ".join(cmd)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.communicate()[0]
ret = p.returncode
if ret != 0:
print "Call to 'crab -report -d %s' failed with return value %d" % (d, ret)
print output
return 1
if opts.verbose:
print output
files.append((d, os.path.join(d, "results", "lumiSummary.json")))
files.extend([(None, f) for f in opts.files])
data = {}
for task, jsonfile in files:
lumicalc = opts.lumicalc
#print
#print "================================================================================"
#print "Dataset %s:" % d
# if lumicalc == "lumiCalc1":
# cmd = ["lumiCalc.py", "-i", jsonfile, "--with-correction", "--nowarning", "overview", "-b", "stable"]
# if lumicalc == "lumiCalc2":
# cmd = ["lumiCalc2.py", "-i", jsonfile, "--nowarning", "overview", "-b", "stable"]
# if lumicalc == "pixelLumiCalc":
# cmd = ["pixelLumiCalc.py", "-i", jsonfile, "--nowarning", "overview"]
#cmd = ["lumiCalc.py", "-c", "frontier://LumiCalc/CMS_LUMI_PROD", "-r", "132440", "--nowarning", "overview"]
#ret = subprocess.call(cmd)
# brilcalc lumi -u /pb -i JSON-file
home = os.environ['HOME']
path = os.path.join(home,".local/bin")
exe = os.path.join(path,"brilcalc")
if not os.path.exists(exe):
print " brilcalc not found, have you installed it?"
print " http://cms-service-lumi.web.cern.ch/cms-service-lumi/brilwsdoc.html"
sys.exit()
cmd = [exe,"lumi","--byls", "-u/pb","-i", jsonfile]
if opts.verbose:
print " ".join(cmd)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.communicate()[0]
ret = p.returncode
if ret != 0:
print "Call to",cmd[0],"failed with return value %d with command" % ret
print " ".join(cmd)
print output
return 1
if opts.verbose:
print output
lines = output.split("\n")
lumi = -1.0
unit = None
for line in lines:
m = unit_re.search(line)
if m:
unit = m.group("unit")
# break
m = lumi_re.search(line)
if m:
lumi = float(m.group("recorded")) # lumiCalc2.py returns pb^-1
if unit == None:
raise Exception("Didn't find unit information from lumiCalc output, command was %s" % " ".join(cmd))
lumi = convertLumi(lumi, unit)
# PileUp
fOUT = os.path.join(task, "results", "PileUp.root")
pucmd = ["pileupCalc.py","-i",jsonfile,"--inputLumiJSON",PileUpJSON,"--calcMode","true","--minBiasXsec","80000","--maxPileupBin","50","--numPileupBins","50",fOUT]
pu = subprocess.Popen(pucmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
puoutput = pu.communicate()[0]
puret = pu.returncode
if puret != 0:
print "Call to",pucmd[0],"failed with return value %d with command" % puret
print " ".join(pucmd)
print puoutput
return puret
if task == None:
print "File %s recorded luminosity %f pb^-1" % (jsonfile, lumi)
else:
print "Task %s recorded luminosity %f pb^-1" % (task, lumi)
data[task] = lumi
# Save the json file after each data task in case of future errors
if len(data) > 0:
f = open(opts.output, "wb")
json.dump(data, f, sort_keys=True, indent=2)
f.close()
if len(data) > 0:
f = open(opts.output, "wb")
json.dump(data, f, sort_keys=True, indent=2)
f.close()
return 0
def main(opts, args):
crabdirs = multicrab.getTaskDirectories(opts)
global re_histos
re_histos.append(re.compile("^output files:.*?(?P<file>%s)" % opts.input))
re_histos.append(re.compile("^\s+file\s+=\s+(?P<file>%s)" % opts.input))
exit_re = re.compile("/results/cmsRun_(?P<exitcode>\d+)\.log\.tar\.gz")
mergedFiles = []
for d in crabdirs:
d = d.replace("/", "")
stdoutFiles = glob.glob(
os.path.join(d, "results", "cmsRun_*.log.tar.gz"))
files = []
exitCodes = []
for f in stdoutFiles:
try:
if opts.filesInSE:
histoFile = getHistogramFileSE(f, opts)
if histoFile != None:
files.append(histoFile)
else:
print "Task %s, skipping job %s: input root file not found from stdout" % (
d, f)
else:
histoFile = getHistogramFile(f, opts)
if histoFile != None:
path = os.path.join(os.path.dirname(f), histoFile)
if os.path.exists(path):
files.append(path)
else:
print "Task %s, skipping job %s: input root file found from stdout, but does not exist" % (
d, f)
else:
print "Task %s, skipping job %s: input root file not found from stdout" % (
d, f)
except multicrab.ExitCodeException, e:
print "Task %s, skipping job %s: %s" % (d, f, str(e))
exit_match = exit_re.search(f)
if exit_match:
exitCodes.append(int(exit_match.group("exitcode")))
if opts.test:
if len(exitCodes) > 0:
print " jobs with problems:", sorted(exitCodes)
continue
if len(files) == 0:
print "Task %s, skipping, no files to merge" % d
continue
for f in files:
if not os.path.isfile(f):
raise Exception(
"File %s is marked as output file in the CMSSW_N.stdout, but does not exist"
% f)
filesSplit = splitFiles(files, opts.filesPerMerge)
if len(filesSplit) == 1:
print "Task %s, merging %d file(s)" % (d, len(files))
else:
print "Task %s, merging %d file(s) to %d files" % (d, len(files),
len(filesSplit))
for index, inputFiles in filesSplit:
tmp = d
if len(filesSplit) > 1:
tmp += "-%d" % index
mergeName = os.path.join(d, "results", opts.output % tmp)
if os.path.exists(mergeName) and not opts.test:
if opts.verbose:
print "mv %s %s" % (mergeName, mergeName + ".backup")
shutil.move(mergeName, mergeName + ".backup")
# FIXME: add here reading of first xrootd file, finding all TTrees, and writing the TList to mergeName file
if opts.filesInSE:
raise Exception("--filesInSE feature is not fully implemented")
if len(inputFiles) == 1:
if opts.verbose:
print "cp %s %s" % (inputFiles[0], mergeName)
if not opts.test:
shutil.copy(inputFiles[0], mergeName)
else:
if opts.fast:
ret = hplusHadd(opts, mergeName, inputFiles)
if ret != 0:
return ret
else:
ret = hadd(opts, mergeName, inputFiles)
if ret != 0:
return ret
if len(filesSplit) > 1:
print " done %d" % index
mergedFiles.append((mergeName, inputFiles))
try:
sanityCheck(mergeName, inputFiles)
except SanityCheckException, e:
print "Task %s: %s; disabling input file deletion" % (d,
str(e))
opts.deleteImmediately = False
opts.delete = False
if opts.deleteImmediately:
for srcFile in inputFiles:
if opts.verbose:
print "rm %s" % srcFile
if not opts.test:
os.remove(srcFile)