def makeLatestJSON(jsonsPath,aDataSet,runRegCfg="runregTemplate.cfg",runRegPy="runregparse.py"):
##Check last analysed run number
maxRun = 0
for fname in os.listdir(jsonsPath):
jsonFile = jsonsPath+"/"+fname
jsonList = LumiList(filename = jsonFile)
runList = jsonList.getRuns()
for run in runList:
if int(run)>maxRun:
maxRun = int(run)
print "Last analysed run is:",maxRun
print "Preparinng JSON for runs:",maxRun+1,999999
file = open(runRegCfg, "r") #Opens the file in read-mode
text = file.read()
file.close()
text1 = text.replace("RunMin=","RunMin=" + str(maxRun+1))
text2 = text1.replace("DBS_PDS=","DBS_PDS=" + aDataSet) # enable cross check with DBS
file = open("runreg.cfg", "w")
file.write(text2)
file.close()
##Create the JSON
os.system(runRegPy)
# os.system(os.getenv("CMSSW_BASE")+"/src/UserCode/L1RpcTriggerAnalysis/test/runregparse.py")
##Rename JSON file to include run range
jsonFile = "GoodRuns.json"
jsonList = LumiList(filename = jsonFile)
runList = jsonList.getRuns()
minRun = int(runList[0])
maxRun = int(runList[len(runList)-1])
jsonFileName = "GoodRuns_"+str(minRun)+"-"+str(maxRun)+".json"
os.system("mv GoodRuns.json "+jsonsPath+"/"+jsonFileName)
return jsonsPath+"/"+jsonFileName
def makeLatestJSON(jsonsPath,
aDataSet,
runRegCfg="runregTemplate.cfg",
runRegPy="runregparse.py"):
##Check last analysed run number
maxRun = 0
for fname in os.listdir(jsonsPath):
jsonFile = jsonsPath + "/" + fname
jsonList = LumiList(filename=jsonFile)
runList = jsonList.getRuns()
for run in runList:
if int(run) > maxRun:
maxRun = int(run)
print "Last analysed run is:", maxRun
print "Preparinng JSON for runs:", maxRun + 1, 999999
file = open(runRegCfg, "r") #Opens the file in read-mode
text = file.read()
file.close()
text1 = text.replace("RunMin=", "RunMin=" + str(maxRun + 1))
text2 = text1.replace("DBS_PDS=",
"DBS_PDS=" + aDataSet) # enable cross check with DBS
file = open("runreg.cfg", "w")
file.write(text2)
file.close()
##Create the JSON
os.system(runRegPy)
# os.system(os.getenv("CMSSW_BASE")+"/src/UserCode/L1RpcTriggerAnalysis/test/runregparse.py")
##Rename JSON file to include run range
jsonFile = "GoodRuns.json"
jsonList = LumiList(filename=jsonFile)
runList = jsonList.getRuns()
minRun = int(runList[0])
maxRun = int(runList[len(runList) - 1])
jsonFileName = "GoodRuns_" + str(minRun) + "-" + str(maxRun) + ".json"
os.system("mv GoodRuns.json " + jsonsPath + "/" + jsonFileName)
return jsonsPath + "/" + jsonFileName
def countFiles(jsonFile, samplePath):
jsonList = LumiList(filename=jsonFile)
runList = jsonList.getRuns()
base = "/castor/cern.ch/cms/"
counter = 0
for run in runList:
path = base + "/" + samplePath + "/" + run[0:3] + "/" + run[3:len(run)]
#command = "nsls "+path+" | grep .root | wc -l"
command1 = "nsls " + path
if commands.getstatusoutput(command1)[0] != 0:
print "Skipping run: " + run
continue
command = command1 + " | grep .root | wc -l"
files = commands.getstatusoutput(command)[1]
counter += int(files)
return counter
def countFiles(jsonFile,samplePath):
jsonList = LumiList(filename = jsonFile)
runList = jsonList.getRuns()
base="/castor/cern.ch/cms/"
counter = 0
for run in runList:
path = base+"/"+samplePath+"/"+run[0:3]+"/"+run[3:len(run)]
#command = "nsls "+path+" | grep .root | wc -l"
command1 = "nsls "+path
if commands.getstatusoutput(command1)[0] != 0:
print "Skipping run: "+run
continue
command = command1+" | grep .root | wc -l"
files = commands.getstatusoutput(command)[1]
counter+=int(files)
return counter
def unpackAndMergeFiles(datasetpath, prefix, castor):
###
pwd = os.getenv("PWD")
minRun = 0
maxRun = 0
sampleName = "None"
## Verify that CRAB directory exists
pathCrabFull = ""
pathCrabShort = ""
for fname in os.listdir(prefix + datasetpath):
if re.search("crab_0_", fname) != None:
print fname
if pathCrabFull == "":
pathCrabFull = prefix + datasetpath + "/" + fname
pathCrabShort = fname
else:
print "ERROR: More than one CRAB directory exists in", prefix + datasetpath
print " Only one is allowed!"
os.abort()
## Unpack and merge the results
if pathCrabFull != "":
#Get min and max number of analysed runs
jsonFile = prefix + datasetpath + "/goodRuns.json"
jsonList = LumiList(filename=jsonFile)
runList = jsonList.getRuns()
minRun = int(runList[0])
maxRun = int(runList[len(runList) - 1])
## Create TEMP dir
pathTMP = "/tmp/" + os.getenv("USER") + "/_HADD_TMP_/"
os.system("rm -rf /tmp/${USER}/_HADD_TMP_")
os.system("mkdir -p " + pathTMP + prefix + datasetpath)
pathCrabResults = pathCrabFull + "/res/"
pathCrabResultsTMP = pathTMP + prefix + datasetpath + "/" + pathCrabShort + "/res/"
print "CRAB results from:", pathCrabResults, " will be unpacked here:", pathCrabResultsTMP
#Copy CRAB directory to TEMP dir
os.system("cp -r " + prefix + "/. " + pathTMP + prefix + "/.")
## Copy all but .tgz files (if any)
##os.system("cd "+pathCrabResults+";for i in efficiencyTree_*.root rawMonitor_*.root synchroAnalysis_*.root crab_*.xml CMSSW_*.stdout CMSSW_*.stderr ; do cp $i /tmp/${USER}/_HADD_TMP_; done; ls -lart ; cd -")
## Extract .tgz files (if any)
##os.system("cd "+pathCrabResults+";for i in out_files_*.tgz; do tar -xzvf $i -C /tmp/${USER}/_HADD_TMP_; done; ls -lart ; cd -")
os.system("cd " + pathCrabResultsTMP +
";for i in out_files_*.tgz; do tar -xzvf $i -C " +
pathCrabResultsTMP + "; done; ls -lart ; cd -")
## Get missing CRAB job outputs (if any)
## After this step all out_files_*tgz will disappear from TMP dir
os.system("cd " + pathTMP + prefix + datasetpath + "; crab -status")
os.system("cd " + pathTMP + prefix + datasetpath + "; crab -get all")
## Get lumi summary report
## NOTE: 'crab -status' and 'crab -get all steps' are essential
## The lumiSummary.json is created in the original submit dir...
os.system("cd " + pathTMP + prefix + datasetpath + "; crab -report")
##Files to merge
jsonFileName = "GoodRuns_" + str(minRun) + "-" + str(maxRun) + ".json"
jsonFileNameLumiSummary = "lumiSummary-" + str(minRun) + "-" + str(
maxRun) + ".json"
sampleName = "efficiencyTree-" + str(minRun) + "-" + str(
maxRun) + ".root"
sampleName2 = "efficiencyHelper-" + str(minRun) + "-" + str(
maxRun) + ".root"
sampleName3 = "synchroAnalysis-" + str(minRun) + "-" + str(
maxRun) + ".root"
sampleName4 = "rawMonitor-" + str(minRun) + "-" + str(maxRun) + ".root"
sampleName5 = "l1demon-" + str(minRun) + "-" + str(maxRun) + ".root"
sampleName6 = "out-" + str(minRun) + "-" + str(maxRun) + ".txt"
print "Doing hadd for ", sampleName
os.system("cd " + pathCrabResultsTMP +
"; hadd /tmp/${USER}/_HADD_TMP_/" + sampleName +
" efficiencyTree_*.root")
print "Doing hadd for ", sampleName2
os.system("cd " + pathCrabResultsTMP +
"; hadd /tmp/${USER}/_HADD_TMP_/" + sampleName2 +
" efficiencyHelper_*.root")
print "Doing hadd for ", sampleName3
os.system("cd " + pathCrabResultsTMP +
"; hadd /tmp/${USER}/_HADD_TMP_/" + sampleName3 +
" synchroAnalysis_*.root")
print "Doing hadd for ", sampleName4
os.system("cd " + pathCrabResultsTMP +
"; hadd /tmp/${USER}/_HADD_TMP_/" + sampleName4 +
" rawMonitor_*.root ; ls -lart")
print "Doing hadd for ", sampleName5
os.system("cd " + pathCrabResultsTMP +
"; hadd /tmp/${USER}/_HADD_TMP_/" + sampleName5 +
" l1demon_*.root ; ls -lart")
print "Preparing out.txt"
os.system(
"cd " + pathCrabResultsTMP + "; rm -f /tmp/${USER}/_HADD_TMP_/" +
sampleName6 +
"; cat C*.stdout|grep -i lb|grep -i mean >> /tmp/${USER}/_HADD_TMP_/"
+ sampleName6)
##Copy results to TMP destination
print "Copying final results to ", pathTMP + prefix, "/ROOT"
os.system("mkdir -p " + pathTMP + prefix + "/ROOT")
os.system("cp " + prefix + datasetpath + "/goodRuns.json" + " " +
pathTMP + prefix + "/ROOT/" + jsonFileName)
os.system("cp " + pathCrabResults + "/lumiSummary.json" + " " +
pathTMP + prefix + "/ROOT/" + jsonFileNameLumiSummary)
os.system("cp " + pathTMP + sampleName + " " + pathTMP + prefix +
"/ROOT/")
os.system("cp " + pathTMP + sampleName2 + " " + pathTMP + prefix +
"/ROOT/")
os.system("cp " + pathTMP + sampleName3 + " " + pathTMP + prefix +
"/ROOT/")
os.system("cp " + pathTMP + sampleName4 + " " + pathTMP + prefix +
"/ROOT/")
os.system("cp " + pathTMP + sampleName5 + " " + pathTMP + prefix +
"/ROOT/")
os.system("gzip " + pathTMP + sampleName6 + "; cp " + pathTMP +
sampleName6 + ".gz " + pathTMP + prefix + "/ROOT/")
##Copy results to final destination
print "Copying final results to ", prefix, "/ROOT"
os.system("mkdir -p " + prefix + "/ROOT")
os.system("cp " + prefix + datasetpath + "/goodRuns.json" + " " +
prefix + "/ROOT/" + jsonFileName)
os.system("cp " + pathCrabResults + "/lumiSummary.json" + " " +
prefix + "/ROOT/" + jsonFileNameLumiSummary)
os.system("cp /tmp/${USER}/_HADD_TMP_/" + sampleName + " " + prefix +
"/ROOT/")
os.system("cp /tmp/${USER}/_HADD_TMP_/" + sampleName2 + " " + prefix +
"/ROOT/")
os.system("cp /tmp/${USER}/_HADD_TMP_/" + sampleName3 + " " + prefix +
"/ROOT/")
os.system("cp /tmp/${USER}/_HADD_TMP_/" + sampleName4 + " " + prefix +
"/ROOT/")
os.system("cp /tmp/${USER}/_HADD_TMP_/" + sampleName5 + " " + prefix +
"/ROOT/")
os.system("gzip /tmp/${USER}/_HADD_TMP_/" + sampleName6 +
"; cp /tmp/${USER}/_HADD_TMP_/" + sampleName6 + ".gz " +
prefix + "/ROOT/")
##Remove trash from tmp
os.system("rm -rf /tmp/${USER}/_HADD_TMP_/*.root")
os.system("rm -rf /tmp/${USER}/_HADD_TMP_/out*.txt.gz")
##Copy crab files to CASTOR
archiveName = (prefix + datasetpath)[2:]
archiveName = archiveName.replace("/", "_")
archiveDir = archiveName[:-1] + "-" + str(minRun) + "-" + str(maxRun)
archiveName = archiveDir + ".tar.gz"
os.system("cd " + pathTMP + "; tar -cvzf " + pathTMP + archiveName +
" " + pathTMP + prefix + datasetpath + "/*")
castor2 = castor + "/" + archiveDir + "/"
print "Copying archive", archiveName, "to CASTOR path: ", castor2
os.system("rfmkdir " + castor2)
command = "cd " + pathTMP + "; rfcp " + archiveName + " " + castor2 + "; cd -"
os.system(command)
print "Copying merged data", sampleName, "to CASTOR path: ", castor2
command = "cd " + pathTMP + prefix + "/ROOT/; rfcp " + sampleName + " " + castor2 + "; cd -"
os.system(command)
print "Copying merged data", sampleName2, "to CASTOR path: ", castor2
command = "cd " + pathTMP + prefix + "/ROOT/; rfcp " + sampleName2 + " " + castor2 + "; cd -"
os.system(command)
print "Copying merged data", sampleName3, "to CASTOR path: ", castor2
command = "cd " + pathTMP + prefix + "/ROOT/; rfcp " + sampleName3 + " " + castor2 + "; cd -"
os.system(command)
print "Copying merged data", sampleName4, "to CASTOR path: ", castor2
command = "cd " + pathTMP + prefix + "/ROOT/; rfcp " + sampleName4 + " " + castor2 + "; cd -"
os.system(command)
print "Copying merged data", sampleName5, "to CASTOR path: ", castor2
command = "cd " + pathTMP + prefix + "/ROOT/; rfcp " + sampleName5 + " " + castor2 + "; cd -"
os.system(command)
print "Copying merged data", sampleName6 + ".gz", "to CASTOR path: ", castor2
command = "cd " + pathTMP + prefix + "/ROOT/; rfcp " + sampleName6 + ".gz " + castor2 + "; cd -"
os.system(command)
print "Copying JSON files:", jsonFileName, ",", jsonFileNameLumiSummary, "to CASTOR path: ", castor2
command = "cd " + pathTMP + prefix + "/ROOT/; rfcp " + jsonFileName + " " + jsonFileNameLumiSummary + " " + castor2 + "; cd -"
os.system(command)
# Remove TEMP DIR
os.system("rm -rf /tmp/${USER}/_HADD_TMP_")
print "Done. " + '\033[93m' + "Please remove " + prefix + datasetpath + " directory manualy!" + '\033[0m'
def unpackAndMergeFiles(datasetpath,prefix,castor):
###
pwd = os.getenv("PWD")
minRun = 0
maxRun = 0
sampleName = "None"
## Verify that CRAB directory exists
pathCrabFull = ""
pathCrabShort = ""
for fname in os.listdir(prefix+datasetpath):
if re.search("crab_0_",fname)!=None:
print fname
if pathCrabFull == "":
pathCrabFull = prefix+datasetpath+"/"+fname
pathCrabShort = fname
else:
print "ERROR: More than one CRAB directory exists in",prefix+datasetpath
print " Only one is allowed!"
os.abort()
## Unpack and merge the results
if pathCrabFull != "" :
#Get min and max number of analysed runs
jsonFile = prefix+datasetpath+"/goodRuns.json"
jsonList = LumiList(filename = jsonFile)
runList = jsonList.getRuns()
minRun = int(runList[0])
maxRun = int(runList[len(runList)-1])
## Create TEMP dir
pathTMP = "/tmp/"+os.getenv("USER")+"/_HADD_TMP_/"
os.system("rm -rf /tmp/${USER}/_HADD_TMP_")
os.system("mkdir -p "+pathTMP+prefix+datasetpath)
pathCrabResults = pathCrabFull+"/res/"
pathCrabResultsTMP = pathTMP+prefix+datasetpath+"/"+pathCrabShort+"/res/"
print "CRAB results from:",pathCrabResults," will be unpacked here:",pathCrabResultsTMP
#Copy CRAB directory to TEMP dir
os.system("cp -r "+prefix+"/. "+pathTMP+prefix+"/.")
## Copy all but .tgz files (if any)
##os.system("cd "+pathCrabResults+";for i in efficiencyTree_*.root rawMonitor_*.root synchroAnalysis_*.root crab_*.xml CMSSW_*.stdout CMSSW_*.stderr ; do cp $i /tmp/${USER}/_HADD_TMP_; done; ls -lart ; cd -")
## Extract .tgz files (if any)
##os.system("cd "+pathCrabResults+";for i in out_files_*.tgz; do tar -xzvf $i -C /tmp/${USER}/_HADD_TMP_; done; ls -lart ; cd -")
os.system("cd "+pathCrabResultsTMP+";for i in out_files_*.tgz; do tar -xzvf $i -C "+pathCrabResultsTMP+"; done; ls -lart ; cd -")
## Get missing CRAB job outputs (if any)
## After this step all out_files_*tgz will disappear from TMP dir
os.system("cd "+pathTMP+prefix+datasetpath+"; crab -status")
os.system("cd "+pathTMP+prefix+datasetpath+"; crab -get all")
## Get lumi summary report
## NOTE: 'crab -status' and 'crab -get all steps' are essential
## The lumiSummary.json is created in the original submit dir...
os.system("cd "+pathTMP+prefix+datasetpath+"; crab -report")
##Files to merge
jsonFileName = "GoodRuns_"+str(minRun)+"-"+str(maxRun)+".json"
jsonFileNameLumiSummary = "lumiSummary-"+str(minRun)+"-"+str(maxRun)+".json"
sampleName = "efficiencyTree-"+str(minRun)+"-"+str(maxRun)+".root"
sampleName2 = "efficiencyHelper-"+str(minRun)+"-"+str(maxRun)+".root"
sampleName3 = "synchroAnalysis-"+str(minRun)+"-"+str(maxRun)+".root"
sampleName4 = "rawMonitor-"+str(minRun)+"-"+str(maxRun)+".root"
sampleName5 = "l1demon-"+str(minRun)+"-"+str(maxRun)+".root"
sampleName6 = "out-"+str(minRun)+"-"+str(maxRun)+".txt"
print "Doing hadd for ",sampleName
os.system("cd "+pathCrabResultsTMP+"; hadd /tmp/${USER}/_HADD_TMP_/"+sampleName+" efficiencyTree_*.root")
print "Doing hadd for ",sampleName2
os.system("cd "+pathCrabResultsTMP+"; hadd /tmp/${USER}/_HADD_TMP_/"+sampleName2+" efficiencyHelper_*.root")
print "Doing hadd for ",sampleName3
os.system("cd "+pathCrabResultsTMP+"; hadd /tmp/${USER}/_HADD_TMP_/"+sampleName3+" synchroAnalysis_*.root")
print "Doing hadd for ",sampleName4
os.system("cd "+pathCrabResultsTMP+"; hadd /tmp/${USER}/_HADD_TMP_/"+sampleName4+" rawMonitor_*.root ; ls -lart")
print "Doing hadd for ",sampleName5
os.system("cd "+pathCrabResultsTMP+"; hadd /tmp/${USER}/_HADD_TMP_/"+sampleName5+" l1demon_*.root ; ls -lart")
print "Preparing out.txt"
os.system("cd "+pathCrabResultsTMP+"; rm -f /tmp/${USER}/_HADD_TMP_/"+sampleName6+"; cat C*.stdout|grep -i lb|grep -i mean >> /tmp/${USER}/_HADD_TMP_/"+sampleName6)
##Copy results to TMP destination
print "Copying final results to ",pathTMP+prefix,"/ROOT"
os.system("mkdir -p "+pathTMP+prefix+"/ROOT")
os.system("cp "+prefix+datasetpath+"/goodRuns.json"+" "+pathTMP+prefix+"/ROOT/"+jsonFileName)
os.system("cp "+pathCrabResults+"/lumiSummary.json"+" "+pathTMP+prefix+"/ROOT/"+jsonFileNameLumiSummary)
os.system("cp "+pathTMP+sampleName+" "+pathTMP+prefix+"/ROOT/")
os.system("cp "+pathTMP+sampleName2+" "+pathTMP+prefix+"/ROOT/")
os.system("cp "+pathTMP+sampleName3+" "+pathTMP+prefix+"/ROOT/")
os.system("cp "+pathTMP+sampleName4+" "+pathTMP+prefix+"/ROOT/")
os.system("cp "+pathTMP+sampleName5+" "+pathTMP+prefix+"/ROOT/")
os.system("gzip "+pathTMP+sampleName6+"; cp "+pathTMP+sampleName6+".gz "+pathTMP+prefix+"/ROOT/")
##Copy results to final destination
print "Copying final results to ",prefix,"/ROOT"
os.system("mkdir -p "+prefix+"/ROOT")
os.system("cp "+prefix+datasetpath+"/goodRuns.json"+" "+ prefix+"/ROOT/"+jsonFileName)
os.system("cp "+pathCrabResults+"/lumiSummary.json"+" "+ prefix+"/ROOT/"+jsonFileNameLumiSummary)
os.system("cp /tmp/${USER}/_HADD_TMP_/"+sampleName+" "+prefix+"/ROOT/")
os.system("cp /tmp/${USER}/_HADD_TMP_/"+sampleName2+" "+prefix+"/ROOT/")
os.system("cp /tmp/${USER}/_HADD_TMP_/"+sampleName3+" "+prefix+"/ROOT/")
os.system("cp /tmp/${USER}/_HADD_TMP_/"+sampleName4+" "+prefix+"/ROOT/")
os.system("cp /tmp/${USER}/_HADD_TMP_/"+sampleName5+" "+prefix+"/ROOT/")
os.system("gzip /tmp/${USER}/_HADD_TMP_/"+sampleName6+"; cp /tmp/${USER}/_HADD_TMP_/"+sampleName6+".gz "+prefix+"/ROOT/")
##Remove trash from tmp
os.system("rm -rf /tmp/${USER}/_HADD_TMP_/*.root")
os.system("rm -rf /tmp/${USER}/_HADD_TMP_/out*.txt.gz")
##Copy crab files to CASTOR
archiveName = (prefix+datasetpath)[2:]
archiveName = archiveName.replace("/","_")
archiveDir = archiveName[:-1]+"-"+str(minRun)+"-"+str(maxRun)
archiveName = archiveDir+".tar.gz"
os.system("cd "+pathTMP+"; tar -cvzf "+pathTMP+archiveName+" "+pathTMP+prefix+datasetpath+"/*")
castor2 = castor+"/"+archiveDir+"/"
print "Copying archive",archiveName,"to CASTOR path: ",castor2
os.system("rfmkdir "+castor2)
command = "cd "+pathTMP+"; rfcp "+archiveName+" "+castor2+"; cd -"
os.system(command)
print "Copying merged data",sampleName,"to CASTOR path: ",castor2
command = "cd "+pathTMP+prefix+"/ROOT/; rfcp "+sampleName+" "+castor2+"; cd -"
os.system(command)
print "Copying merged data",sampleName2,"to CASTOR path: ",castor2
command = "cd "+pathTMP+prefix+"/ROOT/; rfcp "+sampleName2+" "+castor2+"; cd -"
os.system(command)
print "Copying merged data",sampleName3,"to CASTOR path: ",castor2
command = "cd "+pathTMP+prefix+"/ROOT/; rfcp "+sampleName3+" "+castor2+"; cd -"
os.system(command)
print "Copying merged data",sampleName4,"to CASTOR path: ",castor2
command = "cd "+pathTMP+prefix+"/ROOT/; rfcp "+sampleName4+" "+castor2+"; cd -"
os.system(command)
print "Copying merged data",sampleName5,"to CASTOR path: ",castor2
command = "cd "+pathTMP+prefix+"/ROOT/; rfcp "+sampleName5+" "+castor2+"; cd -"
os.system(command)
print "Copying merged data",sampleName6+".gz","to CASTOR path: ",castor2
command = "cd "+pathTMP+prefix+"/ROOT/; rfcp "+sampleName6+".gz "+castor2+"; cd -"
os.system(command)
print "Copying JSON files:",jsonFileName,",",jsonFileNameLumiSummary,"to CASTOR path: ",castor2
command = "cd "+pathTMP+prefix+"/ROOT/; rfcp "+jsonFileName+" "+jsonFileNameLumiSummary+" "+castor2+"; cd -"
os.system(command)
# Remove TEMP DIR
os.system("rm -rf /tmp/${USER}/_HADD_TMP_")
print "Done. "+'\033[93m'+"Please remove "+prefix+datasetpath+" directory manualy!"+'\033[0m'
