def main():
inputpath = "output/ntuple*.root"
sys.path.append(os.path.abspath(os.path.curdir))
from Core import NTupleTools, parseInputArgs, listifyInputFiles
options = parseInputArgs()
try:
inputs = listifyInputFiles(options.inFiles)
except:
inputs = listifyInputFiles(inputpath)
cutflow = NTupleTools.getCutFlowFromHistogram(inputs)
try:
suppress = options.cutFlowSuppress.split(",")
except:
suppress = ["NoControl", "NoSys"]
fullsuppress = []
fulldename = []
for s in suppress:
if s in defaults_suppress:
fullsuppress += defaults_suppress[s]
if s in defaults_dename:
fulldename += defaults_dename[s]
else:
print "ERROR: No such streamlet group:", s
print "See Scripts/PrintCutFlow.py for all streamlet groups or to define new ones"
print "Exiting script..."
sys.exit()
NTupleTools.printCutFlow(cutflow, suppressStreamlet=fullsuppress, denameStreamlet=fulldename)
def main():
inputpath = 'output/ntuple*.root'
sys.path.append(os.path.abspath(os.path.curdir))
from Core import NTupleTools, parseInputArgs, listifyInputFiles
options = parseInputArgs()
try:
inputs = listifyInputFiles(options.inFiles)
except:
inputs = listifyInputFiles(inputpath)
cutflow = NTupleTools.getCutFlowFromHistogram(inputs)
try:
suppress = options.cutFlowSuppress.split(',')
except:
suppress = ['NoControl','NoSys']
fullsuppress = []
fulldename = []
for s in suppress:
if s in defaults_suppress:
fullsuppress += defaults_suppress[s]
if s in defaults_dename:
fulldename += defaults_dename[s]
else:
print "ERROR: No such streamlet group:", s
print "See Scripts/PrintCutFlow.py for all streamlet groups or to define new ones"
print "Exiting script..."
sys.exit()
NTupleTools.printCutFlow(cutflow, suppressStreamlet=fullsuppress, denameStreamlet=fulldename)
def getCutFlowFromHistogram(inputdir):
try:
from ROOT import AnalysisFramework
except:
compileMinimal()
from ROOT import AnalysisFramework
CutFlowHist = AnalysisFramework.CutFlows.CutFlowHist
inputpath = listifyInputFiles(inputdir)
htemp = CutFlowHist("CutFlow", "CutFlow output of AnalysisFramework",
400000, 0, 1)
for d in inputpath:
f = TFile.Open(d)
heach = f.Get("CutFlow")
#for i in range(heach.GetNbinsX()):
# if not heach.GetBinLabel(i+1):
# break
# print i+1, heach.GetBinLabel(i+1)
col = TObjArray()
col.Add(heach)
htemp.Merge(col)
f.Close()
#xaxis = htemp.GetXaxis()
temp = {}
for i in range(htemp.GetNbinsX()):
label = htemp.GetBinLabel(i + 1)
if not label:
continue
flownum = int(label.split('/')[0])
isweighted = label.split('/')[1] == 'W'
flowname = label.split('/')[2]
streamlet = label.split('/')[3]
if isweighted:
raw = 0.
weighted = htemp.GetBinContent(i + 1)
else:
raw = htemp.GetBinContent(i + 1)
weighted = 0.
if not flownum in temp:
temp[flownum] = (flowname, {})
flownametemp, numberstemp = temp[flownum]
if not streamlet in numberstemp:
numberstemp[streamlet] = (raw, weighted)
else:
rawtemp, weightedtemp = numberstemp[streamlet]
numberstemp[streamlet] = (raw + rawtemp, weighted + weightedtemp)
cutflow = []
totalEvents = getTotalEventsHistogram(inputdir)
if totalEvents:
cutflow.append(('OriginalTotalEvents', {
'All': (totalEvents.GetBinContent(1), totalEvents.GetBinContent(2))
}))
for i in sorted(temp.keys()):
cutflow.append(temp[i])
return cutflow
def rerunOne(inputpath, outputpath, samplename, joboptions):
inputpath = [os.path.abspath(p) for p in listifyInputFiles(inputpath)]
inputpath = '"' + ','.join(inputpath) + '"'
command = ' '.join([
'python', joboptions, '-', '-n', '-s', samplename, '-i', inputpath,
'-o', outputpath
])
#subprocess.call(['python', joboptions, '-', '-i', inputpath, '-o', outputpath])
os.system(command)
def getCutFlowFromHistogram(inputdir):
try:
from ROOT import AnalysisFramework
except:
compileMinimal()
from ROOT import AnalysisFramework
CutFlowHist = AnalysisFramework.CutFlows.CutFlowHist
inputpath = listifyInputFiles(inputdir)
htemp = CutFlowHist("CutFlow", "CutFlow output of AnalysisFramework", 400000, 0, 1)
for d in inputpath:
f = TFile.Open(d)
heach = f.Get("CutFlow")
#for i in range(heach.GetNbinsX()):
# if not heach.GetBinLabel(i+1):
# break
# print i+1, heach.GetBinLabel(i+1)
col = TObjArray()
col.Add(heach)
htemp.Merge(col)
f.Close()
#xaxis = htemp.GetXaxis()
temp = {}
for i in range(htemp.GetNbinsX()):
label = htemp.GetBinLabel(i+1)
if not label:
continue
flownum = int(label.split('/')[0])
isweighted = label.split('/')[1] == 'W'
flowname = label.split('/')[2]
streamlet = label.split('/')[3]
if isweighted:
raw = 0.
weighted = htemp.GetBinContent(i+1)
else:
raw = htemp.GetBinContent(i+1)
weighted = 0.
if not flownum in temp:
temp[flownum] = (flowname, {})
flownametemp, numberstemp = temp[flownum]
if not streamlet in numberstemp:
numberstemp[streamlet] = (raw, weighted)
else:
rawtemp, weightedtemp = numberstemp[streamlet]
numberstemp[streamlet] = (raw+rawtemp, weighted+weightedtemp)
cutflow = []
totalEvents = getTotalEventsHistogram(inputdir)
if totalEvents:
cutflow.append( ('OriginalTotalEvents', {'All': (totalEvents.GetBinContent(1), totalEvents.GetBinContent(2))}) )
for i in sorted(temp.keys()):
cutflow.append(temp[i])
return cutflow
def loadNTuple(treename, inputdir):
obs = CodeGenerator.obs
if not obs:
print "Need to setup with a Branch.py (or similar) file first!"
return
location = listifyInputFiles(inputdir)
from ROOT import TChain, AddressOf
tree = TChain(treename)
for l in location:
tree.Add(l)
return tree
def loadNTuple(treename, inputdir):
obs = CodeGenerator.obs
if not obs:
print "Need to setup with a Branch.py (or similar) file first!"
return
location = listifyInputFiles(inputdir)
from ROOT import TChain, AddressOf
tree = TChain(treename)
for l in location:
tree.Add(l)
return tree
def getSystematicNames(inputdir):
inputpath = listifyInputFiles(inputdir)
d = inputpath[0]
f = TFile.Open(d)
systlist = []
l = f.GetDirectory("SystematicsUP")
if l:
keys = l.GetListOfKeys()
for entry in range(keys.GetEntries()):
systlist.append('SystematicsUP/' + keys.At(entry).GetName())
l = f.GetDirectory("SystematicsDOWN")
if l:
keys = l.GetListOfKeys()
for entry in range(keys.GetEntries()):
systlist.append('SystematicsDOWN/' + keys.At(entry).GetName())
return systlist
def getTotalEventsHistogram(inputdir):
inputpath = listifyInputFiles(inputdir)
if outputFile:
outputFile.cd()
totalEventsHistogram = TH1D("TotalEvents", "", 2, 1, 3)
try:
for d in inputpath:
f = TFile.Open(d)
htemp = f.Get("TotalEvents")
if not htemp: htemp = f.Get("cutflow")
else: totalEventsHistogram.SetTitle(htemp.GetTitle())
for i in range(1, 3):
totalEventsHistogram.SetBinContent(i, totalEventsHistogram.GetBinContent(i) + htemp.GetBinContent(i))
f.Close()
return totalEventsHistogram
except:
return None
def getSystematicNames(inputdir):
inputpath = listifyInputFiles(inputdir)
d = inputpath[0]
f = TFile.Open(d)
systlist = []
l = f.GetDirectory("SystematicsUP")
if l:
keys = l.GetListOfKeys()
for entry in range(keys.GetEntries()):
systlist.append('SystematicsUP/' + keys.At(entry).GetName())
l = f.GetDirectory("SystematicsDOWN")
if l:
keys = l.GetListOfKeys()
for entry in range(keys.GetEntries()):
systlist.append('SystematicsDOWN/' + keys.At(entry).GetName())
return systlist
def copyLumi(inputdir):
inputpath = listifyInputFiles(inputdir)
lumidir = outputFile.mkdir("Lumi")
for d in inputpath:
f = TFile.Open(d)
try:
l = f.GetDirectory("Lumi")
keys = l.GetListOfKeys()
for entry in range(keys.GetEntries()):
objstr = l.Get(keys.At(entry).GetName() + ";" + str(keys.At(entry).GetCycle()))
if objstr:
lumidir.cd()
objnew = TObjString(objstr.GetString().Data())
objnew.Write(keys.At(entry).GetName())
except:
pass
f.Close()
outputFile.cd()
def checkDuplicates(inputpath, samples):
original_inputpath = inputpath
inputpath = listifyInputFiles(inputpath)
if not inputpath:
print "ERROR: No inputs here specified!"
return
else:
missingfiles = []
for i in inputpath:
if not os.path.isfile(i):
missingfiles.append(i)
if missingfiles:
print "ERROR: File(s) not found:", ", ".join(missingfiles)
return
cache = {"TOTALLUMI": 0}
errorfiles = []
for i in inputpath:
f = TFile.Open(i)
if not f or f.IsZombie():
errorfiles.append(i)
continue
print "Checking file ", i, "..."
printSampleID(i, samples)
totalevents_EL, totalevents_MetaData, totalweights_MetaData = check(f)
if totalevents_EL != totalevents_MetaData:
print "\t WARNING!\n\t total events from Metadata = ", totalevents_MetaData, "\n\t total sumOfWeights from Metadata = ", totalweights_MetaData, "\n\t total events from EL = ", totalevents_EL, "\n"
else:
print "All good with this sample!\n"
f.Close()
if errorfiles:
print "ERROR in opening the following files:"
for e in errorfiles:
print " ", e
def checkDuplicates(inputpath, samples):
original_inputpath = inputpath
inputpath = listifyInputFiles(inputpath)
if not inputpath:
print "ERROR: No inputs here specified!"
return
else:
missingfiles = []
for i in inputpath:
if not os.path.isfile(i):
missingfiles.append(i)
if missingfiles:
print "ERROR: File(s) not found:", ', '.join(missingfiles)
return
cache={'TOTALLUMI':0}
errorfiles = []
for i in inputpath:
f = TFile.Open(i)
if not f or f.IsZombie():
errorfiles.append(i)
continue
print "Checking file ", i, "..."
printSampleID(i, samples)
totalevents_EL, totalevents_MetaData, totalweights_MetaData = check(f)
if (totalevents_EL != totalevents_MetaData):
print "\t WARNING!\n\t total events from Metadata = ", totalevents_MetaData, "\n\t total sumOfWeights from Metadata = ", totalweights_MetaData,"\n\t total events from EL = ", totalevents_EL, "\n"
else:
print "All good with this sample!\n"
f.Close()
if errorfiles:
print "ERROR in opening the following files:"
for e in errorfiles:
print " ", e
def copyLumi(inputdir):
inputpath = listifyInputFiles(inputdir)
lumidir = outputFile.mkdir("Lumi")
for d in inputpath:
f = TFile.Open(d)
try:
l = f.GetDirectory("Lumi")
keys = l.GetListOfKeys()
for entry in range(keys.GetEntries()):
objstr = l.Get(
keys.At(entry).GetName() + ";" +
str(keys.At(entry).GetCycle()))
if objstr:
lumidir.cd()
objnew = TObjString(objstr.GetString().Data())
objnew.Write(keys.At(entry).GetName())
except:
pass
f.Close()
outputFile.cd()
def getTotalEventsHistogram(inputdir):
inputpath = listifyInputFiles(inputdir)
if outputFile:
outputFile.cd()
totalEventsHistogram = TH1D("TotalEvents", "", 2, 1, 3)
try:
for d in inputpath:
f = TFile.Open(d)
htemp = f.Get("TotalEvents")
if not htemp: htemp = f.Get("cutflow")
else: totalEventsHistogram.SetTitle(htemp.GetTitle())
for i in range(1, 3):
totalEventsHistogram.SetBinContent(
i,
totalEventsHistogram.GetBinContent(i) +
htemp.GetBinContent(i))
f.Close()
return totalEventsHistogram
except:
return None
def mergeOne(inputpath, outputpath, logfile=None, weight=None, cutflow=True):
print("Merging {0} ...\n".format(inputpath)),
original_inputpath = inputpath
inputpath = listifyInputFiles(inputpath)
if not inputpath:
print("ERROR: No inputs here specified!")
if logfile:
logfile.write("ERROR: No inputs found for " + original_inputpath +
"\n")
return
else:
missingfiles = []
for i in inputpath:
if not os.path.isfile(i):
missingfiles.append(i)
if missingfiles:
print("ERROR: File(s) not found:", ', '.join(missingfiles))
if logfile:
logfile.write("ERROR: Missing input files for " +
original_inputpath + ":\n")
for m in missingfiles:
logfile.write(' ' + m + '\n')
return
target = TFile.Open(outputpath, "RECREATE")
path = target.GetPath()
path = path[path.index(':') + 2:]
cache = {'TOTALLUMI': 0}
errorfiles = []
chain = TChain("physics")
for i in inputpath:
f = TFile.Open(i)
if not f or f.IsZombie():
errorfiles.append(i)
continue
# Take tree in file and add it to TChain
chain.Add(i)
print("\nMerging input file: \n{0} \nto target file...".format(i))
recursiveMerge(target, f, path, cache, cutflow)
f.Close()
if errorfiles:
print("ERROR in opening the following files:")
for e in errorfiles:
print " ", e
if logfile:
logfile.write("ERROR: Cannot open input files for " +
original_inputpath + ":\n")
for e in errorfiles:
logfile.write(' ' + e + '\n')
if weight:
totalevents = None
for key in cache:
obj = cache[key]
if (type(obj) == TH1D
or type(obj) == TH1F) and obj.GetName() == 'TotalEventsW':
totalevents = obj.GetBinContent(2)
break
if totalevents:
for key in cache:
obj = cache[key]
if type(obj) == TTree:
obj.SetWeight(weight / totalevents)
print(
"Applying weight - w = ( xsec[fb]*kfactor*filter_eff) / (TOT EVTS W ) = {0}/{1} = {2} [fb]\n"
.format(weight, totalevents, (weight / totalevents)))
target.Write()
del cache
print("Merged {0} files into {1}".format(len(inputpath), outputpath))
merged_tree = target.Get("physics")
if (merged_tree and chain.GetEntries() != merged_tree.GetEntries()):
print(
"******** WARNING! ********\n entries in chain: {0} \n entries in merged TTree: {1} \n*************************"
.format(chain.GetEntries(), merged_tree.GetEntries()))
target.Close()
def mergeOne(inputpath, outputpath, logfile=None, weight=None, cutflow=True):
print "Merging", inputpath, "...",
original_inputpath = inputpath
inputpath = listifyInputFiles(
inputpath) #it solves the wildcards of inputpath in a list of files
if not inputpath:
print "ERROR: No inputs here specified!"
if logfile:
logfile.write("ERROR: No inputs found for " + original_inputpath +
"\n")
return
else:
missingfiles = []
for i in inputpath:
if not os.path.isfile(i):
missingfiles.append(i)
if missingfiles:
print "ERROR: File(s) not found:", ', '.join(missingfiles)
if logfile:
logfile.write("ERROR: Missing input files for " +
original_inputpath + ":\n")
for m in missingfiles:
logfile.write(' ' + m + '\n')
return
target = TFile.Open(outputpath, "RECREATE") #opening output file
path = target.GetPath()
path = path[path.index(':') + 2:]
cache = {'TOTALLUMI': 0}
errorfiles = []
#loop over input files
for i in inputpath:
f = TFile.Open(i)
if not f or f.IsZombie():
errorfiles.append(i)
continue
#print i
recursiveMerge(target, f, path, cache,
cutflow) #function used to merge the files
f.Close()
if errorfiles:
print "ERROR in opening the following files:"
for e in errorfiles:
print " ", e
if logfile:
logfile.write("ERROR: Cannot open input files for " +
original_inputpath + ":\n")
for e in errorfiles:
logfile.write(' ' + e + '\n')
#Setting the ttree weight
if weight:
totalevents = None
for key in cache:
obj = cache[key]
if type(obj) == TH1D and obj.GetName() == 'TotalEvents':
totalevents = obj.GetBinContent(2)
break
if totalevents:
for key in cache:
obj = cache[key]
if type(obj) == TTree:
obj.SetWeight(weight / totalevents)
target.Write()
target.Close()
del cache
print "Merged", len(inputpath), "files into", outputpath
def mergeOne(inputpath, outputpath, logfile=None, weight=None, cutflow=True):
print("Merging {0} ...\n".format(inputpath)),
original_inputpath = inputpath
inputpath = listifyInputFiles(inputpath)
if not inputpath:
print("ERROR: No inputs here specified!")
if logfile:
logfile.write("ERROR: No inputs found for " + original_inputpath + "\n")
return
else:
missingfiles = []
for i in inputpath:
if not os.path.isfile(i):
missingfiles.append(i)
if missingfiles:
print("ERROR: File(s) not found:", ', '.join(missingfiles))
if logfile:
logfile.write("ERROR: Missing input files for " + original_inputpath + ":\n")
for m in missingfiles:
logfile.write(' ' + m + '\n')
return
target = TFile.Open(outputpath, "RECREATE")
path = target.GetPath()
path = path[path.index(':')+2:]
cache={'TOTALLUMI':0}
errorfiles = []
chain = TChain("physics")
for i in inputpath:
f = TFile.Open(i)
if not f or f.IsZombie():
errorfiles.append(i)
continue
# Take tree in file and add it to TChain
chain.Add(i)
print("\nMerging input file: \n{0} \nto target file...".format(i))
recursiveMerge(target, f, path, cache, cutflow)
f.Close()
if errorfiles:
print("ERROR in opening the following files:")
for e in errorfiles:
print " ", e
if logfile:
logfile.write("ERROR: Cannot open input files for " + original_inputpath + ":\n")
for e in errorfiles:
logfile.write(' ' + e + '\n')
if weight:
totalevents = None
for key in cache:
obj = cache[key]
if ( type(obj) == TH1D or type(obj) == TH1F ) and obj.GetName() == 'TotalEventsW':
totalevents = obj.GetBinContent(2)
break
if totalevents:
for key in cache:
obj = cache[key]
if type(obj) == TTree:
obj.SetWeight(weight/totalevents)
print("Applying weight - w = ( xsec[fb]*kfactor*filter_eff) / (TOT EVTS W ) = {0}/{1} = {2} [fb]\n".format(weight,totalevents,(weight/totalevents)))
target.Write()
del cache
print("Merged {0} files into {1}".format(len(inputpath), outputpath))
merged_tree = target.Get("physics")
if ( merged_tree and chain.GetEntries() != merged_tree.GetEntries() ):
print("******** WARNING! ********\n entries in chain: {0} \n entries in merged TTree: {1} \n*************************".format(chain.GetEntries(), merged_tree.GetEntries()))
target.Close()
def rerunOne(inputpath, outputpath, samplename, joboptions):
inputpath = [os.path.abspath(p) for p in listifyInputFiles(inputpath)]
inputpath = '"' + ','.join(inputpath) + '"'
command = ' '.join(['python', joboptions, '-', '-n', '-s', samplename, '-i', inputpath, '-o', outputpath])
#subprocess.call(['python', joboptions, '-', '-i', inputpath, '-o', outputpath])
os.system(command)
def mergeOne(inputpath, outputpath, logfile=None, weight=None, cutflow=True):
print "Merging", inputpath, "...",
original_inputpath = inputpath
inputpath = listifyInputFiles(inputpath)#it solves the wildcards of inputpath in a list of files
if not inputpath:
print "ERROR: No inputs here specified!"
if logfile:
logfile.write("ERROR: No inputs found for " + original_inputpath + "\n")
return
else:
missingfiles = []
for i in inputpath:
if not os.path.isfile(i):
missingfiles.append(i)
if missingfiles:
print "ERROR: File(s) not found:", ', '.join(missingfiles)
if logfile:
logfile.write("ERROR: Missing input files for " + original_inputpath + ":\n")
for m in missingfiles:
logfile.write(' ' + m + '\n')
return
target = TFile.Open(outputpath, "RECREATE")#opening output file
path = target.GetPath()
path = path[path.index(':')+2:]
cache={'TOTALLUMI':0}
errorfiles = []
#loop over input files
for i in inputpath:
f = TFile.Open(i)
if not f or f.IsZombie():
errorfiles.append(i)
continue
#print i
recursiveMerge(target, f, path, cache, cutflow)#function used to merge the files
f.Close()
if errorfiles:
print "ERROR in opening the following files:"
for e in errorfiles:
print " ", e
if logfile:
logfile.write("ERROR: Cannot open input files for " + original_inputpath + ":\n")
for e in errorfiles:
logfile.write(' ' + e + '\n')
#Setting the ttree weight
if weight:
totalevents = None
for key in cache:
obj = cache[key]
if type(obj) == TH1D and obj.GetName() == 'TotalEvents':
totalevents = obj.GetBinContent(2)
break
if totalevents:
for key in cache:
obj = cache[key]
if type(obj) == TTree:
obj.SetWeight(weight / totalevents)
target.Write()
target.Close()
del cache
print "Merged", len(inputpath), "files into", outputpath