def main(): inputpath = "output/ntuple*.root" sys.path.append(os.path.abspath(os.path.curdir)) from Core import NTupleTools, parseInputArgs, listifyInputFiles options = parseInputArgs() try: inputs = listifyInputFiles(options.inFiles) except: inputs = listifyInputFiles(inputpath) cutflow = NTupleTools.getCutFlowFromHistogram(inputs) try: suppress = options.cutFlowSuppress.split(",") except: suppress = ["NoControl", "NoSys"] fullsuppress = [] fulldename = [] for s in suppress: if s in defaults_suppress: fullsuppress += defaults_suppress[s] if s in defaults_dename: fulldename += defaults_dename[s] else: print "ERROR: No such streamlet group:", s print "See Scripts/PrintCutFlow.py for all streamlet groups or to define new ones" print "Exiting script..." sys.exit() NTupleTools.printCutFlow(cutflow, suppressStreamlet=fullsuppress, denameStreamlet=fulldename)
def main(): inputpath = 'output/ntuple*.root' sys.path.append(os.path.abspath(os.path.curdir)) from Core import NTupleTools, parseInputArgs, listifyInputFiles options = parseInputArgs() try: inputs = listifyInputFiles(options.inFiles) except: inputs = listifyInputFiles(inputpath) cutflow = NTupleTools.getCutFlowFromHistogram(inputs) try: suppress = options.cutFlowSuppress.split(',') except: suppress = ['NoControl','NoSys'] fullsuppress = [] fulldename = [] for s in suppress: if s in defaults_suppress: fullsuppress += defaults_suppress[s] if s in defaults_dename: fulldename += defaults_dename[s] else: print "ERROR: No such streamlet group:", s print "See Scripts/PrintCutFlow.py for all streamlet groups or to define new ones" print "Exiting script..." sys.exit() NTupleTools.printCutFlow(cutflow, suppressStreamlet=fullsuppress, denameStreamlet=fulldename)
def setupFlowChain(self, flowlist): from ROOT import AnalysisFramework systematicsPartners = {} systematicsList = [] systematicsOff = [] systematicsOn = [] for syslist in self.systematics: if not type(syslist) in [list, tuple]: syslist = [syslist] else: friendflows = [ flow for flow in flowlist if flow.name in syslist ] for flow in friendflows: systematicsPartners[flow.name] = [ s for s in syslist if not s == flow.name ] sharedVec = makeVectorString( systematicsPartners[flow.name]) flow.systematicsPartners = sharedVec self._tempVec.append(sharedVec) for s in syslist: systematicsList.append(s) self.wb.systematics = makeVectorString(systematicsList) for flow in flowlist: if issubclass(flow.__class__, AnalysisFramework.CutFlows.SystematicsItem): if flow.name in systematicsList: systematicsOn.append(flow.name) if (self.wb.isData and flow.doData) or ( self.wb.isMC and flow.doMC) or (self.wb.isEmbedding and flow.doEmbedding): if flow.name in systematicsPartners: treename = '_'.join( sorted([flow.name] + systematicsPartners[flow.name])) else: treename = flow.name flow.treeDownDelayed = self._tempTree.get( "SystematicsDOWN/" + treename, NTupleTools.makeNTuple( "SystematicsDOWN/" + treename, self.ao, self.outputdump)) flow.treeUpDelayed = self._tempTree.get( "SystematicsUP/" + treename, NTupleTools.makeNTuple("SystematicsUP/" + treename, self.ao, self.outputdump)) self._tempTree["SystematicsDOWN/" + treename] = flow.treeDownDelayed self._tempTree["SystematicsUP/" + treename] = flow.treeUpDelayed else: if not flow.doNominalWhenOff: flow.doNominalInternal = False flow.doNominalExternal = False systematicsOff.append(flow.name) print '@@@@@ The following systematics are enabled:' for syst in systematicsOn: if syst in systematicsPartners: print ' ', syst, 'with', ', '.join( systematicsPartners[syst]) else: print ' ', syst if not systematicsOn: print ' None!' print "@@@@@ The following systematics are disabled:" for syst in systematicsOff: if syst in systematicsPartners: print ' ', syst, 'with', ', '.join( systematicsPartners[syst]) else: print ' ', syst if not systematicsOff: print ' None!' for syst in systematicsList: if not syst in systematicsOff and not syst in systematicsOn: if syst in systematicsPartners: print ' WARNING: Unrecognised systematic:', syst, 'with', ', '.join( systematicsPartners[syst]) else: print ' WARNING: Unrecognised systematic:', syst toRemove = [] correctionsOff = [] correctionsOn = [] for flow in flowlist: if issubclass(flow.__class__, AnalysisFramework.CutFlows.CorrectionItem): if flow.name in self.corrections: correctionsOn.append(flow.name) else: correctionsOff.append(flow.name) toRemove.append(flow) for flow in toRemove: flow.wb = self.wb flow.ao = self.ao flowlist.remove(flow) print "@@@@@ The following corrections are enabled:" for corr in correctionsOn: print ' ', corr if not correctionsOn: print ' None!' print "@@@@@ The following corrections are disabled:" for corr in correctionsOff: print ' ', corr if not correctionsOff: print ' None!' for corr in self.corrections: if not corr in correctionsOff and not corr in correctionsOn: print " WARNING: Unrecognised correction:", corr self.wb.corrections = makeVectorString(correctionsOn) for flow in flowlist: if flow.name == "TotalEvents": self.totalflow = flow break if not self.totalflow: print "@@@@@ ERROR: Must have a flow named 'TotalEvents' to run!" exit() for i in range(len(flowlist)): flow = flowlist[i] try: flows = list(flow.nextList) flow.wb = self.wb flow.ao = self.ao except: flows = [flow] for flow in flows: if i == 0: flow.hasPrevious = False else: prevflow = flowlist[i - 1] flow.previous = prevflow flow.hasPrevious = True if i + 1 < len(flowlist): nextflow = flowlist[i + 1] flow.next = nextflow flow.hasNext = True else: flow.hasNext = False flow.wb = self.wb flow.ao = self.ao return flowlist[0].process
def getDefaultOutputTree(self): if not self.outputtree: self.outputtree = NTupleTools.makeNTuple(self.treename, self.ao, self.outputdump) return self.outputtree
def run(self, entryNumbers=()): if not self.FlowList: raise Exception("Nothing defined in CutFlow!") firstflow_process = self.setupFlowChain(self.FlowList) self.inputtree = NTupleTools.loadNTuple(self.treename, self.inputpath) NTupleTools.setBranchAddresses(self.inputtree, self.ao) num = self.inputtree.GetEntries() if entryNumbers: start, end = entryNumbers if end > num: end = num allentries = xrange(start, end) else: allentries = xrange(num) printmod = int(round(num / 1000)) wb_resetToDefaultValues = self.wb.resetToDefaultValues ao_resetToDefaultValues = self.ao.resetToDefaultValues ao_forceResetToDefaultValues = self.ao.forceResetToDefaultValues inputtree_GetEntry = self.inputtree.GetEntry inputtree_GetTreeNumber = self.inputtree.GetTreeNumber ao_resetSelected = self.ao.resetSelected outputdump = self.outputdump ao_removeUnselected = self.ao.removeUnselected outputtree_Fill = self.outputtree.Fill sys_stdout_flush = sys.stdout.flush print print "Number of input entries:", num try: current_tree_num = 0 for i in allentries: wb_resetToDefaultValues() ao_resetToDefaultValues() if not inputtree_GetTreeNumber() == current_tree_num: current_tree_num = inputtree_GetTreeNumber() print "Switching to tree number %i in TChain..." % ( current_tree_num) NTupleTools.setBranchAddresses(self.inputtree, self.ao) ao_forceResetToDefaultValues() inputtree_GetEntry(i) ao_resetSelected() if printmod and i % printmod == 0: print "\r%.1f%%" % (i * 100. / num), sys_stdout_flush() passflow = firstflow_process() except Exception, e: crashedflow = None for fl in range(len(self.FlowList)): flow = self.FlowList[fl] try: flows = [flow] + list(flow.nextList) except: flows = [flow] for flow in flows: if flow.running: crashedflow = flow break print "-" * 80 if crashedflow: print "@@@@@ Crash detected at entry", i, "in", crashedflow.name, crashedflow print " Passport :", ' > '.join([ fl.name for fl in crashedflow.runningPassport.flowHistory ]) print " Streamlets :", ' > '.join( list(crashedflow.runningPassport.streamletHistory)) else: print "@@@@@ Crash detected at entry", i, "but unable to determine which flow" print " ErrorMessage :", e print "@@@@@ Starting interactive session so you can inspect your variables...use Ctrl-D to exit" print import code vars = globals().copy() vars.update(locals()) shell = code.InteractiveConsole(vars) shell.interact() sys.exit(1)
class Kernel(TNamed): def __init__(self): TNamed.__init__(self, 'Kernel', 'Kernel') self.FlowList = [] self.wb = None self.ao = None self.dataset = None self.treename = None self.samplename = None self.inputpath = None self.outputpath = None self.outputdump = None self.corrections = [] self.systematics = [] self.totalflow = None self.inputtree = None self.outputtree = None self._tempVec = [] self._tempTree = {} def getDefaultOutputTree(self): if not self.outputtree: self.outputtree = NTupleTools.makeNTuple(self.treename, self.ao, self.outputdump) return self.outputtree def setupFlowChain(self, flowlist): from ROOT import AnalysisFramework systematicsPartners = {} systematicsList = [] systematicsOff = [] systematicsOn = [] for syslist in self.systematics: if not type(syslist) in [list, tuple]: syslist = [syslist] else: friendflows = [ flow for flow in flowlist if flow.name in syslist ] for flow in friendflows: systematicsPartners[flow.name] = [ s for s in syslist if not s == flow.name ] sharedVec = makeVectorString( systematicsPartners[flow.name]) flow.systematicsPartners = sharedVec self._tempVec.append(sharedVec) for s in syslist: systematicsList.append(s) self.wb.systematics = makeVectorString(systematicsList) for flow in flowlist: if issubclass(flow.__class__, AnalysisFramework.CutFlows.SystematicsItem): if flow.name in systematicsList: systematicsOn.append(flow.name) if (self.wb.isData and flow.doData) or ( self.wb.isMC and flow.doMC) or (self.wb.isEmbedding and flow.doEmbedding): if flow.name in systematicsPartners: treename = '_'.join( sorted([flow.name] + systematicsPartners[flow.name])) else: treename = flow.name flow.treeDownDelayed = self._tempTree.get( "SystematicsDOWN/" + treename, NTupleTools.makeNTuple( "SystematicsDOWN/" + treename, self.ao, self.outputdump)) flow.treeUpDelayed = self._tempTree.get( "SystematicsUP/" + treename, NTupleTools.makeNTuple("SystematicsUP/" + treename, self.ao, self.outputdump)) self._tempTree["SystematicsDOWN/" + treename] = flow.treeDownDelayed self._tempTree["SystematicsUP/" + treename] = flow.treeUpDelayed else: if not flow.doNominalWhenOff: flow.doNominalInternal = False flow.doNominalExternal = False systematicsOff.append(flow.name) print '@@@@@ The following systematics are enabled:' for syst in systematicsOn: if syst in systematicsPartners: print ' ', syst, 'with', ', '.join( systematicsPartners[syst]) else: print ' ', syst if not systematicsOn: print ' None!' print "@@@@@ The following systematics are disabled:" for syst in systematicsOff: if syst in systematicsPartners: print ' ', syst, 'with', ', '.join( systematicsPartners[syst]) else: print ' ', syst if not systematicsOff: print ' None!' for syst in systematicsList: if not syst in systematicsOff and not syst in systematicsOn: if syst in systematicsPartners: print ' WARNING: Unrecognised systematic:', syst, 'with', ', '.join( systematicsPartners[syst]) else: print ' WARNING: Unrecognised systematic:', syst toRemove = [] correctionsOff = [] correctionsOn = [] for flow in flowlist: if issubclass(flow.__class__, AnalysisFramework.CutFlows.CorrectionItem): if flow.name in self.corrections: correctionsOn.append(flow.name) else: correctionsOff.append(flow.name) toRemove.append(flow) for flow in toRemove: flow.wb = self.wb flow.ao = self.ao flowlist.remove(flow) print "@@@@@ The following corrections are enabled:" for corr in correctionsOn: print ' ', corr if not correctionsOn: print ' None!' print "@@@@@ The following corrections are disabled:" for corr in correctionsOff: print ' ', corr if not correctionsOff: print ' None!' for corr in self.corrections: if not corr in correctionsOff and not corr in correctionsOn: print " WARNING: Unrecognised correction:", corr self.wb.corrections = makeVectorString(correctionsOn) for flow in flowlist: if flow.name == "TotalEvents": self.totalflow = flow break if not self.totalflow: print "@@@@@ ERROR: Must have a flow named 'TotalEvents' to run!" exit() for i in range(len(flowlist)): flow = flowlist[i] try: flows = list(flow.nextList) flow.wb = self.wb flow.ao = self.ao except: flows = [flow] for flow in flows: if i == 0: flow.hasPrevious = False else: prevflow = flowlist[i - 1] flow.previous = prevflow flow.hasPrevious = True if i + 1 < len(flowlist): nextflow = flowlist[i + 1] flow.next = nextflow flow.hasNext = True else: flow.hasNext = False flow.wb = self.wb flow.ao = self.ao return flowlist[0].process def getCutFlowFromFlowChain(self, flowlist): from ROOT import AnalysisFramework cutflow = [] for i in range(len(flowlist)): flow = flowlist[i] try: flows = list(flow.nextList) except: flows = [flow] atleastone = [ flow for flow in flows if issubclass( flow.__class__, AnalysisFramework.CutFlows.CutItem) ] if not atleastone: continue numbers = {} for flow in flows: for streamlet in flow.streamlets: totalEvents = flow.totalEvents[streamlet] if not streamlet: streamlet = "All" numbers[streamlet] = (totalEvents.passed, totalEvents.passedW) if not numbers: numbers["All"] = (0, 0.0) cutflow.append((flow.name, numbers)) return cutflow def run(self, entryNumbers=()): if not self.FlowList: raise Exception("Nothing defined in CutFlow!") firstflow_process = self.setupFlowChain(self.FlowList) self.inputtree = NTupleTools.loadNTuple(self.treename, self.inputpath) NTupleTools.setBranchAddresses(self.inputtree, self.ao) num = self.inputtree.GetEntries() if entryNumbers: start, end = entryNumbers if end > num: end = num allentries = xrange(start, end) else: allentries = xrange(num) printmod = int(round(num / 1000)) wb_resetToDefaultValues = self.wb.resetToDefaultValues ao_resetToDefaultValues = self.ao.resetToDefaultValues ao_forceResetToDefaultValues = self.ao.forceResetToDefaultValues inputtree_GetEntry = self.inputtree.GetEntry inputtree_GetTreeNumber = self.inputtree.GetTreeNumber ao_resetSelected = self.ao.resetSelected outputdump = self.outputdump ao_removeUnselected = self.ao.removeUnselected outputtree_Fill = self.outputtree.Fill sys_stdout_flush = sys.stdout.flush print print "Number of input entries:", num try: current_tree_num = 0 for i in allentries: wb_resetToDefaultValues() ao_resetToDefaultValues() if not inputtree_GetTreeNumber() == current_tree_num: current_tree_num = inputtree_GetTreeNumber() print "Switching to tree number %i in TChain..." % ( current_tree_num) NTupleTools.setBranchAddresses(self.inputtree, self.ao) ao_forceResetToDefaultValues() inputtree_GetEntry(i) ao_resetSelected() if printmod and i % printmod == 0: print "\r%.1f%%" % (i * 100. / num), sys_stdout_flush() passflow = firstflow_process() except Exception, e: crashedflow = None for fl in range(len(self.FlowList)): flow = self.FlowList[fl] try: flows = [flow] + list(flow.nextList) except: flows = [flow] for flow in flows: if flow.running: crashedflow = flow break print "-" * 80 if crashedflow: print "@@@@@ Crash detected at entry", i, "in", crashedflow.name, crashedflow print " Passport :", ' > '.join([ fl.name for fl in crashedflow.runningPassport.flowHistory ]) print " Streamlets :", ' > '.join( list(crashedflow.runningPassport.streamletHistory)) else: print "@@@@@ Crash detected at entry", i, "but unable to determine which flow" print " ErrorMessage :", e print "@@@@@ Starting interactive session so you can inspect your variables...use Ctrl-D to exit" print import code vars = globals().copy() vars.update(locals()) shell = code.InteractiveConsole(vars) shell.interact() sys.exit(1) self.outputtree.SetWeight(self.inputtree.GetWeight()) print " Done!" print print "Setting tree weight to", self.outputtree.GetWeight() print "Number of entries to write in default tree:", self.outputtree.GetEntries( ) if self.treename.startswith( 'SystematicsUP/') or self.treename.startswith( 'SystematicsDOWN/'): # Do the write operation (might take a while if the tree is huge) self.outputtree.Write() return if self._tempTree: print "Number of entries to write in other trees:" for t in sorted(self._tempTree.keys()): self._tempTree[t].SetWeight(self.inputtree.GetWeight()) print ' %30s : ' % (t), self._tempTree[t].GetEntries() # Make (or copy if exist) the total events histogram totalEventsHistogram = NTupleTools.getTotalEventsHistogram( self.inputpath) if totalEventsHistogram: print "TotalEvents histogram already exist. Using it instead." else: totalEventsHistogram = NTupleTools.makeTotalEventsHistogram( self.totalflow) print "Creating TotalEvents histogram. Bin 1 = raw, Bin 2 = weighted." print "Sample name (" + self.samplename + ") stored in TotalEvents.GetTitle()" totalEventsHistogram.SetTitle(self.samplename) # Save the cutflow print "Creating CutFlow histogram. Bins labeled as FlowNumber/FlowName/Streamlet." cutflow = self.getCutFlowFromFlowChain(self.FlowList) cutFlowHist = NTupleTools.makeCutFlowHistogram(cutflow) # Copy the Lumi XML strings across to the new ntuple NTupleTools.copyLumi(self.inputpath) # Do the write operation (might take a while if the tree is huge) NTupleTools.outputFile.Write() # Move the output file to the output location options = parseInputArgs() if not options.noProof: dirname = os.path.dirname(self.outputpath) basename = '.temp.' + os.path.basename(self.outputpath) if dirname and not os.path.isdir(dirname): os.makedirs(dirname) os.rename(basename, self.outputpath)
def setupFlowChain(self, flowlist): from ROOT import AnalysisFramework systematicsPartners = {} systematicsList = [] systematicsOff = [] systematicsOn = [] for syslist in self.systematics: if not type(syslist) in [list, tuple]: syslist = [syslist] else: friendflows = [flow for flow in flowlist if flow.name in syslist] for flow in friendflows: systematicsPartners[flow.name] = [s for s in syslist if not s == flow.name] sharedVec = makeVectorString(systematicsPartners[flow.name]) flow.systematicsPartners = sharedVec self._tempVec.append(sharedVec) for s in syslist: systematicsList.append(s) self.wb.systematics = makeVectorString(systematicsList) for flow in flowlist: if issubclass(flow.__class__, AnalysisFramework.CutFlows.SystematicsItem): if flow.name in systematicsList: systematicsOn.append(flow.name) if (self.wb.isData and flow.doData) or (self.wb.isMC and flow.doMC) or (self.wb.isEmbedding and flow.doEmbedding): if flow.name in systematicsPartners: treename = '_'.join(sorted([flow.name] + systematicsPartners[flow.name])) else: treename = flow.name flow.treeDownDelayed = self._tempTree.get("SystematicsDOWN/"+treename, NTupleTools.makeNTuple("SystematicsDOWN/"+treename, self.ao, self.outputdump)) flow.treeUpDelayed = self._tempTree.get("SystematicsUP/"+treename, NTupleTools.makeNTuple("SystematicsUP/"+treename, self.ao, self.outputdump)) self._tempTree["SystematicsDOWN/"+treename] = flow.treeDownDelayed self._tempTree["SystematicsUP/"+treename] = flow.treeUpDelayed else: if not flow.doNominalWhenOff: flow.doNominalInternal = False flow.doNominalExternal = False systematicsOff.append(flow.name) print '@@@@@ The following systematics are enabled:' for syst in systematicsOn: if syst in systematicsPartners: print ' ', syst, 'with', ', '.join(systematicsPartners[syst]) else: print ' ', syst
def run(self, entryNumbers = ()): if not self.FlowList: raise Exception("Nothing defined in CutFlow!") firstflow_process = self.setupFlowChain(self.FlowList) self.inputtree = NTupleTools.loadNTuple(self.treename, self.inputpath) NTupleTools.setBranchAddresses(self.inputtree, self.ao) num = self.inputtree.GetEntries() if entryNumbers: start, end = entryNumbers if end > num: end = num allentries = xrange(start, end) else: allentries = xrange(num) printmod = int(round(num/1000)) wb_resetToDefaultValues = self.wb.resetToDefaultValues ao_resetToDefaultValues = self.ao.resetToDefaultValues ao_forceResetToDefaultValues = self.ao.forceResetToDefaultValues inputtree_GetEntry = self.inputtree.GetEntry inputtree_GetTreeNumber = self.inputtree.GetTreeNumber ao_resetSelected = self.ao.resetSelected outputdump = self.outputdump ao_removeUnselected = self.ao.removeUnselected outputtree_Fill = self.outputtree.Fill sys_stdout_flush = sys.stdout.flush print print "Number of input entries:", num try: current_tree_num = 0 for i in allentries: wb_resetToDefaultValues() ao_resetToDefaultValues() if not inputtree_GetTreeNumber() == current_tree_num: current_tree_num = inputtree_GetTreeNumber() print "Switching to tree number %i in TChain..." % (current_tree_num) NTupleTools.setBranchAddresses(self.inputtree, self.ao) ao_forceResetToDefaultValues() inputtree_GetEntry(i) ao_resetSelected() if printmod and i % printmod == 0: print "\r%.1f%%" % (i*100./num), sys_stdout_flush() passflow = firstflow_process() except Exception, e: crashedflow = None for fl in range(len(self.FlowList)): flow = self.FlowList[fl] try: flows = [flow] + list(flow.nextList) except: flows = [flow] for flow in flows: if flow.running: crashedflow = flow break print "-" * 80 if crashedflow: print "@@@@@ Crash detected at entry", i, "in", crashedflow.name, crashedflow print " Passport :", ' > '.join([fl.name for fl in crashedflow.runningPassport.flowHistory]) print " Streamlets :", ' > '.join(list(crashedflow.runningPassport.streamletHistory)) else: print "@@@@@ Crash detected at entry", i, "but unable to determine which flow" print " ErrorMessage :", e print "@@@@@ Starting interactive session so you can inspect your variables...use Ctrl-D to exit" print import code vars = globals().copy() vars.update(locals()) shell = code.InteractiveConsole(vars) shell.interact() sys.exit(1)
print "Setting tree weight to", self.outputtree.GetWeight() print "Number of entries to write in default tree:", self.outputtree.GetEntries() if self.treename.startswith('SystematicsUP/') or self.treename.startswith('SystematicsDOWN/'): # Do the write operation (might take a while if the tree is huge) self.outputtree.Write() return if self._tempTree: print "Number of entries to write in other trees:" for t in sorted(self._tempTree.keys()): self._tempTree[t].SetWeight(self.inputtree.GetWeight()) print ' %30s : ' % (t), self._tempTree[t].GetEntries() # Make (or copy if exist) the total events histogram totalEventsHistogram = NTupleTools.getTotalEventsHistogram(self.inputpath) if totalEventsHistogram: print "TotalEvents histogram already exist. Using it instead." else: totalEventsHistogram = NTupleTools.makeTotalEventsHistogram(self.totalflow) print "Creating TotalEvents histogram. Bin 1 = raw, Bin 2 = weighted." print "Sample name (" + self.samplename + ") stored in TotalEvents.GetTitle()" totalEventsHistogram.SetTitle(self.samplename) # Save the cutflow print "Creating CutFlow histogram. Bins labeled as FlowNumber/FlowName/Streamlet." cutflow = self.getCutFlowFromFlowChain(self.FlowList) cutFlowHist = NTupleTools.makeCutFlowHistogram(cutflow) # Copy the Lumi XML strings across to the new ntuple NTupleTools.copyLumi(self.inputpath)