def __openPoolFile(self, fileName): # hack to prevent ROOT from loading graphic libraries and hence bother # our fellow Mac users if self.verbose==True: print "## importing ROOT..." import PyUtils.RootUtils as ru ROOT = ru.import_root() if self.verbose==True: print "## importing ROOT... [DONE]" # prevent ROOT from being too verbose rootMsg = ShutUp() rootMsg.mute() ROOT.gErrorIgnoreLevel = ROOT.kFatal rootMsg.unMute() #import PyCintex; PyCintex.Cintex.Enable() rootMsg.mute() poolFile = None try: poolFile = ROOT.TFile.Open( fileName, PoolOpts.READ_MODE ) except Exception, e: rootMsg.unMute() print "## Failed to open file [%s] !!" % fileName print "## Reason:" print e print "## Bailing out..." raise IOError, "Could not open file [%s]" % fileName
def getTauWPDecoratorEleBDT(): import PyUtils.RootUtils as ru ROOT = ru.import_root() import cppyy cppyy.loadDictionary('xAODTau_cDict') _name = sPrefix + 'TauWPDecoratorEleBDT' from tauRecTools.tauRecToolsConf import TauWPDecorator TauScoreFlatteningTool = TauWPDecorator( name=_name, flatteningFile1Prong="EleBDTFlat1P.root", #update flatteningFile3Prong= "EleBDTFlat3P.root", #update UseEleBDT=True, ScoreName="BDTEleScore", NewScoreName="BDTEleScoreSigTrans", #dynamic DefineWPs=True, CutEnumVals=[ ROOT.xAOD.TauJetParameters.EleBDTLoose, ROOT.xAOD.TauJetParameters.EleBDTMedium, ROOT.xAOD.TauJetParameters.EleBDTTight ], SigEff1P=[0.95, 0.85, 0.75], SigEff3P=[0.95, 0.85, 0.75], ) cached_instances[_name] = TauScoreFlatteningTool return TauScoreFlatteningTool
def initialize(self): self.msg.info('************************************') self.msg.info('==> initialize %s...', self.name()) self.msg.info('Will apply the following cuts:') self.msg.info(' trigger = %r', self.trigger) self.msg.info(' passAll = %r', self.passAll) self.msg.info('************************************') ## Import needed modules import PyUtils.RootUtils as ru ROOT = ru.import_root() ## Get the TrigDecisionTool tool self.tdt = PyAthena.py_tool('Trig::TrigDecisionTool/TrigDecisionTool', iface='Trig::TrigDecisionTool') if not self.tdt: self.msg.warning('Could not retrieve TrigDecisionTool pointer!') return StatusCode.Success # Set description variable in cut flow branch (default is trigger name, but can override in JO) self.setFilterDescription(self.trigger) ## Initialize event pass counters self.nProcessed = 0 self.nEventPassTrigger = 0 return StatusCode.Success
def __openPoolFile(self, fileName): # hack to prevent ROOT from loading graphic libraries and hence bother # our fellow Mac users if self.verbose==True: print "## importing ROOT..." import PyUtils.RootUtils as ru ROOT = ru.import_root() if self.verbose==True: print "## importing ROOT... [DONE]" # prevent ROOT from being too verbose rootMsg = ShutUp() rootMsg.mute() ROOT.gErrorIgnoreLevel = ROOT.kFatal rootMsg.unMute() import PyCintex PyCintex.Cintex.Enable() rootMsg.mute() poolFile = None try: poolFile = ROOT.TFile.Open( fileName, PoolOpts.READ_MODE ) except Exception, e: rootMsg.unMute() print "## Failed to open file [%s] !!" % fileName print "## Reason:" print e print "## Bailing out..." raise IOError, "Could not open file [%s]" % fileName
def __init__(self): import PyUtils.Logging as _L self._msg = _L.logging.getLogger("AthFile") self.set_msg_lvl(_L.logging.INFO) if os.environ.get('ATHFILE_DEBUG', '0') == '1': import PyUtils.Logging as _L self.set_msg_lvl(_L.logging.VERBOSE) pass self.msg().debug('importing ROOT...') import PyUtils.RootUtils as ru self.pyroot = ru.import_root() try: ru._pythonize_tfile() except Exception as err: self.msg().warning('problem during TFile pythonization:\n%s', err) self.msg().debug('importing ROOT... [done]') # a cache of already processed requests self._cache = {} self._do_pers_cache = True self.enable_pers_cache() return
def initialize(self): self.msg.debug('************************************') self.msg.debug('==> initialize %s...', self.name()) self.msg.debug(' inputContainerType = %r', self.inputContainerType) self.msg.debug(' inputContainerNames = %r', self.inputContainerNames) self.msg.debug(' outputContainerName = %r', self.outputContainerName) self.msg.debug(' removeIdentical = %r', self.removeIdentical) self.msg.debug('************************************') ## Also, declare some counters and initialize them to zero self.nProcessed = 0 self.nObjectsProcessed = 0 self.nObjectsPassed = 0 self.nEventAccepted = 0 ## Import needed modules import PyUtils.RootUtils as ru ROOT = ru.import_root() ## Get the StoreGate service self.storeGateSvc = PyAthena.py_svc('StoreGateSvc') if self.storeGateSvc is None: self.msg.error("Problem retrieving StoreGateSvc pointer !!") return StatusCode.Failure ## import some 4-mom utils import FourMomUtils.Bindings self.utils = {'deltaR': PyAthena.P4Helpers.deltaR} return StatusCode.Success
def getTauWPDecoratorJetRNN(): _name = sPrefix + 'TauWPDecoratorJetRNN' if _name in cached_instances: return cached_instances[_name] import PyUtils.RootUtils as ru ROOT = ru.import_root() import cppyy cppyy.loadDictionary('xAODTau_cDict') from AthenaCommon.AppMgr import ToolSvc from tauRecTools.tauRecToolsConf import TauWPDecorator TauWPDecorator = TauWPDecorator( name=_name, flatteningFile0Prong = "rnnid_flat_0p_v4.root", flatteningFile1Prong = "rnnid_flat_1p_v4.root", flatteningFile3Prong = "rnnid_flat_mp_v4.root", CutEnumVals = [ ROOT.xAOD.TauJetParameters.JetRNNSigVeryLoose, ROOT.xAOD.TauJetParameters.JetRNNSigLoose, ROOT.xAOD.TauJetParameters.JetRNNSigMedium, ROOT.xAOD.TauJetParameters.JetRNNSigTight ], SigEff0P = [0.98, 0.90, 0.65, 0.50], SigEff1P = [0.992, 0.99, 0.965, 0.94], SigEff3P = [0.99, 0.98, 0.865, 0.80], ScoreName = "RNNJetScore", NewScoreName = "RNNJetScoreSigTrans", DefineWPs = True, ) ToolSvc += TauWPDecorator cached_instances[_name] = TauWPDecorator return TauWPDecorator
def getTauWPDecoratorJetBDT(): _name = sPrefix + 'TauWPDecoratorJetBDT' if _name in cached_instances: return cached_instances[_name] import PyUtils.RootUtils as ru ROOT = ru.import_root() import cppyy cppyy.loadDictionary('xAODTau_cDict') from AthenaCommon.AppMgr import ToolSvc from tauRecTools.tauRecToolsConf import TauWPDecorator TauWPDecorator = TauWPDecorator( name=_name, flatteningFile1Prong="FlatJetBDT1P_trigger_v1.root", flatteningFile3Prong="FlatJetBDT3P_trigger_v1.root", CutEnumVals=[ ROOT.xAOD.TauJetParameters.JetBDTSigVeryLoose, ROOT.xAOD.TauJetParameters.JetBDTSigLoose, ROOT.xAOD.TauJetParameters.JetBDTSigMedium, ROOT.xAOD.TauJetParameters.JetBDTSigTight ], SigEff1P=[0.995, 0.99, 0.97, 0.90], SigEff3P=[0.995, 0.94, 0.88, 0.78], ScoreName="BDTJetScore", NewScoreName="BDTJetScoreSigTrans", DefineWPs=True) ToolSvc += TauWPDecorator cached_instances[_name] = TauWPDecorator return TauWPDecorator
def getTauTrackClassifier(): _name = sPrefix + 'TauTrackClassifier' if _name in cached_instances: return cached_instances[_name] from AthenaCommon.AppMgr import ToolSvc from tauRecTools.tauRecToolsConf import tauRecTools__TauTrackClassifier as TauTrackClassifier from tauRecTools.tauRecToolsConf import tauRecTools__TrackMVABDT as TrackMVABDT import PyUtils.RootUtils as ru ROOT = ru.import_root() import cppyy cppyy.loadDictionary('xAODTau_cDict') input_file_name = 'EFtracks_BDT_classifier_v0.root' BDTcut = 0.45 deltaZ0 = 1.0 # ========================================================================= EFtrackBDT = TrackMVABDT( name=_name + "_MVABDT", InputWeightsPath=input_file_name, Threshold=BDTcut, DeltaZ0=deltaZ0, ExpectedFlag=ROOT.xAOD.TauJetParameters.unclassified, inTrigger=True) ToolSvc += EFtrackBDT trackclassifier = TauTrackClassifier(name=_name, Classifiers=[EFtrackBDT]) cached_instances[_name] = trackclassifier return trackclassifier
def initialize(self): self.msg.debug('************************************') self.msg.debug('==> initialize %s...', self.name()) self.msg.debug(' inputContainerType = %r', self.inputContainerType) self.msg.debug(' particleType = %r', self.particleTypeName) self.msg.debug(' inputContainerNames = %r', self.inputContainerNames) self.msg.debug(' outputContainerName = %r', self.outputContainerName) self.msg.debug('************************************') ## Also, declare some counters and initialize them to zero self.nProcessed = 0 self.nObjectsProcessed = 0 self.nObjectsPassed = 0 self.nEventAccepted = 0 ## Import needed modules import PyUtils.RootUtils as ru ROOT = ru.import_root() ## Get the StoreGate service self.storeGateSvc = PyAthena.py_svc('StoreGateSvc') if self.storeGateSvc is None: self.msg.error("Problem retrieving StoreGateSvc pointer!") return StatusCode.Failure gc.disable() return StatusCode.Success
def getTauWPDecoratorJetBDT(): import PyUtils.RootUtils as ru ROOT = ru.import_root() import cppyy cppyy.loadDictionary('xAODTau_cDict') _name = sPrefix + 'TauWPDecoratorJetBDT' from tauRecTools.tauRecToolsConf import TauWPDecorator myTauWPDecorator = TauWPDecorator( name=_name, flatteningFile1Prong="FlatJetBDT1Pv2.root", #update flatteningFile3Prong="FlatJetBDT3Pv2.root", #update CutEnumVals=[ ROOT.xAOD.TauJetParameters.JetBDTSigVeryLoose, ROOT.xAOD.TauJetParameters.JetBDTSigLoose, ROOT.xAOD.TauJetParameters.JetBDTSigMedium, ROOT.xAOD.TauJetParameters.JetBDTSigTight ], SigEff1P=[0.95, 0.85, 0.75, 0.60], SigEff3P=[0.95, 0.75, 0.60, 0.45], ScoreName="BDTJetScore", NewScoreName="BDTJetScoreSigTrans", DefineWPs=True, ) cached_instances[_name] = myTauWPDecorator return myTauWPDecorator
def initialize(self): self.msg.info('************************************') self.msg.info('==> initialize %s...', self.name()) self.msg.info('Will apply the following cuts:') self.msg.info(' EtMinEl = %r', self.cutEtMin) self.msg.info(' IsEmEl = %s', self.cutIsEM) self.msg.info(' AuthorEl = %s', self.cutAuthor) self.msg.info(' EtaMax = %r', self.cutEtaMax) self.msg.info(' removeEtaCrack = %r', self.removeEtaCrack) self.msg.info(' crackEtaMin = %r', self.crackEtaMin) self.msg.info(' crackEtaMax = %r', self.crackEtaMax) self.msg.info(' minNumberPassed = %r', self.minNumberPassed) self.msg.info(' passAll = %r', self.passAll) self.msg.info('************************************') ## Also, declare some counters and initialize them to zero self.nProcessed = 0 self.nElectrons = 0 self.nEventElPassEta = 0 self.nEventElPassEt = 0 self.nEventElPassIsEM = 0 self.nEventElPassAuthor = 0 self.nEventMinNumPassed = 0 ## Import needed modules import PyUtils.RootUtils as ru ROOT = ru.import_root() ## Get the StoreGate service self.storeGateSvc = PyAthena.py_svc('StoreGateSvc') if self.storeGateSvc is None: self.msg.error("Problem retrieving StoreGateSvc pointer !!") return StatusCode.Failure return StatusCode.Success
def initialize(self): self.msg.info('************************************') self.msg.info('==> initialize %s...', self.name()) self.msg.info(' jetCollectionType = %r', self.jetCollectionType) self.msg.info(' jetCollectionName = %r', self.jetCollectionName) self.msg.info(' recordGoodJets = %r', self.recordGoodJets) self.msg.info(' goodJetCollectionName = %r', self.goodJetCollectionName) self.msg.info('Will apply the following cuts:') self.msg.info(' useUncalibratedJets = %r', self.useUncalibratedJets) self.msg.info(' EtMin (GeV) = %r', self.cutEtMin / Units.GeV) self.msg.info(' EtMax (GeV) = %r', self.cutEtMax / Units.GeV) self.msg.info(' EtaMin = %r', self.cutEtaMin) self.msg.info(' EtaMax = %r', self.cutEtaMax) self.msg.info(' cutSumPtTrkMax = %r', self.cutSumPtTrkMax) self.msg.info(' removeOverlap = %r', self.removeOverlap) self.msg.info(' deltaR = %r', self.deltaR) self.msg.info(' leptonContainerTypeList = %r', self.leptonContainerTypeList) self.msg.info(' leptonContainerNameList = %r', self.leptonContainerNameList) self.msg.info(' nLeadingsForCheck = %r', self.nLeadingsForCheck) self.msg.info(' minNumberPassed = %r', self.minNumberPassed) self.msg.info(' passAll = %r', self.passAll) self.msg.info('************************************') ## Initialize the counters self.nProcessed = 0 self.nJets = 0 self.nEventPassed = 0 ## Import needed modules import PyUtils.RootUtils as ru ROOT = ru.import_root() ## Get the StoreGate service self.storeGateSvc = PyAthena.py_svc('StoreGateSvc') if self.storeGateSvc is None: self.msg.error("Problem retrieving StoreGateSvc pointer !!") return StatusCode.Failure ## Eagerly load library due to reflex bug (autoloading enums/namespaces doesn't work) PyAthena.load_library('FourMomUtils') ## Get the JetSignalState helper self.jss = PyAthena.SignalStateHelper(PyAthena.P4SignalState.JETFINAL) ## import some 4-mom utils import FourMomUtils.Bindings self.utils = {'deltaR': PyAthena.P4Helpers.deltaR} return StatusCode.Success
def BDTtoolset(self): if TriggerFlags.run2Config == '2016': from TauDiscriminant.TauDiscriminantConf import TauJetBDT bdt_set = TauJetBDT(inTrigger=True, calibFolder='TrigTauRec/00-11-01/', jetBDT="bdt.2016.bin", jetSigBits="trigger.sig.bits.jet.BDT.txt") return [bdt_set] else: from tauRecTools.tauRecToolsConf import TauJetBDTEvaluator # BDT evaluators 1p / mp bdt_1p = TauJetBDTEvaluator( name="TrigTauJetBDT1P", calibFolder='tauRecTools/00-02-00/', weightsFile='vars2016_pt_gamma_1p_isofix.root', inTrigger=True, minNTracks=0, maxNTracks=1) bdt_mp = TauJetBDTEvaluator( name="TrigTauJetBDTMP", calibFolder='tauRecTools/00-02-00/', weightsFile='vars2016_pt_gamma_3p_isofix.root', inTrigger=True, minNTracks=2, maxNTracks=1000) import PyUtils.RootUtils as ru ROOT = ru.import_root() import cppyy cppyy.loadDictionary('xAODTau_cDict') from tauRecTools.tauRecToolsConf import TauWPDecorator # wp creators 1p / mp wp_decorator = TauWPDecorator( name="TrigTauJetWPDecorator", calibFolder='TrigTauRec/00-11-01/', inTrigger=True, flatteningFile1Prong="FlatJetBDT1P_trigger_v1.root", flatteningFile3Prong="FlatJetBDT3P_trigger_v1.root", CutEnumVals=[ ROOT.xAOD.TauJetParameters.JetBDTSigVeryLoose, ROOT.xAOD.TauJetParameters.JetBDTSigLoose, ROOT.xAOD.TauJetParameters.JetBDTSigMedium, ROOT.xAOD.TauJetParameters.JetBDTSigTight ], SigEff1P=[0.995, 0.99, 0.97, 0.90], SigEff3P=[0.995, 0.94, 0.88, 0.78], ScoreName="BDTJetScore", NewScoreName="BDTJetScoreSigTrans", DefineWPs=True) return [bdt_1p, bdt_mp, wp_decorator]
def checkPFCorruption(filename, verbose=False): if not os.access(filename, os.R_OK): print "ERROR can't access file", filename return -1 ROOT = RootUtils.import_root() from ROOT import TFile, TTree import PyCintex try: f = TFile.Open(filename) except: print "Can't open file", filename return -1 nEvents = n = None keys = f.GetListOfKeys() for k in keys: try: tn = k.GetName() t = f.Get(tn) if not isinstance(t, TTree): return except: print "Can't get tree %s from file %s", tn, fn f.Close() return -1 if (verbose): print "Working on tree", tn n = t.GetEntriesFast() for i in range(n): s = t.GetEntry(i) if s <= 0: print "Tree %s: Found corruption in event %i" % (i, n) f.Close() return -2 else: if verbose and i > 0 and i % 100 == 0: print "Checking event", i print "Tree %s: %i event(s) ok" % (tn, n) # Use CollectionTree determine the number of events if tn == 'CollectionTree': nEvents = n pass #end of loop over trees f.Close() print "ROOT file", filename, "looks ok" if n is None: print "Failed to determine number of events in file %s. No tree named 'CollectionTree'" % filename return 0 return n
def addROOTIncludePaths(): """ Fill ROOT include path list for entries for all packages found in CMTPATH """ # Most projects make their headers available to root under # root-include-path. But DetCommon does not. if 'CMTPATH' in os.environ: import glob import PyUtils.RootUtils as ru interp = ru.import_root().gInterpreter for p in os.environ['CMTPATH'].split(':'): if p.find('DetCommon') >= 0: idir = os.path.join(p, 'InstallArea', 'include') for ii in glob.glob(os.path.join(idir, '*')): interp.AddIncludePath(ii) # Also need to make Geant headers available. if 'G4INCLUDE' in os.environ: import PyUtils.RootUtils as ru interp = ru.import_root().gInterpreter interp.AddIncludePath(os.environ['G4INCLUDE'])
def _cleanup_pyroot(self): import PyUtils.RootUtils as ru root = ru.import_root() tfiles = root.gROOT.GetListOfFiles()[:] for i,f in enumerate(tfiles): try: if f: f.Close() del f except Exception as err: self._msg.info('could not close a TFile:\n%s', err) pass tfiles[:] = []
def __init__(self): # import cintex import PyCintex; PyCintex.Cintex.Enable() # import root import PyUtils.RootUtils as ru ROOT = ru.import_root() self._cxx = ROOT.Ath.DsoDb.instance() # load reflex _load_dict = PyCintex.loadDict _load_dict('ReflexRflx') self._rflx = PyCintex.makeNamespace('Reflex') if not self._rflx: self._rflx = PyCintex.makeNamespace('ROOT::Reflex') return
def __init__(self): # import cintex import PyCintex PyCintex.Cintex.Enable() # import root import PyUtils.RootUtils as ru ROOT = ru.import_root() self._cxx = ROOT.Ath.DsoDb.instance() # load reflex _load_dict = PyCintex.loadDict _load_dict('ReflexRflx') self._rflx = PyCintex.makeNamespace('Reflex') if not self._rflx: self._rflx = PyCintex.makeNamespace('ROOT::Reflex') return
def __init__(self): # import cintex # import PyCintex; PyCintex.Cintex.Enable() import cppyy # import root import PyUtils.RootUtils as ru ROOT = ru.import_root() self._cxx = ROOT.Ath.DsoDb.instance() # if int(cppyy.get_version().split('.')[0]) < 6: if hasattr(cppyy, 'hasCintex'): # load reflex for ROOT ver<6 _load_dict = cppyy.loadDict _load_dict('ReflexRflx') self._rflx = cppyy.makeNamespace('Reflex') if not self._rflx: self._rflx = cppyy.makeNamespace('ROOT::Reflex') return
def main(args): """dump the content of a ROOT file into an ASCII format. """ import PyUtils.RootUtils as ru root = ru.import_root() _inspect = root.RootUtils.PyROOTInspector.pyroot_inspect2 import PyUtils.Logging as L msg = L.logging.getLogger('dump-root') msg.setLevel(L.logging.INFO) msg.info('fname: [%s]', args.fname) root_file = root.TFile.Open(args.fname) if (root_file is None or not isinstance(root_file, root.TFile) or not root_file.IsOpen()): msg.error('could not open [%s]', args.fname) return 1 tree_names = [] if args.tree_name: tree_names = args.tree_name.split(',') else: tree_names = [] keys = [k.GetName() for k in root_file.GetListOfKeys()] for k in keys: o = root_file.Get(k) if isinstance(o, root.TTree): tree_names.append(k) msg.info('dumping trees: %s', tree_names) rc = 0 for tree_name in tree_names: f = ru.RootFileDumper(args.fname, tree_name) nentries = f.tree.GetEntries() if args.entries: nentries = args.entries for d in f.dump(tree_name, nentries): tree_name, ientry, name, data = d n = '.'.join(map(str, [tree_name,"%03i"%ientry]+name)) print '%s %r' %(n, data) return 0
def extract_streams_from_tag (fname, tree_name="POOLCollectionTree", nentries=-1, stream_refs=None): """find the GUID(s) of a list of streams which are contained inside a TAG file. @params: `fname` the filename of the TAG file to inspect (can be a LFN or FID) `tree_name` the name of the TTree containing the stream-refs ('CollectionTree' is the usual default) `nentries` the number of entries to inspect, among the ttree entries (-1: all the entries) `stream_refs` a list of stream names (ex: ['StreamAOD_ref', 'Stream1_ref'] or, None - to inspect all the stream refs in the TAG) example: >>> streams = extract_streams_from_tag ('tag.pool') >>> from pprint import pprint >>> pprint (streams) {'Stream1_ref': ['BEE16671-B9F6-DA11-8219-00304871B611'], 'StreamAOD_ref': ['96F3018E-A0AC-DD11-8115-000423D59D52'], 'StreamESD_ref': ['384D0CFC-9FAC-DD11-A905-000423D59D52'], 'StreamRDO_ref': ['22C5BD99-3059-DB11-8D44-0030488365E6']} >>> from PoolFile import PoolFileCatalog as pfc >>> pprint (pfc().pfn(streams['StreamAOD_ref'][0])) ['aod.pool'] """ import sys import PyUtils.RootUtils as ru ROOT = ru.import_root() import PyCintex; PyCintex.Cintex.Enable() print "::: opening file [%s]..." % fname # get the "final" file name (handles all kind of protocols) import PyUtils.AthFile as af try: protocol, fname = af.server.fname(fname) except Exception,err: print "::: warning: problem extracting file name from PoolFileCatalog" print "::: warning: will use [%s]" % fname
def extract_streams_from_tag (fname, tree_name="POOLCollectionTree", nentries=-1, stream_refs=None): """find the GUID(s) of a list of streams which are contained inside a TAG file. @params: `fname` the filename of the TAG file to inspect (can be a LFN or FID) `tree_name` the name of the TTree containing the stream-refs ('CollectionTree' is the usual default) `nentries` the number of entries to inspect, among the ttree entries (-1: all the entries) `stream_refs` a list of stream names (ex: ['StreamAOD_ref', 'Stream1_ref'] or, None - to inspect all the stream refs in the TAG) example: >>> streams = extract_streams_from_tag ('tag.pool') >>> from pprint import pprint >>> pprint (streams) {'Stream1_ref': ['BEE16671-B9F6-DA11-8219-00304871B611'], 'StreamAOD_ref': ['96F3018E-A0AC-DD11-8115-000423D59D52'], 'StreamESD_ref': ['384D0CFC-9FAC-DD11-A905-000423D59D52'], 'StreamRDO_ref': ['22C5BD99-3059-DB11-8D44-0030488365E6']} >>> from PoolFile import PoolFileCatalog as pfc >>> pprint (pfc().pfn(streams['StreamAOD_ref'][0])) ['aod.pool'] """ import sys import PyUtils.RootUtils as ru ROOT = ru.import_root() #import PyCintex; PyCintex.Cintex.Enable() print "::: opening file [%s]..." % fname # get the "final" file name (handles all kind of protocols) import PyUtils.AthFile as af try: protocol, fname = af.server.fname(fname) except Exception,err: print "::: warning: problem extracting file name from PoolFileCatalog" print "::: warning: will use [%s]" % fname
def main(args): """dump the content of a ROOT file into an ASCII format. """ import PyUtils.RootUtils as ru root = ru.import_root() _inspect = root.RootUtils.PyROOTInspector.pyroot_inspect2 import PyUtils.Logging as L msg = L.logging.getLogger('dump-root') msg.setLevel(L.logging.INFO) msg.info('fname: [%s]', args.fname) root_file = root.TFile.Open(args.fname) if (root_file is None or not isinstance(root_file, root.TFile) or not root_file.IsOpen()): msg.error('could not open [%s]', args.fname) return 1 tree_names = [] if args.tree_name: tree_names = args.tree_name.split(',') else: tree_names = [] keys = [k.GetName() for k in root_file.GetListOfKeys()] for k in keys: o = root_file.Get(k) if isinstance(o, root.TTree): tree_names.append(k) msg.info('dumping trees: %s', tree_names) rc = 0 for tree_name in tree_names: f = ru.RootFileDumper(args.fname, tree_name) nentries = f.tree.GetEntries() if args.entries: nentries = args.entries for d in f.dump(tree_name, nentries): tree_name, ientry, name, data = d n = '.'.join(map(str, [tree_name, "%03i" % ientry] + name)) print '%s %r' % (n, data) return 0
def get_runs_from_tagfile(self, fname): # check own cache for this file if fname in self.file_cache.keys(): return self.file_cache[fname] # check file type with AthFile - this should avoid reopening files more times than necessary msg.debug("Checking file %s" % fname) import PyUtils.AthFile as athFile fileinfos = athFile.fopen(fname).fileinfos if not 'TAG' in fileinfos['stream_names']: return [] # this is a TAG file, open it and read all run numbers # fileinfos have only the run number from the first TAG import PyUtils.Helpers as H with H.restricted_ldenviron(projects=['AtlasCore']): import re with H.ShutUp(filters=[ re.compile( 'TClass::TClass:0: RuntimeWarning: no dictionary for.*' ), re.compile('.*duplicate entry.*') ]): msg.debug("Opening TAG file %s" % fname) import PyUtils.RootUtils as ru f = ru.import_root().TFile.Open(fname, "read") if f is None or not f: msg.warning("Failed to open TAG file %s" % fname) return [] coll_tree = f.Get('POOLCollectionTree') run_numbers = set() if coll_tree is not None: for row in xrange(0, coll_tree.GetEntries()): coll_tree.GetEntry(row) run_numbers.add(getattr(coll_tree, self.run_attr_name)) del coll_tree f.Close() del f self.file_cache[fname] = run_numbers msg.info("TAG file: %s, found runs: %s" % (fname, str(run_numbers))) return run_numbers
def initialize(self): self.msg.info('************************************') self.msg.info('==> initialize %s...', self.name()) self.msg.info(' jetCollectionTypeList = %r', self.jetCollectionTypeList) self.msg.info(' jetCollectionNameList = %r', self.jetCollectionNameList) self.msg.info('Will apply the following cuts:') self.msg.info(' MetMin = %r', self.cutMetMin) self.msg.info(' minDeltaPhi = %r', self.minDeltaPhi) self.msg.info(' useLeadingJet = %r', self.useLeadingJet) self.msg.info(' requireMet = %r', self.requireMet) self.msg.info('************************************') ## Get the StoreGate service self.storeGateSvc = PyAthena.py_svc('StoreGateSvc') if self.storeGateSvc is None: self.msg.error("Problem retrieving StoreGateSvc pointer !!") return StatusCode.Failure ## Import needed modules import PyUtils.RootUtils as ru ROOT = ru.import_root() ## Eagerly load library due to reflex bug (autoloading enums/namespaces doesn't work) #PyAthena.load_library ('FourMomUtils') ## Get the JetSignalState helper #self.jss = PyAthena.JetSignalState() self.jss = PyAthena.SignalStateHelper(PyAthena.P4SignalState.JETFINAL) ## import some 4-mom utils import FourMomUtils.Bindings self.utils = { 'deltaR': PyAthena.P4Helpers.deltaR, 'deltaEta': PyAthena.P4Helpers.deltaEta, 'deltaPhi': PyAthena.P4Helpers.deltaPhi } return StatusCode.Success
def __openPoolFile(self, fileName): # hack to prevent ROOT from loading graphic libraries and hence bother # our fellow Mac users if self.verbose is True: print("## importing ROOT...") import PyUtils.RootUtils as ru ROOT = ru.import_root() if self.verbose is True: print("## importing ROOT... [DONE]") # prevent ROOT from being too verbose rootMsg = ShutUp() rootMsg.mute() ROOT.gErrorIgnoreLevel = ROOT.kFatal poolFile = None try: poolFile = ROOT.TFile.Open( fileName, PoolOpts.READ_MODE ) except Exception as e: rootMsg.unMute() print("## Failed to open file [%s] !!" % fileName) print("## Reason:") print(e) print("## Bailing out...") raise IOError("Could not open file [%s]" % fileName) rootMsg.unMute() if poolFile is None: print("## Failed to open file [%s] !!" % fileName) msg = "Could not open file [%s]" % fileName raise IOError(msg) self.poolFile = poolFile assert self.poolFile.IsOpen() and not self.poolFile.IsZombie(), \ "Invalid POOL file or a Zombie one" self._fileInfos = { 'name' : self.poolFile.GetName(), 'size' : self.poolFile.GetSize(), } return
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration # @file PyUtils.scripts.diff_root_files # @purpose check that 2 ROOT files have same content (containers and sizes). # @author Sebastien Binet # @date February 2010 __version__ = "$Revision: 681245 $" __doc__ = "check that 2 ROOT files have same content (containers and sizes)." __author__ = "Sebastien Binet" ### imports ------------------------------------------------------------------- import PyUtils.acmdlib as acmdlib import PyUtils.RootUtils as ru ROOT = ru.import_root() ### globals ------------------------------------------------------------------- g_ALLOWED_MODES = ('summary', 'detailed') g_ALLOWED_ERROR_MODES = ('bailout', 'resilient') g_args = None ### classes ------------------------------------------------------------------- ### functions ----------------------------------------------------------------- def _is_summary(): global g_args return g_args.mode == 'summary' def _is_exit_early():
def main(args): """check that 2 ROOT files have same content (containers and sizes) """ global g_args g_args = args import PyUtils.RootUtils as ru root = ru.import_root() import PyUtils.Logging as L msg = L.logging.getLogger('diff-root') msg.setLevel(L.logging.INFO) from PyUtils.Helpers import ShutUp, ROOT6Setup ROOT6Setup() if args.entries == '': args.entries = -1 msg.info('comparing tree [%s] in files:', args.tree_name) msg.info(' old: [%s]', args.old) msg.info(' new: [%s]', args.new) msg.info('ignore leaves: %s', args.ignore_leaves) msg.info('enforce leaves: %s', args.enforce_leaves) msg.info('hacks: %s', args.known_hacks) msg.info('entries: %s', args.entries) msg.info('mode: %s', args.mode) msg.info('error mode: %s', args.error_mode) import PyUtils.Helpers as H with H.ShutUp(): fold = ru.RootFileDumper(args.old, args.tree_name) fnew = ru.RootFileDumper(args.new, args.tree_name) pass def tree_infos(tree, args): nentries = tree.GetEntriesFast() # l.GetBranch().GetName() gives the full leaf path name leaves = [ l.GetBranch().GetName() for l in tree.GetListOfLeaves() if l.GetBranch().GetName() not in args.ignore_leaves ] return { 'entries': nentries, 'leaves': set(leaves), } def diff_tree(fold, fnew, args): infos = { 'old': tree_infos(fold.tree, args), 'new': tree_infos(fnew.tree, args), } nentries = min(infos['old']['entries'], infos['new']['entries']) itr_entries = nentries if args.entries in (-1, '', '-1'): #msg.info('comparing over [%s] entries...', nentries) itr_entries = nentries if infos['old']['entries'] != infos['new']['entries']: msg.info('different numbers of entries:') msg.info(' old: [%s]', infos['old']['entries']) msg.info(' new: [%s]', infos['new']['entries']) msg.info('=> comparing [%s] first entries...', nentries) else: itr_entries = args.entries pass msg.info('comparing over [%s] entries...', itr_entries) old_leaves = infos['old']['leaves'] - infos['new']['leaves'] if old_leaves: msg.warning('the following variables exist only in the old file !') for l in old_leaves: msg.warning(' - [%s]', l) new_leaves = infos['new']['leaves'] - infos['old']['leaves'] if new_leaves: msg.warning('the following variables exist only in the new file !') for l in new_leaves: msg.warning(' - [%s]', l) skip_leaves = old_leaves | new_leaves | set(args.ignore_leaves) leaves = infos['old']['leaves'] & infos['new']['leaves'] msg.info('comparing [%s] leaves over entries...', len(leaves)) all_good = True n_good = 0 n_bad = 0 import collections from itertools import izip summary = collections.defaultdict(int) old_dump_iter = fold.dump(args.tree_name, itr_entries) new_dump_iter = fnew.dump(args.tree_name, itr_entries) def leafname_fromdump(entry): return '.'.join([s for s in entry[2] if not s.isdigit()]) def reach_next(dump_iter, skip_leaves): keep_reading = True while keep_reading: try: entry = dump_iter.next() except StopIteration: return None entry[2][0] = entry[2][0].rstrip('.\0') # clean branch name name = [] skip = False for n in leafname_fromdump(entry).split('.'): name.append(n) if '.'.join(name) in skip_leaves: skip = True break if not skip: return entry # print 'SKIP:', leafname_fromdump(entry) pass read_old = True read_new = True d_old = None d_new = None while True: if read_old: prev_d_old = d_old d_old = reach_next(old_dump_iter, skip_leaves) if read_new: prev_d_new = d_new d_new = reach_next(new_dump_iter, skip_leaves) if not d_new and not d_old: break read_old = True read_new = True if d_old == d_new: n_good += 1 continue if d_old: tree_name, ientry, name, iold = d_old if d_new: tree_name, ientry, name, inew = d_new # FIXME: that's a plain (temporary?) hack if name[-1] in args.known_hacks: continue n_bad += 1 in_synch = d_old and d_new and d_old[:-1] == d_new[:-1] if not in_synch: if not _is_summary(): if d_old: print '::sync-old %s' % '.'.join(["%03i" % ientry] + map(str, d_old[2])) else: print '::sync-old ABSENT' if d_new: print '::sync-new %s' % '.'.join(["%03i" % ientry] + map(str, d_new[2])) else: print '::sync-new ABSENT' pass # remember for later if not d_old: fold.allgood = False summary[d_new[2][0]] += 1 elif not d_new: fnew.allgood = False summary[d_old[2][0]] += 1 else: branch_old = '.'.join(["%03i" % ientry, d_old[2][0]]) branch_new = '.'.join(["%03i" % ientry, d_new[2][0]]) if branch_old < branch_new: if not _is_summary(): print '::sync-old skipping entry' summary[d_old[2][0]] += 1 fnew.allgood = False read_new = False elif branch_old > branch_new: if not _is_summary(): print '::sync-new skipping entry' summary[d_new[2][0]] += 1 fold.allgood = False read_old = False else: # MN: difference in the leaves prev_leaf_old = leafname_fromdump(prev_d_old) prev_leaf_new = leafname_fromdump(prev_d_new) leaf_old = leafname_fromdump(d_old) leaf_new = leafname_fromdump(d_new) if prev_leaf_old == prev_leaf_new: # array size difference? if leaf_old == leaf_new and leaf_old == prev_leaf_old: # could be a size difference in >1 dim arrays # hard to sync, skipping both pass elif leaf_old == prev_leaf_old: # old has bigger array, skip old entry read_new = False if not _is_summary(): print '::sync-old skipping entry' summary[leaf_old] += 1 elif leaf_new == prev_leaf_new: # new has bigger array, skip new entry read_old = False if not _is_summary(): print '::sync-new skipping entry' summary[leaf_new] += 1 if read_old and read_new: summary[d_new[2][0]] += 1 if not _is_summary(): print '::sync-old+new skipping both entries' fold.allgood = False fnew.allgood = False if _is_exit_early(): print "*** exit on first error ***" break continue n = '.'.join(map(str, ["%03i" % ientry] + name)) diff_value = 'N/A' try: diff_value = 50. * (iold - inew) / (iold + inew) diff_value = '%.8f%%' % (diff_value, ) except Exception: pass if not _is_summary(): print '%s %r -> %r => diff= [%s]' % (n, iold, inew, diff_value) pass summary[leafname_fromdump(d_old)] += 1 if name[0] in args.enforce_leaves: msg.info("don't compare further") all_good = False break pass # loop over events/branches msg.info('Found [%s] identical leaves', n_good) msg.info('Found [%s] different leaves', n_bad) if not _is_summary(): keys = sorted(summary.keys()) for n in keys: v = summary[n] msg.info(' [%s]: %i leaves differ', n, v) pass pass if (not fold.allgood) or (not fnew.allgood): msg.info('NOTE: there were errors during the dump') msg.info('fold.allgood: %s' % fold.allgood) msg.info('fnew.allgood: %s' % fnew.allgood) n_bad += 0.5 return n_bad ndiff = diff_tree(fold, fnew, args) if ndiff != 0: msg.info('files differ!') return 2 msg.info('all good.') return 0
def main(args): """check that 2 ROOT files have same content (containers and sizes) """ global g_args g_args = args import PyUtils.RootUtils as ru root = ru.import_root() import PyUtils.Logging as L msg = L.logging.getLogger('diff-root') msg.setLevel(L.logging.INFO) if args.entries == '': args.entries = -1 msg.info('comparing tree [%s] in files:', args.tree_name) msg.info(' old: [%s]', args.old) msg.info(' new: [%s]', args.new) msg.info('ignore leaves: %s', args.ignore_leaves) msg.info('enforce leaves: %s', args.enforce_leaves) msg.info('hacks: %s', args.known_hacks) msg.info('entries: %s', args.entries) msg.info('mode: %s', args.mode) msg.info('error mode: %s', args.error_mode) import PyUtils.Helpers as H with H.ShutUp() : fold = ru.RootFileDumper(args.old, args.tree_name) fnew = ru.RootFileDumper(args.new, args.tree_name) pass def tree_infos(tree, args): nentries = tree.GetEntriesFast() leaves = [l.GetName() for l in tree.GetListOfLeaves() if l not in args.ignore_leaves] return { 'entries' : nentries, 'leaves': set(leaves), } def diff_tree(fold, fnew, args): infos = { 'old' : tree_infos(fold.tree, args), 'new' : tree_infos(fnew.tree, args), } nentries = min(infos['old']['entries'], infos['new']['entries']) itr_entries = nentries if args.entries in (-1,'','-1'): #msg.info('comparing over [%s] entries...', nentries) itr_entries = nentries if infos['old']['entries'] != infos['new']['entries']: msg.info('different numbers of entries:') msg.info(' old: [%s]', infos['old']['entries']) msg.info(' new: [%s]', infos['new']['entries']) msg.info('=> comparing [%s] first entries...', nentries) else: itr_entries = args.entries pass msg.info('comparing over [%s] entries...', itr_entries) leaves = infos['old']['leaves'] & infos['new']['leaves'] diff_leaves = infos['old']['leaves'] - infos['new']['leaves'] if diff_leaves: msg.info('the following variables exist in only one tree !') for l in diff_leaves: msg.info(' - [%s]', l) leaves = leaves - set(args.ignore_leaves) msg.info('comparing [%s] leaves over entries...', len(leaves)) all_good = True n_good = 0 n_bad = 0 import collections from itertools import izip summary = collections.defaultdict(int) for d in izip(fold.dump(args.tree_name, itr_entries), fnew.dump(args.tree_name, itr_entries)): tree_name, ientry, name, iold = d[0] _, _, _, inew = d[1] name[0] = name[0].rstrip('\0') if ((not (name[0] in leaves)) or # FIXME: that's a plain (temporary?) hack name[-1] in args.known_hacks): continue if d[0] == d[1]: diff = False n_good += 1 continue n_bad += 1 diff = True in_synch = d[0][:-1] == d[1][:-1] if not in_synch: if not _is_summary(): print '::sync-old %s' % \ '.'.join(["%03i"%ientry]+map(str, d[0][2])) print '::sync-new %s' % \ '.'.join(["%03i"%ientry]+map(str, d[1][2])) pass summary[name[0]] += 1 # remember for later fold.allgood = False fnew.allgood = False if _is_exit_early(): print "*** exit on first error ***" break continue n = '.'.join(map(str, ["%03i"%ientry]+name)) diff_value = 'N/A' try: diff_value = 50.*(iold-inew)/(iold+inew) diff_value = '%.8f%%' % (diff_value,) except Exception: pass if not _is_summary(): print '%s %r -> %r => diff= [%s]' %(n, iold, inew, diff_value) pass summary[name[0]] += 1 if name[0] in args.enforce_leaves: msg.info("don't compare further") all_good = False break pass # loop over events/branches msg.info('Found [%s] identical leaves', n_good) msg.info('Found [%s] different leaves', n_bad) if not _is_summary(): keys = sorted(summary.keys()) for n in keys: v = summary[n] msg.info(' [%s]: %i leaves differ', n, v) pass pass if (not fold.allgood) or (not fnew.allgood): msg.info('NOTE: there were errors during the dump') msg.info('fold.allgood: %s' % fold.allgood) msg.info('fnew.allgood: %s' % fnew.allgood) n_bad += 0.5 return n_bad ndiff = diff_tree(fold, fnew, args) if ndiff != 0: msg.info('files differ!') return 2 msg.info('all good.') return 0
def extract_streams_from_tag (fname, tree_name="POOLCollectionTree", nentries=-1, stream_refs=None): """find the GUID(s) of a list of streams which are contained inside a TAG file. @params: `fname` the filename of the TAG file to inspect (can be a LFN or FID) `tree_name` the name of the TTree containing the stream-refs ('CollectionTree' is the usual default) `nentries` the number of entries to inspect, among the ttree entries (-1: all the entries) `stream_refs` a list of stream names (ex: ['StreamAOD_ref', 'Stream1_ref'] or, None - to inspect all the stream refs in the TAG) example: >>> streams = extract_streams_from_tag ('tag.pool') >>> from pprint import pprint >>> pprint (streams) {'Stream1_ref': ['BEE16671-B9F6-DA11-8219-00304871B611'], 'StreamAOD_ref': ['96F3018E-A0AC-DD11-8115-000423D59D52'], 'StreamESD_ref': ['384D0CFC-9FAC-DD11-A905-000423D59D52'], 'StreamRDO_ref': ['22C5BD99-3059-DB11-8D44-0030488365E6']} >>> from PoolFile import PoolFileCatalog as pfc >>> pprint (pfc().pfn(streams['StreamAOD_ref'][0])) ['aod.pool'] """ import PyUtils.RootUtils as ru ROOT = ru.import_root() print("::: opening file [%s]..." % fname) # get the "final" file name (handles all kind of protocols) import PyUtils.AthFile as af try: protocol, fname = af.server.fname(fname) except Exception: print("::: warning: problem extracting file name from PoolFileCatalog") print("::: warning: will use [%s]" % fname) f = ROOT.TFile.Open (fname, "read") assert not f.IsZombie() and f.IsOpen(), \ "problem opening POOL file [%s]"%fname # backward compat: # in 15.2.0 TAG file got a new tree name 'POOLCollectionTree' # it was named 'CollectionTree' before that... keys = [k.GetName() for k in f.GetListOfKeys()] if tree_name not in keys and "CollectionTree" not in keys: err= "::: error: neither [%s] nor [CollectionTree] in file [%s]" % ( tree_name, fname) print(err) raise RuntimeError(err) # try the backward compat. hack if tree_name not in keys: tree_name = "CollectionTree" t = f.Get(tree_name) assert isinstance(t, ROOT.TTree), \ "could not retrieve tree [%s]" % tree_name # interesting branch names branches = [str(b.GetName()) for b in t.GetListOfBranches() if b.GetName().endswith ("_ref")] if stream_refs is None: stream_refs = branches else: _streams = stream_refs[:] stream_refs = [] for ref in _streams: if ref not in branches: print("::: discarding [%s] from file chasing..."%ref) else: stream_refs.append (ref) if nentries <= 0: nentries = t.GetEntries() else: nentries = min (nentries, t.GetEntries()) print("::: chasing streams: %s" % stream_refs) print("::: ...over entries: %r" % nentries) # disable everything... t.SetBranchStatus ("*", 0) streams = dict() for ref in stream_refs: streams[ref] = list() # list of FileIDs, according to POOL # ex: 'B2B485E1-BB37-DD11-984C-0030487A17BA' # but (re-)enable the branches we are interested in t.SetBranchStatus (ref, 1) import re # Pool token are of the form: # '[DB=7CCD8D32-BC37-DD11-967E-0030487CD916]\ # [CNT=POOLContainer_DataHeader]\ # [CLID=72FBBC6F-C8BE-4122-8790-DC627696C176]\ # [TECH=00000202]\ # [OID=0000008C-000002BA]' token = re.compile (r'[[]DB=(?P<FID>.*?)[]]' r'[[]CNT=(?P<CNT>.*?)[]]' r'[[]CLID=(?P<CLID>.*?)[]]' r'[[]TECH=(?P<TECH>.*?)[]]' r'[[]OID=(?P<OID>.*?)[]]') for i in range(nentries): t.GetEntry (i) for ref in stream_refs: try: token_str = getattr(t, ref) except (AttributeError, TypeError): # MN: TypeError is a bug in ROOT 5.34.25, fixed in 5.34.30 # filthy work-around... try: token_branch = t.GetBranch (ref) token_branch.GetEntry(i) token_str = token_branch.GetLeaf("Token").GetValueString() except Exception: print("::: could not access stream-ref [%s] (entry #%i)", (ref, i)) continue tok = token.match (token_str) if not tok: print("::: invalid POOL token: [%s]" % token_str) continue streams[ref].append (tok.group('FID')) for ref in stream_refs: streams[ref] = list(set(streams[ref])) return streams
def main(args): """diff two ROOT files (containers and sizes)""" global g_args g_args = args import PyUtils.RootUtils as ru root = ru.import_root() # noqa: F841 import PyUtils.Logging as L msg = L.logging.getLogger('diff-root') if args.verbose: msg.setLevel(L.logging.VERBOSE) else: msg.setLevel(L.logging.INFO) from PyUtils.Helpers import ShutUp # noqa: F401 if args.entries == '': args.entries = -1 msg.info('comparing tree [%s] in files:', args.tree_name) msg.info(' old: [%s]', args.old) msg.info(' new: [%s]', args.new) msg.info('ignore leaves: %s', args.ignore_leaves) msg.info('enforce leaves: %s', args.enforce_leaves) msg.info('leaves prefix: %s', args.leaves_prefix) msg.info('hacks: %s', args.known_hacks) msg.info('entries: %s', args.entries) msg.info('mode: %s', args.mode) msg.info('error mode: %s', args.error_mode) msg.info('order trees: %s', args.order_trees) import PyUtils.Helpers as H with H.ShutUp(): fold = ru.RootFileDumper(args.old, args.tree_name) fnew = ru.RootFileDumper(args.new, args.tree_name) pass def tree_infos(tree, args): nentries = tree.GetEntriesFast() # l.GetBranch().GetName() gives the full leaf path name leaves = [ l.GetBranch().GetName() for l in tree.GetListOfLeaves() if l.GetBranch().GetName() not in args.ignore_leaves ] if args.leaves_prefix: leaves = [l.replace(args.leaves_prefix, '') for l in leaves] return { 'entries': nentries, 'leaves': set(leaves), } def ordered_indices(tree, reverse_order=False): from collections import OrderedDict import operator dict_in = {} nevts = tree.GetEntriesFast() for idx in range(0, nevts): if idx % 100 == 0: msg.debug('Read {} events from the input so far'.format(idx)) tree.GetEntry(idx) if hasattr(tree, 'xAOD::EventAuxInfo_v2_EventInfoAux.'): event_info = getattr(tree, 'xAOD::EventAuxInfo_v2_EventInfoAux.') event_number = event_info.eventNumber elif hasattr(tree, 'xAOD::EventAuxInfo_v1_EventInfoAux.'): event_info = getattr(tree, 'xAOD::EventAuxInfo_v1_EventInfoAux.') event_number = event_info.eventNumber elif hasattr(tree, 'EventInfoAux.'): event_info = getattr(tree, 'EventInfoAux.') event_number = event_info.eventNumber elif hasattr(tree, 'EventInfo_p4_McEventInfo'): event_info = getattr(tree, 'EventInfo_p4_McEventInfo') event_number = event_info.m_event_ID.m_event_number elif hasattr(tree, 'EventInfo_p4_ByteStreamEventInfo'): event_info = getattr(tree, 'EventInfo_p4_ByteStreamEventInfo') event_number = event_info.m_event_ID.m_event_number elif hasattr(tree, 'ByteStreamEventInfo'): event_info = getattr(tree, 'ByteStreamEventInfo') event_number = event_info.m_event_ID.m_event_number else: msg.error('Cannot read event info, will bail out.') break msg.debug('Idx : EvtNum {:10d} : {}'.format(idx, event_number)) dict_in[idx] = event_number # Sort the dictionary by event numbers dict_out = OrderedDict( sorted(dict_in.items(), key=operator.itemgetter(1), reverse=reverse_order)) # Write out the ordered index list return [idx for idx in dict_out] def diff_tree(fold, fnew, args): infos = { 'old': tree_infos(fold.tree, args), 'new': tree_infos(fnew.tree, args), } nentries = min(infos['old']['entries'], infos['new']['entries']) itr_entries = nentries if args.entries in (-1, '', '-1'): #msg.info('comparing over [%s] entries...', nentries) itr_entries = nentries if infos['old']['entries'] != infos['new']['entries']: msg.info('different numbers of entries:') msg.info(' old: [%s]', infos['old']['entries']) msg.info(' new: [%s]', infos['new']['entries']) msg.info('=> comparing [%s] first entries...', nentries) else: itr_entries = args.entries pass msg.info('comparing over [%s] entries...', itr_entries) old_leaves = infos['old']['leaves'] - infos['new']['leaves'] if old_leaves: msg.warning('the following variables exist only in the old file !') for l in old_leaves: msg.warning(' - [%s]', l) new_leaves = infos['new']['leaves'] - infos['old']['leaves'] if new_leaves: msg.warning('the following variables exist only in the new file !') for l in new_leaves: msg.warning(' - [%s]', l) # need to remove trailing dots as they confuse reach_next() skip_leaves = [ l.rstrip('.') for l in old_leaves | new_leaves | set(args.ignore_leaves) ] for l in skip_leaves: msg.debug('skipping [%s]', l) leaves = infos['old']['leaves'] & infos['new']['leaves'] msg.info('comparing [%s] leaves over entries...', len(leaves)) n_good = 0 n_bad = 0 import collections summary = collections.defaultdict(int) if args.order_trees: slice_max = int(itr_entries) if int(itr_entries) > 0 else None itr_entries_old = ordered_indices(fold.tree)[0:slice_max] itr_entries_new = ordered_indices(fnew.tree)[0:slice_max] msg.debug('List of old indices {}'.format(itr_entries_old)) msg.debug('List of new indices {}'.format(itr_entries_new)) else: itr_entries_old = itr_entries itr_entries_new = itr_entries old_dump_iter = fold.dump(args.tree_name, itr_entries_old) new_dump_iter = fnew.dump(args.tree_name, itr_entries_new) def leafname_fromdump(entry): return '.'.join([s for s in entry[2] if not s.isdigit()]) def reach_next(dump_iter, skip_leaves, leaves_prefix=None): keep_reading = True while keep_reading: try: entry = next(dump_iter) except StopIteration: return None entry[2][0] = entry[2][0].rstrip('.\0') # clean branch name if leaves_prefix: entry[2][0] = entry[2][0].replace(leaves_prefix, '') name = [] skip = False for n in leafname_fromdump(entry).split('.'): name.append(n) if '.'.join(name) in skip_leaves or n in skip_leaves: skip = True break if not skip: return entry # print('SKIP:', leafname_fromdump(entry)) pass read_old = True read_new = True d_old = None d_new = None while True: if read_old: prev_d_old = d_old d_old = reach_next(old_dump_iter, skip_leaves, args.leaves_prefix) if read_new: prev_d_new = d_new d_new = reach_next(new_dump_iter, skip_leaves, args.leaves_prefix) if not d_new and not d_old: break read_old = True read_new = True if (args.order_trees and d_old and d_new and d_old[-1] == d_new[-1]) or d_old == d_new: n_good += 1 continue if d_old: tree_name, ientry, name, iold = d_old if d_new: tree_name, jentry, name, inew = d_new # for regression testing we should have NAN == NAN if args.nan_equal: if all( [isinstance(x, Real) and isnan(x) for x in [iold, inew]]): n_good += 1 continue # FIXME: that's a plain (temporary?) hack if name[-1] in args.known_hacks: continue n_bad += 1 if not args.order_trees: in_synch = d_old and d_new and d_old[:-1] == d_new[:-1] else: in_synch = d_old and d_new and d_old[0] == d_new[0] and d_old[ 2] == d_new[2] if not in_synch: if _is_detailed(): if d_old: print('::sync-old %s' % '.'.join(["%03i" % ientry] + list(map(str, d_old[2])))) else: print('::sync-old ABSENT') if d_new: print('::sync-new %s' % '.'.join(["%03i" % jentry] + list(map(str, d_new[2])))) else: print('::sync-new ABSENT') pass # remember for later if not d_old: fold.allgood = False summary[d_new[2][0]] += 1 elif not d_new: fnew.allgood = False summary[d_old[2][0]] += 1 else: branch_old = '.'.join(["%03i" % ientry, d_old[2][0]]) branch_new = '.'.join(["%03i" % jentry, d_new[2][0]]) if branch_old < branch_new: if _is_detailed(): print('::sync-old skipping entry') summary[d_old[2][0]] += 1 fnew.allgood = False read_new = False elif branch_old > branch_new: if _is_detailed(): print('::sync-new skipping entry') summary[d_new[2][0]] += 1 fold.allgood = False read_old = False else: # MN: difference in the leaves prev_leaf_old = leafname_fromdump(prev_d_old) prev_leaf_new = leafname_fromdump(prev_d_new) leaf_old = leafname_fromdump(d_old) leaf_new = leafname_fromdump(d_new) if prev_leaf_old == prev_leaf_new: # array size difference? if leaf_old == leaf_new and leaf_old == prev_leaf_old: # could be a size difference in >1 dim arrays # hard to sync, skipping both pass elif leaf_old == prev_leaf_old: # old has bigger array, skip old entry read_new = False if _is_detailed(): print('::sync-old skipping entry') summary[leaf_old] += 1 elif leaf_new == prev_leaf_new: # new has bigger array, skip new entry read_old = False if _is_detailed(): print('::sync-new skipping entry') summary[leaf_new] += 1 if read_old and read_new: summary[d_new[2][0]] += 1 if _is_detailed(): print('::sync-old+new skipping both entries') fold.allgood = False fnew.allgood = False if _is_exit_early(): print('*** exit on first error ***') break continue if not args.order_trees: n = '.'.join(list(map(str, ["%03i" % ientry] + name))) else: n = '.'.join( list(map(str, ["%03i.%03i" % (ientry, jentry)] + name))) diff_value = 'N/A' try: diff_value = 50. * (iold - inew) / (iold + inew) diff_value = '%.8f%%' % (diff_value, ) except Exception: pass if _is_detailed(): print('%s %r -> %r => diff= [%s]' % (n, iold, inew, diff_value)) pass summary[leafname_fromdump(d_old)] += 1 if name[0] in args.enforce_leaves: msg.info("don't compare further") break pass # loop over events/branches msg.info('Found [%s] identical leaves', n_good) msg.info('Found [%s] different leaves', n_bad) if not _is_summary(): keys = sorted(summary.keys()) for n in keys: v = summary[n] msg.info(' [%s]: %i leaves differ', n, v) pass pass if (not fold.allgood) or (not fnew.allgood): msg.info('NOTE: there were errors during the dump') msg.info('fold.allgood: %s', fold.allgood) msg.info('fnew.allgood: %s', fnew.allgood) n_bad += 0.5 return n_bad ndiff = diff_tree(fold, fnew, args) if ndiff != 0: msg.info('files differ!') return 2 msg.info('all good.') return 0
def initialize(self): self.msg.info( '************************************' ) self.msg.info( '==> initialize %s...', self.name() ) self.msg.info( 'Will apply the following electron cuts:' ) self.msg.info( ' EtMinEl = %r', self.cutEtMinEl ) self.msg.info( ' IsEmEl = %s', self.cutIsEM ) self.msg.info( ' AuthorEl = %s', self.cutElAuthor ) self.msg.info( ' ElEtaMax = %r', self.cutElEtaMax ) self.msg.info( ' removeEtaCrack = %r', self.removeEtaCrack ) self.msg.info( ' crackEtaMin = %r', self.crackEtaMin ) self.msg.info( ' crackEtaMax = %r', self.crackEtaMax ) self.msg.info( ' minNumberPassedEl = %r', self.minNumberPassedEl ) self.msg.info( ' passAllEl = %r', self.passAllEl ) self.msg.info( 'And the following muon cuts:' ) self.msg.info( ' PtMinMu = %r', self.cutPtMinMu ) self.msg.info( ' AuthorMu = %s', self.muonAuthors ) self.msg.info( ' MuEtaMax = %r', self.cutMuEtaMax ) if self.doEtconeMu: self.msg.info( ' muon etcone cut = %r', self.etconeMuMax ) self.msg.info( ' passAllMu = %r', self.passAllMu ) self.msg.info( ' minNumberPassedMu = %r', self.minNumberPassedMu ) self.msg.info( '************************************' ) ## Also, declare some counters and initialize them to zero self.nProcessed = 0 self.nEventMinNumPassed = 0 self.nElectrons = 0 self.nEventElPassEta = 0 self.nEventElPassEt = 0 self.nEventElPassIsEM = 0 self.nEventElPassAuthor = 0 self.nMuons = 0 self.nEventMuPassEta = 0 self.nEventMuPassPt = 0 self.nEventMuPassAuthor = 0 self.nEventMuPassPrescale = 0 self.nEventMuPassIsolation = 0 ## Import needed modules import PyUtils.RootUtils as ru ROOT = ru.import_root() if self.doNoTrackIsolMu or self.doPtTrackIsolMu or self.doCaloIsolMu : if self.muonIsolationTool is None: self.msg.warning("Problem retrieving MuonIsolationTool pointer ") self.msg.warning("setting Track and Calo isolation to False ") self.doNoTrackIsolMu = False self.doPtTrackIsolMu = False self.doCaloIsolMu = False ## import some 4-mom utils import FourMomUtils.Bindings self.utils = { 'deltaR' : PyAthena.P4Helpers.deltaR } _append = self.muContainerNames.append if self.usingAOD : if self.muonContainers.__contains__('staco') or self.muonContainers is 'all': _append("StacoMuonCollection") if self.muonContainers.__contains__('muid') or self.muonContainers is 'all': _append("MuidMuonCollection") if self.muonContainers.__contains__('calo') or self.muonContainers is 'all': _append("CaloMuonCollection") else : if self.muonContainers.__contains__('staco') or self.muonContainers is 'all': _append("StacoESDMuonCollection") if self.muonAuthors is 'combined' or self.muonAuthors is 'all' : _append("StacoCombinedMuonContainer") if self.muonAuthors is 'lowpt' or self.muonAuthors is 'all' : _append("StacoCombinedMuonContainerLowPt") if self.muonAuthors is 'standalone' or self.muonAuthors is 'all': _append("MboyESDMuonContainer") if self.muonContainers.__contains__('muid') or self.muonContainers is 'all': _append("MuidESDMuonCollection") if self.muonAuthors is 'combined' or self.muonAuthors is 'all' : _append("MuidCombinedMuonContainer") if self.muonAuthors is 'lowpt' or self.muonAuthors is 'all' : _append("MuGirlCombinedMuonContainer") if self.muonAuthors is 'standalone' or self.muonAuthors is 'all' : _append("MuidExtrMuonContainer") if self.muonContainers.__contains__('calo') or self.muonContainers is 'all': if self.muonAuthors is 'calo' or self.muonAuthors is 'all' : _append("CaloMuonIdContainer") _append("CaloESDMuonCollection") del _append ## Get the StoreGate service self.storeGateSvc = PyAthena.py_svc('StoreGateSvc') if self.storeGateSvc is None: self.msg.error("Problem retrieving StoreGateSvc pointer !!") return StatusCode.Failure return StatusCode.Success
# @author: Carlos Chavez from __future__ import print_function import logging msg = logging.getLogger("PyJobTransforms." + __name__) import os import re import eformat from TrigTransform.dbgEventInfo import dbgEventInfo from TrigConfStorage.TriggerCoolUtil import TriggerCoolUtil from PyCool import cool from PyUtils import RootUtils ROOT = RootUtils.import_root() from ROOT import TFile def dbgPreRun(inputFileList,outputFileList): msg.info('Running debug_stream analysis PreRun operations on files :{0} '.format(inputFileList)) msg.info('Running debug_stream analysis PreRun, histogram output in :{0} '.format(outputFileList)) total = 0 #open root output file out_file = outputFileList[0] hfile = TFile( out_file , 'RECREATE' ) #inicialize dbgEventInfo, this is the main event analysis class eventInfo = dbgEventInfo("_Pre",inputFileList.value[0]) data = [] l1Info = []
# @file PyUtils.scripts.diff_root_files # @purpose check that 2 ROOT files have same content (containers and sizes). # @author Sebastien Binet # @date February 2010 __version__ = "$Revision$" __doc__ = "check that 2 ROOT files have same content (containers and sizes)." __author__ = "Sebastien Binet" ### imports ------------------------------------------------------------------- import PyUtils.acmdlib as acmdlib import PyUtils.RootUtils as ru ROOT = ru.import_root() ### globals ------------------------------------------------------------------- g_ALLOWED_MODES = ('summary', 'detailed') g_ALLOWED_ERROR_MODES = ('bailout', 'resilient') g_args = None ### classes ------------------------------------------------------------------- ### functions ----------------------------------------------------------------- def _is_summary(): global g_args return g_args.mode == 'summary' def _is_exit_early(): global g_args return g_args.error_mode == 'bailout' @acmdlib.command(name='diff-root')