def __init__(self):
import PyUtils.Logging as _L
self._msg = _L.logging.getLogger("AthFile")
self.set_msg_lvl(_L.logging.INFO)
if os.environ.get('ATHFILE_DEBUG', '0') == '1':
import PyUtils.Logging as _L
self.set_msg_lvl(_L.logging.VERBOSE)
pass
self.msg().debug('importing ROOT...')
import PyUtils.RootUtils as ru
self.pyroot = ru.import_root()
try:
ru._pythonize_tfile()
except Exception as err:
self.msg().warning('problem during TFile pythonization:\n%s', err)
self.msg().debug('importing ROOT... [done]')
# a cache of already processed requests
self._cache = {}
self._do_pers_cache = True
self.enable_pers_cache()
return
def getTauWPDecoratorJetBDT():
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
import cppyy
cppyy.loadDictionary('xAODTau_cDict')
_name = sPrefix + 'TauWPDecoratorJetBDT'
from tauRecTools.tauRecToolsConf import TauWPDecorator
myTauWPDecorator = TauWPDecorator(
name=_name,
flatteningFile1Prong="FlatJetBDT1Pv2.root", #update
flatteningFile3Prong="FlatJetBDT3Pv2.root", #update
CutEnumVals=[
ROOT.xAOD.TauJetParameters.JetBDTSigVeryLoose,
ROOT.xAOD.TauJetParameters.JetBDTSigLoose,
ROOT.xAOD.TauJetParameters.JetBDTSigMedium,
ROOT.xAOD.TauJetParameters.JetBDTSigTight
],
SigEff1P=[0.95, 0.85, 0.75, 0.60],
SigEff3P=[0.95, 0.75, 0.60, 0.45],
ScoreName="BDTJetScore",
NewScoreName="BDTJetScoreSigTrans",
DefineWPs=True,
)
cached_instances[_name] = myTauWPDecorator
return myTauWPDecorator
def __openPoolFile(self, fileName):
# hack to prevent ROOT from loading graphic libraries and hence bother
# our fellow Mac users
if self.verbose==True:
print "## importing ROOT..."
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
if self.verbose==True:
print "## importing ROOT... [DONE]"
# prevent ROOT from being too verbose
rootMsg = ShutUp()
rootMsg.mute()
ROOT.gErrorIgnoreLevel = ROOT.kFatal
rootMsg.unMute()
#import PyCintex; PyCintex.Cintex.Enable()
rootMsg.mute()
poolFile = None
try:
poolFile = ROOT.TFile.Open( fileName, PoolOpts.READ_MODE )
except Exception, e:
rootMsg.unMute()
print "## Failed to open file [%s] !!" % fileName
print "## Reason:"
print e
print "## Bailing out..."
raise IOError, "Could not open file [%s]" % fileName
def initialize(self):
self.msg.info('************************************')
self.msg.info('==> initialize %s...', self.name())
self.msg.info('Will apply the following cuts:')
self.msg.info(' trigger = %r', self.trigger)
self.msg.info(' passAll = %r', self.passAll)
self.msg.info('************************************')
## Import needed modules
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
## Get the TrigDecisionTool tool
self.tdt = PyAthena.py_tool('Trig::TrigDecisionTool/TrigDecisionTool',
iface='Trig::TrigDecisionTool')
if not self.tdt:
self.msg.warning('Could not retrieve TrigDecisionTool pointer!')
return StatusCode.Success
# Set description variable in cut flow branch (default is trigger name, but can override in JO)
self.setFilterDescription(self.trigger)
## Initialize event pass counters
self.nProcessed = 0
self.nEventPassTrigger = 0
return StatusCode.Success
def __openPoolFile(self, fileName):
# hack to prevent ROOT from loading graphic libraries and hence bother
# our fellow Mac users
if self.verbose==True:
print "## importing ROOT..."
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
if self.verbose==True:
print "## importing ROOT... [DONE]"
# prevent ROOT from being too verbose
rootMsg = ShutUp()
rootMsg.mute()
ROOT.gErrorIgnoreLevel = ROOT.kFatal
rootMsg.unMute()
import PyCintex
PyCintex.Cintex.Enable()
rootMsg.mute()
poolFile = None
try:
poolFile = ROOT.TFile.Open( fileName, PoolOpts.READ_MODE )
except Exception, e:
rootMsg.unMute()
print "## Failed to open file [%s] !!" % fileName
print "## Reason:"
print e
print "## Bailing out..."
raise IOError, "Could not open file [%s]" % fileName
def initialize(self):
self.msg.debug('************************************')
self.msg.debug('==> initialize %s...', self.name())
self.msg.debug(' inputContainerType = %r', self.inputContainerType)
self.msg.debug(' inputContainerNames = %r', self.inputContainerNames)
self.msg.debug(' outputContainerName = %r', self.outputContainerName)
self.msg.debug(' removeIdentical = %r', self.removeIdentical)
self.msg.debug('************************************')
## Also, declare some counters and initialize them to zero
self.nProcessed = 0
self.nObjectsProcessed = 0
self.nObjectsPassed = 0
self.nEventAccepted = 0
## Import needed modules
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
## Get the StoreGate service
self.storeGateSvc = PyAthena.py_svc('StoreGateSvc')
if self.storeGateSvc is None:
self.msg.error("Problem retrieving StoreGateSvc pointer !!")
return StatusCode.Failure
## import some 4-mom utils
import FourMomUtils.Bindings
self.utils = {'deltaR': PyAthena.P4Helpers.deltaR}
return StatusCode.Success
def getTauWPDecoratorJetRNN():
_name = sPrefix + 'TauWPDecoratorJetRNN'
if _name in cached_instances:
return cached_instances[_name]
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
import cppyy
cppyy.loadDictionary('xAODTau_cDict')
from AthenaCommon.AppMgr import ToolSvc
from tauRecTools.tauRecToolsConf import TauWPDecorator
TauWPDecorator = TauWPDecorator( name=_name,
flatteningFile0Prong = "rnnid_flat_0p_v4.root",
flatteningFile1Prong = "rnnid_flat_1p_v4.root",
flatteningFile3Prong = "rnnid_flat_mp_v4.root",
CutEnumVals =
[ ROOT.xAOD.TauJetParameters.JetRNNSigVeryLoose, ROOT.xAOD.TauJetParameters.JetRNNSigLoose,
ROOT.xAOD.TauJetParameters.JetRNNSigMedium, ROOT.xAOD.TauJetParameters.JetRNNSigTight ],
SigEff0P = [0.98, 0.90, 0.65, 0.50],
SigEff1P = [0.992, 0.99, 0.965, 0.94],
SigEff3P = [0.99, 0.98, 0.865, 0.80],
ScoreName = "RNNJetScore",
NewScoreName = "RNNJetScoreSigTrans",
DefineWPs = True,
)
ToolSvc += TauWPDecorator
cached_instances[_name] = TauWPDecorator
return TauWPDecorator
def getTauWPDecoratorJetBDT():
_name = sPrefix + 'TauWPDecoratorJetBDT'
if _name in cached_instances:
return cached_instances[_name]
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
import cppyy
cppyy.loadDictionary('xAODTau_cDict')
from AthenaCommon.AppMgr import ToolSvc
from tauRecTools.tauRecToolsConf import TauWPDecorator
TauWPDecorator = TauWPDecorator(
name=_name,
flatteningFile1Prong="FlatJetBDT1P_trigger_v1.root",
flatteningFile3Prong="FlatJetBDT3P_trigger_v1.root",
CutEnumVals=[
ROOT.xAOD.TauJetParameters.JetBDTSigVeryLoose,
ROOT.xAOD.TauJetParameters.JetBDTSigLoose,
ROOT.xAOD.TauJetParameters.JetBDTSigMedium,
ROOT.xAOD.TauJetParameters.JetBDTSigTight
],
SigEff1P=[0.995, 0.99, 0.97, 0.90],
SigEff3P=[0.995, 0.94, 0.88, 0.78],
ScoreName="BDTJetScore",
NewScoreName="BDTJetScoreSigTrans",
DefineWPs=True)
ToolSvc += TauWPDecorator
cached_instances[_name] = TauWPDecorator
return TauWPDecorator
def getTauTrackClassifier():
_name = sPrefix + 'TauTrackClassifier'
if _name in cached_instances:
return cached_instances[_name]
from AthenaCommon.AppMgr import ToolSvc
from tauRecTools.tauRecToolsConf import tauRecTools__TauTrackClassifier as TauTrackClassifier
from tauRecTools.tauRecToolsConf import tauRecTools__TrackMVABDT as TrackMVABDT
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
import cppyy
cppyy.loadDictionary('xAODTau_cDict')
input_file_name = 'EFtracks_BDT_classifier_v0.root'
BDTcut = 0.45
deltaZ0 = 1.0
# =========================================================================
EFtrackBDT = TrackMVABDT(
name=_name + "_MVABDT",
InputWeightsPath=input_file_name,
Threshold=BDTcut,
DeltaZ0=deltaZ0,
ExpectedFlag=ROOT.xAOD.TauJetParameters.unclassified,
inTrigger=True)
ToolSvc += EFtrackBDT
trackclassifier = TauTrackClassifier(name=_name, Classifiers=[EFtrackBDT])
cached_instances[_name] = trackclassifier
return trackclassifier
def initialize(self):
self.msg.debug('************************************')
self.msg.debug('==> initialize %s...', self.name())
self.msg.debug(' inputContainerType = %r', self.inputContainerType)
self.msg.debug(' particleType = %r', self.particleTypeName)
self.msg.debug(' inputContainerNames = %r', self.inputContainerNames)
self.msg.debug(' outputContainerName = %r', self.outputContainerName)
self.msg.debug('************************************')
## Also, declare some counters and initialize them to zero
self.nProcessed = 0
self.nObjectsProcessed = 0
self.nObjectsPassed = 0
self.nEventAccepted = 0
## Import needed modules
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
## Get the StoreGate service
self.storeGateSvc = PyAthena.py_svc('StoreGateSvc')
if self.storeGateSvc is None:
self.msg.error("Problem retrieving StoreGateSvc pointer!")
return StatusCode.Failure
gc.disable()
return StatusCode.Success
def getTauWPDecoratorEleBDT():
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
import cppyy
cppyy.loadDictionary('xAODTau_cDict')
_name = sPrefix + 'TauWPDecoratorEleBDT'
from tauRecTools.tauRecToolsConf import TauWPDecorator
TauScoreFlatteningTool = TauWPDecorator(
name=_name,
flatteningFile1Prong="EleBDTFlat1P.root", #update
flatteningFile3Prong=
"EleBDTFlat3P.root", #update
UseEleBDT=True,
ScoreName="BDTEleScore",
NewScoreName="BDTEleScoreSigTrans", #dynamic
DefineWPs=True,
CutEnumVals=[
ROOT.xAOD.TauJetParameters.EleBDTLoose,
ROOT.xAOD.TauJetParameters.EleBDTMedium,
ROOT.xAOD.TauJetParameters.EleBDTTight
],
SigEff1P=[0.95, 0.85, 0.75],
SigEff3P=[0.95, 0.85, 0.75],
)
cached_instances[_name] = TauScoreFlatteningTool
return TauScoreFlatteningTool
def initialize(self):
self.msg.info('************************************')
self.msg.info('==> initialize %s...', self.name())
self.msg.info('Will apply the following cuts:')
self.msg.info(' EtMinEl = %r', self.cutEtMin)
self.msg.info(' IsEmEl = %s', self.cutIsEM)
self.msg.info(' AuthorEl = %s', self.cutAuthor)
self.msg.info(' EtaMax = %r', self.cutEtaMax)
self.msg.info(' removeEtaCrack = %r', self.removeEtaCrack)
self.msg.info(' crackEtaMin = %r', self.crackEtaMin)
self.msg.info(' crackEtaMax = %r', self.crackEtaMax)
self.msg.info(' minNumberPassed = %r', self.minNumberPassed)
self.msg.info(' passAll = %r', self.passAll)
self.msg.info('************************************')
## Also, declare some counters and initialize them to zero
self.nProcessed = 0
self.nElectrons = 0
self.nEventElPassEta = 0
self.nEventElPassEt = 0
self.nEventElPassIsEM = 0
self.nEventElPassAuthor = 0
self.nEventMinNumPassed = 0
## Import needed modules
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
## Get the StoreGate service
self.storeGateSvc = PyAthena.py_svc('StoreGateSvc')
if self.storeGateSvc is None:
self.msg.error("Problem retrieving StoreGateSvc pointer !!")
return StatusCode.Failure
return StatusCode.Success
def main(args):
"""dump the content of a ROOT file into an ASCII format.
"""
import PyUtils.RootUtils as ru
root = ru.import_root()
_inspect = root.RootUtils.PyROOTInspector.pyroot_inspect2
import PyUtils.Logging as L
msg = L.logging.getLogger('dump-root')
msg.setLevel(L.logging.INFO)
msg.info('fname: [%s]', args.fname)
root_file = root.TFile.Open(args.fname)
if (root_file is None or not isinstance(root_file, root.TFile)
or not root_file.IsOpen()):
msg.error('could not open [%s]', args.fname)
return 1
tree_names = []
if args.tree_name:
tree_names = args.tree_name.split(',')
else:
tree_names = []
keys = [k.GetName() for k in root_file.GetListOfKeys()]
for k in keys:
o = root_file.Get(k)
if isinstance(o, root.TTree):
tree_names.append(k)
msg.info('dumping trees: %s', tree_names)
rc = 0
for tree_name in tree_names:
f = ru.RootFileDumper(args.fname, tree_name)
nentries = f.tree.GetEntries()
if args.entries:
nentries = args.entries
for d in f.dump(tree_name, nentries):
tree_name, ientry, name, data = d
n = '.'.join(map(str, [tree_name, "%03i" % ientry] + name))
print '%s %r' % (n, data)
return 0
def initialize(self):
self.msg.info('************************************')
self.msg.info('==> initialize %s...', self.name())
self.msg.info(' jetCollectionType = %r', self.jetCollectionType)
self.msg.info(' jetCollectionName = %r', self.jetCollectionName)
self.msg.info(' recordGoodJets = %r', self.recordGoodJets)
self.msg.info(' goodJetCollectionName = %r',
self.goodJetCollectionName)
self.msg.info('Will apply the following cuts:')
self.msg.info(' useUncalibratedJets = %r',
self.useUncalibratedJets)
self.msg.info(' EtMin (GeV) = %r',
self.cutEtMin / Units.GeV)
self.msg.info(' EtMax (GeV) = %r',
self.cutEtMax / Units.GeV)
self.msg.info(' EtaMin = %r', self.cutEtaMin)
self.msg.info(' EtaMax = %r', self.cutEtaMax)
self.msg.info(' cutSumPtTrkMax = %r', self.cutSumPtTrkMax)
self.msg.info(' removeOverlap = %r', self.removeOverlap)
self.msg.info(' deltaR = %r', self.deltaR)
self.msg.info(' leptonContainerTypeList = %r',
self.leptonContainerTypeList)
self.msg.info(' leptonContainerNameList = %r',
self.leptonContainerNameList)
self.msg.info(' nLeadingsForCheck = %r', self.nLeadingsForCheck)
self.msg.info(' minNumberPassed = %r', self.minNumberPassed)
self.msg.info(' passAll = %r', self.passAll)
self.msg.info('************************************')
## Initialize the counters
self.nProcessed = 0
self.nJets = 0
self.nEventPassed = 0
## Import needed modules
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
## Get the StoreGate service
self.storeGateSvc = PyAthena.py_svc('StoreGateSvc')
if self.storeGateSvc is None:
self.msg.error("Problem retrieving StoreGateSvc pointer !!")
return StatusCode.Failure
## Eagerly load library due to reflex bug (autoloading enums/namespaces doesn't work)
PyAthena.load_library('FourMomUtils')
## Get the JetSignalState helper
self.jss = PyAthena.SignalStateHelper(PyAthena.P4SignalState.JETFINAL)
## import some 4-mom utils
import FourMomUtils.Bindings
self.utils = {'deltaR': PyAthena.P4Helpers.deltaR}
return StatusCode.Success
def BDTtoolset(self):
if TriggerFlags.run2Config == '2016':
from TauDiscriminant.TauDiscriminantConf import TauJetBDT
bdt_set = TauJetBDT(inTrigger=True,
calibFolder='TrigTauRec/00-11-01/',
jetBDT="bdt.2016.bin",
jetSigBits="trigger.sig.bits.jet.BDT.txt")
return [bdt_set]
else:
from tauRecTools.tauRecToolsConf import TauJetBDTEvaluator
# BDT evaluators 1p / mp
bdt_1p = TauJetBDTEvaluator(
name="TrigTauJetBDT1P",
calibFolder='tauRecTools/00-02-00/',
weightsFile='vars2016_pt_gamma_1p_isofix.root',
inTrigger=True,
minNTracks=0,
maxNTracks=1)
bdt_mp = TauJetBDTEvaluator(
name="TrigTauJetBDTMP",
calibFolder='tauRecTools/00-02-00/',
weightsFile='vars2016_pt_gamma_3p_isofix.root',
inTrigger=True,
minNTracks=2,
maxNTracks=1000)
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
import cppyy
cppyy.loadDictionary('xAODTau_cDict')
from tauRecTools.tauRecToolsConf import TauWPDecorator
# wp creators 1p / mp
wp_decorator = TauWPDecorator(
name="TrigTauJetWPDecorator",
calibFolder='TrigTauRec/00-11-01/',
inTrigger=True,
flatteningFile1Prong="FlatJetBDT1P_trigger_v1.root",
flatteningFile3Prong="FlatJetBDT3P_trigger_v1.root",
CutEnumVals=[
ROOT.xAOD.TauJetParameters.JetBDTSigVeryLoose,
ROOT.xAOD.TauJetParameters.JetBDTSigLoose,
ROOT.xAOD.TauJetParameters.JetBDTSigMedium,
ROOT.xAOD.TauJetParameters.JetBDTSigTight
],
SigEff1P=[0.995, 0.99, 0.97, 0.90],
SigEff3P=[0.995, 0.94, 0.88, 0.78],
ScoreName="BDTJetScore",
NewScoreName="BDTJetScoreSigTrans",
DefineWPs=True)
return [bdt_1p, bdt_mp, wp_decorator]
def checkPFCorruption(filename, verbose=False):
if not os.access(filename, os.R_OK):
print "ERROR can't access file", filename
return -1
ROOT = RootUtils.import_root()
from ROOT import TFile, TTree
import PyCintex
try:
f = TFile.Open(filename)
except:
print "Can't open file", filename
return -1
nEvents = n = None
keys = f.GetListOfKeys()
for k in keys:
try:
tn = k.GetName()
t = f.Get(tn)
if not isinstance(t, TTree): return
except:
print "Can't get tree %s from file %s", tn, fn
f.Close()
return -1
if (verbose): print "Working on tree", tn
n = t.GetEntriesFast()
for i in range(n):
s = t.GetEntry(i)
if s <= 0:
print "Tree %s: Found corruption in event %i" % (i, n)
f.Close()
return -2
else:
if verbose and i > 0 and i % 100 == 0:
print "Checking event", i
print "Tree %s: %i event(s) ok" % (tn, n)
# Use CollectionTree determine the number of events
if tn == 'CollectionTree':
nEvents = n
pass #end of loop over trees
f.Close()
print "ROOT file", filename, "looks ok"
if n is None:
print "Failed to determine number of events in file %s. No tree named 'CollectionTree'" % filename
return 0
return n
def addROOTIncludePaths():
"""
Fill ROOT include path list for entries for all packages found in CMTPATH
"""
# Most projects make their headers available to root under
# root-include-path. But DetCommon does not.
if 'CMTPATH' in os.environ:
import glob
import PyUtils.RootUtils as ru
interp = ru.import_root().gInterpreter
for p in os.environ['CMTPATH'].split(':'):
if p.find('DetCommon') >= 0:
idir = os.path.join(p, 'InstallArea', 'include')
for ii in glob.glob(os.path.join(idir, '*')):
interp.AddIncludePath(ii)
# Also need to make Geant headers available.
if 'G4INCLUDE' in os.environ:
import PyUtils.RootUtils as ru
interp = ru.import_root().gInterpreter
interp.AddIncludePath(os.environ['G4INCLUDE'])
def _cleanup_pyroot(self):
import PyUtils.RootUtils as ru
root = ru.import_root()
tfiles = root.gROOT.GetListOfFiles()[:]
for i,f in enumerate(tfiles):
try:
if f:
f.Close()
del f
except Exception as err:
self._msg.info('could not close a TFile:\n%s', err)
pass
tfiles[:] = []
def __init__(self):
# import cintex
import PyCintex; PyCintex.Cintex.Enable()
# import root
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
self._cxx = ROOT.Ath.DsoDb.instance()
# load reflex
_load_dict = PyCintex.loadDict
_load_dict('ReflexRflx')
self._rflx = PyCintex.makeNamespace('Reflex')
if not self._rflx:
self._rflx = PyCintex.makeNamespace('ROOT::Reflex')
return
def __init__(self):
# import cintex
import PyCintex
PyCintex.Cintex.Enable()
# import root
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
self._cxx = ROOT.Ath.DsoDb.instance()
# load reflex
_load_dict = PyCintex.loadDict
_load_dict('ReflexRflx')
self._rflx = PyCintex.makeNamespace('Reflex')
if not self._rflx:
self._rflx = PyCintex.makeNamespace('ROOT::Reflex')
return
def __init__(self):
# import cintex
# import PyCintex; PyCintex.Cintex.Enable()
import cppyy
# import root
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
self._cxx = ROOT.Ath.DsoDb.instance()
# if int(cppyy.get_version().split('.')[0]) < 6:
if hasattr(cppyy, 'hasCintex'):
# load reflex for ROOT ver<6
_load_dict = cppyy.loadDict
_load_dict('ReflexRflx')
self._rflx = cppyy.makeNamespace('Reflex')
if not self._rflx:
self._rflx = cppyy.makeNamespace('ROOT::Reflex')
return
def main(args):
"""dump the content of a ROOT file into an ASCII format.
"""
import PyUtils.RootUtils as ru
root = ru.import_root()
_inspect = root.RootUtils.PyROOTInspector.pyroot_inspect2
import PyUtils.Logging as L
msg = L.logging.getLogger('dump-root')
msg.setLevel(L.logging.INFO)
msg.info('fname: [%s]', args.fname)
root_file = root.TFile.Open(args.fname)
if (root_file is None or
not isinstance(root_file, root.TFile) or
not root_file.IsOpen()):
msg.error('could not open [%s]', args.fname)
return 1
tree_names = []
if args.tree_name:
tree_names = args.tree_name.split(',')
else:
tree_names = []
keys = [k.GetName() for k in root_file.GetListOfKeys()]
for k in keys:
o = root_file.Get(k)
if isinstance(o, root.TTree):
tree_names.append(k)
msg.info('dumping trees: %s', tree_names)
rc = 0
for tree_name in tree_names:
f = ru.RootFileDumper(args.fname, tree_name)
nentries = f.tree.GetEntries()
if args.entries:
nentries = args.entries
for d in f.dump(tree_name, nentries):
tree_name, ientry, name, data = d
n = '.'.join(map(str, [tree_name,"%03i"%ientry]+name))
print '%s %r' %(n, data)
return 0
def extract_streams_from_tag (fname,
tree_name="POOLCollectionTree",
nentries=-1,
stream_refs=None):
"""find the GUID(s) of a list of streams which are contained inside a TAG
file.
@params:
`fname` the filename of the TAG file to inspect
(can be a LFN or FID)
`tree_name` the name of the TTree containing the stream-refs
('CollectionTree' is the usual default)
`nentries` the number of entries to inspect, among the ttree entries
(-1: all the entries)
`stream_refs` a list of stream names
(ex: ['StreamAOD_ref', 'Stream1_ref'] or,
None - to inspect all the stream refs in the TAG)
example:
>>> streams = extract_streams_from_tag ('tag.pool')
>>> from pprint import pprint
>>> pprint (streams)
{'Stream1_ref': ['BEE16671-B9F6-DA11-8219-00304871B611'],
'StreamAOD_ref': ['96F3018E-A0AC-DD11-8115-000423D59D52'],
'StreamESD_ref': ['384D0CFC-9FAC-DD11-A905-000423D59D52'],
'StreamRDO_ref': ['22C5BD99-3059-DB11-8D44-0030488365E6']}
>>> from PoolFile import PoolFileCatalog as pfc
>>> pprint (pfc().pfn(streams['StreamAOD_ref'][0]))
['aod.pool']
"""
import sys
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
import PyCintex; PyCintex.Cintex.Enable()
print "::: opening file [%s]..." % fname
# get the "final" file name (handles all kind of protocols)
import PyUtils.AthFile as af
try:
protocol, fname = af.server.fname(fname)
except Exception,err:
print "::: warning: problem extracting file name from PoolFileCatalog"
print "::: warning: will use [%s]" % fname
def extract_streams_from_tag (fname,
tree_name="POOLCollectionTree",
nentries=-1,
stream_refs=None):
"""find the GUID(s) of a list of streams which are contained inside a TAG
file.
@params:
`fname` the filename of the TAG file to inspect
(can be a LFN or FID)
`tree_name` the name of the TTree containing the stream-refs
('CollectionTree' is the usual default)
`nentries` the number of entries to inspect, among the ttree entries
(-1: all the entries)
`stream_refs` a list of stream names
(ex: ['StreamAOD_ref', 'Stream1_ref'] or,
None - to inspect all the stream refs in the TAG)
example:
>>> streams = extract_streams_from_tag ('tag.pool')
>>> from pprint import pprint
>>> pprint (streams)
{'Stream1_ref': ['BEE16671-B9F6-DA11-8219-00304871B611'],
'StreamAOD_ref': ['96F3018E-A0AC-DD11-8115-000423D59D52'],
'StreamESD_ref': ['384D0CFC-9FAC-DD11-A905-000423D59D52'],
'StreamRDO_ref': ['22C5BD99-3059-DB11-8D44-0030488365E6']}
>>> from PoolFile import PoolFileCatalog as pfc
>>> pprint (pfc().pfn(streams['StreamAOD_ref'][0]))
['aod.pool']
"""
import sys
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
#import PyCintex; PyCintex.Cintex.Enable()
print "::: opening file [%s]..." % fname
# get the "final" file name (handles all kind of protocols)
import PyUtils.AthFile as af
try:
protocol, fname = af.server.fname(fname)
except Exception,err:
print "::: warning: problem extracting file name from PoolFileCatalog"
print "::: warning: will use [%s]" % fname
def get_runs_from_tagfile(self, fname):
# check own cache for this file
if fname in self.file_cache.keys():
return self.file_cache[fname]
# check file type with AthFile - this should avoid reopening files more times than necessary
msg.debug("Checking file %s" % fname)
import PyUtils.AthFile as athFile
fileinfos = athFile.fopen(fname).fileinfos
if not 'TAG' in fileinfos['stream_names']:
return []
# this is a TAG file, open it and read all run numbers
# fileinfos have only the run number from the first TAG
import PyUtils.Helpers as H
with H.restricted_ldenviron(projects=['AtlasCore']):
import re
with H.ShutUp(filters=[
re.compile(
'TClass::TClass:0: RuntimeWarning: no dictionary for.*'
),
re.compile('.*duplicate entry.*')
]):
msg.debug("Opening TAG file %s" % fname)
import PyUtils.RootUtils as ru
f = ru.import_root().TFile.Open(fname, "read")
if f is None or not f:
msg.warning("Failed to open TAG file %s" % fname)
return []
coll_tree = f.Get('POOLCollectionTree')
run_numbers = set()
if coll_tree is not None:
for row in xrange(0, coll_tree.GetEntries()):
coll_tree.GetEntry(row)
run_numbers.add(getattr(coll_tree, self.run_attr_name))
del coll_tree
f.Close()
del f
self.file_cache[fname] = run_numbers
msg.info("TAG file: %s, found runs: %s" %
(fname, str(run_numbers)))
return run_numbers
def initialize(self):
self.msg.info('************************************')
self.msg.info('==> initialize %s...', self.name())
self.msg.info(' jetCollectionTypeList = %r',
self.jetCollectionTypeList)
self.msg.info(' jetCollectionNameList = %r',
self.jetCollectionNameList)
self.msg.info('Will apply the following cuts:')
self.msg.info(' MetMin = %r', self.cutMetMin)
self.msg.info(' minDeltaPhi = %r', self.minDeltaPhi)
self.msg.info(' useLeadingJet = %r', self.useLeadingJet)
self.msg.info(' requireMet = %r', self.requireMet)
self.msg.info('************************************')
## Get the StoreGate service
self.storeGateSvc = PyAthena.py_svc('StoreGateSvc')
if self.storeGateSvc is None:
self.msg.error("Problem retrieving StoreGateSvc pointer !!")
return StatusCode.Failure
## Import needed modules
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
## Eagerly load library due to reflex bug (autoloading enums/namespaces doesn't work)
#PyAthena.load_library ('FourMomUtils')
## Get the JetSignalState helper
#self.jss = PyAthena.JetSignalState()
self.jss = PyAthena.SignalStateHelper(PyAthena.P4SignalState.JETFINAL)
## import some 4-mom utils
import FourMomUtils.Bindings
self.utils = {
'deltaR': PyAthena.P4Helpers.deltaR,
'deltaEta': PyAthena.P4Helpers.deltaEta,
'deltaPhi': PyAthena.P4Helpers.deltaPhi
}
return StatusCode.Success
def __openPoolFile(self, fileName):
# hack to prevent ROOT from loading graphic libraries and hence bother
# our fellow Mac users
if self.verbose is True:
print("## importing ROOT...")
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
if self.verbose is True:
print("## importing ROOT... [DONE]")
# prevent ROOT from being too verbose
rootMsg = ShutUp()
rootMsg.mute()
ROOT.gErrorIgnoreLevel = ROOT.kFatal
poolFile = None
try:
poolFile = ROOT.TFile.Open( fileName, PoolOpts.READ_MODE )
except Exception as e:
rootMsg.unMute()
print("## Failed to open file [%s] !!" % fileName)
print("## Reason:")
print(e)
print("## Bailing out...")
raise IOError("Could not open file [%s]" % fileName)
rootMsg.unMute()
if poolFile is None:
print("## Failed to open file [%s] !!" % fileName)
msg = "Could not open file [%s]" % fileName
raise IOError(msg)
self.poolFile = poolFile
assert self.poolFile.IsOpen() and not self.poolFile.IsZombie(), \
"Invalid POOL file or a Zombie one"
self._fileInfos = {
'name' : self.poolFile.GetName(),
'size' : self.poolFile.GetSize(),
}
return
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
# @file PyUtils.scripts.diff_root_files
# @purpose check that 2 ROOT files have same content (containers and sizes).
# @author Sebastien Binet
# @date February 2010
__version__ = "$Revision: 681245 $"
__doc__ = "check that 2 ROOT files have same content (containers and sizes)."
__author__ = "Sebastien Binet"
### imports -------------------------------------------------------------------
import PyUtils.acmdlib as acmdlib
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
### globals -------------------------------------------------------------------
g_ALLOWED_MODES = ('summary', 'detailed')
g_ALLOWED_ERROR_MODES = ('bailout', 'resilient')
g_args = None
### classes -------------------------------------------------------------------
### functions -----------------------------------------------------------------
def _is_summary():
global g_args
return g_args.mode == 'summary'
def _is_exit_early():
def main(args):
"""check that 2 ROOT files have same content (containers and sizes)
"""
global g_args
g_args = args
import PyUtils.RootUtils as ru
root = ru.import_root()
import PyUtils.Logging as L
msg = L.logging.getLogger('diff-root')
msg.setLevel(L.logging.INFO)
from PyUtils.Helpers import ShutUp, ROOT6Setup
ROOT6Setup()
if args.entries == '':
args.entries = -1
msg.info('comparing tree [%s] in files:', args.tree_name)
msg.info(' old: [%s]', args.old)
msg.info(' new: [%s]', args.new)
msg.info('ignore leaves: %s', args.ignore_leaves)
msg.info('enforce leaves: %s', args.enforce_leaves)
msg.info('hacks: %s', args.known_hacks)
msg.info('entries: %s', args.entries)
msg.info('mode: %s', args.mode)
msg.info('error mode: %s', args.error_mode)
import PyUtils.Helpers as H
with H.ShutUp():
fold = ru.RootFileDumper(args.old, args.tree_name)
fnew = ru.RootFileDumper(args.new, args.tree_name)
pass
def tree_infos(tree, args):
nentries = tree.GetEntriesFast()
# l.GetBranch().GetName() gives the full leaf path name
leaves = [
l.GetBranch().GetName() for l in tree.GetListOfLeaves()
if l.GetBranch().GetName() not in args.ignore_leaves
]
return {
'entries': nentries,
'leaves': set(leaves),
}
def diff_tree(fold, fnew, args):
infos = {
'old': tree_infos(fold.tree, args),
'new': tree_infos(fnew.tree, args),
}
nentries = min(infos['old']['entries'], infos['new']['entries'])
itr_entries = nentries
if args.entries in (-1, '', '-1'):
#msg.info('comparing over [%s] entries...', nentries)
itr_entries = nentries
if infos['old']['entries'] != infos['new']['entries']:
msg.info('different numbers of entries:')
msg.info(' old: [%s]', infos['old']['entries'])
msg.info(' new: [%s]', infos['new']['entries'])
msg.info('=> comparing [%s] first entries...', nentries)
else:
itr_entries = args.entries
pass
msg.info('comparing over [%s] entries...', itr_entries)
old_leaves = infos['old']['leaves'] - infos['new']['leaves']
if old_leaves:
msg.warning('the following variables exist only in the old file !')
for l in old_leaves:
msg.warning(' - [%s]', l)
new_leaves = infos['new']['leaves'] - infos['old']['leaves']
if new_leaves:
msg.warning('the following variables exist only in the new file !')
for l in new_leaves:
msg.warning(' - [%s]', l)
skip_leaves = old_leaves | new_leaves | set(args.ignore_leaves)
leaves = infos['old']['leaves'] & infos['new']['leaves']
msg.info('comparing [%s] leaves over entries...', len(leaves))
all_good = True
n_good = 0
n_bad = 0
import collections
from itertools import izip
summary = collections.defaultdict(int)
old_dump_iter = fold.dump(args.tree_name, itr_entries)
new_dump_iter = fnew.dump(args.tree_name, itr_entries)
def leafname_fromdump(entry):
return '.'.join([s for s in entry[2] if not s.isdigit()])
def reach_next(dump_iter, skip_leaves):
keep_reading = True
while keep_reading:
try:
entry = dump_iter.next()
except StopIteration:
return None
entry[2][0] = entry[2][0].rstrip('.\0') # clean branch name
name = []
skip = False
for n in leafname_fromdump(entry).split('.'):
name.append(n)
if '.'.join(name) in skip_leaves:
skip = True
break
if not skip:
return entry
# print 'SKIP:', leafname_fromdump(entry)
pass
read_old = True
read_new = True
d_old = None
d_new = None
while True:
if read_old:
prev_d_old = d_old
d_old = reach_next(old_dump_iter, skip_leaves)
if read_new:
prev_d_new = d_new
d_new = reach_next(new_dump_iter, skip_leaves)
if not d_new and not d_old:
break
read_old = True
read_new = True
if d_old == d_new:
n_good += 1
continue
if d_old:
tree_name, ientry, name, iold = d_old
if d_new:
tree_name, ientry, name, inew = d_new
# FIXME: that's a plain (temporary?) hack
if name[-1] in args.known_hacks:
continue
n_bad += 1
in_synch = d_old and d_new and d_old[:-1] == d_new[:-1]
if not in_synch:
if not _is_summary():
if d_old:
print '::sync-old %s' % '.'.join(["%03i" % ientry] +
map(str, d_old[2]))
else:
print '::sync-old ABSENT'
if d_new:
print '::sync-new %s' % '.'.join(["%03i" % ientry] +
map(str, d_new[2]))
else:
print '::sync-new ABSENT'
pass
# remember for later
if not d_old:
fold.allgood = False
summary[d_new[2][0]] += 1
elif not d_new:
fnew.allgood = False
summary[d_old[2][0]] += 1
else:
branch_old = '.'.join(["%03i" % ientry, d_old[2][0]])
branch_new = '.'.join(["%03i" % ientry, d_new[2][0]])
if branch_old < branch_new:
if not _is_summary():
print '::sync-old skipping entry'
summary[d_old[2][0]] += 1
fnew.allgood = False
read_new = False
elif branch_old > branch_new:
if not _is_summary():
print '::sync-new skipping entry'
summary[d_new[2][0]] += 1
fold.allgood = False
read_old = False
else:
# MN: difference in the leaves
prev_leaf_old = leafname_fromdump(prev_d_old)
prev_leaf_new = leafname_fromdump(prev_d_new)
leaf_old = leafname_fromdump(d_old)
leaf_new = leafname_fromdump(d_new)
if prev_leaf_old == prev_leaf_new:
# array size difference?
if leaf_old == leaf_new and leaf_old == prev_leaf_old:
# could be a size difference in >1 dim arrays
# hard to sync, skipping both
pass
elif leaf_old == prev_leaf_old:
# old has bigger array, skip old entry
read_new = False
if not _is_summary():
print '::sync-old skipping entry'
summary[leaf_old] += 1
elif leaf_new == prev_leaf_new:
# new has bigger array, skip new entry
read_old = False
if not _is_summary():
print '::sync-new skipping entry'
summary[leaf_new] += 1
if read_old and read_new:
summary[d_new[2][0]] += 1
if not _is_summary():
print '::sync-old+new skipping both entries'
fold.allgood = False
fnew.allgood = False
if _is_exit_early():
print "*** exit on first error ***"
break
continue
n = '.'.join(map(str, ["%03i" % ientry] + name))
diff_value = 'N/A'
try:
diff_value = 50. * (iold - inew) / (iold + inew)
diff_value = '%.8f%%' % (diff_value, )
except Exception:
pass
if not _is_summary():
print '%s %r -> %r => diff= [%s]' % (n, iold, inew, diff_value)
pass
summary[leafname_fromdump(d_old)] += 1
if name[0] in args.enforce_leaves:
msg.info("don't compare further")
all_good = False
break
pass # loop over events/branches
msg.info('Found [%s] identical leaves', n_good)
msg.info('Found [%s] different leaves', n_bad)
if not _is_summary():
keys = sorted(summary.keys())
for n in keys:
v = summary[n]
msg.info(' [%s]: %i leaves differ', n, v)
pass
pass
if (not fold.allgood) or (not fnew.allgood):
msg.info('NOTE: there were errors during the dump')
msg.info('fold.allgood: %s' % fold.allgood)
msg.info('fnew.allgood: %s' % fnew.allgood)
n_bad += 0.5
return n_bad
ndiff = diff_tree(fold, fnew, args)
if ndiff != 0:
msg.info('files differ!')
return 2
msg.info('all good.')
return 0
def main(args):
"""check that 2 ROOT files have same content (containers and sizes)
"""
global g_args
g_args = args
import PyUtils.RootUtils as ru
root = ru.import_root()
import PyUtils.Logging as L
msg = L.logging.getLogger('diff-root')
msg.setLevel(L.logging.INFO)
if args.entries == '':
args.entries = -1
msg.info('comparing tree [%s] in files:', args.tree_name)
msg.info(' old: [%s]', args.old)
msg.info(' new: [%s]', args.new)
msg.info('ignore leaves: %s', args.ignore_leaves)
msg.info('enforce leaves: %s', args.enforce_leaves)
msg.info('hacks: %s', args.known_hacks)
msg.info('entries: %s', args.entries)
msg.info('mode: %s', args.mode)
msg.info('error mode: %s', args.error_mode)
import PyUtils.Helpers as H
with H.ShutUp() :
fold = ru.RootFileDumper(args.old, args.tree_name)
fnew = ru.RootFileDumper(args.new, args.tree_name)
pass
def tree_infos(tree, args):
nentries = tree.GetEntriesFast()
leaves = [l.GetName() for l in tree.GetListOfLeaves()
if l not in args.ignore_leaves]
return {
'entries' : nentries,
'leaves': set(leaves),
}
def diff_tree(fold, fnew, args):
infos = {
'old' : tree_infos(fold.tree, args),
'new' : tree_infos(fnew.tree, args),
}
nentries = min(infos['old']['entries'],
infos['new']['entries'])
itr_entries = nentries
if args.entries in (-1,'','-1'):
#msg.info('comparing over [%s] entries...', nentries)
itr_entries = nentries
if infos['old']['entries'] != infos['new']['entries']:
msg.info('different numbers of entries:')
msg.info(' old: [%s]', infos['old']['entries'])
msg.info(' new: [%s]', infos['new']['entries'])
msg.info('=> comparing [%s] first entries...', nentries)
else:
itr_entries = args.entries
pass
msg.info('comparing over [%s] entries...', itr_entries)
leaves = infos['old']['leaves'] & infos['new']['leaves']
diff_leaves = infos['old']['leaves'] - infos['new']['leaves']
if diff_leaves:
msg.info('the following variables exist in only one tree !')
for l in diff_leaves:
msg.info(' - [%s]', l)
leaves = leaves - set(args.ignore_leaves)
msg.info('comparing [%s] leaves over entries...', len(leaves))
all_good = True
n_good = 0
n_bad = 0
import collections
from itertools import izip
summary = collections.defaultdict(int)
for d in izip(fold.dump(args.tree_name, itr_entries),
fnew.dump(args.tree_name, itr_entries)):
tree_name, ientry, name, iold = d[0]
_, _, _, inew = d[1]
name[0] = name[0].rstrip('\0')
if ((not (name[0] in leaves)) or
# FIXME: that's a plain (temporary?) hack
name[-1] in args.known_hacks):
continue
if d[0] == d[1]:
diff = False
n_good += 1
continue
n_bad += 1
diff = True
in_synch = d[0][:-1] == d[1][:-1]
if not in_synch:
if not _is_summary():
print '::sync-old %s' % \
'.'.join(["%03i"%ientry]+map(str, d[0][2]))
print '::sync-new %s' % \
'.'.join(["%03i"%ientry]+map(str, d[1][2]))
pass
summary[name[0]] += 1
# remember for later
fold.allgood = False
fnew.allgood = False
if _is_exit_early():
print "*** exit on first error ***"
break
continue
n = '.'.join(map(str, ["%03i"%ientry]+name))
diff_value = 'N/A'
try:
diff_value = 50.*(iold-inew)/(iold+inew)
diff_value = '%.8f%%' % (diff_value,)
except Exception:
pass
if not _is_summary():
print '%s %r -> %r => diff= [%s]' %(n, iold, inew, diff_value)
pass
summary[name[0]] += 1
if name[0] in args.enforce_leaves:
msg.info("don't compare further")
all_good = False
break
pass # loop over events/branches
msg.info('Found [%s] identical leaves', n_good)
msg.info('Found [%s] different leaves', n_bad)
if not _is_summary():
keys = sorted(summary.keys())
for n in keys:
v = summary[n]
msg.info(' [%s]: %i leaves differ', n, v)
pass
pass
if (not fold.allgood) or (not fnew.allgood):
msg.info('NOTE: there were errors during the dump')
msg.info('fold.allgood: %s' % fold.allgood)
msg.info('fnew.allgood: %s' % fnew.allgood)
n_bad += 0.5
return n_bad
ndiff = diff_tree(fold, fnew, args)
if ndiff != 0:
msg.info('files differ!')
return 2
msg.info('all good.')
return 0
def extract_streams_from_tag (fname,
tree_name="POOLCollectionTree",
nentries=-1,
stream_refs=None):
"""find the GUID(s) of a list of streams which are contained inside a TAG
file.
@params:
`fname` the filename of the TAG file to inspect
(can be a LFN or FID)
`tree_name` the name of the TTree containing the stream-refs
('CollectionTree' is the usual default)
`nentries` the number of entries to inspect, among the ttree entries
(-1: all the entries)
`stream_refs` a list of stream names
(ex: ['StreamAOD_ref', 'Stream1_ref'] or,
None - to inspect all the stream refs in the TAG)
example:
>>> streams = extract_streams_from_tag ('tag.pool')
>>> from pprint import pprint
>>> pprint (streams)
{'Stream1_ref': ['BEE16671-B9F6-DA11-8219-00304871B611'],
'StreamAOD_ref': ['96F3018E-A0AC-DD11-8115-000423D59D52'],
'StreamESD_ref': ['384D0CFC-9FAC-DD11-A905-000423D59D52'],
'StreamRDO_ref': ['22C5BD99-3059-DB11-8D44-0030488365E6']}
>>> from PoolFile import PoolFileCatalog as pfc
>>> pprint (pfc().pfn(streams['StreamAOD_ref'][0]))
['aod.pool']
"""
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
print("::: opening file [%s]..." % fname)
# get the "final" file name (handles all kind of protocols)
import PyUtils.AthFile as af
try:
protocol, fname = af.server.fname(fname)
except Exception:
print("::: warning: problem extracting file name from PoolFileCatalog")
print("::: warning: will use [%s]" % fname)
f = ROOT.TFile.Open (fname, "read")
assert not f.IsZombie() and f.IsOpen(), \
"problem opening POOL file [%s]"%fname
# backward compat:
# in 15.2.0 TAG file got a new tree name 'POOLCollectionTree'
# it was named 'CollectionTree' before that...
keys = [k.GetName() for k in f.GetListOfKeys()]
if tree_name not in keys and "CollectionTree" not in keys:
err= "::: error: neither [%s] nor [CollectionTree] in file [%s]" % (
tree_name, fname)
print(err)
raise RuntimeError(err)
# try the backward compat. hack
if tree_name not in keys:
tree_name = "CollectionTree"
t = f.Get(tree_name)
assert isinstance(t, ROOT.TTree), \
"could not retrieve tree [%s]" % tree_name
# interesting branch names
branches = [str(b.GetName()) for b in t.GetListOfBranches()
if b.GetName().endswith ("_ref")]
if stream_refs is None:
stream_refs = branches
else:
_streams = stream_refs[:]
stream_refs = []
for ref in _streams:
if ref not in branches:
print("::: discarding [%s] from file chasing..."%ref)
else:
stream_refs.append (ref)
if nentries <= 0: nentries = t.GetEntries()
else: nentries = min (nentries, t.GetEntries())
print("::: chasing streams: %s" % stream_refs)
print("::: ...over entries: %r" % nentries)
# disable everything...
t.SetBranchStatus ("*", 0)
streams = dict()
for ref in stream_refs:
streams[ref] = list() # list of FileIDs, according to POOL
# ex: 'B2B485E1-BB37-DD11-984C-0030487A17BA'
# but (re-)enable the branches we are interested in
t.SetBranchStatus (ref, 1)
import re
# Pool token are of the form:
# '[DB=7CCD8D32-BC37-DD11-967E-0030487CD916]\
# [CNT=POOLContainer_DataHeader]\
# [CLID=72FBBC6F-C8BE-4122-8790-DC627696C176]\
# [TECH=00000202]\
# [OID=0000008C-000002BA]'
token = re.compile (r'[[]DB=(?P<FID>.*?)[]]'
r'[[]CNT=(?P<CNT>.*?)[]]'
r'[[]CLID=(?P<CLID>.*?)[]]'
r'[[]TECH=(?P<TECH>.*?)[]]'
r'[[]OID=(?P<OID>.*?)[]]')
for i in range(nentries):
t.GetEntry (i)
for ref in stream_refs:
try:
token_str = getattr(t, ref)
except (AttributeError, TypeError):
# MN: TypeError is a bug in ROOT 5.34.25, fixed in 5.34.30
# filthy work-around...
try:
token_branch = t.GetBranch (ref)
token_branch.GetEntry(i)
token_str = token_branch.GetLeaf("Token").GetValueString()
except Exception:
print("::: could not access stream-ref [%s] (entry #%i)",
(ref, i))
continue
tok = token.match (token_str)
if not tok:
print("::: invalid POOL token: [%s]" % token_str)
continue
streams[ref].append (tok.group('FID'))
for ref in stream_refs:
streams[ref] = list(set(streams[ref]))
return streams
def main(args):
"""diff two ROOT files (containers and sizes)"""
global g_args
g_args = args
import PyUtils.RootUtils as ru
root = ru.import_root() # noqa: F841
import PyUtils.Logging as L
msg = L.logging.getLogger('diff-root')
if args.verbose:
msg.setLevel(L.logging.VERBOSE)
else:
msg.setLevel(L.logging.INFO)
from PyUtils.Helpers import ShutUp # noqa: F401
if args.entries == '':
args.entries = -1
msg.info('comparing tree [%s] in files:', args.tree_name)
msg.info(' old: [%s]', args.old)
msg.info(' new: [%s]', args.new)
msg.info('ignore leaves: %s', args.ignore_leaves)
msg.info('enforce leaves: %s', args.enforce_leaves)
msg.info('leaves prefix: %s', args.leaves_prefix)
msg.info('hacks: %s', args.known_hacks)
msg.info('entries: %s', args.entries)
msg.info('mode: %s', args.mode)
msg.info('error mode: %s', args.error_mode)
msg.info('order trees: %s', args.order_trees)
import PyUtils.Helpers as H
with H.ShutUp():
fold = ru.RootFileDumper(args.old, args.tree_name)
fnew = ru.RootFileDumper(args.new, args.tree_name)
pass
def tree_infos(tree, args):
nentries = tree.GetEntriesFast()
# l.GetBranch().GetName() gives the full leaf path name
leaves = [
l.GetBranch().GetName() for l in tree.GetListOfLeaves()
if l.GetBranch().GetName() not in args.ignore_leaves
]
if args.leaves_prefix:
leaves = [l.replace(args.leaves_prefix, '') for l in leaves]
return {
'entries': nentries,
'leaves': set(leaves),
}
def ordered_indices(tree, reverse_order=False):
from collections import OrderedDict
import operator
dict_in = {}
nevts = tree.GetEntriesFast()
for idx in range(0, nevts):
if idx % 100 == 0:
msg.debug('Read {} events from the input so far'.format(idx))
tree.GetEntry(idx)
if hasattr(tree, 'xAOD::EventAuxInfo_v2_EventInfoAux.'):
event_info = getattr(tree,
'xAOD::EventAuxInfo_v2_EventInfoAux.')
event_number = event_info.eventNumber
elif hasattr(tree, 'xAOD::EventAuxInfo_v1_EventInfoAux.'):
event_info = getattr(tree,
'xAOD::EventAuxInfo_v1_EventInfoAux.')
event_number = event_info.eventNumber
elif hasattr(tree, 'EventInfoAux.'):
event_info = getattr(tree, 'EventInfoAux.')
event_number = event_info.eventNumber
elif hasattr(tree, 'EventInfo_p4_McEventInfo'):
event_info = getattr(tree, 'EventInfo_p4_McEventInfo')
event_number = event_info.m_event_ID.m_event_number
elif hasattr(tree, 'EventInfo_p4_ByteStreamEventInfo'):
event_info = getattr(tree, 'EventInfo_p4_ByteStreamEventInfo')
event_number = event_info.m_event_ID.m_event_number
elif hasattr(tree, 'ByteStreamEventInfo'):
event_info = getattr(tree, 'ByteStreamEventInfo')
event_number = event_info.m_event_ID.m_event_number
else:
msg.error('Cannot read event info, will bail out.')
break
msg.debug('Idx : EvtNum {:10d} : {}'.format(idx, event_number))
dict_in[idx] = event_number
# Sort the dictionary by event numbers
dict_out = OrderedDict(
sorted(dict_in.items(),
key=operator.itemgetter(1),
reverse=reverse_order))
# Write out the ordered index list
return [idx for idx in dict_out]
def diff_tree(fold, fnew, args):
infos = {
'old': tree_infos(fold.tree, args),
'new': tree_infos(fnew.tree, args),
}
nentries = min(infos['old']['entries'], infos['new']['entries'])
itr_entries = nentries
if args.entries in (-1, '', '-1'):
#msg.info('comparing over [%s] entries...', nentries)
itr_entries = nentries
if infos['old']['entries'] != infos['new']['entries']:
msg.info('different numbers of entries:')
msg.info(' old: [%s]', infos['old']['entries'])
msg.info(' new: [%s]', infos['new']['entries'])
msg.info('=> comparing [%s] first entries...', nentries)
else:
itr_entries = args.entries
pass
msg.info('comparing over [%s] entries...', itr_entries)
old_leaves = infos['old']['leaves'] - infos['new']['leaves']
if old_leaves:
msg.warning('the following variables exist only in the old file !')
for l in old_leaves:
msg.warning(' - [%s]', l)
new_leaves = infos['new']['leaves'] - infos['old']['leaves']
if new_leaves:
msg.warning('the following variables exist only in the new file !')
for l in new_leaves:
msg.warning(' - [%s]', l)
# need to remove trailing dots as they confuse reach_next()
skip_leaves = [
l.rstrip('.')
for l in old_leaves | new_leaves | set(args.ignore_leaves)
]
for l in skip_leaves:
msg.debug('skipping [%s]', l)
leaves = infos['old']['leaves'] & infos['new']['leaves']
msg.info('comparing [%s] leaves over entries...', len(leaves))
n_good = 0
n_bad = 0
import collections
summary = collections.defaultdict(int)
if args.order_trees:
slice_max = int(itr_entries) if int(itr_entries) > 0 else None
itr_entries_old = ordered_indices(fold.tree)[0:slice_max]
itr_entries_new = ordered_indices(fnew.tree)[0:slice_max]
msg.debug('List of old indices {}'.format(itr_entries_old))
msg.debug('List of new indices {}'.format(itr_entries_new))
else:
itr_entries_old = itr_entries
itr_entries_new = itr_entries
old_dump_iter = fold.dump(args.tree_name, itr_entries_old)
new_dump_iter = fnew.dump(args.tree_name, itr_entries_new)
def leafname_fromdump(entry):
return '.'.join([s for s in entry[2] if not s.isdigit()])
def reach_next(dump_iter, skip_leaves, leaves_prefix=None):
keep_reading = True
while keep_reading:
try:
entry = next(dump_iter)
except StopIteration:
return None
entry[2][0] = entry[2][0].rstrip('.\0') # clean branch name
if leaves_prefix:
entry[2][0] = entry[2][0].replace(leaves_prefix, '')
name = []
skip = False
for n in leafname_fromdump(entry).split('.'):
name.append(n)
if '.'.join(name) in skip_leaves or n in skip_leaves:
skip = True
break
if not skip:
return entry
# print('SKIP:', leafname_fromdump(entry))
pass
read_old = True
read_new = True
d_old = None
d_new = None
while True:
if read_old:
prev_d_old = d_old
d_old = reach_next(old_dump_iter, skip_leaves,
args.leaves_prefix)
if read_new:
prev_d_new = d_new
d_new = reach_next(new_dump_iter, skip_leaves,
args.leaves_prefix)
if not d_new and not d_old:
break
read_old = True
read_new = True
if (args.order_trees and d_old and d_new
and d_old[-1] == d_new[-1]) or d_old == d_new:
n_good += 1
continue
if d_old:
tree_name, ientry, name, iold = d_old
if d_new:
tree_name, jentry, name, inew = d_new
# for regression testing we should have NAN == NAN
if args.nan_equal:
if all(
[isinstance(x, Real) and isnan(x) for x in [iold, inew]]):
n_good += 1
continue
# FIXME: that's a plain (temporary?) hack
if name[-1] in args.known_hacks:
continue
n_bad += 1
if not args.order_trees:
in_synch = d_old and d_new and d_old[:-1] == d_new[:-1]
else:
in_synch = d_old and d_new and d_old[0] == d_new[0] and d_old[
2] == d_new[2]
if not in_synch:
if _is_detailed():
if d_old:
print('::sync-old %s' %
'.'.join(["%03i" % ientry] +
list(map(str, d_old[2]))))
else:
print('::sync-old ABSENT')
if d_new:
print('::sync-new %s' %
'.'.join(["%03i" % jentry] +
list(map(str, d_new[2]))))
else:
print('::sync-new ABSENT')
pass
# remember for later
if not d_old:
fold.allgood = False
summary[d_new[2][0]] += 1
elif not d_new:
fnew.allgood = False
summary[d_old[2][0]] += 1
else:
branch_old = '.'.join(["%03i" % ientry, d_old[2][0]])
branch_new = '.'.join(["%03i" % jentry, d_new[2][0]])
if branch_old < branch_new:
if _is_detailed():
print('::sync-old skipping entry')
summary[d_old[2][0]] += 1
fnew.allgood = False
read_new = False
elif branch_old > branch_new:
if _is_detailed():
print('::sync-new skipping entry')
summary[d_new[2][0]] += 1
fold.allgood = False
read_old = False
else:
# MN: difference in the leaves
prev_leaf_old = leafname_fromdump(prev_d_old)
prev_leaf_new = leafname_fromdump(prev_d_new)
leaf_old = leafname_fromdump(d_old)
leaf_new = leafname_fromdump(d_new)
if prev_leaf_old == prev_leaf_new:
# array size difference?
if leaf_old == leaf_new and leaf_old == prev_leaf_old:
# could be a size difference in >1 dim arrays
# hard to sync, skipping both
pass
elif leaf_old == prev_leaf_old:
# old has bigger array, skip old entry
read_new = False
if _is_detailed():
print('::sync-old skipping entry')
summary[leaf_old] += 1
elif leaf_new == prev_leaf_new:
# new has bigger array, skip new entry
read_old = False
if _is_detailed():
print('::sync-new skipping entry')
summary[leaf_new] += 1
if read_old and read_new:
summary[d_new[2][0]] += 1
if _is_detailed():
print('::sync-old+new skipping both entries')
fold.allgood = False
fnew.allgood = False
if _is_exit_early():
print('*** exit on first error ***')
break
continue
if not args.order_trees:
n = '.'.join(list(map(str, ["%03i" % ientry] + name)))
else:
n = '.'.join(
list(map(str, ["%03i.%03i" % (ientry, jentry)] + name)))
diff_value = 'N/A'
try:
diff_value = 50. * (iold - inew) / (iold + inew)
diff_value = '%.8f%%' % (diff_value, )
except Exception:
pass
if _is_detailed():
print('%s %r -> %r => diff= [%s]' %
(n, iold, inew, diff_value))
pass
summary[leafname_fromdump(d_old)] += 1
if name[0] in args.enforce_leaves:
msg.info("don't compare further")
break
pass # loop over events/branches
msg.info('Found [%s] identical leaves', n_good)
msg.info('Found [%s] different leaves', n_bad)
if not _is_summary():
keys = sorted(summary.keys())
for n in keys:
v = summary[n]
msg.info(' [%s]: %i leaves differ', n, v)
pass
pass
if (not fold.allgood) or (not fnew.allgood):
msg.info('NOTE: there were errors during the dump')
msg.info('fold.allgood: %s', fold.allgood)
msg.info('fnew.allgood: %s', fnew.allgood)
n_bad += 0.5
return n_bad
ndiff = diff_tree(fold, fnew, args)
if ndiff != 0:
msg.info('files differ!')
return 2
msg.info('all good.')
return 0
def initialize(self):
self.msg.info( '************************************' )
self.msg.info( '==> initialize %s...', self.name() )
self.msg.info( 'Will apply the following electron cuts:' )
self.msg.info( ' EtMinEl = %r', self.cutEtMinEl )
self.msg.info( ' IsEmEl = %s', self.cutIsEM )
self.msg.info( ' AuthorEl = %s', self.cutElAuthor )
self.msg.info( ' ElEtaMax = %r', self.cutElEtaMax )
self.msg.info( ' removeEtaCrack = %r', self.removeEtaCrack )
self.msg.info( ' crackEtaMin = %r', self.crackEtaMin )
self.msg.info( ' crackEtaMax = %r', self.crackEtaMax )
self.msg.info( ' minNumberPassedEl = %r', self.minNumberPassedEl )
self.msg.info( ' passAllEl = %r', self.passAllEl )
self.msg.info( 'And the following muon cuts:' )
self.msg.info( ' PtMinMu = %r', self.cutPtMinMu )
self.msg.info( ' AuthorMu = %s', self.muonAuthors )
self.msg.info( ' MuEtaMax = %r', self.cutMuEtaMax )
if self.doEtconeMu: self.msg.info( ' muon etcone cut = %r', self.etconeMuMax )
self.msg.info( ' passAllMu = %r', self.passAllMu )
self.msg.info( ' minNumberPassedMu = %r', self.minNumberPassedMu )
self.msg.info( '************************************' )
## Also, declare some counters and initialize them to zero
self.nProcessed = 0
self.nEventMinNumPassed = 0
self.nElectrons = 0
self.nEventElPassEta = 0
self.nEventElPassEt = 0
self.nEventElPassIsEM = 0
self.nEventElPassAuthor = 0
self.nMuons = 0
self.nEventMuPassEta = 0
self.nEventMuPassPt = 0
self.nEventMuPassAuthor = 0
self.nEventMuPassPrescale = 0
self.nEventMuPassIsolation = 0
## Import needed modules
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
if self.doNoTrackIsolMu or self.doPtTrackIsolMu or self.doCaloIsolMu :
if self.muonIsolationTool is None:
self.msg.warning("Problem retrieving MuonIsolationTool pointer ")
self.msg.warning("setting Track and Calo isolation to False ")
self.doNoTrackIsolMu = False
self.doPtTrackIsolMu = False
self.doCaloIsolMu = False
## import some 4-mom utils
import FourMomUtils.Bindings
self.utils = { 'deltaR' : PyAthena.P4Helpers.deltaR }
_append = self.muContainerNames.append
if self.usingAOD :
if self.muonContainers.__contains__('staco') or self.muonContainers is 'all':
_append("StacoMuonCollection")
if self.muonContainers.__contains__('muid') or self.muonContainers is 'all':
_append("MuidMuonCollection")
if self.muonContainers.__contains__('calo') or self.muonContainers is 'all':
_append("CaloMuonCollection")
else :
if self.muonContainers.__contains__('staco') or self.muonContainers is 'all':
_append("StacoESDMuonCollection")
if self.muonAuthors is 'combined' or self.muonAuthors is 'all' :
_append("StacoCombinedMuonContainer")
if self.muonAuthors is 'lowpt' or self.muonAuthors is 'all' :
_append("StacoCombinedMuonContainerLowPt")
if self.muonAuthors is 'standalone' or self.muonAuthors is 'all':
_append("MboyESDMuonContainer")
if self.muonContainers.__contains__('muid') or self.muonContainers is 'all':
_append("MuidESDMuonCollection")
if self.muonAuthors is 'combined' or self.muonAuthors is 'all' :
_append("MuidCombinedMuonContainer")
if self.muonAuthors is 'lowpt' or self.muonAuthors is 'all' :
_append("MuGirlCombinedMuonContainer")
if self.muonAuthors is 'standalone' or self.muonAuthors is 'all' :
_append("MuidExtrMuonContainer")
if self.muonContainers.__contains__('calo') or self.muonContainers is 'all':
if self.muonAuthors is 'calo' or self.muonAuthors is 'all' :
_append("CaloMuonIdContainer")
_append("CaloESDMuonCollection")
del _append
## Get the StoreGate service
self.storeGateSvc = PyAthena.py_svc('StoreGateSvc')
if self.storeGateSvc is None:
self.msg.error("Problem retrieving StoreGateSvc pointer !!")
return StatusCode.Failure
return StatusCode.Success
# @author: Carlos Chavez
from __future__ import print_function
import logging
msg = logging.getLogger("PyJobTransforms." + __name__)
import os
import re
import eformat
from TrigTransform.dbgEventInfo import dbgEventInfo
from TrigConfStorage.TriggerCoolUtil import TriggerCoolUtil
from PyCool import cool
from PyUtils import RootUtils
ROOT = RootUtils.import_root()
from ROOT import TFile
def dbgPreRun(inputFileList,outputFileList):
msg.info('Running debug_stream analysis PreRun operations on files :{0} '.format(inputFileList))
msg.info('Running debug_stream analysis PreRun, histogram output in :{0} '.format(outputFileList))
total = 0
#open root output file
out_file = outputFileList[0]
hfile = TFile( out_file , 'RECREATE' )
#inicialize dbgEventInfo, this is the main event analysis class
eventInfo = dbgEventInfo("_Pre",inputFileList.value[0])
data = []
l1Info = []
# @file PyUtils.scripts.diff_root_files
# @purpose check that 2 ROOT files have same content (containers and sizes).
# @author Sebastien Binet
# @date February 2010
__version__ = "$Revision$"
__doc__ = "check that 2 ROOT files have same content (containers and sizes)."
__author__ = "Sebastien Binet"
### imports -------------------------------------------------------------------
import PyUtils.acmdlib as acmdlib
import PyUtils.RootUtils as ru
ROOT = ru.import_root()
### globals -------------------------------------------------------------------
g_ALLOWED_MODES = ('summary', 'detailed')
g_ALLOWED_ERROR_MODES = ('bailout', 'resilient')
g_args = None
### classes -------------------------------------------------------------------
### functions -----------------------------------------------------------------
def _is_summary():
global g_args
return g_args.mode == 'summary'
def _is_exit_early():
global g_args
return g_args.error_mode == 'bailout'
@acmdlib.command(name='diff-root')