def _setupRoot6IfNeeded():
import cppyy
try:
# try to touch ROOT5-only attribute (success means we have ROOT5)
cppyy.Cintex.Debug
except AttributeError:
# ROOT 6
from PyUtils.Helpers import ROOT6Setup
ROOT6Setup()
import re
import ROOT
import cppyy
import string
from AthenaROOTAccess.persTypeToTransType import persTypeToTransType
from AthenaROOTAccess.gaudiSvcExists import gaudiSvcExists
from RootUtils import PyROOTFixes
import ROOT
import glob
import sys
import os
from PyUtils.Helpers import ROOT6Setup
ROOT6Setup()
# Turn off annoying dict auto-generation --- it doesn't work anyway.
ROOT.gInterpreter.ProcessLine(".autodict")
# Make sure the proper dictionaries are loaded.
cppyy.loadDictionary('libAtlasSTLAddReflexDict')
# Make sure abstract base classes have streaminfos built.
# Otherwise, we can get crashes from TTree::Scan.
# Prevent AthenaBarCodeImpl from trying to create JobIDSvc.
import os
import uuid
os.environ['_ATHENABARCODEIMPL_JOBUUID'] = uuid.uuid1().hex
def main(args):
"""check that 2 ROOT files have same content (containers and sizes)
"""
global g_args
g_args = args
import PyUtils.RootUtils as ru
root = ru.import_root()
import PyUtils.Logging as L
msg = L.logging.getLogger('diff-root')
msg.setLevel(L.logging.INFO)
from PyUtils.Helpers import ShutUp, ROOT6Setup
ROOT6Setup()
if args.entries == '':
args.entries = -1
msg.info('comparing tree [%s] in files:', args.tree_name)
msg.info(' old: [%s]', args.old)
msg.info(' new: [%s]', args.new)
msg.info('ignore leaves: %s', args.ignore_leaves)
msg.info('enforce leaves: %s', args.enforce_leaves)
msg.info('hacks: %s', args.known_hacks)
msg.info('entries: %s', args.entries)
msg.info('mode: %s', args.mode)
msg.info('error mode: %s', args.error_mode)
import PyUtils.Helpers as H
with H.ShutUp():
fold = ru.RootFileDumper(args.old, args.tree_name)
fnew = ru.RootFileDumper(args.new, args.tree_name)
pass
def tree_infos(tree, args):
nentries = tree.GetEntriesFast()
# l.GetBranch().GetName() gives the full leaf path name
leaves = [
l.GetBranch().GetName() for l in tree.GetListOfLeaves()
if l.GetBranch().GetName() not in args.ignore_leaves
]
return {
'entries': nentries,
'leaves': set(leaves),
}
def diff_tree(fold, fnew, args):
infos = {
'old': tree_infos(fold.tree, args),
'new': tree_infos(fnew.tree, args),
}
nentries = min(infos['old']['entries'], infos['new']['entries'])
itr_entries = nentries
if args.entries in (-1, '', '-1'):
#msg.info('comparing over [%s] entries...', nentries)
itr_entries = nentries
if infos['old']['entries'] != infos['new']['entries']:
msg.info('different numbers of entries:')
msg.info(' old: [%s]', infos['old']['entries'])
msg.info(' new: [%s]', infos['new']['entries'])
msg.info('=> comparing [%s] first entries...', nentries)
else:
itr_entries = args.entries
pass
msg.info('comparing over [%s] entries...', itr_entries)
old_leaves = infos['old']['leaves'] - infos['new']['leaves']
if old_leaves:
msg.warning('the following variables exist only in the old file !')
for l in old_leaves:
msg.warning(' - [%s]', l)
new_leaves = infos['new']['leaves'] - infos['old']['leaves']
if new_leaves:
msg.warning('the following variables exist only in the new file !')
for l in new_leaves:
msg.warning(' - [%s]', l)
skip_leaves = old_leaves | new_leaves | set(args.ignore_leaves)
leaves = infos['old']['leaves'] & infos['new']['leaves']
msg.info('comparing [%s] leaves over entries...', len(leaves))
all_good = True
n_good = 0
n_bad = 0
import collections
from itertools import izip
summary = collections.defaultdict(int)
old_dump_iter = fold.dump(args.tree_name, itr_entries)
new_dump_iter = fnew.dump(args.tree_name, itr_entries)
def leafname_fromdump(entry):
return '.'.join([s for s in entry[2] if not s.isdigit()])
def reach_next(dump_iter, skip_leaves):
keep_reading = True
while keep_reading:
try:
entry = dump_iter.next()
except StopIteration:
return None
entry[2][0] = entry[2][0].rstrip('.\0') # clean branch name
name = []
skip = False
for n in leafname_fromdump(entry).split('.'):
name.append(n)
if '.'.join(name) in skip_leaves:
skip = True
break
if not skip:
return entry
# print 'SKIP:', leafname_fromdump(entry)
pass
read_old = True
read_new = True
d_old = None
d_new = None
while True:
if read_old:
prev_d_old = d_old
d_old = reach_next(old_dump_iter, skip_leaves)
if read_new:
prev_d_new = d_new
d_new = reach_next(new_dump_iter, skip_leaves)
if not d_new and not d_old:
break
read_old = True
read_new = True
if d_old == d_new:
n_good += 1
continue
if d_old:
tree_name, ientry, name, iold = d_old
if d_new:
tree_name, ientry, name, inew = d_new
# FIXME: that's a plain (temporary?) hack
if name[-1] in args.known_hacks:
continue
n_bad += 1
in_synch = d_old and d_new and d_old[:-1] == d_new[:-1]
if not in_synch:
if not _is_summary():
if d_old:
print '::sync-old %s' % '.'.join(["%03i" % ientry] +
map(str, d_old[2]))
else:
print '::sync-old ABSENT'
if d_new:
print '::sync-new %s' % '.'.join(["%03i" % ientry] +
map(str, d_new[2]))
else:
print '::sync-new ABSENT'
pass
# remember for later
if not d_old:
fold.allgood = False
summary[d_new[2][0]] += 1
elif not d_new:
fnew.allgood = False
summary[d_old[2][0]] += 1
else:
branch_old = '.'.join(["%03i" % ientry, d_old[2][0]])
branch_new = '.'.join(["%03i" % ientry, d_new[2][0]])
if branch_old < branch_new:
if not _is_summary():
print '::sync-old skipping entry'
summary[d_old[2][0]] += 1
fnew.allgood = False
read_new = False
elif branch_old > branch_new:
if not _is_summary():
print '::sync-new skipping entry'
summary[d_new[2][0]] += 1
fold.allgood = False
read_old = False
else:
# MN: difference in the leaves
prev_leaf_old = leafname_fromdump(prev_d_old)
prev_leaf_new = leafname_fromdump(prev_d_new)
leaf_old = leafname_fromdump(d_old)
leaf_new = leafname_fromdump(d_new)
if prev_leaf_old == prev_leaf_new:
# array size difference?
if leaf_old == leaf_new and leaf_old == prev_leaf_old:
# could be a size difference in >1 dim arrays
# hard to sync, skipping both
pass
elif leaf_old == prev_leaf_old:
# old has bigger array, skip old entry
read_new = False
if not _is_summary():
print '::sync-old skipping entry'
summary[leaf_old] += 1
elif leaf_new == prev_leaf_new:
# new has bigger array, skip new entry
read_old = False
if not _is_summary():
print '::sync-new skipping entry'
summary[leaf_new] += 1
if read_old and read_new:
summary[d_new[2][0]] += 1
if not _is_summary():
print '::sync-old+new skipping both entries'
fold.allgood = False
fnew.allgood = False
if _is_exit_early():
print "*** exit on first error ***"
break
continue
n = '.'.join(map(str, ["%03i" % ientry] + name))
diff_value = 'N/A'
try:
diff_value = 50. * (iold - inew) / (iold + inew)
diff_value = '%.8f%%' % (diff_value, )
except Exception:
pass
if not _is_summary():
print '%s %r -> %r => diff= [%s]' % (n, iold, inew, diff_value)
pass
summary[leafname_fromdump(d_old)] += 1
if name[0] in args.enforce_leaves:
msg.info("don't compare further")
all_good = False
break
pass # loop over events/branches
msg.info('Found [%s] identical leaves', n_good)
msg.info('Found [%s] different leaves', n_bad)
if not _is_summary():
keys = sorted(summary.keys())
for n in keys:
v = summary[n]
msg.info(' [%s]: %i leaves differ', n, v)
pass
pass
if (not fold.allgood) or (not fnew.allgood):
msg.info('NOTE: there were errors during the dump')
msg.info('fold.allgood: %s' % fold.allgood)
msg.info('fnew.allgood: %s' % fnew.allgood)
n_bad += 0.5
return n_bad
ndiff = diff_tree(fold, fnew, args)
if ndiff != 0:
msg.info('files differ!')
return 2
msg.info('all good.')
return 0
def setupCommonServices():
from AthenaCommon import CfgMgr
from AthenaCommon.Logging import logging
from AthenaCommon.Constants import INFO
from AthenaCommon.AppMgr import ServiceMgr as svcMgr, theApp
from AthenaCommon.ConcurrencyFlags import jobproperties as jps
# Setup messaging for Python and C++
from AthenaCommon.Logging import log
log.setFormat("%(asctime)s Py:%(name)-31s %(levelname)7s %(message)s")
# Create our own logger
log = logging.getLogger('TriggerUnixStandardSetup::setupCommonServices:')
from TrigServices.TrigServicesConfig import setupMessageSvc
setupMessageSvc()
# Do the default Atlas job configuration first
import AthenaCommon.AtlasUnixStandardJob # noqa: F401
# Now do HLT/thread specific configuration (see e.g. AtlasThreadedJob.py)
from StoreGate.StoreGateConf import SG__HiveMgrSvc
svcMgr += SG__HiveMgrSvc("EventDataSvc",
NSlots=jps.ConcurrencyFlags.NumConcurrentEvents())
import StoreGate.StoreGateConf as StoreGateConf
svcMgr += StoreGateConf.StoreGateSvc("ConditionStore")
# Configure the CoreDumpSvc
if not hasattr(svcMgr, "CoreDumpSvc"):
from AthenaServices.Configurables import CoreDumpSvc
svcMgr += CoreDumpSvc()
# ThreadPoolService thread local initialization
from GaudiHive.GaudiHiveConf import ThreadPoolSvc
svcMgr += ThreadPoolSvc("ThreadPoolSvc")
svcMgr.ThreadPoolSvc.ThreadInitTools = ["ThreadInitTool"]
from GaudiHive.GaudiHiveConf import AlgResourcePool
svcMgr += AlgResourcePool(OutputLevel=INFO,
TopAlg=["AthSequencer/AthMasterSeq"])
from AthenaCommon.AlgSequence import AlgSequence
from SGComps.SGCompsConf import SGInputLoader
topSequence = AlgSequence()
topSequence += SGInputLoader(
FailIfNoProxy=False) # change to True eventually
from AthenaCommon.AlgScheduler import AlgScheduler
AlgScheduler.ShowDataDependencies(False)
AlgScheduler.ShowControlFlow(False)
AlgScheduler.setDataLoaderAlg('SGInputLoader')
# Setup SGCommitAuditor to sweep new DataObjects at end of Alg execute
theApp.AuditAlgorithms = True
from SGComps.SGCompsConf import SGCommitAuditor
svcMgr.AuditorSvc += SGCommitAuditor()
# setup ROOT6
from PyUtils.Helpers import ROOT6Setup
ROOT6Setup()
# Setup online THistSvc unless specifically configured otherwise
# setup the THistSvc early and force the creation of the THistSvc
# so that it can be used by infrastructure services to book histograms
# (to avoid problems e.g. with histograms in ROBDataProviderSvc)
if _Conf.useOnlineTHistSvc:
if hasattr(svcMgr, 'THistSvc'):
log.fatal(
"The offline histogramming THistSvc is already in place.")
raise RuntimeError(
"Cannot setup online histogramming TrigMonTHistSvc")
log.debug("Using online histogramming service (TrigMonTHistSvc)")
from TrigServices.TrigServicesConf import TrigMonTHistSvc
svcMgr += TrigMonTHistSvc("THistSvc")
else:
log.debug("Using offline histogramming service (THistSvc)")
from GaudiSvc.GaudiSvcConf import THistSvc
svcMgr += THistSvc()
# StoreGateSvc
svcMgr.StoreGateSvc.ActivateHistory = False
# ProxyProviderSvc services configuration
svcMgr += CfgMgr.ProxyProviderSvc()
# --- ByteStreamAddressProviderSvc configuration
svcMgr += CfgMgr.ByteStreamAddressProviderSvc()
svcMgr.ProxyProviderSvc.ProviderNames += ["ByteStreamAddressProviderSvc"]
theApp.CreateSvc += [svcMgr.ByteStreamAddressProviderSvc.getFullName()]
# Initialization of DetDescrCnvSvc
svcMgr += CfgMgr.DetDescrCnvSvc(
# specify primary Identifier dictionary to be used
IdDictName="IdDictParser/ATLAS_IDS.xml")
theApp.CreateSvc += [svcMgr.DetDescrCnvSvc.getFullName()]
svcMgr.EventPersistencySvc.CnvServices += ["DetDescrCnvSvc"]
# Online services for ByteStream input/output
from TrigByteStreamCnvSvc.TrigByteStreamCnvSvcConf import TrigEventSelectorByteStream
from TrigByteStreamCnvSvc.TrigByteStreamCnvSvcConfig import TrigByteStreamInputSvc, TrigByteStreamCnvSvc
svcMgr += TrigByteStreamCnvSvc(
"ByteStreamCnvSvc") # this name is hard-coded in some converters
svcMgr.EventPersistencySvc.CnvServices += ["ByteStreamCnvSvc"]
svcMgr += TrigByteStreamInputSvc("ByteStreamInputSvc")
svcMgr += TrigEventSelectorByteStream(
"EventSelector", ByteStreamInputSvc=svcMgr.ByteStreamInputSvc)
theApp.EvtSel = "EventSelector"
# Online event loop manager
from TrigServices.TrigServicesConfig import HltEventLoopMgr
loopMgr = HltEventLoopMgr("HltEventLoopMgr")
loopMgr.WhiteboardSvc = "EventDataSvc"
loopMgr.SchedulerSvc = AlgScheduler.getScheduler().getName()
loopMgr.EvtSel = svcMgr.EventSelector
loopMgr.OutputCnvSvc = svcMgr.ByteStreamCnvSvc
svcMgr += loopMgr
theApp.EventLoop = loopMgr.name()
from TrigOutputHandling.TrigOutputHandlingConfig import HLTResultMTMakerCfg
svcMgr.HltEventLoopMgr.ResultMaker = HLTResultMTMakerCfg()
# Configuration of Interval of Validity Service
svcMgr += CfgMgr.IOVSvc()
# Configure COOL update helper tool
from TrigServices.TrigServicesConfig import TrigCOOLUpdateHelper
svcMgr.HltEventLoopMgr.CoolUpdateTool = TrigCOOLUpdateHelper()
# Configure the online ROB data provider service
from TrigServices.TrigServicesConfig import HltROBDataProviderSvc
svcMgr += HltROBDataProviderSvc()
# Explicitly set a few OutputLevels (needed because some services are created in
# different order when running with the PSC)
svcMgr.IncidentSvc.OutputLevel = theApp.OutputLevel
svcMgr.ProxyProviderSvc.OutputLevel = theApp.OutputLevel
svcMgr.StoreGateSvc.OutputLevel = theApp.OutputLevel
return