def loadJobOptionsCatalogue( cfg_fname ):
"""Load properties from a pickle file, previously dumped by
storeConfiguration, back into the JobOptionsSvc.
"""
# read jobopt catalogue dump and pycomps back in
cfg = open( cfg_fname, 'rb' )
jocat = pickle.load( cfg )
jocfg = pickle.load( cfg )
pycomps = pickle.load( cfg )
cfg.close()
kw = jocfg[ 'ApplicationMgr' ]
from AppMgr import theApp
theApp.JobOptionsSvcType = kw[ 'JobOptionsSvcType' ]
theApp.MessageSvcType = kw[ 'MessageSvcType' ]
handle = theApp.getHandle( kw )
del jocfg[ 'ApplicationMgr' ]
# no longer want to call setup(), since there are no Configurables to
# setup; it would be a no-op, already, but __build_master_sequence is
# broken, so make sure this can't be run ...
def noop( self ):
pass
theApp.__class__.setup = noop
import AthenaPython.PyAthena as PyAthena
josvc = PyAthena.py_svc( 'JobOptionsSvc', createIf = False, iface = 'IJobOptionsSvc' )
# restore job catalogue entries
import GaudiPython.Bindings as gaudi
for client in jocat:
for n,v in jocat[ client ].iteritems():
p = gaudi.StringProperty( n, v )
if not josvc.addPropertyToCatalogue( client, p ).isSuccess():
raise RuntimeError( 'could not add property [%s.%s = %s]' % (client, n, v) )
# restore special services properties
for client in jocfg:
svc = PyAthena.py_svc( client, createIf = False, iface='IProperty' )
for n,v in jocfg[ client ].iteritems():
p = gaudi.StringProperty( n, v )
svc.setProperty( p )
# pycomps hack-around
if pycomps:
import AthenaPython.Configurables as _C
_C.PyComponents.instances = dict( (p.name, p) for p in pycomps )
for p in pycomps:
if hasattr( p, 'setup' ):
if callable( p.setup ):
p.setup()
def initialize(self):
self.msg.info("initializing [%s]", self.name())
self.storeGateSvc = PyAthena.py_svc('StoreGateSvc')
if self.storeGateSvc is None:
self.msg.error("Problem retrieving storegate !!")
return PyAthena.StatusCode.Failure
self.tool = PyAthena.py_tool('EGammaAmbiguityTool', iface='IEGammaAmbiguityTool')
if not self.tool:
self.msg.error("Problem retrieving EgammaAmbiguityTool !!")
return PyAthena.StatusCode.Failure
return StatusCode.Success
def initialize(self):
self.msg.info( "Initializing %s...", self.name() )
self.sg = PyAthena.py_svc("StoreGateSvc")
if not self.sg:
self.msg.error ("could not retrieve event store")
return StatusCode.Failure
return StatusCode.Success
def initialize(self):
_info = self.msg.info
_info("==> initialize...")
self.hsvc = PyAthena.py_svc('THistSvc/THistSvc')
if not self.hsvc:
self.msg.error('Could not retrieve THistSvc')
return StatusCode.Failure
def print_properties(h):
_info(' -%-20s: %i <mean>=%8.3f rms=%8.3f',
h.GetName(), h.GetEntries(), h.GetMean(), h.GetRMS())
# load histos from stream and print some of their properties
o = self.hsvc.load('/read1/xxx/gauss1d', oid_type='hist')
print_properties(o)
o = self.hsvc.load('/read2/gauss2d', oid_type='hist')
print_properties(o)
o = self.hsvc.load('/read2/gauss3d', oid_type='hist')
print_properties(o)
o = self.hsvc.load('/read2/profile', oid_type='hist')
print_properties(o)
## FIXME: 'THistSvc::getTrees' (always) segfaults
## https://savannah.cern.ch/bugs/index.php?36379
try:
o = self.hsvc.load('/read2/trees/stuff/tree1', oid_type='tree')
_info(' -%-20s: %i', o.GetName(), o.GetEntries())
except KeyError,err:
self.msg.error(err)
self.msg.error('bug #36379 still not fixed...')
def initialize(self):
self.msg.info( "Initializing %s", self.name() )
self.sg = PyAthena.py_svc ("StoreGateSvc")
if not self.sg:
self.msg.error ("could not retrieve event store")
return StatusCode.Failure
#PyROOTFixes.fix_dv_container( "SgTests::PayLoadDv" )
return StatusCode.Success
def initialize(self):
log.info("Initialize AOD2A4")
#self.stvf_algo = make_METRefAlg(_suffix='_STVF')
self.sg = PyAthena.py_svc("StoreGateSvc")
self.sum_mc_event_weights = 0.0
self.number_events = 0
self.init()
return PyAthena.StatusCode.Success
def initialize(self):
self.sg = PyAthena.py_svc("StoreGateSvc")
self.msg.info( "==> initializing [%s]...", self.name() )
self.msg.info( "eta: %r",self.eta )
self.msg.info( "pt: %r",self.pt )
self.msg.info( "px: %r",self.px )
self.mytool.counter += 1
self.msg.info( "tool:%r %r",self.mytool.counter, self.mytool.name() )
return StatusCode.Success
def initialize(self):
_info = self.msg.info
_info('==> initialize...')
# the ugly intertwined part: get some properties from
# the AresEventSelector
from AthenaCommon.AppMgr import ServiceMgr as svcMgr
_evtSel = svcMgr.EventSelector
self.inputFiles = _evtSel.InputCollections[:]
self.persTreeName = _evtSel.TupleName
## self.dhTreeName = _evtSel.DhTreeName
## self.dhBranchName = _evtSel.DhBranchName
## self.branchNames = _evtSel.BranchNames or dict()
## del _evtSel
_info('reading files: %r', self.inputFiles)
# try to load the ttree speed-ups
try:
from RootUtils.PyROOTFixes import enable_tree_speedups
enable_tree_speedups()
except ImportError:
_info('could NOT install the ttree speedups...')
pass
import ROOT, PyCintex;
# load an ARA chain (and fool checkreq)
Ara = __import__ ('AthenaROOTAccess').transientTree
TChainROOTAccess = ROOT.AthenaROOTAccess.TChainROOTAccess
self.__ara_chain = TChainROOTAccess(self.persTreeName)
for f in self.inputFiles:
self.msg.debug('adding [%r]', f)
self.__ara_chain.Add (f)
_info('loading ara...')
self.ara_tree = Ara.makeTree(self.__ara_chain,
persTreeName=self.persTreeName,
dhTreeName=self.dhTreeName,
dhBranchName=self.dhBranchName,
branchNames=self.branchNames)
assert isinstance(self.ara_tree, ROOT.TTree),\
"problem while creating transient tree w/ ara"
self.nTotEntries = self.ara_tree.GetEntriesFast()
_info('#entries: %i', self.nTotEntries)
self.hsvc = PyAthena.py_svc('THistSvc')
if not self.hsvc:
self.msg.error('could not retrieve THistSvc')
return StatusCode.Failure
ares_tree_name = self.__ares_tree_name%self.persTreeName
_info('registering transient tree with ITHistSvc: [%s]',
ares_tree_name)
self.hsvc[ares_tree_name]=self.ara_tree
return StatusCode.Success
def initialize(self):
self.msg.info ('retrieving the coredump svc...')
self._svc = PyAthena.py_svc('CoreDumpSvc', iface='IService')
if self._svc is None:
self.msg.error ('could not retrieve the coredump svc')
return StatusCode.Failure
import signal
import os
self.msg.info('sending SIGBUS to pid [%s]...', os.getpid())
os.kill(os.getpid(), signal.SIGBUS)
return StatusCode.Success
def initialize(self):
# register with the incident svc
svc = PyAthena.py_svc('IncidentSvc', iface='IIncidentSvc')
if not svc:
self.msg.error('unable to get the incident svc')
return StatusCode.Failure
for incident in ('EndEvent',
'BeginInputFile',):
svc.addListener(self, incident)
pass
return StatusCode.Success
def initialize(self):
self.thSvc = PyAthena.py_svc('THistSvc', iface='ITHistSvc')
if not self.thSvc:
self.msg.error("Could not retrieve THistSvc !")
return StatusCode.Failure
self.tree = self.thSvc.get('/temp/TTreeStream/egamma',
klass='TTree')
if not self.tree:
self.msg.error('could not retrieve tree from THistSvc')
return StatusCode.Failure
return StatusCode.Success
def initialize(self):
# Check the validitly of self.cutDict
if len(self.cutDict["pTCuts"])!=self.nObj:
self.msg.fatal("Lengths of pTCuts and types are different!!")
return StatusCode.Failure
if len(self.cutDict["qualityCuts"])!=self.nObj:
self.msg.fatal("Lengths of qualityCuts and types are different!!")
return StatusCode.Failure
if len(self.cutDict["collections"])!=self.nObj:
self.msg.fatal("Lengths of collections and types are different!!")
return StatusCode.Failure
## Get the StoreGate service
self.storeGateSvc = PyAthena.py_svc('StoreGateSvc')
if self.storeGateSvc is None:
self.msg.fatal("Problem retrieving StoreGateSvc pointer !!")
return StatusCode.Failure
# AthElectronLikelihoodTool_VeryLoose
self.electronLikelihoodTool_VeryLoose = PyAthena.py_tool('AthElectronLikelihoodTool/AthElectronLikelihoodTool_VeryLoose')
if self.electronLikelihoodTool_VeryLoose is None:
self.msg.error("Problem retrieving AthElectronLikelihoodTool/AthElectronLikelihoodTool_VeryLoose pointer !!")
return StatusCode.Recoverable
self.msg.info("AthElectronLikelihoodTool/AthElectronLikelihoodTool_VeryLoose retrieved.")
# AthElectronLikelihoodTool_Loose
self.electronLikelihoodTool_Loose = PyAthena.py_tool('AthElectronLikelihoodTool/AthElectronLikelihoodTool_Loose')
if self.electronLikelihoodTool_Loose is None:
self.msg.error("Problem retrieving AthElectronLikelihoodTool/AthElectronLikelihoodTool_Loose pointer !!")
return StatusCode.Recoverable
self.msg.info("AthElectronLikelihoodTool/AthElectronLikelihoodTool_Loose retrieved.")
# AthElectronLikelihoodTool_Medium
self.electronLikelihoodTool_Medium = PyAthena.py_tool('AthElectronLikelihoodTool/AthElectronLikelihoodTool_Medium')
if self.electronLikelihoodTool_Medium is None:
self.msg.error("Problem retrieving AthElectronLikelihoodTool/AthElectronLikelihoodTool_Medium pointer !!")
return StatusCode.Recoverable
self.msg.info("AthElectronLikelihoodTool/AthElectronLikelihoodTool_Medium retrieved.")
return StatusCode.Success
def initialize(self):
self.msg.info("initializing...")
self.sg = PyAthena.py_svc('StoreGateSvc')
if not self.sg:
self.msg.error("Could not retrieve StoreGateSvc !")
return StatusCode.Failure
_info = self.msg.info
_info("Configuration:")
_info(" - ints: [%s]", self.intsName)
_info(" - uints: [%s]", self.uintsName)
_info(" - floats: [%s]", self.floatsName)
_info(" - doubles: [%s]", self.doublesName)
self._test_matrix = {
'std::vector<int>' : self.intsName,
'std::vector<unsigned int>' : self.uintsName,
'std::vector<float>' : self.floatsName,
'std::vector<double>' : self.doublesName,
}
return StatusCode.Success
def initialize(self):
_info = self.msg.info
_info('==> initialize...')
self.hsvc = PyAthena.py_svc('THistSvc/THistSvc')
if not self.hsvc:
self.msg.error('could not retrieve THistSvc/THistSvc')
return StatusCode.Failure
from ROOT import TH1F, TH2F, TH3F, TProfile, TTree
# helpers
def th1(n,t): return TH1F(n,t,100,0.,100.)
def th2(n,t): return TH2F(n,t,100,-50.,50.,100,-50.,50.)
def th3(n,t): return TH3F(n,t,100,-50.,50.,100,-50.,50.,100,-50.,50.)
def tp (n,t): return TProfile(n,t,100,-50.,50.)
# temporary trees
self.hsvc['/temp/h1'] = th1('h1', 'Temporary hist 1')
self.hsvc['/temp/other/h1a'] = th1('h1a', 'Temporary hist 1a')
# write to stream 'new'
self.hsvc['/new/hists/h1'] = th1('h1', 'Persistent hist 1')
# update to stream 'upd', dir '/xxx'
self.hsvc['/upd/xxx/gauss1d'] = TH1F('gauss1d', '1D gaussian',
100,-50.,50.)
# recreate stuff in '/'
self.hsvc['/rec/gauss2d'] = th2('gauss2d','2D gaussian')
self.hsvc['/rec/gauss3d'] = th3('gauss3d','3D gaussian')
self.hsvc['/rec/prof'] = tp ('profile','profile')
# tree with branches in '/trees/stuff'
self.hsvc['/rec/trees/stuff/tree1'] = TTree('tree1','tree title')
self._n = -1
return StatusCode.Success
# initialize Athena, then go to first event
#--------------------------------------------------------------
theApp.initialize()
theApp.nextEvent()
# IdHelpers
onlineID = PyKernel.retrieveDet(g.LArOnlineID, "LArOnlineID")
caloCellID = PyKernel.retrieveDet(g.CaloCell_ID, "CaloCell_ID")
emID = PyKernel.retrieveDet(g.LArEM_ID, "LArEM_ID")
hecID = PyKernel.retrieveDet(g.LArHEC_ID, "LArHEC_ID")
fcalID = PyKernel.retrieveDet(g.LArFCAL_ID, "LArFCAL_ID")
# CablingService
import AthenaPython.PyAthena as PyAthena
larCablingSvc = PyAthena.py_tool("LArCablingService")
#--------------------------------------------------------------
# access by identifier
#--------------------------------------------------------------
# Identifier = PyLCGDict.makeClass('Identifier')
be = 0 # barrel
pn = 1 # positive negative
ft = 0 # feedthrough
slot = 10 # slot
chan = 0 # channel
print "*****************************"
print " "
def configure(self, joboptions=None, commands=None,
dllname=None,
factname=None,
extra_options=None):
if not (self.app is None):
self.msg.info('C++ application already configured')
return self.app
self.msg.info('configuring application...')
usr_cfg = AthCfg()
self.cfg.seek(0)
usr_cfg << self.cfg.read()
# reset
self.cfg = AthCfg()
if commands:
self.cfg << commands+'\n'
# common configuration
self.cfg << """
# basic job configuration
include('AthenaCommon/Atlas.UnixStandardJob.py')
include.block('AthenaCommon/Atlas.UnixStandardJob.py')
if not (not %(run_batch)s and
theApp.EventLoop == 'PyAthenaEventLoopMgr'):
# make SIG_INT fatal
svcMgr.CoreDumpSvc.FatalHandler = -1
""" % {'run_batch' : self.options.run_batch}
self.cfg << """
# user level configuration
try:
include('$HOME/.athenarc')
except IncludeError:
pass
"""
# another user level configuration
usr_cfg.seek(0)
self.cfg << usr_cfg.read()
if isinstance(joboptions, (list,tuple)):
for jobo_name in joboptions:
self.cfg.include(jobo_name)
if not self.options.run_batch:
self.cfg << """
theApp.EventLoop = 'PyAthenaEventLoopMgr'
svcMgr += CfgMgr.PyAthenaEventLoopMgr()
"""
self.cfg << """
### logging and messages ---------
from AthenaCommon.Logging import *
_msg = log
_msg.setLevel(getattr(logging, '%(output_level)s'))
import AthenaCommon.Constants as Lvl
theApp.setOutputLevel(%(output_level)s)
theApp.OutputLevel = Lvl.%(output_level)s
from AthenaCommon.AppMgr import ServiceMgr as svcMgr
svcMgr.MessageSvc.OutputLevel = Lvl.%(output_level)s
""" % dict(output_level=self.options.msg_lvl)
self.cfg << """
from AthenaCommon.Include import Include, IncludeError, include
include.setShowIncludes(%(showincludes)s)
if %(showincludes)s:
import AthenaCommon.Include as AthCIncMod
AthCIncMod.marker=' -#-' # distinguish bootstrap from other jo-code
""" % dict(showincludes=self.options.showincludes)
cfg_name = self.cfg._jobo.name.replace('.py','.pkl')
self.msg.info('dumping job-configuration into [%s]...',
cfg_name)
# run configuration in a forked-subprocess...
sc = _app_configure(self.cfg, cfg_name, extra_options)
if sc:
err = 'could not configure application [sc=%d]' % sc
self.msg.error(err)
raise RuntimeError(err)
self.msg.info('configuring application w/ [%s]', cfg_name)
import os
self.cfg._jobo.close()
os.remove(self.cfg._jobo.name)
import PyCintex
PyCintex.Cintex.Enable()
gbl = PyCintex.makeNamespace('')
import GaudiPython.Bindings as gaudi
# remove the gaudimodule exit handler as to prevent them from clobering
import atexit
for hdlr in reversed(atexit._exithandlers[:]):
module_name = hdlr[0].__module__
if ('GaudiPython' in module_name or
'gaudimodule' in module_name):
atexit._exithandlers.remove(hdlr)
del hdlr
# install our own exit handler (if needed)
import sys
if hasattr(sys, 'ps1'): # ie: is interactive
atexit.register(self.exit)
del atexit
from . import ResourceLimits
ResourceLimits.SetMaxLimits()
try: import cPickle as pickle
except ImportError: import pickle
import PyUtils.dbsqlite as dbs
db = dbs.open(cfg_name, 'r')
jobo_cfg = db['jobopts']
kw = jobo_cfg['ApplicationMgr']
for k in ('Go', 'Exit', 'AuditInitialize', 'AuditFinalize'):
if k in kw:
del kw[k]
outputlevel = jobo_cfg['ApplicationMgr']['OutputLevel']
self.app = gaudi.AppMgr(outputlevel=outputlevel,
selfoptions=kw,
dllname=dllname,
factname=factname)
# open the pycomps folder
pycomps = db.get('pycomps', None) # just opening it should do
if pycomps:
import AthenaPython.Configurables as _C
_C.PyComponents.instances = dict((p.name, p) for p in pycomps)
#_C.PyComponents.instances = pycomps
for p in pycomps:
if hasattr(p, 'setup'):
if callable(p.setup):
p.setup()
setattr(self, '_pycomps', pycomps)
import AthenaPython.PyAthena as PyAthena
josvc = PyAthena.py_svc('JobOptionsSvc',
createIf=False,
iface='IJobOptionsSvc')
assert josvc is not None
for client in jobo_cfg:
if client == 'ApplicationMgr':
continue
for n,v in jobo_cfg[client].iteritems():
p = gaudi.StringProperty(n, v)
if not josvc.addPropertyToCatalogue(client, p).isSuccess():
self.msg.error(
'could not add property [%s.%s = %s]',
client, n, v
)
if client in ('MessageSvc', 'JobOptionsSvc'):
svc = PyAthena.py_svc(client, iface='IProperty')
svc.setProperty(p)
db.close()
import os
if os.path.exists(cfg_name):
os.remove(cfg_name)
pass
#import AthenaCommon.Debugging as dbg
#dbg.hookDebugger()
return self.app
def tagStore(self):
import AthenaPython.PyAthena as PyAthena
return PyAthena.py_svc('StoreGateSvc/TagMetaDataStore')
def initialize(self):
self.msg.info("SkimDecisionsContainer name: '%s'" %
self.SkimDecisionsContainerName)
self.sg = PyAthena.py_svc("StoreGateSvc")
return BookkeepingWriterBase.initialize(self)
def execute(self):
self.nProcessed += 1
self.msg.debug('==> execute %s on %r. event...' %
(self.name(), self.nProcessed))
nElPassEta = 0
nElPassEt = 0
nElPassIsEM = 0
nElPassAuthor = 0
# Create a class-member list of all electrons that passed all cuts.
# This can then be used by other filters to do overlap removal, etc.
VIEW_ELEMENTS = 1
goodElectrons = PyAthena.ElectronContainer(VIEW_ELEMENTS)
#goodElectrons = []
## If passAll is selected, accept all events
if self.passAll:
self.msg.debug('%s event passed because passAll is true' %
self.name())
self.setFilterPassed(True)
return StatusCode.Success
#Then... here we go!
# Get the electron collection from StoreGate
try:
electronCollection = self.storeGateSvc.retrieve(
self.electronCollectionType, self.electronCollectionName)
pass
except LookupError:
self.msg.warning('Collection %s not found' %
self.electronCollectionName)
self.setFilterPassed(True)
return StatusCode.Success
# Loop over all electrons and make the selections cuts
for electron in electronCollection:
self.nElectrons += 1
# Use Et of CaloCluster.
cluster = None
cluster = electron.cluster()
if cluster == None:
self.msg.warning(
'Could not get the cluster for this electron!')
pass
else:
if abs(cluster.etaBE(2)) < self.cutEtaMax:
# Check if the EM calorimeter barrel-encap crack should be removed.
if not self.removeEtaCrack or ( self.removeEtaCrack and \
( abs(cluster.etaBE(2)) < self.crackEtaMin
or abs(cluster.etaBE(2)) > self.crackEtaMax ) ):
nElPassEta += 1
# If the Et of this electron is larger than the minimum required Et
# of the high-Et cut, then the high-Et electron is found.
if cluster.et() > self.cutEtMin:
nElPassEt += 1
# Check that the electron in question comes from the right reconstruction algorithm using PrimaryDPDHelpers
if PrimaryDPDHelpers.checkEgammaAuthor(
electron, self.cutAuthor):
nElPassAuthor += 1
# Check the quality of the electron using PrimaryDPDHelpers
if PrimaryDPDHelpers.checkElectronIsEM(
electron, self.cutIsEM):
nElPassIsEM += 1
# goodElectrons.append( electron )
goodElectrons.push_back(electron)
pass
pass
pass
pass
pass
pass
pass
## Translate the electron pass counters into event pass counters
if nElPassEta > 0: self.nEventElPassEta += 1
if nElPassEt > 0: self.nEventElPassEt += 1
if nElPassIsEM > 0: self.nEventElPassIsEM += 1
if nElPassAuthor > 0: self.nEventElPassAuthor += 1
## Record the good electrons into StoreGate so that they can be retrieved by other algorithms
#VIEW_ELEMENTS = 1
#goodElectronColl = PyAthena.ElectronContainer(VIEW_ELEMENTS)
#goodElectronColl.m_ownPolicy = 1 # This sets it as a VIEW_CONTAINER
#for el in goodElectrons:
# goodElectronColl.push_back(el)
# pass
if self.storeGateSvc.record(
goodElectrons,
self.goodElectronCollectionName) != StatusCode.Success:
self.msg.error(
'Could not record the goodElectrons into StoreGate with the key = '
% self.goodElectronCollectionName)
pass
## Check if the event is accepted
if goodElectrons.__len__() >= self.minNumberPassed:
self.nEventMinNumPassed += 1
self.msg.debug('%s event passed.' % self.name())
self.setFilterPassed(True)
pass
else:
self.msg.debug('%s event failed.' % self.name())
self.setFilterPassed(False)
pass
return StatusCode.Success
def mon_push_back(sgname='StoreGateSvc'):
"""
Helper method to crawl through collections in a StoreGateSvc instance and
assess how much memory they are wasting b/c they 'forgot' to reserve
enough space before doing 'push_back'.
This task is performed by just comparing what Klass::size() and
Klass::capacity() are returning as a value.
@param `sgname` the fully qualified Gaudi name of the StoreGateSvc instance
@return a dictionary of { <sgkey> : (size, capacity, clid) }
"""
wasted = {}
try:
from AthenaPython import PyAthena
except ImportError:
# arf... release 13...
return wasted
sg = PyAthena.py_svc(sgname)
cl = PyAthena.py_svc('ClassIDSvc')
# retrieve all containers from StoreGate
proxies = sg.proxies()
for dp in proxies:
k = dp.name()
# no double counting from symlinks
# FIXME: it is actually valid to have 2 different collections
# (=/= CLIDs) with the same key...
if wasted.has_key(k):
continue
clid = dp.clID()
klass = "%s" % cl.typename(clid)
# 'handle' classes for which we were not able to fetch
# a C++ typename from the SG::DataProxy classID (e.g. HistoryObjects)
if klass == "None":
continue
try:
data = sg.retrieve(klass, k)
except Exception:
# don't bother user: probably no Reflex dictionary...
continue
if not data:
continue
# reject objects we can't inspect or not interested in
if not hasattr(data, 'size') or \
not hasattr(data, 'capacity'):
continue
sz = data.size()
cp = data.capacity()
rt = 0.
if cp != 0.:
rt = sz / float(cp) * 100.
# simple minded filter
if rt == 0. or rt >= 90.:
continue
wasted[k] = (sz, cp, clid)
pass
return wasted
fileName=arg
if options.fileName == None and len(fileName) == 0:
str(parser.print_help() or "")
sys.exit(1)
ofileName=""
if options.outFileName == None:
ofileName=fileName + "_optBS"
else:
ofileName=options.outFileName
import AthenaPython.PyAthena as PyAthena
print "Optimizing file: ", fileName
print "Output file: ", ofileName
trans = PyAthena.cobs(fileName, ofileName)
if options.skip != None:
trees=options.skip.split(",")
for t in trees:
print "Skipping tree: ", t
trans.setTreeToSkip(t)
if options.compressionF != None:
trans.setCompressionLevel(options.compressionF)
print 'Compression factor:', options.compressionF
else:
print 'Compression factor: 7'
if options.Memory != None:
trans.setDedicatedMemory(options.Memory)
theApp.run() # runs until theApp.EvtMax events reached
from AthenaCommon.Debugging import hookDebugger, DbgStage
if DbgStage.value == "fini":
hookDebugger()
except:
# print a stack trace to know something bad happened !
import traceback, sys
traceback.print_exc(file=sys.stdout)
# update the exit-code
import AthenaCommon.ExitCodes as ath_codes
if theApp._exitstate == ath_codes.ALL_OK:
theApp._exitstate = ath_codes.UNKNOWN_EXCEPTION
# re-raise, in case somebody wants to do something about it
raise
finally:
import AthenaCommon.ExitCodes as ath_codes
if theApp._exitstate != ath_codes.ALL_OK:
# trigger the AthenaSummarySvc
try:
from AthenaCommon.AppMgr import ServiceMgr as svcMgr
if hasattr(svcMgr, 'AthenaSummarySvc'):
from AthenaPython import PyAthena
aths = PyAthena.py_svc('AthenaSummarySvc',
iface='IAthenaSummarySvc')
aths.setStatus(theApp._exitstate)
aths.createSummary()
except ImportError:
print "import of PyAthena failed: unable to trigger AthenaSummarySvc"
theApp.exit() # exits program, yields theApp._exitstate on shell,
# ok if re-thrown C++ exception: try to exit clean
theApp.HistogramPersistency = "ROOT"
from GaudiSvc.GaudiSvcConf import THistSvc
svcMgr += THistSvc()
svcMgr.THistSvc.Output += [
"AANT DATAFILE='" + CafJobOutputs[0] + "' OPT='RECREATE'"
]
from AthenaCommon.AppMgr import ServiceMgr as svcMgr
theAuditorSvc = svcMgr.AuditorSvc
theAuditorSvc.Auditors += ["ChronoAuditor"]
#svcMgr.ChronoStatSvc.ChronoDestinationCout = True
svcMgr.ChronoStatSvc.PrintUserTime = True
svcMgr.ChronoStatSvc.PrintSystemTime = True
svcMgr.ChronoStatSvc.PrintEllapsedTime = True
#svcMgr.ChronoStatSvc.NumberOfSkippedEventsForMemStat = 1
#svcMgr.ChronoStatSvc.AsciiStatsOutputFile = "chronoStats.ascii"
svcMgr.AthenaPoolCnvSvc.UseDetailChronoStat = True
theApp.AuditAlgorithms = True
#Hack to load the right dict to write vector< vector < X > > to TTRee
import AthenaPython.PyAthena as PyAthena
PyAthena.load_library('AtlasSTLAddReflexDict')
# allow the use of the pythonized properties interface
def __getattr__( self, attr ):
try: from GaudiPython.Bindings import iProperty
except ImportError: from gaudimodule import iProperty
return getattr( iProperty("StoreGateSvc", self), attr )
def __setattr__( self, attr, value ):
try: from GaudiPython.Bindings import iProperty
except ImportError: from gaudimodule import iProperty
return setattr( iProperty("StoreGateSvc", self), attr, value )
StoreGateSvc.__getattr__ = __getattr__
StoreGateSvc.__setattr__ = __setattr__
import AthenaPython.PyAthena as PyAthena
StoreGateSvc._pyclidsvc = PyAthena.py_svc('ClassIDSvc')
def keys(self, clid=None, allKeys=False):
"""return the list of keys for a given CLID (int, type or class_name)
if clid is None, it will return the list of all keys.
When 'allKeys' is True it will also return the key aliases.
"""
if isinstance(clid, str):
clid = self._pyclidsvc.clid(clid)
if isinstance(clid, type):
clid = self._pyclidsvc.clid(clid.__name__)
if clid is None:
return [p.name() for p in self.proxies()]
return list(self._cpp_keys(clid, allKeys))
StoreGateSvc._cpp_keys = StoreGateSvc.keys
StoreGateSvc.keys = keys
def initialize(self):
print "==> initializing ", self.name
print "MinDigitADC: ", self.MinDigitADC
print "MaxDeltaT: ", self.MaxDeltaT
print "NtupleFileName: ", self.NtupleFileName
print "TriggerLines: ", self.TriggerLines
#
self.sg = PyAthena.py_svc("StoreGateSvc")
self.det = PyAthena.StoreGate.pointer("DetectorStore")
self.LArOID = self.det.retrieve("LArOnlineID", "LArOnlineID")
self.lcs = PyAthena.py_tool('LArCablingService')
self.cdd = PyAthena.CaloDetDescrManager.instance()
self.cid = self.cdd.getCaloCell_ID()
self.tdt = PyAthena.py_tool('Trig::TrigDecisionTool/TrigDecisionTool')
self.ntfile = TFile(self.NtupleFileName, "RECREATE")
self.hectree = TTree("HECNoise", "HECNoise")
self.iRun = array('i', [0])
self.iEvent = array('L', [0])
self.iEventCount = array('i', [0])
self.fPrescale = {}
self.iTrigger = {}
for tl in self.TriggerLines:
self.fPrescale[tl] = array('f', [0.0])
self.iTrigger[tl] = array('i', [0])
pass
self.iTime = array('i', [0])
self.iLB = array('i', [0])
self.iBCID = array('i', [0])
self.avgMu = array('f', [0.0])
self.actMu = array('f', [0.0])
self.iGain = array('i', [0])
self.iOID = array('L', [0])
self.iSide = array('i', [0])
self.iSamp = array('i', [0])
self.iReg = array('i', [0])
self.iEta = array('i', [0])
self.iPhi = array('i', [0])
self.iQuality = array('i', [0])
self.e = array('f', [0.0])
self.t = array('f', [0.0])
self.eta = array('f', [0.0])
self.phi = array('f', [0.0])
self.z = array('f', [0.0])
self.r = array('f', [0.0])
self.Ped = array('f', [0.0])
self.PedRMS = array('f', [0.0])
self.iDigi = array('i', 32 * [0])
self.iMax = array('i', [0])
self.iMin = array('i', [0])
#
self.hectree.Branch("iRun", self.iRun, "iRun/I")
self.hectree.Branch("iEvent", self.iEvent, "iEvent/I")
self.hectree.Branch("iEventCount", self.iEventCount, "iEventCount/I")
for tl in self.TriggerLines:
self.hectree.Branch(tl + "_Prescale", self.fPrescale[tl],
tl + "_Prescale/F")
self.hectree.Branch(tl + "_Trigger", self.iTrigger[tl],
tl + "_Trigger/I")
pass
self.hectree.Branch("iTime", self.iTime, "iTime/I")
self.hectree.Branch("iLB", self.iLB, "iLB/I")
self.hectree.Branch("iBCID", self.iBCID, "iBCID/I")
self.hectree.Branch("avgMu", self.avgMu, "avgMu/F")
self.hectree.Branch("actMu", self.actMu, "actMu/F")
self.hectree.Branch("iGain", self.iGain, "iGain/I")
self.hectree.Branch("iOID", self.iOID, "iOID/l")
self.hectree.Branch("iSide", self.iSide, "iSide/I")
self.hectree.Branch("iSamp", self.iSamp, "iSamp/I")
self.hectree.Branch("iReg", self.iReg, "iReg/I")
self.hectree.Branch("iEta", self.iEta, "iEta/I")
self.hectree.Branch("iPhi", self.iPhi, "iPhi/I")
self.hectree.Branch("iQuality", self.iQuality, "iQuality/I")
self.hectree.Branch("e", self.e, "e/F")
self.hectree.Branch("t", self.t, "t/F")
self.hectree.Branch("eta", self.eta, "eta/F")
self.hectree.Branch("phi", self.phi, "phi/F")
self.hectree.Branch("z", self.z, "z/F")
self.hectree.Branch("r", self.r, "r/F")
self.hectree.Branch("Ped", self.Ped, "Ped/F")
self.hectree.Branch("PedRMS", self.PedRMS, "PedRMS/F")
self.hectree.Branch("iDigi", self.iDigi, "iDigi[32]/I")
self.hectree.Branch("iMax", self.iMax, "iMax/I")
self.hectree.Branch("iMin", self.iMin, "iMin/I")
#
return True
)
svcMgr.IOVDbSvc.Folders.append(
'<db>COOLONL_TRIGGER/CONDBR2</db> /TRIGGER/L1Calo/V1/Calibration/PpmDeadChannels <tag>HEAD</tag>'
)
svcMgr.IOVDbSvc.Folders.append(
'<db>COOLONL_TRIGGER/CONDBR2</db> /TRIGGER/L1Calo/V2/Configuration/PprChanDefaults <tag>HEAD</tag>'
)
svcMgr.IOVDbSvc.Folders.append(
'<db>COOLONL_TRIGGER/CONDBR2</db> /TRIGGER/L1Calo/V1/Configuration/PprChanDefaults <tag>HEAD</tag>'
)
svcMgr.IOVDbSvc.forceTimestamp = 1446047363
svcMgr.IOVDbSvc.forceRunNumber = 283851
from AthenaPython import PyAthena
theApp.initialize()
s = PyAthena.py_svc('L1CaloCondSvc', True, 'L1CaloCondSvc')
theApp.nextEvent()
# test L1CaloPprChanCalib (default, i.e. V2)
c = cppyy.gbl.L1CaloPprChanCalibContainer()
s.retrieve(c)
chan = c.pprChanCalib(0x100000)
assert chan.channelId() == 0x100000
assert chan.pedValue() == 33
assert chan.firCoeff1() == 2
assert chan.lutCpOffset() == 32400
assert chan.lutCpSlope() == 1138
# test L1CaloPprChanCalib (V1)
c = cppyy.gbl.L1CaloPprChanCalibV1Container()
#-----------------------------------------#
# Script for extra MissingEt calculations #
#-----------------------------------------#
import AthenaPython.PyAthena as PyAthena
PyAthena.load_library('egammaEnumsDict')
import MissingET.METRefGetter_plup
#from MissingET.METRefGetter_plup import *
# MissingEt with tight++ electrons and EMJES AntiKt4 jets
#TightPPMissingEtAlg = MissingET.METRefGetter_plup.make_METRefAlg(_suffix = "_em_tightpp")
#TightPPMissingEtAlg.ele_EleInputCollectionKey = "ElectronAODCollection"
#TightPPMissingEtAlg.ele_noCut = False
#TightPPMissingEtAlg.ele_isEM = PyAthena.egammaPID.ElectronIDTightPP
#TightPPMissingEtAlg.ele_calibType = "RefCalib"
#TightPPMissingEtAlg.photon_doPhotonTool = False
#TightPPMissingEtAlg.gamma_isEM = PyAthena.egammaPID.PhotonIDTight
#TightPPMissingEtAlg.gamma_calibType = "EmScale"
#TightPPMissingEtAlg.tau_doTauTool = False
#TightPPMissingEtAlg.tau_calibType = "EmScale"
#TightPPMissingEtAlg.jet_JetInputCollectionKey = "AntiKt4TopoEMNewJets"
#TightPPMissingEtAlg.jet_JetPtCut = 20.0*GeV
#TightPPMissingEtAlg.jet_JetMaxPtCut = 1000000.0*GeV
#TightPPMissingEtAlg.jet_ApplyJetScale = "Yes"
#TightPPMissingEtAlg.jet_UseJetMomentForScale = True
#TightPPMissingEtAlg.jet_JetMomentForScale = "JES"
#TightPPMissingEtAlg.jet_calibType = "EmScale"
#TightPPMissingEtAlg.jet_RunSoftJetsTool = True
def initialize(self):
self.msg.info('************************************')
self.msg.info('==> initialize %s...', self.name())
self.msg.info('************************************')
self.cutFlowSvc().setFilterDescription(self.cutID, self.cmdstring)
# look if parentheses are matched
if self.cmdstring.count("(") != self.cmdstring.count(")"):
self.msg.fatal("Mismatched parentheses in filter string: %s" %
self.cmdstring)
return False
# these parentheses are not logically correct
if self.cmdstring.count("{") != 0 or \
self.cmdstring.count("}") != 0 or \
self.cmdstring.count("[") != 0 or \
self.cmdstring.count("]") != 0:
self.msg.fatal("Wrong type of parentheses in filter string: %s" %
self.cmdstring)
return False
try:
tokenobj = tokenize.generate_tokens(
StringIO(self.cmdstring).readline)
self.algdict = {}
result = []
for toknum, tokval, _, _, _ in tokenobj:
if toknum == tokenize.NAME and \
tokval != 'or' and \
tokval != 'not' and \
tokval != 'and' and \
tokval != 'True' and \
tokval != 'False':
if self.prefix != '':
tokval = "%s_%s" % (self.prefix, tokval)
self.msg.info("Adding algorithm: %s", tokval)
_alg = None
_alg = PyAthena.py_alg(tokval)
self.algdict.update({tokval: _alg})
if not _alg:
self.msg.error("Algorithm %s not found", tokval)
return False
else:
self.msg.debug("Found algorithm: %s -> %s" %
(tokval, _alg))
exec('self.%s = _alg' % tokval)
self.cutFlowSvc().declareChildFilter(tokval, self.cutID)
result.extend([(tokenize.STRING, 'self.%s' % tokval),
(tokenize.OP, '.'),
(tokenize.STRING, 'filterPassed'),
(tokenize.OP, '('), (tokenize.OP, ')')])
else:
result.append((toknum, tokval))
self.cmd = tokenize.untokenize(result)
self.msg.debug("String changed internally to:\n%s", self.cmd)
#execute command once to validate
response = bool(eval(self.cmd))
except Exception as e:
self.msg.fatal("Not a valid Python string. Exception: %s" % e)
import traceback
self.msg.fatal(traceback.format_exc())
return False
self.msg.info("Filter string validated")
return True
def cutFlowSvc(self):
if not hasattr(self, '_cutflowsvc'):
import AthenaPython.PyAthena as PyAthena
self._cutflowsvc = PyAthena.py_svc('CutFlowSvc',
iface='ICutFlowSvc')
return self._cutflowsvc
def initialize (self):
self.sg = PyAthena.py_svc('StoreGateSvc')
self.cs = PyAthena.py_svc('ClassIDSvc')
return 1
def detStore(self):
if not self._pyath_detstore:
import AthenaPython.PyAthena as PyAthena
self._pyath_detstore = PyAthena.py_svc('StoreGateSvc/DetectorStore')
return self._pyath_detstore
# example:
# for standard AOD file this would be a reasonable command line:
# orderFile -o ofilename ifilename
#
###########################################################################################################
__version__ = "$Revision: 226925 $"
__author__ = "Ilija Vukotic <*****@*****.**>"
import sys
import os
from optparse import OptionParser
import AthenaPython.PyAthena as PyAthena
trans = PyAthena.cobs('/data/ilija/AOD.067184.big.pool.root',
'/data/ilija/ordered.pool.root')
if __name__ == "__main__":
parser = OptionParser(usage="usage: %prog [options] my.file.pool.root")
p = parser.add_option
p("-f",
"--file",
dest="fileName",
help="The path to the POOL file to analyze")
p("-m",
"--mode",
dest="orderingMode",
default=2,
help="ordering mode 1 - by Offset, 2 - byEntry, 3 - byBranch ")
p("-o",
def initialize(self):
## note that we are using the python logging service
## and that the PyAthena.Alg base class has already initialized
## it for us
# Get DetectorStore...
from StoreGateBindings.Bindings import StoreGate
self._detStore = StoreGate.pointer("DetectorStore")
if self._detStore is None:
self.msg.error("Failed to get DetectorStore")
return StatusCode.Failure
# Get LArOnlineID helper class
self.onlineID = self._detStore.retrieve("LArOnlineID", "LArOnlineID")
if self.onlineID is None:
self.msg.error("Failed to get LArOnlineID")
return StatusCode.Failure
# Get CaloCell_ID
self.offlineID = self._detStore.retrieve("CaloCell_ID", "CaloCell_ID")
if self.offlineID is None:
self.msg.error("Failed to get CaloCell_ID")
return StatusCode.Failure
# -----------------------------------------------------------
# Initialize LArCabling service
self.larCablingSvc = PyAthena.py_tool("LArCablingLegacyService")
if self.larCablingSvc is None:
self.msg.error('Problem retrieving LArCablingService pointer !')
return StatusCode.Failure
else:
self.msg.info('retrieved [%s]', self.larCablingSvc.name())
self.class_larBCBitPacking = cppyy.makeClass("LArBadChanBitPacking")
self.bc_packing = self.class_larBCBitPacking()
self.noisepattern = 0
for n in ("lowNoiseHG", "highNoiseHG", "unstableNoiseHG", "lowNoiseMG",
"highNoiseMG", "unstableNoiseMG", "lowNoiseLG",
"highNoiseLG", "unstableNoiseLG", "sporadicBurstNoise"):
stat = self.bc_packing.enumName(n)
if stat[0]:
self.noisepattern |= 1 << stat[1]
self.deadpattern = 0
for n in ("deadReadout", "deadPhys", "almostDead"):
stat = self.bc_packing.enumName(n)
if stat[0]:
self.deadpattern |= 1 << stat[1]
self.noid = Identifier()
if self.includeLocation:
try:
self.caloDDM = CaloDetDescrManager.instance()
except:
print("Failed to retrieve CaloDDM")
return StatusCode.Failure
self.larPedestal = None
self.larMphysOverMcal = None
self.larRamp = None
self.larDAC2uA = None
self.laruA2MeV = None
self.larhvScaleCorr = None
self.larDSPThr = None
return StatusCode.Success
def _evtSeek(self):
""" retrieve a handle to the IEventSeek interface of the event loop mgr
"""
import AthenaPython.PyAthena as PyAthena
return PyAthena.py_svc(self.EventLoop, iface='IEventSeek')
def _setup():
import cppyy
# StoreGate bindings from dictionary
cppyy.loadDictionary("libAthenaPythonDict") # for clidsvc
cppyy.loadDictionary("libStoreGateBindingsDict") # for storegatesvc
cppyy.loadDictionary("libStoreGateBindings"
) # not linked from libStoreGateBindingsDict in ROOT6
# make sure the global C++ namespace has been created
gbl = cppyy.makeNamespace('') # noqa: F841
_ath = cppyy.makeNamespace('AthenaInternal')
# ROOT6 workaround, kick the loading of headers
_ath.ROOT6_AthenaPython_WorkAround_Dummy
_ath.ROOT6_StoreGateBindings_WorkAround_Dummy
# end workaround
global py_retrieve
py_retrieve = cppyy.gbl.AthenaInternal.retrieveObjectFromStore
global py_record
py_record = cppyy.gbl.AthenaInternal.recordObjectToStore
global py_sg_contains
py_sg_contains = cppyy.gbl.AthenaInternal.py_sg_contains
global py_sg_getitem
py_sg_getitem = cppyy.gbl.AthenaInternal.py_sg_getitem
# retrieve the StoreGateSvc class
global StoreGate, StoreGateSvc
StoreGateSvc = cppyy.gbl.StoreGateSvc
StoreGate = cppyy.gbl.StoreGate
# add specialized retrieve method
def retrieve(self, klass, key=None):
ret = py_retrieve(self, klass, key)
if ret and hasattr(ret, 'setStore') and not ret.hasStore():
if not hasattr(ret, 'trackIndices') or ret.trackIndices():
if py_sg_contains(self, 'SG::IConstAuxStore', key + 'Aux.'):
aux = py_retrieve(self, 'SG::IConstAuxStore', key + 'Aux.')
ret.setStore(aux)
return ret
StoreGateSvc.retrieve = retrieve
# add specialized record method
def record(self, obj, key, allowMods=True, resetOnly=True, noHist=False):
return py_record(self, obj, key, allowMods, resetOnly, noHist)
StoreGateSvc.record = record
# add specialized contains method
def contains(self, klass_or_clid, key):
print("---- StoreGateSvc.contains() ", klass_or_clid, key)
from builtins import int
if isinstance(klass_or_clid, str):
try:
clid = int(klass_or_clid)
klass = self._pyclidsvc.typename(clid)
except ValueError:
klass = str(klass_or_clid)
pass
elif isinstance(klass_or_clid, int):
klass = self._pyclidsvc.typename(klass_or_clid)
elif isinstance(klass_or_clid, type):
klass = klass_or_clid.__name__
else:
raise TypeError(
'argument 2 must be a typename, a clid or a type (got %r)' %
type(klass_or_clid))
return py_sg_contains(self, klass, key)
StoreGateSvc.contains = contains
# dict-pythonization of storegate: __setitem__
def __setitem__(self, key, obj):
return py_record(self, obj, key, True, True, False)
StoreGateSvc.__setitem__ = __setitem__
# dict-pythonization of storegate: __getitem__
def __getitem__(self, key):
try:
ret = py_sg_getitem(self, key.encode())
except LookupError as err:
raise KeyError(str(err))
if ret and hasattr(ret, 'setStore') and not ret.hasStore():
if not hasattr(ret, 'trackIndices') or ret.trackIndices():
if py_sg_contains(self, 'SG::IConstAuxStore', key + 'Aux.'):
aux = py_retrieve(self, 'SG::IConstAuxStore', key + 'Aux.')
ret.setStore(aux)
return ret
StoreGateSvc.__getitem__ = __getitem__
# dict-pythonization of storegate: __len__
def __len__(self):
return len(self.keys())
StoreGateSvc.__len__ = __len__
# make dump print, rather than return string
def dump(self, fd=None):
if fd is None:
import sys
fd = sys.stdout
print(self.__class__._dump(self), file=fd)
StoreGateSvc._dump = StoreGateSvc.dump
StoreGateSvc.dump = dump
# allow the use of the pythonized properties interface
def __getattr__(self, attr):
try:
from GaudiPython.Bindings import iProperty
except ImportError:
from gaudimodule import iProperty
return getattr(iProperty("StoreGateSvc", self), attr)
def __setattr__(self, attr, value):
try:
from GaudiPython.Bindings import iProperty
except ImportError:
from gaudimodule import iProperty
return setattr(iProperty("StoreGateSvc", self), attr, value)
StoreGateSvc.__getattr__ = __getattr__
StoreGateSvc.__setattr__ = __setattr__
import AthenaPython.PyAthena as PyAthena
StoreGateSvc._pyclidsvc = PyAthena.py_svc('ClassIDSvc')
def keys(self, clid=None, allKeys=False):
"""return the list of keys for a given CLID (int, type or class_name)
if clid is None, it will return the list of all keys.
When 'allKeys' is True it will also return the key aliases.
"""
if isinstance(clid, str):
clid = self._pyclidsvc.clid(clid)
if isinstance(clid, type):
clid = self._pyclidsvc.clid(clid.__name__)
if clid is None:
return [p.name() for p in self.proxies()]
return list(self._cpp_keys(clid, allKeys))
StoreGateSvc._cpp_keys = StoreGateSvc.keys
StoreGateSvc.keys = keys
return
except ImportError:
from gaudimodule import iProperty
return getattr(iProperty("StoreGateSvc", self), attr)
def __setattr__(self, attr, value):
try:
from GaudiPython.Bindings import iProperty
except ImportError:
from gaudimodule import iProperty
return setattr(iProperty("StoreGateSvc", self), attr, value)
StoreGateSvc.__getattr__ = __getattr__
StoreGateSvc.__setattr__ = __setattr__
import AthenaPython.PyAthena as PyAthena
StoreGateSvc._pyclidsvc = PyAthena.py_svc('ClassIDSvc')
def keys(self, clid=None, allKeys=False):
"""return the list of keys for a given CLID (int, type or class_name)
if clid is None, it will return the list of all keys.
When 'allKeys' is True it will also return the key aliases.
"""
if isinstance(clid, str):
clid = self._pyclidsvc.clid(clid)
if isinstance(clid, type):
clid = self._pyclidsvc.clid(clid.__name__)
if clid is None:
return [p.name() for p in self.proxies()]
return list(self._cpp_keys(clid, allKeys))
StoreGateSvc._cpp_keys = StoreGateSvc.keys
def initialize(self):
self.sg = PyAthena.py_svc("StoreGateSvc")
self.msg.info( "==> initializing [%s]...", self.name() )
self.msg.info( "cnt: %r",self.counter )
return StatusCode.Success
def initialize(self):
self.sg = PyAthena.py_svc("StoreGateSvc")
return StatusCode.Success
def store_configuration(cfg_fname=None):
"""helper function to inspect the 'configured' JobOptionsSvc and dump
the properties of each component into a (sqlite) shelve.
it will eventually dump the properties of py-components too.
"""
jobo_cfg = defaultdict(dict)
if cfg_fname is None:
import tempfile
tmpfile = tempfile.NamedTemporaryFile(suffix='-jobo.pkl')
cfg_fname = tmpfile.name
tmpfile.close()
import os
if os.path.exists(cfg_fname):
os.remove(cfg_fname)
assert cfg_fname
from AthenaCommon.AppMgr import theApp
def _fill_cfg(client, props):
for p in props:
n = p[0]
v = p[1]
if hasattr(v, 'toStringProperty'):
v = str(v.toStringProperty().toString())
elif hasattr(v, 'toString'):
v = str(v.toString())
else:
v = str(v)
jobo_cfg[client][n] = v
from AthenaCommon.AppMgr import ServiceMgr as svcMgr
# special cases: joboptionsvc and messagesvc
def _fill_props(svcname):
if not hasattr(svcMgr, svcname):
return
svc = getattr(svcMgr, svcname)
props = []
for k,v in svc.properties().iteritems():
if v == svc.propertyNoValue:
v = svc.getDefaultProperty(k)
props.append((k,v))
_fill_cfg(svcname, props)
_fill_props('JobOptionsSvc')
_fill_props('MessageSvc')
# tickle C++ and configure the whole app
theApp.setup()
app_props = [(k,v.value())
for k,v in theApp.getHandle().properties().iteritems()]
_fill_cfg(theApp.name(), app_props)
import AthenaPython.PyAthena as PyAthena
josvc = PyAthena.py_svc('JobOptionsSvc', iface='IJobOptionsSvc')
assert josvc is not None
clients = list(josvc.getClients())
for client in clients:
props = josvc.getProperties(client)
for prop in props:
n = prop.name()
v = prop.toString()
jobo_cfg[client][n] = v
cfg = dbs.open(cfg_fname, 'w')
cfg['jobopts'] = jobo_cfg
pycomps = []
import sys
from .AppMgr import ServiceMgr as svcMgr
# all other pycomps
from .Configurable import Configurable as C
for c in C.allConfigurables.itervalues():
if not isinstance(c, (PyAthena.Alg,
PyAthena.AlgTool,
PyAthena.Svc,
PyAthena.Aud)):
continue
# FIXME: should check if the component is still 'active'
# ie: is it in one of the TopAlg,SvcMgr,ToolSvc and children ?
if hasattr(c, 'msg'):
delattr(c, 'msg')
pycomps.append(c)
pass
# folder for the pyathena-components
cfg['pycomps'] = pycomps
cfg.close()
return cfg_fname
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
import AthenaPython.PyAthena as PyAthena
trans = PyAthena.cobs('/data/ilija/AOD.067184.big.pool.root','/data/ilija/tmp.pool.root')
# trans.mkProject()
# resizing BS and adding CL=7
trans.setTreeToSkip('##Links')
trans.setTreeToSkip('##Shapes')
trans.setTreeToSkip('##Params')
trans.setTreeMemory(10*1024,'POOLContainer_DataHeader')
trans.resizeBaskets()
trans1 = PyAthena.cobs('/data/ilija/tmp.pool.root','/data/ilija/oBS_orderedByEvent.pool.root')
trans1.order(2)
def evtStore(self):
import AthenaPython.PyAthena as PyAthena
return PyAthena.py_svc('StoreGateSvc/StoreGateSvc')
def _evtSize(self):
""" retrieve a handle to the ICollectionSize interface of the event loop mgr
"""
import AthenaPython.PyAthena as PyAthena
return PyAthena.py_svc(self.EventLoop, iface='ICollectionSize')
def inputStore(self):
import AthenaPython.PyAthena as PyAthena
return PyAthena.py_svc('StoreGateSvc/InputMetaDataStore')
def __init__(self,
name,
seq=topSequence,
file=None,
stream=None,
tuplename=None,
TuplePath=None,
preD3PDAlgSeqName=D3PDMakerFlags.PreD3PDAlgSeqName(),
streamNameRoot=None,
clevel=D3PDMakerFlags.CompressionLevel(),
**kwargs):
"""MakerAlg constructor. See the class documentation for a full description.
"""
if streamNameRoot == None:
streamNameRoot = 'D3PD'
# Work around initialization order issue.
if seq:
seq.__iadd__(D3PDMakerCoreComps.DummyInitAlg(name + 'DummyInit'),
index=0)
# If the tuple path wasn't supplied, build it from the other args.
if TuplePath == None:
# tuple name defaults to the algorithm name.
if tuplename == None:
tuplename = name
if stream == None:
# If no stream was given, infer it from the file.
# This creates the stream if needed.
if file == None:
raise TypeError("Neither stream nor file specified "
"for tuple %s" % tuplename)
stream = _stream_from_file(file, seq, tuplename,
streamNameRoot, clevel)
TuplePath = '/%s/%s' % (stream, tuplename)
# Create the algorithm Configurable.
D3PDMakerCoreCompsConf.D3PD__MakerAlg.__init__(self,
name,
TuplePath=TuplePath,
**kwargs)
# Ensure configuration parameters are set.
if (D3PDMakerFlags.AutoFlush() != -1
and self.D3PDSvc.getFullName() == 'D3PD::RootD3PDSvc'):
from D3PDMakerRoot.D3PDMakerRootConf import D3PD__RootD3PDSvc
rootsvc = D3PD__RootD3PDSvc()
rootsvc.AutoFlush = D3PDMakerFlags.AutoFlush()
from AthenaCommon.AppMgr import ServiceMgr
ServiceMgr += rootsvc
# Add to the supplied sequence.
if seq:
# But first, add a sequence for algorithms that should run
# before D3PD making, if it's not already there.
preseq = AlgSequence(preD3PDAlgSeqName)
if not hasattr(seq, preD3PDAlgSeqName):
seq += [preseq]
# We don't want to do filtering in the presequence.
preseq.StopOverride = True
# Now set up another sequence for filtering.
# Unlike the presequence, there should be a unique one of these
# per algorithm. We also need to break out an additional
# sequence to which users can add, and to wrap the whole
# thing in a sequence to prevent a failed filter
# decision from stopping other algorithms.
# Like this:
#
# ALG_FilterAlgorithmsWrap (StopOverride = True)
# ALG_FilterAlgorithmsHolder
# ALG_FilterAlgorithms
# ALG
# Dummy alg, to reset filter flag
suffix = D3PDMakerFlags.FilterAlgSeqSuffix()
wrap = AlgSequence(name + suffix + 'Wrap', StopOverride=True)
holder = AlgSequence(name + suffix + 'Holder')
self.filterSeq = AlgSequence(name + suffix)
holder += self.filterSeq
holder += self
wrap += holder
wrap += PyAthena.Alg(name + 'Dummy')
seq += wrap
# Create a unique collection getter registry tool for this tree.
from AthenaCommon.AppMgr import ToolSvc
self._registry = \
D3PDMakerCoreComps.CollectionGetterRegistryTool (self.name() +
'_CollectionGetterRegistry')
ToolSvc += self._registry
return
#Create instance of converter alg, configuring a property of it
#Note that this procedure will hopefully be improved soon
alg = ROOT.AthAnalysisHelper.createAlgorithm("xAODMaker::xAODTruthCnvAlg/myAlg")
myalg = InterfaceCast('IProperty').cast(alg)
myalg.setProperty( "AODContainerName" , "GEN_EVENT" )
alg.initialize()
tool = ROOT.AthAnalysisHelper.createTool("DerivationFramework::TruthDressingTool/TruthDressing")
#this is how to cast the tool in python so you have access to the addBranches method
tool = InterfaceCast('DerivationFramework::IAugmentationTool').cast(tool)
tool.initialize()
#these next two lines are needed for now,
#until python bindings for retrieve method of TEvent is working
from AthenaPython import PyAthena
sg = PyAthena.py_svc("StoreGateSvc")
m_ll = [ ROOT.TH1D("m_ll","Dilepton mass",100,0,200) ];
m_ll += [ ROOT.TH1D("m_ll_dressed","Dressed Dilepton mass",100,0,200) ];
for i in range(0,evt.getEntries()):
evt.getEntry(i)
alg.execute() #converts HepMC into xAOD
tool.addBranches() #decorates leptons with pt_dressed etc
mc = sg['TruthParticles'] #retrieve objects via the sg storegate obj
elplus = [[],[]]
elminus = [[],[]]
muplus = [[],[]]
muminus = [[],[]]
for p in mc:
if p.status() != 1: continue
evLoopName = theApp.EventLoop.split('/')[-1]
evLoop = getattr( svcMgr, evLoopName )
props = []
for k,v in evLoop.properties().iteritems():
if v != C.propertyNoValue:
props.append( (k,v) )
_fillCfg( evLoopName, props )
except AttributeError, a:
pass # no properties defined for EventLoop type
# get the values for all other components (these may contain duplicates of
# the ones above in cfgSvcs, and there may even be conflicts)
import AthenaPython.PyAthena as PyAthena
josvc = PyAthena.py_svc( 'JobOptionsSvc', iface = 'IJobOptionsSvc' )
clients = list( josvc.getClients() )
for client in clients:
props = josvc.getProperties( client )
for prop in props:
n = prop.name()
v = prop.toString()
jocat[ client ][ n ] = v
# take care of some ancient history
for k in ( 'Go', 'Exit' ):
if k in jocfg[ 'ApplicationMgr' ]:
del jocfg[ 'ApplicationMgr' ][k]
# workaround for pycomps
def evtStore(self):
if not self._pyath_evtstore:
import AthenaPython.PyAthena as PyAthena
self._pyath_evtstore = PyAthena.py_svc('StoreGateSvc/StoreGateSvc')
return self._pyath_evtstore
def initialize(self):
import AthenaPython.PyAthena as PyAthena
_info = self.msg.info
_info("POOL2EI::initialize")
_info("## DoProvenanceRef: {}".format(self.DoProvenanceRef))
_info("## DoTriggerInfo: {}".format(self.DoTriggerInfo))
_info("## HaveHlt: {}".format(self.HaveHlt))
_info("## HaveXHlt: {}".format(self.HaveXHlt))
_info("## RUN1: {}".format(self.RUN1))
_info("## SendToBroker: {}".format(self.SendToBroker))
self._dsname = "Unknown.Input.Dataset.Name" # default fake value
if self.EiDsName is not None:
_info("## EiDsName: {}".format(self.EiDsName))
self._dsname = self.EiDsName
else:
# try to get dataset name from pathena INDS environment variable
import os
inds = os.getenv('INDS')
if inds is not None:
_info("## INDS: {}".format(inds))
self._dsname = inds
else:
# else, try to use job definition
try:
import newJobDef
processingType = newJobDef.job['processingType']
transformation = newJobDef.job['transformation']
dsSource = 'realDatasetsIn' # take dataset name from input
if processingType == 'merge' and transformation != 'POOLtoEI_tf.py':
dsSource = 'realDatasets' # take dataset name from output
datasets = newJobDef.job[dsSource].split(',')
_info("## {}[0]: {}".format(dsSource, datasets[0]))
self._dsname = datasets[0]
# remove _tid and _sub parts from dsname
import re
self._dsname = re.sub('_tid[0-9]{8}_[0-9]{2}', '',
self._dsname)
self._dsname = re.sub('_sub[0-9]{10}', '', self._dsname)
self._dsname = re.sub('\\/$', '', self._dsname)
except:
_info(
'## Unable to get dataset name from realDatasetsIn or realDatasets'
)
# token match regex
import re
self._re_pool_token = re.compile(r'[[]DB=(?P<db>.*?)[]]' \
r'[[]CNT=(?P<cnt>.*?)[]]' \
r'[[]CLID=(?P<clid>.*?)[]]' \
r'[[]TECH=(?P<tech>.*?)[]]' \
r'[[]OID=(?P<oid>.*?)[]]').match
# load our pythonizations:
for cls_name in ('EventStreamInfo', 'EventType', 'PyEventType'):
cls = getattr(PyAthena, cls_name)
_info("retrieving various stores...")
for store_name in ('evtStore', 'inputStore', 'detStore', 'tagStore',
'metaStore'):
_info("retrieving [{}]...".format(store_name))
o = getattr(self, store_name)
_info("retrieving [{}]... [done]".format(store_name))
_info("retrieving various stores... [done]")
if self.HaveHlt:
# load trigger decision tool
from TriggerJobOpts.TriggerFlags import TriggerFlags
TriggerFlags.configurationSourceList = ['ds']
import AthenaPython.PyAthena as PyAthena
self.trigDec = PyAthena.py_tool(
'Trig::TrigDecisionTool/TrigDecisionTool')
self.trigDec.ExperimentalAndExpertMethods().enable()
if self.HaveXHlt:
self.trigDec.setProperty("ConfigTool",
"TrigConf::xAODConfigTool")
self.trigDec.setProperty("TrigDecisionKey", "xTrigDecision")
else:
self.trigDec.setProperty("ConfigTool",
"TrigConf::AODConfigTool")
self.trigDec.setProperty("TrigDecisionKey", "TrigDecision")
## open output pkl file
import os
if self.Out is not None:
oname = self.Out
else:
oname = "output.ei.pkl"
oname = os.path.expanduser(os.path.expandvars(oname))
self._eifname = oname
_info('Opening EI file [{}]...'.format(oname))
if os.path.exists(oname):
os.remove(oname)
import PyUtils.dbsqlite as dbsqlite
try:
self._eif = dbsqlite.open(oname, flags='w')
except:
pass
if self._eif is None:
self.msg.fatal(
"Unable to open EI output file {} exapnded as {}".format(
self.Out, oname))
raise RuntimeError("Unable to open EI output file")
# get taskid and jobid
if hasattr(self, 'TaskID') and hasattr(
self, 'JobID'
) and self.TaskID is not None and self.JobID is not None:
self._eif['TaskID'] = "{}.T".format(self.TaskID)
if hasattr(self,
'AttemptNumber') and self.AttemptNumber is not None:
self._eif['JobID'] = "{}.{}".format(self.JobID,
self.AttemptNumber)
else:
self._eif['JobID'] = "{}.0".format(self.JobID)
else:
# get them from job info
try:
import newJobDef
self._eif['TaskID'] = "{}.G".format(newJobDef.job['taskID'])
self._eif['JobID'] = "{}.{}".format(newJobDef.job['PandaID'],
newJobDef.job['attemptNr'])
except:
self._eif['TaskID'] = "{}.G".format(
os.getenv('PanDA_TaskID', 0))
self._eif['JobID'] = "{}.0".format(os.getenv('PandaID', 0))
# initial information
self._eif['StartProcTime'] = int(time.time() * 1000)
self._eif['Schema'] = EIrecord().getRecSchema()
self._eif['Version'] = EIrecord().getVersion()
self._eif['InputDsName'] = self._dsname
#processing options
self._eif['ProvenanceRef'] = self.DoProvenanceRef
self._eif['TriggerInfo'] = self.DoTriggerInfo
return StatusCode.Success
def initialize(self):
self.evtstore = PyAthena.py_svc('StoreGateSvc')
return StatusCode.Success
def _evtSize(self):
""" retrieve a handle to the ICollectionSize interface of the event loop mgr
"""
import AthenaPython.PyAthena as PyAthena
return PyAthena.py_svc(self.EventLoop, iface='ICollectionSize')
def _evtSeek(self):
""" retrieve a handle to the IEventSeek interface of the event loop mgr
"""
import AthenaPython.PyAthena as PyAthena
return PyAthena.py_svc(self.EventLoop, iface='IEventSeek')
def initialize(self):
self.sg = PyAthena.py_svc('StoreGateSvc')
self.rootfile = ROOT.TFile("out.root", "RECREATE")
self.h_pids = ROOT.TH1F("pids", "", 1201, -600.5, 600.5)
self.h_pids12 = ROOT.TH1F("pids12", "", 1201, -600.5, 600.5)
return PyAthena.StatusCode.Success
def detStore(self):
import AthenaPython.PyAthena as PyAthena
return PyAthena.py_svc('StoreGateSvc/DetectorStore')
def inputStore(self):
import AthenaPython.PyAthena as PyAthena
return PyAthena.py_svc('StoreGateSvc/InputMetaDataStore')
def evtStore(self):
import AthenaPython.PyAthena as PyAthena
return PyAthena.py_svc('StoreGateSvc/StoreGateSvc')
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
import AthenaPython.PyAthena as PyAthena
trans = PyAthena.cobs('/data/ilija/AOD.067184.big.pool.root',
'/data/ilija/split0.pool.root')
# trans.mkProject()
trans.setSplitLevel(0)
trans.changeSplitLevel()
def tagStore(self):
import AthenaPython.PyAthena as PyAthena
return PyAthena.py_svc('StoreGateSvc/TagMetaDataStore')
def initialize(self):
_info = self.msg.info
_error = self.msg.error
_info('==> initialize...')
# check our properties and which modus operandi is active
if self.evt_list is None and self.filter_fct is None:
_error ('invalid properties: evt_list *and* filter_fct are None !')
_error ('usage:\n%s', self.__doc__)
return StatusCode.Failure
if (not (self.evt_list is None)) and \
(not (self.filter_fct is None)):
_error ('invalid properties: evt_list *and* filter_fct '
'are not None !')
_error ('usage:\n%s', self.__doc__)
return StatusCode.Failure
if not (self.filter_fct is None):
# checking the filtering function is sound
import inspect
args = inspect.getargspec (self.filter_fct)[0]
# FIXME: class' methods have the hidden 'self' argument...
if len(args) != 2:
_error ('filter_fct has NOT the expected signature')
_error (' nbr of arguments: %i', len(args))
_error (' expected: 2')
return StatusCode.Failure
if not (self.filter_policy in ('reject', 'accept')):
_error ('invalid value for filter_policy: %r',
self.filter_policy)
_error ("valid values are 'reject' or 'accept'")
return StatusCode.Failure
else:
# transform the list of events into a set of pairs (run,evt)
evt_list = self.evt_list[:]
self.evt_list = set()
for i in evt_list:
if isinstance(i, tuple): irun, ievt = i
else: irun, ievt = None, i
self.evt_list.add ((irun, ievt))
_info ('EventInfo name: %s',
self.evt_info
if self.evt_info else '<any>')
if self.evt_list:
_info("filtering mode: evt_list")
_info("filtering policy: %s", self.filter_policy)
self.filter_policy = False if self.filter_policy == 'reject' else \
True
else:
_info ("filtering mode: filter_fct")
self.sg = PyAthena.py_svc ('StoreGateSvc')
if self.sg is None:
_error ('could not retrieve event store')
return StatusCode.Failure
self._evt_cnt = 0 # total number of events processed
self._evt_acc_cnt = 0 # total number of events accepted by the filter
self._evt_rej_cnt = 0 # total number of events rejected by the filter
return StatusCode.Success
