def cnv(self):
data = {}
import tempfile, os, glob
origdir = os.getcwd()
tmpdir = self._tmpdir
try:
os.chdir(tmpdir)
import tarfile
tar = tarfile.open(os.path.abspath(self._fileName), 'r')
tar.extractall(path=tmpdir)
tar.close()
fname = glob.glob("*.dat")[0]
if open(fname, 'r').read(1024).startswith('SQLite format'):
import PyUtils.dbsqlite as dbs
db = dbs.open(fname, 'r')
else:
db = shelve.open(fname)
data['meta'] = {}
for k in db.iterkeys():
data['meta'][k] = db[k]
db.close()
## print "version:",data['meta']['version_id']
from PyRootLib import importRoot
ROOT = importRoot()
root = ROOT.fopen(glob.glob("*.root")[0], "read")
for k in ('ini', 'evt', 'fin'):
data[k] = root.Get("perfmon/%s" % k)
data['meta']['rootFile'] = root
finally:
os.chdir(origdir)
return data
def __write_out_pmon_data(self, outFiles):
headerFile = outFiles[0]
_msg = self.msg
db_infos = ('err',) # lshosts_infos()
if db_infos[0] != 'ok':
host_infos = { 'si2k' : 0.,
'cpuf' : 0. }
else:
host_infos = db_infos[1]
host_infos['si2k'] = float(host_infos['cpuf'])*1000.
#stats = self.stats['evt']
_msg.info("Si2k: %r => <cpu time>: %10.3f kSi2k",
host_infos['si2k'],
host_infos['si2k']*self._evt_cpu_mean/1000.)
self.meta['hosts_infos'] = host_infos
self.meta['pmon_version']= __version__
self.meta['algs_names'] = getattr(self, '_algs_names', ['<N/A>'])
_msg.info("Writing out collected data... [%s]", self.outFileName)
## write out meta-data
import PyUtils.dbsqlite as dbs
meta = dbs.open(headerFile, 'n')
for k,v in six.iteritems (self.meta): meta[k] = v
meta['version_id'] = '0.4.0' # stream-format + header file
meta['pmon_tuple_files'] = map( os.path.basename, outFiles[1:] )
import socket
try: meta['hostname'] = socket.gethostname()
except Exception: meta['hostname'] = '<unknown>'
meta.close()
cwd = os.getcwd()
## compress/package everything
import tarfile
tar = tarfile.open(self.outFileName, 'w:gz')
try:
for outFile in outFiles:
outFileDirName = os.path.dirname(outFile)
try: os.chdir(outFileDirName)
except OSError as err: pass
outFile = os.path.basename(outFile)
_msg.info(' --> [%s] => %8.3f kb',
outFile,
os.stat(outFile).st_size/Units.kB)
tar.add(outFile)
os.remove(outFile)
os.chdir(cwd)
pass
except Exception as err:
_msg.warning(err)
pass
os.chdir(cwd)
tar.close()
_msg.info ("Writing out collected data... [%s] => %8.3f kb",
self.outFileName,
os.stat(self.outFileName).st_size / Units.kB)
return
def _load_db_cache(self, fname):
"""load file informations from a sqlite file"""
import PyUtils.dbsqlite as dbsqlite
cache = dbsqlite.open(fname)
d = {}
for k,v in six.iteritems (cache):
d[k] = AthFile.from_infos(v)
return d
def retrieve_pmon_svc(fname):
import PyUtils.dbsqlite as dbs
db = dbs.open(fname, 'r')
svc = db['pmon']
db.close()
import os
os.remove(fname)
return svc
def main():
# logger
global log
log = logging.getLogger('RAW2EI.py')
# get options
opt = options(sys.argv)
if opt.verbose > 0:
log.setLevel(logging.INFO)
if opt.debug > 0:
log.setLevel(logging.DEBUG)
# open EI file
if os.path.exists(opt.output):
os.remove(opt.output)
eif = dbsqlite.open(opt.output, flags='w')
eif['StartProcTime'] = int(time.time() * 1000)
eif['Schema'] = EIrecord().getRecSchema()
eif['Version'] = EIrecord().getVersion()
eif['TaskID'] = os.getenv('TaskID', 0)
eif['JobID'] = os.getenv('JobID', 0)
if opt.eidsname is not None:
dsname = opt.eidsname
else:
dsname = os.getenv('INDS', "Unknown.Input.Dataset.Name")
dsname = re.sub('_tid[0-9]{8}_[0-9]{2}', '', dsname)
dsname = re.sub('_sub[0-9]{10}', '', dsname)
dsname = re.sub('\/$', '', dsname)
eif['InputDsName'] = dsname
#processing options
eif['ProvenanceRef'] = False
eif['TriggerInfo'] = True
## process RAW files
if type(opt.input) is str:
filelist = [opt.input]
else:
filelist = opt.input
nfile = 0
nevents = 0
for f in filelist:
nevents += processRAW(f, eif, nfile, nevents, opt.evtmax)
nfile += 1
if opt.evtmax > 0 and nevents >= opt.evtmax:
break
eif['Nentries'] = nevents
eif['Nfiles'] = nfile
eif['EndProcTime'] = int(time.time() * 1000)
eif.close()
def _save_db_cache(self, fname):
"""save file informations using sqlite"""
import PyUtils.dbsqlite as dbsqlite
db = dbsqlite.open(fname,flags='w')
cache = self._cache
for k in cache:
db[k] = cache[k].fileinfos
db.close()
return
def __init__(self, fname):
try:
self._db = dbsqlite.open(fname, flags='r')
except:
log.error("Unable to open Event Index file {}".format(fname))
raise
try:
self._nentries = self._db['Nentries']
except:
log.error("Unable to get nentries from Event Index file {}".format(
fname))
raise
def finalize(self):
_info = self.msg.info
peeked_data = dict(self.peeked_data)
if self.outfname:
oname = self.outfname
import os
oname = os.path.expanduser(os.path.expandvars(oname))
_info('storing peeked file infos into [%s]...', oname)
if os.path.exists(oname):
os.remove(oname)
try:
import cPickle as pickle
except ImportError:
import pickle
import PyUtils.dbsqlite as dbsqlite
db = dbsqlite.open(oname, flags='w')
# merge and collect data from all processed events (if any)
if self._peeked_events:
peeked_data = self._peeked_events.pop(0)
for d in self._peeked_events:
for k in (
'run_number',
'evt_number',
'lumi_block',
'run_type',
'beam_type',
'beam_energy',
'stream_tags',
):
v = d[k]
if isinstance(v, list) and len(v) > 0:
v = v[0]
peeked_data[k].append(v)
#_info('peeked_data:')
#_info(str(peeked_data))
db['fileinfos'] = peeked_data
db.close()
_info('storing peeked file infos into [%s]... [done]', oname)
self.print_summary(peeked_data)
return StatusCode.Success
def finalize(self):
_info = self.msg.info
peeked_data = dict(self.peeked_data)
if self.outfname:
oname = self.outfname
import os
oname = os.path.expanduser(os.path.expandvars(oname))
_info('storing peeked file infos into [%s]...', oname)
if os.path.exists(oname):
os.remove(oname)
try:
import cPickle as pickle
except ImportError:
import pickle
import PyUtils.dbsqlite as dbsqlite
db = dbsqlite.open(oname,flags='w')
# merge and collect data from all processed events (if any)
if self._peeked_events:
peeked_data = self._peeked_events.pop(0)
for d in self._peeked_events:
for k in ('run_number',
'evt_number',
'lumi_block',
'run_type',
'beam_type',
'beam_energy',
'stream_tags',):
v = d[k]
if isinstance(v, list) and len(v)>0:
v = v[0]
peeked_data[k].append(v)
#_info('peeked_data:')
#_info(str(peeked_data))
db['fileinfos'] = peeked_data
db.close()
_info('storing peeked file infos into [%s]... [done]', oname)
self.print_summary(peeked_data)
return StatusCode.Success
def dict_from_shelve(fname):
db = {}
if open(fname, 'rb').read(1024).startswith(b'SQLite format'):
import PyUtils.dbsqlite as dbs
db = dbs.open(fname, 'r')
else:
import shelve
try:
db = shelve.open(fname)
except Exception:
pass
pass
out = dict(db)
if not out:
return None
if only_summary:
if 'perf_summary' in out.keys():
return {'perf_summary': out['perf_summary']}
else:
return None
return out
def _save_db_cache(self):
"""save file informations using sqlite"""
import tempfile
fd_pkl, out_pkl_fname = tempfile.mkstemp(suffix='.pkl')
# import os
os.close(fd_pkl)
if os.path.exists(out_pkl_fname):
os.remove(out_pkl_fname)
oname = os.path.expanduser(os.path.expandvars(out_pkl_fname))
if os.path.exists(oname):
os.remove(oname)
import PyUtils.dbsqlite as dbsqlite
db = dbsqlite.open(oname, flags='w')
db['fileinfos'] = self.peeked_data
db.close()
if os.path.exists(oname) and len(self.peeked_data) > 0:
return 0, out_pkl_fname
else:
os.remove(oname)
return 1, out_pkl_fname
def configure(self,
joboptions=None,
commands=None,
dllname=None,
factname=None,
extra_options=None):
if not (self.app is None):
self.msg.info('C++ application already configured')
return self.app
self.msg.info('configuring application...')
usr_cfg = AthCfg()
self.cfg.seek(0)
usr_cfg << self.cfg.read()
# reset
self.cfg = AthCfg()
if commands:
self.cfg << commands + '\n'
# common configuration
self.cfg << """
# basic job configuration
include('AthenaCommon/Atlas.UnixStandardJob.py')
include.block('AthenaCommon/Atlas.UnixStandardJob.py')
if not (not %(run_batch)s and
theApp.EventLoop == 'PyAthenaEventLoopMgr'):
# make SIG_INT fatal
svcMgr.CoreDumpSvc.FatalHandler = -1
""" % {
'run_batch': self.options.run_batch
}
self.cfg << """
# user level configuration
try:
include('$HOME/.athenarc')
except IncludeError:
pass
"""
# another user level configuration
usr_cfg.seek(0)
self.cfg << usr_cfg.read()
if isinstance(joboptions, (list, tuple)):
for jobo_name in joboptions:
self.cfg.include(jobo_name)
if not self.options.run_batch:
self.cfg << """
theApp.EventLoop = 'PyAthenaEventLoopMgr'
svcMgr += CfgMgr.PyAthenaEventLoopMgr()
"""
self.cfg << """
### logging and messages ---------
from AthenaCommon.Logging import *
_msg = log
_msg.setLevel(getattr(logging, '%(output_level)s'))
import AthenaCommon.Constants as Lvl
theApp.setOutputLevel(%(output_level)s)
theApp.OutputLevel = Lvl.%(output_level)s
from AthenaCommon.AppMgr import ServiceMgr as svcMgr
svcMgr.MessageSvc.OutputLevel = Lvl.%(output_level)s
""" % dict(output_level=self.options.msg_lvl)
self.cfg << """
from AthenaCommon.Include import Include, IncludeError, include
include.setShowIncludes(%(showincludes)s)
if %(showincludes)s:
import AthenaCommon.Include as AthCIncMod
AthCIncMod.marker=' -#-' # distinguish bootstrap from other jo-code
""" % dict(showincludes=self.options.showincludes)
cfg_name = self.cfg._jobo.name.replace('.py', '.pkl')
self.msg.info('dumping job-configuration into [%s]...', cfg_name)
# run configuration in a forked-subprocess...
sc = _app_configure(self.cfg, cfg_name, extra_options)
if sc:
err = 'could not configure application [sc=%d]' % sc
self.msg.error(err)
raise RuntimeError(err)
self.msg.info('configuring application w/ [%s]', cfg_name)
import os
self.cfg._jobo.close()
os.remove(self.cfg._jobo.name)
import PyCintex
PyCintex.Cintex.Enable()
gbl = PyCintex.makeNamespace('')
import GaudiPython.Bindings as gaudi
# remove the gaudimodule exit handler as to prevent them from clobering
import atexit
for hdlr in reversed(atexit._exithandlers[:]):
module_name = hdlr[0].__module__
if ('GaudiPython' in module_name or 'gaudimodule' in module_name):
atexit._exithandlers.remove(hdlr)
del hdlr
# install our own exit handler (if needed)
import sys
if hasattr(sys, 'ps1'): # ie: is interactive
atexit.register(self.exit)
del atexit
from . import ResourceLimits
ResourceLimits.SetMaxLimits()
try:
import cPickle as pickle
except ImportError:
import pickle
import PyUtils.dbsqlite as dbs
db = dbs.open(cfg_name, 'r')
jobo_cfg = db['jobopts']
kw = jobo_cfg['ApplicationMgr']
for k in ('Go', 'Exit', 'AuditInitialize', 'AuditFinalize'):
if k in kw:
del kw[k]
outputlevel = jobo_cfg['ApplicationMgr']['OutputLevel']
self.app = gaudi.AppMgr(outputlevel=outputlevel,
selfoptions=kw,
dllname=dllname,
factname=factname)
# open the pycomps folder
pycomps = db.get('pycomps', None) # just opening it should do
if pycomps:
import AthenaPython.Configurables as _C
_C.PyComponents.instances = dict((p.name, p) for p in pycomps)
#_C.PyComponents.instances = pycomps
for p in pycomps:
if hasattr(p, 'setup'):
if callable(p.setup):
p.setup()
setattr(self, '_pycomps', pycomps)
import AthenaPython.PyAthena as PyAthena
josvc = PyAthena.py_svc('JobOptionsSvc',
createIf=False,
iface='IJobOptionsSvc')
assert josvc is not None
for client in jobo_cfg:
if client == 'ApplicationMgr':
continue
for n, v in jobo_cfg[client].iteritems():
p = gaudi.StringProperty(n, v)
if not josvc.addPropertyToCatalogue(client, p).isSuccess():
self.msg.error('could not add property [%s.%s = %s]',
client, n, v)
if client in ('MessageSvc', 'JobOptionsSvc'):
svc = PyAthena.py_svc(client, iface='IProperty')
svc.setProperty(p)
db.close()
import os
if os.path.exists(cfg_name):
os.remove(cfg_name)
pass
#import AthenaCommon.Debugging as dbg
#dbg.hookDebugger()
return self.app
def store_configuration(cfg_fname=None):
"""helper function to inspect the 'configured' JobOptionsSvc and dump
the properties of each component into a (sqlite) shelve.
it will eventually dump the properties of py-components too.
"""
jobo_cfg = defaultdict(dict)
if cfg_fname is None:
import tempfile
tmpfile = tempfile.NamedTemporaryFile(suffix='-jobo.pkl')
cfg_fname = tmpfile.name
tmpfile.close()
import os
if os.path.exists(cfg_fname):
os.remove(cfg_fname)
assert cfg_fname
from AthenaCommon.AppMgr import theApp
def _fill_cfg(client, props):
for p in props:
n = p[0]
v = p[1]
if hasattr(v, 'toStringProperty'):
v = str(v.toStringProperty().toString())
elif hasattr(v, 'toString'):
v = str(v.toString())
else:
v = str(v)
jobo_cfg[client][n] = v
from AthenaCommon.AppMgr import ServiceMgr as svcMgr
# special cases: joboptionsvc and messagesvc
def _fill_props(svcname):
if not hasattr(svcMgr, svcname):
return
svc = getattr(svcMgr, svcname)
props = []
for k, v in svc.properties().iteritems():
if v == svc.propertyNoValue:
v = svc.getDefaultProperty(k)
props.append((k, v))
_fill_cfg(svcname, props)
_fill_props('JobOptionsSvc')
_fill_props('MessageSvc')
# tickle C++ and configure the whole app
theApp.setup()
app_props = [(k, v.value())
for k, v in theApp.getHandle().properties().iteritems()]
_fill_cfg(theApp.name(), app_props)
import AthenaPython.PyAthena as PyAthena
josvc = PyAthena.py_svc('JobOptionsSvc', iface='IJobOptionsSvc')
assert josvc is not None
clients = list(josvc.getClients())
for client in clients:
props = josvc.getProperties(client)
for prop in props:
n = prop.name()
v = prop.toString()
jobo_cfg[client][n] = v
cfg = dbs.open(cfg_fname, 'w')
cfg['jobopts'] = jobo_cfg
pycomps = []
import sys
from .AppMgr import ServiceMgr as svcMgr
# all other pycomps
from .Configurable import Configurable as C
for c in C.allConfigurables.itervalues():
if not isinstance(
c,
(PyAthena.Alg, PyAthena.AlgTool, PyAthena.Svc, PyAthena.Aud)):
continue
# FIXME: should check if the component is still 'active'
# ie: is it in one of the TopAlg,SvcMgr,ToolSvc and children ?
if hasattr(c, 'msg'):
delattr(c, 'msg')
pycomps.append(c)
pass
# folder for the pyathena-components
cfg['pycomps'] = pycomps
cfg.close()
return cfg_fname
def main():
# analyze options
opt = options(sys.argv)
fname = opt.eifile
try:
db = dbsqlite.open(fname, flags='r')
except:
print >> sys.stderr, "Unable to open peeker file {}".format(fname)
sys.exit(1)
try:
nentries = db['Nentries']
except:
print >> sys.stderr, "Unable to get nentries from EI file {}".format(
fname)
sys.exit(1)
#print "Stored keys: "
#print " ",db.keys()
print ""
print " version: ", db['Version']
print " schema: ", db['Schema']
print " #input files: ", db['Nfiles']
print " number of events: ", nentries
print " StartProcTime: ", db['StartProcTime']
print " EndProcTime: ", db['EndProcTime']
print " TaskID: ", db['TaskID']
print " JobID: ", db['JobID']
print " InputDsName: ", db['InputDsName']
print " Includes Provenance: ", db['ProvenanceRef']
print " Includes Trigger: ", db['TriggerInfo']
triggerinfo = db['TriggerInfo']
provenanceinfo = db['ProvenanceRef']
for i in xrange(db['Nfiles']):
print ""
print "File {:d}".format(i)
if 'Nentries_{:d}'.format(i) in db:
print " Events in this file: {}".format(
db['Nentries_{:d}'.format(i)])
print " StartProcTime: ", db['StartProcTime_{:d}'.format(i)]
if 'EndProcTime_{:d}'.format(i) in db:
print " EndProcTime: ", db['EndProcTime_{:d}'.format(i)]
else:
print " EndProcTime: ", db['EndProcTime']
if "AMITag_{:d}".format(i) in db:
print " AMITag: ", db['AMITag_{:d}'.format(i)]
if "TrigStream_{:d}".format(i) in db:
print " TrigStream: ", db['TrigStream_{:d}'.format(i)]
if "ProjName_{:d}".format(i) in db:
print " ProjName: ", db['ProjName_{:d}'.format(i)]
if "GUID_{:d}".format(i) in db:
print " GUID: ", db['GUID_{:d}'.format(i)]
print ""
print "Summary: (info for {:d} events max)".format(opt.evtmax)
print ""
schema = db['Schema']
evtnumber_idx = schema.index('EventNumber')
runnumber_idx = schema.index('RunNumber')
nevt_shown = 0
if opt.evtnumber == 0:
sw0 = True
else:
sw0 = False
for i in xrange(nentries):
evt = db['Entry_{:d}'.format(i)]
data = zip(schema, evt)
if opt.evtnumber == evt[evtnumber_idx]:
if opt.runnumber == 0 or opt.runnumber == evt[runnumber_idx]:
sw0 = True
if not sw0:
continue
if nevt_shown >= opt.evtmax:
break
nevt_shown += 1
trigL1 = None
trigL2 = None
trigEF = None
print "{:d} ============================================================".format(
nevt_shown)
for k, v in data:
print " {:<20s}: {}".format(k, v)
if k in "L1PassedTrigMask','L2PassedTrigMask','EFPassedTrigMask":
if triggerinfo:
v = v.split(";")[0]
## trigger
if v is not None and v != '':
dv = decompressB64(v)
if opt.decodetrig1 > 0:
print " {:<20s}: {}".format(k, dv)
if k == "L1PassedTrigMask":
trigL1_U = v
trigL1 = dv
if k == "L2PassedTrigMask":
trigL2_U = v
trigL2 = dv
if k == "EFPassedTrigMask":
trigEF_U = v
trigEF = dv
#trigger bits fired
if trigL1 is not None:
if opt.decodetrig2 > 0:
l1tl = getL1TriggerList(trigL1_U)
print " L1PassedTrigMaskTBP: ", l1tl['TBP']
print " L1PassedTrigMaskTAP: ", l1tl['TAP']
print " L1PassedTrigMaskTAV: ", l1tl['TAV']
for i in xrange(8):
tw = trigL1[32 * i:32 * (i + 1)][::-1]
v = int(tw, 2)
if v != 0 and opt.decodetrig0 > 0:
print " L1PassedTrigMaskTBP{:d} = {:d} (0x{:08x})".format(
i, v, v)
for i in xrange(8, 16):
tw = trigL1[32 * i:32 * (i + 1)][::-1]
v = int(tw, 2)
if v != 0 and opt.decodetrig0 > 0:
print " L1PassedTrigMaskTAP{:d} = {:d} (0x{:08x})".format(
i - 8, v, v)
for i in xrange(16, 24):
tw = trigL1[32 * i:32 * (i + 1)][::-1]
v = int(tw, 2)
if v != 0 and opt.decodetrig0 > 0:
print " L1PassedTrigMaskTAV{:d} = {:d} (0x{:08x})".format(
i - 16, v, v)
if trigL2 is not None:
if opt.decodetrig2 > 0:
print " L2PassedTrigMask: ", getTriggerList(trigL2_U)
for i in xrange(len(trigL2) / 32):
tw = trigL2[32 * i:32 * (i + 1)][::-1]
v = int(tw, 2)
if v != 0 and opt.decodetrig0 > 0:
print " L2PassedTrigMask{:d} = {:d} (0x{:08x})".format(
i, v, v)
if trigEF is not None:
if opt.decodetrig2 > 0:
print " EFPassedTrigMask: ", getTriggerList(trigEF_U)
for i in xrange(len(trigEF) / 32):
tw = trigEF[32 * i:32 * (i + 1)][::-1]
v = int(tw, 2)
if v != 0 and opt.decodetrig0 > 0:
print " EFPassedTrigMask{:d} = {:d} (0x{:08x}) ".format(
i, v, v)
db.close()
def _process_call(self, fname, evtmax, projects=['AtlasCore']):
msg = self.msg()
f = _create_file_infos()
protocol, _ = self.server.fname(fname)
f_raw = self._root_open(fname, raw=True)
if f_raw is None or not f_raw:
raise IOError(
errno.ENOENT,
'No such file or directory',
fname)
f_root = f_raw
try:
file_type, file_name = self.server.ftype(f_raw)
protocol,file_name = self.server.fname(fname)
f['file_md5sum'] = self.server.md5sum(f_raw)
f['file_name'] = file_name
f['file_type'] = file_type
f['file_size'] = f_raw.GetSize()
if file_type == 'pool':
f_root = self._root_open(fname, raw=False)
# POOL files are most nutritious when known to PoolFileCatalog.xml
# FIXME: best would be to do that in athfile_peeker.py but
# athena.py closes sys.stdin when in batch, which confuses
# PyUtils:subprocess.getstatusoutput
#
# ATEAM-192: avoid the PoolFileCatalog.xml conflict
#cmd = ['pool_insertFileToCatalog.py',
# file_name,]
#subprocess.call(cmd, env=self._sub_env)
#
if True:
is_tag, tag_ref, tag_guid, nentries, runs, evts = self._is_tag_file(f_root, evtmax)
if is_tag:
f['stream_names'] = ['TAG']
f['file_guid'] = tag_guid
f['nentries'] = nentries
f['run_number'] = runs
f['evt_number'] = evts
else:
import tempfile
fd_pkl,out_pkl_fname = tempfile.mkstemp(suffix='.pkl')
os.close(fd_pkl)
if os.path.exists(out_pkl_fname):
os.remove(out_pkl_fname)
print ("\n --------- running Athena peeker")
print (os.environ.get('CMTPATH',''))
import AthenaCommon.ChapPy as api
app = api.AthenaApp(cmdlineargs=["--nprocs=0"])
app << """
FNAME = %s
""" % str([file_name])
app << """
import os
# prevent from running athena in interactive mode (and freeze)
if 'PYTHONINSPECT' in os.environ:
del os.environ['PYTHONINSPECT']
include('AthenaPython/athfile_peeker.py')
from AthenaCommon.AlgSequence import AlgSequence
job = AlgSequence()
# we don't really need this...
job.peeker.outfname='%(outfname)s'
job.peeker.infname='%(infname)s'
# metadata + taginfo
import IOVDbSvc.IOVDb
# evt-max
theApp.EvtMax = %(evtmax)i
""" % {
'infname' : file_name,
'outfname': out_pkl_fname,
'evtmax': evtmax,
}
import uuid
stdout_fname = (
'athfile-%i-%s.log.txt' %
(os.getpid(), uuid.uuid4())
)
stdout = open(stdout_fname, "w")
print ("="*80, file=stdout)
print (self._sub_env, file=stdout)
print ("="*80, file=stdout)
stdout.flush()
if DEFAULT_AF_RUN:
sc = app.run(stdout=stdout, env=self._sub_env)
else:
import PyUtils.FilePeekerTool as fpt
fp = fpt.FilePeekerTool(f_root)
sc, fp_pkl_fname = fp.run()
# revert to athena sub-process in case of file with old schema
if sc == 0:
out_pkl_fname = fp_pkl_fname
else:
sc = app.run(stdout=stdout, env=self._sub_env)
stdout.flush()
stdout.close()
import AthenaCommon.ExitCodes as ath_codes
if sc == 0:
#import shelve
import PyUtils.dbsqlite as dbsqlite
msg.info('extracting infos from [%s]...',
out_pkl_fname)
db = dbsqlite.open(out_pkl_fname)
msg.info('keys: %s',db.keys())
f.update(db['fileinfos'])
db.close()
msg.info('extracting infos from [%s]... [ok]',
out_pkl_fname)
os.remove(stdout.name)
else:
# maybe an empty file
# trust but verify
if not self._is_empty_pool_file(f_root):
# actually a problem in athena !
from textwrap import dedent
err = dedent("""
%s
problem running chappy!
code: [%s (%s)]
what: [%s]
=> corrupted input file ?
%s
logfile: [%s]
"""% (":"*25,
sc,errno.errorcode.get(sc,sc),
ath_codes.codes.get(sc,sc),
":"*25,
stdout.name
))
msg.error(err)
raise IOError(sc, err)
msg.info('athena failed to initialize.')
msg.info('=> probably an empty input POOL file')
# TAG-file
else: # bytestream
bs_fileinfos = self._process_bs_file(file_name,
evtmax=evtmax,
full_details=False)
del bs_fileinfos['file_name']
del bs_fileinfos['file_size']
del bs_fileinfos['file_type']
del bs_fileinfos['file_md5sum']
f.update(bs_fileinfos)
finally:
try:
f_raw.Close()
f_root.Close()
del f_raw
del f_root
except Exception as err:
msg.warning(
'problem while closing raw and root file handles:\n%s',
err
)
return f
def _loadFileInfo(self):
db = dbsqlite.open(self._infoOutputFile)
self._metadata = db['fileinfos']
def initialize(self):
import AthenaPython.PyAthena as PyAthena
_info = self.msg.info
_info("POOL2EI::initialize")
_info("## DoProvenanceRef: {}".format(self.DoProvenanceRef))
_info("## DoTriggerInfo: {}".format(self.DoTriggerInfo))
_info("## HaveHlt: {}".format(self.HaveHlt))
_info("## HaveXHlt: {}".format(self.HaveXHlt))
_info("## RUN1: {}".format(self.RUN1))
_info("## SendToBroker: {}".format(self.SendToBroker))
self._dsname = "Unknown.Input.Dataset.Name" # default fake value
if self.EiDsName is not None:
_info("## EiDsName: {}".format(self.EiDsName))
self._dsname = self.EiDsName
else:
# try to get dataset name from pathena INDS environment variable
import os
inds = os.getenv('INDS')
if inds is not None:
_info("## INDS: {}".format(inds))
self._dsname = inds
else:
# else, try to use job definition
try:
import newJobDef
processingType = newJobDef.job['processingType']
transformation = newJobDef.job['transformation']
dsSource = 'realDatasetsIn' # take dataset name from input
if processingType == 'merge' and transformation != 'POOLtoEI_tf.py':
dsSource = 'realDatasets' # take dataset name from output
datasets = newJobDef.job[dsSource].split(',')
_info("## {}[0]: {}".format(dsSource, datasets[0]))
self._dsname = datasets[0]
# remove _tid and _sub parts from dsname
import re
self._dsname = re.sub('_tid[0-9]{8}_[0-9]{2}', '',
self._dsname)
self._dsname = re.sub('_sub[0-9]{10}', '', self._dsname)
self._dsname = re.sub('\\/$', '', self._dsname)
except:
_info(
'## Unable to get dataset name from realDatasetsIn or realDatasets'
)
# token match regex
import re
self._re_pool_token = re.compile(r'[[]DB=(?P<db>.*?)[]]' \
r'[[]CNT=(?P<cnt>.*?)[]]' \
r'[[]CLID=(?P<clid>.*?)[]]' \
r'[[]TECH=(?P<tech>.*?)[]]' \
r'[[]OID=(?P<oid>.*?)[]]').match
# load our pythonizations:
for cls_name in ('EventStreamInfo', 'EventType', 'PyEventType'):
cls = getattr(PyAthena, cls_name)
_info("retrieving various stores...")
for store_name in ('evtStore', 'inputStore', 'detStore', 'tagStore',
'metaStore'):
_info("retrieving [{}]...".format(store_name))
o = getattr(self, store_name)
_info("retrieving [{}]... [done]".format(store_name))
_info("retrieving various stores... [done]")
if self.HaveHlt:
# load trigger decision tool
from TriggerJobOpts.TriggerFlags import TriggerFlags
TriggerFlags.configurationSourceList = ['ds']
import AthenaPython.PyAthena as PyAthena
self.trigDec = PyAthena.py_tool(
'Trig::TrigDecisionTool/TrigDecisionTool')
self.trigDec.ExperimentalAndExpertMethods().enable()
if self.HaveXHlt:
self.trigDec.setProperty("ConfigTool",
"TrigConf::xAODConfigTool")
self.trigDec.setProperty("TrigDecisionKey", "xTrigDecision")
else:
self.trigDec.setProperty("ConfigTool",
"TrigConf::AODConfigTool")
self.trigDec.setProperty("TrigDecisionKey", "TrigDecision")
## open output pkl file
import os
if self.Out is not None:
oname = self.Out
else:
oname = "output.ei.pkl"
oname = os.path.expanduser(os.path.expandvars(oname))
self._eifname = oname
_info('Opening EI file [{}]...'.format(oname))
if os.path.exists(oname):
os.remove(oname)
import PyUtils.dbsqlite as dbsqlite
try:
self._eif = dbsqlite.open(oname, flags='w')
except:
pass
if self._eif is None:
self.msg.fatal(
"Unable to open EI output file {} exapnded as {}".format(
self.Out, oname))
raise RuntimeError("Unable to open EI output file")
# get taskid and jobid
if hasattr(self, 'TaskID') and hasattr(
self, 'JobID'
) and self.TaskID is not None and self.JobID is not None:
self._eif['TaskID'] = "{}.T".format(self.TaskID)
if hasattr(self,
'AttemptNumber') and self.AttemptNumber is not None:
self._eif['JobID'] = "{}.{}".format(self.JobID,
self.AttemptNumber)
else:
self._eif['JobID'] = "{}.0".format(self.JobID)
else:
# get them from job info
try:
import newJobDef
self._eif['TaskID'] = "{}.G".format(newJobDef.job['taskID'])
self._eif['JobID'] = "{}.{}".format(newJobDef.job['PandaID'],
newJobDef.job['attemptNr'])
except:
self._eif['TaskID'] = "{}.G".format(
os.getenv('PanDA_TaskID', 0))
self._eif['JobID'] = "{}.0".format(os.getenv('PandaID', 0))
# initial information
self._eif['StartProcTime'] = int(time.time() * 1000)
self._eif['Schema'] = EIrecord().getRecSchema()
self._eif['Version'] = EIrecord().getVersion()
self._eif['InputDsName'] = self._dsname
#processing options
self._eif['ProvenanceRef'] = self.DoProvenanceRef
self._eif['TriggerInfo'] = self.DoTriggerInfo
return StatusCode.Success
val = jobproperties.PerfMonFlags.enableDoubleCountingCorrection()
if val:
handle.EnableDoubleCountingCorrection = True
else:
handle.EnableDoubleCountingCorrection = False
pmon_flags = {}
import AthenaCommon.JobProperties as acjp
for n, v in jobproperties.PerfMonFlags.__dict__.iteritems():
if isinstance(v, acjp.JobProperty):
pmon_flags[n] = v()
## create PyPerfMon.Svc object
from .PyPerfMon import Svc
svc = Svc(name=handle.getName(),
properties={
'outFileName': "%s" % handle.OutFileName,
'components': componentNames,
'ioContainers': ioContainers,
'pmon_flags': pmon_flags,
})
## store the py-perfmonsvc object into a sqlite-pickle...
import PyUtils.dbsqlite as dbs
pmon_props = dbs.open(handle.OutFileName + '.props.pkl', 'w')
pmon_props['pmon'] = svc
pmon_props.close()
return
pass # class PerfMonSvc
def setDefaults( cls, handle ):
## get a handle on the service manager
from AthenaCommon.AppMgr import ServiceMgr as svcMgr
## don't allow the service to be used in MT>1 jobs
from AthenaCommon.ConcurrencyFlags import jobproperties as jp
if jp.ConcurrencyFlags.NumThreads() > 1:
print(" >> PerfMonSvc should NOT be used in MT>1 jobs, self removing...")
if hasattr(svcMgr, 'PerfMonSvc'):
del svcMgr.PerfMonSvc
return
## import the PerfMonFlags
from PerfMonComps.PerfMonFlags import jobproperties
## continue, only if we are required to:
if not jobproperties.PerfMonFlags.doMonitoring():
return
## continue, only if it is our Configurable
if not isinstance(handle, PerfMonSvc):
return
from AthenaCommon import CfgMgr
## get a handle on the application manager
from AthenaCommon.AppMgr import theApp
if not jobproperties.PerfMonFlags.doFastMon() or jobproperties.PerfMonFlags.doSemiDetailedMonitoring():
theApp.AuditAlgorithms = True
theApp.AuditTools = True
theApp.AuditServices = True
## get output levels
import AthenaCommon.Constants as Lvl
## make sure the application manager explicitly creates the service
if hasattr(handle, "getFullJobOptName") :
handleName = handle.getFullJobOptName()
if not handleName in theApp.CreateSvc:
# we want to be the very first service to be initialized
# so we can monitor what happens during initialize
theApp.CreateSvc = [ handleName ] + theApp.CreateSvc
fname = handle.getDefaultProperty('OutFileName')
## provide a default tuple file name from the jobproperties
if not hasattr(handle, "OutFileName") and \
hasattr(handle, "getDefaultProperty") :
fname = jobproperties.PerfMonFlags.OutputFile()
if fname.endswith('.pmon.gz'):
fname = fname.replace('.pmon.gz','')
elif fname.endswith('.root'):
fname = fname.replace('.root','')
handle.OutFileName = fname
## configure the auditor service if not done already...
if not hasattr(svcMgr, 'AuditorSvc'):
from GaudiSvc.GaudiSvcConf import AuditorSvc
svcMgr += AuditorSvc()
## configure the chronostat service
if not hasattr(svcMgr, 'ChronoStatSvc'):
from GaudiSvc.GaudiSvcConf import ChronoStatSvc
svcMgr += ChronoStatSvc( OutputLevel = Lvl.WARNING,
ChronoPrintOutTable = False )
## communicate the whole list of Algorithms if no default provided
## by user: we ask for all the ConfigurableAlgorithm's instances
componentNames = []
if not hasattr(handle, "ProfiledAlgs") or \
(hasattr(handle, "ProfiledAlgs") and len(handle.ProfiledAlgs) == 0):
from AthenaCommon.Configurable import (Configurable,
ConfigurableAlgorithm,
ConfigurableAlgTool,
ConfigurableService)
for k,v in Configurable.allConfigurables.items():
if isinstance( v, ( ConfigurableAlgorithm,
ConfigurableAlgTool,
ConfigurableService ) ) \
and not k in ('PerfMonSvc',
'AthenaEventLoopMgr',):
componentNames.append (v.getJobOptName())
pass
pass
else:
componentNames = [k for k in handle.ProfiledAlgs[:]]
## try to fetch the list of containers to be written out from the
## registered OutputStreams (and if we are configured to do such a
## thing)
ioContainers = []
if (jobproperties.PerfMonFlags.doPersistencyMonitoring() and
not jobproperties.PerfMonFlags.doFastMon()) :
## make sure the AthenaPoolCnvSvc is correctly configured
try: svcMgr.AthenaPoolCnvSvc.UseDetailChronoStat = True
except AttributeError:
# ok... maybe not a 'writeAthenaPool' job...
pass
## add our ioContainers list if none has been provided
if not hasattr(handle, "IoContainers") or \
( hasattr(handle, "IoContainers") and \
len(handle.IoContainers) == 0 ):
# first let us ask the Streams...
from AthenaCommon.Configurable import Configurable
if 'Streams' in Configurable.configurables:
outStreams = Configurable.configurables['Streams']
for o in outStreams:
if not hasattr(o, 'ItemList'): continue
ioContainers += [ i for i in o.ItemList
if not i.endswith("#*") ]
pass
# then, what about objKeyStore ?
from AthenaCommon.KeyStore import CfgKeyStore
keystore = CfgKeyStore.instances.get('KeyStore')
if keystore:
ioLabels = [ "streamRDO","streamESD",
"streamAOD","streamTAG",
"inputBackNav","inputFile" ]
for l in ioLabels:
try:
ioContainers.extend(keystore[l].list())
except AttributeError:
for k,v in keystore[l].items():
ioContainers += [ "%s#%s" % (k,c) for c in v ]
pass
## collect everything
handle.IoContainers = ioContainers
pass
pass
## communicate the granularity of monitoring from the job-flags
if jobproperties.PerfMonFlags.doFullMon():
handle.MonLvl = -1
elif jobproperties.PerfMonFlags.doFastMon():
handle.MonLvl = 0
elif jobproperties.PerfMonFlags.doMonitoring():
handle.MonLvl = 1
else:
handle.MonLvl = -1
pass
if jobproperties.PerfMonFlags.doSemiDetailedMonitoring():
if jobproperties.PerfMonFlags.doSemiDetailedMonitoringFullPrint():
handle.SemiDetailedMonitoring=2
else:
handle.SemiDetailedMonitoring=1
#Collect the start-time (in jiffies) of the configuration process
try:
j=open('/proc/self/stat').readline().split()[21]
except Exception as err:
#Not worth stopping the job over missing diagnostics.
#We should notice the missing info in any case.
j=""
print("PMonSD WARNING: Could not determine job start time:",err)
pass
handle.JobStartJiffies=j
handle.ExtraPrintouts=jobproperties.PerfMonFlags.doExtraPrintouts()
## communicate the double-counting correction from the job-flags
val = jobproperties.PerfMonFlags.enableDoubleCountingCorrection()
if val:
handle.EnableDoubleCountingCorrection = True
else:
handle.EnableDoubleCountingCorrection = False
pmon_flags = {}
import AthenaCommon.JobProperties as acjp
for n,v in six.iteritems (jobproperties.PerfMonFlags.__dict__):
if isinstance(v, acjp.JobProperty):
pmon_flags[n] = v()
## create PyPerfMon.Svc object
from .PyPerfMon import Svc
svc = Svc(
name=handle.getName(),
properties = {
'outFileName' : "%s" % handle.OutFileName,
'components' : componentNames,
'ioContainers' : ioContainers,
'pmon_flags' : pmon_flags,
})
## store the py-perfmonsvc object into a sqlite-pickle...
import PyUtils.dbsqlite as dbs
pmon_props = dbs.open(handle.OutFileName+'.props.pkl', 'w')
pmon_props['pmon'] = svc
pmon_props.close()
return
def store_configuration(cfg_fname=None):
"""helper function to inspect the 'configured' JobOptionsSvc and dump
the properties of each component into a (sqlite) shelve.
it will eventually dump the properties of py-components too.
"""
jobo_cfg = defaultdict(dict)
if cfg_fname is None:
import tempfile
tmpfile = tempfile.NamedTemporaryFile(suffix='-jobo.pkl')
cfg_fname = tmpfile.name
tmpfile.close()
import os
if os.path.exists(cfg_fname):
os.remove(cfg_fname)
assert cfg_fname
from AthenaCommon.AppMgr import theApp
def _fill_cfg(client, props):
for p in props:
n = p[0]
v = p[1]
if hasattr(v, 'toStringProperty'):
v = str(v.toStringProperty().toString())
elif hasattr(v, 'toString'):
v = str(v.toString())
else:
v = str(v)
jobo_cfg[client][n] = v
from AthenaCommon.AppMgr import ServiceMgr as svcMgr
# special cases: joboptionsvc and messagesvc
def _fill_props(svcname):
if not hasattr(svcMgr, svcname):
return
svc = getattr(svcMgr, svcname)
props = []
for k,v in svc.properties().iteritems():
if v == svc.propertyNoValue:
v = svc.getDefaultProperty(k)
props.append((k,v))
_fill_cfg(svcname, props)
_fill_props('JobOptionsSvc')
_fill_props('MessageSvc')
# tickle C++ and configure the whole app
theApp.setup()
app_props = [(k,v.value())
for k,v in theApp.getHandle().properties().iteritems()]
_fill_cfg(theApp.name(), app_props)
import AthenaPython.PyAthena as PyAthena
josvc = PyAthena.py_svc('JobOptionsSvc', iface='IJobOptionsSvc')
assert josvc is not None
clients = list(josvc.getClients())
for client in clients:
props = josvc.getProperties(client)
for prop in props:
n = prop.name()
v = prop.toString()
jobo_cfg[client][n] = v
cfg = dbs.open(cfg_fname, 'w')
cfg['jobopts'] = jobo_cfg
pycomps = []
import sys
from .AppMgr import ServiceMgr as svcMgr
# all other pycomps
from .Configurable import Configurable as C
for c in C.allConfigurables.itervalues():
if not isinstance(c, (PyAthena.Alg,
PyAthena.AlgTool,
PyAthena.Svc,
PyAthena.Aud)):
continue
# FIXME: should check if the component is still 'active'
# ie: is it in one of the TopAlg,SvcMgr,ToolSvc and children ?
if hasattr(c, 'msg'):
delattr(c, 'msg')
pycomps.append(c)
pass
# folder for the pyathena-components
cfg['pycomps'] = pycomps
cfg.close()
return cfg_fname
def configure(self, joboptions=None, commands=None,
dllname=None,
factname=None,
extra_options=None):
if not (self.app is None):
self.msg.info('C++ application already configured')
return self.app
self.msg.info('configuring application...')
usr_cfg = AthCfg()
self.cfg.seek(0)
usr_cfg << self.cfg.read()
# reset
self.cfg = AthCfg()
if commands:
self.cfg << commands+'\n'
# common configuration
self.cfg << """
# basic job configuration
include('AthenaCommon/Atlas.UnixStandardJob.py')
include.block('AthenaCommon/Atlas.UnixStandardJob.py')
if not (not %(run_batch)s and
theApp.EventLoop == 'PyAthenaEventLoopMgr'):
# make SIG_INT fatal
svcMgr.CoreDumpSvc.FatalHandler = -1
""" % {'run_batch' : self.options.run_batch}
self.cfg << """
# user level configuration
try:
include('$HOME/.athenarc')
except IncludeError:
pass
"""
# another user level configuration
usr_cfg.seek(0)
self.cfg << usr_cfg.read()
if isinstance(joboptions, (list,tuple)):
for jobo_name in joboptions:
self.cfg.include(jobo_name)
if not self.options.run_batch:
self.cfg << """
theApp.EventLoop = 'PyAthenaEventLoopMgr'
svcMgr += CfgMgr.PyAthenaEventLoopMgr()
"""
self.cfg << """
### logging and messages ---------
from AthenaCommon.Logging import *
_msg = log
_msg.setLevel(getattr(logging, '%(output_level)s'))
import AthenaCommon.Constants as Lvl
theApp.setOutputLevel(%(output_level)s)
theApp.OutputLevel = Lvl.%(output_level)s
from AthenaCommon.AppMgr import ServiceMgr as svcMgr
svcMgr.MessageSvc.OutputLevel = Lvl.%(output_level)s
""" % dict(output_level=self.options.msg_lvl)
self.cfg << """
from AthenaCommon.Include import Include, IncludeError, include
include.setShowIncludes(%(showincludes)s)
if %(showincludes)s:
import AthenaCommon.Include as AthCIncMod
AthCIncMod.marker=' -#-' # distinguish bootstrap from other jo-code
""" % dict(showincludes=self.options.showincludes)
cfg_name = self.cfg._jobo.name.replace('.py','.pkl')
self.msg.info('dumping job-configuration into [%s]...',
cfg_name)
# run configuration in a forked-subprocess...
sc = _app_configure(self.cfg, cfg_name, extra_options)
if sc:
err = 'could not configure application [sc=%d]' % sc
self.msg.error(err)
raise RuntimeError(err)
self.msg.info('configuring application w/ [%s]', cfg_name)
import os
self.cfg._jobo.close()
os.remove(self.cfg._jobo.name)
import PyCintex
PyCintex.Cintex.Enable()
gbl = PyCintex.makeNamespace('')
import GaudiPython.Bindings as gaudi
# remove the gaudimodule exit handler as to prevent them from clobering
import atexit
for hdlr in reversed(atexit._exithandlers[:]):
module_name = hdlr[0].__module__
if ('GaudiPython' in module_name or
'gaudimodule' in module_name):
atexit._exithandlers.remove(hdlr)
del hdlr
# install our own exit handler (if needed)
import sys
if hasattr(sys, 'ps1'): # ie: is interactive
atexit.register(self.exit)
del atexit
from . import ResourceLimits
ResourceLimits.SetMaxLimits()
try: import cPickle as pickle
except ImportError: import pickle
import PyUtils.dbsqlite as dbs
db = dbs.open(cfg_name, 'r')
jobo_cfg = db['jobopts']
kw = jobo_cfg['ApplicationMgr']
for k in ('Go', 'Exit', 'AuditInitialize', 'AuditFinalize'):
if k in kw:
del kw[k]
outputlevel = jobo_cfg['ApplicationMgr']['OutputLevel']
self.app = gaudi.AppMgr(outputlevel=outputlevel,
selfoptions=kw,
dllname=dllname,
factname=factname)
# open the pycomps folder
pycomps = db.get('pycomps', None) # just opening it should do
if pycomps:
import AthenaPython.Configurables as _C
_C.PyComponents.instances = dict((p.name, p) for p in pycomps)
#_C.PyComponents.instances = pycomps
for p in pycomps:
if hasattr(p, 'setup'):
if callable(p.setup):
p.setup()
setattr(self, '_pycomps', pycomps)
import AthenaPython.PyAthena as PyAthena
josvc = PyAthena.py_svc('JobOptionsSvc',
createIf=False,
iface='IJobOptionsSvc')
assert josvc is not None
for client in jobo_cfg:
if client == 'ApplicationMgr':
continue
for n,v in jobo_cfg[client].iteritems():
p = gaudi.StringProperty(n, v)
if not josvc.addPropertyToCatalogue(client, p).isSuccess():
self.msg.error(
'could not add property [%s.%s = %s]',
client, n, v
)
if client in ('MessageSvc', 'JobOptionsSvc'):
svc = PyAthena.py_svc(client, iface='IProperty')
svc.setProperty(p)
db.close()
import os
if os.path.exists(cfg_name):
os.remove(cfg_name)
pass
#import AthenaCommon.Debugging as dbg
#dbg.hookDebugger()
return self.app
def main():
# get options
opt = options(sys.argv)
if opt.input is None or opt.output is None:
pass
# open tagfile
(tg, pyl, cm) = Tagfile(opt.input)
nentries = tg.GetEntries()
# open database
if os.path.exists(opt.output):
os.remove(opt.output)
eif = dbsqlite.open(opt.output, flags='w')
eif['StartProcTime'] = int(time.time() * 1000)
eif['StartProcTime_0'] = int(time.time() * 1000)
eif['Schema'] = EIrecord().getRecSchema()
eif['Version'] = EIrecord().getVersion()
eif['TaskID'] = os.getenv('TaskID', 0)
eif['JobID'] = os.getenv('JobID', 0)
#processing options
eif['ProvenanceRef'] = True
eif['TriggerInfo'] = True
## process TAG file
iov = IOV()
# get HLTPSK,L1PSK,SMK from metadata
# [188921,1] - [188921,215]
iov_interval = re.compile(r'[[](?P<r0>[0-9]+?),(?P<e0>[0-9]+?)[]] - '\
r'[[](?P<r1>[0-9]+?),(?P<e1>[0-9]+?)[]]').match
#pdb.set_trace()
Project = "-"
physicsShort = "-"
TAG = "-"
for k, v in cm.iteritems():
n = k.find('|')
if n != -1:
k2 = k[n + 1:]
pfn = k[:n + 1][4:]
else:
k2 = k
pfn = ""
#print k,v
if k2.startswith("Key@HLTPSK;") or k2.startswith(
"Key@L1PSK;") or k2.startswith("Key@SMK;"):
parts = k2.split(";")
k1 = parts[0][4:].rstrip(";")
iov1 = parts[1][5:].rstrip("}")
match = iov_interval(iov1)
if not match:
err = "BAD IOV"
raise RuntimeError(err)
d = match.groupdict()
iov.add(k1, int(v),
(int(d['r0']), int(d['e0']), int(d['r1']), int(d['e1'])))
# asume that all events in TAG file belongs to same Project,datasetNumber,physicsShort,prodStep,dataType,TAG
if k2 == 'POOLCollectionID':
try:
Project, datasetNumber, physicsShort, prodStep, dataType, TAG = pfn.split(
'.')[:6]
iund = physicsShort.find('_')
physicsShort = physicsShort[iund + 1:]
#print pfn.split('.')[:6]
except:
pass
#print iov.dump()
eif['ProjName_0'] = Project
eif['TrigStream_0'] = physicsShort
eif['AMITag_0'] = TAG
# pool token has the format:
# [DB=dbID][CNT=cntID][CLID=classID][TECH=technology][OID=oid.first-oid.second]
# eg.:
# [DB=FEAD2AD8-111F-11E2-8080-1CCD8E80BEEF][CNT=POOLContainer(DataHeader)]
# [CLID=D82968A1-CF91-4320-B2DD-E0F739CBC7E6][TECH=00000202][OID=00000123-00000008]
pool_token = re.compile(r'[[]DB=(?P<db>.*?)[]]' \
r'[[]CNT=(?P<cnt>.*?)[]]' \
r'[[]CLID=(?P<clid>.*?)[]]' \
r'[[]TECH=(?P<tech>.*?)[]]' \
r'[[]OID=(?P<oid1>.*?)-(?P<oid2>.*?)[]]' \
).match
# event loop
tnow0 = int(time.time() * 1000)
tnow = tnow0
for evt_idx in xrange(nentries):
if opt.evtmax != 0 and evt_idx >= opt.evtmax:
break
if opt.verbose > 0 and evt_idx % 1000 == 0:
tnow1 = int(time.time() * 1000)
sys.stderr.write("{:8d} {:5d} msecs\n".format(
evt_idx, tnow1 - tnow))
tnow = tnow1
tg.GetEntry(evt_idx)
eirec = EIrecord()
run_number = int(pyl.RunNumber.GetCnvValue())
event_number = int(pyl.EventNumber.GetCnvValue())
lumi_block = pyl.LumiBlockN.GetCnvValue()
#print "run, evt", run_number,event_number
eirec['RunNumber'] = run_number
eirec['EventNumber'] = event_number
eirec['LumiBlockN'] = lumi_block
eirec["BunchId"] = pyl.BunchId.GetCnvValue()
eirec['EventTime'] = pyl.EventTime.GetCnvValue()
eirec['EventTimeNanoSec'] = pyl.EventTimeNanoSec.GetCnvValue()
eirec['EventWeight'] = pyl.EventWeight.GetCnvValue()
eirec['McChannelNumber'] = 0
eirec['Lvl1ID'] = 0
eirec['IsSimulation'] = int(pyl.IsSimulation.GetValue())
eirec['IsTestBeam'] = int(pyl.IsTestBeam.GetValue())
eirec['IsCalibration'] = int(pyl.IsCalibration.GetValue())
SMK = iov.get('SMK', (run_number, event_number))
L1PSK = iov.get('L1PSK', (run_number, lumi_block))
HLTPSK = iov.get('HLTPSK', (run_number, lumi_block))
#print '## SMK ', SMK
#print '## L1PSK ', L1PSK
#print '## HLTPSK ', HLTPSK
eirec['SMK'] = SMK
eirec['L1PSK'] = L1PSK
eirec['HLTPSK'] = HLTPSK
try:
eirec['Sref0'] = pyl.StreamAOD_ref.GetCnvValue().rstrip('\0')
eirec['Snam0'] = "StreamAOD"
except:
pass
try:
eirec['Sref0'] = pyl.Token.GetCnvValue().rstrip('\0')
eirec['Snam0'] = "StreamAOD"
except:
pass
try:
eirec['Sref1'] = pyl.StreamRAW_ref.GetCnvValue().rstrip('\0')
eirec['Snam1'] = "StreamRAW"
except:
pass
try:
eirec['Sref2'] = pyl.StreamESD_ref.GetCnvValue().rstrip('\0')
eirec['Snam2'] = "StreamESD"
except:
pass
trigL1 = ""
trigL2 = ""
trigEF = ""
for k in range(0, 8):
v = getattr(pyl, 'L1PassedTrigMaskTBP{:d}'.format(k)).GetCnvValue()
trigL1 += "{0:032b}".format(v)[::-1]
for k in range(0, 8):
v = getattr(pyl, 'L1PassedTrigMaskTAP{:d}'.format(k)).GetCnvValue()
trigL1 += "{0:032b}".format(v)[::-1]
for k in range(0, 8):
v = getattr(pyl, 'L1PassedTrigMaskTAV{:d}'.format(k)).GetCnvValue()
trigL1 += "{0:032b}".format(v)[::-1]
for k in range(0, 32):
v = getattr(pyl, 'L2PassedTrigMask{:d}'.format(k)).GetCnvValue()
trigL2 += "{0:032b}".format(v)[::-1]
for k in range(0, 32):
v = getattr(pyl, 'EFPassedTrigMask{:d}'.format(k)).GetCnvValue()
vs = "{0:064b}".format(v)
ef_phys = vs[:32]
ef_inclusive = vs[32:]
trigEF += ef_inclusive[::-1]
trigL1 = compressB64(trigL1)
trigL2 = compressB64(trigL2)
trigEF = compressB64(trigEF)
eirec['L1PassedTrigMask'] = trigL1
eirec['L2PassedTrigMask'] = trigL2
eirec['EFPassedTrigMask'] = trigEF
# write to db
eif['Entry_{:d}'.format(evt_idx)] = eirec.getRec()
eif['Nentries_0'] = evt_idx
eif['EndProcTime_0'] = int(time.time() * 1000)
eif['Nentries'] = evt_idx
eif['Nfiles'] = 1
eif['EndProcTime'] = int(time.time() * 1000)
eif.close()
tnow = int(time.time() * 1000)
sys.stderr.write("Processed {:d} events in {:f} seconds\n".format(
evt_idx, (tnow - tnow0) / 1000.))
