def cnv(self): data = {} import tempfile, os, glob origdir = os.getcwd() tmpdir = self._tmpdir try: os.chdir(tmpdir) import tarfile tar = tarfile.open(os.path.abspath(self._fileName), 'r') tar.extractall(path=tmpdir) tar.close() fname = glob.glob("*.dat")[0] if open(fname, 'r').read(1024).startswith('SQLite format'): import PyUtils.dbsqlite as dbs db = dbs.open(fname, 'r') else: db = shelve.open(fname) data['meta'] = {} for k in db.iterkeys(): data['meta'][k] = db[k] db.close() ## print "version:",data['meta']['version_id'] from PyRootLib import importRoot ROOT = importRoot() root = ROOT.fopen(glob.glob("*.root")[0], "read") for k in ('ini', 'evt', 'fin'): data[k] = root.Get("perfmon/%s" % k) data['meta']['rootFile'] = root finally: os.chdir(origdir) return data
def __write_out_pmon_data(self, outFiles): headerFile = outFiles[0] _msg = self.msg db_infos = ('err',) # lshosts_infos() if db_infos[0] != 'ok': host_infos = { 'si2k' : 0., 'cpuf' : 0. } else: host_infos = db_infos[1] host_infos['si2k'] = float(host_infos['cpuf'])*1000. #stats = self.stats['evt'] _msg.info("Si2k: %r => <cpu time>: %10.3f kSi2k", host_infos['si2k'], host_infos['si2k']*self._evt_cpu_mean/1000.) self.meta['hosts_infos'] = host_infos self.meta['pmon_version']= __version__ self.meta['algs_names'] = getattr(self, '_algs_names', ['<N/A>']) _msg.info("Writing out collected data... [%s]", self.outFileName) ## write out meta-data import PyUtils.dbsqlite as dbs meta = dbs.open(headerFile, 'n') for k,v in six.iteritems (self.meta): meta[k] = v meta['version_id'] = '0.4.0' # stream-format + header file meta['pmon_tuple_files'] = map( os.path.basename, outFiles[1:] ) import socket try: meta['hostname'] = socket.gethostname() except Exception: meta['hostname'] = '<unknown>' meta.close() cwd = os.getcwd() ## compress/package everything import tarfile tar = tarfile.open(self.outFileName, 'w:gz') try: for outFile in outFiles: outFileDirName = os.path.dirname(outFile) try: os.chdir(outFileDirName) except OSError as err: pass outFile = os.path.basename(outFile) _msg.info(' --> [%s] => %8.3f kb', outFile, os.stat(outFile).st_size/Units.kB) tar.add(outFile) os.remove(outFile) os.chdir(cwd) pass except Exception as err: _msg.warning(err) pass os.chdir(cwd) tar.close() _msg.info ("Writing out collected data... [%s] => %8.3f kb", self.outFileName, os.stat(self.outFileName).st_size / Units.kB) return
def _load_db_cache(self, fname): """load file informations from a sqlite file""" import PyUtils.dbsqlite as dbsqlite cache = dbsqlite.open(fname) d = {} for k,v in six.iteritems (cache): d[k] = AthFile.from_infos(v) return d
def retrieve_pmon_svc(fname): import PyUtils.dbsqlite as dbs db = dbs.open(fname, 'r') svc = db['pmon'] db.close() import os os.remove(fname) return svc
def main(): # logger global log log = logging.getLogger('RAW2EI.py') # get options opt = options(sys.argv) if opt.verbose > 0: log.setLevel(logging.INFO) if opt.debug > 0: log.setLevel(logging.DEBUG) # open EI file if os.path.exists(opt.output): os.remove(opt.output) eif = dbsqlite.open(opt.output, flags='w') eif['StartProcTime'] = int(time.time() * 1000) eif['Schema'] = EIrecord().getRecSchema() eif['Version'] = EIrecord().getVersion() eif['TaskID'] = os.getenv('TaskID', 0) eif['JobID'] = os.getenv('JobID', 0) if opt.eidsname is not None: dsname = opt.eidsname else: dsname = os.getenv('INDS', "Unknown.Input.Dataset.Name") dsname = re.sub('_tid[0-9]{8}_[0-9]{2}', '', dsname) dsname = re.sub('_sub[0-9]{10}', '', dsname) dsname = re.sub('\/$', '', dsname) eif['InputDsName'] = dsname #processing options eif['ProvenanceRef'] = False eif['TriggerInfo'] = True ## process RAW files if type(opt.input) is str: filelist = [opt.input] else: filelist = opt.input nfile = 0 nevents = 0 for f in filelist: nevents += processRAW(f, eif, nfile, nevents, opt.evtmax) nfile += 1 if opt.evtmax > 0 and nevents >= opt.evtmax: break eif['Nentries'] = nevents eif['Nfiles'] = nfile eif['EndProcTime'] = int(time.time() * 1000) eif.close()
def _save_db_cache(self, fname): """save file informations using sqlite""" import PyUtils.dbsqlite as dbsqlite db = dbsqlite.open(fname,flags='w') cache = self._cache for k in cache: db[k] = cache[k].fileinfos db.close() return
def __init__(self, fname): try: self._db = dbsqlite.open(fname, flags='r') except: log.error("Unable to open Event Index file {}".format(fname)) raise try: self._nentries = self._db['Nentries'] except: log.error("Unable to get nentries from Event Index file {}".format( fname)) raise
def finalize(self): _info = self.msg.info peeked_data = dict(self.peeked_data) if self.outfname: oname = self.outfname import os oname = os.path.expanduser(os.path.expandvars(oname)) _info('storing peeked file infos into [%s]...', oname) if os.path.exists(oname): os.remove(oname) try: import cPickle as pickle except ImportError: import pickle import PyUtils.dbsqlite as dbsqlite db = dbsqlite.open(oname, flags='w') # merge and collect data from all processed events (if any) if self._peeked_events: peeked_data = self._peeked_events.pop(0) for d in self._peeked_events: for k in ( 'run_number', 'evt_number', 'lumi_block', 'run_type', 'beam_type', 'beam_energy', 'stream_tags', ): v = d[k] if isinstance(v, list) and len(v) > 0: v = v[0] peeked_data[k].append(v) #_info('peeked_data:') #_info(str(peeked_data)) db['fileinfos'] = peeked_data db.close() _info('storing peeked file infos into [%s]... [done]', oname) self.print_summary(peeked_data) return StatusCode.Success
def finalize(self): _info = self.msg.info peeked_data = dict(self.peeked_data) if self.outfname: oname = self.outfname import os oname = os.path.expanduser(os.path.expandvars(oname)) _info('storing peeked file infos into [%s]...', oname) if os.path.exists(oname): os.remove(oname) try: import cPickle as pickle except ImportError: import pickle import PyUtils.dbsqlite as dbsqlite db = dbsqlite.open(oname,flags='w') # merge and collect data from all processed events (if any) if self._peeked_events: peeked_data = self._peeked_events.pop(0) for d in self._peeked_events: for k in ('run_number', 'evt_number', 'lumi_block', 'run_type', 'beam_type', 'beam_energy', 'stream_tags',): v = d[k] if isinstance(v, list) and len(v)>0: v = v[0] peeked_data[k].append(v) #_info('peeked_data:') #_info(str(peeked_data)) db['fileinfos'] = peeked_data db.close() _info('storing peeked file infos into [%s]... [done]', oname) self.print_summary(peeked_data) return StatusCode.Success
def dict_from_shelve(fname): db = {} if open(fname, 'rb').read(1024).startswith(b'SQLite format'): import PyUtils.dbsqlite as dbs db = dbs.open(fname, 'r') else: import shelve try: db = shelve.open(fname) except Exception: pass pass out = dict(db) if not out: return None if only_summary: if 'perf_summary' in out.keys(): return {'perf_summary': out['perf_summary']} else: return None return out
def _save_db_cache(self): """save file informations using sqlite""" import tempfile fd_pkl, out_pkl_fname = tempfile.mkstemp(suffix='.pkl') # import os os.close(fd_pkl) if os.path.exists(out_pkl_fname): os.remove(out_pkl_fname) oname = os.path.expanduser(os.path.expandvars(out_pkl_fname)) if os.path.exists(oname): os.remove(oname) import PyUtils.dbsqlite as dbsqlite db = dbsqlite.open(oname, flags='w') db['fileinfos'] = self.peeked_data db.close() if os.path.exists(oname) and len(self.peeked_data) > 0: return 0, out_pkl_fname else: os.remove(oname) return 1, out_pkl_fname
def configure(self, joboptions=None, commands=None, dllname=None, factname=None, extra_options=None): if not (self.app is None): self.msg.info('C++ application already configured') return self.app self.msg.info('configuring application...') usr_cfg = AthCfg() self.cfg.seek(0) usr_cfg << self.cfg.read() # reset self.cfg = AthCfg() if commands: self.cfg << commands + '\n' # common configuration self.cfg << """ # basic job configuration include('AthenaCommon/Atlas.UnixStandardJob.py') include.block('AthenaCommon/Atlas.UnixStandardJob.py') if not (not %(run_batch)s and theApp.EventLoop == 'PyAthenaEventLoopMgr'): # make SIG_INT fatal svcMgr.CoreDumpSvc.FatalHandler = -1 """ % { 'run_batch': self.options.run_batch } self.cfg << """ # user level configuration try: include('$HOME/.athenarc') except IncludeError: pass """ # another user level configuration usr_cfg.seek(0) self.cfg << usr_cfg.read() if isinstance(joboptions, (list, tuple)): for jobo_name in joboptions: self.cfg.include(jobo_name) if not self.options.run_batch: self.cfg << """ theApp.EventLoop = 'PyAthenaEventLoopMgr' svcMgr += CfgMgr.PyAthenaEventLoopMgr() """ self.cfg << """ ### logging and messages --------- from AthenaCommon.Logging import * _msg = log _msg.setLevel(getattr(logging, '%(output_level)s')) import AthenaCommon.Constants as Lvl theApp.setOutputLevel(%(output_level)s) theApp.OutputLevel = Lvl.%(output_level)s from AthenaCommon.AppMgr import ServiceMgr as svcMgr svcMgr.MessageSvc.OutputLevel = Lvl.%(output_level)s """ % dict(output_level=self.options.msg_lvl) self.cfg << """ from AthenaCommon.Include import Include, IncludeError, include include.setShowIncludes(%(showincludes)s) if %(showincludes)s: import AthenaCommon.Include as AthCIncMod AthCIncMod.marker=' -#-' # distinguish bootstrap from other jo-code """ % dict(showincludes=self.options.showincludes) cfg_name = self.cfg._jobo.name.replace('.py', '.pkl') self.msg.info('dumping job-configuration into [%s]...', cfg_name) # run configuration in a forked-subprocess... sc = _app_configure(self.cfg, cfg_name, extra_options) if sc: err = 'could not configure application [sc=%d]' % sc self.msg.error(err) raise RuntimeError(err) self.msg.info('configuring application w/ [%s]', cfg_name) import os self.cfg._jobo.close() os.remove(self.cfg._jobo.name) import PyCintex PyCintex.Cintex.Enable() gbl = PyCintex.makeNamespace('') import GaudiPython.Bindings as gaudi # remove the gaudimodule exit handler as to prevent them from clobering import atexit for hdlr in reversed(atexit._exithandlers[:]): module_name = hdlr[0].__module__ if ('GaudiPython' in module_name or 'gaudimodule' in module_name): atexit._exithandlers.remove(hdlr) del hdlr # install our own exit handler (if needed) import sys if hasattr(sys, 'ps1'): # ie: is interactive atexit.register(self.exit) del atexit from . import ResourceLimits ResourceLimits.SetMaxLimits() try: import cPickle as pickle except ImportError: import pickle import PyUtils.dbsqlite as dbs db = dbs.open(cfg_name, 'r') jobo_cfg = db['jobopts'] kw = jobo_cfg['ApplicationMgr'] for k in ('Go', 'Exit', 'AuditInitialize', 'AuditFinalize'): if k in kw: del kw[k] outputlevel = jobo_cfg['ApplicationMgr']['OutputLevel'] self.app = gaudi.AppMgr(outputlevel=outputlevel, selfoptions=kw, dllname=dllname, factname=factname) # open the pycomps folder pycomps = db.get('pycomps', None) # just opening it should do if pycomps: import AthenaPython.Configurables as _C _C.PyComponents.instances = dict((p.name, p) for p in pycomps) #_C.PyComponents.instances = pycomps for p in pycomps: if hasattr(p, 'setup'): if callable(p.setup): p.setup() setattr(self, '_pycomps', pycomps) import AthenaPython.PyAthena as PyAthena josvc = PyAthena.py_svc('JobOptionsSvc', createIf=False, iface='IJobOptionsSvc') assert josvc is not None for client in jobo_cfg: if client == 'ApplicationMgr': continue for n, v in jobo_cfg[client].iteritems(): p = gaudi.StringProperty(n, v) if not josvc.addPropertyToCatalogue(client, p).isSuccess(): self.msg.error('could not add property [%s.%s = %s]', client, n, v) if client in ('MessageSvc', 'JobOptionsSvc'): svc = PyAthena.py_svc(client, iface='IProperty') svc.setProperty(p) db.close() import os if os.path.exists(cfg_name): os.remove(cfg_name) pass #import AthenaCommon.Debugging as dbg #dbg.hookDebugger() return self.app
def store_configuration(cfg_fname=None): """helper function to inspect the 'configured' JobOptionsSvc and dump the properties of each component into a (sqlite) shelve. it will eventually dump the properties of py-components too. """ jobo_cfg = defaultdict(dict) if cfg_fname is None: import tempfile tmpfile = tempfile.NamedTemporaryFile(suffix='-jobo.pkl') cfg_fname = tmpfile.name tmpfile.close() import os if os.path.exists(cfg_fname): os.remove(cfg_fname) assert cfg_fname from AthenaCommon.AppMgr import theApp def _fill_cfg(client, props): for p in props: n = p[0] v = p[1] if hasattr(v, 'toStringProperty'): v = str(v.toStringProperty().toString()) elif hasattr(v, 'toString'): v = str(v.toString()) else: v = str(v) jobo_cfg[client][n] = v from AthenaCommon.AppMgr import ServiceMgr as svcMgr # special cases: joboptionsvc and messagesvc def _fill_props(svcname): if not hasattr(svcMgr, svcname): return svc = getattr(svcMgr, svcname) props = [] for k, v in svc.properties().iteritems(): if v == svc.propertyNoValue: v = svc.getDefaultProperty(k) props.append((k, v)) _fill_cfg(svcname, props) _fill_props('JobOptionsSvc') _fill_props('MessageSvc') # tickle C++ and configure the whole app theApp.setup() app_props = [(k, v.value()) for k, v in theApp.getHandle().properties().iteritems()] _fill_cfg(theApp.name(), app_props) import AthenaPython.PyAthena as PyAthena josvc = PyAthena.py_svc('JobOptionsSvc', iface='IJobOptionsSvc') assert josvc is not None clients = list(josvc.getClients()) for client in clients: props = josvc.getProperties(client) for prop in props: n = prop.name() v = prop.toString() jobo_cfg[client][n] = v cfg = dbs.open(cfg_fname, 'w') cfg['jobopts'] = jobo_cfg pycomps = [] import sys from .AppMgr import ServiceMgr as svcMgr # all other pycomps from .Configurable import Configurable as C for c in C.allConfigurables.itervalues(): if not isinstance( c, (PyAthena.Alg, PyAthena.AlgTool, PyAthena.Svc, PyAthena.Aud)): continue # FIXME: should check if the component is still 'active' # ie: is it in one of the TopAlg,SvcMgr,ToolSvc and children ? if hasattr(c, 'msg'): delattr(c, 'msg') pycomps.append(c) pass # folder for the pyathena-components cfg['pycomps'] = pycomps cfg.close() return cfg_fname
def main(): # analyze options opt = options(sys.argv) fname = opt.eifile try: db = dbsqlite.open(fname, flags='r') except: print >> sys.stderr, "Unable to open peeker file {}".format(fname) sys.exit(1) try: nentries = db['Nentries'] except: print >> sys.stderr, "Unable to get nentries from EI file {}".format( fname) sys.exit(1) #print "Stored keys: " #print " ",db.keys() print "" print " version: ", db['Version'] print " schema: ", db['Schema'] print " #input files: ", db['Nfiles'] print " number of events: ", nentries print " StartProcTime: ", db['StartProcTime'] print " EndProcTime: ", db['EndProcTime'] print " TaskID: ", db['TaskID'] print " JobID: ", db['JobID'] print " InputDsName: ", db['InputDsName'] print " Includes Provenance: ", db['ProvenanceRef'] print " Includes Trigger: ", db['TriggerInfo'] triggerinfo = db['TriggerInfo'] provenanceinfo = db['ProvenanceRef'] for i in xrange(db['Nfiles']): print "" print "File {:d}".format(i) if 'Nentries_{:d}'.format(i) in db: print " Events in this file: {}".format( db['Nentries_{:d}'.format(i)]) print " StartProcTime: ", db['StartProcTime_{:d}'.format(i)] if 'EndProcTime_{:d}'.format(i) in db: print " EndProcTime: ", db['EndProcTime_{:d}'.format(i)] else: print " EndProcTime: ", db['EndProcTime'] if "AMITag_{:d}".format(i) in db: print " AMITag: ", db['AMITag_{:d}'.format(i)] if "TrigStream_{:d}".format(i) in db: print " TrigStream: ", db['TrigStream_{:d}'.format(i)] if "ProjName_{:d}".format(i) in db: print " ProjName: ", db['ProjName_{:d}'.format(i)] if "GUID_{:d}".format(i) in db: print " GUID: ", db['GUID_{:d}'.format(i)] print "" print "Summary: (info for {:d} events max)".format(opt.evtmax) print "" schema = db['Schema'] evtnumber_idx = schema.index('EventNumber') runnumber_idx = schema.index('RunNumber') nevt_shown = 0 if opt.evtnumber == 0: sw0 = True else: sw0 = False for i in xrange(nentries): evt = db['Entry_{:d}'.format(i)] data = zip(schema, evt) if opt.evtnumber == evt[evtnumber_idx]: if opt.runnumber == 0 or opt.runnumber == evt[runnumber_idx]: sw0 = True if not sw0: continue if nevt_shown >= opt.evtmax: break nevt_shown += 1 trigL1 = None trigL2 = None trigEF = None print "{:d} ============================================================".format( nevt_shown) for k, v in data: print " {:<20s}: {}".format(k, v) if k in "L1PassedTrigMask','L2PassedTrigMask','EFPassedTrigMask": if triggerinfo: v = v.split(";")[0] ## trigger if v is not None and v != '': dv = decompressB64(v) if opt.decodetrig1 > 0: print " {:<20s}: {}".format(k, dv) if k == "L1PassedTrigMask": trigL1_U = v trigL1 = dv if k == "L2PassedTrigMask": trigL2_U = v trigL2 = dv if k == "EFPassedTrigMask": trigEF_U = v trigEF = dv #trigger bits fired if trigL1 is not None: if opt.decodetrig2 > 0: l1tl = getL1TriggerList(trigL1_U) print " L1PassedTrigMaskTBP: ", l1tl['TBP'] print " L1PassedTrigMaskTAP: ", l1tl['TAP'] print " L1PassedTrigMaskTAV: ", l1tl['TAV'] for i in xrange(8): tw = trigL1[32 * i:32 * (i + 1)][::-1] v = int(tw, 2) if v != 0 and opt.decodetrig0 > 0: print " L1PassedTrigMaskTBP{:d} = {:d} (0x{:08x})".format( i, v, v) for i in xrange(8, 16): tw = trigL1[32 * i:32 * (i + 1)][::-1] v = int(tw, 2) if v != 0 and opt.decodetrig0 > 0: print " L1PassedTrigMaskTAP{:d} = {:d} (0x{:08x})".format( i - 8, v, v) for i in xrange(16, 24): tw = trigL1[32 * i:32 * (i + 1)][::-1] v = int(tw, 2) if v != 0 and opt.decodetrig0 > 0: print " L1PassedTrigMaskTAV{:d} = {:d} (0x{:08x})".format( i - 16, v, v) if trigL2 is not None: if opt.decodetrig2 > 0: print " L2PassedTrigMask: ", getTriggerList(trigL2_U) for i in xrange(len(trigL2) / 32): tw = trigL2[32 * i:32 * (i + 1)][::-1] v = int(tw, 2) if v != 0 and opt.decodetrig0 > 0: print " L2PassedTrigMask{:d} = {:d} (0x{:08x})".format( i, v, v) if trigEF is not None: if opt.decodetrig2 > 0: print " EFPassedTrigMask: ", getTriggerList(trigEF_U) for i in xrange(len(trigEF) / 32): tw = trigEF[32 * i:32 * (i + 1)][::-1] v = int(tw, 2) if v != 0 and opt.decodetrig0 > 0: print " EFPassedTrigMask{:d} = {:d} (0x{:08x}) ".format( i, v, v) db.close()
def _process_call(self, fname, evtmax, projects=['AtlasCore']): msg = self.msg() f = _create_file_infos() protocol, _ = self.server.fname(fname) f_raw = self._root_open(fname, raw=True) if f_raw is None or not f_raw: raise IOError( errno.ENOENT, 'No such file or directory', fname) f_root = f_raw try: file_type, file_name = self.server.ftype(f_raw) protocol,file_name = self.server.fname(fname) f['file_md5sum'] = self.server.md5sum(f_raw) f['file_name'] = file_name f['file_type'] = file_type f['file_size'] = f_raw.GetSize() if file_type == 'pool': f_root = self._root_open(fname, raw=False) # POOL files are most nutritious when known to PoolFileCatalog.xml # FIXME: best would be to do that in athfile_peeker.py but # athena.py closes sys.stdin when in batch, which confuses # PyUtils:subprocess.getstatusoutput # # ATEAM-192: avoid the PoolFileCatalog.xml conflict #cmd = ['pool_insertFileToCatalog.py', # file_name,] #subprocess.call(cmd, env=self._sub_env) # if True: is_tag, tag_ref, tag_guid, nentries, runs, evts = self._is_tag_file(f_root, evtmax) if is_tag: f['stream_names'] = ['TAG'] f['file_guid'] = tag_guid f['nentries'] = nentries f['run_number'] = runs f['evt_number'] = evts else: import tempfile fd_pkl,out_pkl_fname = tempfile.mkstemp(suffix='.pkl') os.close(fd_pkl) if os.path.exists(out_pkl_fname): os.remove(out_pkl_fname) print ("\n --------- running Athena peeker") print (os.environ.get('CMTPATH','')) import AthenaCommon.ChapPy as api app = api.AthenaApp(cmdlineargs=["--nprocs=0"]) app << """ FNAME = %s """ % str([file_name]) app << """ import os # prevent from running athena in interactive mode (and freeze) if 'PYTHONINSPECT' in os.environ: del os.environ['PYTHONINSPECT'] include('AthenaPython/athfile_peeker.py') from AthenaCommon.AlgSequence import AlgSequence job = AlgSequence() # we don't really need this... job.peeker.outfname='%(outfname)s' job.peeker.infname='%(infname)s' # metadata + taginfo import IOVDbSvc.IOVDb # evt-max theApp.EvtMax = %(evtmax)i """ % { 'infname' : file_name, 'outfname': out_pkl_fname, 'evtmax': evtmax, } import uuid stdout_fname = ( 'athfile-%i-%s.log.txt' % (os.getpid(), uuid.uuid4()) ) stdout = open(stdout_fname, "w") print ("="*80, file=stdout) print (self._sub_env, file=stdout) print ("="*80, file=stdout) stdout.flush() if DEFAULT_AF_RUN: sc = app.run(stdout=stdout, env=self._sub_env) else: import PyUtils.FilePeekerTool as fpt fp = fpt.FilePeekerTool(f_root) sc, fp_pkl_fname = fp.run() # revert to athena sub-process in case of file with old schema if sc == 0: out_pkl_fname = fp_pkl_fname else: sc = app.run(stdout=stdout, env=self._sub_env) stdout.flush() stdout.close() import AthenaCommon.ExitCodes as ath_codes if sc == 0: #import shelve import PyUtils.dbsqlite as dbsqlite msg.info('extracting infos from [%s]...', out_pkl_fname) db = dbsqlite.open(out_pkl_fname) msg.info('keys: %s',db.keys()) f.update(db['fileinfos']) db.close() msg.info('extracting infos from [%s]... [ok]', out_pkl_fname) os.remove(stdout.name) else: # maybe an empty file # trust but verify if not self._is_empty_pool_file(f_root): # actually a problem in athena ! from textwrap import dedent err = dedent(""" %s problem running chappy! code: [%s (%s)] what: [%s] => corrupted input file ? %s logfile: [%s] """% (":"*25, sc,errno.errorcode.get(sc,sc), ath_codes.codes.get(sc,sc), ":"*25, stdout.name )) msg.error(err) raise IOError(sc, err) msg.info('athena failed to initialize.') msg.info('=> probably an empty input POOL file') # TAG-file else: # bytestream bs_fileinfos = self._process_bs_file(file_name, evtmax=evtmax, full_details=False) del bs_fileinfos['file_name'] del bs_fileinfos['file_size'] del bs_fileinfos['file_type'] del bs_fileinfos['file_md5sum'] f.update(bs_fileinfos) finally: try: f_raw.Close() f_root.Close() del f_raw del f_root except Exception as err: msg.warning( 'problem while closing raw and root file handles:\n%s', err ) return f
def _loadFileInfo(self): db = dbsqlite.open(self._infoOutputFile) self._metadata = db['fileinfos']
def initialize(self): import AthenaPython.PyAthena as PyAthena _info = self.msg.info _info("POOL2EI::initialize") _info("## DoProvenanceRef: {}".format(self.DoProvenanceRef)) _info("## DoTriggerInfo: {}".format(self.DoTriggerInfo)) _info("## HaveHlt: {}".format(self.HaveHlt)) _info("## HaveXHlt: {}".format(self.HaveXHlt)) _info("## RUN1: {}".format(self.RUN1)) _info("## SendToBroker: {}".format(self.SendToBroker)) self._dsname = "Unknown.Input.Dataset.Name" # default fake value if self.EiDsName is not None: _info("## EiDsName: {}".format(self.EiDsName)) self._dsname = self.EiDsName else: # try to get dataset name from pathena INDS environment variable import os inds = os.getenv('INDS') if inds is not None: _info("## INDS: {}".format(inds)) self._dsname = inds else: # else, try to use job definition try: import newJobDef processingType = newJobDef.job['processingType'] transformation = newJobDef.job['transformation'] dsSource = 'realDatasetsIn' # take dataset name from input if processingType == 'merge' and transformation != 'POOLtoEI_tf.py': dsSource = 'realDatasets' # take dataset name from output datasets = newJobDef.job[dsSource].split(',') _info("## {}[0]: {}".format(dsSource, datasets[0])) self._dsname = datasets[0] # remove _tid and _sub parts from dsname import re self._dsname = re.sub('_tid[0-9]{8}_[0-9]{2}', '', self._dsname) self._dsname = re.sub('_sub[0-9]{10}', '', self._dsname) self._dsname = re.sub('\\/$', '', self._dsname) except: _info( '## Unable to get dataset name from realDatasetsIn or realDatasets' ) # token match regex import re self._re_pool_token = re.compile(r'[[]DB=(?P<db>.*?)[]]' \ r'[[]CNT=(?P<cnt>.*?)[]]' \ r'[[]CLID=(?P<clid>.*?)[]]' \ r'[[]TECH=(?P<tech>.*?)[]]' \ r'[[]OID=(?P<oid>.*?)[]]').match # load our pythonizations: for cls_name in ('EventStreamInfo', 'EventType', 'PyEventType'): cls = getattr(PyAthena, cls_name) _info("retrieving various stores...") for store_name in ('evtStore', 'inputStore', 'detStore', 'tagStore', 'metaStore'): _info("retrieving [{}]...".format(store_name)) o = getattr(self, store_name) _info("retrieving [{}]... [done]".format(store_name)) _info("retrieving various stores... [done]") if self.HaveHlt: # load trigger decision tool from TriggerJobOpts.TriggerFlags import TriggerFlags TriggerFlags.configurationSourceList = ['ds'] import AthenaPython.PyAthena as PyAthena self.trigDec = PyAthena.py_tool( 'Trig::TrigDecisionTool/TrigDecisionTool') self.trigDec.ExperimentalAndExpertMethods().enable() if self.HaveXHlt: self.trigDec.setProperty("ConfigTool", "TrigConf::xAODConfigTool") self.trigDec.setProperty("TrigDecisionKey", "xTrigDecision") else: self.trigDec.setProperty("ConfigTool", "TrigConf::AODConfigTool") self.trigDec.setProperty("TrigDecisionKey", "TrigDecision") ## open output pkl file import os if self.Out is not None: oname = self.Out else: oname = "output.ei.pkl" oname = os.path.expanduser(os.path.expandvars(oname)) self._eifname = oname _info('Opening EI file [{}]...'.format(oname)) if os.path.exists(oname): os.remove(oname) import PyUtils.dbsqlite as dbsqlite try: self._eif = dbsqlite.open(oname, flags='w') except: pass if self._eif is None: self.msg.fatal( "Unable to open EI output file {} exapnded as {}".format( self.Out, oname)) raise RuntimeError("Unable to open EI output file") # get taskid and jobid if hasattr(self, 'TaskID') and hasattr( self, 'JobID' ) and self.TaskID is not None and self.JobID is not None: self._eif['TaskID'] = "{}.T".format(self.TaskID) if hasattr(self, 'AttemptNumber') and self.AttemptNumber is not None: self._eif['JobID'] = "{}.{}".format(self.JobID, self.AttemptNumber) else: self._eif['JobID'] = "{}.0".format(self.JobID) else: # get them from job info try: import newJobDef self._eif['TaskID'] = "{}.G".format(newJobDef.job['taskID']) self._eif['JobID'] = "{}.{}".format(newJobDef.job['PandaID'], newJobDef.job['attemptNr']) except: self._eif['TaskID'] = "{}.G".format( os.getenv('PanDA_TaskID', 0)) self._eif['JobID'] = "{}.0".format(os.getenv('PandaID', 0)) # initial information self._eif['StartProcTime'] = int(time.time() * 1000) self._eif['Schema'] = EIrecord().getRecSchema() self._eif['Version'] = EIrecord().getVersion() self._eif['InputDsName'] = self._dsname #processing options self._eif['ProvenanceRef'] = self.DoProvenanceRef self._eif['TriggerInfo'] = self.DoTriggerInfo return StatusCode.Success
val = jobproperties.PerfMonFlags.enableDoubleCountingCorrection() if val: handle.EnableDoubleCountingCorrection = True else: handle.EnableDoubleCountingCorrection = False pmon_flags = {} import AthenaCommon.JobProperties as acjp for n, v in jobproperties.PerfMonFlags.__dict__.iteritems(): if isinstance(v, acjp.JobProperty): pmon_flags[n] = v() ## create PyPerfMon.Svc object from .PyPerfMon import Svc svc = Svc(name=handle.getName(), properties={ 'outFileName': "%s" % handle.OutFileName, 'components': componentNames, 'ioContainers': ioContainers, 'pmon_flags': pmon_flags, }) ## store the py-perfmonsvc object into a sqlite-pickle... import PyUtils.dbsqlite as dbs pmon_props = dbs.open(handle.OutFileName + '.props.pkl', 'w') pmon_props['pmon'] = svc pmon_props.close() return pass # class PerfMonSvc
def setDefaults( cls, handle ): ## get a handle on the service manager from AthenaCommon.AppMgr import ServiceMgr as svcMgr ## don't allow the service to be used in MT>1 jobs from AthenaCommon.ConcurrencyFlags import jobproperties as jp if jp.ConcurrencyFlags.NumThreads() > 1: print(" >> PerfMonSvc should NOT be used in MT>1 jobs, self removing...") if hasattr(svcMgr, 'PerfMonSvc'): del svcMgr.PerfMonSvc return ## import the PerfMonFlags from PerfMonComps.PerfMonFlags import jobproperties ## continue, only if we are required to: if not jobproperties.PerfMonFlags.doMonitoring(): return ## continue, only if it is our Configurable if not isinstance(handle, PerfMonSvc): return from AthenaCommon import CfgMgr ## get a handle on the application manager from AthenaCommon.AppMgr import theApp if not jobproperties.PerfMonFlags.doFastMon() or jobproperties.PerfMonFlags.doSemiDetailedMonitoring(): theApp.AuditAlgorithms = True theApp.AuditTools = True theApp.AuditServices = True ## get output levels import AthenaCommon.Constants as Lvl ## make sure the application manager explicitly creates the service if hasattr(handle, "getFullJobOptName") : handleName = handle.getFullJobOptName() if not handleName in theApp.CreateSvc: # we want to be the very first service to be initialized # so we can monitor what happens during initialize theApp.CreateSvc = [ handleName ] + theApp.CreateSvc fname = handle.getDefaultProperty('OutFileName') ## provide a default tuple file name from the jobproperties if not hasattr(handle, "OutFileName") and \ hasattr(handle, "getDefaultProperty") : fname = jobproperties.PerfMonFlags.OutputFile() if fname.endswith('.pmon.gz'): fname = fname.replace('.pmon.gz','') elif fname.endswith('.root'): fname = fname.replace('.root','') handle.OutFileName = fname ## configure the auditor service if not done already... if not hasattr(svcMgr, 'AuditorSvc'): from GaudiSvc.GaudiSvcConf import AuditorSvc svcMgr += AuditorSvc() ## configure the chronostat service if not hasattr(svcMgr, 'ChronoStatSvc'): from GaudiSvc.GaudiSvcConf import ChronoStatSvc svcMgr += ChronoStatSvc( OutputLevel = Lvl.WARNING, ChronoPrintOutTable = False ) ## communicate the whole list of Algorithms if no default provided ## by user: we ask for all the ConfigurableAlgorithm's instances componentNames = [] if not hasattr(handle, "ProfiledAlgs") or \ (hasattr(handle, "ProfiledAlgs") and len(handle.ProfiledAlgs) == 0): from AthenaCommon.Configurable import (Configurable, ConfigurableAlgorithm, ConfigurableAlgTool, ConfigurableService) for k,v in Configurable.allConfigurables.items(): if isinstance( v, ( ConfigurableAlgorithm, ConfigurableAlgTool, ConfigurableService ) ) \ and not k in ('PerfMonSvc', 'AthenaEventLoopMgr',): componentNames.append (v.getJobOptName()) pass pass else: componentNames = [k for k in handle.ProfiledAlgs[:]] ## try to fetch the list of containers to be written out from the ## registered OutputStreams (and if we are configured to do such a ## thing) ioContainers = [] if (jobproperties.PerfMonFlags.doPersistencyMonitoring() and not jobproperties.PerfMonFlags.doFastMon()) : ## make sure the AthenaPoolCnvSvc is correctly configured try: svcMgr.AthenaPoolCnvSvc.UseDetailChronoStat = True except AttributeError: # ok... maybe not a 'writeAthenaPool' job... pass ## add our ioContainers list if none has been provided if not hasattr(handle, "IoContainers") or \ ( hasattr(handle, "IoContainers") and \ len(handle.IoContainers) == 0 ): # first let us ask the Streams... from AthenaCommon.Configurable import Configurable if 'Streams' in Configurable.configurables: outStreams = Configurable.configurables['Streams'] for o in outStreams: if not hasattr(o, 'ItemList'): continue ioContainers += [ i for i in o.ItemList if not i.endswith("#*") ] pass # then, what about objKeyStore ? from AthenaCommon.KeyStore import CfgKeyStore keystore = CfgKeyStore.instances.get('KeyStore') if keystore: ioLabels = [ "streamRDO","streamESD", "streamAOD","streamTAG", "inputBackNav","inputFile" ] for l in ioLabels: try: ioContainers.extend(keystore[l].list()) except AttributeError: for k,v in keystore[l].items(): ioContainers += [ "%s#%s" % (k,c) for c in v ] pass ## collect everything handle.IoContainers = ioContainers pass pass ## communicate the granularity of monitoring from the job-flags if jobproperties.PerfMonFlags.doFullMon(): handle.MonLvl = -1 elif jobproperties.PerfMonFlags.doFastMon(): handle.MonLvl = 0 elif jobproperties.PerfMonFlags.doMonitoring(): handle.MonLvl = 1 else: handle.MonLvl = -1 pass if jobproperties.PerfMonFlags.doSemiDetailedMonitoring(): if jobproperties.PerfMonFlags.doSemiDetailedMonitoringFullPrint(): handle.SemiDetailedMonitoring=2 else: handle.SemiDetailedMonitoring=1 #Collect the start-time (in jiffies) of the configuration process try: j=open('/proc/self/stat').readline().split()[21] except Exception as err: #Not worth stopping the job over missing diagnostics. #We should notice the missing info in any case. j="" print("PMonSD WARNING: Could not determine job start time:",err) pass handle.JobStartJiffies=j handle.ExtraPrintouts=jobproperties.PerfMonFlags.doExtraPrintouts() ## communicate the double-counting correction from the job-flags val = jobproperties.PerfMonFlags.enableDoubleCountingCorrection() if val: handle.EnableDoubleCountingCorrection = True else: handle.EnableDoubleCountingCorrection = False pmon_flags = {} import AthenaCommon.JobProperties as acjp for n,v in six.iteritems (jobproperties.PerfMonFlags.__dict__): if isinstance(v, acjp.JobProperty): pmon_flags[n] = v() ## create PyPerfMon.Svc object from .PyPerfMon import Svc svc = Svc( name=handle.getName(), properties = { 'outFileName' : "%s" % handle.OutFileName, 'components' : componentNames, 'ioContainers' : ioContainers, 'pmon_flags' : pmon_flags, }) ## store the py-perfmonsvc object into a sqlite-pickle... import PyUtils.dbsqlite as dbs pmon_props = dbs.open(handle.OutFileName+'.props.pkl', 'w') pmon_props['pmon'] = svc pmon_props.close() return
def store_configuration(cfg_fname=None): """helper function to inspect the 'configured' JobOptionsSvc and dump the properties of each component into a (sqlite) shelve. it will eventually dump the properties of py-components too. """ jobo_cfg = defaultdict(dict) if cfg_fname is None: import tempfile tmpfile = tempfile.NamedTemporaryFile(suffix='-jobo.pkl') cfg_fname = tmpfile.name tmpfile.close() import os if os.path.exists(cfg_fname): os.remove(cfg_fname) assert cfg_fname from AthenaCommon.AppMgr import theApp def _fill_cfg(client, props): for p in props: n = p[0] v = p[1] if hasattr(v, 'toStringProperty'): v = str(v.toStringProperty().toString()) elif hasattr(v, 'toString'): v = str(v.toString()) else: v = str(v) jobo_cfg[client][n] = v from AthenaCommon.AppMgr import ServiceMgr as svcMgr # special cases: joboptionsvc and messagesvc def _fill_props(svcname): if not hasattr(svcMgr, svcname): return svc = getattr(svcMgr, svcname) props = [] for k,v in svc.properties().iteritems(): if v == svc.propertyNoValue: v = svc.getDefaultProperty(k) props.append((k,v)) _fill_cfg(svcname, props) _fill_props('JobOptionsSvc') _fill_props('MessageSvc') # tickle C++ and configure the whole app theApp.setup() app_props = [(k,v.value()) for k,v in theApp.getHandle().properties().iteritems()] _fill_cfg(theApp.name(), app_props) import AthenaPython.PyAthena as PyAthena josvc = PyAthena.py_svc('JobOptionsSvc', iface='IJobOptionsSvc') assert josvc is not None clients = list(josvc.getClients()) for client in clients: props = josvc.getProperties(client) for prop in props: n = prop.name() v = prop.toString() jobo_cfg[client][n] = v cfg = dbs.open(cfg_fname, 'w') cfg['jobopts'] = jobo_cfg pycomps = [] import sys from .AppMgr import ServiceMgr as svcMgr # all other pycomps from .Configurable import Configurable as C for c in C.allConfigurables.itervalues(): if not isinstance(c, (PyAthena.Alg, PyAthena.AlgTool, PyAthena.Svc, PyAthena.Aud)): continue # FIXME: should check if the component is still 'active' # ie: is it in one of the TopAlg,SvcMgr,ToolSvc and children ? if hasattr(c, 'msg'): delattr(c, 'msg') pycomps.append(c) pass # folder for the pyathena-components cfg['pycomps'] = pycomps cfg.close() return cfg_fname
def configure(self, joboptions=None, commands=None, dllname=None, factname=None, extra_options=None): if not (self.app is None): self.msg.info('C++ application already configured') return self.app self.msg.info('configuring application...') usr_cfg = AthCfg() self.cfg.seek(0) usr_cfg << self.cfg.read() # reset self.cfg = AthCfg() if commands: self.cfg << commands+'\n' # common configuration self.cfg << """ # basic job configuration include('AthenaCommon/Atlas.UnixStandardJob.py') include.block('AthenaCommon/Atlas.UnixStandardJob.py') if not (not %(run_batch)s and theApp.EventLoop == 'PyAthenaEventLoopMgr'): # make SIG_INT fatal svcMgr.CoreDumpSvc.FatalHandler = -1 """ % {'run_batch' : self.options.run_batch} self.cfg << """ # user level configuration try: include('$HOME/.athenarc') except IncludeError: pass """ # another user level configuration usr_cfg.seek(0) self.cfg << usr_cfg.read() if isinstance(joboptions, (list,tuple)): for jobo_name in joboptions: self.cfg.include(jobo_name) if not self.options.run_batch: self.cfg << """ theApp.EventLoop = 'PyAthenaEventLoopMgr' svcMgr += CfgMgr.PyAthenaEventLoopMgr() """ self.cfg << """ ### logging and messages --------- from AthenaCommon.Logging import * _msg = log _msg.setLevel(getattr(logging, '%(output_level)s')) import AthenaCommon.Constants as Lvl theApp.setOutputLevel(%(output_level)s) theApp.OutputLevel = Lvl.%(output_level)s from AthenaCommon.AppMgr import ServiceMgr as svcMgr svcMgr.MessageSvc.OutputLevel = Lvl.%(output_level)s """ % dict(output_level=self.options.msg_lvl) self.cfg << """ from AthenaCommon.Include import Include, IncludeError, include include.setShowIncludes(%(showincludes)s) if %(showincludes)s: import AthenaCommon.Include as AthCIncMod AthCIncMod.marker=' -#-' # distinguish bootstrap from other jo-code """ % dict(showincludes=self.options.showincludes) cfg_name = self.cfg._jobo.name.replace('.py','.pkl') self.msg.info('dumping job-configuration into [%s]...', cfg_name) # run configuration in a forked-subprocess... sc = _app_configure(self.cfg, cfg_name, extra_options) if sc: err = 'could not configure application [sc=%d]' % sc self.msg.error(err) raise RuntimeError(err) self.msg.info('configuring application w/ [%s]', cfg_name) import os self.cfg._jobo.close() os.remove(self.cfg._jobo.name) import PyCintex PyCintex.Cintex.Enable() gbl = PyCintex.makeNamespace('') import GaudiPython.Bindings as gaudi # remove the gaudimodule exit handler as to prevent them from clobering import atexit for hdlr in reversed(atexit._exithandlers[:]): module_name = hdlr[0].__module__ if ('GaudiPython' in module_name or 'gaudimodule' in module_name): atexit._exithandlers.remove(hdlr) del hdlr # install our own exit handler (if needed) import sys if hasattr(sys, 'ps1'): # ie: is interactive atexit.register(self.exit) del atexit from . import ResourceLimits ResourceLimits.SetMaxLimits() try: import cPickle as pickle except ImportError: import pickle import PyUtils.dbsqlite as dbs db = dbs.open(cfg_name, 'r') jobo_cfg = db['jobopts'] kw = jobo_cfg['ApplicationMgr'] for k in ('Go', 'Exit', 'AuditInitialize', 'AuditFinalize'): if k in kw: del kw[k] outputlevel = jobo_cfg['ApplicationMgr']['OutputLevel'] self.app = gaudi.AppMgr(outputlevel=outputlevel, selfoptions=kw, dllname=dllname, factname=factname) # open the pycomps folder pycomps = db.get('pycomps', None) # just opening it should do if pycomps: import AthenaPython.Configurables as _C _C.PyComponents.instances = dict((p.name, p) for p in pycomps) #_C.PyComponents.instances = pycomps for p in pycomps: if hasattr(p, 'setup'): if callable(p.setup): p.setup() setattr(self, '_pycomps', pycomps) import AthenaPython.PyAthena as PyAthena josvc = PyAthena.py_svc('JobOptionsSvc', createIf=False, iface='IJobOptionsSvc') assert josvc is not None for client in jobo_cfg: if client == 'ApplicationMgr': continue for n,v in jobo_cfg[client].iteritems(): p = gaudi.StringProperty(n, v) if not josvc.addPropertyToCatalogue(client, p).isSuccess(): self.msg.error( 'could not add property [%s.%s = %s]', client, n, v ) if client in ('MessageSvc', 'JobOptionsSvc'): svc = PyAthena.py_svc(client, iface='IProperty') svc.setProperty(p) db.close() import os if os.path.exists(cfg_name): os.remove(cfg_name) pass #import AthenaCommon.Debugging as dbg #dbg.hookDebugger() return self.app
def main(): # get options opt = options(sys.argv) if opt.input is None or opt.output is None: pass # open tagfile (tg, pyl, cm) = Tagfile(opt.input) nentries = tg.GetEntries() # open database if os.path.exists(opt.output): os.remove(opt.output) eif = dbsqlite.open(opt.output, flags='w') eif['StartProcTime'] = int(time.time() * 1000) eif['StartProcTime_0'] = int(time.time() * 1000) eif['Schema'] = EIrecord().getRecSchema() eif['Version'] = EIrecord().getVersion() eif['TaskID'] = os.getenv('TaskID', 0) eif['JobID'] = os.getenv('JobID', 0) #processing options eif['ProvenanceRef'] = True eif['TriggerInfo'] = True ## process TAG file iov = IOV() # get HLTPSK,L1PSK,SMK from metadata # [188921,1] - [188921,215] iov_interval = re.compile(r'[[](?P<r0>[0-9]+?),(?P<e0>[0-9]+?)[]] - '\ r'[[](?P<r1>[0-9]+?),(?P<e1>[0-9]+?)[]]').match #pdb.set_trace() Project = "-" physicsShort = "-" TAG = "-" for k, v in cm.iteritems(): n = k.find('|') if n != -1: k2 = k[n + 1:] pfn = k[:n + 1][4:] else: k2 = k pfn = "" #print k,v if k2.startswith("Key@HLTPSK;") or k2.startswith( "Key@L1PSK;") or k2.startswith("Key@SMK;"): parts = k2.split(";") k1 = parts[0][4:].rstrip(";") iov1 = parts[1][5:].rstrip("}") match = iov_interval(iov1) if not match: err = "BAD IOV" raise RuntimeError(err) d = match.groupdict() iov.add(k1, int(v), (int(d['r0']), int(d['e0']), int(d['r1']), int(d['e1']))) # asume that all events in TAG file belongs to same Project,datasetNumber,physicsShort,prodStep,dataType,TAG if k2 == 'POOLCollectionID': try: Project, datasetNumber, physicsShort, prodStep, dataType, TAG = pfn.split( '.')[:6] iund = physicsShort.find('_') physicsShort = physicsShort[iund + 1:] #print pfn.split('.')[:6] except: pass #print iov.dump() eif['ProjName_0'] = Project eif['TrigStream_0'] = physicsShort eif['AMITag_0'] = TAG # pool token has the format: # [DB=dbID][CNT=cntID][CLID=classID][TECH=technology][OID=oid.first-oid.second] # eg.: # [DB=FEAD2AD8-111F-11E2-8080-1CCD8E80BEEF][CNT=POOLContainer(DataHeader)] # [CLID=D82968A1-CF91-4320-B2DD-E0F739CBC7E6][TECH=00000202][OID=00000123-00000008] pool_token = re.compile(r'[[]DB=(?P<db>.*?)[]]' \ r'[[]CNT=(?P<cnt>.*?)[]]' \ r'[[]CLID=(?P<clid>.*?)[]]' \ r'[[]TECH=(?P<tech>.*?)[]]' \ r'[[]OID=(?P<oid1>.*?)-(?P<oid2>.*?)[]]' \ ).match # event loop tnow0 = int(time.time() * 1000) tnow = tnow0 for evt_idx in xrange(nentries): if opt.evtmax != 0 and evt_idx >= opt.evtmax: break if opt.verbose > 0 and evt_idx % 1000 == 0: tnow1 = int(time.time() * 1000) sys.stderr.write("{:8d} {:5d} msecs\n".format( evt_idx, tnow1 - tnow)) tnow = tnow1 tg.GetEntry(evt_idx) eirec = EIrecord() run_number = int(pyl.RunNumber.GetCnvValue()) event_number = int(pyl.EventNumber.GetCnvValue()) lumi_block = pyl.LumiBlockN.GetCnvValue() #print "run, evt", run_number,event_number eirec['RunNumber'] = run_number eirec['EventNumber'] = event_number eirec['LumiBlockN'] = lumi_block eirec["BunchId"] = pyl.BunchId.GetCnvValue() eirec['EventTime'] = pyl.EventTime.GetCnvValue() eirec['EventTimeNanoSec'] = pyl.EventTimeNanoSec.GetCnvValue() eirec['EventWeight'] = pyl.EventWeight.GetCnvValue() eirec['McChannelNumber'] = 0 eirec['Lvl1ID'] = 0 eirec['IsSimulation'] = int(pyl.IsSimulation.GetValue()) eirec['IsTestBeam'] = int(pyl.IsTestBeam.GetValue()) eirec['IsCalibration'] = int(pyl.IsCalibration.GetValue()) SMK = iov.get('SMK', (run_number, event_number)) L1PSK = iov.get('L1PSK', (run_number, lumi_block)) HLTPSK = iov.get('HLTPSK', (run_number, lumi_block)) #print '## SMK ', SMK #print '## L1PSK ', L1PSK #print '## HLTPSK ', HLTPSK eirec['SMK'] = SMK eirec['L1PSK'] = L1PSK eirec['HLTPSK'] = HLTPSK try: eirec['Sref0'] = pyl.StreamAOD_ref.GetCnvValue().rstrip('\0') eirec['Snam0'] = "StreamAOD" except: pass try: eirec['Sref0'] = pyl.Token.GetCnvValue().rstrip('\0') eirec['Snam0'] = "StreamAOD" except: pass try: eirec['Sref1'] = pyl.StreamRAW_ref.GetCnvValue().rstrip('\0') eirec['Snam1'] = "StreamRAW" except: pass try: eirec['Sref2'] = pyl.StreamESD_ref.GetCnvValue().rstrip('\0') eirec['Snam2'] = "StreamESD" except: pass trigL1 = "" trigL2 = "" trigEF = "" for k in range(0, 8): v = getattr(pyl, 'L1PassedTrigMaskTBP{:d}'.format(k)).GetCnvValue() trigL1 += "{0:032b}".format(v)[::-1] for k in range(0, 8): v = getattr(pyl, 'L1PassedTrigMaskTAP{:d}'.format(k)).GetCnvValue() trigL1 += "{0:032b}".format(v)[::-1] for k in range(0, 8): v = getattr(pyl, 'L1PassedTrigMaskTAV{:d}'.format(k)).GetCnvValue() trigL1 += "{0:032b}".format(v)[::-1] for k in range(0, 32): v = getattr(pyl, 'L2PassedTrigMask{:d}'.format(k)).GetCnvValue() trigL2 += "{0:032b}".format(v)[::-1] for k in range(0, 32): v = getattr(pyl, 'EFPassedTrigMask{:d}'.format(k)).GetCnvValue() vs = "{0:064b}".format(v) ef_phys = vs[:32] ef_inclusive = vs[32:] trigEF += ef_inclusive[::-1] trigL1 = compressB64(trigL1) trigL2 = compressB64(trigL2) trigEF = compressB64(trigEF) eirec['L1PassedTrigMask'] = trigL1 eirec['L2PassedTrigMask'] = trigL2 eirec['EFPassedTrigMask'] = trigEF # write to db eif['Entry_{:d}'.format(evt_idx)] = eirec.getRec() eif['Nentries_0'] = evt_idx eif['EndProcTime_0'] = int(time.time() * 1000) eif['Nentries'] = evt_idx eif['Nfiles'] = 1 eif['EndProcTime'] = int(time.time() * 1000) eif.close() tnow = int(time.time() * 1000) sys.stderr.write("Processed {:d} events in {:f} seconds\n".format( evt_idx, (tnow - tnow0) / 1000.))