def Reader( readerType, filename, qacross, qToEngine ) :
#
# Process for reading a file
# One process for reading Serial File, another for Parallel File
#
# First the order of events is determined, (parallel != serial, usually)
#
# Then the events are run *in order* using AppMgr().runSelectedEvents(pfn, evtNumber)
# on both Serial-Reader and Parallel-Reader processes.
#
# The string repr of everything in the TES is placed in a dictionary and
# sent to the comparison Process, which compares the two dictionaries
#
a = AppMgr()
sel = a.evtsel()
evt = a.evtsvc()
header = '/Event/Rec/Header'
sel.open( filename )
ct = 0
order = {}
fname = filename[4:] # runSelectedEvents doesn't need the "PFN:" prefix
# determine the ordering
while True :
a.run( 1 )
if evt[header] :
eNumber = int( evt[header].evtNumber() )
order[eNumber] = ct
ct += 1
else : break
if readerType == SER :
# send the ordering details to the parallel-reader
order = switchDict( order )
qacross.put( order )
qacross.put( None )
# changeName
serOrder = order
elif readerType == PAR :
# receive the serial ordering from queue, and send ordering to SerialReader
for serOrder in iter( qacross.get, None ) : pass
lsks = len( serOrder.keys() )
lpks = len( order.keys() )
print 'Events in Files (serial/parallel) : %i / %i' % ( lsks, lpks )
# now run files in the order specified by the serial ordering
# and send them one by one to the comparison engine
for i in iter( serOrder.keys() ) :
if readerType == PAR : i = order[serOrder[i]]
a.runSelectedEvents( fname, i )
lst = evt.getList()
lst.sort()
ascii = dict( [ ( l, ( evt[l].__class__.__name__, evt[l].__repr__() ) ) for l in lst ] )
qToEngine.put( ascii )
qToEngine.put( None )
print '%s Reader Finished' % ( readerType )
def main():
parser = optparse.OptionParser(usage="usage: %prog [options]")
parser.add_option("--debug",
action="store_true",
dest="debug",
default=False,
help="Debug?")
parser.add_option("-d",
"--datatype",
action="store",
dest="DataType",
default="2015",
help="DataType to run on.")
parser.add_option("-n",
"--evtmax",
type="int",
action="store",
dest="EvtMax",
default=10000,
help="Number of events to run")
parser.add_option("--dddbtag",
action="store",
dest="DDDBtag",
default='dddb-20150526',
help="DDDBTag to use")
parser.add_option("--conddbtag",
action="store",
dest="CondDBtag",
default='cond-20150617',
help="CondDBtag to use")
parser.add_option("--settings",
action="store",
dest="ThresholdSettings",
default='Physics_25ns_September2015',
help="ThresholdSettings to use")
parser.add_option("--TCK",
action="store",
dest="TCK",
default='',
help="HLT TCK. If unspecified,then run from settings")
parser.add_option("--simulation",
action="store_true",
dest="Simulation",
default=False,
help="Run on simulated data")
parser.add_option("--input_rate",
action="store",
dest="input_rate",
default=1.e6,
help="Input rate from L0 in Hz")
parser.add_option("--tuplefile",
action="store",
dest="tuplefile",
default="",
help="Output root file")
parser.add_option("--inputdata",
action="store",
dest="inputdata",
default="Physics1600TestNode",
help="Name of inputdata")
# Parse the arguments
(options, args) = parser.parse_args()
#### configured from the arguments
input_rate = options.input_rate
Moore().EvtMax = options.EvtMax
Moore().DDDBtag = options.DDDBtag
Moore().CondDBtag = options.CondDBtag
Moore().Simulation = options.Simulation
Moore().DataType = options.DataType
if options.TCK != "":
Moore().UseTCK = True
Moore().InitialTCK = options.TCK
else:
Moore().ThresholdSettings = options.ThresholdSettings
Moore().UseTCK = False
#### hard coded here
Moore().ForceSingleL0Configuration = False
Moore().OutputLevel = 6
Moore().RemoveInputHltRawBanks = True
Moore().Simulation = False
Moore().Split = ""
Moore().CheckOdin = False
from Configurables import CondDB
CondDB().IgnoreHeartBeat = True
CondDB().EnableRunChangeHandler = True
EventSelector().PrintFreq = 100
#### configure raw data
importOptions('$STDOPTS/DecodeRawEvent.py')
EventPersistencySvc().CnvServices.append('LHCb::RawDataCnvSvc')
from Configurables import DataOnDemandSvc
DataOnDemandSvc().AlgMap['Hlt/DecReports'] = "HltDecReportsDecoder"
from GaudiConf import IOHelper
if options.inputdata == "Physics1600":
IOHelper("MDF").inputFiles([
"mdf:root://eoslhcb.cern.ch//eos/lhcb/wg/HLT/BWdivData/Run164440_L0Filtered_0x00A2_Mika/2015NB_25ns_0x00A2_0.mdf"
])
elif options.inputdata == "Physics1600TestNode":
IOHelper("MDF").inputFiles([
"/localdisk/bw_division/run164440_L0Filtered_0x00A2_Mika/2015NB_25ns_0x00A2_0.mdf",
"/localdisk/bw_division/run164440_L0Filtered_0x00A2_Mika/2015NB_25ns_0x00A2_1.mdf",
"/localdisk/bw_division/run164440_L0Filtered_0x00A2_Mika/2015NB_25ns_0x00A2_2.mdf",
"/localdisk/bw_division/run164440_L0Filtered_0x00A2_Mika/2015NB_25ns_0x00A2_3.mdf",
"/localdisk/bw_division/run164440_L0Filtered_0x00A2_Mika/2015NB_25ns_0x00A2_4.mdf"
])
### getting ready for the event loop
gaudi = AppMgr(outputlevel=4)
gaudi.ExtSvc += ['ToolSvc']
gaudi.ExtSvc.append('DataOnDemandSvc')
gaudi.initialize()
### get the list of active line names
Lines = {}
for level in ["Hlt1", "Hlt2"]:
Lines[level] = set()
for m in Sequence(level).Members:
Lines[level].add(m.name())
### remove certain lines from the accounting
remove = set(
["Hlt1MBNoBias", "Hlt2Transparent", "Hlt1Global", "Hlt2Global"])
Lines["Hlt1"] -= remove
Lines["Hlt2"] -= remove
### this is what I would prefer to do...
#appendPostConfigAction( setLines(Lines["Hlt1"],Lines["Hlt2"]))
### option to create a tuple with all of the decisions
if options.tuplefile != "":
from ROOT import (TTree, TFile)
from array import array
TF = TFile(options.tuplefile, "RECREATE")
DecMaps = {} ## dicto of branches for the TTrees
DecTrees = {} ## dicto of TTrees
for level in ["Hlt1", "Hlt2"]:
DecTrees[level] = TTree('Tuple%s' % level, 'Tuple%s' % level)
DecMaps[level] = {}
for l in Lines[level]:
DecMaps[level][l] = array('i', [0])
DecTrees[level].Branch(l, DecMaps[level][l],
'%sDecision/I' % l)
### this will be dictionary of lines and their counters for the rates
line_stats = {}
for line in Lines["Hlt1"].union(Lines["Hlt2"]).union(
set(["Hlt1Global", "Hlt2Global"])):
line_stats[line] = {"passed_incl": 0, "passed_excl": 0}
### counters for various regex
stream_stats = {
"Turbo": {
"filter": "Hlt2.(?!.*?TurboCalib).*Turbo"
},
"Turcal": {
"filter": "Hlt2.*TurboCalib"
},
"Full": {
"filter": "Hlt2.(?!.*?Turbo).(?!.*?TurboCalib)"
}
}
for k, v in stream_stats.iteritems():
v["pass_this_event"] = False
v["passed"] = 0
print '*' * 100
print Lines
print line_stats
print stream_stats
print '*' * 100
i = 0
processed = 0
#### start of the event loop
while i < Moore().EvtMax:
i += 1
# run the sequences on this event
gaudi.run(1)
processed += 1
if not gaudi.evtsvc()['Hlt1/DecReports']: break
### reset the stream counters
for s in stream_stats.keys():
stream_stats[s]["pass_this_event"] = False
### this is a nasty hack since I can't seem to switch
### off some of the Hlt1 lines with a post config action
### so I have to do my own book-keeping to see if one
### of the good hlt1 lines fired.
### I only consider events in Hlt2 if they pass my subset
### of good hlt1 lines.
PassMyGlobal = {"Hlt1": False, "Hlt2": False}
# loop over levels
for level in ["Hlt1", "Hlt2"]:
if level == "Hlt2" and not PassMyGlobal["Hlt1"]: continue
# do the dec reports exist?
# for Hlt2 this check does the job of telling us whether the event passed Hlt1
if gaudi.evtsvc()['%s/DecReports' % level]:
# get the dec reports
reps = gaudi.evtsvc()['%s/DecReports' % level]
# get the list of lines from the reports
#lines = reps.decReports().keys()
# counter for how many hlt lines fired in this event
nPassed = 0
# loop over all lines
for line in Lines[level]:
# protection. why is this needed though?
if not line + "Decision" in reps.decReports().keys():
print '%s not in %s' % (line, reps.decReports().keys())
continue
# just check this once
LINE_FIRED = reps.decReport(line + "Decision").decision()
# my global counter
if LINE_FIRED: PassMyGlobal[level] = True
# does this event fire any lines that match my "streams"?
if LINE_FIRED and level == "Hlt2" and not line == "Hlt2Global": # and not line in remove:
for s in stream_stats.keys():
if re.match(stream_stats[s]["filter"],
line,
flags=0):
stream_stats[s]["pass_this_event"] = True
# set the variable to be stored in the tuple
if options.tuplefile != "":
if LINE_FIRED:
DecMaps[level][line][0] = 1
else:
DecMaps[level][line][0] = 0
# if this is the first fired event then
# need to initialise the dictionary entry
#if not line in line_stats.keys():
# line_stats[line] = {"passed_incl":0,
# "passed_excl":0}
# increment the counter for this line
if LINE_FIRED:
line_stats[line]["passed_incl"] += 1
if not "Global" in line:
nPassed += 1 ### for the exclusives
# my own global counter
if PassMyGlobal[level]:
line_stats["%sGlobal" % level]["passed_incl"] += 1
# now go back and count the number of exclusive fires of this line
# just need to ignore HltXGlobal
for line in Lines[level]:
if not line + "Decision" in reps.decReports().keys():
continue # protection
if reps.decReport(line +
"Decision").decision() and nPassed == 1:
if not "Global" in line:
line_stats[line]["passed_excl"] += 1
# fill the tree
if options.tuplefile != "":
DecTrees[level].Fill()
# stream accounting
for s in stream_stats.keys():
if stream_stats[s]["pass_this_event"] == True:
stream_stats[s]["passed"] += 1
# write the root file
if options.tuplefile != "":
TF.Write()
# I was hoping that this would dump all of the gaudi related
# printout before I print the summary that I'm interested in.
# Apparently this doesn't work, but it's only really a cosmetic thing.
#gaudi.finalize()
sys.stdout.flush()
#############################################
###### print the summary tables #############
#############################################
for k, v in line_stats.iteritems():
v["processed"] = processed
for k, v in stream_stats.iteritems():
v["processed"] = processed
GlobalRates = {}
print '-' * 100
print 'HLT rates summary starts here'
print '-' * 100
#### print the global rates
print 'removed lines: %s' % remove
print 'processed: %s' % processed
print '%s Hlt1Lines' % (len(Lines['Hlt1']))
print '%s Hlt2Lines' % (len(Lines['Hlt2']))
for level in ['Hlt1', 'Hlt2']:
rate = getrate(1.e-3 * input_rate,
line_stats["%sGlobal" % level]["passed_incl"],
line_stats["%sGlobal" % level]["processed"])
print '%sGlobal rate = (%s+-%s)kHz' % (level, rate[0], rate[1])
### print the stream rates
for k, v in stream_stats.iteritems():
print '%s rate = %s kHz' % (k, v["passed"] / float(v["processed"]))
#### order by inclusive rate
for level in ['Hlt1', 'Hlt2']:
iLine = -1 ## counter for line
table_row("**", "*Line*", "*Incl.*", "*Excl.*")
OrderedStats = {}
for key, value in line_stats.iteritems():
if level in key:
OrderedStats[key] = value["passed_incl"]
for line_name, rate in sorted(OrderedStats.iteritems(),
key=lambda (v, k): (k, v),
reverse=True):
iLine += 1
rate_incl = getrate(1.e-3 * input_rate,
line_stats[line_name]["passed_incl"],
line_stats[line_name]["processed"])
rate_excl = getrate(1.e-3 * input_rate,
line_stats[line_name]["passed_excl"],
line_stats[line_name]["processed"])
if "Global" in line_name:
GlobalRates[level] = rate_incl
else:
table_row(iLine, line_name.replace("Decision", ""),
"%.2f+-%.2f" % (rate_incl[0], rate_incl[1]),
"%.2f+-%.2f" % (rate_excl[0], rate_excl[1]))
## do we pass the test??
MaxRates = {"Hlt1": 400., "Hlt2": 40.}
for level in ['Hlt1', 'Hlt2']:
if GlobalRates[level][0] > MaxRates[level]:
print '%s rate = %s > max = %s' % (level, GlobalRates[level][0],
MaxRates[level])
elif GlobalRates[level][0] == 0:
print '%s rate is zero' % level
else:
print '%s rates OK' % level
print '-' * 100
print 'HLT rates summary ends here'
print '-' * 100
sys.stdout.flush()
class EventReporter(Task):
def __init__(self, name, queues, condition):
Task.__init__(self, name)
self._config = dict()
self._inQueue = queues[0]
self._outQueue = queues[1]
self._condition = condition
def configure(self, configuration):
from Configurables import LHCbApp
app = LHCbApp()
for (attr, value) in configuration.items():
if hasattr(app, attr):
setattr(app, attr, value)
self._config[attr] = value
EventSelector().Input = self._config['Input']
FileCatalog().Catalogs = self._config['Catalogs']
EventDataSvc().RootCLID = 1
from Configurables import LHCb__RawDataCnvSvc as RawDataCnvSvc
EventPersistencySvc().CnvServices.append(
RawDataCnvSvc('RawDataCnvSvc'))
EventSelector().PrintFreq = 100
from Configurables import GaudiSequencer as Sequence
from Configurables import createODIN
seq = Sequence("OdinSequence")
co = createODIN()
seq.Members = [co]
ApplicationMgr().TopAlg = [seq]
def initialize(self):
self._appMgr = AppMgr()
self._appMgr.initialize()
def run(self):
evt = self._appMgr.evtsvc()
nEvents = self._config['EvtMax']
event = 1
while True:
self._condition.acquire()
self._appMgr.run(1)
# Get the ODIN
odin = evt['DAQ/ODIN']
info = None
if odin:
info = (odin.runNumber(), odin.eventNumber())
else:
self.done()
break
# Put the event info on the queue
self._outQueue.put(info)
event += 1
if event == nEvents:
self.done()
break
else:
self._condition.wait()
self._condition.release()
def finalize(self):
self._appMgr.exit()
def done(self):
# Max events reached, signal done to the main process
self._outQueue.put('DONE')
self._condition.release()
class DecisionReporter(Task):
def __init__(self, name, queues, condition):
Task.__init__(self, name)
self._config = dict()
self._inQueue = queues[0]
self._outQueue = queues[1]
self._condition = condition
def configure(self, configuration):
from Configurables import Moore
moore = Moore()
for (attr, value) in configuration.items():
if attr in moore.__slots__:
setattr(moore, attr, value)
self._config[attr] = value
hlt1Lines = self._config['Hlt1Lines']
hlt2Lines = self._config['Hlt2Lines']
#if 'L0' in self._config and self._config[ 'L0' ]:
#from Configurables import L0MuonAlg
#L0MuonAlg( "L0Muon" ).L0DUConfigProviderType = "L0DUConfigProvider"
if 'Dataset' in self._config:
from PRConfig import TestFileDB
TestFileDB.test_file_db[self._config['Dataset']].run(
configurable=Moore())
else:
EventSelector().Input = self._config['Input']
EventSelector().PrintFreq = 100
FileCatalog().Catalogs = self._config['Catalogs']
config = Config(hlt1Lines, hlt2Lines)
appendPostConfigAction(config.postConfigAction)
def initialize(self):
self._appMgr = AppMgr()
self._appMgr.initialize()
def run(self):
evt = self._appMgr.evtsvc()
nEvents = self._config['EvtMax']
event = 1
while True:
if self.wait():
self._condition.acquire()
self._appMgr.run(1)
# Check if there is still event data
if not bool(evt['/Event']):
self.done()
break
odin = evt['DAQ/ODIN']
reports = dict()
reports['event'] = odin.eventNumber()
reports['run'] = odin.runNumber()
# Grab the HltDecReports and put the decisions in a dict by line name
if evt['Hlt1/DecReports']:
decReports1 = evt['Hlt1/DecReports']
names1 = decReports1.decisionNames()
for name in names1:
reports[name] = decReports1.decReport(name).decision()
if evt['Hlt2/DecReports']:
decReports2 = evt['Hlt2/DecReports']
names2 = decReports2.decisionNames()
for name in names2:
reports[name] = decReports2.decReport(name).decision()
# Put our dict on the queue
self._outQueue.put(reports)
event += 1
if event == nEvents:
self.done()
break
elif self.wait():
self._condition.wait()
self._condition.release()
def finalize(self):
self._appMgr.exit()
def done(self):
# Max events reached, signal done to the main process
self._outQueue.put('DONE')
if self.wait():
self._condition.release()
def wait(self):
if 'Wait' in self._config:
return self._config['Wait']
else:
return True
