def evaluate(self, runlist):
# provide pickled dictionary of query result
with timer('CreateResultDict'):
from CoolRunQuery.output.AtlRunQuerySave import CreateResultDict
dic, dicsum = CreateResultDict(runlist)
# create XML file (and return pretty html output for print)
if self.prodgrl:
with timer('CreateXMLFile'):
print("Producing XML file")
from CoolRunQuery.output.AtlRunQueryXML import CreateXMLFile
from .AtlRunQueryVersion import SvnVersion
xmlhtmlstr = CreateXMLFile(runlist, self.cmdlineOptions,
self.origQuery, self.datapath,
self.xmlFileName, self.xmlFileLabel,
SvnVersion)
else:
xmlhtmlstr = None
print("Creation of GRL disabled")
if self.dictroot and len(runlist) > 0:
with timer('CreateRootFile'):
# create root file from dictionary - and make plots
from CoolRunQuery.output.AtlRunQueryRoot import CreateRootFile
rootfilename, roothtmlstr = CreateRootFile(dic)
else:
roothtmlstr = None
print("Creation of root file disabled")
return (dic, dicsum, xmlhtmlstr, roothtmlstr)
def output(self, runlist, dic, dicsum, xmlhtmlstr, roothtmlstr):
if self.html:
with timer('GetOKS link info'):
from AtlRunQuerySFO import SetOKSLinks
SetOKSLinks(runlist)
# create web page
from .html.AtlRunQueryHTML import ResultPageMaker
#try:
pageinfo = {
'datapath': self.datapath,
'origQuery': self.origQuery,
'fullQuery': self.parsedstring,
'runlist': runlist,
'dic': dic,
'dicsum': dicsum,
'makeSummary': self.makeSummary,
'makeDQeff': self.makeDQeff,
'makeDQSummary': self.makeDQSummary,
'makeDQPlots': self.makeDQPlots,
'roothtmlstr': roothtmlstr,
'xmlfilename': self.xmlFileName,
'xmlhtmlstr': xmlhtmlstr,
'querytime': time() - self.querystart,
'selout': self.selectionOutput,
}
with timer("run ResultPageMaker makePage"):
ResultPageMaker.makePage(pageinfo)
#except ImportError, ie:
# print ("Can't import pagemaker, no web page made",ie)
else:
print(
'---------------------------------------------------------------------'
)
print(Run.header())
runlist.reverse()
for r in runlist:
print(r)
from CoolRunQuery.utils.AtlRunQueryUtils import addKommaToNumber, filesize
print('Summary:')
for data_key, summary in dicsum.items():
if data_key.Type == DataKey.STREAM:
key = data_key.ResultKey.replace('STR:', '')
val = '%s (%s)' % (addKommaToNumber(
summary[0]), filesize(summary[1]))
else:
key = data_key.ResultKey
val = addKommaToNumber(summary)
print('%20s : %s' % (key, val))
duration = time() - self.querystart
print("%20s : %g sec" % ('Total execution time', duration))
print(
'---------------------------------------------------------------------'
)
def select(self, runlist):
# some preparation: compile the show patterns
start = time()
print self,
sys.stdout.flush()
newrunlist = []
connection = coolDbConn.GetSFODBConnection()
cursor = connection.cursor()
cursor.arraysize = 1000
runnrlist = [r.runNr for r in runlist]
from CoolRunQuery.AtlRunQuerySFO import GetSFO_NphysicseventsAll
with timer("GetSFO_NphysicseventsAll"):
events = GetSFO_NphysicseventsAll(
cursor, runnrlist) # { runnr: nPhysEvents }
for run in runlist:
iov = IOVRange(runStart=run.runNr,
lbStart=1,
runEnd=run.runNr + 1,
lbEnd=0)
for k in self.ResultKey():
if not run.runNr in events:
# the OVERLAP_EVENTS information is not yet available in the SFO (before Nov 15, 2009)
# use the inclusive number instead
run.addResult(k, "n.a.", iov, reject=False)
run.showDataIncomplete = True
continue
nev = events[run.runNr]
if self.ApplySelection(k) and not self.passes(nev, k):
run.addResult(k,
self.prettyValue(nev, k),
iov,
reject=True)
continue
run.addResult(k, self.prettyValue(nev, k), iov)
rej = self.rejectRun(run)
if not rej:
newrunlist += [run.runNr]
runlist = [r for r in runlist if r.runNr in newrunlist]
duration = time() - start
if self.applySelection:
print " ==> %i runs found (%.2f sec)" % (len(runlist), duration)
else:
print " ==> Done (%g sec)" % duration
return runlist
def _makeDQeff( cls, pageinfo ):
for x in ["dicsum", "dic", "datapath", "makeDQeff"]:
exec("%s = pageinfo['%s']" % (x,x) )
if dicsum and makeDQeff:
with timer("make the DQ efficiency summary"):
from CoolRunQuery.utils.AtlRunQueryDQUtils import MakeDQeffHtml
return MakeDQeffHtml( dic, dicsum, datapath )
else:
return ''
def __init__(self,
options,
readoracle=False,
loglevel=1,
html="AUTO",
origQuery="",
datapath='data',
parsedstring=""):
with timer('total'):
with timer('config'):
self.config(options, readoracle, loglevel, html, origQuery,
datapath, parsedstring)
with timer('run all'):
runlist = self.run()
with timer('evaluate all'):
(dic, dicsum, xmlhtmlstr, roothtmlstr) = self.evaluate(runlist)
with timer('output all'):
self.output(runlist, dic, dicsum, xmlhtmlstr, roothtmlstr)
with timer('finalize'):
self.finalize()
from CoolRunQuery.utils.AtlRunQueryTimer import TimerStats as TS
TS.printTimeSummary()
TS.printTimeFlat()
def _makeResultsTable( cls, pageinfo ):
for x in pageinfo.keys():
exec("%s = pageinfo['%s']" % (x,x) )
# run results table
resultstable = '<table class="resulttable" id="resulttable" style="margin-left: 13px">\n'
# table head
resultstable += Run.header()
# runs
with timer("print the runs"):
for r in runlist:
with timer("print run %i" % r.runNr):
resultstable += str(r)
# summary
headlist = Run.headerkeys()
resultstable += " <tr class=\"space\"><td style=\"text-align: left;\" colspan=\"%i\"><font size=-2><i>Summary</i>:</font></td></tr>\n" % (len(headlist)-1)
resultstable += " <tr class=\"sum\">" + sumstr + "</tr>\n"
resultstable += "</table>\n"
return resultstable
def _OverallSummary( cls, pageinfo ):
for x in pageinfo.keys():
exec("%s = pageinfo['%s']" % (x,x) )
with timer("make the summary"):
from CoolRunQuery.html.AtlRunQuerySummary import MakeSummaryHtml
overallsummarystr = MakeSummaryHtml( dic, dicsum, datapath )
if overallsummarystr == '': return ''
s = '''<table width="95%" cellpadding="5" style="margin-left: 13px">
<tr><td colspan=2 bgcolor=gainsboro><font size=+1><b>Search Result Summary</b></font></td></tr>
</table>'''
s += '<p></p>'
s += overallsummarystr
s += '<p></p>'
return s
def _defineToolTips( cls, pageinfo, globalOnly=False ):
for x in pageinfo.keys():
exec("%s = pageinfo['%s']" % (x,x) )
tooltips = u''
with timer("print the tooltips"):
dw_call = ['<script type="text/javascript">']
dw_call += ["if(dw_Tooltip.content_vars==undefined) {dw_Tooltip.content_vars = {}; };"]
dw_call += Run.GlobalTooltips
if not globalOnly:
for r in runlist:
dw_call += r.tooltips
dw_call += ['</script>']
for l in dw_call:
tooltips += unicode(l) + '\n'
return tooltips
def CreateRunStreamAndNeventsList(runlist):
# open SFO DB connection
from CoolRunQuery.utils.AtlRunQueryUtils import coolDbConn
from CoolRunQuery.AtlRunQuerySFO import GetSFO_NeventsAllPhysics
cursor = coolDbConn.GetSFODBConnection().cursor()
cursor.arraysize = 1000
# find streams
runnrlist = [r.runNr for r in runlist]
with timer('get SFO number of events AllPhysics'):
runstreamevents = GetSFO_NeventsAllPhysics(
cursor,
runnrlist) # { runnr: { stream: [(LUMIBLOCKNR, NREVENTS)] } }
return runstreamevents, ",".join(['%i' % r for r in runnrlist])
def _makeYellowLine( cls, pageinfo ):
for x in ["dicsum"]:
exec("%s = pageinfo['%s']" % (x,x) )
with timer("make the yellow summary line"):
sumdic = {}
for key, summary in dicsum.items():
if key.Type == DataKey.STREAM:
s = addKommaToNumber(summary[0])+' <BR><font size="-2">(%s)</font>' % filesize(summary[1])
elif key=='Run':
s = addKommaToNumber(summary) + " runs"
else:
s = addKommaToNumber(summary)
sumdic[key.ResultKey] = s.strip()
sumstr = ''
headlist = Run.headerkeys()
for title in headlist:
sumstr += '<td style="text-align:right;">%s</td>' % (sumdic[title] if title in sumdic else '')
return sumstr
def _DQSummaryWrap( cls, pageinfo, doPickle=True ):
from CoolRunQuery.utils.AtlRunQueryUtils import runsOnServer
if not runsOnServer() and doPickle:
import pickle
from CoolRunQuery.AtlRunQueryQueryConfig import QC
f = open('%s/dqsum_pi.pickle' % QC.datapath, "w")
pickle.dump(pageinfo, f)
f.close()
for x in pageinfo.keys():
exec("%s = pageinfo['%s']" % (x,x) )
with timer("make the DQ summary"):
from CoolRunQuery.html.AtlRunQueryDQSummary import DQSummary
dqsummary = DQSummary.makeHTML( dic, dicsum, doDQSummary=makeDQSummary, doDQPlots=makeDQPlots )
if dqsummary == '': return ''
s = '''<table width="95%" cellpadding="5" style="margin-left: 13px">
<tr><td colspan=2 bgcolor=gainsboro><font size=+1><b>Data Quality Summary</b></font></td></tr>
</table>'''
s += '<p></p>'
s += dqsummary
s += '<p></p>'
return s
def run(self):
# is last run still open ?
from CoolRunQuery.AtlRunQuerySFO import GetSFO_lastNruns
sfoconnection = coolDbConn.GetSFODBConnection()
sfocursor = sfoconnection.cursor()
retlist = GetSFO_lastNruns(sfocursor, 1)
if 'OPENED' in retlist[0][1]: Run.runnropen = retlist[0][0]
if self.cmdlineOptions.runlist != None:
runlist = self.cmdlineOptions.runlist
if any('last' in x for x in self.cmdlineOptions.runlist):
for idx in xrange(len(runlist)):
rrange = runlist[idx]
if not 'last' in rrange: continue
rrange = rrange.replace('last', '')
self.maxNumOfRuns = int(rrange)
# list of last n runs run numbers, where at least one stream is not calibration
retlist = GetSFO_lastNruns(sfocursor,
1.1 * self.maxNumOfRuns)
runlist[idx] = '%i+' % (retlist[-1][0])
elif self.cmdlineOptions.timelist != None:
# translate time into run-range
if 'last' in self.cmdlineOptions.timelist[0]:
# contains 'last'
from CoolRunQuery.utils.AtlRunQueryUtils import get_runs_last_dt
rlist = get_runs_last_dt(self.cmdlineOptions.timelist[0])
runlist = ['%i+' %
rlist[-1]] if (len(rlist) > 0) else ['99999999+']
else:
# translate time into run-range
from utils.AtlRunQueryUtils import timeStringToSecondsUTC, secondsToTimeStringUTC, get_run_range2, GetTimeRanges
timelist = ','.join(self.cmdlineOptions.timelist)
timeranges, timerangesHR = GetTimeRanges(
timelist, intRepFnc=timeStringToSecondsUTC, maxval=time())
timerangesAsString = [
map(secondsToTimeStringUTC, tr) for tr in timeranges
]
runranges = [("%s-%s" % get_run_range2(tr[0], tr[1]))
for tr in timerangesAsString]
runlist = [','.join(runranges)]
# the run selector
rtSel = RunTimeSelector(name='runtime', runlist=runlist)
# find the begin and end of each interesting run
runlist = rtSel.select()
# if Selector.condDB()=="CONDBR2":
# self.cmdlineOptions.show = [x for x in self.cmdlineOptions.show if x not in ["lhc","olclumi","olcfillparams","olclbdata"]]
# self.cmdlineOptions.partition=None
# self.cmdlineOptions.projecttag=None
# print ("Pre-run2 selection removed partition and projecttag from querying")
# print ("Pre-run2 selection removed lhc, olc from showing. Modified show list: %r" % self.cmdlineOptions.show)
self.selectionOutput += ["%s" % rtSel]
from .AtlRunQuerySelectorWorker import SelectorWorker
# create all selectors that are actively select
SelectorWorker.parseSelectorOptions(self.cmdlineOptions)
# parse show option:
SelectorWorker.parseShowOption(self.cmdlineOptions)
# call setApply for all selectors that actively select
SelectorWorker.setApplySelection()
# call setShowOutput for all selectors that show
SelectorWorker.setShowOutput()
SelectorWorker.getRetrieveSelector('trigkey', 'TrigKeySelector')
# Selectors can implement an initialize function which runs after any constructor and setShow
for s in SelectorWorker.getOrderedSelectorList():
s = s.selector
if hasattr(s, 'initialize'):
with timer("initializing selector '%s'" % s.name):
s.verbose = True
s.initialize()
# apply the selectors to initial run list
for s in SelectorWorker.selectors():
with timer("run selector '%s'" % s.name):
s.verbose = True
runlist = s.select(runlist)
self.selectionOutput += ["%s" % s.__str__()]
with timer("run AfterQuery for selector '%s'" % s.name):
s.runAfterQuery(runlist)
#print ([r.runNr for r in runlist])
# reverse list for presentation: newest run on top
runlist.reverse()
# if "last" option used, the runlist has to be truncated
if self.maxNumOfRuns > 0:
print('SELOUT Selecting the %i most recent runs' %
self.maxNumOfRuns)
runlist = runlist[0:self.maxNumOfRuns]
self.selectionOutput += [
'SELOUT Selecting the %i most recent runs' % self.maxNumOfRuns
]
return runlist
def select(self, runlist):
# some preparation: compile the show patterns
start = time()
if '.*' in self.showstreampatterns:
compiledShowPatterns = [re.compile('.*')]
else:
compiledShowPatterns = [
re.compile(p) for p in self.showstreampatterns
]
# we disable the access to everything that is not needed if we only select
ShowStreams = False
useTier0 = False
if len(compiledShowPatterns) > 0:
ShowStreams = True
useTier0 = True
print(self, end='')
sys.stdout.flush()
newrunlist = []
allStreams = Set(
) # list of all the streams that are in the selected runs
connection = coolDbConn.GetSFODBConnection()
cursor = connection.cursor()
cursor.arraysize = 1000
runnrlist = [r.runNr for r in runlist]
from CoolRunQuery.AtlRunQuerySFO import GetSFO_streamsAll, GetSFO_filesAll
with timer('GetSFO_streamsAll'):
streamsall = GetSFO_streamsAll(cursor,
runnrlist) # { runnr: [streams] }
with timer('GetSFO_filesAll'):
filesall = GetSFO_filesAll(
cursor,
runnrlist) # [(COUNT(FILESIZE), SUM(FILESIZE), SUM(NREVENTS))]
if ShowStreams:
from CoolRunQuery.AtlRunQuerySFO import GetSFO_LBsAll, GetSFO_NeventsAll, GetSFO_overlapAll
with timer('GetSFO_LBsAll'):
lbinfoall = GetSFO_LBsAll(
cursor, runnrlist
) # [(MIN(LUMIBLOCKNR), MAX(LUMIBLOCKNR), #LUMIBLOCKS)]
with timer('GetSFO_overlapAll'):
overlapall = GetSFO_overlapAll(
cursor, runnrlist) # [(SUM(OVERLAP_EVENTS))]
smallrunnrlist = []
for r in runnrlist: # go through old runlist and see
if not r in streamsall: continue
for s in streamsall[r]:
if r in lbinfoall and s in lbinfoall[
r] and lbinfoall[r][s][1] > 0:
smallrunnrlist += [r]
break
with timer('GetSFO_NeventsAll'):
neventsall = GetSFO_NeventsAll(
cursor, smallrunnrlist) # [(LUMIBLOCKNR, NREVENTS)]
for run in runlist: # go through old runlist and see
streams = [] # names of all streams in this run
strsize = [] # size of streams in this run
strevents = [] # nr of events in this run
if run.runNr in streamsall:
streams = streamsall[run.runNr]
for s in streams:
try:
nfiles, size, events = filesall[run.runNr][s]
except:
nfiles, size, events = (0, 0, 0)
strsize += [size]
strevents += [events]
if ShowStreams:
from CoolRunQuery.utils.AtlRunQueryUtils import Matrix
nst = len(streams)
stovmat = Matrix(nst, nst) # overlap matrix
for i, s in enumerate(streams):
run.stats['STR:' + s] = {}
# fill overlaps into matrix
for j, s2 in enumerate(streams):
try:
eventsij = overlapall[run.runNr][s][s2]
except KeyError:
eventsij = 0
if i == j and events:
eventsij = events
stovmat.setitem(i, j, float(eventsij))
stovmat.setitem(j, i,
float(eventsij)) # symmetrise matrix
# read number of events per LB
minlb, maxlb, lbs = (0, 0, 1)
try:
minlb, maxlb, lbs = lbinfoall[run.runNr][s]
except KeyError:
pass
# if minlb==maxlb==0 -> no file closure at LB boundary
if minlb == 0 and maxlb == 0:
run.stats['STR:' + s]['LBRecInfo'] = None
continue
else:
lbevcount = '<tr>'
result = neventsall[run.runNr][s] #[ (lb,nev),... ]
lbold = -1
allnev = 0
ic = 0
ice = 0
allic = 0
lastElement = False
firstcall = True
for ice, (lb, nev) in enumerate(result):
if ice == len(result):
lastElement = True
allnev += nev
if lb != lbold or lastElement:
if lbold != -1:
ic += 1
allic += 1
if allic < 101:
if ic == 9:
ic = 1
lbevcount += '</tr><tr>'
lbevcount += '<td style="font-size:75%%">%i (%s)</td>' % (
lbold, allnev)
else:
if firstcall:
lbevcount += '</tr><tr>'
lbevcount += '<td style="font-size:75%%" colspan="8">... <i>too many LBs (> 100) to show</i></td>'
firstcall = False
allnev = nev
lbold = lb
else:
allnev += nev
lbevcount += '</tr>'
run.stats['STR:' + s]['LBRecInfo'] = lbevcount
run.stats['STR:' + s]['LBRecInfo'] = result
# add overlap information to the run stats
for i in xrange(nst):
statkey = 'STR:' + streams[i]
run.stats[statkey]['StrOverlap'] = []
denom = stovmat.getitem(i, i)
if denom == 0: continue
for j in xrange(nst):
if i == j or stovmat.getitem(i, j) == 0: continue
fraction = 100
if stovmat.getitem(i, j) != denom:
fraction = float(stovmat.getitem(
i, j)) / float(denom) * 100.0
run.stats[statkey]['StrOverlap'] += [(streams[j],
fraction)]
# selection...
if not self.passes(zip(streams, strevents), 0): continue
newrunlist += [run.runNr]
allStreams.update(streams)
for k, v, s in zip(streams, strevents, strsize):
run.addResult('STR:' + k, (v, s))
allStreams = ['STR:' + s for s in allStreams]
allStreams.sort(lambda x, y: 2 * cmp(y[4], x[4]) + cmp(x[5:], y[5:]))
# fill the gaps
for run in runlist:
for s in allStreams:
if not s in run.result:
run.addResult(s, 'n.a.')
runlist = [r for r in runlist if r.runNr in newrunlist]
# check if the streams in 'allStreams' match the show patterns
for s in allStreams:
if any([p.match(s[4:]) != None for p in compiledShowPatterns]):
Run.AddToShowOrder(DataKey(s, keytype=DataKey.STREAM))
if useTier0:
# retrieve Tier-0 information
from CoolRunQuery.AtlRunQueryTier0 import GetTier0_datasetsAndTypes
tier0connection = coolDbConn.GetTier0DBConnection()
cursor = tier0connection.cursor()
cursor.arraysize = 1000
tier0retdico = GetTier0_datasetsAndTypes(cursor, runnrlist)
# add Tier0 information
for run in runlist:
for s in allStreams:
if run.result[s] == 'n.a.': continue
run.stats[s]['StrTier0TypesRAW'] = {}
run.stats[s]['StrTier0TypesESD'] = {}
run.stats[s]['StrTier0AMI'] = {}
if run.runNr in tier0retdico.keys():
for dsname, t, pstates in tier0retdico[run.runNr]:
if s.replace('STR:', '') in dsname:
if '.RAW' in dsname:
if '.merge' in dsname:
prodstep = 'StrTier0TypesESD'
t = '(RAW)'
else:
prodstep = 'StrTier0TypesRAW'
t = '(RAW)'
else:
if '.recon.' in dsname:
prodstep = 'StrTier0TypesRAW'
else:
prodstep = 'StrTier0TypesESD'
if not run.stats[s]['StrTier0AMI'].has_key(
prodstep):
dsnamesplit = dsname.split('.')
if len(dsnamesplit) > 5:
amitag = dsnamesplit[5]
run.stats[s]['StrTier0AMI'][
prodstep] = amitag
else:
amitag = ''
# check if on CAF
oncaf = False
if pstates and 'replicate:done' in pstates:
oncaf = True
# fill the run stats
if not run.stats[s][prodstep].has_key(t):
run.stats[s][prodstep][t] = oncaf
# Done
duration = time() - start
if len(self.streams) != 0:
print(" ==> %i runs found (%.2f sec)" % (len(runlist), duration))
else:
print(" ==> Done (%g sec)" % duration)
return runlist
def CreateXMLFile(runlist,
options,
origQuery,
datapath,
xmlfname,
xmllabel,
svnversion='Unknown'):
"""
"""
with timer('create RunStreamAndNeventsList'):
# show number of events per stream per run ?
ShowNumberOfEventsPerStreamPerRun = False
ShowNumberOfEventsPerStreamSummary = True
# find streams
runstreamevents, runnrliststr = CreateRunStreamAndNeventsList(
runlist) # { runnr: { stream: [(LUMIBLOCKNR, NREVENTS)] } }
with timer('prepare document'):
####################################
## create XML file of GRL
####################################
doc = Document()
docType = DocumentType('LumiRangeCollection')
docType.systemId = 'http://atlas-runquery.cern.ch/LumiRangeCollection.dtd'
doc.appendChild(docType)
# number of comments
txt = ' Good-runs-list created by %s on %s ' % (
sys.argv[0].split('/')[-1], str(datetime.datetime.now()))
doc.appendChild(doc.createComment(txt))
# root element
lrc = doc.createElement('LumiRangeCollection')
doc.appendChild(lrc)
# NamedLumiRange
namedLR = doc.createElement('NamedLumiRange')
lrc.appendChild(namedLR)
# name of NamedLumiRange
namedLR.appendChild(TextElement('Name', xmllabel, doc))
# version of NamedLumiRange
namedLR.appendChild(TextElement('Version', '2.1', doc))
# metadata of NamedLumiRange
metadata = {
'Query': origQuery.split('/')[0],
'RQTSVNVersion': svnversion,
'RunList': runnrliststr
}
for md in metadata:
mdelm = TextElement('Metadata', metadata[md], doc)
mdelm.setAttribute('Name', md)
namedLR.appendChild(mdelm)
with timer('ShowNumberOfEventsPerStreamSummary'):
if ShowNumberOfEventsPerStreamSummary:
strsummdelm = doc.createElement('Metadata')
strsummdelm.setAttribute('Name', 'StreamListInfo')
namedLR.appendChild(strsummdelm)
# lumiblock collections of NamedLumiRange
streams_sum = {}
streams_byrun = {}
with timer('Loop over all runs'):
for run in runlist:
lbc = doc.createElement('LumiBlockCollection')
# run number
runnrelm = TextElement('Run', str(run.runNr), doc)
if len(run.stats['SMK']['random']) == 2:
(rd0, rd1) = run.stats['SMK']['random'][0:2]
# protect against missing information
if rd0 == 'n.a.': rd0 = 0
if rd1 == 'n.a.': rd1 = 0
runnrelm.setAttribute('PrescaleRD0', 0x1 << (3 + rd0))
runnrelm.setAttribute('PrescaleRD1', 0x1 << (3 + rd1))
else:
(rd0, rd1, rd2, rd3) = run.stats['SMK']['random'][0:4]
# protect against missing information
if rd0 == 'n.a.': rd0 = 0
if rd1 == 'n.a.': rd1 = 0
if rd2 == 'n.a.': rd2 = 0
if rd3 == 'n.a.': rd3 = 0
runnrelm.setAttribute('Cut0', rd0)
runnrelm.setAttribute('Cut1', rd1)
runnrelm.setAttribute('Cut2', rd2)
runnrelm.setAttribute('Cut3', rd3)
lbc.appendChild(runnrelm)
# streams (initialisation)
streams = {}
streams_byrun[run.runNr] = streams
if runstreamevents.has_key(
run.runNr
): # protection in case the run does not have any stream
for stream in runstreamevents[run.runNr].keys():
if 'physics_' == stream[:8]:
streams[stream] = [0, 0] # only for physics streams
if not streams_sum.has_key(stream):
streams_sum[stream] = [0, 0]
# total number of events in stream
for (nlb, nev) in runstreamevents[run.runNr][stream]:
streams[stream][0] += nev
# lumiblock ranges
for lbrange in run.data.getLBRanges(activeOnly=True):
lbrelm = TextElement('LBRange', '', doc)
lbrelm.setAttribute('Start', lbrange[1])
lbrelm.setAttribute('End', lbrange[2] - 1)
lbc.appendChild(lbrelm)
# count nevents in streams
if runstreamevents.has_key(
run.runNr
): # protection in case the run does not have any stream
for stream, lbnevts in runstreamevents[run.runNr].items():
if 'physics_' == stream[:8]:
for (nlb, nev) in lbnevts:
if nlb >= lbrange[1] and nlb < lbrange[2]:
streams[stream][1] += nev
# append stream element
s = streams.keys()
s.sort()
strselm = doc.createElement('StreamsInRun')
for stream in s:
nevts = streams[stream]
if ShowNumberOfEventsPerStreamPerRun:
strelm = TextElement('Stream', '', doc)
strelm.setAttribute('Name', stream)
strelm.setAttribute('TotalNumOfEvents', nevts[0])
strelm.setAttribute('NumOfSelectedEvents', nevts[1])
strselm.appendChild(strelm)
eff = 0
if nevts[0] > 0: eff = nevts[1] / float(nevts[0]) * 100.0
# collect total number of events
streams_sum[stream][0] += nevts[0]
streams_sum[stream][1] += nevts[1]
# append streams
if ShowNumberOfEventsPerStreamPerRun: lbc.appendChild(strselm)
# append LumiBlickCollection
namedLR.appendChild(lbc)
with timer('Streams'):
for stream in sorted(streams_sum.keys()):
nevts = streams_sum[stream]
if ShowNumberOfEventsPerStreamSummary:
strelm = TextElement('Stream', '', doc)
strelm.setAttribute('Name', stream)
strelm.setAttribute('TotalNumOfEvents', nevts[0])
strelm.setAttribute('NumOfSelectedEvents', nevts[1])
strsummdelm.appendChild(strelm)
with timer('Save GRL'):
filename = '%s/%s' % (datapath, xmlfname)
#print "Writing",filename
xmlfile = open(filename, mode="w")
xmlfile.write(doc.toprettyxml(' '))
xmlfile.close()
with timer('Create HTML'):
####################################
## create HTML
####################################
# provide also pretty html text output
htmltext = ''
hspace = ' '
htmltext += '<table style="width: auto; border: 0px solid; border-width: margin: 0 0 0 0; 0px; border-spacing: 0px; border-collapse: separate; padding: 0px;" font-family: sans-serif; font-size: 85%%">\n'
htmltext += '<tr><td colspan="2"><b><font color="#999999">' + txt.strip(
) + '</font></b></td></tr>\n'
htmltext += '<tr><td style="vertical-align:top">SVN Version: </td><td> ' + svnversion + '</td></tr>\n'
htmltext += '<tr><td style="vertical-align:top">Query string:</td><td><b>' + origQuery.split(
'/')[0] + '</b></td></tr>\n'
htmltext += '<tr><td style="vertical-align:top">Run list:</td><td>' + runnrliststr + '</td></tr>\n'
htmltext += '</table>'
htmltext += '<hr color="#000000" size=1><font color="#777777">\n'
htmltext += '<table style="width: auto; border: 0px solid; border-width: margin: 0 0 0 0; 0px; border-spacing: 0px; border-collapse: separate; padding: 0px;" font-family: sans-serif; font-size: 90%%">\n'
# lumiblock collections of NamedLumiRange
for run in runlist:
# run number
htmltext += '<tr><td style="text-align:left;height:25px;vertical-align:bottom">Run <b>%i</b>:</td><td></td></tr>\n' % run.runNr
# lumiblock ranges
for lbrange in run.data.getLBRanges(activeOnly=True):
htmltext += '<tr><td></td><td style="text-align:left">LB range: [%5i-%5i]</td></tr>\n' % (
lbrange[1], lbrange[2] - 1)
# append stream element
htmltext += '<tr><td></td><td style="text-align:left"></td></tr>'
htmltext += '<tr><td></td><td style="text-align:left"><table style="width: auto; border: 0px solid; border-width: margin: 0 0 0 0; 0px; border-spacing: 0px; border-collapse: separate; padding: 0px;font-family: sans-serif; font-size: 90%"><tr><td>Stream name</td><td>#Events total</td><td> #Events selected</td><td> Sel. fraction (%)</td></tr>\n'
streams = streams_byrun[run.runNr]
for stream in sorted(streams.keys()):
nevts = streams[stream]
eff = (nevts[1] / float(nevts[0]) *
100.0) if (nevts[0] > 0) else 0
htmltext += '<tr><td style="text-align:left"><i>%s</i></td><td style="text-align:right">%s</td><td style="text-align:right">%s</td><td style="text-align:right">%.4g</td></tr>\n' % (
stream, prettyNumber(nevts[0]), prettyNumber(
nevts[1]), eff)
htmltext += '</table></td></tr>\n'
# append stream element
htmltext += '</table>'
htmltext += '<hr color="#000000" size=1><font color="#777777">\n'
htmltext += '<b>Stream summary for all selected runs:</b><br>\n'
htmltext += '<table style="width: auto; border: 0px solid; border-width: margin: 0 0 0 0; 0px; border-spacing: 0px; border-collapse: separate; padding: 0px;font-family: sans-serif; font-size: 95%"><tr><td>Stream name</td><td>#Events total</td><td> #Events selected</td><td> Sel. fraction (%)</td></tr>\n'
for stream in sorted(streams_sum.keys()):
nevts = streams_sum[stream]
eff = 0
if nevts[0] > 0: eff = nevts[1] / float(nevts[0]) * 100.0
htmltext += '<tr><td style="text-align:left"><i>%s</i></td><td style="text-align:right">%s</td><td style="text-align:right">%s</td><td style="text-align:right">%.4g</td></tr>\n' % (
stream, prettyNumber(nevts[0]), prettyNumber(nevts[1]), eff)
htmltext += '</table>\n'
#print """========================================================
#%r
#===========================================================
#""" % htmltext
# provide also text output
return htmltext
def runAfterQuery(self, runlist):
import cPickle
for k in self.ResultKey():
with timer("olc afterquery blocks prepare for: %s" % k):
for run in runlist:
run.stats[k] = {}
if not k in run.data: continue
blocks = []
for entry in run.data[k]:
v = entry.value
if k != 'olc:bcidmask':
v = int(v)
if len(blocks) > 0 and blocks[-1][0] == v and blocks[
-1][2] == entry.startlb:
blocks[-1][2] = entry.endlb
else:
blocks += [[v, entry.startlb, entry.endlb]]
run.stats[k] = {
"blocks": blocks,
"first": run.data[k][0].value
}
#f = open("olc.pickle", "w")
#cPickle.dump([run.data,run.stats],f)
#f.close()
with timer("olc afterquery rest"):
from CoolRunQuery.utils.AtlRunQueryUtils import unpackRun1BCIDMask, unpackRun2BCIDMask
for run in runlist:
if run.runNr < 151260:
continue # OLC information was not in COOL before this run
# these contain the number of bunches for each IOV [(nb,lbstart,lbend)(...)...]
xb1 = run.stats['olc:beam1bunches']['blocks']
xb2 = run.stats['olc:beam2bunches']['blocks']
xbc = run.stats['olc:collbunches']['blocks']
#print "JOERG XB1 ",xb1
#print "JOERG XB2 ",xb2
#print "JOERG XBC ",xbc
# loop over BCID mask
bcidmask = run.stats['olc:bcidmask']['blocks']
for i in xrange(len(bcidmask)):
(bcidblob, lbstart, lbend) = bcidmask[i]
# number of bunches
for nb1, b, e in xb1:
if lbstart >= b and lbstart < e: break
for nb2, b, e in xb2:
if lbstart >= b and lbstart < e: break
for nbc, b, e in xbc:
if lbstart >= b and lbstart < e: break
bcidBlobLength = len(bcidblob)
if bcidBlobLength == 3564:
# run 2
beam1, beam2, beam12 = unpackRun2BCIDMask(bcidblob)
#print "BC beam 1: %i, beam 2: %i, coll: %i" % (len(beam1), len(beam2), len(beam12))
else:
# unpack BCID mask
if len(bcidblob) == 2 * (nb1 + nb2 + nbc):
beam1, beam2, beam12 = unpackRun1BCIDMask(
bcidblob, nb1, nb2, nbc)
#print "BC beam 1: %i, beam 2: %i, coll: %i" % (len(beam1), len(beam2), len(beam12))
else:
print "WARNING, bcidMask inconsistent", nb1, nb2, nbc, "should add up to half of", len(
bcidblob)
beam1, beam2, beam12 = ([], [], [])
# refill run stats
bcidmask[i] = ((nb1, nb2, nbc), (beam1, beam2, beam12),
lbstart, lbend)