def output(self, runlist, dic, dicsum, xmlhtmlstr, roothtmlstr):
if self.html:
with timer('GetOKS link info'):
from AtlRunQuerySFO import SetOKSLinks
SetOKSLinks(runlist)
# create web page
from .html.AtlRunQueryHTML import ResultPageMaker
#try:
pageinfo = {
'datapath': self.datapath,
'origQuery': self.origQuery,
'fullQuery': self.parsedstring,
'runlist': runlist,
'dic': dic,
'dicsum': dicsum,
'makeSummary': self.makeSummary,
'makeDQeff': self.makeDQeff,
'makeDQSummary': self.makeDQSummary,
'makeDQPlots': self.makeDQPlots,
'roothtmlstr': roothtmlstr,
'xmlfilename': self.xmlFileName,
'xmlhtmlstr': xmlhtmlstr,
'querytime': time() - self.querystart,
'selout': self.selectionOutput,
}
with timer("run ResultPageMaker makePage"):
ResultPageMaker.makePage(pageinfo)
#except ImportError, ie:
# print "Can't import pagemaker, no web page made",ie
else:
print '---------------------------------------------------------------------'
print Run.header()
runlist.reverse()
for r in runlist:
print r
from CoolRunQuery.utils.AtlRunQueryUtils import addKommaToNumber, filesize
print 'Summary:'
for data_key, summary in dicsum.items():
if data_key.Type == DataKey.STREAM:
key = data_key.ResultKey.replace('STR:', '')
val = '%s (%s)' % (addKommaToNumber(
summary[0]), filesize(summary[1]))
else:
key = data_key.ResultKey
val = addKommaToNumber(summary)
print '%20s : %s' % (key, val)
duration = time() - self.querystart
print "%20s : %g sec" % ('Total execution time', duration)
print '---------------------------------------------------------------------'
def _makeResultsTable( cls, pageinfo ):
for x in pageinfo.keys():
exec("%s = pageinfo['%s']" % (x,x) )
# run results table
resultstable = '<table class="resulttable" id="resulttable" style="margin-left: 13px">\n'
# table head
resultstable += Run.header()
# runs
with timer("print the runs"):
for r in runlist:
with timer("print run %i" % r.runNr):
resultstable += str(r)
# summary
headlist = Run.headerkeys()
resultstable += " <tr class=\"space\"><td style=\"text-align: left;\" colspan=\"%i\"><font size=-2><i>Summary</i>:</font></td></tr>\n" % (len(headlist)-1)
resultstable += " <tr class=\"sum\">" + sumstr + "</tr>\n"
resultstable += "</table>\n"
return resultstable
def runAfterQuery(self, runlist):
# do show selection here
from CoolRunQuery.AtlRunQueryTier0 import GetTier0_allDatasets
runnrlist = [r.runNr for r in runlist]
tier0connection = coolDbConn.GetTier0DBConnection()
tier0retdico = GetTier0_allDatasets(tier0connection.cursor(),
runnrlist, self.selpattern)
for run in runlist: # go through old runlist and see
if tier0retdico.has_key(run.runNr):
run.addResult('Datasets', tier0retdico[run.runNr])
else:
run.addResult('Datasets', {})
Run.AddToShowOrder(DataKey('Datasets'))
Run.showCAFLinks = self.showCAFLinks
def _makeYellowLine( cls, pageinfo ):
for x in ["dicsum"]:
exec("%s = pageinfo['%s']" % (x,x) )
with timer("make the yellow summary line"):
sumdic = {}
for key, summary in dicsum.items():
if key.Type == DataKey.STREAM:
s = addKommaToNumber(summary[0])+' <BR><font size="-2">(%s)</font>' % filesize(summary[1])
elif key=='Run':
s = addKommaToNumber(summary) + " runs"
else:
s = addKommaToNumber(summary)
sumdic[key.ResultKey] = s.strip()
sumstr = ''
headlist = Run.headerkeys()
for title in headlist:
sumstr += '<td style="text-align:right;">%s</td>' % (sumdic[title] if title in sumdic else '')
return sumstr
def runAfterQuery(self, runlist):
# do show selection here
from CoolRunQuery.AtlRunQueryPVSS import GetPVSS_BPMs
pvssdb = coolDbConn.GetPVSSDBConnection()
cursor = pvssdb.cursor()
cursor.arraysize = 1000
pvssretdico = {}
for r in runlist:
t = gmtime(r.sor / 1.E9)
sor = "%02i-%02i-%4i %02i:%02i:%02i" % (t[2], t[1], t[0], t[3],
t[4], t[5])
t = gmtime(r.eor / 1.E9)
eor = "%02i-%02i-%4i %02i:%02i:%02i" % (t[2], t[1], t[0], t[3],
t[4], t[5])
# retrieve values
res = GetPVSS_BPMs(cursor, sor, eor)
r.addResult('BPM', res)
Run.AddToShowOrder(DataKey('BPM'))
def select(self):
start = time()
runlist = []
folder = coolDbConn.GetDBConn(
schema="COOLONL_TRIGGER",
db=Selector.condDB()).getFolder('/TRIGGER/LUMI/LBTIME')
print('SELOUT Checking for runs in time range "%s"' % self.timelist,
end='')
sys.stdout.flush()
ranges = GetRanges(self.timelist, maxval=long(time() * 1E09))
currentRun = None
for rr in ranges:
objs = folder.browseObjects(rr[0], rr[1] + 86400000000000,
cool.ChannelSelection(0))
while objs.goToNext():
obj = objs.currentRef()
payload = obj.payload()
runNr = int(payload['Run'])
if runNr == 0: continue # mistakenly runnr=0 was stored
if runNr > 1 << 30:
# there is a problem with runs between
# usetimes 2009-04-14:00:00:00 2009-04-16:13:00:00
# there the runnumbers are off the chart (> 1<<30)
continue
if not currentRun or runNr != currentRun.runNr:
if currentRun:
currentRun.eor = currentEOR
runlist.append(currentRun)
currentRun = Run(runNr)
currentRun.sor = obj.since()
lbNr = int(payload['LumiBlock'])
currentRun.lbtimes.extend([(0, 0)] *
(lbNr - len(currentRun.lbtimes)))
currentRun.lbtimes[lbNr - 1] = (obj.since(), obj.until())
currentRun.lastlb = lbNr
currentEOR = obj.until()
if currentRun:
currentRun.eor = currentEOR
runlist.append(currentRun)
runlist.sort()
duration = time() - start
print(" ==> %i runs selected (%g sec)" % (len(runlist), duration))
return runlist
def select(self):
if len(self.runranges) == 0: # no run specified
return []
runlist = []
firstRun = self.runranges[0][0]
start = time()
folder = coolDbConn.GetDBConn(
schema="COOLONL_TRIGGER",
db=Selector.condDB(firstRun)).getFolder('/TRIGGER/LUMI/LBLB')
print(self, end='')
sys.stdout.flush()
currentRun = None
for rr in self.runranges:
objs = folder.browseObjects(rr[0] << 32, ((rr[1] + 1) << 32) - 1,
cool.ChannelSelection(0))
while objs.goToNext():
obj = objs.currentRef()
payload = obj.payload()
runNr, lbNr = RunTimeSelector.runlb(obj.since())
if lbNr == 0:
lbNr = 1 # this is an aweful hack to make MC work (there only one LB exists in LBLB and it is nr 0) need to rethink this
if not currentRun or runNr > currentRun.runNr:
if currentRun:
currentRun.eor = currentEOR
runlist.append(currentRun)
currentRun = Run(runNr)
currentRun.sor = payload['StartTime']
currentRun.lbtimes.extend([(0, 0)] *
(lbNr - len(currentRun.lbtimes)))
currentRun.lbtimes[lbNr - 1] = (payload['StartTime'],
payload['EndTime'])
currentRun.lastlb = lbNr
currentEOR = payload['EndTime']
if currentRun:
currentRun.eor = currentEOR
runlist.append(currentRun)
runlist.sort()
duration = time() - start
print(" ==> %i runs found (%.2f sec)" % (len(runlist), duration))
return runlist
def select(self, runlist):
# some preparation: compile the show patterns
start = time()
if '.*' in self.showstreampatterns:
compiledShowPatterns = [re.compile('.*')]
else:
compiledShowPatterns = [
re.compile(p) for p in self.showstreampatterns
]
# we disable the access to everything that is not needed if we only select
ShowStreams = False
useTier0 = False
if len(compiledShowPatterns) > 0:
ShowStreams = True
useTier0 = True
print(self, end='')
sys.stdout.flush()
newrunlist = []
allStreams = Set(
) # list of all the streams that are in the selected runs
connection = coolDbConn.GetSFODBConnection()
cursor = connection.cursor()
cursor.arraysize = 1000
runnrlist = [r.runNr for r in runlist]
from CoolRunQuery.AtlRunQuerySFO import GetSFO_streamsAll, GetSFO_filesAll
with timer('GetSFO_streamsAll'):
streamsall = GetSFO_streamsAll(cursor,
runnrlist) # { runnr: [streams] }
with timer('GetSFO_filesAll'):
filesall = GetSFO_filesAll(
cursor,
runnrlist) # [(COUNT(FILESIZE), SUM(FILESIZE), SUM(NREVENTS))]
if ShowStreams:
from CoolRunQuery.AtlRunQuerySFO import GetSFO_LBsAll, GetSFO_NeventsAll, GetSFO_overlapAll
with timer('GetSFO_LBsAll'):
lbinfoall = GetSFO_LBsAll(
cursor, runnrlist
) # [(MIN(LUMIBLOCKNR), MAX(LUMIBLOCKNR), #LUMIBLOCKS)]
with timer('GetSFO_overlapAll'):
overlapall = GetSFO_overlapAll(
cursor, runnrlist) # [(SUM(OVERLAP_EVENTS))]
smallrunnrlist = []
for r in runnrlist: # go through old runlist and see
if not r in streamsall: continue
for s in streamsall[r]:
if r in lbinfoall and s in lbinfoall[
r] and lbinfoall[r][s][1] > 0:
smallrunnrlist += [r]
break
with timer('GetSFO_NeventsAll'):
neventsall = GetSFO_NeventsAll(
cursor, smallrunnrlist) # [(LUMIBLOCKNR, NREVENTS)]
for run in runlist: # go through old runlist and see
streams = [] # names of all streams in this run
strsize = [] # size of streams in this run
strevents = [] # nr of events in this run
if run.runNr in streamsall:
streams = streamsall[run.runNr]
for s in streams:
try:
nfiles, size, events = filesall[run.runNr][s]
except:
nfiles, size, events = (0, 0, 0)
strsize += [size]
strevents += [events]
if ShowStreams:
from CoolRunQuery.utils.AtlRunQueryUtils import Matrix
nst = len(streams)
stovmat = Matrix(nst, nst) # overlap matrix
for i, s in enumerate(streams):
run.stats['STR:' + s] = {}
# fill overlaps into matrix
for j, s2 in enumerate(streams):
try:
eventsij = overlapall[run.runNr][s][s2]
except KeyError:
eventsij = 0
if i == j and events:
eventsij = events
stovmat.setitem(i, j, float(eventsij))
stovmat.setitem(j, i,
float(eventsij)) # symmetrise matrix
# read number of events per LB
minlb, maxlb, lbs = (0, 0, 1)
try:
minlb, maxlb, lbs = lbinfoall[run.runNr][s]
except KeyError:
pass
# if minlb==maxlb==0 -> no file closure at LB boundary
if minlb == 0 and maxlb == 0:
run.stats['STR:' + s]['LBRecInfo'] = None
continue
else:
lbevcount = '<tr>'
result = neventsall[run.runNr][s] #[ (lb,nev),... ]
lbold = -1
allnev = 0
ic = 0
ice = 0
allic = 0
lastElement = False
firstcall = True
for ice, (lb, nev) in enumerate(result):
if ice == len(result):
lastElement = True
allnev += nev
if lb != lbold or lastElement:
if lbold != -1:
ic += 1
allic += 1
if allic < 101:
if ic == 9:
ic = 1
lbevcount += '</tr><tr>'
lbevcount += '<td style="font-size:75%%">%i (%s)</td>' % (
lbold, allnev)
else:
if firstcall:
lbevcount += '</tr><tr>'
lbevcount += '<td style="font-size:75%%" colspan="8">... <i>too many LBs (> 100) to show</i></td>'
firstcall = False
allnev = nev
lbold = lb
else:
allnev += nev
lbevcount += '</tr>'
run.stats['STR:' + s]['LBRecInfo'] = lbevcount
run.stats['STR:' + s]['LBRecInfo'] = result
# add overlap information to the run stats
for i in xrange(nst):
statkey = 'STR:' + streams[i]
run.stats[statkey]['StrOverlap'] = []
denom = stovmat.getitem(i, i)
if denom == 0: continue
for j in xrange(nst):
if i == j or stovmat.getitem(i, j) == 0: continue
fraction = 100
if stovmat.getitem(i, j) != denom:
fraction = float(stovmat.getitem(
i, j)) / float(denom) * 100.0
run.stats[statkey]['StrOverlap'] += [(streams[j],
fraction)]
# selection...
if not self.passes(zip(streams, strevents), 0): continue
newrunlist += [run.runNr]
allStreams.update(streams)
for k, v, s in zip(streams, strevents, strsize):
run.addResult('STR:' + k, (v, s))
allStreams = ['STR:' + s for s in allStreams]
allStreams.sort(lambda x, y: 2 * cmp(y[4], x[4]) + cmp(x[5:], y[5:]))
# fill the gaps
for run in runlist:
for s in allStreams:
if not s in run.result:
run.addResult(s, 'n.a.')
runlist = [r for r in runlist if r.runNr in newrunlist]
# check if the streams in 'allStreams' match the show patterns
for s in allStreams:
if any([p.match(s[4:]) != None for p in compiledShowPatterns]):
Run.AddToShowOrder(DataKey(s, keytype=DataKey.STREAM))
if useTier0:
# retrieve Tier-0 information
from CoolRunQuery.AtlRunQueryTier0 import GetTier0_datasetsAndTypes
tier0connection = coolDbConn.GetTier0DBConnection()
cursor = tier0connection.cursor()
cursor.arraysize = 1000
tier0retdico = GetTier0_datasetsAndTypes(cursor, runnrlist)
# add Tier0 information
for run in runlist:
for s in allStreams:
if run.result[s] == 'n.a.': continue
run.stats[s]['StrTier0TypesRAW'] = {}
run.stats[s]['StrTier0TypesESD'] = {}
run.stats[s]['StrTier0AMI'] = {}
if run.runNr in tier0retdico.keys():
for dsname, t, pstates in tier0retdico[run.runNr]:
if s.replace('STR:', '') in dsname:
if '.RAW' in dsname:
if '.merge' in dsname:
prodstep = 'StrTier0TypesESD'
t = '(RAW)'
else:
prodstep = 'StrTier0TypesRAW'
t = '(RAW)'
else:
if '.recon.' in dsname:
prodstep = 'StrTier0TypesRAW'
else:
prodstep = 'StrTier0TypesESD'
if not run.stats[s]['StrTier0AMI'].has_key(
prodstep):
dsnamesplit = dsname.split('.')
if len(dsnamesplit) > 5:
amitag = dsnamesplit[5]
run.stats[s]['StrTier0AMI'][
prodstep] = amitag
else:
amitag = ''
# check if on CAF
oncaf = False
if pstates and 'replicate:done' in pstates:
oncaf = True
# fill the run stats
if not run.stats[s][prodstep].has_key(t):
run.stats[s][prodstep][t] = oncaf
# Done
duration = time() - start
if len(self.streams) != 0:
print(" ==> %i runs found (%.2f sec)" % (len(runlist), duration))
else:
print(" ==> Done (%g sec)" % duration)
return runlist
def updateShowOrder(self):
from CoolRunQuery.AtlRunQueryRun import Run
for cd in self.ChannelDesc():
if cd.SelectShowRetrieve<2: continue
Run.AddToShowOrder(cd)
def makeHTML(cls,
dic,
dicsum,
doPickle=True,
doDQSummary=True,
doDQPlots=True):
""" method returns a string (unicode is fine) of html code, with out tag <div>...</div> """
#####################################################################
# Jessica, your main code goes here #
# (note that all methods in the class are 'static' -> @classmethod #
#####################################################################
### Global Variables ###
with timer("DQSummary"):
### loop once over runs to decide between pb/nb###
global usenb
usenb = False
global livetrigger
livetrigger = 'L1_EM30'
for r, run in enumerate(dic[DataKey('Run')]):
ptag = dic[DataKey('Project tag')][r][0].value
if 'data13' in ptag:
usenb = True
livetrigger = 'L1_EM5'
elif 'data15' in ptag or 'data16' in ptag or 'data17' in ptag or 'data18' in ptag:
usenb = False
livetrigger = 'L1_EM12'
global unit
unit = 'pb'
if usenb: unit = 'nb'
### WARNING messages
warning = ''
### Total Lumi : 0 = Recorded 1 = Loss (in ATLAS Ready)
dicsum[DataKey('TotalLumi')] = [0., 0.]
### ATLAS not Ready Total/Per run (in stable Beams)
dicsum[DataKey('TotalNotReady')] = 0.
dicsum[DataKey('TotalBusy')] = 0.
### Initialize Detectors Variables
detectors = [
'ALFA', 'LCD', 'ZDC', 'BCM', 'PIXEL', 'SCT', 'TRT', 'LAR',
'TILE', 'MS_CSC', 'MS_RPC', 'MS_TGC', 'MS_MDT', 'MBTS', 'TRIG',
'GLOBAL'
]
detectors_color = {
'ALFA': kYellow - 9,
'LCD': kYellow - 7,
'ZDC': kYellow - 4,
'BCM': kOrange + 1,
'PIXEL': kOrange - 3,
'SCT': kOrange + 7,
'TRT': kRed - 3,
'LAR': kRed + 2,
'TILE': kPink - 7,
'MS_CSC': kBlue + 4,
'MS_RPC': kBlue - 2,
'MS_MDT': kAzure + 3,
'MS_TGC': kBlue - 9,
'MBTS': kBlue - 6,
'TRIG': kGreen - 3,
'GLOBAL': kGreen + 3,
'_TOTAL': TColor.GetColor("#585858")
}
detectors_lumiloss = {}
detectors_affectedLBs = {}
for d in detectors:
detectors_lumiloss[d] = {}
detectors_affectedLBs[d] = {}
## Additional field for TOTAL
detectors_lumiloss['_TOTAL'] = {}
detectors_affectedLBs['_TOTAL'] = {}
### Initialize Performances Variables
performances = [
'LUMI', 'ID', 'CALO', 'TAU', 'EGAMMA', 'JET', 'MET', 'MCP',
'BTAG'
]
performances_color = {
'LUMI': kYellow - 9,
'ID': kOrange + 7,
'CALO': kRed + 2,
'TAU': kBlue + 2,
'EGAMMA': kGreen - 3,
'JET': kAzure + 3,
'MET': kGreen - 6,
'MCP': kBlue - 3,
'BTAG': kPink,
'_TOTAL': TColor.GetColor("#585858")
}
performances_lumiloss = {}
performances_affectedLBs = {}
for p in performances:
performances_lumiloss[p] = {}
performances_affectedLBs[p] = {}
## Additional field for TOTAL
performances_lumiloss['_TOTAL'] = {}
performances_affectedLBs['_TOTAL'] = {}
### Initialize table content // Considered systems ###
columns = []
global_systems = []
columns = ['Run Info', 'ES1', 'BLK']
global_systems = ['Calo', 'Tracking', 'Muon', 'Trigger & Lumi']
columns += global_systems
with timer('Create summary results table'):
### create big results table ###
summarytable = '<div><table id="resulttable" class="resulttable" style="margin-left: 13px; margin-right: 13px;" >'
### create first title row ###
summarytable += '<tr>'
summarytable += '<th colspan="1"></th>' # run info column
#summarytable += '<th colspan="2">Tier0 Reco Status</th>'
summarytable += '<th colspan="2">Missing Sign-Off</th>'
summarytable += '<th colspan="%i">Defects in ATLAS Ready<br>' % len(
global_systems)
summarytable += '<div style="font-size: xx-small; cursor: pointer;" onclick="toggle_dq(this)">[show tolerable]</div></th>'
summarytable += '</tr>'
### create subtitle row ###
summarytable += '<tr>'
#for item in columns: summarytable += '<th>%s</th>'%item.split(":")[0]
for item in columns:
summarytable += '<th>%s</th>' % item
summarytable += '</tr>'
totalNumberOfReadyLB = 0
### loop over runs ###
for r, run in enumerate(dic[DataKey('Run')]):
## Get list of ATLAS Ready LBs
global readylb
readylb = GetLBReady(dic, r)
totalNumberOfReadyLB += len(readylb)
## If no ATLAS ready LB: skip the run
if len(readylb) == 0:
warning += '<center><font color="red">WARNING! Run %s has 0 ATLAS Ready LB</font></center><br>' % run
continue
## Get lumi per LB and live fraction for the current run
lumiperlb = GetLBLumi(dic, r)
livefrac = GetLBLiveFraction(dic, r)
if len(lumiperlb) == 0:
warning += '<center><font color="red">WARNING! Run %s has no offline lumi info</font></center><br>' % run
continue
## Correct lumi per LB with live fraction
#print 'DEBUG',run,len(lumiperlb),len(livefrac),len(readylb)
for l in lumiperlb:
# IF statement used when len(lumiperlb)!= len(livefrac)
if l not in readylb: continue
if l not in livefrac:
print "--> Warning: live fraction not available for LB %i. Setting live fraction to 1." % l
else:
lumiperlb[l] *= livefrac[l]
## Initialize columns content for current run
content = {}
for i, item in enumerate(columns):
content[item] = ''
## Retrieve and store run info
dp = '?'
ptag = dic[DataKey('Project tag')][r][0].value
year = '20' + ptag.split('_')[0][4:]
DP = ARQ_COMA.get_periods_for_run(run)
if len(DP) > 0: dp = DP[0]
content[
'Run Info'] = '<b>%s</b> <font style="font-size:10px;">data period <b>%s</b></font><br>' % (
str(run), dp)
### In case some systems should be removed
tobeignored = ['ALFA', 'ZDC', 'LCD']
import re
if re.match('data1[3-9]', ptag[0:6]): tobeignored = []
## useful links
target = 'target="blank"'
somelinks = '<font style="font-size:10px;color:#0000FF">'
somelinks += '<a href="https://atlas-tagservices.cern.ch/tagservices/RunBrowser/runBrowserReport/runBrowserReport.php?runs=%s&pn=%s&fnt=%s" %s>COMA</a>' % (
run, dp, ptag, target)
somelinks += ', <a href="https://atlasdqm.web.cern.ch/atlasdqm/DQBrowser/makeMatrix.php?selection=All&run=%s&runup=&dbinstance=CONDBR2_DCS&tag=DEFAULT" %s>DCS</a>' % (
run, target)
somelinks += ', <a href="https://atlas.web.cern.ch/Atlas/GROUPS/DATAPREPARATION/DataSummary/%s/run.py?run=%s" %s>DS</a>' % (
year, run, target)
somelinks += ', <a href="https://atlasdqm.cern.ch/defectentry/?run=%s" %s>defects</a>' % (
run, target)
somelinks += '<br><a href="https://atlasdqm.cern.ch/dqsignoff/%s/" %s>DQlogbook</a>' % (
run, target)
somelinks += ', <a href="https://atlasdqm.cern.ch/webdisplay/tier0?lowrun=%s&highrun=%s&show_all=on" %s>DQWebDisplay</a>' % (
run, run, target)
somelinks += '</font><br>'
content['Run Info'] += somelinks
## Start and End time
rundate = dic[DataKey('Start and endtime')][r].split(",")
tstart = time.strftime("%a, %d %b %Y %H:%M:%S",
time.localtime(float(rundate[0])))
tend = time.strftime("%a, %d %b %Y %H:%M:%S",
time.localtime(float(rundate[1])))
content['Run Info'] += '<font style="font-size:10px;">'
content[
'Run Info'] += '<br>Start: <font style="color:#4C7D7E;">%s</font>' % tstart
content[
'Run Info'] += '<br>End: <font style="color:#4C7D7E;">%s</font>' % tend
content['Run Info'] += '</font>'
#nevents = str(dic[DataKey('#Events')][r][0].value)
#content['Run Info'] += 'Recorded events: '+nevents+'<br>'
#content['Run Info'] += 'Number of LBs : '+dic[DataKey('#LB')][r][0]+'<br>'
#content['Run Info'] += 'LHC Fill : %s<br>'%(dic[DataKey('lhc:fillnumber')][r][0].value)+'<br>'
## Add stable Beam / ATLAS Ready info in lhctable
lhctable = '<br><font style="font-size:10px;">'
lhctable += '<table><tr><td>'
lhctable += '<b>Stable Beam</b><br>%s' % '<br>'.join([
'%s-%s: <font style="color:#4C7D7E">%s</font>' %
(b.startlb, b.endlb, b.value)
for b in dic[DataKey('lhc:stablebeams')][r]
])
lhctable += '</td><td>'
lhctable += '<b>ATLAS Ready</b><br>%s' % '<br>'.join([
'%s-%s: <font style="color:#4C7D7E">%s</font>' %
(b.startlb, b.endlb, b.value)
for b in dic[DataKey('Ready for physics')][r]
])
lhctable += '</td></tr></table>'
lhctable += '</font>'
content['Run Info'] += lhctable
## Add Total Lumi
lumitot = ComputeRunLumi(dic, r, lumiperlb)
dicsum[DataKey('TotalLumi')][0] += lumitot
content['Run Info'] += '<font style="font-size:10px;">'
content[
'Run Info'] += 'Ready Recorded: <b>%.2f</b> %s<sup>-1</sup>' % (
lumitot, unit)
content['Run Info'] += '</font><br>'
## Retrieve DQ defects
defects = dic[DataKey('DQ')][r]
## Select only primary flags
defects_primary = [d for d in defects if d.value.primary]
## Initialize list of affected LBs per detector/cp
## Initialize lumi losses due to intolerable defects
for d in detectors:
detectors_lumiloss[d][run] = 0.
detectors_affectedLBs[d][run] = []
for p in performances:
performances_lumiloss[p][run] = 0.
performances_affectedLBs[p][run] = []
# Total per run
detectors_lumiloss['_TOTAL'][run] = 0.
detectors_affectedLBs['_TOTAL'][run] = []
performances_lumiloss['_TOTAL'][run] = 0.
performances_affectedLBs['_TOTAL'][run] = []
# And the big sums
total_affectedLBs = []
GlobalNotReady = []
GlobalBusy = []
## Store list of defects and affected LBs to print systems table
ListOfIntolerableDefects = {}
ListOfTolerableDefects = {}
## Loop over defects
for item in defects_primary:
defect = item.value.defect
comment = item.value.comment
user = item.value.user
startlb = item.startlb
endlb = item.endlb
tolerable = item.value.tolerable
system = MapToSystem(defect)
if 'GLOBAL_NOTREADY' in defect:
GlobalNotReady += [lb for lb in range(startlb, endlb)]
if 'GLOBAL_BUSY' in defect:
GlobalBusy += [
lb for lb in range(startlb, endlb) if lb in readylb
]
## Missing Sign-Offs ##
## FINAL sign-off
if 'UNCHECKED_FINAL' in defect: continue
## BULK sign-off
if 'BULK_UNCHECKED' in defect:
short = defect.split('_UNCHECKED')[0]
content[
'BLK'] += '<font style="font-size:8px;font-weight:bold;">%s</font><br>' % short
continue
## ES sign-off
if 'UNCHECKED' in defect:
short = defect.split('_UNCHECKED')[0]
content[
'ES1'] += '<font style="font-size:8px;font-weight:bold;">%s</font><br>' % short
continue
## Some cross-checks
if system == '':
print 'run %s: this defect is fishy %s ' % (run,
defect)
continue
## Some cross-checks
word = defect.split('_')
cpdet = word[0]
if word[0] == 'MS': cpdet += "_" + word[1] # MS systems
if not cpdet in detectors and not cpdet in performances:
print 'This system is not included: %s (%s)' % (cpdet,
defect)
continue
## Store intolerable defects if in ATLAS Ready LBs only
if not tolerable:
RangeDefect = [
lb for lb in range(startlb, endlb) if lb in readylb
]
if len(RangeDefect) > 0:
if not defect in ListOfIntolerableDefects:
ListOfIntolerableDefects[defect] = [[], '']
ListOfIntolerableDefects[defect][
0] = RangeDefect
ListOfIntolerableDefects[defect][
1] = user + ':' + comment
else:
ListOfIntolerableDefects[defect][
0] += RangeDefect
if comment not in ListOfIntolerableDefects[
defect][1]:
ListOfIntolerableDefects[defect][
1] += ' ' + user + ':' + comment
# This is used to compute lumilosses
# Here, we do not include systems "to be ignored"
if cpdet in tobeignored: continue
# Store list of affected LBs per detector/cp group
# we can have a double counting of LBs if defects overlap - fixed later
if cpdet in detectors:
detectors_affectedLBs[cpdet][
run] += RangeDefect
if cpdet in performances:
performances_affectedLBs[cpdet][
run] += RangeDefect
total_affectedLBs += RangeDefect
## Store tolerable defects
else:
RangeDefect = [
lb for lb in range(startlb, endlb) if lb in readylb
]
if len(RangeDefect) > 0:
if not defect in ListOfTolerableDefects:
ListOfTolerableDefects[defect] = [[], '']
ListOfTolerableDefects[defect][0] = RangeDefect
ListOfTolerableDefects[defect][
1] = '[%s]:%s ' % (user, comment)
else:
ListOfTolerableDefects[defect][
0] += RangeDefect
if comment not in ListOfTolerableDefects[
defect][1]:
ListOfTolerableDefects[defect][
1] += '[%s]:%s ' % (user, comment)
## end of defects loop ##
## Create defects table for each system
for item in global_systems:
content[item] = '<table class="dqdefects">'
for item in ListOfIntolerableDefects:
system = MapToSystem(item)
if not system in global_systems: continue
lbs = listify(ListOfIntolerableDefects[item][0])
tip = ListOfIntolerableDefects[item][1]
tipkey = "dqcomment_int_%s_%s" % (item, run)
Run.addGlobalToolTip(tipkey, tip)
content[system] += '<tr class="showTip %s" >' % (tipkey)
content[system] += '<td class="intolerable">%s</td>' % (
item)
content[system] += '<td class="lb">%s</td>' % (lbs)
content[system] += '</tr>'
for item in ListOfTolerableDefects:
system = MapToSystem(item)
if not system in global_systems: continue
lbs = listify(ListOfTolerableDefects[item][0])
tip = ListOfTolerableDefects[item][1]
tipkey = "dqcomment_tol_%s_%s" % (item, run)
Run.addGlobalToolTip(tipkey, tip)
content[
system] += '<tr class="showTip %s tolerable" style="visibility:collapse;">' % (
tipkey)
content[system] += '<td>%s</td>' % (item)
content[system] += '<td class="lb">%s</td>' % (lbs)
content[system] += '</tr>'
## close systems defects tables
for item in global_systems:
content[item] += '</table><br>'
## Add defects plots in each system column
thumbsize = 70
imgsize = 600
for sys in global_systems:
tol = {}
int = {}
for defect in ListOfTolerableDefects:
# remove systems to be ignored from the plots
word = defect.split('_')
cpdet = word[0]
if word[0] == 'MS':
cpdet += "_" + word[1] # MS systems
if cpdet in tobeignored: continue
if sys == MapToSystem(defect):
tol[defect] = ListOfTolerableDefects[defect][0]
for defect in ListOfIntolerableDefects:
# remove systems to be ignored from the plots
word = defect.split('_')
cpdet = word[0]
if word[0] == 'MS':
cpdet += "_" + word[1] # MS systems
if cpdet in tobeignored: continue
if sys == MapToSystem(defect):
int[defect] = ListOfIntolerableDefects[defect][0]
hname = MakePlot_DefectsPerSystem(sys, int, tol, dic, r)
for h in hname:
if len(h) == 0: continue
title = "Click to zoom"
wincontent = "<img src="%s" height=%s/>" % (
h, imgsize)
openwin = "javascript:openLargeWindow('Print1','"
openwin += "<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">"
openwin += "<html xmlns:"my"><head><title>Defects for run %s</title></head>" % run
openwin += "<body style="background-color:#ffffff">%s</body></html>" % wincontent
openwin += "')"
content[
sys] += '<a title="%s" href="%s" ><img src="%s" height=%s/></a>' % (
title, openwin, h, thumbsize)
## Compute Global not ready ##
l = 0.
for lb in GlobalNotReady:
l += lumiperlb[lb]
dicsum[DataKey('TotalNotReady')] += l
content['Run Info'] += '<font style="font-size:10px;">'
content[
'Run Info'] += 'Global Not Ready: <b>%.2f</b> %s<sup>-1</sup>' % (
l, unit)
content['Run Info'] += '</font><br>'
## Compute Global Busy ##
l = 0.
for lb in GlobalBusy:
l += lumiperlb[lb]
dicsum[DataKey('TotalBusy')] += l
content['Run Info'] += '<font style="font-size:10px;">'
content[
'Run Info'] += 'Global Busy: <b>%.2f</b> %s<sup>-1</sup>' % (
l, unit)
content['Run Info'] += '</font><br>'
## Compute cp/det lumilosses for current run
for d in detectors:
if len(detectors_affectedLBs[d][run]) == 0: continue
# Remove double counting (defects overlap)
dll = list(set(detectors_affectedLBs[d][run]))
detectors_affectedLBs['_TOTAL'][run] += dll
for lb in dll:
detectors_lumiloss[d][run] += lumiperlb[lb]
for p in performances:
if len(performances_affectedLBs[p][run]) == 0: continue
# Remove double counting (defects overlap)
pll = list(set(performances_affectedLBs[p][run]))
performances_affectedLBs['_TOTAL'][run] += pll
for lb in pll:
performances_lumiloss[p][run] += lumiperlb[lb]
## Compute total Lumiloss for this run
## Remove double counting (defects overlap)
detectors_affectedLBs['_TOTAL'][run] = list(
set(detectors_affectedLBs['_TOTAL'][run]))
performances_affectedLBs['_TOTAL'][run] = list(
set(performances_affectedLBs['_TOTAL'][run]))
total_affectedLBs = list(set(total_affectedLBs))
totallossperrun = 0.
# Store the values
for lb in total_affectedLBs:
totallossperrun += lumiperlb[lb]
dicsum[DataKey('TotalLumi')][1] += totallossperrun
# Store the values
for lb in detectors_affectedLBs['_TOTAL'][run]:
detectors_lumiloss['_TOTAL'][run] += lumiperlb[lb]
# Store the values
for lb in performances_affectedLBs['_TOTAL'][run]:
performances_lumiloss['_TOTAL'][run] += lumiperlb[lb]
## Add Total LumiLoss in run info column
content['Run Info'] += '<font style="font-size:10px;">'
content[
'Run Info'] += 'DQ Lumi Loss: <b>%.2f</b> %s<sup>-1</sup>' % (
totallossperrun, unit)
content['Run Info'] += '</font><br>'
### Print run row ###
summarytable += '<tr class="out2">'
for item in columns:
summarytable += '<td>%s</td>' % content[item]
summarytable += '</tr>'
### end of run loop ###
summarytable += '</table></div><br>'
### end of results table ###
##########################
## Create Summary Plots ##
##########################
summaryplot = ''
from CoolRunQuery.AtlRunQueryQueryConfig import QC
plots = []
plots.append(
MakePlot_SummaryLumiLoss(detectors_lumiloss, detectors_color,
dicsum, 'detectors'))
plots.append(
MakePlot_SummaryLumiLoss(performances_lumiloss, performances_color,
dicsum, 'performances'))
plots.append(
MakePlot_PerRunLumiLoss(detectors_lumiloss, detectors_color,
dicsum, 'detectors'))
plots.append(
MakePlot_PerRunLumiLoss(performances_lumiloss, performances_color,
dicsum, 'performances'))
imgsize = 500
thumbsize = 300
title = "Click to zoom"
## Start plots table at the top of the page
summaryplot += '<table align="center" width="90%">'
summaryplot += '<th>'
for p in plots:
wincontent = "<img src="%s" width=%s/>" % (p, imgsize)
#openwincmd = "javascript:openLargeWindow('Print%i','"%(r)
openwincmd = "javascript:openWindow('Print%i','" % (run)
openwincmd += "<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">"
openwincmd += "<html xmlns:"my"><head><title>DQ Summary Plot - Run %s</title></head>" % (
run)
openwincmd += "<body style="background-color:#ffffff">%s</body></html>" % wincontent
openwincmd += "')"
summaryplot += '<td><a title="%s" href="%s" ><img src="%s" height=%s/></a></td>' % (
title, openwincmd, p, thumbsize)
## End plots table
summaryplot += '</th></table>'
## debug ##
#s = "<div>%s</div>" % '<br>'.join([ "%s: %s" % (k.ResultKey,v) for k,v in dicsum.items()])
#s = "<div>%s</div>" % '<br>'.join([ "%s: %s" % (k.ResultKey,v) for k,v in dic.items()])
#print s
###########################
## Print Summary numbers ##
###########################
totalLumiRecorded = dicsum[DataKey('TotalLumi')][0]
totalLumiLoss = dicsum[DataKey('TotalLumi')][1]
lumiLossFraction = (100. * totalLumiLoss /
totalLumiRecorded) if totalLumiRecorded > 0 else 0
print '+++++++++++++++++++++++++ Summary +++++++++++++++++++'
print ' Total Ready Recorded Luminosity: %.2f %s-1' % (
totalLumiRecorded, unit)
print ' Total Luminosity Loss (ATLAS Ready) : %.2f %s-1' % (
totalLumiLoss, unit)
print ' Total Global Not Ready (Stable Beams): %.2f %s-1' % (
dicsum[DataKey('TotalNotReady')], unit)
print ' Total Global Busy (Stable Beams): %.2f %s-1' % (
dicsum[DataKey('TotalBusy')], unit)
print '+++++++++++++++++++++++++++++++++++++++++++++++++++++'
global lumifolder
if totalNumberOfReadyLB > 0:
summaryinfo = '<table align="center" style="font-size:80%;"><tr>'
summaryinfo += '<td> Total Luminosity Loss: <b>%.2f %s<sup>-1</sup></b> (%.2f%%)' % (
totalLumiLoss, unit, lumiLossFraction)
summaryinfo += '<br>Excluded Systems: %s</td></tr>' % tobeignored
summaryinfo += '<tr><td style="font-size:70%%">using %s // %s</td></tr>' % (
lumifolder.split(':')[2], livetrigger)
summaryinfo += '</table>'
else:
summaryinfo = '<table align="center" style="font-size:80%;"><tr>'
summaryinfo += '<td> No ready lumi block </td>'
summaryinfo += '</tr></table>'
#############################
## return web page content ##
#############################
pagecontent = summaryplot + '<br>'
pagecontent += summaryinfo + '<br>'
if doDQSummary: pagecontent += summarytable
pagecontent += warning
return pagecontent