def runAfterQuery(self, runlist):
# do show selection here
from CoolRunQuery.AtlRunQueryTier0 import GetTier0_allDatasets
runnrlist = [r.runNr for r in runlist]
tier0connection = coolDbConn.GetTier0DBConnection()
tier0retdico = GetTier0_allDatasets(tier0connection.cursor(),
runnrlist, self.selpattern)
for run in runlist: # go through old runlist and see
if tier0retdico.has_key(run.runNr):
run.addResult('Datasets', tier0retdico[run.runNr])
else:
run.addResult('Datasets', {})
Run.AddToShowOrder(DataKey('Datasets'))
Run.showCAFLinks = self.showCAFLinks
def runAfterQuery(self, runlist):
# do show selection here
from CoolRunQuery.AtlRunQueryPVSS import GetPVSS_BPMs
pvssdb = coolDbConn.GetPVSSDBConnection()
cursor = pvssdb.cursor()
cursor.arraysize = 1000
pvssretdico = {}
for r in runlist:
t = gmtime(r.sor / 1.E9)
sor = "%02i-%02i-%4i %02i:%02i:%02i" % (t[2], t[1], t[0], t[3],
t[4], t[5])
t = gmtime(r.eor / 1.E9)
eor = "%02i-%02i-%4i %02i:%02i:%02i" % (t[2], t[1], t[0], t[3],
t[4], t[5])
# retrieve values
res = GetPVSS_BPMs(cursor, sor, eor)
r.addResult('BPM', res)
Run.AddToShowOrder(DataKey('BPM'))
def select(self, runlist):
# some preparation: compile the show patterns
start = time()
if '.*' in self.showstreampatterns:
compiledShowPatterns = [re.compile('.*')]
else:
compiledShowPatterns = [
re.compile(p) for p in self.showstreampatterns
]
# we disable the access to everything that is not needed if we only select
ShowStreams = False
useTier0 = False
if len(compiledShowPatterns) > 0:
ShowStreams = True
useTier0 = True
print(self, end='')
sys.stdout.flush()
newrunlist = []
allStreams = Set(
) # list of all the streams that are in the selected runs
connection = coolDbConn.GetSFODBConnection()
cursor = connection.cursor()
cursor.arraysize = 1000
runnrlist = [r.runNr for r in runlist]
from CoolRunQuery.AtlRunQuerySFO import GetSFO_streamsAll, GetSFO_filesAll
with timer('GetSFO_streamsAll'):
streamsall = GetSFO_streamsAll(cursor,
runnrlist) # { runnr: [streams] }
with timer('GetSFO_filesAll'):
filesall = GetSFO_filesAll(
cursor,
runnrlist) # [(COUNT(FILESIZE), SUM(FILESIZE), SUM(NREVENTS))]
if ShowStreams:
from CoolRunQuery.AtlRunQuerySFO import GetSFO_LBsAll, GetSFO_NeventsAll, GetSFO_overlapAll
with timer('GetSFO_LBsAll'):
lbinfoall = GetSFO_LBsAll(
cursor, runnrlist
) # [(MIN(LUMIBLOCKNR), MAX(LUMIBLOCKNR), #LUMIBLOCKS)]
with timer('GetSFO_overlapAll'):
overlapall = GetSFO_overlapAll(
cursor, runnrlist) # [(SUM(OVERLAP_EVENTS))]
smallrunnrlist = []
for r in runnrlist: # go through old runlist and see
if not r in streamsall: continue
for s in streamsall[r]:
if r in lbinfoall and s in lbinfoall[
r] and lbinfoall[r][s][1] > 0:
smallrunnrlist += [r]
break
with timer('GetSFO_NeventsAll'):
neventsall = GetSFO_NeventsAll(
cursor, smallrunnrlist) # [(LUMIBLOCKNR, NREVENTS)]
for run in runlist: # go through old runlist and see
streams = [] # names of all streams in this run
strsize = [] # size of streams in this run
strevents = [] # nr of events in this run
if run.runNr in streamsall:
streams = streamsall[run.runNr]
for s in streams:
try:
nfiles, size, events = filesall[run.runNr][s]
except:
nfiles, size, events = (0, 0, 0)
strsize += [size]
strevents += [events]
if ShowStreams:
from CoolRunQuery.utils.AtlRunQueryUtils import Matrix
nst = len(streams)
stovmat = Matrix(nst, nst) # overlap matrix
for i, s in enumerate(streams):
run.stats['STR:' + s] = {}
# fill overlaps into matrix
for j, s2 in enumerate(streams):
try:
eventsij = overlapall[run.runNr][s][s2]
except KeyError:
eventsij = 0
if i == j and events:
eventsij = events
stovmat.setitem(i, j, float(eventsij))
stovmat.setitem(j, i,
float(eventsij)) # symmetrise matrix
# read number of events per LB
minlb, maxlb, lbs = (0, 0, 1)
try:
minlb, maxlb, lbs = lbinfoall[run.runNr][s]
except KeyError:
pass
# if minlb==maxlb==0 -> no file closure at LB boundary
if minlb == 0 and maxlb == 0:
run.stats['STR:' + s]['LBRecInfo'] = None
continue
else:
lbevcount = '<tr>'
result = neventsall[run.runNr][s] #[ (lb,nev),... ]
lbold = -1
allnev = 0
ic = 0
ice = 0
allic = 0
lastElement = False
firstcall = True
for ice, (lb, nev) in enumerate(result):
if ice == len(result):
lastElement = True
allnev += nev
if lb != lbold or lastElement:
if lbold != -1:
ic += 1
allic += 1
if allic < 101:
if ic == 9:
ic = 1
lbevcount += '</tr><tr>'
lbevcount += '<td style="font-size:75%%">%i (%s)</td>' % (
lbold, allnev)
else:
if firstcall:
lbevcount += '</tr><tr>'
lbevcount += '<td style="font-size:75%%" colspan="8">... <i>too many LBs (> 100) to show</i></td>'
firstcall = False
allnev = nev
lbold = lb
else:
allnev += nev
lbevcount += '</tr>'
run.stats['STR:' + s]['LBRecInfo'] = lbevcount
run.stats['STR:' + s]['LBRecInfo'] = result
# add overlap information to the run stats
for i in xrange(nst):
statkey = 'STR:' + streams[i]
run.stats[statkey]['StrOverlap'] = []
denom = stovmat.getitem(i, i)
if denom == 0: continue
for j in xrange(nst):
if i == j or stovmat.getitem(i, j) == 0: continue
fraction = 100
if stovmat.getitem(i, j) != denom:
fraction = float(stovmat.getitem(
i, j)) / float(denom) * 100.0
run.stats[statkey]['StrOverlap'] += [(streams[j],
fraction)]
# selection...
if not self.passes(zip(streams, strevents), 0): continue
newrunlist += [run.runNr]
allStreams.update(streams)
for k, v, s in zip(streams, strevents, strsize):
run.addResult('STR:' + k, (v, s))
allStreams = ['STR:' + s for s in allStreams]
allStreams.sort(lambda x, y: 2 * cmp(y[4], x[4]) + cmp(x[5:], y[5:]))
# fill the gaps
for run in runlist:
for s in allStreams:
if not s in run.result:
run.addResult(s, 'n.a.')
runlist = [r for r in runlist if r.runNr in newrunlist]
# check if the streams in 'allStreams' match the show patterns
for s in allStreams:
if any([p.match(s[4:]) != None for p in compiledShowPatterns]):
Run.AddToShowOrder(DataKey(s, keytype=DataKey.STREAM))
if useTier0:
# retrieve Tier-0 information
from CoolRunQuery.AtlRunQueryTier0 import GetTier0_datasetsAndTypes
tier0connection = coolDbConn.GetTier0DBConnection()
cursor = tier0connection.cursor()
cursor.arraysize = 1000
tier0retdico = GetTier0_datasetsAndTypes(cursor, runnrlist)
# add Tier0 information
for run in runlist:
for s in allStreams:
if run.result[s] == 'n.a.': continue
run.stats[s]['StrTier0TypesRAW'] = {}
run.stats[s]['StrTier0TypesESD'] = {}
run.stats[s]['StrTier0AMI'] = {}
if run.runNr in tier0retdico.keys():
for dsname, t, pstates in tier0retdico[run.runNr]:
if s.replace('STR:', '') in dsname:
if '.RAW' in dsname:
if '.merge' in dsname:
prodstep = 'StrTier0TypesESD'
t = '(RAW)'
else:
prodstep = 'StrTier0TypesRAW'
t = '(RAW)'
else:
if '.recon.' in dsname:
prodstep = 'StrTier0TypesRAW'
else:
prodstep = 'StrTier0TypesESD'
if not run.stats[s]['StrTier0AMI'].has_key(
prodstep):
dsnamesplit = dsname.split('.')
if len(dsnamesplit) > 5:
amitag = dsnamesplit[5]
run.stats[s]['StrTier0AMI'][
prodstep] = amitag
else:
amitag = ''
# check if on CAF
oncaf = False
if pstates and 'replicate:done' in pstates:
oncaf = True
# fill the run stats
if not run.stats[s][prodstep].has_key(t):
run.stats[s][prodstep][t] = oncaf
# Done
duration = time() - start
if len(self.streams) != 0:
print(" ==> %i runs found (%.2f sec)" % (len(runlist), duration))
else:
print(" ==> Done (%g sec)" % duration)
return runlist
def updateShowOrder(self):
from CoolRunQuery.AtlRunQueryRun import Run
for cd in self.ChannelDesc():
if cd.SelectShowRetrieve<2: continue
Run.AddToShowOrder(cd)