def select(self, runlist):
# some preparation: compile the show patterns
start = time()
print self,
sys.stdout.flush()
newrunlist = []
connection = coolDbConn.GetSFODBConnection()
cursor = connection.cursor()
cursor.arraysize = 1000
runnrlist = [r.runNr for r in runlist]
from CoolRunQuery.AtlRunQuerySFO import GetSFO_NphysicseventsAll
with timer("GetSFO_NphysicseventsAll"):
events = GetSFO_NphysicseventsAll(
cursor, runnrlist) # { runnr: nPhysEvents }
for run in runlist:
iov = IOVRange(runStart=run.runNr,
lbStart=1,
runEnd=run.runNr + 1,
lbEnd=0)
for k in self.ResultKey():
if not run.runNr in events:
# the OVERLAP_EVENTS information is not yet available in the SFO (before Nov 15, 2009)
# use the inclusive number instead
run.addResult(k, "n.a.", iov, reject=False)
run.showDataIncomplete = True
continue
nev = events[run.runNr]
if self.ApplySelection(k) and not self.passes(nev, k):
run.addResult(k,
self.prettyValue(nev, k),
iov,
reject=True)
continue
run.addResult(k, self.prettyValue(nev, k), iov)
rej = self.rejectRun(run)
if not rej:
newrunlist += [run.runNr]
runlist = [r for r in runlist if r.runNr in newrunlist]
duration = time() - start
if self.applySelection:
print " ==> %i runs found (%.2f sec)" % (len(runlist), duration)
else:
print " ==> Done (%g sec)" % duration
return runlist
def CreateRunStreamAndNeventsList(runlist):
# open SFO DB connection
from CoolRunQuery.utils.AtlRunQueryUtils import coolDbConn
from CoolRunQuery.AtlRunQuerySFO import GetSFO_NeventsAllPhysics
cursor = coolDbConn.GetSFODBConnection().cursor()
cursor.arraysize = 1000
# find streams
runnrlist = [r.runNr for r in runlist]
with timer('get SFO number of events AllPhysics'):
runstreamevents = GetSFO_NeventsAllPhysics(
cursor,
runnrlist) # { runnr: { stream: [(LUMIBLOCKNR, NREVENTS)] } }
return runstreamevents, ",".join(['%i' % r for r in runnrlist])
def select(self, runlist):
# some preparation: compile the show patterns
start = time()
if '.*' in self.showstreampatterns:
compiledShowPatterns = [re.compile('.*')]
else:
compiledShowPatterns = [
re.compile(p) for p in self.showstreampatterns
]
# we disable the access to everything that is not needed if we only select
ShowStreams = False
useTier0 = False
if len(compiledShowPatterns) > 0:
ShowStreams = True
useTier0 = True
print(self, end='')
sys.stdout.flush()
newrunlist = []
allStreams = Set(
) # list of all the streams that are in the selected runs
connection = coolDbConn.GetSFODBConnection()
cursor = connection.cursor()
cursor.arraysize = 1000
runnrlist = [r.runNr for r in runlist]
from CoolRunQuery.AtlRunQuerySFO import GetSFO_streamsAll, GetSFO_filesAll
with timer('GetSFO_streamsAll'):
streamsall = GetSFO_streamsAll(cursor,
runnrlist) # { runnr: [streams] }
with timer('GetSFO_filesAll'):
filesall = GetSFO_filesAll(
cursor,
runnrlist) # [(COUNT(FILESIZE), SUM(FILESIZE), SUM(NREVENTS))]
if ShowStreams:
from CoolRunQuery.AtlRunQuerySFO import GetSFO_LBsAll, GetSFO_NeventsAll, GetSFO_overlapAll
with timer('GetSFO_LBsAll'):
lbinfoall = GetSFO_LBsAll(
cursor, runnrlist
) # [(MIN(LUMIBLOCKNR), MAX(LUMIBLOCKNR), #LUMIBLOCKS)]
with timer('GetSFO_overlapAll'):
overlapall = GetSFO_overlapAll(
cursor, runnrlist) # [(SUM(OVERLAP_EVENTS))]
smallrunnrlist = []
for r in runnrlist: # go through old runlist and see
if not r in streamsall: continue
for s in streamsall[r]:
if r in lbinfoall and s in lbinfoall[
r] and lbinfoall[r][s][1] > 0:
smallrunnrlist += [r]
break
with timer('GetSFO_NeventsAll'):
neventsall = GetSFO_NeventsAll(
cursor, smallrunnrlist) # [(LUMIBLOCKNR, NREVENTS)]
for run in runlist: # go through old runlist and see
streams = [] # names of all streams in this run
strsize = [] # size of streams in this run
strevents = [] # nr of events in this run
if run.runNr in streamsall:
streams = streamsall[run.runNr]
for s in streams:
try:
nfiles, size, events = filesall[run.runNr][s]
except:
nfiles, size, events = (0, 0, 0)
strsize += [size]
strevents += [events]
if ShowStreams:
from CoolRunQuery.utils.AtlRunQueryUtils import Matrix
nst = len(streams)
stovmat = Matrix(nst, nst) # overlap matrix
for i, s in enumerate(streams):
run.stats['STR:' + s] = {}
# fill overlaps into matrix
for j, s2 in enumerate(streams):
try:
eventsij = overlapall[run.runNr][s][s2]
except KeyError:
eventsij = 0
if i == j and events:
eventsij = events
stovmat.setitem(i, j, float(eventsij))
stovmat.setitem(j, i,
float(eventsij)) # symmetrise matrix
# read number of events per LB
minlb, maxlb, lbs = (0, 0, 1)
try:
minlb, maxlb, lbs = lbinfoall[run.runNr][s]
except KeyError:
pass
# if minlb==maxlb==0 -> no file closure at LB boundary
if minlb == 0 and maxlb == 0:
run.stats['STR:' + s]['LBRecInfo'] = None
continue
else:
lbevcount = '<tr>'
result = neventsall[run.runNr][s] #[ (lb,nev),... ]
lbold = -1
allnev = 0
ic = 0
ice = 0
allic = 0
lastElement = False
firstcall = True
for ice, (lb, nev) in enumerate(result):
if ice == len(result):
lastElement = True
allnev += nev
if lb != lbold or lastElement:
if lbold != -1:
ic += 1
allic += 1
if allic < 101:
if ic == 9:
ic = 1
lbevcount += '</tr><tr>'
lbevcount += '<td style="font-size:75%%">%i (%s)</td>' % (
lbold, allnev)
else:
if firstcall:
lbevcount += '</tr><tr>'
lbevcount += '<td style="font-size:75%%" colspan="8">... <i>too many LBs (> 100) to show</i></td>'
firstcall = False
allnev = nev
lbold = lb
else:
allnev += nev
lbevcount += '</tr>'
run.stats['STR:' + s]['LBRecInfo'] = lbevcount
run.stats['STR:' + s]['LBRecInfo'] = result
# add overlap information to the run stats
for i in xrange(nst):
statkey = 'STR:' + streams[i]
run.stats[statkey]['StrOverlap'] = []
denom = stovmat.getitem(i, i)
if denom == 0: continue
for j in xrange(nst):
if i == j or stovmat.getitem(i, j) == 0: continue
fraction = 100
if stovmat.getitem(i, j) != denom:
fraction = float(stovmat.getitem(
i, j)) / float(denom) * 100.0
run.stats[statkey]['StrOverlap'] += [(streams[j],
fraction)]
# selection...
if not self.passes(zip(streams, strevents), 0): continue
newrunlist += [run.runNr]
allStreams.update(streams)
for k, v, s in zip(streams, strevents, strsize):
run.addResult('STR:' + k, (v, s))
allStreams = ['STR:' + s for s in allStreams]
allStreams.sort(lambda x, y: 2 * cmp(y[4], x[4]) + cmp(x[5:], y[5:]))
# fill the gaps
for run in runlist:
for s in allStreams:
if not s in run.result:
run.addResult(s, 'n.a.')
runlist = [r for r in runlist if r.runNr in newrunlist]
# check if the streams in 'allStreams' match the show patterns
for s in allStreams:
if any([p.match(s[4:]) != None for p in compiledShowPatterns]):
Run.AddToShowOrder(DataKey(s, keytype=DataKey.STREAM))
if useTier0:
# retrieve Tier-0 information
from CoolRunQuery.AtlRunQueryTier0 import GetTier0_datasetsAndTypes
tier0connection = coolDbConn.GetTier0DBConnection()
cursor = tier0connection.cursor()
cursor.arraysize = 1000
tier0retdico = GetTier0_datasetsAndTypes(cursor, runnrlist)
# add Tier0 information
for run in runlist:
for s in allStreams:
if run.result[s] == 'n.a.': continue
run.stats[s]['StrTier0TypesRAW'] = {}
run.stats[s]['StrTier0TypesESD'] = {}
run.stats[s]['StrTier0AMI'] = {}
if run.runNr in tier0retdico.keys():
for dsname, t, pstates in tier0retdico[run.runNr]:
if s.replace('STR:', '') in dsname:
if '.RAW' in dsname:
if '.merge' in dsname:
prodstep = 'StrTier0TypesESD'
t = '(RAW)'
else:
prodstep = 'StrTier0TypesRAW'
t = '(RAW)'
else:
if '.recon.' in dsname:
prodstep = 'StrTier0TypesRAW'
else:
prodstep = 'StrTier0TypesESD'
if not run.stats[s]['StrTier0AMI'].has_key(
prodstep):
dsnamesplit = dsname.split('.')
if len(dsnamesplit) > 5:
amitag = dsnamesplit[5]
run.stats[s]['StrTier0AMI'][
prodstep] = amitag
else:
amitag = ''
# check if on CAF
oncaf = False
if pstates and 'replicate:done' in pstates:
oncaf = True
# fill the run stats
if not run.stats[s][prodstep].has_key(t):
run.stats[s][prodstep][t] = oncaf
# Done
duration = time() - start
if len(self.streams) != 0:
print(" ==> %i runs found (%.2f sec)" % (len(runlist), duration))
else:
print(" ==> Done (%g sec)" % duration)
return runlist