def runAfterQuery(self, runlist): # do show selection here from CoolRunQuery.AtlRunQueryTier0 import GetTier0_allDatasets runnrlist = [r.runNr for r in runlist] tier0connection = coolDbConn.GetTier0DBConnection() tier0retdico = GetTier0_allDatasets(tier0connection.cursor(), runnrlist, self.selpattern) for run in runlist: # go through old runlist and see if tier0retdico.has_key(run.runNr): run.addResult('Datasets', tier0retdico[run.runNr]) else: run.addResult('Datasets', {}) Run.AddToShowOrder(DataKey('Datasets')) Run.showCAFLinks = self.showCAFLinks
def runAfterQuery(self, runlist): # do show selection here from CoolRunQuery.AtlRunQueryPVSS import GetPVSS_BPMs pvssdb = coolDbConn.GetPVSSDBConnection() cursor = pvssdb.cursor() cursor.arraysize = 1000 pvssretdico = {} for r in runlist: t = gmtime(r.sor / 1.E9) sor = "%02i-%02i-%4i %02i:%02i:%02i" % (t[2], t[1], t[0], t[3], t[4], t[5]) t = gmtime(r.eor / 1.E9) eor = "%02i-%02i-%4i %02i:%02i:%02i" % (t[2], t[1], t[0], t[3], t[4], t[5]) # retrieve values res = GetPVSS_BPMs(cursor, sor, eor) r.addResult('BPM', res) Run.AddToShowOrder(DataKey('BPM'))
def select(self, runlist): # some preparation: compile the show patterns start = time() if '.*' in self.showstreampatterns: compiledShowPatterns = [re.compile('.*')] else: compiledShowPatterns = [ re.compile(p) for p in self.showstreampatterns ] # we disable the access to everything that is not needed if we only select ShowStreams = False useTier0 = False if len(compiledShowPatterns) > 0: ShowStreams = True useTier0 = True print(self, end='') sys.stdout.flush() newrunlist = [] allStreams = Set( ) # list of all the streams that are in the selected runs connection = coolDbConn.GetSFODBConnection() cursor = connection.cursor() cursor.arraysize = 1000 runnrlist = [r.runNr for r in runlist] from CoolRunQuery.AtlRunQuerySFO import GetSFO_streamsAll, GetSFO_filesAll with timer('GetSFO_streamsAll'): streamsall = GetSFO_streamsAll(cursor, runnrlist) # { runnr: [streams] } with timer('GetSFO_filesAll'): filesall = GetSFO_filesAll( cursor, runnrlist) # [(COUNT(FILESIZE), SUM(FILESIZE), SUM(NREVENTS))] if ShowStreams: from CoolRunQuery.AtlRunQuerySFO import GetSFO_LBsAll, GetSFO_NeventsAll, GetSFO_overlapAll with timer('GetSFO_LBsAll'): lbinfoall = GetSFO_LBsAll( cursor, runnrlist ) # [(MIN(LUMIBLOCKNR), MAX(LUMIBLOCKNR), #LUMIBLOCKS)] with timer('GetSFO_overlapAll'): overlapall = GetSFO_overlapAll( cursor, runnrlist) # [(SUM(OVERLAP_EVENTS))] smallrunnrlist = [] for r in runnrlist: # go through old runlist and see if not r in streamsall: continue for s in streamsall[r]: if r in lbinfoall and s in lbinfoall[ r] and lbinfoall[r][s][1] > 0: smallrunnrlist += [r] break with timer('GetSFO_NeventsAll'): neventsall = GetSFO_NeventsAll( cursor, smallrunnrlist) # [(LUMIBLOCKNR, NREVENTS)] for run in runlist: # go through old runlist and see streams = [] # names of all streams in this run strsize = [] # size of streams in this run strevents = [] # nr of events in this run if run.runNr in streamsall: streams = streamsall[run.runNr] for s in streams: try: nfiles, size, events = filesall[run.runNr][s] except: nfiles, size, events = (0, 0, 0) strsize += [size] strevents += [events] if ShowStreams: from CoolRunQuery.utils.AtlRunQueryUtils import Matrix nst = len(streams) stovmat = Matrix(nst, nst) # overlap matrix for i, s in enumerate(streams): run.stats['STR:' + s] = {} # fill overlaps into matrix for j, s2 in enumerate(streams): try: eventsij = overlapall[run.runNr][s][s2] except KeyError: eventsij = 0 if i == j and events: eventsij = events stovmat.setitem(i, j, float(eventsij)) stovmat.setitem(j, i, float(eventsij)) # symmetrise matrix # read number of events per LB minlb, maxlb, lbs = (0, 0, 1) try: minlb, maxlb, lbs = lbinfoall[run.runNr][s] except KeyError: pass # if minlb==maxlb==0 -> no file closure at LB boundary if minlb == 0 and maxlb == 0: run.stats['STR:' + s]['LBRecInfo'] = None continue else: lbevcount = '<tr>' result = neventsall[run.runNr][s] #[ (lb,nev),... ] lbold = -1 allnev = 0 ic = 0 ice = 0 allic = 0 lastElement = False firstcall = True for ice, (lb, nev) in enumerate(result): if ice == len(result): lastElement = True allnev += nev if lb != lbold or lastElement: if lbold != -1: ic += 1 allic += 1 if allic < 101: if ic == 9: ic = 1 lbevcount += '</tr><tr>' lbevcount += '<td style="font-size:75%%">%i (%s)</td>' % ( lbold, allnev) else: if firstcall: lbevcount += '</tr><tr>' lbevcount += '<td style="font-size:75%%" colspan="8">... <i>too many LBs (> 100) to show</i></td>' firstcall = False allnev = nev lbold = lb else: allnev += nev lbevcount += '</tr>' run.stats['STR:' + s]['LBRecInfo'] = lbevcount run.stats['STR:' + s]['LBRecInfo'] = result # add overlap information to the run stats for i in xrange(nst): statkey = 'STR:' + streams[i] run.stats[statkey]['StrOverlap'] = [] denom = stovmat.getitem(i, i) if denom == 0: continue for j in xrange(nst): if i == j or stovmat.getitem(i, j) == 0: continue fraction = 100 if stovmat.getitem(i, j) != denom: fraction = float(stovmat.getitem( i, j)) / float(denom) * 100.0 run.stats[statkey]['StrOverlap'] += [(streams[j], fraction)] # selection... if not self.passes(zip(streams, strevents), 0): continue newrunlist += [run.runNr] allStreams.update(streams) for k, v, s in zip(streams, strevents, strsize): run.addResult('STR:' + k, (v, s)) allStreams = ['STR:' + s for s in allStreams] allStreams.sort(lambda x, y: 2 * cmp(y[4], x[4]) + cmp(x[5:], y[5:])) # fill the gaps for run in runlist: for s in allStreams: if not s in run.result: run.addResult(s, 'n.a.') runlist = [r for r in runlist if r.runNr in newrunlist] # check if the streams in 'allStreams' match the show patterns for s in allStreams: if any([p.match(s[4:]) != None for p in compiledShowPatterns]): Run.AddToShowOrder(DataKey(s, keytype=DataKey.STREAM)) if useTier0: # retrieve Tier-0 information from CoolRunQuery.AtlRunQueryTier0 import GetTier0_datasetsAndTypes tier0connection = coolDbConn.GetTier0DBConnection() cursor = tier0connection.cursor() cursor.arraysize = 1000 tier0retdico = GetTier0_datasetsAndTypes(cursor, runnrlist) # add Tier0 information for run in runlist: for s in allStreams: if run.result[s] == 'n.a.': continue run.stats[s]['StrTier0TypesRAW'] = {} run.stats[s]['StrTier0TypesESD'] = {} run.stats[s]['StrTier0AMI'] = {} if run.runNr in tier0retdico.keys(): for dsname, t, pstates in tier0retdico[run.runNr]: if s.replace('STR:', '') in dsname: if '.RAW' in dsname: if '.merge' in dsname: prodstep = 'StrTier0TypesESD' t = '(RAW)' else: prodstep = 'StrTier0TypesRAW' t = '(RAW)' else: if '.recon.' in dsname: prodstep = 'StrTier0TypesRAW' else: prodstep = 'StrTier0TypesESD' if not run.stats[s]['StrTier0AMI'].has_key( prodstep): dsnamesplit = dsname.split('.') if len(dsnamesplit) > 5: amitag = dsnamesplit[5] run.stats[s]['StrTier0AMI'][ prodstep] = amitag else: amitag = '' # check if on CAF oncaf = False if pstates and 'replicate:done' in pstates: oncaf = True # fill the run stats if not run.stats[s][prodstep].has_key(t): run.stats[s][prodstep][t] = oncaf # Done duration = time() - start if len(self.streams) != 0: print(" ==> %i runs found (%.2f sec)" % (len(runlist), duration)) else: print(" ==> Done (%g sec)" % duration) return runlist
def updateShowOrder(self): from CoolRunQuery.AtlRunQueryRun import Run for cd in self.ChannelDesc(): if cd.SelectShowRetrieve<2: continue Run.AddToShowOrder(cd)