def evaluate(self, runlist): # provide pickled dictionary of query result with timer('CreateResultDict'): from CoolRunQuery.output.AtlRunQuerySave import CreateResultDict dic, dicsum = CreateResultDict(runlist) # create XML file (and return pretty html output for print) if self.prodgrl: with timer('CreateXMLFile'): print("Producing XML file") from CoolRunQuery.output.AtlRunQueryXML import CreateXMLFile from .AtlRunQueryVersion import SvnVersion xmlhtmlstr = CreateXMLFile(runlist, self.cmdlineOptions, self.origQuery, self.datapath, self.xmlFileName, self.xmlFileLabel, SvnVersion) else: xmlhtmlstr = None print("Creation of GRL disabled") if self.dictroot and len(runlist) > 0: with timer('CreateRootFile'): # create root file from dictionary - and make plots from CoolRunQuery.output.AtlRunQueryRoot import CreateRootFile rootfilename, roothtmlstr = CreateRootFile(dic) else: roothtmlstr = None print("Creation of root file disabled") return (dic, dicsum, xmlhtmlstr, roothtmlstr)
def output(self, runlist, dic, dicsum, xmlhtmlstr, roothtmlstr): if self.html: with timer('GetOKS link info'): from AtlRunQuerySFO import SetOKSLinks SetOKSLinks(runlist) # create web page from .html.AtlRunQueryHTML import ResultPageMaker #try: pageinfo = { 'datapath': self.datapath, 'origQuery': self.origQuery, 'fullQuery': self.parsedstring, 'runlist': runlist, 'dic': dic, 'dicsum': dicsum, 'makeSummary': self.makeSummary, 'makeDQeff': self.makeDQeff, 'makeDQSummary': self.makeDQSummary, 'makeDQPlots': self.makeDQPlots, 'roothtmlstr': roothtmlstr, 'xmlfilename': self.xmlFileName, 'xmlhtmlstr': xmlhtmlstr, 'querytime': time() - self.querystart, 'selout': self.selectionOutput, } with timer("run ResultPageMaker makePage"): ResultPageMaker.makePage(pageinfo) #except ImportError, ie: # print ("Can't import pagemaker, no web page made",ie) else: print( '---------------------------------------------------------------------' ) print(Run.header()) runlist.reverse() for r in runlist: print(r) from CoolRunQuery.utils.AtlRunQueryUtils import addKommaToNumber, filesize print('Summary:') for data_key, summary in dicsum.items(): if data_key.Type == DataKey.STREAM: key = data_key.ResultKey.replace('STR:', '') val = '%s (%s)' % (addKommaToNumber( summary[0]), filesize(summary[1])) else: key = data_key.ResultKey val = addKommaToNumber(summary) print('%20s : %s' % (key, val)) duration = time() - self.querystart print("%20s : %g sec" % ('Total execution time', duration)) print( '---------------------------------------------------------------------' )
def select(self, runlist): # some preparation: compile the show patterns start = time() print self, sys.stdout.flush() newrunlist = [] connection = coolDbConn.GetSFODBConnection() cursor = connection.cursor() cursor.arraysize = 1000 runnrlist = [r.runNr for r in runlist] from CoolRunQuery.AtlRunQuerySFO import GetSFO_NphysicseventsAll with timer("GetSFO_NphysicseventsAll"): events = GetSFO_NphysicseventsAll( cursor, runnrlist) # { runnr: nPhysEvents } for run in runlist: iov = IOVRange(runStart=run.runNr, lbStart=1, runEnd=run.runNr + 1, lbEnd=0) for k in self.ResultKey(): if not run.runNr in events: # the OVERLAP_EVENTS information is not yet available in the SFO (before Nov 15, 2009) # use the inclusive number instead run.addResult(k, "n.a.", iov, reject=False) run.showDataIncomplete = True continue nev = events[run.runNr] if self.ApplySelection(k) and not self.passes(nev, k): run.addResult(k, self.prettyValue(nev, k), iov, reject=True) continue run.addResult(k, self.prettyValue(nev, k), iov) rej = self.rejectRun(run) if not rej: newrunlist += [run.runNr] runlist = [r for r in runlist if r.runNr in newrunlist] duration = time() - start if self.applySelection: print " ==> %i runs found (%.2f sec)" % (len(runlist), duration) else: print " ==> Done (%g sec)" % duration return runlist
def _makeDQeff( cls, pageinfo ): for x in ["dicsum", "dic", "datapath", "makeDQeff"]: exec("%s = pageinfo['%s']" % (x,x) ) if dicsum and makeDQeff: with timer("make the DQ efficiency summary"): from CoolRunQuery.utils.AtlRunQueryDQUtils import MakeDQeffHtml return MakeDQeffHtml( dic, dicsum, datapath ) else: return ''
def __init__(self, options, readoracle=False, loglevel=1, html="AUTO", origQuery="", datapath='data', parsedstring=""): with timer('total'): with timer('config'): self.config(options, readoracle, loglevel, html, origQuery, datapath, parsedstring) with timer('run all'): runlist = self.run() with timer('evaluate all'): (dic, dicsum, xmlhtmlstr, roothtmlstr) = self.evaluate(runlist) with timer('output all'): self.output(runlist, dic, dicsum, xmlhtmlstr, roothtmlstr) with timer('finalize'): self.finalize() from CoolRunQuery.utils.AtlRunQueryTimer import TimerStats as TS TS.printTimeSummary() TS.printTimeFlat()
def _makeResultsTable( cls, pageinfo ): for x in pageinfo.keys(): exec("%s = pageinfo['%s']" % (x,x) ) # run results table resultstable = '<table class="resulttable" id="resulttable" style="margin-left: 13px">\n' # table head resultstable += Run.header() # runs with timer("print the runs"): for r in runlist: with timer("print run %i" % r.runNr): resultstable += str(r) # summary headlist = Run.headerkeys() resultstable += " <tr class=\"space\"><td style=\"text-align: left;\" colspan=\"%i\"><font size=-2><i>Summary</i>:</font></td></tr>\n" % (len(headlist)-1) resultstable += " <tr class=\"sum\">" + sumstr + "</tr>\n" resultstable += "</table>\n" return resultstable
def _OverallSummary( cls, pageinfo ): for x in pageinfo.keys(): exec("%s = pageinfo['%s']" % (x,x) ) with timer("make the summary"): from CoolRunQuery.html.AtlRunQuerySummary import MakeSummaryHtml overallsummarystr = MakeSummaryHtml( dic, dicsum, datapath ) if overallsummarystr == '': return '' s = '''<table width="95%" cellpadding="5" style="margin-left: 13px"> <tr><td colspan=2 bgcolor=gainsboro><font size=+1><b>Search Result Summary</b></font></td></tr> </table>''' s += '<p></p>' s += overallsummarystr s += '<p></p>' return s
def _defineToolTips( cls, pageinfo, globalOnly=False ): for x in pageinfo.keys(): exec("%s = pageinfo['%s']" % (x,x) ) tooltips = u'' with timer("print the tooltips"): dw_call = ['<script type="text/javascript">'] dw_call += ["if(dw_Tooltip.content_vars==undefined) {dw_Tooltip.content_vars = {}; };"] dw_call += Run.GlobalTooltips if not globalOnly: for r in runlist: dw_call += r.tooltips dw_call += ['</script>'] for l in dw_call: tooltips += unicode(l) + '\n' return tooltips
def CreateRunStreamAndNeventsList(runlist): # open SFO DB connection from CoolRunQuery.utils.AtlRunQueryUtils import coolDbConn from CoolRunQuery.AtlRunQuerySFO import GetSFO_NeventsAllPhysics cursor = coolDbConn.GetSFODBConnection().cursor() cursor.arraysize = 1000 # find streams runnrlist = [r.runNr for r in runlist] with timer('get SFO number of events AllPhysics'): runstreamevents = GetSFO_NeventsAllPhysics( cursor, runnrlist) # { runnr: { stream: [(LUMIBLOCKNR, NREVENTS)] } } return runstreamevents, ",".join(['%i' % r for r in runnrlist])
def _makeYellowLine( cls, pageinfo ): for x in ["dicsum"]: exec("%s = pageinfo['%s']" % (x,x) ) with timer("make the yellow summary line"): sumdic = {} for key, summary in dicsum.items(): if key.Type == DataKey.STREAM: s = addKommaToNumber(summary[0])+' <BR><font size="-2">(%s)</font>' % filesize(summary[1]) elif key=='Run': s = addKommaToNumber(summary) + " runs" else: s = addKommaToNumber(summary) sumdic[key.ResultKey] = s.strip() sumstr = '' headlist = Run.headerkeys() for title in headlist: sumstr += '<td style="text-align:right;">%s</td>' % (sumdic[title] if title in sumdic else '') return sumstr
def _DQSummaryWrap( cls, pageinfo, doPickle=True ): from CoolRunQuery.utils.AtlRunQueryUtils import runsOnServer if not runsOnServer() and doPickle: import pickle from CoolRunQuery.AtlRunQueryQueryConfig import QC f = open('%s/dqsum_pi.pickle' % QC.datapath, "w") pickle.dump(pageinfo, f) f.close() for x in pageinfo.keys(): exec("%s = pageinfo['%s']" % (x,x) ) with timer("make the DQ summary"): from CoolRunQuery.html.AtlRunQueryDQSummary import DQSummary dqsummary = DQSummary.makeHTML( dic, dicsum, doDQSummary=makeDQSummary, doDQPlots=makeDQPlots ) if dqsummary == '': return '' s = '''<table width="95%" cellpadding="5" style="margin-left: 13px"> <tr><td colspan=2 bgcolor=gainsboro><font size=+1><b>Data Quality Summary</b></font></td></tr> </table>''' s += '<p></p>' s += dqsummary s += '<p></p>' return s
def run(self): # is last run still open ? from CoolRunQuery.AtlRunQuerySFO import GetSFO_lastNruns sfoconnection = coolDbConn.GetSFODBConnection() sfocursor = sfoconnection.cursor() retlist = GetSFO_lastNruns(sfocursor, 1) if 'OPENED' in retlist[0][1]: Run.runnropen = retlist[0][0] if self.cmdlineOptions.runlist != None: runlist = self.cmdlineOptions.runlist if any('last' in x for x in self.cmdlineOptions.runlist): for idx in xrange(len(runlist)): rrange = runlist[idx] if not 'last' in rrange: continue rrange = rrange.replace('last', '') self.maxNumOfRuns = int(rrange) # list of last n runs run numbers, where at least one stream is not calibration retlist = GetSFO_lastNruns(sfocursor, 1.1 * self.maxNumOfRuns) runlist[idx] = '%i+' % (retlist[-1][0]) elif self.cmdlineOptions.timelist != None: # translate time into run-range if 'last' in self.cmdlineOptions.timelist[0]: # contains 'last' from CoolRunQuery.utils.AtlRunQueryUtils import get_runs_last_dt rlist = get_runs_last_dt(self.cmdlineOptions.timelist[0]) runlist = ['%i+' % rlist[-1]] if (len(rlist) > 0) else ['99999999+'] else: # translate time into run-range from utils.AtlRunQueryUtils import timeStringToSecondsUTC, secondsToTimeStringUTC, get_run_range2, GetTimeRanges timelist = ','.join(self.cmdlineOptions.timelist) timeranges, timerangesHR = GetTimeRanges( timelist, intRepFnc=timeStringToSecondsUTC, maxval=time()) timerangesAsString = [ map(secondsToTimeStringUTC, tr) for tr in timeranges ] runranges = [("%s-%s" % get_run_range2(tr[0], tr[1])) for tr in timerangesAsString] runlist = [','.join(runranges)] # the run selector rtSel = RunTimeSelector(name='runtime', runlist=runlist) # find the begin and end of each interesting run runlist = rtSel.select() # if Selector.condDB()=="CONDBR2": # self.cmdlineOptions.show = [x for x in self.cmdlineOptions.show if x not in ["lhc","olclumi","olcfillparams","olclbdata"]] # self.cmdlineOptions.partition=None # self.cmdlineOptions.projecttag=None # print ("Pre-run2 selection removed partition and projecttag from querying") # print ("Pre-run2 selection removed lhc, olc from showing. Modified show list: %r" % self.cmdlineOptions.show) self.selectionOutput += ["%s" % rtSel] from .AtlRunQuerySelectorWorker import SelectorWorker # create all selectors that are actively select SelectorWorker.parseSelectorOptions(self.cmdlineOptions) # parse show option: SelectorWorker.parseShowOption(self.cmdlineOptions) # call setApply for all selectors that actively select SelectorWorker.setApplySelection() # call setShowOutput for all selectors that show SelectorWorker.setShowOutput() SelectorWorker.getRetrieveSelector('trigkey', 'TrigKeySelector') # Selectors can implement an initialize function which runs after any constructor and setShow for s in SelectorWorker.getOrderedSelectorList(): s = s.selector if hasattr(s, 'initialize'): with timer("initializing selector '%s'" % s.name): s.verbose = True s.initialize() # apply the selectors to initial run list for s in SelectorWorker.selectors(): with timer("run selector '%s'" % s.name): s.verbose = True runlist = s.select(runlist) self.selectionOutput += ["%s" % s.__str__()] with timer("run AfterQuery for selector '%s'" % s.name): s.runAfterQuery(runlist) #print ([r.runNr for r in runlist]) # reverse list for presentation: newest run on top runlist.reverse() # if "last" option used, the runlist has to be truncated if self.maxNumOfRuns > 0: print('SELOUT Selecting the %i most recent runs' % self.maxNumOfRuns) runlist = runlist[0:self.maxNumOfRuns] self.selectionOutput += [ 'SELOUT Selecting the %i most recent runs' % self.maxNumOfRuns ] return runlist
def select(self, runlist): # some preparation: compile the show patterns start = time() if '.*' in self.showstreampatterns: compiledShowPatterns = [re.compile('.*')] else: compiledShowPatterns = [ re.compile(p) for p in self.showstreampatterns ] # we disable the access to everything that is not needed if we only select ShowStreams = False useTier0 = False if len(compiledShowPatterns) > 0: ShowStreams = True useTier0 = True print(self, end='') sys.stdout.flush() newrunlist = [] allStreams = Set( ) # list of all the streams that are in the selected runs connection = coolDbConn.GetSFODBConnection() cursor = connection.cursor() cursor.arraysize = 1000 runnrlist = [r.runNr for r in runlist] from CoolRunQuery.AtlRunQuerySFO import GetSFO_streamsAll, GetSFO_filesAll with timer('GetSFO_streamsAll'): streamsall = GetSFO_streamsAll(cursor, runnrlist) # { runnr: [streams] } with timer('GetSFO_filesAll'): filesall = GetSFO_filesAll( cursor, runnrlist) # [(COUNT(FILESIZE), SUM(FILESIZE), SUM(NREVENTS))] if ShowStreams: from CoolRunQuery.AtlRunQuerySFO import GetSFO_LBsAll, GetSFO_NeventsAll, GetSFO_overlapAll with timer('GetSFO_LBsAll'): lbinfoall = GetSFO_LBsAll( cursor, runnrlist ) # [(MIN(LUMIBLOCKNR), MAX(LUMIBLOCKNR), #LUMIBLOCKS)] with timer('GetSFO_overlapAll'): overlapall = GetSFO_overlapAll( cursor, runnrlist) # [(SUM(OVERLAP_EVENTS))] smallrunnrlist = [] for r in runnrlist: # go through old runlist and see if not r in streamsall: continue for s in streamsall[r]: if r in lbinfoall and s in lbinfoall[ r] and lbinfoall[r][s][1] > 0: smallrunnrlist += [r] break with timer('GetSFO_NeventsAll'): neventsall = GetSFO_NeventsAll( cursor, smallrunnrlist) # [(LUMIBLOCKNR, NREVENTS)] for run in runlist: # go through old runlist and see streams = [] # names of all streams in this run strsize = [] # size of streams in this run strevents = [] # nr of events in this run if run.runNr in streamsall: streams = streamsall[run.runNr] for s in streams: try: nfiles, size, events = filesall[run.runNr][s] except: nfiles, size, events = (0, 0, 0) strsize += [size] strevents += [events] if ShowStreams: from CoolRunQuery.utils.AtlRunQueryUtils import Matrix nst = len(streams) stovmat = Matrix(nst, nst) # overlap matrix for i, s in enumerate(streams): run.stats['STR:' + s] = {} # fill overlaps into matrix for j, s2 in enumerate(streams): try: eventsij = overlapall[run.runNr][s][s2] except KeyError: eventsij = 0 if i == j and events: eventsij = events stovmat.setitem(i, j, float(eventsij)) stovmat.setitem(j, i, float(eventsij)) # symmetrise matrix # read number of events per LB minlb, maxlb, lbs = (0, 0, 1) try: minlb, maxlb, lbs = lbinfoall[run.runNr][s] except KeyError: pass # if minlb==maxlb==0 -> no file closure at LB boundary if minlb == 0 and maxlb == 0: run.stats['STR:' + s]['LBRecInfo'] = None continue else: lbevcount = '<tr>' result = neventsall[run.runNr][s] #[ (lb,nev),... ] lbold = -1 allnev = 0 ic = 0 ice = 0 allic = 0 lastElement = False firstcall = True for ice, (lb, nev) in enumerate(result): if ice == len(result): lastElement = True allnev += nev if lb != lbold or lastElement: if lbold != -1: ic += 1 allic += 1 if allic < 101: if ic == 9: ic = 1 lbevcount += '</tr><tr>' lbevcount += '<td style="font-size:75%%">%i (%s)</td>' % ( lbold, allnev) else: if firstcall: lbevcount += '</tr><tr>' lbevcount += '<td style="font-size:75%%" colspan="8">... <i>too many LBs (> 100) to show</i></td>' firstcall = False allnev = nev lbold = lb else: allnev += nev lbevcount += '</tr>' run.stats['STR:' + s]['LBRecInfo'] = lbevcount run.stats['STR:' + s]['LBRecInfo'] = result # add overlap information to the run stats for i in xrange(nst): statkey = 'STR:' + streams[i] run.stats[statkey]['StrOverlap'] = [] denom = stovmat.getitem(i, i) if denom == 0: continue for j in xrange(nst): if i == j or stovmat.getitem(i, j) == 0: continue fraction = 100 if stovmat.getitem(i, j) != denom: fraction = float(stovmat.getitem( i, j)) / float(denom) * 100.0 run.stats[statkey]['StrOverlap'] += [(streams[j], fraction)] # selection... if not self.passes(zip(streams, strevents), 0): continue newrunlist += [run.runNr] allStreams.update(streams) for k, v, s in zip(streams, strevents, strsize): run.addResult('STR:' + k, (v, s)) allStreams = ['STR:' + s for s in allStreams] allStreams.sort(lambda x, y: 2 * cmp(y[4], x[4]) + cmp(x[5:], y[5:])) # fill the gaps for run in runlist: for s in allStreams: if not s in run.result: run.addResult(s, 'n.a.') runlist = [r for r in runlist if r.runNr in newrunlist] # check if the streams in 'allStreams' match the show patterns for s in allStreams: if any([p.match(s[4:]) != None for p in compiledShowPatterns]): Run.AddToShowOrder(DataKey(s, keytype=DataKey.STREAM)) if useTier0: # retrieve Tier-0 information from CoolRunQuery.AtlRunQueryTier0 import GetTier0_datasetsAndTypes tier0connection = coolDbConn.GetTier0DBConnection() cursor = tier0connection.cursor() cursor.arraysize = 1000 tier0retdico = GetTier0_datasetsAndTypes(cursor, runnrlist) # add Tier0 information for run in runlist: for s in allStreams: if run.result[s] == 'n.a.': continue run.stats[s]['StrTier0TypesRAW'] = {} run.stats[s]['StrTier0TypesESD'] = {} run.stats[s]['StrTier0AMI'] = {} if run.runNr in tier0retdico.keys(): for dsname, t, pstates in tier0retdico[run.runNr]: if s.replace('STR:', '') in dsname: if '.RAW' in dsname: if '.merge' in dsname: prodstep = 'StrTier0TypesESD' t = '(RAW)' else: prodstep = 'StrTier0TypesRAW' t = '(RAW)' else: if '.recon.' in dsname: prodstep = 'StrTier0TypesRAW' else: prodstep = 'StrTier0TypesESD' if not run.stats[s]['StrTier0AMI'].has_key( prodstep): dsnamesplit = dsname.split('.') if len(dsnamesplit) > 5: amitag = dsnamesplit[5] run.stats[s]['StrTier0AMI'][ prodstep] = amitag else: amitag = '' # check if on CAF oncaf = False if pstates and 'replicate:done' in pstates: oncaf = True # fill the run stats if not run.stats[s][prodstep].has_key(t): run.stats[s][prodstep][t] = oncaf # Done duration = time() - start if len(self.streams) != 0: print(" ==> %i runs found (%.2f sec)" % (len(runlist), duration)) else: print(" ==> Done (%g sec)" % duration) return runlist
def CreateXMLFile(runlist, options, origQuery, datapath, xmlfname, xmllabel, svnversion='Unknown'): """ """ with timer('create RunStreamAndNeventsList'): # show number of events per stream per run ? ShowNumberOfEventsPerStreamPerRun = False ShowNumberOfEventsPerStreamSummary = True # find streams runstreamevents, runnrliststr = CreateRunStreamAndNeventsList( runlist) # { runnr: { stream: [(LUMIBLOCKNR, NREVENTS)] } } with timer('prepare document'): #################################### ## create XML file of GRL #################################### doc = Document() docType = DocumentType('LumiRangeCollection') docType.systemId = 'http://atlas-runquery.cern.ch/LumiRangeCollection.dtd' doc.appendChild(docType) # number of comments txt = ' Good-runs-list created by %s on %s ' % ( sys.argv[0].split('/')[-1], str(datetime.datetime.now())) doc.appendChild(doc.createComment(txt)) # root element lrc = doc.createElement('LumiRangeCollection') doc.appendChild(lrc) # NamedLumiRange namedLR = doc.createElement('NamedLumiRange') lrc.appendChild(namedLR) # name of NamedLumiRange namedLR.appendChild(TextElement('Name', xmllabel, doc)) # version of NamedLumiRange namedLR.appendChild(TextElement('Version', '2.1', doc)) # metadata of NamedLumiRange metadata = { 'Query': origQuery.split('/')[0], 'RQTSVNVersion': svnversion, 'RunList': runnrliststr } for md in metadata: mdelm = TextElement('Metadata', metadata[md], doc) mdelm.setAttribute('Name', md) namedLR.appendChild(mdelm) with timer('ShowNumberOfEventsPerStreamSummary'): if ShowNumberOfEventsPerStreamSummary: strsummdelm = doc.createElement('Metadata') strsummdelm.setAttribute('Name', 'StreamListInfo') namedLR.appendChild(strsummdelm) # lumiblock collections of NamedLumiRange streams_sum = {} streams_byrun = {} with timer('Loop over all runs'): for run in runlist: lbc = doc.createElement('LumiBlockCollection') # run number runnrelm = TextElement('Run', str(run.runNr), doc) if len(run.stats['SMK']['random']) == 2: (rd0, rd1) = run.stats['SMK']['random'][0:2] # protect against missing information if rd0 == 'n.a.': rd0 = 0 if rd1 == 'n.a.': rd1 = 0 runnrelm.setAttribute('PrescaleRD0', 0x1 << (3 + rd0)) runnrelm.setAttribute('PrescaleRD1', 0x1 << (3 + rd1)) else: (rd0, rd1, rd2, rd3) = run.stats['SMK']['random'][0:4] # protect against missing information if rd0 == 'n.a.': rd0 = 0 if rd1 == 'n.a.': rd1 = 0 if rd2 == 'n.a.': rd2 = 0 if rd3 == 'n.a.': rd3 = 0 runnrelm.setAttribute('Cut0', rd0) runnrelm.setAttribute('Cut1', rd1) runnrelm.setAttribute('Cut2', rd2) runnrelm.setAttribute('Cut3', rd3) lbc.appendChild(runnrelm) # streams (initialisation) streams = {} streams_byrun[run.runNr] = streams if runstreamevents.has_key( run.runNr ): # protection in case the run does not have any stream for stream in runstreamevents[run.runNr].keys(): if 'physics_' == stream[:8]: streams[stream] = [0, 0] # only for physics streams if not streams_sum.has_key(stream): streams_sum[stream] = [0, 0] # total number of events in stream for (nlb, nev) in runstreamevents[run.runNr][stream]: streams[stream][0] += nev # lumiblock ranges for lbrange in run.data.getLBRanges(activeOnly=True): lbrelm = TextElement('LBRange', '', doc) lbrelm.setAttribute('Start', lbrange[1]) lbrelm.setAttribute('End', lbrange[2] - 1) lbc.appendChild(lbrelm) # count nevents in streams if runstreamevents.has_key( run.runNr ): # protection in case the run does not have any stream for stream, lbnevts in runstreamevents[run.runNr].items(): if 'physics_' == stream[:8]: for (nlb, nev) in lbnevts: if nlb >= lbrange[1] and nlb < lbrange[2]: streams[stream][1] += nev # append stream element s = streams.keys() s.sort() strselm = doc.createElement('StreamsInRun') for stream in s: nevts = streams[stream] if ShowNumberOfEventsPerStreamPerRun: strelm = TextElement('Stream', '', doc) strelm.setAttribute('Name', stream) strelm.setAttribute('TotalNumOfEvents', nevts[0]) strelm.setAttribute('NumOfSelectedEvents', nevts[1]) strselm.appendChild(strelm) eff = 0 if nevts[0] > 0: eff = nevts[1] / float(nevts[0]) * 100.0 # collect total number of events streams_sum[stream][0] += nevts[0] streams_sum[stream][1] += nevts[1] # append streams if ShowNumberOfEventsPerStreamPerRun: lbc.appendChild(strselm) # append LumiBlickCollection namedLR.appendChild(lbc) with timer('Streams'): for stream in sorted(streams_sum.keys()): nevts = streams_sum[stream] if ShowNumberOfEventsPerStreamSummary: strelm = TextElement('Stream', '', doc) strelm.setAttribute('Name', stream) strelm.setAttribute('TotalNumOfEvents', nevts[0]) strelm.setAttribute('NumOfSelectedEvents', nevts[1]) strsummdelm.appendChild(strelm) with timer('Save GRL'): filename = '%s/%s' % (datapath, xmlfname) #print "Writing",filename xmlfile = open(filename, mode="w") xmlfile.write(doc.toprettyxml(' ')) xmlfile.close() with timer('Create HTML'): #################################### ## create HTML #################################### # provide also pretty html text output htmltext = '' hspace = ' ' htmltext += '<table style="width: auto; border: 0px solid; border-width: margin: 0 0 0 0; 0px; border-spacing: 0px; border-collapse: separate; padding: 0px;" font-family: sans-serif; font-size: 85%%">\n' htmltext += '<tr><td colspan="2"><b><font color="#999999">' + txt.strip( ) + '</font></b></td></tr>\n' htmltext += '<tr><td style="vertical-align:top">SVN Version: </td><td> ' + svnversion + '</td></tr>\n' htmltext += '<tr><td style="vertical-align:top">Query string:</td><td><b>' + origQuery.split( '/')[0] + '</b></td></tr>\n' htmltext += '<tr><td style="vertical-align:top">Run list:</td><td>' + runnrliststr + '</td></tr>\n' htmltext += '</table>' htmltext += '<hr color="#000000" size=1><font color="#777777">\n' htmltext += '<table style="width: auto; border: 0px solid; border-width: margin: 0 0 0 0; 0px; border-spacing: 0px; border-collapse: separate; padding: 0px;" font-family: sans-serif; font-size: 90%%">\n' # lumiblock collections of NamedLumiRange for run in runlist: # run number htmltext += '<tr><td style="text-align:left;height:25px;vertical-align:bottom">Run <b>%i</b>:</td><td></td></tr>\n' % run.runNr # lumiblock ranges for lbrange in run.data.getLBRanges(activeOnly=True): htmltext += '<tr><td></td><td style="text-align:left">LB range: [%5i-%5i]</td></tr>\n' % ( lbrange[1], lbrange[2] - 1) # append stream element htmltext += '<tr><td></td><td style="text-align:left"></td></tr>' htmltext += '<tr><td></td><td style="text-align:left"><table style="width: auto; border: 0px solid; border-width: margin: 0 0 0 0; 0px; border-spacing: 0px; border-collapse: separate; padding: 0px;font-family: sans-serif; font-size: 90%"><tr><td>Stream name</td><td>#Events total</td><td> #Events selected</td><td> Sel. fraction (%)</td></tr>\n' streams = streams_byrun[run.runNr] for stream in sorted(streams.keys()): nevts = streams[stream] eff = (nevts[1] / float(nevts[0]) * 100.0) if (nevts[0] > 0) else 0 htmltext += '<tr><td style="text-align:left"><i>%s</i></td><td style="text-align:right">%s</td><td style="text-align:right">%s</td><td style="text-align:right">%.4g</td></tr>\n' % ( stream, prettyNumber(nevts[0]), prettyNumber( nevts[1]), eff) htmltext += '</table></td></tr>\n' # append stream element htmltext += '</table>' htmltext += '<hr color="#000000" size=1><font color="#777777">\n' htmltext += '<b>Stream summary for all selected runs:</b><br>\n' htmltext += '<table style="width: auto; border: 0px solid; border-width: margin: 0 0 0 0; 0px; border-spacing: 0px; border-collapse: separate; padding: 0px;font-family: sans-serif; font-size: 95%"><tr><td>Stream name</td><td>#Events total</td><td> #Events selected</td><td> Sel. fraction (%)</td></tr>\n' for stream in sorted(streams_sum.keys()): nevts = streams_sum[stream] eff = 0 if nevts[0] > 0: eff = nevts[1] / float(nevts[0]) * 100.0 htmltext += '<tr><td style="text-align:left"><i>%s</i></td><td style="text-align:right">%s</td><td style="text-align:right">%s</td><td style="text-align:right">%.4g</td></tr>\n' % ( stream, prettyNumber(nevts[0]), prettyNumber(nevts[1]), eff) htmltext += '</table>\n' #print """======================================================== #%r #=========================================================== #""" % htmltext # provide also text output return htmltext
def runAfterQuery(self, runlist): import cPickle for k in self.ResultKey(): with timer("olc afterquery blocks prepare for: %s" % k): for run in runlist: run.stats[k] = {} if not k in run.data: continue blocks = [] for entry in run.data[k]: v = entry.value if k != 'olc:bcidmask': v = int(v) if len(blocks) > 0 and blocks[-1][0] == v and blocks[ -1][2] == entry.startlb: blocks[-1][2] = entry.endlb else: blocks += [[v, entry.startlb, entry.endlb]] run.stats[k] = { "blocks": blocks, "first": run.data[k][0].value } #f = open("olc.pickle", "w") #cPickle.dump([run.data,run.stats],f) #f.close() with timer("olc afterquery rest"): from CoolRunQuery.utils.AtlRunQueryUtils import unpackRun1BCIDMask, unpackRun2BCIDMask for run in runlist: if run.runNr < 151260: continue # OLC information was not in COOL before this run # these contain the number of bunches for each IOV [(nb,lbstart,lbend)(...)...] xb1 = run.stats['olc:beam1bunches']['blocks'] xb2 = run.stats['olc:beam2bunches']['blocks'] xbc = run.stats['olc:collbunches']['blocks'] #print "JOERG XB1 ",xb1 #print "JOERG XB2 ",xb2 #print "JOERG XBC ",xbc # loop over BCID mask bcidmask = run.stats['olc:bcidmask']['blocks'] for i in xrange(len(bcidmask)): (bcidblob, lbstart, lbend) = bcidmask[i] # number of bunches for nb1, b, e in xb1: if lbstart >= b and lbstart < e: break for nb2, b, e in xb2: if lbstart >= b and lbstart < e: break for nbc, b, e in xbc: if lbstart >= b and lbstart < e: break bcidBlobLength = len(bcidblob) if bcidBlobLength == 3564: # run 2 beam1, beam2, beam12 = unpackRun2BCIDMask(bcidblob) #print "BC beam 1: %i, beam 2: %i, coll: %i" % (len(beam1), len(beam2), len(beam12)) else: # unpack BCID mask if len(bcidblob) == 2 * (nb1 + nb2 + nbc): beam1, beam2, beam12 = unpackRun1BCIDMask( bcidblob, nb1, nb2, nbc) #print "BC beam 1: %i, beam 2: %i, coll: %i" % (len(beam1), len(beam2), len(beam12)) else: print "WARNING, bcidMask inconsistent", nb1, nb2, nbc, "should add up to half of", len( bcidblob) beam1, beam2, beam12 = ([], [], []) # refill run stats bcidmask[i] = ((nb1, nb2, nbc), (beam1, beam2, beam12), lbstart, lbend)