def insertV2(dbsession,runnum,inputpathnames,inputdata): ''' inputpathnames: [pathname] inputdata: {cmslsnum:[presc,presc...]} ''' branchrevision_id=DATABRANCH_ID try: pathnamesClob=','.join(inputpathnames) hltrundata=[pathnamesClob,'text file'] (hltrevid,hltentryid,hltdataid)=dataDML.addHLTRunDataToBranch(dbsession.nominalSchema(),runnum,hltrundata,(branchrevision_id,'DATA')) hltlsdata={} for cmslsnum,perlsdata in inputdata.items(): prescaleArray=array.array('I') hltcountArray=array.array('I') hltacceptArray=array.array('I') for presc in perlsdata: thiscount=0 thisaccept=0 prescaleArray.append(presc) hltcountArray.append(thiscount) hltacceptArray.append(thisaccept) prescaleblob=CommonUtil.packArraytoBlob(prescaleArray) hltcountblob=CommonUtil.packArraytoBlob(hltcountArray) hltacceptblob=CommonUtil.packArraytoBlob(hltacceptArray) hltlsdata[cmslsnum]=[hltcountblob,hltacceptblob,prescaleblob] dataDML.bulkInsertHltLSData(dbsession,runnum,hltdataid,hltlsdata,500) except: raise
def fieldvalues(self, fieldname, fieldtype): ''' given the input result field name and typem return the list of values ''' fieldidx = None result = [] try: fieldidx = self.__inputResultHeader.index(fieldname) except: print 'field ', fieldname, ' not found' raise for r in self.__inputResult: stringvalue = r[fieldidx] if fieldtype in ['int', 'unsigned int']: if not CommonUtil.is_intstr(stringvalue): print 'field ', fieldname, ' is not integer type' raise else: result.append(int(stringvalue)) continue elif fieldtype in ['float']: if not CommonUtil.is_floatstr(stringvalue): print 'field ', fieldname, ' is not float type' raise else: result.append(float(stringvalue)) contine elif fieldtype in ['string', 'str']: result.append(stringvalue) else: raise 'unsupported type ', fieldtype return result
def insertLumiDetaildata(dbsession,perlsrawdata,perbunchrawdata,summaryidlsmap): dataDef=[] dataDef.append(('LUMISUMMARY_ID','unsigned long long')) dataDef.append(('LUMIDETAIL_ID','unsigned long long')) dataDef.append(('BXLUMIVALUE','blob')) dataDef.append(('BXLUMIERROR','blob')) dataDef.append(('BXLUMIQUALITY','blob')) dataDef.append(('ALGONAME','string')) perbunchiData=[] dbsession.transaction().start(False) iddealer=idDealer.idDealer(dbsession.nominalSchema()) db=dbUtil.dbUtil(dbsession.nominalSchema()) print('to insert lumidetail ') for algoname in ['OCC1','OCC2','ET']: for cmslsnum,instlumi in perlsrawdata.items(): lumisummary_id=summaryidlsmap[cmslsnum] lumidetail_id=iddealer.generateNextIDForTable('LUMIDETAIL') bxdata=array.array('f') bxerror=array.array('f') bxquality=array.array('h') for bxidx in range(1,3565): lumifraction=0.0 if bxidx in perbunchrawdata: lumifraction=perbunchrawdata[bxidx] bxlumivalue=float(instlumi*lumifraction)/float(bunchnorm) bxdata.append(bxlumivalue) bxerror.append(0.0) bxquality.append(1) bxdataBlob=CommonUtil.packArraytoBlob(bxdata) bxerrorBlob=CommonUtil.packArraytoBlob(bxerror) bxqualityBlob=CommonUtil.packArraytoBlob(bxquality) perbunchiData.append([('LUMISUMMARY_ID',lumisummary_id),('LUMIDETAIL_ID',lumidetail_id),('BXLUMIVALUE',bxdataBlob),('BXLUMIERROR',bxerrorBlob),('BXLUMIQUALITY',bxqualityBlob),('ALGONAME',algoname)]) db.bulkInsert('LUMIDETAIL',dataDef,perbunchiData) dbsession.transaction().commit() return
def fieldvalues(self,fieldname,fieldtype): ''' given the input result field name and typem return the list of values ''' fieldidx=None result=[] try: fieldidx=self.__inputResultHeader.index(fieldname) except: print('field ',fieldname,' not found') raise RuntimeError('field') for r in self.__inputResult: stringvalue=r[fieldidx] if fieldtype in ['int','unsigned int']: if not CommonUtil.is_intstr(stringvalue): print('field ',fieldname,' is not integer type') raise RuntimeError('field') else: result.append(int(stringvalue)) continue elif fieldtype in ['float']: if not CommonUtil.is_floatstr(stringvalue): print('field ',fieldname,' is not float type') raise RuntimeError('field') else: result.append(float(stringvalue)) contine elif fieldtype in ['string','str']: result.append(stringvalue) else: raise RuntimeError('unsupported type '+fieldtype) return result
def hlt(schema, nls): ''' input: output: [datasource,pathnameclob,{cmslsnum:[inputcountBlob,acceptcountBlob,prescaleBlob]}] ''' o = ['oracle://cms_orcon_prod/cms_runinfo'] pathnameclob = 'HLT_PixelTracks_Multiplicity70,HLT_PixelTracks_Multiplicity85,HLT_PixelTracks_Multiplicity100,HLT_GlobalRunHPDNoise,HLT_TechTrigHCALNoise' o.append(pathnameclob) perlsdata = {} for cmslsnum in range(1, nls + 1): inputcounts = array.array('I') acceptcounts = array.array('I') prescalecounts = array.array('I') for i in range(1, 201): inputcounts.append(6677889) acceptcounts.append(3344565) prescalecounts.append(17) inputcountsBlob = CommonUtil.packArraytoBlob(inputcounts) acceptcountsBlob = CommonUtil.packArraytoBlob(acceptcounts) prescalecountsBlob = CommonUtil.packArraytoBlob(prescalecounts) if cmslsnum not in perlsdata: perlsdata[cmslsnum] = [] perlsdata[cmslsnum].extend( [inputcountsBlob, acceptcountsBlob, prescalecountsBlob]) o.append(perlsdata) return o
def insertLumiDetaildata(dbsession,perlsrawdata,perbunchrawdata,summaryidlsmap): dataDef=[] dataDef.append(('LUMISUMMARY_ID','unsigned long long')) dataDef.append(('LUMIDETAIL_ID','unsigned long long')) dataDef.append(('BXLUMIVALUE','blob')) dataDef.append(('BXLUMIERROR','blob')) dataDef.append(('BXLUMIQUALITY','blob')) dataDef.append(('ALGONAME','string')) perbunchiData=[] dbsession.transaction().start(False) iddealer=idDealer.idDealer(dbsession.nominalSchema()) db=dbUtil.dbUtil(dbsession.nominalSchema()) print('to insert lumidetail ') for algoname in ['OCC1','OCC2','ET']: for cmslsnum,instlumi in perlsrawdata.items(): lumisummary_id=summaryidlsmap[cmslsnum] lumidetail_id=iddealer.generateNextIDForTable('LUMIDETAIL') bxdata=array.array('f') bxerror=array.array('f') bxquality=array.array('h') for bxidx in range(1,3565): lumifraction=0.0 if bxidx in perbunchrawdata: lumifraction=perbunchrawdata[bxidx] bxlumivalue=float(instlumi*lumifraction)/float(bunchnorm) bxdata.append(bxlumivalue) bxerror.append(0.0) bxquality.append(1) bxdataBlob=CommonUtil.packArraytoBlob(bxdata) bxerrorBlob=CommonUtil.packArraytoBlob(bxerror) bxqualityBlob=CommonUtil.packArraytoBlob(bxquality) perbunchiData.append([('LUMISUMMARY_ID',lumisummary_id),('LUMIDETAIL_ID',lumidetail_id),('BXLUMIVALUE',bxdataBlob),('BXLUMIERROR',bxerrorBlob),('BXLUMIQUALITY',bxqualityBlob),('ALGONAME',algoname)]) db.bulkInsert('LUMIDETAIL',dataDef,perbunchiData) dbsession.transaction().commit() return
def trg(schema, nls): ''' input: output: [datasource,bitzeroname,bitnameclob,{cmslsnum:[deadtime,bitzerocount,bitzeroprescale,trgcountBlob,trgprescaleBlob]}] ''' o = ['oracle://cms_orcon_prod/cms_gtmon', 'L1_ZeroBias'] bitnameclob = 'L1_ZeroBias,False,L1_SingleHfBitCountsRing1_1,L1_SingleHfBitCountsRing2_1,L1_SingleMu15,L1SingleJet,Jura' o.append(bitnameclob) perlsdata = {} for cmslsnum in range(1, nls + 1): deadtime = 99 + cmslsnum bitzerocount = 897865 bitzeroprescale = 17 trgcounts = array.array('I') prescalecounts = array.array('I') for i in range(1, 192): trgcounts.append(778899 + i) prescalecounts.append(17) trgcountsBlob = CommonUtil.packArraytoBlob(trgcounts) prescalecountsBlob = CommonUtil.packArraytoBlob(prescalecounts) if cmslsnum not in perlsdata: perlsdata[cmslsnum] = [] perlsdata[cmslsnum].extend([ deadtime, bitzerocount, bitzeroprescale, trgcountsBlob, prescalecountsBlob ]) o.append(perlsdata) return o
def hlt(schema,nls): ''' input: output: [datasource,pathnameclob,{cmslsnum:[inputcountBlob,acceptcountBlob,prescaleBlob]}] ''' o=['oracle://cms_orcon_prod/cms_runinfo'] pathnameclob='HLT_PixelTracks_Multiplicity70,HLT_PixelTracks_Multiplicity85,HLT_PixelTracks_Multiplicity100,HLT_GlobalRunHPDNoise,HLT_TechTrigHCALNoise' o.append(pathnameclob) perlsdata={} for cmslsnum in range(1,nls+1): inputcounts=array.array('I') acceptcounts=array.array('I') prescalecounts=array.array('I') for i in range(1,201): inputcounts.append(6677889 ) acceptcounts.append(3344565) prescalecounts.append(17) inputcountsBlob=CommonUtil.packArraytoBlob(inputcounts) acceptcountsBlob=CommonUtil.packArraytoBlob(acceptcounts) prescalecountsBlob=CommonUtil.packArraytoBlob(prescalecounts) if not perlsdata.has_key(cmslsnum): perlsdata[cmslsnum]=[] perlsdata[cmslsnum].extend([inputcountsBlob,acceptcountsBlob,prescalecountsBlob]) o.append(perlsdata) return o
def trg(schema,nls): ''' input: output: [datasource,bitzeroname,bitnameclob,{cmslsnum:[deadtime,bitzerocount,bitzeroprescale,trgcountBlob,trgprescaleBlob]}] ''' o=['oracle://cms_orcon_prod/cms_gtmon','L1_ZeroBias'] bitnameclob='L1_ZeroBias,False,L1_SingleHfBitCountsRing1_1,L1_SingleHfBitCountsRing2_1,L1_SingleMu15,L1SingleJet,Jura' o.append(bitnameclob) perlsdata={} for cmslsnum in range(1,nls+1): deadtime=99+cmslsnum bitzerocount=897865 bitzeroprescale=17 trgcounts=array.array('I') prescalecounts=array.array('I') for i in range(1,192): trgcounts.append(778899+i) prescalecounts.append(17) trgcountsBlob=CommonUtil.packArraytoBlob(trgcounts) prescalecountsBlob=CommonUtil.packArraytoBlob(prescalecounts) if not perlsdata.has_key(cmslsnum): perlsdata[cmslsnum]=[] perlsdata[cmslsnum].extend([deadtime,bitzerocount,bitzeroprescale,trgcountsBlob,prescalecountsBlob]) o.append(perlsdata) return o
def insertV2(dbsession,runnum,inputpathnames,inputdata): ''' inputpathnames: [pathname] inputdata: {cmslsnum:[presc,presc...]} ''' branchrevision_id=DATABRANCH_ID try: pathnamesClob=','.join(inputpathnames) hltrundata=[pathnamesClob,'text file'] (hltrevid,hltentryid,hltdataid)=dataDML.addHLTRunDataToBranch(dbsession.nominalSchema(),runnum,hltrundata,(branchrevision_id,'DATA')) hltlsdata={} for cmslsnum,perlsdata in inputdata.items(): prescaleArray=array.array('I') hltcountArray=array.array('I') hltacceptArray=array.array('I') for presc in perlsdata: thiscount=0 thisaccept=0 prescaleArray.append(presc) hltcountArray.append(thiscount) hltacceptArray.append(thisaccept) prescaleblob=CommonUtil.packArraytoBlob(prescaleArray) hltcountblob=CommonUtil.packArraytoBlob(hltcountArray) hltacceptblob=CommonUtil.packArraytoBlob(hltacceptArray) hltlsdata[cmslsnum]=[hltcountblob,hltacceptblob,prescaleblob] dataDML.bulkInsertHltLSData(dbsession,runnum,hltdataid,hltlsdata,500) except: raise
def fieldtotal(self,fieldname,fieldtype): ''' given the input result field name and type, return the total ''' fieldidx=None result=0 try: fieldidx=self.__inputResultHeader.index(fieldname) except: print('field ',fieldname,' not found') raise for r in self.__inputResult: stringvalue=r[fieldidx] if fieldtype in ['int','unsigned int']: if not CommonUtil.is_intstr(stringvalue): print('field ',fieldname,' is not integer type') raise else: result=int(result)+int(stringvalue) continue elif fieldtype in ['float'] : if not CommonUtil.is_floatstr(stringvalue): print('field ',fieldname,' is not float type') raise else: result=float(result)+float(stringvalue) continue else: raise RunTimeError('cannot sum types other than int ,float') return result
def fieldtotal(self, fieldname, fieldtype): ''' given the input result field name and type, return the total ''' fieldidx = None result = 0 try: fieldidx = self.__inputResultHeader.index(fieldname) except: print 'field ', fieldname, ' not found' raise for r in self.__inputResult: stringvalue = r[fieldidx] if fieldtype in ['int', 'unsigned int']: if not CommonUtil.is_intstr(stringvalue): print 'field ', fieldname, ' is not integer type' raise else: result = int(result) + int(stringvalue) continue elif fieldtype in ['float']: if not CommonUtil.is_floatstr(stringvalue): print 'field ', fieldname, ' is not float type' raise else: result = float(result) + float(stringvalue) continue else: raise 'cannot sum types other than int ,float' return result
def insertLumiDetaildata(dbsession, perlsrawdata, perbunchrawdata, summaryidlsmap): dataDef = [] dataDef.append(("LUMISUMMARY_ID", "unsigned long long")) dataDef.append(("LUMIDETAIL_ID", "unsigned long long")) dataDef.append(("BXLUMIVALUE", "blob")) dataDef.append(("BXLUMIERROR", "blob")) dataDef.append(("BXLUMIQUALITY", "blob")) dataDef.append(("ALGONAME", "string")) perbunchiData = [] dbsession.transaction().start(False) iddealer = idDealer.idDealer(dbsession.nominalSchema()) db = dbUtil.dbUtil(dbsession.nominalSchema()) print "to insert lumidetail " for algoname in ["OCC1", "OCC2", "ET"]: for cmslsnum, instlumi in perlsrawdata.items(): lumisummary_id = summaryidlsmap[cmslsnum] lumidetail_id = iddealer.generateNextIDForTable("LUMIDETAIL") print "cmslsnum ", lumidetail_id, lumisummary_id bxdataocc1 = array.array("f") bxdataocc2 = array.array("f") bxdataet = array.array("f") bxerror = array.array("f") bxquality = array.array("h") for bxidx in range(1, 3565): lumifraction = 0.0 if perbunchrawdata.has_key(bxidx): lumifraction = perbunchrawdata[bxidx] bxlumivalue = float(instlumi * lumifraction) bxdataocc1.append(bxlumivalue) bxdataocc2.append(bxlumivalue) bxdataet.append(bxlumivalue) bxdataocc1Blob = CommonUtil.packArraytoBlob(bxdataocc1) bxdataocc2Blob = CommonUtil.packArraytoBlob(bxdataocc2) bxdataetBlob = CommonUtil.packArraytoBlob(bxdataet) bxqualityBlob = CommonUtil.packArraytoBlob(bxquality) perbunchiData.append( [ ("LUMISUMMARY_ID", lumisummary_id), ("LUMIDETAIL_ID", lumidetail_id), ("BXLUMIVALUE", bxdataocc1Blob), ("BXLUMIERROR", bxdataocc2Blob), ("BXLUMIQUALITY", bxqualityBlob), ("ALGONAME", algoname), ] ) db.bulkInsert("LUMIDETAIL", dataDef, perbunchiData) print perbunchiData dbsession.transaction().commit() return
def _getnorm(schema,norm): if isinstance(norm,int) or isinstance(norm,float) or CommonUtil.is_floatstr(norm) or CommonUtil.is_intstr(norm): return float(norm) if not isinstance(norm,str): raise ValueError('wrong parameter type') normdataid=dataDML.guessnormIdByName(schema,norm) normresult=dataDML.luminormById(schema,normdataid) return normresult[2]
def toCSVLumiByLSXing(lumidata,scalefactor,filename): ''' input:{run:[lumilsnum(0),cmslsnum(1),timestamp(2),beamstatus(3),beamenergy(4),deliveredlumi(5),recordedlumi(6),calibratedlumierror(7),{hltpath:[l1name,l1prescale,hltprescale,efflumi]},bxdata,beamdata]} output: fieldnames=['Run','CMSLS','Delivered(/ub)','Recorded(/ub)','BX'] ''' result=[] assert(filename) fieldnames=['run','ls','delivered(/ub)','recorded(/ub)','bx'] for run in sorted(lumidata): rundata=lumidata[run] if rundata is None: result.append([run,'n/a','n/a','n/a','n/a']) continue for lsdata in rundata: cmslsnum=lsdata[1] if cmslsnum==0: continue deliveredlumi=lsdata[5] recordedlumi=lsdata[6] (bxidxlist,bxvaluelist,bxerrorlist)=lsdata[8] bxresult=[] if bxidxlist and bxvaluelist: bxinfo=CommonUtil.transposed([bxidxlist,bxvaluelist]) bxresult=CommonUtil.flatten([run,cmslsnum,deliveredlumi*scalefactor,recordedlumi*scalefactor,bxinfo]) result.append(bxresult) else: result.append([run,cmslsnum,deliveredlumi*scalefactor,recordedlumi*scalefactor]) r=None if filename.upper()=='STDOUT': r=sys.stdout r.write(','.join(fieldnames)+'\n') for l in result: r.write(str(l)+'\n') else: r=csvReporter.csvReporter(filename) r.writeRow(fieldnames) r.writeRows(result)
def runsWithresult(self): ''' output: [run,run,...] ''' result = {} for f in self.__inputresultfiles: csvReader = csv.reader(open(f), delimiter=',') for row in csvReader: field0 = str(row[0]).strip() if not CommonUtil.is_intstr(field0): continue runnumber = int(field0) if runnumber not in result: result[runnumber] = None return result.keys()
def runsWithresult(self): ''' output: [run,run,...] ''' result={} for f in self.__inputresultfiles: csvReader=csv.reader(open(f),delimiter=',') for row in csvReader: field0=str(row[0]).strip() if not CommonUtil.is_intstr(field0): continue runnumber=int(field0) if runnumber not in result: result[runnumber]=None return result.keys()
def lumiSummary(schema, nlumils): ''' input: output: [datasource,{lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beamintensityblob_1,beamintensitublob_2,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]}] ''' o = ['file:fake.root'] perlsdata = {} for lumilsnum in range(1, nlumils + 1): cmslsnum = 0 if lumilsnum < 100: cmslsnum = lumilsnum instlumi = 2.37 instlumierror = 0.56 instlumiquality = 2 beamstatus = 'STABLE BEAMS' beamenergy = 3.5e03 numorbit = 12345 startorbit = numorbit * lumilsnum if cmslsnum == 0: cmsbxindex = None beam1intensity = None beam2intensity = None else: cmsbxindex = array.array('I') beam1intensity = array.array('f') beam2intensity = array.array('f') for idx in range(1, 3565): cmsbxindex.append(idx) beam1intensity.append(1.5e09) beam2intensity.append(5.5e09) cmsbxindexBlob = CommonUtil.packArraytoBlob(cmsbxindex) beam1intensityBlob = CommonUtil.packArraytoBlob(beam1intensity) beam2intensityBlob = CommonUtil.packArraytoBlob(beam2intensity) bxlumivalue = array.array('f') bxlumierror = array.array('f') bxlumiquality = array.array('I') for idx in range(1, 3565): bxlumivalue.append(2.3) bxlumierror.append(0.4) bxlumiquality.append(2) bxlumivalueBlob = CommonUtil.packArraytoBlob(bxlumivalue) bxlumierrorBlob = CommonUtil.packArraytoBlob(bxlumierror) bxlumiqualityBlob = CommonUtil.packArraytoBlob(bxlumiquality) if lumilsnum not in perlsdata: perlsdata[lumilsnum] = [] perlsdata[lumilsnum].extend([ cmslsnum, instlumi, instlumierror, instlumiquality, beamstatus, beamenergy, numorbit, startorbit, cmsbxindexBlob, beam1intensityBlob, beam2intensityBlob, bxlumivalueBlob, bxlumierrorBlob, bxlumiqualityBlob, bxlumivalueBlob, bxlumierrorBlob, bxlumiqualityBlob, bxlumivalueBlob, bxlumierrorBlob, bxlumiqualityBlob ]) o.append(perlsdata) return o
def toCSVOverview(lumidata,filename,resultlines,scalefactor,isverbose): ''' input: lumidata {run:[lumilsnum,cmslsnum,timestamp,beamstatus,beamenergy,deliveredlumi,recordedlumi,calibratedlumierror,(bxidx,bxvalues,bxerrs),(bxidx,b1intensities,b2intensities)]} resultlines [[resultrow1],[resultrow2],...,] existing result row ''' result=[] fieldnames = ['Run', 'DeliveredLS', 'Delivered(/ub)','SelectedLS','Recorded(/ub)'] r=csvReporter.csvReporter(filename) for rline in resultlines: result.append(rline) for run in lumidata.keys(): lsdata=lumidata[run] if lsdata is None: result.append([run,'n/a','n/a','n/a','n/a']) continue nls=len(lsdata) deliveredData=[x[5] for x in lsdata] recordedData=[x[6] for x in lsdata if x[6] is not None] totdeliveredlumi=0.0 totrecordedlumi=0.0 if len(deliveredData)!=0: totdeliveredlumi=sum(deliveredData) if len(recordedData)!=0: totrecordedlumi=sum(recordedData) selectedcmsls=[x[1] for x in lsdata if x[1]!=0] if len(selectedcmsls)==0: selectedlsStr='n/a' else: selectedlsStr = CommonUtil.splitlistToRangeString(selectedcmsls) result.append([run,nls,totdeliveredlumi*scalefactor,selectedlsStr,totrecordedlumi*scalefactor]) sortedresult=sorted(result,key=lambda x : int(x[0])) r=None assert(filename) if filename.upper()=='STDOUT': r=sys.stdout r.write(','.join(fieldnames)+'\n') for l in sortedresult: r.write(str(l)+'\n') else: r=csvReporter.csvReporter(filename) r.writeRow(fieldnames) r.writeRows(sortedresult)
def getInstLumiPerLS(dbsession,c,runList,selectionDict,beamstatus=None,beamenergy=None,beamenergyfluctuation=0.09,finecorrections=None): ''' input: runList[runnum], selectionDict{runnum:[ls]} output:[[runnumber,lsnumber,deliveredInst,recordedInst,norbit,startorbit,runstarttime,runstoptime]] ''' result=[] selectedRunlist=runList if len(selectionDict)!=0: selectedRunlist=[] allruns=runlist+selectionDict.keys() dups=CommonUtil.count_dups(allruns) for runnum,dupcount in dups: if dupcount==2: selectedRunlist.append(runnum) dbsession.transaction().start(True) for run in selectedRunlist: q=dbsession.nominalSchema().newQuery() runsummary=lumiQueryAPI.runsummaryByrun(q,run) del q runstarttime=runsummary[3] runstoptime=runsummary[4] q=dbsession.nominalSchema().newQuery() if finecorrections and finecorrections[run]: lumiperrun=lumiQueryAPI.lumisummaryByrun(q,run,c.LUMIVERSION,beamstatus,beamenergy,beamenergyfluctuation,finecorrections=finecorrections[run]) else: lumiperrun=lumiQueryAPI.lumisummaryByrun(q,run,c.LUMIVERSION,beamstatus,beamenergy,beamenergyfluctuation) del q if len(lumiperrun)==0: #no result for this run result.append([run,1,0.0,0.0,0,0,runstarttime,runstoptime]) else: for lumiperls in lumiperrun: cmslsnum=lumiperls[0] instlumi=lumiperls[1] recordedlumi=0.0 numorbit=lumiperls[2] startorbit=lumiperls[3] deadcount=0 bitzero=0 result.append([run,cmslsnum,instlumi,recordedlumi,numorbit,startorbit,runstarttime,runstoptime]) dbsession.transaction().commit() if c.VERBOSE: print result return result
def lumiSummary(schema,nlumils): ''' input: output: [datasource,{lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beamintensityblob_1,beamintensitublob_2,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]}] ''' o=['file:fake.root'] perlsdata={} for lumilsnum in range(1,nlumils+1): cmslsnum=0 if lumilsnum<100: cmslsnum=lumilsnum instlumi=2.37 instlumierror=0.56 instlumiquality=2 beamstatus='STABLE BEAMS' beamenergy=3.5e03 numorbit=12345 startorbit=numorbit*lumilsnum if cmslsnum==0: cmsbxindex=None beam1intensity=None beam2intensity=None else: cmsbxindex=array.array('I') beam1intensity=array.array('f') beam2intensity=array.array('f') for idx in range(1,3565): cmsbxindex.append(idx) beam1intensity.append(1.5e09) beam2intensity.append(5.5e09) cmsbxindexBlob=CommonUtil.packArraytoBlob(cmsbxindex) beam1intensityBlob=CommonUtil.packArraytoBlob(beam1intensity) beam2intensityBlob=CommonUtil.packArraytoBlob(beam2intensity) bxlumivalue=array.array('f') bxlumierror=array.array('f') bxlumiquality=array.array('I') for idx in range(1,3565): bxlumivalue.append(2.3) bxlumierror.append(0.4) bxlumiquality.append(2) bxlumivalueBlob=CommonUtil.packArraytoBlob(bxlumivalue) bxlumierrorBlob=CommonUtil.packArraytoBlob(bxlumierror) bxlumiqualityBlob=CommonUtil.packArraytoBlob(bxlumiquality) if not perlsdata.has_key(lumilsnum): perlsdata[lumilsnum]=[] perlsdata[lumilsnum].extend([cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexBlob,beam1intensityBlob,beam2intensityBlob,bxlumivalueBlob,bxlumierrorBlob,bxlumiqualityBlob,bxlumivalueBlob,bxlumierrorBlob,bxlumiqualityBlob,bxlumivalueBlob,bxlumierrorBlob,bxlumiqualityBlob]) o.append(perlsdata) return o
def normValueById(schema, normid): ''' select * from luminormsv2data where data_id=normid output: {since:[corrector(0),{paramname:paramvalue}(1),amodetag(2),egev(3),comment(4)]} ''' result = {} l = nameDealer.luminormv2dataTableName() paramdict = {} qHandle = schema.newQuery() try: qHandle.addToTableList(l) qConditionStr = 'DATA_ID=:normid' qCondition = coral.AttributeList() qCondition.extend('normid', 'unsigned long long') qCondition['normid'].setData(normid) qResult = coral.AttributeList() qHandle.setCondition(qConditionStr, qCondition) cursor = qHandle.execute() while cursor.next(): since = cursor.currentRow()['SINCE'].data() corrector = cursor.currentRow()['CORRECTOR'].data() amodetag = cursor.currentRow()['AMODETAG'].data() nominalegev = cursor.currentRow()['NOMINALEGEV'].data() comment = '' if not cursor.currentRow()['COMMENT'].isNull(): comment = cursor.currentRow()['COMMENT'].data() (correctorfunc, params) = CommonUtil.parselumicorrector(corrector) paramdict = {} for param in params: paramvalue = 0.0 if not cursor.currentRow()[param.upper()].isNull(): paramvalue = cursor.currentRow()[param.upper()].data() paramdict[param] = paramvalue result[since] = [ correctorfunc, paramdict, amodetag, nominalegev, comment ] except: raise return result
def normValueById(schema,normid): ''' select * from luminormsv2data where data_id=normid output: {since:[corrector(0),{paramname:paramvalue}(1),amodetag(2),egev(3),comment(4)]} ''' result={} l=nameDealer.luminormv2dataTableName() paramdict={} qHandle=schema.newQuery() try: qHandle.addToTableList(l) qConditionStr='DATA_ID=:normid' qCondition=coral.AttributeList() qCondition.extend('normid','unsigned long long') qCondition['normid'].setData(normid) qResult=coral.AttributeList() qHandle.setCondition(qConditionStr,qCondition) cursor=qHandle.execute() while cursor.next(): since=cursor.currentRow()['SINCE'].data() corrector=cursor.currentRow()['CORRECTOR'].data() amodetag=cursor.currentRow()['AMODETAG'].data() nominalegev=cursor.currentRow()['NOMINALEGEV'].data() comment='' if not cursor.currentRow()['COMMENT'].isNull(): comment=cursor.currentRow()['COMMENT'].data() (correctorfunc,params)=CommonUtil.parselumicorrector(corrector) paramdict={} for param in params: paramvalue=0.0 if not cursor.currentRow()[param.upper()].isNull(): paramvalue=cursor.currentRow()[param.upper()].data() paramdict[param]=paramvalue result[since]=[correctorfunc,paramdict,amodetag,nominalegev,comment] except: raise return result
def fetchOldData(schema, oldlumidataid): ''' fetch old perbunch data if the run exists select CMSBXINDEXBLOB,BEAMINTENSITYBLOB_1,BEAMINTENSITYBLOB_2,BXLUMIVALUE_OCC1,BXLUMIVALUE_OCC2,BXLUMIVALUE_ET,BXLUMIERROR_OCC1,BXLUMIERROR_OCC2,BXLUMIERROR_ET,BXLUMIQUALITY_OCC1,BXLUMIQUALITY_OCC2,BXLUMIQUALITY_ET from lumisummaryv2 where data_id=:oldlumidataid; output: {lumilsnum:[bxvalueblob_occ1(0),bxerrblob_occ1(1),bxqualityblob_occ1(2),bxvalueblob_occ2(3),bxerrblob_occ2(4),bxqualityblob_occ2(5),bxvalueblob_et(6),bxerrblob_et(7),bxqualityblob_et(8),bxindexblob(9),beam1intensity(10),beam2intensity(11),beamstatus(12),beamenergy(13),instlumierror(14),instlumiquality(15)],...} ''' result = {} qHandle = schema.newQuery() try: qHandle.addToTableList('LUMISUMMARYV2') qHandle.addToOutputList('LUMILSNUM', 'lumilsnum') qHandle.addToOutputList('BEAMSTATUS', 'beamstatus') qHandle.addToOutputList('BEAMENERGY', 'beamenergy') qHandle.addToOutputList('INSTLUMIERROR', 'instlumierror') qHandle.addToOutputList('INSTLUMIQUALITY', 'instlumiquality') qHandle.addToOutputList('BXLUMIVALUE_OCC1', 'bxvalue_occ1') qHandle.addToOutputList('BXLUMIERROR_OCC1', 'bxerror_occ1') qHandle.addToOutputList('BXLUMIQUALITY_OCC1', 'bxquality_occ1') qHandle.addToOutputList('BXLUMIVALUE_OCC2', 'bxvalue_occ2') qHandle.addToOutputList('BXLUMIERROR_OCC2', 'bxerror_occ2') qHandle.addToOutputList('BXLUMIQUALITY_OCC2', 'bxquality_occ2') qHandle.addToOutputList('BXLUMIVALUE_ET', 'bxvalue_et') qHandle.addToOutputList('BXLUMIERROR_ET', 'bxerror_et') qHandle.addToOutputList('BXLUMIQUALITY_ET', 'bxquality_et') qHandle.addToOutputList('CMSBXINDEXBLOB', 'bxindexblob') qHandle.addToOutputList('BEAMINTENSITYBLOB_1', 'beam1intensity') qHandle.addToOutputList('BEAMINTENSITYBLOB_2', 'beam2intensity') qConditionStr = 'DATA_ID=:dataid' qCondition = coral.AttributeList() qCondition.extend('dataid', 'unsigned long long') print('oldlumidataid ', oldlumidataid) qCondition['dataid'].setData(int(oldlumidataid)) qResult = coral.AttributeList() qResult.extend('lumilsnum', 'unsigned int') qResult.extend('beamstatus', 'string') qResult.extend('beamenergy', 'float') qResult.extend('instlumierror', 'float') qResult.extend('instlumiquality', 'short') qResult.extend('bxvalue_occ1', 'blob') qResult.extend('bxerror_occ1', 'blob') qResult.extend('bxquality_occ1', 'blob') qResult.extend('bxvalue_occ2', 'blob') qResult.extend('bxerror_occ2', 'blob') qResult.extend('bxquality_occ2', 'blob') qResult.extend('bxvalue_et', 'blob') qResult.extend('bxerror_et', 'blob') qResult.extend('bxquality_et', 'blob') qResult.extend('bxindexblob', 'blob') qResult.extend('beam1intensity', 'blob') qResult.extend('beam2intensity', 'blob') qHandle.defineOutput(qResult) qHandle.setCondition(qConditionStr, qCondition) cursor = qHandle.execute() while next(cursor): lumilsnum = cursor.currentRow()['lumilsnum'].data() beamstatus = cursor.currentRow()['beamstatus'].data() beamenergy = cursor.currentRow()['beamenergy'].data() instlumierror = cursor.currentRow()['instlumierror'].data() instlumiquality = cursor.currentRow()['instlumiquality'].data() bxvalueblob_occ1 = cursor.currentRow()['bxvalue_occ1'].data() bxvalueblob_occ1Array = CommonUtil.unpackBlobtoArray( bxvalueblob_occ1, 'f') bxerrblob_occ1 = cursor.currentRow()['bxerror_occ1'].data() bxqualityblob_occ1 = cursor.currentRow()['bxquality_occ1'].data() bxvalueblob_occ2 = cursor.currentRow()['bxvalue_occ2'].data() bxerrblob_occ2 = cursor.currentRow()['bxerror_occ2'].data() bxqualityblob_occ2 = cursor.currentRow()['bxquality_occ2'].data() bxvalueblob_et = cursor.currentRow()['bxvalue_et'].data() bxerrblob_et = cursor.currentRow()['bxerror_et'].data() bxqualityblob_et = cursor.currentRow()['bxquality_et'].data() bxindexblob = cursor.currentRow()['bxindexblob'].data() beam1intensity = cursor.currentRow()['beam1intensity'].data() beam2intensity = cursor.currentRow()['beam2intensity'].data() result[lumilsnum] = [ bxvalueblob_occ1, bxerrblob_occ1, bxqualityblob_occ1, bxvalueblob_occ2, bxerrblob_occ2, bxqualityblob_occ2, bxvalueblob_et, bxerrblob_et, bxqualityblob_et, bxindexblob, beam1intensity, beam2intensity, beamstatus, beamenergy, instlumierror, instlumiquality ] except: del qHandle raise del qHandle return result
def generateLumidata(lumirundatafromfile, lsdatafromfile, rundatafromdb, lsdatafromdb, replacelsMin, replacelsMax): ''' input: perrunresultfromfile=[]#source,starttime,stoptime,nls perlsresultfromfile={} #{lumilsnum:instlumiub} lumirundatafromdb=[] #[source,nominalegev,ncollidingbunches,starttime,stoptime,nls] lumilsdatafromdb={}#{lumilsnum:[cmslsnum(0),instlumi(1),instlumierror(2),instlumiquality(3),beamstatus(4),beamenergy(5),numorbit(6),startorbit(7),cmsbxindexblob(8),beamintensityblob_1(9),beamintensityblob_2(10),bxlumivalue_occ1(11),bxlumierror_occ1(12),bxlumiquality_occ1(13),bxlumivalue_occ2(14),bxlumierror_occ2(15),bxlumiquality_occ2(16),bxlumivalue_et(17),bxlumierror_et(18),bxlumiquality_et(19)]} ''' lumip = lumiParameters.ParametersObject() numorbit = lumip.numorbit startorbit = 0 fakebeamenergy = 4000. fakebeamstatus = 'STABLE BEAMS' fakefloatArray = array.array('f') fakeidxArray = array.array('h') fakeshortArray = array.array('h') for bxidx in range(1, 3565): fakeidxArray.append(bxidx) fakefloatArray.append(0.) fakeshortArray.append(0) lumirundata = [] lumilsdata = {} if rundatafromdb: lumirundata = rundatafromdb lumirundata[0] = rundatafromdb[0] + '+file:' + lumirundatafromfile[0] else: lu = lumiTime.lumiTime() source = '+file:' + lumirundatafromfile[0] nominalegev = fakebeamenergy ncollidingbunches = 72 starttime = lumirundatafromfile[1] stoptime = lumirundatafromfile[2] starttimeT = lu.timestampTodatetimeUTC(starttime) stoptimeT = lu.timestampTodatetimeUTC(stoptime) print(starttimeT.day, starttimeT.month, starttimeT.year) starttimeT_coral = coral.TimeStamp(starttimeT.year, starttimeT.month, starttimeT.day, starttimeT.hour, starttimeT.minute, starttimeT.second, 0) stoptimeT_coral = coral.TimeStamp(stoptimeT.year, stoptimeT.month, stoptimeT.day, stoptimeT.hour, stoptimeT.minute, stoptimeT.second, 0) nls = lumirundatafromfile[3] lumirundata = [ source, nominalegev, ncollidingbunches, starttimeT_coral, stoptimeT_coral, nls ] if lsdatafromdb: lumilsdata = lsdatafromdb if replacelsMin > len(lsdatafromdb): print('[INFO]Operation: extend an existing run from LS=', replacelsMin) lumirundata[5] += len(lsdatafromfile) else: print( '[INFO]Operation: replace instlumi in an existing run LS range=', replacelsMin, replacelsMax) else: print('[INFO]Operation: insert a new fake run') for lumilsnum in range(replacelsMin, replacelsMax + 1): instlumi = lsdatafromfile[lumilsnum] if lumilsnum in lsdatafromdb.keys(): #if this is a hole lumilsdata[lumilsnum][1] = instlumi else: #if this is an extension instlumierror = 0.0 instlumiquality = 0 startorbit = (lumilsnum - 1) * numorbit cmsbxindexblob = CommonUtil.packArraytoBlob(fakeshortArray) beamintensityblob_1 = CommonUtil.packArraytoBlob(fakefloatArray) beamintensityblob_2 = CommonUtil.packArraytoBlob(fakefloatArray) bxlumivalue_occ1 = CommonUtil.packArraytoBlob(fakefloatArray) bxlumierror_occ1 = CommonUtil.packArraytoBlob(fakefloatArray) bxlumiquality_occ1 = CommonUtil.packArraytoBlob(fakeshortArray) bxlumivalue_occ2 = CommonUtil.packArraytoBlob(fakefloatArray) bxlumierror_occ2 = CommonUtil.packArraytoBlob(fakefloatArray) bxlumiquality_occ2 = CommonUtil.packArraytoBlob(fakeshortArray) bxlumivalue_et = CommonUtil.packArraytoBlob(fakefloatArray) bxlumierror_et = CommonUtil.packArraytoBlob(fakefloatArray) bxlumiquality_et = CommonUtil.packArraytoBlob(fakeshortArray) lumilsdata[lumilsnum] = [ 0, instlumi, instlumierror, instlumiquality, fakebeamstatus, fakebeamenergy, numorbit, startorbit, cmsbxindexblob, beamintensityblob_1, beamintensityblob_2, bxlumivalue_occ1, bxlumierror_occ1, bxlumiquality_occ1, bxlumivalue_occ2, bxlumierror_occ2, bxlumiquality_occ2, bxlumivalue_et, bxlumierror_et, bxlumiquality_et ] return (lumirundata, lumilsdata)
def main(): allowedscales=['linear','log','both'] c=constants() parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description="Plot integrated luminosity as function of the time variable of choice",formatter_class=argparse.ArgumentDefaultsHelpFormatter) # add required arguments parser.add_argument('-c',dest='connect',action='store',required=True,help='connect string to lumiDB') # add optional arguments parser.add_argument('-P',dest='authpath',action='store',help='path to authentication file') parser.add_argument('-n',dest='normfactor',action='store',help='normalization factor (optional, default to 1.0)') parser.add_argument('-i',dest='inputfile',action='store',help='lumi range selection file (optional)') parser.add_argument('-o',dest='outputfile',action='store',help='csv outputfile name (optional)') parser.add_argument('-lumiversion',dest='lumiversion',default='0001',action='store',required=False,help='lumi data version') parser.add_argument('-begin',dest='begin',action='store',help='begin value of x-axi (required)') parser.add_argument('-end',dest='end',action='store',help='end value of x-axi (optional). Default to the maximum exists DB') parser.add_argument('-beamenergy',dest='beamenergy',action='store',type=float,required=False,help='beamenergy (in GeV) selection criteria,e.g. 3.5e3') parser.add_argument('-beamfluctuation',dest='beamfluctuation',action='store',type=float,required=False,help='allowed fraction of beamenergy to fluctuate, e.g. 0.1') parser.add_argument('-beamstatus',dest='beamstatus',action='store',required=False,help='selection criteria beam status,e.g. STABLE BEAMS') parser.add_argument('-yscale',dest='yscale',action='store',required=False,default='linear',help='y_scale') parser.add_argument('-hltpath',dest='hltpath',action='store',help='specific hltpath to calculate the recorded luminosity. If specified aoverlays the recorded luminosity for the hltpath on the plot') parser.add_argument('-batch',dest='batch',action='store',help='graphical mode to produce PNG file. Specify graphical file here. Default to lumiSum.png') parser.add_argument('--annotateboundary',dest='annotateboundary',action='store_true',help='annotate boundary run numbers') parser.add_argument('--interactive',dest='interactive',action='store_true',help='graphical mode to draw plot in a TK pannel.') parser.add_argument('-timeformat',dest='timeformat',action='store',help='specific python timeformat string (optional). Default mm/dd/yy hh:min:ss.00') parser.add_argument('-siteconfpath',dest='siteconfpath',action='store',help='specific path to site-local-config.xml file, default to $CMS_PATH/SITECONF/local/JobConfig, if path undefined, fallback to cern proxy&server') parser.add_argument('action',choices=['run','fill','time','perday'],help='x-axis data type of choice') #graphical mode options parser.add_argument('--with-correction',dest='withFineCorrection',action='store_true',help='with fine correction') parser.add_argument('--verbose',dest='verbose',action='store_true',help='verbose mode, print result also to screen') parser.add_argument('--debug',dest='debug',action='store_true',help='debug') # parse arguments batchmode=True args=parser.parse_args() connectstring=args.connect begvalue=args.begin endvalue=args.end beamstatus=args.beamstatus beamenergy=args.beamenergy beamfluctuation=args.beamfluctuation xaxitype='run' connectparser=connectstrParser.connectstrParser(connectstring) connectparser.parse() usedefaultfrontierconfig=False cacheconfigpath='' if connectparser.needsitelocalinfo(): if not args.siteconfpath: cacheconfigpath=os.environ['CMS_PATH'] if cacheconfigpath: cacheconfigpath=os.path.join(cacheconfigpath,'SITECONF','local','JobConfig','site-local-config.xml') else: usedefaultfrontierconfig=True else: cacheconfigpath=args.siteconfpath cacheconfigpath=os.path.join(cacheconfigpath,'site-local-config.xml') p=cacheconfigParser.cacheconfigParser() if usedefaultfrontierconfig: p.parseString(c.defaultfrontierConfigString) else: p.parse(cacheconfigpath) connectstring=connectparser.fullfrontierStr(connectparser.schemaname(),p.parameterdict()) #print 'connectstring',connectstring runnumber=0 svc = coral.ConnectionService() hltpath='' if args.hltpath: hltpath=args.hltpath if args.debug : msg=coral.MessageStream('') msg.setMsgVerbosity(coral.message_Level_Debug) ifilename='' ofilename='integratedlumi.png' timeformat='' if args.authpath and len(args.authpath)!=0: os.environ['CORAL_AUTH_PATH']=args.authpath if args.normfactor: c.NORM=float(args.normfactor) if args.lumiversion: c.LUMIVERSION=args.lumiversion if args.verbose: c.VERBOSE=True if args.inputfile: ifilename=args.inputfile if args.batch: opicname=args.batch if args.outputfile: ofilename=args.outputfile if args.timeformat: timeformat=args.timeformat session=svc.connect(connectstring,accessMode=coral.access_Update) session.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)") session.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)") inputfilecontent='' fileparsingResult='' runList=[] runDict={} fillDict={} selectionDict={} minTime='' maxTime='' startRunTime='' stopRunTime='' #if len(ifilename)!=0 : # f=open(ifilename,'r') # inputfilecontent=f.read() # sparser=selectionParser.selectionParser(inputfilecontent) # runsandls=sparser.runsandls() # keylist=runsandls.keys() # keylist.sort() # for run in keylist: # if selectionDict.has_key(run): # lslist=runsandls[run] # lslist.sort() # selectionDict[run]=lslist if len(ifilename)!=0: ifparser=inputFilesetParser.inputFilesetParser(ifilename) runsandls=ifparser.runsandls() keylist=runsandls.keys() keylist.sort() for run in keylist: if not selectionDict.has_key(run): lslist=runsandls[run] lslist.sort() selectionDict[run]=lslist if args.action == 'run': if not args.end: session.transaction().start(True) schema=session.nominalSchema() lastrun=max(lumiQueryAPI.allruns(schema,requireRunsummary=True,requireLumisummary=True,requireTrg=True,requireHlt=False)) session.transaction().commit() else: lastrun=int(args.end) for r in range(int(args.begin),lastrun+1): runList.append(r) runList.sort() elif args.action == 'fill': session.transaction().start(True) maxfill=None if not args.end: qHandle=session.nominalSchema().newQuery() maxfill=max(lumiQueryAPI.allfills(qHandle,filtercrazy=True)) del qHandle else: maxfill=int(args.end) qHandle=session.nominalSchema().newQuery() fillDict=lumiQueryAPI.runsByfillrange(qHandle,int(args.begin),maxfill) del qHandle session.transaction().commit() #print 'fillDict ',fillDict for fill in range(int(args.begin),maxfill+1): if fillDict.has_key(fill): #fill exists for run in fillDict[fill]: runList.append(run) runList.sort() elif args.action == 'time' or args.action == 'perday': session.transaction().start(True) t=lumiTime.lumiTime() minTime=t.StrToDatetime(args.begin,timeformat) if not args.end: maxTime=datetime.datetime.utcnow() #to now else: maxTime=t.StrToDatetime(args.end,timeformat) #print minTime,maxTime qHandle=session.nominalSchema().newQuery() runDict=lumiQueryAPI.runsByTimerange(qHandle,minTime,maxTime)#xrawdata session.transaction().commit() runList=runDict.keys() del qHandle runList.sort() if len(runList)!=0: runmin=min(runList) runmax=max(runList) startRunTime=runDict[runmin][0] stopRunTime=runDict[runmax][1] #print 'run list: ',runDict else: print 'unsupported action ',args.action exit finecorrections=None if args.withFineCorrection: schema=session.nominalSchema() session.transaction().start(True) finecorrections=lumiCorrections.correctionsForRange(schema,runList) session.transaction().commit() #print 'runList ',runList #print 'runDict ', runDict fig=Figure(figsize=(7.2,5.4),dpi=120) m=matplotRender.matplotRender(fig) logfig=Figure(figsize=(7.5,5.7),dpi=135) mlog=matplotRender.matplotRender(logfig) if args.action == 'run': result={} result=getLumiInfoForRuns(session,c,runList,selectionDict,hltpath,beamstatus=beamstatus,beamenergy=beamenergy,beamfluctuation=beamfluctuation,finecorrections=finecorrections) xdata=[] ydata={} ydata['Delivered']=[] ydata['Recorded']=[] keylist=result.keys() keylist.sort() #must be sorted in order if args.outputfile: reporter=csvReporter.csvReporter(ofilename) fieldnames=['run','delivered','recorded'] reporter.writeRow(fieldnames) for run in keylist: xdata.append(run) delivered=result[run][0] recorded=result[run][1] ydata['Delivered'].append(delivered) ydata['Recorded'].append(recorded) if args.outputfile and (delivered!=0 or recorded!=0): reporter.writeRow([run,result[run][0],result[run][1]]) m.plotSumX_Run(xdata,ydata,yscale='linear') mlog.plotSumX_Run(xdata,ydata,yscale='log') elif args.action == 'fill': lumiDict={} lumiDict=getLumiInfoForRuns(session,c,runList,selectionDict,hltpath,beamstatus=beamstatus,beamenergy=beamenergy,beamfluctuation=beamfluctuation,finecorrections=finecorrections) xdata=[] ydata={} ydata['Delivered']=[] ydata['Recorded']=[] #keylist=lumiDict.keys() #keylist.sort() if args.outputfile: reporter=csvReporter.csvReporter(ofilename) fieldnames=['fill','run','delivered','recorded'] reporter.writeRow(fieldnames) fills=fillDict.keys() fills.sort() for fill in fills: runs=fillDict[fill] runs.sort() for run in runs: xdata.append(run) ydata['Delivered'].append(lumiDict[run][0]) ydata['Recorded'].append(lumiDict[run][1]) if args.outputfile : reporter.writeRow([fill,run,lumiDict[run][0],lumiDict[run][1]]) #print 'input fillDict ',len(fillDict.keys()),fillDict m.plotSumX_Fill(xdata,ydata,fillDict,yscale='linear') mlog.plotSumX_Fill(xdata,ydata,fillDict,yscale='log') elif args.action == 'time' : lumiDict={} lumiDict=getLumiInfoForRuns(session,c,runList,selectionDict,hltpath,beamstatus=beamstatus,beamenergy=beamenergy,beamfluctuation=beamfluctuation,finecorrections=finecorrections) #lumiDict=getLumiInfoForRuns(session,c,runList,selectionDict,hltpath,beamstatus='STABLE BEAMS') xdata={}#{run:[starttime,stoptime]} ydata={} ydata['Delivered']=[] ydata['Recorded']=[] keylist=lumiDict.keys() keylist.sort() if args.outputfile: reporter=csvReporter.csvReporter(ofilename) fieldnames=['run','starttime','stoptime','delivered','recorded'] reporter.writeRow(fieldnames) for run in keylist: ydata['Delivered'].append(lumiDict[run][0]) ydata['Recorded'].append(lumiDict[run][1]) starttime=runDict[run][0] stoptime=runDict[run][1] xdata[run]=[starttime,stoptime] if args.outputfile : reporter.writeRow([run,starttime,stoptime,lumiDict[run][0],lumiDict[run][1]]) m.plotSumX_Time(xdata,ydata,startRunTime,stopRunTime,hltpath=hltpath,annotateBoundaryRunnum=args.annotateboundary,yscale='linear') mlog.plotSumX_Time(xdata,ydata,startRunTime,stopRunTime,hltpath=hltpath,annotateBoundaryRunnum=args.annotateboundary,yscale='log') elif args.action == 'perday': daydict={}#{day:[[run,cmslsnum,lsstarttime,delivered,recorded]]} lumibyls=getLumiOrderByLS(session,c,runList,selectionDict,hltpath,beamstatus=beamstatus,beamenergy=beamenergy,beamfluctuation=beamfluctuation,finecorrections=finecorrections) #lumibyls [[runnumber,runstarttime,lsnum,lsstarttime,delivered,recorded,recordedinpath]] if args.outputfile: reporter=csvReporter.csvReporter(ofilename) fieldnames=['day','begrunls','endrunls','delivered','recorded'] reporter.writeRow(fieldnames) beginfo=[lumibyls[0][3],str(lumibyls[0][0])+':'+str(lumibyls[0][2])] endinfo=[lumibyls[-1][3],str(lumibyls[-1][0])+':'+str(lumibyls[-1][2])] for perlsdata in lumibyls: lsstarttime=perlsdata[3] delivered=perlsdata[4] recorded=perlsdata[5] day=lsstarttime.toordinal() if not daydict.has_key(day): daydict[day]=[] daydict[day].append([delivered,recorded]) days=daydict.keys() days.sort() daymin=days[0] daymax=days[-1] #alldays=range(daymin,daymax+1) resultbyday={} resultbyday['Delivered']=[] resultbyday['Recorded']=[] #for day in days: #print 'day min ',daymin #print 'day max ',daymax for day in range(daymin,daymax+1): if not daydict.has_key(day): delivered=0.0 recorded=0.0 else: daydata=daydict[day] mytransposed=CommonUtil.transposed(daydata,defaultval=0.0) delivered=sum(mytransposed[0]) recorded=sum(mytransposed[1]) resultbyday['Delivered'].append(delivered) resultbyday['Recorded'].append(recorded) if args.outputfile: reporter.writeRow([day,beginfo[1],endinfo[1],delivered,recorded]) #print 'beginfo ',beginfo #print 'endinfo ',endinfo #print resultbyday m.plotPerdayX_Time( range(daymin,daymax+1) ,resultbyday,startRunTime,stopRunTime,boundaryInfo=[beginfo,endinfo],annotateBoundaryRunnum=args.annotateboundary,yscale='linear') mlog.plotPerdayX_Time( range(daymin,daymax+1),resultbyday,startRunTime,stopRunTime,boundaryInfo=[beginfo,endinfo],annotateBoundaryRunnum=args.annotateboundary,yscale='log') else: raise Exception,'must specify the type of x-axi' del session del svc if args.batch and args.yscale=='linear': m.drawPNG(args.batch) elif args.batch and args.yscale=='log': mlog.drawPNG(args.batch) elif args.batch and args.yscale=='both': m.drawPNG(args.batch) basename,extension=os.path.splitext(args.batch) logfilename=basename+'_log'+extension mlog.drawPNG(logfilename) else: if not args.interactive: return if args.interactive is True and args.yscale=='linear': m.drawInteractive() elif args.interactive is True and args.yscale=='log': mlog.drawInteractive() else: raise Exception('unsupported yscale for interactive mode : '+args.yscale)
runlist=[] irunlsdict={} if options.runnumber: # if runnumber specified, do not go through other run selection criteria irunlsdict[options.runnumber]=None runlist=irunlsdict.keys() else: if options.inputfile: (irunlsdict,iresults)=parseInputFiles(options.inputfile) runlist=irunlsdict.keys() session.transaction().start(True) (hf_tagid,hf_tagname)=revisionDML.currentDataTag(session.nominalSchema(),lumitype='HF') hfdataidmap=revisionDML.dataIdsByTagId(session.nominalSchema(),hf_tagid,runlist,withcomment=False,lumitype='HF') lumirundata=dataDML.lumiRunByIds(session.nominalSchema(),hfdataidmap,lumitype='HF') #{runnum: (datasource(1),nominalegev(2),ncollidingbunches(3)} GrunsummaryData=lumiCalcAPI.runsummaryMap(session.nominalSchema(),irunlsdict) session.transaction().commit() for runnum in lumirundata.keys(): session.transaction().start(True) hfdataidinfo=hfdataidmap[runnum] hflumidata=lumiCalcAPI.instLumiForIds(session.nominalSchema(),irunlsdict,hfdataidmap,GrunsummaryData,beamstatusfilter=None,withBXInfo=True,bxAlgo='OCC1') for perlsdata in hflumidata[runnum]: lumilsnum=perlsdata[0] cmslsnum=perlsdata[1] bxinfo=perlsdata[9] bxdistro=getdistribution(cmslsnum,bxinfo[0],bxinfo[1]) print(lumilsnum,cmslsnum,CommonUtil.flatten(bxdistro)) session.transaction().commit() del session del svc
def fetchOldData(schema,oldlumidataid): ''' fetch old perbunch data if the run exists select CMSBXINDEXBLOB,BEAMINTENSITYBLOB_1,BEAMINTENSITYBLOB_2,BXLUMIVALUE_OCC1,BXLUMIVALUE_OCC2,BXLUMIVALUE_ET,BXLUMIERROR_OCC1,BXLUMIERROR_OCC2,BXLUMIERROR_ET,BXLUMIQUALITY_OCC1,BXLUMIQUALITY_OCC2,BXLUMIQUALITY_ET from lumisummaryv2 where data_id=:oldlumidataid; output: {lumilsnum:[bxvalueblob_occ1(0),bxerrblob_occ1(1),bxqualityblob_occ1(2),bxvalueblob_occ2(3),bxerrblob_occ2(4),bxqualityblob_occ2(5),bxvalueblob_et(6),bxerrblob_et(7),bxqualityblob_et(8),bxindexblob(9),beam1intensity(10),beam2intensity(11),beamstatus(12),beamenergy(13),instlumierror(14),instlumiquality(15)],...} ''' result={} qHandle=schema.newQuery() try: qHandle.addToTableList('LUMISUMMARYV2') qHandle.addToOutputList('LUMILSNUM','lumilsnum') qHandle.addToOutputList('BEAMSTATUS','beamstatus') qHandle.addToOutputList('BEAMENERGY','beamenergy') qHandle.addToOutputList('INSTLUMIERROR','instlumierror') qHandle.addToOutputList('INSTLUMIQUALITY','instlumiquality') qHandle.addToOutputList('BXLUMIVALUE_OCC1','bxvalue_occ1') qHandle.addToOutputList('BXLUMIERROR_OCC1','bxerror_occ1') qHandle.addToOutputList('BXLUMIQUALITY_OCC1','bxquality_occ1') qHandle.addToOutputList('BXLUMIVALUE_OCC2','bxvalue_occ2') qHandle.addToOutputList('BXLUMIERROR_OCC2','bxerror_occ2') qHandle.addToOutputList('BXLUMIQUALITY_OCC2','bxquality_occ2') qHandle.addToOutputList('BXLUMIVALUE_ET','bxvalue_et') qHandle.addToOutputList('BXLUMIERROR_ET','bxerror_et') qHandle.addToOutputList('BXLUMIQUALITY_ET','bxquality_et') qHandle.addToOutputList('CMSBXINDEXBLOB','bxindexblob') qHandle.addToOutputList('BEAMINTENSITYBLOB_1','beam1intensity') qHandle.addToOutputList('BEAMINTENSITYBLOB_2','beam2intensity') qConditionStr='DATA_ID=:dataid' qCondition=coral.AttributeList() qCondition.extend('dataid','unsigned long long') print('oldlumidataid ',oldlumidataid) qCondition['dataid'].setData(int(oldlumidataid)) qResult=coral.AttributeList() qResult.extend('lumilsnum','unsigned int') qResult.extend('beamstatus','string') qResult.extend('beamenergy','float') qResult.extend('instlumierror','float') qResult.extend('instlumiquality','short') qResult.extend('bxvalue_occ1','blob') qResult.extend('bxerror_occ1','blob') qResult.extend('bxquality_occ1','blob') qResult.extend('bxvalue_occ2','blob') qResult.extend('bxerror_occ2','blob') qResult.extend('bxquality_occ2','blob') qResult.extend('bxvalue_et','blob') qResult.extend('bxerror_et','blob') qResult.extend('bxquality_et','blob') qResult.extend('bxindexblob','blob') qResult.extend('beam1intensity','blob') qResult.extend('beam2intensity','blob') qHandle.defineOutput(qResult) qHandle.setCondition(qConditionStr,qCondition) cursor=qHandle.execute() while next(cursor): lumilsnum=cursor.currentRow()['lumilsnum'].data() beamstatus=cursor.currentRow()['beamstatus'].data() beamenergy=cursor.currentRow()['beamenergy'].data() instlumierror=cursor.currentRow()['instlumierror'].data() instlumiquality=cursor.currentRow()['instlumiquality'].data() bxvalueblob_occ1=cursor.currentRow()['bxvalue_occ1'].data() bxvalueblob_occ1Array=CommonUtil.unpackBlobtoArray(bxvalueblob_occ1,'f') bxerrblob_occ1=cursor.currentRow()['bxerror_occ1'].data() bxqualityblob_occ1=cursor.currentRow()['bxquality_occ1'].data() bxvalueblob_occ2=cursor.currentRow()['bxvalue_occ2'].data() bxerrblob_occ2=cursor.currentRow()['bxerror_occ2'].data() bxqualityblob_occ2=cursor.currentRow()['bxquality_occ2'].data() bxvalueblob_et=cursor.currentRow()['bxvalue_et'].data() bxerrblob_et=cursor.currentRow()['bxerror_et'].data() bxqualityblob_et=cursor.currentRow()['bxquality_et'].data() bxindexblob=cursor.currentRow()['bxindexblob'].data() beam1intensity=cursor.currentRow()['beam1intensity'].data() beam2intensity=cursor.currentRow()['beam2intensity'].data() result[lumilsnum]=[bxvalueblob_occ1,bxerrblob_occ1,bxqualityblob_occ1,bxvalueblob_occ2,bxerrblob_occ2,bxqualityblob_occ2,bxvalueblob_et,bxerrblob_et,bxqualityblob_et,bxindexblob,beam1intensity,beam2intensity,beamstatus,beamenergy,instlumierror,instlumiquality] except : del qHandle raise del qHandle return result
def lumidb(run_lumi_dict, action='delivered', lumi_report=False): "Call lumidb to get luminosity numbers" actions = ['overview', 'delivered', 'recorded', 'lumibyls', 'lumibylsXing'] if action not in actions: raise Exception('Unsupported action="%s", please check from %s' % (action, actions)) beamModeChoices = ['stable'] amodetagChoices = ['PROTPHYS', 'IONPHYS', 'PAPHYS' ] xingAlgoChoices = ['OCC1', 'OCC2', 'ET'] # # parse arguments # connect='frontier://LumiCalc/CMS_LUMI_PROD' authpath = None scalefactor = 1.0 beamfluctuation = 0.2 minbiasxsec = 69300.0 # minbias cross-secvtion in ub xingMinLum = 1e-03 # Minimum perbunch luminosity to print, default=1e-03/ub xingAlgo = 'OCC1' hltpath = None outputfile = None siteconfpath = None withoutNorm = False withoutCheckforupdate = False fillnum = None nowarning = True debug = False beamenergy = None amodetag = None reqrunmin=None reqfillmin=None reqtimemin=None reqrunmax=None reqfillmax=None reqtimemax=None timeFilter=[None,None] pbeammode = None iresults=[] reqTrg=False reqHlt=False svc = sessionManager.sessionManager(connect, authpath, siteconfpath, debugON=debug) session=svc.openSession(isReadOnly=True, cpp2sqltype=[('unsigned int', 'NUMBER(10)'), ('unsigned long long', 'NUMBER(20)')]) ############################################################## # check run/ls list ############################################################## irunlsdict={} rruns=[] session.transaction().start(True) irunlsdict = run_lumi_dict iresults = [] runlist=lumiCalcAPI.runList(session.nominalSchema(), fillnum, runmin=reqrunmin, runmax=reqrunmax, fillmin=reqfillmin, fillmax=reqfillmax, startT=reqtimemin, stopT=reqtimemax, l1keyPattern=None, hltkeyPattern=None, amodetag=amodetag, nominalEnergy=beamenergy, energyFlut=beamfluctuation, requiretrg=False, requirehlt=False) rruns = [val for val in runlist if val in irunlsdict.keys()] for selectedrun in irunlsdict.keys():#if there's further filter on the runlist,clean input dict if selectedrun not in rruns: del irunlsdict[selectedrun] ############################################################## # check datatag # ############################################################# datatagid, datatagname = revisionDML.currentDataTag(session.nominalSchema()) dataidmap=revisionDML.dataIdsByTagId(\ session.nominalSchema(), datatagid, runlist=rruns, withcomment=False) session.transaction().commit() if not dataidmap: print_warning('No data found in LumiDB for given set run lumi section') return 0, '/ub' # return lumi, units normvalueDict={} ################## # ls level # ################## session.transaction().start(True) GrunsummaryData=lumiCalcAPI.runsummaryMap(session.nominalSchema(),irunlsdict) if action == 'delivered': result=lumiCalcAPI.deliveredLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,lumitype='HF') if lumi_report: lumiReport.toScreenTotDelivered(result,iresults,scalefactor,irunlsdict=irunlsdict,noWarning=nowarning,toFile=outputfile) # result {run:[lumilsnum(0),cmslsnum(1),timestamp(2),beamstatus(3),beamenergy(4),deliveredlumi(5),calibratedlumierr(6),(bxidxlist,bxvalues,bxerrs)(7),(bxidx,b1intensities,b2intensities)(8),fillnum(9),pu(10)]} totlumi = 0 for run, val in result.items(): for llist in val: if irunlsdict[run]: if llist[0] in irunlsdict[run]: # select only lumis from input run ls dict totlumi += llist[5] else: totlumi += llist[5] totlumival, lumiunit = CommonUtil.guessUnit(totlumi) return totlumival, lumiunit if action == 'overview': result=lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,lumitype='HF') lumiReport.toScreenOverview(result,iresults,scalefactor,irunlsdict=irunlsdict,noWarning=nowarning,toFile=outputfile) if action == 'lumibyls': if not hltpath: result=lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,lumitype='HF',minbiasXsec=minbiasxsec) lumiReport.toScreenLumiByLS(result,iresults,scalefactor,irunlsdict=irunlsdict,noWarning=nowarning,toFile=outputfile) else: hltname=hltpath hltpat=None if hltname=='*' or hltname=='all': hltname=None elif 1 in [c in hltname for c in '*?[]']: #is a fnmatch pattern hltpat=hltname hltname=None result=lumiCalcAPI.effectiveLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,hltpathname=hltname,hltpathpattern=hltpat,withBXInfo=False,bxAlgo=None,xingMinLum=xingMinLum,withBeamIntensity=False,lumitype='HF') lumiReport.toScreenLSEffective(result,iresults,scalefactor,irunlsdict=irunlsdict,noWarning=nowarning,toFile=outputfile) if action == 'recorded':#recorded actually means effective because it needs to show all the hltpaths... hltname=hltpath hltpat=None if hltname is not None: if hltname=='*' or hltname=='all': hltname=None elif 1 in [c in hltname for c in '*?[]']: #is a fnmatch pattern hltpat=hltname hltname=None result=lumiCalcAPI.effectiveLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,hltpathname=hltname,hltpathpattern=hltpat,withBXInfo=False,bxAlgo=None,xingMinLum=xingMinLum,withBeamIntensity=False,lumitype='HF') lumiReport.toScreenTotEffective(result,iresults,scalefactor,irunlsdict=irunlsdict,noWarning=nowarning,toFile=outputfile) if action == 'lumibylsXing': result=lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,withBXInfo=True,bxAlgo=xingAlgo,xingMinLum=xingMinLum,withBeamIntensity=False,lumitype='HF') outfile=outputfile if not outfile: print '[WARNING] no output file given. lumibylsXing writes per-bunch lumi only to default file lumibylsXing.csv' outfile='lumibylsXing.csv' lumiReport.toCSVLumiByLSXing(result,scalefactor,outfile,irunlsdict=irunlsdict,noWarning=nowarning) session.transaction().commit() del session del svc
lumils = sorted(lumiLSdata.keys()) for lsnum in lumils: print('lumilsnum,cmslsnum,instlumi ', lsnum, lumiLSdata[lsnum][0], lumiLSdata[lsnum][1] * normval) (runnum, trgLSdata) = dataDML.trgLSById(schema, trgdataid) cmsls = trgLSdata.keys() cmsls.sort() for lsnum in cmsls: print('cmslsnum,deadtime,bizerocount,bitzeroprescale,deadfrac ', lsnum, trgLSdata[lsnum][0], trgLSdata[lsnum][1], trgLSdata[lsnum][2], trgLSdata[lsnum][3]) [runnum, datasource, npath, pathnames] = dataDML.hltRunById(schema, hltdataid) print('npath,pathnames ', npath, pathnames) pathnameList = pathnames.split(',') (runnum, hltLSdata) = dataDML.hltLSById(schema, hltdataid) cmsls = hltLSdata.keys() cmsls.sort() for lsnum in cmsls: prescaleblob = hltLSdata[lsnum][0] print('lsnum ', lsnum) if prescaleblob: hltprescales = CommonUtil.unpackBlobtoArray(prescaleblob, 'h') for pidx, pathname in enumerate(pathnameList): print('pathname, hltprescales ', pathname, hltprescales[pidx]) session.transaction().commit() del session print('')
def patchV2(dbsession,runnum,inputpathnames,inputdata): ''' inputpathnames: [pathname,] inputdata: {cmslsnum:[presc,presc...]} update the most recent version of lshlt data if some ls exist if none old ls exists insert new hlt records ''' try: dbsession.transaction().start(True) oldhltdataid=dataDML.guessHltDataIdByRun(dbsession.nominalSchema(),runnum) existingrundata=dataDML.hltRunById(dbsession.nominalSchema(),oldhltdataid) dbsession.transaction().commit() if not oldhltdataid:#no data at all dbsession.transaction().start(False) insertV2(dbsession,runnum,inputpathnames,inputdata) dbsession.transaction().commit() return hltnamedict=existingrundata[3]#[(pathidx,hltname),(pathidx,hltname)...] dbsession.transaction().start(True) existinglsdata=dataDML.hltLSById(dbsession.nominalSchema(),oldhltdataid) dbsession.transaction().commit() oldlsdata=existinglsdata[1] existinglslist=oldlsdata.keys() toupdate={}#{cmslsnum:[presc,presc...]} toinsert={}#{cmslsnum:[presc,presc...]} if existinglslist and len(existinglslist)!=0:#there are some existing data for cmslsnum,oldlscontent in oldlsdata.items(): if cmslsnum in inputdata.keys(): # if overlap with new data, update old data with new toupdate[cmslsnum]=inputdata[cmslsnum] for cmslsnum,lshltcontent in inputdata.items(): if cmslsnum in toupdate: continue #it's to update not to insert toinsert[cmslsnum]=inputdata[cmslsnum] # # insert into lshlt(data_id,runnum,cmslsnum,prescaleblob,hltcountblob,hltacceptblob) values() # dbsession.transaction().start(False) tabrowDefDict={'DATA_ID':'unsigned long long','RUNNUM':'unsigned int','CMSLSNUM':'unsigned int','PRESCALEBLOB':'blob','HLTCOUNTBLOB':'blob','HLTACCEPTBLOB':'blob'} for cmslsnum,perlsdata in toinsert.items(): prescaleArray=array.array('I') hltcountArray=array.array('I') hltacceptArray=array.array('I') for (pathidx,hltname) in hltnamedict: thispathIdx=inputpathnames.index(hltname) thispresc=perlsdata[thispathIdx] thiscount=0 thisaccept=0 prescaleArray.append(thispresc) hltcountArray.append(thiscount) hltacceptArray.append(thisaccept) prescaleblob=CommonUtil.packArraytoBlob(prescaleArray) hltcountblob=CommonUtil.packArraytoBlob(hltcountArray) hltacceptblob=CommonUtil.packArraytoBlob(hltacceptArray) tabrowValueDict={'DATA_ID':oldhltdataid,'RUNNUM':int(runnum),'CMSLSNUM':int(cmslsnum),'PRESCALEBLOB':prescaleblob,'HLTCOUNTBLOB':hltcountblob,'HLTACCEPTBLOB':hltacceptblob} db=dbUtil.dbUtil(dbsession.nominalSchema()) db.insertOneRow(nameDealer.lshltTableName(),tabrowDefDict,tabrowValueDict) # # update lshlt set prescaleblob=:prescaleblob,hltcoutblob=:hltcountblob,hltacceptblob=:hltacceptblob where data_id=:olddata_id and cmslsnum=:cmslsnum; # setClause='PRESCALEBLOB=:prescaleblob,HLTCOUNTBLOB=:hltcountblob,HLTACCEPTBLOB=:hltacceptblob' updateCondition='DATA_ID=:oldhltdataid and CMSLSNUM=:cmslsnum' for cmslsnum,perlsdata in toupdate.items(): prescaleArray=array.array('I') hltcountArray=array.array('I') hltacceptArray=array.array('I') for (pathidx,hltname) in hltnamedict: thispathIdx=inputpathnames.index(hltname) thispresc=perlsdata[thispathIdx] thiscount=0 thisaccept=0 prescaleArray.append(thispresc) hltcountArray.append(thiscount) hltacceptArray.append(thisaccept) prescaleblob=CommonUtil.packArraytoBlob(prescaleArray) hltcountblob=CommonUtil.packArraytoBlob(hltcountArray) hltacceptblob=CommonUtil.packArraytoBlob(hltacceptArray) iData=coral.AttributeList() iData.extend('prescaleblob','blob') iData.extend('hltcountblob','blob') iData.extend('hltacceptblob','blob') iData.extend('olddata_id','unsigned int') iData.extend('cmslsnum','unsigned int') iData['prescaleblob'].setData(prescaleblob) iData['hltcountblob'].setData(hltcountblob) iData['hltacceptblob'].setData(hltacceptblob) iData['olddata_id'].setData(int(olddata_id)) iData['cmslsnum'].setData(int(cmslsnum)) db=dbUtil.dbUtil(schema) db.singleUpdate(nameDealer.lshltTableName(),setClause,updateCondition,iData) dbsession.transaction().commit() #dbsession.transaction().rollback() except : raise
(lumidataid,trgdataid,hltdataid)=dataDML.guessDataIdByRun(schema,myrun) print 'all dataids ',lumidataid,trgdataid,hltdataid (runnum,lumiLSdata)=dataDML.lumiLSById(schema,lumidataid) lumils=lumiLSdata.keys() lumils.sort() for lsnum in lumils: print 'lumilsnum,cmslsnum,instlumi ',lsnum,lumiLSdata[lsnum][0],lumiLSdata[lsnum][1]*normval (runnum,trgLSdata)=dataDML.trgLSById(schema,trgdataid) cmsls=trgLSdata.keys() cmsls.sort() for lsnum in cmsls: print 'cmslsnum,deadtime,bizerocount,bitzeroprescale,deadfrac ',lsnum,trgLSdata[lsnum][0],trgLSdata[lsnum][1],trgLSdata[lsnum][2],trgLSdata[lsnum][3] [runnum,datasource,npath,pathnames]=dataDML.hltRunById(schema,hltdataid) print 'npath,pathnames ',npath,pathnames pathnameList=pathnames.split(',') (runnum,hltLSdata)=dataDML.hltLSById(schema,hltdataid) cmsls=hltLSdata.keys() cmsls.sort() for lsnum in cmsls: prescaleblob=hltLSdata[lsnum][0] print 'lsnum ',lsnum if prescaleblob: hltprescales=CommonUtil.unpackBlobtoArray(prescaleblob,'h') for pidx,pathname in enumerate(pathnameList): print 'pathname, hltprescales ',pathname,hltprescales[pidx] session.transaction().commit() del session print ''
def generateLumidata(lumirundatafromfile,lsdatafromfile,rundatafromdb,lsdatafromdb,replacelsMin,replacelsMax): ''' input: perrunresultfromfile=[]#source,starttime,stoptime,nls perlsresultfromfile={} #{lumilsnum:instlumiub} lumirundatafromdb=[] #[source,nominalegev,ncollidingbunches,starttime,stoptime,nls] lumilsdatafromdb={}#{lumilsnum:[cmslsnum(0),instlumi(1),instlumierror(2),instlumiquality(3),beamstatus(4),beamenergy(5),numorbit(6),startorbit(7),cmsbxindexblob(8),beamintensityblob_1(9),beamintensityblob_2(10),bxlumivalue_occ1(11),bxlumierror_occ1(12),bxlumiquality_occ1(13),bxlumivalue_occ2(14),bxlumierror_occ2(15),bxlumiquality_occ2(16),bxlumivalue_et(17),bxlumierror_et(18),bxlumiquality_et(19)]} ''' lumip=lumiParameters.ParametersObject() numorbit=lumip.numorbit startorbit=0 fakebeamenergy=4000. fakebeamstatus='STABLE BEAMS' fakefloatArray=array.array('f') fakeidxArray=array.array('h') fakeshortArray=array.array('h') for bxidx in range(1,3565): fakeidxArray.append(bxidx) fakefloatArray.append(0.) fakeshortArray.append(0) lumirundata=[] lumilsdata={} if rundatafromdb: lumirundata=rundatafromdb lumirundata[0]=rundatafromdb[0]+'+file:'+lumirundatafromfile[0] else: lu=lumiTime.lumiTime() source='+file:'+lumirundatafromfile[0] nominalegev=fakebeamenergy ncollidingbunches=72 starttime=lumirundatafromfile[1] stoptime=lumirundatafromfile[2] starttimeT=lu.timestampTodatetimeUTC(starttime) stoptimeT=lu.timestampTodatetimeUTC(stoptime) print(starttimeT.day,starttimeT.month,starttimeT.year) starttimeT_coral=coral.TimeStamp(starttimeT.year,starttimeT.month,starttimeT.day,starttimeT.hour,starttimeT.minute,starttimeT.second,0) stoptimeT_coral=coral.TimeStamp(stoptimeT.year,stoptimeT.month,stoptimeT.day,stoptimeT.hour,stoptimeT.minute,stoptimeT.second,0) nls=lumirundatafromfile[3] lumirundata=[source,nominalegev,ncollidingbunches,starttimeT_coral,stoptimeT_coral,nls] if lsdatafromdb: lumilsdata=lsdatafromdb if replacelsMin>len(lsdatafromdb): print('[INFO]Operation: extend an existing run from LS=',replacelsMin) lumirundata[5]+=len(lsdatafromfile) else: print('[INFO]Operation: replace instlumi in an existing run LS range=',replacelsMin,replacelsMax) else: print('[INFO]Operation: insert a new fake run') for lumilsnum in range(replacelsMin,replacelsMax+1): instlumi=lsdatafromfile[lumilsnum] if lumilsnum in lsdatafromdb.keys(): #if this is a hole lumilsdata[lumilsnum][1]=instlumi else: #if this is an extension instlumierror=0.0 instlumiquality=0 startorbit=(lumilsnum-1)*numorbit cmsbxindexblob=CommonUtil.packArraytoBlob(fakeshortArray) beamintensityblob_1=CommonUtil.packArraytoBlob(fakefloatArray) beamintensityblob_2=CommonUtil.packArraytoBlob(fakefloatArray) bxlumivalue_occ1=CommonUtil.packArraytoBlob(fakefloatArray) bxlumierror_occ1=CommonUtil.packArraytoBlob(fakefloatArray) bxlumiquality_occ1=CommonUtil.packArraytoBlob(fakeshortArray) bxlumivalue_occ2=CommonUtil.packArraytoBlob(fakefloatArray) bxlumierror_occ2=CommonUtil.packArraytoBlob(fakefloatArray) bxlumiquality_occ2=CommonUtil.packArraytoBlob(fakeshortArray) bxlumivalue_et=CommonUtil.packArraytoBlob(fakefloatArray) bxlumierror_et=CommonUtil.packArraytoBlob(fakefloatArray) bxlumiquality_et=CommonUtil.packArraytoBlob(fakeshortArray) lumilsdata[lumilsnum]=[0,instlumi,instlumierror,instlumiquality,fakebeamstatus,fakebeamenergy,numorbit,startorbit,cmsbxindexblob,beamintensityblob_1,beamintensityblob_2,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et] return (lumirundata,lumilsdata)
def specificlumiTofile(fillnum, filldata, outdir): # #input : fillnum # filldata: {bxidx:[[lstime,beamstatusfrac,lumivalue,lumierror,speclumi,speclumierr]],[]} #sorted by bxidx, sorted by lstime inside list #check outdir/fillnum subdir exists; if not, create it; else outdir=outdir/fillnum # if not filldata: print 'empty input data, do nothing for fill ', fillnum return timedict = { } #{lstime:[[stablebeamfrac,lumi,lumierr,speclumi,speclumierr]]} filloutdir = os.path.join(outdir, str(fillnum)) if not os.path.exists(filloutdir): os.mkdir(filloutdir) for cmsbxidx, perbxdata in filldata.items(): lhcbucket = 0 if cmsbxidx != 0: lhcbucket = (cmsbxidx - 1) * 10 + 1 a = sorted(perbxdata, key=lambda x: x[0]) filename = str(fillnum) + '_lumi_' + str(lhcbucket) + '_CMS.txt' linedata = [] for perlsdata in a: ts = int(perlsdata[0]) beamstatusfrac = perlsdata[1] lumi = perlsdata[2] lumierror = perlsdata[3] #beam1intensity=perlsdata[4] #beam2intensity=perlsdata[5] speclumi = perlsdata[4] speclumierror = perlsdata[5] if lumi > 0: linedata.append([ ts, beamstatusfrac, lumi, lumierror, speclumi, speclumierror ]) if not timedict.has_key(ts): timedict[ts] = [] timedict[ts].append( [beamstatusfrac, lumi, lumierror, speclumi, speclumierror]) if len(linedata) > 10: #at least 10 good ls f = open(os.path.join(filloutdir, filename), 'w') for line in linedata: print >> f, '%d\t%e\t%e\t%e\t%e\t%e' % ( line[0], line[1], line[2], line[3], line[4], line[5]) f.close() #print 'writing avg file' summaryfilename = str(fillnum) + '_lumi_CMS.txt' f = None lstimes = timedict.keys() lstimes.sort() fillseg = [] lscounter = 0 for lstime in lstimes: allvalues = timedict[lstime] transposedvalues = CommonUtil.transposed(allvalues, 0.0) bstatfrac = transposedvalues[0][ 0] #beamstatus does not change with bx position lumivals = transposedvalues[1] lumitot = sum(lumivals) if bstatfrac == 1.0: fillseg.append([lstime, lumitot]) lumierrs = transposedvalues[2] lumierrortot = math.sqrt(sum(map(lambda x: x**2, lumierrs))) specificvals = transposedvalues[3] specificavg = sum(specificvals) / float( len(specificvals)) #avg spec lumi specificerrs = transposedvalues[4] specifictoterr = math.sqrt(sum(map(lambda x: x**2, specificerrs))) specificerravg = specifictoterr / float(len(specificvals)) if lscounter == 0: f = open(os.path.join(filloutdir, summaryfilename), 'w') lscounter += 1 print >> f, '%d\t%e\t%e\t%e\t%e\t%e' % (lstime, bstatfrac, lumitot, lumierrortot, specificavg, specificerravg) if f is not None: f.close() #print 'writing summary file' fillsummaryfilename = str(fillnum) + '_bxsum_CMS.txt' f = open(os.path.join(filloutdir, fillsummaryfilename), 'w') if len(fillseg) == 0: print >> f, '%s' % ('#no stable beams') f.close() return previoustime = fillseg[0][0] boundarytime = fillseg[0][0] #print 'boundary time ',boundarytime summaryls = {} summaryls[boundarytime] = [] for [lstime, lumitot] in fillseg: #fillseg is everything with stable beam flag if lstime - previoustime > 50.0: boundarytime = lstime #print 'found new boundary ',boundarytime summaryls[boundarytime] = [] # print 'appending ',boundarytime,lstime,lumitot summaryls[boundarytime].append([lstime, lumitot]) previoustime = lstime #print summaryls summarylstimes = summaryls.keys() summarylstimes.sort() lumip = lumiParameters.ParametersObject() for bts in summarylstimes: startts = bts tsdatainseg = summaryls[bts] #print 'tsdatainseg ',tsdatainseg stopts = tsdatainseg[-1][0] plu = max(CommonUtil.transposed(tsdatainseg, 0.0)[1]) lui = sum(CommonUtil.transposed(tsdatainseg, 0.0)[1]) * lumip.lslengthsec() print >> f, '%d\t%d\t%e\t%e' % (startts, stopts, plu, lui) f.close()
reqTrg=False reqHlt=False if options.beammode=='stable': pbeammode = 'STABLE BEAMS' if options.action=='trgbyls': reqTrg=True if options.action=='hltbyls': reqHlt=True if options.runnumber: reqrunmax=options.runnumber reqrunmin=options.runnumber if options.fillnum: reqfillmin=options.fillnum reqfillmax=options.fillnum if options.begin: (reqrunmin,reqfillmin,reqtimemin)=CommonUtil.parseTime(options.begin) if reqtimemin: lute=lumiTime.lumiTime() reqtimeminT=lute.StrToDatetime(reqtimemin,customfm='%m/%d/%y %H:%M:%S') timeFilter[0]=reqtimeminT if options.end: (reqrunmax,reqfillmax,reqtimemax)=CommonUtil.parseTime(options.end) if reqtimemax: lute=lumiTime.lumiTime() reqtimemaxT=lute.StrToDatetime(reqtimemax,customfm='%m/%d/%y %H:%M:%S') timeFilter[1]=reqtimemaxT sname=options.name isdetail=False spattern=None if sname is not None: isdetail=True
def lumidb(run_lumi_dict, action='delivered', lumi_report=False): "Call lumidb to get luminosity numbers" actions = ['overview', 'delivered', 'recorded', 'lumibyls', 'lumibylsXing'] if action not in actions: raise Exception('Unsupported action="%s", please check from %s' % (action, actions)) beamModeChoices = ['stable'] amodetagChoices = ['PROTPHYS', 'IONPHYS', 'PAPHYS'] xingAlgoChoices = ['OCC1', 'OCC2', 'ET'] # # parse arguments # connect = 'frontier://LumiCalc/CMS_LUMI_PROD' authpath = None scalefactor = 1.0 beamfluctuation = 0.2 minbiasxsec = 69300.0 # minbias cross-secvtion in ub xingMinLum = 1e-03 # Minimum perbunch luminosity to print, default=1e-03/ub xingAlgo = 'OCC1' hltpath = None outputfile = None siteconfpath = None withoutNorm = False withoutCheckforupdate = False fillnum = None nowarning = True debug = False beamenergy = None amodetag = None reqrunmin = None reqfillmin = None reqtimemin = None reqrunmax = None reqfillmax = None reqtimemax = None timeFilter = [None, None] pbeammode = None iresults = [] reqTrg = False reqHlt = False svc = sessionManager.sessionManager(connect, authpath, siteconfpath, debugON=debug) session = svc.openSession(isReadOnly=True, cpp2sqltype=[('unsigned int', 'NUMBER(10)'), ('unsigned long long', 'NUMBER(20)') ]) ############################################################## # check run/ls list ############################################################## irunlsdict = {} rruns = [] session.transaction().start(True) irunlsdict = run_lumi_dict iresults = [] runlist = lumiCalcAPI.runList(session.nominalSchema(), fillnum, runmin=reqrunmin, runmax=reqrunmax, fillmin=reqfillmin, fillmax=reqfillmax, startT=reqtimemin, stopT=reqtimemax, l1keyPattern=None, hltkeyPattern=None, amodetag=amodetag, nominalEnergy=beamenergy, energyFlut=beamfluctuation, requiretrg=False, requirehlt=False) rruns = [val for val in runlist if val in irunlsdict.keys()] for selectedrun in irunlsdict.keys( ): #if there's further filter on the runlist,clean input dict if selectedrun not in rruns: del irunlsdict[selectedrun] ############################################################## # check datatag # ############################################################# datatagid, datatagname = revisionDML.currentDataTag( session.nominalSchema()) dataidmap=revisionDML.dataIdsByTagId(\ session.nominalSchema(), datatagid, runlist=rruns, withcomment=False) session.transaction().commit() if not dataidmap: print_warning('No data found in LumiDB for given set run lumi section') return 0, '/ub' # return lumi, units normvalueDict = {} ################## # ls level # ################## session.transaction().start(True) GrunsummaryData = lumiCalcAPI.runsummaryMap(session.nominalSchema(), irunlsdict) if action == 'delivered': result = lumiCalcAPI.deliveredLumiForIds(session.nominalSchema(), irunlsdict, dataidmap, runsummaryMap=GrunsummaryData, beamstatusfilter=pbeammode, timeFilter=timeFilter, normmap=normvalueDict, lumitype='HF') if lumi_report: lumiReport.toScreenTotDelivered(result, iresults, scalefactor, irunlsdict=irunlsdict, noWarning=nowarning, toFile=outputfile) # result {run:[lumilsnum(0),cmslsnum(1),timestamp(2),beamstatus(3),beamenergy(4),deliveredlumi(5),calibratedlumierr(6),(bxidxlist,bxvalues,bxerrs)(7),(bxidx,b1intensities,b2intensities)(8),fillnum(9),pu(10)]} totlumi = 0 for run, val in result.items(): for llist in val: if irunlsdict[run]: if llist[0] in irunlsdict[ run]: # select only lumis from input run ls dict totlumi += llist[5] else: totlumi += llist[5] totlumival, lumiunit = CommonUtil.guessUnit(totlumi) return totlumival, lumiunit if action == 'overview': result = lumiCalcAPI.lumiForIds(session.nominalSchema(), irunlsdict, dataidmap, runsummaryMap=GrunsummaryData, beamstatusfilter=pbeammode, timeFilter=timeFilter, normmap=normvalueDict, lumitype='HF') lumiReport.toScreenOverview(result, iresults, scalefactor, irunlsdict=irunlsdict, noWarning=nowarning, toFile=outputfile) if action == 'lumibyls': if not hltpath: result = lumiCalcAPI.lumiForIds(session.nominalSchema(), irunlsdict, dataidmap, runsummaryMap=GrunsummaryData, beamstatusfilter=pbeammode, timeFilter=timeFilter, normmap=normvalueDict, lumitype='HF', minbiasXsec=minbiasxsec) lumiReport.toScreenLumiByLS(result, iresults, scalefactor, irunlsdict=irunlsdict, noWarning=nowarning, toFile=outputfile) else: hltname = hltpath hltpat = None if hltname == '*' or hltname == 'all': hltname = None elif 1 in [c in hltname for c in '*?[]']: #is a fnmatch pattern hltpat = hltname hltname = None result = lumiCalcAPI.effectiveLumiForIds( session.nominalSchema(), irunlsdict, dataidmap, runsummaryMap=GrunsummaryData, beamstatusfilter=pbeammode, timeFilter=timeFilter, normmap=normvalueDict, hltpathname=hltname, hltpathpattern=hltpat, withBXInfo=False, bxAlgo=None, xingMinLum=xingMinLum, withBeamIntensity=False, lumitype='HF') lumiReport.toScreenLSEffective(result, iresults, scalefactor, irunlsdict=irunlsdict, noWarning=nowarning, toFile=outputfile) if action == 'recorded': #recorded actually means effective because it needs to show all the hltpaths... hltname = hltpath hltpat = None if hltname is not None: if hltname == '*' or hltname == 'all': hltname = None elif 1 in [c in hltname for c in '*?[]']: #is a fnmatch pattern hltpat = hltname hltname = None result = lumiCalcAPI.effectiveLumiForIds(session.nominalSchema(), irunlsdict, dataidmap, runsummaryMap=GrunsummaryData, beamstatusfilter=pbeammode, timeFilter=timeFilter, normmap=normvalueDict, hltpathname=hltname, hltpathpattern=hltpat, withBXInfo=False, bxAlgo=None, xingMinLum=xingMinLum, withBeamIntensity=False, lumitype='HF') lumiReport.toScreenTotEffective(result, iresults, scalefactor, irunlsdict=irunlsdict, noWarning=nowarning, toFile=outputfile) if action == 'lumibylsXing': result = lumiCalcAPI.lumiForIds(session.nominalSchema(), irunlsdict, dataidmap, runsummaryMap=GrunsummaryData, beamstatusfilter=pbeammode, timeFilter=timeFilter, normmap=normvalueDict, withBXInfo=True, bxAlgo=xingAlgo, xingMinLum=xingMinLum, withBeamIntensity=False, lumitype='HF') outfile = outputfile if not outfile: print '[WARNING] no output file given. lumibylsXing writes per-bunch lumi only to default file lumibylsXing.csv' outfile = 'lumibylsXing.csv' lumiReport.toCSVLumiByLSXing(result, scalefactor, outfile, irunlsdict=irunlsdict, noWarning=nowarning) session.transaction().commit() del session del svc
else: normdata_id=normDML.normIdByName(dbsession.nominalSchema(),normname) for normvalueDict in normvalues: if 'corrector' not in normvalueDict or not normvalueDict['corrector']: raise RuntimeError('parameter corrector is required for create/insert action') if 'since' not in normvalueDict or not normvalueDict['since']: raise RuntimeError('parameter since is required for create/insert action') correctorStr=normvalueDict['corrector'] sincerun=int(normvalueDict['since']) if options.firstsince: if sincerun<int(options.firstsince): continue amodetag=normvalueDict['amodetag'] egev=int(normvalueDict['egev']) detailcomment=normvalueDict['comment'] (correctorname,parameterlist)=CommonUtil.parselumicorrector(correctorStr) parameterDict={} for param in parameterlist: parameterDict[param]=normvalueDict[param] normDML.insertValueToNormId(dbsession.nominalSchema(),normdata_id,sincerun,correctorStr,amodetag,egev,parameterDict,comment=detailcomment) dbsession.transaction().commit() ############################## # setdefault/unsetdefault ############################## if options.action in ['setdefault','unsetdefault']: dbsession=svc.openSession(isReadOnly=False,cpp2sqltype=[('unsigned int','NUMBER(10)'),('unsigned long long','NUMBER(20)')]) dbsession.transaction().start(False) if options.action=='setdefault': normDML.promoteNormToTypeDefault(dbsession.nominalSchema(),options.normname,options.lumitype) if options.action=='unsetdefault':
def insertLumischemaV2(dbsession,runnum,datasource,perlsrawdata,perbunchrawdata): ''' input: lumirundata[datasource] perlsrawdata: {cmslsnum:instlumi} perbunchrawdata: {bxidx:lumifraction} lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]} ''' branchrevision_id=3#databranch_id branchinfo=(branchrevision_id,'DATA') lumirundata=[datasource] lumilsdata={} for cmslsnum,instlumi in perlsrawdata.items(): mystartorbit=startorbit+numorbit*(cmslsnum-1) bxdataArray=array.array('f') bxerrorArray=array.array('f') bxqualityArray=array.array('h') cmsbxindexArray=array.array('h') beam1intensityArray=array.array('f') beam2intensityArray=array.array('f') for bxidx in range(1,3565): lumifraction=0.0 if bxidx in perbunchrawdata: lumifraction=perbunchrawdata[bxidx] bxlumivalue=float(instlumi*lumifraction)/float(bunchnorm) bxdataArray.append(bxlumivalue) beam1intensityArray.append(9124580336.0) beam1intensityArray.append(8932813306.0) cmsbxindexArray.append(bxidx) bxqualityArray.append(1) bxerrorArray.append(0.0) bxdataocc1blob=CommonUtil.packArraytoBlob(bxdataArray) bxdataocc2blob=CommonUtil.packArraytoBlob(bxdataArray) bxdataetblob=CommonUtil.packArraytoBlob(bxdataArray) bxerrorocc1blob=CommonUtil.packArraytoBlob(bxerrorArray) bxerrorocc2blob=CommonUtil.packArraytoBlob(bxerrorArray) bxerroretblob=CommonUtil.packArraytoBlob(bxerrorArray) bxqualityocc1blob=CommonUtil.packArraytoBlob(bxqualityArray) bxqualityocc2blob=CommonUtil.packArraytoBlob(bxqualityArray) bxqualityetblob=CommonUtil.packArraytoBlob(bxqualityArray) cmsbxindexblob=CommonUtil.packArraytoBlob(cmsbxindexArray) beam1intensityblob=CommonUtil.packArraytoBlob(beam1intensityArray) beam2intensityblob=CommonUtil.packArraytoBlob(beam2intensityArray) perlsdata=[cmslsnum,float(instlumi)/float(6370),0.0,1,'STABLE BEAMS',beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob] lumilsdata[cmslsnum]=perlsdata print(lumilsdata) dbsession.transaction().start(False) (revision_id,entry_id,data_id)=dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(),runnum,lumirundata,branchinfo) dataDML.bulkInsertLumiLSSummary(dbsession,runnum,data_id,lumilsdata) dbsession.transaction().commit()
def insertLumischemaV2(dbsession,runnum,datasource,perlsrawdata,perbunchrawdata,deliveredonly=False): ''' input: lumirundata[datasource] perlsrawdata: {cmslsnum:instlumi} perbunchrawdata: {bxidx:lumifraction} lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]} ''' branchrevision_id=3#databranch_id branchinfo=(branchrevision_id,'DATA') lumirundata=[datasource] lumilsdata={} dbsession.transaction().start(True) oldlumidataid=dataDML.guessLumiDataIdByRun(dbsession.nominalSchema(),runnum) dbsession.transaction().commit() for cmslsnum,instlumi in perlsrawdata.items(): mystartorbit=startorbit+numorbit*(cmslsnum-1) bxdataocc1blob=None bxdataocc2blob=None bxdataetblob=None bxerrorocc1blob=None bxerrorocc2blob=None bxerroretblob=None bxqualityocc1blob=None bxqualityocc2blob=None bxqualityetblob=None cmsbxindexblob=None beam1intensityblob=None beam2intensityblob=None if perbunchrawdata: bxdataArray=array.array('f') bxerrorArray=array.array('f') bxqualityArray=array.array('h') cmsbxindexArray=array.array('h') beam1intensityArray=array.array('f') beam2intensityArray=array.array('f') for bxidx in range(1,3565): lumifraction=0.0 if perbunchrawdata.has_key(bxidx): lumifraction=perbunchrawdata[bxidx] bxlumivalue=float(instlumi*lumifraction)/float(bunchnorm) bxdataArray.append(bxlumivalue) beam1intensityArray.append(9124580336.0) beam1intensityArray.append(8932813306.0) cmsbxindexArray.append(bxidx) bxqualityArray.append(1) bxerrorArray.append(0.0) bxdataocc1blob=CommonUtil.packArraytoBlob(bxdataArray) bxdataocc2blob=CommonUtil.packArraytoBlob(bxdataArray) bxdataetblob=CommonUtil.packArraytoBlob(bxdataArray) bxerrorocc1blob=CommonUtil.packArraytoBlob(bxerrorArray) bxerrorocc2blob=CommonUtil.packArraytoBlob(bxerrorArray) bxerroretblob=CommonUtil.packArraytoBlob(bxerrorArray) bxqualityocc1blob=CommonUtil.packArraytoBlob(bxqualityArray) bxqualityocc2blob=CommonUtil.packArraytoBlob(bxqualityArray) bxqualityetblob=CommonUtil.packArraytoBlob(bxqualityArray) cmsbxindexblob=CommonUtil.packArraytoBlob(cmsbxindexArray) beam1intensityblob=CommonUtil.packArraytoBlob(beam1intensityArray) beam2intensityblob=CommonUtil.packArraytoBlob(beam2intensityArray) if deliveredonly: perlsdata=[0,float(instlumi)/float(6370),0.0,1,'STABLE BEAMS',beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob] else: perlsdata=[cmslsnum,float(instlumi)/float(6370),0.0,1,'STABLE BEAMS',beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob] lumilsdata[cmslsnum]=perlsdata print 'toinsert from scratch',lumilsdata dbsession.transaction().start(False) (revision_id,entry_id,data_id)=dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(),runnum,lumirundata,branchinfo) newlumilsnumMin=min(lumilsdata.keys()) newlumilsnumMax=max(lumilsdata.keys()) #update id of existing to new #update lumisummaryv2 set DATA_ID=:data_id where DATA_ID=:oldid and lumilsnum<newlumilsnumMin or lumilsnum>newlumilsnumMax #lumidataid is not None update lumilsdata set lumidataid=:newlumidataid where runnum=:run a inputData=coral.AttributeList() inputData.extend('dataid','unsigned long long') inputData.extend('oldid','unsigned long long') inputData.extend('lumilsnumMin','unsigned int') inputData.extend('lumilsnumMax','unsigned int') inputData['dataid'].setData( data_id ) inputData['oldid'].setData( oldlumidataid ) inputData['lumilsnumMin'].setData( newlumilsnumMin ) inputData['lumilsnumMax'].setData( newlumilsnumMax ) db=dbUtil.dbUtil(dbsession.nominalSchema()) db.singleUpdate('LUMISUMMARYV2','DATA_ID=:dataid','DATA_ID=:oldid AND LUMILSNUM<:lumilsnumMin OR lumilsnum>:lumilsnumMax',inputData) print 'to update existing id ',oldlumidataid,' outside region ',newlumilsnumMin,' , ',newlumilsnumMax dataDML.bulkInsertLumiLSSummary(dbsession,runnum,data_id,lumilsdata) dbsession.transaction().commit()
if options.action=='overview' or options.action=='lumibyls': reqTrg=True if options.action=='lumibyls' and options.hltpath: reqHlt=True if options.action=='recorded': reqTrg=True reqHlt=True if options.runnumber: reqrunmax=options.runnumber reqrunmin=options.runnumber if options.fillnum: reqfillmin=options.fillnum reqfillmax=options.fillnum if options.begin: (runbeg,fillbeg,timebeg)=CommonUtil.parseTime(options.begin) if runbeg: #there's --begin runnum #priority run,fill,time if not reqrunmin:# there's no -r, then take this reqrunmin=runbeg elif fillbeg: if not reqfillmin: reqfillmin=fillbeg elif timebeg: reqtimemin=timebeg if reqtimemin: lute=lumiTime.lumiTime() reqtimeminT=lute.StrToDatetime(reqtimemin,customfm='%m/%d/%y %H:%M:%S') timeFilter[0]=reqtimeminT if options.end: (runend,fillend,timeend)=CommonUtil.parseTime(options.end) if runend:
if options.action=='overview' or options.action=='lumibyls': reqTrg=True if options.action=='lumibyls' and options.hltpath: reqHlt=True if options.action=='recorded': reqTrg=True reqHlt=True if options.runnumber: reqrunmax=options.runnumber reqrunmin=options.runnumber if options.fillnum: reqfillmin=options.fillnum reqfillmax=options.fillnum if options.begin: (runbeg,fillbeg,timebeg)=CommonUtil.parseTime(options.begin) if runbeg: #there's --begin runnum #priority run,fill,time if not reqrunmin:# there's no -r, then take this reqrunmin=runbeg elif fillbeg: if not reqfillmin: reqfillmin=fillbeg elif timebeg: reqtimemin=timebeg if reqtimemin: lute=lumiTime.lumiTime() reqtimeminT=lute.StrToDatetime(reqtimemin,customfm='%m/%d/%y %H:%M:%S') timeFilter[0]=reqtimeminT if options.end: (runend,fillend,timeend)=CommonUtil.parseTime(options.end) if runend:
def main(): parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description = "Lumi Validation",formatter_class=argparse.ArgumentDefaultsHelpFormatter) allowedActions = ['batchupdate','update','dump'] allowedFlags = ['UNKNOWN','GOOD','BAD','SUSPECT'] # parse arguments parser.add_argument('action',choices=allowedActions,help='command actions') parser.add_argument('-c',dest='connect',action='store',required=True,help='connect string to lumiDB') parser.add_argument('-P',dest='authpath',action='store',required=True,help='path to authentication file') parser.add_argument('-i',dest='inputfile',action='store',help='lumi range selection file,required for batchupdate action') parser.add_argument('-o',dest='outputfile',action='store',help='output to csv file') parser.add_argument('-r',dest='runnumber',action='store',type=int,help='run number,optional') parser.add_argument('-runls',dest='runls',action='store',help='selection string,optional. Example [1234:[],4569:[1,1],[2,100]]') parser.add_argument('-flag',dest='flag',action='store',default='UNKNOWN',help='flag string,optional') parser.add_argument('--verbose',dest='verbose',action='store_true',help='verbose mode for printing' ) parser.add_argument('--debug',dest='debug',action='store_true',help='debug') options=parser.parse_args() if options.flag.upper() not in allowedFlags: print 'unrecognised flag ',options.flag.upper() raise os.environ['CORAL_AUTH_PATH'] = options.authpath connectstring=options.connect svc = coral.ConnectionService() msg=coral.MessageStream('') if options.debug: msg.setMsgVerbosity(coral.message_Level_Debug) else: msg.setMsgVerbosity(coral.message_Level_Error) session=svc.connect(connectstring,accessMode=coral.access_Update) session.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)") session.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)") result={}#parsing result {run:[[ls,status,comment]]} if options.debug : msg=coral.MessageStream('') msg.setMsgVerbosity(coral.message_Level_Debug) if options.action=='batchupdate': #populate from csv file, require -i argument if not options.inputfile: print 'inputfile -i option is required for batchupdate' raise csvReader=csv.reader(open(options.inputfile),delimiter=',') for row in csvReader: if len(row)==0: continue fieldrun=str(row[0]).strip() fieldls=str(row[1]).strip() fieldstatus=row[2] fieldcomment=row[3] if not result.has_key(int(fieldrun)): result[int(fieldrun)]=[] result[int(fieldrun)].append([int(fieldls),fieldstatus,fieldcomment]) insertupdateValidationData(session,result) if options.action=='update': #update flag interactively, require -runls argument #runls={run:[]} populate all CMSLSNUM found in LUMISUMMARY #runls={run:[[1,1],[2,5]],run:[[1,1],[2,5]]} #default value if not options.runls and not options.runnumber: print 'option -runls or -r is required for update' raise if not options.flag: print 'option -flag is required for update' raise if options.flag.upper() not in allowedFlags: print 'unrecognised flag ',options.flag raise if options.runnumber: runlsjson='{"'+str(options.runnumber)+'":[]}' elif options.runls: runlsjson=CommonUtil.tolegalJSON(options.runls) sparser=selectionParser.selectionParser(runlsjson) runsandls=sparser.runsandls() commentStr='NA' statusStr=options.flag for run,lslist in runsandls.items(): if not result.has_key(run): result[run]=[] for ls in lslist: result[run].append([ls,statusStr,commentStr]) insertupdateValidationData(session,result) if options.action=='dump': if options.runls or options.inputfile: if options.runls: runlsjson=CommonUtil.tolegalJSON(options.runls) sparser=selectionParser.selectionParser(runlsjson) runsandls=sparser.runsandls() if options.inputfile: p=inputFilesetParser.inputFilesetParser(options.inputfile) runsandls=p.runsandls() for runnum,lslist in runsandls.items(): dataperrun=getValidationData(session,run=runnum,cmsls=lslist) if dataperrun.has_key(runnum): result[runnum]=dataperrun[runnum] else: result[runnum]=[] else: result=getValidationData(session,run=options.runnumber) runs=result.keys() runs.sort() if options.outputfile: r=csvReporter.csvReporter(options.outputfile) for run in runs: for perrundata in result[run]: r.writeRow([str(run),str(perrundata[0]),perrundata[1],perrundata[2]]) else: for run in runs: print '== =' if len(result[run])==0: print str(run),'no validation data' continue for lsdata in result[run]: print str(run)+','+str(lsdata[0])+','+lsdata[1]+','+lsdata[2] del session del svc
def insertLumischemaV2(dbsession,runnum,datasource,perlsrawdata,perbunchrawdata,bxdistribution,withDetails=False,deliveredonly=False): ''' input: lumirundata[datasource] perlsrawdata: {cmslsnum:instlumi} perbunchrawdata: {bxidx:lumifraction} lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]} ''' branchrevision_id=3#databranch_id branchinfo=(branchrevision_id,'DATA') lumirundata=[datasource] lumilsdata={} for cmslsnum,instlumi in perlsrawdata.items(): mystartorbit=startorbit+numorbit*(cmslsnum-1) bxdataocc1blob=None bxdataocc2blob=None bxdataetblob=None bxerrorocc1blob=None bxerrorocc2blob=None bxerroretblob=None bxqualityocc1blob=None bxqualityocc2blob=None bxqualityetblob=None cmsbxindexblob=None beam1intensityblob=None beam2intensityblob=None beamstatus='STABLE BEAMS' beamenergy=4000. instlumierror=0. instlumiquality=1 if perbunchrawdata: withDetails=True bxdataocc1blob=perbunchrawdata[cmslsnum][0] bxerrorocc1blob=perbunchrawdata[cmslsnum][1] bxqualityocc1blob=perbunchrawdata[cmslsnum][2] bxdataocc2blob=perbunchrawdata[cmslsnum][3] bxerrorocc2blob=perbunchrawdata[cmslsnum][4] bxqualityocc2blob=perbunchrawdata[cmslsnum][5] bxdataetblob=perbunchrawdata[cmslsnum][6] bxerroretblob=perbunchrawdata[cmslsnum][7] bxqualityetblob=perbunchrawdata[cmslsnum][8] bxindexblob=perbunchrawdata[cmslsnum][9] beam1intensityblob=perbunchrawdata[cmslsnum][10] beam2intensityblob=perbunchrawdata[cmslsnum][11] beamstatus=perbunchrawdata[cmslsnum][12] beamenergy=perbunchrawdata[cmslsnum][13] instlumierror=perbunchrawdata[cmslsnum][14] instlumiquality=perbunchrawdata[cmslsnum][15] elif bxdistribution: withDetails=True bxdataArray=array.array('f') bxerrorArray=array.array('f') bxqualityArray=array.array('h') cmsbxindexArray=array.array('h') beam1intensityArray=array.array('f') beam2intensityArray=array.array('f') for bxidx in range(1,3565): lumifraction=0.0 if bxidx in perbunchrawdata: lumifraction=perbunchrawdata[bxidx] bxlumivalue=float(instlumi*lumifraction) bxdataArray.append(bxlumivalue) beam1intensityArray.append(9124580336.0) beam1intensityArray.append(8932813306.0) cmsbxindexArray.append(bxidx) bxqualityArray.append(1) bxerrorArray.append(0.0) bxdataocc1blob=CommonUtil.packArraytoBlob(bxdataArray) bxdataocc2blob=CommonUtil.packArraytoBlob(bxdataArray) bxdataetblob=CommonUtil.packArraytoBlob(bxdataArray) bxerrorocc1blob=CommonUtil.packArraytoBlob(bxerrorArray) bxerrorocc2blob=CommonUtil.packArraytoBlob(bxerrorArray) bxerroretblob=CommonUtil.packArraytoBlob(bxerrorArray) bxqualityocc1blob=CommonUtil.packArraytoBlob(bxqualityArray) bxqualityocc2blob=CommonUtil.packArraytoBlob(bxqualityArray) bxqualityetblob=CommonUtil.packArraytoBlob(bxqualityArray) cmsbxindexblob=CommonUtil.packArraytoBlob(cmsbxindexArray) beam1intensityblob=CommonUtil.packArraytoBlob(beam1intensityArray) beam2intensityblob=CommonUtil.packArraytoBlob(beam2intensityArray) if deliveredonly: perlsdata=[0,float(instlumi),instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob] else: perlsdata=[cmslsnum,float(instlumi),instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob] lumilsdata[cmslsnum]=perlsdata #print 'toinsert from scratch',lumilsdata print(lumilsdata) dbsession.transaction().start(False) (revision_id,entry_id,data_id)=dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(),runnum,lumirundata,branchinfo,'LUMIDATA') dataDML.bulkInsertLumiLSSummary(dbsession,runnum,data_id,lumilsdata,'LUMISUMMARYV2',withDetails=withDetails) dbsession.transaction().commit()
reqtimemax = None timeFilter = [None, None] pbeammode = None if options.beammode == 'stable': pbeammode = 'STABLE BEAMS' iresults = [] reqTrg = False reqHlt = False if options.action == 'overview' or options.action == 'lumibyls' or options.action == 'lumibylsXing': reqTrg = True if options.action == 'recorded': reqTrg = True reqHlt = True if options.begin: (reqrunmin, reqfillmin, reqtimemin) = CommonUtil.parseTime(options.begin) if reqtimemin: lute = lumiTime.lumiTime() reqtimeminT = lute.StrToDatetime(reqtimemin, customfm='%m/%d/%y %H:%M:%S') timeFilter[0] = reqtimeminT if options.end: (reqrunmax, reqfillmax, reqtimemax) = CommonUtil.parseTime(options.end) if reqtimemax: lute = lumiTime.lumiTime() reqtimemaxT = lute.StrToDatetime(reqtimemax, customfm='%m/%d/%y %H:%M:%S') timeFilter[1] = reqtimemaxT ############################################################## # check working environment
def insertLumischemaV2(dbsession, runnum, datasource, perlsrawdata, perbunchrawdata): ''' input: lumirundata[datasource] perlsrawdata: {cmslsnum:instlumi} perbunchrawdata: {bxidx:lumifraction} lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]} ''' branchrevision_id = 3 #databranch_id branchinfo = (branchrevision_id, 'DATA') lumirundata = [datasource] lumilsdata = {} for cmslsnum, instlumi in perlsrawdata.items(): mystartorbit = startorbit + numorbit * (cmslsnum - 1) bxdataArray = array.array('f') bxerrorArray = array.array('f') bxqualityArray = array.array('h') cmsbxindexArray = array.array('h') beam1intensityArray = array.array('f') beam2intensityArray = array.array('f') for bxidx in range(1, 3565): lumifraction = 0.0 if bxidx in perbunchrawdata: lumifraction = perbunchrawdata[bxidx] bxlumivalue = float(instlumi * lumifraction) / float(bunchnorm) bxdataArray.append(bxlumivalue) beam1intensityArray.append(9124580336.0) beam1intensityArray.append(8932813306.0) cmsbxindexArray.append(bxidx) bxqualityArray.append(1) bxerrorArray.append(0.0) bxdataocc1blob = CommonUtil.packArraytoBlob(bxdataArray) bxdataocc2blob = CommonUtil.packArraytoBlob(bxdataArray) bxdataetblob = CommonUtil.packArraytoBlob(bxdataArray) bxerrorocc1blob = CommonUtil.packArraytoBlob(bxerrorArray) bxerrorocc2blob = CommonUtil.packArraytoBlob(bxerrorArray) bxerroretblob = CommonUtil.packArraytoBlob(bxerrorArray) bxqualityocc1blob = CommonUtil.packArraytoBlob(bxqualityArray) bxqualityocc2blob = CommonUtil.packArraytoBlob(bxqualityArray) bxqualityetblob = CommonUtil.packArraytoBlob(bxqualityArray) cmsbxindexblob = CommonUtil.packArraytoBlob(cmsbxindexArray) beam1intensityblob = CommonUtil.packArraytoBlob(beam1intensityArray) beam2intensityblob = CommonUtil.packArraytoBlob(beam2intensityArray) perlsdata = [ cmslsnum, float(instlumi) / float(6370), 0.0, 1, 'STABLE BEAMS', beamenergy, numorbit, mystartorbit, cmsbxindexblob, beam1intensityblob, beam2intensityblob, bxdataocc1blob, bxerrorocc1blob, bxqualityocc1blob, bxdataocc2blob, bxerrorocc2blob, bxqualityocc2blob, bxdataetblob, bxerroretblob, bxqualityetblob ] lumilsdata[cmslsnum] = perlsdata print(lumilsdata) dbsession.transaction().start(False) (revision_id, entry_id, data_id) = dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(), runnum, lumirundata, branchinfo) dataDML.bulkInsertLumiLSSummary(dbsession, runnum, data_id, lumilsdata) dbsession.transaction().commit()
def specificlumiTofile(fillnum,filldata,outdir): # #input : fillnum # filldata: {bxidx:[[lstime,beamstatusfrac,lumivalue,lumierror,speclumi,speclumierr]],[]} #sorted by bxidx, sorted by lstime inside list #check outdir/fillnum subdir exists; if not, create it; else outdir=outdir/fillnum # if not filldata: print('empty input data, do nothing for fill ',fillnum) return timedict={}#{lstime:[[stablebeamfrac,lumi,lumierr,speclumi,speclumierr]]} filloutdir=os.path.join(outdir,str(fillnum)) if not os.path.exists(filloutdir): os.mkdir(filloutdir) for cmsbxidx,perbxdata in filldata.items(): lhcbucket=0 if cmsbxidx!=0: lhcbucket=(cmsbxidx-1)*10+1 a=sorted(perbxdata,key=lambda x:x[0]) filename=str(fillnum)+'_lumi_'+str(lhcbucket)+'_CMS.txt' linedata=[] for perlsdata in a: ts=int(perlsdata[0]) beamstatusfrac=perlsdata[1] lumi=perlsdata[2] lumierror=perlsdata[3] #beam1intensity=perlsdata[4] #beam2intensity=perlsdata[5] speclumi=perlsdata[4] speclumierror= perlsdata[5] if lumi>0: linedata.append([ts,beamstatusfrac,lumi,lumierror,speclumi,speclumierror]) if ts not in timedict: timedict[ts]=[] timedict[ts].append([beamstatusfrac,lumi,lumierror,speclumi,speclumierror]) if len(linedata)>10:#at least 10 good ls f=open(os.path.join(filloutdir,filename),'w') for line in linedata: print('%d\t%e\t%e\t%e\t%e\t%e'%(line[0],line[1],line[2],line[3],line[4],line[5]), file=f) f.close() #print 'writing avg file' summaryfilename=str(fillnum)+'_lumi_CMS.txt' f=None lstimes=sorted(timedict.keys()) fillseg=[] lscounter=0 for lstime in lstimes: allvalues=timedict[lstime] transposedvalues=CommonUtil.transposed(allvalues,0.0) bstatfrac=transposedvalues[0][0]#beamstatus does not change with bx position lumivals=transposedvalues[1] lumitot=sum(lumivals) if bstatfrac==1.0 : fillseg.append([lstime,lumitot]) lumierrs=transposedvalues[2] lumierrortot=math.sqrt(sum(map(lambda x:x**2,lumierrs))) specificvals=transposedvalues[3] specificavg=sum(specificvals)/float(len(specificvals))#avg spec lumi specificerrs=transposedvalues[4] specifictoterr=math.sqrt(sum(map(lambda x:x**2,specificerrs))) specificerravg=specifictoterr/float(len(specificvals)) if lscounter==0: f=open(os.path.join(filloutdir,summaryfilename),'w') lscounter+=1 print('%d\t%e\t%e\t%e\t%e\t%e'%(lstime,bstatfrac,lumitot,lumierrortot,specificavg,specificerravg), file=f) if f is not None: f.close() #print 'writing summary file' fillsummaryfilename=str(fillnum)+'_bxsum_CMS.txt' f=open(os.path.join(filloutdir,fillsummaryfilename),'w') if len(fillseg)==0: print('%s'%('#no stable beams'), file=f) f.close() return previoustime=fillseg[0][0] boundarytime=fillseg[0][0] #print 'boundary time ',boundarytime summaryls={} summaryls[boundarytime]=[] for [lstime,lumitot] in fillseg:#fillseg is everything with stable beam flag if lstime-previoustime>50.0: boundarytime=lstime #print 'found new boundary ',boundarytime summaryls[boundarytime]=[] # print 'appending ',boundarytime,lstime,lumitot summaryls[boundarytime].append([lstime,lumitot]) previoustime=lstime #print summaryls summarylstimes=summaryls.keys() summarylstimes.sort() lumip=lumiParameters.ParametersObject() for bts in summarylstimes: startts=bts tsdatainseg=summaryls[bts] #print 'tsdatainseg ',tsdatainseg stopts=tsdatainseg[-1][0] plu=max(CommonUtil.transposed(tsdatainseg,0.0)[1]) lui=sum(CommonUtil.transposed(tsdatainseg,0.0)[1])*lumip.lslengthsec() print('%d\t%d\t%e\t%e'%(startts,stopts,plu,lui), file=f) f.close()
def patchV2(dbsession,runnum,inputpathnames,inputdata): ''' inputpathnames: [pathname,] inputdata: {cmslsnum:[presc,presc...]} update the most recent version of lshlt data if some ls exist if none old ls exists insert new hlt records ''' try: dbsession.transaction().start(True) oldhltdataid=dataDML.guessHltDataIdByRun(dbsession.nominalSchema(),runnum) existingrundata=dataDML.hltRunById(dbsession.nominalSchema(),oldhltdataid) dbsession.transaction().commit() if not oldhltdataid:#no data at all dbsession.transaction().start(False) insertV2(dbsession,runnum,inputpathnames,inputdata) dbsession.transaction().commit() return hltnamedict=existingrundata[3]#[(pathidx,hltname),(pathidx,hltname)...] dbsession.transaction().start(True) existinglsdata=dataDML.hltLSById(dbsession.nominalSchema(),oldhltdataid) dbsession.transaction().commit() oldlsdata=existinglsdata[1] existinglslist=oldlsdata.keys() toupdate={}#{cmslsnum:[presc,presc...]} toinsert={}#{cmslsnum:[presc,presc...]} if existinglslist and len(existinglslist)!=0:#there are some existing data for cmslsnum,oldlscontent in oldlsdata.items(): if cmslsnum in inputdata.keys(): # if overlap with new data, update old data with new toupdate[cmslsnum]=inputdata[cmslsnum] for cmslsnum,lshltcontent in inputdata.items(): if cmslsnum in toupdate: continue #it's to update not to insert toinsert[cmslsnum]=inputdata[cmslsnum] # # insert into lshlt(data_id,runnum,cmslsnum,prescaleblob,hltcountblob,hltacceptblob) values() # dbsession.transaction().start(False) tabrowDefDict={'DATA_ID':'unsigned long long','RUNNUM':'unsigned int','CMSLSNUM':'unsigned int','PRESCALEBLOB':'blob','HLTCOUNTBLOB':'blob','HLTACCEPTBLOB':'blob'} for cmslsnum,perlsdata in toinsert.items(): prescaleArray=array.array('I') hltcountArray=array.array('I') hltacceptArray=array.array('I') for (pathidx,hltname) in hltnamedict: thispathIdx=inputpathnames.index(hltname) thispresc=perlsdata[thispathIdx] thiscount=0 thisaccept=0 prescaleArray.append(thispresc) hltcountArray.append(thiscount) hltacceptArray.append(thisaccept) prescaleblob=CommonUtil.packArraytoBlob(prescaleArray) hltcountblob=CommonUtil.packArraytoBlob(hltcountArray) hltacceptblob=CommonUtil.packArraytoBlob(hltacceptArray) tabrowValueDict={'DATA_ID':oldhltdataid,'RUNNUM':int(runnum),'CMSLSNUM':int(cmslsnum),'PRESCALEBLOB':prescaleblob,'HLTCOUNTBLOB':hltcountblob,'HLTACCEPTBLOB':hltacceptblob} db=dbUtil.dbUtil(dbsession.nominalSchema()) db.insertOneRow(nameDealer.lshltTableName(),tabrowDefDict,tabrowValueDict) # # update lshlt set prescaleblob=:prescaleblob,hltcoutblob=:hltcountblob,hltacceptblob=:hltacceptblob where data_id=:olddata_id and cmslsnum=:cmslsnum; # setClause='PRESCALEBLOB=:prescaleblob,HLTCOUNTBLOB=:hltcountblob,HLTACCEPTBLOB=:hltacceptblob' updateCondition='DATA_ID=:oldhltdataid and CMSLSNUM=:cmslsnum' for cmslsnum,perlsdata in toupdate.items(): prescaleArray=array.array('I') hltcountArray=array.array('I') hltacceptArray=array.array('I') for (pathidx,hltname) in hltnamedict: thispathIdx=inputpathnames.index(hltname) thispresc=perlsdata[thispathIdx] thiscount=0 thisaccept=0 prescaleArray.append(thispresc) hltcountArray.append(thiscount) hltacceptArray.append(thisaccept) prescaleblob=CommonUtil.packArraytoBlob(prescaleArray) hltcountblob=CommonUtil.packArraytoBlob(hltcountArray) hltacceptblob=CommonUtil.packArraytoBlob(hltacceptArray) iData=coral.AttributeList() iData.extend('prescaleblob','blob') iData.extend('hltcountblob','blob') iData.extend('hltacceptblob','blob') iData.extend('olddata_id','unsigned int') iData.extend('cmslsnum','unsigned int') iData['prescaleblob'].setData(prescaleblob) iData['hltcountblob'].setData(hltcountblob) iData['hltacceptblob'].setData(hltacceptblob) iData['olddata_id'].setData(int(olddata_id)) iData['cmslsnum'].setData(int(cmslsnum)) db=dbUtil.dbUtil(schema) db.singleUpdate(nameDealer.lshltTableName(),setClause,updateCondition,iData) dbsession.transaction().commit() #dbsession.transaction().rollback() except : raise
runlist, withcomment=False, lumitype='HF') lumirundata = dataDML.lumiRunByIds(session.nominalSchema(), hfdataidmap, lumitype='HF') #{runnum: (datasource(1),nominalegev(2),ncollidingbunches(3)} GrunsummaryData = lumiCalcAPI.runsummaryMap(session.nominalSchema(), irunlsdict) session.transaction().commit() for runnum in lumirundata.keys(): session.transaction().start(True) hfdataidinfo = hfdataidmap[runnum] hflumidata = lumiCalcAPI.instLumiForIds(session.nominalSchema(), irunlsdict, hfdataidmap, GrunsummaryData, beamstatusfilter=None, withBXInfo=True, bxAlgo='OCC1') for perlsdata in hflumidata[runnum]: lumilsnum = perlsdata[0] cmslsnum = perlsdata[1] bxinfo = perlsdata[9] bxdistro = getdistribution(cmslsnum, bxinfo[0], bxinfo[1]) print(lumilsnum, cmslsnum, CommonUtil.flatten(bxdistro)) session.transaction().commit() del session del svc
def insertLumischemaV2(dbsession, runnum, datasource, perlsrawdata, perbunchrawdata, bxdistribution, withDetails=False, deliveredonly=False): ''' input: lumirundata[datasource] perlsrawdata: {cmslsnum:instlumi} perbunchrawdata: {bxidx:lumifraction} lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]} ''' branchrevision_id = 3 #databranch_id branchinfo = (branchrevision_id, 'DATA') lumirundata = [datasource] lumilsdata = {} for cmslsnum, instlumi in perlsrawdata.items(): mystartorbit = startorbit + numorbit * (cmslsnum - 1) bxdataocc1blob = None bxdataocc2blob = None bxdataetblob = None bxerrorocc1blob = None bxerrorocc2blob = None bxerroretblob = None bxqualityocc1blob = None bxqualityocc2blob = None bxqualityetblob = None cmsbxindexblob = None beam1intensityblob = None beam2intensityblob = None beamstatus = 'STABLE BEAMS' beamenergy = 4000. instlumierror = 0. instlumiquality = 1 if perbunchrawdata: withDetails = True bxdataocc1blob = perbunchrawdata[cmslsnum][0] bxerrorocc1blob = perbunchrawdata[cmslsnum][1] bxqualityocc1blob = perbunchrawdata[cmslsnum][2] bxdataocc2blob = perbunchrawdata[cmslsnum][3] bxerrorocc2blob = perbunchrawdata[cmslsnum][4] bxqualityocc2blob = perbunchrawdata[cmslsnum][5] bxdataetblob = perbunchrawdata[cmslsnum][6] bxerroretblob = perbunchrawdata[cmslsnum][7] bxqualityetblob = perbunchrawdata[cmslsnum][8] bxindexblob = perbunchrawdata[cmslsnum][9] beam1intensityblob = perbunchrawdata[cmslsnum][10] beam2intensityblob = perbunchrawdata[cmslsnum][11] beamstatus = perbunchrawdata[cmslsnum][12] beamenergy = perbunchrawdata[cmslsnum][13] instlumierror = perbunchrawdata[cmslsnum][14] instlumiquality = perbunchrawdata[cmslsnum][15] elif bxdistribution: withDetails = True bxdataArray = array.array('f') bxerrorArray = array.array('f') bxqualityArray = array.array('h') cmsbxindexArray = array.array('h') beam1intensityArray = array.array('f') beam2intensityArray = array.array('f') for bxidx in range(1, 3565): lumifraction = 0.0 if bxidx in perbunchrawdata: lumifraction = perbunchrawdata[bxidx] bxlumivalue = float(instlumi * lumifraction) bxdataArray.append(bxlumivalue) beam1intensityArray.append(9124580336.0) beam1intensityArray.append(8932813306.0) cmsbxindexArray.append(bxidx) bxqualityArray.append(1) bxerrorArray.append(0.0) bxdataocc1blob = CommonUtil.packArraytoBlob(bxdataArray) bxdataocc2blob = CommonUtil.packArraytoBlob(bxdataArray) bxdataetblob = CommonUtil.packArraytoBlob(bxdataArray) bxerrorocc1blob = CommonUtil.packArraytoBlob(bxerrorArray) bxerrorocc2blob = CommonUtil.packArraytoBlob(bxerrorArray) bxerroretblob = CommonUtil.packArraytoBlob(bxerrorArray) bxqualityocc1blob = CommonUtil.packArraytoBlob(bxqualityArray) bxqualityocc2blob = CommonUtil.packArraytoBlob(bxqualityArray) bxqualityetblob = CommonUtil.packArraytoBlob(bxqualityArray) cmsbxindexblob = CommonUtil.packArraytoBlob(cmsbxindexArray) beam1intensityblob = CommonUtil.packArraytoBlob( beam1intensityArray) beam2intensityblob = CommonUtil.packArraytoBlob( beam2intensityArray) if deliveredonly: perlsdata = [ 0, float(instlumi), instlumierror, instlumiquality, beamstatus, beamenergy, numorbit, mystartorbit, cmsbxindexblob, beam1intensityblob, beam2intensityblob, bxdataocc1blob, bxerrorocc1blob, bxqualityocc1blob, bxdataocc2blob, bxerrorocc2blob, bxqualityocc2blob, bxdataetblob, bxerroretblob, bxqualityetblob ] else: perlsdata = [ cmslsnum, float(instlumi), instlumierror, instlumiquality, beamstatus, beamenergy, numorbit, mystartorbit, cmsbxindexblob, beam1intensityblob, beam2intensityblob, bxdataocc1blob, bxerrorocc1blob, bxqualityocc1blob, bxdataocc2blob, bxerrorocc2blob, bxqualityocc2blob, bxdataetblob, bxerroretblob, bxqualityetblob ] lumilsdata[cmslsnum] = perlsdata #print 'toinsert from scratch',lumilsdata print(lumilsdata) dbsession.transaction().start(False) (revision_id, entry_id, data_id) = dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(), runnum, lumirundata, branchinfo, 'LUMIDATA') dataDML.bulkInsertLumiLSSummary(dbsession, runnum, data_id, lumilsdata, 'LUMISUMMARYV2', withDetails=withDetails) dbsession.transaction().commit()
raise RuntimeError( 'parameter corrector is required for create/insert action') if not normvalueDict.has_key( 'since') or not normvalueDict['since']: raise RuntimeError( 'parameter since is required for create/insert action') correctorStr = normvalueDict['corrector'] sincerun = int(normvalueDict['since']) if options.firstsince: if sincerun < int(options.firstsince): continue amodetag = normvalueDict['amodetag'] egev = int(normvalueDict['egev']) detailcomment = normvalueDict['comment'] (correctorname, parameterlist) = CommonUtil.parselumicorrector(correctorStr) parameterDict = {} for param in parameterlist: parameterDict[param] = normvalueDict[param] normDML.insertValueToNormId(dbsession.nominalSchema(), normdata_id, sincerun, correctorStr, amodetag, egev, parameterDict, comment=detailcomment) dbsession.transaction().commit() ############################## # setdefault/unsetdefault
if options.beammode=='stable': pbeammode = 'STABLE BEAMS' if options.verbose: print 'General configuration' print '\tconnect: ',options.connect print '\tauthpath: ',options.authpath print '\tlumi data version: ',options.lumiversion print '\tsiteconfpath: ',options.siteconfpath print '\toutputfile: ',options.outputfile print '\tscalefactor: ',options.scalefactor if options.action=='recorded' and options.hltpath: print 'Action: effective luminosity in hltpath: ',options.hltpath else: print 'Action: ',options.action if options.normfactor: if CommonUtil.is_floatstr(normfactor): print '\tuse norm factor value ',normfactor else: print '\tuse specific norm factor name ',normfactor else: print '\tuse norm factor in context (amodetag,beamenergy)' if options.runnumber: # if runnumber specified, do not go through other run selection criteria print '\tselect specific run== ',options.runnumber else: print '\trun selections == ' print '\tinput selection file: ',options.inputfile print '\tbeam mode: ',options.beammode print '\tfill: ',options.fillnum print '\tamodetag: ',options.amodetag print '\tbegin: ',options.begin print '\tend: ',options.end