def hltForIds(schema, irunlsdict, dataidmap, hltpathname=None, hltpathpattern=None, withL1Pass=False, withHLTAccept=False): ''' input: irunlsdict: {run:[cmsls]} (required) dataidmap: {run:(lumiid,trgid,hltid)} hltpathname: exact match hltpathname (optional) hltpathpattern: regex match hltpathpattern (optional) withL1Pass: with L1 pass count withHLTAccept: with HLT accept output: {runnumber:[(cmslsnum,[(hltpath,hltprescale,l1pass,hltaccept),...]),(cmslsnum,[])})} ''' result = {} for run in irunlsdict.keys(): lslist = irunlsdict[run] if lslist is not None and len(lslist) == 0: result[run] = [] #if no LS is selected for a run continue if not dataidmap.has_key(run): continue hltdataid = dataidmap[run][2] if hltdataid is None: result[run] = None continue #run non exist hltdata = dataDML.hltLSById(schema, hltdataid, hltpathname=hltpathname, hltpathpattern=hltpathpattern, withL1Pass=withL1Pass, withHLTAccept=withHLTAccept) #(runnum,{cmslsnum:[(pathname,prescale,l1pass,hltaccept),...]}) result[run] = [] if hltdata and hltdata[1]: lsdict = {} for cmslsnum in sorted(hltdata[1]): if lslist is not None and cmslsnum not in lslist: continue lsdata = [] for perpathdata in hltdata[1][cmslsnum]: pathname = perpathdata[0] prescale = perpathdata[1] l1pass = None hltaccept = None if withL1Pass: l1pass = perpathdata[2] if withHLTAccept: hltaccept = perpathdata[3] lsdata.append((pathname, prescale, l1pass, hltaccept)) result[run].append((cmslsnum, lsdata)) return result
def hltForIds(schema,irunlsdict,dataidmap,hltpathname=None,hltpathpattern=None,withL1Pass=False,withHLTAccept=False): ''' input: irunlsdict: {run:[cmsls]} (required) dataidmap: {run:(lumiid,trgid,hltid)} hltpathname: exact match hltpathname (optional) hltpathpattern: regex match hltpathpattern (optional) withL1Pass: with L1 pass count withHLTAccept: with HLT accept output: {runnumber:[(cmslsnum,[(hltpath,hltprescale,l1pass,hltaccept),...]),(cmslsnum,[])})} ''' result={} for run in irunlsdict.keys(): lslist=irunlsdict[run] if lslist is not None and len(lslist)==0: result[run]=[]#if no LS is selected for a run continue if run not in dataidmap: continue hltdataid=dataidmap[run][2] if hltdataid is None: result[run]=None continue #run non exist hltdata=dataDML.hltLSById(schema,hltdataid,hltpathname=hltpathname,hltpathpattern=hltpathpattern,withL1Pass=withL1Pass,withHLTAccept=withHLTAccept) #(runnum,{cmslsnum:[(pathname,prescale,l1pass,hltaccept),...]}) result[run]=[] if hltdata and hltdata[1]: lsdict={} for cmslsnum in sorted(hltdata[1]): if lslist is not None and cmslsnum not in lslist: continue lsdata=[] for perpathdata in hltdata[1][cmslsnum]: pathname=perpathdata[0] prescale=perpathdata[1] l1pass=None hltaccept=None if withL1Pass: l1pass=perpathdata[2] if withHLTAccept: hltaccept=perpathdata[3] lsdata.append((pathname,prescale,l1pass,hltaccept)) result[run].append((cmslsnum,lsdata)) return result
def hltForRange(schema,inputRange,hltpathname=None,hltpathpattern=None,withL1Pass=False,withHLTAccept=False,tableName=None,branchName=None): ''' input: inputRange: {run:[cmsls]} (required) hltpathname: exact match hltpathname (optional) hltpathpattern: regex match hltpathpattern (optional) branchName : data version output: {runnumber:[(cmslsnum,[(hltpath,hltprescale,l1pass,hltaccept),...]),(cmslsnum,[])})} ''' #if tableName is None: # tableName=nameDealer.hltdataTableName() #if branchName is None: # branchName='DATA' result={} for run in inputRange.keys(): lslist=inputRange[run] if lslist is not None and len(lslist)==0: result[run]=[]#if no LS is selected for a run continue hltdataid=dataDML.guessHltDataIdByRun(schema,run) if hltdataid is None: result[run]=None continue #run non exist hltdata=dataDML.hltLSById(schema,hltdataid,hltpathname=hltpathname,hltpathpattern=hltpathpattern,withL1Pass=withL1Pass,withHLTAccept=withHLTAccept) #(runnum,{cmslsnum:[(pathname,prescale,l1pass,hltaccept),...]}) result[run]=[] if hltdata and hltdata[1]: for cmslsnum in sorted(hltdata[1]): if lslist is not None and cmslsnum not in lslist: continue lsdata=[] for perpathdata in hltdata[1][cmslsnum]: pathname=perpathdata[0] prescale=perpathdata[1] l1pass=None hltaccept=None if withL1Pass: l1pass=perpathdata[2] if withHLTAccept: hltaccept=perpathdata[3] lsdata.append((pathname,prescale,l1pass,hltaccept)) result[run].append((cmslsnum,lsdata)) return result
lumils = sorted(lumiLSdata.keys()) for lsnum in lumils: print('lumilsnum,cmslsnum,instlumi ', lsnum, lumiLSdata[lsnum][0], lumiLSdata[lsnum][1] * normval) (runnum, trgLSdata) = dataDML.trgLSById(schema, trgdataid) cmsls = trgLSdata.keys() cmsls.sort() for lsnum in cmsls: print('cmslsnum,deadtime,bizerocount,bitzeroprescale,deadfrac ', lsnum, trgLSdata[lsnum][0], trgLSdata[lsnum][1], trgLSdata[lsnum][2], trgLSdata[lsnum][3]) [runnum, datasource, npath, pathnames] = dataDML.hltRunById(schema, hltdataid) print('npath,pathnames ', npath, pathnames) pathnameList = pathnames.split(',') (runnum, hltLSdata) = dataDML.hltLSById(schema, hltdataid) cmsls = hltLSdata.keys() cmsls.sort() for lsnum in cmsls: prescaleblob = hltLSdata[lsnum][0] print('lsnum ', lsnum) if prescaleblob: hltprescales = CommonUtil.unpackBlobtoArray(prescaleblob, 'h') for pidx, pathname in enumerate(pathnameList): print('pathname, hltprescales ', pathname, hltprescales[pidx]) session.transaction().commit() del session print('')
def patchV2(dbsession,runnum,inputpathnames,inputdata): ''' inputpathnames: [pathname,] inputdata: {cmslsnum:[presc,presc...]} update the most recent version of lshlt data if some ls exist if none old ls exists insert new hlt records ''' try: dbsession.transaction().start(True) oldhltdataid=dataDML.guessHltDataIdByRun(dbsession.nominalSchema(),runnum) existingrundata=dataDML.hltRunById(dbsession.nominalSchema(),oldhltdataid) dbsession.transaction().commit() if not oldhltdataid:#no data at all dbsession.transaction().start(False) insertV2(dbsession,runnum,inputpathnames,inputdata) dbsession.transaction().commit() return hltnamedict=existingrundata[3]#[(pathidx,hltname),(pathidx,hltname)...] dbsession.transaction().start(True) existinglsdata=dataDML.hltLSById(dbsession.nominalSchema(),oldhltdataid) dbsession.transaction().commit() oldlsdata=existinglsdata[1] existinglslist=oldlsdata.keys() toupdate={}#{cmslsnum:[presc,presc...]} toinsert={}#{cmslsnum:[presc,presc...]} if existinglslist and len(existinglslist)!=0:#there are some existing data for cmslsnum,oldlscontent in oldlsdata.items(): if cmslsnum in inputdata.keys(): # if overlap with new data, update old data with new toupdate[cmslsnum]=inputdata[cmslsnum] for cmslsnum,lshltcontent in inputdata.items(): if cmslsnum in toupdate: continue #it's to update not to insert toinsert[cmslsnum]=inputdata[cmslsnum] # # insert into lshlt(data_id,runnum,cmslsnum,prescaleblob,hltcountblob,hltacceptblob) values() # dbsession.transaction().start(False) tabrowDefDict={'DATA_ID':'unsigned long long','RUNNUM':'unsigned int','CMSLSNUM':'unsigned int','PRESCALEBLOB':'blob','HLTCOUNTBLOB':'blob','HLTACCEPTBLOB':'blob'} for cmslsnum,perlsdata in toinsert.items(): prescaleArray=array.array('I') hltcountArray=array.array('I') hltacceptArray=array.array('I') for (pathidx,hltname) in hltnamedict: thispathIdx=inputpathnames.index(hltname) thispresc=perlsdata[thispathIdx] thiscount=0 thisaccept=0 prescaleArray.append(thispresc) hltcountArray.append(thiscount) hltacceptArray.append(thisaccept) prescaleblob=CommonUtil.packArraytoBlob(prescaleArray) hltcountblob=CommonUtil.packArraytoBlob(hltcountArray) hltacceptblob=CommonUtil.packArraytoBlob(hltacceptArray) tabrowValueDict={'DATA_ID':oldhltdataid,'RUNNUM':int(runnum),'CMSLSNUM':int(cmslsnum),'PRESCALEBLOB':prescaleblob,'HLTCOUNTBLOB':hltcountblob,'HLTACCEPTBLOB':hltacceptblob} db=dbUtil.dbUtil(dbsession.nominalSchema()) db.insertOneRow(nameDealer.lshltTableName(),tabrowDefDict,tabrowValueDict) # # update lshlt set prescaleblob=:prescaleblob,hltcoutblob=:hltcountblob,hltacceptblob=:hltacceptblob where data_id=:olddata_id and cmslsnum=:cmslsnum; # setClause='PRESCALEBLOB=:prescaleblob,HLTCOUNTBLOB=:hltcountblob,HLTACCEPTBLOB=:hltacceptblob' updateCondition='DATA_ID=:oldhltdataid and CMSLSNUM=:cmslsnum' for cmslsnum,perlsdata in toupdate.items(): prescaleArray=array.array('I') hltcountArray=array.array('I') hltacceptArray=array.array('I') for (pathidx,hltname) in hltnamedict: thispathIdx=inputpathnames.index(hltname) thispresc=perlsdata[thispathIdx] thiscount=0 thisaccept=0 prescaleArray.append(thispresc) hltcountArray.append(thiscount) hltacceptArray.append(thisaccept) prescaleblob=CommonUtil.packArraytoBlob(prescaleArray) hltcountblob=CommonUtil.packArraytoBlob(hltcountArray) hltacceptblob=CommonUtil.packArraytoBlob(hltacceptArray) iData=coral.AttributeList() iData.extend('prescaleblob','blob') iData.extend('hltcountblob','blob') iData.extend('hltacceptblob','blob') iData.extend('olddata_id','unsigned int') iData.extend('cmslsnum','unsigned int') iData['prescaleblob'].setData(prescaleblob) iData['hltcountblob'].setData(hltcountblob) iData['hltacceptblob'].setData(hltacceptblob) iData['olddata_id'].setData(int(olddata_id)) iData['cmslsnum'].setData(int(cmslsnum)) db=dbUtil.dbUtil(schema) db.singleUpdate(nameDealer.lshltTableName(),setClause,updateCondition,iData) dbsession.transaction().commit() #dbsession.transaction().rollback() except : raise
(lumidataid,trgdataid,hltdataid)=dataDML.guessDataIdByRun(schema,myrun) print 'all dataids ',lumidataid,trgdataid,hltdataid (runnum,lumiLSdata)=dataDML.lumiLSById(schema,lumidataid) lumils=lumiLSdata.keys() lumils.sort() for lsnum in lumils: print 'lumilsnum,cmslsnum,instlumi ',lsnum,lumiLSdata[lsnum][0],lumiLSdata[lsnum][1]*normval (runnum,trgLSdata)=dataDML.trgLSById(schema,trgdataid) cmsls=trgLSdata.keys() cmsls.sort() for lsnum in cmsls: print 'cmslsnum,deadtime,bizerocount,bitzeroprescale,deadfrac ',lsnum,trgLSdata[lsnum][0],trgLSdata[lsnum][1],trgLSdata[lsnum][2],trgLSdata[lsnum][3] [runnum,datasource,npath,pathnames]=dataDML.hltRunById(schema,hltdataid) print 'npath,pathnames ',npath,pathnames pathnameList=pathnames.split(',') (runnum,hltLSdata)=dataDML.hltLSById(schema,hltdataid) cmsls=hltLSdata.keys() cmsls.sort() for lsnum in cmsls: prescaleblob=hltLSdata[lsnum][0] print 'lsnum ',lsnum if prescaleblob: hltprescales=CommonUtil.unpackBlobtoArray(prescaleblob,'h') for pidx,pathname in enumerate(pathnameList): print 'pathname, hltprescales ',pathname,hltprescales[pidx] session.transaction().commit() del session print ''
def effectiveLumiForRange(schema,inputRange,hltpathname=None,hltpathpattern=None,amodetag=None,beamstatus=None,egev=None,withBXInfo=False,xingMinLum=0.0,bxAlgo=None,withBeamIntensity=False,norm=None,finecorrections=None,driftcorrections=None,usecorrectionv2=False,lumitype='HF',branchName=None): ''' input: inputRange {run:[cmsls]} (required) hltpathname: selected hltpathname hltpathpattern: regex select hltpaths amodetag: amodetag for picking norm(optional) egev: beamenergy for picking norm(optional) withBXInfo: get per bunch info (optional) bxAlgo: algoname for bx values (optional) ['OCC1','OCC2','ET'] xingMinLum: cut on bx lumi value (optional) withBeamIntensity: get beam intensity info (optional) normname: norm factor name to use (optional) branchName: data version output: result {run:[lumilsnum(0),cmslsnum(1),timestamp(2),beamstatus(3),beamenergy(4),deliveredlumi(5),recordedlumi(6),calibratedlumierror(7),{hltpath:[l1name,l1prescale,hltprescale,efflumi]},bxdata,beamdata,fillnum]} lumi unit: 1/ub ''' if lumitype not in ['HF','PIXEL']: raise ValueError('unknown lumitype '+lumitype) if branchName is None: branchName='DATA' lumitableName='' lumilstableName='' if lumitype=='HF': lumitableName=nameDealer.lumidataTableName() lumilstableName=nameDealer.lumisummaryv2TableName() else: lumitableName=nameDealer.pixellumidataTableName() lumilstableName=nameDealer.pixellumisummaryv2TableName() numbx=3564 result = {} normval=None perbunchnormval=None if norm: normval=_getnorm(schema,norm) perbunchnormval=float(normval)/float(1000) elif amodetag and egev: normval=_decidenormFromContext(schema,amodetag,egev) perbunchnormval=float(normval)/float(1000) c=lumiTime.lumiTime() lumip=lumiParameters.ParametersObject() for run in inputRange.keys(): lslist=inputRange[run] if lslist is not None and len(lslist)==0:#no selected ls, do nothing for this run result[run]=[] continue cmsrunsummary=dataDML.runsummary(schema,run) if len(cmsrunsummary)==0:#non existing run result[run]=None continue startTimeStr=cmsrunsummary[6] fillnum=cmsrunsummary[4] lumidataid=None trgdataid=None hltdataid=None lumidataid=dataDML.guessLumiDataIdByRun(schema,run,lumitableName) trgdataid=dataDML.guessTrgDataIdByRun(schema,run) hltdataid=dataDML.guessHltDataIdByRun(schema,run) if lumidataid is None or trgdataid is None or hltdataid is None: result[run]=None continue (lumirunnum,lumidata)=dataDML.lumiLSById(schema,lumidataid,beamstatus,tableName=lumilstableName) (trgrunnum,trgdata)=dataDML.trgLSById(schema,trgdataid,withPrescale=True) (hltrunnum,hltdata)=dataDML.hltLSById(schema,hltdataid,hltpathname=hltpathname,hltpathpattern=hltpathpattern) hlttrgmap=dataDML.hlttrgMappingByrun(schema,run) if not normval:#if norm cannot be decided , look for it according to context per run normval=_decidenormForRun(schema,run) perbunchnormval=float(normval)/float(1000) if not normval:#still not found? resort to global default (should never come here) normval=6370 perbunchnormval=6.37 print '[Warning] using default normalization '+str(normval) perrunresult=[] for lumilsnum,perlsdata in lumidata.items(): cmslsnum=perlsdata[0] triggeredls=perlsdata[0] if lslist is not None and cmslsnum not in lslist: #cmslsnum=0 triggeredls=0 recordedlumi=0.0 instlumi=perlsdata[1] instlumierror=perlsdata[2] avglumi=instlumi*normval calibratedlumi=avglumi if lumitype=='HF' and finecorrections and finecorrections[run]: if usecorrectionv2: if driftcorrections and driftcorrections[run]: calibratedlumi=lumiCorrections.applyfinecorrectionV2(avglumi,finecorrections[run][0],finecorrections[run][1],finecorrections[run][2],finecorrections[run][3],finecorrections[run][4],driftcorrections[run]) else: calibratedlumi=lumiCorrections.applyfinecorrectionV2(avglumi,finecorrections[run][0],finecorrections[run][1],finecorrections[run][2],finecorrections[run][3],finecorrections[run][4],1.0) else: calibratedlumi=lumiCorrections.applyfinecorrection(avglumi,finecorrections[run][0],finecorrections[run][1],finecorrections[run][2]) if lumitype=='PIXEL' and finecorrections is not None: calibratedlumi=finecorrections[run]*avglumi calibratedlumierror=instlumierror*normval bstatus=perlsdata[4] begev=perlsdata[5] numorbit=perlsdata[6] startorbit=perlsdata[7] timestamp=c.OrbitToTime(startTimeStr,startorbit,0) lslen=lumip.lslengthsec() deliveredlumi=calibratedlumi*lslen recordedlumi=0.0 trgprescalemap={}#trgprescalemap for this ls efflumidict={} if triggeredls!=0: if not trgdata.has_key(cmslsnum): #triggeredls=0 #if no trigger, set back to non-cms-active ls recordedlumi=0.0 # no trigger->nobeam recordedlumi=None else: deadcount=trgdata[cmslsnum][0] ##subject to change !! bitzerocount=trgdata[cmslsnum][1] bitzeroprescale=trgdata[cmslsnum][2] deadfrac=trgdata[cmslsnum][3] if deadfrac<0 or deadfrac>1.0: deadfrac=1.0 #if float(bitzerocount)*float(bitzeroprescale)==0.0: # deadfrac=1.0 #else: # deadfrac=float(deadcount)/(float(bitzerocount)*float(bitzeroprescale)) #if deadfrac>1.0: # deadfrac=1.0 #artificial correction in case of deadfrac>1 recordedlumi=deliveredlumi*(1.0-deadfrac) l1bitinfo=trgdata[cmslsnum][4] if l1bitinfo: for thisbitinfo in l1bitinfo: thisbitname=thisbitinfo[0] thisbitprescale=thisbitinfo[2] #trgprescalemap['"'+thisbitname+'"']=thisbitprescale#note:need to double quote bit name! trgprescalemap['"'+thisbitname+'"']=thisbitprescale del trgdata[cmslsnum][:] if hltdata.has_key(cmslsnum): hltpathdata=hltdata[cmslsnum] #print 'hltpathdata ',hltpathdata for pathidx,thispathinfo in enumerate(hltpathdata): efflumi=0.0 thispathname=thispathinfo[0] thisprescale=thispathinfo[1] thisl1seed=None l1bitname=None l1prescale=None try: thisl1seed=hlttrgmap[thispathname] except KeyError: thisl1seed=None # print 'hltpath, l1seed, hltprescale ',thispathname,thisl1seed,thisprescale if thisl1seed: try: l1bitname=hltTrgSeedMapper.findUniqueSeed(thispathname,thisl1seed) if l1bitname : l1prescale=trgprescalemap[l1bitname]#need to match double quoted string! else: l1prescale=None except KeyError: l1prescale=None if l1prescale and thisprescale :#normal both prescaled efflumi=recordedlumi/(float(l1prescale)*float(thisprescale)) efflumidict[thispathname]=[l1bitname,l1prescale,thisprescale,efflumi] elif l1prescale and thisprescale==0: #hltpath in menu but masked efflumi=0.0 efflumidict[thispathname]=[l1bitname,l1prescale,thisprescale,efflumi] else: efflumi=0.0 efflumidict[thispathname]=[None,0,thisprescale,efflumi] bxvaluelist=[] bxerrorlist=[] bxdata=None beamdata=None if withBXInfo: bxinfo=lumidata[8] bxvalueArray=None bxerrArray=None if bxinfo: bxvalueArray=bxinfo[0] bxerrArray=bxinfo[1] for idx,bxval in enumerate(bxvalueArray): if finecorrections and finecorrections[run]: if usecorrectionv2: if driftcorrections and driftcorrections[run]: mybxval=lumiCorrections.applyfinecorrectionBXV2(bxval,avglumi,perbunchnormval,finecorrections[run][0],finecorrections[run][1],finecorrections[run][2],finecorrections[run][3],finecorrections[run][4],driftcorrections[run]) else: mybxval=lumiCorrections.applyfinecorrectionBXV2(bxval,avglumi,perbunchnormval,finecorrections[run][0],finecorrections[run][1],finecorrections[run][2],finecorrections[run][3],finecorrections[run][4],1.0) else: mybxval=lumiCorrections.applyfinecorrectionBX(bxval,avglumi,perbunchnormval,finecorrections[run][0],finecorrections[run][1],finecorrections[run][2]) else: mybxval=bxval*perbunchnormval if mybxval>xingMinLum: bxidxlist.append(idx) bxvaluelist.append(bxval) bxerrorlist.append(bxerrArray[idx]) del bxvalueArray[:] del bxerrArray[:] bxdata=(bxidxlist,bxvaluelist,bxerrorlist) if withBeamIntensity: beaminfo=perlsdata[9] bxindexlist=[] b1intensitylist=[] b2intensitylist=[] if beaminfo: bxindexarray=beaminfo[0] beam1intensityarray=beaminfo[1] beam2intensityarray=beaminfo[2] bxindexlist=bxindexarray.tolist() b1intensitylist=beam1intensityarray.tolist() b2intensitylist=beam2intensityarray.tolist() del bxindexarray[:] del beam1intensityarray[:] del beam2intensityarray[:] beamdata=(bxindexlist,b1intensitylist,b2intensitylist) # print cmslsnum,deliveredlumi,recordedlumi,efflumidict perrunresult.append([lumilsnum,triggeredls,timestamp,bstatus,begev,deliveredlumi,recordedlumi,calibratedlumierror,efflumidict,bxdata,beamdata,fillnum]) del perlsdata[:] result[run]=perrunresult #print result return result