def insertV2(dbsession,runnum,inputpathnames,inputdata):
'''
inputpathnames: [pathname]
inputdata: {cmslsnum:[presc,presc...]}
'''
branchrevision_id=DATABRANCH_ID
try:
pathnamesClob=','.join(inputpathnames)
hltrundata=[pathnamesClob,'text file']
(hltrevid,hltentryid,hltdataid)=dataDML.addHLTRunDataToBranch(dbsession.nominalSchema(),runnum,hltrundata,(branchrevision_id,'DATA'))
hltlsdata={}
for cmslsnum,perlsdata in inputdata.items():
prescaleArray=array.array('I')
hltcountArray=array.array('I')
hltacceptArray=array.array('I')
for presc in perlsdata:
thiscount=0
thisaccept=0
prescaleArray.append(presc)
hltcountArray.append(thiscount)
hltacceptArray.append(thisaccept)
prescaleblob=CommonUtil.packArraytoBlob(prescaleArray)
hltcountblob=CommonUtil.packArraytoBlob(hltcountArray)
hltacceptblob=CommonUtil.packArraytoBlob(hltacceptArray)
hltlsdata[cmslsnum]=[hltcountblob,hltacceptblob,prescaleblob]
dataDML.bulkInsertHltLSData(dbsession,runnum,hltdataid,hltlsdata,500)
except:
raise
def fieldvalues(self, fieldname, fieldtype):
'''
given the input result field name and typem return the list of values
'''
fieldidx = None
result = []
try:
fieldidx = self.__inputResultHeader.index(fieldname)
except:
print 'field ', fieldname, ' not found'
raise
for r in self.__inputResult:
stringvalue = r[fieldidx]
if fieldtype in ['int', 'unsigned int']:
if not CommonUtil.is_intstr(stringvalue):
print 'field ', fieldname, ' is not integer type'
raise
else:
result.append(int(stringvalue))
continue
elif fieldtype in ['float']:
if not CommonUtil.is_floatstr(stringvalue):
print 'field ', fieldname, ' is not float type'
raise
else:
result.append(float(stringvalue))
contine
elif fieldtype in ['string', 'str']:
result.append(stringvalue)
else:
raise 'unsupported type ', fieldtype
return result
def insertLumiDetaildata(dbsession,perlsrawdata,perbunchrawdata,summaryidlsmap):
dataDef=[]
dataDef.append(('LUMISUMMARY_ID','unsigned long long'))
dataDef.append(('LUMIDETAIL_ID','unsigned long long'))
dataDef.append(('BXLUMIVALUE','blob'))
dataDef.append(('BXLUMIERROR','blob'))
dataDef.append(('BXLUMIQUALITY','blob'))
dataDef.append(('ALGONAME','string'))
perbunchiData=[]
dbsession.transaction().start(False)
iddealer=idDealer.idDealer(dbsession.nominalSchema())
db=dbUtil.dbUtil(dbsession.nominalSchema())
print('to insert lumidetail ')
for algoname in ['OCC1','OCC2','ET']:
for cmslsnum,instlumi in perlsrawdata.items():
lumisummary_id=summaryidlsmap[cmslsnum]
lumidetail_id=iddealer.generateNextIDForTable('LUMIDETAIL')
bxdata=array.array('f')
bxerror=array.array('f')
bxquality=array.array('h')
for bxidx in range(1,3565):
lumifraction=0.0
if bxidx in perbunchrawdata:
lumifraction=perbunchrawdata[bxidx]
bxlumivalue=float(instlumi*lumifraction)/float(bunchnorm)
bxdata.append(bxlumivalue)
bxerror.append(0.0)
bxquality.append(1)
bxdataBlob=CommonUtil.packArraytoBlob(bxdata)
bxerrorBlob=CommonUtil.packArraytoBlob(bxerror)
bxqualityBlob=CommonUtil.packArraytoBlob(bxquality)
perbunchiData.append([('LUMISUMMARY_ID',lumisummary_id),('LUMIDETAIL_ID',lumidetail_id),('BXLUMIVALUE',bxdataBlob),('BXLUMIERROR',bxerrorBlob),('BXLUMIQUALITY',bxqualityBlob),('ALGONAME',algoname)])
db.bulkInsert('LUMIDETAIL',dataDef,perbunchiData)
dbsession.transaction().commit()
return
def fieldvalues(self,fieldname,fieldtype):
'''
given the input result field name and typem return the list of values
'''
fieldidx=None
result=[]
try:
fieldidx=self.__inputResultHeader.index(fieldname)
except:
print('field ',fieldname,' not found')
raise RuntimeError('field')
for r in self.__inputResult:
stringvalue=r[fieldidx]
if fieldtype in ['int','unsigned int']:
if not CommonUtil.is_intstr(stringvalue):
print('field ',fieldname,' is not integer type')
raise RuntimeError('field')
else:
result.append(int(stringvalue))
continue
elif fieldtype in ['float']:
if not CommonUtil.is_floatstr(stringvalue):
print('field ',fieldname,' is not float type')
raise RuntimeError('field')
else:
result.append(float(stringvalue))
contine
elif fieldtype in ['string','str']:
result.append(stringvalue)
else:
raise RuntimeError('unsupported type '+fieldtype)
return result
def hlt(schema, nls):
'''
input:
output: [datasource,pathnameclob,{cmslsnum:[inputcountBlob,acceptcountBlob,prescaleBlob]}]
'''
o = ['oracle://cms_orcon_prod/cms_runinfo']
pathnameclob = 'HLT_PixelTracks_Multiplicity70,HLT_PixelTracks_Multiplicity85,HLT_PixelTracks_Multiplicity100,HLT_GlobalRunHPDNoise,HLT_TechTrigHCALNoise'
o.append(pathnameclob)
perlsdata = {}
for cmslsnum in range(1, nls + 1):
inputcounts = array.array('I')
acceptcounts = array.array('I')
prescalecounts = array.array('I')
for i in range(1, 201):
inputcounts.append(6677889)
acceptcounts.append(3344565)
prescalecounts.append(17)
inputcountsBlob = CommonUtil.packArraytoBlob(inputcounts)
acceptcountsBlob = CommonUtil.packArraytoBlob(acceptcounts)
prescalecountsBlob = CommonUtil.packArraytoBlob(prescalecounts)
if cmslsnum not in perlsdata:
perlsdata[cmslsnum] = []
perlsdata[cmslsnum].extend(
[inputcountsBlob, acceptcountsBlob, prescalecountsBlob])
o.append(perlsdata)
return o
def insertLumiDetaildata(dbsession,perlsrawdata,perbunchrawdata,summaryidlsmap):
dataDef=[]
dataDef.append(('LUMISUMMARY_ID','unsigned long long'))
dataDef.append(('LUMIDETAIL_ID','unsigned long long'))
dataDef.append(('BXLUMIVALUE','blob'))
dataDef.append(('BXLUMIERROR','blob'))
dataDef.append(('BXLUMIQUALITY','blob'))
dataDef.append(('ALGONAME','string'))
perbunchiData=[]
dbsession.transaction().start(False)
iddealer=idDealer.idDealer(dbsession.nominalSchema())
db=dbUtil.dbUtil(dbsession.nominalSchema())
print('to insert lumidetail ')
for algoname in ['OCC1','OCC2','ET']:
for cmslsnum,instlumi in perlsrawdata.items():
lumisummary_id=summaryidlsmap[cmslsnum]
lumidetail_id=iddealer.generateNextIDForTable('LUMIDETAIL')
bxdata=array.array('f')
bxerror=array.array('f')
bxquality=array.array('h')
for bxidx in range(1,3565):
lumifraction=0.0
if bxidx in perbunchrawdata:
lumifraction=perbunchrawdata[bxidx]
bxlumivalue=float(instlumi*lumifraction)/float(bunchnorm)
bxdata.append(bxlumivalue)
bxerror.append(0.0)
bxquality.append(1)
bxdataBlob=CommonUtil.packArraytoBlob(bxdata)
bxerrorBlob=CommonUtil.packArraytoBlob(bxerror)
bxqualityBlob=CommonUtil.packArraytoBlob(bxquality)
perbunchiData.append([('LUMISUMMARY_ID',lumisummary_id),('LUMIDETAIL_ID',lumidetail_id),('BXLUMIVALUE',bxdataBlob),('BXLUMIERROR',bxerrorBlob),('BXLUMIQUALITY',bxqualityBlob),('ALGONAME',algoname)])
db.bulkInsert('LUMIDETAIL',dataDef,perbunchiData)
dbsession.transaction().commit()
return
def trg(schema, nls):
'''
input:
output: [datasource,bitzeroname,bitnameclob,{cmslsnum:[deadtime,bitzerocount,bitzeroprescale,trgcountBlob,trgprescaleBlob]}]
'''
o = ['oracle://cms_orcon_prod/cms_gtmon', 'L1_ZeroBias']
bitnameclob = 'L1_ZeroBias,False,L1_SingleHfBitCountsRing1_1,L1_SingleHfBitCountsRing2_1,L1_SingleMu15,L1SingleJet,Jura'
o.append(bitnameclob)
perlsdata = {}
for cmslsnum in range(1, nls + 1):
deadtime = 99 + cmslsnum
bitzerocount = 897865
bitzeroprescale = 17
trgcounts = array.array('I')
prescalecounts = array.array('I')
for i in range(1, 192):
trgcounts.append(778899 + i)
prescalecounts.append(17)
trgcountsBlob = CommonUtil.packArraytoBlob(trgcounts)
prescalecountsBlob = CommonUtil.packArraytoBlob(prescalecounts)
if cmslsnum not in perlsdata:
perlsdata[cmslsnum] = []
perlsdata[cmslsnum].extend([
deadtime, bitzerocount, bitzeroprescale, trgcountsBlob,
prescalecountsBlob
])
o.append(perlsdata)
return o
def hlt(schema,nls):
'''
input:
output: [datasource,pathnameclob,{cmslsnum:[inputcountBlob,acceptcountBlob,prescaleBlob]}]
'''
o=['oracle://cms_orcon_prod/cms_runinfo']
pathnameclob='HLT_PixelTracks_Multiplicity70,HLT_PixelTracks_Multiplicity85,HLT_PixelTracks_Multiplicity100,HLT_GlobalRunHPDNoise,HLT_TechTrigHCALNoise'
o.append(pathnameclob)
perlsdata={}
for cmslsnum in range(1,nls+1):
inputcounts=array.array('I')
acceptcounts=array.array('I')
prescalecounts=array.array('I')
for i in range(1,201):
inputcounts.append(6677889 )
acceptcounts.append(3344565)
prescalecounts.append(17)
inputcountsBlob=CommonUtil.packArraytoBlob(inputcounts)
acceptcountsBlob=CommonUtil.packArraytoBlob(acceptcounts)
prescalecountsBlob=CommonUtil.packArraytoBlob(prescalecounts)
if not perlsdata.has_key(cmslsnum):
perlsdata[cmslsnum]=[]
perlsdata[cmslsnum].extend([inputcountsBlob,acceptcountsBlob,prescalecountsBlob])
o.append(perlsdata)
return o
def trg(schema,nls):
'''
input:
output: [datasource,bitzeroname,bitnameclob,{cmslsnum:[deadtime,bitzerocount,bitzeroprescale,trgcountBlob,trgprescaleBlob]}]
'''
o=['oracle://cms_orcon_prod/cms_gtmon','L1_ZeroBias']
bitnameclob='L1_ZeroBias,False,L1_SingleHfBitCountsRing1_1,L1_SingleHfBitCountsRing2_1,L1_SingleMu15,L1SingleJet,Jura'
o.append(bitnameclob)
perlsdata={}
for cmslsnum in range(1,nls+1):
deadtime=99+cmslsnum
bitzerocount=897865
bitzeroprescale=17
trgcounts=array.array('I')
prescalecounts=array.array('I')
for i in range(1,192):
trgcounts.append(778899+i)
prescalecounts.append(17)
trgcountsBlob=CommonUtil.packArraytoBlob(trgcounts)
prescalecountsBlob=CommonUtil.packArraytoBlob(prescalecounts)
if not perlsdata.has_key(cmslsnum):
perlsdata[cmslsnum]=[]
perlsdata[cmslsnum].extend([deadtime,bitzerocount,bitzeroprescale,trgcountsBlob,prescalecountsBlob])
o.append(perlsdata)
return o
def insertV2(dbsession,runnum,inputpathnames,inputdata):
'''
inputpathnames: [pathname]
inputdata: {cmslsnum:[presc,presc...]}
'''
branchrevision_id=DATABRANCH_ID
try:
pathnamesClob=','.join(inputpathnames)
hltrundata=[pathnamesClob,'text file']
(hltrevid,hltentryid,hltdataid)=dataDML.addHLTRunDataToBranch(dbsession.nominalSchema(),runnum,hltrundata,(branchrevision_id,'DATA'))
hltlsdata={}
for cmslsnum,perlsdata in inputdata.items():
prescaleArray=array.array('I')
hltcountArray=array.array('I')
hltacceptArray=array.array('I')
for presc in perlsdata:
thiscount=0
thisaccept=0
prescaleArray.append(presc)
hltcountArray.append(thiscount)
hltacceptArray.append(thisaccept)
prescaleblob=CommonUtil.packArraytoBlob(prescaleArray)
hltcountblob=CommonUtil.packArraytoBlob(hltcountArray)
hltacceptblob=CommonUtil.packArraytoBlob(hltacceptArray)
hltlsdata[cmslsnum]=[hltcountblob,hltacceptblob,prescaleblob]
dataDML.bulkInsertHltLSData(dbsession,runnum,hltdataid,hltlsdata,500)
except:
raise
def fieldtotal(self,fieldname,fieldtype):
'''
given the input result field name and type, return the total
'''
fieldidx=None
result=0
try:
fieldidx=self.__inputResultHeader.index(fieldname)
except:
print('field ',fieldname,' not found')
raise
for r in self.__inputResult:
stringvalue=r[fieldidx]
if fieldtype in ['int','unsigned int']:
if not CommonUtil.is_intstr(stringvalue):
print('field ',fieldname,' is not integer type')
raise
else:
result=int(result)+int(stringvalue)
continue
elif fieldtype in ['float'] :
if not CommonUtil.is_floatstr(stringvalue):
print('field ',fieldname,' is not float type')
raise
else:
result=float(result)+float(stringvalue)
continue
else:
raise RunTimeError('cannot sum types other than int ,float')
return result
def fieldtotal(self, fieldname, fieldtype):
'''
given the input result field name and type, return the total
'''
fieldidx = None
result = 0
try:
fieldidx = self.__inputResultHeader.index(fieldname)
except:
print 'field ', fieldname, ' not found'
raise
for r in self.__inputResult:
stringvalue = r[fieldidx]
if fieldtype in ['int', 'unsigned int']:
if not CommonUtil.is_intstr(stringvalue):
print 'field ', fieldname, ' is not integer type'
raise
else:
result = int(result) + int(stringvalue)
continue
elif fieldtype in ['float']:
if not CommonUtil.is_floatstr(stringvalue):
print 'field ', fieldname, ' is not float type'
raise
else:
result = float(result) + float(stringvalue)
continue
else:
raise 'cannot sum types other than int ,float'
return result
def insertLumiDetaildata(dbsession, perlsrawdata, perbunchrawdata, summaryidlsmap):
dataDef = []
dataDef.append(("LUMISUMMARY_ID", "unsigned long long"))
dataDef.append(("LUMIDETAIL_ID", "unsigned long long"))
dataDef.append(("BXLUMIVALUE", "blob"))
dataDef.append(("BXLUMIERROR", "blob"))
dataDef.append(("BXLUMIQUALITY", "blob"))
dataDef.append(("ALGONAME", "string"))
perbunchiData = []
dbsession.transaction().start(False)
iddealer = idDealer.idDealer(dbsession.nominalSchema())
db = dbUtil.dbUtil(dbsession.nominalSchema())
print "to insert lumidetail "
for algoname in ["OCC1", "OCC2", "ET"]:
for cmslsnum, instlumi in perlsrawdata.items():
lumisummary_id = summaryidlsmap[cmslsnum]
lumidetail_id = iddealer.generateNextIDForTable("LUMIDETAIL")
print "cmslsnum ", lumidetail_id, lumisummary_id
bxdataocc1 = array.array("f")
bxdataocc2 = array.array("f")
bxdataet = array.array("f")
bxerror = array.array("f")
bxquality = array.array("h")
for bxidx in range(1, 3565):
lumifraction = 0.0
if perbunchrawdata.has_key(bxidx):
lumifraction = perbunchrawdata[bxidx]
bxlumivalue = float(instlumi * lumifraction)
bxdataocc1.append(bxlumivalue)
bxdataocc2.append(bxlumivalue)
bxdataet.append(bxlumivalue)
bxdataocc1Blob = CommonUtil.packArraytoBlob(bxdataocc1)
bxdataocc2Blob = CommonUtil.packArraytoBlob(bxdataocc2)
bxdataetBlob = CommonUtil.packArraytoBlob(bxdataet)
bxqualityBlob = CommonUtil.packArraytoBlob(bxquality)
perbunchiData.append(
[
("LUMISUMMARY_ID", lumisummary_id),
("LUMIDETAIL_ID", lumidetail_id),
("BXLUMIVALUE", bxdataocc1Blob),
("BXLUMIERROR", bxdataocc2Blob),
("BXLUMIQUALITY", bxqualityBlob),
("ALGONAME", algoname),
]
)
db.bulkInsert("LUMIDETAIL", dataDef, perbunchiData)
print perbunchiData
dbsession.transaction().commit()
return
def _getnorm(schema,norm):
if isinstance(norm,int) or isinstance(norm,float) or CommonUtil.is_floatstr(norm) or CommonUtil.is_intstr(norm):
return float(norm)
if not isinstance(norm,str):
raise ValueError('wrong parameter type')
normdataid=dataDML.guessnormIdByName(schema,norm)
normresult=dataDML.luminormById(schema,normdataid)
return normresult[2]
def toCSVLumiByLSXing(lumidata,scalefactor,filename):
'''
input:{run:[lumilsnum(0),cmslsnum(1),timestamp(2),beamstatus(3),beamenergy(4),deliveredlumi(5),recordedlumi(6),calibratedlumierror(7),{hltpath:[l1name,l1prescale,hltprescale,efflumi]},bxdata,beamdata]}
output:
fieldnames=['Run','CMSLS','Delivered(/ub)','Recorded(/ub)','BX']
'''
result=[]
assert(filename)
fieldnames=['run','ls','delivered(/ub)','recorded(/ub)','bx']
for run in sorted(lumidata):
rundata=lumidata[run]
if rundata is None:
result.append([run,'n/a','n/a','n/a','n/a'])
continue
for lsdata in rundata:
cmslsnum=lsdata[1]
if cmslsnum==0:
continue
deliveredlumi=lsdata[5]
recordedlumi=lsdata[6]
(bxidxlist,bxvaluelist,bxerrorlist)=lsdata[8]
bxresult=[]
if bxidxlist and bxvaluelist:
bxinfo=CommonUtil.transposed([bxidxlist,bxvaluelist])
bxresult=CommonUtil.flatten([run,cmslsnum,deliveredlumi*scalefactor,recordedlumi*scalefactor,bxinfo])
result.append(bxresult)
else:
result.append([run,cmslsnum,deliveredlumi*scalefactor,recordedlumi*scalefactor])
r=None
if filename.upper()=='STDOUT':
r=sys.stdout
r.write(','.join(fieldnames)+'\n')
for l in result:
r.write(str(l)+'\n')
else:
r=csvReporter.csvReporter(filename)
r.writeRow(fieldnames)
r.writeRows(result)
def runsWithresult(self):
'''
output: [run,run,...]
'''
result = {}
for f in self.__inputresultfiles:
csvReader = csv.reader(open(f), delimiter=',')
for row in csvReader:
field0 = str(row[0]).strip()
if not CommonUtil.is_intstr(field0):
continue
runnumber = int(field0)
if runnumber not in result:
result[runnumber] = None
return result.keys()
def runsWithresult(self):
'''
output: [run,run,...]
'''
result={}
for f in self.__inputresultfiles:
csvReader=csv.reader(open(f),delimiter=',')
for row in csvReader:
field0=str(row[0]).strip()
if not CommonUtil.is_intstr(field0):
continue
runnumber=int(field0)
if runnumber not in result:
result[runnumber]=None
return result.keys()
def lumiSummary(schema, nlumils):
'''
input:
output: [datasource,{lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beamintensityblob_1,beamintensitublob_2,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]}]
'''
o = ['file:fake.root']
perlsdata = {}
for lumilsnum in range(1, nlumils + 1):
cmslsnum = 0
if lumilsnum < 100:
cmslsnum = lumilsnum
instlumi = 2.37
instlumierror = 0.56
instlumiquality = 2
beamstatus = 'STABLE BEAMS'
beamenergy = 3.5e03
numorbit = 12345
startorbit = numorbit * lumilsnum
if cmslsnum == 0:
cmsbxindex = None
beam1intensity = None
beam2intensity = None
else:
cmsbxindex = array.array('I')
beam1intensity = array.array('f')
beam2intensity = array.array('f')
for idx in range(1, 3565):
cmsbxindex.append(idx)
beam1intensity.append(1.5e09)
beam2intensity.append(5.5e09)
cmsbxindexBlob = CommonUtil.packArraytoBlob(cmsbxindex)
beam1intensityBlob = CommonUtil.packArraytoBlob(beam1intensity)
beam2intensityBlob = CommonUtil.packArraytoBlob(beam2intensity)
bxlumivalue = array.array('f')
bxlumierror = array.array('f')
bxlumiquality = array.array('I')
for idx in range(1, 3565):
bxlumivalue.append(2.3)
bxlumierror.append(0.4)
bxlumiquality.append(2)
bxlumivalueBlob = CommonUtil.packArraytoBlob(bxlumivalue)
bxlumierrorBlob = CommonUtil.packArraytoBlob(bxlumierror)
bxlumiqualityBlob = CommonUtil.packArraytoBlob(bxlumiquality)
if lumilsnum not in perlsdata:
perlsdata[lumilsnum] = []
perlsdata[lumilsnum].extend([
cmslsnum, instlumi, instlumierror, instlumiquality, beamstatus,
beamenergy, numorbit, startorbit, cmsbxindexBlob,
beam1intensityBlob, beam2intensityBlob, bxlumivalueBlob,
bxlumierrorBlob, bxlumiqualityBlob, bxlumivalueBlob,
bxlumierrorBlob, bxlumiqualityBlob, bxlumivalueBlob,
bxlumierrorBlob, bxlumiqualityBlob
])
o.append(perlsdata)
return o
def toCSVOverview(lumidata,filename,resultlines,scalefactor,isverbose):
'''
input:
lumidata {run:[lumilsnum,cmslsnum,timestamp,beamstatus,beamenergy,deliveredlumi,recordedlumi,calibratedlumierror,(bxidx,bxvalues,bxerrs),(bxidx,b1intensities,b2intensities)]}
resultlines [[resultrow1],[resultrow2],...,] existing result row
'''
result=[]
fieldnames = ['Run', 'DeliveredLS', 'Delivered(/ub)','SelectedLS','Recorded(/ub)']
r=csvReporter.csvReporter(filename)
for rline in resultlines:
result.append(rline)
for run in lumidata.keys():
lsdata=lumidata[run]
if lsdata is None:
result.append([run,'n/a','n/a','n/a','n/a'])
continue
nls=len(lsdata)
deliveredData=[x[5] for x in lsdata]
recordedData=[x[6] for x in lsdata if x[6] is not None]
totdeliveredlumi=0.0
totrecordedlumi=0.0
if len(deliveredData)!=0:
totdeliveredlumi=sum(deliveredData)
if len(recordedData)!=0:
totrecordedlumi=sum(recordedData)
selectedcmsls=[x[1] for x in lsdata if x[1]!=0]
if len(selectedcmsls)==0:
selectedlsStr='n/a'
else:
selectedlsStr = CommonUtil.splitlistToRangeString(selectedcmsls)
result.append([run,nls,totdeliveredlumi*scalefactor,selectedlsStr,totrecordedlumi*scalefactor])
sortedresult=sorted(result,key=lambda x : int(x[0]))
r=None
assert(filename)
if filename.upper()=='STDOUT':
r=sys.stdout
r.write(','.join(fieldnames)+'\n')
for l in sortedresult:
r.write(str(l)+'\n')
else:
r=csvReporter.csvReporter(filename)
r.writeRow(fieldnames)
r.writeRows(sortedresult)
def getInstLumiPerLS(dbsession,c,runList,selectionDict,beamstatus=None,beamenergy=None,beamenergyfluctuation=0.09,finecorrections=None):
'''
input: runList[runnum], selectionDict{runnum:[ls]}
output:[[runnumber,lsnumber,deliveredInst,recordedInst,norbit,startorbit,runstarttime,runstoptime]]
'''
result=[]
selectedRunlist=runList
if len(selectionDict)!=0:
selectedRunlist=[]
allruns=runlist+selectionDict.keys()
dups=CommonUtil.count_dups(allruns)
for runnum,dupcount in dups:
if dupcount==2:
selectedRunlist.append(runnum)
dbsession.transaction().start(True)
for run in selectedRunlist:
q=dbsession.nominalSchema().newQuery()
runsummary=lumiQueryAPI.runsummaryByrun(q,run)
del q
runstarttime=runsummary[3]
runstoptime=runsummary[4]
q=dbsession.nominalSchema().newQuery()
if finecorrections and finecorrections[run]:
lumiperrun=lumiQueryAPI.lumisummaryByrun(q,run,c.LUMIVERSION,beamstatus,beamenergy,beamenergyfluctuation,finecorrections=finecorrections[run])
else:
lumiperrun=lumiQueryAPI.lumisummaryByrun(q,run,c.LUMIVERSION,beamstatus,beamenergy,beamenergyfluctuation)
del q
if len(lumiperrun)==0: #no result for this run
result.append([run,1,0.0,0.0,0,0,runstarttime,runstoptime])
else:
for lumiperls in lumiperrun:
cmslsnum=lumiperls[0]
instlumi=lumiperls[1]
recordedlumi=0.0
numorbit=lumiperls[2]
startorbit=lumiperls[3]
deadcount=0
bitzero=0
result.append([run,cmslsnum,instlumi,recordedlumi,numorbit,startorbit,runstarttime,runstoptime])
dbsession.transaction().commit()
if c.VERBOSE:
print result
return result
def lumiSummary(schema,nlumils):
'''
input:
output: [datasource,{lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beamintensityblob_1,beamintensitublob_2,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]}]
'''
o=['file:fake.root']
perlsdata={}
for lumilsnum in range(1,nlumils+1):
cmslsnum=0
if lumilsnum<100:
cmslsnum=lumilsnum
instlumi=2.37
instlumierror=0.56
instlumiquality=2
beamstatus='STABLE BEAMS'
beamenergy=3.5e03
numorbit=12345
startorbit=numorbit*lumilsnum
if cmslsnum==0:
cmsbxindex=None
beam1intensity=None
beam2intensity=None
else:
cmsbxindex=array.array('I')
beam1intensity=array.array('f')
beam2intensity=array.array('f')
for idx in range(1,3565):
cmsbxindex.append(idx)
beam1intensity.append(1.5e09)
beam2intensity.append(5.5e09)
cmsbxindexBlob=CommonUtil.packArraytoBlob(cmsbxindex)
beam1intensityBlob=CommonUtil.packArraytoBlob(beam1intensity)
beam2intensityBlob=CommonUtil.packArraytoBlob(beam2intensity)
bxlumivalue=array.array('f')
bxlumierror=array.array('f')
bxlumiquality=array.array('I')
for idx in range(1,3565):
bxlumivalue.append(2.3)
bxlumierror.append(0.4)
bxlumiquality.append(2)
bxlumivalueBlob=CommonUtil.packArraytoBlob(bxlumivalue)
bxlumierrorBlob=CommonUtil.packArraytoBlob(bxlumierror)
bxlumiqualityBlob=CommonUtil.packArraytoBlob(bxlumiquality)
if not perlsdata.has_key(lumilsnum):
perlsdata[lumilsnum]=[]
perlsdata[lumilsnum].extend([cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexBlob,beam1intensityBlob,beam2intensityBlob,bxlumivalueBlob,bxlumierrorBlob,bxlumiqualityBlob,bxlumivalueBlob,bxlumierrorBlob,bxlumiqualityBlob,bxlumivalueBlob,bxlumierrorBlob,bxlumiqualityBlob])
o.append(perlsdata)
return o
def normValueById(schema, normid):
'''
select * from luminormsv2data where data_id=normid
output:
{since:[corrector(0),{paramname:paramvalue}(1),amodetag(2),egev(3),comment(4)]}
'''
result = {}
l = nameDealer.luminormv2dataTableName()
paramdict = {}
qHandle = schema.newQuery()
try:
qHandle.addToTableList(l)
qConditionStr = 'DATA_ID=:normid'
qCondition = coral.AttributeList()
qCondition.extend('normid', 'unsigned long long')
qCondition['normid'].setData(normid)
qResult = coral.AttributeList()
qHandle.setCondition(qConditionStr, qCondition)
cursor = qHandle.execute()
while cursor.next():
since = cursor.currentRow()['SINCE'].data()
corrector = cursor.currentRow()['CORRECTOR'].data()
amodetag = cursor.currentRow()['AMODETAG'].data()
nominalegev = cursor.currentRow()['NOMINALEGEV'].data()
comment = ''
if not cursor.currentRow()['COMMENT'].isNull():
comment = cursor.currentRow()['COMMENT'].data()
(correctorfunc, params) = CommonUtil.parselumicorrector(corrector)
paramdict = {}
for param in params:
paramvalue = 0.0
if not cursor.currentRow()[param.upper()].isNull():
paramvalue = cursor.currentRow()[param.upper()].data()
paramdict[param] = paramvalue
result[since] = [
correctorfunc, paramdict, amodetag, nominalegev, comment
]
except:
raise
return result
def normValueById(schema,normid):
'''
select * from luminormsv2data where data_id=normid
output:
{since:[corrector(0),{paramname:paramvalue}(1),amodetag(2),egev(3),comment(4)]}
'''
result={}
l=nameDealer.luminormv2dataTableName()
paramdict={}
qHandle=schema.newQuery()
try:
qHandle.addToTableList(l)
qConditionStr='DATA_ID=:normid'
qCondition=coral.AttributeList()
qCondition.extend('normid','unsigned long long')
qCondition['normid'].setData(normid)
qResult=coral.AttributeList()
qHandle.setCondition(qConditionStr,qCondition)
cursor=qHandle.execute()
while cursor.next():
since=cursor.currentRow()['SINCE'].data()
corrector=cursor.currentRow()['CORRECTOR'].data()
amodetag=cursor.currentRow()['AMODETAG'].data()
nominalegev=cursor.currentRow()['NOMINALEGEV'].data()
comment=''
if not cursor.currentRow()['COMMENT'].isNull():
comment=cursor.currentRow()['COMMENT'].data()
(correctorfunc,params)=CommonUtil.parselumicorrector(corrector)
paramdict={}
for param in params:
paramvalue=0.0
if not cursor.currentRow()[param.upper()].isNull():
paramvalue=cursor.currentRow()[param.upper()].data()
paramdict[param]=paramvalue
result[since]=[correctorfunc,paramdict,amodetag,nominalegev,comment]
except:
raise
return result
def fetchOldData(schema, oldlumidataid):
'''
fetch old perbunch data if the run exists
select CMSBXINDEXBLOB,BEAMINTENSITYBLOB_1,BEAMINTENSITYBLOB_2,BXLUMIVALUE_OCC1,BXLUMIVALUE_OCC2,BXLUMIVALUE_ET,BXLUMIERROR_OCC1,BXLUMIERROR_OCC2,BXLUMIERROR_ET,BXLUMIQUALITY_OCC1,BXLUMIQUALITY_OCC2,BXLUMIQUALITY_ET from lumisummaryv2 where data_id=:oldlumidataid;
output:
{lumilsnum:[bxvalueblob_occ1(0),bxerrblob_occ1(1),bxqualityblob_occ1(2),bxvalueblob_occ2(3),bxerrblob_occ2(4),bxqualityblob_occ2(5),bxvalueblob_et(6),bxerrblob_et(7),bxqualityblob_et(8),bxindexblob(9),beam1intensity(10),beam2intensity(11),beamstatus(12),beamenergy(13),instlumierror(14),instlumiquality(15)],...}
'''
result = {}
qHandle = schema.newQuery()
try:
qHandle.addToTableList('LUMISUMMARYV2')
qHandle.addToOutputList('LUMILSNUM', 'lumilsnum')
qHandle.addToOutputList('BEAMSTATUS', 'beamstatus')
qHandle.addToOutputList('BEAMENERGY', 'beamenergy')
qHandle.addToOutputList('INSTLUMIERROR', 'instlumierror')
qHandle.addToOutputList('INSTLUMIQUALITY', 'instlumiquality')
qHandle.addToOutputList('BXLUMIVALUE_OCC1', 'bxvalue_occ1')
qHandle.addToOutputList('BXLUMIERROR_OCC1', 'bxerror_occ1')
qHandle.addToOutputList('BXLUMIQUALITY_OCC1', 'bxquality_occ1')
qHandle.addToOutputList('BXLUMIVALUE_OCC2', 'bxvalue_occ2')
qHandle.addToOutputList('BXLUMIERROR_OCC2', 'bxerror_occ2')
qHandle.addToOutputList('BXLUMIQUALITY_OCC2', 'bxquality_occ2')
qHandle.addToOutputList('BXLUMIVALUE_ET', 'bxvalue_et')
qHandle.addToOutputList('BXLUMIERROR_ET', 'bxerror_et')
qHandle.addToOutputList('BXLUMIQUALITY_ET', 'bxquality_et')
qHandle.addToOutputList('CMSBXINDEXBLOB', 'bxindexblob')
qHandle.addToOutputList('BEAMINTENSITYBLOB_1', 'beam1intensity')
qHandle.addToOutputList('BEAMINTENSITYBLOB_2', 'beam2intensity')
qConditionStr = 'DATA_ID=:dataid'
qCondition = coral.AttributeList()
qCondition.extend('dataid', 'unsigned long long')
print('oldlumidataid ', oldlumidataid)
qCondition['dataid'].setData(int(oldlumidataid))
qResult = coral.AttributeList()
qResult.extend('lumilsnum', 'unsigned int')
qResult.extend('beamstatus', 'string')
qResult.extend('beamenergy', 'float')
qResult.extend('instlumierror', 'float')
qResult.extend('instlumiquality', 'short')
qResult.extend('bxvalue_occ1', 'blob')
qResult.extend('bxerror_occ1', 'blob')
qResult.extend('bxquality_occ1', 'blob')
qResult.extend('bxvalue_occ2', 'blob')
qResult.extend('bxerror_occ2', 'blob')
qResult.extend('bxquality_occ2', 'blob')
qResult.extend('bxvalue_et', 'blob')
qResult.extend('bxerror_et', 'blob')
qResult.extend('bxquality_et', 'blob')
qResult.extend('bxindexblob', 'blob')
qResult.extend('beam1intensity', 'blob')
qResult.extend('beam2intensity', 'blob')
qHandle.defineOutput(qResult)
qHandle.setCondition(qConditionStr, qCondition)
cursor = qHandle.execute()
while next(cursor):
lumilsnum = cursor.currentRow()['lumilsnum'].data()
beamstatus = cursor.currentRow()['beamstatus'].data()
beamenergy = cursor.currentRow()['beamenergy'].data()
instlumierror = cursor.currentRow()['instlumierror'].data()
instlumiquality = cursor.currentRow()['instlumiquality'].data()
bxvalueblob_occ1 = cursor.currentRow()['bxvalue_occ1'].data()
bxvalueblob_occ1Array = CommonUtil.unpackBlobtoArray(
bxvalueblob_occ1, 'f')
bxerrblob_occ1 = cursor.currentRow()['bxerror_occ1'].data()
bxqualityblob_occ1 = cursor.currentRow()['bxquality_occ1'].data()
bxvalueblob_occ2 = cursor.currentRow()['bxvalue_occ2'].data()
bxerrblob_occ2 = cursor.currentRow()['bxerror_occ2'].data()
bxqualityblob_occ2 = cursor.currentRow()['bxquality_occ2'].data()
bxvalueblob_et = cursor.currentRow()['bxvalue_et'].data()
bxerrblob_et = cursor.currentRow()['bxerror_et'].data()
bxqualityblob_et = cursor.currentRow()['bxquality_et'].data()
bxindexblob = cursor.currentRow()['bxindexblob'].data()
beam1intensity = cursor.currentRow()['beam1intensity'].data()
beam2intensity = cursor.currentRow()['beam2intensity'].data()
result[lumilsnum] = [
bxvalueblob_occ1, bxerrblob_occ1, bxqualityblob_occ1,
bxvalueblob_occ2, bxerrblob_occ2, bxqualityblob_occ2,
bxvalueblob_et, bxerrblob_et, bxqualityblob_et, bxindexblob,
beam1intensity, beam2intensity, beamstatus, beamenergy,
instlumierror, instlumiquality
]
except:
del qHandle
raise
del qHandle
return result
def generateLumidata(lumirundatafromfile, lsdatafromfile, rundatafromdb,
lsdatafromdb, replacelsMin, replacelsMax):
'''
input:
perrunresultfromfile=[]#source,starttime,stoptime,nls
perlsresultfromfile={} #{lumilsnum:instlumiub}
lumirundatafromdb=[] #[source,nominalegev,ncollidingbunches,starttime,stoptime,nls]
lumilsdatafromdb={}#{lumilsnum:[cmslsnum(0),instlumi(1),instlumierror(2),instlumiquality(3),beamstatus(4),beamenergy(5),numorbit(6),startorbit(7),cmsbxindexblob(8),beamintensityblob_1(9),beamintensityblob_2(10),bxlumivalue_occ1(11),bxlumierror_occ1(12),bxlumiquality_occ1(13),bxlumivalue_occ2(14),bxlumierror_occ2(15),bxlumiquality_occ2(16),bxlumivalue_et(17),bxlumierror_et(18),bxlumiquality_et(19)]}
'''
lumip = lumiParameters.ParametersObject()
numorbit = lumip.numorbit
startorbit = 0
fakebeamenergy = 4000.
fakebeamstatus = 'STABLE BEAMS'
fakefloatArray = array.array('f')
fakeidxArray = array.array('h')
fakeshortArray = array.array('h')
for bxidx in range(1, 3565):
fakeidxArray.append(bxidx)
fakefloatArray.append(0.)
fakeshortArray.append(0)
lumirundata = []
lumilsdata = {}
if rundatafromdb:
lumirundata = rundatafromdb
lumirundata[0] = rundatafromdb[0] + '+file:' + lumirundatafromfile[0]
else:
lu = lumiTime.lumiTime()
source = '+file:' + lumirundatafromfile[0]
nominalegev = fakebeamenergy
ncollidingbunches = 72
starttime = lumirundatafromfile[1]
stoptime = lumirundatafromfile[2]
starttimeT = lu.timestampTodatetimeUTC(starttime)
stoptimeT = lu.timestampTodatetimeUTC(stoptime)
print(starttimeT.day, starttimeT.month, starttimeT.year)
starttimeT_coral = coral.TimeStamp(starttimeT.year, starttimeT.month,
starttimeT.day, starttimeT.hour,
starttimeT.minute,
starttimeT.second, 0)
stoptimeT_coral = coral.TimeStamp(stoptimeT.year, stoptimeT.month,
stoptimeT.day, stoptimeT.hour,
stoptimeT.minute, stoptimeT.second,
0)
nls = lumirundatafromfile[3]
lumirundata = [
source, nominalegev, ncollidingbunches, starttimeT_coral,
stoptimeT_coral, nls
]
if lsdatafromdb:
lumilsdata = lsdatafromdb
if replacelsMin > len(lsdatafromdb):
print('[INFO]Operation: extend an existing run from LS=',
replacelsMin)
lumirundata[5] += len(lsdatafromfile)
else:
print(
'[INFO]Operation: replace instlumi in an existing run LS range=',
replacelsMin, replacelsMax)
else:
print('[INFO]Operation: insert a new fake run')
for lumilsnum in range(replacelsMin, replacelsMax + 1):
instlumi = lsdatafromfile[lumilsnum]
if lumilsnum in lsdatafromdb.keys(): #if this is a hole
lumilsdata[lumilsnum][1] = instlumi
else: #if this is an extension
instlumierror = 0.0
instlumiquality = 0
startorbit = (lumilsnum - 1) * numorbit
cmsbxindexblob = CommonUtil.packArraytoBlob(fakeshortArray)
beamintensityblob_1 = CommonUtil.packArraytoBlob(fakefloatArray)
beamintensityblob_2 = CommonUtil.packArraytoBlob(fakefloatArray)
bxlumivalue_occ1 = CommonUtil.packArraytoBlob(fakefloatArray)
bxlumierror_occ1 = CommonUtil.packArraytoBlob(fakefloatArray)
bxlumiquality_occ1 = CommonUtil.packArraytoBlob(fakeshortArray)
bxlumivalue_occ2 = CommonUtil.packArraytoBlob(fakefloatArray)
bxlumierror_occ2 = CommonUtil.packArraytoBlob(fakefloatArray)
bxlumiquality_occ2 = CommonUtil.packArraytoBlob(fakeshortArray)
bxlumivalue_et = CommonUtil.packArraytoBlob(fakefloatArray)
bxlumierror_et = CommonUtil.packArraytoBlob(fakefloatArray)
bxlumiquality_et = CommonUtil.packArraytoBlob(fakeshortArray)
lumilsdata[lumilsnum] = [
0, instlumi, instlumierror, instlumiquality, fakebeamstatus,
fakebeamenergy, numorbit, startorbit, cmsbxindexblob,
beamintensityblob_1, beamintensityblob_2, bxlumivalue_occ1,
bxlumierror_occ1, bxlumiquality_occ1, bxlumivalue_occ2,
bxlumierror_occ2, bxlumiquality_occ2, bxlumivalue_et,
bxlumierror_et, bxlumiquality_et
]
return (lumirundata, lumilsdata)
def main():
allowedscales=['linear','log','both']
c=constants()
parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description="Plot integrated luminosity as function of the time variable of choice",formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# add required arguments
parser.add_argument('-c',dest='connect',action='store',required=True,help='connect string to lumiDB')
# add optional arguments
parser.add_argument('-P',dest='authpath',action='store',help='path to authentication file')
parser.add_argument('-n',dest='normfactor',action='store',help='normalization factor (optional, default to 1.0)')
parser.add_argument('-i',dest='inputfile',action='store',help='lumi range selection file (optional)')
parser.add_argument('-o',dest='outputfile',action='store',help='csv outputfile name (optional)')
parser.add_argument('-lumiversion',dest='lumiversion',default='0001',action='store',required=False,help='lumi data version')
parser.add_argument('-begin',dest='begin',action='store',help='begin value of x-axi (required)')
parser.add_argument('-end',dest='end',action='store',help='end value of x-axi (optional). Default to the maximum exists DB')
parser.add_argument('-beamenergy',dest='beamenergy',action='store',type=float,required=False,help='beamenergy (in GeV) selection criteria,e.g. 3.5e3')
parser.add_argument('-beamfluctuation',dest='beamfluctuation',action='store',type=float,required=False,help='allowed fraction of beamenergy to fluctuate, e.g. 0.1')
parser.add_argument('-beamstatus',dest='beamstatus',action='store',required=False,help='selection criteria beam status,e.g. STABLE BEAMS')
parser.add_argument('-yscale',dest='yscale',action='store',required=False,default='linear',help='y_scale')
parser.add_argument('-hltpath',dest='hltpath',action='store',help='specific hltpath to calculate the recorded luminosity. If specified aoverlays the recorded luminosity for the hltpath on the plot')
parser.add_argument('-batch',dest='batch',action='store',help='graphical mode to produce PNG file. Specify graphical file here. Default to lumiSum.png')
parser.add_argument('--annotateboundary',dest='annotateboundary',action='store_true',help='annotate boundary run numbers')
parser.add_argument('--interactive',dest='interactive',action='store_true',help='graphical mode to draw plot in a TK pannel.')
parser.add_argument('-timeformat',dest='timeformat',action='store',help='specific python timeformat string (optional). Default mm/dd/yy hh:min:ss.00')
parser.add_argument('-siteconfpath',dest='siteconfpath',action='store',help='specific path to site-local-config.xml file, default to $CMS_PATH/SITECONF/local/JobConfig, if path undefined, fallback to cern proxy&server')
parser.add_argument('action',choices=['run','fill','time','perday'],help='x-axis data type of choice')
#graphical mode options
parser.add_argument('--with-correction',dest='withFineCorrection',action='store_true',help='with fine correction')
parser.add_argument('--verbose',dest='verbose',action='store_true',help='verbose mode, print result also to screen')
parser.add_argument('--debug',dest='debug',action='store_true',help='debug')
# parse arguments
batchmode=True
args=parser.parse_args()
connectstring=args.connect
begvalue=args.begin
endvalue=args.end
beamstatus=args.beamstatus
beamenergy=args.beamenergy
beamfluctuation=args.beamfluctuation
xaxitype='run'
connectparser=connectstrParser.connectstrParser(connectstring)
connectparser.parse()
usedefaultfrontierconfig=False
cacheconfigpath=''
if connectparser.needsitelocalinfo():
if not args.siteconfpath:
cacheconfigpath=os.environ['CMS_PATH']
if cacheconfigpath:
cacheconfigpath=os.path.join(cacheconfigpath,'SITECONF','local','JobConfig','site-local-config.xml')
else:
usedefaultfrontierconfig=True
else:
cacheconfigpath=args.siteconfpath
cacheconfigpath=os.path.join(cacheconfigpath,'site-local-config.xml')
p=cacheconfigParser.cacheconfigParser()
if usedefaultfrontierconfig:
p.parseString(c.defaultfrontierConfigString)
else:
p.parse(cacheconfigpath)
connectstring=connectparser.fullfrontierStr(connectparser.schemaname(),p.parameterdict())
#print 'connectstring',connectstring
runnumber=0
svc = coral.ConnectionService()
hltpath=''
if args.hltpath:
hltpath=args.hltpath
if args.debug :
msg=coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Debug)
ifilename=''
ofilename='integratedlumi.png'
timeformat=''
if args.authpath and len(args.authpath)!=0:
os.environ['CORAL_AUTH_PATH']=args.authpath
if args.normfactor:
c.NORM=float(args.normfactor)
if args.lumiversion:
c.LUMIVERSION=args.lumiversion
if args.verbose:
c.VERBOSE=True
if args.inputfile:
ifilename=args.inputfile
if args.batch:
opicname=args.batch
if args.outputfile:
ofilename=args.outputfile
if args.timeformat:
timeformat=args.timeformat
session=svc.connect(connectstring,accessMode=coral.access_Update)
session.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)")
session.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)")
inputfilecontent=''
fileparsingResult=''
runList=[]
runDict={}
fillDict={}
selectionDict={}
minTime=''
maxTime=''
startRunTime=''
stopRunTime=''
#if len(ifilename)!=0 :
# f=open(ifilename,'r')
# inputfilecontent=f.read()
# sparser=selectionParser.selectionParser(inputfilecontent)
# runsandls=sparser.runsandls()
# keylist=runsandls.keys()
# keylist.sort()
# for run in keylist:
# if selectionDict.has_key(run):
# lslist=runsandls[run]
# lslist.sort()
# selectionDict[run]=lslist
if len(ifilename)!=0:
ifparser=inputFilesetParser.inputFilesetParser(ifilename)
runsandls=ifparser.runsandls()
keylist=runsandls.keys()
keylist.sort()
for run in keylist:
if not selectionDict.has_key(run):
lslist=runsandls[run]
lslist.sort()
selectionDict[run]=lslist
if args.action == 'run':
if not args.end:
session.transaction().start(True)
schema=session.nominalSchema()
lastrun=max(lumiQueryAPI.allruns(schema,requireRunsummary=True,requireLumisummary=True,requireTrg=True,requireHlt=False))
session.transaction().commit()
else:
lastrun=int(args.end)
for r in range(int(args.begin),lastrun+1):
runList.append(r)
runList.sort()
elif args.action == 'fill':
session.transaction().start(True)
maxfill=None
if not args.end:
qHandle=session.nominalSchema().newQuery()
maxfill=max(lumiQueryAPI.allfills(qHandle,filtercrazy=True))
del qHandle
else:
maxfill=int(args.end)
qHandle=session.nominalSchema().newQuery()
fillDict=lumiQueryAPI.runsByfillrange(qHandle,int(args.begin),maxfill)
del qHandle
session.transaction().commit()
#print 'fillDict ',fillDict
for fill in range(int(args.begin),maxfill+1):
if fillDict.has_key(fill): #fill exists
for run in fillDict[fill]:
runList.append(run)
runList.sort()
elif args.action == 'time' or args.action == 'perday':
session.transaction().start(True)
t=lumiTime.lumiTime()
minTime=t.StrToDatetime(args.begin,timeformat)
if not args.end:
maxTime=datetime.datetime.utcnow() #to now
else:
maxTime=t.StrToDatetime(args.end,timeformat)
#print minTime,maxTime
qHandle=session.nominalSchema().newQuery()
runDict=lumiQueryAPI.runsByTimerange(qHandle,minTime,maxTime)#xrawdata
session.transaction().commit()
runList=runDict.keys()
del qHandle
runList.sort()
if len(runList)!=0:
runmin=min(runList)
runmax=max(runList)
startRunTime=runDict[runmin][0]
stopRunTime=runDict[runmax][1]
#print 'run list: ',runDict
else:
print 'unsupported action ',args.action
exit
finecorrections=None
if args.withFineCorrection:
schema=session.nominalSchema()
session.transaction().start(True)
finecorrections=lumiCorrections.correctionsForRange(schema,runList)
session.transaction().commit()
#print 'runList ',runList
#print 'runDict ', runDict
fig=Figure(figsize=(7.2,5.4),dpi=120)
m=matplotRender.matplotRender(fig)
logfig=Figure(figsize=(7.5,5.7),dpi=135)
mlog=matplotRender.matplotRender(logfig)
if args.action == 'run':
result={}
result=getLumiInfoForRuns(session,c,runList,selectionDict,hltpath,beamstatus=beamstatus,beamenergy=beamenergy,beamfluctuation=beamfluctuation,finecorrections=finecorrections)
xdata=[]
ydata={}
ydata['Delivered']=[]
ydata['Recorded']=[]
keylist=result.keys()
keylist.sort() #must be sorted in order
if args.outputfile:
reporter=csvReporter.csvReporter(ofilename)
fieldnames=['run','delivered','recorded']
reporter.writeRow(fieldnames)
for run in keylist:
xdata.append(run)
delivered=result[run][0]
recorded=result[run][1]
ydata['Delivered'].append(delivered)
ydata['Recorded'].append(recorded)
if args.outputfile and (delivered!=0 or recorded!=0):
reporter.writeRow([run,result[run][0],result[run][1]])
m.plotSumX_Run(xdata,ydata,yscale='linear')
mlog.plotSumX_Run(xdata,ydata,yscale='log')
elif args.action == 'fill':
lumiDict={}
lumiDict=getLumiInfoForRuns(session,c,runList,selectionDict,hltpath,beamstatus=beamstatus,beamenergy=beamenergy,beamfluctuation=beamfluctuation,finecorrections=finecorrections)
xdata=[]
ydata={}
ydata['Delivered']=[]
ydata['Recorded']=[]
#keylist=lumiDict.keys()
#keylist.sort()
if args.outputfile:
reporter=csvReporter.csvReporter(ofilename)
fieldnames=['fill','run','delivered','recorded']
reporter.writeRow(fieldnames)
fills=fillDict.keys()
fills.sort()
for fill in fills:
runs=fillDict[fill]
runs.sort()
for run in runs:
xdata.append(run)
ydata['Delivered'].append(lumiDict[run][0])
ydata['Recorded'].append(lumiDict[run][1])
if args.outputfile :
reporter.writeRow([fill,run,lumiDict[run][0],lumiDict[run][1]])
#print 'input fillDict ',len(fillDict.keys()),fillDict
m.plotSumX_Fill(xdata,ydata,fillDict,yscale='linear')
mlog.plotSumX_Fill(xdata,ydata,fillDict,yscale='log')
elif args.action == 'time' :
lumiDict={}
lumiDict=getLumiInfoForRuns(session,c,runList,selectionDict,hltpath,beamstatus=beamstatus,beamenergy=beamenergy,beamfluctuation=beamfluctuation,finecorrections=finecorrections)
#lumiDict=getLumiInfoForRuns(session,c,runList,selectionDict,hltpath,beamstatus='STABLE BEAMS')
xdata={}#{run:[starttime,stoptime]}
ydata={}
ydata['Delivered']=[]
ydata['Recorded']=[]
keylist=lumiDict.keys()
keylist.sort()
if args.outputfile:
reporter=csvReporter.csvReporter(ofilename)
fieldnames=['run','starttime','stoptime','delivered','recorded']
reporter.writeRow(fieldnames)
for run in keylist:
ydata['Delivered'].append(lumiDict[run][0])
ydata['Recorded'].append(lumiDict[run][1])
starttime=runDict[run][0]
stoptime=runDict[run][1]
xdata[run]=[starttime,stoptime]
if args.outputfile :
reporter.writeRow([run,starttime,stoptime,lumiDict[run][0],lumiDict[run][1]])
m.plotSumX_Time(xdata,ydata,startRunTime,stopRunTime,hltpath=hltpath,annotateBoundaryRunnum=args.annotateboundary,yscale='linear')
mlog.plotSumX_Time(xdata,ydata,startRunTime,stopRunTime,hltpath=hltpath,annotateBoundaryRunnum=args.annotateboundary,yscale='log')
elif args.action == 'perday':
daydict={}#{day:[[run,cmslsnum,lsstarttime,delivered,recorded]]}
lumibyls=getLumiOrderByLS(session,c,runList,selectionDict,hltpath,beamstatus=beamstatus,beamenergy=beamenergy,beamfluctuation=beamfluctuation,finecorrections=finecorrections)
#lumibyls [[runnumber,runstarttime,lsnum,lsstarttime,delivered,recorded,recordedinpath]]
if args.outputfile:
reporter=csvReporter.csvReporter(ofilename)
fieldnames=['day','begrunls','endrunls','delivered','recorded']
reporter.writeRow(fieldnames)
beginfo=[lumibyls[0][3],str(lumibyls[0][0])+':'+str(lumibyls[0][2])]
endinfo=[lumibyls[-1][3],str(lumibyls[-1][0])+':'+str(lumibyls[-1][2])]
for perlsdata in lumibyls:
lsstarttime=perlsdata[3]
delivered=perlsdata[4]
recorded=perlsdata[5]
day=lsstarttime.toordinal()
if not daydict.has_key(day):
daydict[day]=[]
daydict[day].append([delivered,recorded])
days=daydict.keys()
days.sort()
daymin=days[0]
daymax=days[-1]
#alldays=range(daymin,daymax+1)
resultbyday={}
resultbyday['Delivered']=[]
resultbyday['Recorded']=[]
#for day in days:
#print 'day min ',daymin
#print 'day max ',daymax
for day in range(daymin,daymax+1):
if not daydict.has_key(day):
delivered=0.0
recorded=0.0
else:
daydata=daydict[day]
mytransposed=CommonUtil.transposed(daydata,defaultval=0.0)
delivered=sum(mytransposed[0])
recorded=sum(mytransposed[1])
resultbyday['Delivered'].append(delivered)
resultbyday['Recorded'].append(recorded)
if args.outputfile:
reporter.writeRow([day,beginfo[1],endinfo[1],delivered,recorded])
#print 'beginfo ',beginfo
#print 'endinfo ',endinfo
#print resultbyday
m.plotPerdayX_Time( range(daymin,daymax+1) ,resultbyday,startRunTime,stopRunTime,boundaryInfo=[beginfo,endinfo],annotateBoundaryRunnum=args.annotateboundary,yscale='linear')
mlog.plotPerdayX_Time( range(daymin,daymax+1),resultbyday,startRunTime,stopRunTime,boundaryInfo=[beginfo,endinfo],annotateBoundaryRunnum=args.annotateboundary,yscale='log')
else:
raise Exception,'must specify the type of x-axi'
del session
del svc
if args.batch and args.yscale=='linear':
m.drawPNG(args.batch)
elif args.batch and args.yscale=='log':
mlog.drawPNG(args.batch)
elif args.batch and args.yscale=='both':
m.drawPNG(args.batch)
basename,extension=os.path.splitext(args.batch)
logfilename=basename+'_log'+extension
mlog.drawPNG(logfilename)
else:
if not args.interactive:
return
if args.interactive is True and args.yscale=='linear':
m.drawInteractive()
elif args.interactive is True and args.yscale=='log':
mlog.drawInteractive()
else:
raise Exception('unsupported yscale for interactive mode : '+args.yscale)
runlist=[]
irunlsdict={}
if options.runnumber: # if runnumber specified, do not go through other run selection criteria
irunlsdict[options.runnumber]=None
runlist=irunlsdict.keys()
else:
if options.inputfile:
(irunlsdict,iresults)=parseInputFiles(options.inputfile)
runlist=irunlsdict.keys()
session.transaction().start(True)
(hf_tagid,hf_tagname)=revisionDML.currentDataTag(session.nominalSchema(),lumitype='HF')
hfdataidmap=revisionDML.dataIdsByTagId(session.nominalSchema(),hf_tagid,runlist,withcomment=False,lumitype='HF')
lumirundata=dataDML.lumiRunByIds(session.nominalSchema(),hfdataidmap,lumitype='HF')
#{runnum: (datasource(1),nominalegev(2),ncollidingbunches(3)}
GrunsummaryData=lumiCalcAPI.runsummaryMap(session.nominalSchema(),irunlsdict)
session.transaction().commit()
for runnum in lumirundata.keys():
session.transaction().start(True)
hfdataidinfo=hfdataidmap[runnum]
hflumidata=lumiCalcAPI.instLumiForIds(session.nominalSchema(),irunlsdict,hfdataidmap,GrunsummaryData,beamstatusfilter=None,withBXInfo=True,bxAlgo='OCC1')
for perlsdata in hflumidata[runnum]:
lumilsnum=perlsdata[0]
cmslsnum=perlsdata[1]
bxinfo=perlsdata[9]
bxdistro=getdistribution(cmslsnum,bxinfo[0],bxinfo[1])
print(lumilsnum,cmslsnum,CommonUtil.flatten(bxdistro))
session.transaction().commit()
del session
del svc
def fetchOldData(schema,oldlumidataid):
'''
fetch old perbunch data if the run exists
select CMSBXINDEXBLOB,BEAMINTENSITYBLOB_1,BEAMINTENSITYBLOB_2,BXLUMIVALUE_OCC1,BXLUMIVALUE_OCC2,BXLUMIVALUE_ET,BXLUMIERROR_OCC1,BXLUMIERROR_OCC2,BXLUMIERROR_ET,BXLUMIQUALITY_OCC1,BXLUMIQUALITY_OCC2,BXLUMIQUALITY_ET from lumisummaryv2 where data_id=:oldlumidataid;
output:
{lumilsnum:[bxvalueblob_occ1(0),bxerrblob_occ1(1),bxqualityblob_occ1(2),bxvalueblob_occ2(3),bxerrblob_occ2(4),bxqualityblob_occ2(5),bxvalueblob_et(6),bxerrblob_et(7),bxqualityblob_et(8),bxindexblob(9),beam1intensity(10),beam2intensity(11),beamstatus(12),beamenergy(13),instlumierror(14),instlumiquality(15)],...}
'''
result={}
qHandle=schema.newQuery()
try:
qHandle.addToTableList('LUMISUMMARYV2')
qHandle.addToOutputList('LUMILSNUM','lumilsnum')
qHandle.addToOutputList('BEAMSTATUS','beamstatus')
qHandle.addToOutputList('BEAMENERGY','beamenergy')
qHandle.addToOutputList('INSTLUMIERROR','instlumierror')
qHandle.addToOutputList('INSTLUMIQUALITY','instlumiquality')
qHandle.addToOutputList('BXLUMIVALUE_OCC1','bxvalue_occ1')
qHandle.addToOutputList('BXLUMIERROR_OCC1','bxerror_occ1')
qHandle.addToOutputList('BXLUMIQUALITY_OCC1','bxquality_occ1')
qHandle.addToOutputList('BXLUMIVALUE_OCC2','bxvalue_occ2')
qHandle.addToOutputList('BXLUMIERROR_OCC2','bxerror_occ2')
qHandle.addToOutputList('BXLUMIQUALITY_OCC2','bxquality_occ2')
qHandle.addToOutputList('BXLUMIVALUE_ET','bxvalue_et')
qHandle.addToOutputList('BXLUMIERROR_ET','bxerror_et')
qHandle.addToOutputList('BXLUMIQUALITY_ET','bxquality_et')
qHandle.addToOutputList('CMSBXINDEXBLOB','bxindexblob')
qHandle.addToOutputList('BEAMINTENSITYBLOB_1','beam1intensity')
qHandle.addToOutputList('BEAMINTENSITYBLOB_2','beam2intensity')
qConditionStr='DATA_ID=:dataid'
qCondition=coral.AttributeList()
qCondition.extend('dataid','unsigned long long')
print('oldlumidataid ',oldlumidataid)
qCondition['dataid'].setData(int(oldlumidataid))
qResult=coral.AttributeList()
qResult.extend('lumilsnum','unsigned int')
qResult.extend('beamstatus','string')
qResult.extend('beamenergy','float')
qResult.extend('instlumierror','float')
qResult.extend('instlumiquality','short')
qResult.extend('bxvalue_occ1','blob')
qResult.extend('bxerror_occ1','blob')
qResult.extend('bxquality_occ1','blob')
qResult.extend('bxvalue_occ2','blob')
qResult.extend('bxerror_occ2','blob')
qResult.extend('bxquality_occ2','blob')
qResult.extend('bxvalue_et','blob')
qResult.extend('bxerror_et','blob')
qResult.extend('bxquality_et','blob')
qResult.extend('bxindexblob','blob')
qResult.extend('beam1intensity','blob')
qResult.extend('beam2intensity','blob')
qHandle.defineOutput(qResult)
qHandle.setCondition(qConditionStr,qCondition)
cursor=qHandle.execute()
while next(cursor):
lumilsnum=cursor.currentRow()['lumilsnum'].data()
beamstatus=cursor.currentRow()['beamstatus'].data()
beamenergy=cursor.currentRow()['beamenergy'].data()
instlumierror=cursor.currentRow()['instlumierror'].data()
instlumiquality=cursor.currentRow()['instlumiquality'].data()
bxvalueblob_occ1=cursor.currentRow()['bxvalue_occ1'].data()
bxvalueblob_occ1Array=CommonUtil.unpackBlobtoArray(bxvalueblob_occ1,'f')
bxerrblob_occ1=cursor.currentRow()['bxerror_occ1'].data()
bxqualityblob_occ1=cursor.currentRow()['bxquality_occ1'].data()
bxvalueblob_occ2=cursor.currentRow()['bxvalue_occ2'].data()
bxerrblob_occ2=cursor.currentRow()['bxerror_occ2'].data()
bxqualityblob_occ2=cursor.currentRow()['bxquality_occ2'].data()
bxvalueblob_et=cursor.currentRow()['bxvalue_et'].data()
bxerrblob_et=cursor.currentRow()['bxerror_et'].data()
bxqualityblob_et=cursor.currentRow()['bxquality_et'].data()
bxindexblob=cursor.currentRow()['bxindexblob'].data()
beam1intensity=cursor.currentRow()['beam1intensity'].data()
beam2intensity=cursor.currentRow()['beam2intensity'].data()
result[lumilsnum]=[bxvalueblob_occ1,bxerrblob_occ1,bxqualityblob_occ1,bxvalueblob_occ2,bxerrblob_occ2,bxqualityblob_occ2,bxvalueblob_et,bxerrblob_et,bxqualityblob_et,bxindexblob,beam1intensity,beam2intensity,beamstatus,beamenergy,instlumierror,instlumiquality]
except :
del qHandle
raise
del qHandle
return result
def lumidb(run_lumi_dict, action='delivered', lumi_report=False):
"Call lumidb to get luminosity numbers"
actions = ['overview', 'delivered', 'recorded', 'lumibyls', 'lumibylsXing']
if action not in actions:
raise Exception('Unsupported action="%s", please check from %s' % (action, actions))
beamModeChoices = ['stable']
amodetagChoices = ['PROTPHYS', 'IONPHYS', 'PAPHYS' ]
xingAlgoChoices = ['OCC1', 'OCC2', 'ET']
#
# parse arguments
#
connect='frontier://LumiCalc/CMS_LUMI_PROD'
authpath = None
scalefactor = 1.0
beamfluctuation = 0.2
minbiasxsec = 69300.0 # minbias cross-secvtion in ub
xingMinLum = 1e-03 # Minimum perbunch luminosity to print, default=1e-03/ub
xingAlgo = 'OCC1'
hltpath = None
outputfile = None
siteconfpath = None
withoutNorm = False
withoutCheckforupdate = False
fillnum = None
nowarning = True
debug = False
beamenergy = None
amodetag = None
reqrunmin=None
reqfillmin=None
reqtimemin=None
reqrunmax=None
reqfillmax=None
reqtimemax=None
timeFilter=[None,None]
pbeammode = None
iresults=[]
reqTrg=False
reqHlt=False
svc = sessionManager.sessionManager(connect, authpath, siteconfpath, debugON=debug)
session=svc.openSession(isReadOnly=True,
cpp2sqltype=[('unsigned int', 'NUMBER(10)'), ('unsigned long long', 'NUMBER(20)')])
##############################################################
# check run/ls list
##############################################################
irunlsdict={}
rruns=[]
session.transaction().start(True)
irunlsdict = run_lumi_dict
iresults = []
runlist=lumiCalcAPI.runList(session.nominalSchema(),
fillnum, runmin=reqrunmin, runmax=reqrunmax,
fillmin=reqfillmin, fillmax=reqfillmax,
startT=reqtimemin, stopT=reqtimemax,
l1keyPattern=None, hltkeyPattern=None,
amodetag=amodetag, nominalEnergy=beamenergy,
energyFlut=beamfluctuation,
requiretrg=False, requirehlt=False)
rruns = [val for val in runlist if val in irunlsdict.keys()]
for selectedrun in irunlsdict.keys():#if there's further filter on the runlist,clean input dict
if selectedrun not in rruns:
del irunlsdict[selectedrun]
##############################################################
# check datatag
# #############################################################
datatagid, datatagname = revisionDML.currentDataTag(session.nominalSchema())
dataidmap=revisionDML.dataIdsByTagId(\
session.nominalSchema(), datatagid,
runlist=rruns, withcomment=False)
session.transaction().commit()
if not dataidmap:
print_warning('No data found in LumiDB for given set run lumi section')
return 0, '/ub' # return lumi, units
normvalueDict={}
##################
# ls level #
##################
session.transaction().start(True)
GrunsummaryData=lumiCalcAPI.runsummaryMap(session.nominalSchema(),irunlsdict)
if action == 'delivered':
result=lumiCalcAPI.deliveredLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,lumitype='HF')
if lumi_report:
lumiReport.toScreenTotDelivered(result,iresults,scalefactor,irunlsdict=irunlsdict,noWarning=nowarning,toFile=outputfile)
# result {run:[lumilsnum(0),cmslsnum(1),timestamp(2),beamstatus(3),beamenergy(4),deliveredlumi(5),calibratedlumierr(6),(bxidxlist,bxvalues,bxerrs)(7),(bxidx,b1intensities,b2intensities)(8),fillnum(9),pu(10)]}
totlumi = 0
for run, val in result.items():
for llist in val:
if irunlsdict[run]:
if llist[0] in irunlsdict[run]: # select only lumis from input run ls dict
totlumi += llist[5]
else:
totlumi += llist[5]
totlumival, lumiunit = CommonUtil.guessUnit(totlumi)
return totlumival, lumiunit
if action == 'overview':
result=lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,lumitype='HF')
lumiReport.toScreenOverview(result,iresults,scalefactor,irunlsdict=irunlsdict,noWarning=nowarning,toFile=outputfile)
if action == 'lumibyls':
if not hltpath:
result=lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,lumitype='HF',minbiasXsec=minbiasxsec)
lumiReport.toScreenLumiByLS(result,iresults,scalefactor,irunlsdict=irunlsdict,noWarning=nowarning,toFile=outputfile)
else:
hltname=hltpath
hltpat=None
if hltname=='*' or hltname=='all':
hltname=None
elif 1 in [c in hltname for c in '*?[]']: #is a fnmatch pattern
hltpat=hltname
hltname=None
result=lumiCalcAPI.effectiveLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,hltpathname=hltname,hltpathpattern=hltpat,withBXInfo=False,bxAlgo=None,xingMinLum=xingMinLum,withBeamIntensity=False,lumitype='HF')
lumiReport.toScreenLSEffective(result,iresults,scalefactor,irunlsdict=irunlsdict,noWarning=nowarning,toFile=outputfile)
if action == 'recorded':#recorded actually means effective because it needs to show all the hltpaths...
hltname=hltpath
hltpat=None
if hltname is not None:
if hltname=='*' or hltname=='all':
hltname=None
elif 1 in [c in hltname for c in '*?[]']: #is a fnmatch pattern
hltpat=hltname
hltname=None
result=lumiCalcAPI.effectiveLumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,hltpathname=hltname,hltpathpattern=hltpat,withBXInfo=False,bxAlgo=None,xingMinLum=xingMinLum,withBeamIntensity=False,lumitype='HF')
lumiReport.toScreenTotEffective(result,iresults,scalefactor,irunlsdict=irunlsdict,noWarning=nowarning,toFile=outputfile)
if action == 'lumibylsXing':
result=lumiCalcAPI.lumiForIds(session.nominalSchema(),irunlsdict,dataidmap,runsummaryMap=GrunsummaryData,beamstatusfilter=pbeammode,timeFilter=timeFilter,normmap=normvalueDict,withBXInfo=True,bxAlgo=xingAlgo,xingMinLum=xingMinLum,withBeamIntensity=False,lumitype='HF')
outfile=outputfile
if not outfile:
print '[WARNING] no output file given. lumibylsXing writes per-bunch lumi only to default file lumibylsXing.csv'
outfile='lumibylsXing.csv'
lumiReport.toCSVLumiByLSXing(result,scalefactor,outfile,irunlsdict=irunlsdict,noWarning=nowarning)
session.transaction().commit()
del session
del svc
lumils = sorted(lumiLSdata.keys())
for lsnum in lumils:
print('lumilsnum,cmslsnum,instlumi ', lsnum, lumiLSdata[lsnum][0],
lumiLSdata[lsnum][1] * normval)
(runnum, trgLSdata) = dataDML.trgLSById(schema, trgdataid)
cmsls = trgLSdata.keys()
cmsls.sort()
for lsnum in cmsls:
print('cmslsnum,deadtime,bizerocount,bitzeroprescale,deadfrac ', lsnum,
trgLSdata[lsnum][0], trgLSdata[lsnum][1], trgLSdata[lsnum][2],
trgLSdata[lsnum][3])
[runnum, datasource, npath,
pathnames] = dataDML.hltRunById(schema, hltdataid)
print('npath,pathnames ', npath, pathnames)
pathnameList = pathnames.split(',')
(runnum, hltLSdata) = dataDML.hltLSById(schema, hltdataid)
cmsls = hltLSdata.keys()
cmsls.sort()
for lsnum in cmsls:
prescaleblob = hltLSdata[lsnum][0]
print('lsnum ', lsnum)
if prescaleblob:
hltprescales = CommonUtil.unpackBlobtoArray(prescaleblob, 'h')
for pidx, pathname in enumerate(pathnameList):
print('pathname, hltprescales ', pathname, hltprescales[pidx])
session.transaction().commit()
del session
print('')
def patchV2(dbsession,runnum,inputpathnames,inputdata):
'''
inputpathnames: [pathname,]
inputdata: {cmslsnum:[presc,presc...]}
update the most recent version of lshlt data if some ls exist
if none old ls exists
insert new hlt records
'''
try:
dbsession.transaction().start(True)
oldhltdataid=dataDML.guessHltDataIdByRun(dbsession.nominalSchema(),runnum)
existingrundata=dataDML.hltRunById(dbsession.nominalSchema(),oldhltdataid)
dbsession.transaction().commit()
if not oldhltdataid:#no data at all
dbsession.transaction().start(False)
insertV2(dbsession,runnum,inputpathnames,inputdata)
dbsession.transaction().commit()
return
hltnamedict=existingrundata[3]#[(pathidx,hltname),(pathidx,hltname)...]
dbsession.transaction().start(True)
existinglsdata=dataDML.hltLSById(dbsession.nominalSchema(),oldhltdataid)
dbsession.transaction().commit()
oldlsdata=existinglsdata[1]
existinglslist=oldlsdata.keys()
toupdate={}#{cmslsnum:[presc,presc...]}
toinsert={}#{cmslsnum:[presc,presc...]}
if existinglslist and len(existinglslist)!=0:#there are some existing data
for cmslsnum,oldlscontent in oldlsdata.items():
if cmslsnum in inputdata.keys(): # if overlap with new data, update old data with new
toupdate[cmslsnum]=inputdata[cmslsnum]
for cmslsnum,lshltcontent in inputdata.items():
if cmslsnum in toupdate: continue #it's to update not to insert
toinsert[cmslsnum]=inputdata[cmslsnum]
#
# insert into lshlt(data_id,runnum,cmslsnum,prescaleblob,hltcountblob,hltacceptblob) values()
#
dbsession.transaction().start(False)
tabrowDefDict={'DATA_ID':'unsigned long long','RUNNUM':'unsigned int','CMSLSNUM':'unsigned int','PRESCALEBLOB':'blob','HLTCOUNTBLOB':'blob','HLTACCEPTBLOB':'blob'}
for cmslsnum,perlsdata in toinsert.items():
prescaleArray=array.array('I')
hltcountArray=array.array('I')
hltacceptArray=array.array('I')
for (pathidx,hltname) in hltnamedict:
thispathIdx=inputpathnames.index(hltname)
thispresc=perlsdata[thispathIdx]
thiscount=0
thisaccept=0
prescaleArray.append(thispresc)
hltcountArray.append(thiscount)
hltacceptArray.append(thisaccept)
prescaleblob=CommonUtil.packArraytoBlob(prescaleArray)
hltcountblob=CommonUtil.packArraytoBlob(hltcountArray)
hltacceptblob=CommonUtil.packArraytoBlob(hltacceptArray)
tabrowValueDict={'DATA_ID':oldhltdataid,'RUNNUM':int(runnum),'CMSLSNUM':int(cmslsnum),'PRESCALEBLOB':prescaleblob,'HLTCOUNTBLOB':hltcountblob,'HLTACCEPTBLOB':hltacceptblob}
db=dbUtil.dbUtil(dbsession.nominalSchema())
db.insertOneRow(nameDealer.lshltTableName(),tabrowDefDict,tabrowValueDict)
#
# update lshlt set prescaleblob=:prescaleblob,hltcoutblob=:hltcountblob,hltacceptblob=:hltacceptblob where data_id=:olddata_id and cmslsnum=:cmslsnum;
#
setClause='PRESCALEBLOB=:prescaleblob,HLTCOUNTBLOB=:hltcountblob,HLTACCEPTBLOB=:hltacceptblob'
updateCondition='DATA_ID=:oldhltdataid and CMSLSNUM=:cmslsnum'
for cmslsnum,perlsdata in toupdate.items():
prescaleArray=array.array('I')
hltcountArray=array.array('I')
hltacceptArray=array.array('I')
for (pathidx,hltname) in hltnamedict:
thispathIdx=inputpathnames.index(hltname)
thispresc=perlsdata[thispathIdx]
thiscount=0
thisaccept=0
prescaleArray.append(thispresc)
hltcountArray.append(thiscount)
hltacceptArray.append(thisaccept)
prescaleblob=CommonUtil.packArraytoBlob(prescaleArray)
hltcountblob=CommonUtil.packArraytoBlob(hltcountArray)
hltacceptblob=CommonUtil.packArraytoBlob(hltacceptArray)
iData=coral.AttributeList()
iData.extend('prescaleblob','blob')
iData.extend('hltcountblob','blob')
iData.extend('hltacceptblob','blob')
iData.extend('olddata_id','unsigned int')
iData.extend('cmslsnum','unsigned int')
iData['prescaleblob'].setData(prescaleblob)
iData['hltcountblob'].setData(hltcountblob)
iData['hltacceptblob'].setData(hltacceptblob)
iData['olddata_id'].setData(int(olddata_id))
iData['cmslsnum'].setData(int(cmslsnum))
db=dbUtil.dbUtil(schema)
db.singleUpdate(nameDealer.lshltTableName(),setClause,updateCondition,iData)
dbsession.transaction().commit()
#dbsession.transaction().rollback()
except :
raise
(lumidataid,trgdataid,hltdataid)=dataDML.guessDataIdByRun(schema,myrun)
print 'all dataids ',lumidataid,trgdataid,hltdataid
(runnum,lumiLSdata)=dataDML.lumiLSById(schema,lumidataid)
lumils=lumiLSdata.keys()
lumils.sort()
for lsnum in lumils:
print 'lumilsnum,cmslsnum,instlumi ',lsnum,lumiLSdata[lsnum][0],lumiLSdata[lsnum][1]*normval
(runnum,trgLSdata)=dataDML.trgLSById(schema,trgdataid)
cmsls=trgLSdata.keys()
cmsls.sort()
for lsnum in cmsls:
print 'cmslsnum,deadtime,bizerocount,bitzeroprescale,deadfrac ',lsnum,trgLSdata[lsnum][0],trgLSdata[lsnum][1],trgLSdata[lsnum][2],trgLSdata[lsnum][3]
[runnum,datasource,npath,pathnames]=dataDML.hltRunById(schema,hltdataid)
print 'npath,pathnames ',npath,pathnames
pathnameList=pathnames.split(',')
(runnum,hltLSdata)=dataDML.hltLSById(schema,hltdataid)
cmsls=hltLSdata.keys()
cmsls.sort()
for lsnum in cmsls:
prescaleblob=hltLSdata[lsnum][0]
print 'lsnum ',lsnum
if prescaleblob:
hltprescales=CommonUtil.unpackBlobtoArray(prescaleblob,'h')
for pidx,pathname in enumerate(pathnameList):
print 'pathname, hltprescales ',pathname,hltprescales[pidx]
session.transaction().commit()
del session
print ''
def generateLumidata(lumirundatafromfile,lsdatafromfile,rundatafromdb,lsdatafromdb,replacelsMin,replacelsMax):
'''
input:
perrunresultfromfile=[]#source,starttime,stoptime,nls
perlsresultfromfile={} #{lumilsnum:instlumiub}
lumirundatafromdb=[] #[source,nominalegev,ncollidingbunches,starttime,stoptime,nls]
lumilsdatafromdb={}#{lumilsnum:[cmslsnum(0),instlumi(1),instlumierror(2),instlumiquality(3),beamstatus(4),beamenergy(5),numorbit(6),startorbit(7),cmsbxindexblob(8),beamintensityblob_1(9),beamintensityblob_2(10),bxlumivalue_occ1(11),bxlumierror_occ1(12),bxlumiquality_occ1(13),bxlumivalue_occ2(14),bxlumierror_occ2(15),bxlumiquality_occ2(16),bxlumivalue_et(17),bxlumierror_et(18),bxlumiquality_et(19)]}
'''
lumip=lumiParameters.ParametersObject()
numorbit=lumip.numorbit
startorbit=0
fakebeamenergy=4000.
fakebeamstatus='STABLE BEAMS'
fakefloatArray=array.array('f')
fakeidxArray=array.array('h')
fakeshortArray=array.array('h')
for bxidx in range(1,3565):
fakeidxArray.append(bxidx)
fakefloatArray.append(0.)
fakeshortArray.append(0)
lumirundata=[]
lumilsdata={}
if rundatafromdb:
lumirundata=rundatafromdb
lumirundata[0]=rundatafromdb[0]+'+file:'+lumirundatafromfile[0]
else:
lu=lumiTime.lumiTime()
source='+file:'+lumirundatafromfile[0]
nominalegev=fakebeamenergy
ncollidingbunches=72
starttime=lumirundatafromfile[1]
stoptime=lumirundatafromfile[2]
starttimeT=lu.timestampTodatetimeUTC(starttime)
stoptimeT=lu.timestampTodatetimeUTC(stoptime)
print(starttimeT.day,starttimeT.month,starttimeT.year)
starttimeT_coral=coral.TimeStamp(starttimeT.year,starttimeT.month,starttimeT.day,starttimeT.hour,starttimeT.minute,starttimeT.second,0)
stoptimeT_coral=coral.TimeStamp(stoptimeT.year,stoptimeT.month,stoptimeT.day,stoptimeT.hour,stoptimeT.minute,stoptimeT.second,0)
nls=lumirundatafromfile[3]
lumirundata=[source,nominalegev,ncollidingbunches,starttimeT_coral,stoptimeT_coral,nls]
if lsdatafromdb:
lumilsdata=lsdatafromdb
if replacelsMin>len(lsdatafromdb):
print('[INFO]Operation: extend an existing run from LS=',replacelsMin)
lumirundata[5]+=len(lsdatafromfile)
else:
print('[INFO]Operation: replace instlumi in an existing run LS range=',replacelsMin,replacelsMax)
else:
print('[INFO]Operation: insert a new fake run')
for lumilsnum in range(replacelsMin,replacelsMax+1):
instlumi=lsdatafromfile[lumilsnum]
if lumilsnum in lsdatafromdb.keys(): #if this is a hole
lumilsdata[lumilsnum][1]=instlumi
else: #if this is an extension
instlumierror=0.0
instlumiquality=0
startorbit=(lumilsnum-1)*numorbit
cmsbxindexblob=CommonUtil.packArraytoBlob(fakeshortArray)
beamintensityblob_1=CommonUtil.packArraytoBlob(fakefloatArray)
beamintensityblob_2=CommonUtil.packArraytoBlob(fakefloatArray)
bxlumivalue_occ1=CommonUtil.packArraytoBlob(fakefloatArray)
bxlumierror_occ1=CommonUtil.packArraytoBlob(fakefloatArray)
bxlumiquality_occ1=CommonUtil.packArraytoBlob(fakeshortArray)
bxlumivalue_occ2=CommonUtil.packArraytoBlob(fakefloatArray)
bxlumierror_occ2=CommonUtil.packArraytoBlob(fakefloatArray)
bxlumiquality_occ2=CommonUtil.packArraytoBlob(fakeshortArray)
bxlumivalue_et=CommonUtil.packArraytoBlob(fakefloatArray)
bxlumierror_et=CommonUtil.packArraytoBlob(fakefloatArray)
bxlumiquality_et=CommonUtil.packArraytoBlob(fakeshortArray)
lumilsdata[lumilsnum]=[0,instlumi,instlumierror,instlumiquality,fakebeamstatus,fakebeamenergy,numorbit,startorbit,cmsbxindexblob,beamintensityblob_1,beamintensityblob_2,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]
return (lumirundata,lumilsdata)
def specificlumiTofile(fillnum, filldata, outdir):
#
#input : fillnum
# filldata: {bxidx:[[lstime,beamstatusfrac,lumivalue,lumierror,speclumi,speclumierr]],[]}
#sorted by bxidx, sorted by lstime inside list
#check outdir/fillnum subdir exists; if not, create it; else outdir=outdir/fillnum
#
if not filldata:
print 'empty input data, do nothing for fill ', fillnum
return
timedict = {
} #{lstime:[[stablebeamfrac,lumi,lumierr,speclumi,speclumierr]]}
filloutdir = os.path.join(outdir, str(fillnum))
if not os.path.exists(filloutdir):
os.mkdir(filloutdir)
for cmsbxidx, perbxdata in filldata.items():
lhcbucket = 0
if cmsbxidx != 0:
lhcbucket = (cmsbxidx - 1) * 10 + 1
a = sorted(perbxdata, key=lambda x: x[0])
filename = str(fillnum) + '_lumi_' + str(lhcbucket) + '_CMS.txt'
linedata = []
for perlsdata in a:
ts = int(perlsdata[0])
beamstatusfrac = perlsdata[1]
lumi = perlsdata[2]
lumierror = perlsdata[3]
#beam1intensity=perlsdata[4]
#beam2intensity=perlsdata[5]
speclumi = perlsdata[4]
speclumierror = perlsdata[5]
if lumi > 0:
linedata.append([
ts, beamstatusfrac, lumi, lumierror, speclumi,
speclumierror
])
if not timedict.has_key(ts):
timedict[ts] = []
timedict[ts].append(
[beamstatusfrac, lumi, lumierror, speclumi, speclumierror])
if len(linedata) > 10: #at least 10 good ls
f = open(os.path.join(filloutdir, filename), 'w')
for line in linedata:
print >> f, '%d\t%e\t%e\t%e\t%e\t%e' % (
line[0], line[1], line[2], line[3], line[4], line[5])
f.close()
#print 'writing avg file'
summaryfilename = str(fillnum) + '_lumi_CMS.txt'
f = None
lstimes = timedict.keys()
lstimes.sort()
fillseg = []
lscounter = 0
for lstime in lstimes:
allvalues = timedict[lstime]
transposedvalues = CommonUtil.transposed(allvalues, 0.0)
bstatfrac = transposedvalues[0][
0] #beamstatus does not change with bx position
lumivals = transposedvalues[1]
lumitot = sum(lumivals)
if bstatfrac == 1.0:
fillseg.append([lstime, lumitot])
lumierrs = transposedvalues[2]
lumierrortot = math.sqrt(sum(map(lambda x: x**2, lumierrs)))
specificvals = transposedvalues[3]
specificavg = sum(specificvals) / float(
len(specificvals)) #avg spec lumi
specificerrs = transposedvalues[4]
specifictoterr = math.sqrt(sum(map(lambda x: x**2, specificerrs)))
specificerravg = specifictoterr / float(len(specificvals))
if lscounter == 0:
f = open(os.path.join(filloutdir, summaryfilename), 'w')
lscounter += 1
print >> f, '%d\t%e\t%e\t%e\t%e\t%e' % (lstime, bstatfrac, lumitot,
lumierrortot, specificavg,
specificerravg)
if f is not None:
f.close()
#print 'writing summary file'
fillsummaryfilename = str(fillnum) + '_bxsum_CMS.txt'
f = open(os.path.join(filloutdir, fillsummaryfilename), 'w')
if len(fillseg) == 0:
print >> f, '%s' % ('#no stable beams')
f.close()
return
previoustime = fillseg[0][0]
boundarytime = fillseg[0][0]
#print 'boundary time ',boundarytime
summaryls = {}
summaryls[boundarytime] = []
for [lstime,
lumitot] in fillseg: #fillseg is everything with stable beam flag
if lstime - previoustime > 50.0:
boundarytime = lstime
#print 'found new boundary ',boundarytime
summaryls[boundarytime] = []
# print 'appending ',boundarytime,lstime,lumitot
summaryls[boundarytime].append([lstime, lumitot])
previoustime = lstime
#print summaryls
summarylstimes = summaryls.keys()
summarylstimes.sort()
lumip = lumiParameters.ParametersObject()
for bts in summarylstimes:
startts = bts
tsdatainseg = summaryls[bts]
#print 'tsdatainseg ',tsdatainseg
stopts = tsdatainseg[-1][0]
plu = max(CommonUtil.transposed(tsdatainseg, 0.0)[1])
lui = sum(CommonUtil.transposed(tsdatainseg,
0.0)[1]) * lumip.lslengthsec()
print >> f, '%d\t%d\t%e\t%e' % (startts, stopts, plu, lui)
f.close()
reqTrg=False
reqHlt=False
if options.beammode=='stable':
pbeammode = 'STABLE BEAMS'
if options.action=='trgbyls':
reqTrg=True
if options.action=='hltbyls':
reqHlt=True
if options.runnumber:
reqrunmax=options.runnumber
reqrunmin=options.runnumber
if options.fillnum:
reqfillmin=options.fillnum
reqfillmax=options.fillnum
if options.begin:
(reqrunmin,reqfillmin,reqtimemin)=CommonUtil.parseTime(options.begin)
if reqtimemin:
lute=lumiTime.lumiTime()
reqtimeminT=lute.StrToDatetime(reqtimemin,customfm='%m/%d/%y %H:%M:%S')
timeFilter[0]=reqtimeminT
if options.end:
(reqrunmax,reqfillmax,reqtimemax)=CommonUtil.parseTime(options.end)
if reqtimemax:
lute=lumiTime.lumiTime()
reqtimemaxT=lute.StrToDatetime(reqtimemax,customfm='%m/%d/%y %H:%M:%S')
timeFilter[1]=reqtimemaxT
sname=options.name
isdetail=False
spattern=None
if sname is not None:
isdetail=True
def lumidb(run_lumi_dict, action='delivered', lumi_report=False):
"Call lumidb to get luminosity numbers"
actions = ['overview', 'delivered', 'recorded', 'lumibyls', 'lumibylsXing']
if action not in actions:
raise Exception('Unsupported action="%s", please check from %s' %
(action, actions))
beamModeChoices = ['stable']
amodetagChoices = ['PROTPHYS', 'IONPHYS', 'PAPHYS']
xingAlgoChoices = ['OCC1', 'OCC2', 'ET']
#
# parse arguments
#
connect = 'frontier://LumiCalc/CMS_LUMI_PROD'
authpath = None
scalefactor = 1.0
beamfluctuation = 0.2
minbiasxsec = 69300.0 # minbias cross-secvtion in ub
xingMinLum = 1e-03 # Minimum perbunch luminosity to print, default=1e-03/ub
xingAlgo = 'OCC1'
hltpath = None
outputfile = None
siteconfpath = None
withoutNorm = False
withoutCheckforupdate = False
fillnum = None
nowarning = True
debug = False
beamenergy = None
amodetag = None
reqrunmin = None
reqfillmin = None
reqtimemin = None
reqrunmax = None
reqfillmax = None
reqtimemax = None
timeFilter = [None, None]
pbeammode = None
iresults = []
reqTrg = False
reqHlt = False
svc = sessionManager.sessionManager(connect,
authpath,
siteconfpath,
debugON=debug)
session = svc.openSession(isReadOnly=True,
cpp2sqltype=[('unsigned int', 'NUMBER(10)'),
('unsigned long long', 'NUMBER(20)')
])
##############################################################
# check run/ls list
##############################################################
irunlsdict = {}
rruns = []
session.transaction().start(True)
irunlsdict = run_lumi_dict
iresults = []
runlist = lumiCalcAPI.runList(session.nominalSchema(),
fillnum,
runmin=reqrunmin,
runmax=reqrunmax,
fillmin=reqfillmin,
fillmax=reqfillmax,
startT=reqtimemin,
stopT=reqtimemax,
l1keyPattern=None,
hltkeyPattern=None,
amodetag=amodetag,
nominalEnergy=beamenergy,
energyFlut=beamfluctuation,
requiretrg=False,
requirehlt=False)
rruns = [val for val in runlist if val in irunlsdict.keys()]
for selectedrun in irunlsdict.keys(
): #if there's further filter on the runlist,clean input dict
if selectedrun not in rruns:
del irunlsdict[selectedrun]
##############################################################
# check datatag
# #############################################################
datatagid, datatagname = revisionDML.currentDataTag(
session.nominalSchema())
dataidmap=revisionDML.dataIdsByTagId(\
session.nominalSchema(), datatagid,
runlist=rruns, withcomment=False)
session.transaction().commit()
if not dataidmap:
print_warning('No data found in LumiDB for given set run lumi section')
return 0, '/ub' # return lumi, units
normvalueDict = {}
##################
# ls level #
##################
session.transaction().start(True)
GrunsummaryData = lumiCalcAPI.runsummaryMap(session.nominalSchema(),
irunlsdict)
if action == 'delivered':
result = lumiCalcAPI.deliveredLumiForIds(session.nominalSchema(),
irunlsdict,
dataidmap,
runsummaryMap=GrunsummaryData,
beamstatusfilter=pbeammode,
timeFilter=timeFilter,
normmap=normvalueDict,
lumitype='HF')
if lumi_report:
lumiReport.toScreenTotDelivered(result,
iresults,
scalefactor,
irunlsdict=irunlsdict,
noWarning=nowarning,
toFile=outputfile)
# result {run:[lumilsnum(0),cmslsnum(1),timestamp(2),beamstatus(3),beamenergy(4),deliveredlumi(5),calibratedlumierr(6),(bxidxlist,bxvalues,bxerrs)(7),(bxidx,b1intensities,b2intensities)(8),fillnum(9),pu(10)]}
totlumi = 0
for run, val in result.items():
for llist in val:
if irunlsdict[run]:
if llist[0] in irunlsdict[
run]: # select only lumis from input run ls dict
totlumi += llist[5]
else:
totlumi += llist[5]
totlumival, lumiunit = CommonUtil.guessUnit(totlumi)
return totlumival, lumiunit
if action == 'overview':
result = lumiCalcAPI.lumiForIds(session.nominalSchema(),
irunlsdict,
dataidmap,
runsummaryMap=GrunsummaryData,
beamstatusfilter=pbeammode,
timeFilter=timeFilter,
normmap=normvalueDict,
lumitype='HF')
lumiReport.toScreenOverview(result,
iresults,
scalefactor,
irunlsdict=irunlsdict,
noWarning=nowarning,
toFile=outputfile)
if action == 'lumibyls':
if not hltpath:
result = lumiCalcAPI.lumiForIds(session.nominalSchema(),
irunlsdict,
dataidmap,
runsummaryMap=GrunsummaryData,
beamstatusfilter=pbeammode,
timeFilter=timeFilter,
normmap=normvalueDict,
lumitype='HF',
minbiasXsec=minbiasxsec)
lumiReport.toScreenLumiByLS(result,
iresults,
scalefactor,
irunlsdict=irunlsdict,
noWarning=nowarning,
toFile=outputfile)
else:
hltname = hltpath
hltpat = None
if hltname == '*' or hltname == 'all':
hltname = None
elif 1 in [c in hltname for c in '*?[]']: #is a fnmatch pattern
hltpat = hltname
hltname = None
result = lumiCalcAPI.effectiveLumiForIds(
session.nominalSchema(),
irunlsdict,
dataidmap,
runsummaryMap=GrunsummaryData,
beamstatusfilter=pbeammode,
timeFilter=timeFilter,
normmap=normvalueDict,
hltpathname=hltname,
hltpathpattern=hltpat,
withBXInfo=False,
bxAlgo=None,
xingMinLum=xingMinLum,
withBeamIntensity=False,
lumitype='HF')
lumiReport.toScreenLSEffective(result,
iresults,
scalefactor,
irunlsdict=irunlsdict,
noWarning=nowarning,
toFile=outputfile)
if action == 'recorded': #recorded actually means effective because it needs to show all the hltpaths...
hltname = hltpath
hltpat = None
if hltname is not None:
if hltname == '*' or hltname == 'all':
hltname = None
elif 1 in [c in hltname for c in '*?[]']: #is a fnmatch pattern
hltpat = hltname
hltname = None
result = lumiCalcAPI.effectiveLumiForIds(session.nominalSchema(),
irunlsdict,
dataidmap,
runsummaryMap=GrunsummaryData,
beamstatusfilter=pbeammode,
timeFilter=timeFilter,
normmap=normvalueDict,
hltpathname=hltname,
hltpathpattern=hltpat,
withBXInfo=False,
bxAlgo=None,
xingMinLum=xingMinLum,
withBeamIntensity=False,
lumitype='HF')
lumiReport.toScreenTotEffective(result,
iresults,
scalefactor,
irunlsdict=irunlsdict,
noWarning=nowarning,
toFile=outputfile)
if action == 'lumibylsXing':
result = lumiCalcAPI.lumiForIds(session.nominalSchema(),
irunlsdict,
dataidmap,
runsummaryMap=GrunsummaryData,
beamstatusfilter=pbeammode,
timeFilter=timeFilter,
normmap=normvalueDict,
withBXInfo=True,
bxAlgo=xingAlgo,
xingMinLum=xingMinLum,
withBeamIntensity=False,
lumitype='HF')
outfile = outputfile
if not outfile:
print '[WARNING] no output file given. lumibylsXing writes per-bunch lumi only to default file lumibylsXing.csv'
outfile = 'lumibylsXing.csv'
lumiReport.toCSVLumiByLSXing(result,
scalefactor,
outfile,
irunlsdict=irunlsdict,
noWarning=nowarning)
session.transaction().commit()
del session
del svc
else:
normdata_id=normDML.normIdByName(dbsession.nominalSchema(),normname)
for normvalueDict in normvalues:
if 'corrector' not in normvalueDict or not normvalueDict['corrector']:
raise RuntimeError('parameter corrector is required for create/insert action')
if 'since' not in normvalueDict or not normvalueDict['since']:
raise RuntimeError('parameter since is required for create/insert action')
correctorStr=normvalueDict['corrector']
sincerun=int(normvalueDict['since'])
if options.firstsince:
if sincerun<int(options.firstsince):
continue
amodetag=normvalueDict['amodetag']
egev=int(normvalueDict['egev'])
detailcomment=normvalueDict['comment']
(correctorname,parameterlist)=CommonUtil.parselumicorrector(correctorStr)
parameterDict={}
for param in parameterlist:
parameterDict[param]=normvalueDict[param]
normDML.insertValueToNormId(dbsession.nominalSchema(),normdata_id,sincerun,correctorStr,amodetag,egev,parameterDict,comment=detailcomment)
dbsession.transaction().commit()
##############################
# setdefault/unsetdefault
##############################
if options.action in ['setdefault','unsetdefault']:
dbsession=svc.openSession(isReadOnly=False,cpp2sqltype=[('unsigned int','NUMBER(10)'),('unsigned long long','NUMBER(20)')])
dbsession.transaction().start(False)
if options.action=='setdefault':
normDML.promoteNormToTypeDefault(dbsession.nominalSchema(),options.normname,options.lumitype)
if options.action=='unsetdefault':
def insertLumischemaV2(dbsession,runnum,datasource,perlsrawdata,perbunchrawdata):
'''
input:
lumirundata[datasource]
perlsrawdata: {cmslsnum:instlumi}
perbunchrawdata: {bxidx:lumifraction}
lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]}
'''
branchrevision_id=3#databranch_id
branchinfo=(branchrevision_id,'DATA')
lumirundata=[datasource]
lumilsdata={}
for cmslsnum,instlumi in perlsrawdata.items():
mystartorbit=startorbit+numorbit*(cmslsnum-1)
bxdataArray=array.array('f')
bxerrorArray=array.array('f')
bxqualityArray=array.array('h')
cmsbxindexArray=array.array('h')
beam1intensityArray=array.array('f')
beam2intensityArray=array.array('f')
for bxidx in range(1,3565):
lumifraction=0.0
if bxidx in perbunchrawdata:
lumifraction=perbunchrawdata[bxidx]
bxlumivalue=float(instlumi*lumifraction)/float(bunchnorm)
bxdataArray.append(bxlumivalue)
beam1intensityArray.append(9124580336.0)
beam1intensityArray.append(8932813306.0)
cmsbxindexArray.append(bxidx)
bxqualityArray.append(1)
bxerrorArray.append(0.0)
bxdataocc1blob=CommonUtil.packArraytoBlob(bxdataArray)
bxdataocc2blob=CommonUtil.packArraytoBlob(bxdataArray)
bxdataetblob=CommonUtil.packArraytoBlob(bxdataArray)
bxerrorocc1blob=CommonUtil.packArraytoBlob(bxerrorArray)
bxerrorocc2blob=CommonUtil.packArraytoBlob(bxerrorArray)
bxerroretblob=CommonUtil.packArraytoBlob(bxerrorArray)
bxqualityocc1blob=CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityocc2blob=CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityetblob=CommonUtil.packArraytoBlob(bxqualityArray)
cmsbxindexblob=CommonUtil.packArraytoBlob(cmsbxindexArray)
beam1intensityblob=CommonUtil.packArraytoBlob(beam1intensityArray)
beam2intensityblob=CommonUtil.packArraytoBlob(beam2intensityArray)
perlsdata=[cmslsnum,float(instlumi)/float(6370),0.0,1,'STABLE BEAMS',beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob]
lumilsdata[cmslsnum]=perlsdata
print(lumilsdata)
dbsession.transaction().start(False)
(revision_id,entry_id,data_id)=dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(),runnum,lumirundata,branchinfo)
dataDML.bulkInsertLumiLSSummary(dbsession,runnum,data_id,lumilsdata)
dbsession.transaction().commit()
def insertLumischemaV2(dbsession,runnum,datasource,perlsrawdata,perbunchrawdata,deliveredonly=False):
'''
input:
lumirundata[datasource]
perlsrawdata: {cmslsnum:instlumi}
perbunchrawdata: {bxidx:lumifraction}
lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]}
'''
branchrevision_id=3#databranch_id
branchinfo=(branchrevision_id,'DATA')
lumirundata=[datasource]
lumilsdata={}
dbsession.transaction().start(True)
oldlumidataid=dataDML.guessLumiDataIdByRun(dbsession.nominalSchema(),runnum)
dbsession.transaction().commit()
for cmslsnum,instlumi in perlsrawdata.items():
mystartorbit=startorbit+numorbit*(cmslsnum-1)
bxdataocc1blob=None
bxdataocc2blob=None
bxdataetblob=None
bxerrorocc1blob=None
bxerrorocc2blob=None
bxerroretblob=None
bxqualityocc1blob=None
bxqualityocc2blob=None
bxqualityetblob=None
cmsbxindexblob=None
beam1intensityblob=None
beam2intensityblob=None
if perbunchrawdata:
bxdataArray=array.array('f')
bxerrorArray=array.array('f')
bxqualityArray=array.array('h')
cmsbxindexArray=array.array('h')
beam1intensityArray=array.array('f')
beam2intensityArray=array.array('f')
for bxidx in range(1,3565):
lumifraction=0.0
if perbunchrawdata.has_key(bxidx):
lumifraction=perbunchrawdata[bxidx]
bxlumivalue=float(instlumi*lumifraction)/float(bunchnorm)
bxdataArray.append(bxlumivalue)
beam1intensityArray.append(9124580336.0)
beam1intensityArray.append(8932813306.0)
cmsbxindexArray.append(bxidx)
bxqualityArray.append(1)
bxerrorArray.append(0.0)
bxdataocc1blob=CommonUtil.packArraytoBlob(bxdataArray)
bxdataocc2blob=CommonUtil.packArraytoBlob(bxdataArray)
bxdataetblob=CommonUtil.packArraytoBlob(bxdataArray)
bxerrorocc1blob=CommonUtil.packArraytoBlob(bxerrorArray)
bxerrorocc2blob=CommonUtil.packArraytoBlob(bxerrorArray)
bxerroretblob=CommonUtil.packArraytoBlob(bxerrorArray)
bxqualityocc1blob=CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityocc2blob=CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityetblob=CommonUtil.packArraytoBlob(bxqualityArray)
cmsbxindexblob=CommonUtil.packArraytoBlob(cmsbxindexArray)
beam1intensityblob=CommonUtil.packArraytoBlob(beam1intensityArray)
beam2intensityblob=CommonUtil.packArraytoBlob(beam2intensityArray)
if deliveredonly:
perlsdata=[0,float(instlumi)/float(6370),0.0,1,'STABLE BEAMS',beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob]
else:
perlsdata=[cmslsnum,float(instlumi)/float(6370),0.0,1,'STABLE BEAMS',beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob]
lumilsdata[cmslsnum]=perlsdata
print 'toinsert from scratch',lumilsdata
dbsession.transaction().start(False)
(revision_id,entry_id,data_id)=dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(),runnum,lumirundata,branchinfo)
newlumilsnumMin=min(lumilsdata.keys())
newlumilsnumMax=max(lumilsdata.keys())
#update id of existing to new
#update lumisummaryv2 set DATA_ID=:data_id where DATA_ID=:oldid and lumilsnum<newlumilsnumMin or lumilsnum>newlumilsnumMax
#lumidataid is not None update lumilsdata set lumidataid=:newlumidataid where runnum=:run a
inputData=coral.AttributeList()
inputData.extend('dataid','unsigned long long')
inputData.extend('oldid','unsigned long long')
inputData.extend('lumilsnumMin','unsigned int')
inputData.extend('lumilsnumMax','unsigned int')
inputData['dataid'].setData( data_id )
inputData['oldid'].setData( oldlumidataid )
inputData['lumilsnumMin'].setData( newlumilsnumMin )
inputData['lumilsnumMax'].setData( newlumilsnumMax )
db=dbUtil.dbUtil(dbsession.nominalSchema())
db.singleUpdate('LUMISUMMARYV2','DATA_ID=:dataid','DATA_ID=:oldid AND LUMILSNUM<:lumilsnumMin OR lumilsnum>:lumilsnumMax',inputData)
print 'to update existing id ',oldlumidataid,' outside region ',newlumilsnumMin,' , ',newlumilsnumMax
dataDML.bulkInsertLumiLSSummary(dbsession,runnum,data_id,lumilsdata)
dbsession.transaction().commit()
if options.action=='overview' or options.action=='lumibyls':
reqTrg=True
if options.action=='lumibyls' and options.hltpath:
reqHlt=True
if options.action=='recorded':
reqTrg=True
reqHlt=True
if options.runnumber:
reqrunmax=options.runnumber
reqrunmin=options.runnumber
if options.fillnum:
reqfillmin=options.fillnum
reqfillmax=options.fillnum
if options.begin:
(runbeg,fillbeg,timebeg)=CommonUtil.parseTime(options.begin)
if runbeg: #there's --begin runnum #priority run,fill,time
if not reqrunmin:# there's no -r, then take this
reqrunmin=runbeg
elif fillbeg:
if not reqfillmin:
reqfillmin=fillbeg
elif timebeg:
reqtimemin=timebeg
if reqtimemin:
lute=lumiTime.lumiTime()
reqtimeminT=lute.StrToDatetime(reqtimemin,customfm='%m/%d/%y %H:%M:%S')
timeFilter[0]=reqtimeminT
if options.end:
(runend,fillend,timeend)=CommonUtil.parseTime(options.end)
if runend:
if options.action=='overview' or options.action=='lumibyls':
reqTrg=True
if options.action=='lumibyls' and options.hltpath:
reqHlt=True
if options.action=='recorded':
reqTrg=True
reqHlt=True
if options.runnumber:
reqrunmax=options.runnumber
reqrunmin=options.runnumber
if options.fillnum:
reqfillmin=options.fillnum
reqfillmax=options.fillnum
if options.begin:
(runbeg,fillbeg,timebeg)=CommonUtil.parseTime(options.begin)
if runbeg: #there's --begin runnum #priority run,fill,time
if not reqrunmin:# there's no -r, then take this
reqrunmin=runbeg
elif fillbeg:
if not reqfillmin:
reqfillmin=fillbeg
elif timebeg:
reqtimemin=timebeg
if reqtimemin:
lute=lumiTime.lumiTime()
reqtimeminT=lute.StrToDatetime(reqtimemin,customfm='%m/%d/%y %H:%M:%S')
timeFilter[0]=reqtimeminT
if options.end:
(runend,fillend,timeend)=CommonUtil.parseTime(options.end)
if runend:
def main():
parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description = "Lumi Validation",formatter_class=argparse.ArgumentDefaultsHelpFormatter)
allowedActions = ['batchupdate','update','dump']
allowedFlags = ['UNKNOWN','GOOD','BAD','SUSPECT']
# parse arguments
parser.add_argument('action',choices=allowedActions,help='command actions')
parser.add_argument('-c',dest='connect',action='store',required=True,help='connect string to lumiDB')
parser.add_argument('-P',dest='authpath',action='store',required=True,help='path to authentication file')
parser.add_argument('-i',dest='inputfile',action='store',help='lumi range selection file,required for batchupdate action')
parser.add_argument('-o',dest='outputfile',action='store',help='output to csv file')
parser.add_argument('-r',dest='runnumber',action='store',type=int,help='run number,optional')
parser.add_argument('-runls',dest='runls',action='store',help='selection string,optional. Example [1234:[],4569:[1,1],[2,100]]')
parser.add_argument('-flag',dest='flag',action='store',default='UNKNOWN',help='flag string,optional')
parser.add_argument('--verbose',dest='verbose',action='store_true',help='verbose mode for printing' )
parser.add_argument('--debug',dest='debug',action='store_true',help='debug')
options=parser.parse_args()
if options.flag.upper() not in allowedFlags:
print 'unrecognised flag ',options.flag.upper()
raise
os.environ['CORAL_AUTH_PATH'] = options.authpath
connectstring=options.connect
svc = coral.ConnectionService()
msg=coral.MessageStream('')
if options.debug:
msg.setMsgVerbosity(coral.message_Level_Debug)
else:
msg.setMsgVerbosity(coral.message_Level_Error)
session=svc.connect(connectstring,accessMode=coral.access_Update)
session.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)")
session.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)")
result={}#parsing result {run:[[ls,status,comment]]}
if options.debug :
msg=coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Debug)
if options.action=='batchupdate':
#populate from csv file, require -i argument
if not options.inputfile:
print 'inputfile -i option is required for batchupdate'
raise
csvReader=csv.reader(open(options.inputfile),delimiter=',')
for row in csvReader:
if len(row)==0:
continue
fieldrun=str(row[0]).strip()
fieldls=str(row[1]).strip()
fieldstatus=row[2]
fieldcomment=row[3]
if not result.has_key(int(fieldrun)):
result[int(fieldrun)]=[]
result[int(fieldrun)].append([int(fieldls),fieldstatus,fieldcomment])
insertupdateValidationData(session,result)
if options.action=='update':
#update flag interactively, require -runls argument
#runls={run:[]} populate all CMSLSNUM found in LUMISUMMARY
#runls={run:[[1,1],[2,5]],run:[[1,1],[2,5]]}
#default value
if not options.runls and not options.runnumber:
print 'option -runls or -r is required for update'
raise
if not options.flag:
print 'option -flag is required for update'
raise
if options.flag.upper() not in allowedFlags:
print 'unrecognised flag ',options.flag
raise
if options.runnumber:
runlsjson='{"'+str(options.runnumber)+'":[]}'
elif options.runls:
runlsjson=CommonUtil.tolegalJSON(options.runls)
sparser=selectionParser.selectionParser(runlsjson)
runsandls=sparser.runsandls()
commentStr='NA'
statusStr=options.flag
for run,lslist in runsandls.items():
if not result.has_key(run):
result[run]=[]
for ls in lslist:
result[run].append([ls,statusStr,commentStr])
insertupdateValidationData(session,result)
if options.action=='dump':
if options.runls or options.inputfile:
if options.runls:
runlsjson=CommonUtil.tolegalJSON(options.runls)
sparser=selectionParser.selectionParser(runlsjson)
runsandls=sparser.runsandls()
if options.inputfile:
p=inputFilesetParser.inputFilesetParser(options.inputfile)
runsandls=p.runsandls()
for runnum,lslist in runsandls.items():
dataperrun=getValidationData(session,run=runnum,cmsls=lslist)
if dataperrun.has_key(runnum):
result[runnum]=dataperrun[runnum]
else:
result[runnum]=[]
else:
result=getValidationData(session,run=options.runnumber)
runs=result.keys()
runs.sort()
if options.outputfile:
r=csvReporter.csvReporter(options.outputfile)
for run in runs:
for perrundata in result[run]:
r.writeRow([str(run),str(perrundata[0]),perrundata[1],perrundata[2]])
else:
for run in runs:
print '== ='
if len(result[run])==0:
print str(run),'no validation data'
continue
for lsdata in result[run]:
print str(run)+','+str(lsdata[0])+','+lsdata[1]+','+lsdata[2]
del session
del svc
def insertLumischemaV2(dbsession,runnum,datasource,perlsrawdata,perbunchrawdata,bxdistribution,withDetails=False,deliveredonly=False):
'''
input:
lumirundata[datasource]
perlsrawdata: {cmslsnum:instlumi}
perbunchrawdata: {bxidx:lumifraction}
lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]}
'''
branchrevision_id=3#databranch_id
branchinfo=(branchrevision_id,'DATA')
lumirundata=[datasource]
lumilsdata={}
for cmslsnum,instlumi in perlsrawdata.items():
mystartorbit=startorbit+numorbit*(cmslsnum-1)
bxdataocc1blob=None
bxdataocc2blob=None
bxdataetblob=None
bxerrorocc1blob=None
bxerrorocc2blob=None
bxerroretblob=None
bxqualityocc1blob=None
bxqualityocc2blob=None
bxqualityetblob=None
cmsbxindexblob=None
beam1intensityblob=None
beam2intensityblob=None
beamstatus='STABLE BEAMS'
beamenergy=4000.
instlumierror=0.
instlumiquality=1
if perbunchrawdata:
withDetails=True
bxdataocc1blob=perbunchrawdata[cmslsnum][0]
bxerrorocc1blob=perbunchrawdata[cmslsnum][1]
bxqualityocc1blob=perbunchrawdata[cmslsnum][2]
bxdataocc2blob=perbunchrawdata[cmslsnum][3]
bxerrorocc2blob=perbunchrawdata[cmslsnum][4]
bxqualityocc2blob=perbunchrawdata[cmslsnum][5]
bxdataetblob=perbunchrawdata[cmslsnum][6]
bxerroretblob=perbunchrawdata[cmslsnum][7]
bxqualityetblob=perbunchrawdata[cmslsnum][8]
bxindexblob=perbunchrawdata[cmslsnum][9]
beam1intensityblob=perbunchrawdata[cmslsnum][10]
beam2intensityblob=perbunchrawdata[cmslsnum][11]
beamstatus=perbunchrawdata[cmslsnum][12]
beamenergy=perbunchrawdata[cmslsnum][13]
instlumierror=perbunchrawdata[cmslsnum][14]
instlumiquality=perbunchrawdata[cmslsnum][15]
elif bxdistribution:
withDetails=True
bxdataArray=array.array('f')
bxerrorArray=array.array('f')
bxqualityArray=array.array('h')
cmsbxindexArray=array.array('h')
beam1intensityArray=array.array('f')
beam2intensityArray=array.array('f')
for bxidx in range(1,3565):
lumifraction=0.0
if bxidx in perbunchrawdata:
lumifraction=perbunchrawdata[bxidx]
bxlumivalue=float(instlumi*lumifraction)
bxdataArray.append(bxlumivalue)
beam1intensityArray.append(9124580336.0)
beam1intensityArray.append(8932813306.0)
cmsbxindexArray.append(bxidx)
bxqualityArray.append(1)
bxerrorArray.append(0.0)
bxdataocc1blob=CommonUtil.packArraytoBlob(bxdataArray)
bxdataocc2blob=CommonUtil.packArraytoBlob(bxdataArray)
bxdataetblob=CommonUtil.packArraytoBlob(bxdataArray)
bxerrorocc1blob=CommonUtil.packArraytoBlob(bxerrorArray)
bxerrorocc2blob=CommonUtil.packArraytoBlob(bxerrorArray)
bxerroretblob=CommonUtil.packArraytoBlob(bxerrorArray)
bxqualityocc1blob=CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityocc2blob=CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityetblob=CommonUtil.packArraytoBlob(bxqualityArray)
cmsbxindexblob=CommonUtil.packArraytoBlob(cmsbxindexArray)
beam1intensityblob=CommonUtil.packArraytoBlob(beam1intensityArray)
beam2intensityblob=CommonUtil.packArraytoBlob(beam2intensityArray)
if deliveredonly:
perlsdata=[0,float(instlumi),instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob]
else:
perlsdata=[cmslsnum,float(instlumi),instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob]
lumilsdata[cmslsnum]=perlsdata
#print 'toinsert from scratch',lumilsdata
print(lumilsdata)
dbsession.transaction().start(False)
(revision_id,entry_id,data_id)=dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(),runnum,lumirundata,branchinfo,'LUMIDATA')
dataDML.bulkInsertLumiLSSummary(dbsession,runnum,data_id,lumilsdata,'LUMISUMMARYV2',withDetails=withDetails)
dbsession.transaction().commit()
reqtimemax = None
timeFilter = [None, None]
pbeammode = None
if options.beammode == 'stable':
pbeammode = 'STABLE BEAMS'
iresults = []
reqTrg = False
reqHlt = False
if options.action == 'overview' or options.action == 'lumibyls' or options.action == 'lumibylsXing':
reqTrg = True
if options.action == 'recorded':
reqTrg = True
reqHlt = True
if options.begin:
(reqrunmin, reqfillmin,
reqtimemin) = CommonUtil.parseTime(options.begin)
if reqtimemin:
lute = lumiTime.lumiTime()
reqtimeminT = lute.StrToDatetime(reqtimemin,
customfm='%m/%d/%y %H:%M:%S')
timeFilter[0] = reqtimeminT
if options.end:
(reqrunmax, reqfillmax, reqtimemax) = CommonUtil.parseTime(options.end)
if reqtimemax:
lute = lumiTime.lumiTime()
reqtimemaxT = lute.StrToDatetime(reqtimemax,
customfm='%m/%d/%y %H:%M:%S')
timeFilter[1] = reqtimemaxT
##############################################################
# check working environment
def insertLumischemaV2(dbsession, runnum, datasource, perlsrawdata,
perbunchrawdata):
'''
input:
lumirundata[datasource]
perlsrawdata: {cmslsnum:instlumi}
perbunchrawdata: {bxidx:lumifraction}
lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]}
'''
branchrevision_id = 3 #databranch_id
branchinfo = (branchrevision_id, 'DATA')
lumirundata = [datasource]
lumilsdata = {}
for cmslsnum, instlumi in perlsrawdata.items():
mystartorbit = startorbit + numorbit * (cmslsnum - 1)
bxdataArray = array.array('f')
bxerrorArray = array.array('f')
bxqualityArray = array.array('h')
cmsbxindexArray = array.array('h')
beam1intensityArray = array.array('f')
beam2intensityArray = array.array('f')
for bxidx in range(1, 3565):
lumifraction = 0.0
if bxidx in perbunchrawdata:
lumifraction = perbunchrawdata[bxidx]
bxlumivalue = float(instlumi * lumifraction) / float(bunchnorm)
bxdataArray.append(bxlumivalue)
beam1intensityArray.append(9124580336.0)
beam1intensityArray.append(8932813306.0)
cmsbxindexArray.append(bxidx)
bxqualityArray.append(1)
bxerrorArray.append(0.0)
bxdataocc1blob = CommonUtil.packArraytoBlob(bxdataArray)
bxdataocc2blob = CommonUtil.packArraytoBlob(bxdataArray)
bxdataetblob = CommonUtil.packArraytoBlob(bxdataArray)
bxerrorocc1blob = CommonUtil.packArraytoBlob(bxerrorArray)
bxerrorocc2blob = CommonUtil.packArraytoBlob(bxerrorArray)
bxerroretblob = CommonUtil.packArraytoBlob(bxerrorArray)
bxqualityocc1blob = CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityocc2blob = CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityetblob = CommonUtil.packArraytoBlob(bxqualityArray)
cmsbxindexblob = CommonUtil.packArraytoBlob(cmsbxindexArray)
beam1intensityblob = CommonUtil.packArraytoBlob(beam1intensityArray)
beam2intensityblob = CommonUtil.packArraytoBlob(beam2intensityArray)
perlsdata = [
cmslsnum,
float(instlumi) / float(6370), 0.0, 1, 'STABLE BEAMS', beamenergy,
numorbit, mystartorbit, cmsbxindexblob, beam1intensityblob,
beam2intensityblob, bxdataocc1blob, bxerrorocc1blob,
bxqualityocc1blob, bxdataocc2blob, bxerrorocc2blob,
bxqualityocc2blob, bxdataetblob, bxerroretblob, bxqualityetblob
]
lumilsdata[cmslsnum] = perlsdata
print(lumilsdata)
dbsession.transaction().start(False)
(revision_id, entry_id,
data_id) = dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(),
runnum, lumirundata, branchinfo)
dataDML.bulkInsertLumiLSSummary(dbsession, runnum, data_id, lumilsdata)
dbsession.transaction().commit()
def specificlumiTofile(fillnum,filldata,outdir):
#
#input : fillnum
# filldata: {bxidx:[[lstime,beamstatusfrac,lumivalue,lumierror,speclumi,speclumierr]],[]}
#sorted by bxidx, sorted by lstime inside list
#check outdir/fillnum subdir exists; if not, create it; else outdir=outdir/fillnum
#
if not filldata:
print('empty input data, do nothing for fill ',fillnum)
return
timedict={}#{lstime:[[stablebeamfrac,lumi,lumierr,speclumi,speclumierr]]}
filloutdir=os.path.join(outdir,str(fillnum))
if not os.path.exists(filloutdir):
os.mkdir(filloutdir)
for cmsbxidx,perbxdata in filldata.items():
lhcbucket=0
if cmsbxidx!=0:
lhcbucket=(cmsbxidx-1)*10+1
a=sorted(perbxdata,key=lambda x:x[0])
filename=str(fillnum)+'_lumi_'+str(lhcbucket)+'_CMS.txt'
linedata=[]
for perlsdata in a:
ts=int(perlsdata[0])
beamstatusfrac=perlsdata[1]
lumi=perlsdata[2]
lumierror=perlsdata[3]
#beam1intensity=perlsdata[4]
#beam2intensity=perlsdata[5]
speclumi=perlsdata[4]
speclumierror= perlsdata[5]
if lumi>0:
linedata.append([ts,beamstatusfrac,lumi,lumierror,speclumi,speclumierror])
if ts not in timedict:
timedict[ts]=[]
timedict[ts].append([beamstatusfrac,lumi,lumierror,speclumi,speclumierror])
if len(linedata)>10:#at least 10 good ls
f=open(os.path.join(filloutdir,filename),'w')
for line in linedata:
print('%d\t%e\t%e\t%e\t%e\t%e'%(line[0],line[1],line[2],line[3],line[4],line[5]), file=f)
f.close()
#print 'writing avg file'
summaryfilename=str(fillnum)+'_lumi_CMS.txt'
f=None
lstimes=sorted(timedict.keys())
fillseg=[]
lscounter=0
for lstime in lstimes:
allvalues=timedict[lstime]
transposedvalues=CommonUtil.transposed(allvalues,0.0)
bstatfrac=transposedvalues[0][0]#beamstatus does not change with bx position
lumivals=transposedvalues[1]
lumitot=sum(lumivals)
if bstatfrac==1.0 :
fillseg.append([lstime,lumitot])
lumierrs=transposedvalues[2]
lumierrortot=math.sqrt(sum(map(lambda x:x**2,lumierrs)))
specificvals=transposedvalues[3]
specificavg=sum(specificvals)/float(len(specificvals))#avg spec lumi
specificerrs=transposedvalues[4]
specifictoterr=math.sqrt(sum(map(lambda x:x**2,specificerrs)))
specificerravg=specifictoterr/float(len(specificvals))
if lscounter==0:
f=open(os.path.join(filloutdir,summaryfilename),'w')
lscounter+=1
print('%d\t%e\t%e\t%e\t%e\t%e'%(lstime,bstatfrac,lumitot,lumierrortot,specificavg,specificerravg), file=f)
if f is not None:
f.close()
#print 'writing summary file'
fillsummaryfilename=str(fillnum)+'_bxsum_CMS.txt'
f=open(os.path.join(filloutdir,fillsummaryfilename),'w')
if len(fillseg)==0:
print('%s'%('#no stable beams'), file=f)
f.close()
return
previoustime=fillseg[0][0]
boundarytime=fillseg[0][0]
#print 'boundary time ',boundarytime
summaryls={}
summaryls[boundarytime]=[]
for [lstime,lumitot] in fillseg:#fillseg is everything with stable beam flag
if lstime-previoustime>50.0:
boundarytime=lstime
#print 'found new boundary ',boundarytime
summaryls[boundarytime]=[]
# print 'appending ',boundarytime,lstime,lumitot
summaryls[boundarytime].append([lstime,lumitot])
previoustime=lstime
#print summaryls
summarylstimes=summaryls.keys()
summarylstimes.sort()
lumip=lumiParameters.ParametersObject()
for bts in summarylstimes:
startts=bts
tsdatainseg=summaryls[bts]
#print 'tsdatainseg ',tsdatainseg
stopts=tsdatainseg[-1][0]
plu=max(CommonUtil.transposed(tsdatainseg,0.0)[1])
lui=sum(CommonUtil.transposed(tsdatainseg,0.0)[1])*lumip.lslengthsec()
print('%d\t%d\t%e\t%e'%(startts,stopts,plu,lui), file=f)
f.close()
def patchV2(dbsession,runnum,inputpathnames,inputdata):
'''
inputpathnames: [pathname,]
inputdata: {cmslsnum:[presc,presc...]}
update the most recent version of lshlt data if some ls exist
if none old ls exists
insert new hlt records
'''
try:
dbsession.transaction().start(True)
oldhltdataid=dataDML.guessHltDataIdByRun(dbsession.nominalSchema(),runnum)
existingrundata=dataDML.hltRunById(dbsession.nominalSchema(),oldhltdataid)
dbsession.transaction().commit()
if not oldhltdataid:#no data at all
dbsession.transaction().start(False)
insertV2(dbsession,runnum,inputpathnames,inputdata)
dbsession.transaction().commit()
return
hltnamedict=existingrundata[3]#[(pathidx,hltname),(pathidx,hltname)...]
dbsession.transaction().start(True)
existinglsdata=dataDML.hltLSById(dbsession.nominalSchema(),oldhltdataid)
dbsession.transaction().commit()
oldlsdata=existinglsdata[1]
existinglslist=oldlsdata.keys()
toupdate={}#{cmslsnum:[presc,presc...]}
toinsert={}#{cmslsnum:[presc,presc...]}
if existinglslist and len(existinglslist)!=0:#there are some existing data
for cmslsnum,oldlscontent in oldlsdata.items():
if cmslsnum in inputdata.keys(): # if overlap with new data, update old data with new
toupdate[cmslsnum]=inputdata[cmslsnum]
for cmslsnum,lshltcontent in inputdata.items():
if cmslsnum in toupdate: continue #it's to update not to insert
toinsert[cmslsnum]=inputdata[cmslsnum]
#
# insert into lshlt(data_id,runnum,cmslsnum,prescaleblob,hltcountblob,hltacceptblob) values()
#
dbsession.transaction().start(False)
tabrowDefDict={'DATA_ID':'unsigned long long','RUNNUM':'unsigned int','CMSLSNUM':'unsigned int','PRESCALEBLOB':'blob','HLTCOUNTBLOB':'blob','HLTACCEPTBLOB':'blob'}
for cmslsnum,perlsdata in toinsert.items():
prescaleArray=array.array('I')
hltcountArray=array.array('I')
hltacceptArray=array.array('I')
for (pathidx,hltname) in hltnamedict:
thispathIdx=inputpathnames.index(hltname)
thispresc=perlsdata[thispathIdx]
thiscount=0
thisaccept=0
prescaleArray.append(thispresc)
hltcountArray.append(thiscount)
hltacceptArray.append(thisaccept)
prescaleblob=CommonUtil.packArraytoBlob(prescaleArray)
hltcountblob=CommonUtil.packArraytoBlob(hltcountArray)
hltacceptblob=CommonUtil.packArraytoBlob(hltacceptArray)
tabrowValueDict={'DATA_ID':oldhltdataid,'RUNNUM':int(runnum),'CMSLSNUM':int(cmslsnum),'PRESCALEBLOB':prescaleblob,'HLTCOUNTBLOB':hltcountblob,'HLTACCEPTBLOB':hltacceptblob}
db=dbUtil.dbUtil(dbsession.nominalSchema())
db.insertOneRow(nameDealer.lshltTableName(),tabrowDefDict,tabrowValueDict)
#
# update lshlt set prescaleblob=:prescaleblob,hltcoutblob=:hltcountblob,hltacceptblob=:hltacceptblob where data_id=:olddata_id and cmslsnum=:cmslsnum;
#
setClause='PRESCALEBLOB=:prescaleblob,HLTCOUNTBLOB=:hltcountblob,HLTACCEPTBLOB=:hltacceptblob'
updateCondition='DATA_ID=:oldhltdataid and CMSLSNUM=:cmslsnum'
for cmslsnum,perlsdata in toupdate.items():
prescaleArray=array.array('I')
hltcountArray=array.array('I')
hltacceptArray=array.array('I')
for (pathidx,hltname) in hltnamedict:
thispathIdx=inputpathnames.index(hltname)
thispresc=perlsdata[thispathIdx]
thiscount=0
thisaccept=0
prescaleArray.append(thispresc)
hltcountArray.append(thiscount)
hltacceptArray.append(thisaccept)
prescaleblob=CommonUtil.packArraytoBlob(prescaleArray)
hltcountblob=CommonUtil.packArraytoBlob(hltcountArray)
hltacceptblob=CommonUtil.packArraytoBlob(hltacceptArray)
iData=coral.AttributeList()
iData.extend('prescaleblob','blob')
iData.extend('hltcountblob','blob')
iData.extend('hltacceptblob','blob')
iData.extend('olddata_id','unsigned int')
iData.extend('cmslsnum','unsigned int')
iData['prescaleblob'].setData(prescaleblob)
iData['hltcountblob'].setData(hltcountblob)
iData['hltacceptblob'].setData(hltacceptblob)
iData['olddata_id'].setData(int(olddata_id))
iData['cmslsnum'].setData(int(cmslsnum))
db=dbUtil.dbUtil(schema)
db.singleUpdate(nameDealer.lshltTableName(),setClause,updateCondition,iData)
dbsession.transaction().commit()
#dbsession.transaction().rollback()
except :
raise
runlist,
withcomment=False,
lumitype='HF')
lumirundata = dataDML.lumiRunByIds(session.nominalSchema(),
hfdataidmap,
lumitype='HF')
#{runnum: (datasource(1),nominalegev(2),ncollidingbunches(3)}
GrunsummaryData = lumiCalcAPI.runsummaryMap(session.nominalSchema(),
irunlsdict)
session.transaction().commit()
for runnum in lumirundata.keys():
session.transaction().start(True)
hfdataidinfo = hfdataidmap[runnum]
hflumidata = lumiCalcAPI.instLumiForIds(session.nominalSchema(),
irunlsdict,
hfdataidmap,
GrunsummaryData,
beamstatusfilter=None,
withBXInfo=True,
bxAlgo='OCC1')
for perlsdata in hflumidata[runnum]:
lumilsnum = perlsdata[0]
cmslsnum = perlsdata[1]
bxinfo = perlsdata[9]
bxdistro = getdistribution(cmslsnum, bxinfo[0], bxinfo[1])
print(lumilsnum, cmslsnum, CommonUtil.flatten(bxdistro))
session.transaction().commit()
del session
del svc
def insertLumischemaV2(dbsession,
runnum,
datasource,
perlsrawdata,
perbunchrawdata,
bxdistribution,
withDetails=False,
deliveredonly=False):
'''
input:
lumirundata[datasource]
perlsrawdata: {cmslsnum:instlumi}
perbunchrawdata: {bxidx:lumifraction}
lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]}
'''
branchrevision_id = 3 #databranch_id
branchinfo = (branchrevision_id, 'DATA')
lumirundata = [datasource]
lumilsdata = {}
for cmslsnum, instlumi in perlsrawdata.items():
mystartorbit = startorbit + numorbit * (cmslsnum - 1)
bxdataocc1blob = None
bxdataocc2blob = None
bxdataetblob = None
bxerrorocc1blob = None
bxerrorocc2blob = None
bxerroretblob = None
bxqualityocc1blob = None
bxqualityocc2blob = None
bxqualityetblob = None
cmsbxindexblob = None
beam1intensityblob = None
beam2intensityblob = None
beamstatus = 'STABLE BEAMS'
beamenergy = 4000.
instlumierror = 0.
instlumiquality = 1
if perbunchrawdata:
withDetails = True
bxdataocc1blob = perbunchrawdata[cmslsnum][0]
bxerrorocc1blob = perbunchrawdata[cmslsnum][1]
bxqualityocc1blob = perbunchrawdata[cmslsnum][2]
bxdataocc2blob = perbunchrawdata[cmslsnum][3]
bxerrorocc2blob = perbunchrawdata[cmslsnum][4]
bxqualityocc2blob = perbunchrawdata[cmslsnum][5]
bxdataetblob = perbunchrawdata[cmslsnum][6]
bxerroretblob = perbunchrawdata[cmslsnum][7]
bxqualityetblob = perbunchrawdata[cmslsnum][8]
bxindexblob = perbunchrawdata[cmslsnum][9]
beam1intensityblob = perbunchrawdata[cmslsnum][10]
beam2intensityblob = perbunchrawdata[cmslsnum][11]
beamstatus = perbunchrawdata[cmslsnum][12]
beamenergy = perbunchrawdata[cmslsnum][13]
instlumierror = perbunchrawdata[cmslsnum][14]
instlumiquality = perbunchrawdata[cmslsnum][15]
elif bxdistribution:
withDetails = True
bxdataArray = array.array('f')
bxerrorArray = array.array('f')
bxqualityArray = array.array('h')
cmsbxindexArray = array.array('h')
beam1intensityArray = array.array('f')
beam2intensityArray = array.array('f')
for bxidx in range(1, 3565):
lumifraction = 0.0
if bxidx in perbunchrawdata:
lumifraction = perbunchrawdata[bxidx]
bxlumivalue = float(instlumi * lumifraction)
bxdataArray.append(bxlumivalue)
beam1intensityArray.append(9124580336.0)
beam1intensityArray.append(8932813306.0)
cmsbxindexArray.append(bxidx)
bxqualityArray.append(1)
bxerrorArray.append(0.0)
bxdataocc1blob = CommonUtil.packArraytoBlob(bxdataArray)
bxdataocc2blob = CommonUtil.packArraytoBlob(bxdataArray)
bxdataetblob = CommonUtil.packArraytoBlob(bxdataArray)
bxerrorocc1blob = CommonUtil.packArraytoBlob(bxerrorArray)
bxerrorocc2blob = CommonUtil.packArraytoBlob(bxerrorArray)
bxerroretblob = CommonUtil.packArraytoBlob(bxerrorArray)
bxqualityocc1blob = CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityocc2blob = CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityetblob = CommonUtil.packArraytoBlob(bxqualityArray)
cmsbxindexblob = CommonUtil.packArraytoBlob(cmsbxindexArray)
beam1intensityblob = CommonUtil.packArraytoBlob(
beam1intensityArray)
beam2intensityblob = CommonUtil.packArraytoBlob(
beam2intensityArray)
if deliveredonly:
perlsdata = [
0,
float(instlumi), instlumierror, instlumiquality, beamstatus,
beamenergy, numorbit, mystartorbit, cmsbxindexblob,
beam1intensityblob, beam2intensityblob, bxdataocc1blob,
bxerrorocc1blob, bxqualityocc1blob, bxdataocc2blob,
bxerrorocc2blob, bxqualityocc2blob, bxdataetblob,
bxerroretblob, bxqualityetblob
]
else:
perlsdata = [
cmslsnum,
float(instlumi), instlumierror, instlumiquality, beamstatus,
beamenergy, numorbit, mystartorbit, cmsbxindexblob,
beam1intensityblob, beam2intensityblob, bxdataocc1blob,
bxerrorocc1blob, bxqualityocc1blob, bxdataocc2blob,
bxerrorocc2blob, bxqualityocc2blob, bxdataetblob,
bxerroretblob, bxqualityetblob
]
lumilsdata[cmslsnum] = perlsdata
#print 'toinsert from scratch',lumilsdata
print(lumilsdata)
dbsession.transaction().start(False)
(revision_id, entry_id,
data_id) = dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(),
runnum, lumirundata, branchinfo,
'LUMIDATA')
dataDML.bulkInsertLumiLSSummary(dbsession,
runnum,
data_id,
lumilsdata,
'LUMISUMMARYV2',
withDetails=withDetails)
dbsession.transaction().commit()
raise RuntimeError(
'parameter corrector is required for create/insert action')
if not normvalueDict.has_key(
'since') or not normvalueDict['since']:
raise RuntimeError(
'parameter since is required for create/insert action')
correctorStr = normvalueDict['corrector']
sincerun = int(normvalueDict['since'])
if options.firstsince:
if sincerun < int(options.firstsince):
continue
amodetag = normvalueDict['amodetag']
egev = int(normvalueDict['egev'])
detailcomment = normvalueDict['comment']
(correctorname,
parameterlist) = CommonUtil.parselumicorrector(correctorStr)
parameterDict = {}
for param in parameterlist:
parameterDict[param] = normvalueDict[param]
normDML.insertValueToNormId(dbsession.nominalSchema(),
normdata_id,
sincerun,
correctorStr,
amodetag,
egev,
parameterDict,
comment=detailcomment)
dbsession.transaction().commit()
##############################
# setdefault/unsetdefault
if options.beammode=='stable':
pbeammode = 'STABLE BEAMS'
if options.verbose:
print 'General configuration'
print '\tconnect: ',options.connect
print '\tauthpath: ',options.authpath
print '\tlumi data version: ',options.lumiversion
print '\tsiteconfpath: ',options.siteconfpath
print '\toutputfile: ',options.outputfile
print '\tscalefactor: ',options.scalefactor
if options.action=='recorded' and options.hltpath:
print 'Action: effective luminosity in hltpath: ',options.hltpath
else:
print 'Action: ',options.action
if options.normfactor:
if CommonUtil.is_floatstr(normfactor):
print '\tuse norm factor value ',normfactor
else:
print '\tuse specific norm factor name ',normfactor
else:
print '\tuse norm factor in context (amodetag,beamenergy)'
if options.runnumber: # if runnumber specified, do not go through other run selection criteria
print '\tselect specific run== ',options.runnumber
else:
print '\trun selections == '
print '\tinput selection file: ',options.inputfile
print '\tbeam mode: ',options.beammode
print '\tfill: ',options.fillnum
print '\tamodetag: ',options.amodetag
print '\tbegin: ',options.begin
print '\tend: ',options.end
