def main():
from RecoLuminosity.LumiDB import sessionManager,queryDataSource
parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description="migrate lumidb schema",formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-c',dest='connect',action='store',required=False,default='oracle://devdb10/cms_xiezhen_dev',help='connect string to dest db(required)')
parser.add_argument('-lumisource',dest='lumisource',action='store',required=False,default='oracle://cms_orcoff_prod/CMS_LUMI_PROD',help='connect string to source lumi db')
#parser.add_argument('-runinfo',dest='runinfo',action='store',required=False,default='oracle://cms_orcoff_prod/CMS_RUNINFO',help='connect string to runinfo db')
parser.add_argument('-P',dest='authpath',action='store',required=False,default='/afs/cern.ch/user/x/xiezhen',help='path to authentication file')
parser.add_argument('-r',dest='runnumber',action='store',required=True,help='run number')
parser.add_argument('--debug',dest='debug',action='store_true',help='debug')
args=parser.parse_args()
runnumber=int(args.runnumber)
print('processing run ',runnumber)
#runinfosvc=sessionManager.sessionManager(args.runinfo,authpath=args.authpath,debugON=args.debug)
lumisvc=sessionManager.sessionManager(args.lumisource,authpath=args.authpath,debugON=args.debug)
destsvc=sessionManager.sessionManager(args.connect,authpath=args.authpath,debugON=args.debug)
#runinfosession=runinfosvc.openSession(isReadOnly=True,cpp2sqltype=[('unsigned int','NUMBER(10)'),('unsigned long long','NUMBER(20)')])
lumisession=lumisvc.openSession(isReadOnly=True,cpp2sqltype=[('unsigned int','NUMBER(10)'),('unsigned long long','NUMBER(20)')])
try:
#[l1key,amodetag,egev,sequence,hltkey,fillnum,starttime,stoptime]=queryDataSource.runsummary(runinfosession,'CMS_RUNINFO',runnumber,complementalOnly=False)
#print 'runsummary ',[l1key,amodetag,egev,sequence,hltkey,fillnum,starttime,stoptime]
lumidata=queryDataSource.uncalibratedlumiFromOldLumi(lumisession,runnumber)
#print 'lumidata ',lumidata
[bitnames,trglsdata]=queryDataSource.trgFromOldLumi(lumisession,runnumber)
#print 'trg data ',bitnames,trglsdata
[pathnames,hltlsdata]=queryDataSource.hltFromOldLumi(lumisession,runnumber)
#print 'hlt data ',pathnames,hltlsdata
lumisession.transaction().commit()
#runinfosession.transaction().commit()
del lumisession
del lumisvc
#del runinfosession
#del runinfosvc
destsession=destsvc.openSession(isReadOnly=False,cpp2sqltype=[('unsigned int','NUMBER(10)'),('unsigned long long','NUMBER(20)')])
destsession.transaction().start(False)
branchrevision_id=DATABRANCH_ID
#print 'data branchid ',branchrevision_id
#dataDML.insertRunSummaryData(destsession.nominalSchema(),runnumber,[l1key,amodetag,egev,sequence,hltkey,fillnum,starttime,stoptime],complementalOnly=False)
(lumirevid,lumientryid,lumidataid)=dataDML.addLumiRunDataToBranch(destsession.nominalSchema(),runnumber,[args.lumisource],(branchrevision_id,'DATA'))
bitzeroname=bitnames.split(',')[0]
trgrundata=[args.lumisource,bitzeroname,bitnames]
(trgrevid,trgentryid,trgdataid)=dataDML.addTrgRunDataToBranch(destsession.nominalSchema(),runnumber,trgrundata,(branchrevision_id,'DATA'))
hltrundata=[pathnames,args.lumisource]
(hltrevid,hltentryid,hltdataid)=dataDML.addHLTRunDataToBranch(destsession.nominalSchema(),runnumber,hltrundata,(branchrevision_id,'DATA'))
destsession.transaction().commit()
dataDML.bulkInsertLumiLSSummary(destsession,runnumber,lumidataid,lumidata,500)
#
dataDML.bulkInsertTrgLSData(destsession,runnumber,trgdataid,trglsdata,500)
dataDML.bulkInsertHltLSData(destsession,runnumber,hltdataid,hltlsdata,500)
del destsession
del destsvc
except:
raise
def insertLumischemaV2(dbsession,runnum,datasource,perlsrawdata,perbunchrawdata):
'''
input:
lumirundata[datasource]
perlsrawdata: {cmslsnum:instlumi}
perbunchrawdata: {bxidx:lumifraction}
lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]}
'''
branchrevision_id=3#databranch_id
branchinfo=(branchrevision_id,'DATA')
lumirundata=[datasource]
lumilsdata={}
for cmslsnum,instlumi in perlsrawdata.items():
mystartorbit=startorbit+numorbit*(cmslsnum-1)
bxdataArray=array.array('f')
bxerrorArray=array.array('f')
bxqualityArray=array.array('h')
cmsbxindexArray=array.array('h')
beam1intensityArray=array.array('f')
beam2intensityArray=array.array('f')
for bxidx in range(1,3565):
lumifraction=0.0
if bxidx in perbunchrawdata:
lumifraction=perbunchrawdata[bxidx]
bxlumivalue=float(instlumi*lumifraction)/float(bunchnorm)
bxdataArray.append(bxlumivalue)
beam1intensityArray.append(9124580336.0)
beam1intensityArray.append(8932813306.0)
cmsbxindexArray.append(bxidx)
bxqualityArray.append(1)
bxerrorArray.append(0.0)
bxdataocc1blob=CommonUtil.packArraytoBlob(bxdataArray)
bxdataocc2blob=CommonUtil.packArraytoBlob(bxdataArray)
bxdataetblob=CommonUtil.packArraytoBlob(bxdataArray)
bxerrorocc1blob=CommonUtil.packArraytoBlob(bxerrorArray)
bxerrorocc2blob=CommonUtil.packArraytoBlob(bxerrorArray)
bxerroretblob=CommonUtil.packArraytoBlob(bxerrorArray)
bxqualityocc1blob=CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityocc2blob=CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityetblob=CommonUtil.packArraytoBlob(bxqualityArray)
cmsbxindexblob=CommonUtil.packArraytoBlob(cmsbxindexArray)
beam1intensityblob=CommonUtil.packArraytoBlob(beam1intensityArray)
beam2intensityblob=CommonUtil.packArraytoBlob(beam2intensityArray)
perlsdata=[cmslsnum,float(instlumi)/float(6370),0.0,1,'STABLE BEAMS',beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob]
lumilsdata[cmslsnum]=perlsdata
print(lumilsdata)
dbsession.transaction().start(False)
(revision_id,entry_id,data_id)=dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(),runnum,lumirundata,branchinfo)
dataDML.bulkInsertLumiLSSummary(dbsession,runnum,data_id,lumilsdata)
dbsession.transaction().commit()
def insertLumischemaV2(dbsession,runnum,datasource,perlsrawdata,perbunchrawdata):
'''
input:
lumirundata[datasource]
perlsrawdata: {cmslsnum:instlumi}
perbunchrawdata: {bxidx:lumifraction}
lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]}
'''
branchrevision_id=3#databranch_id
branchinfo=(branchrevision_id,'DATA')
lumirundata=[datasource]
lumilsdata={}
for cmslsnum,instlumi in perlsrawdata.items():
mystartorbit=startorbit+numorbit*(cmslsnum-1)
bxdataArray=array.array('f')
bxerrorArray=array.array('f')
bxqualityArray=array.array('h')
cmsbxindexArray=array.array('h')
beam1intensityArray=array.array('f')
beam2intensityArray=array.array('f')
for bxidx in range(1,3565):
lumifraction=0.0
if bxidx in perbunchrawdata:
lumifraction=perbunchrawdata[bxidx]
bxlumivalue=float(instlumi*lumifraction)/float(bunchnorm)
bxdataArray.append(bxlumivalue)
beam1intensityArray.append(9124580336.0)
beam1intensityArray.append(8932813306.0)
cmsbxindexArray.append(bxidx)
bxqualityArray.append(1)
bxerrorArray.append(0.0)
bxdataocc1blob=CommonUtil.packArraytoBlob(bxdataArray)
bxdataocc2blob=CommonUtil.packArraytoBlob(bxdataArray)
bxdataetblob=CommonUtil.packArraytoBlob(bxdataArray)
bxerrorocc1blob=CommonUtil.packArraytoBlob(bxerrorArray)
bxerrorocc2blob=CommonUtil.packArraytoBlob(bxerrorArray)
bxerroretblob=CommonUtil.packArraytoBlob(bxerrorArray)
bxqualityocc1blob=CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityocc2blob=CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityetblob=CommonUtil.packArraytoBlob(bxqualityArray)
cmsbxindexblob=CommonUtil.packArraytoBlob(cmsbxindexArray)
beam1intensityblob=CommonUtil.packArraytoBlob(beam1intensityArray)
beam2intensityblob=CommonUtil.packArraytoBlob(beam2intensityArray)
perlsdata=[cmslsnum,float(instlumi)/float(6370),0.0,1,'STABLE BEAMS',beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob]
lumilsdata[cmslsnum]=perlsdata
print(lumilsdata)
dbsession.transaction().start(False)
(revision_id,entry_id,data_id)=dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(),runnum,lumirundata,branchinfo)
dataDML.bulkInsertLumiLSSummary(dbsession,runnum,data_id,lumilsdata)
dbsession.transaction().commit()
hfdataidmap=revisionDML.dataIdsByTagId(session.nominalSchema(),hf_tagid,runlist,withcomment=False,lumitype='HF')
beamstatusdata=dataDML.beamstatusByIds(session.nominalSchema(),hfdataidmap)
#print 'beamstatusdata ',beamstatusdata
lumirundata=dataDML.lumiRunByIds(session.nominalSchema(),hfdataidmap,lumitype='HF')#{runnum: (nominalegev(0),ncollidingbunches(1),starttime(2),stoptime(3),nls(4)}
session.transaction().commit()
#print 'lumirundata ', lumirundata
alllumirundata=generateLumiRundata(inputfilename,lumirundata,runlist)
alllumilsdata={}
for runnum,perrundata in parseresult.items():
pixellumidataid=0
session.transaction().start(False)
#session.transaction().start(True)
hfdataidinfo=hfdataidmap[runnum]
hflumidataid=hfdataidinfo[0]
trgdataid=hfdataidinfo[1]
hltdataid=hfdataidinfo[2]
beamsta={}
if runnum in beamstatusdata:
beamsta=beamstatusdata[runnum]
alllumilsdata[runnum]=generateLumiLSdataForRun(perrundata,alllumirundata[runnum],beamsta)#lumirundata [datasource,nominalenergy,ncollidingbunches,starttime,stoptime,nls]
pixellumirundata=alllumirundata[runnum]
(pixellumirevid,pixellumientryid,pixellumidataid)=dataDML.addLumiRunDataToBranch(session.nominalSchema(),runnum,pixellumirundata,pixellumibranchinfo,nameDealer.pixellumidataTableName())
pixellumilsdata=alllumilsdata[runnum]
revisionDML.addRunToCurrentDataTag(session.nominalSchema(),runnum,pixellumidataid,trgdataid,hltdataid,lumitype='PIXEL')
session.transaction().commit()
dataDML.bulkInsertLumiLSSummary(session,runnum,pixellumidataid,pixellumilsdata,nameDealer.pixellumisummaryv2TableName(),withDetails=False)
del session
del svc
session.transaction().start(False)
#session.transaction().start(True)
hfdataidinfo = hfdataidmap[runnum]
hflumidataid = hfdataidinfo[0]
trgdataid = hfdataidinfo[1]
hltdataid = hfdataidinfo[2]
beamsta = {}
if beamstatusdata.has_key(runnum):
beamsta = beamstatusdata[runnum]
alllumilsdata[runnum] = generateLumiLSdataForRun(
perrundata, alllumirundata[runnum], beamsta
) #lumirundata [datasource,nominalenergy,ncollidingbunches,starttime,stoptime,nls]
pixellumirundata = alllumirundata[runnum]
(pixellumirevid, pixellumientryid,
pixellumidataid) = dataDML.addLumiRunDataToBranch(
session.nominalSchema(), runnum, pixellumirundata,
pixellumibranchinfo, nameDealer.pixellumidataTableName())
pixellumilsdata = alllumilsdata[runnum]
revisionDML.addRunToCurrentDataTag(session.nominalSchema(),
runnum,
pixellumidataid,
trgdataid,
hltdataid,
lumitype='PIXEL')
session.transaction().commit()
dataDML.bulkInsertLumiLSSummary(
session,
runnum,
pixellumidataid,
pixellumilsdata,
nameDealer.pixellumisummaryv2TableName(),
def insertLumischemaV2(dbsession,
runnum,
datasource,
perlsrawdata,
perbunchrawdata,
bxdistribution,
withDetails=False,
deliveredonly=False):
'''
input:
lumirundata[datasource]
perlsrawdata: {cmslsnum:instlumi}
perbunchrawdata: {bxidx:lumifraction}
lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]}
'''
branchrevision_id = 3 #databranch_id
branchinfo = (branchrevision_id, 'DATA')
lumirundata = [datasource]
lumilsdata = {}
for cmslsnum, instlumi in perlsrawdata.items():
mystartorbit = startorbit + numorbit * (cmslsnum - 1)
bxdataocc1blob = None
bxdataocc2blob = None
bxdataetblob = None
bxerrorocc1blob = None
bxerrorocc2blob = None
bxerroretblob = None
bxqualityocc1blob = None
bxqualityocc2blob = None
bxqualityetblob = None
cmsbxindexblob = None
beam1intensityblob = None
beam2intensityblob = None
beamstatus = 'STABLE BEAMS'
beamenergy = 4000.
instlumierror = 0.
instlumiquality = 1
if perbunchrawdata:
withDetails = True
bxdataocc1blob = perbunchrawdata[cmslsnum][0]
bxerrorocc1blob = perbunchrawdata[cmslsnum][1]
bxqualityocc1blob = perbunchrawdata[cmslsnum][2]
bxdataocc2blob = perbunchrawdata[cmslsnum][3]
bxerrorocc2blob = perbunchrawdata[cmslsnum][4]
bxqualityocc2blob = perbunchrawdata[cmslsnum][5]
bxdataetblob = perbunchrawdata[cmslsnum][6]
bxerroretblob = perbunchrawdata[cmslsnum][7]
bxqualityetblob = perbunchrawdata[cmslsnum][8]
bxindexblob = perbunchrawdata[cmslsnum][9]
beam1intensityblob = perbunchrawdata[cmslsnum][10]
beam2intensityblob = perbunchrawdata[cmslsnum][11]
beamstatus = perbunchrawdata[cmslsnum][12]
beamenergy = perbunchrawdata[cmslsnum][13]
instlumierror = perbunchrawdata[cmslsnum][14]
instlumiquality = perbunchrawdata[cmslsnum][15]
elif bxdistribution:
withDetails = True
bxdataArray = array.array('f')
bxerrorArray = array.array('f')
bxqualityArray = array.array('h')
cmsbxindexArray = array.array('h')
beam1intensityArray = array.array('f')
beam2intensityArray = array.array('f')
for bxidx in range(1, 3565):
lumifraction = 0.0
if bxidx in perbunchrawdata:
lumifraction = perbunchrawdata[bxidx]
bxlumivalue = float(instlumi * lumifraction)
bxdataArray.append(bxlumivalue)
beam1intensityArray.append(9124580336.0)
beam1intensityArray.append(8932813306.0)
cmsbxindexArray.append(bxidx)
bxqualityArray.append(1)
bxerrorArray.append(0.0)
bxdataocc1blob = CommonUtil.packArraytoBlob(bxdataArray)
bxdataocc2blob = CommonUtil.packArraytoBlob(bxdataArray)
bxdataetblob = CommonUtil.packArraytoBlob(bxdataArray)
bxerrorocc1blob = CommonUtil.packArraytoBlob(bxerrorArray)
bxerrorocc2blob = CommonUtil.packArraytoBlob(bxerrorArray)
bxerroretblob = CommonUtil.packArraytoBlob(bxerrorArray)
bxqualityocc1blob = CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityocc2blob = CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityetblob = CommonUtil.packArraytoBlob(bxqualityArray)
cmsbxindexblob = CommonUtil.packArraytoBlob(cmsbxindexArray)
beam1intensityblob = CommonUtil.packArraytoBlob(
beam1intensityArray)
beam2intensityblob = CommonUtil.packArraytoBlob(
beam2intensityArray)
if deliveredonly:
perlsdata = [
0,
float(instlumi), instlumierror, instlumiquality, beamstatus,
beamenergy, numorbit, mystartorbit, cmsbxindexblob,
beam1intensityblob, beam2intensityblob, bxdataocc1blob,
bxerrorocc1blob, bxqualityocc1blob, bxdataocc2blob,
bxerrorocc2blob, bxqualityocc2blob, bxdataetblob,
bxerroretblob, bxqualityetblob
]
else:
perlsdata = [
cmslsnum,
float(instlumi), instlumierror, instlumiquality, beamstatus,
beamenergy, numorbit, mystartorbit, cmsbxindexblob,
beam1intensityblob, beam2intensityblob, bxdataocc1blob,
bxerrorocc1blob, bxqualityocc1blob, bxdataocc2blob,
bxerrorocc2blob, bxqualityocc2blob, bxdataetblob,
bxerroretblob, bxqualityetblob
]
lumilsdata[cmslsnum] = perlsdata
#print 'toinsert from scratch',lumilsdata
print(lumilsdata)
dbsession.transaction().start(False)
(revision_id, entry_id,
data_id) = dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(),
runnum, lumirundata, branchinfo,
'LUMIDATA')
dataDML.bulkInsertLumiLSSummary(dbsession,
runnum,
data_id,
lumilsdata,
'LUMISUMMARYV2',
withDetails=withDetails)
dbsession.transaction().commit()
def insertLumischemaV2(dbsession,runnum,datasource,perlsrawdata,perbunchrawdata,bxdistribution,withDetails=False,deliveredonly=False):
'''
input:
lumirundata[datasource]
perlsrawdata: {cmslsnum:instlumi}
perbunchrawdata: {bxidx:lumifraction}
lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]}
'''
branchrevision_id=3#databranch_id
branchinfo=(branchrevision_id,'DATA')
lumirundata=[datasource]
lumilsdata={}
for cmslsnum,instlumi in perlsrawdata.items():
mystartorbit=startorbit+numorbit*(cmslsnum-1)
bxdataocc1blob=None
bxdataocc2blob=None
bxdataetblob=None
bxerrorocc1blob=None
bxerrorocc2blob=None
bxerroretblob=None
bxqualityocc1blob=None
bxqualityocc2blob=None
bxqualityetblob=None
cmsbxindexblob=None
beam1intensityblob=None
beam2intensityblob=None
beamstatus='STABLE BEAMS'
beamenergy=4000.
instlumierror=0.
instlumiquality=1
if perbunchrawdata:
withDetails=True
bxdataocc1blob=perbunchrawdata[cmslsnum][0]
bxerrorocc1blob=perbunchrawdata[cmslsnum][1]
bxqualityocc1blob=perbunchrawdata[cmslsnum][2]
bxdataocc2blob=perbunchrawdata[cmslsnum][3]
bxerrorocc2blob=perbunchrawdata[cmslsnum][4]
bxqualityocc2blob=perbunchrawdata[cmslsnum][5]
bxdataetblob=perbunchrawdata[cmslsnum][6]
bxerroretblob=perbunchrawdata[cmslsnum][7]
bxqualityetblob=perbunchrawdata[cmslsnum][8]
bxindexblob=perbunchrawdata[cmslsnum][9]
beam1intensityblob=perbunchrawdata[cmslsnum][10]
beam2intensityblob=perbunchrawdata[cmslsnum][11]
beamstatus=perbunchrawdata[cmslsnum][12]
beamenergy=perbunchrawdata[cmslsnum][13]
instlumierror=perbunchrawdata[cmslsnum][14]
instlumiquality=perbunchrawdata[cmslsnum][15]
elif bxdistribution:
withDetails=True
bxdataArray=array.array('f')
bxerrorArray=array.array('f')
bxqualityArray=array.array('h')
cmsbxindexArray=array.array('h')
beam1intensityArray=array.array('f')
beam2intensityArray=array.array('f')
for bxidx in range(1,3565):
lumifraction=0.0
if bxidx in perbunchrawdata:
lumifraction=perbunchrawdata[bxidx]
bxlumivalue=float(instlumi*lumifraction)
bxdataArray.append(bxlumivalue)
beam1intensityArray.append(9124580336.0)
beam1intensityArray.append(8932813306.0)
cmsbxindexArray.append(bxidx)
bxqualityArray.append(1)
bxerrorArray.append(0.0)
bxdataocc1blob=CommonUtil.packArraytoBlob(bxdataArray)
bxdataocc2blob=CommonUtil.packArraytoBlob(bxdataArray)
bxdataetblob=CommonUtil.packArraytoBlob(bxdataArray)
bxerrorocc1blob=CommonUtil.packArraytoBlob(bxerrorArray)
bxerrorocc2blob=CommonUtil.packArraytoBlob(bxerrorArray)
bxerroretblob=CommonUtil.packArraytoBlob(bxerrorArray)
bxqualityocc1blob=CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityocc2blob=CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityetblob=CommonUtil.packArraytoBlob(bxqualityArray)
cmsbxindexblob=CommonUtil.packArraytoBlob(cmsbxindexArray)
beam1intensityblob=CommonUtil.packArraytoBlob(beam1intensityArray)
beam2intensityblob=CommonUtil.packArraytoBlob(beam2intensityArray)
if deliveredonly:
perlsdata=[0,float(instlumi),instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob]
else:
perlsdata=[cmslsnum,float(instlumi),instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob]
lumilsdata[cmslsnum]=perlsdata
#print 'toinsert from scratch',lumilsdata
print(lumilsdata)
dbsession.transaction().start(False)
(revision_id,entry_id,data_id)=dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(),runnum,lumirundata,branchinfo,'LUMIDATA')
dataDML.bulkInsertLumiLSSummary(dbsession,runnum,data_id,lumilsdata,'LUMISUMMARYV2',withDetails=withDetails)
dbsession.transaction().commit()
print('(hf_tagid,hf_tagname) ', (hf_tagid, hf_tagname))
hfdataidmap = revisionDML.dataIdsByTagId(destsession.nominalSchema(),
hf_tagid, [int(options.runnum)],
withcomment=False,
lumitype='HF')
destsession.transaction().commit()
print('dest hfdataidmap ', hfdataidmap)
if int(options.runnum) in hfdataidmap:
print('existing old hf data in destdb of run ', options.runnum,
hfdataidmap[int(options.runnum)])
destsession.transaction().start(False)
[destlumidataid, desttrgdataid,
desthltdataid] = hfdataidmap[int(options.runnum)]
(lumirevid, lumientryid, lumidataid) = dataDML.addLumiRunDataToBranch(
destsession.nominalSchema(), int(options.runnum), rundat,
branchinfo, nameDealer.lumidataTableName())
dataDML.bulkInsertLumiLSSummary(destsession, int(options.runnum),
lumidataid, lsdat,
nameDealer.lumisummaryv2TableName())
destsession.transaction().commit()
destsession.transaction().start(False)
revisionDML.addRunToCurrentDataTag(destsession.nominalSchema(),
int(options.runnum),
lumidataid,
desttrgdataid,
desthltdataid,
lumitype='HF',
comment=options.comment)
destsession.transaction().commit()
else:
def main():
from RecoLuminosity.LumiDB import sessionManager, queryDataSource
parser = argparse.ArgumentParser(
prog=os.path.basename(sys.argv[0]),
description="migrate lumidb schema",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-c',
dest='connect',
action='store',
required=False,
default='oracle://devdb10/cms_xiezhen_dev',
help='connect string to dest db(required)')
parser.add_argument('-lumisource',
dest='lumisource',
action='store',
required=False,
default='oracle://cms_orcoff_prod/CMS_LUMI_PROD',
help='connect string to source lumi db')
#parser.add_argument('-runinfo',dest='runinfo',action='store',required=False,default='oracle://cms_orcoff_prod/CMS_RUNINFO',help='connect string to runinfo db')
parser.add_argument('-P',
dest='authpath',
action='store',
required=False,
default='/afs/cern.ch/user/x/xiezhen',
help='path to authentication file')
parser.add_argument('-r',
dest='runnumber',
action='store',
required=True,
help='run number')
parser.add_argument('--debug',
dest='debug',
action='store_true',
help='debug')
args = parser.parse_args()
runnumber = int(args.runnumber)
print 'processing run ', runnumber
#runinfosvc=sessionManager.sessionManager(args.runinfo,authpath=args.authpath,debugON=args.debug)
lumisvc = sessionManager.sessionManager(args.lumisource,
authpath=args.authpath,
debugON=args.debug)
destsvc = sessionManager.sessionManager(args.connect,
authpath=args.authpath,
debugON=args.debug)
#runinfosession=runinfosvc.openSession(isReadOnly=True,cpp2sqltype=[('unsigned int','NUMBER(10)'),('unsigned long long','NUMBER(20)')])
lumisession = lumisvc.openSession(isReadOnly=True,
cpp2sqltype=[
('unsigned int', 'NUMBER(10)'),
('unsigned long long', 'NUMBER(20)')
])
try:
#[l1key,amodetag,egev,sequence,hltkey,fillnum,starttime,stoptime]=queryDataSource.runsummary(runinfosession,'CMS_RUNINFO',runnumber,complementalOnly=False)
#print 'runsummary ',[l1key,amodetag,egev,sequence,hltkey,fillnum,starttime,stoptime]
lumidata = queryDataSource.uncalibratedlumiFromOldLumi(
lumisession, runnumber)
#print 'lumidata ',lumidata
[bitnames,
trglsdata] = queryDataSource.trgFromOldLumi(lumisession, runnumber)
#print 'trg data ',bitnames,trglsdata
[pathnames,
hltlsdata] = queryDataSource.hltFromOldLumi(lumisession, runnumber)
#print 'hlt data ',pathnames,hltlsdata
lumisession.transaction().commit()
#runinfosession.transaction().commit()
del lumisession
del lumisvc
#del runinfosession
#del runinfosvc
destsession = destsvc.openSession(isReadOnly=False,
cpp2sqltype=[('unsigned int',
'NUMBER(10)'),
('unsigned long long',
'NUMBER(20)')])
destsession.transaction().start(False)
branchrevision_id = DATABRANCH_ID
#print 'data branchid ',branchrevision_id
#dataDML.insertRunSummaryData(destsession.nominalSchema(),runnumber,[l1key,amodetag,egev,sequence,hltkey,fillnum,starttime,stoptime],complementalOnly=False)
(lumirevid, lumientryid, lumidataid) = dataDML.addLumiRunDataToBranch(
destsession.nominalSchema(), runnumber, [args.lumisource],
(branchrevision_id, 'DATA'))
bitzeroname = bitnames.split(',')[0]
trgrundata = [args.lumisource, bitzeroname, bitnames]
(trgrevid, trgentryid, trgdataid) = dataDML.addTrgRunDataToBranch(
destsession.nominalSchema(), runnumber, trgrundata,
(branchrevision_id, 'DATA'))
hltrundata = [pathnames, args.lumisource]
(hltrevid, hltentryid, hltdataid) = dataDML.addHLTRunDataToBranch(
destsession.nominalSchema(), runnumber, hltrundata,
(branchrevision_id, 'DATA'))
destsession.transaction().commit()
dataDML.bulkInsertLumiLSSummary(destsession, runnumber, lumidataid,
lumidata, 500)
#
dataDML.bulkInsertTrgLSData(destsession, runnumber, trgdataid,
trglsdata, 500)
dataDML.bulkInsertHltLSData(destsession, runnumber, hltdataid,
hltlsdata, 500)
del destsession
del destsvc
except:
raise
destsession.transaction().start(False)
(lumibranchid,lumibranchparent)=revisionDML.branchInfoByName(destsession.nominalSchema(),'DATA')
branchinfo=(lumibranchid,'DATA')
print('branchinfo ',branchinfo)
(hf_tagid,hf_tagname)=revisionDML.currentDataTag(destsession.nominalSchema(),lumitype='HF')
print('(hf_tagid,hf_tagname) ',(hf_tagid,hf_tagname))
hfdataidmap=revisionDML.dataIdsByTagId(destsession.nominalSchema(),hf_tagid,[int(options.runnum)],withcomment=False,lumitype='HF')
destsession.transaction().commit()
print('dest hfdataidmap ',hfdataidmap)
if int(options.runnum) in hfdataidmap:
print('existing old hf data in destdb of run ',options.runnum,hfdataidmap[int(options.runnum)])
destsession.transaction().start(False)
[destlumidataid,desttrgdataid,desthltdataid]=hfdataidmap[int(options.runnum)]
(lumirevid,lumientryid,lumidataid)=dataDML.addLumiRunDataToBranch(destsession.nominalSchema(),int(options.runnum),rundat,branchinfo,nameDealer.lumidataTableName())
dataDML.bulkInsertLumiLSSummary(destsession,int(options.runnum),lumidataid,lsdat,nameDealer.lumisummaryv2TableName())
destsession.transaction().commit()
destsession.transaction().start(False)
revisionDML.addRunToCurrentDataTag(destsession.nominalSchema(),int(options.runnum),lumidataid,desttrgdataid,desthltdataid,lumitype='HF',comment=options.comment)
destsession.transaction().commit()
else:
print('non-existing old hf data in destdb of run ',int(options.runnum))
destsession.transaction().start(False)
(lumirevid,lumientryid,lumidataid)=dataDML.addLumiRunDataToBranch(destsession.nominalSchema(),int(options.runnum),rundat,branchinfo,nameDealer.lumidataTableName())
print((lumirevid,lumientryid,lumidataid))
dataDML.bulkInsertLumiLSSummary(destsession,int(options.runnum),lumidataid,lsdat,nameDealer.lumisummaryv2TableName())
destsession.transaction().commit()
destsession.transaction().start(False)
revisionDML.addRunToCurrentDataTag(destsession.nominalSchema(),int(options.runnum),lumidataid,0,0,lumitype='HF',comment=options.comment)
destsession.transaction().commit()
def insertLumischemaV2(dbsession,runnum,datasource,perlsrawdata,perbunchrawdata,deliveredonly=False):
'''
input:
lumirundata[datasource]
perlsrawdata: {cmslsnum:instlumi}
perbunchrawdata: {bxidx:lumifraction}
lumilsdata {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxlumivalue_occ1,bxlumierror_occ1,bxlumiquality_occ1,bxlumivalue_occ2,bxlumierror_occ2,bxlumiquality_occ2,bxlumivalue_et,bxlumierror_et,bxlumiquality_et]}
'''
branchrevision_id=3#databranch_id
branchinfo=(branchrevision_id,'DATA')
lumirundata=[datasource]
lumilsdata={}
dbsession.transaction().start(True)
oldlumidataid=dataDML.guessLumiDataIdByRun(dbsession.nominalSchema(),runnum)
dbsession.transaction().commit()
for cmslsnum,instlumi in perlsrawdata.items():
mystartorbit=startorbit+numorbit*(cmslsnum-1)
bxdataocc1blob=None
bxdataocc2blob=None
bxdataetblob=None
bxerrorocc1blob=None
bxerrorocc2blob=None
bxerroretblob=None
bxqualityocc1blob=None
bxqualityocc2blob=None
bxqualityetblob=None
cmsbxindexblob=None
beam1intensityblob=None
beam2intensityblob=None
if perbunchrawdata:
bxdataArray=array.array('f')
bxerrorArray=array.array('f')
bxqualityArray=array.array('h')
cmsbxindexArray=array.array('h')
beam1intensityArray=array.array('f')
beam2intensityArray=array.array('f')
for bxidx in range(1,3565):
lumifraction=0.0
if perbunchrawdata.has_key(bxidx):
lumifraction=perbunchrawdata[bxidx]
bxlumivalue=float(instlumi*lumifraction)/float(bunchnorm)
bxdataArray.append(bxlumivalue)
beam1intensityArray.append(9124580336.0)
beam1intensityArray.append(8932813306.0)
cmsbxindexArray.append(bxidx)
bxqualityArray.append(1)
bxerrorArray.append(0.0)
bxdataocc1blob=CommonUtil.packArraytoBlob(bxdataArray)
bxdataocc2blob=CommonUtil.packArraytoBlob(bxdataArray)
bxdataetblob=CommonUtil.packArraytoBlob(bxdataArray)
bxerrorocc1blob=CommonUtil.packArraytoBlob(bxerrorArray)
bxerrorocc2blob=CommonUtil.packArraytoBlob(bxerrorArray)
bxerroretblob=CommonUtil.packArraytoBlob(bxerrorArray)
bxqualityocc1blob=CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityocc2blob=CommonUtil.packArraytoBlob(bxqualityArray)
bxqualityetblob=CommonUtil.packArraytoBlob(bxqualityArray)
cmsbxindexblob=CommonUtil.packArraytoBlob(cmsbxindexArray)
beam1intensityblob=CommonUtil.packArraytoBlob(beam1intensityArray)
beam2intensityblob=CommonUtil.packArraytoBlob(beam2intensityArray)
if deliveredonly:
perlsdata=[0,float(instlumi)/float(6370),0.0,1,'STABLE BEAMS',beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob]
else:
perlsdata=[cmslsnum,float(instlumi)/float(6370),0.0,1,'STABLE BEAMS',beamenergy,numorbit,mystartorbit,cmsbxindexblob,beam1intensityblob,beam2intensityblob,bxdataocc1blob,bxerrorocc1blob,bxqualityocc1blob,bxdataocc2blob,bxerrorocc2blob,bxqualityocc2blob,bxdataetblob,bxerroretblob,bxqualityetblob]
lumilsdata[cmslsnum]=perlsdata
print 'toinsert from scratch',lumilsdata
dbsession.transaction().start(False)
(revision_id,entry_id,data_id)=dataDML.addLumiRunDataToBranch(dbsession.nominalSchema(),runnum,lumirundata,branchinfo)
newlumilsnumMin=min(lumilsdata.keys())
newlumilsnumMax=max(lumilsdata.keys())
#update id of existing to new
#update lumisummaryv2 set DATA_ID=:data_id where DATA_ID=:oldid and lumilsnum<newlumilsnumMin or lumilsnum>newlumilsnumMax
#lumidataid is not None update lumilsdata set lumidataid=:newlumidataid where runnum=:run a
inputData=coral.AttributeList()
inputData.extend('dataid','unsigned long long')
inputData.extend('oldid','unsigned long long')
inputData.extend('lumilsnumMin','unsigned int')
inputData.extend('lumilsnumMax','unsigned int')
inputData['dataid'].setData( data_id )
inputData['oldid'].setData( oldlumidataid )
inputData['lumilsnumMin'].setData( newlumilsnumMin )
inputData['lumilsnumMax'].setData( newlumilsnumMax )
db=dbUtil.dbUtil(dbsession.nominalSchema())
db.singleUpdate('LUMISUMMARYV2','DATA_ID=:dataid','DATA_ID=:oldid AND LUMILSNUM<:lumilsnumMin OR lumilsnum>:lumilsnumMax',inputData)
print 'to update existing id ',oldlumidataid,' outside region ',newlumilsnumMin,' , ',newlumilsnumMax
dataDML.bulkInsertLumiLSSummary(dbsession,runnum,data_id,lumilsdata)
dbsession.transaction().commit()