def generateLumiLSdataForRun(lsdata, lumirundata, beamsta):
'''
input:
lsdata: [(cmslsnum,instlumi),...]
lumirundata: [datasource,nominalegev,ncollidingbunches,ncollidingbunches,starttime,stoptime,o]
beamstatus {cmslsnum:beamstatus}
output:
i.e. bulkInsertLumiLSSummary expected input: {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit]}
'''
lumip = lumiParameters.ParametersObject()
result = {}
beamstatus = 'STABLE BEAMS'
beamenergy = lumirundata[1]
numorbit = lumip.numorbit
startorbit = 0
for (cmslsnum, instlumi) in lsdata:
lumilsnum = cmslsnum
instlumierror = 0.0
instlumiquality = 0
startorbit = (cmslsnum - 1) * numorbit
if beamsta and beamsta.has_key(cmslsnum):
beamstatus = beamsta[cmslsnum]
result[lumilsnum] = [
cmslsnum, instlumi, instlumierror, instlumiquality, beamstatus,
beamenergy, numorbit, startorbit
]
return result
def toinstlumi(i):
'''
input: luminosity integrated in ls
output: avg instlumi in ls Hz/ub
'''
lumip = lumiParameters.ParametersObject()
lslength = lumip.lslengthsec()
return float(i) / lslength
action='store_true',
help='debug')
parser.add_argument('action', choices=allowedActions, help='type of plots')
options = parser.parse_args()
if options.yscale == 'both' and options.interactive:
print(
'--interactive mode can draw either yscale log or linear at a time'
)
exit(0)
outplotfilename = options.outplot
if not outplotfilename:
outplotfilename = actiontofilebasemap[options.action]
outtextfilename = outplotfilename + '.csv'
if options.withoutTextoutput:
outtextfilename = None
lumip = lumiParameters.ParametersObject()
svc = sessionManager.sessionManager(options.connect,
authpath=options.authpath,
siteconfpath=options.siteconfpath,
debugON=options.debug)
session = svc.openSession(isReadOnly=True,
cpp2sqltype=[('unsigned int', 'NUMBER(10)'),
('unsigned long long', 'NUMBER(20)')
])
lslength = lumip.lslengthsec()
begtime = options.begintime
endtime = options.endtime
lut = lumiTime.lumiTime()
if not endtime:
endtime = lut.DatetimeToStr(datetime.datetime.utcnow(),
customfm='%m/%d/%y %H:%M:%S')
def deliveredLumiForIds(schema,irunlsdict,dataidmap,runsummaryMap,beamstatusfilter=None,timeFilter=None,normmap=None,withBXInfo=False,bxAlgo=None,xingMinLum=None,withBeamIntensity=False,lumitype='HF',minbiasXsec=None):
'''
delivered lumi (including calibration,time integral)
input:
irunlsdict: {run:[lsnum]}, where [lsnum]==None means all ; [lsnum]==[] means selected ls
dataidmap : {run:(lumiid,trgid,hltid)}
runsummaryMap: {run:[l1key(0),amodetag(1),egev(2),hltkey(3),fillnum(4),fillscheme(5),starttime(6),stoptime(7)]}
beamstatus: LS filter on beamstatus
normmap: {since:[corrector(0),{paramname:paramvalue}(1),amodetag(2),egev(3),comment(4)]} if normmap empty, means without-correction , if notnormmap means without-correction
withBXInfo: get per bunch info (optional)
bxAlgo: algoname for bx values (optional) ['OCC1','OCC2','ET']
xingMinLum: cut on bx lumi value (optional)
withBeamIntensity: get beam intensity info (optional)
lumitype: luminosity source
output:
result {run:[lumilsnum(0),cmslsnum(1),timestamp(2),beamstatus(3),beamenergy(4),deliveredlumi(5),calibratedlumierr(6),(bxidxlist,bxvalues,bxerrs)(7),(bxidx,b1intensities,b2intensities)(8),fillnum(9),pu(10)]}
special meanings:
{run:None} None means no run in lumiDB,
{run:[]} [] means no lumi for this run in lumiDB
{run:cmslsnum(1)==0} means either not cmslsnum or iscms but not selected
lumi unit: /ub
'''
result = {}
lumip=lumiParameters.ParametersObject()
lumirundata=dataDML.lumiRunByIds(schema,dataidmap,lumitype=lumitype)
instresult=instLumiForIds(schema,irunlsdict,dataidmap,runsummaryMap,beamstatusfilter=beamstatusfilter,timeFilter=timeFilter,withBXInfo=withBXInfo,bxAlgo=bxAlgo,withBeamIntensity=withBeamIntensity,lumitype=lumitype)
intglumimap={}
if lumitype=='HF':
intglumimap=dataDML.intglumiForRange(schema,irunlsdict.keys())#some runs need drift correction
allsince=[]
if normmap:
allsince=normmap.keys()
allsince.sort()
correctorname='fPoly' #HF default
correctionparams={'a0':1.0}
runfillschemeMap={}
fillschemePatternMap={}
if lumitype=='PIXEL':
correctorname='fPolyScheme' #PIXEL default
fillschemePatternMap=dataDML.fillschemePatternMap(schema,'PIXEL')
for run,perrundata in instresult.items():
if perrundata is None:
result[run]=None
continue
intglumi=0.
if normmap and intglumimap and run in intglumimap and intglumimap[run]:
intglumi=intglumimap[run]
nBXs=0
if normmap and lumirundata and run in lumirundata and lumirundata[run][2]:
nBXs=lumirundata[run][2]
fillschemeStr=''
if normmap and runsummaryMap and run in runsummaryMap and runsummaryMap[run][5]:
fillschemeStr=runsummaryMap[run][5]
if allsince:
lastsince=allsince[0]
for since in allsince:
if run>=since:
lastsince=since
correctorname=normmap[lastsince][0]
correctionparams=normmap[lastsince][1]
correctioninput=[0.,intglumi,nBXs,fillschemeStr,fillschemePatternMap]
result[run]=[]
for perlsdata in perrundata:#loop over ls
lumilsnum=perlsdata[0]
cmslsnum=perlsdata[1]
timestamp=perlsdata[2]
bs=perlsdata[3]
beamenergy=perlsdata[4]
instluminonorm=perlsdata[5]
correctioninput[0]=instluminonorm
totcorrectionFac=normFunctors.normFunctionCaller(correctorname,*correctioninput,**correctionparams)
fillnum=perlsdata[11]
instcorrectedlumi=totcorrectionFac*instluminonorm
numorbit=perlsdata[8]
numbx=lumip.NBX
lslen=lumip.lslengthsec()
deliveredlumi=instcorrectedlumi*lslen
calibratedbxdata=None
beamdata=None
pu=0.#avgPU
if nBXs and minbiasXsec:
pu=(instcorrectedlumi/nBXs)*minbiasXsec/lumip.rotationRate
if withBXInfo:
(bxidxData,bxvaluesData,bxerrsData)=perlsdata[9]
if lumitype=='HF':
if xingMinLum:
bxidxList=[]
bxvalueList=[]
bxerrList=[]
for idx,bxval in enumerate(bxvaluesData):
correctedbxintlumi=totcorrectionFac*bxval
correctedbxintlumierr=totcorrectionFac*bxerrsData[idx]
if correctedbxintlumi>xingMinLum:
bxidxList.append(bxidxData[idx])
bxvalueList.append(correctedbxintlumi)
bxerrList.append(correctedbxintlumierr)
calibratedbxdata=(bxidxList,bxvalueList,bxerrList)
else:
calibratedbxvalue=[totcorrectionFac*x for x in bxvaluesData]
calibratedlumierr=[totcorrectionFac*x for x in bxerrsData]
calibratedbxdata=(bxidxData,calibratedbxvalue,calibratedlumierr)
if withBeamIntensity:
beamdata=perlsdata[10]
calibratedlumierr=0.0
result[run].append([lumilsnum,cmslsnum,timestamp,bs,beamenergy,deliveredlumi,calibratedlumierr,calibratedbxdata,beamdata,fillnum,pu])
del perlsdata[:]
return result
def generateLumidata(lumirundatafromfile, lsdatafromfile, rundatafromdb,
lsdatafromdb, replacelsMin, replacelsMax):
'''
input:
perrunresultfromfile=[]#source,starttime,stoptime,nls
perlsresultfromfile={} #{lumilsnum:instlumiub}
lumirundatafromdb=[] #[source,nominalegev,ncollidingbunches,starttime,stoptime,nls]
lumilsdatafromdb={}#{lumilsnum:[cmslsnum(0),instlumi(1),instlumierror(2),instlumiquality(3),beamstatus(4),beamenergy(5),numorbit(6),startorbit(7),cmsbxindexblob(8),beamintensityblob_1(9),beamintensityblob_2(10),bxlumivalue_occ1(11),bxlumierror_occ1(12),bxlumiquality_occ1(13),bxlumivalue_occ2(14),bxlumierror_occ2(15),bxlumiquality_occ2(16),bxlumivalue_et(17),bxlumierror_et(18),bxlumiquality_et(19)]}
'''
lumip = lumiParameters.ParametersObject()
numorbit = lumip.numorbit
startorbit = 0
fakebeamenergy = 4000.
fakebeamstatus = 'STABLE BEAMS'
fakefloatArray = array.array('f')
fakeidxArray = array.array('h')
fakeshortArray = array.array('h')
for bxidx in range(1, 3565):
fakeidxArray.append(bxidx)
fakefloatArray.append(0.)
fakeshortArray.append(0)
lumirundata = []
lumilsdata = {}
if rundatafromdb:
lumirundata = rundatafromdb
lumirundata[0] = rundatafromdb[0] + '+file:' + lumirundatafromfile[0]
else:
lu = lumiTime.lumiTime()
source = '+file:' + lumirundatafromfile[0]
nominalegev = fakebeamenergy
ncollidingbunches = 72
starttime = lumirundatafromfile[1]
stoptime = lumirundatafromfile[2]
starttimeT = lu.timestampTodatetimeUTC(starttime)
stoptimeT = lu.timestampTodatetimeUTC(stoptime)
print(starttimeT.day, starttimeT.month, starttimeT.year)
starttimeT_coral = coral.TimeStamp(starttimeT.year, starttimeT.month,
starttimeT.day, starttimeT.hour,
starttimeT.minute,
starttimeT.second, 0)
stoptimeT_coral = coral.TimeStamp(stoptimeT.year, stoptimeT.month,
stoptimeT.day, stoptimeT.hour,
stoptimeT.minute, stoptimeT.second,
0)
nls = lumirundatafromfile[3]
lumirundata = [
source, nominalegev, ncollidingbunches, starttimeT_coral,
stoptimeT_coral, nls
]
if lsdatafromdb:
lumilsdata = lsdatafromdb
if replacelsMin > len(lsdatafromdb):
print('[INFO]Operation: extend an existing run from LS=',
replacelsMin)
lumirundata[5] += len(lsdatafromfile)
else:
print(
'[INFO]Operation: replace instlumi in an existing run LS range=',
replacelsMin, replacelsMax)
else:
print('[INFO]Operation: insert a new fake run')
for lumilsnum in range(replacelsMin, replacelsMax + 1):
instlumi = lsdatafromfile[lumilsnum]
if lumilsnum in lsdatafromdb.keys(): #if this is a hole
lumilsdata[lumilsnum][1] = instlumi
else: #if this is an extension
instlumierror = 0.0
instlumiquality = 0
startorbit = (lumilsnum - 1) * numorbit
cmsbxindexblob = CommonUtil.packArraytoBlob(fakeshortArray)
beamintensityblob_1 = CommonUtil.packArraytoBlob(fakefloatArray)
beamintensityblob_2 = CommonUtil.packArraytoBlob(fakefloatArray)
bxlumivalue_occ1 = CommonUtil.packArraytoBlob(fakefloatArray)
bxlumierror_occ1 = CommonUtil.packArraytoBlob(fakefloatArray)
bxlumiquality_occ1 = CommonUtil.packArraytoBlob(fakeshortArray)
bxlumivalue_occ2 = CommonUtil.packArraytoBlob(fakefloatArray)
bxlumierror_occ2 = CommonUtil.packArraytoBlob(fakefloatArray)
bxlumiquality_occ2 = CommonUtil.packArraytoBlob(fakeshortArray)
bxlumivalue_et = CommonUtil.packArraytoBlob(fakefloatArray)
bxlumierror_et = CommonUtil.packArraytoBlob(fakefloatArray)
bxlumiquality_et = CommonUtil.packArraytoBlob(fakeshortArray)
lumilsdata[lumilsnum] = [
0, instlumi, instlumierror, instlumiquality, fakebeamstatus,
fakebeamenergy, numorbit, startorbit, cmsbxindexblob,
beamintensityblob_1, beamintensityblob_2, bxlumivalue_occ1,
bxlumierror_occ1, bxlumiquality_occ1, bxlumivalue_occ2,
bxlumierror_occ2, bxlumiquality_occ2, bxlumivalue_et,
bxlumierror_et, bxlumiquality_et
]
return (lumirundata, lumilsdata)
def getSpecificLumi(schema,
fillnum,
inputdir,
dataidmap,
normmap,
xingMinLum=0.0,
amodetag='PROTPHYS',
bxAlgo='OCC1'):
'''
specific lumi in 1e-30 (ub-1s-1) unit
lumidetail occlumi in 1e-27
1309_lumi_401_CMS.txt
time(in seconds since January 1,2011,00:00:00 UTC) stab(fraction of time spent in stable beams for this time bin) l(lumi in Hz/ub) dl(point-to-point error on lumi in Hz/ub) sl(specific lumi in Hz/ub) dsl(error on specific lumi)
20800119.0 1 -0.889948 0.00475996848729 0.249009 0.005583287562 -0.68359 6.24140208607 0.0 0.0 0.0 0.0 0.0 0.0 0.0383576 0.00430892097862 0.0479095 0.00430892097862 66.6447 4.41269758764 0.0 0.0 0.0
result [(time,beamstatusfrac,lumi,lumierror,speclumi,speclumierror)]
'''
t = lumiTime.lumiTime()
fillbypos = {
} #{bxidx:[[ts,beamstatusfrac,lumi,lumierror,spec1,specerror],[]]}
runtimesInFill = getFillFromDB(schema, fillnum) #{runnum:starttimestr}
runlist = runtimesInFill.keys()
if not runlist: return fillbypos
irunlsdict = dict(zip(runlist, [None] * len(runlist)))
#prirunlsdict
GrunsummaryData = lumiCalcAPI.runsummaryMap(session.nominalSchema(),
irunlsdict)
lumidetails = lumiCalcAPI.deliveredLumiForIds(schema,
irunlsdict,
dataidmap,
GrunsummaryData,
beamstatusfilter=None,
normmap=normmap,
withBXInfo=True,
bxAlgo=bxAlgo,
xingMinLum=xingMinLum,
withBeamIntensity=True,
lumitype='HF')
#
#output: {run:[lumilsnum(0),cmslsnum(1),timestamp(2),beamstatus(3),beamenergy(4),deliveredlumi(5),calibratedlumierr(6),(bxvalues,bxerrs)(7),(bxidx,b1intensities,b2intensities)(8),fillnum(9)]}
#
totalstablebeamls = 0
orderedrunlist = sorted(lumidetails)
for run in orderedrunlist:
perrundata = lumidetails[run]
for perlsdata in perrundata:
beamstatus = perlsdata[3]
if beamstatus == 'STABLE BEAMS':
totalstablebeamls += 1
#print 'totalstablebeamls in fill ',totalstablebeamls
if totalstablebeamls < 10: #less than 10 LS in a fill has 'stable beam', it's no a good fill
print 'fill ', fillnum, ' , having less than 10 stable beam lS, is not good, skip'
return fillbypos
lumiparam = lumiParameters.ParametersObject()
for run in orderedrunlist:
perrundata = lumidetails[run]
for perlsdata in perrundata:
beamstatusfrac = 0.0
tsdatetime = perlsdata[2]
ts = calendar.timegm(tsdatetime.utctimetuple())
beamstatus = perlsdata[3]
if beamstatus == 'STABLE BEAMS':
beamstatusfrac = 1.0
(bxidxlist, bxvaluelist, bxerrolist) = perlsdata[7]
#instbxvaluelist=[x/lumiparam.lslengthsec() for x in bxvaluelist if x]
instbxvaluelist = [x for x in bxvaluelist if x]
maxlumi = 0.0
if len(instbxvaluelist) != 0:
maxlumi = max(instbxvaluelist)
avginstlumi = 0.0
if len(instbxvaluelist) != 0:
avginstlumi = sum(instbxvaluelist)
(intbxidxlist, b1intensities,
b2intensities) = perlsdata[8] #contains only non-zero bx
for bxidx in bxidxlist:
idx = bxidxlist.index(bxidx)
instbxvalue = bxvaluelist[idx]
bxerror = bxerrolist[idx]
if instbxvalue < max(xingMinLum, maxlumi * 0.2):
continue
bintensityPos = -1
try:
bintensityPos = intbxidxlist.index(bxidx)
except ValueError:
pass
if bintensityPos <= 0:
fillbypos.setdefault(bxidx, []).append(
[ts, beamstatusfrac, instbxvalue, bxerror, 0.0, 0.0])
continue
b1intensity = b1intensities[bintensityPos]
b2intensity = b2intensities[bintensityPos]
speclumi = calculateSpecificLumi(instbxvalue, bxerror,
b1intensity, 0.0, b2intensity,
0.0)
fillbypos.setdefault(bxidx, []).append([
ts, beamstatusfrac, instbxvalue, bxerror, speclumi[0],
speclumi[1]
])
return fillbypos
def specificlumiTofile(fillnum, filldata, outdir):
#
#input : fillnum
# filldata: {bxidx:[[lstime,beamstatusfrac,lumivalue,lumierror,speclumi,speclumierr]],[]}
#sorted by bxidx, sorted by lstime inside list
#check outdir/fillnum subdir exists; if not, create it; else outdir=outdir/fillnum
#
if not filldata:
print 'empty input data, do nothing for fill ', fillnum
return
timedict = {
} #{lstime:[[stablebeamfrac,lumi,lumierr,speclumi,speclumierr]]}
filloutdir = os.path.join(outdir, str(fillnum))
if not os.path.exists(filloutdir):
os.mkdir(filloutdir)
for cmsbxidx, perbxdata in filldata.items():
lhcbucket = 0
if cmsbxidx != 0:
lhcbucket = (cmsbxidx - 1) * 10 + 1
a = sorted(perbxdata, key=lambda x: x[0])
filename = str(fillnum) + '_lumi_' + str(lhcbucket) + '_CMS.txt'
linedata = []
for perlsdata in a:
ts = int(perlsdata[0])
beamstatusfrac = perlsdata[1]
lumi = perlsdata[2]
lumierror = perlsdata[3]
#beam1intensity=perlsdata[4]
#beam2intensity=perlsdata[5]
speclumi = perlsdata[4]
speclumierror = perlsdata[5]
if lumi > 0:
linedata.append([
ts, beamstatusfrac, lumi, lumierror, speclumi,
speclumierror
])
if not timedict.has_key(ts):
timedict[ts] = []
timedict[ts].append(
[beamstatusfrac, lumi, lumierror, speclumi, speclumierror])
if len(linedata) > 10: #at least 10 good ls
f = open(os.path.join(filloutdir, filename), 'w')
for line in linedata:
print >> f, '%d\t%e\t%e\t%e\t%e\t%e' % (
line[0], line[1], line[2], line[3], line[4], line[5])
f.close()
#print 'writing avg file'
summaryfilename = str(fillnum) + '_lumi_CMS.txt'
f = None
lstimes = timedict.keys()
lstimes.sort()
fillseg = []
lscounter = 0
for lstime in lstimes:
allvalues = timedict[lstime]
transposedvalues = CommonUtil.transposed(allvalues, 0.0)
bstatfrac = transposedvalues[0][
0] #beamstatus does not change with bx position
lumivals = transposedvalues[1]
lumitot = sum(lumivals)
if bstatfrac == 1.0:
fillseg.append([lstime, lumitot])
lumierrs = transposedvalues[2]
lumierrortot = math.sqrt(sum(map(lambda x: x**2, lumierrs)))
specificvals = transposedvalues[3]
specificavg = sum(specificvals) / float(
len(specificvals)) #avg spec lumi
specificerrs = transposedvalues[4]
specifictoterr = math.sqrt(sum(map(lambda x: x**2, specificerrs)))
specificerravg = specifictoterr / float(len(specificvals))
if lscounter == 0:
f = open(os.path.join(filloutdir, summaryfilename), 'w')
lscounter += 1
print >> f, '%d\t%e\t%e\t%e\t%e\t%e' % (lstime, bstatfrac, lumitot,
lumierrortot, specificavg,
specificerravg)
if f is not None:
f.close()
#print 'writing summary file'
fillsummaryfilename = str(fillnum) + '_bxsum_CMS.txt'
f = open(os.path.join(filloutdir, fillsummaryfilename), 'w')
if len(fillseg) == 0:
print >> f, '%s' % ('#no stable beams')
f.close()
return
previoustime = fillseg[0][0]
boundarytime = fillseg[0][0]
#print 'boundary time ',boundarytime
summaryls = {}
summaryls[boundarytime] = []
for [lstime,
lumitot] in fillseg: #fillseg is everything with stable beam flag
if lstime - previoustime > 50.0:
boundarytime = lstime
#print 'found new boundary ',boundarytime
summaryls[boundarytime] = []
# print 'appending ',boundarytime,lstime,lumitot
summaryls[boundarytime].append([lstime, lumitot])
previoustime = lstime
#print summaryls
summarylstimes = summaryls.keys()
summarylstimes.sort()
lumip = lumiParameters.ParametersObject()
for bts in summarylstimes:
startts = bts
tsdatainseg = summaryls[bts]
#print 'tsdatainseg ',tsdatainseg
stopts = tsdatainseg[-1][0]
plu = max(CommonUtil.transposed(tsdatainseg, 0.0)[1])
lui = sum(CommonUtil.transposed(tsdatainseg,
0.0)[1]) * lumip.lslengthsec()
print >> f, '%d\t%d\t%e\t%e' % (startts, stopts, plu, lui)
f.close()
