def __init__(self,inputfilename):
filelist=inputfilename.split('+')
self.__inputresultfiles=filelist[0:-1]
self.__inputselectionfile=filelist[-1]
self.__inputResultHeader=[]
self.__inputResult=[]
self.__inputSelectionFileparsingResult=None
if len(self.__inputselectionfile)!=0:
basename,extension=os.path.splitext(self.__inputselectionfile)
if extension=='.csv':#if file ends with .csv,use csv parser,else parse as json file
self.__inputSelectionFileparsingResult=csvSelectionParser.csvSelectionParser(self.__inputselectionfile)
else:
selectf=open(self.__inputselectionfile,'r')
inputfilecontent=selectf.read()
self.__inputSelectionFileparsingResult=selectionParser.selectionParser(inputfilecontent)
if len(self.__inputresultfiles)!=0:
header=''
for f in self.__inputresultfiles:
ifile=open(f)
hasHeader=filehasHeader(ifile)
#hasHeader=csv.Sniffer().has_header(ifile.read(1024)) #sniffer doesn't work well , replace with custom
ifile.seek(0)
csvReader=csv.reader(ifile,delimiter=',')
irow=0
for row in csvReader:
if hasHeader and irow==0:
self.__inputResultHeader=row
else:
self.__inputResult.append(row)
irow=irow+1
ifile.close()
def __init__(self, inputfilename):
filelist = inputfilename.split('+')
self.__inputresultfiles = filelist[0:-1]
self.__inputselectionfile = filelist[-1]
self.__inputResultHeader = []
self.__inputResult = []
self.__inputSelectionFileparsingResult = None
if len(self.__inputselectionfile) != 0:
basename, extension = os.path.splitext(self.__inputselectionfile)
if extension == '.csv': #if file ends with .csv,use csv parser,else parse as json file
self.__inputSelectionFileparsingResult = csvSelectionParser.csvSelectionParser(
self.__inputselectionfile)
else:
selectf = open(self.__inputselectionfile, 'r')
inputfilecontent = selectf.read()
self.__inputSelectionFileparsingResult = selectionParser.selectionParser(
inputfilecontent)
if len(self.__inputresultfiles) != 0:
header = ''
for f in self.__inputresultfiles:
ifile = open(f)
hasHeader = filehasHeader(ifile)
#hasHeader=csv.Sniffer().has_header(ifile.read(1024)) #sniffer doesn't work well , replace with custom
ifile.seek(0)
csvReader = csv.reader(ifile, delimiter=',')
irow = 0
for row in csvReader:
if hasHeader and irow == 0:
self.__inputResultHeader = row
else:
self.__inputResult.append(row)
irow = irow + 1
ifile.close()
def main():
c=constants()
parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description="Patch LumiData")
parser.add_argument('-c',dest='destination',action='store',required=True,help='destination lumi db (required)')
parser.add_argument('-s',dest='source',action='store',required=False,help='source db (required except for lumicalib)')
parser.add_argument('-P',dest='authpath',action='store',required=True,help='path to authentication file (required)')
parser.add_argument('-r',dest='runnumber',action='store',required=False,help='run number (optional)')
parser.add_argument('-i',dest='inputfile',action='store',required=False,help='run selection file(optional)')
parser.add_argument('-delta',dest='delta',action='store',required=False,help='calibration factor wrt old data in lumiDB (required for lumicalib)')
parser.add_argument('action',choices=['deadtimeGT','deadtimeWBM','lumicalib','runtimestamp'],help='deadtimeGT: patch deadtime to deadtimebeamactive,\ndeadtimeWBM: patch deadtimeWBM to deadtimebeamactive,\nlumicalib: recalibrate inst lumi by delta where delta>1\n runtimestamp: add start,stop run timestamp where empty')
parser.add_argument('--dryrun',dest='dryrun',action='store_true',help='only print datasource query result, do not update destination')
parser.add_argument('--debug',dest='debug',action='store_true',help='debug')
args=parser.parse_args()
runnumber=args.runnumber
destConnect=args.destination
sourceConnect=args.source
if args.authpath and len(args.authpath)!=0:
os.environ['CORAL_AUTH_PATH']=args.authpath
svc=coral.ConnectionService()
sourcesession=None
if sourceConnect:
sourcesession=svc.connect(sourceConnect,accessMode=coral.access_ReadOnly)
sourcesession.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)")
sourcesession.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)")
destsession=svc.connect(destConnect,accessMode=coral.access_Update)
destsession.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)")
destsession.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)")
if args.debug:
msg=coral.MessageStream('')
msg.setMsgVerbosity(coral.message_Level_Debug)
if args.dryrun:
c.isdryrun=True
else:
c.isdryrun=False
deadresult={}
if args.action == 'deadtimeGT':
if not sourceConnect:
raise Exception('deadtimeGT action requies -s option for source connection string')
deadresult=GTdeadtimeBeamActiveForRun(sourcesession,c,runnumber)
print 'reading from ',sourceConnect
print 'run : ',runnumber
print 'LS:deadtimebeamactive'
#print deadresult
if deadresult and len(deadresult)!=0:
for cmsls,deadtimebeamactive in deadresult.items():
print cmsls,deadtimebeamactive
else:
print 'no deadtime found for run ',runnumber
print 'exit'
return
print 'total LS: ',len(deadresult)
# if len(deadresult)!=max( [ (deadresult[x],x) for x in deadresult] )[1]:
if len(deadresult)!=max( [ x for x in deadresult.keys() ] ):
print 'total ls: ',len(deadresult)
#print 'max key: ',max( [ x for x in deadresult.keys()])
print 'alert: missing Lumi Sections in the middle'
for x in range(1,max( [ x for x in deadresult.keys()] ) ):
if x not in deadresult:
print 'filling up LS deadtime with 0: LS : ',x
deadresult[x]=0
#print deadresult
if not args.dryrun:
print 'updating ',destConnect
nupdated=patchDeadtimeForRun(destsession,c,int(runnumber),deadresult)
print 'number of updated rows ',nupdated
elif args.action == 'deadtimeWBM':
if not sourceConnect:
raise Exception('deadtimeWBM action requies -s option for source connection string')
deadresult=WBMdeadtimeBeamActiveForRun(sourcesession,c,runnumber)
print 'reading from ',sourceConnect
print 'run : ',runnumber
print 'LS:deadtimebeamactive'
#print deadresult
if deadresult and len(deadresult)!=0:
for cmsls,deadtimebeamactive in deadresult.items():
print cmsls,deadtimebeamactive
else:
print 'no deadtime found for run ',runnumber
print 'exit'
return
print 'total LS: ',len(deadresult)
if len(deadresult)!=max( [ (deadresult[x],x) for x in deadresult])[1]:
print 'alert: missing Lumi Sections in the middle'
for x in range(1,max( [ (deadresult[x],x) for x in deadresult])[1]):
if x not in deadresult:
print 'filling up LS deadtime with 0: LS : ',x
deadresult[x]=0
print deadresult
if not args.dryrun:
print 'updating ',destConnect
nupdated=patchDeadtimeForRun(destsession,c,int(runnumber),deadresult)
print 'number of updated rows ',nupdated
elif args.action == 'lumicalib':
if not args.delta or args.delta==0:
raise Exception('Must provide non-zero -delta argument')
runnums=[]
if args.runnumber:
runnums.append(args.runnumber)
elif args.inputfile:
basename,extension=os.path.splitext(args.inputfile)
if extension=='.csv':#if file ends with .csv,use csv parser,else parse as json file
fileparsingResult=csvSelectionParser.csvSelectionParser(args.inputfile)
else:
f=open(args.inputfile,'r')
inputfilecontent=f.read()
fileparsingResult=selectionParser.selectionParser(inputfilecontent)
if not fileparsingResult:
raise Exception('failed to parse the input file '+ifilename)
#print fileparsingResult.runsandls()
runnums=fileparsingResult.runs()
#print runnums
else:
raise Exception('Must provide -r or -i argument as input')
nupdated=recalibrateLumiForRun(destsession,c,args.delta,runnums)
elif args.action == 'runtimestamp':
if not sourceConnect:
raise Exception('runtimestamp action requies -s option for source connection string')
if not args.runnumber and not args.inputfile: #if no runnumber nor input file specified, check all
runnums=missingTimeRuns(destsession,c)
print 'these runs miss start/stop time: ',runnums
print 'total : ',len(runnums)
elif args.runnumber:
runnums=[int(args.runnumber)]
elif args.inputfile:
basename,extension=os.path.splitext(args.inputfile)
if extension=='.csv':#if file ends with .csv,use csv parser,else parse as json file
fileparsingResult=csvSelectionParser.csvSelectionParser(args.inputfile)
else:
f=open(args.inputfile,'r')
inputfilecontent=f.read()
fileparsingResult=selectionParser.selectionParser(inputfilecontent)
if not fileparsingResult:
raise Exception('failed to parse the input file '+ifilename)
runnums=fileparsingResult.runs()
result=getTimeForRun(sourcesession,c,runnums)
#for run,(startTimeT,stopTimeT) in result.items():
#print 'run: ',run
#if not startTimeT or not stopTimeT:
#print 'None'
#else:
#print 'start: ',startTimeT
#print 'stop: ',stopTimeT
addTimeForRun(destsession,c,result)
if sourcesession:
del sourcesession
del destsession
del svc
[cmslsnum, delivered, recorded, (xingIdx, xingVal)])
#{run:[[cmslsnum,delivered,recorded,xingInstlumiArray]..]}
events.close()
else:
session = svc.openSession(isReadOnly=True,
cpp2sqltype=[('unsigned int', 'NUMBER(10)'),
('unsigned long long',
'NUMBER(20)')])
finecorrections = None
if options.runnumber:
inputRange = {int(options.runnumber): None}
else:
basename, extension = os.path.splitext(options.inputfile)
if extension == '.csv': # if file ends with .csv, use csv
# parser, else parse as json file
fileparsingResult = csvSelectionParser.csvSelectionParser(
options.inputfile)
else:
f = open(options.inputfile, 'r')
inputfilecontent = f.read()
inputRange = selectionParser.selectionParser(
inputfilecontent).runsandls()
if not inputRange:
print('failed to parse the input file', options.inputfile)
raise
if not options.withoutFineCorrection:
rruns = inputRange.keys()
schema = session.nominalSchema()
session.transaction().start(True)
finecorrections = lumiCorrections.correctionsForRange(
schema, rruns)
session.transaction().commit()
raise RuntimeError, "Could not open '%s' as an output root file" % output
pileupHist.Write()
for hist in histList:
hist.Write()
histFile.Close()
# pprint (csvDict)
sys.exit()
## Get input source
if options.runnumber:
inputRange = options.runnumber
else:
basename, extension = os.path.splitext(options.inputfile)
if extension == ".csv": # if file ends with .csv, use csv
# parser, else parse as json file
fileparsingResult = csvSelectionParser.csvSelectionParser(options.inputfile)
else:
f = open(options.inputfile, "r")
inputfilecontent = f.read()
inputRange = selectionParser.selectionParser(inputfilecontent)
if not inputRange:
print "failed to parse the input file", options.inputfile
raise
recordedData = LumiQueryAPI.recordedLumiForRange(session, parameters, inputRange)
## pprint (recordedData)
for runDTarray in recordedData:
runNumber = runDTarray[0]
deadTable = runDTarray[2]
if options.saveRuns:
hist = fillPileupHistogram(deadTable, parameters, runNumber=runNumber, debug=options.debugLumi)
