def __init__(self,inputfilename): filelist=inputfilename.split('+') self.__inputresultfiles=filelist[0:-1] self.__inputselectionfile=filelist[-1] self.__inputResultHeader=[] self.__inputResult=[] self.__inputSelectionFileparsingResult=None if len(self.__inputselectionfile)!=0: basename,extension=os.path.splitext(self.__inputselectionfile) if extension=='.csv':#if file ends with .csv,use csv parser,else parse as json file self.__inputSelectionFileparsingResult=csvSelectionParser.csvSelectionParser(self.__inputselectionfile) else: selectf=open(self.__inputselectionfile,'r') inputfilecontent=selectf.read() self.__inputSelectionFileparsingResult=selectionParser.selectionParser(inputfilecontent) if len(self.__inputresultfiles)!=0: header='' for f in self.__inputresultfiles: ifile=open(f) hasHeader=filehasHeader(ifile) #hasHeader=csv.Sniffer().has_header(ifile.read(1024)) #sniffer doesn't work well , replace with custom ifile.seek(0) csvReader=csv.reader(ifile,delimiter=',') irow=0 for row in csvReader: if hasHeader and irow==0: self.__inputResultHeader=row else: self.__inputResult.append(row) irow=irow+1 ifile.close()
def __init__(self, inputfilename): filelist = inputfilename.split('+') self.__inputresultfiles = filelist[0:-1] self.__inputselectionfile = filelist[-1] self.__inputResultHeader = [] self.__inputResult = [] self.__inputSelectionFileparsingResult = None if len(self.__inputselectionfile) != 0: basename, extension = os.path.splitext(self.__inputselectionfile) if extension == '.csv': #if file ends with .csv,use csv parser,else parse as json file self.__inputSelectionFileparsingResult = csvSelectionParser.csvSelectionParser( self.__inputselectionfile) else: selectf = open(self.__inputselectionfile, 'r') inputfilecontent = selectf.read() self.__inputSelectionFileparsingResult = selectionParser.selectionParser( inputfilecontent) if len(self.__inputresultfiles) != 0: header = '' for f in self.__inputresultfiles: ifile = open(f) hasHeader = filehasHeader(ifile) #hasHeader=csv.Sniffer().has_header(ifile.read(1024)) #sniffer doesn't work well , replace with custom ifile.seek(0) csvReader = csv.reader(ifile, delimiter=',') irow = 0 for row in csvReader: if hasHeader and irow == 0: self.__inputResultHeader = row else: self.__inputResult.append(row) irow = irow + 1 ifile.close()
def main(): c=constants() parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]),description="Patch LumiData") parser.add_argument('-c',dest='destination',action='store',required=True,help='destination lumi db (required)') parser.add_argument('-s',dest='source',action='store',required=False,help='source db (required except for lumicalib)') parser.add_argument('-P',dest='authpath',action='store',required=True,help='path to authentication file (required)') parser.add_argument('-r',dest='runnumber',action='store',required=False,help='run number (optional)') parser.add_argument('-i',dest='inputfile',action='store',required=False,help='run selection file(optional)') parser.add_argument('-delta',dest='delta',action='store',required=False,help='calibration factor wrt old data in lumiDB (required for lumicalib)') parser.add_argument('action',choices=['deadtimeGT','deadtimeWBM','lumicalib','runtimestamp'],help='deadtimeGT: patch deadtime to deadtimebeamactive,\ndeadtimeWBM: patch deadtimeWBM to deadtimebeamactive,\nlumicalib: recalibrate inst lumi by delta where delta>1\n runtimestamp: add start,stop run timestamp where empty') parser.add_argument('--dryrun',dest='dryrun',action='store_true',help='only print datasource query result, do not update destination') parser.add_argument('--debug',dest='debug',action='store_true',help='debug') args=parser.parse_args() runnumber=args.runnumber destConnect=args.destination sourceConnect=args.source if args.authpath and len(args.authpath)!=0: os.environ['CORAL_AUTH_PATH']=args.authpath svc=coral.ConnectionService() sourcesession=None if sourceConnect: sourcesession=svc.connect(sourceConnect,accessMode=coral.access_ReadOnly) sourcesession.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)") sourcesession.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)") destsession=svc.connect(destConnect,accessMode=coral.access_Update) destsession.typeConverter().setCppTypeForSqlType("unsigned int","NUMBER(10)") destsession.typeConverter().setCppTypeForSqlType("unsigned long long","NUMBER(20)") if args.debug: msg=coral.MessageStream('') msg.setMsgVerbosity(coral.message_Level_Debug) if args.dryrun: c.isdryrun=True else: c.isdryrun=False deadresult={} if args.action == 'deadtimeGT': if not sourceConnect: raise Exception('deadtimeGT action requies -s option for source connection string') deadresult=GTdeadtimeBeamActiveForRun(sourcesession,c,runnumber) print 'reading from ',sourceConnect print 'run : ',runnumber print 'LS:deadtimebeamactive' #print deadresult if deadresult and len(deadresult)!=0: for cmsls,deadtimebeamactive in deadresult.items(): print cmsls,deadtimebeamactive else: print 'no deadtime found for run ',runnumber print 'exit' return print 'total LS: ',len(deadresult) # if len(deadresult)!=max( [ (deadresult[x],x) for x in deadresult] )[1]: if len(deadresult)!=max( [ x for x in deadresult.keys() ] ): print 'total ls: ',len(deadresult) #print 'max key: ',max( [ x for x in deadresult.keys()]) print 'alert: missing Lumi Sections in the middle' for x in range(1,max( [ x for x in deadresult.keys()] ) ): if x not in deadresult: print 'filling up LS deadtime with 0: LS : ',x deadresult[x]=0 #print deadresult if not args.dryrun: print 'updating ',destConnect nupdated=patchDeadtimeForRun(destsession,c,int(runnumber),deadresult) print 'number of updated rows ',nupdated elif args.action == 'deadtimeWBM': if not sourceConnect: raise Exception('deadtimeWBM action requies -s option for source connection string') deadresult=WBMdeadtimeBeamActiveForRun(sourcesession,c,runnumber) print 'reading from ',sourceConnect print 'run : ',runnumber print 'LS:deadtimebeamactive' #print deadresult if deadresult and len(deadresult)!=0: for cmsls,deadtimebeamactive in deadresult.items(): print cmsls,deadtimebeamactive else: print 'no deadtime found for run ',runnumber print 'exit' return print 'total LS: ',len(deadresult) if len(deadresult)!=max( [ (deadresult[x],x) for x in deadresult])[1]: print 'alert: missing Lumi Sections in the middle' for x in range(1,max( [ (deadresult[x],x) for x in deadresult])[1]): if x not in deadresult: print 'filling up LS deadtime with 0: LS : ',x deadresult[x]=0 print deadresult if not args.dryrun: print 'updating ',destConnect nupdated=patchDeadtimeForRun(destsession,c,int(runnumber),deadresult) print 'number of updated rows ',nupdated elif args.action == 'lumicalib': if not args.delta or args.delta==0: raise Exception('Must provide non-zero -delta argument') runnums=[] if args.runnumber: runnums.append(args.runnumber) elif args.inputfile: basename,extension=os.path.splitext(args.inputfile) if extension=='.csv':#if file ends with .csv,use csv parser,else parse as json file fileparsingResult=csvSelectionParser.csvSelectionParser(args.inputfile) else: f=open(args.inputfile,'r') inputfilecontent=f.read() fileparsingResult=selectionParser.selectionParser(inputfilecontent) if not fileparsingResult: raise Exception('failed to parse the input file '+ifilename) #print fileparsingResult.runsandls() runnums=fileparsingResult.runs() #print runnums else: raise Exception('Must provide -r or -i argument as input') nupdated=recalibrateLumiForRun(destsession,c,args.delta,runnums) elif args.action == 'runtimestamp': if not sourceConnect: raise Exception('runtimestamp action requies -s option for source connection string') if not args.runnumber and not args.inputfile: #if no runnumber nor input file specified, check all runnums=missingTimeRuns(destsession,c) print 'these runs miss start/stop time: ',runnums print 'total : ',len(runnums) elif args.runnumber: runnums=[int(args.runnumber)] elif args.inputfile: basename,extension=os.path.splitext(args.inputfile) if extension=='.csv':#if file ends with .csv,use csv parser,else parse as json file fileparsingResult=csvSelectionParser.csvSelectionParser(args.inputfile) else: f=open(args.inputfile,'r') inputfilecontent=f.read() fileparsingResult=selectionParser.selectionParser(inputfilecontent) if not fileparsingResult: raise Exception('failed to parse the input file '+ifilename) runnums=fileparsingResult.runs() result=getTimeForRun(sourcesession,c,runnums) #for run,(startTimeT,stopTimeT) in result.items(): #print 'run: ',run #if not startTimeT or not stopTimeT: #print 'None' #else: #print 'start: ',startTimeT #print 'stop: ',stopTimeT addTimeForRun(destsession,c,result) if sourcesession: del sourcesession del destsession del svc
[cmslsnum, delivered, recorded, (xingIdx, xingVal)]) #{run:[[cmslsnum,delivered,recorded,xingInstlumiArray]..]} events.close() else: session = svc.openSession(isReadOnly=True, cpp2sqltype=[('unsigned int', 'NUMBER(10)'), ('unsigned long long', 'NUMBER(20)')]) finecorrections = None if options.runnumber: inputRange = {int(options.runnumber): None} else: basename, extension = os.path.splitext(options.inputfile) if extension == '.csv': # if file ends with .csv, use csv # parser, else parse as json file fileparsingResult = csvSelectionParser.csvSelectionParser( options.inputfile) else: f = open(options.inputfile, 'r') inputfilecontent = f.read() inputRange = selectionParser.selectionParser( inputfilecontent).runsandls() if not inputRange: print('failed to parse the input file', options.inputfile) raise if not options.withoutFineCorrection: rruns = inputRange.keys() schema = session.nominalSchema() session.transaction().start(True) finecorrections = lumiCorrections.correctionsForRange( schema, rruns) session.transaction().commit()
raise RuntimeError, "Could not open '%s' as an output root file" % output pileupHist.Write() for hist in histList: hist.Write() histFile.Close() # pprint (csvDict) sys.exit() ## Get input source if options.runnumber: inputRange = options.runnumber else: basename, extension = os.path.splitext(options.inputfile) if extension == ".csv": # if file ends with .csv, use csv # parser, else parse as json file fileparsingResult = csvSelectionParser.csvSelectionParser(options.inputfile) else: f = open(options.inputfile, "r") inputfilecontent = f.read() inputRange = selectionParser.selectionParser(inputfilecontent) if not inputRange: print "failed to parse the input file", options.inputfile raise recordedData = LumiQueryAPI.recordedLumiForRange(session, parameters, inputRange) ## pprint (recordedData) for runDTarray in recordedData: runNumber = runDTarray[0] deadTable = runDTarray[2] if options.saveRuns: hist = fillPileupHistogram(deadTable, parameters, runNumber=runNumber, debug=options.debugLumi)