def execute(self): # Handle command-line switches self.parseOpts() # Open outfile if desired if self.outFile != None: f = open(self.outFile, 'w') # Get our COOL folder lblb = CoolDataReader(self.onlLumiDB, self.onlLBLBFolder) # Load data for each run specified for run in self.runList: lblb.setIOVRangeFromRun(run) if not lblb.readData(): print 'RunLumiTime - No LBLB data found for run %d!' % run continue for obj in lblb.data: # IOV is equal to (Run << 32) + LB number. run = obj.since() >> 32 lb = obj.since() & 0xFFFFFFFF # Time is UTC nanoseconds startTime = obj.payload()['StartTime'] endTime = obj.payload()['EndTime'] # Write this out as seconds outstr = "%d %d %f %f" % (run, lb, float(startTime)/1E9, float(endTime)/1E9) if self.outFile != None: f.write(outstr+'\n') else: print outstr
class LumiDeadtimeHandler: # Constructor def __init__(self): self.verbose = False # Data readers self.vetoReader = None self.menuReader = None self.countsReader = None # Trigger channels to read for live fractions self.trigList = ['L1_EM30'] # Storage of per-BCID deadtime (index by [runlb][bcid]) self.liveFracBCID = dict() # Storage of L1 trigger counts deadtime (index by [runlb][trigname]) self.liveFracTrig = dict() # Storage for L1 trigger name -> value mapping self.trigChan = dict() def calculateAll(self, run): self.loadData(run) self.findBCIDDeadtime() def loadData(self, run): self.loadVetoData(run) self.loadTrigChannels(run) self.loadTrigCounts(run) def loadVetoData(self, run): if self.verbose: print('Loading trigger veto data') # Instantiate new COOL data reader if not already done if self.vetoReader is None: self.vetoReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/PerBcidDeadtime') self.vetoReader.setIOVRangeFromRun(run) self.vetoReader.readData() if self.verbose: print('Read %d Trig veto records' % len(self.vetoReader.data)) # Read trigger channel mappings # Fills self.trigChan based on values in self.trigList def loadTrigChannels(self, run): if self.verbose: print('Loading trigger channel data') self.trigChan = dict() for trig in self.trigList: self.trigChan[trig] = -1 if self.menuReader is None: self.menuReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LVL1/Menu') self.menuReader.setIOVRangeFromRun(run) self.menuReader.readData() for obj in self.menuReader.data: trigName = obj.payload()['ItemName'] if trigName in self.trigList: self.trigChan[trigName] = int(obj.channelId()) for trig in self.trigList: if self.trigChan[trig] == -1: print("Couldn't find", trig, "in run", run) if self.verbose: print('Found', trig, 'in channel', self.trigChan[trig]) # Load all trigger counts for the given run # Fills counts for all triggers with channels found in self.trigChan def loadTrigCounts(self, run): if self.verbose: print('loading Trigger Counts data') self.liveFracTrig = dict() if self.countsReader is None: self.countsReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/LVL1COUNTERS') self.countsReader.setIOVRangeFromRun(run) # Build channel list chanList = self.trigChan.values() chanList.sort() # Skip any trigger we didn't find tmpList = [] for chan in chanList: if chan < 0: continue tmpList.append(chan) chanList = tmpList self.countsReader.setChannel(chanList) self.countsReader.readData() for obj in self.countsReader.data: #if self.verbose: # print obj.since()>>32, '/', obj.since()&0xFFFFFFFF, obj.channelId(), obj.payload()['BeforePrescale'], obj.payload()['AfterPrescale'], obj.payload()['L1Accept'] # use the string as the dictionary key ss = obj.since() if ss not in self.liveFracTrig: self.liveFracTrig[ss] = dict() for (trig, chan) in self.trigChan.iteritems(): if chan != obj.channelId(): continue ratio = 0. if obj.payload()['AfterPrescale'] > 0: ratio = float(obj.payload()['L1Accept']) / obj.payload( )['AfterPrescale'] self.liveFracTrig[ss][trig] = ratio if self.verbose: print(obj.since() >> 32, '/', obj.since() & 0xFFFFFFFF, trig, ratio) def findBCIDDeadtime(self): if self.verbose: print('Calculating per-BCID deadtime') # First dictionary index is lumiblock IOV self.liveFracBCID = dict() # Loop over each lumi block for obj in self.vetoReader.data: key = obj.since() if self.verbose: run = key >> 32 lb = key & 0xFFFFFFFF bloblength = obj.payload()['HighPriority'].size() print('%d %d Found trigger counter blob of length %d' % (run, lb, bloblength)) # Unpack High Priority blob here liveVec = unpackLiveFraction(obj.payload()) self.liveFracBCID[key] = liveVec
class MuHistMaker: def __init__(self): # Control output level self.verbose = True # Luminosity Channel self.lumiChan = 103 # Lucid_HitOR # self.lumiChan = 201 # BCM Event OR # self.lumiChan = 102 # Lucid Event OR # self.lumiChan = 0 # Use preferred (doesn't work?) # Stable only self.stableOnly = True # Ready only self.readyOnly = True # Apply deadtime self.deadtime = False # List of (integer) run numbers specified on the command line self.runList = [] # Dict of (lblo, lbhi) pairs giving extent of StableBeams self.stableDict = dict() # Dict of (lblo, lbhi) pairs giving extent of ATLAS Ready self.readyDict = dict() # Utlity routine for handling COOL connections self.dbHandler = LumiDBHandler() # Data readers self.maskReader = None self.lumiReader = None self.caliReader = None self.lblbReader = None self.lhcReader = None self.trigReader = None self.bgReader = None self.rdyReader = None # Object which stores the luminosity calibration self.caliObj = LumiCalib() # Object which stores the BCID information self.maskObj = BCIDMask() # Object to store the bunch group definition self.bgObj = BunchGroup() # Histogram parameters self.nbins = 500 self.mulo = 0. self.muhi = 50. # Explicitly clean up our DB connections def __del__(self): self.dbHandler.closeAllDB() # Called in command-line mode def execute(self): # Handle command-line switches self.parseOpts() # Process each run in self.runList for run in self.runList: # Load all data from COOL self.loadBCIDData(run) # Skip if run doesn't exist if len(self.lblbReader.data) == 0: continue # Find stable beams, output goes to self.stableTime self.findStable() # Find ATLAS ready self.findReady() # Skip if no stable beams if self.stableOnly and len(self.stableTime) == 0: continue # Skip of no ATLAS ready if self.readyOnly and len(self.readyTime) == 0: continue # Open new output file filename = 'run%d_mu.root' % run rootfile = TFile(filename, 'recreate') # Precompute deadtime into dictionary if self.deadtime: self.findDeadtime() if self.readyOnly: htitle = 'readyMuDist' elif self.stableOnly: htitle = 'stableMuDist' else: htitle = 'allMuDist' hall = TH1D(htitle, htitle, self.nbins, self.mulo, self.muhi) delivered = 0. recorded = 0. average = 0. navg = 0 # Loop over lumi blocks and calibrate for obj in self.lumiReader.data: startTime = obj.since() endTime = obj.until() runlb = obj.payload()['RunLB'] run = runlb >> 32 lb = runlb & 0xFFFFFFFF valid = obj.payload()['Valid'] & 0x3FF dtime = (endTime-startTime)/1E9 if self.verbose: print 'Run %d LB %d ' % (run, lb), # Check whether this is in stable beams if self.stableOnly: isStable = False for stable in self.stableTime: if not stable[0] <= startTime < stable[1]: continue isStable = True break if not isStable: continue if self.verbose: print 'stable ', # Check whether this is in ATLAS ready if self.readyOnly: isReady = False for ready in self.readyTime: if not ready[0] <= runlb < ready[1]: continue isReady = True break if not isReady: if self.verbose: print continue if self.verbose: print 'ready', if run == 189610 and lb < 310: if self.verbose: print ' - skipped' continue # Global timing shift if run == 201494 and lb > 220 and lb < 241: if self.verbose: print ' - skipped' continue if self.verbose: print # Make sure lumi data is valid #if valid != 0: # print 'Invalid data %d found in Run %d LB %d!' % (valid, run, lb) # continue # These all change slowly, so checking them every run/lb is OK # Make sure calibration is valid if not self.updateCalibration(startTime): print 'Error finding calibration for Run %d LB %d!' % (run, lb) continue # Make sure BCID mask is valid if not self.updateBCIDMask(startTime): print "Couldn't find valid BCID mask data for Run %d LB %d!" % (run, lb) continue # Make sure bunch group is valid (keyed by run/lb) if not self.updateBunchGroup(runlb): print "Couldn't find valid Bunch Group definition for Run %d LB %d!" % (run, lb) continue # Now we want to start extracting bunch-by-bunch information # Get raw lumi normValue = obj.payload()['AverageRawInstLum'] blobValue = obj.payload()['BunchRawInstLum'] bcidVec, rawLumi = unpackBCIDValues(blobValue, self.maskObj.coll, normValue) bcidavg = 0. nbcid = 0 h = TH1D(str(lb), '', self.nbins, self.mulo, self.muhi) for i in range(len(rawLumi)): # Check if this is in our bunch group bcid = bcidVec[i] # Off by one, check if *following* BCID is in bunch group # If so, this previous bcid is where the actual data is if run >= 189639 and run <= 189660 and bcid > 1: bcid += 1 if run == 191933 and bcid > 1: bcid += 1 if not bcid in self.bgObj.bg: continue ldel = self.caliObj.calibrate(rawLumi[i]) muval = ldel / self.caliObj.muToLumi lumi = ldel * dtime delivered += lumi if self.deadtime: lrec = ldel * self.liveFrac[runlb][bcid] lumi = lrec * dtime recorded += lumi bcidavg += self.liveFrac[runlb][bcid] nbcid += 1 # Note, mu only depends on delivered # Weighting of recorded data depends on recorded h.Fill(muval, lumi) hall.Fill(muval, lumi) if nbcid > 0: bcidavg /= nbcid average += bcidavg navg += 1 h.Write() # Close output file hall.Write() rootfile.Close() print 'Lumi Delivered:', delivered print 'Lumi Recorded:', recorded if delivered > 0.: print 'True live fraction:', recorded/delivered else: print 'True live fraction: 0.0' if navg > 0: print 'BCID average frac: ', average/navg else: print 'BCID average frac: 0.0' # Load all data necessary to make bunch-by-bunch lumi for a single run def loadBCIDData(self, runnum): print 'MuHistMaker.loadBCIDData(%s) called' % runnum # Many folders are indexed by time stamp. Load RunLB -> time conversion here self.loadLBLB(runnum) # Check if this run exists if len(self.lblbReader.data) == 0: return # Determine start and end time of this run startTime = self.lblbReader.data[0].payload()['StartTime'] endTime = self.lblbReader.data[-1].payload()['EndTime'] iov = (startTime, endTime) # Read LHC Data (for stable beams) self.loadLHCData(iov) # Read ATLAS ready data self.loadReadyData(runnum) # Read the bunch group (so we sum over the right thing) self.loadBGData(runnum) # Read BCID Data self.loadBCIDMask(iov) self.loadBCIDCali(iov) self.loadBCIDLumi(iov) # Read turn counters (for livetime) if self.deadtime: self.loadTrigData(runnum) def loadLBLB(self, run): if self.verbose: print 'Loading LBLB data' # Instantiate new COOL data reader if not already done if self.lblbReader == None: self.lblbReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/LBLB') self.lblbReader.setIOVRangeFromRun(run) self.lblbReader.readData() if self.verbose: print 'Read %d LBLB records' % len(self.lblbReader.data) if len(self.lblbReader.data) > 0: print 'First LB %d/%d' % (self.lblbReader.data[0].since() >> 32, self.lblbReader.data[0].since() & 0xFFFFFFFF) if len(self.lblbReader.data) > 1: print 'Last LB %d/%d' % (self.lblbReader.data[-1].since() >> 32, self.lblbReader.data[-1].since() & 0xFFFFFFFF) def loadTrigData(self, run): if self.verbose: print 'Loading Trigger data' # Instantiate new COOL data reader if not already done if self.trigReader == None: self.trigReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/PerBcidDeadtime') self.trigReader.setIOVRangeFromRun(run) self.trigReader.readData() if self.verbose: print 'Read %d Trig records' % len(self.trigReader.data) def loadBGData(self, run): if self.verbose: print 'Loading Bunch group data' # Instantiate new COOL reader if not already done if self.bgReader == None: self.bgReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LVL1/BunchGroupContent') self.bgReader.setIOVRangeFromRun(run) self.bgReader.readData() if self.verbose: print 'Read %d bunch group records' % len(self.bgReader.data) def loadBCIDMask(self, iov): if self.verbose: print 'Loading BCID masks' # Instantiate new COOL data reader if not already done if self.maskReader == None: self.maskReader = CoolDataReader('COOLONL_TDAQ/COMP200', '/TDAQ/OLC/LHC/FILLPARAMS') self.maskReader.setIOVRange(iov[0], iov[1]) self.maskReader.readData() if self.verbose: print 'Read %d BCID Mask records' % len(self.maskReader.data) def loadBCIDCali(self, iov): if self.verbose: print 'Loading BCID luminosity calibrations' # Instantiate new COOL data reader if not already done if self.caliReader == None: self.caliReader = CoolDataReader('COOLONL_TDAQ/COMP200', '/TDAQ/OLC/CALIBRATIONS') self.caliReader.setIOVRange(iov[0], iov[1]) self.caliReader.setChannel([self.lumiChan]) self.caliReader.readData() if self.verbose: print 'Read %d Calibration records' % len(self.caliReader.data) def loadBCIDLumi(self, iov): if self.verbose: print 'Loading BCID luminosity values' # Instantiate new COOL data reader if not already done if self.lumiReader == None: # self.lumiReader = CoolDataReader('COOLONL_TDAQ/MONP200', '/TDAQ/OLC/BUNCHLUMIS') self.lumiReader = CoolDataReader('COOLONL_TDAQ/COMP200', '/TDAQ/OLC/BUNCHLUMIS') self.lumiReader.setIOVRange(iov[0], iov[1]) self.lumiReader.setChannel([self.lumiChan]) self.lumiReader.readData() if self.verbose: print 'Read %d Lumi records' % len(self.lumiReader.data) # Information about stable beams def loadLHCData(self, iov): if self.verbose: print 'Loading LHC information' # Instantiate new COOL data reader if not already done if self.lhcReader == None: self.lhcReader = CoolDataReader('COOLOFL_DCS/COMP200', '/LHC/DCS/FILLSTATE') self.lhcReader.setIOVRange(iov[0], iov[1]) self.lhcReader.readData() if self.verbose: print 'Read %d LHC records' % len(self.lhcReader.data) # Information about ATLAS ready def loadReadyData(self, run): if self.verbose: print 'Loading ATLAS ready information' # Instantiate new COOL data reader if not already done if self.rdyReader == None: self.rdyReader = CoolDataReader('COOLONL_TDAQ/COMP200', '/TDAQ/RunCtrl/DataTakingMode') self.rdyReader.setIOVRangeFromRun(run) self.rdyReader.readData() if self.verbose: print 'Read %d ATLAS ready records' % len(self.rdyReader.data) # Fill the stable beam information in the OflLumiRunData object def findStable(self): if self.verbose: print 'Finding stable beam periods' # First, fill stable beam time periods for this run tlo = cool.ValidityKeyMax thi = cool.ValidityKeyMin self.stableTime = [] for obj in self.lhcReader.data: if obj.payload()['StableBeams'] == 0: continue if tlo > thi: # First stable beams tlo = obj.since() thi = obj.until() elif thi == obj.since(): # Extension of existing stable beams thi = obj.until() else: # Not contiguous, add old to list and start again self.stableTime.append((tlo, thi)) tlo = obj.since() thi = obj.until() if tlo < thi: self.stableTime.append((tlo, thi)) if self.verbose: print 'Stable beam periods found:', self.stableTime # Fill the stable beam information in the OflLumiRunData object def findReady(self): if self.verbose: print 'Finding ATLAS ready beam periods' # First, fill stable beam time periods for this run tlo = cool.ValidityKeyMax thi = cool.ValidityKeyMin self.readyTime = [] for obj in self.rdyReader.data: if obj.payload()['ReadyForPhysics'] == 0: continue if tlo > thi: # First ready tlo = obj.since() thi = obj.until() elif thi == obj.since(): # Extension of existing ready thi = obj.until() else: # Not contiguous, add old to list and start again self.readyTime.append((tlo, thi)) tlo = obj.since() thi = obj.until() if tlo < thi: self.readyTime.append((tlo, thi)) if self.verbose: print 'ATLAS ready periods found:', self.readyTime # Precompute the deadtime for this run into a dictionary indexed by lumi block and BCID def findDeadtime(self): if self.verbose: print 'Calculating per-BCID deadtime' # First dictionary index is lumiblock IOV self.liveFrac = dict() # Loop over each lumi block for obj in self.trigReader.data: key = obj.since() run = key >> 32 lb = key & 0xFFFFFFFF bloblength = obj.payload()['HighPriority'].size() if self.verbose: print '%d %d Found trigger counter blob of length %d' % (run, lb, bloblength) # Unpack High Priority blob here liveVec = unpackLiveFraction(obj.payload()) self.liveFrac[key] = liveVec # Each BCID is one 24-bit integer if self.verbose: for i in range(10): print 'BICD: %d Live: %f' % (i+1, liveVec[i]) def updateBCIDMask(self, startTime): if not self.maskObj.isValid(startTime): self.maskObj.clearValidity() # Find the proper mask object maskData = None for mask in self.maskReader.data: if not mask.since() <= startTime < mask.until(): continue self.maskObj.setMask(mask) break if not self.maskObj.isValid(startTime): return False return True def updateCalibration(self, startTime): if not self.caliObj.isValid(startTime): self.caliObj.clearValidity() # Find the proper calibration object for cali in self.caliReader.data: if not cali.since() <= startTime < cali.until(): continue self.caliObj.setCalibration(cali) break if not self.caliObj.isValid(startTime): return False return True def updateBunchGroup(self, runlb): if not self.bgObj.isValid(runlb): self.bgObj.clearValidity() # Find the proper BG object for bg in self.bgReader.data: if not bg.since() <= runlb < bg.until(): continue self.bgObj.setBG(bg) break if not self.bgObj.isValid(runlb): return False return True def parseOpts(self): parser = OptionParser(usage="usage: %prog [options]", add_help_option=False) parser.add_option("-?", "--usage", action="store_true", default=False, dest="help", help="show this help message and exit") parser.add_option("-v", "--verbose", action="store_true", default=self.verbose, dest="verbose", help="turn on verbose output") parser.add_option("-r", "--updateRun", dest="runlist", metavar="RUN", help="update specific run, or comma separated list") (options, args) = parser.parse_args() if options.help: parser.print_help() sys.exit() self.verbose = options.verbose # Parse run list if options.runlist != None: # Clear the old one self.runList = [] # Can be comma-separated list of run ranges runlist = options.runlist.split(',') if len(runlist) == 0: print 'Invalid run list specified!' sys.exit() # Go through and check for run ranges for runstr in runlist: subrunlist = runstr.split('-') if len(subrunlist) == 1: # Single run self.runList.append(int(subrunlist[0])) elif len(subrunlist) == 2: # Range of runs for runnum in range(int(subrunlist[0]), int(subrunlist[1])+1): self.runList.append(runnum) else: # Too many parameters print 'Invalid run list segment found:', runstr sys.exit() self.runList.sort() if self.verbose: print 'Finished parsing run list:', for runnum in self.runList: print runnum, print
else: print 'Invalid run list segment found:', runstr sys.exit() runList.sort() else: runList = data2010 print len(runList) countlb = 0 countpbd = 0 for run in runList: print run lblbReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/LBLB') lblbReader.setIOVRangeFromRun(run) lblbReader.readData() iovSince = lblbReader.data[0].payload()['StartTime'] iovUntil = lblbReader.data[-1].payload()['EndTime'] countlb = 0 for obj in lblbReader.data: countlb += 1 bunchPrbcidDeadtime = CoolDataReader("COOLONL_TRIGGER/COMP200", "/TRIGGER/LUMI/PerBcidDeadtime") bunchPrbcidDeadtime.setIOVRangeFromRun(run) bunchPrbcidDeadtime.readData() countbpd = 0
class TriggerHandler: def __init__(self): # Database parameters self.menuReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LVL1/Menu') self.countsReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/LVL1COUNTERS') self.lbtimeReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/LBTIME') self.lblbReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/LBLB') self.verbose = False #self.verbose = True # Dict of all TrigL1Data objects for the given time interval (keyed by RunLB IOV) self.trigL1Dict = dict() self.allL1Triggers = False # List of all trigger items to read self.trigList = ['L1_MBTS_2', 'L1_EM30'] # Dictionary of trigger channel number keyed by trigger name self.trigChan = dict() self.chanTrig = dict() # reverse order # Store the lumi block times self.lblbDict = dict() # Clear all data def clear(self): # Clear trigger dict self.trigL1Dict.clear() # Find trigger information in iovrange by time def loadData(self, startIOV, endIOV): self.clear() # Runlist holds specific runs in this time range self.runlist = [] if self.verbose: print 'Searching for trigger information for max IOVRange', timeString( startIOV), timeString(endIOV) # Load the run based information as we fundamentally need to do this by run number # Prepare the lbtime reader self.lbtimeReader.setIOVRange(startIOV, endIOV) self.lbtimeReader.readData() for obj in self.lbtimeReader.data: runnum = int(obj.payload()['Run']) if not runnum in self.runlist: self.runlist.append(runnum) # Loop over each run, getting the trigger counts/Lumi # Must do this by run, as trigger menu can change # Here we are storing this in a separate list for runnum in self.runlist: self.loadDataByRun(runnum, clear=False) # Find trigger information by run def loadDataByRun(self, runnum, clear=True): if self.verbose: print 'TriggerHandler.loadDataByRun(%d) called' % runnum if clear: self.clear() # Figure out the channel mappings for the L1 trigger items self.loadTrigChannels(runnum) # Get the LB durations self.loadLBLBData(runnum) # Third, get the trigger counts self.loadTrigCounts(runnum) # Read LBLB data def loadLBLBData(self, runnum): if self.verbose: print 'TriggerHandler.loadLBLBData(%d) called' % runnum self.lblbDict.clear() self.lblbReader.setIOVRangeFromRun(runnum) self.lblbReader.readData() for obj in self.lblbReader.data: self.lblbDict[obj.since()] = (obj.payload()['StartTime'], obj.payload()['EndTime']) # Read trigger channel mappings # Fills self.trigChan based on values in self.trigList def loadTrigChannels(self, runnum): if self.verbose: print 'TriggerHandler.loadTrigChannels(%d) called' % runnum # Trigger channels keyed by name self.trigChan = dict() # Trigger name keyed by channel self.chanTrig = dict() for trig in self.trigList: self.trigChan[trig] = -1 self.menuReader.setIOVRangeFromRun(runnum) self.menuReader.readData() for obj in self.menuReader.data: if self.verbose or True: print int(obj.channelId()), obj.payload()['ItemName'] trigName = obj.payload()['ItemName'] trigChan = int(obj.channelId()) if self.allL1Triggers or self.trigList.count(trigName) > 0: self.trigChan[trigName] = trigChan self.chanTrig[trigChan] = trigName for trig in self.trigList: if self.trigChan[trig] == -1: print "Couldn't find", trig, "in run", str(runnum) if self.verbose: for (trig, chan) in self.trigChan.iteritems(): print 'Found', trig, 'in channel', chan # Load all trigger counts for the given run # Fills counts for all triggers with channels found in self.trigChan def loadTrigCounts(self, runnum): if self.verbose: print 'TriggerHandler.loadTrigCounts(%d) called' % runnum self.countsReader.setIOVRangeFromRun(runnum) # Build channel list chanList = self.trigChan.values() chanList.sort() nMaxChan = 50 nChanBlock = 0 chanBlock = [] nChannels = 0 # Skip any trigger we didn't find tmpList = [] for chan in chanList: if chan < 0: continue tmpList.append(chan) chanList = tmpList if self.verbose: print 'breaking up', len( chanList), 'into', nMaxChan, 'for run', runnum # There is a 50 item limit somehow hardcoded into browseObjects. # Use this code from Elliot to get around the limitation. # Break up list of indices into blocks: for x in range(0, len(chanList), nMaxChan): top = min([x + nMaxChan, len(chanList)]) if self.verbose: print 'Initializing block [%d] from %d to %d' % (nChanBlock, x, top) chanBlock.append(chanList[x:top]) nChanBlock += 1 for x in range(nChanBlock): if self.verbose: print 'Channel Selector', chanBlock[x] self.countsReader.setChannel(chanBlock[x]) self.countsReader.readData() for obj in self.countsReader.data: since = obj.since() until = obj.until() if self.verbose: print runLBString(since), runLBString( until), obj.channelId(), obj.payload( )['BeforePrescale'], obj.payload( )['AfterPrescale'], obj.payload()['L1Accept'] # use the string as the dictionary key ss = since chan = int(obj.channelId()) trig = self.chanTrig.get(chan, "") if len(trig) == 0: print 'TriggerHandler.loadTrigCounts(%d) - found unknown channel %d in %s!' % ( runnum, chan, runLBString(ss)) continue if ss not in self.trigL1Dict: self.trigL1Dict[ss] = TriggerL1Data() self.trigL1Dict[ss].runlb = obj.since() self.trigL1Dict[ss].startTime = self.lblbDict.get( ss, (0., 0.))[0] self.trigL1Dict[ss].endTime = self.lblbDict.get( ss, (0., 0.))[1] self.trigL1Dict[ss].dtime = ( self.trigL1Dict[ss].endTime - self.trigL1Dict[ss].startTime) / 1.E9 self.trigL1Dict[ss].TBP[trig] = obj.payload()['BeforePrescale'] self.trigL1Dict[ss].TAP[trig] = obj.payload()['AfterPrescale'] self.trigL1Dict[ss].TAV[trig] = obj.payload()['L1Accept']
class LumiInspector: def __init__(self): # Control output level self.verbose = True # List of (integer) run numbers specified on the command line self.runList = [] # List of (integer, integer) COOL-format IOVs to print out self.iovList = [] # List of channel numbers found in calibration data self.lumiChanList = [] # Dict of (lblo, lbhi) pairs giving extent of StableBeams self.stableDict = dict() # Utlity routine for handling COOL connections self.dbHandler = LumiDBHandler() # Data readers self.maskReader = None self.oflReader = None self.lumiReader = None self.caliReader = None self.lblbReader = None self.currReader = None self.lhcReader = None self.bgReader = None self.pmtReader = None # Object which stores the luminosity calibration self.caliObj = dict() # LumiCalib() # Object which stores the BCID information self.maskObj = BCIDMask() # Object to store the bunch group definition self.bgObj = BunchGroup() # Offline tag self.lumiTag = 'OflLumi-8TeV-002' # Use online DB self.online = False self.nBCID = 3564 # Explicitly clean up our DB connections def __del__(self): self.dbHandler.closeAllDB() # Called in command-line mode def execute(self): # Handle command-line switches self.parseOpts() # Process each run in self.runList for run in self.runList: # Load data from COOL self.loadData(run) # Skip if run doesn't exist if len(self.lblbReader.data) == 0: continue # Find stable beams, output goes to self.stableTime self.findStable() # Loop over all calibration records for obj in self.lblbReader.data: runlb = obj.since() # Check if this is in our IOV list found = False for iov in self.iovList: if not (iov[0] <= runlb < iov[1]): continue found = True break if not found: continue run = runlb >> 32 lb = runlb & 0xFFFFFFFF startTime = obj.payload()['StartTime'] endTime = obj.payload()['EndTime'] dtime = (endTime-startTime)/1E9 # Check whether this is in stable beams isStable = False for stable in self.stableTime: if not (stable[0] <= startTime < stable[1]): continue isStable = True break startStr = time.strftime('%y/%m/%d, %H:%M:%S', time.gmtime(startTime/1E9)) endStr = time.strftime('%y/%m/%d, %H:%M:%S', time.gmtime(endTime/1E9)) print 'Found Run/LB %d/%d (%s - %s) Length %.1f sec' % (run, lb, startStr, endStr, dtime), if isStable: print ' -> stable', print # These change slowly, so checking them every run/lb is OK # Make sure bunch group is valid (keyed by run/lb) for chan in self.lumiChanList: # Make sure calibration is valid if not self.updateCalibration(startTime, chan): print 'Error finding calibration for Run %d LB %d Chan %d!' % (run, lb, rawChan) continue # End of loop over channels # End of loop over LBs # End of loop over runs # Load all data necessary to make bunch-by-bunch lumi for a single run def loadData(self, runnum): print 'calibrationInspector.loadData(%s) called' % runnum # Many folders are indexed by time stamp. Load RunLB -> time conversion here self.loadLBLB(runnum) if len(self.lblbReader.data) == 0: return # Determine start and end time of this run startTime = self.lblbReader.data[0].payload()['StartTime'] endTime = self.lblbReader.data[-1].payload()['EndTime'] iov = (startTime, endTime) # Read LHC Data (for stable beams) self.loadLHCData(iov) # Read calibration data self.loadBCIDCali(iov) if len(self.lumiChanList) == 0: for obj in self.caliReader.data: if obj.channelId() not in self.lumiChanList: self.lumiChanList.append(obj.channelId()) def loadLBLB(self, run): if self.verbose: print 'Loading LBLB data' # Instantiate new COOL data reader if not already done if self.lblbReader == None: self.lblbReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/LBLB') self.lblbReader.setIOVRangeFromRun(run) self.lblbReader.readData() if self.verbose: print 'Read %d LBLB records' % len(self.lblbReader.data) if len(self.lblbReader.data) > 0: print 'First LB %d/%d' % (self.lblbReader.data[0].since() >> 32, self.lblbReader.data[0].since() & 0xFFFFFFFF) print 'Last LB %d/%d' % (self.lblbReader.data[-1].since() >> 32, self.lblbReader.data[-1].since() & 0xFFFFFFFF) def loadBCIDCali(self, iov): if self.verbose: print 'Loading BCID luminosity calibrations' # Instantiate new COOL data reader if not already done if self.caliReader == None: self.caliReader = CoolDataReader('COOLONL_TDAQ/COMP200', '/TDAQ/OLC/CALIBRATIONS') self.caliReader.setIOVRange(iov[0], iov[1]) self.caliReader.readData() if self.verbose: print 'Read %d Calibration records' % len(self.caliReader.data) # Information about stable beams def loadLHCData(self, iov): if self.verbose: print 'Loading LHC information' # Instantiate new COOL data reader if not already done if self.lhcReader == None: self.lhcReader = CoolDataReader('COOLOFL_DCS/COMP200', '/LHC/DCS/FILLSTATE') self.lhcReader.setIOVRange(iov[0], iov[1]) self.lhcReader.readData() if self.verbose: print 'Read %d LHC records' % len(self.lhcReader.data) # Fill the stable beam information in the OflLumiRunData object def findStable(self): if self.verbose: print 'Finding stable beam periods' # First, fill stable beam time periods for this run tlo = cool.ValidityKeyMax thi = cool.ValidityKeyMin self.stableTime = [] for obj in self.lhcReader.data: if obj.payload()['StableBeams'] == 0: continue if tlo > thi: # First stable beams tlo = obj.since() thi = obj.until() elif thi == obj.since(): # Extension of existing stable beams thi = obj.until() else: # Not contiguous, add old to list and start again self.stableTime.append((tlo, thi)) tlo = obj.since() thi = obj.until() if tlo < thi: self.stableTime.append((tlo, thi)) if self.verbose: print 'Stable beam periods found:', self.stableTime def updateCalibration(self, startTime, chan): if not chan in self.caliObj: self.caliObj[chan] = LumiCalib() self.caliObj[chan].verbose = True if not self.caliObj[chan].isValid(startTime): self.caliObj[chan].clearValidity() # Find the proper calibration object for cali in self.caliReader.data: if cali.channelId() != chan: continue if not cali.since() <= startTime < cali.until(): continue print 'New Calibration for channel %d (%s)' % (chan, LumiChannelDefs().name(chan)) self.caliObj[chan].setCalibration(cali) if not self.caliObj[chan].isValid(startTime): return False return True def updateBunchGroup(self, runlb): if not self.bgObj.isValid(runlb): self.bgObj.clearValidity() # Find the proper BG object for bg in self.bgReader.data: if not bg.since() <= runlb < bg.until(): continue self.bgObj.setBG(bg) break if not self.bgObj.isValid(runlb): return False return True def parseOpts(self): parser = OptionParser(usage="usage: %prog [options]", add_help_option=False) parser.add_option("-?", "--usage", action="store_true", default=False, dest="help", help="show this help message and exit") parser.add_option("-v", "--verbose", action="store_true", default=self.verbose, dest="verbose", help="turn on verbose output") parser.add_option("-r", "--run", dest="runlist", metavar="RUN", help="process specific run, run/lb, range, or comma separated list") parser.add_option("--channel", dest="chanlist", metavar="CHAN", help="select specific channel") (options, args) = parser.parse_args() if options.help: parser.print_help() sys.exit() self.verbose = options.verbose # Parse run list if options.runlist != None: self.parseRunList(options.runlist) if options.chanlist != None: self.lumiChanList = [int(options.chanlist)] # Parse text-based run list # Can specify runs alone or run/lb lumi blocks # Ranges are specified with a dash, and commas separate instances def parseRunList(self,runstr): # Clear the old one self.runList = [] # Each run gets one entry here self.iovList = [] # Pair of IOVs in COOL format # Can be comma-separated list of run ranges runlist = runstr.split(',') if len(runlist) == 0: print 'Invalid run list specified!' sys.exit() # Further parse each comma-separated item for str in runlist: # Check for ranges subrunlist = str.split('-') if len(subrunlist) == 1: # Single item # Check for lumi block or not lblist = str.split('/') if len(lblist) == 1: # Single run runnum = int(subrunlist[0]) self.iovList.append(((runnum << 32), ((runnum+1) << 32))) elif len(lblist) == 2: # Run/LB runnum = int(lblist[0]) lbnum = int(lblist[1]) self.iovList.append(((runnum << 32) + lbnum, (runnum << 32) + lbnum + 1)) else: # Too many parameters print 'Invalid run list item found:', str sys.exit() if runnum not in self.runList: self.runList.append(runnum) elif len(subrunlist) == 2: # Range # Parse starting item lblist = subrunlist[0].split('/') if len(lblist) == 1: # Single run startrun = int(lblist[0]) startiov = startrun << 32 elif len(lblist) == 2: # Run/LB startrun = int(lblist[0]) lb = int(lblist[1]) startiov = (startrun << 32) + lb else: # Too many parameters print 'Invalid run list item found:', str sys.exit() # Parse ending item lblist = subrunlist[1].split('/') if len(lblist) == 1: # Single run endrun = int(lblist[0]) endiov = (endrun+1) << 32 elif len(lblist) == 2: # Run/LB endrun = int(lblist[0]) lb = int(lblist[1]) endiov = (endrun << 32) + lb + 1 else: # Too many parameters print 'Invalid run list item found:', str sys.exit() self.iovList.append((startiov, endiov)) for runnum in range(startrun, endrun+1): if runnum not in self.runList: self.runList.append(runnum) else: # Too many parameters print 'Invalid run list item found:', str sys.exit() self.runList.sort() if self.verbose: print 'Finished parsing run list:', for runnum in self.runList: print runnum, print self.iovList.sort() if self.verbose: for iov in self.iovList: runlo = iov[0] >> 32 lblo = iov[0] & 0xFFFFFFFF print '%d/%d - %d/%d' % (iov[0] >> 32, iov[0] & 0xFFFFFFFF, iov[1] >> 32, iov[1] & 0xFFFFFFFF)
class CompareLumi: def __init__(self): # Control output level self.verbose = False # Channels for comparision self.lumi1Channel = 0 self.lumi2Channel = 261 self.lumi1Tag = 'OflLumi-7TeV-002' self.lumi2Tag = 'OflLumi-7TeV-003' # Difference (in percent) to complain about self.threshold = 5. # Stable only self.stableOnly = True # List of IOVRanges with stable beams self.stableTime = [] # ATLAS Ready only self.readyOnly = True #Dict of (runlblo, runlblbhi) pairs giving extent of ATLAS ready self.readyTime = [] # List of (integer) run numbers specified on the command line self.runList = [] # Utlity routine for handling COOL connections self.dbHandler = LumiDBHandler() # Data readers self.lumi1Reader = None self.lumi2Reader = None self.lblbReader = None self.lhcReader = None self.rdyReader = None # Explicitly clean up our DB connections def __del__(self): self.dbHandler.closeAllDB() # Called in command-line mode def execute(self): # Handle command-line switches self.parseOpts() lumi1Sum = dict() lumi2Sum = dict() # Process each run in self.runList for run in self.runList: # Load all data from COOL self.loadCOOLData(run) # Skip if run doesn't exist if len(self.lblbReader.data) == 0: continue # Find stable beams, output goes to self.stableTime self.findStable() # Find ATLAS ready, result goes to self.readyTime self.findReady() if self.stableOnly and len(self.stableTime) == 0: continue if self.readyOnly and len(self.readyTime) == 0: continue # Storage objects keyed by [lb] lumi1Dict = dict() lumi2Dict = dict() lumi1Sum[run] = 0. lumi2Sum[run] = 0. # Keep track of LB range lbLo = 99999 lbHi = 0 # Loop over all objects and sort by runlb and algo for obj in self.lumi1Reader.data: runlb = obj.since() lumi1Dict[runlb] = obj.payload() for obj in self.lumi2Reader.data: runlb = obj.since() lumi2Dict[runlb] = obj.payload() # if self.verbose: # run = runlb >> 32 # lb = runlb & 0xFFFFFFFF # print 'Found Run %d LB %d chan %d' % (run, lb, channel) # Loop over defined data for obj in self.lblbReader.data: runlb = obj.since() run = runlb >> 32 lb = runlb & 0xFFFFFFFF startTime = obj.payload()['StartTime'] endTime = obj.payload()['EndTime'] dtime = (endTime - startTime) / 1E9 # Check whether this is in stable beams isStable = False for stable in self.stableTime: if not stable[0] <= startTime < stable[1]: continue isStable = True break if self.stableOnly and not isStable: continue # Check whether this in in ATLAS ready isReady = False for ready in self.readyTime: if not ready[0] <= runlb < ready[1]: continue isReady = True break #if self.verbose: # print 'Found stable Run %d LB %d' % (run, lb) if lb < lbLo: lbLo = lb if lb > lbHi: lbHi = lb # Check if lumi record exists missing1 = False missing2 = False if runlb not in lumi1Dict: missing1 = True if runlb not in lumi2Dict: missing2 = True if missing1 and missing2: if isStable: print 'Run: %d LB: %4d does not have any luminosity record in stable beams!' % ( run, lb) else: print 'Run: %d LB: %4d does not have any luminosity record!' % ( run, lb) continue elif missing1: if isStable: print 'Run: %d LB: %4d does not have first lumi record in stable beams!' % ( run, lb) else: print 'Run: %d LB: %4d does not have first lumi record!' % ( run, lb) elif missing2: if isStable: print 'Run: %d LB: %4d does not have second lumi record in stable beams!' % ( run, lb) else: print 'Run: %d LB: %4d does not have second lumi record!' % ( run, lb) # Get lumi lumi1 = 0. valid1 = 999 chan1 = self.lumi1Channel lumi2 = 0. valid2 = 999 chan2 = self.lumi2Channel if not missing1: lumi1 = lumi1Dict[runlb]['LBAvInstLumi'] valid1 = lumi1Dict[runlb]['Valid'] & 0x3FF if chan1 == 0: chan1 = lumi1Dict[runlb]['Valid'] >> 22 lumi1Sum[run] += (lumi1 * dtime) / 1.E6 if not missing2: lumi2 = lumi2Dict[runlb]['LBAvInstLumi'] valid2 = lumi2Dict[runlb]['Valid'] & 0x3FF if chan2 == 0: chan2 = lumi2Dict[runlb]['Valid'] >> 22 lumi2Sum[run] += (lumi2 * dtime) / 1.E6 try: ratio = 100 * (lumi2 / lumi1 - 1.) except ZeroDivisionError: ratio = -99. if self.verbose: print 'Run: %6d LB: %4d Lumi1: %6.1f Chan1: %d Valid1: %d Lumi2: %6.1f Chan2: %d Valid2: %d Ratio: %.3f%%' % ( run, lb, lumi1, chan1, valid1, lumi2, chan2, valid2, ratio) # Compare if (abs(ratio) > self.threshold and (lumi1 > 1. or lumi2 > 1.)): # or valid1 != 0 or valid2 != 0: print '>>> Run: %6d LB: %4d dTime: %f Lumi1: %6.1f Chan1: %d Valid1: %d Lumi2: %6.1f Chan2: %d Valid2: %d Ratio: %.3f%%' % ( run, lb, dtime, lumi1, chan1, valid1, lumi2, chan2, valid2, ratio) # End of loop over LBs try: ratio = 100 * (lumi2Sum[run] / lumi1Sum[run] - 1.) except ZeroDivisionError: ratio = -99. print 'Run: %6d LBLo: %4d LBHi: %4d IntLumi1: %8.3f IntLumi2: %8.3f Ratio: %.3f%%' % ( run, lbLo, lbHi, lumi1Sum[run], lumi2Sum[run], ratio) # End of loop over runs totalLumi1 = 0. totalLumi2 = 0. for lumi in lumi1Sum.itervalues(): totalLumi1 += lumi for lumi in lumi2Sum.itervalues(): totalLumi2 += lumi try: ratio = 100 * (totalLumi2 / totalLumi1 - 1.) except ZeroDivisionError: ratio = -99. print 'Total Luminosity1: %8.3f Luminosity2: %8.3f => 2/1 = %.3f%%' % ( totalLumi1, totalLumi2, ratio) # Load all data necessary to make bunch-by-bunch lumi for a single run def loadCOOLData(self, runnum): if self.verbose: print 'LumiChecker.loadCOOLData(%s) called' % runnum # Many folders are indexed by time stamp. Load RunLB -> time conversion here self.loadLBLB(runnum) if len(self.lblbReader.data) == 0: return # Determine start and end time of this run startTime = self.lblbReader.data[0].payload()['StartTime'] endTime = self.lblbReader.data[-1].payload()['EndTime'] iov = (startTime, endTime) # Read LHC Data (for stable beams) self.loadLHCData(iov) # Read Ready Data self.loadReadyData(runnum) # Read the luminosity self.loadLumi(runnum) def loadLBLB(self, run): if self.verbose: print 'Loading LBLB data' # Instantiate new COOL data reader if not already done if self.lblbReader == None: self.lblbReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/LBLB') self.lblbReader.setIOVRangeFromRun(run) self.lblbReader.readData() if self.verbose: print 'Read %d LBLB records' % len(self.lblbReader.data) if len(self.lblbReader.data) > 0: print 'First LB %d/%d' % ( self.lblbReader.data[0].since() >> 32, self.lblbReader.data[0].since() & 0xFFFFFFFF) print 'Last LB %d/%d' % ( self.lblbReader.data[-1].since() >> 32, self.lblbReader.data[-1].since() & 0xFFFFFFFF) def loadLumi(self, runnum): if self.verbose: print 'Loading luminosity values' # Instantiate new COOL data reader if not already done if self.lumi1Reader == None: self.lumi1Reader = CoolDataReader('COOLOFL_TRIGGER/COMP200', '/TRIGGER/OFLLUMI/LBLESTOFL') self.lumi1Reader.setIOVRangeFromRun(runnum) self.lumi1Reader.setChannel([self.lumi1Channel]) self.lumi1Reader.setTag(self.lumi1Tag) self.lumi1Reader.readData() if self.verbose: print 'Read %d Lumi records' % len(self.lumi1Reader.data) # Instantiate new COOL data reader if not already done if self.lumi2Reader == None: self.lumi2Reader = CoolDataReader('COOLOFL_TRIGGER/COMP200', '/TRIGGER/OFLLUMI/LBLESTOFL') self.lumi2Reader.setIOVRangeFromRun(runnum) self.lumi2Reader.setChannel([self.lumi2Channel]) self.lumi2Reader.setTag(self.lumi2Tag) self.lumi2Reader.readData() if self.verbose: print 'Read %d Lumi records' % len(self.lumi2Reader.data) # Information about stable beams def loadLHCData(self, iov): if self.verbose: print 'Loading LHC information' # Instantiate new COOL data reader if not already done if self.lhcReader == None: self.lhcReader = CoolDataReader('COOLOFL_DCS/COMP200', '/LHC/DCS/FILLSTATE') self.lhcReader.setIOVRange(iov[0], iov[1]) self.lhcReader.readData() if self.verbose: print 'Read %d LHC records' % len(self.lhcReader.data) # Information about ATLAS ready def loadReadyData(self, run): if self.verbose: print 'Loading ATLAS ready information' # Instantiate new COOL data reader if not already done if self.rdyReader == None: self.rdyReader = CoolDataReader('COOLONL_TDAQ/COMP200', '/TDAQ/RunCtrl/DataTakingMode') self.rdyReader.setIOVRangeFromRun(run) self.rdyReader.readData() if self.verbose: print 'Read %d ATLAS ready records' % len(self.rdyReader.data) # Fill the stable beam information in the OflLumiRunData object def findStable(self): if self.verbose: print 'Finding stable beam periods' # First, fill stable beam time periods for this run tlo = cool.ValidityKeyMax thi = cool.ValidityKeyMin self.stableTime = [] for obj in self.lhcReader.data: if obj.payload()['StableBeams'] == 0: continue if tlo > thi: # First stable beams tlo = obj.since() thi = obj.until() elif thi == obj.since(): # Extension of existing stable beams thi = obj.until() else: # Not contiguous, add old to list and start again self.stableTime.append((tlo, thi)) tlo = obj.since() thi = obj.until() if tlo < thi: self.stableTime.append((tlo, thi)) if self.verbose: print 'Stable beam periods found:', self.stableTime # Fill the stable beam information in the OflLumiRunData object def findReady(self): if self.verbose: print 'Finding ATLAS ready beam periods' # First, fill stable beam time periods for this run tlo = cool.ValidityKeyMax thi = cool.ValidityKeyMin self.readyTime = [] for obj in self.rdyReader.data: if obj.payload()['ReadyForPhysics'] == 0: continue if tlo > thi: # First ready tlo = obj.since() thi = obj.until() elif thi == obj.since(): # Extension of existing ready thi = obj.until() else: # Not contiguous, add old to list and start again self.readyTime.append((tlo, thi)) tlo = obj.since() thi = obj.until() if tlo < thi: self.readyTime.append((tlo, thi)) if self.verbose: print 'ATLAS ready periods found:', self.readyTime def parseOpts(self): parser = OptionParser(usage="usage: %prog [options]", add_help_option=False) parser.add_option("-?", "--usage", action="store_true", default=False, dest="help", help="show this help message and exit") parser.add_option("-v", "--verbose", action="store_true", default=self.verbose, dest="verbose", help="turn on verbose output") parser.add_option("-r", "--updateRun", dest="runlist", metavar="RUN", help="update specific run, or comma separated list") parser.add_option("--tag1", dest="lumi1tag", metavar="TAG", default=self.lumi1Tag, help='first luminosity tag (default: %s)' % self.lumi1Tag) parser.add_option("--chan1", dest="lumi1chan", metavar="CHAN", default=self.lumi1Channel, help='first luminosity channel (default: %d)' % self.lumi1Channel) parser.add_option("--tag2", dest="lumi2tag", metavar="TAG", default=self.lumi2Tag, help='first luminosity tag (default: %s)' % self.lumi2Tag) parser.add_option("--chan2", dest="lumi2chan", metavar="CHAN", default=self.lumi2Channel, help='second luminosity channel (default: %d)' % self.lumi2Channel) parser.add_option( "--noStable", action="store_false", default=self.stableOnly, dest="stable", help="turn off stable beams requirement (default: stable only)") parser.add_option( "--noReady", action="store_false", default=self.readyOnly, dest="ready", help="turn off ATLAS ready requirement (default: ready only)") (options, args) = parser.parse_args() if options.help: parser.print_help() sys.exit() self.verbose = options.verbose self.lumi1Tag = options.lumi1tag self.lumi2Tag = options.lumi2tag self.stableOnly = options.stable self.readyOnly = options.ready self.lumi1Channel = int(options.lumi1chan) self.lumi2Channel = int(options.lumi2chan) # Parse run list if options.runlist != None: # Clear the old one self.runList = [] # Can be comma-separated list of run ranges runlist = options.runlist.split(',') if len(runlist) == 0: print 'Invalid run list specified!' sys.exit() # Go through and check for run ranges for runstr in runlist: subrunlist = runstr.split('-') if len(subrunlist) == 1: # Single run self.runList.append(int(subrunlist[0])) elif len(subrunlist) == 2: # Range of runs for runnum in range(int(subrunlist[0]), int(subrunlist[1]) + 1): self.runList.append(runnum) else: # Too many parameters print 'Invalid run list segment found:', runstr sys.exit() self.runList.sort() if self.verbose: print 'Finished parsing run list:', for runnum in self.runList: print runnum, print
class LumiDumper: def __init__(self): # Control output level self.verbose = True # Channel to chose (preferred by default) self.lumiChan = 0 # Run list self.runList = [] # Luminosity tag self.lumiTag = 'OflLumi-7TeV-002' # Use online instead self.online = False # Instantiate the LumiDBHandler, so we can cleanup all COOL connections in the destructor self.dbHandler = LumiDBHandler() # Output file (default stdout) self.outFile = None # Output directory (default none - pwd) self.outDir = None # Write stable beams only self.checkStable = True # Predefine data readers self.lumi = None self.fill = None self.ardy = None self.lhc = None self.lblb = None # Explicitly clean up our DB connections def __del__(self): self.dbHandler.closeAllDB() # Called in command-line mode def execute(self): # Handle command-line switches self.parseOpts() # Process each run in the runlist for runnum in self.runList: # Read all COOL data self.readData(runnum) self.printData(runnum) def parseOpts(self): parser = OptionParser(usage="usage: %prog [options]", add_help_option=False) parser.add_option("-?", "--usage", action="store_true", default=False, dest="help", help="show this help message and exit") parser.add_option("-v", "--verbose", action="store_true", default=self.verbose, dest="verbose", help="turn on verbose output") parser.add_option( "-r", "--run", dest="runlist", metavar="RUN", help="Specific run, range, or comma-separated list of both") parser.add_option("--channel", dest='chan', metavar='N', default=self.lumiChan, help='specify luminosity channel (default: %d)' % self.lumiChan) parser.add_option("--lumiTag", dest='lumitag', metavar='TAG', default=self.lumiTag, help='specify luminosity tag (default: %s)' % self.lumiTag) parser.add_option("--online", action='store_true', default=self.online, dest='online', help='use online luminosity (default: use offline)') parser.add_option("--outDir", dest='outdir', metavar='DIR', default=self.outDir, help='change default output directory') parser.add_option( "--noStable", action="store_false", default=self.checkStable, dest="checkstable", help="write non-stable beam luminosity (default: stable only)") (options, args) = parser.parse_args() if options.help: parser.print_help() sys.exit() if options.verbose: self.verbose = options.verbose self.lumiChan = int(options.chan) self.lumiTag = options.lumitag self.online = options.online self.outDir = options.outdir self.checkStable = options.checkstable # Parse run list if options.runlist != None: # Clear the old one self.runList = [] # Can be comma-separated list of run ranges runlist = options.runlist.split(',') if len(runlist) == 0: print 'Invalid run list specified!' sys.exit() # Go through and check for run ranges for runstr in runlist: subrunlist = runstr.split('-') if len(subrunlist) == 1: # Single run self.runList.append(int(subrunlist[0])) elif len(subrunlist) == 2: # Range of runs for runnum in range(int(subrunlist[0]), int(subrunlist[1]) + 1): self.runList.append(runnum) else: # Too many parameters print 'Invalid run list segment found:', runstr sys.exit() self.runList.sort() if self.verbose: print 'Finished parsing run list:', for runnum in self.runList: print runnum, print def readData(self, runnum): # Luminosity folder if self.verbose: print 'Reading lumi for run %d' % runnum if self.lumi == None: if self.online: self.lumi = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/LBLESTONL') else: self.lumi = CoolDataReader('COOLOFL_TRIGGER/COMP200', '/TRIGGER/OFLLUMI/LBLESTOFL') self.lumi.setTag(self.lumiTag) self.lumi.setChannelId(self.lumiChan) self.lumi.setIOVRangeFromRun(runnum) self.lumi.readData() if self.verbose: print 'Read %d Luminosity records' % len(self.lumi.data) # Load stable beam flag ( if self.verbose: print 'Reading ATLAS ready flag' if self.ardy == None: self.ardy = LumiDBCache('COOLONL_TDAQ/COMP200', '/TDAQ/RunCtrl/DataTakingMode') self.ardy.reader.setIOVRangeFromRun(runnum) self.ardy.reader.readData() if self.verbose: print 'Read %d ATLAS ready records' % len(self.ardy.reader.data) # Load LBLB data (needed to convert to time) if self.verbose: print 'Reading LBLB data' if self.lblb == None: self.lblb = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/LBLB') self.lblb.setIOVRangeFromRun(runnum) self.lblb.readData() if self.verbose: print 'Read %d LBLB records' % len(self.lblb.data) if len(self.lblb.data) > 0: print 'First LB %d/%d' % (self.lblb.data[0].since() >> 32, self.lblb.data[0].since() & 0xFFFFFFFF) print 'Last LB %d/%d' % (self.lblb.data[-1].since() >> 32, self.lblb.data[-1].since() & 0xFFFFFFFF) # Now figure out starting/ending time if len(self.lblb.data) < 1: print 'No LBLB data found!' return # Nothing to work with tlo = self.lblb.data[0].payload()['StartTime'] thi = self.lblb.data[-1].payload()['EndTime'] # Fillparams (slow) if self.verbose: print 'Reading Fillparams' if self.fill == None: self.fill = LumiDBCache('COOLONL_TDAQ/COMP200', '/TDAQ/OLC/LHC/FILLPARAMS') self.fill.reader.setIOVRange(tlo, thi) self.fill.reader.readData() if self.verbose: print 'Read %d FILLPARAMS records' % len(self.fill.reader.data) # LHC information (for stable beams) if self.verbose: print 'Reading LHC information' if self.lhc == None: self.lhc = LumiDBCache('COOLOFL_DCS/COMP200', '/LHC/DCS/FILLSTATE') self.lhc.reader.setIOVRange(tlo, thi) self.lhc.reader.readData() if self.verbose: print 'Read %d LHC records' % len(self.lhc.reader.data) def printData(self, runnum): # Only proceed if we have actual lumi data if len(self.lumi.data) == 0: return f = None # Make time map lblbMap = dict() for obj in self.lblb.data: lblbMap[obj.since()] = (obj.payload()['StartTime'], obj.payload()['EndTime']) # OK, now we want to go through the luminosity records and match the other data for obj in self.lumi.data: run = obj.since() >> 32 lb = obj.since() & 0xFFFFFFFF startTime = lblbMap.get(obj.since(), (0., 0.))[0] endTime = lblbMap.get(obj.since(), (0., 0.))[1] lumi = obj.payload()['LBAvInstLumi'] mu = obj.payload()['LBAvEvtsPerBX'] payload = self.fill.getPayload(startTime) if payload == None: ncol = 0 else: ncol = payload['LuminousBunches'] payload = self.lhc.getPayload(startTime) if payload == None: stable = 0 else: stable = payload['StableBeams'] payload = self.ardy.getPayload(obj.since()) if self.ardy == None: ready = 0 else: ready = payload['ReadyForPhysics'] if self.checkStable and not stable: continue # Open file if not open already if f == None: # Open if self.outDir != None: self.outFile = '%s/run%d.txt' % (self.outDir, runnum) else: self.outFile = 'run%d.txt' % runnum print 'Writing file %s' % self.outFile f = open(self.outFile, 'w') # OK, print it out print >> f, run, lb, startTime / 1.E9, endTime / 1.E9, lumi, mu, ncol, ready # Close file if f != None: f.close()
def main(): f1 = "%s::%s" % (db1, options.folderBS) f2 = "%s::%s" % (db2, options.folderLumi) print("=" * 100) print("Comparing: ") print(" * ", f1, options.tagBS) print(" * ", f2, options.tagLumi) print("=" * 100) requiredForNtuple = ['posX', 'posY', 'posZ', 'sigmaX', 'sigmaY', 'sigmaZ'] checkNtupleProd = all(item in varColl for item in requiredForNtuple) if not checkNtupleProd: print('Ntuple will not be filled missing vars') #Open up required databases from PyCool import cool from CoolConvUtilities import AtlCoolLib cooldbBS = AtlCoolLib.indirectOpen(db1, True, True, False) cooldbLumi = AtlCoolLib.indirectOpen(db2, True, True, False) folderBS = cooldbBS.getFolder(options.folderBS) folderLumi = cooldbLumi.getFolder(options.folderLumi) from InDetBeamSpotExample.COOLUtils import COOLQuery coolQuery = COOLQuery() if options.runMin is not None: iov1 = options.runMin << 32 if options.runMax is not None: iov2 = (options.runMax + 1) << 32 else: iov2 = (options.runMin + 1) << 32 print('Plotting runs %i to %i ' % (iov1, iov2)) else: print('No run selected -- ERROR') return if (iov2 > cool.ValidityKeyMax): iov2 = cool.ValidityKeyMax print("Reading data from database") itrBS = folderBS.browseObjects(iov1, iov2, cool.ChannelSelection.all(), options.tagBS) print("...finished getting BS data") lbDict = dict() startRLB = 0x7FFFFFFFFFFFFFFF endRLB = 0 outfile = ROOT.TFile("BeamspotLumi_%i.root" % (options.runMin), "recreate") ntuple = ROOT.TNtupleD( 'BeamspotLumi', 'BeamSpotLumi', "x:y:z:sigma_x:sigma_y:sigma_z:run:mu:lumi:year:month:day:hour:minute:epoch" ) runs = set() while itrBS.goToNext(): obj = itrBS.currentRef() since = obj.since() runBegin = since >> 32 lumiBegin = since & 0xFFFFFFFF until = obj.until() runUntil = until >> 32 lumiUntil = until & 0xFFFFFFFF status = int(obj.payloadValue('status')) if status != 59: continue runs.add(runBegin) if since < startRLB: startRLB = since if until > endRLB: endRLB = until values = {} for var in varColl: values[var] = float(obj.payloadValue(var)) values[var + 'Err'] = float(obj.payloadValue(var + 'Err')) values['until'] = until lbDict[since] = values print('Runs: ', runs) lumi = array('d') xd = array('d') exd = array('d') ydDict = {} eydDict = {} ydDict2 = {} sqtrt2pi = math.sqrt(2 * math.pi) lblb = CoolDataReader('COOLONL_TRIGGER/CONDBR2', '/TRIGGER/LUMI/LBLB') from DQUtils.sugar import RANGEIOV_VAL, RunLumi from DQUtils import IOVSet grlIOVs = IOVSet.from_grl( "data15_13TeV.periodAllYear_DetStatus-v89-pro21-02_Unknown_PHYS_StandardGRL_All_Good_25ns.xml" ) grlIOVs += IOVSet.from_grl( "data16_13TeV.periodAllYear_DetStatus-v89-pro21-01_DQDefects-00-02-04_PHYS_StandardGRL_All_Good_25ns.xml" ) grlIOVs += IOVSet.from_grl( "data17_13TeV.periodAllYear_DetStatus-v99-pro22-01_Unknown_PHYS_StandardGRL_All_Good_25ns_Triggerno17e33prim.xml" ) grlIOVs += IOVSet.from_grl( "data18_13TeV.periodAllYear_DetStatus-v102-pro22-04_Unknown_PHYS_StandardGRL_All_Good_25ns_Triggerno17e33prim.xml" ) for run in runs: iov1 = run << 32 iov2 = (run + 1) << 32 itrLumi = folderLumi.browseObjects(iov1, iov2, cool.ChannelSelection.all(), options.tagLumi) print("...finished getting Lumi data for run %i" % run) lblb.setIOVRangeFromRun(run) lblb.readData() if len(lblb.data) < 1: print('No LBLB data found!') continue # Make time map lblbMap = dict() for obj in lblb.data: lblbMap[obj.since()] = (obj.payload()['StartTime'], obj.payload()['EndTime']) while itrLumi.goToNext(): obj = itrLumi.currentRef() since = obj.since() runBegin = since >> 32 lumiBegin = since & 0xFFFFFFFF until = obj.until() runUntil = until >> 32 lumiUntil = until & 0xFFFFFFFF inGRL = False for sinceGRL, untilGRL, grl_states in process_iovs(grlIOVs): if grl_states[0].since == None: continue if (sinceGRL.run <= runBegin and untilGRL.run >= runUntil and sinceGRL.lumi <= lumiBegin and untilGRL.lumi >= lumiUntil): inGRL = True break if not inGRL: continue mu = float(obj.payloadValue('LBAvEvtsPerBX')) instlumi = float(obj.payloadValue('LBAvInstLumi')) #if( mu < 10 or mu > 65 ): #print 'Mu: %2.1f Run : %d LB: %d - %d Lumi: %f' % (mu,runBegin,lumiBegin,lumiUntil,instlumi) if since in lbDict: if lbDict[since]['sigmaX'] > 0.1: continue startTime = lblbMap.get(obj.since(), (0., 0.))[0] endTime = lblbMap.get(lbDict[since]['until'], (0., 0.))[0] #[1] end of lumiblock mylumi = (endTime - startTime) / 1e9 * instlumi / 1e9 thisTime = time.gmtime(startTime / 1.e9) year = thisTime[0] month = thisTime[1] day = thisTime[2] hour = thisTime[3] mins = thisTime[4] sec = thisTime[5] lumi.append(mylumi) # in fb^-1 xd.append(mu) exd.append(0) if options.plotSomething: for var in varColl: if not var in ydDict: ydDict[var] = array('d') ydDict2[var] = array('d') eydDict[var] = array('d') ydDict2[var].append(mu / (lbDict[since][var] * sqtrt2pi)) ydDict[var].append(lbDict[since][var]) eydDict[var].append(lbDict[since][var + 'Err']) if checkNtupleProd and lbDict[since]['sigmaZErr'] < 5: ntuple.Fill(lbDict[since]['posX'], lbDict[since]['posY'], lbDict[since]['posZ'], lbDict[since]['sigmaX'], lbDict[since]['sigmaY'], lbDict[since]['sigmaZ'], runBegin, mu, mylumi, year, month, day, hour, mins, startTime / 1.e9) runStart = startRLB >> 32 runEnd = endRLB >> 32 fillStart = fillEnd = 0 timeStart = timeEnd = 0 beamEnergy = 13 try: timeStart = coolQuery.lbTime(int(startRLB >> 32), int(startRLB & 0xFFFFFFFF))[0] except: pass try: timeEnd = coolQuery.lbTime(int(endRLB >> 32), int(endRLB & 0xFFFFFFFF) - 1)[1] except: pass try: fillStart = coolQuery.getLHCInfo(timeStart).get('FillNumber', 0) except: pass try: fillEnd = coolQuery.getLHCInfo(timeEnd).get('FillNumber', 0) except: pass try: beamEnergy = coolQuery.getLHCInfo(timeStart).get('BeamEnergyGeV', 0) beamEnergy *= 2e-3 except: pass ntuple.Write() if not options.plotSomething: return from InDetBeamSpotExample import ROOTUtils ROOTUtils.setStyle() canvas = ROOT.TCanvas('BeamSpotComparison', 'BeamSpotComparison', 1600, 1200) canvas.cd() ROOT.gPad.SetTopMargin(0.05) ROOT.gPad.SetLeftMargin(0.15) ROOT.gPad.SetRightMargin(0.05) if not options.plotGraph: ROOT.gPad.SetRightMargin(0.15) #Plot each variable for var in varColl: if var not in ydDict: print('Missing yd: ', var) if var not in eydDict: print('Missing eyd: ', var) continue gr = ROOT.TGraphErrors(len(xd), xd, ydDict[var], exd, eydDict[var]) xmin = min(xd) xmax = max(xd) ymin = min(ydDict[var]) ymax = max(ydDict[var]) h = (ymax - ymin) ymin -= 0.25 * h ymaxSmall = ymax + 0.25 * h ymax += 0.75 * h ymin2 = min(ydDict2[var]) ymax2 = max(ydDict2[var]) h = (ymax2 - ymin2) ymin2 -= 0.25 * h ymax2 += 0.75 * h h = (xmax - xmin) xmin -= 0.05 * h xmax += 0.05 * h #This histogram is made just to make it easier to manipulate the margins histo = ROOT.TH2D('hd' + var, 'hd' + var, 100, xmin, xmax, 100, ymin, ymax) histo.GetYaxis().SetTitle(varDef(var, 'atit', var)) histo.GetXaxis().SetTitle('Average interactions per bunch crossing') histo.GetZaxis().SetTitle('Entries') histo2 = ROOT.TH2D('hd2' + var, 'hd2' + var, 100, xmin, xmax, 100, ymin2, ymax2) histo2.GetYaxis().SetTitle( "<Interaction density> @ z=0 [interactions/mm]") histo2.GetXaxis().SetTitle('Average interactions per bunch crossing') histo2.GetZaxis().SetTitle('Entries') histo2W = ROOT.TH2D('hd3' + var, 'hd3' + var, 100, xmin, xmax, 100, ymin2, ymax2) histo2W.GetYaxis().SetTitle( "<Interaction density> @ z=0 [interactions/mm]") histo2W.GetXaxis().SetTitle('Average interactions per bunch crossing') histo2W.GetZaxis().SetTitle('Integrated Luminosity (fb^{-1}/bin)') histoW = ROOT.TH2D('hdW' + var, 'hdW' + var, 100, xmin, xmax, 100, ymin, ymax) histoW.GetYaxis().SetTitle(varDef(var, 'atit', var)) histoW.GetXaxis().SetTitle('Average interactions per bunch crossing') histoW.GetZaxis().SetTitle('Integrated Luminosity (fb^{-1}/bin)') histoW1D = ROOT.TH1D('hd1D' + var, 'hd1D' + var, 100, ymin, ymaxSmall) histoW1D.GetXaxis().SetTitle(varDef(var, 'atit', var)) histoW1D.GetYaxis().SetTitle('Integrated Luminosity (fb^{-1}/bin)') histo.Draw() if options.plotGraph: gr.Draw("p") else: for mu, x, l in zip(xd, ydDict[var], lumi): histo.Fill(mu, x) histoW.Fill(mu, x, l) histoW1D.Fill(x, l) for mu, x, l in zip(xd, ydDict2[var], lumi): histo2.Fill(mu, x) histo2W.Fill(mu, x, l) histo.Draw("colz") histo.Write() histoW.Write() histo2.Write() histo2W.Write() histoW1D.Write() # Add some information to the graph ROOTUtils.atlasLabel(0.53, 0.87, False, offset=0.12, isForApproval=False, customstring="Internal", energy='%2.0f' % beamEnergy, size=0.055) ROOTUtils.drawText(0.18, 0.87, 0.055, varDef(var, 'title', var)) comments = [] if runStart == runEnd: comments.append('Run %i' % runStart) else: comments.append('Runs %i - %i' % (runStart, runEnd)) if fillStart == fillEnd: comments.append('Fill %i' % fillStart) else: comments.append('Fills %i - %i' % (fillStart, fillEnd)) t1 = time.strftime('%d %b %Y', time.localtime(timeStart)) t2 = time.strftime('%d %b %Y', time.localtime(timeEnd)) if t1 == t2: comments.append(t1) else: comments.append('%s - %s' % (t1, t2)) ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42) canvas.Print("Run_%d_%sVsMu.png" % (options.runMin, var)) canvas.Print("Run_%d_%sVsMu.pdf" % (options.runMin, var)) if not options.plotGraph: canvas.SetLogz(True) canvas.Print("Run_%d_%sVsMuLog.png" % (options.runMin, var)) canvas.Print("Run_%d_%sVsMuLog.pdf" % (options.runMin, var)) canvas.SetLogz(False) histo2.Draw("colz") ROOTUtils.atlasLabel(0.53, 0.87, False, offset=0.12, isForApproval=False, customstring="Internal", energy='%2.0f' % beamEnergy, size=0.055) ROOTUtils.drawText(0.18, 0.87, 0.055, "Interaction density") ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42) canvas.Print("Run_%d_Mu%sVsMu.png" % (options.runMin, var)) canvas.Print("Run_%d_Mu%sVsMu.pdf" % (options.runMin, var)) canvas.SetLogz(True) canvas.Print("Run_%d_Mu%sVsMuLog.png" % (options.runMin, var)) canvas.Print("Run_%d_Mu%sVsMuLog.pdf" % (options.runMin, var)) canvas.SetLogz(False) histoW.Draw("colz") histoW.SetMinimum(0.005) ROOTUtils.atlasLabel(0.53, 0.87, False, offset=0.12, isForApproval=False, customstring="Internal", energy='%2.0f' % beamEnergy, size=0.055) ROOTUtils.drawText(0.18, 0.87, 0.055, varDef(var, 'title', var)) ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42) canvas.Print("Run_%d_%sVsMuW.png" % (options.runMin, var)) canvas.Print("Run_%d_%sVsMuW.pdf" % (options.runMin, var)) canvas.SetLogz(True) canvas.Print("Run_%d_%sVsMuWLog.png" % (options.runMin, var)) canvas.Print("Run_%d_%sVsMuWLog.pdf" % (options.runMin, var)) canvas.SetLogz(False) histo2W.Draw("colz") histo2W.SetMinimum(0.01) ROOTUtils.atlasLabel(0.53, 0.87, False, offset=0.12, isForApproval=False, customstring="Internal", energy='%2.0f' % beamEnergy, size=0.055) ROOTUtils.drawText(0.18, 0.87, 0.055, "Interaction density") ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42) canvas.Print("Run_%d_Mu%sVsMuW.png" % (options.runMin, var)) canvas.Print("Run_%d_Mu%sVsMuW.pdf" % (options.runMin, var)) canvas.SetLogz(True) canvas.Print("Run_%d_Mu%sVsMuWLog.png" % (options.runMin, var)) canvas.Print("Run_%d_Mu%sVsMuWLog.pdf" % (options.runMin, var)) canvas.SetLogz(False) histoW1D.Draw("colz") ROOTUtils.atlasLabel(0.53, 0.87, False, offset=0.12, isForApproval=False, customstring="Internal", energy='%2.0f' % beamEnergy, size=0.055) ROOTUtils.drawText(0.18, 0.87, 0.055, varDef(var, 'title', var)) ROOTUtils.drawText(0.18, 0.81, 0.05, "#mu=%2.4f RMS=%2.4f" % (histoW1D.GetMean(), histoW1D.GetRMS()), font=42) canvas.Print("Run_%d_%s1D.png" % (options.runMin, var)) canvas.Print("Run_%d_%s1D.pdf" % (options.runMin, var)) canvas.SetLogy(True) canvas.Print("Run_%d_%s1DLog.png" % (options.runMin, var)) canvas.Print("Run_%d_%s1DLog.pdf" % (options.runMin, var)) canvas.SetLogy(False)
class CalibNtupleMaker: def __init__(self): # Control output level self.verbose = False # Luminosity Channels self.lumiChanList = [] # Luminosity Channel Map self.lumiChanNames = dict() self.lumiMapFile = 'defaultChannels.txt' # Stable only self.stableOnly = False # Write output ntuple self.ntuple = True # Write extra current information self.writeExtraCurrents = True # List of (integer) run numbers specified on the command line self.runList = [] # Dict of (lblo, lbhi) pairs giving extent of StableBeams self.stableDict = dict() # Utlity routine for handling COOL connections self.dbHandler = LumiDBHandler() # Data readers self.maskReader = None self.lumiReader = None self.caliReader = None self.lblbReader = None self.currReader = None self.lhcReader = None self.bgReader = None self.pmtReader = None self.lbdataReader = None # Object which stores the BCID information self.maskObj = BCIDMask() # Object to store the bunch group definition self.bgObj = BunchGroup() self.currentRun = 0 self.nBCID = 3564 self.outdir = '.' # Explicitly clean up our DB connections def __del__(self): self.dbHandler.closeAllDB() # Called in command-line mode def execute(self): # Handle command-line switches self.parseOpts() # Fill channel list self.fillChannelList() # Process each run in self.runList for run in self.runList: # Load all data from COOL self.loadBCIDData(run) # Skip if run doesn't exist if len(self.lblbReader.data) == 0: continue # Find stable beams, output goes to self.stableTime self.findStable() if self.stableOnly and len(self.stableTime) == 0: continue # Open new output file and init ntuple if self.ntuple: nt = NtupleHandler() nt.chanMap = self.lumiChanNames nt.writeExtraCurrents = self.writeExtraCurrents nt.fileName = '%s/r%d.root' % (self.outdir, run) nt.open() nt.init() # Storage objects keyed by [runlb][algo] objDict = dict() runlbList = [] # Loop over all objects and sort by runlb and algo for obj in self.lumiReader.data: channel = obj.channelId() runlb = obj.payload()['RunLB'] if runlb not in objDict: objDict[runlb] = dict() runlbList.append(runlb) objDict[runlb][channel] = obj # if self.verbose: # run = runlb >> 32 # lb = runlb & 0xFFFFFFFF # print 'Found Run %d LB %d chan %d' % (run, lb, channel) # LHC Current objects keyed by [runlb] currDict = dict() for obj in self.currReader.data: runlb = obj.payload()['RunLB'] if runlb not in currDict: currDict[runlb] = dict() currDict[runlb][obj.channelId()] = obj #if self.writeExtraCurrents: lbdataDict = dict() for obj in self.lbdataReader.data: runlb = obj.payload()['RunLB'] if runlb not in lbdataDict: lbdataDict[runlb] = dict() lbdataDict[runlb][obj.channelId()] = obj # Loop over all lumi blocks and calibrate each algorithm runlbList.sort() for runlb in runlbList: run = runlb >> 32 lb = runlb & 0xFFFFFFFF self.currentRun = run # Make sure bunch group is valid (keyed by run/lb) if not self.updateBunchGroup(runlb): print "Couldn't find valid Bunch Group definition for Run %d LB %d!" % ( run, lb) continue # Get integer to make average mu value below nBunch = len(self.bgObj.bg) if nBunch < 1: nBunch = 1 # Zero all storage locations nt.clearBCIDData() first = True lbSum = dict() for chan in self.lumiChanList: if chan not in objDict[runlb]: print "Can't find channel", chan, "in run/lb", run, '/', lb continue obj = objDict[runlb][chan] if chan != obj.channelId(): print 'Channel', chan, '!=', obj.channelId(), '!' continue if runlb != obj.payload()['RunLB']: print 'RunLB', runlb, '!=', obj.payload()['RunLB'], '!' continue startTime = obj.since() endTime = obj.until() dtime = (endTime - startTime) / 1E9 valid = obj.payload()['Valid'] & 0x03 # Hack for lucid validity if 100 < chan < 200 and valid == 1: valid = 0 # Check whether this is in stable beams isStable = False for stable in self.stableTime: if not stable[0] <= startTime < stable[1]: continue isStable = True break if self.stableOnly and not isStable: continue # Clear lumi block sum counters lbSum[chan] = 0. # Only do this once per lumi block if first: first = False if self.verbose: print 'Found stable Run %d LB %d' % (run, lb) # Fill general LB information if self.ntuple: nt.fillLBData(obj) nt.lbDataStruct.fStable = isStable # These change slowly, so checking them every run/lb is OK # Make sure BCID mask is valid if not self.updateBCIDMask(startTime): print "Couldn't find valid BCID mask data for Run %d LB %d!" % ( run, lb) continue # Do this here so the BCID mask is up to date if self.ntuple: if runlb in currDict: if 1 in currDict[runlb]: nt.fillCurrentData(currDict[runlb], self.maskObj, 1) if 0 in currDict[ runlb] and self.writeExtraCurrents: nt.fillCurrentData(currDict[runlb], self.maskObj, 0) if runlb in lbdataDict: nt.fillMoreCurrentData(lbdataDict[runlb]) # Now we want to start extracting bunch-by-bunch information # Get raw lumi normValue = obj.payload()['AverageRawInstLum'] blobValue = obj.payload()['BunchRawInstLum'] bcidVec, rawLumi = unpackBCIDValues(blobValue) # dict to hold BCID lumi keyed by BCID bcidLumi = dict() for i in range(len(rawLumi)): # Check if this is in our bunch group (only really need to fill this once) bcid = bcidVec[i] # Protect against any weird values if bcid >= self.nBCID: print 'BCID %d found >= %d!' % (bcid, self.nBCID) continue if bcid in self.bgObj.bg: nt.bcidArray['Status'][bcid] = 1 else: nt.bcidArray['Status'][bcid] = 0 # Now need to save bcid, mu, and calibLumi lraw = rawLumi[i] nt.fillBCIDData(chan, bcid, lraw) # End loop over BCIDs # End of loop over channels nt.tree.Fill() # End of loop over LBs nt.close() # End of loop over runs # Load all data necessary to make bunch-by-bunch lumi for a single run def loadBCIDData(self, runnum): print 'MuHistMaker.loadBCIDData(%s) called' % runnum # Many folders are indexed by time stamp. Load RunLB -> time conversion here self.loadLBLB(runnum) if len(self.lblbReader.data) == 0: return # Determine start and end time of this run startTime = self.lblbReader.data[0].payload()['StartTime'] endTime = self.lblbReader.data[-1].payload()['EndTime'] iov = (startTime, endTime) # Read LHC Data (for stable beams) self.loadLHCData(iov) # Read the bunch group (so we sum over the right thing) self.loadBGData(runnum) # Read BCID Data self.loadBCIDMask(iov) self.loadBCIDLumi(iov) self.loadBCIDCurrents(iov) self.loadOtherCurrents(iov) def loadLBLB(self, run): if self.verbose: print 'Loading LBLB data' # Instantiate new COOL data reader if not already done if self.lblbReader == None: self.lblbReader = CoolDataReader('COOLONL_TRIGGER/CONDBR2', '/TRIGGER/LUMI/LBLB') self.lblbReader.setIOVRangeFromRun(run) self.lblbReader.readData() if self.verbose: print 'Read %d LBLB records' % len(self.lblbReader.data) if len(self.lblbReader.data) > 0: print 'First LB %d/%d' % ( self.lblbReader.data[0].since() >> 32, self.lblbReader.data[0].since() & 0xFFFFFFFF) print 'Last LB %d/%d' % ( self.lblbReader.data[-1].since() >> 32, self.lblbReader.data[-1].since() & 0xFFFFFFFF) def loadBGData(self, run): if self.verbose: print 'Loading Bunch group data' # Instantiate new COOL reader if not already done if self.bgReader == None: self.bgReader = CoolDataReader('COOLONL_TRIGGER/CONDBR2', '/TRIGGER/LVL1/BunchGroupContent') self.bgReader.setIOVRangeFromRun(run) self.bgReader.readData() if self.verbose: print 'Read %d bunch group records' % len(self.bgReader.data) def loadBCIDMask(self, iov): if self.verbose: print 'Loading BCID masks' # Instantiate new COOL data reader if not already done if self.maskReader == None: self.maskReader = CoolDataReader('COOLONL_TDAQ/CONDBR2', '/TDAQ/OLC/LHC/FILLPARAMS') self.maskReader.setIOVRange(iov[0], iov[1]) self.maskReader.readData() if self.verbose: print 'Read %d BCID Mask records' % len(self.maskReader.data) def loadBCIDLumi(self, iov): if self.verbose: print 'Loading BCID luminosity values' # Instantiate new COOL data reader if not already done if self.lumiReader == None: # Switch at start of September self.lumiReader = CoolDataReader('COOLONL_TDAQ/CONDBR2', '/TDAQ/OLC/BUNCHLUMIS') #self.lumiReader.verbose = True self.lumiReader.setIOVRange(iov[0], iov[1]) self.lumiReader.setChannel(self.lumiChanList) self.lumiReader.readData() #self.lumiReader.verbose = False if self.verbose: print 'Read %d Lumi records' % len(self.lumiReader.data) # Bunch currents def loadBCIDCurrents(self, iov): if self.verbose: print 'Loading Bunch Current information' if self.currReader == None: # self.currReader = CoolDataReader('COOLONL_TDAQ/MONP200', '/TDAQ/OLC/LHC/BUNCHDATA') self.currReader = CoolDataReader('COOLONL_TDAQ/CONDBR2', '/TDAQ/OLC/LHC/BUNCHDATA') self.currReader.setIOVRange(iov[0], iov[1]) if self.writeExtraCurrents: self.currReader.setChannel([0, 1]) # 0 = BPTX, 1 = Fast BCT else: self.currReader.setChannelId(1) # 0 = BPTX, 1 = Fast BCT self.currReader.readData() if self.verbose: print 'Read %d Current records' % len(self.currReader.data) def loadOtherCurrents(self, iov): if self.verbose: print 'Loading LBDATA Bunch Current information' if self.lbdataReader == None: self.lbdataReader = CoolDataReader('COOLONL_TDAQ/CONDBR2', '/TDAQ/OLC/LHC/LBDATA') self.lbdataReader.setIOVRange(iov[0], iov[1]) self.lbdataReader.setChannel([0, 1, 2, 3]) # 0 = BPTX, 1 = Fast BCT self.lbdataReader.readData() if self.verbose: print 'Read %d LBDATA Current records' % len( self.lbdataReader.data) # Information about stable beams def loadLHCData(self, iov): if self.verbose: print 'Loading LHC information' # Instantiate new COOL data reader if not already done if self.lhcReader == None: self.lhcReader = CoolDataReader('COOLOFL_DCS/CONDBR2', '/LHC/DCS/FILLSTATE') self.lhcReader.setIOVRange(iov[0], iov[1]) self.lhcReader.readData() if self.verbose: print 'Read %d LHC records' % len(self.lhcReader.data) # Fill the stable beam information in the OflLumiRunData object def findStable(self): if self.verbose: print 'Finding stable beam periods' # First, fill stable beam time periods for this run tlo = cool.ValidityKeyMax thi = cool.ValidityKeyMin self.stableTime = [] for obj in self.lhcReader.data: if obj.payload()['StableBeams'] == 0: continue if tlo > thi: # First stable beams tlo = obj.since() thi = obj.until() elif thi == obj.since(): # Extension of existing stable beams thi = obj.until() else: # Not contiguous, add old to list and start again self.stableTime.append((tlo, thi)) tlo = obj.since() thi = obj.until() if tlo < thi: self.stableTime.append((tlo, thi)) if self.verbose: print 'Stable beam periods found:', self.stableTime def updateBCIDMask(self, startTime): if not self.maskObj.isValid(startTime): self.maskObj.clearValidity() # Find the proper mask object maskData = None for mask in self.maskReader.data: if not mask.since() <= startTime < mask.until(): continue self.maskObj.setMask(mask) break if not self.maskObj.isValid(startTime): return False return True def updateBunchGroup(self, runlb): if not self.bgObj.isValid(runlb): self.bgObj.clearValidity() # Find the proper BG object for bg in self.bgReader.data: if not bg.since() <= runlb < bg.until(): continue self.bgObj.setBG(bg) break if not self.bgObj.isValid(runlb): return False return True def parseOpts(self): parser = OptionParser(usage="usage: %prog [options]", add_help_option=False) parser.add_option("-?", "--usage", action="store_true", default=False, dest="help", help="show this help message and exit") parser.add_option("-v", "--verbose", action="store_true", default=self.verbose, dest="verbose", help="turn on verbose output") parser.add_option("-r", "--updateRun", dest="runlist", metavar="RUN", help="update specific run, or comma separated list") parser.add_option("--noNtuple", action="store_false", default=self.ntuple, dest='ntuple', help="Don't store output ntuple (default: Do)") parser.add_option("-o", "--outputDir", dest="outdir", metavar="DIR", default=self.outdir, help='directory for output ntuple (default: %s)' % self.outdir) parser.add_option( "--noStable", action="store_false", default=self.stableOnly, dest="stable", help="turn off stable beams requirements (default: stable only)") parser.add_option("--channelList", dest='chanlist', metavar="FILE", default=self.lumiMapFile, help='file to read channel list from (default: %s)' % self.lumiMapFile) (options, args) = parser.parse_args() if options.help: parser.print_help() sys.exit() self.verbose = options.verbose self.outdir = options.outdir self.stableOnly = options.stable self.lumiMapFile = options.chanlist # Parse run list if options.runlist != None: # Clear the old one self.runList = [] # Can be comma-separated list of run ranges runlist = options.runlist.split(',') if len(runlist) == 0: print 'Invalid run list specified!' sys.exit() # Go through and check for run ranges for runstr in runlist: subrunlist = runstr.split('-') if len(subrunlist) == 1: # Single run self.runList.append(int(subrunlist[0])) elif len(subrunlist) == 2: # Range of runs for runnum in range(int(subrunlist[0]), int(subrunlist[1]) + 1): self.runList.append(runnum) else: # Too many parameters print 'Invalid run list segment found:', runstr sys.exit() self.runList.sort() if self.verbose: print 'Finished parsing run list:', for runnum in self.runList: print runnum, print def fillChannelList(self): print 'Reading channel list from', self.lumiMapFile # Make sure these are empty self.lumiChanList = [] self.lumiChanNames = dict() f = open(self.lumiMapFile) for line in f.readlines(): line = string.lstrip(line) # Skip obvious comments if len(line) == 0: continue if line[0] == "!": continue if line[0] == "#": continue # Now parse tokens = line.split() chanID = int(tokens[0]) chanName = tokens[1] print 'Found Channel %d: %s' % (chanID, chanName) self.lumiChanList.append(chanID) self.lumiChanNames[chanID] = chanName # End of loop over channels f.close()
to.setValue(2.) print to.getValue(), '-> should be 2.0' to.append(1.) to.append(2.) v = to.getList() print v[0], '-> should be 1.0' print v[1], '-> should be 2.0' ##### FillParamsUtil print 'Testing FillParamsUtil' from ROOT import FillParamsUtil # Find start/end time of run lblbReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/LBLB') lblbReader.setIOVRangeFromRun(215589) lblbReader.readData() iovSince = lblbReader.data[0].payload()['StartTime'] iovUntil = lblbReader.data[-1].payload()['EndTime'] #print iovSince, '-> starting time' print(iovUntil - iovSince) / (10**9 * 3600), '-> end time' print "years after 1970", (iovUntil) / (10**9 * 3600 * 24 * 365) count = 0 fillParamsReader = CoolDataReader("COOLONL_TDAQ/COMP200", "/TDAQ/OLC/LHC/FILLPARAMS") fillParamsReader.setIOVRange(iovSince, iovUntil) fillParamsReader.readData()
def execute(self): self.parseOpts() # Read offline luminosity lumiReader = CoolDataReader('COOLOFL_TRIGGER/COMP200', '/TRIGGER/OFLLUMI/LBLESTOFL') lumiReader.setChannelId(self.lumiChannel) lumiReader.setTag(self.lumiTag) # Read LAr noise bursts larReader = CoolDataReader('COOLOFL_LAR/COMP200', '/LAR/BadChannelsOfl/EventVeto') larReader.setTag('LARBadChannelsOflEventVeto-UPD4-04') # Time to use with LAr noise # lbtimeReader = CoolDataReaderCache('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/LBTIME') lblbReader = CoolDataReader('COOLONL_TRIGGER/COMP200', '/TRIGGER/LUMI/LBLB') # Read ATLAS ready flag readyReader = LumiDBCache('COOLONL_TDAQ/COMP200', '/TDAQ/RunCtrl/DataTakingMode') for run in self.runList: print print 'Generating run', run rootfile = self.outdir + '/run' + str(run) + '.root' # Define ntuple - will open existing file if present nt = TrigNtupleHandler() nt.fileName = rootfile nt.open(update=False) nt.initLBData() nt.initL1TrigData() # Load run information print 'Load trigger information' th = TriggerHandler() th.allL1Triggers = True th.trigList = [] th.loadDataByRun(run) # Load lumi information print 'Load lumi information' lumiReader.setIOVRangeFromRun(run) lumiReader.readData() # Read ATLAS ready information print 'Load ATLAS ready information' readyReader.reader.setIOVRangeFromRun(run) readyReader.reader.readData() # Load time stamps print 'Load LBLB information' lblbReader.setIOVRangeFromRun(run) lblbReader.readData() startTime = lblbReader.data[0].payload()["StartTime"] endTime = lblbReader.data[-1].payload()["EndTime"] # Read bad LAr periods print 'Load LAr information' larReader.setIOVRange(startTime, endTime) larReader.readData() # Now make a list of bad lumi blocks print 'Finding bad LBs' badLB = set() for larData in larReader.data: if larData.payload()["EventVeto"] == 0: continue tlo = larData.since() thi = larData.until() # Find all lumi blocks spanned by this range for lb in lblbReader.data: if lb.payload()["EndTime"] <= tlo: continue ss = lb.since() if lb.payload()["StartTime"] < tlo: badLB.add(ss) print runLBString(ss) if lb.payload()["StartTime"] < thi: badLB.add(ss) print runLBString(ss) if lb.payload()["StartTime"] > thi: break # Process for obj in lumiReader.data: ss = obj.since() lumi = obj.payload()["LBAvInstLumi"] mu = obj.payload()["LBAvEvtsPerBX"] if ss not in th.trigL1Dict: continue trigcount = th.trigL1Dict[ss].TBP[self.trigChan] dtime = th.trigL1Dict[ss].dtime if (dtime > 0.): trigrate = trigcount / dtime else: trigrate = 0. atlasReady = False readypay = readyReader.getPayload(obj.since()) if readypay != None: atlasReady = readypay['ReadyForPhysics'] print runLBString( ss), atlasReady, lumi, mu, trigcount, trigrate, if trigrate > 0.: print lumi / trigrate else: print # ATLAS Ready only if self.ready and (not atlasReady): continue nt.clear() nt.lbData.coolStartTime = th.trigL1Dict[ss].startTime nt.lbData.coolEndTime = th.trigL1Dict[ss].endTime nt.lbData.startTime = nt.lbData.coolStartTime / 1.E9 nt.lbData.endTime = nt.lbData.coolEndTime / 1.E9 nt.lbData.lbTime = dtime nt.lbData.run = obj.since() >> 32 nt.lbData.lb = obj.since() & 0xFFFFFFFF nt.lbData.onlInstLum = lumi nt.lbData.onlEvtsPerBX = mu nt.lbData.ready = atlasReady nt.lbData.larVeto = (ss in badLB) # And save trigger counts nt.fillL1Trig(th.trigL1Dict[ss], th.trigChan) nt.save() #for chan in range(256): # print chan, nt.l1TBP[chan] nt.close()