def __init__(self): # type: () -> None self.parser = DBParser() # The lists all have the same number of elements, e.g.: len(self.lumi_data[trg][run]) == len(self.pu_data[trg][run]) self.ls_data = {} # {'name': { run_number: [LS] } } self.rate_data = {} # {'name': { run_number: { LS: raw_rates } } } self.ps_data = {} # {'name': { run_number: { LS: prescale } } } self.pu_data = {} # {'name': { run_number: { LS: PU } } } self.lumi_data = {} # {'name': { run_number: { LS: iLumi } } } self.det_data = {} # {'name': { run_number: { LS: detecotr_ready } } } self.bw_data = {} # {'name': { run_number: { LS: bandwidth } } } self.size_data = {} # {'name': { run_number: { LS: size } } } self.lumi_info = {} # {run_number: [ (LS,ilum,psi,phys,cms_ready) ] } self.bunch_map = {} # {run_number: nBunches } self.hlt_triggers = [] # List of specific HLT triggers we want to get rates for, if empty --> get all HLT rates self.l1_triggers = [] # List of specific L1 triggers we want to get rates for, if empty --> get all L1 rates self.runs_used = [] self.runs_skipped = [] self.name_list = [] # List of all named objects for which we have data, e.g. triggers, datasets, streams, etc... self.psi_filter = [] self.type_map = {} # Maps each object name to a type: trigger, dataset, stream, or L1A # NOTE: Still need to handle the case where if two objects share the same name, but diff type # NOTE2: This approach should be fine, since DataParser owns the nameing, will need to be careful self.ls_veto = {} # {run_number:[LS list]} - LS to ignore self.name_veto = [] # List of paths/objects to remove from consideration self.use_prescaled_rate = False # If true, then rates are not un-prescaled self.use_cross_section = False # If true, then divide the rate by inst. lumi (only for L1 and HLT trigger data) self.normalize_bunches = True # Normalize by the number of colliding bunches self.correct_for_DT = True self.convert_output = True # Flag to convert data from { LS: data } to [ data ], used in the data getters self.l1_rate_cut = 10e6 # Ignore L1 rates above this threshold self.max_deadtime = 10. self.min_ls = -1 self.max_ls = 9999999 self.use_L1_triggers = False # Gets L1 rates self.use_HLT_triggers = False # Gets HLT rates self.use_streams = False # Gets stream rates self.use_datasets = False # Gets dataset rates self.use_L1A_rate = False # Gets the L1A rates self.use_ps_mask = False # Collects data only for LS in the specified prescale indices self.use_best_lumi = True # Uses best luminosity as determined by BRIL self.use_PLTZ_lumi = False # Uses luminosity reading from PLTZ self.use_HF_lumi = False # Uses luminosity reading from HF self.verbose = True
def __init__(self): # type: () -> None self.parser = DBParser() self.rate_monitor = RateMonitor() self.do_cron_job = False # Set the default state for the rate_monitor and plotter to produce plots for triggers self.rate_monitor.object_list = [] #self.set_plotter_fits = False self.rate_monitor.plotter.set_plotter_fits = False self.rate_monitor.plotter.compare_fits = False self.rate_monitor.use_fills = False self.rate_monitor.use_pileup = True self.rate_monitor.use_lumi = False self.rate_monitor.make_fits = False self.rate_monitor.update_online_fits = False self.rate_monitor.data_parser.normalize_bunches = True self.rate_monitor.data_parser.use_prescaled_rate = False self.rate_monitor.data_parser.use_cross_section = False self.rate_monitor.data_parser.use_L1_triggers = True self.rate_monitor.data_parser.use_HLT_triggers = True self.rate_monitor.data_parser.use_streams = False self.rate_monitor.data_parser.use_datasets = False self.rate_monitor.data_parser.use_L1A_rate = False self.rate_monitor.fitter.use_best_fit = False self.rate_monitor.plotter.color_by_fill = False # Determines how to color the plots self.rate_monitor.plotter.use_fit = False self.rate_monitor.plotter.use_multi_fit = False self.rate_monitor.plotter.show_errors = False self.rate_monitor.plotter.show_eq = False self.rate_monitor.plotter.save_png = True self.rate_monitor.plotter.save_root_file = True self.rate_monitor.plotter.show_fit_runs = False self.rate_monitor.plotter.root_file_name = "rate_plots.root" self.rate_monitor.plotter.label_Y = "pre-deadtime unprescaled rate / num colliding bx [Hz]" self.rate_monitor.plotter.name_X = "< PU >" self.rate_monitor.plotter.units_X = "" self.rate_monitor.plotter.ls_options['show_bad_ls'] = False self.rate_monitor.plotter.ls_options['rm_bad_beams'] = False self.rate_monitor.plotter.ls_options['rm_bad_det'] = False
def DBparser(): output = request.get_json() outputdto = output["dto"] rs = outputdto["DBlist"] outputlist = DBParser.DBParser(rs) outputdto["DBparsinglist"] = outputlist outputdto["DBlist"] = "" return output
def DBconnector(): output = request.get_json() outputdto = output["dto"] ip = outputdto["IP"] user = outputdto["USER"] password = outputdto["PASSWORD"] outputdto["DBlist"] = DBParser.DBConn(ip, user, password) outputdto["USER"] = "" outputdto["PASSWORD"] = "" return output
from DBParser import * dbparser = DBParser() recentRunsFill = dbparser.getRecentRuns() lastFill = str(recentRunsFill[1]) print lastFill runList = [] for item in recentRunsFill[0]: runList.append(str(item)) print " ".join(runList)
def __init__(self): # Suppress root warnings ROOT.gErrorIgnoreLevel = 7000 # Fits and fit files self.fitFile = "Fits/2016/FOG.pkl" # The fit file, can contain both HLT and L1 triggers self.InputFitHLT = None # The fit information for the HLT triggers self.InputFitL1 = None # The fit information for the L1 triggers # DBParser self.parser = DBParser() # A database parser # Rates self.HLTRates = None # HLT rates self.L1Rates = None # L1 rates self.Rates = None # Combined L1 and HLT rates self.deadTimeData = {} # initializing deadTime dict # Run control self.runNumber = -1 # The number of the current run self.numBunches = [-1, -1] # Number of [target, colliding] bunches # Lumisection control self.lastLS = 1 # The last LS that was processed last segment self.currentLS = 1 # The latest LS written to the DB self.slidingLS = -1 # The number of LS to average over, use -1 for no sliding LS self.useLSRange = False # Only look at LS in a certain range # Mode self.triggerMode = None # The trigger mode self.mode = None # Mode: cosmics, circulate, physics # Columns header self.displayRawRates = False # display raw rates, to display prescaled rates, set = True self.pileUp = True # derive expected rate as a function of the pileUp, and not the luminosity # Triggers self.cosmics_triggerList = "monitorlist_COSMICS.list" #default list used when in cosmics mode self.collisions_triggerList = "monitorlist_COLLISIONS.list" #default list used when in collision mode self.triggerList = "" # A list of all the L1 and HLT triggers we want to monitor self.userSpecTrigList = False # User specified trigger list self.usableHLTTriggers = [] # HLT Triggers active during the run that we have fits for (and are in the HLT trigger list if it exists) self.otherHLTTriggers = [] # HLT Triggers active during the run that are not usable triggers self.usableL1Triggers = [] # L1 Triggers active during the run that have fits for (and are in the L1 trigger list if it exists) self.otherL1Triggers = [] # L1 Triggers active during that run that are not usable triggers self.redoTList = True # Whether we need to update the trigger lists self.useAll = False # If true, we will print out the rates for all the HLT triggers self.useL1 = False # If true, we will print out the rates for all the L1 triggers self.totalHLTTriggers = 0 # The total number of HLT Triggers on the menu this run self.totalL1Triggers = 0 # The total number of L1 Triggers on the menu this run self.fullL1HLTMenu = [] self.ignoreStrings = ["Calibration","L1Tech"] # Restrictions self.removeZeros = False # If true, we don't show triggers that have zero rate self.requireLumi = False # If true, we only display tables when aveLumi is not None # Trigger behavior self.percAccept = 50.0 # The acceptence for % diff self.devAccept = 5 # The acceptance for deviation self.badRates = {} # A dictionary: [ trigger name ] { num consecutive bad , whether the trigger was bad last time we checked, rate, expected, dev } self.recordAllBadTriggers = {} # A dictionary: [ trigger name ] < total times the trigger was bad > self.maxCBR = 3 # The maximum consecutive db queries a trigger is allowed to deviate from prediction by specified amount before it's printed out self.displayBadRates = -1 # The number of bad rates we should show in the summary. We use -1 for all self.usePerDiff = False # Whether we should identify bad triggers by perc diff or deviatoin self.sortRates = True # Whether we should sort triggers by their rates self.maxHLTRate = 500 # The maximum prescaled rate we allow an HLT Trigger to have self.maxL1Rate = 30000 # The maximum prescaled rate we allow an L1 Trigger to have # Other options self.quiet = False # Prints fewer messages in this mode self.noColors = False # Special formatting for if we want to dump the table to a file self.sendMailAlerts_static = True # Whether we should send alert mails self.sendMailAlerts_dynamic = self.sendMailAlerts_static self.sendAudioAlerts = False # Whether we should send audio warning messages in the control room (CAUTION) self.isUpdating = True # flag to determine whether or not we're receiving new LS self.showStreams = False # Whether we should print stream information self.showPDs = False # Whether we should print pd information self.totalStreams = 0 # The total number of streams self.maxStreamRate = 1000000 # The maximum rate we allow a "good" stream to have self.maxPDRate = 10000000 # The maximum rate we allow a "good" pd to have self.lumi_ave = "NONE" self.pu_ave = "NONE" self.deadTimeCorrection = True # correct the rates for dead time self.scale_sleeptime = 1.0 # Scales the length of time to wait before sending another query (1.0 = 60sec, 2.0 = 120sec, etc) self.badTriggers = {} #[trigger name] <List of trigger infos>
class ShiftMonitor: def __init__(self): # Suppress root warnings ROOT.gErrorIgnoreLevel = 7000 # Fits and fit files self.fitFile = "Fits/2016/FOG.pkl" # The fit file, can contain both HLT and L1 triggers self.InputFitHLT = None # The fit information for the HLT triggers self.InputFitL1 = None # The fit information for the L1 triggers # DBParser self.parser = DBParser() # A database parser # Rates self.HLTRates = None # HLT rates self.L1Rates = None # L1 rates self.Rates = None # Combined L1 and HLT rates self.deadTimeData = {} # initializing deadTime dict # Run control self.runNumber = -1 # The number of the current run self.numBunches = [-1, -1] # Number of [target, colliding] bunches # Lumisection control self.lastLS = 1 # The last LS that was processed last segment self.currentLS = 1 # The latest LS written to the DB self.slidingLS = -1 # The number of LS to average over, use -1 for no sliding LS self.useLSRange = False # Only look at LS in a certain range # Mode self.triggerMode = None # The trigger mode self.mode = None # Mode: cosmics, circulate, physics # Columns header self.displayRawRates = False # display raw rates, to display prescaled rates, set = True self.pileUp = True # derive expected rate as a function of the pileUp, and not the luminosity # Triggers self.cosmics_triggerList = "monitorlist_COSMICS.list" #default list used when in cosmics mode self.collisions_triggerList = "monitorlist_COLLISIONS.list" #default list used when in collision mode self.triggerList = "" # A list of all the L1 and HLT triggers we want to monitor self.userSpecTrigList = False # User specified trigger list self.usableHLTTriggers = [] # HLT Triggers active during the run that we have fits for (and are in the HLT trigger list if it exists) self.otherHLTTriggers = [] # HLT Triggers active during the run that are not usable triggers self.usableL1Triggers = [] # L1 Triggers active during the run that have fits for (and are in the L1 trigger list if it exists) self.otherL1Triggers = [] # L1 Triggers active during that run that are not usable triggers self.redoTList = True # Whether we need to update the trigger lists self.useAll = False # If true, we will print out the rates for all the HLT triggers self.useL1 = False # If true, we will print out the rates for all the L1 triggers self.totalHLTTriggers = 0 # The total number of HLT Triggers on the menu this run self.totalL1Triggers = 0 # The total number of L1 Triggers on the menu this run self.fullL1HLTMenu = [] self.ignoreStrings = ["Calibration","L1Tech"] # Restrictions self.removeZeros = False # If true, we don't show triggers that have zero rate self.requireLumi = False # If true, we only display tables when aveLumi is not None # Trigger behavior self.percAccept = 50.0 # The acceptence for % diff self.devAccept = 5 # The acceptance for deviation self.badRates = {} # A dictionary: [ trigger name ] { num consecutive bad , whether the trigger was bad last time we checked, rate, expected, dev } self.recordAllBadTriggers = {} # A dictionary: [ trigger name ] < total times the trigger was bad > self.maxCBR = 3 # The maximum consecutive db queries a trigger is allowed to deviate from prediction by specified amount before it's printed out self.displayBadRates = -1 # The number of bad rates we should show in the summary. We use -1 for all self.usePerDiff = False # Whether we should identify bad triggers by perc diff or deviatoin self.sortRates = True # Whether we should sort triggers by their rates self.maxHLTRate = 500 # The maximum prescaled rate we allow an HLT Trigger to have self.maxL1Rate = 30000 # The maximum prescaled rate we allow an L1 Trigger to have # Other options self.quiet = False # Prints fewer messages in this mode self.noColors = False # Special formatting for if we want to dump the table to a file self.sendMailAlerts_static = True # Whether we should send alert mails self.sendMailAlerts_dynamic = self.sendMailAlerts_static self.sendAudioAlerts = False # Whether we should send audio warning messages in the control room (CAUTION) self.isUpdating = True # flag to determine whether or not we're receiving new LS self.showStreams = False # Whether we should print stream information self.showPDs = False # Whether we should print pd information self.totalStreams = 0 # The total number of streams self.maxStreamRate = 1000000 # The maximum rate we allow a "good" stream to have self.maxPDRate = 10000000 # The maximum rate we allow a "good" pd to have self.lumi_ave = "NONE" self.pu_ave = "NONE" self.deadTimeCorrection = True # correct the rates for dead time self.scale_sleeptime = 1.0 # Scales the length of time to wait before sending another query (1.0 = 60sec, 2.0 = 120sec, etc) self.badTriggers = {} #[trigger name] <List of trigger infos> # Use: Opens a file containing a list of trigger names and adds them to the RateMonitor class's trigger list # Note: We do not clear the trigger list, this way we could add triggers from multiple files to the trigger list # -- fileName: The name of the file that trigger names are contained in # Returns: (void) def loadTriggersFromFile(self, fileName): try: file = open(fileName, 'r') except: print "File", fileName, "(a trigger list file) failed to open." return allTriggerNames = file.read().split() # Get all the words, no argument -> split on any whitespace TriggerList = [] for triggerName in allTriggerNames: # Recognize comments if triggerName[0]=='#': continue try: if not str(triggerName) in TriggerList: TriggerList.append(stripVersion(str(triggerName))) except: print "Error parsing trigger name in file", fileName return TriggerList # Use: Formats the header string def getHeader(self): # Define spacing and header maxNameHLT = 0 maxNameL1 = 0 if len(self.usableHLTTriggers)>0 or len(self.otherHLTTriggers)>0: maxNameHLT = max([len(trigger) for trigger in self.usableHLTTriggers+self.otherHLTTriggers]) if len(self.usableL1Triggers)>0 or len(self.otherL1Triggers)>0: maxNameL1 = max([len(trigger) for trigger in self.usableL1Triggers+self.otherL1Triggers]) # Make the name spacing at least 90 maxName = max([maxNameHLT, maxNameL1, 90]) self.spacing = [maxName + 5, 14, 14, 14, 14, 14, 0] self.spacing[5] = max( [ 181 - sum(self.spacing), 0 ] ) self.header = "" # The table header self.header += stringSegment("* TRIGGER NAME", self.spacing[0]) if (self.displayRawRates): self.header += stringSegment("* RAW [Hz]", self.spacing[1]) else: self.header += stringSegment("* ACTUAL [Hz]", self.spacing[1]) self.header += stringSegment("* EXPECTED", self.spacing[2]) self.header += stringSegment("* % DIFF", self.spacing[3]) self.header += stringSegment("* DEVIATION", self.spacing[4]) self.header += stringSegment("* AVE PS", self.spacing[5]) self.header += stringSegment("* COMMENTS", self.spacing[6]) self.hlength = 181 #sum(self.spacing) # Use: Runs the program # Returns: (void) def run(self): # Load the fit and trigger list if self.fitFile != "": inputFit = self.loadFit(self.fitFile) for triggerName in inputFit: if triggerName[0:3] == "L1_": if self.InputFitL1 is None: self.InputFitL1 = {} self.InputFitL1[triggerName] = inputFit[triggerName] elif triggerName[0:4] == "HLT_": if self.InputFitHLT is None: self.InputFitHLT = {} self.InputFitHLT[triggerName] = inputFit[triggerName] # Sort trigger list into HLT and L1 trigger lists if self.triggerList != "": for triggerName in self.triggerList: if triggerName[0:3] == "L1_": self.TriggerListL1.append(triggerName) else: self.TriggerListHLT.append(triggerName) # If there aren't preset trigger lists, use all the triggers that we can fit else: if not self.InputFitHLT is None: self.TriggerListHLT = self.InputFitHLT.keys() if not self.InputFitL1 is None: self.TriggerListL1 = self.InputFitL1.keys() while True: try: previous_run_number = self.runNumber #self.updateRunInfo(previous_run_number) self.runNumber, _, _, mode = self.parser.getLatestRunInfo() self.triggerMode = mode[0] ## ---Run loop start--- is_new_run = ( previous_run_number != self.runNumber or self.runNumber < 0 ) if is_new_run: print "Starting a new run: Run %s" % (self.runNumber) self.lastLS = 1 self.currentLS = 0 self.setMode() physicsActive,avgLumi,avgDeadTime,avgL1rate,PScol = self.queryDB() self.redoTriggerLists() else: physicsActive,avgLumi,avgDeadTime,avgL1rate,PScol = self.queryDB() self.lumi_ave = avgLumi #Construct (or reconstruct) trigger lists if self.redoTList: self.redoTriggerLists() if len(self.HLTRates) == 0 or len(self.L1Rates) == 0: self.redoTList = True # Make sure there is info to use if len(self.HLTRates) == 0 and len(self.L1Rates) == 0: print "No new information can be retrieved. Waiting... (There may be no new LS, or run active may be false)" continue trigger_table = self.getTriggerTable(physicsActive,avgLumi) self.checkTriggers(avgLumi,trigger_table) # If there are lumisection to show, print info for them if self.currentLS > self.lastLS: self.printTable(trigger_table,physicsActive,avgLumi,avgDeadTime,avgL1rate,PScol) self.isUpdating = True else: self.isUpdating = False print "Not enough lumisections. Last LS was %s, current LS is %s. Waiting." % (self.lastLS, self.currentLS) ## ---Run loop start--- self.mailSender() self.sleepWait() except KeyboardInterrupt: print "Quitting. Bye." break # Queries the Database and updates trigger entries def queryDB(self): # Get Rates: [triggerName][LS] { raw rate, prescale } if not self.useLSRange: self.HLTRates = self.parser.getRawRates(self.runNumber, self.lastLS) self.L1Rates = self.parser.getL1RawRates(self.runNumber, self.lastLS) self.streamData = self.parser.getStreamData(self.runNumber, self.lastLS) self.pdData = self.parser.getPrimaryDatasets(self.runNumber, self.lastLS) else: self.HLTRates = self.parser.getRawRates(self.runNumber, self.LSRange[0], self.LSRange[1]) self.L1Rates = self.parser.getL1RawRates(self.runNumber, self.LSRange[0], self.LSRange[1]) self.streamData = self.parser.getStreamData(self.runNumber, self.LSRange[0], self.LSRange[1]) self.pdData = self.parser.getPrimaryDatasets(self.runNumber, self.LSRange[0], self.LSRange[1]) self.totalStreams = len(self.streamData.keys()) self.Rates = {} self.Rates.update(self.HLTRates) self.Rates.update(self.L1Rates) self.totalHLTTriggers = len(self.HLTRates.keys()) self.totalL1Triggers = len(self.L1Rates.keys()) lslist = [] for trig in self.Rates.keys(): if len(self.Rates[trig]) > 0: lslist.append(max(self.Rates[trig])) # Update lastLS self.lastLS = self.currentLS # Update current LS if len(lslist) > 0: self.currentLS = max(lslist) if self.useLSRange: # Adjust runs so we only look at those in our range self.slidingLS = -1 # No sliding LS window self.lastLS = max( [self.lastLS, self.LSRange[0]] ) self.currentLS = min( [self.currentLS, self.LSRange[1] ] ) # Get the inst lumi avgDeadTime = 0 try: self.deadTimeData = self.parser.getDeadTime(self.runNumber) avgDeadTime = 0 except: self.deadTimeData = {} avgDeadTime = None print "Error getting deadtime data" # Get total L1 rate l1rate = 0 try: l1rateData = self.parser.getL1rate(self.runNumber) avgL1rate = 0 except: l1rateData = {} avgL1rate = None print "Error getting total L1 rate data" self.startLS = self.lastLS physicsActive = False avgLumi = 0 PScol = -1 if self.mode != "cosmics": lumiData = self.parser.getLumiInfo(self.runNumber, self.startLS, self.currentLS) self.numBunches = self.parser.getNumberCollidingBunches(self.runNumber) # Find the average lumi since we last checked count = 0 # Get luminosity (only for non-cosmic runs) #for LS, instLumi, psi, physics in lumiData: for LS, instLumi, psi, physics, all_subSys_good in lumiData: # If we are watching a certain range, throw out other LS if self.useLSRange and (LS < self.LSRange[0] or LS > self.LSRange[1]): continue # Average our instLumi if not instLumi is None and physics: physicsActive = True PScol = psi if not avgDeadTime is None and self.deadTimeData.has_key(LS): avgDeadTime += self.deadTimeData[LS] else: avgDeadTime = 0 if not avgL1rate is None and l1rateData.has_key(LS): avgL1rate += l1rateData[LS] else: avgL1rate = 0 avgLumi += instLumi count += 1 if count == 0: avgLumi = None else: avgLumi /= float(count) avgDeadTime /= float(count) avgL1rate /= float(count) else: count = 0 avgLumi = None for LS in l1rateData.keys(): if not avgDeadTime is None and self.deadTimeData.has_key(LS): avgDeadTime += self.deadTimeData[LS] else: avgDeadTime = 0 if not avgL1rate is None and l1rateData.has_key(LS): avgL1rate += l1rateData[LS] else: avgL1rate = 0 if not count == 0: avgDeadTime /= float(count) avgL1rate /= float(count) if self.numBunches[0] > 0 and not avgLumi == None: self.pu_ave = avgLumi/self.numBunches[0]*ppInelXsec/orbitsPerSec else: self.pu_ave = "NONE" return physicsActive,avgLumi,avgDeadTime,avgL1rate,PScol def getTriggerTable(self, physicsActive, avgLumi): # A list of tuples, each a row in the table: # ( { trigger, rate, predicted rate, sign of % diff, abs % diff, sign of sigma, abs sigma, # ave PS, doPred, comment } ) table = {} #Calculate trigger information #for trigger in self.triggerList: tmp_trigger_list = [] tmp_trigger_list += self.HLTRates.keys() tmp_trigger_list += self.L1Rates.keys() #for trigger in self.fullL1HLTMenu: for trigger in tmp_trigger_list: #for trigger in self.Rates.keys(): perc = 0 dev = 0 aveRate = 0 properAvePSRate = 0 avePS = 0 avePSExpected = 0 count = 0 comment = "" doPred = physicsActive and self.mode == "collisions" if not self.Rates.has_key(trigger): continue # If cosmics, don't do predictions if self.mode == "cosmics": doPred = False # Calculate rate if self.mode != "cosmics" and doPred: if not avgLumi is None: if trigger in self.TriggerListL1: self.L1 = True else: self.L1 = False expected = self.calculateRate(trigger, avgLumi) if expected < 0: expected = 0 # Don't let expected value be negative avePSExpected = expected # Get the mean square error (standard deviation) mse = self.getMSE(trigger) else: expected = None avePSExpected = None mse = None correct_for_deadtime = self.deadTimeCorrection if trigger[0:3]=="L1_": correct_for_deadtime = False for LS in self.Rates[trigger].keys(): if LS < self.startLS or LS > self.currentLS: continue prescale = self.Rates[trigger][LS][1] rate = self.Rates[trigger][LS][0] try: deadTime = self.deadTimeData[LS] except: print "trouble getting deadtime for LS: ", LS," setting DT to zero" deadTime = 0.0 if correct_for_deadtime: rate *= 1. + (deadTime/100.) if prescale > 0: properAvePSRate += rate/prescale else: properAvePSRate += rate aveRate += rate count += 1 avePS += prescale if count > 0: if aveRate == 0: comment += "0 counts " aveRate /= count properAvePSRate /= count avePS /= count else: #comment += "PS=0" comment += "No rate yet " doPred = False if doPred and not avePSExpected is None and avePS > 1: avePSExpected /= avePS # skips if we are not making predictions for this trigger and we are throwing zeros #if not doPred and self.removeZeros and aveRate == 0: # continue if self.displayRawRates: rate_val = aveRate else: rate_val = properAvePSRate if doPred and not expected is None: #Do nothing xkcd = "" else: avePSExpected = "" if doPred: if expected == None: perc = "UNDEF" dev = "UNDEF" sign = 1 else: diff = aveRate - expected if expected != 0: perc = 100*diff/expected else: perc = "INF" if mse != 0: dev = diff/mse if abs(dev)>1000000: dev = ">1E6" else: dev = "INF" if perc > 0: sign = 1 else: sign = -1 if perc != "INF": perc = abs(perc) if dev != "INF": dev = abs(dev) else: sign = "" # No prediction, so no sign of a % diff perc = "" # No prediction, so no % diff dev = "" # No prediction, so no deviation table[trigger] = [rate_val,avePSExpected,sign,perc,sign,dev,avePS,doPred,comment] return table def checkTriggers(self,avgLumi,trigger_table): # Reset counting variable self.normal = 0 self.bad = 0 tmp_list = [] for elem in self.triggerList: if elem in tmp_list: continue else: tmp_list.append(elem) #Remove any duplicates for elem in self.fullL1HLTMenu: if elem in tmp_list: continue else: tmp_list.append(elem) #Check if the trigger is good or bad for trigger in tmp_list: #table[trigger] = [rate_val,avePSExpected,sign,perc,sign,dev,avPS,doPred,comment] #if not self.Rates.has_key(trigger): continue if not trigger_table.has_key(trigger): print "Table missing triggers: %s" % trigger continue properAvePSRate = trigger_table[trigger][0] avePSExpected = trigger_table[trigger][1] perc = trigger_table[trigger][3] dev = trigger_table[trigger][5] avePS = trigger_table[trigger][6] doPred = trigger_table[trigger][7] if doPred: # Check for bad rates. if self.isBadTrigger(perc, dev, properAvePSRate, trigger[0:3]=="L1_"): self.bad += 1 # Record if a trigger was bad if not self.recordAllBadRates.has_key(trigger): self.recordAllBadRates[trigger] = 0 self.recordAllBadRates[trigger] += 1 # Record consecutive bad rates if not self.badRates.has_key(trigger): self.badRates[trigger] = [1, True, properAvePSRate, avePSExpected, dev, avePS ] else: last = self.badRates[trigger] self.badRates[trigger] = [ last[0]+1, True, properAvePSRate, avePSExpected, dev, avePS ] else: self.normal += 1 # Remove warning from badRates if self.badRates.has_key(trigger): del self.badRates[trigger] else: if self.isBadTrigger("", "", properAvePSRate, trigger[0:3]=="L1_"): self.bad += 1 # Record if a trigger was bad if not self.recordAllBadRates.has_key(trigger): self.recordAllBadRates[trigger] = 0 self.recordAllBadRates[trigger] += 1 # Record consecutive bad rates if not self.badRates.has_key(trigger): self.badRates[trigger] = [ 1, True, properAvePSRate, -999, -999, -999 ] else: last = self.badRates[trigger] self.badRates[trigger] = [ last[0]+1, True, properAvePSRate, -999, -999, -999 ] else: self.normal += 1 # Remove warning from badRates if self.badRates.has_key(trigger): del self.badRates[trigger] # Use: Prints a section of a table, i.e. all the triggers in a trigger list (like usableHLTTriggers, otherHLTTriggers, etc) def printTableSection(self, triggerList, trigger_table, hasPred, avgLumi=0): tmp_dict = {} # Find the correct triggers for trigger in trigger_table: if trigger in triggerList: tmp_dict[trigger] = trigger_table[trigger] sorted_list = tmp_dict.items() #print "Trigger_table: ",trigger_table.keys() #table --> [trigger]<rate_val,avePSExpected,sign,perc,sign,dev,avPS,doPred,comment> if hasPred: if self.usePerDiff: #Sort by % diff sorted_list.sort(key=lambda tup: tup[1][3], reverse = True) else: #Sort by deviation sorted_list.sort(key=lambda tup: tup[1][5], reverse = True) else: sorted_list.sort(key=lambda tup: tup[1][0], reverse = True) #for trigger, rate, pred, sign, perdiff, dsign, dev, avePS, comment in self.tableData: for tup in sorted_list: trigger = tup[0] rate = trigger_table[trigger][0] pred = trigger_table[trigger][1] sign = trigger_table[trigger][2] perdiff = trigger_table[trigger][3] dsign = trigger_table[trigger][4] dev = trigger_table[trigger][5] avePS = trigger_table[trigger][6] doPred = trigger_table[trigger][7] comment = trigger_table[trigger][8] info = stringSegment("* "+trigger, self.spacing[0]) info += stringSegment("* "+"{0:.2f}".format(rate), self.spacing[1]) if pred != "": info += stringSegment("* "+"{0:.2f}".format(pred), self.spacing[2]) else: info += stringSegment("", self.spacing[2]) if perdiff == "": info += stringSegment("", self.spacing[3]) elif perdiff == "INF": info += stringSegment("* INF", self.spacing[3]) else: info += stringSegment("* "+"{0:.2f}".format(sign*perdiff), self.spacing[3]) if dev == "": info += stringSegment("", self.spacing[4]) elif dev == "INF" or dev == ">1E6": info += stringSegment("* "+dev, self.spacing[4]) else: info += stringSegment("* "+"{0:.2f}".format(dsign*dev), self.spacing[4]) info += stringSegment("* "+"{0:.2f}".format(avePS), self.spacing[5]) info += stringSegment("* "+comment, self.spacing[6]) # Color the bad triggers with warning colors #if avePS != 0 and self.isBadTrigger(perdiff, dev, rate/avePS, trigger[0:3]=="L1_"): if avePS != 0 and self.badRates.has_key(trigger): if not self.noColors: write(bcolors.WARNING) # Write colored text print info if not self.noColors: write(bcolors.ENDC) # Stop writing colored text # Don't color normal triggers else: print info def printTable(self,trigger_table,physicsActive,avgLumi,avgDeadTime,avgL1rate,PScol): self.printHeader() hasPred = physicsActive and self.mode == "collisions" # Print the triggers that we can make predictions for anytriggers = False if len(self.usableHLTTriggers) > 0: print '*' * self.hlength print "Predictable HLT Triggers (ones we have a fit for)" print '*' * self.hlength anytriggers = True self.L1 = False self.printTableSection(self.usableHLTTriggers,trigger_table,hasPred,avgLumi) if len(self.usableL1Triggers) > 0: print '*' * self.hlength print "Predictable L1 Triggers (ones we have a fit for)" print '*' * self.hlength anytriggers = True self.L1 = True self.printTableSection(self.usableL1Triggers,trigger_table,hasPred,avgLumi) #check the full menu for paths deviating past thresholds fullMenu_fits = False #for trigger in self.fullL1HLTMenu: self.getTriggerData(trigger, fullMenu_fits, avgLumi) # Print the triggers that we can't make predictions for (only for certain modes) if self.useAll or self.mode != "collisions" or self.InputFitHLT is None: print '*' * self.hlength print "Unpredictable HLT Triggers (ones we have no fit for or do not try to fit)" print '*' * self.hlength self.L1 = False self.printTableSection(self.otherHLTTriggers,trigger_table,False) self.printTableSection(self.otherL1Triggers,trigger_table,False) anytriggers = True if self.useL1: print '*' * self.hlength print "Unpredictable L1 Triggers (ones we have no fit for or do not try to fit)" print '*' * self.hlength self.L1 = True self.printTableSection(self.otherL1Triggers,trigger_table,False) anytriggers = True if not anytriggers: print '*' * self.hlength print "\n --- No useable triggers --- \n" # Print stream data if self.showStreams: print '*' * self.hlength streamSpacing = [ 50, 20, 25, 25, 25, 25 ] head = stringSegment("* Stream name", streamSpacing[0]) head += stringSegment("* NLumis", streamSpacing[1]) head += stringSegment("* Events", streamSpacing[2]) head += stringSegment("* Stream rate [Hz]", streamSpacing[3]) head += stringSegment("* File size [GB]", streamSpacing[4]) head += stringSegment("* Stream bandwidth [GB/s]", streamSpacing[5]) print head print '*' * self.hlength for name in sorted(self.streamData.keys()): count = 0 streamsize = 0 aveBandwidth = 0 aveRate = 0 for LS, rate, size, bandwidth in self.streamData[name]: streamsize += size aveRate += rate aveBandwidth += bandwidth count += 1 if count > 0: aveRate /= count streamsize /= (count*1000000000.0) aveBandwidth /= (count*1000000000.0) row = stringSegment("* "+name, streamSpacing[0]) row += stringSegment("* "+str(int(count)), streamSpacing[1]) row += stringSegment("* "+str(int(aveRate*23.3*count)), streamSpacing[2]) row += stringSegment("* "+"{0:.2f}".format(aveRate), streamSpacing[3]) row += stringSegment("* "+"{0:.2f}".format(streamsize), streamSpacing[4]) row += stringSegment("* "+"{0:.5f}".format(aveBandwidth), streamSpacing[5]) if not self.noColors and aveRate > self.maxStreamRate: write(bcolors.WARNING) # Write colored text print row if not self.noColors and aveRate > self.maxStreamRate: write(bcolors.ENDC) # Stop writing colored text else: pass # Print PD data if self.showPDs: print '*' * self.hlength pdSpacing = [ 50, 20, 25, 25] head = stringSegment("* Primary Dataset name", pdSpacing[0]) head += stringSegment("* NLumis", pdSpacing[1]) head += stringSegment("* Events", pdSpacing[2]) head += stringSegment("* Dataset rate [Hz]", pdSpacing[3]) print head print '*' * self.hlength for name in self.pdData.keys(): count = 0 aveRate = 0 for LS, rate in self.pdData[name]: aveRate += rate count += 1 if count > 0: aveRate /= count row = stringSegment("* "+name, pdSpacing[0]) row += stringSegment("* "+str(int(count)), pdSpacing[1]) row += stringSegment("* "+str(int(aveRate*23.3*count)), pdSpacing[2]) row += stringSegment("* "+"{0:.2f}".format(aveRate), pdSpacing[3]) if not self.noColors and aveRate > self.maxPDRate: write(bcolors.WARNING) # Write colored text print row if not self.noColors and aveRate > self.maxPDRate: write(bcolors.ENDC) # Stop writing colored text else: pass # Closing information print '*' * self.hlength print "SUMMARY:" if self.mode == "collisions": print "Triggers in Normal Range: %s | Triggers outside Normal Range: %s" % (self.normal, self.bad) if self.mode == "collisions": print "Prescale column index:", if PScol == 0: if not self.noColors and PScol == 0 and self.mode != "other": write(bcolors.WARNING) # Write colored text print PScol, "\t0 - Column 0 is an emergency column in collision mode, please select the proper column" if not self.noColors and PScol == 0 and self.mode != "other": write(bcolors.ENDC) # Stop writing colored text else: print PScol try: print "Average inst. lumi: %.0f x 10^30 cm-2 s-1" % (avgLumi) except: print "Average inst. lumi: Not available" print "Total L1 rate: %.0f Hz" % (avgL1rate) print "Average dead time: %.2f %%" % (avgDeadTime) try: print "Average PU: %.2f" % (self.pu_ave) except: print "Average PU: %s" % (self.pu_ave) print '*' * self.hlength def updateRunInfo(self, previous_run): #updates: #run number #trigger mode (and trigger lists) #self.runNumber, self.triggerMode = self.parser.getLatestRunInfo() self.runNumber, _, _, mode = self.parser.getLatestRunInfo() self.triggerMode = mode[0] is_new_run = ( previous_run != self.runNumber or self.runNumber < 0 ) if is_new_run: print "Starting a new run: Run %s" % (self.runNumber) self.lastLS = 1 self.currentLS = 0 self.setMode() self.redoTriggerLists() def setMode(self): self.sendMailAlerts_dynamic = self.sendMailAlerts_static try: self.triggerMode = self.parser.getTriggerMode(self.runNumber)[0] except: self.triggerMode = "Other" if self.triggerMode.find("cosmics") > -1: self.mode = "cosmics" elif self.triggerMode.find("circulate") > -1: self.mode = "circulate" elif self.triggerMode.find("collisions") > -1: self.mode = "collisions" elif self.triggerMode == "MANUAL": self.mode = "MANUAL" elif self.triggerMode.find("highrate") > -1: self.mode = "other" #self.maxHLTRate = 100000 #self.maxL1Rate = 100000 else: self.mode = "other" # Use: Remakes the trigger lists def redoTriggerLists(self): self.redoTList = False # Reset the trigger lists self.usableHLTTriggers = [] self.otherHLTTriggers = [] self.usableL1Triggers = [] self.otherL1Triggers = [] self.fullL1HLTMenu = [] # Reset bad rate records self.badRates = {} # A dictionary: [ trigger name ] { num consecutive bad, trigger bad last check, rate, expected, dev } self.recordAllBadRates = {} # A dictionary: [ trigger name ] < total times the trigger was bad > #set trigger lists automatically based on mode if not self.useAll and not self.userSpecTrigList: if self.mode == "cosmics" or self.mode == "circulate": self.triggerList = self.loadTriggersFromFile(self.cosmics_triggerList) print "monitoring triggers in: ", self.cosmics_triggerList elif self.mode == "collisions": self.triggerList = self.loadTriggersFromFile(self.collisions_triggerList) print "monitoring triggers in: ", self.collisions_triggerList else: self.triggerList = "" print "No lists to monitor: trigger mode not recognized" self.TriggerListL1 = [] self.TriggerListHLT = [] for triggerName in self.triggerList: if triggerName[0:3]=="L1_": self.TriggerListL1.append(triggerName) elif triggerName[0:4]=="HLT_": self.TriggerListHLT.append(triggerName) # Re-make trigger lists #print self.InputFitHLT.keys() for trigger in self.HLTRates.keys(): if (not self.InputFitHLT is None and self.InputFitHLT.has_key(trigger)) and \ (len(self.TriggerListHLT) !=0 and trigger in self.TriggerListHLT): self.usableHLTTriggers.append(trigger) elif trigger[0:4] == "HLT_" and (self.triggerList == "" or trigger in self.TriggerListHLT): self.otherHLTTriggers.append(trigger) elif (trigger[0:4] == "HLT_"): self.fullL1HLTMenu.append(trigger) for trigger in self.L1Rates.keys(): if (not self.InputFitL1 is None and self.InputFitL1.has_key(trigger)) and \ (len(self.TriggerListL1) != 0 and trigger in self.TriggerListL1): self.usableL1Triggers.append(trigger) elif trigger[0:3] == "L1_" and (self.triggerList =="" or trigger in self.TriggerListL1): self.otherL1Triggers.append(trigger) elif (trigger[0:3] == "L1_"): self.fullL1HLTMenu.append(trigger) self.getHeader() # Use: Prints the table header def printHeader(self): print "\n\n", '*' * self.hlength print "INFORMATION:" print "Run Number: %s" % (self.runNumber) print "LS Range: %s - %s" % (self.startLS, self.currentLS) print "Latest LHC Status: %s" % self.parser.getLHCStatus()[1] print "Number of colliding bunches: %s" % self.numBunches[0] print "Trigger Mode: %s (%s)" % (self.triggerMode, self.mode) print "Number of HLT Triggers: %s \nNumber of L1 Triggers: %s" % (self.totalHLTTriggers, self.totalL1Triggers) print "Number of streams:", self.totalStreams print '*' * self.hlength print self.header # Use: Returns whether a given trigger is bad # Returns: Whether the trigger is bad def isBadTrigger(self, perdiff, dev, psrate, isL1): if psrate == 0.0: return False if ( (self.usePerDiff and perdiff!="INF" and perdiff!="" and abs(perdiff)>self.percAccept) or (dev!="INF" and dev!="" and (dev==">1E6" or abs(dev)>self.devAccept)))\ or (perdiff!="INF" and perdiff!="" and abs(perdiff)>self.percAccept and dev!="INF" and dev!="" and abs(dev)>self.devAccept)\ or (isL1 and psrate>self.maxL1Rate)\ or (not isL1 and psrate>self.maxHLTRate): return True return False def mailSender(self): if self.displayBadRates != 0: count = 0 if self.displayBadRates != -1: write("First %s triggers that are bad: " % (self.displayBadRates)) elif len(self.badRates) > 0 : write("All triggers deviating past thresholds from fit and/or L1 rate > %s Hz, HLT rate > %s Hz: " %(self.maxL1Rate,self.maxHLTRate)) for trigger in self.badRates: if self.badRates[trigger][1]: count += 1 write(trigger) if count != self.displayBadRates-1: write(", ") if count == self.displayBadRates: write(".....") break print "" # Print warnings for triggers that have been repeatedly misbehaving mailTriggers = [] # A list of triggers that we should mail alerts about for trigger in self.badRates: if self.badRates[trigger][1]: if self.badRates[trigger][0] >= 1: print "Trigger %s has been out of line for more than %.1f minutes" % (trigger, float(self.badRates[trigger][0])*self.scale_sleeptime) # We want to mail an alert whenever a trigger exits the acceptable threshold envelope if self.badRates[trigger][0] == self.maxCBR: mailTriggers.append( [ trigger, self.badRates[trigger][2], self.badRates[trigger][3], self.badRates[trigger][4], self.badRates[trigger][5] ] ) # Send mail alerts if len(mailTriggers) > 0 and self.isUpdating: if self.sendMailAlerts_static and self.sendMailAlerts_dynamic: self.sendMail(mailTriggers) if self.sendAudioAlerts: audioAlert() # Use: Sleeps and prints out waiting dots def sleepWait(self): if not self.quiet: print "Sleeping for %.1f sec before next query" % (60.0*self.scale_sleeptime) for iSleep in range(20): if not self.quiet: write(".") sys.stdout.flush() time.sleep(3.0*self.scale_sleeptime) sys.stdout.flush() print "" # Use: Loads the fit data from the fit file # Parameters: # -- fitFile: The file that the fit data is stored in (a pickle file) # Returns: The input fit data def loadFit(self, fileName): if fileName == "": return None InputFit = {} # Initialize InputFit (as an empty dictionary) # Try to open the file containing the fit info try: pkl_file = open(fileName, 'rb') InputFit = pickle.load(pkl_file) pkl_file.close() except: # File failed to open print "Error: could not open fit file: %s" % (fileName) return InputFit # Use: Calculates the expected rate for a trigger at a given ilumi based on our input fit def calculateRate(self, triggerName, ilum): # Make sure we have a fit for the trigger if not self.L1 and (self.InputFitHLT is None or not self.InputFitHLT.has_key(triggerName)): return 0 elif self.L1 and ((self.InputFitL1 is None) or not self.InputFitL1.has_key(triggerName)): return 0 # Get the param list if self.L1: paramlist = self.InputFitL1[triggerName] else: paramlist = self.InputFitHLT[triggerName] # Calculate the rate if paramlist[0]=="exp": funcStr = "%s + %s*expo(%s+%s*x)" % (paramlist[1], paramlist[2], paramlist[3], paramlist[4]) # Exponential else: funcStr = "%s+x*(%s+ x*(%s+x*%s))" % (paramlist[1], paramlist[2], paramlist[3], paramlist[4]) # Polynomial fitFunc = TF1("Fit_"+triggerName, funcStr) if self.pileUp: if self.numBunches[0] > 0: return self.numBunches[0]*fitFunc.Eval(ilum/self.numBunches[0]*ppInelXsec/orbitsPerSec) else: return 0 return fitFunc.Eval(ilum) # Use: Gets the MSE of the fit def getMSE(self, triggerName): if not self.L1 and (self.InputFitHLT is None or not self.InputFitHLT.has_key(triggerName)): return 0 elif self.L1 and ((self.InputFitL1 is None) or not self.InputFitL1.has_key(triggerName)): return 0 if self.L1: paramlist = self.InputFitL1[triggerName] else: paramlist = self.InputFitHLT[triggerName] if self.pileUp: return self.numBunches[0]*paramlist[5] return paramlist[5] # The MSE # Use: Sends an email alert # Parameters: # -- mailTriggers: A list of triggers that we should include in the mail, ( { triggerName, aveRate, expected rate, standard dev } ) # Returns: (void) def sendMail(self, mailTriggers): mail = "Run: %d, Lumisections: %s - %s \n" % (self.runNumber, self.lastLS, self.currentLS) try: mail += "Average inst. lumi: %.0f x 10^30 cm-2 s-1\n" % (self.lumi_ave) except: mail += "Average inst. lumi: %s x 10^30 cm-2 s-1\n" % (self.lumi_ave) try: mail += "Average PU: %.2f\n \n" % (self.pu_ave) except: mail += "Average PU: %s\n \n" % (self.pu_ave) mail += "Trigger rates deviating from acceptable and/or expected values: \n\n" for triggerName, rate, expected, dev, ps in mailTriggers: if self.numBunches[0] == 0: mail += "\n %s: Actual: %s Hz\n" % (stringSegment(triggerName, 35), rate) else: if expected > 0: try: mail += "\n %s: Expected: %.1f Hz, Actual: %.1f Hz, Unprescaled Expected/nBunches: %.5f Hz, Unprescaled Actual/nBunches: %.5f Hz, Deviation: %.1f\n" % (stringSegment(triggerName, 35), expected, rate, expected*ps/self.numBunches[0], rate*ps/self.numBunches[0], dev) except: mail += "\n %s: Expected: %s Hz, Actual: %s Hz, Unprescaled Expected/nBunches: %s Hz, Unprescaled Actual/nBunches: %s Hz, Deviation: %s\n" % (stringSegment(triggerName, 35), expected, rate, expected*ps/self.numBunches[0], rate*ps/self.numBunches[0], dev) mail += " *referenced fit: <https://raw.githubusercontent.com/cms-tsg-fog/RateMon/master/Fits/2016/plots/%s.png>\n" % (triggerName) else: try: mail += "\n %s: Actual: %.1f Hz\n" % (stringSegment(triggerName, 35), rate) except: mail += "\n %s: Actual: %s Hz\n" % (stringSegment(triggerName, 35), rate) try: wbm_url = self.parser.getWbmUrl(self.runNumber,triggerName,self.currentLS) if not wbm_url == "-": mail += " *WBM rate: <%s>\n" % (wbm_url) except: print "WBM plot url query failed" mail += "\nWBM Run Summary: <https://cmswbm.web.cern.ch/cmswbm/cmsdb/servlet/RunSummary?RUN=%s> \n\n" % (self.runNumber) mail += "Email warnings triggered when: \n" mail += " - L1 or HLT rates deviate by more than %s standard deviations from fit \n" % (self.devAccept) mail += " - HLT rates > %s Hz \n" % (self.maxHLTRate) mail += " - L1 rates > %s Hz \n" % (self.maxL1Rate) print "--- SENDING MAIL ---\n"+mail+"\n--------------------" mailAlert(mail)
class MonitorController: def __init__(self): # type: () -> None self.parser = DBParser() self.rate_monitor = RateMonitor() self.do_cron_job = False # Set the default state for the rate_monitor and plotter to produce plots for triggers self.rate_monitor.object_list = [] self.set_plotter_fits = False self.compareFits = False self.rate_monitor.use_fills = False self.rate_monitor.use_pileup = True self.rate_monitor.use_lumi = False self.rate_monitor.make_fits = False self.rate_monitor.update_online_fits = False self.rate_monitor.data_parser.normalize_bunches = True self.rate_monitor.data_parser.use_prescaled_rate = False self.rate_monitor.data_parser.use_cross_section = False self.rate_monitor.data_parser.use_L1_triggers = True self.rate_monitor.data_parser.use_HLT_triggers = True self.rate_monitor.data_parser.use_streams = False self.rate_monitor.data_parser.use_datasets = False self.rate_monitor.data_parser.use_L1A_rate = False self.rate_monitor.fitter.use_best_fit = False self.rate_monitor.plotter.color_by_fill = False # Determines how to color the plots self.rate_monitor.plotter.use_fit = False self.rate_monitor.plotter.use_multi_fit = False self.rate_monitor.plotter.show_errors = False self.rate_monitor.plotter.show_eq = False self.rate_monitor.plotter.save_png = True self.rate_monitor.plotter.save_root_file = True self.rate_monitor.plotter.root_file_name = "rate_plots.root" self.rate_monitor.plotter.label_Y = "pre-deadtime unprescaled rate / num colliding bx [Hz]" self.rate_monitor.plotter.name_X = "< PU >" self.rate_monitor.plotter.units_X = "" self.rate_monitor.plotter.ls_options['show_bad_ls'] = False self.rate_monitor.plotter.ls_options['rm_bad_beams'] = False self.rate_monitor.plotter.ls_options['rm_bad_det'] = False # Use: Parses arguments from the command line and sets class variables # Returns: True if parsing was successful, False if not def parseArgs(self): # type: () -> bool # Get the command line arguments try: opt, args = getopt.getopt(sys.argv[1:], "", [ "fitFile=", "triggerList=", "saveDirectory=", "useFit=", "psFilter=", "lsVeto=", "pathVeto=", "Secondary", "datasetRate", "L1ARate", "streamRate", "streamBandwidth", "streamSize", "cronJob", "updateOnlineFits", "createFit", "multiFit", "bestFit", "nonLinear", "vsInstLumi", "vsLS", "useCrossSection", "useFills", "useBunches", "compareFits=" ]) except: print "Error getting options: command unrecognized. Exiting." return False self.rate_monitor.ops = opt for label, op in opt: if label == "--fitFile": # Specify the .pkl file to be used to extract fits from # TODO: Needs to be updated to account for the fact that the plotter is expecting a different input fits = self.readFits(str(op)) #self.rate_monitor.plotter.setFits(fits) self.set_plotter_fits = True self.rate_monitor.plotter.use_fit = True self.rate_monitor.plotter.show_errors = True self.rate_monitor.plotter.show_eq = True elif label == "--triggerList": # Specify the .list file that determines which triggers will be plotted trigger_list = self.readTriggerList(str(op)) self.rate_monitor.object_list = trigger_list self.rate_monitor.data_parser.hlt_triggers = [] self.rate_monitor.data_parser.l1_triggers = [] for name in trigger_list: if name[0:4] == "HLT_": self.rate_monitor.data_parser.hlt_triggers.append(name) elif name[0:3] == "L1_": self.rate_monitor.data_parser.l1_triggers.append(name) elif label == "--saveDirectory": # Specify the directory that the plots will be saved to # NOTE: Doesn't work with --Secondary option self.rate_monitor.save_dir = str(op) elif label == "--useFit": # Use a specific fit type as the 'default' fit # NEEDS TO BE IMPLEMENTED/TESTED self.rate_monitor.make_fits = True self.rate_monitor.plotter.default_fit = str(op) self.rate_monitor.plotter.use_multi_fit = False self.rate_monitor.plotter.use_fit = True self.rate_monitor.plotter.show_errors = True self.rate_monitor.plotter.show_eq = True elif label == "--psFilter": # Specify which prescale indicies to use, ex: '--psFilter=1,2,3' will only use PSI 1,2,3 self.rate_monitor.data_parser.use_ps_mask = True self.rate_monitor.data_parser.psi_filter = [ int(x) for x in str(op).split(',') ] elif label == "--lsVeto": # Specifiy certain LS to veto/remove from consideration ls_veto = self.readLSVetoFile(str(op)) self.rate_monitor.data_parser.ls_veto = ls_veto elif label == "--pathVeto": # Specify certain paths to veto/remove from consideration path_veto_list = self.readTriggerList(str(op)) self.rate_monitor.data_parser.name_veto = path_veto_list elif label == "--Secondary": # Set the code to produce certification plots # NOTE: Still need to specify --fitFile and --triggerList # NEEDS TO BE IMPLEMENTED/TESTED self.rate_monitor.certify_mode = True self.rate_monitor.use_pileup = False self.rate_monitor.use_lumi = False self.rate_monitor.use_fills = False self.rate_monitor.use_LS = True self.rate_monitor.make_fits = False self.rate_monitor.data_parser.use_L1_triggers = True self.rate_monitor.data_parser.use_HLT_triggers = True self.rate_monitor.data_parser.normalize_bunches = False self.rate_monitor.data_parser.use_prescaled_rate = False self.rate_monitor.data_parser.max_deadtime = 100. self.rate_monitor.plotter.use_fit = True self.rate_monitor.plotter.save_root_file = True elif label == "--datasetRate": # Make plots of dataset rates self.rate_monitor.data_parser.use_L1_triggers = False self.rate_monitor.data_parser.use_HLT_triggers = False self.rate_monitor.data_parser.use_streams = False self.rate_monitor.data_parser.use_datasets = True self.rate_monitor.data_parser.use_L1A_rate = False self.rate_monitor.plotter.root_file_name = "Dataset_Rates.root" #self.rate_monitor.plotter.label_Y = "dataset rate / num colliding bx [Hz]" elif label == "--L1ARate": # Make plots of the L1A rate self.rate_monitor.data_parser.use_L1_triggers = False self.rate_monitor.data_parser.use_HLT_triggers = False self.rate_monitor.data_parser.use_streams = False self.rate_monitor.data_parser.use_datasets = False self.rate_monitor.data_parser.use_L1A_rate = True self.rate_monitor.plotter.root_file_name = "L1A_Rates.root" #self.rate_monitor.plotter.label_Y = "L1A rate / num colliding bx [Hz]" elif label == "--streamRate": # Make plots of the stream rates self.rate_monitor.data_parser.use_L1_triggers = False self.rate_monitor.data_parser.use_HLT_triggers = False self.rate_monitor.data_parser.use_streams = True self.rate_monitor.data_parser.use_datasets = False self.rate_monitor.data_parser.use_L1A_rate = False self.rate_monitor.plotter.root_file_name = "Stream_Rates.root" #self.rate_monitor.plotter.label_Y = "stream rate / num colliding bx [Hz]" elif label == "--streamBandwidth": # Make plots of the stream bandwidths # NEEDS TO BE TESTED self.rate_monitor.use_stream_bandwidth = True self.rate_monitor.data_parser.use_L1_triggers = False self.rate_monitor.data_parser.use_HLT_triggers = False self.rate_monitor.data_parser.use_streams = True self.rate_monitor.data_parser.use_datasets = False self.rate_monitor.data_parser.use_L1A_rate = False self.rate_monitor.data_parser.normalize_bunches = False self.rate_monitor.plotter.root_file_name = "Stream_Bandwidth.root" elif label == "--streamSize": # Make plots of stream sizes # NEEDS TO BE TESTED self.rate_monitor.use_stream_size = True self.rate_monitor.data_parser.use_L1_triggers = False self.rate_monitor.data_parser.use_HLT_triggers = False self.rate_monitor.data_parser.use_streams = True self.rate_monitor.data_parser.use_datasets = False self.rate_monitor.data_parser.use_L1A_rate = False self.rate_monitor.data_parser.normalize_bunches = False self.rate_monitor.plotter.root_file_name = "Stream_Size.root" elif label == "--cronJob": # Set the code to produce plots for the cron jobs # NOTE: Still need to specify --triggerList, --saveDirectory, and --fitFile # NEEDS MORE TESTING self.do_cron_job = True self.rate_monitor.use_pileup = True self.rate_monitor.use_fills = False self.rate_monitor.make_fits = False self.rate_monitor.use_grouping = True self.rate_monitor.data_parser.use_L1_triggers = True self.rate_monitor.data_parser.use_HLT_triggers = True self.rate_monitor.data_parser.use_streams = True self.rate_monitor.data_parser.use_datasets = True #self.rate_monitor.data_parser.use_L1A_rate = False self.rate_monitor.data_parser.use_L1A_rate = True self.rate_monitor.plotter.color_by_fill = False self.rate_monitor.plotter.use_fit = True self.rate_monitor.plotter.show_errors = True self.rate_monitor.plotter.show_eq = True self.rate_monitor.plotter.ls_options['show_bad_ls'] = True self.rate_monitor.plotter.ls_options['rm_bad_beams'] = True self.rate_monitor.plotter.ls_options['rm_bad_det'] = False self.rate_monitor.plotter.root_file_name = "Cron_Job_Rates.root" #self.rate_monitor.plotter.label_Y = "pre-deadtime unprescaled rate / num colliding bx [Hz]" self.rate_monitor.plotter.name_X = "< PU >" self.rate_monitor.plotter.label_Y = "< PU >" elif label == "--updateOnlineFits": # Creates fits and saves them to the Fits directory # NEEDS TO BE IMPLEMENTED/TESTED self.rate_monitor.update_online_fits = True self.rate_monitor.use_pileup = True self.rate_monitor.make_fits = False # We make this false, since we need to make more then one fit file self.rate_monitor.data_parser.use_L1_triggers = True self.rate_monitor.data_parser.use_HLT_triggers = True self.rate_monitor.data_parser.use_streams = False self.rate_monitor.data_parser.use_datasets = False self.rate_monitor.data_parser.use_L1A_rate = False self.rate_monitor.plotter.use_fit = True self.rate_monitor.plotter.show_errors = True self.rate_monitor.plotter.show_eq = True self.rate_monitor.plotter.save_root_file = False self.rate_monitor.object_list = self.readTriggerList( "monitorlist_COLLISIONS.list") elif label == "--createFit": # Specify that we should create fits self.rate_monitor.make_fits = True self.rate_monitor.plotter.use_fit = True self.rate_monitor.plotter.show_errors = True self.rate_monitor.plotter.show_eq = True elif label == "--multiFit": # Specify that we should plot all of the fit functions on the same plot #self.rate_monitor.make_fits = True self.rate_monitor.plotter.use_fit = True self.rate_monitor.plotter.use_multi_fit = True self.rate_monitor.plotter.show_errors = False self.rate_monitor.plotter.show_eq = False elif label == "--bestFit": # Specify that only the best fit is to be used (as opposed to only the default one) self.rate_monitor.make_fits = True self.rate_monitor.fitter.use_best_fit = True self.rate_monitor.plotter.use_fit = True self.rate_monitor.plotter.show_errors = True self.rate_monitor.plotter.show_eq = True elif label == "--vsInstLumi": # Plot vs the instantaenous luminosity self.rate_monitor.use_pileup = False self.rate_monitor.use_lumi = True self.rate_monitor.use_LS = False elif label == "--vsLS": # Plot vs the LS self.rate_monitor.use_pileup = False self.rate_monitor.use_lumi = False self.rate_monitor.use_LS = True elif label == "--useCrossSection": # Plot the (rate/inst. lumi) vs. <PU> self.rate_monitor.data_parser.normalize_bunches = False self.rate_monitor.data_parser.use_cross_section = True elif label == "--useFills": # Specify that the data should fetched by fill number self.rate_monitor.use_fills = True self.rate_monitor.plotter.color_by_fill = True # Might want to make this an optional switch elif label == "--useBunches": # Don't try to normalize the rates by colliding bunches self.rate_monitor.data_parser.normalize_bunches = False elif label == "--compareFits": data_dict = self.readDataListTextFile(str(op)) self.rate_monitor.data_dict = data_dict self.compareFits = True else: print "Unimplemented option '%s'." % label return False # Process Arguments if len(args) > 0: # There are arguments to look at arg_list = [] for item in args: arg_list.append(int(item)) if self.rate_monitor.use_fills: self.rate_monitor.fill_list = arg_list #self.rate_monitor.run_list = self.getRuns(arg_list) self.rate_monitor.data_dict['user_input'] = self.getRuns( arg_list) else: self.rate_monitor.fill_list = [] #self.rate_monitor.run_list = arg_list self.rate_monitor.data_dict['user_input'] = arg_list # Append the user specified fills or runs to the dictionary made from the compareFits text file unique_runs = set() for data_group, runs in self.rate_monitor.data_dict.iteritems(): unique_runs = unique_runs.union(runs) self.rate_monitor.run_list = list(unique_runs) #print self.rate_monitor.run_list if len(self.rate_monitor.run_list) == 0: print "ERROR: No runs specified!" return False if self.set_plotter_fits: self.rate_monitor.plotter.setFits(fits) # This needs to be done after we have our run_list, otherwise we can't get the run_list! if self.do_cron_job: if len(self.rate_monitor.plotter.fits.keys()) == 0: print "WARNING: No fit file specified!" self.rate_monitor.plotter.use_fit = False if len(self.rate_monitor.object_list) == 0: print "WARNING: No trigger list specified! Plotting all triggers..." run_list = sorted(self.rate_monitor.run_list) # We check that each run used a physics menu #tmp_list = [] #for run in run_list: # if self.parser.getRunInfo(run): # if self.parser.HLT_Key[:14] == "/cdaq/physics/": # tmp_list.append(run) #run_list = list(tmp_list) if len(run_list) == 0: print "No valid runs. Exiting" return False ## Find which HLT paths are included in which streams #try: # # {'stream': ['trigger_name'] } # #stream_map = self.parser.getPathsInStreams(run_list[-1]) # Use the most recent run to generate the map # stream_map = {} # for run in run_list: # tmp_map = self.parser.getPathsInStreams(run) # for stream in tmp_map: # if not stream_map.has_key(stream): # stream_map[stream] = set() # stream_map[stream] = stream_map[stream] | set(tmp_map[stream]) # for stream in stream_map: # stream_map[stream] = list(stream_map[stream]) #except: # print "ERROR: Failed to get stream map" # return False ## Find which HLT paths are included in which datasets #try: # dataset_map = {} # for run in run_list: # tmp_map = self.parser.getPathsInDatasets(run) # for dataset in tmp_map: # if not dataset_map.has_key(dataset): # dataset_map[dataset] = set() # dataset_map[dataset] = dataset_map[dataset] | set(tmp_map[dataset]) # for dataset in dataset_map: # dataset_map[dataset] = list(dataset_map[dataset]) #except: # print "ERROR: Failed to get dataset map" # return False # Find which HLT paths are included in which streams/datasets try: path_mapping = {} for run in run_list: tmp_map = self.parser.getPathsInDatasets(run) #tmp_map = self.parser.getPathsInStreams(run) for group_name in tmp_map: if not path_mapping.has_key(group_name): path_mapping[group_name] = set() path_mapping[group_name] = path_mapping[ group_name] | set(tmp_map[group_name]) for group_name in path_mapping: path_mapping[group_name] = list(path_mapping[group_name]) except: print "ERROR: Failed to get stream/dataset map" return False group_map = {} # Add specific triggers to monitor to the group map, list of all objects from .list file group_map["Monitored_Triggers"] = list( self.rate_monitor.object_list) # We look for triggers in all runs to ensure we don't miss any (this should be unnecessary for the cron job though) L1_triggers = set() for run in sorted(run_list): tmp_list = self.parser.getL1Triggers(run) for item in tmp_list: L1_triggers.add(item) # Add L1 triggers to the group map, list of all L1 triggers group_map["L1_Triggers"] = list(L1_triggers) self.rate_monitor.data_parser.l1_triggers = list(L1_triggers) # Add HLT triggers to the group map hlt_triggers = set() for path_owner in path_mapping: group_map[path_owner] = path_mapping[path_owner] for item in path_mapping[path_owner]: hlt_triggers.add(item) # Make a group for all HLT triggers group_map["HLT_Triggers"] = list(hlt_triggers) self.rate_monitor.data_parser.hlt_triggers = list(hlt_triggers) # Update the object_list to include all the L1/HLT triggers in the menu self.rate_monitor.object_list += list(L1_triggers) self.rate_monitor.object_list += list(hlt_triggers) self.rate_monitor.group_map = group_map return True def readFits(self, fit_file): # type: (str) -> Dict[str,Dict[str,List[Any]]] fits = {} # Try to open the file containing the fit info print "Reading fit file: %s" % (fit_file) try: pkl_file = open(fit_file, 'rb') fits = pickle.load(pkl_file) # {'obj': fit_params} pkl_file.close() for trig in fits: if type(fits[trig]) is list: fits_format = 'dict_of_lists' if type(fits[trig]) is dict: fits_format = 'nested_dict' if fits_format is 'dict_of_lists': tmp_dict = {} # {'obj': {'fit_type': fit_params } } for obj in fits: if fits[obj] is None: # We were unable to generate a proper fit for this trigger continue fit_type = fits[obj][0] tmp_dict[obj] = {} tmp_dict[obj][fit_type] = fits[obj] fits = tmp_dict return fits if fits_format is 'nested_dict': return fits except: # File failed to open print "Error: could not open fit file: %s" % (fit_file) print "Info:", sys.exc_info()[0] return {} def readTriggerList(self, trigger_file): # type: (str) -> List[str] path = trigger_file f = open(path, 'r') print "Reading trigger file: %s" % path output_list = [] for line in f: line = line.strip() # Remove whitespace/EOL chars if line[0] == "#": continue output_list.append(line) f.close() return output_list # Creates a filter for the DataParser to exlude specific LS def readLSVetoFile(self, ls_veto_file): json_file = open(ls_veto_file) json_data = json.load(json_file) ls_veto = {} for run_number in json_data: ls_veto[int(run_number)] = [] for ls_low, ls_high in json_data[run_number]: ls_veto[int(run_number)] += [ x for x in range(ls_low, ls_high + 1) ] return ls_veto # Gets the runs from each fill specified in arg_list def getRuns(self, arg_list): # type: (List[int]) -> List[int] if len(arg_list) == 0: print "No fills specified!" return [] run_list = [] fill_map = {} # {run_number: fill_number} for fill in sorted(arg_list): print "Getting runs from fill %d..." % fill new_runs = self.parser.getFillRuns(fill) if len(new_runs) == 0: print "\tFill %d has no eligible runs!" continue for run in new_runs: fill_map[run] = fill run_list += new_runs self.rate_monitor.plotter.fill_map = fill_map return run_list # Use: Runs the rateMonitor object using parameters supplied as command line arguments def run(self): # type: () -> None if self.parseArgs(): self.rate_monitor.run() # Read text file of lists of runs def readDataListTextFile(self, datalist_file): path = datalist_file f = open(path, 'r') dict1 = {} i = 1 for line in f: key = 'data%i' % (i) data1 = [] for run in line.split(): data1.append(int(run)) dict1[key] = data1 i = i + 1 if self.rate_monitor.use_fills == True: for key in dict1: dict1[key] = self.getRuns(dict1[key]) f.close() return dict1
def __init__(self): # Set ROOT properties gROOT.SetBatch(True) # Use batch mode so plots are not spit out in X11 as we make them gStyle.SetPadRightMargin(0.2) # Set the canvas right margin so the legend can be bigger ROOT.gErrorIgnoreLevel = 7000 # Suppress info message from TCanvas.Print # Member variables self.runFile = "" # The name of the file that a list of runs is contained in self.runList = [] # A list of runs to process self.jsonFilter = False self.jsonFile = "" self.jsonData = {} self.fitFile = "" # The name of the file that the fit info is contained in self.logFile = "" #self.colorList = [602, 856, 410, 419, 801, 798, 881, 803, 626, 920, 922] #[2,3,4,6,7,8,9,28,38,30,40,46] # List of colors that we can use for graphing #self.colorList = [2,3,4,6,7,8,9,28,38,30,40,46] # List of colors that we can use for graphing self.colorList = [4,6,8,7,9,419,46,20,28,862,874,38,32,40,41,5,3] # List of colors that we can use for graphing self.processAll = False # If true, we process all the runs in the run list self.varX = "instLumi" # Plot the instantaneous luminosity on the x axis self.varY = "rawRate" # Plot the unprescaled rate on the y axis self.labelX = "instantaneous lumi" self.labelY = "unprescaled rate [Hz]" self.saveName = "" # A name that we save the root file as self.saveDirectory = "" # A directory that we can save all our files in if we are in batch mode self.updateOnlineFits = False #Update the online fits in the git directory self.ops = [] #options for log file self.parser = DBParser() # A database parser self.TriggerList = [] # The list of triggers to consider in plot-making # Trigger Options self.L1Triggers = False # If True, then we get the L1 trigger Data self.HLTTriggers = True # If True, then we get the HLT trigger Data self.correctForDT = True # Correct rates for deadtime self.savedAFile = False # True if we saved at least one file # Stream + PD Options self.plotStreams = False # If true, we plot the streams self.plotDatasets = False # If true, we plot the primary datasets # Error File Options self.makeErrFile = False # If true, we will write an error file self.errFileName = "" # The name of the error file self.errFile = None # A file to output errors to self.certifyMode = False # False -> Primary mode, True -> Secondary mode self.certifyDir = None self.runsToProcess = 12 # How many runs we are about to process self.sigmas = 3.0 # How many sigmas the error bars should be self.errorBands = True # display error self.sigmas bands on the rate vs inst lumi plot self.png = True # If true, create png images of all plots # Fitting self.allRates = {} # Retain a copy of rates to use for validating lumisections later on: [ runNumber ] [ triggerName ] [ LS ] { rawRate, ps } self.predictionRec = {} # A dictionary used to store predictions and prediction errors: [ triggerName ] { ( LS ), ( prediction ), (error) } self.minStatistics = 10 # The minimum number of points that we will allow for a run and still consider it self.fit = False # If true, we fit the data to fit functions self.InputFit = None # The fit from the fit file that we open self.OutputFit = None # The fit that we can make in primary mode self.outFitFile = "" # The name of the file that we will save an output fit to self.fitFinder = FitFinder() # A fit finder object self.pileUp = True self.bunches = 1 # The number of colliding bunches if divByBunches is true, 1 otherwise self.showEq = True # Whether we should show the fit equation on the plot self.dataCol = 0 # The column of the input data that we want to use as our y values # Batch mode variables self.plot_steam_rates = False self.steamRates = {} #misc. self.minNum = 0 self.maxNum = 0 # Cuts self.minPointsToFit = 10 # The minimum number of points we need to make a fit self.maxDeadTime = 10. # the maximum % acceptable deadtime, if deadtime is > maxDeadTime, we do not plot or fit that lumi # self.useFit: # If False, no fit will be plotted and all possible triggers will be used in graph making. # If True, only triggers in the fit file will be considered, and the fit will be plotted self.useFit = True # self.useTrigList # If False, modify self.triggerList as neccessary to include as many triggers as possible # If True, only use triggers in self.triggerList self.useTrigList = False
class RateMonitor: # Default constructor for RateMonitor class def __init__(self): # Set ROOT properties gROOT.SetBatch(True) # Use batch mode so plots are not spit out in X11 as we make them gStyle.SetPadRightMargin(0.2) # Set the canvas right margin so the legend can be bigger ROOT.gErrorIgnoreLevel = 7000 # Suppress info message from TCanvas.Print # Member variables self.runFile = "" # The name of the file that a list of runs is contained in self.runList = [] # A list of runs to process self.jsonFilter = False self.jsonFile = "" self.jsonData = {} self.fitFile = "" # The name of the file that the fit info is contained in self.logFile = "" #self.colorList = [602, 856, 410, 419, 801, 798, 881, 803, 626, 920, 922] #[2,3,4,6,7,8,9,28,38,30,40,46] # List of colors that we can use for graphing #self.colorList = [2,3,4,6,7,8,9,28,38,30,40,46] # List of colors that we can use for graphing self.colorList = [4,6,8,7,9,419,46,20,28,862,874,38,32,40,41,5,3] # List of colors that we can use for graphing self.processAll = False # If true, we process all the runs in the run list self.varX = "instLumi" # Plot the instantaneous luminosity on the x axis self.varY = "rawRate" # Plot the unprescaled rate on the y axis self.labelX = "instantaneous lumi" self.labelY = "unprescaled rate [Hz]" self.saveName = "" # A name that we save the root file as self.saveDirectory = "" # A directory that we can save all our files in if we are in batch mode self.updateOnlineFits = False #Update the online fits in the git directory self.ops = [] #options for log file self.parser = DBParser() # A database parser self.TriggerList = [] # The list of triggers to consider in plot-making # Trigger Options self.L1Triggers = False # If True, then we get the L1 trigger Data self.HLTTriggers = True # If True, then we get the HLT trigger Data self.correctForDT = True # Correct rates for deadtime self.savedAFile = False # True if we saved at least one file # Stream + PD Options self.plotStreams = False # If true, we plot the streams self.plotDatasets = False # If true, we plot the primary datasets # Error File Options self.makeErrFile = False # If true, we will write an error file self.errFileName = "" # The name of the error file self.errFile = None # A file to output errors to self.certifyMode = False # False -> Primary mode, True -> Secondary mode self.certifyDir = None self.runsToProcess = 12 # How many runs we are about to process self.sigmas = 3.0 # How many sigmas the error bars should be self.errorBands = True # display error self.sigmas bands on the rate vs inst lumi plot self.png = True # If true, create png images of all plots # Fitting self.allRates = {} # Retain a copy of rates to use for validating lumisections later on: [ runNumber ] [ triggerName ] [ LS ] { rawRate, ps } self.predictionRec = {} # A dictionary used to store predictions and prediction errors: [ triggerName ] { ( LS ), ( prediction ), (error) } self.minStatistics = 10 # The minimum number of points that we will allow for a run and still consider it self.fit = False # If true, we fit the data to fit functions self.InputFit = None # The fit from the fit file that we open self.OutputFit = None # The fit that we can make in primary mode self.outFitFile = "" # The name of the file that we will save an output fit to self.fitFinder = FitFinder() # A fit finder object self.pileUp = True self.bunches = 1 # The number of colliding bunches if divByBunches is true, 1 otherwise self.showEq = True # Whether we should show the fit equation on the plot self.dataCol = 0 # The column of the input data that we want to use as our y values # Batch mode variables self.plot_steam_rates = False self.steamRates = {} #misc. self.minNum = 0 self.maxNum = 0 # Cuts self.minPointsToFit = 10 # The minimum number of points we need to make a fit self.maxDeadTime = 10. # the maximum % acceptable deadtime, if deadtime is > maxDeadTime, we do not plot or fit that lumi # self.useFit: # If False, no fit will be plotted and all possible triggers will be used in graph making. # If True, only triggers in the fit file will be considered, and the fit will be plotted self.useFit = True # self.useTrigList # If False, modify self.triggerList as neccessary to include as many triggers as possible # If True, only use triggers in self.triggerList self.useTrigList = False # Use: sets up the variables before the main loop in run() # Returns: (void) def setUp(self): print "" # Formatting length = len(self.runList) try: self.runList.sort() except: print "Unable to sort runs" return self.bunches = 1 # Reset bunches, just in case if self.certifyMode: self.varX = "LS" # We are in secondary mode self.savedAFile = False # Reset self.savedAFile # Read JSON file if self.jsonFilter: with open(self.jsonFile) as jsonfile: self.jsonData = json.load(jsonfile) if not len(self.jsonData) > 0: print "JSON file is empty or not valid" self.jsonFilter = False else: print "JSON file list of runs: ", for k in self.jsonData.keys(): print "%s" % k, print "\n" # Apply run pre-filtering if self.jsonFilter: self.runList = [x for x in self.runList if "%d" % x in self.jsonData] self.minNum = self.runList[0] self.maxNum = self.runList[-1] # If we are supposed to, get the fit, a dictionary: [ triggername ] [ ( fit parameters ) ] if self.useFit or self.certifyMode: # Always try to load a fit in secondary mode self.InputFit = self.loadFit() if not self.useTrigList and not self.InputFit is None: self.TriggerList = sorted(self.InputFit) if not self.certifyMode and self.saveDirectory == "": self.saveDirectory = "fits__"+str(self.minNum) + "-" + str(self.maxNum) if not self.certifyMode: if os.path.exists(self.saveDirectory): shutil.rmtree(self.saveDirectory) print "Removing existing directory %s " % (self.saveDirectory) os.mkdir(self.saveDirectory) if self.png: os.chdir(self.saveDirectory) os.mkdir("png") os.chdir("../") print "Created directory %s " % (self.saveDirectory) self.saveName = self.saveDirectory + "/" + self.saveName # File names and name templates RootNameTemplate = "HLT_%s_vs_%s_%s_Run%s-%s_Tot%s_cert.root" if self.outFitFile=="": self.outFitFile = self.saveDirectory+"/HLT_Fit_Run%s-%s_Tot%s_fit.pkl" % (self.minNum, self.maxNum, self.runsToProcess) if self.useFit or self.fit or (self.certifyMode and not self.InputFit is None): fitOpt = "Fitted" else: fitOpt = "NoFit" self.saveName = self.saveDirectory+"/"+RootNameTemplate % (self.varX, self.varY, fitOpt, self.minNum, self.maxNum, self.runsToProcess) if self.certifyMode: self.saveName = self.certifyDir+"/"+self.saveName # Remove any root files that already have that name if os.path.exists(self.saveName): os.remove(self.saveName) # Open a file for writing errors if self.makeErrFile: self.errFileName = "rateGrapher_%s_%s.err" % (self.minNum, self.maxNum) # Define the error file name try: self.errFile = open(self.errFileName, 'w') except: print "Could not open error file." # If we are going to save fit find debug graph, delete any old ones if self.fitFinder.saveDebug and os.path.exists("Debug.root"): os.remove("Debug.root") if not self.certifyMode: ###dump a log file with the exact command used to produce the results self.logFile = self.saveDirectory+"/command_line.txt" command_line_str = "Results produced with:\n" command_line_str+="python plotTriggerRates.py " for tuple in self.ops: if tuple[0].find('--updateOnlineFits') > -1: continue #never record when we update online fits if len(tuple[1]) == 0: command_line_str+= "%s " % (tuple[0]) else: command_line_str+= "%s=%s " % (tuple[0],tuple[1]) for run in self.runList: command_line_str+= "%d " % (run) command_line_str+="\n" command_line_log_file = open(self.logFile, "w") command_line_log_file.write(command_line_str) command_line_log_file.close() def runBatch(self): if self.certifyMode: self.certifyDir = "Certification_%sruns_%s_%s" % (len(self.runList),str(datetime.datetime.now()).split()[0],str(datetime.datetime.now()).split()[1].split(':')[0] +"_"+ str(datetime.datetime.now()).split()[1].split(':')[1]) if os.path.exists(self.certifyDir): shutil.rmtree(self.certifyDir) os.mkdir(self.certifyDir) plottingData = {} # A dictionary [ trigger name ] [ run number ] { ( inst lumi's || LS ), ( data ) } self.setUp() # Set up parameters and data structures for run_number in self.runList: self.run(run_number, plottingData) if self.certifyMode: self.makeFits(plottingData) plottingData = {} print "-----" # Newline for formatting if self.certifyMode: self.doChecks() else: self.makeFits(plottingData) # Use: Created graphs based on the information stored in the class (list of runs, fit file, etc) # Returns: (void) def run(self,runNumber,plottingData): print "" # Print a newline (just for formatting) print "\nProcessing run: %d" % (runNumber) if self.certifyMode: #should probably find a better place for this... self.saveDirectory = "run%s" % (str(runNumber)) if os.path.exists(self.saveDirectory): shutil.rmtree(self.saveDirectory) print "Removing existing directory %s " % (self.saveDirectory) os.chdir(self.certifyDir) os.mkdir(self.saveDirectory) if self.png: os.chdir(self.saveDirectory) os.mkdir("png") os.chdir("../../") print "Created directory %s " % (self.saveDirectory) self.saveName = self.saveDirectory + "/" + self.saveName # File names and name templates RootNameTemplate = "HLT_%s_vs_%s_%s_Run%s_CERTIFICATION.root" if self.useFit or self.fit or (not self.InputFit is None): fitOpt = "Fitted" else: fitOpt = "NoFit" self.saveName = self.saveDirectory+"/"+RootNameTemplate % (self.varX, self.varY, fitOpt, runNumber) self.saveName = self.certifyDir+"/"+self.saveName # Remove any root files that already have that name if os.path.exists(self.saveName): os.remove(self.saveName) if self.pileUp: self.bunches = self.parser.getNumberCollidingBunches(runNumber)[0] if self.bunches is None or self.bunches is 0: print "Cannot get number of bunches: skipping this run.\n" return print "(%s colliding bunches)" % (self.bunches) # Get run info in a dictionary: [ trigger name ] { ( inst lumi's ), ( raw rates ) } dataList = self.getData(runNumber) if dataList == {}: # The run does not exist (or some other critical error occured) print "Fatal error for run %s, could not retrieve data. Probably Lumi was None or physics was not active. Moving on." % (runNumber) # Info message return # Make plots for each trigger if not self.plotStreams and not self.plotDatasets: for triggerName in self.TriggerList: if dataList.has_key(triggerName): # Add this run to plottingData[triggerName] # Make sure the is an entry for this trigger in plottingData if not plottingData.has_key(triggerName): plottingData[triggerName] = {} plottingData[triggerName][runNumber] = dataList[triggerName] elif self.makeErrFile: # The trigger data was not taken from the DB or does not exist # This should not occur if useFit is false, all triggers should be processed message = "For run %s Trigger %s could not be processed\n" % (runNumber, triggerName) self.errFile.write(message) elif not self.plotDatasets: # Otherwise, make plots for each stream sumPhysics = "Sum_Physics_Streams" for streamName in dataList: if not plottingData.has_key(streamName): plottingData[streamName] = {} plottingData[streamName][runNumber] = dataList[streamName] if not plottingData.has_key(sumPhysics): plottingData[sumPhysics] = {} if (streamName[0:7] =="Physics" or streamName[0:9] =="HIPhysics") and not plottingData[sumPhysics].has_key(runNumber): plottingData[sumPhysics][runNumber] = plottingData[streamName][runNumber] elif (streamName[0:7] =="Physics" or streamName[0:9] =="HIPhysics"): if plottingData[sumPhysics] != {}: ls_number =0 for rate in plottingData[streamName][runNumber][1]: plottingData[sumPhysics][runNumber][1][ls_number] += rate ls_number +=1 else: # Otherwise, make plots for each dataset for pdName in dataList: if not plottingData.has_key(pdName): plottingData[pdName] = {} plottingData[pdName][runNumber] = dataList[pdName] def makeFits(self, plottingData): if self.fit: self.findFit(plottingData) print "\n" # We have all our data, now plot it if self.useFit or (self.certifyMode and not self.InputFit is None): fitparams = self.InputFit elif self.fit: fitparams = self.OutputFit # Plot the fit that we made else: fitparams = None if self.plot_steam_rates: self.getSteamRates() for name in sorted(plottingData): if fitparams is None or not fitparams.has_key(name): fit = None else: fit = fitparams[name] self.graphAllData(plottingData[name], fit, name) # Try to close the error file if self.makeErrFile: try: self.errFile.close() # Close the error file print "Error file saved to", self.errFileName # Info message except: print "Could not save error file." if self.fitFinder.saveDebug and self.fitFinder.usePointSelection: print "Fit finder debug file saved to Debug.root.\n" # Info message if self.savedAFile: print "File saved as %s" % (self.saveName) # Info message else: print "No files were saved. Perhaps none of the triggers you requested were in use for this run." if self.png: self.printHtml(plottingData) if self.updateOnlineFits: #online files/dirs online_git_dir = "Fits/2016/" online_plots_dir = online_git_dir+"plots/" online_fit_file = online_git_dir+"FOG.pkl" online_cmd_line_file = online_git_dir+"command_line.txt" #files and dirs to be copied self.outFitFile self.logFile png_dir = self.saveDirectory+"/png/" shutil.rmtree(online_plots_dir) shutil.copytree(png_dir,online_plots_dir) shutil.copy(self.outFitFile,online_fit_file) #copy fit shutil.copy(self.logFile,online_cmd_line_file) #copy command line # Use: Gets the data we desire in primary mode (rawrate vs inst lumi) or secondary mode (rawrate vs LS) # Parameters: # -- runNumber: The number of the run we want data from # Returns: A dictionary: [ trigger name ] { ( inst lumi's || LS ), ( data ) } def getData(self, runNumber): Rates = {} if self.HLTTriggers: Rates = self.parser.getRawRates(runNumber) # Get the HLT raw rate vs LS if self.correctForDT: self.correctForDeadtime(Rates, runNumber) # Correct HLT Rates for deadtime if self.L1Triggers: L1Rates = self.parser.getL1RawRates(runNumber) # Get the L1 raw rate vs LS Rates.update(L1Rates) if Rates == {}: print "trouble fetching rates from db" return {} # The run (probably) doesn't exist # JSON filtering # Removes from the Rates dictionary the lumisections not included in the JSON file, if present. if self.jsonFilter: runNumberStr = "%d" % runNumber # Check for run number if not runNumberStr in self.jsonData: print "Run", runNumberStr, "is not included in the JSON file" return {} else: print "Run", runNumberStr, "and lumisections", self.jsonData[runNumberStr], "are included in the JSON file" # Remove lumisections for trigger, lumi in Rates.iteritems(): lumis = lumi.keys() for i, ls in enumerate(lumis): if not any(l <= ls <= u for [l, u] in self.jsonData[runNumberStr]): del Rates[trigger][ls] iLumi = self.parser.getLumiInfo(runNumber) # If we are in primary mode, we need luminosity info, otherwise, we just need the physics bit #iLumi = self.getBrilCalcLumi(runNumber) #for special studies using brilcalc lumis # Get the trigger list if useFit is false and we want to see all triggers (self.useTrigList is false) if not self.useFit and not self.useTrigList and not self.certifyMode: for triggerName in sorted(Rates): if not triggerName in self.TriggerList: self.TriggerList.append(triggerName) # Store Rates for this run self.allRates[runNumber] = Rates # Get stream data if self.plotStreams: # Stream Data [ stream name ] { LS, rate, size, bandwidth } streamData = self.parser.getStreamData(runNumber) Data = {} # Format the data correcly: [ stream name ] [ LS ] = { rate, size, bandwidth } for name in streamData: Data[name] = {} for LS, rate, size, bandwidth in streamData[name]: Data[name][LS] = [ rate, size, bandwidth ] # Get PD data elif self.plotDatasets: # pdData [ pd name ] { LS, rate } pdData = self.parser.getPrimaryDatasets(runNumber) Data = {} # Format the data correcly: [ pd name ] [ LS ] = {rate} for name in pdData: Data[name] = {} for LS, rate in pdData[name]: Data[name][LS] = [ rate ] else: Data = Rates # Depending on the mode, we return different pairs of data if not self.certifyMode: return self.combineInfo(Data, iLumi) # Combine the rates and lumi into one dictionary, [ trigger name ] { ( inst lumi's ), ( raw rates ) } and return else: # self.certifyMode == True return self.sortData(Data, iLumi) # Use: Modifies the rates in Rates, correcting them for deadtime # Parameters: # -- Rates: A dictionary [ triggerName ] [ LS ] { raw rate, prescale } # Returns: (void), directly modifies Rates def correctForDeadtime(self, Rates, runNumber): # Get the deadtime deadTime = self.parser.getDeadTime(runNumber) for LS in deadTime: for triggerName in Rates: if Rates[triggerName].has_key(LS): # Sometimes, LS's are missing Rates[triggerName][LS][0] *= (1. + deadTime[LS]/100.) if deadTime[LS] > self.maxDeadTime and not self.certifyMode: del Rates[triggerName][LS] #do not plot lumis where deadtime is greater than # Use: Combines the Rate data and instant luminosity data into a form that we can make a graph from # Parameters: # -- Data: A dictionary [ triggerName ] [ LS ] { col 0, col 1, ... } # -- iLumi: A list ( { LS, instLumi, cms active } ) # Returns: A dictionary: [ name ] { ( inst lumi's ), ( Data[...][col] ) } def combineInfo(self, Data, iLumi, col=0): # Create a dictionary [ trigger name ] { ( inst lumi's ), ( Data[...][col] ) } dataList = {} # For each trigger in Rates for name in Data: iLuminosity = array.array('f') yvals = array.array('f') detector_ready = array.array('f') for LS, ilum, psi, phys, cms_ready in iLumi: if Data[name].has_key(LS) and phys and not ilum is None: # Normalize ilumi here, if we aren't normalizing, self.bunches is set to 1 normedILumi = ilum/self.bunches # Extract the required data from Data data = Data[name][LS][self.dataCol] # We apply our cuts here if they are called for detector_ready.append(cms_ready) if self.pileUp: PU = (ilum/self.bunches*ppInelXsec/orbitsPerSec) iLuminosity.append(PU) yvals.append(data/self.bunches) else: iLuminosity.append(ilum) # Add the instantaneous luminosity for this LS yvals.append(data) # Add the correspoinding raw rate iLuminosity, yvals = self.fitFinder.getGoodPoints(iLuminosity, yvals) #filter out points that differ greatly from the averages on a trigger,run basis if len(iLuminosity) > 0: dataList[name] = [iLuminosity, yvals, detector_ready] else: pass return dataList # Use: Combines the Data from an array of the shown format and the instant luminosity data into a form that we can make a graph from # Parameters: # -- Data: A dictionary [ name ] [ LS ] { col 0, col 1, ... } # Returns: A dictionary: [ name ] { ( LS ), ( Data[LS][col] ) } def sortData(self, Data, iLumi): # Create a dictionary [ trigger name ] { ( LS ), ( Data[LS][col] ) } dataList = {} for name in Data: lumisecs = array.array('f') yvals = array.array('f') detector_ready = array.array('f') for LS, ilum, psi, phys, cms_ready in iLumi: if phys and not ilum is None and Data[name].has_key(LS): lumisecs.append(LS) detector_ready.append(cms_ready) yvals.append(Data[name][LS][self.dataCol]) dataList[name] = [lumisecs, yvals, detector_ready] return dataList # Use: Graphs the data from all runs and triggers onto graphs and saves them to the root file # Parameters: # -- plottingData: A dictionary [ run number ] { ( inst lumi's ), ( raw rates ) } # -- paramlist: A tuple: { FitType, X0, X1, X2, X3, sigma, meanrawrate, X0err, X1err, X2err, X3err } # -- triggerName: The name of the trigger that we are examining # Returns: (void) def graphAllData(self, plottingData, paramlist, triggerName): # Find max and min values maximumRR = array.array('f') maximumVals = array.array('f') minimumVals = array.array('f') yVals = array.array('f') xVals = array.array('f') # Find minima and maxima so we create graphs of the right size for runNumber in plottingData: xVals, yVals = self.fitFinder.getGoodPoints(plottingData[runNumber][0], plottingData[runNumber][1]) if len(xVals) > 0: maximumRR.append(max(yVals)) maximumVals.append(max(xVals)) minimumVals.append(min(xVals)) if len(maximumRR) > 0: max_yaxis_value = max(maximumRR) else: return if len(maximumVals) > 0: max_xaxis_val = max(maximumVals) min_xaxis_val = min(minimumVals) else: return if self.plot_steam_rates: if self.steamRates.has_key(triggerName.split('_v')[0]): steam_xVal = array.array('f'); steam_xVal.append(15.) steam_xVal_err = array.array('f'); steam_xVal_err.append(0.) steam_yVal = array.array('f'); steam_yVal.append(self.steamRates[triggerName.split('_v')[0]][0]) steam_yVal_err = array.array('f'); steam_yVal_err.append(self.steamRates[triggerName.split('_v')[0]][1]) steamGraph = TGraphErrors(1, steam_xVal, steam_yVal, steam_xVal_err, steam_yVal_err) steamGraph.SetMarkerStyle(29) steamGraph.SetMarkerSize(2.6) steamGraph.SetMarkerColor(1) if max(steam_yVal) > max_yaxis_value: max_yaxis_value = max(steam_yVal) else: return if max_xaxis_val==0 or max_yaxis_value==0: return # Set axis names/units, create canvas if self.pileUp: nameX = "< PU >" xunits = "" self.labelY = "unprescaled rate / num colliding bx [Hz]" else: xunits = "[10^{30} Hz/cm^{2}]" nameX = "instantaneous luminosity" if self.certifyMode: xunits = "" nameX = "lumisection" self.labelY = "unprescaled rate [Hz]" canvas = TCanvas((self.varX+" "+xunits), self.varY, 1000, 600) canvas.SetName(triggerName+"_"+self.varX+"_vs_"+self.varY) plotFuncStr = "" funcStr = "" if (self.useFit or self.fit) and not paramlist is None: if self.certifyMode: # Make a prediction graph of raw rate vs LS for values between min_xaxis_val and max_xaxis_val runNum_cert = plottingData.keys()[0] predictionTGraph = self.makePredictionTGraph(paramlist, min_xaxis_val, max_xaxis_val, triggerName, runNum_cert) maxPred = self.predictionRec[triggerName][runNum_cert][0][1] if maxPred > max_yaxis_value: max_yaxis_value = maxPred else: #primary mode if paramlist[0]=="exp": plotFuncStr = "%.5f + %.5f*exp( %.5f+%.5f*x )" % (paramlist[1], paramlist[2], paramlist[3], paramlist[4]) # Exponential funcStr = "%.5f + %.5f*exp( %.5f+%.5f*x )" % (paramlist[1], paramlist[2], paramlist[3], paramlist[4]) elif paramlist[0]=="linear": plotFuncStr = "%.15f + x*%.15f" % (paramlist[1], paramlist[2]) funcStr = "%.5f + x*%.5f" % (paramlist[1], paramlist[2]) else: plotFuncStr = "%.15f+x*(%.15f+ x*(%.15f+x*%.15f))" % (paramlist[1], paramlist[2], paramlist[3], paramlist[4])#Polynomial funcStr = "%.5f+x*(%.5f+ x*(%.5f+x*%.5f))" % (paramlist[1], paramlist[2], paramlist[3], paramlist[4]) #max_xaxis_val = 40 #Extend x-axis to 40 fitFunc = TF1("Fit_"+triggerName, plotFuncStr, 0., 1.1*max_xaxis_val) #if fitFunc.Eval(max_xaxis_val) > max_yaxis_value: max_yaxis_value = fitFunc.Eval(max_xaxis_val) #extend y-axis to maximum fit value if self.errorBands: xVal = array.array('f') yVal = array.array('f') yeVal = array.array('f') xeVal = array.array('f') xMin = fitFunc.GetXmin() xMax = fitFunc.GetXmax() xrange = xMax-xMin nPoints = 1000 stepSize = xrange/nPoints xCoord = xMin while xCoord <= xMax: xVal.append(xCoord) yVal.append(fitFunc.Eval(xCoord)) yeVal.append(self.sigmas*paramlist[5]) xeVal.append(0) xCoord += stepSize fitErrorBand = TGraphErrors(len(xVal),xVal,yVal,xeVal,yeVal) fitErrorBand.SetFillColor(2) fitErrorBand.SetFillStyle(3003) counter = 0 # This is the only way I have found to get an arbitrary number of graphs to be plotted on the same canvas. This took a while to get to work. graphList = [] # Create legend left = 0.81; right = 0.98; top = 0.9; scaleFactor = 0.05; minimum = 0.1 bottom = max( [top-scaleFactor*(len(plottingData)+1), minimum]) # Height we desire for the legend, adjust for number of entries legend = TLegend(left,top,right,bottom) for runNumber in sorted(plottingData): numLS = len(plottingData[runNumber][0]) bunchesForLegend = self.parser.getNumberCollidingBunches(runNumber)[0] if numLS == 0: continue graphList.append(TGraph(numLS, plottingData[runNumber][0], plottingData[runNumber][1])) # Set some stylistic settings for dataGraph graphColor = self.colorList[counter % len(self.colorList)]# + (counter // len(self.colorList)) # If we have more runs then colors, we just reuse colors (instead of crashing the program) graphList[-1].SetMarkerStyle(7) graphList[-1].SetMarkerSize(1.0) graphList[-1].SetLineColor(graphColor) graphList[-1].SetFillColor(graphColor) graphList[-1].SetMarkerColor(graphColor) graphList[-1].SetLineWidth(2) graphList[-1].GetXaxis().SetTitle(nameX+" "+xunits) graphList[-1].GetXaxis().SetLimits(0, 1.1*max_xaxis_val) graphList[-1].GetYaxis().SetTitle(self.labelY) graphList[-1].GetYaxis().SetTitleOffset(1.2) graphList[-1].SetMinimum(0) graphList[-1].SetMaximum(1.2*max_yaxis_value) graphList[-1].SetTitle(triggerName) if counter == 0: graphList[-1].Draw("AP") else: graphList[-1].Draw("P") canvas.Update() if bunchesForLegend > 0: legend.AddEntry(graphList[-1], "%s (%s b)" %(runNumber,bunchesForLegend), "f") else: legend.AddEntry(graphList[-1], "%s (- b)" %(runNumber), "f") counter += 1 if (self.useFit or self.fit) and not paramlist is None: if self.certifyMode: predictionTGraph.Draw("PZ3") canvas.Update() legend.AddEntry(predictionTGraph, "Fit ( %s \sigma )" % (self.sigmas)) else: # Primary Mode if self.errorBands: fitErrorBand.Draw("3") legend.AddEntry(fitFunc, "Fit ( %s \sigma )" % (self.sigmas)) fitFunc.Draw("same") # Draw the fit function on the same graph if self.plot_steam_rates: if self.steamRates.has_key(triggerName.split('_v')[0]): steamGraph.Draw("P") legend.AddEntry(steamGraph,"STEAM estimate","p") if not funcStr == "" and self.showEq: # Draw function string on the plot funcLeg = TLegend(.146, .71, .47, .769) funcLeg.SetHeader("f(x) = " + funcStr) funcLeg.SetFillColor(0) funcLeg.Draw() canvas.Update() # draw text latex = TLatex() latex.SetNDC() latex.SetTextColor(1) latex.SetTextAlign(11) latex.SetTextFont(62) latex.SetTextSize(0.05) latex.DrawLatex(0.15, 0.84, "CMS") latex.SetTextSize(0.035) latex.SetTextFont(52) latex.DrawLatex(0.15, 0.80, "Rate Monitoring") canvas.SetGridx(1); canvas.SetGridy(1); canvas.Update() # Draw Legend legend.SetHeader("%s runs:" % (len(plottingData))) legend.SetFillColor(0) legend.Draw() canvas.Update() # Update root file file = TFile(self.saveName, "UPDATE") canvas.Modified() if self.png: if not self.certifyMode: canvas.Print(self.saveDirectory+"/png/"+triggerName+".png", "png") else: canvas.Print(self.certifyDir+"/"+self.saveDirectory+"/png/"+triggerName+".png", "png") canvas.Write() file.Close() self.savedAFile = True # Use: Get a fit for all the data that we have collected # Parameters: # -- plottingData: A dictionary [triggerName] [ run number ] { ( inst lumi's ), ( raw rates ) } # Returns: (void) def findFit(self, plottingData): self.OutputFit = {} for name in sorted(plottingData): instLumis = array.array('f') yvals = array.array('f') detector_ready = array.array('f') for runNumber in sorted(plottingData[name]): #if runNumber == 273447 and ( name.find('DoubleMu') > -1 or name.find('QuadMu') > -1 or name.find('TripleMu') > -1 ): print "Skimming some Runs and Paths"; continue detector_ready = plottingData[name][runNumber][2] for x,y,cms_ready in zip(plottingData[name][runNumber][0],plottingData[name][runNumber][1],detector_ready): if cms_ready: #only fit data points when ALL subsystems are IN. instLumis.append(x) yvals.append(y) if len(instLumis) > self.minPointsToFit: try: self.OutputFit[name] = self.fitFinder.findFit(instLumis, yvals, name) except: continue self.saveFit() # Use: Save a fit to a file def saveFit(self): outputFile = open(self.outFitFile, "wb") pickle.dump(self.OutputFit, outputFile, 2) outputFile.close() self.sortFit() print "\nFit file saved to", self.outFitFile # Use: Sorts trigger fits by their chi squared value and writes it to a file def sortFit(self): outputFile = open(self.saveDirectory+"/sortedChiSqr.txt", "wb") chisqrDict = {} for trigger in self.OutputFit: _,_,_,_,_,_,_,_,_,_,_,chisqr = self.OutputFit[trigger] chisqrDict[chisqr] = trigger for chisqr in sorted(chisqrDict): outputFile.write(chisqrDict[chisqr] + ": " + str(chisqr) + "\n") outputFile.close print "Sorted chi-square saved to:"+self.saveDirectory+"/sortedChiSqr.txt" def printHtml(self,plottingData): try: if not self.certifyMode: htmlFile = open(self.saveDirectory+"/index.html", "wb") else: htmlFile = open(self.certifyDir+"/"+self.saveDirectory+"/index.html", "wb") htmlFile.write("<!DOCTYPE html>\n") htmlFile.write("<html>\n") htmlFile.write("<style>.image { float:right; margin: 5px; clear:justify; font-size: 6px; font-family: Verdana, Arial, sans-serif; text-align: center;}</style>\n") for pathName in sorted(plottingData): fileName = "%s/png/%s.png" % (self.saveDirectory,pathName) if self.certifyMode: fileName = "%s/%s/png/%s.png" % (self.certifyDir,self.saveDirectory,pathName) if os.access(fileName,os.F_OK): htmlFile.write("<div class=image><a href=\'png/%s.png\'><img width=398 height=229 border=0 src=\'png/%s.png\'></a><div style=\'width:398px\'>%s</div></div>\n" % (pathName,pathName,pathName)) htmlFile.write("</html>\n") htmlFile.close except: print "Unable to write index.html file" # Use: Creates a graph of predicted raw rate vs lumisection data # Parameters: # -- paramlist: A tuple of parameters { FitType, X0, X1, X2, X3, sigma, meanrawrate, X0err, X1err, X2err, X3err, ChiSqr } # -- min_xaxis_val: The minimum LS # -- max_xaxis_val: The maximum LS # -- iLumi: A list: ( { LS, instLumi } ) # -- triggerName: The name of the trigger we are making a fit for # Returns: A TGraph of predicted values def makePredictionTGraph(self, paramlist, min_xaxis_val, max_xaxis_val, triggerName, runNumber): # Initialize our point arrays lumisecs = array.array('f') predictions = array.array('f') lsError = array.array('f') predError = array.array('f') # Unpack values type, X0, X1, X2, X3, sigma, meanraw, X0err, X1err, X2err, X3err, ChiSqr = paramlist # Create our point arrays iLumi = self.parser.getLumiInfo(runNumber) # iLumi is a list: ( { LS, instLumi } ) for LS, ilum, psi, phys, cms_ready in iLumi: if not ilum is None and phys: lumisecs.append(LS) pu = (ilum * ppInelXsec) / ( self.bunches * orbitsPerSec ) # Either we have an exponential fit, or a polynomial fit if type == "exp": rr = self.bunches * (X0 + X1*math.exp(X2+X3*pu)) else: rr = self.bunches * (X0 + pu*X1 + (pu**2)*X2 + (pu**3)*X3) if rr<0: rr=0 # Make sure prediction is non negative predictions.append(rr) lsError.append(0) predError.append(self.bunches*self.sigmas*sigma) # Record for the purpose of doing checks self.predictionRec.setdefault(triggerName,{})[runNumber] = zip(lumisecs, predictions, predError) # Set some graph options fitGraph = TGraphErrors(len(lumisecs), lumisecs, predictions, lsError, predError) fitGraph.SetTitle("Fit (%s sigma)" % (self.sigmas)) fitGraph.SetMarkerStyle(8) fitGraph.SetMarkerSize(0.8) fitGraph.SetMarkerColor(2) # Red fitGraph.SetFillColor(4) fitGraph.SetFillStyle(3003) fitGraph.GetXaxis().SetLimits(min_xaxis_val, 1.1*max_xaxis_val) return fitGraph # Use: Loads the fit data from the fit file # Parameters: # -- fitFile: The file that the fit data is stored in (a pickle file) # Returns: The input fit data def loadFit(self): if self.fitFile == "": print "No fit file specified." return None InputFit = {} # Initialize InputFit (as an empty dictionary) # Try to open the file containing the fit info try: pkl_file = open(self.fitFile, 'rb') InputFit = pickle.load(pkl_file) pkl_file.close() except: # File failed to open print "Error: could not open fit file: %s" % (self.fitFile) return InputFit def getSteamRates(self): import csv file = "steamRate_forRateMon.csv" with open(file) as csvfile: steamReader=csv.reader(csvfile) for line in steamReader: path = line[0].split("_v")[0] if path.find("HLT_")!=-1: try: rate = float(line[6]) rateErr = float(line[7]) except: rate -1 rateErr = -1 if rate >0.: self.steamRates[path] = [rate, rateErr] def getBrilCalcLumi(self, run_number): import csv file = "lumi.csv" lumi_array = [] with open(file) as csvfile: fileReader = csv.reader(csvfile) for line in fileReader: run_number_file = int(line[0].split(":")[0]) if run_number_file != run_number: continue lumi_section = int(line[1].split(":")[0]) delivered = float(line[5])/23.3 recorded = float(line[6])/23.3 lumi_array.append((lumi_section,recorded,0,1,1)) return lumi_array # Use: Check raw rates in lumisections against the prediction, take note if any are outside a certain sigma range # Returns: (void) def doChecks(self): eprint = ErrorPrinter() eprint.saveDirectory = self.certifyDir # Look at all lumisections for each trigger for each run. Check which ones are behaving badly for triggerName in self.TriggerList: # We may want to look at the triggers from somewhere else, but for now I assume this will work for runNumber in self.allRates: if self.predictionRec.has_key(triggerName): if self.allRates[runNumber].has_key(triggerName): # In case some run did not contain this trigger data = self.allRates[runNumber][triggerName] if self.predictionRec[triggerName].has_key(runNumber): predList = self.predictionRec[triggerName][runNumber] else: continue for LS, pred, err in predList: if data.has_key(LS): # In case this LS is missing if not eprint.run_allLs.has_key(runNumber): eprint.run_allLs[runNumber] = {} if not eprint.run_allLs[runNumber].has_key(triggerName): eprint.run_allLs[runNumber][triggerName] = [] eprint.run_allLs[runNumber][triggerName].append(LS) if(abs(data[LS][0] - pred) > err): if err != 0: errStr = str((data[LS][0]-pred)/err) else: errStr = "inf" # Add data to eprint.run_ls_trig if not eprint.run_ls_trig.has_key(runNumber): eprint.run_ls_trig[runNumber] = {} if not eprint.run_ls_trig[runNumber].has_key(int(LS)): eprint.run_ls_trig[runNumber][int(LS)] = [] eprint.run_ls_trig[runNumber][LS].append(triggerName) # Add data to eprint.run_trig_ls if not eprint.run_trig_ls.has_key(runNumber): eprint.run_trig_ls[runNumber] = {} if not eprint.run_trig_ls[runNumber].has_key(triggerName): eprint.run_trig_ls[runNumber][triggerName] = [] eprint.run_trig_ls[runNumber][triggerName].append(int(LS)) eprint.outputErrors()
def __init__(self): # Suppress root warnings ROOT.gErrorIgnoreLevel = 7000 # Fits and fit files self.fitFile = "Fits/2016/FOG.pkl" # The fit file, can contain both HLT and L1 triggers #self.fitFile = "../HLT_Fit_Run275911-276244_Tot12_fit.pkl" # self.fitFile = ""#fits__273013-273017/HLT_Fit_Run273013-273017_Tot12_fit.pkl" # The fit file, can contain both HLT and L1 triggers self.InputFitHLT = None # The fit information for the HLT triggers self.InputFitL1 = None # The fit information for the L1 triggers # DBParser self.parser = DBParser() # A database parser # Rates self.HLTRates = None # HLT rates self.L1Rates = None # L1 rates self.Rates = None # Combined L1 and HLT rates self.deadTimeData = {} # initializing deadTime dict # Run control self.lastRunNumber = -2 # The run number during the last segment self.runNumber = -1 # The number of the current run self.numBunches = [-1, -1] # Number of [target, colliding] bunches # Running over a previouly done run self.assignedNum = False # If true, we look at the assigned run, not the latest run self.LSRange = [] # If we want to only look at a range of LS from the run ##self.simulate = False # Simulate running through and monitoring a previous run # Lumisection control self.lastLS = 1 # The last LS that was processed last segment self.currentLS = 1 # The latest LS written to the DB self.slidingLS = -1 # The number of LS to average over, use -1 for no sliding LS self.useLSRange = False # Only look at LS in a certain range # Mode self.triggerMode = None # The trigger mode self.mode = None # Mode: cosmics, circulate, physics # Columns header self.displayRawRates = False # display raw rates, to display prescaled rates, set = True self.pileUp = True # derive expected rate as a function of the pileUp, and not the luminosity # Triggers self.cosmics_triggerList = "monitorlist_COSMICS.list" #default list used when in cosmics mode self.collisions_triggerList = "monitorlist_COLLISIONS.list" #default list used when in collision mode self.triggerList = "" # A list of all the L1 and HLT triggers we want to monitor self.userSpecTrigList = False # User specified trigger list self.usableHLTTriggers = [] # HLT Triggers active during the run that we have fits for (and are in the HLT trigger list if it exists) self.otherHLTTriggers = [] # HLT Triggers active during the run that are not usable triggers self.usableL1Triggers = [] # L1 Triggers active during the run that have fits for (and are in the L1 trigger list if it exists) self.otherL1Triggers = [] # L1 Triggers active during that run that are not usable triggers self.redoTList = True # Whether we need to update the trigger lists self.useAll = False # If true, we will print out the rates for all the HLT triggers self.useL1 = False # If true, we will print out the rates for all the L1 triggers self.totalHLTTriggers = 0 # The total number of HLT Triggers on the menu this run self.totalL1Triggers = 0 # The total number of L1 Triggers on the menu this run self.fullL1HLTMenu = [] self.ignoreStrings = ["Calibration","L1Tech","BPTX","Bptx"] # Restrictions self.removeZeros = False # If true, we don't show triggers that have zero rate # Trigger behavior self.percAccept = 50.0 # The acceptence for % diff self.devAccept = 5 # The acceptance for deviation self.badRates = {} # A dictionary: [ trigger name ] { num consecutive bad , whether the trigger was bad last time we checked, rate, expected, dev } self.recordAllBadTriggers = {} # A dictionary: [ trigger name ] < total times the trigger was bad > self.maxCBR = 3 # The maximum consecutive db queries a trigger is allowed to deviate from prediction by specified amount before it's printed out self.displayBadRates = -1 # The number of bad rates we should show in the summary. We use -1 for all self.usePerDiff = False # Whether we should identify bad triggers by perc diff or deviatoin self.sortRates = True # Whether we should sort triggers by their rates self.maxHLTRate = 500 # The maximum prescaled rate we allow an HLT Trigger to have self.maxL1Rate = 30000 # The maximum prescaled rate we allow an L1 Trigger to have # Other options self.quiet = False # Prints fewer messages in this mode self.noColors = False # Special formatting for if we want to dump the table to a file self.sendMailAlerts_static = True # Whether we should send alert mails self.sendMailAlerts_dynamic = self.sendMailAlerts_static self.sendAudioAlerts = False # Whether we should send audio warning messages in the control room (CAUTION) self.isUpdating = True # flag to determine whether or not we're receiving new LS self.showStreams = False # Whether we should print stream information self.showPDs = False # Whether we should print pd information self.totalStreams = 0 # The total number of streams self.maxStreamRate = 1000000 # The maximum rate we allow a "good" stream to have self.maxPDRate = 250 # The maximum rate we allow a "good" pd to have self.lumi_ave = "NONE" self.pu_ave = "NONE" self.deadTimeCorrection = True # correct the rates for dead time self.scale_sleeptime = 2.0 # Scales the length of time to wait before sending another query (1.0 = 60sec, 2.0 = 120sec, etc)
class ShiftMonitor: def __init__(self): # Suppress root warnings ROOT.gErrorIgnoreLevel = 7000 # Fits and fit files self.fitFile = "Fits/2016/FOG.pkl" # The fit file, can contain both HLT and L1 triggers #self.fitFile = "../HLT_Fit_Run275911-276244_Tot12_fit.pkl" # self.fitFile = ""#fits__273013-273017/HLT_Fit_Run273013-273017_Tot12_fit.pkl" # The fit file, can contain both HLT and L1 triggers self.InputFitHLT = None # The fit information for the HLT triggers self.InputFitL1 = None # The fit information for the L1 triggers # DBParser self.parser = DBParser() # A database parser # Rates self.HLTRates = None # HLT rates self.L1Rates = None # L1 rates self.Rates = None # Combined L1 and HLT rates self.deadTimeData = {} # initializing deadTime dict # Run control self.lastRunNumber = -2 # The run number during the last segment self.runNumber = -1 # The number of the current run self.numBunches = [-1, -1] # Number of [target, colliding] bunches # Running over a previouly done run self.assignedNum = False # If true, we look at the assigned run, not the latest run self.LSRange = [] # If we want to only look at a range of LS from the run ##self.simulate = False # Simulate running through and monitoring a previous run # Lumisection control self.lastLS = 1 # The last LS that was processed last segment self.currentLS = 1 # The latest LS written to the DB self.slidingLS = -1 # The number of LS to average over, use -1 for no sliding LS self.useLSRange = False # Only look at LS in a certain range # Mode self.triggerMode = None # The trigger mode self.mode = None # Mode: cosmics, circulate, physics # Columns header self.displayRawRates = False # display raw rates, to display prescaled rates, set = True self.pileUp = True # derive expected rate as a function of the pileUp, and not the luminosity # Triggers self.cosmics_triggerList = "monitorlist_COSMICS.list" #default list used when in cosmics mode self.collisions_triggerList = "monitorlist_COLLISIONS.list" #default list used when in collision mode self.triggerList = "" # A list of all the L1 and HLT triggers we want to monitor self.userSpecTrigList = False # User specified trigger list self.usableHLTTriggers = [] # HLT Triggers active during the run that we have fits for (and are in the HLT trigger list if it exists) self.otherHLTTriggers = [] # HLT Triggers active during the run that are not usable triggers self.usableL1Triggers = [] # L1 Triggers active during the run that have fits for (and are in the L1 trigger list if it exists) self.otherL1Triggers = [] # L1 Triggers active during that run that are not usable triggers self.redoTList = True # Whether we need to update the trigger lists self.useAll = False # If true, we will print out the rates for all the HLT triggers self.useL1 = False # If true, we will print out the rates for all the L1 triggers self.totalHLTTriggers = 0 # The total number of HLT Triggers on the menu this run self.totalL1Triggers = 0 # The total number of L1 Triggers on the menu this run self.fullL1HLTMenu = [] self.ignoreStrings = ["Calibration","L1Tech","BPTX","Bptx"] # Restrictions self.removeZeros = False # If true, we don't show triggers that have zero rate # Trigger behavior self.percAccept = 50.0 # The acceptence for % diff self.devAccept = 5 # The acceptance for deviation self.badRates = {} # A dictionary: [ trigger name ] { num consecutive bad , whether the trigger was bad last time we checked, rate, expected, dev } self.recordAllBadTriggers = {} # A dictionary: [ trigger name ] < total times the trigger was bad > self.maxCBR = 3 # The maximum consecutive db queries a trigger is allowed to deviate from prediction by specified amount before it's printed out self.displayBadRates = -1 # The number of bad rates we should show in the summary. We use -1 for all self.usePerDiff = False # Whether we should identify bad triggers by perc diff or deviatoin self.sortRates = True # Whether we should sort triggers by their rates self.maxHLTRate = 500 # The maximum prescaled rate we allow an HLT Trigger to have self.maxL1Rate = 30000 # The maximum prescaled rate we allow an L1 Trigger to have # Other options self.quiet = False # Prints fewer messages in this mode self.noColors = False # Special formatting for if we want to dump the table to a file self.sendMailAlerts_static = True # Whether we should send alert mails self.sendMailAlerts_dynamic = self.sendMailAlerts_static self.sendAudioAlerts = False # Whether we should send audio warning messages in the control room (CAUTION) self.isUpdating = True # flag to determine whether or not we're receiving new LS self.showStreams = False # Whether we should print stream information self.showPDs = False # Whether we should print pd information self.totalStreams = 0 # The total number of streams self.maxStreamRate = 1000000 # The maximum rate we allow a "good" stream to have self.maxPDRate = 250 # The maximum rate we allow a "good" pd to have self.lumi_ave = "NONE" self.pu_ave = "NONE" self.deadTimeCorrection = True # correct the rates for dead time self.scale_sleeptime = 2.0 # Scales the length of time to wait before sending another query (1.0 = 60sec, 2.0 = 120sec, etc) # Use: Opens a file containing a list of trigger names and adds them to the RateMonitor class's trigger list # Note: We do not clear the trigger list, this way we could add triggers from multiple files to the trigger list # -- fileName: The name of the file that trigger names are contained in # Returns: (void) def loadTriggersFromFile(self, fileName): try: file = open(fileName, 'r') except: print "File", fileName, "(a trigger list file) failed to open." return allTriggerNames = file.read().split() # Get all the words, no argument -> split on any whitespace TriggerList = [] for triggerName in allTriggerNames: # Recognize comments if triggerName[0]=='#': continue try: if not str(triggerName) in TriggerList: TriggerList.append(stripVersion(str(triggerName))) except: print "Error parsing trigger name in file", fileName return TriggerList # Use: Formats the header string # Returns: (void) def getHeader(self): # Define spacing and header maxNameHLT = 0 maxNameL1 = 0 if len(self.usableHLTTriggers)>0 or len(self.otherHLTTriggers)>0: maxNameHLT = max([len(trigger) for trigger in self.usableHLTTriggers+self.otherHLTTriggers]) if len(self.usableL1Triggers)>0 or len(self.otherL1Triggers)>0: maxNameL1 = max([len(trigger) for trigger in self.usableL1Triggers+self.otherL1Triggers]) # Make the name spacing at least 90 maxName = max([maxNameHLT, maxNameL1, 90]) self.spacing = [maxName + 5, 14, 14, 14, 14, 14, 0] self.spacing[5] = max( [ 181 - sum(self.spacing), 0 ] ) self.header = "" # The table header self.header += stringSegment("* TRIGGER NAME", self.spacing[0]) if (self.displayRawRates): self.header += stringSegment("* RAW [Hz]", self.spacing[1]) else: self.header += stringSegment("* ACTUAL [Hz]", self.spacing[1]) self.header += stringSegment("* EXPECTED", self.spacing[2]) self.header += stringSegment("* % DIFF", self.spacing[3]) self.header += stringSegment("* DEVIATION", self.spacing[4]) self.header += stringSegment("* AVE PS", self.spacing[5]) self.header += stringSegment("* COMMENTS", self.spacing[6]) self.hlength = 181 #sum(self.spacing) # Use: Runs the program # Returns: (void) def run(self): # Load the fit and trigger list if self.fitFile!="": inputFit = self.loadFit(self.fitFile) for triggerName in inputFit: if triggerName[0:3]=="L1_": if self.InputFitL1 is None: self.InputFitL1 = {} self.InputFitL1[stripVersion(triggerName)] = inputFit[triggerName] elif triggerName[0:4] =="HLT_": if self.InputFitHLT is None: self.InputFitHLT = {} self.InputFitHLT[stripVersion(triggerName)] = inputFit[triggerName] # Sort trigger list into HLT and L1 trigger lists if self.triggerList!="": for triggerName in self.triggerList: if triggerName[0:3]=="L1_": self.TriggerListL1.append(triggerName) else: self.TriggerListHLT.append(triggerName) # If there aren't preset trigger lists, use all the triggers that we can fit else: if not self.InputFitHLT is None: self.TriggerListHLT = self.InputFitHLT.keys() if not self.InputFitL1 is None: self.TriggerListL1 = self.InputFitL1.keys() # Get run info: The current run number, if the detector is collecting (?), if the data is good (?), and the trigger mode if not self.assignedNum: self.runNumber, _, _, mode = self.parser.getLatestRunInfo() self.triggerMode = mode[0] # Info message print "The current run number is %s." % (self.runNumber) # If we are observing a single run from the past ##elif not self.simulate: else: try: self.triggerMode = self.parser.getTriggerMode(self.runNumber)[0] except: self.triggerMode = "Other" self.getRates() self.runLoop() self.checkTriggers() return # If we are simulating a previous run #if self.simulate: # self.simulateRun() # return # Run as long as we can self.setMode() self.redoTList = True while True: try: # Check if we are still in the same run, get trigger mode self.lastRunNumber = self.runNumber self.runNumber, _, _, mode = self.parser.getLatestRunInfo() self.runLoop() self.checkTriggers() self.sleepWait() except KeyboardInterrupt: print "Quitting. Bye." break ##def simulateRun(self): ## modTime = 0 ## # Get the rates ## self.triggerMode = self.parser.getTriggerMode(self.runNumber)[0] ## # Set the trigger mode ## self.setMode() ## # Get the rates for the entire run ## self.getRates() ## # Find the max LS for that run ## trig = self.Rates.keys()[0] ## self.lastLS = self.currentLS ## maxLS = max(self.Rates[trig].keys()) ## # Simulate the run ## self.lastRunNumber = self.runNumber ## self.lastLS = 1 ## while self.currentLS < maxLS: ## modTime += 23.3 ## self.currentLS += 1 ## if modTime > 60 or self.currentLS == maxLS: ## modTime -= 60 ## # Print table ## self.runLoop() ## # Check for bad triggers ## self.checkTriggers() ## # We would sleep here if this was an online run ## if not self.quiet: print "Simulating 60 s of sleep..." ## self.lastLS = self.currentLS ## print "End of simulation" # Use: The main body of the main loop, checks the mode, creates trigger lists, prints table # Returns: (void) def runLoop(self): # Reset counting variable self.normal = 0 self.bad = 0 # If we have started a new run if self.lastRunNumber != self.runNumber: print "Starting a new run: Run %s" % (self.runNumber) self.lastLS = 1 self.currentLS = 0 # Check what mode we are in self.setMode() self.getRates() self.redoTriggerLists() # Get Rates: [triggerName][LS] { raw rate, prescale } ##if not self.simulate: self.getRates() self.getRates() #Construct (or reconstruct) trigger lists if self.redoTList: self.redoTriggerLists() # Make sure there is info to use if len(self.HLTRates) == 0 or len(self.L1Rates) == 0: self.redoTList = True if len(self.HLTRates) == 0 and len(self.L1Rates) == 0: print "No new information can be retrieved. Waiting... (There may be no new LS, or run active may be false)" self.redoTList = True return # If we are not simulating a previous run. Otherwise, we already set lastLS and currentLS ##if not self.simulate: lslist = [] for trig in self.Rates.keys(): if len(self.Rates[trig])>0: lslist.append(max(self.Rates[trig])) # Update lastLS self.lastLS = self.currentLS # Update current LS if len(lslist)>0: self.currentLS = max(lslist) if self.useLSRange: # Adjust runs so we only look at those in our range self.slidingLS = -1 # No sliding LS window self.lastLS = max( [self.lastLS, self.LSRange[0]] ) self.currentLS = min( [self.currentLS, self.LSRange[1] ] ) # If there are lumisection to show, print info for them if self.currentLS > self.lastLS: self.printTable() self.isUpdating = True else: self.isUpdating = False print "Not enough lumisections. Last LS was %s, current LS is %s. Waiting." % (self.lastLS, self.currentLS) def setMode(self): self.sendMailAlerts_dynamic = self.sendMailAlerts_static try: self.triggerMode = self.parser.getTriggerMode(self.runNumber)[0] except: self.triggerMode = "Other" if self.triggerMode.find("cosmics") > -1: self.mode = "cosmics" elif self.triggerMode.find("circulate") > -1: self.mode = "circulate" elif self.triggerMode.find("collisions") > -1: self.mode = "collisions" elif self.triggerMode == "MANUAL": self.mode = "MANUAL" elif self.triggerMode.find("highrate") > -1: self.mode = "other" self.sendMailAlerts_dynamic = False else: self.mode = "other" # Use: Remakes the trigger lists def redoTriggerLists(self): self.redoTList = False # Reset the trigger lists self.usableHLTTriggers = [] self.otherHLTTriggers = [] self.usableL1Triggers = [] self.otherL1Triggers = [] self.fullL1HLTMenu = [] # Reset bad rate records self.badRates = {} # A dictionary: [ trigger name ] { num consecutive bad, trigger bad last check, rate, expected, dev } self.recordAllBadRates = {} # A dictionary: [ trigger name ] < total times the trigger was bad > #set trigger lists automatically based on mode if not self.useAll and not self.userSpecTrigList: if self.mode == "cosmics" or self.mode == "circulate": self.triggerList = self.loadTriggersFromFile(self.cosmics_triggerList) print "monitoring triggers in: ", self.cosmics_triggerList elif self.mode == "collisions": self.triggerList = self.loadTriggersFromFile(self.collisions_triggerList) print "monitoring triggers in: ", self.collisions_triggerList else: self.triggerList = "" print "No lists to monitor: trigger mode not recognized" self.TriggerListL1 = [] self.TriggerListHLT = [] for triggerName in self.triggerList: if triggerName[0:3]=="L1_": self.TriggerListL1.append(triggerName) elif triggerName[0:4]=="HLT_": self.TriggerListHLT.append(triggerName) # Re-make trigger lists for trigger in self.HLTRates.keys(): if (not self.InputFitHLT is None and self.InputFitHLT.has_key(trigger)) and \ (len(self.TriggerListHLT) !=0 and trigger in self.TriggerListHLT): self.usableHLTTriggers.append(trigger) elif trigger[0:4] == "HLT_" and (self.triggerList == "" or trigger in self.TriggerListHLT): self.otherHLTTriggers.append(trigger) elif (trigger[0:4] == "HLT_"): self.fullL1HLTMenu.append(trigger) for trigger in self.L1Rates.keys(): if (not self.InputFitL1 is None and self.InputFitL1.has_key(trigger)) and \ (len(self.TriggerListL1) != 0 and trigger in self.TriggerListL1): self.usableL1Triggers.append(trigger) elif trigger[0:3] == "L1_" and (self.triggerList =="" or trigger in self.TriggerListL1): self.otherL1Triggers.append(trigger) elif (trigger[0:3] == "L1_"): self.fullL1HLTMenu.append(trigger) self.getHeader() # Use: Gets the rates for the lumisections we want def getRates(self): if not self.useLSRange: self.HLTRates = self.parser.getRawRates(self.runNumber, self.lastLS) self.L1Rates = self.parser.getL1RawRates(self.runNumber) self.streamData = self.parser.getStreamData(self.runNumber, self.lastLS) self.pdData = self.parser.getPrimaryDatasets(self.runNumber, self.lastLS) else: self.HLTRates = self.parser.getRawRates(self.runNumber, self.LSRange[0], self.LSRange[1]) self.L1Rates = self.parser.getL1RawRates(self.runNumber) self.streamData = self.parser.getStreamData(self.runNumber, self.LSRange[0], self.LSRange[1]) self.pdData = self.parser.getPrimaryDatasets(self.runNumber, self.LSRange[0], self.LSRange[1]) self.totalStreams = len(self.streamData.keys()) self.Rates = {} self.Rates.update(self.HLTRates) self.Rates.update(self.L1Rates) self.totalHLTTriggers = len(self.HLTRates.keys()) self.totalL1Triggers = len(self.L1Rates.keys()) # Use: Retrieves information and prints it in table form def printTable(self): if self.slidingLS == -1: self.startLS = self.lastLS else: self.startLS = max( [0, self.currentLS-self.slidingLS ] )+1 # Reset variable self.normal = 0 self.bad = 0 PScol = -1 # Get the inst lumi aveLumi = 0 try: self.deadTimeData = self.parser.getDeadTime(self.runNumber) aveDeadTime = 0 except: self.deadTimeData = {} aveDeadTime = None print "Error getting deadtime data" # Get total L1 rate l1rate = 0 try: l1rateData = self.parser.getL1rate(self.runNumber) aveL1rate = 0 except: l1rateData = {} aveL1rate = None print "Error getting total L1 rate data" physicsActive = False # True if we have at least 1 LS with lumi and physics bit true if self.mode != "cosmics": lumiData = self.parser.getLumiInfo(self.runNumber, self.startLS, self.currentLS) self.numBunches = self.parser.getNumberCollidingBunches(self.runNumber) # Find the average lumi since we last checked count = 0 # Get luminosity (only for non-cosmic runs) for LS, instLumi, psi, physics, all_subSys_good in lumiData: # If we are watching a certain range, throw out other LS if self.useLSRange and (LS < self.LSRange[0] or LS > self.LSRange[1]): continue # Average our instLumi if not instLumi is None and physics: physicsActive = True PScol = psi if not aveDeadTime is None and self.deadTimeData.has_key(LS): aveDeadTime += self.deadTimeData[LS] else: aveDeadTime = 0 if not aveL1rate is None and l1rateData.has_key(LS): aveL1rate += l1rateData[LS] else: aveL1rate = 0 aveLumi += instLumi count += 1 if count == 0: aveLumi = "NONE" expected = "NONE" else: aveLumi /= float(count) aveDeadTime /= float(count) aveL1rate /= float(count) else: count = 0 for LS in l1rateData.keys(): if self.useLSRange and (LS < self.LSRange[0] or LS > self.LSRange[1]): continue if not aveDeadTime is None and self.deadTimeData.has_key(LS): aveDeadTime += self.deadTimeData[LS] else: aveDeadTime = 0 if not aveL1rate is None and l1rateData.has_key(LS): aveL1rate += l1rateData[LS] else: aveL1rate = 0 if not count == 0: aveDeadTime /= float(count) aveL1rate /= float(count) self.lumi_ave = aveLumi if self.numBunches[0] > 0 and not aveLumi == "NONE": self.pu_ave = aveLumi/self.numBunches[0]*ppInelXsec/orbitsPerSec else: self.pu_ave = "NONE" # We only do predictions when there were physics active LS in a collisions run doPred = physicsActive and self.mode=="collisions" # Print the header self.printHeader() # Print the triggers that we can make predictions for anytriggers = False if len(self.usableHLTTriggers)>0: print '*' * self.hlength print "Predictable HLT Triggers (ones we have a fit for)" print '*' * self.hlength anytriggers = True self.L1 = False self.printTableSection(self.usableHLTTriggers, doPred, aveLumi) if len(self.usableL1Triggers)>0: print '*' * self.hlength print "Predictable L1 Triggers (ones we have a fit for)" print '*' * self.hlength anytriggers = True self.L1 = True self.printTableSection(self.usableL1Triggers, doPred, aveLumi) #check the full menu for paths deviating past thresholds fullMenu_fits = False for trigger in self.fullL1HLTMenu: self.getTriggerData(trigger, fullMenu_fits, aveLumi) # Print the triggers that we can't make predictions for if self.useAll or self.mode != "collisions" or self.InputFitHLT is None: print '*' * self.hlength print "Unpredictable HLT Triggers (ones we have no fit for or do not try to fit)" print '*' * self.hlength self.L1 = False self.printTableSection(self.otherHLTTriggers, False) self.printTableSection(self.otherL1Triggers, False) anytriggers = True if self.useL1: print '*' * self.hlength print "Unpredictable L1 Triggers (ones we have no fit for or do not try to fit)" print '*' * self.hlength self.L1 = True self.printTableSection(self.otherL1Triggers, False) anytriggers = True if not anytriggers: print '*' * self.hlength print "\n --- No useable triggers --- \n" # Print stream data if self.showStreams: print '*' * self.hlength streamSpacing = [ 50, 20, 25, 25, 25, 25 ] head = stringSegment("* Stream name", streamSpacing[0]) head += stringSegment("* NLumis", streamSpacing[1]) head += stringSegment("* Events", streamSpacing[2]) head += stringSegment("* Stream rate [Hz]", streamSpacing[3]) head += stringSegment("* File size [GB]", streamSpacing[4]) head += stringSegment("* Stream bandwidth [GB/s]", streamSpacing[5]) print head print '*' * self.hlength for name in sorted(self.streamData.keys()): count = 0 streamsize = 0 aveBandwidth = 0 aveRate = 0 for LS, rate, size, bandwidth in self.streamData[name]: streamsize += size aveRate += rate aveBandwidth += bandwidth count += 1 if count > 0: aveRate /= count streamsize /= (count*1000000000.0) aveBandwidth /= (count*1000000000.0) row = stringSegment("* "+name, streamSpacing[0]) row += stringSegment("* "+str(int(count)), streamSpacing[1]) row += stringSegment("* "+str(int(aveRate*23.3*count)), streamSpacing[2]) row += stringSegment("* "+"{0:.2f}".format(aveRate), streamSpacing[3]) row += stringSegment("* "+"{0:.2f}".format(streamsize), streamSpacing[4]) row += stringSegment("* "+"{0:.5f}".format(aveBandwidth), streamSpacing[5]) if not self.noColors and aveRate > self.maxStreamRate and self.mode != "other": write(bcolors.WARNING) # Write colored text print row if not self.noColors and aveRate > self.maxStreamRate and self.mode != "other": write(bcolors.ENDC) # Stop writing colored text else: pass # Print PD data if self.showPDs: print '*' * self.hlength pdSpacing = [ 50, 20, 25, 25] head = stringSegment("* Primary Dataset name", pdSpacing[0]) head += stringSegment("* NLumis", pdSpacing[1]) head += stringSegment("* Events", pdSpacing[2]) head += stringSegment("* Dataset rate [Hz]", pdSpacing[3]) print head print '*' * self.hlength for name in self.pdData.keys(): count = 0 aveRate = 0 for LS, rate in self.pdData[name]: aveRate += rate count += 1 if count > 0: aveRate /= count row = stringSegment("* "+name, pdSpacing[0]) row += stringSegment("* "+str(int(count)), pdSpacing[1]) row += stringSegment("* "+str(int(aveRate*23.3*count)), pdSpacing[2]) row += stringSegment("* "+"{0:.2f}".format(aveRate), pdSpacing[3]) if not self.noColors and aveRate > self.maxPDRate and self.mode != "other": write(bcolors.WARNING) # Write colored text print row if not self.noColors and aveRate > self.maxPDRate and self.mode != "other": write(bcolors.ENDC) # Stop writing colored text else: pass # Closing information print '*' * self.hlength print "SUMMARY:" if self.mode=="collisions": print "Triggers in Normal Range: %s | Triggers outside Normal Range: %s" % (self.normal, self.bad) if self.mode=="collisions": print "Prescale column index:", if PScol == 0: if not self.noColors and PScol == 0 and self.mode != "other": write(bcolors.WARNING) # Write colored text print PScol, "\t0 - Column 0 is an emergency column in collision mode, please select the proper column" if not self.noColors and PScol == 0 and self.mode != "other": write(bcolors.ENDC) # Stop writing colored text else: print PScol try: print "Average inst. lumi: %.0f x 10^30 cm-2 s-1" % (aveLumi) except: print "Average inst. lumi: Not available" print "Total L1 rate: %.0f Hz" % (aveL1rate) print "Average dead time: %.2f %%" % (aveDeadTime) try: print "Average PU: %.2f" % (self.pu_ave) except: print "Average PU: %s" % (self.pu_ave) print '*' * self.hlength # Use: Prints the table header def printHeader(self): print "\n\n", '*' * self.hlength print "INFORMATION:" print "Run Number: %s" % (self.runNumber) print "LS Range: %s - %s" % (self.startLS, self.currentLS) print "Latest LHC Status: %s" % self.parser.getLHCStatus()[1] print "Number of colliding bunches: %s" % self.numBunches[0] print "Trigger Mode: %s (%s)" % (self.triggerMode, self.mode) print "Number of HLT Triggers: %s \nNumber of L1 Triggers: %s" % (self.totalHLTTriggers, self.totalL1Triggers) print "Number of streams:", self.totalStreams print '*' * self.hlength print self.header # Use: Prints a section of a table, ie all the triggers in a trigger list (like usableHLTTriggers, otherHLTTriggers, etc) def printTableSection(self, triggerList, doPred, aveLumi=0): self.tableData = [] # A list of tuples, each a row in the table: ( { trigger, rate, predicted rate, sign of % diff, abs % diff, sign of sigma, abs sigma, ave PS, comment } ) # Get the trigger data for trigger in triggerList: self.getTriggerData(trigger, doPred, aveLumi) # Sort by % diff if need be if doPred: # [4] is % diff, [6] is deviation if self.usePerDiff: self.tableData.sort(key=lambda tup : tup[4], reverse = True) else: self.tableData.sort(key=lambda tup : tup[6], reverse = True) elif self.sortRates: self.tableData.sort(key=lambda tup: tup[1], reverse = True) for trigger, rate, pred, sign, perdiff, dsign, dev, avePS, comment in self.tableData: info = stringSegment("* "+trigger, self.spacing[0]) info += stringSegment("* "+"{0:.2f}".format(rate), self.spacing[1]) if pred!="": info += stringSegment("* "+"{0:.2f}".format(pred), self.spacing[2]) else: info += stringSegment("", self.spacing[2]) if perdiff=="": info += stringSegment("", self.spacing[3]) elif perdiff=="INF": info += stringSegment("* INF", self.spacing[3]) else: info += stringSegment("* "+"{0:.2f}".format(sign*perdiff), self.spacing[3]) if dev=="": info += stringSegment("", self.spacing[4]) elif dev=="INF" or dev==">1E6": info += stringSegment("* "+dev, self.spacing[4]) else: info += stringSegment("* "+"{0:.2f}".format(dsign*dev), self.spacing[4]) info += stringSegment("* "+"{0:.2f}".format(avePS), self.spacing[5]) info += stringSegment("* "+comment, self.spacing[6]) # Color the bad triggers with warning colors if avePS != 0 and self.isBadTrigger(perdiff, dev, rate, trigger[0:3]=="L1_"): if not self.noColors and self.mode != "other": write(bcolors.WARNING) # Write colored text print info if not self.noColors and self.mode != "other": write(bcolors.ENDC) # Stop writing colored text # Don't color normal triggers else: print info # Use: Returns whether a given trigger is bad # Returns: Whether the trigger is bad def isBadTrigger(self, perdiff, dev, psrate, isL1): if psrate == 0: return False if self.mode == "other": return False if ( (self.usePerDiff and perdiff!="INF" and perdiff!="" and abs(perdiff)>self.percAccept) or (dev!="INF" and dev!="" and (dev==">1E6" or abs(dev)>self.devAccept)))\ or (perdiff!="INF" and perdiff!="" and abs(perdiff)>self.percAccept and dev!="INF" and dev!="" and abs(dev)>self.devAccept)\ or (isL1 and psrate>self.maxL1Rate)\ or (not isL1 and psrate>self.maxHLTRate): return True return False # Use: Gets a row of the table, self.tableData: ( { trigger, rate, predicted rate, sign of % diff, abs % diff, ave PS, comment } ) # Parameters: # -- trigger : The name of the trigger # -- doPred : Whether we want to make a prediction for this trigger # -- aveLumi : The average luminosity during the LS in question # Returns: (void) def getTriggerData(self, trigger, doPred, aveLumi): # In case of critical error (this shouldn't occur) if not self.Rates.has_key(trigger): return # If cosmics, don't do predictions if self.mode == "cosmics": doPred = False # Calculate rate if self.mode != "cosmics" and doPred: if not aveLumi is None: expected = self.calculateRate(trigger, aveLumi) if expected<0: expected = 0 # Don't let expected value be negative avePSExpected = expected # Get the mean square error (standard deviation) mse = self.getMSE(trigger) else: expected = None avePSExpected = None mse = None # Find the ave rate since the last time we checked aveRate = 0 properAvePSRate = 0 avePS = 0 aveDeadTime = 0 count = 0 comment = "" correct_for_deadtime = self.deadTimeCorrection if trigger[0:3]=="L1_": correct_for_deadtime = False for LS in self.Rates[trigger].keys(): if self.useLSRange and (LS < self.LSRange[0] or LS > self.LSRange[1]): continue elif LS < self.startLS or LS > self.currentLS: continue prescale = self.Rates[trigger][LS][1] rate = self.Rates[trigger][LS][0] try: deadTime = self.deadTimeData[LS] except: print "trouble getting deadtime for LS: ", LS," setting DT to zero" deadTime = 0 if correct_for_deadtime: rate *= 1. + (deadTime/100.) if prescale > 0: properAvePSRate += rate/prescale else: properAvePSRate += rate aveRate += rate count += 1 avePS += prescale aveDeadTime += deadTime if count > 0: if aveRate == 0: comment += "0 counts " aveRate /= count properAvePSRate /= count avePS /= count aveDeadTime /= count else: #comment += "PS=0" comment += "No rate yet " doPred = False if doPred and not avePSExpected is None and avePS > 1: avePSExpected /= avePS if not doPred and self.removeZeros and aveRate==0: return # Returns if we are not making predictions for this trigger and we are throwing zeros # We want this trigger to be in the table row = [trigger] if self.displayRawRates: row.append(aveRate) else: row.append(properAvePSRate) if doPred and not expected is None: row.append(avePSExpected) else: row.append("") # No predicted rate # Find the % diff if doPred: if expected == "NONE": perc = "UNDEF" dev = "UNDEF" row.append(1) # Sign of % diff row.append(perc) # abs % diff row.append(1) # Sign of deviation row.append(dev) # abs deviation else: diff = aveRate-expected if expected!=0: perc = 100*diff/expected else: perc = "INF" if mse!=0: dev = diff/mse if abs(dev)>1000000: dev = ">1E6" else: dev = "INF" if perc>0: sign=1 else: sign=-1 row.append(sign) # Sign of % diff if perc!="INF": row.append(abs(perc)) # abs % diff else: row.append("INF") #if mse>0: sign=1 #else: sign=-1 row.append(sign) # Sign of the deviation if dev!="INF" and dev!=">1E6": row.append(abs(dev)) # abs deviation else: row.append(dev) else: row.append("") # No prediction, so no sign of a % diff row.append("") # No prediction, so no % diff row.append("") # No prediction, so no sign of deviation row.append("") # No prediction, so no deviation # Add the rest of the info to the row row.append(avePS) row.append(comment) # Add row to the table data if doPred: if expected > 0: self.tableData.append(row) else: self.tableData.append(row) #do not warn on specific triggers for vetoString in self.ignoreStrings: if trigger.find(vetoString) > -1: return # Check if the trigger is bad if doPred: # Check for bad rates. #if (self.usePerDiff and perc!="INF" and perc>self.percAccept) or \ #(not self.usePerDiff and dev!="INF" and (dev==">1E6" or dev>self.devAccept)): if self.isBadTrigger(perc, dev, properAvePSRate, trigger[0:3]=="L1_"): self.bad += 1 # Record if a trigger was bad if not self.recordAllBadRates.has_key(trigger): self.recordAllBadRates[trigger] = 0 self.recordAllBadRates[trigger] += 1 # Record consecutive bad rates if not self.badRates.has_key(trigger): self.badRates[trigger] = [1, True, properAvePSRate, avePSExpected, dev, avePS ] else: last = self.badRates[trigger] self.badRates[trigger] = [ last[0]+1, True, properAvePSRate, avePSExpected, dev, avePS ] else: self.normal += 1 # Remove warning from badRates if self.badRates.has_key(trigger): del self.badRates[trigger] else: if self.isBadTrigger("", "", properAvePSRate, trigger[0:3]=="L1_") and avePS > 0: self.bad += 1 # Record if a trigger was bad if not self.recordAllBadRates.has_key(trigger): self.recordAllBadRates[trigger] = 0 self.recordAllBadRates[trigger] += 1 # Record consecutive bad rates if not self.badRates.has_key(trigger): self.badRates[trigger] = [ 1, True, properAvePSRate, -999, -999, -999 ] else: last = self.badRates[trigger] self.badRates[trigger] = [ last[0]+1, True, properAvePSRate, -999, -999, -999 ] else: self.normal += 1 # Remove warning from badRates if self.badRates.has_key(trigger): del self.badRates[trigger] # Use: Checks triggers to make sure none have been bad for to long def checkTriggers(self): if self.displayBadRates != 0: count = 0 if self.displayBadRates != -1: write("First %s triggers that are bad: " % (self.displayBadRates)) elif len(self.badRates) > 0 : write("All triggers deviating past thresholds from fit and/or L1 rate > %s Hz, HLT rate > %s Hz: " %(self.maxL1Rate,self.maxHLTRate)) for trigger in self.badRates: if self.badRates[trigger][1]: count += 1 write(trigger) if count != self.displayBadRates-1: write(", ") if count == self.displayBadRates: write(".....") break print "" # Print warnings for triggers that have been repeatedly misbehaving mailTriggers = [] # A list of triggers that we should mail alerts about for trigger in self.badRates: if self.badRates[trigger][1]: if self.badRates[trigger][0] >= 1: print "Trigger %s has been out of line for more than %.1f minutes" % (trigger, float(self.badRates[trigger][0])*self.scale_sleeptime) # We want to mail an alert whenever a trigger exits the acceptable threshold envelope if self.badRates[trigger][0] == self.maxCBR: mailTriggers.append( [ trigger, self.badRates[trigger][2], self.badRates[trigger][3], self.badRates[trigger][4], self.badRates[trigger][5] ] ) # Send mail alerts if len(mailTriggers)>0 and self.isUpdating: if self.sendMailAlerts_static and self.sendMailAlerts_dynamic: self.sendMail(mailTriggers) if self.sendAudioAlerts: audioAlert() # Use: Sleeps and prints out waiting dots def sleepWait(self): if not self.quiet: print "Sleeping for %.1f sec before next query" % (60.0*self.scale_sleeptime) for iSleep in range(20): if not self.quiet: write(".") sys.stdout.flush() time.sleep(3.0*self.scale_sleeptime) sys.stdout.flush() print "" # Use: Loads the fit data from the fit file # Parameters: # -- fitFile: The file that the fit data is stored in (a pickle file) # Returns: The input fit data def loadFit(self, fileName): if fileName == "": return None InputFit = {} # Initialize InputFit (as an empty dictionary) # Try to open the file containing the fit info try: pkl_file = open(fileName, 'rb') InputFit = pickle.load(pkl_file) pkl_file.close() except: # File failed to open print "Error: could not open fit file: %s" % (fileName) return InputFit # Use: Calculates the expected rate for a trigger at a given ilumi based on our input fit def calculateRate(self, triggerName, ilum): # Make sure we have a fit for the trigger if not self.L1 and (self.InputFitHLT is None or not self.InputFitHLT.has_key(triggerName)): return 0 elif self.L1 and ((self.InputFitL1 is None) or not self.InputFitL1.has_key(triggerName)): return 0 # Get the param list if self.L1: paramlist = self.InputFitL1[triggerName] else: paramlist = self.InputFitHLT[triggerName] # Calculate the rate if paramlist[0]=="exp": funcStr = "%s + %s*expo(%s+%s*x)" % (paramlist[1], paramlist[2], paramlist[3], paramlist[4]) # Exponential else: funcStr = "%s+x*(%s+ x*(%s+x*%s))" % (paramlist[1], paramlist[2], paramlist[3], paramlist[4]) # Polynomial fitFunc = TF1("Fit_"+triggerName, funcStr) if self.pileUp: if self.numBunches[0] > 0: return self.numBunches[0]*fitFunc.Eval(ilum/self.numBunches[0]*ppInelXsec/orbitsPerSec) else: return 0 return fitFunc.Eval(ilum) # Use: Gets the MSE of the fit def getMSE(self, triggerName): if not self.L1 and (self.InputFitHLT is None or not self.InputFitHLT.has_key(triggerName)): return 0 elif self.L1 and ((self.InputFitL1 is None) or not self.InputFitL1.has_key(triggerName)): return 0 if self.L1: paramlist = self.InputFitL1[triggerName] else: paramlist = self.InputFitHLT[triggerName] if self.pileUp: return self.numBunches[0]*paramlist[5] return paramlist[5] # The MSE # Use: Sends an email alert # Parameters: # -- mailTriggers: A list of triggers that we should include in the mail, ( { triggerName, aveRate, expected rate, standard dev } ) # Returns: (void) def sendMail(self, mailTriggers): mail = "Run: %d, Lumisections: %s - %s \n" % (self.runNumber, self.lastLS, self.currentLS) try: mail += "Average inst. lumi: %.0f x 10^30 cm-2 s-1\n" % (self.lumi_ave) except: mail += "Average inst. lumi: %s x 10^30 cm-2 s-1\n" % (self.lumi_ave) try: mail += "Average PU: %.2f\n \n" % (self.pu_ave) except: mail += "Average PU: %s\n \n" % (self.pu_ave) mail += "Trigger rates deviating from acceptable and/or expected values: \n\n" for triggerName, rate, expected, dev, ps in mailTriggers: if self.numBunches[0] == 0: mail += "\n %s: Actual: %s Hz\n" % (stringSegment(triggerName, 35), rate) else: if expected > 0: try: mail += "\n %s: Expected: %.1f Hz, Actual: %.1f Hz, Unprescaled Expected/nBunches: %.5f Hz, Unprescaled Actual/nBunches: %.5f Hz, Deviation: %.1f\n" % (stringSegment(triggerName, 35), expected, rate, expected*ps/self.numBunches[0], rate*ps/self.numBunches[0], dev) except: mail += "\n %s: Expected: %s Hz, Actual: %s Hz, Unprescaled Expected/nBunches: %s Hz, Unprescaled Actual/nBunches: %s Hz, Deviation: %s\n" % (stringSegment(triggerName, 35), expected, rate, expected*ps/self.numBunches[0], rate*ps/self.numBunches[0], dev) mail += " *referenced fit: <https://raw.githubusercontent.com/cms-tsg-fog/RateMon/master/Fits/2016/plots/%s.png>\n" % (triggerName) else: try: mail += "\n %s: Actual: %.1f Hz\n" % (stringSegment(triggerName, 35), rate) except: mail += "\n %s: Actual: %s Hz\n" % (stringSegment(triggerName, 35), rate) try: wbm_url = self.parser.getWbmUrl(self.runNumber,triggerName,self.currentLS) if not wbm_url == "-": mail += " *WBM rate: <%s>\n" % (wbm_url) except: print "WBM plot url query failed" mail += "\nWBM Run Summary: <https://cmswbm.web.cern.ch/cmswbm/cmsdb/servlet/RunSummary?RUN=%s> \n\n" % (self.runNumber) mail += "Email warnings triggered when: \n" mail += " - L1 or HLT rates deviate by more than %s standard deviations from fit \n" % (self.devAccept) mail += " - HLT rates > %s Hz \n" % (self.maxHLTRate) mail += " - L1 rates > %s Hz \n" % (self.maxL1Rate) print "--- SENDING MAIL ---\n"+mail+"\n--------------------" mailAlert(mail)
import partition import DBParser import createfile import generalize data = DBParser.makeListOfDictionaries() clusters = partition.algorithm(data,4000) for i in range(1,4000): genOfSex = generalize.generalizeSex(clusters[i]) genOfMaritalStat = generalize.generalizeMaritalStat(clusters[i]) genOfCountry = generalize.generalizeCountry(clusters[i]) genOfWorkclass = generalize.generalizeWorkclass(clusters[i]) createfile.createAnonDataFile(clusters[i],genOfSex,genOfWorkclass,genOfCountry,genOfMaritalStat) print(genOfSex,genOfWorkclass,genOfCountry,genOfMaritalStat)
class DataParser: # This is an interface for DBParser() to select and manage the data returned by DBParser() def __init__(self): # type: () -> None self.parser = DBParser() # The lists all have the same number of elements, e.g.: len(self.lumi_data[trg][run]) == len(self.pu_data[trg][run]) self.ls_data = {} # {'name': { run_number: [LS] } } self.rate_data = {} # {'name': { run_number: { LS: raw_rates } } } self.ps_data = {} # {'name': { run_number: { LS: prescale } } } self.pu_data = {} # {'name': { run_number: { LS: PU } } } self.lumi_data = {} # {'name': { run_number: { LS: iLumi } } } self.det_data = {} # {'name': { run_number: { LS: detecotr_ready } } } self.phys_data = {} # {'name': { run_number: { LS: phys_active } } } self.bw_data = {} # {'name': { run_number: { LS: bandwidth } } } self.size_data = {} # {'name': { run_number: { LS: size } } } self.lumi_info = {} # {run_number: [ (LS,ilum,psi,phys,cms_ready) ] } self.bunch_map = {} # {run_number: nBunches } self.hlt_triggers = [ ] # List of specific HLT triggers we want to get rates for, if empty --> get all HLT rates self.l1_triggers = [ ] # List of specific L1 triggers we want to get rates for, if empty --> get all L1 rates self.runs_used = [ ] # List of runs which had rate info for queried objects self.runs_skipped = [ ] # List of runs which did not have rate info for queried objects self.name_list = [ ] # List of all named objects for which we have data, e.g. triggers, datasets, streams, etc... self.psi_filter = [] self.type_map = { } # Maps each object name to a type: trigger, dataset, stream, or L1A # NOTE: Still need to handle the case where if two objects share the same name, but diff type # NOTE2: This approach should be fine, since DataParser owns the nameing, will need to be careful self.ls_veto = {} # {run_number:[LS list]} - LS to ignore self.name_veto = [ ] # List of paths/objects to remove from consideration self.use_prescaled_rate = False # If true, then rates are not un-prescaled self.use_cross_section = False # If true, then divide the rate by inst. lumi (only for L1 and HLT trigger data) self.normalize_bunches = True # Normalize by the number of colliding bunches self.correct_for_DT = True self.convert_output = True # Flag to convert data from { LS: data } to [ data ], used in the data getters self.skip_l1_triggers = False # Flag to skip collecting rates for L1 triggers self.skip_hlt_triggers = False # Flag to skip collecting rates for HLT triggers self.l1_rate_cut = 25e6 # Ignore L1 rates above this threshold self.max_deadtime = 10. self.min_ls = -1 self.max_ls = 9999999 self.use_L1_triggers = False # Gets L1 rates self.use_HLT_triggers = False # Gets HLT rates self.use_streams = False # Gets stream rates self.use_datasets = False # Gets dataset rates self.use_L1A_rate = False # Gets the L1A rates self.use_ps_mask = False # Collects data only for LS in the specified prescale indices self.use_best_lumi = True # Uses best luminosity as determined by BRIL self.use_PLTZ_lumi = False # Uses luminosity reading from PLTZ self.use_HF_lumi = False # Uses luminosity reading from HF self.verbose = True def parseRuns(self, run_list): # type: (List[int]) -> None if len(self.hlt_triggers) == 0 and len(self.l1_triggers) > 0: self.skip_hlt_triggers = True if len(self.hlt_triggers) > 0 and len(self.l1_triggers) == 0: self.skip_l1_triggers = True counter = 1 for run in sorted(run_list): if self.verbose: print "Processing run: %d (%d/%d)" % (run, counter, len(run_list)) counter += 1 bunches = self.parser.getNumberCollidingBunches(run)[0] if bunches is None or bunches is 0: bunches = 1 lumi_info = self.parseLumiInfo( run) # [( LS,ilum,psi,phys,cms_ready ) ] run_data = self.getRunData(run, bunches, lumi_info) if len( run_data.keys() ) == 0: # i.e. no triggers/streams/datasets had enough valid rates self.runs_skipped.append(run) continue else: self.runs_used.append(run) self.bunch_map[run] = bunches self.lumi_info[run] = lumi_info for name in run_data: if name in self.name_veto: continue ls_array = run_data[name]["LS"] rate = run_data[name]["rate"] prescale = run_data[name]["prescale"] pu = run_data[name]["PU"] lumi = run_data[name]["ilumi"] det_status = run_data[name]["status"] phys = run_data[name]["phys"] bw = run_data[name]["bandwidth"] size = run_data[name]["size"] if not name in self.name_list: self.name_list.append(name) self.ls_data[name] = {} self.rate_data[name] = {} self.ps_data[name] = {} self.pu_data[name] = {} self.lumi_data[name] = {} self.det_data[name] = {} self.phys_data[name] = {} self.bw_data[name] = {} self.size_data[name] = {} self.ls_data[name][run] = ls_array self.rate_data[name][run] = rate self.ps_data[name][run] = prescale self.pu_data[name][run] = pu self.lumi_data[name][run] = lumi self.det_data[name][run] = det_status self.phys_data[name][run] = phys self.bw_data[name][run] = bw self.size_data[name][run] = size def parseLumiInfo(self, run): # [( LS,ilum,psi,phys,cms_ready ) ] lumi_info = [] trigger_mode = self.parser.getTriggerMode(run)[0] if trigger_mode.find('cosmics') > 0: # This is a cosmics menu --> No luminosity info if self.verbose: print "\tDetected cosmics run..." print "\tGetting lumi info..." for LS, psi in self.parser.getLSInfo(run): # We hard code phys and cms_ready to both be true for all LS in the run lumi_info.append([LS, 0.0, psi, 1, 1]) elif trigger_mode.find('collisions') > 0: # This is a collisions menu if self.verbose: print "\tDetected collisions run..." print "\tGetting lumi info..." if self.use_best_lumi: lumi_info = self.parser.getQuickLumiInfo(run, minLS=self.min_ls, maxLS=self.max_ls) elif self.use_PLTZ_lumi: lumi_info = self.parser.getLumiInfo(run, minLS=self.min_ls, maxLS=self.max_ls, lumi_source=1) elif self.use_HF_lumi: lumi_info = self.parser.getLumiInfo(run, minLS=self.min_ls, maxLS=self.max_ls, lumi_source=2) else: # Unknown menu --> For now we assume it's compatibale with collisions type menus if self.verbose: print "\tUnknown run type: %s" % (trigger_mode) print "\tGetting lumi info..." if self.use_best_lumi: lumi_info = self.parser.getQuickLumiInfo(run, minLS=self.min_ls, maxLS=self.max_ls) elif self.use_PLTZ_lumi: lumi_info = self.parser.getLumiInfo(run, minLS=self.min_ls, maxLS=self.max_ls, lumi_source=1) elif self.use_HF_lumi: lumi_info = self.parser.getLumiInfo(run, minLS=self.min_ls, maxLS=self.max_ls, lumi_source=2) if self.ls_veto.has_key(run): new_lumi_info = [] for LS, ilum, psi, phys, cms_ready in lumi_info: if LS in self.ls_veto[run]: continue new_lumi_info.append([LS, ilum, psi, phys, cms_ready]) lumi_info = new_lumi_info return lumi_info # This might be excessive, should think about reworking this section # ------ # TODO: We need to ensure that none of the object names overlap with one another # (i.e. dataset names that overlap with stream names) for the rate data. def getRunData(self, run, bunches, lumi_info): # type: (int,int,List[Tuple[int,float,int,bool,bool]]) -> Dict[str: object] run_data = {} if bunches is None or bunches is 0: if self.verbose: print "Unable to get bunches" return {} if self.use_streams: run_data.update(self.getStreamData(run, bunches, lumi_info)) if self.use_datasets: run_data.update(self.getDatasetData(run, bunches, lumi_info)) if self.use_L1A_rate: run_data.update(self.getL1AData(run, bunches, lumi_info)) if self.use_HLT_triggers: run_data.update(self.getHLTTriggerData(run, bunches, lumi_info)) if self.use_L1_triggers: run_data.update(self.getL1TriggerData(run, bunches, lumi_info)) return run_data # Returns information related to L1 triggers def getL1TriggerData(self, run, bunches, lumi_info): # type: (int,int,List[Tuple[int,float,int,bool,bool]]) -> Dict[str: object] if self.skip_l1_triggers: return {} if self.verbose: print "\tGetting L1 rates..." L1_rates = self.parser.getL1Rates(run, minLS=self.min_ls, maxLS=self.max_ls, scaler_type=1) run_data = {} # {'object': {"LS": list, "rate": {...}, ... } } for trigger in L1_rates: self.type_map[trigger] = "trigger" run_data[trigger] = {} ls_array = array.array('f') rate_dict = {} ps_dict = {} pu_dict = {} lumi_dict = {} det_dict = {} phys_dict = {} bw_dict = {} size_dict = {} for LS, ilum, psi, phys, cms_ready in lumi_info: if psi not in self.psi_filter and self.use_ps_mask: continue if not ilum is None and L1_rates[trigger].has_key(LS): pu = (ilum / bunches * ppInelXsec / orbitsPerSec) rate = L1_rates[trigger][LS][0] prescale = L1_rates[trigger][LS][1] if rate > self.l1_rate_cut: continue if self.normalize_bunches: rate = rate / bunches if self.use_prescaled_rate: if prescale != 0: rate = rate / prescale #else: # rate = 0 if self.use_cross_section: rate = rate / ilum ls_array.append(LS) rate_dict[LS] = rate ps_dict[LS] = prescale pu_dict[LS] = pu lumi_dict[LS] = ilum det_dict[LS] = cms_ready phys_dict[LS] = phys bw_dict[LS] = None size_dict[LS] = None run_data[trigger]["LS"] = ls_array run_data[trigger]["rate"] = rate_dict run_data[trigger]["prescale"] = ps_dict run_data[trigger]["PU"] = pu_dict run_data[trigger]["ilumi"] = lumi_dict run_data[trigger]["status"] = det_dict run_data[trigger]["phys"] = phys_dict run_data[trigger]["bandwidth"] = bw_dict run_data[trigger]["size"] = size_dict return run_data # Returns information related to HLT triggers def getHLTTriggerData(self, run, bunches, lumi_info): # type: (int,int,List[Tuple[int,float,int,bool,bool]]) -> Dict[str: object] if self.skip_hlt_triggers: return {} if self.verbose: print "\tGetting HLT rates..." HLT_rates = self.parser.getHLTRates(run, self.hlt_triggers, minLS=self.min_ls, maxLS=self.max_ls) if self.correct_for_DT: self.correctForDeadtime(HLT_rates, run) run_data = {} # {'object': {"LS": list, "rate": {...}, ... } } for trigger in HLT_rates: self.type_map[trigger] = "trigger" run_data[trigger] = {} ls_array = array.array('f') rate_dict = {} ps_dict = {} pu_dict = {} lumi_dict = {} det_dict = {} phys_dict = {} bw_dict = {} size_dict = {} for LS, ilum, psi, phys, cms_ready in lumi_info: if psi not in self.psi_filter and self.use_ps_mask: continue if not ilum is None and HLT_rates[trigger].has_key(LS): pu = (ilum / bunches * ppInelXsec / orbitsPerSec) rate = HLT_rates[trigger][LS][0] prescale = HLT_rates[trigger][LS][1] if self.normalize_bunches: rate = rate / bunches if self.use_prescaled_rate: if prescale != 0: rate = rate / prescale #else: # rate = 0 if self.use_cross_section: rate = rate / ilum ls_array.append(LS) rate_dict[LS] = rate ps_dict[LS] = prescale pu_dict[LS] = pu lumi_dict[LS] = ilum det_dict[LS] = cms_ready phys_dict[LS] = phys bw_dict[LS] = None size_dict[LS] = None run_data[trigger]["LS"] = ls_array run_data[trigger]["rate"] = rate_dict run_data[trigger]["prescale"] = ps_dict run_data[trigger]["PU"] = pu_dict run_data[trigger]["ilumi"] = lumi_dict run_data[trigger]["status"] = det_dict run_data[trigger]["phys"] = phys_dict run_data[trigger]["bandwidth"] = bw_dict run_data[trigger]["size"] = size_dict return run_data def getStreamData(self, run, bunches, lumi_info): # type: (int,int,List[Tuple[int,float,int,bool,bool]]) -> Dict[str: object] if self.verbose: print "\tGetting Stream rates..." data = self.parser.getStreamData( run, minLS=self.min_ls, maxLS=self.max_ls) # {'stream': [ (LS,rate,size,bandwidth) ] } stream_rates = {} # {'stream': {LS: (rate,size,bandwidth) } } # Format the output from DBParser() for name in data: stream_rates[name] = {} for LS, rate, size, bandwidth in data[name]: stream_rates[name][LS] = [rate, size, bandwidth] run_data = {} # {'object': {"LS": list, "rate": {...}, ... } } blacklist = [ "PhysicsEndOfFill", "PhysicsMinimumBias0", "PhysicsMinimumBias1", "PhysicsMinimumBias2" ] sum_list = [] for _object in stream_rates: self.type_map[_object] = "stream" run_data[_object] = {} if _object[:7] == "Physics" and not _object in blacklist: sum_list.append(_object) ls_array = array.array('f') rate_dict = {} ps_dict = {} pu_dict = {} lumi_dict = {} det_dict = {} phys_dict = {} bw_dict = {} size_dict = {} for LS, ilum, psi, phys, cms_ready in lumi_info: if psi not in self.psi_filter and self.use_ps_mask: continue if not ilum is None and stream_rates[_object].has_key(LS): pu = (ilum / bunches * ppInelXsec / orbitsPerSec) rate = stream_rates[_object][LS][0] size = stream_rates[_object][LS][1] bandwidth = stream_rates[_object][LS][2] if self.normalize_bunches: rate = rate / bunches ls_array.append(LS) rate_dict[LS] = rate ps_dict[LS] = None pu_dict[LS] = pu lumi_dict[LS] = ilum det_dict[LS] = cms_ready phys_dict[LS] = phys bw_dict[LS] = bandwidth size_dict[LS] = size run_data[_object]["LS"] = ls_array run_data[_object]["rate"] = rate_dict run_data[_object]["prescale"] = ps_dict run_data[_object]["PU"] = pu_dict run_data[_object]["ilumi"] = lumi_dict run_data[_object]["status"] = det_dict run_data[_object]["phys"] = phys_dict run_data[_object]["bandwidth"] = bw_dict run_data[_object]["size"] = size_dict self.sumObjects(run_data=run_data, new_name="Combined_Physics_Streams", sum_list=sum_list, obj_type="stream") return run_data def getDatasetData(self, run, bunches, lumi_info): # type: (int,int,List[Tuple[int,float,int,bool,bool]]) -> Dict[str: object] if self.verbose: print "\tGetting Dataset rates..." data = self.parser.getPrimaryDatasets( run, minLS=self.min_ls, maxLS=self.max_ls) # {'dataset': [ (LS,rate) ] } dataset_rates = {} # {'dataset': {LS: (rate) } } # Format the output from DBParser() for name in data: dataset_rates[name] = {} for LS, rate in data[name]: dataset_rates[name][LS] = [rate] run_data = {} # {'object': {"LS": list, "rate": {...}, ... } } for _object in dataset_rates: self.type_map[_object] = "dataset" run_data[_object] = {} ls_array = array.array('f') rate_dict = {} ps_dict = {} pu_dict = {} lumi_dict = {} det_dict = {} phys_dict = {} bw_dict = {} size_dict = {} for LS, ilum, psi, phys, cms_ready in lumi_info: if psi not in self.psi_filter and self.use_ps_mask: continue if not ilum is None and dataset_rates[_object].has_key(LS): pu = (ilum / bunches * ppInelXsec / orbitsPerSec) rate = dataset_rates[_object][LS][0] if self.normalize_bunches: rate = rate / bunches ls_array.append(LS) rate_dict[LS] = rate ps_dict[LS] = None pu_dict[LS] = pu lumi_dict[LS] = ilum det_dict[LS] = cms_ready phys_dict[LS] = phys bw_dict[LS] = None size_dict[LS] = None run_data[_object]["LS"] = ls_array run_data[_object]["rate"] = rate_dict run_data[_object]["prescale"] = ps_dict run_data[_object]["PU"] = pu_dict run_data[_object]["ilumi"] = lumi_dict run_data[_object]["status"] = det_dict run_data[_object]["phys"] = phys_dict run_data[_object]["bandwidth"] = bw_dict run_data[_object]["size"] = size_dict return run_data # NOTE: L1A_rates has a slightly different dict format, the value-pair for the LS keys is NOT a list def getL1AData(self, run, bunches, lumi_info): # type: (int,int,List[Tuple[int,float,int,bool,bool]]) -> Dict[str: object] L1A_rates = {} # {'L1A': {LS: rate } } if self.verbose: print "\tGetting L1ATotal rates..." L1A_rates["L1ATotal"] = self.parser.getL1rate(run) if self.verbose: print "\tGetting L1APhysics rates..." L1A_rates["L1APhysics"] = self.parser.getL1APhysics(run) if self.verbose: print "\tGetting L1APhysicsLost rates..." L1A_rates["L1APhysicsLost"] = self.parser.getL1APhysicsLost(run) run_data = {} # {'object': {"LS": list, "rate": {...}, ... } } for _object in L1A_rates: self.type_map[_object] = "L1A" run_data[_object] = {} ls_array = array.array('f') rate_dict = {} ps_dict = {} pu_dict = {} lumi_dict = {} det_dict = {} phys_dict = {} bw_dict = {} size_dict = {} for LS, ilum, psi, phys, cms_ready in lumi_info: if psi not in self.psi_filter and self.use_ps_mask: continue if not ilum is None and L1A_rates[_object].has_key(LS): pu = (ilum / bunches * ppInelXsec / orbitsPerSec) rate = L1A_rates[_object][LS] if self.normalize_bunches: rate = rate / bunches ls_array.append(LS) rate_dict[LS] = rate ps_dict[LS] = None pu_dict[LS] = pu lumi_dict[LS] = ilum det_dict[LS] = cms_ready phys_dict[LS] = phys bw_dict[LS] = None size_dict[LS] = None run_data[_object]["LS"] = ls_array run_data[_object]["rate"] = rate_dict run_data[_object]["prescale"] = ps_dict run_data[_object]["PU"] = pu_dict run_data[_object]["ilumi"] = lumi_dict run_data[_object]["status"] = det_dict run_data[_object]["phys"] = phys_dict run_data[_object]["bandwidth"] = bw_dict run_data[_object]["size"] = size_dict return run_data # Use: Modifies the rates in Rates, correcting them for deadtime # Parameters: # -- Rates: A dict - {'trigger': {LS: (raw_rate,prescale) } } def correctForDeadtime(self, Rates, run_number): # type: (Dict[str,object],int) -> None dead_time = self.parser.getDeadTime(run_number) for LS in dead_time: for trigger in Rates: if Rates[trigger].has_key(LS): # Sometimes, LS's are missing Rates[trigger][LS][0] *= (1. + dead_time[LS] / 100.) if dead_time[ LS] > self.max_deadtime: # Do not plot lumis where deadtime is greater than 10% del Rates[trigger][LS] # Creates a new dictionary key, that corresponds to the summed rates of all the specified objects # data: {'object': {"LS": list, "rate": {...}, ... } } def sumObjects(self, run_data, new_name, sum_list, obj_type): # type: (Dict[str,object],str,List[str],str) -> bool if not set(sum_list) <= set(run_data.keys()): if self.verbose: print "\tERROR: Specified objects that aren't in the run_data" return False if (len(sum_list) == 0): print "\tERROR: sum_list has size=0 (see sumObjects in DataParser.py). May be that there were no streams for this run." return False ref_name = sum_list[0] ls_array = array.array('f') rate_dict = {} ps_dict = {} pu_dict = {} lumi_dict = {} det_dict = {} phys_dict = {} bw_dict = {} size_dict = {} # We only use LS that are in *all* of the objects ls_set = set(run_data[ref_name]["LS"]) for name in sum_list: ls_set = ls_set & set(run_data[name]["LS"]) ls_array.extend(sorted(ls_set)) for LS in ls_array: total_rate = 0 total_bw = 0 total_size = 0 for name in sum_list: total_rate += run_data[name]["rate"][LS] try: total_bw += run_data[name]["bandwidth"][LS] except: total_bw = None try: total_size += run_data[name]["size"][LS] except: total_size = None rate_dict[LS] = total_rate bw_dict[LS] = total_bw size_dict[LS] = total_size ps_dict[LS] = None pu_dict[LS] = run_data[ref_name]["PU"][LS] lumi_dict[LS] = run_data[ref_name]["ilumi"][LS] det_dict[LS] = run_data[ref_name]["status"][LS] phys_dict[LS] = run_data[ref_name]["phys"][LS] self.type_map[new_name] = obj_type run_data[new_name] = {} run_data[new_name]["LS"] = ls_array run_data[new_name]["rate"] = rate_dict run_data[new_name]["prescale"] = ps_dict run_data[new_name]["PU"] = pu_dict run_data[new_name]["ilumi"] = lumi_dict run_data[new_name]["status"] = det_dict run_data[new_name]["phys"] = phys_dict run_data[new_name]["bandwidth"] = bw_dict run_data[new_name]["size"] = size_dict return True # Converts input: {'name': { run_number: { LS: data } } } --> {'name': {run_number: [ data ] } } def convertOutput(self, _input): # type: (Dict[str,object]) -> Dict[str,object] output = {} for name in _input: output[name] = {} for run in _input[name]: output[name][run] = array.array('f') for LS in sorted(_input[name][run].keys( )): # iterating over *sorted* LS is extremely important here output[name][run].append(_input[name][run][LS]) return output def resetData(self): # type: () -> None self.ls_data = {} # {'name': { run_number: [LS] } } self.rate_data = {} # {'name': { run_number: { LS: raw_rates } } } self.ps_data = {} # {'name': { run_number: { LS: prescale } } } self.pu_data = {} # {'name': { run_number: { LS: PU } } } self.lumi_data = {} # {'name': { run_number: { LS: iLumi } } } self.det_data = {} # {'name': { run_number: { LS: detecotr_ready } } } self.phys_data = {} # {'name': { run_number: { LS: phys_active } } } self.bw_data = {} # {'name': { run_number: { LS: bandwidth } } } self.size_data = {} # {'name': { run_number: { LS: size } } } self.lumi_info = {} # {run_number: [ (LS,ilum,psi,phys,cms_ready) ] } self.bunch_map = {} # {run_number: nBunches } self.runs_used = [] self.runs_skipped = [] self.name_list = [] self.type_map = {} #################################################################################################### # --- All the 'getters' --- def getLSData(self): # type: () -> Dict[str,object] return self.ls_data def getRateData(self): # type: () -> Dict[str,object] if self.convert_output: output = self.convertOutput(self.rate_data) else: output = self.rate_data return output def getPSData(self): # type: () -> Dict[str,object] if self.convert_output: output = self.convertOutput(self.ps_data) else: output = self.ps_data return output def getPUData(self): # type: () -> Dict[str,object] if self.convert_output: output = self.convertOutput(self.pu_data) else: output = self.pu_data return output def getLumiData(self): # type: () -> Dict[str,object] if self.convert_output: output = self.convertOutput(self.lumi_data) else: output = self.lumi_data return output def getDetectorStatus(self): # type: () -> Dict[str,object] if self.convert_output: output = self.convertOutput(self.det_data) else: output = self.det_data return output def getPhysStatus(self): # type: () -> Dict[str,object] if self.convert_output: output = self.convertOutput(self.phys_data) else: output = self.phys_data return output def getBandwidthData(self): # type: () -> Dict[str,object] if self.convert_output: output = self.convertOutput(self.bw_data) else: output = self.bw_data return output def getSizeData(self): # type: () -> Dict[str,object] if self.convert_output: output = self.convertOutput(self.size_data) else: output = self.size_data return output def getLumiInfo(self): # type: () -> Dict[int,List[Tuple]] return self.lumi_info def getBunchMap(self): # type: () -> Dict[int,int] return self.bunch_map def getRunsUsed(self): # type: () -> List[int] return self.runs_used def getNameList(self): # type: () -> List[int] return self.name_list # Returns all the objects of type obj_type we have rate for def getObjectList(self, obj_type): # type: (str) -> List[str] _list = [] for obj in self.name_list: if self.type_map[obj] == obj_type: _list.append(obj) return _list def getTypeMap(self): # type: () -> Dict[str,str] return self.type_map # Return the latest LS for a given run or -1 def getLastLS(self, run): # type: (int) -> int if not run in self.runs_used: return -1 index = LUMI_INFO_MAP["LS"] return max(self.lumi_info[run], key=lambda x: x[index])[index] # Return the latest run for which we have data or -1 def getLastRun(self): # type: () -> int if len(self.runs_used) == 0: return -1 return max(self.runs_used)