class ShiftMonitor:
def __init__(self):
# Suppress root warnings
ROOT.gErrorIgnoreLevel = 7000
# Fits and fit files
self.fitFile = "Fits/2016/FOG.pkl" # The fit file, can contain both HLT and L1 triggers
#self.fitFile = "../HLT_Fit_Run275911-276244_Tot12_fit.pkl"
# self.fitFile = ""#fits__273013-273017/HLT_Fit_Run273013-273017_Tot12_fit.pkl" # The fit file, can contain both HLT and L1 triggers
self.InputFitHLT = None # The fit information for the HLT triggers
self.InputFitL1 = None # The fit information for the L1 triggers
# DBParser
self.parser = DBParser() # A database parser
# Rates
self.HLTRates = None # HLT rates
self.L1Rates = None # L1 rates
self.Rates = None # Combined L1 and HLT rates
self.deadTimeData = {} # initializing deadTime dict
# Run control
self.lastRunNumber = -2 # The run number during the last segment
self.runNumber = -1 # The number of the current run
self.numBunches = [-1, -1] # Number of [target, colliding] bunches
# Running over a previouly done run
self.assignedNum = False # If true, we look at the assigned run, not the latest run
self.LSRange = [] # If we want to only look at a range of LS from the run
##self.simulate = False # Simulate running through and monitoring a previous run
# Lumisection control
self.lastLS = 1 # The last LS that was processed last segment
self.currentLS = 1 # The latest LS written to the DB
self.slidingLS = -1 # The number of LS to average over, use -1 for no sliding LS
self.useLSRange = False # Only look at LS in a certain range
# Mode
self.triggerMode = None # The trigger mode
self.mode = None # Mode: cosmics, circulate, physics
# Columns header
self.displayRawRates = False # display raw rates, to display prescaled rates, set = True
self.pileUp = True # derive expected rate as a function of the pileUp, and not the luminosity
# Triggers
self.cosmics_triggerList = "monitorlist_COSMICS.list" #default list used when in cosmics mode
self.collisions_triggerList = "monitorlist_COLLISIONS.list" #default list used when in collision mode
self.triggerList = "" # A list of all the L1 and HLT triggers we want to monitor
self.userSpecTrigList = False # User specified trigger list
self.usableHLTTriggers = [] # HLT Triggers active during the run that we have fits for (and are in the HLT trigger list if it exists)
self.otherHLTTriggers = [] # HLT Triggers active during the run that are not usable triggers
self.usableL1Triggers = [] # L1 Triggers active during the run that have fits for (and are in the L1 trigger list if it exists)
self.otherL1Triggers = [] # L1 Triggers active during that run that are not usable triggers
self.redoTList = True # Whether we need to update the trigger lists
self.useAll = False # If true, we will print out the rates for all the HLT triggers
self.useL1 = False # If true, we will print out the rates for all the L1 triggers
self.totalHLTTriggers = 0 # The total number of HLT Triggers on the menu this run
self.totalL1Triggers = 0 # The total number of L1 Triggers on the menu this run
self.fullL1HLTMenu = []
self.ignoreStrings = ["Calibration","L1Tech","BPTX","Bptx"]
# Restrictions
self.removeZeros = False # If true, we don't show triggers that have zero rate
# Trigger behavior
self.percAccept = 50.0 # The acceptence for % diff
self.devAccept = 5 # The acceptance for deviation
self.badRates = {} # A dictionary: [ trigger name ] { num consecutive bad , whether the trigger was bad last time we checked, rate, expected, dev }
self.recordAllBadTriggers = {} # A dictionary: [ trigger name ] < total times the trigger was bad >
self.maxCBR = 3 # The maximum consecutive db queries a trigger is allowed to deviate from prediction by specified amount before it's printed out
self.displayBadRates = -1 # The number of bad rates we should show in the summary. We use -1 for all
self.usePerDiff = False # Whether we should identify bad triggers by perc diff or deviatoin
self.sortRates = True # Whether we should sort triggers by their rates
self.maxHLTRate = 500 # The maximum prescaled rate we allow an HLT Trigger to have
self.maxL1Rate = 30000 # The maximum prescaled rate we allow an L1 Trigger to have
# Other options
self.quiet = False # Prints fewer messages in this mode
self.noColors = False # Special formatting for if we want to dump the table to a file
self.sendMailAlerts_static = True # Whether we should send alert mails
self.sendMailAlerts_dynamic = self.sendMailAlerts_static
self.sendAudioAlerts = False # Whether we should send audio warning messages in the control room (CAUTION)
self.isUpdating = True # flag to determine whether or not we're receiving new LS
self.showStreams = False # Whether we should print stream information
self.showPDs = False # Whether we should print pd information
self.totalStreams = 0 # The total number of streams
self.maxStreamRate = 1000000 # The maximum rate we allow a "good" stream to have
self.maxPDRate = 250 # The maximum rate we allow a "good" pd to have
self.lumi_ave = "NONE"
self.pu_ave = "NONE"
self.deadTimeCorrection = True # correct the rates for dead time
self.scale_sleeptime = 2.0 # Scales the length of time to wait before sending another query (1.0 = 60sec, 2.0 = 120sec, etc)
# Use: Opens a file containing a list of trigger names and adds them to the RateMonitor class's trigger list
# Note: We do not clear the trigger list, this way we could add triggers from multiple files to the trigger list
# -- fileName: The name of the file that trigger names are contained in
# Returns: (void)
def loadTriggersFromFile(self, fileName):
try:
file = open(fileName, 'r')
except:
print "File", fileName, "(a trigger list file) failed to open."
return
allTriggerNames = file.read().split() # Get all the words, no argument -> split on any whitespace
TriggerList = []
for triggerName in allTriggerNames:
# Recognize comments
if triggerName[0]=='#': continue
try:
if not str(triggerName) in TriggerList:
TriggerList.append(stripVersion(str(triggerName)))
except:
print "Error parsing trigger name in file", fileName
return TriggerList
# Use: Formats the header string
# Returns: (void)
def getHeader(self):
# Define spacing and header
maxNameHLT = 0
maxNameL1 = 0
if len(self.usableHLTTriggers)>0 or len(self.otherHLTTriggers)>0:
maxNameHLT = max([len(trigger) for trigger in self.usableHLTTriggers+self.otherHLTTriggers])
if len(self.usableL1Triggers)>0 or len(self.otherL1Triggers)>0:
maxNameL1 = max([len(trigger) for trigger in self.usableL1Triggers+self.otherL1Triggers])
# Make the name spacing at least 90
maxName = max([maxNameHLT, maxNameL1, 90])
self.spacing = [maxName + 5, 14, 14, 14, 14, 14, 0]
self.spacing[5] = max( [ 181 - sum(self.spacing), 0 ] )
self.header = "" # The table header
self.header += stringSegment("* TRIGGER NAME", self.spacing[0])
if (self.displayRawRates): self.header += stringSegment("* RAW [Hz]", self.spacing[1])
else: self.header += stringSegment("* ACTUAL [Hz]", self.spacing[1])
self.header += stringSegment("* EXPECTED", self.spacing[2])
self.header += stringSegment("* % DIFF", self.spacing[3])
self.header += stringSegment("* DEVIATION", self.spacing[4])
self.header += stringSegment("* AVE PS", self.spacing[5])
self.header += stringSegment("* COMMENTS", self.spacing[6])
self.hlength = 181 #sum(self.spacing)
# Use: Runs the program
# Returns: (void)
def run(self):
# Load the fit and trigger list
if self.fitFile!="":
inputFit = self.loadFit(self.fitFile)
for triggerName in inputFit:
if triggerName[0:3]=="L1_":
if self.InputFitL1 is None: self.InputFitL1 = {}
self.InputFitL1[stripVersion(triggerName)] = inputFit[triggerName]
elif triggerName[0:4] =="HLT_":
if self.InputFitHLT is None: self.InputFitHLT = {}
self.InputFitHLT[stripVersion(triggerName)] = inputFit[triggerName]
# Sort trigger list into HLT and L1 trigger lists
if self.triggerList!="":
for triggerName in self.triggerList:
if triggerName[0:3]=="L1_":
self.TriggerListL1.append(triggerName)
else:
self.TriggerListHLT.append(triggerName)
# If there aren't preset trigger lists, use all the triggers that we can fit
else:
if not self.InputFitHLT is None: self.TriggerListHLT = self.InputFitHLT.keys()
if not self.InputFitL1 is None: self.TriggerListL1 = self.InputFitL1.keys()
# Get run info: The current run number, if the detector is collecting (?), if the data is good (?), and the trigger mode
if not self.assignedNum:
self.runNumber, _, _, mode = self.parser.getLatestRunInfo()
self.triggerMode = mode[0]
# Info message
print "The current run number is %s." % (self.runNumber)
# If we are observing a single run from the past
##elif not self.simulate:
else:
try:
self.triggerMode = self.parser.getTriggerMode(self.runNumber)[0]
except:
self.triggerMode = "Other"
self.getRates()
self.runLoop()
self.checkTriggers()
return
# If we are simulating a previous run
#if self.simulate:
# self.simulateRun()
# return
# Run as long as we can
self.setMode()
self.redoTList = True
while True:
try:
# Check if we are still in the same run, get trigger mode
self.lastRunNumber = self.runNumber
self.runNumber, _, _, mode = self.parser.getLatestRunInfo()
self.runLoop()
self.checkTriggers()
self.sleepWait()
except KeyboardInterrupt:
print "Quitting. Bye."
break
##def simulateRun(self):
## modTime = 0
## # Get the rates
## self.triggerMode = self.parser.getTriggerMode(self.runNumber)[0]
## # Set the trigger mode
## self.setMode()
## # Get the rates for the entire run
## self.getRates()
## # Find the max LS for that run
## trig = self.Rates.keys()[0]
## self.lastLS = self.currentLS
## maxLS = max(self.Rates[trig].keys())
## # Simulate the run
## self.lastRunNumber = self.runNumber
## self.lastLS = 1
## while self.currentLS < maxLS:
## modTime += 23.3
## self.currentLS += 1
## if modTime > 60 or self.currentLS == maxLS:
## modTime -= 60
## # Print table
## self.runLoop()
## # Check for bad triggers
## self.checkTriggers()
## # We would sleep here if this was an online run
## if not self.quiet: print "Simulating 60 s of sleep..."
## self.lastLS = self.currentLS
## print "End of simulation"
# Use: The main body of the main loop, checks the mode, creates trigger lists, prints table
# Returns: (void)
def runLoop(self):
# Reset counting variable
self.normal = 0
self.bad = 0
# If we have started a new run
if self.lastRunNumber != self.runNumber:
print "Starting a new run: Run %s" % (self.runNumber)
self.lastLS = 1
self.currentLS = 0
# Check what mode we are in
self.setMode()
self.getRates()
self.redoTriggerLists()
# Get Rates: [triggerName][LS] { raw rate, prescale }
##if not self.simulate: self.getRates()
self.getRates()
#Construct (or reconstruct) trigger lists
if self.redoTList:
self.redoTriggerLists()
# Make sure there is info to use
if len(self.HLTRates) == 0 or len(self.L1Rates) == 0:
self.redoTList = True
if len(self.HLTRates) == 0 and len(self.L1Rates) == 0:
print "No new information can be retrieved. Waiting... (There may be no new LS, or run active may be false)"
self.redoTList = True
return
# If we are not simulating a previous run. Otherwise, we already set lastLS and currentLS
##if not self.simulate:
lslist = []
for trig in self.Rates.keys():
if len(self.Rates[trig])>0: lslist.append(max(self.Rates[trig]))
# Update lastLS
self.lastLS = self.currentLS
# Update current LS
if len(lslist)>0: self.currentLS = max(lslist)
if self.useLSRange: # Adjust runs so we only look at those in our range
self.slidingLS = -1 # No sliding LS window
self.lastLS = max( [self.lastLS, self.LSRange[0]] )
self.currentLS = min( [self.currentLS, self.LSRange[1] ] )
# If there are lumisection to show, print info for them
if self.currentLS > self.lastLS:
self.printTable()
self.isUpdating = True
else:
self.isUpdating = False
print "Not enough lumisections. Last LS was %s, current LS is %s. Waiting." % (self.lastLS, self.currentLS)
def setMode(self):
self.sendMailAlerts_dynamic = self.sendMailAlerts_static
try:
self.triggerMode = self.parser.getTriggerMode(self.runNumber)[0]
except:
self.triggerMode = "Other"
if self.triggerMode.find("cosmics") > -1:
self.mode = "cosmics"
elif self.triggerMode.find("circulate") > -1:
self.mode = "circulate"
elif self.triggerMode.find("collisions") > -1:
self.mode = "collisions"
elif self.triggerMode == "MANUAL":
self.mode = "MANUAL"
elif self.triggerMode.find("highrate") > -1:
self.mode = "other"
self.sendMailAlerts_dynamic = False
else: self.mode = "other"
# Use: Remakes the trigger lists
def redoTriggerLists(self):
self.redoTList = False
# Reset the trigger lists
self.usableHLTTriggers = []
self.otherHLTTriggers = []
self.usableL1Triggers = []
self.otherL1Triggers = []
self.fullL1HLTMenu = []
# Reset bad rate records
self.badRates = {} # A dictionary: [ trigger name ] { num consecutive bad, trigger bad last check, rate, expected, dev }
self.recordAllBadRates = {} # A dictionary: [ trigger name ] < total times the trigger was bad >
#set trigger lists automatically based on mode
if not self.useAll and not self.userSpecTrigList:
if self.mode == "cosmics" or self.mode == "circulate":
self.triggerList = self.loadTriggersFromFile(self.cosmics_triggerList)
print "monitoring triggers in: ", self.cosmics_triggerList
elif self.mode == "collisions":
self.triggerList = self.loadTriggersFromFile(self.collisions_triggerList)
print "monitoring triggers in: ", self.collisions_triggerList
else:
self.triggerList = ""
print "No lists to monitor: trigger mode not recognized"
self.TriggerListL1 = []
self.TriggerListHLT = []
for triggerName in self.triggerList:
if triggerName[0:3]=="L1_":
self.TriggerListL1.append(triggerName)
elif triggerName[0:4]=="HLT_":
self.TriggerListHLT.append(triggerName)
# Re-make trigger lists
for trigger in self.HLTRates.keys():
if (not self.InputFitHLT is None and self.InputFitHLT.has_key(trigger)) and \
(len(self.TriggerListHLT) !=0 and trigger in self.TriggerListHLT):
self.usableHLTTriggers.append(trigger)
elif trigger[0:4] == "HLT_" and (self.triggerList == "" or trigger in self.TriggerListHLT):
self.otherHLTTriggers.append(trigger)
elif (trigger[0:4] == "HLT_"): self.fullL1HLTMenu.append(trigger)
for trigger in self.L1Rates.keys():
if (not self.InputFitL1 is None and self.InputFitL1.has_key(trigger)) and \
(len(self.TriggerListL1) != 0 and trigger in self.TriggerListL1):
self.usableL1Triggers.append(trigger)
elif trigger[0:3] == "L1_" and (self.triggerList =="" or trigger in self.TriggerListL1):
self.otherL1Triggers.append(trigger)
elif (trigger[0:3] == "L1_"): self.fullL1HLTMenu.append(trigger)
self.getHeader()
# Use: Gets the rates for the lumisections we want
def getRates(self):
if not self.useLSRange:
self.HLTRates = self.parser.getRawRates(self.runNumber, self.lastLS)
self.L1Rates = self.parser.getL1RawRates(self.runNumber)
self.streamData = self.parser.getStreamData(self.runNumber, self.lastLS)
self.pdData = self.parser.getPrimaryDatasets(self.runNumber, self.lastLS)
else:
self.HLTRates = self.parser.getRawRates(self.runNumber, self.LSRange[0], self.LSRange[1])
self.L1Rates = self.parser.getL1RawRates(self.runNumber)
self.streamData = self.parser.getStreamData(self.runNumber, self.LSRange[0], self.LSRange[1])
self.pdData = self.parser.getPrimaryDatasets(self.runNumber, self.LSRange[0], self.LSRange[1])
self.totalStreams = len(self.streamData.keys())
self.Rates = {}
self.Rates.update(self.HLTRates)
self.Rates.update(self.L1Rates)
self.totalHLTTriggers = len(self.HLTRates.keys())
self.totalL1Triggers = len(self.L1Rates.keys())
# Use: Retrieves information and prints it in table form
def printTable(self):
if self.slidingLS == -1:
self.startLS = self.lastLS
else: self.startLS = max( [0, self.currentLS-self.slidingLS ] )+1
# Reset variable
self.normal = 0
self.bad = 0
PScol = -1
# Get the inst lumi
aveLumi = 0
try:
self.deadTimeData = self.parser.getDeadTime(self.runNumber)
aveDeadTime = 0
except:
self.deadTimeData = {}
aveDeadTime = None
print "Error getting deadtime data"
# Get total L1 rate
l1rate = 0
try:
l1rateData = self.parser.getL1rate(self.runNumber)
aveL1rate = 0
except:
l1rateData = {}
aveL1rate = None
print "Error getting total L1 rate data"
physicsActive = False # True if we have at least 1 LS with lumi and physics bit true
if self.mode != "cosmics":
lumiData = self.parser.getLumiInfo(self.runNumber, self.startLS, self.currentLS)
self.numBunches = self.parser.getNumberCollidingBunches(self.runNumber)
# Find the average lumi since we last checked
count = 0
# Get luminosity (only for non-cosmic runs)
for LS, instLumi, psi, physics, all_subSys_good in lumiData:
# If we are watching a certain range, throw out other LS
if self.useLSRange and (LS < self.LSRange[0] or LS > self.LSRange[1]): continue
# Average our instLumi
if not instLumi is None and physics:
physicsActive = True
PScol = psi
if not aveDeadTime is None and self.deadTimeData.has_key(LS): aveDeadTime += self.deadTimeData[LS]
else: aveDeadTime = 0
if not aveL1rate is None and l1rateData.has_key(LS): aveL1rate += l1rateData[LS]
else: aveL1rate = 0
aveLumi += instLumi
count += 1
if count == 0:
aveLumi = "NONE"
expected = "NONE"
else:
aveLumi /= float(count)
aveDeadTime /= float(count)
aveL1rate /= float(count)
else:
count = 0
for LS in l1rateData.keys():
if self.useLSRange and (LS < self.LSRange[0] or LS > self.LSRange[1]): continue
if not aveDeadTime is None and self.deadTimeData.has_key(LS): aveDeadTime += self.deadTimeData[LS]
else: aveDeadTime = 0
if not aveL1rate is None and l1rateData.has_key(LS): aveL1rate += l1rateData[LS]
else: aveL1rate = 0
if not count == 0:
aveDeadTime /= float(count)
aveL1rate /= float(count)
self.lumi_ave = aveLumi
if self.numBunches[0] > 0 and not aveLumi == "NONE":
self.pu_ave = aveLumi/self.numBunches[0]*ppInelXsec/orbitsPerSec
else:
self.pu_ave = "NONE"
# We only do predictions when there were physics active LS in a collisions run
doPred = physicsActive and self.mode=="collisions"
# Print the header
self.printHeader()
# Print the triggers that we can make predictions for
anytriggers = False
if len(self.usableHLTTriggers)>0:
print '*' * self.hlength
print "Predictable HLT Triggers (ones we have a fit for)"
print '*' * self.hlength
anytriggers = True
self.L1 = False
self.printTableSection(self.usableHLTTriggers, doPred, aveLumi)
if len(self.usableL1Triggers)>0:
print '*' * self.hlength
print "Predictable L1 Triggers (ones we have a fit for)"
print '*' * self.hlength
anytriggers = True
self.L1 = True
self.printTableSection(self.usableL1Triggers, doPred, aveLumi)
#check the full menu for paths deviating past thresholds
fullMenu_fits = False
for trigger in self.fullL1HLTMenu: self.getTriggerData(trigger, fullMenu_fits, aveLumi)
# Print the triggers that we can't make predictions for
if self.useAll or self.mode != "collisions" or self.InputFitHLT is None:
print '*' * self.hlength
print "Unpredictable HLT Triggers (ones we have no fit for or do not try to fit)"
print '*' * self.hlength
self.L1 = False
self.printTableSection(self.otherHLTTriggers, False)
self.printTableSection(self.otherL1Triggers, False)
anytriggers = True
if self.useL1:
print '*' * self.hlength
print "Unpredictable L1 Triggers (ones we have no fit for or do not try to fit)"
print '*' * self.hlength
self.L1 = True
self.printTableSection(self.otherL1Triggers, False)
anytriggers = True
if not anytriggers:
print '*' * self.hlength
print "\n --- No useable triggers --- \n"
# Print stream data
if self.showStreams:
print '*' * self.hlength
streamSpacing = [ 50, 20, 25, 25, 25, 25 ]
head = stringSegment("* Stream name", streamSpacing[0])
head += stringSegment("* NLumis", streamSpacing[1])
head += stringSegment("* Events", streamSpacing[2])
head += stringSegment("* Stream rate [Hz]", streamSpacing[3])
head += stringSegment("* File size [GB]", streamSpacing[4])
head += stringSegment("* Stream bandwidth [GB/s]", streamSpacing[5])
print head
print '*' * self.hlength
for name in sorted(self.streamData.keys()):
count = 0
streamsize = 0
aveBandwidth = 0
aveRate = 0
for LS, rate, size, bandwidth in self.streamData[name]:
streamsize += size
aveRate += rate
aveBandwidth += bandwidth
count += 1
if count > 0:
aveRate /= count
streamsize /= (count*1000000000.0)
aveBandwidth /= (count*1000000000.0)
row = stringSegment("* "+name, streamSpacing[0])
row += stringSegment("* "+str(int(count)), streamSpacing[1])
row += stringSegment("* "+str(int(aveRate*23.3*count)), streamSpacing[2])
row += stringSegment("* "+"{0:.2f}".format(aveRate), streamSpacing[3])
row += stringSegment("* "+"{0:.2f}".format(streamsize), streamSpacing[4])
row += stringSegment("* "+"{0:.5f}".format(aveBandwidth), streamSpacing[5])
if not self.noColors and aveRate > self.maxStreamRate and self.mode != "other": write(bcolors.WARNING) # Write colored text
print row
if not self.noColors and aveRate > self.maxStreamRate and self.mode != "other": write(bcolors.ENDC) # Stop writing colored text
else: pass
# Print PD data
if self.showPDs:
print '*' * self.hlength
pdSpacing = [ 50, 20, 25, 25]
head = stringSegment("* Primary Dataset name", pdSpacing[0])
head += stringSegment("* NLumis", pdSpacing[1])
head += stringSegment("* Events", pdSpacing[2])
head += stringSegment("* Dataset rate [Hz]", pdSpacing[3])
print head
print '*' * self.hlength
for name in self.pdData.keys():
count = 0
aveRate = 0
for LS, rate in self.pdData[name]:
aveRate += rate
count += 1
if count > 0:
aveRate /= count
row = stringSegment("* "+name, pdSpacing[0])
row += stringSegment("* "+str(int(count)), pdSpacing[1])
row += stringSegment("* "+str(int(aveRate*23.3*count)), pdSpacing[2])
row += stringSegment("* "+"{0:.2f}".format(aveRate), pdSpacing[3])
if not self.noColors and aveRate > self.maxPDRate and self.mode != "other": write(bcolors.WARNING) # Write colored text
print row
if not self.noColors and aveRate > self.maxPDRate and self.mode != "other": write(bcolors.ENDC) # Stop writing colored text
else: pass
# Closing information
print '*' * self.hlength
print "SUMMARY:"
if self.mode=="collisions": print "Triggers in Normal Range: %s | Triggers outside Normal Range: %s" % (self.normal, self.bad)
if self.mode=="collisions":
print "Prescale column index:",
if PScol == 0:
if not self.noColors and PScol == 0 and self.mode != "other": write(bcolors.WARNING) # Write colored text
print PScol, "\t0 - Column 0 is an emergency column in collision mode, please select the proper column"
if not self.noColors and PScol == 0 and self.mode != "other": write(bcolors.ENDC) # Stop writing colored text
else:
print PScol
try:
print "Average inst. lumi: %.0f x 10^30 cm-2 s-1" % (aveLumi)
except:
print "Average inst. lumi: Not available"
print "Total L1 rate: %.0f Hz" % (aveL1rate)
print "Average dead time: %.2f %%" % (aveDeadTime)
try:
print "Average PU: %.2f" % (self.pu_ave)
except:
print "Average PU: %s" % (self.pu_ave)
print '*' * self.hlength
# Use: Prints the table header
def printHeader(self):
print "\n\n", '*' * self.hlength
print "INFORMATION:"
print "Run Number: %s" % (self.runNumber)
print "LS Range: %s - %s" % (self.startLS, self.currentLS)
print "Latest LHC Status: %s" % self.parser.getLHCStatus()[1]
print "Number of colliding bunches: %s" % self.numBunches[0]
print "Trigger Mode: %s (%s)" % (self.triggerMode, self.mode)
print "Number of HLT Triggers: %s \nNumber of L1 Triggers: %s" % (self.totalHLTTriggers, self.totalL1Triggers)
print "Number of streams:", self.totalStreams
print '*' * self.hlength
print self.header
# Use: Prints a section of a table, ie all the triggers in a trigger list (like usableHLTTriggers, otherHLTTriggers, etc)
def printTableSection(self, triggerList, doPred, aveLumi=0):
self.tableData = [] # A list of tuples, each a row in the table: ( { trigger, rate, predicted rate, sign of % diff, abs % diff, sign of sigma, abs sigma, ave PS, comment } )
# Get the trigger data
for trigger in triggerList: self.getTriggerData(trigger, doPred, aveLumi)
# Sort by % diff if need be
if doPred:
# [4] is % diff, [6] is deviation
if self.usePerDiff: self.tableData.sort(key=lambda tup : tup[4], reverse = True)
else: self.tableData.sort(key=lambda tup : tup[6], reverse = True)
elif self.sortRates:
self.tableData.sort(key=lambda tup: tup[1], reverse = True)
for trigger, rate, pred, sign, perdiff, dsign, dev, avePS, comment in self.tableData:
info = stringSegment("* "+trigger, self.spacing[0])
info += stringSegment("* "+"{0:.2f}".format(rate), self.spacing[1])
if pred!="": info += stringSegment("* "+"{0:.2f}".format(pred), self.spacing[2])
else: info += stringSegment("", self.spacing[2])
if perdiff=="": info += stringSegment("", self.spacing[3])
elif perdiff=="INF": info += stringSegment("* INF", self.spacing[3])
else: info += stringSegment("* "+"{0:.2f}".format(sign*perdiff), self.spacing[3])
if dev=="": info += stringSegment("", self.spacing[4])
elif dev=="INF" or dev==">1E6": info += stringSegment("* "+dev, self.spacing[4])
else: info += stringSegment("* "+"{0:.2f}".format(dsign*dev), self.spacing[4])
info += stringSegment("* "+"{0:.2f}".format(avePS), self.spacing[5])
info += stringSegment("* "+comment, self.spacing[6])
# Color the bad triggers with warning colors
if avePS != 0 and self.isBadTrigger(perdiff, dev, rate, trigger[0:3]=="L1_"):
if not self.noColors and self.mode != "other": write(bcolors.WARNING) # Write colored text
print info
if not self.noColors and self.mode != "other": write(bcolors.ENDC) # Stop writing colored text
# Don't color normal triggers
else:
print info
# Use: Returns whether a given trigger is bad
# Returns: Whether the trigger is bad
def isBadTrigger(self, perdiff, dev, psrate, isL1):
if psrate == 0: return False
if self.mode == "other": return False
if ( (self.usePerDiff and perdiff!="INF" and perdiff!="" and abs(perdiff)>self.percAccept) or (dev!="INF" and dev!="" and (dev==">1E6" or abs(dev)>self.devAccept)))\
or (perdiff!="INF" and perdiff!="" and abs(perdiff)>self.percAccept and dev!="INF" and dev!="" and abs(dev)>self.devAccept)\
or (isL1 and psrate>self.maxL1Rate)\
or (not isL1 and psrate>self.maxHLTRate): return True
return False
# Use: Gets a row of the table, self.tableData: ( { trigger, rate, predicted rate, sign of % diff, abs % diff, ave PS, comment } )
# Parameters:
# -- trigger : The name of the trigger
# -- doPred : Whether we want to make a prediction for this trigger
# -- aveLumi : The average luminosity during the LS in question
# Returns: (void)
def getTriggerData(self, trigger, doPred, aveLumi):
# In case of critical error (this shouldn't occur)
if not self.Rates.has_key(trigger): return
# If cosmics, don't do predictions
if self.mode == "cosmics": doPred = False
# Calculate rate
if self.mode != "cosmics" and doPred:
if not aveLumi is None:
expected = self.calculateRate(trigger, aveLumi)
if expected<0: expected = 0 # Don't let expected value be negative
avePSExpected = expected
# Get the mean square error (standard deviation)
mse = self.getMSE(trigger)
else:
expected = None
avePSExpected = None
mse = None
# Find the ave rate since the last time we checked
aveRate = 0
properAvePSRate = 0
avePS = 0
aveDeadTime = 0
count = 0
comment = ""
correct_for_deadtime = self.deadTimeCorrection
if trigger[0:3]=="L1_": correct_for_deadtime = False
for LS in self.Rates[trigger].keys():
if self.useLSRange and (LS < self.LSRange[0] or LS > self.LSRange[1]): continue
elif LS < self.startLS or LS > self.currentLS: continue
prescale = self.Rates[trigger][LS][1]
rate = self.Rates[trigger][LS][0]
try:
deadTime = self.deadTimeData[LS]
except:
print "trouble getting deadtime for LS: ", LS," setting DT to zero"
deadTime = 0
if correct_for_deadtime: rate *= 1. + (deadTime/100.)
if prescale > 0: properAvePSRate += rate/prescale
else: properAvePSRate += rate
aveRate += rate
count += 1
avePS += prescale
aveDeadTime += deadTime
if count > 0:
if aveRate == 0: comment += "0 counts "
aveRate /= count
properAvePSRate /= count
avePS /= count
aveDeadTime /= count
else:
#comment += "PS=0"
comment += "No rate yet "
doPred = False
if doPred and not avePSExpected is None and avePS > 1: avePSExpected /= avePS
if not doPred and self.removeZeros and aveRate==0: return # Returns if we are not making predictions for this trigger and we are throwing zeros
# We want this trigger to be in the table
row = [trigger]
if self.displayRawRates:
row.append(aveRate)
else:
row.append(properAvePSRate)
if doPred and not expected is None: row.append(avePSExpected)
else: row.append("") # No predicted rate
# Find the % diff
if doPred:
if expected == "NONE":
perc = "UNDEF"
dev = "UNDEF"
row.append(1) # Sign of % diff
row.append(perc) # abs % diff
row.append(1) # Sign of deviation
row.append(dev) # abs deviation
else:
diff = aveRate-expected
if expected!=0: perc = 100*diff/expected
else: perc = "INF"
if mse!=0:
dev = diff/mse
if abs(dev)>1000000: dev = ">1E6"
else: dev = "INF"
if perc>0: sign=1
else: sign=-1
row.append(sign) # Sign of % diff
if perc!="INF": row.append(abs(perc)) # abs % diff
else: row.append("INF")
#if mse>0: sign=1
#else: sign=-1
row.append(sign) # Sign of the deviation
if dev!="INF" and dev!=">1E6":
row.append(abs(dev)) # abs deviation
else: row.append(dev)
else:
row.append("") # No prediction, so no sign of a % diff
row.append("") # No prediction, so no % diff
row.append("") # No prediction, so no sign of deviation
row.append("") # No prediction, so no deviation
# Add the rest of the info to the row
row.append(avePS)
row.append(comment)
# Add row to the table data
if doPred:
if expected > 0:
self.tableData.append(row)
else:
self.tableData.append(row)
#do not warn on specific triggers
for vetoString in self.ignoreStrings:
if trigger.find(vetoString) > -1: return
# Check if the trigger is bad
if doPred:
# Check for bad rates.
#if (self.usePerDiff and perc!="INF" and perc>self.percAccept) or \
#(not self.usePerDiff and dev!="INF" and (dev==">1E6" or dev>self.devAccept)):
if self.isBadTrigger(perc, dev, properAvePSRate, trigger[0:3]=="L1_"):
self.bad += 1
# Record if a trigger was bad
if not self.recordAllBadRates.has_key(trigger):
self.recordAllBadRates[trigger] = 0
self.recordAllBadRates[trigger] += 1
# Record consecutive bad rates
if not self.badRates.has_key(trigger):
self.badRates[trigger] = [1, True, properAvePSRate, avePSExpected, dev, avePS ]
else:
last = self.badRates[trigger]
self.badRates[trigger] = [ last[0]+1, True, properAvePSRate, avePSExpected, dev, avePS ]
else:
self.normal += 1
# Remove warning from badRates
if self.badRates.has_key(trigger): del self.badRates[trigger]
else:
if self.isBadTrigger("", "", properAvePSRate, trigger[0:3]=="L1_") and avePS > 0:
self.bad += 1
# Record if a trigger was bad
if not self.recordAllBadRates.has_key(trigger):
self.recordAllBadRates[trigger] = 0
self.recordAllBadRates[trigger] += 1
# Record consecutive bad rates
if not self.badRates.has_key(trigger):
self.badRates[trigger] = [ 1, True, properAvePSRate, -999, -999, -999 ]
else:
last = self.badRates[trigger]
self.badRates[trigger] = [ last[0]+1, True, properAvePSRate, -999, -999, -999 ]
else:
self.normal += 1
# Remove warning from badRates
if self.badRates.has_key(trigger): del self.badRates[trigger]
# Use: Checks triggers to make sure none have been bad for to long
def checkTriggers(self):
if self.displayBadRates != 0:
count = 0
if self.displayBadRates != -1: write("First %s triggers that are bad: " % (self.displayBadRates))
elif len(self.badRates) > 0 : write("All triggers deviating past thresholds from fit and/or L1 rate > %s Hz, HLT rate > %s Hz: " %(self.maxL1Rate,self.maxHLTRate))
for trigger in self.badRates:
if self.badRates[trigger][1]:
count += 1
write(trigger)
if count != self.displayBadRates-1:
write(", ")
if count == self.displayBadRates:
write(".....")
break
print ""
# Print warnings for triggers that have been repeatedly misbehaving
mailTriggers = [] # A list of triggers that we should mail alerts about
for trigger in self.badRates:
if self.badRates[trigger][1]:
if self.badRates[trigger][0] >= 1:
print "Trigger %s has been out of line for more than %.1f minutes" % (trigger, float(self.badRates[trigger][0])*self.scale_sleeptime)
# We want to mail an alert whenever a trigger exits the acceptable threshold envelope
if self.badRates[trigger][0] == self.maxCBR:
mailTriggers.append( [ trigger, self.badRates[trigger][2], self.badRates[trigger][3], self.badRates[trigger][4], self.badRates[trigger][5] ] )
# Send mail alerts
if len(mailTriggers)>0 and self.isUpdating:
if self.sendMailAlerts_static and self.sendMailAlerts_dynamic: self.sendMail(mailTriggers)
if self.sendAudioAlerts: audioAlert()
# Use: Sleeps and prints out waiting dots
def sleepWait(self):
if not self.quiet: print "Sleeping for %.1f sec before next query" % (60.0*self.scale_sleeptime)
for iSleep in range(20):
if not self.quiet: write(".")
sys.stdout.flush()
time.sleep(3.0*self.scale_sleeptime)
sys.stdout.flush()
print ""
# Use: Loads the fit data from the fit file
# Parameters:
# -- fitFile: The file that the fit data is stored in (a pickle file)
# Returns: The input fit data
def loadFit(self, fileName):
if fileName == "":
return None
InputFit = {} # Initialize InputFit (as an empty dictionary)
# Try to open the file containing the fit info
try:
pkl_file = open(fileName, 'rb')
InputFit = pickle.load(pkl_file)
pkl_file.close()
except:
# File failed to open
print "Error: could not open fit file: %s" % (fileName)
return InputFit
# Use: Calculates the expected rate for a trigger at a given ilumi based on our input fit
def calculateRate(self, triggerName, ilum):
# Make sure we have a fit for the trigger
if not self.L1 and (self.InputFitHLT is None or not self.InputFitHLT.has_key(triggerName)):
return 0
elif self.L1 and ((self.InputFitL1 is None) or not self.InputFitL1.has_key(triggerName)):
return 0
# Get the param list
if self.L1: paramlist = self.InputFitL1[triggerName]
else: paramlist = self.InputFitHLT[triggerName]
# Calculate the rate
if paramlist[0]=="exp": funcStr = "%s + %s*expo(%s+%s*x)" % (paramlist[1], paramlist[2], paramlist[3], paramlist[4]) # Exponential
else: funcStr = "%s+x*(%s+ x*(%s+x*%s))" % (paramlist[1], paramlist[2], paramlist[3], paramlist[4]) # Polynomial
fitFunc = TF1("Fit_"+triggerName, funcStr)
if self.pileUp:
if self.numBunches[0] > 0:
return self.numBunches[0]*fitFunc.Eval(ilum/self.numBunches[0]*ppInelXsec/orbitsPerSec)
else:
return 0
return fitFunc.Eval(ilum)
# Use: Gets the MSE of the fit
def getMSE(self, triggerName):
if not self.L1 and (self.InputFitHLT is None or not self.InputFitHLT.has_key(triggerName)):
return 0
elif self.L1 and ((self.InputFitL1 is None) or not self.InputFitL1.has_key(triggerName)):
return 0
if self.L1: paramlist = self.InputFitL1[triggerName]
else: paramlist = self.InputFitHLT[triggerName]
if self.pileUp:
return self.numBunches[0]*paramlist[5]
return paramlist[5] # The MSE
# Use: Sends an email alert
# Parameters:
# -- mailTriggers: A list of triggers that we should include in the mail, ( { triggerName, aveRate, expected rate, standard dev } )
# Returns: (void)
def sendMail(self, mailTriggers):
mail = "Run: %d, Lumisections: %s - %s \n" % (self.runNumber, self.lastLS, self.currentLS)
try: mail += "Average inst. lumi: %.0f x 10^30 cm-2 s-1\n" % (self.lumi_ave)
except: mail += "Average inst. lumi: %s x 10^30 cm-2 s-1\n" % (self.lumi_ave)
try: mail += "Average PU: %.2f\n \n" % (self.pu_ave)
except: mail += "Average PU: %s\n \n" % (self.pu_ave)
mail += "Trigger rates deviating from acceptable and/or expected values: \n\n"
for triggerName, rate, expected, dev, ps in mailTriggers:
if self.numBunches[0] == 0:
mail += "\n %s: Actual: %s Hz\n" % (stringSegment(triggerName, 35), rate)
else:
if expected > 0:
try: mail += "\n %s: Expected: %.1f Hz, Actual: %.1f Hz, Unprescaled Expected/nBunches: %.5f Hz, Unprescaled Actual/nBunches: %.5f Hz, Deviation: %.1f\n" % (stringSegment(triggerName, 35), expected, rate, expected*ps/self.numBunches[0], rate*ps/self.numBunches[0], dev)
except: mail += "\n %s: Expected: %s Hz, Actual: %s Hz, Unprescaled Expected/nBunches: %s Hz, Unprescaled Actual/nBunches: %s Hz, Deviation: %s\n" % (stringSegment(triggerName, 35), expected, rate, expected*ps/self.numBunches[0], rate*ps/self.numBunches[0], dev)
mail += " *referenced fit: <https://raw.githubusercontent.com/cms-tsg-fog/RateMon/master/Fits/2016/plots/%s.png>\n" % (triggerName)
else:
try: mail += "\n %s: Actual: %.1f Hz\n" % (stringSegment(triggerName, 35), rate)
except: mail += "\n %s: Actual: %s Hz\n" % (stringSegment(triggerName, 35), rate)
try:
wbm_url = self.parser.getWbmUrl(self.runNumber,triggerName,self.currentLS)
if not wbm_url == "-": mail += " *WBM rate: <%s>\n" % (wbm_url)
except:
print "WBM plot url query failed"
mail += "\nWBM Run Summary: <https://cmswbm.web.cern.ch/cmswbm/cmsdb/servlet/RunSummary?RUN=%s> \n\n" % (self.runNumber)
mail += "Email warnings triggered when: \n"
mail += " - L1 or HLT rates deviate by more than %s standard deviations from fit \n" % (self.devAccept)
mail += " - HLT rates > %s Hz \n" % (self.maxHLTRate)
mail += " - L1 rates > %s Hz \n" % (self.maxL1Rate)
print "--- SENDING MAIL ---\n"+mail+"\n--------------------"
mailAlert(mail)
class ShiftMonitor:
def __init__(self):
# Suppress root warnings
ROOT.gErrorIgnoreLevel = 7000
# Fits and fit files
self.fitFile = "Fits/2016/FOG.pkl" # The fit file, can contain both HLT and L1 triggers
self.InputFitHLT = None # The fit information for the HLT triggers
self.InputFitL1 = None # The fit information for the L1 triggers
# DBParser
self.parser = DBParser() # A database parser
# Rates
self.HLTRates = None # HLT rates
self.L1Rates = None # L1 rates
self.Rates = None # Combined L1 and HLT rates
self.deadTimeData = {} # initializing deadTime dict
# Run control
self.runNumber = -1 # The number of the current run
self.numBunches = [-1, -1] # Number of [target, colliding] bunches
# Lumisection control
self.lastLS = 1 # The last LS that was processed last segment
self.currentLS = 1 # The latest LS written to the DB
self.slidingLS = -1 # The number of LS to average over, use -1 for no sliding LS
self.useLSRange = False # Only look at LS in a certain range
# Mode
self.triggerMode = None # The trigger mode
self.mode = None # Mode: cosmics, circulate, physics
# Columns header
self.displayRawRates = False # display raw rates, to display prescaled rates, set = True
self.pileUp = True # derive expected rate as a function of the pileUp, and not the luminosity
# Triggers
self.cosmics_triggerList = "monitorlist_COSMICS.list" #default list used when in cosmics mode
self.collisions_triggerList = "monitorlist_COLLISIONS.list" #default list used when in collision mode
self.triggerList = "" # A list of all the L1 and HLT triggers we want to monitor
self.userSpecTrigList = False # User specified trigger list
self.usableHLTTriggers = [] # HLT Triggers active during the run that we have fits for (and are in the HLT trigger list if it exists)
self.otherHLTTriggers = [] # HLT Triggers active during the run that are not usable triggers
self.usableL1Triggers = [] # L1 Triggers active during the run that have fits for (and are in the L1 trigger list if it exists)
self.otherL1Triggers = [] # L1 Triggers active during that run that are not usable triggers
self.redoTList = True # Whether we need to update the trigger lists
self.useAll = False # If true, we will print out the rates for all the HLT triggers
self.useL1 = False # If true, we will print out the rates for all the L1 triggers
self.totalHLTTriggers = 0 # The total number of HLT Triggers on the menu this run
self.totalL1Triggers = 0 # The total number of L1 Triggers on the menu this run
self.fullL1HLTMenu = []
self.ignoreStrings = ["Calibration","L1Tech"]
# Restrictions
self.removeZeros = False # If true, we don't show triggers that have zero rate
self.requireLumi = False # If true, we only display tables when aveLumi is not None
# Trigger behavior
self.percAccept = 50.0 # The acceptence for % diff
self.devAccept = 5 # The acceptance for deviation
self.badRates = {} # A dictionary: [ trigger name ] { num consecutive bad , whether the trigger was bad last time we checked, rate, expected, dev }
self.recordAllBadTriggers = {} # A dictionary: [ trigger name ] < total times the trigger was bad >
self.maxCBR = 3 # The maximum consecutive db queries a trigger is allowed to deviate from prediction by specified amount before it's printed out
self.displayBadRates = -1 # The number of bad rates we should show in the summary. We use -1 for all
self.usePerDiff = False # Whether we should identify bad triggers by perc diff or deviatoin
self.sortRates = True # Whether we should sort triggers by their rates
self.maxHLTRate = 500 # The maximum prescaled rate we allow an HLT Trigger to have
self.maxL1Rate = 30000 # The maximum prescaled rate we allow an L1 Trigger to have
# Other options
self.quiet = False # Prints fewer messages in this mode
self.noColors = False # Special formatting for if we want to dump the table to a file
self.sendMailAlerts_static = True # Whether we should send alert mails
self.sendMailAlerts_dynamic = self.sendMailAlerts_static
self.sendAudioAlerts = False # Whether we should send audio warning messages in the control room (CAUTION)
self.isUpdating = True # flag to determine whether or not we're receiving new LS
self.showStreams = False # Whether we should print stream information
self.showPDs = False # Whether we should print pd information
self.totalStreams = 0 # The total number of streams
self.maxStreamRate = 1000000 # The maximum rate we allow a "good" stream to have
self.maxPDRate = 10000000 # The maximum rate we allow a "good" pd to have
self.lumi_ave = "NONE"
self.pu_ave = "NONE"
self.deadTimeCorrection = True # correct the rates for dead time
self.scale_sleeptime = 1.0 # Scales the length of time to wait before sending another query (1.0 = 60sec, 2.0 = 120sec, etc)
self.badTriggers = {} #[trigger name] <List of trigger infos>
# Use: Opens a file containing a list of trigger names and adds them to the RateMonitor class's trigger list
# Note: We do not clear the trigger list, this way we could add triggers from multiple files to the trigger list
# -- fileName: The name of the file that trigger names are contained in
# Returns: (void)
def loadTriggersFromFile(self, fileName):
try:
file = open(fileName, 'r')
except:
print "File", fileName, "(a trigger list file) failed to open."
return
allTriggerNames = file.read().split() # Get all the words, no argument -> split on any whitespace
TriggerList = []
for triggerName in allTriggerNames:
# Recognize comments
if triggerName[0]=='#': continue
try:
if not str(triggerName) in TriggerList:
TriggerList.append(stripVersion(str(triggerName)))
except:
print "Error parsing trigger name in file", fileName
return TriggerList
# Use: Formats the header string
def getHeader(self):
# Define spacing and header
maxNameHLT = 0
maxNameL1 = 0
if len(self.usableHLTTriggers)>0 or len(self.otherHLTTriggers)>0:
maxNameHLT = max([len(trigger) for trigger in self.usableHLTTriggers+self.otherHLTTriggers])
if len(self.usableL1Triggers)>0 or len(self.otherL1Triggers)>0:
maxNameL1 = max([len(trigger) for trigger in self.usableL1Triggers+self.otherL1Triggers])
# Make the name spacing at least 90
maxName = max([maxNameHLT, maxNameL1, 90])
self.spacing = [maxName + 5, 14, 14, 14, 14, 14, 0]
self.spacing[5] = max( [ 181 - sum(self.spacing), 0 ] )
self.header = "" # The table header
self.header += stringSegment("* TRIGGER NAME", self.spacing[0])
if (self.displayRawRates): self.header += stringSegment("* RAW [Hz]", self.spacing[1])
else: self.header += stringSegment("* ACTUAL [Hz]", self.spacing[1])
self.header += stringSegment("* EXPECTED", self.spacing[2])
self.header += stringSegment("* % DIFF", self.spacing[3])
self.header += stringSegment("* DEVIATION", self.spacing[4])
self.header += stringSegment("* AVE PS", self.spacing[5])
self.header += stringSegment("* COMMENTS", self.spacing[6])
self.hlength = 181 #sum(self.spacing)
# Use: Runs the program
# Returns: (void)
def run(self):
# Load the fit and trigger list
if self.fitFile != "":
inputFit = self.loadFit(self.fitFile)
for triggerName in inputFit:
if triggerName[0:3] == "L1_":
if self.InputFitL1 is None: self.InputFitL1 = {}
self.InputFitL1[triggerName] = inputFit[triggerName]
elif triggerName[0:4] == "HLT_":
if self.InputFitHLT is None: self.InputFitHLT = {}
self.InputFitHLT[triggerName] = inputFit[triggerName]
# Sort trigger list into HLT and L1 trigger lists
if self.triggerList != "":
for triggerName in self.triggerList:
if triggerName[0:3] == "L1_":
self.TriggerListL1.append(triggerName)
else:
self.TriggerListHLT.append(triggerName)
# If there aren't preset trigger lists, use all the triggers that we can fit
else:
if not self.InputFitHLT is None: self.TriggerListHLT = self.InputFitHLT.keys()
if not self.InputFitL1 is None: self.TriggerListL1 = self.InputFitL1.keys()
while True:
try:
previous_run_number = self.runNumber
#self.updateRunInfo(previous_run_number)
self.runNumber, _, _, mode = self.parser.getLatestRunInfo()
self.triggerMode = mode[0]
## ---Run loop start---
is_new_run = ( previous_run_number != self.runNumber or self.runNumber < 0 )
if is_new_run:
print "Starting a new run: Run %s" % (self.runNumber)
self.lastLS = 1
self.currentLS = 0
self.setMode()
physicsActive,avgLumi,avgDeadTime,avgL1rate,PScol = self.queryDB()
self.redoTriggerLists()
else:
physicsActive,avgLumi,avgDeadTime,avgL1rate,PScol = self.queryDB()
self.lumi_ave = avgLumi
#Construct (or reconstruct) trigger lists
if self.redoTList:
self.redoTriggerLists()
if len(self.HLTRates) == 0 or len(self.L1Rates) == 0:
self.redoTList = True
# Make sure there is info to use
if len(self.HLTRates) == 0 and len(self.L1Rates) == 0:
print "No new information can be retrieved. Waiting... (There may be no new LS, or run active may be false)"
continue
trigger_table = self.getTriggerTable(physicsActive,avgLumi)
self.checkTriggers(avgLumi,trigger_table)
# If there are lumisection to show, print info for them
if self.currentLS > self.lastLS:
self.printTable(trigger_table,physicsActive,avgLumi,avgDeadTime,avgL1rate,PScol)
self.isUpdating = True
else:
self.isUpdating = False
print "Not enough lumisections. Last LS was %s, current LS is %s. Waiting." % (self.lastLS, self.currentLS)
## ---Run loop start---
self.mailSender()
self.sleepWait()
except KeyboardInterrupt:
print "Quitting. Bye."
break
# Queries the Database and updates trigger entries
def queryDB(self):
# Get Rates: [triggerName][LS] { raw rate, prescale }
if not self.useLSRange:
self.HLTRates = self.parser.getRawRates(self.runNumber, self.lastLS)
self.L1Rates = self.parser.getL1RawRates(self.runNumber, self.lastLS)
self.streamData = self.parser.getStreamData(self.runNumber, self.lastLS)
self.pdData = self.parser.getPrimaryDatasets(self.runNumber, self.lastLS)
else:
self.HLTRates = self.parser.getRawRates(self.runNumber, self.LSRange[0], self.LSRange[1])
self.L1Rates = self.parser.getL1RawRates(self.runNumber, self.LSRange[0], self.LSRange[1])
self.streamData = self.parser.getStreamData(self.runNumber, self.LSRange[0], self.LSRange[1])
self.pdData = self.parser.getPrimaryDatasets(self.runNumber, self.LSRange[0], self.LSRange[1])
self.totalStreams = len(self.streamData.keys())
self.Rates = {}
self.Rates.update(self.HLTRates)
self.Rates.update(self.L1Rates)
self.totalHLTTriggers = len(self.HLTRates.keys())
self.totalL1Triggers = len(self.L1Rates.keys())
lslist = []
for trig in self.Rates.keys():
if len(self.Rates[trig]) > 0: lslist.append(max(self.Rates[trig]))
# Update lastLS
self.lastLS = self.currentLS
# Update current LS
if len(lslist) > 0: self.currentLS = max(lslist)
if self.useLSRange: # Adjust runs so we only look at those in our range
self.slidingLS = -1 # No sliding LS window
self.lastLS = max( [self.lastLS, self.LSRange[0]] )
self.currentLS = min( [self.currentLS, self.LSRange[1] ] )
# Get the inst lumi
avgDeadTime = 0
try:
self.deadTimeData = self.parser.getDeadTime(self.runNumber)
avgDeadTime = 0
except:
self.deadTimeData = {}
avgDeadTime = None
print "Error getting deadtime data"
# Get total L1 rate
l1rate = 0
try:
l1rateData = self.parser.getL1rate(self.runNumber)
avgL1rate = 0
except:
l1rateData = {}
avgL1rate = None
print "Error getting total L1 rate data"
self.startLS = self.lastLS
physicsActive = False
avgLumi = 0
PScol = -1
if self.mode != "cosmics":
lumiData = self.parser.getLumiInfo(self.runNumber, self.startLS, self.currentLS)
self.numBunches = self.parser.getNumberCollidingBunches(self.runNumber)
# Find the average lumi since we last checked
count = 0
# Get luminosity (only for non-cosmic runs)
#for LS, instLumi, psi, physics in lumiData:
for LS, instLumi, psi, physics, all_subSys_good in lumiData:
# If we are watching a certain range, throw out other LS
if self.useLSRange and (LS < self.LSRange[0] or LS > self.LSRange[1]): continue
# Average our instLumi
if not instLumi is None and physics:
physicsActive = True
PScol = psi
if not avgDeadTime is None and self.deadTimeData.has_key(LS):
avgDeadTime += self.deadTimeData[LS]
else:
avgDeadTime = 0
if not avgL1rate is None and l1rateData.has_key(LS):
avgL1rate += l1rateData[LS]
else:
avgL1rate = 0
avgLumi += instLumi
count += 1
if count == 0:
avgLumi = None
else:
avgLumi /= float(count)
avgDeadTime /= float(count)
avgL1rate /= float(count)
else:
count = 0
avgLumi = None
for LS in l1rateData.keys():
if not avgDeadTime is None and self.deadTimeData.has_key(LS):
avgDeadTime += self.deadTimeData[LS]
else:
avgDeadTime = 0
if not avgL1rate is None and l1rateData.has_key(LS):
avgL1rate += l1rateData[LS]
else:
avgL1rate = 0
if not count == 0:
avgDeadTime /= float(count)
avgL1rate /= float(count)
if self.numBunches[0] > 0 and not avgLumi == None:
self.pu_ave = avgLumi/self.numBunches[0]*ppInelXsec/orbitsPerSec
else:
self.pu_ave = "NONE"
return physicsActive,avgLumi,avgDeadTime,avgL1rate,PScol
def getTriggerTable(self, physicsActive, avgLumi):
# A list of tuples, each a row in the table:
# ( { trigger, rate, predicted rate, sign of % diff, abs % diff, sign of sigma, abs sigma,
# ave PS, doPred, comment } )
table = {}
#Calculate trigger information
#for trigger in self.triggerList:
tmp_trigger_list = []
tmp_trigger_list += self.HLTRates.keys()
tmp_trigger_list += self.L1Rates.keys()
#for trigger in self.fullL1HLTMenu:
for trigger in tmp_trigger_list:
#for trigger in self.Rates.keys():
perc = 0
dev = 0
aveRate = 0
properAvePSRate = 0
avePS = 0
avePSExpected = 0
count = 0
comment = ""
doPred = physicsActive and self.mode == "collisions"
if not self.Rates.has_key(trigger): continue
# If cosmics, don't do predictions
if self.mode == "cosmics": doPred = False
# Calculate rate
if self.mode != "cosmics" and doPred:
if not avgLumi is None:
if trigger in self.TriggerListL1:
self.L1 = True
else:
self.L1 = False
expected = self.calculateRate(trigger, avgLumi)
if expected < 0: expected = 0 # Don't let expected value be negative
avePSExpected = expected
# Get the mean square error (standard deviation)
mse = self.getMSE(trigger)
else:
expected = None
avePSExpected = None
mse = None
correct_for_deadtime = self.deadTimeCorrection
if trigger[0:3]=="L1_": correct_for_deadtime = False
for LS in self.Rates[trigger].keys():
if LS < self.startLS or LS > self.currentLS: continue
prescale = self.Rates[trigger][LS][1]
rate = self.Rates[trigger][LS][0]
try:
deadTime = self.deadTimeData[LS]
except:
print "trouble getting deadtime for LS: ", LS," setting DT to zero"
deadTime = 0.0
if correct_for_deadtime: rate *= 1. + (deadTime/100.)
if prescale > 0:
properAvePSRate += rate/prescale
else:
properAvePSRate += rate
aveRate += rate
count += 1
avePS += prescale
if count > 0:
if aveRate == 0: comment += "0 counts "
aveRate /= count
properAvePSRate /= count
avePS /= count
else:
#comment += "PS=0"
comment += "No rate yet "
doPred = False
if doPred and not avePSExpected is None and avePS > 1:
avePSExpected /= avePS
# skips if we are not making predictions for this trigger and we are throwing zeros
#if not doPred and self.removeZeros and aveRate == 0:
# continue
if self.displayRawRates:
rate_val = aveRate
else:
rate_val = properAvePSRate
if doPred and not expected is None:
#Do nothing
xkcd = ""
else:
avePSExpected = ""
if doPred:
if expected == None:
perc = "UNDEF"
dev = "UNDEF"
sign = 1
else:
diff = aveRate - expected
if expected != 0:
perc = 100*diff/expected
else:
perc = "INF"
if mse != 0:
dev = diff/mse
if abs(dev)>1000000: dev = ">1E6"
else:
dev = "INF"
if perc > 0:
sign = 1
else:
sign = -1
if perc != "INF":
perc = abs(perc)
if dev != "INF":
dev = abs(dev)
else:
sign = "" # No prediction, so no sign of a % diff
perc = "" # No prediction, so no % diff
dev = "" # No prediction, so no deviation
table[trigger] = [rate_val,avePSExpected,sign,perc,sign,dev,avePS,doPred,comment]
return table
def checkTriggers(self,avgLumi,trigger_table):
# Reset counting variable
self.normal = 0
self.bad = 0
tmp_list = []
for elem in self.triggerList:
if elem in tmp_list:
continue
else:
tmp_list.append(elem)
#Remove any duplicates
for elem in self.fullL1HLTMenu:
if elem in tmp_list:
continue
else:
tmp_list.append(elem)
#Check if the trigger is good or bad
for trigger in tmp_list:
#table[trigger] = [rate_val,avePSExpected,sign,perc,sign,dev,avPS,doPred,comment]
#if not self.Rates.has_key(trigger): continue
if not trigger_table.has_key(trigger):
print "Table missing triggers: %s" % trigger
continue
properAvePSRate = trigger_table[trigger][0]
avePSExpected = trigger_table[trigger][1]
perc = trigger_table[trigger][3]
dev = trigger_table[trigger][5]
avePS = trigger_table[trigger][6]
doPred = trigger_table[trigger][7]
if doPred:
# Check for bad rates.
if self.isBadTrigger(perc, dev, properAvePSRate, trigger[0:3]=="L1_"):
self.bad += 1
# Record if a trigger was bad
if not self.recordAllBadRates.has_key(trigger):
self.recordAllBadRates[trigger] = 0
self.recordAllBadRates[trigger] += 1
# Record consecutive bad rates
if not self.badRates.has_key(trigger):
self.badRates[trigger] = [1, True, properAvePSRate, avePSExpected, dev, avePS ]
else:
last = self.badRates[trigger]
self.badRates[trigger] = [ last[0]+1, True, properAvePSRate, avePSExpected, dev, avePS ]
else:
self.normal += 1
# Remove warning from badRates
if self.badRates.has_key(trigger): del self.badRates[trigger]
else:
if self.isBadTrigger("", "", properAvePSRate, trigger[0:3]=="L1_"):
self.bad += 1
# Record if a trigger was bad
if not self.recordAllBadRates.has_key(trigger):
self.recordAllBadRates[trigger] = 0
self.recordAllBadRates[trigger] += 1
# Record consecutive bad rates
if not self.badRates.has_key(trigger):
self.badRates[trigger] = [ 1, True, properAvePSRate, -999, -999, -999 ]
else:
last = self.badRates[trigger]
self.badRates[trigger] = [ last[0]+1, True, properAvePSRate, -999, -999, -999 ]
else:
self.normal += 1
# Remove warning from badRates
if self.badRates.has_key(trigger): del self.badRates[trigger]
# Use: Prints a section of a table, i.e. all the triggers in a trigger list (like usableHLTTriggers, otherHLTTriggers, etc)
def printTableSection(self, triggerList, trigger_table, hasPred, avgLumi=0):
tmp_dict = {}
# Find the correct triggers
for trigger in trigger_table:
if trigger in triggerList:
tmp_dict[trigger] = trigger_table[trigger]
sorted_list = tmp_dict.items()
#print "Trigger_table: ",trigger_table.keys()
#table --> [trigger]<rate_val,avePSExpected,sign,perc,sign,dev,avPS,doPred,comment>
if hasPred:
if self.usePerDiff:
#Sort by % diff
sorted_list.sort(key=lambda tup: tup[1][3], reverse = True)
else:
#Sort by deviation
sorted_list.sort(key=lambda tup: tup[1][5], reverse = True)
else:
sorted_list.sort(key=lambda tup: tup[1][0], reverse = True)
#for trigger, rate, pred, sign, perdiff, dsign, dev, avePS, comment in self.tableData:
for tup in sorted_list:
trigger = tup[0]
rate = trigger_table[trigger][0]
pred = trigger_table[trigger][1]
sign = trigger_table[trigger][2]
perdiff = trigger_table[trigger][3]
dsign = trigger_table[trigger][4]
dev = trigger_table[trigger][5]
avePS = trigger_table[trigger][6]
doPred = trigger_table[trigger][7]
comment = trigger_table[trigger][8]
info = stringSegment("* "+trigger, self.spacing[0])
info += stringSegment("* "+"{0:.2f}".format(rate), self.spacing[1])
if pred != "":
info += stringSegment("* "+"{0:.2f}".format(pred), self.spacing[2])
else:
info += stringSegment("", self.spacing[2])
if perdiff == "":
info += stringSegment("", self.spacing[3])
elif perdiff == "INF":
info += stringSegment("* INF", self.spacing[3])
else:
info += stringSegment("* "+"{0:.2f}".format(sign*perdiff), self.spacing[3])
if dev == "":
info += stringSegment("", self.spacing[4])
elif dev == "INF" or dev == ">1E6":
info += stringSegment("* "+dev, self.spacing[4])
else:
info += stringSegment("* "+"{0:.2f}".format(dsign*dev), self.spacing[4])
info += stringSegment("* "+"{0:.2f}".format(avePS), self.spacing[5])
info += stringSegment("* "+comment, self.spacing[6])
# Color the bad triggers with warning colors
#if avePS != 0 and self.isBadTrigger(perdiff, dev, rate/avePS, trigger[0:3]=="L1_"):
if avePS != 0 and self.badRates.has_key(trigger):
if not self.noColors: write(bcolors.WARNING) # Write colored text
print info
if not self.noColors: write(bcolors.ENDC) # Stop writing colored text
# Don't color normal triggers
else:
print info
def printTable(self,trigger_table,physicsActive,avgLumi,avgDeadTime,avgL1rate,PScol):
self.printHeader()
hasPred = physicsActive and self.mode == "collisions"
# Print the triggers that we can make predictions for
anytriggers = False
if len(self.usableHLTTriggers) > 0:
print '*' * self.hlength
print "Predictable HLT Triggers (ones we have a fit for)"
print '*' * self.hlength
anytriggers = True
self.L1 = False
self.printTableSection(self.usableHLTTriggers,trigger_table,hasPred,avgLumi)
if len(self.usableL1Triggers) > 0:
print '*' * self.hlength
print "Predictable L1 Triggers (ones we have a fit for)"
print '*' * self.hlength
anytriggers = True
self.L1 = True
self.printTableSection(self.usableL1Triggers,trigger_table,hasPred,avgLumi)
#check the full menu for paths deviating past thresholds
fullMenu_fits = False
#for trigger in self.fullL1HLTMenu: self.getTriggerData(trigger, fullMenu_fits, avgLumi)
# Print the triggers that we can't make predictions for (only for certain modes)
if self.useAll or self.mode != "collisions" or self.InputFitHLT is None:
print '*' * self.hlength
print "Unpredictable HLT Triggers (ones we have no fit for or do not try to fit)"
print '*' * self.hlength
self.L1 = False
self.printTableSection(self.otherHLTTriggers,trigger_table,False)
self.printTableSection(self.otherL1Triggers,trigger_table,False)
anytriggers = True
if self.useL1:
print '*' * self.hlength
print "Unpredictable L1 Triggers (ones we have no fit for or do not try to fit)"
print '*' * self.hlength
self.L1 = True
self.printTableSection(self.otherL1Triggers,trigger_table,False)
anytriggers = True
if not anytriggers:
print '*' * self.hlength
print "\n --- No useable triggers --- \n"
# Print stream data
if self.showStreams:
print '*' * self.hlength
streamSpacing = [ 50, 20, 25, 25, 25, 25 ]
head = stringSegment("* Stream name", streamSpacing[0])
head += stringSegment("* NLumis", streamSpacing[1])
head += stringSegment("* Events", streamSpacing[2])
head += stringSegment("* Stream rate [Hz]", streamSpacing[3])
head += stringSegment("* File size [GB]", streamSpacing[4])
head += stringSegment("* Stream bandwidth [GB/s]", streamSpacing[5])
print head
print '*' * self.hlength
for name in sorted(self.streamData.keys()):
count = 0
streamsize = 0
aveBandwidth = 0
aveRate = 0
for LS, rate, size, bandwidth in self.streamData[name]:
streamsize += size
aveRate += rate
aveBandwidth += bandwidth
count += 1
if count > 0:
aveRate /= count
streamsize /= (count*1000000000.0)
aveBandwidth /= (count*1000000000.0)
row = stringSegment("* "+name, streamSpacing[0])
row += stringSegment("* "+str(int(count)), streamSpacing[1])
row += stringSegment("* "+str(int(aveRate*23.3*count)), streamSpacing[2])
row += stringSegment("* "+"{0:.2f}".format(aveRate), streamSpacing[3])
row += stringSegment("* "+"{0:.2f}".format(streamsize), streamSpacing[4])
row += stringSegment("* "+"{0:.5f}".format(aveBandwidth), streamSpacing[5])
if not self.noColors and aveRate > self.maxStreamRate: write(bcolors.WARNING) # Write colored text
print row
if not self.noColors and aveRate > self.maxStreamRate: write(bcolors.ENDC) # Stop writing colored text
else: pass
# Print PD data
if self.showPDs:
print '*' * self.hlength
pdSpacing = [ 50, 20, 25, 25]
head = stringSegment("* Primary Dataset name", pdSpacing[0])
head += stringSegment("* NLumis", pdSpacing[1])
head += stringSegment("* Events", pdSpacing[2])
head += stringSegment("* Dataset rate [Hz]", pdSpacing[3])
print head
print '*' * self.hlength
for name in self.pdData.keys():
count = 0
aveRate = 0
for LS, rate in self.pdData[name]:
aveRate += rate
count += 1
if count > 0:
aveRate /= count
row = stringSegment("* "+name, pdSpacing[0])
row += stringSegment("* "+str(int(count)), pdSpacing[1])
row += stringSegment("* "+str(int(aveRate*23.3*count)), pdSpacing[2])
row += stringSegment("* "+"{0:.2f}".format(aveRate), pdSpacing[3])
if not self.noColors and aveRate > self.maxPDRate: write(bcolors.WARNING) # Write colored text
print row
if not self.noColors and aveRate > self.maxPDRate: write(bcolors.ENDC) # Stop writing colored text
else: pass
# Closing information
print '*' * self.hlength
print "SUMMARY:"
if self.mode == "collisions": print "Triggers in Normal Range: %s | Triggers outside Normal Range: %s" % (self.normal, self.bad)
if self.mode == "collisions":
print "Prescale column index:",
if PScol == 0:
if not self.noColors and PScol == 0 and self.mode != "other": write(bcolors.WARNING) # Write colored text
print PScol, "\t0 - Column 0 is an emergency column in collision mode, please select the proper column"
if not self.noColors and PScol == 0 and self.mode != "other": write(bcolors.ENDC) # Stop writing colored text
else:
print PScol
try:
print "Average inst. lumi: %.0f x 10^30 cm-2 s-1" % (avgLumi)
except:
print "Average inst. lumi: Not available"
print "Total L1 rate: %.0f Hz" % (avgL1rate)
print "Average dead time: %.2f %%" % (avgDeadTime)
try:
print "Average PU: %.2f" % (self.pu_ave)
except:
print "Average PU: %s" % (self.pu_ave)
print '*' * self.hlength
def updateRunInfo(self, previous_run):
#updates:
#run number
#trigger mode (and trigger lists)
#self.runNumber, self.triggerMode = self.parser.getLatestRunInfo()
self.runNumber, _, _, mode = self.parser.getLatestRunInfo()
self.triggerMode = mode[0]
is_new_run = ( previous_run != self.runNumber or self.runNumber < 0 )
if is_new_run:
print "Starting a new run: Run %s" % (self.runNumber)
self.lastLS = 1
self.currentLS = 0
self.setMode()
self.redoTriggerLists()
def setMode(self):
self.sendMailAlerts_dynamic = self.sendMailAlerts_static
try:
self.triggerMode = self.parser.getTriggerMode(self.runNumber)[0]
except:
self.triggerMode = "Other"
if self.triggerMode.find("cosmics") > -1:
self.mode = "cosmics"
elif self.triggerMode.find("circulate") > -1:
self.mode = "circulate"
elif self.triggerMode.find("collisions") > -1:
self.mode = "collisions"
elif self.triggerMode == "MANUAL":
self.mode = "MANUAL"
elif self.triggerMode.find("highrate") > -1:
self.mode = "other"
#self.maxHLTRate = 100000
#self.maxL1Rate = 100000
else: self.mode = "other"
# Use: Remakes the trigger lists
def redoTriggerLists(self):
self.redoTList = False
# Reset the trigger lists
self.usableHLTTriggers = []
self.otherHLTTriggers = []
self.usableL1Triggers = []
self.otherL1Triggers = []
self.fullL1HLTMenu = []
# Reset bad rate records
self.badRates = {} # A dictionary: [ trigger name ] { num consecutive bad, trigger bad last check, rate, expected, dev }
self.recordAllBadRates = {} # A dictionary: [ trigger name ] < total times the trigger was bad >
#set trigger lists automatically based on mode
if not self.useAll and not self.userSpecTrigList:
if self.mode == "cosmics" or self.mode == "circulate":
self.triggerList = self.loadTriggersFromFile(self.cosmics_triggerList)
print "monitoring triggers in: ", self.cosmics_triggerList
elif self.mode == "collisions":
self.triggerList = self.loadTriggersFromFile(self.collisions_triggerList)
print "monitoring triggers in: ", self.collisions_triggerList
else:
self.triggerList = ""
print "No lists to monitor: trigger mode not recognized"
self.TriggerListL1 = []
self.TriggerListHLT = []
for triggerName in self.triggerList:
if triggerName[0:3]=="L1_":
self.TriggerListL1.append(triggerName)
elif triggerName[0:4]=="HLT_":
self.TriggerListHLT.append(triggerName)
# Re-make trigger lists
#print self.InputFitHLT.keys()
for trigger in self.HLTRates.keys():
if (not self.InputFitHLT is None and self.InputFitHLT.has_key(trigger)) and \
(len(self.TriggerListHLT) !=0 and trigger in self.TriggerListHLT):
self.usableHLTTriggers.append(trigger)
elif trigger[0:4] == "HLT_" and (self.triggerList == "" or trigger in self.TriggerListHLT):
self.otherHLTTriggers.append(trigger)
elif (trigger[0:4] == "HLT_"): self.fullL1HLTMenu.append(trigger)
for trigger in self.L1Rates.keys():
if (not self.InputFitL1 is None and self.InputFitL1.has_key(trigger)) and \
(len(self.TriggerListL1) != 0 and trigger in self.TriggerListL1):
self.usableL1Triggers.append(trigger)
elif trigger[0:3] == "L1_" and (self.triggerList =="" or trigger in self.TriggerListL1):
self.otherL1Triggers.append(trigger)
elif (trigger[0:3] == "L1_"): self.fullL1HLTMenu.append(trigger)
self.getHeader()
# Use: Prints the table header
def printHeader(self):
print "\n\n", '*' * self.hlength
print "INFORMATION:"
print "Run Number: %s" % (self.runNumber)
print "LS Range: %s - %s" % (self.startLS, self.currentLS)
print "Latest LHC Status: %s" % self.parser.getLHCStatus()[1]
print "Number of colliding bunches: %s" % self.numBunches[0]
print "Trigger Mode: %s (%s)" % (self.triggerMode, self.mode)
print "Number of HLT Triggers: %s \nNumber of L1 Triggers: %s" % (self.totalHLTTriggers, self.totalL1Triggers)
print "Number of streams:", self.totalStreams
print '*' * self.hlength
print self.header
# Use: Returns whether a given trigger is bad
# Returns: Whether the trigger is bad
def isBadTrigger(self, perdiff, dev, psrate, isL1):
if psrate == 0.0: return False
if ( (self.usePerDiff and perdiff!="INF" and perdiff!="" and abs(perdiff)>self.percAccept) or (dev!="INF" and dev!="" and (dev==">1E6" or abs(dev)>self.devAccept)))\
or (perdiff!="INF" and perdiff!="" and abs(perdiff)>self.percAccept and dev!="INF" and dev!="" and abs(dev)>self.devAccept)\
or (isL1 and psrate>self.maxL1Rate)\
or (not isL1 and psrate>self.maxHLTRate): return True
return False
def mailSender(self):
if self.displayBadRates != 0:
count = 0
if self.displayBadRates != -1: write("First %s triggers that are bad: " % (self.displayBadRates))
elif len(self.badRates) > 0 : write("All triggers deviating past thresholds from fit and/or L1 rate > %s Hz, HLT rate > %s Hz: " %(self.maxL1Rate,self.maxHLTRate))
for trigger in self.badRates:
if self.badRates[trigger][1]:
count += 1
write(trigger)
if count != self.displayBadRates-1:
write(", ")
if count == self.displayBadRates:
write(".....")
break
print ""
# Print warnings for triggers that have been repeatedly misbehaving
mailTriggers = [] # A list of triggers that we should mail alerts about
for trigger in self.badRates:
if self.badRates[trigger][1]:
if self.badRates[trigger][0] >= 1:
print "Trigger %s has been out of line for more than %.1f minutes" % (trigger, float(self.badRates[trigger][0])*self.scale_sleeptime)
# We want to mail an alert whenever a trigger exits the acceptable threshold envelope
if self.badRates[trigger][0] == self.maxCBR:
mailTriggers.append( [ trigger, self.badRates[trigger][2], self.badRates[trigger][3], self.badRates[trigger][4], self.badRates[trigger][5] ] )
# Send mail alerts
if len(mailTriggers) > 0 and self.isUpdating:
if self.sendMailAlerts_static and self.sendMailAlerts_dynamic: self.sendMail(mailTriggers)
if self.sendAudioAlerts: audioAlert()
# Use: Sleeps and prints out waiting dots
def sleepWait(self):
if not self.quiet: print "Sleeping for %.1f sec before next query" % (60.0*self.scale_sleeptime)
for iSleep in range(20):
if not self.quiet: write(".")
sys.stdout.flush()
time.sleep(3.0*self.scale_sleeptime)
sys.stdout.flush()
print ""
# Use: Loads the fit data from the fit file
# Parameters:
# -- fitFile: The file that the fit data is stored in (a pickle file)
# Returns: The input fit data
def loadFit(self, fileName):
if fileName == "":
return None
InputFit = {} # Initialize InputFit (as an empty dictionary)
# Try to open the file containing the fit info
try:
pkl_file = open(fileName, 'rb')
InputFit = pickle.load(pkl_file)
pkl_file.close()
except:
# File failed to open
print "Error: could not open fit file: %s" % (fileName)
return InputFit
# Use: Calculates the expected rate for a trigger at a given ilumi based on our input fit
def calculateRate(self, triggerName, ilum):
# Make sure we have a fit for the trigger
if not self.L1 and (self.InputFitHLT is None or not self.InputFitHLT.has_key(triggerName)):
return 0
elif self.L1 and ((self.InputFitL1 is None) or not self.InputFitL1.has_key(triggerName)):
return 0
# Get the param list
if self.L1: paramlist = self.InputFitL1[triggerName]
else: paramlist = self.InputFitHLT[triggerName]
# Calculate the rate
if paramlist[0]=="exp": funcStr = "%s + %s*expo(%s+%s*x)" % (paramlist[1], paramlist[2], paramlist[3], paramlist[4]) # Exponential
else: funcStr = "%s+x*(%s+ x*(%s+x*%s))" % (paramlist[1], paramlist[2], paramlist[3], paramlist[4]) # Polynomial
fitFunc = TF1("Fit_"+triggerName, funcStr)
if self.pileUp:
if self.numBunches[0] > 0:
return self.numBunches[0]*fitFunc.Eval(ilum/self.numBunches[0]*ppInelXsec/orbitsPerSec)
else:
return 0
return fitFunc.Eval(ilum)
# Use: Gets the MSE of the fit
def getMSE(self, triggerName):
if not self.L1 and (self.InputFitHLT is None or not self.InputFitHLT.has_key(triggerName)):
return 0
elif self.L1 and ((self.InputFitL1 is None) or not self.InputFitL1.has_key(triggerName)):
return 0
if self.L1: paramlist = self.InputFitL1[triggerName]
else: paramlist = self.InputFitHLT[triggerName]
if self.pileUp:
return self.numBunches[0]*paramlist[5]
return paramlist[5] # The MSE
# Use: Sends an email alert
# Parameters:
# -- mailTriggers: A list of triggers that we should include in the mail, ( { triggerName, aveRate, expected rate, standard dev } )
# Returns: (void)
def sendMail(self, mailTriggers):
mail = "Run: %d, Lumisections: %s - %s \n" % (self.runNumber, self.lastLS, self.currentLS)
try: mail += "Average inst. lumi: %.0f x 10^30 cm-2 s-1\n" % (self.lumi_ave)
except: mail += "Average inst. lumi: %s x 10^30 cm-2 s-1\n" % (self.lumi_ave)
try: mail += "Average PU: %.2f\n \n" % (self.pu_ave)
except: mail += "Average PU: %s\n \n" % (self.pu_ave)
mail += "Trigger rates deviating from acceptable and/or expected values: \n\n"
for triggerName, rate, expected, dev, ps in mailTriggers:
if self.numBunches[0] == 0:
mail += "\n %s: Actual: %s Hz\n" % (stringSegment(triggerName, 35), rate)
else:
if expected > 0:
try: mail += "\n %s: Expected: %.1f Hz, Actual: %.1f Hz, Unprescaled Expected/nBunches: %.5f Hz, Unprescaled Actual/nBunches: %.5f Hz, Deviation: %.1f\n" % (stringSegment(triggerName, 35), expected, rate, expected*ps/self.numBunches[0], rate*ps/self.numBunches[0], dev)
except: mail += "\n %s: Expected: %s Hz, Actual: %s Hz, Unprescaled Expected/nBunches: %s Hz, Unprescaled Actual/nBunches: %s Hz, Deviation: %s\n" % (stringSegment(triggerName, 35), expected, rate, expected*ps/self.numBunches[0], rate*ps/self.numBunches[0], dev)
mail += " *referenced fit: <https://raw.githubusercontent.com/cms-tsg-fog/RateMon/master/Fits/2016/plots/%s.png>\n" % (triggerName)
else:
try: mail += "\n %s: Actual: %.1f Hz\n" % (stringSegment(triggerName, 35), rate)
except: mail += "\n %s: Actual: %s Hz\n" % (stringSegment(triggerName, 35), rate)
try:
wbm_url = self.parser.getWbmUrl(self.runNumber,triggerName,self.currentLS)
if not wbm_url == "-": mail += " *WBM rate: <%s>\n" % (wbm_url)
except:
print "WBM plot url query failed"
mail += "\nWBM Run Summary: <https://cmswbm.web.cern.ch/cmswbm/cmsdb/servlet/RunSummary?RUN=%s> \n\n" % (self.runNumber)
mail += "Email warnings triggered when: \n"
mail += " - L1 or HLT rates deviate by more than %s standard deviations from fit \n" % (self.devAccept)
mail += " - HLT rates > %s Hz \n" % (self.maxHLTRate)
mail += " - L1 rates > %s Hz \n" % (self.maxL1Rate)
print "--- SENDING MAIL ---\n"+mail+"\n--------------------"
mailAlert(mail)
class DataParser:
# This is an interface for DBParser() to select and manage the data returned by DBParser()
def __init__(self):
# type: () -> None
self.parser = DBParser()
# The lists all have the same number of elements, e.g.: len(self.lumi_data[trg][run]) == len(self.pu_data[trg][run])
self.ls_data = {} # {'name': { run_number: [LS] } }
self.rate_data = {} # {'name': { run_number: { LS: raw_rates } } }
self.ps_data = {} # {'name': { run_number: { LS: prescale } } }
self.pu_data = {} # {'name': { run_number: { LS: PU } } }
self.lumi_data = {} # {'name': { run_number: { LS: iLumi } } }
self.det_data = {} # {'name': { run_number: { LS: detecotr_ready } } }
self.phys_data = {} # {'name': { run_number: { LS: phys_active } } }
self.bw_data = {} # {'name': { run_number: { LS: bandwidth } } }
self.size_data = {} # {'name': { run_number: { LS: size } } }
self.lumi_info = {} # {run_number: [ (LS,ilum,psi,phys,cms_ready) ] }
self.bunch_map = {} # {run_number: nBunches }
self.hlt_triggers = [
] # List of specific HLT triggers we want to get rates for, if empty --> get all HLT rates
self.l1_triggers = [
] # List of specific L1 triggers we want to get rates for, if empty --> get all L1 rates
self.runs_used = [
] # List of runs which had rate info for queried objects
self.runs_skipped = [
] # List of runs which did not have rate info for queried objects
self.name_list = [
] # List of all named objects for which we have data, e.g. triggers, datasets, streams, etc...
self.psi_filter = []
self.type_map = {
} # Maps each object name to a type: trigger, dataset, stream, or L1A
# NOTE: Still need to handle the case where if two objects share the same name, but diff type
# NOTE2: This approach should be fine, since DataParser owns the nameing, will need to be careful
self.ls_veto = {} # {run_number:[LS list]} - LS to ignore
self.name_veto = [
] # List of paths/objects to remove from consideration
self.use_prescaled_rate = False # If true, then rates are not un-prescaled
self.use_cross_section = False # If true, then divide the rate by inst. lumi (only for L1 and HLT trigger data)
self.normalize_bunches = True # Normalize by the number of colliding bunches
self.correct_for_DT = True
self.convert_output = True # Flag to convert data from { LS: data } to [ data ], used in the data getters
self.skip_l1_triggers = False # Flag to skip collecting rates for L1 triggers
self.skip_hlt_triggers = False # Flag to skip collecting rates for HLT triggers
self.l1_rate_cut = 25e6 # Ignore L1 rates above this threshold
self.max_deadtime = 10.
self.min_ls = -1
self.max_ls = 9999999
self.use_L1_triggers = False # Gets L1 rates
self.use_HLT_triggers = False # Gets HLT rates
self.use_streams = False # Gets stream rates
self.use_datasets = False # Gets dataset rates
self.use_L1A_rate = False # Gets the L1A rates
self.use_ps_mask = False # Collects data only for LS in the specified prescale indices
self.use_best_lumi = True # Uses best luminosity as determined by BRIL
self.use_PLTZ_lumi = False # Uses luminosity reading from PLTZ
self.use_HF_lumi = False # Uses luminosity reading from HF
self.verbose = True
def parseRuns(self, run_list):
# type: (List[int]) -> None
if len(self.hlt_triggers) == 0 and len(self.l1_triggers) > 0:
self.skip_hlt_triggers = True
if len(self.hlt_triggers) > 0 and len(self.l1_triggers) == 0:
self.skip_l1_triggers = True
counter = 1
for run in sorted(run_list):
if self.verbose:
print "Processing run: %d (%d/%d)" % (run, counter,
len(run_list))
counter += 1
bunches = self.parser.getNumberCollidingBunches(run)[0]
if bunches is None or bunches is 0:
bunches = 1
lumi_info = self.parseLumiInfo(
run) # [( LS,ilum,psi,phys,cms_ready ) ]
run_data = self.getRunData(run, bunches, lumi_info)
if len(
run_data.keys()
) == 0: # i.e. no triggers/streams/datasets had enough valid rates
self.runs_skipped.append(run)
continue
else:
self.runs_used.append(run)
self.bunch_map[run] = bunches
self.lumi_info[run] = lumi_info
for name in run_data:
if name in self.name_veto:
continue
ls_array = run_data[name]["LS"]
rate = run_data[name]["rate"]
prescale = run_data[name]["prescale"]
pu = run_data[name]["PU"]
lumi = run_data[name]["ilumi"]
det_status = run_data[name]["status"]
phys = run_data[name]["phys"]
bw = run_data[name]["bandwidth"]
size = run_data[name]["size"]
if not name in self.name_list:
self.name_list.append(name)
self.ls_data[name] = {}
self.rate_data[name] = {}
self.ps_data[name] = {}
self.pu_data[name] = {}
self.lumi_data[name] = {}
self.det_data[name] = {}
self.phys_data[name] = {}
self.bw_data[name] = {}
self.size_data[name] = {}
self.ls_data[name][run] = ls_array
self.rate_data[name][run] = rate
self.ps_data[name][run] = prescale
self.pu_data[name][run] = pu
self.lumi_data[name][run] = lumi
self.det_data[name][run] = det_status
self.phys_data[name][run] = phys
self.bw_data[name][run] = bw
self.size_data[name][run] = size
def parseLumiInfo(self, run):
# [( LS,ilum,psi,phys,cms_ready ) ]
lumi_info = []
trigger_mode = self.parser.getTriggerMode(run)[0]
if trigger_mode.find('cosmics') > 0:
# This is a cosmics menu --> No luminosity info
if self.verbose:
print "\tDetected cosmics run..."
print "\tGetting lumi info..."
for LS, psi in self.parser.getLSInfo(run):
# We hard code phys and cms_ready to both be true for all LS in the run
lumi_info.append([LS, 0.0, psi, 1, 1])
elif trigger_mode.find('collisions') > 0:
# This is a collisions menu
if self.verbose:
print "\tDetected collisions run..."
print "\tGetting lumi info..."
if self.use_best_lumi:
lumi_info = self.parser.getQuickLumiInfo(run,
minLS=self.min_ls,
maxLS=self.max_ls)
elif self.use_PLTZ_lumi:
lumi_info = self.parser.getLumiInfo(run,
minLS=self.min_ls,
maxLS=self.max_ls,
lumi_source=1)
elif self.use_HF_lumi:
lumi_info = self.parser.getLumiInfo(run,
minLS=self.min_ls,
maxLS=self.max_ls,
lumi_source=2)
else:
# Unknown menu --> For now we assume it's compatibale with collisions type menus
if self.verbose:
print "\tUnknown run type: %s" % (trigger_mode)
print "\tGetting lumi info..."
if self.use_best_lumi:
lumi_info = self.parser.getQuickLumiInfo(run,
minLS=self.min_ls,
maxLS=self.max_ls)
elif self.use_PLTZ_lumi:
lumi_info = self.parser.getLumiInfo(run,
minLS=self.min_ls,
maxLS=self.max_ls,
lumi_source=1)
elif self.use_HF_lumi:
lumi_info = self.parser.getLumiInfo(run,
minLS=self.min_ls,
maxLS=self.max_ls,
lumi_source=2)
if self.ls_veto.has_key(run):
new_lumi_info = []
for LS, ilum, psi, phys, cms_ready in lumi_info:
if LS in self.ls_veto[run]:
continue
new_lumi_info.append([LS, ilum, psi, phys, cms_ready])
lumi_info = new_lumi_info
return lumi_info
# This might be excessive, should think about reworking this section
# ------
# TODO: We need to ensure that none of the object names overlap with one another
# (i.e. dataset names that overlap with stream names) for the rate data.
def getRunData(self, run, bunches, lumi_info):
# type: (int,int,List[Tuple[int,float,int,bool,bool]]) -> Dict[str: object]
run_data = {}
if bunches is None or bunches is 0:
if self.verbose: print "Unable to get bunches"
return {}
if self.use_streams:
run_data.update(self.getStreamData(run, bunches, lumi_info))
if self.use_datasets:
run_data.update(self.getDatasetData(run, bunches, lumi_info))
if self.use_L1A_rate:
run_data.update(self.getL1AData(run, bunches, lumi_info))
if self.use_HLT_triggers:
run_data.update(self.getHLTTriggerData(run, bunches, lumi_info))
if self.use_L1_triggers:
run_data.update(self.getL1TriggerData(run, bunches, lumi_info))
return run_data
# Returns information related to L1 triggers
def getL1TriggerData(self, run, bunches, lumi_info):
# type: (int,int,List[Tuple[int,float,int,bool,bool]]) -> Dict[str: object]
if self.skip_l1_triggers:
return {}
if self.verbose: print "\tGetting L1 rates..."
L1_rates = self.parser.getL1Rates(run,
minLS=self.min_ls,
maxLS=self.max_ls,
scaler_type=1)
run_data = {} # {'object': {"LS": list, "rate": {...}, ... } }
for trigger in L1_rates:
self.type_map[trigger] = "trigger"
run_data[trigger] = {}
ls_array = array.array('f')
rate_dict = {}
ps_dict = {}
pu_dict = {}
lumi_dict = {}
det_dict = {}
phys_dict = {}
bw_dict = {}
size_dict = {}
for LS, ilum, psi, phys, cms_ready in lumi_info:
if psi not in self.psi_filter and self.use_ps_mask:
continue
if not ilum is None and L1_rates[trigger].has_key(LS):
pu = (ilum / bunches * ppInelXsec / orbitsPerSec)
rate = L1_rates[trigger][LS][0]
prescale = L1_rates[trigger][LS][1]
if rate > self.l1_rate_cut:
continue
if self.normalize_bunches:
rate = rate / bunches
if self.use_prescaled_rate:
if prescale != 0:
rate = rate / prescale
#else:
# rate = 0
if self.use_cross_section:
rate = rate / ilum
ls_array.append(LS)
rate_dict[LS] = rate
ps_dict[LS] = prescale
pu_dict[LS] = pu
lumi_dict[LS] = ilum
det_dict[LS] = cms_ready
phys_dict[LS] = phys
bw_dict[LS] = None
size_dict[LS] = None
run_data[trigger]["LS"] = ls_array
run_data[trigger]["rate"] = rate_dict
run_data[trigger]["prescale"] = ps_dict
run_data[trigger]["PU"] = pu_dict
run_data[trigger]["ilumi"] = lumi_dict
run_data[trigger]["status"] = det_dict
run_data[trigger]["phys"] = phys_dict
run_data[trigger]["bandwidth"] = bw_dict
run_data[trigger]["size"] = size_dict
return run_data
# Returns information related to HLT triggers
def getHLTTriggerData(self, run, bunches, lumi_info):
# type: (int,int,List[Tuple[int,float,int,bool,bool]]) -> Dict[str: object]
if self.skip_hlt_triggers:
return {}
if self.verbose: print "\tGetting HLT rates..."
HLT_rates = self.parser.getHLTRates(run,
self.hlt_triggers,
minLS=self.min_ls,
maxLS=self.max_ls)
if self.correct_for_DT:
self.correctForDeadtime(HLT_rates, run)
run_data = {} # {'object': {"LS": list, "rate": {...}, ... } }
for trigger in HLT_rates:
self.type_map[trigger] = "trigger"
run_data[trigger] = {}
ls_array = array.array('f')
rate_dict = {}
ps_dict = {}
pu_dict = {}
lumi_dict = {}
det_dict = {}
phys_dict = {}
bw_dict = {}
size_dict = {}
for LS, ilum, psi, phys, cms_ready in lumi_info:
if psi not in self.psi_filter and self.use_ps_mask:
continue
if not ilum is None and HLT_rates[trigger].has_key(LS):
pu = (ilum / bunches * ppInelXsec / orbitsPerSec)
rate = HLT_rates[trigger][LS][0]
prescale = HLT_rates[trigger][LS][1]
if self.normalize_bunches:
rate = rate / bunches
if self.use_prescaled_rate:
if prescale != 0:
rate = rate / prescale
#else:
# rate = 0
if self.use_cross_section:
rate = rate / ilum
ls_array.append(LS)
rate_dict[LS] = rate
ps_dict[LS] = prescale
pu_dict[LS] = pu
lumi_dict[LS] = ilum
det_dict[LS] = cms_ready
phys_dict[LS] = phys
bw_dict[LS] = None
size_dict[LS] = None
run_data[trigger]["LS"] = ls_array
run_data[trigger]["rate"] = rate_dict
run_data[trigger]["prescale"] = ps_dict
run_data[trigger]["PU"] = pu_dict
run_data[trigger]["ilumi"] = lumi_dict
run_data[trigger]["status"] = det_dict
run_data[trigger]["phys"] = phys_dict
run_data[trigger]["bandwidth"] = bw_dict
run_data[trigger]["size"] = size_dict
return run_data
def getStreamData(self, run, bunches, lumi_info):
# type: (int,int,List[Tuple[int,float,int,bool,bool]]) -> Dict[str: object]
if self.verbose: print "\tGetting Stream rates..."
data = self.parser.getStreamData(
run, minLS=self.min_ls,
maxLS=self.max_ls) # {'stream': [ (LS,rate,size,bandwidth) ] }
stream_rates = {} # {'stream': {LS: (rate,size,bandwidth) } }
# Format the output from DBParser()
for name in data:
stream_rates[name] = {}
for LS, rate, size, bandwidth in data[name]:
stream_rates[name][LS] = [rate, size, bandwidth]
run_data = {} # {'object': {"LS": list, "rate": {...}, ... } }
blacklist = [
"PhysicsEndOfFill", "PhysicsMinimumBias0", "PhysicsMinimumBias1",
"PhysicsMinimumBias2"
]
sum_list = []
for _object in stream_rates:
self.type_map[_object] = "stream"
run_data[_object] = {}
if _object[:7] == "Physics" and not _object in blacklist:
sum_list.append(_object)
ls_array = array.array('f')
rate_dict = {}
ps_dict = {}
pu_dict = {}
lumi_dict = {}
det_dict = {}
phys_dict = {}
bw_dict = {}
size_dict = {}
for LS, ilum, psi, phys, cms_ready in lumi_info:
if psi not in self.psi_filter and self.use_ps_mask:
continue
if not ilum is None and stream_rates[_object].has_key(LS):
pu = (ilum / bunches * ppInelXsec / orbitsPerSec)
rate = stream_rates[_object][LS][0]
size = stream_rates[_object][LS][1]
bandwidth = stream_rates[_object][LS][2]
if self.normalize_bunches:
rate = rate / bunches
ls_array.append(LS)
rate_dict[LS] = rate
ps_dict[LS] = None
pu_dict[LS] = pu
lumi_dict[LS] = ilum
det_dict[LS] = cms_ready
phys_dict[LS] = phys
bw_dict[LS] = bandwidth
size_dict[LS] = size
run_data[_object]["LS"] = ls_array
run_data[_object]["rate"] = rate_dict
run_data[_object]["prescale"] = ps_dict
run_data[_object]["PU"] = pu_dict
run_data[_object]["ilumi"] = lumi_dict
run_data[_object]["status"] = det_dict
run_data[_object]["phys"] = phys_dict
run_data[_object]["bandwidth"] = bw_dict
run_data[_object]["size"] = size_dict
self.sumObjects(run_data=run_data,
new_name="Combined_Physics_Streams",
sum_list=sum_list,
obj_type="stream")
return run_data
def getDatasetData(self, run, bunches, lumi_info):
# type: (int,int,List[Tuple[int,float,int,bool,bool]]) -> Dict[str: object]
if self.verbose: print "\tGetting Dataset rates..."
data = self.parser.getPrimaryDatasets(
run, minLS=self.min_ls,
maxLS=self.max_ls) # {'dataset': [ (LS,rate) ] }
dataset_rates = {} # {'dataset': {LS: (rate) } }
# Format the output from DBParser()
for name in data:
dataset_rates[name] = {}
for LS, rate in data[name]:
dataset_rates[name][LS] = [rate]
run_data = {} # {'object': {"LS": list, "rate": {...}, ... } }
for _object in dataset_rates:
self.type_map[_object] = "dataset"
run_data[_object] = {}
ls_array = array.array('f')
rate_dict = {}
ps_dict = {}
pu_dict = {}
lumi_dict = {}
det_dict = {}
phys_dict = {}
bw_dict = {}
size_dict = {}
for LS, ilum, psi, phys, cms_ready in lumi_info:
if psi not in self.psi_filter and self.use_ps_mask:
continue
if not ilum is None and dataset_rates[_object].has_key(LS):
pu = (ilum / bunches * ppInelXsec / orbitsPerSec)
rate = dataset_rates[_object][LS][0]
if self.normalize_bunches:
rate = rate / bunches
ls_array.append(LS)
rate_dict[LS] = rate
ps_dict[LS] = None
pu_dict[LS] = pu
lumi_dict[LS] = ilum
det_dict[LS] = cms_ready
phys_dict[LS] = phys
bw_dict[LS] = None
size_dict[LS] = None
run_data[_object]["LS"] = ls_array
run_data[_object]["rate"] = rate_dict
run_data[_object]["prescale"] = ps_dict
run_data[_object]["PU"] = pu_dict
run_data[_object]["ilumi"] = lumi_dict
run_data[_object]["status"] = det_dict
run_data[_object]["phys"] = phys_dict
run_data[_object]["bandwidth"] = bw_dict
run_data[_object]["size"] = size_dict
return run_data
# NOTE: L1A_rates has a slightly different dict format, the value-pair for the LS keys is NOT a list
def getL1AData(self, run, bunches, lumi_info):
# type: (int,int,List[Tuple[int,float,int,bool,bool]]) -> Dict[str: object]
L1A_rates = {} # {'L1A': {LS: rate } }
if self.verbose: print "\tGetting L1ATotal rates..."
L1A_rates["L1ATotal"] = self.parser.getL1rate(run)
if self.verbose: print "\tGetting L1APhysics rates..."
L1A_rates["L1APhysics"] = self.parser.getL1APhysics(run)
if self.verbose: print "\tGetting L1APhysicsLost rates..."
L1A_rates["L1APhysicsLost"] = self.parser.getL1APhysicsLost(run)
run_data = {} # {'object': {"LS": list, "rate": {...}, ... } }
for _object in L1A_rates:
self.type_map[_object] = "L1A"
run_data[_object] = {}
ls_array = array.array('f')
rate_dict = {}
ps_dict = {}
pu_dict = {}
lumi_dict = {}
det_dict = {}
phys_dict = {}
bw_dict = {}
size_dict = {}
for LS, ilum, psi, phys, cms_ready in lumi_info:
if psi not in self.psi_filter and self.use_ps_mask:
continue
if not ilum is None and L1A_rates[_object].has_key(LS):
pu = (ilum / bunches * ppInelXsec / orbitsPerSec)
rate = L1A_rates[_object][LS]
if self.normalize_bunches:
rate = rate / bunches
ls_array.append(LS)
rate_dict[LS] = rate
ps_dict[LS] = None
pu_dict[LS] = pu
lumi_dict[LS] = ilum
det_dict[LS] = cms_ready
phys_dict[LS] = phys
bw_dict[LS] = None
size_dict[LS] = None
run_data[_object]["LS"] = ls_array
run_data[_object]["rate"] = rate_dict
run_data[_object]["prescale"] = ps_dict
run_data[_object]["PU"] = pu_dict
run_data[_object]["ilumi"] = lumi_dict
run_data[_object]["status"] = det_dict
run_data[_object]["phys"] = phys_dict
run_data[_object]["bandwidth"] = bw_dict
run_data[_object]["size"] = size_dict
return run_data
# Use: Modifies the rates in Rates, correcting them for deadtime
# Parameters:
# -- Rates: A dict - {'trigger': {LS: (raw_rate,prescale) } }
def correctForDeadtime(self, Rates, run_number):
# type: (Dict[str,object],int) -> None
dead_time = self.parser.getDeadTime(run_number)
for LS in dead_time:
for trigger in Rates:
if Rates[trigger].has_key(LS): # Sometimes, LS's are missing
Rates[trigger][LS][0] *= (1. + dead_time[LS] / 100.)
if dead_time[
LS] > self.max_deadtime: # Do not plot lumis where deadtime is greater than 10%
del Rates[trigger][LS]
# Creates a new dictionary key, that corresponds to the summed rates of all the specified objects
# data: {'object': {"LS": list, "rate": {...}, ... } }
def sumObjects(self, run_data, new_name, sum_list, obj_type):
# type: (Dict[str,object],str,List[str],str) -> bool
if not set(sum_list) <= set(run_data.keys()):
if self.verbose:
print "\tERROR: Specified objects that aren't in the run_data"
return False
if (len(sum_list) == 0):
print "\tERROR: sum_list has size=0 (see sumObjects in DataParser.py). May be that there were no streams for this run."
return False
ref_name = sum_list[0]
ls_array = array.array('f')
rate_dict = {}
ps_dict = {}
pu_dict = {}
lumi_dict = {}
det_dict = {}
phys_dict = {}
bw_dict = {}
size_dict = {}
# We only use LS that are in *all* of the objects
ls_set = set(run_data[ref_name]["LS"])
for name in sum_list:
ls_set = ls_set & set(run_data[name]["LS"])
ls_array.extend(sorted(ls_set))
for LS in ls_array:
total_rate = 0
total_bw = 0
total_size = 0
for name in sum_list:
total_rate += run_data[name]["rate"][LS]
try:
total_bw += run_data[name]["bandwidth"][LS]
except:
total_bw = None
try:
total_size += run_data[name]["size"][LS]
except:
total_size = None
rate_dict[LS] = total_rate
bw_dict[LS] = total_bw
size_dict[LS] = total_size
ps_dict[LS] = None
pu_dict[LS] = run_data[ref_name]["PU"][LS]
lumi_dict[LS] = run_data[ref_name]["ilumi"][LS]
det_dict[LS] = run_data[ref_name]["status"][LS]
phys_dict[LS] = run_data[ref_name]["phys"][LS]
self.type_map[new_name] = obj_type
run_data[new_name] = {}
run_data[new_name]["LS"] = ls_array
run_data[new_name]["rate"] = rate_dict
run_data[new_name]["prescale"] = ps_dict
run_data[new_name]["PU"] = pu_dict
run_data[new_name]["ilumi"] = lumi_dict
run_data[new_name]["status"] = det_dict
run_data[new_name]["phys"] = phys_dict
run_data[new_name]["bandwidth"] = bw_dict
run_data[new_name]["size"] = size_dict
return True
# Converts input: {'name': { run_number: { LS: data } } } --> {'name': {run_number: [ data ] } }
def convertOutput(self, _input):
# type: (Dict[str,object]) -> Dict[str,object]
output = {}
for name in _input:
output[name] = {}
for run in _input[name]:
output[name][run] = array.array('f')
for LS in sorted(_input[name][run].keys(
)): # iterating over *sorted* LS is extremely important here
output[name][run].append(_input[name][run][LS])
return output
def resetData(self):
# type: () -> None
self.ls_data = {} # {'name': { run_number: [LS] } }
self.rate_data = {} # {'name': { run_number: { LS: raw_rates } } }
self.ps_data = {} # {'name': { run_number: { LS: prescale } } }
self.pu_data = {} # {'name': { run_number: { LS: PU } } }
self.lumi_data = {} # {'name': { run_number: { LS: iLumi } } }
self.det_data = {} # {'name': { run_number: { LS: detecotr_ready } } }
self.phys_data = {} # {'name': { run_number: { LS: phys_active } } }
self.bw_data = {} # {'name': { run_number: { LS: bandwidth } } }
self.size_data = {} # {'name': { run_number: { LS: size } } }
self.lumi_info = {} # {run_number: [ (LS,ilum,psi,phys,cms_ready) ] }
self.bunch_map = {} # {run_number: nBunches }
self.runs_used = []
self.runs_skipped = []
self.name_list = []
self.type_map = {}
####################################################################################################
# --- All the 'getters' ---
def getLSData(self):
# type: () -> Dict[str,object]
return self.ls_data
def getRateData(self):
# type: () -> Dict[str,object]
if self.convert_output:
output = self.convertOutput(self.rate_data)
else:
output = self.rate_data
return output
def getPSData(self):
# type: () -> Dict[str,object]
if self.convert_output:
output = self.convertOutput(self.ps_data)
else:
output = self.ps_data
return output
def getPUData(self):
# type: () -> Dict[str,object]
if self.convert_output:
output = self.convertOutput(self.pu_data)
else:
output = self.pu_data
return output
def getLumiData(self):
# type: () -> Dict[str,object]
if self.convert_output:
output = self.convertOutput(self.lumi_data)
else:
output = self.lumi_data
return output
def getDetectorStatus(self):
# type: () -> Dict[str,object]
if self.convert_output:
output = self.convertOutput(self.det_data)
else:
output = self.det_data
return output
def getPhysStatus(self):
# type: () -> Dict[str,object]
if self.convert_output:
output = self.convertOutput(self.phys_data)
else:
output = self.phys_data
return output
def getBandwidthData(self):
# type: () -> Dict[str,object]
if self.convert_output:
output = self.convertOutput(self.bw_data)
else:
output = self.bw_data
return output
def getSizeData(self):
# type: () -> Dict[str,object]
if self.convert_output:
output = self.convertOutput(self.size_data)
else:
output = self.size_data
return output
def getLumiInfo(self):
# type: () -> Dict[int,List[Tuple]]
return self.lumi_info
def getBunchMap(self):
# type: () -> Dict[int,int]
return self.bunch_map
def getRunsUsed(self):
# type: () -> List[int]
return self.runs_used
def getNameList(self):
# type: () -> List[int]
return self.name_list
# Returns all the objects of type obj_type we have rate for
def getObjectList(self, obj_type):
# type: (str) -> List[str]
_list = []
for obj in self.name_list:
if self.type_map[obj] == obj_type:
_list.append(obj)
return _list
def getTypeMap(self):
# type: () -> Dict[str,str]
return self.type_map
# Return the latest LS for a given run or -1
def getLastLS(self, run):
# type: (int) -> int
if not run in self.runs_used:
return -1
index = LUMI_INFO_MAP["LS"]
return max(self.lumi_info[run], key=lambda x: x[index])[index]
# Return the latest run for which we have data or -1
def getLastRun(self):
# type: () -> int
if len(self.runs_used) == 0:
return -1
return max(self.runs_used)