def calculate_dead_fraction_all(self, output_channel, local_variables):
log.debug("Calculating dead fractions for output: %i", output_channel)
# local_variables is a list of IOVSets (one for each input channel),
# for this output channel.
# Why would you call it local_variables?
#prev_states = []
dead_frac_iovs = IOVSet()
calc_dead_frac = self.calculate_dead_fraction
# loop over smallest IOV chunks for this output channel
for since, until, states in process_iovs(self.run_iovs, *local_variables):
run_iov = states[0]
# state_iovs is now a list of iovs, one for each input channel mapped
# to this output channel
state_iovs = states[1:]
states = [s.good for s in state_iovs]
if run_iov._is_empty:
# Ignore regions outside runs.
continue
iov_state = calc_dead_frac(since, until, output_channel,
states, state_iovs)
dead_frac_iovs.add(since, until, output_channel, *iov_state)
return dead_frac_iovs.solidify(DCSOFL_IOV)
def merge_input_information(self, channel, *inputs):
"""
Join up the information which was used to make a decision across
multiple variables.
"""
result = IOVSet()
for since, until, states in process_iovs(*inputs):
info = tuple(state._orig_iov[3:] for state in states)
result.add(since, until, channel, info)
return result.solidify(GoodIOV)
def merge_inputs(self, channel, *inputs):
"""
Merge multiple variables together for one input channel.
Each 'inputs' arg is an IOVSet that corresponds to this input channel.
"""
# inputs must correspond to and be in sync with subdetector.variables...?
result = IOVSet()
# combine the IOVSets into smallest chunks using process_iovs
for since, until, states in process_iovs(*inputs):
# Get the worst state for the list of vectors
state = self.merge_variable_states(states)
result.add(since, until, channel, state)
return result.solidify(GoodIOV)
def make_good_iovs(self, iovs):
# Filter out channels which are permanently dead
excluded_channels = self.excluded_channels
if excluded_channels:
iovs = iovs.empty(i for i in iovs
if i.channel not in excluded_channels)
chans, iovsets = iovs.chans_iovsets
result = IOVSet()
for since, until, states in process_iovs(*iovsets):
if not all(s and s.powerStatus for s in states):
result.add(since, until, self.defect_name, True, "")
return result.solidify(DefectIOV)
def merge_globals(self, output_channel, dead_frac_iovs, global_variables):
"""
Merge together global states to decide a final code
If the dead fraction is unavailable, writes -1.
"""
result_iovs = IOVSet()
if self.run_iovs is not None:
# run_iovs are used to constrain to what runs the calculator will
# write. If there is a hole in `run_iovs`, then no records are emitted.
state_ranges = process_iovs(self.run_iovs, dead_frac_iovs,
*global_variables)
else:
state_ranges = process_iovs(dead_frac_iovs, *global_variables)
for since, until, states in state_ranges:
if self.run_iovs:
# No run_iovs were specified, so run for all available input data
run_iov, dead_frac_iov = states[:2]
if run_iov._is_empty:
# We're outside of a run, don't write an IoV.
continue
states = states[1:]
else:
dead_frac_iov = states[0]
if not dead_frac_iov._is_empty:
dead_fraction, thrust, n_config, n_working = dead_frac_iov[4:]
else:
dead_fraction, thrust, n_config, n_working = -1., 0., -1, -1
states = states[1:]
code = self.dq_worst(states)
state = dead_fraction, thrust, n_config, n_working
if code is WHITE:
# Don't write empty regions
continue
result_iovs.add(since, until, output_channel, code, *state)
return result_iovs.solidify(DCSOFL_IOV)
def calculate_good_iovs(self, lbtime, subdetector):
magnets = Magnets()
result = magnets.run(lbtime)
by_defect = result.by_channel
toroid_ramp = by_defect.get("GLOBAL_TOROID_RAMPING", IOVSet())
solenoid_ramp = by_defect.get("GLOBAL_SOLENOID_RAMPING", IOVSet())
self.input_hashes = [hash(toroid_ramp), hash(solenoid_ramp)]
result = IOVSet()
events = process_iovs(toroid_ramp, solenoid_ramp)
for since, until, (toroid, solenoid) in events:
if toroid or solenoid:
result.add(since, until, "LCD_MAGNETS_RAMPING", True, "")
self.iovs = result.solidify(DefectIOV)
return self
def make_good_iovs(self, iovs):
"""
The absence of an IoV means that channel is out of config.
"""
seen_channels = set()
result = IOVSet()
for since, until, channel, (state, ) in process_iovs_mc(iovs):
seen_channels.add(channel)
if state._is_empty:
# There is an IoV hole for this channel.
result.add(since, until, channel, OUT_OF_CONFIG)
# Need to deal with channels which were not seen at all for the query
# range, since they will not appear in the above loop
all_channels = self.subdetector.input_channel_set
for missing_channel in (all_channels - seen_channels):
result.add(0, 2**63 - 1, missing_channel, OUT_OF_CONFIG)
return result.solidify(GoodIOV)
class IDBSDefectWriter:
"""
Class for writing BS defects to an sqlite file
"""
def __init__(self,
fileName,
forceNew=False,
dbName='IDBSDQ',
tag='nominal',
user='******'):
"""
Initialise database connection
"""
self.defect = None
self.iovs = IOVSet()
self.user = user
#self.allowedDefects = DefectsDB('').defect_names
if not fileName: fileName = 'tmp.' + str(os.getpid()) + '.db'
self.connect(fileName, forceNew, dbName, tag)
pass
def __del__(self):
"""
Delete db to clear connection
"""
os.system('[[ -f tmp.%s.db ]] && rm tmp.%s.db' %
(os.getpid(), os.getpid()))
del self.db
pass
def connect(self,
fileName,
forceNew=False,
dbName='IDBSDQ',
tag='nominal'):
"""
Open connection to defect DB
"""
connString = 'sqlite://;schema=%s;dbname=%s' % (fileName, dbName)
if forceNew and os.path.exists(fileName):
os.remove(fileName)
self.db = DefectsDB(
connString, read_only=False, create=True, tag=(tag, 'HEAD')
) # Second tag is for virtual defects, which we are not interested in
self.officialDb = DefectsDB()
return
def defectType(self, t):
"""
Set defect type
"""
self.defect = t
def add(self,
runMin=0,
runMax=(1 << 31) - 1,
lbMin=0,
lbMax=(1 << 32) - 1):
"""
Add iovs which are NOT defective to the list
Note, lbMax is exclusive here (and inclusive when shown on defect web page).
"""
# Make since and until and convert to syntactially nice RunLumiType
since = RunLumiType((runMin << 32) + lbMin)
until = RunLumiType((runMax << 32) + lbMax)
# IoVs which are not defective (i.e. beamspot good)
self.iovs.add(since, until, self.defect, False)
return
def complete(self, runMin, runMax):
"""
Complete a list of IoVs to cover all LBs in a run, treating empty ones as having 'emptyState'
"""
# Create an IOV set covering the entire run(s)
run_lbs = fetch_iovs("EOR",
runs=(runMin, runMax),
what=[],
with_channel=False)
# run_lbs = IOVSet()
# lbMin = 1
# lbMax = (1 << 32) -1 # Note, lbMax is exclusive
# since = RunLumiType((runMin << 32)+lbMin)
# until = RunLumiType((runMax << 32)+lbMax)
# run_lbs.add(since, until)
if not len(run_lbs):
print "WARNING: No LBs in run according to EOR_Params - are we running before the run has finished?"
def lbsStartAtOne(iov):
"Change LBs starting at 0 to start at 1"
return iov._replace(since=RunLumi(iov.since.run, 1))
# Start LBs from 1 rather than 0 (at request of P. Onyisi) as long as run has more than one LB (else skip)
run_lbs = [lbsStartAtOne(iov) for iov in run_lbs if iov.until.lumi > 1]
# Empty IOV set for adding full list of LBs too
iovs = IOVSet()
# Ask official DB if an IoV is currently defective so can unset only those if doing reprocessing
defectsCurrentlyInDb = self.officialDb.retrieve(
(runMin, 0), (runMax + 1, 0), [self.defect])
# Order IOVs to avoid since > until
self.iovs = IOVSet(sorted(self.iovs))
#for since, until, (run_lb, iov, dbDefect) in process_iovs(run_lbs.solidify(RANGEIOV_VAL), self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb):
for since, until, (run_lb, iov, dbDefect) in process_iovs(
run_lbs, self.iovs.solidify(DEFECTIOV_VAL),
defectsCurrentlyInDb):
if not run_lb: continue # Check valid
# # Start LBs from 1 rather than 0 (at request of P. Onyisi)
# # If this makes since==until, i.e. it was a [0->1) LB range then skip
# if since.lumi==0:
# since = RunLumiType((since.run << 32)+since.lumi+1)
# if since==until: continue
# Add iovs from self.iovs treating empty ones as defective (i.e. beamspot bad/missing)
if iov:
# These are not defective
if dbDefect and dbDefect.present:
# Only store not defective IoVs if they are present on the Db so we can unset them
# (don't want to write not present defects to Db unless really chaning an existing defect)
iovs.add(since, until, self.defect, False)
else:
# These are defective
iovs.add(since, until, self.defect, True)
self.iovs = iovs
return
def writeDefects(self, tag='nominal', nonpresent=False):
"""
Write all defects to the database. If 'nonpresent' is True then write the absent ones too
"""
with self.db.storage_buffer:
for since, until, defect, present in self.iovs:
if not present and not nonpresent: continue
#print since, until, present
self._writeDefect(defect, since, until, present=present)
def _writeDefect(self,
defect,
since,
until,
tag='nominal',
description='',
comment='',
present=True,
recoverable=False):
"""
Write a single defect to the database
"""
if not defect in self.db.defect_names:
self.db.create_defect(defect, description)
self.db.insert(defect, since, until, comment, self.user, present,
recoverable)
return
def dump(self, filename=None):
"""
Dump defects to a file given by filename or stdout if no filename given
"""
from DQUtils.utils import pprint_objects
if filename is not None:
f = open(filename.replace('.db', '.txt'), "w")
# If not defects then nothing will be in the database and we write an empty file
if len(self.iovs):
pprint_objects(
self.db.retrieve(primary_only=True, nonpresent=True), f)
f.close()
else:
if len(self.iovs):
self.iovs.pprint()
else:
print '\nNo DQ defects'
# with open(filename.replace('.db', '.txt'), "w") as f:
# pprint_objects(self.db.retrieve(), f)
return
def complete(self, runMin, runMax):
"""
Complete a list of IoVs to cover all LBs in a run, treating empty ones as having 'emptyState'
"""
# Create an IOV set covering the entire run(s)
run_lbs = fetch_iovs("EOR",
runs=(runMin, runMax),
what=[],
with_channel=False)
# run_lbs = IOVSet()
# lbMin = 1
# lbMax = (1 << 32) -1 # Note, lbMax is exclusive
# since = RunLumiType((runMin << 32)+lbMin)
# until = RunLumiType((runMax << 32)+lbMax)
# run_lbs.add(since, until)
if not len(run_lbs):
print "WARNING: No LBs in run according to EOR_Params - are we running before the run has finished?"
def lbsStartAtOne(iov):
"Change LBs starting at 0 to start at 1"
return iov._replace(since=RunLumi(iov.since.run, 1))
# Start LBs from 1 rather than 0 (at request of P. Onyisi) as long as run has more than one LB (else skip)
run_lbs = [lbsStartAtOne(iov) for iov in run_lbs if iov.until.lumi > 1]
# Empty IOV set for adding full list of LBs too
iovs = IOVSet()
# Ask official DB if an IoV is currently defective so can unset only those if doing reprocessing
defectsCurrentlyInDb = self.officialDb.retrieve(
(runMin, 0), (runMax + 1, 0), [self.defect])
# Order IOVs to avoid since > until
self.iovs = IOVSet(sorted(self.iovs))
#for since, until, (run_lb, iov, dbDefect) in process_iovs(run_lbs.solidify(RANGEIOV_VAL), self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb):
for since, until, (run_lb, iov, dbDefect) in process_iovs(
run_lbs, self.iovs.solidify(DEFECTIOV_VAL),
defectsCurrentlyInDb):
if not run_lb: continue # Check valid
# # Start LBs from 1 rather than 0 (at request of P. Onyisi)
# # If this makes since==until, i.e. it was a [0->1) LB range then skip
# if since.lumi==0:
# since = RunLumiType((since.run << 32)+since.lumi+1)
# if since==until: continue
# Add iovs from self.iovs treating empty ones as defective (i.e. beamspot bad/missing)
if iov:
# These are not defective
if dbDefect and dbDefect.present:
# Only store not defective IoVs if they are present on the Db so we can unset them
# (don't want to write not present defects to Db unless really chaning an existing defect)
iovs.add(since, until, self.defect, False)
else:
# These are defective
iovs.add(since, until, self.defect, True)
self.iovs = iovs
return
class Pixels(DCSC_DefectTranslate_Subdetector):
def __init__(self, *args, **kwargs):
super(Pixels, self).__init__(*args, **kwargs)
self.translators = [Pixels.pix_color_to_defect_translator]
@staticmethod
def pix_color_to_defect_translator(flag_iovs):
from DCSCalculator2.consts import GREEN
from DCSCalculator2.variable import DefectIOV
from DQUtils import process_iovs
rv = []
defect_mapping = {
101: "PIXEL_BARREL_STANDBY",
102: "PIXEL_LAYER0_STANDBY",
103: "PIXEL_IBL_STANDBY",
104: "PIXEL_ENDCAPA_STANDBY",
105: "PIXEL_ENDCAPC_STANDBY"
}
for since, until, states in process_iovs(
*flag_iovs.by_channel.values()):
#print states
for state in states:
if state.Code != GREEN:
badfrac = 'Standby module fraction: ' + str(state.deadFrac)
rv.append(
DefectIOV(since=since,
until=until,
channel=defect_mapping[state.channel],
present=True,
comment=badfrac))
return rv
input_db = "COOLOFL_DCS/CONDBR2"
folder_base = "/PIXEL/DCS"
cid_barrel, cid_blayer, cid_ibl = 101, 102, 103
cid_endcapa, cid_endcapb = 104, 105
mapping = {
101: range(722, 1892),
102: range(436, 722),
103: range(156, 436),
104: range(1892, 2036),
105: range(12, 156),
}
variables = [
DCSC_Variable("FSMSTATUS", lambda iov: iov.FSM_status in
("OK", "WARNING")),
DCSC_Variable("FSMSTATE", lambda iov: iov.FSM_state == "READY"),
]
# Note barrel and blayer use different deadfraction_caution. Implemented in
# calculate_iov below.
dead_fraction_caution = 0.2
dead_fraction_caution_barrel = 0.05
dead_fraction_bad = 0.9
assert dead_fraction_caution_barrel <= dead_fraction_caution, (
"logic needs changing in calculate_iov before removing this assert")
def start(self):
self.bad_modules = IOVSet()
def tally_additional_info(self, since, until, output_channel, states,
state_iovs):
"""
Figure out which modules are OOC when
"""
ooc_modules = self.get_ids_which_are(output_channel, states,
OUT_OF_CONFIG)
ooc_modules = tuple(sorted(ooc_modules))
self.bad_modules.add(since, until, output_channel, ooc_modules)
def done(self):
if logger.isEnabledFor(logging.DEBUG):
print "The following ranges indicate bad modules:"
#from pprint import pprint
#pprint(list(self.bad_modules))
def calculate_dead_fraction(self, since, until, output_channel, states,
state_iovs):
"""
Compute the dead fraction differently for the barrel and blayer.
This function is written with the assumption that
`dead_fraction_caution_barrel` is smaller than `dead_fraction_caution`
because the new logic is only run if the code is better than Yellow.
"""
cdf = super(Pixels, self).calculate_dead_fraction
result = cdf(since, until, output_channel, states, state_iovs)
self.tally_additional_info(since, until, output_channel, states,
state_iovs)
code, dead_fraction, thrust, n_config, n_working = result
if (output_channel not in [self.cid_barrel, self.cid_blayer]
or code <= YELLOW):
# Result is already correct
return result
# Need to check whether code needs modifying.
if self.dead_fraction_caution_barrel < dead_fraction < self.dead_fraction_bad:
code = YELLOW
return code, dead_fraction, thrust, n_config, n_working
def initialize_name_mapping(self):
from DQUtils.db import Databases, get_channel_ids_names
f = Databases.get_folder("/TDAQ/EnabledResources/ATLAS/PIXEL/Modules",
"COOLONL_TDAQ")
cids, cnames, cmap = get_channel_ids_names(f)
self.name_mapping = dict(zip(cids, cnames))
self.name_mapping_initialized = True
# TODO: is this ok??
def get_name_for_input_channel(self, input_channel):
"""
Transform an input channelid into a name
"""
#if not getattr(self, "name_mapping_initialized", False):
# self.initialize_name_mapping()
#if input_channel in self.name_mapping:
# return (input_channel, self.name_mapping[input_channel])
return input_channel
