def calculate_dead_fraction_all(self, output_channel, local_variables):
log.debug("Calculating dead fractions for output: %i", output_channel)
# local_variables is a list of IOVSets (one for each input channel),
# for this output channel.
# Why would you call it local_variables?
#prev_states = []
dead_frac_iovs = IOVSet()
calc_dead_frac = self.calculate_dead_fraction
# loop over smallest IOV chunks for this output channel
for since, until, states in process_iovs(self.run_iovs, *local_variables):
run_iov = states[0]
# state_iovs is now a list of iovs, one for each input channel mapped
# to this output channel
state_iovs = states[1:]
states = [s.good for s in state_iovs]
if run_iov._is_empty:
# Ignore regions outside runs.
continue
iov_state = calc_dead_frac(since, until, output_channel,
states, state_iovs)
dead_frac_iovs.add(since, until, output_channel, *iov_state)
return dead_frac_iovs.solidify(DCSOFL_IOV)
def map_channels(iovs, mapping, folder):
"""
Remap the input channel identifiers. Returns a new IOVSet
with the channel number changed according to the provided mapping
"""
# look for unmapped channels
bad_channels = set()
def has_channel(c):
result = c in mapping
if not result:
bad_channels.add(c)
return result
IOVSet = iovs.empty
iovs = IOVSet(
iov._replace(channel=mapping[iov.channel]) for iov in iovs
if has_channel(iov.channel))
if bad_channels:
log.debug("WARNING: %s has %i unmapped channels %r" %
(folder, len(bad_channels), repr(bad_channels)))
# Remove lists from channel field of iovs
iovs = flatten_channels(iovs)
# Traditional COOL ordering
iovs.sort(key=lambda iov: (iov.channel, iov.since))
return IOVSet(iovs)
def merge_input_information(self, channel, *inputs):
"""
Join up the information which was used to make a decision across
multiple variables.
"""
result = IOVSet()
for since, until, states in process_iovs(*inputs):
info = tuple(state._orig_iov[3:] for state in states)
result.add(since, until, channel, info)
return result.solidify(GoodIOV)
def run_sequential(systems, lbtime, run_iovs):
result_iovs = IOVSet()
for system in systems:
log.info(" -- running for %s", system)
system_result = run_one(system, lbtime, run_iovs)
print_system_iovs(system_result)
if system_result:
result_iovs.extend(system_result)
return result_iovs
def merge_input_variables(self, inputs):
"""
Merge multiple variables together for many channels.
Takes a list of IOVSets, one for each DCSC_Variable.
"""
result = []
info_states = IOVSet(iov_type=GoodIOV)
# Reassign inputs to be a list of dictionaries with
# Key=channel_id and Val=IOVSet
inputs = [iovs.by_channel for iovs in inputs]
# set of channel ids
all_channels = sorted(set(y for x in inputs for y in x.keys()))
tally_system_states = config.opts.tally_system_states
for channel in all_channels:
# Handle one channel at a time for variable merging
c_inputs = [x[channel] for x in inputs]
# Merge "good" state across multiple variables
result.append(self.merge_inputs(channel, *c_inputs))
if tally_system_states:
# Merge Input information across multiple variables
info_state = self.merge_input_information(channel, *c_inputs)
info_states.extend(info_state)
if tally_system_states:
# Print a tally of the states for the different systems
from DQUtils.ext import tally
def pretty(state):
return "/".join(x[0] for x in state)
chans, iovs = zip(*sorted(six.iteritems(info_states.by_channel)))
for since, until, states in process_iovs(self.run_iovs, *iovs):
if states[0]._is_empty:
# Not inside a run
continue
statetally = tally(pretty(x.good) for x in states[1:])
print(since, until, statetally)
return result
def calculate_result(self, inputs_by_output, global_variables):
"""
Terrible name for a method.
Calculate the iov extents and dead fractions for all output channels.
In other words, the IoVs to be written to DCSOFL for this subdetector.
"""
result = IOVSet(iov_type=DCSOFL_IOV)
# loop over output channel dictionary
for channel, input_iovs in sorted(six.iteritems(inputs_by_output)):
these_globals = self.select_globals(channel, global_variables)
args = channel, input_iovs, these_globals
result.extend(self.calculate_result_for_output(*args))
return result
def run_parallel(systems, lbtime, run_iovs, N):
pool = Pool(N if N > 0 else len(systems))
results = []
for system in systems:
result = pool.apply_async(run_one, (system, lbtime, run_iovs))
results.append(result)
all_results = IOVSet()
for system, result in zip(systems, map(lambda x: x.get(), results)):
log.info(" -- result for %s", system)
print_system_iovs(result)
if result:
all_results.extend(result)
return all_results
def merge_inputs(self, channel, *inputs):
"""
Merge multiple variables together for one input channel.
Each 'inputs' arg is an IOVSet that corresponds to this input channel.
"""
# inputs must correspond to and be in sync with subdetector.variables...?
result = IOVSet()
# combine the IOVSets into smallest chunks using process_iovs
for since, until, states in process_iovs(*inputs):
# Get the worst state for the list of vectors
state = self.merge_variable_states(states)
result.add(since, until, channel, state)
return result.solidify(GoodIOV)
def diff_folder(iov_range, folder, tag_1, tag_2, interesting_channels):
other_args = dict(named_channels=True)
iovs_1 = fetch_iovs(folder, tag=tag_1, *iov_range, **other_args).by_channel
iovs_2 = fetch_iovs(folder, tag=tag_2, *iov_range, **other_args).by_channel
all_channels = set(iovs_1.keys() + iovs_2.keys())
for channel in sorted(all_channels):
if channel not in interesting_channels:
continue
print "Considering", channel
iovs_1_c = iovs_1.get(channel, IOVSet())
iovs_2_c = iovs_2.get(channel, IOVSet())
diff_channel(channel, iovs_1_c, iovs_2_c)
def make_good_iovs(self, iovs):
# Filter out channels which are permanently dead
excluded_channels = self.excluded_channels
if excluded_channels:
iovs = iovs.empty(i for i in iovs
if i.channel not in excluded_channels)
chans, iovsets = iovs.chans_iovsets
result = IOVSet()
for since, until, states in process_iovs(*iovsets):
if not all(s and s.powerStatus for s in states):
result.add(since, until, self.defect_name, True, "")
return result.solidify(DefectIOV)
def quantize(self, lbtime, iovs):
iovs = [iovs_ for c, iovs_ in sorted(iovs.by_channel.iteritems())]
result = quantize_iovs_slow_mc(
lbtime, iovs, DCSC_Defect_Global_Variable.quantizing_function)
return IOVSet(
DefectIOV(*iov, comment='Automatically set') for iovi in result
for iov in iovi if iov[0].run == iov[1].run)
def calculate_virtual_defects(primary_iovs, evaluation_order,
virtual_output_channels, primary_output_channels,
since, until, ignore):
"""
Returns a list of IoVs for a given query in the normal COOL order
(sorted by channelID, then by since)
"""
if since == None: since = 0
if until == None: until = 2**63 - 1
since, until = RunLumi(since), RunLumi(until)
result = defaultdict(IOVSet)
primary_by_channel = primary_iovs.by_channel
# Copy desired primary channels to the result
for primary_channel, primary_iovs in primary_by_channel.iteritems():
if primary_channel in primary_output_channels:
if ignore and primary_channel in ignore:
continue
result[primary_channel] = primary_iovs
args = primary_by_channel, evaluation_order, since, until, ignore
# Skip vfgen loop if there is no virtual logic to compute
vfgen = [] if not evaluation_order else generate_virtual_defects(*args)
for since, until, virtual_iovs in vfgen:
if bad_iov(since, until):
continue
for output_name in virtual_output_channels:
# Consider the state of each desired output for this
# (since, until) and write it to output_iovs
iov = virtual_iovs.get(output_name)
if iov and iov.present:
result[output_name].add(since, until, *iov[2:])
# Sort them by traditional COOL sort ordering (by channelId first,
# then by since. `iovs` are already ordered by since.)
result_list = IOVSet()
for channel, iovs in sorted(result.iteritems()):
result_list.extend(iovs.solidify(DEFECT_IOV))
return result_list
def merge_globals(self, output_channel, dead_frac_iovs, global_variables):
"""
Merge together global states to decide a final code
If the dead fraction is unavailable, writes -1.
"""
result_iovs = IOVSet()
if self.run_iovs is not None:
# run_iovs are used to constrain to what runs the calculator will
# write. If there is a hole in `run_iovs`, then no records are emitted.
state_ranges = process_iovs(self.run_iovs, dead_frac_iovs,
*global_variables)
else:
state_ranges = process_iovs(dead_frac_iovs, *global_variables)
for since, until, states in state_ranges:
if self.run_iovs:
# No run_iovs were specified, so run for all available input data
run_iov, dead_frac_iov = states[:2]
if run_iov._is_empty:
# We're outside of a run, don't write an IoV.
continue
states = states[1:]
else:
dead_frac_iov = states[0]
if not dead_frac_iov._is_empty:
dead_fraction, thrust, n_config, n_working = dead_frac_iov[4:]
else:
dead_fraction, thrust, n_config, n_working = -1., 0., -1, -1
states = states[1:]
code = self.dq_worst(states)
state = dead_fraction, thrust, n_config, n_working
if code is WHITE:
# Don't write empty regions
continue
result_iovs.add(since, until, output_channel, code, *state)
return result_iovs.solidify(DCSOFL_IOV)
def quantize(self, lbtime, iovs):
"""
Needs a different quantizer. (The default DQ quantizer will do)
"""
iovs = [iovs_ for c, iovs_ in sorted(iovs.by_channel.iteritems())]
# Custom quantizer not needed
result = quantize_iovs_slow_mc(lbtime, iovs)
return IOVSet(
CodeIOV(*iov) for iovs in result for iov in iovs
if iov[0].run == iov[1].run)
def __init__(self,
fileName,
forceNew=False,
dbName='IDBSDQ',
tag='nominal',
user='******'):
"""
Initialise database connection
"""
self.defect = None
self.iovs = IOVSet()
self.user = user
#self.allowedDefects = DefectsDB('').defect_names
if not fileName: fileName = 'tmp.' + str(os.getpid()) + '.db'
self.connect(fileName, forceNew, dbName, tag)
pass
def filter_atlas_runs(iovs):
iov_runs = set(iov.since.run for iov in iovs)
first, last = min(iov_runs), max(iov_runs)
rows = make_atlas_partition_query()
rows = rows.where(first <= run_table.c.RUNNUMBER <= last)
atlas_runs = set(row.RUNNUMBER for row in rows.execute().fetchall())
keep_runs = atlas_runs.intersection(iov_runs)
return IOVSet(iov for iov in iovs if iov.since.run in keep_runs)
def quantize(self, lbtime, iovs):
"""
Quantize "good state" timewise-iovs to lumiblocks.
OUT_OF_CONFIG gets priority over BAD if BAD and OUT_OF_CONFIG overlap
the same lumiblock.
"""
IOVSet = iovs.empty
iovs = [iovs_ for c, iovs_ in sorted(iovs.by_channel.iteritems())]
def quantizer(iovs):
return min(i.good for i in iovs) if iovs else None
result = quantize_iovs_slow_mc(lbtime, iovs, quantizer)
return IOVSet(
GoodIOV(*iov) for iovs in result for iov in iovs
if iov[0].run == iov[1].run)
def make_good_iovs(self, iovs):
"""
Beware, deceptive function name. Here we are really
making OUT_OF_CONFIG iovs, rather than 'good' ones.
The resulting IoVs are in terms of detector serial number. These get
mapped onto the HV channel IDs with the `sct_sn_to_cool_ids` mapping.
"""
iovs = [iov for iov in iovs if iov.elements]
#assert all(len(iov.elements) == 1 for iov in iovs)
iovs = [
GoodIOV(iov.since, iov.until, iov.elements[0].id, OUT_OF_CONFIG)
for iov in iovs if iov.elements[0].group == -1
]
return IOVSet(iovs)
def dcs_summary(grl_iovs, dcs_iovs, quantity="dead_fraction"):
"""
Calculate minimum, maximum and weighted average for `quantity` from
the `dcs_iovs` iov sets, for lumiblocks within the `grl_iovs`.
"""
channels, dcs_iovs = zip(*sorted(dcs_iovs.by_channel.iteritems()))
# Somewhere to store IoVs which lie within a GRL
iov_sets = dict((channel, IOVSet()) for channel in channels)
# Fetch iov information from dcs_iovs within a GRL
# process_iovs pulls out common (since, until, state) efficiently, where
# `state` is a list of IoVs which are currently active, and in the same
# order as the iov sets which are passed to it.
for since, until, iovs in process_iovs(grl_iovs, *dcs_iovs):
grl_iov, dcs_states = iovs[0], iovs[1:]
if grl_iov is None:
# `grl_iov` unset? => We are outside a GRL IoV
continue
# Copy dead fraction information for iovs lying within a GRL
for channel, dcs_iov in zip(channels, dcs_states):
properties = dcs_iov.deadFrac, dcs_iov.NConfig
iov_sets[channel].add(since, until, *properties)
@define_iov_type
def DCS_IOV(dead_fraction, n_in_config):
"Represent a dead fraction"
# 'solidify' required once an iov_set has been filled
iov_sets = dict((channel, iov_set.solidify(DCS_IOV))
for channel, iov_set in iov_sets.iteritems())
result = {}
# Consider channels, then for a given channel consider all runs.
for channel in channels:
for run, iovs in sorted(iov_sets[channel].by_run.iteritems()):
lowest, average, highest = compute_min_max_avg(iovs, quantity)
run_state = RunInfo(lowest, average, highest)
result.setdefault(channel, []).append((run, run_state))
return result
def calculate_good_iovs(self, lbtime, subdetector):
magnets = Magnets()
result = magnets.run(lbtime)
by_defect = result.by_channel
toroid_ramp = by_defect.get("GLOBAL_TOROID_RAMPING", IOVSet())
solenoid_ramp = by_defect.get("GLOBAL_SOLENOID_RAMPING", IOVSet())
self.input_hashes = [hash(toroid_ramp), hash(solenoid_ramp)]
result = IOVSet()
events = process_iovs(toroid_ramp, solenoid_ramp)
for since, until, (toroid, solenoid) in events:
if toroid or solenoid:
result.add(since, until, "LCD_MAGNETS_RAMPING", True, "")
self.iovs = result.solidify(DefectIOV)
return self
def calculate_result_for_output(self, output_channel,
local_variables, global_variables):
"""
Calculate the iov extents and dead fractions for one output channel
* If there are 'non-global' variables, evaluate the dead fraction, which
effectively becomes a new global variable.
* If there are no global variables, return the above as a result
* If there are global variables, merge them together.
"""
dead_frac_iovs = IOVSet(iov_type=DCSOFL_IOV)
if local_variables:
dead_frac_iovs = self.calculate_dead_fraction_all(output_channel,
local_variables)
if not global_variables:
# There are no globals, we're done.
return dead_frac_iovs
return self.merge_globals(output_channel, dead_frac_iovs, global_variables)
def make_good_iovs(self, iovs):
"""
The absence of an IoV means that channel is out of config.
"""
seen_channels = set()
result = IOVSet()
for since, until, channel, (state, ) in process_iovs_mc(iovs):
seen_channels.add(channel)
if state._is_empty:
# There is an IoV hole for this channel.
result.add(since, until, channel, OUT_OF_CONFIG)
# Need to deal with channels which were not seen at all for the query
# range, since they will not appear in the above loop
all_channels = self.subdetector.input_channel_set
for missing_channel in (all_channels - seen_channels):
result.add(0, 2**63 - 1, missing_channel, OUT_OF_CONFIG)
return result.solidify(GoodIOV)
def main():
f1 = "%s::%s" % (db1, options.folderBS)
f2 = "%s::%s" % (db2, options.folderLumi)
print("=" * 100)
print("Comparing: ")
print(" * ", f1, options.tagBS)
print(" * ", f2, options.tagLumi)
print("=" * 100)
requiredForNtuple = ['posX', 'posY', 'posZ', 'sigmaX', 'sigmaY', 'sigmaZ']
checkNtupleProd = all(item in varColl for item in requiredForNtuple)
if not checkNtupleProd:
print('Ntuple will not be filled missing vars')
#Open up required databases
from PyCool import cool
from CoolConvUtilities import AtlCoolLib
cooldbBS = AtlCoolLib.indirectOpen(db1, True, True, False)
cooldbLumi = AtlCoolLib.indirectOpen(db2, True, True, False)
folderBS = cooldbBS.getFolder(options.folderBS)
folderLumi = cooldbLumi.getFolder(options.folderLumi)
from InDetBeamSpotExample.COOLUtils import COOLQuery
coolQuery = COOLQuery()
if options.runMin is not None:
iov1 = options.runMin << 32
if options.runMax is not None:
iov2 = (options.runMax + 1) << 32
else:
iov2 = (options.runMin + 1) << 32
print('Plotting runs %i to %i ' % (iov1, iov2))
else:
print('No run selected -- ERROR')
return
if (iov2 > cool.ValidityKeyMax):
iov2 = cool.ValidityKeyMax
print("Reading data from database")
itrBS = folderBS.browseObjects(iov1, iov2, cool.ChannelSelection.all(),
options.tagBS)
print("...finished getting BS data")
lbDict = dict()
startRLB = 0x7FFFFFFFFFFFFFFF
endRLB = 0
outfile = ROOT.TFile("BeamspotLumi_%i.root" % (options.runMin), "recreate")
ntuple = ROOT.TNtupleD(
'BeamspotLumi', 'BeamSpotLumi',
"x:y:z:sigma_x:sigma_y:sigma_z:run:mu:lumi:year:month:day:hour:minute:epoch"
)
runs = set()
while itrBS.goToNext():
obj = itrBS.currentRef()
since = obj.since()
runBegin = since >> 32
lumiBegin = since & 0xFFFFFFFF
until = obj.until()
runUntil = until >> 32
lumiUntil = until & 0xFFFFFFFF
status = int(obj.payloadValue('status'))
if status != 59:
continue
runs.add(runBegin)
if since < startRLB:
startRLB = since
if until > endRLB:
endRLB = until
values = {}
for var in varColl:
values[var] = float(obj.payloadValue(var))
values[var + 'Err'] = float(obj.payloadValue(var + 'Err'))
values['until'] = until
lbDict[since] = values
print('Runs: ', runs)
lumi = array('d')
xd = array('d')
exd = array('d')
ydDict = {}
eydDict = {}
ydDict2 = {}
sqtrt2pi = math.sqrt(2 * math.pi)
lblb = CoolDataReader('COOLONL_TRIGGER/CONDBR2', '/TRIGGER/LUMI/LBLB')
from DQUtils.sugar import RANGEIOV_VAL, RunLumi
from DQUtils import IOVSet
grlIOVs = IOVSet.from_grl(
"data15_13TeV.periodAllYear_DetStatus-v89-pro21-02_Unknown_PHYS_StandardGRL_All_Good_25ns.xml"
)
grlIOVs += IOVSet.from_grl(
"data16_13TeV.periodAllYear_DetStatus-v89-pro21-01_DQDefects-00-02-04_PHYS_StandardGRL_All_Good_25ns.xml"
)
grlIOVs += IOVSet.from_grl(
"data17_13TeV.periodAllYear_DetStatus-v99-pro22-01_Unknown_PHYS_StandardGRL_All_Good_25ns_Triggerno17e33prim.xml"
)
grlIOVs += IOVSet.from_grl(
"data18_13TeV.periodAllYear_DetStatus-v102-pro22-04_Unknown_PHYS_StandardGRL_All_Good_25ns_Triggerno17e33prim.xml"
)
for run in runs:
iov1 = run << 32
iov2 = (run + 1) << 32
itrLumi = folderLumi.browseObjects(iov1, iov2,
cool.ChannelSelection.all(),
options.tagLumi)
print("...finished getting Lumi data for run %i" % run)
lblb.setIOVRangeFromRun(run)
lblb.readData()
if len(lblb.data) < 1:
print('No LBLB data found!')
continue
# Make time map
lblbMap = dict()
for obj in lblb.data:
lblbMap[obj.since()] = (obj.payload()['StartTime'],
obj.payload()['EndTime'])
while itrLumi.goToNext():
obj = itrLumi.currentRef()
since = obj.since()
runBegin = since >> 32
lumiBegin = since & 0xFFFFFFFF
until = obj.until()
runUntil = until >> 32
lumiUntil = until & 0xFFFFFFFF
inGRL = False
for sinceGRL, untilGRL, grl_states in process_iovs(grlIOVs):
if grl_states[0].since == None:
continue
if (sinceGRL.run <= runBegin and untilGRL.run >= runUntil
and sinceGRL.lumi <= lumiBegin
and untilGRL.lumi >= lumiUntil):
inGRL = True
break
if not inGRL:
continue
mu = float(obj.payloadValue('LBAvEvtsPerBX'))
instlumi = float(obj.payloadValue('LBAvInstLumi'))
#if( mu < 10 or mu > 65 ):
#print 'Mu: %2.1f Run : %d LB: %d - %d Lumi: %f' % (mu,runBegin,lumiBegin,lumiUntil,instlumi)
if since in lbDict:
if lbDict[since]['sigmaX'] > 0.1:
continue
startTime = lblbMap.get(obj.since(), (0., 0.))[0]
endTime = lblbMap.get(lbDict[since]['until'],
(0., 0.))[0] #[1] end of lumiblock
mylumi = (endTime - startTime) / 1e9 * instlumi / 1e9
thisTime = time.gmtime(startTime / 1.e9)
year = thisTime[0]
month = thisTime[1]
day = thisTime[2]
hour = thisTime[3]
mins = thisTime[4]
sec = thisTime[5]
lumi.append(mylumi)
# in fb^-1
xd.append(mu)
exd.append(0)
if options.plotSomething:
for var in varColl:
if not var in ydDict:
ydDict[var] = array('d')
ydDict2[var] = array('d')
eydDict[var] = array('d')
ydDict2[var].append(mu /
(lbDict[since][var] * sqtrt2pi))
ydDict[var].append(lbDict[since][var])
eydDict[var].append(lbDict[since][var + 'Err'])
if checkNtupleProd and lbDict[since]['sigmaZErr'] < 5:
ntuple.Fill(lbDict[since]['posX'], lbDict[since]['posY'],
lbDict[since]['posZ'], lbDict[since]['sigmaX'],
lbDict[since]['sigmaY'],
lbDict[since]['sigmaZ'], runBegin, mu, mylumi,
year, month, day, hour, mins, startTime / 1.e9)
runStart = startRLB >> 32
runEnd = endRLB >> 32
fillStart = fillEnd = 0
timeStart = timeEnd = 0
beamEnergy = 13
try:
timeStart = coolQuery.lbTime(int(startRLB >> 32),
int(startRLB & 0xFFFFFFFF))[0]
except:
pass
try:
timeEnd = coolQuery.lbTime(int(endRLB >> 32),
int(endRLB & 0xFFFFFFFF) - 1)[1]
except:
pass
try:
fillStart = coolQuery.getLHCInfo(timeStart).get('FillNumber', 0)
except:
pass
try:
fillEnd = coolQuery.getLHCInfo(timeEnd).get('FillNumber', 0)
except:
pass
try:
beamEnergy = coolQuery.getLHCInfo(timeStart).get('BeamEnergyGeV', 0)
beamEnergy *= 2e-3
except:
pass
ntuple.Write()
if not options.plotSomething:
return
from InDetBeamSpotExample import ROOTUtils
ROOTUtils.setStyle()
canvas = ROOT.TCanvas('BeamSpotComparison', 'BeamSpotComparison', 1600,
1200)
canvas.cd()
ROOT.gPad.SetTopMargin(0.05)
ROOT.gPad.SetLeftMargin(0.15)
ROOT.gPad.SetRightMargin(0.05)
if not options.plotGraph:
ROOT.gPad.SetRightMargin(0.15)
#Plot each variable
for var in varColl:
if var not in ydDict:
print('Missing yd: ', var)
if var not in eydDict:
print('Missing eyd: ', var)
continue
gr = ROOT.TGraphErrors(len(xd), xd, ydDict[var], exd, eydDict[var])
xmin = min(xd)
xmax = max(xd)
ymin = min(ydDict[var])
ymax = max(ydDict[var])
h = (ymax - ymin)
ymin -= 0.25 * h
ymaxSmall = ymax + 0.25 * h
ymax += 0.75 * h
ymin2 = min(ydDict2[var])
ymax2 = max(ydDict2[var])
h = (ymax2 - ymin2)
ymin2 -= 0.25 * h
ymax2 += 0.75 * h
h = (xmax - xmin)
xmin -= 0.05 * h
xmax += 0.05 * h
#This histogram is made just to make it easier to manipulate the margins
histo = ROOT.TH2D('hd' + var, 'hd' + var, 100, xmin, xmax, 100, ymin,
ymax)
histo.GetYaxis().SetTitle(varDef(var, 'atit', var))
histo.GetXaxis().SetTitle('Average interactions per bunch crossing')
histo.GetZaxis().SetTitle('Entries')
histo2 = ROOT.TH2D('hd2' + var, 'hd2' + var, 100, xmin, xmax, 100,
ymin2, ymax2)
histo2.GetYaxis().SetTitle(
"<Interaction density> @ z=0 [interactions/mm]")
histo2.GetXaxis().SetTitle('Average interactions per bunch crossing')
histo2.GetZaxis().SetTitle('Entries')
histo2W = ROOT.TH2D('hd3' + var, 'hd3' + var, 100, xmin, xmax, 100,
ymin2, ymax2)
histo2W.GetYaxis().SetTitle(
"<Interaction density> @ z=0 [interactions/mm]")
histo2W.GetXaxis().SetTitle('Average interactions per bunch crossing')
histo2W.GetZaxis().SetTitle('Integrated Luminosity (fb^{-1}/bin)')
histoW = ROOT.TH2D('hdW' + var, 'hdW' + var, 100, xmin, xmax, 100,
ymin, ymax)
histoW.GetYaxis().SetTitle(varDef(var, 'atit', var))
histoW.GetXaxis().SetTitle('Average interactions per bunch crossing')
histoW.GetZaxis().SetTitle('Integrated Luminosity (fb^{-1}/bin)')
histoW1D = ROOT.TH1D('hd1D' + var, 'hd1D' + var, 100, ymin, ymaxSmall)
histoW1D.GetXaxis().SetTitle(varDef(var, 'atit', var))
histoW1D.GetYaxis().SetTitle('Integrated Luminosity (fb^{-1}/bin)')
histo.Draw()
if options.plotGraph:
gr.Draw("p")
else:
for mu, x, l in zip(xd, ydDict[var], lumi):
histo.Fill(mu, x)
histoW.Fill(mu, x, l)
histoW1D.Fill(x, l)
for mu, x, l in zip(xd, ydDict2[var], lumi):
histo2.Fill(mu, x)
histo2W.Fill(mu, x, l)
histo.Draw("colz")
histo.Write()
histoW.Write()
histo2.Write()
histo2W.Write()
histoW1D.Write()
# Add some information to the graph
ROOTUtils.atlasLabel(0.53,
0.87,
False,
offset=0.12,
isForApproval=False,
customstring="Internal",
energy='%2.0f' % beamEnergy,
size=0.055)
ROOTUtils.drawText(0.18, 0.87, 0.055, varDef(var, 'title', var))
comments = []
if runStart == runEnd:
comments.append('Run %i' % runStart)
else:
comments.append('Runs %i - %i' % (runStart, runEnd))
if fillStart == fillEnd:
comments.append('Fill %i' % fillStart)
else:
comments.append('Fills %i - %i' % (fillStart, fillEnd))
t1 = time.strftime('%d %b %Y', time.localtime(timeStart))
t2 = time.strftime('%d %b %Y', time.localtime(timeEnd))
if t1 == t2:
comments.append(t1)
else:
comments.append('%s - %s' % (t1, t2))
ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42)
canvas.Print("Run_%d_%sVsMu.png" % (options.runMin, var))
canvas.Print("Run_%d_%sVsMu.pdf" % (options.runMin, var))
if not options.plotGraph:
canvas.SetLogz(True)
canvas.Print("Run_%d_%sVsMuLog.png" % (options.runMin, var))
canvas.Print("Run_%d_%sVsMuLog.pdf" % (options.runMin, var))
canvas.SetLogz(False)
histo2.Draw("colz")
ROOTUtils.atlasLabel(0.53,
0.87,
False,
offset=0.12,
isForApproval=False,
customstring="Internal",
energy='%2.0f' % beamEnergy,
size=0.055)
ROOTUtils.drawText(0.18, 0.87, 0.055, "Interaction density")
ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42)
canvas.Print("Run_%d_Mu%sVsMu.png" % (options.runMin, var))
canvas.Print("Run_%d_Mu%sVsMu.pdf" % (options.runMin, var))
canvas.SetLogz(True)
canvas.Print("Run_%d_Mu%sVsMuLog.png" % (options.runMin, var))
canvas.Print("Run_%d_Mu%sVsMuLog.pdf" % (options.runMin, var))
canvas.SetLogz(False)
histoW.Draw("colz")
histoW.SetMinimum(0.005)
ROOTUtils.atlasLabel(0.53,
0.87,
False,
offset=0.12,
isForApproval=False,
customstring="Internal",
energy='%2.0f' % beamEnergy,
size=0.055)
ROOTUtils.drawText(0.18, 0.87, 0.055, varDef(var, 'title', var))
ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42)
canvas.Print("Run_%d_%sVsMuW.png" % (options.runMin, var))
canvas.Print("Run_%d_%sVsMuW.pdf" % (options.runMin, var))
canvas.SetLogz(True)
canvas.Print("Run_%d_%sVsMuWLog.png" % (options.runMin, var))
canvas.Print("Run_%d_%sVsMuWLog.pdf" % (options.runMin, var))
canvas.SetLogz(False)
histo2W.Draw("colz")
histo2W.SetMinimum(0.01)
ROOTUtils.atlasLabel(0.53,
0.87,
False,
offset=0.12,
isForApproval=False,
customstring="Internal",
energy='%2.0f' % beamEnergy,
size=0.055)
ROOTUtils.drawText(0.18, 0.87, 0.055, "Interaction density")
ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42)
canvas.Print("Run_%d_Mu%sVsMuW.png" % (options.runMin, var))
canvas.Print("Run_%d_Mu%sVsMuW.pdf" % (options.runMin, var))
canvas.SetLogz(True)
canvas.Print("Run_%d_Mu%sVsMuWLog.png" % (options.runMin, var))
canvas.Print("Run_%d_Mu%sVsMuWLog.pdf" % (options.runMin, var))
canvas.SetLogz(False)
histoW1D.Draw("colz")
ROOTUtils.atlasLabel(0.53,
0.87,
False,
offset=0.12,
isForApproval=False,
customstring="Internal",
energy='%2.0f' % beamEnergy,
size=0.055)
ROOTUtils.drawText(0.18, 0.87, 0.055, varDef(var, 'title', var))
ROOTUtils.drawText(0.18,
0.81,
0.05,
"#mu=%2.4f RMS=%2.4f" %
(histoW1D.GetMean(), histoW1D.GetRMS()),
font=42)
canvas.Print("Run_%d_%s1D.png" % (options.runMin, var))
canvas.Print("Run_%d_%s1D.pdf" % (options.runMin, var))
canvas.SetLogy(True)
canvas.Print("Run_%d_%s1DLog.png" % (options.runMin, var))
canvas.Print("Run_%d_%s1DLog.pdf" % (options.runMin, var))
canvas.SetLogy(False)
def make_good_iovs(self, iovs):
atlsol_iovs = self.magnet_iov_generator(iovs, 'GLOBAL_SOLENOID', 1, 2,
TOLERANCE_SOLENOID)
atltor_iovs = self.magnet_iov_generator(iovs, 'GLOBAL_TOROID', 3, 4,
TOLERANCE_TOROID)
return IOVSet(list(atlsol_iovs) + list(atltor_iovs))
def make_good_iovs(self, iovs):
"""
Determine whether each iov signifies a good or bad state.
"""
make_good_iov = self.make_good_iov
return IOVSet(make_good_iov(iov) for iov in iovs)
def start(self):
self.bad_modules = IOVSet()
class Pixels(DCSC_DefectTranslate_Subdetector):
def __init__(self, *args, **kwargs):
super(Pixels, self).__init__(*args, **kwargs)
self.translators = [Pixels.pix_color_to_defect_translator]
@staticmethod
def pix_color_to_defect_translator(flag_iovs):
from DCSCalculator2.consts import GREEN
from DCSCalculator2.variable import DefectIOV
from DQUtils import process_iovs
rv = []
defect_mapping = {
101: "PIXEL_BARREL_STANDBY",
102: "PIXEL_LAYER0_STANDBY",
103: "PIXEL_IBL_STANDBY",
104: "PIXEL_ENDCAPA_STANDBY",
105: "PIXEL_ENDCAPC_STANDBY"
}
for since, until, states in process_iovs(
*flag_iovs.by_channel.values()):
#print states
for state in states:
if state.Code != GREEN:
badfrac = 'Standby module fraction: ' + str(state.deadFrac)
rv.append(
DefectIOV(since=since,
until=until,
channel=defect_mapping[state.channel],
present=True,
comment=badfrac))
return rv
input_db = "COOLOFL_DCS/CONDBR2"
folder_base = "/PIXEL/DCS"
cid_barrel, cid_blayer, cid_ibl = 101, 102, 103
cid_endcapa, cid_endcapb = 104, 105
mapping = {
101: range(722, 1892),
102: range(436, 722),
103: range(156, 436),
104: range(1892, 2036),
105: range(12, 156),
}
variables = [
DCSC_Variable("FSMSTATUS", lambda iov: iov.FSM_status in
("OK", "WARNING")),
DCSC_Variable("FSMSTATE", lambda iov: iov.FSM_state == "READY"),
]
# Note barrel and blayer use different deadfraction_caution. Implemented in
# calculate_iov below.
dead_fraction_caution = 0.2
dead_fraction_caution_barrel = 0.05
dead_fraction_bad = 0.9
assert dead_fraction_caution_barrel <= dead_fraction_caution, (
"logic needs changing in calculate_iov before removing this assert")
def start(self):
self.bad_modules = IOVSet()
def tally_additional_info(self, since, until, output_channel, states,
state_iovs):
"""
Figure out which modules are OOC when
"""
ooc_modules = self.get_ids_which_are(output_channel, states,
OUT_OF_CONFIG)
ooc_modules = tuple(sorted(ooc_modules))
self.bad_modules.add(since, until, output_channel, ooc_modules)
def done(self):
if logger.isEnabledFor(logging.DEBUG):
print "The following ranges indicate bad modules:"
#from pprint import pprint
#pprint(list(self.bad_modules))
def calculate_dead_fraction(self, since, until, output_channel, states,
state_iovs):
"""
Compute the dead fraction differently for the barrel and blayer.
This function is written with the assumption that
`dead_fraction_caution_barrel` is smaller than `dead_fraction_caution`
because the new logic is only run if the code is better than Yellow.
"""
cdf = super(Pixels, self).calculate_dead_fraction
result = cdf(since, until, output_channel, states, state_iovs)
self.tally_additional_info(since, until, output_channel, states,
state_iovs)
code, dead_fraction, thrust, n_config, n_working = result
if (output_channel not in [self.cid_barrel, self.cid_blayer]
or code <= YELLOW):
# Result is already correct
return result
# Need to check whether code needs modifying.
if self.dead_fraction_caution_barrel < dead_fraction < self.dead_fraction_bad:
code = YELLOW
return code, dead_fraction, thrust, n_config, n_working
def initialize_name_mapping(self):
from DQUtils.db import Databases, get_channel_ids_names
f = Databases.get_folder("/TDAQ/EnabledResources/ATLAS/PIXEL/Modules",
"COOLONL_TDAQ")
cids, cnames, cmap = get_channel_ids_names(f)
self.name_mapping = dict(zip(cids, cnames))
self.name_mapping_initialized = True
# TODO: is this ok??
def get_name_for_input_channel(self, input_channel):
"""
Transform an input channelid into a name
"""
#if not getattr(self, "name_mapping_initialized", False):
# self.initialize_name_mapping()
#if input_channel in self.name_mapping:
# return (input_channel, self.name_mapping[input_channel])
return input_channel
def run(self, lbtime, run_iovs):
self.evaluate_inputs(lbtime)
result = IOVSet()
for variable in self.variables:
result.extend(variable.iovs)
return result
class IDBSDefectWriter:
"""
Class for writing BS defects to an sqlite file
"""
def __init__(self,
fileName,
forceNew=False,
dbName='IDBSDQ',
tag='nominal',
user='******'):
"""
Initialise database connection
"""
self.defect = None
self.iovs = IOVSet()
self.user = user
#self.allowedDefects = DefectsDB('').defect_names
if not fileName: fileName = 'tmp.' + str(os.getpid()) + '.db'
self.connect(fileName, forceNew, dbName, tag)
pass
def __del__(self):
"""
Delete db to clear connection
"""
os.system('[[ -f tmp.%s.db ]] && rm tmp.%s.db' %
(os.getpid(), os.getpid()))
del self.db
pass
def connect(self,
fileName,
forceNew=False,
dbName='IDBSDQ',
tag='nominal'):
"""
Open connection to defect DB
"""
connString = 'sqlite://;schema=%s;dbname=%s' % (fileName, dbName)
if forceNew and os.path.exists(fileName):
os.remove(fileName)
self.db = DefectsDB(
connString, read_only=False, create=True, tag=(tag, 'HEAD')
) # Second tag is for virtual defects, which we are not interested in
self.officialDb = DefectsDB()
return
def defectType(self, t):
"""
Set defect type
"""
self.defect = t
def add(self,
runMin=0,
runMax=(1 << 31) - 1,
lbMin=0,
lbMax=(1 << 32) - 1):
"""
Add iovs which are NOT defective to the list
Note, lbMax is exclusive here (and inclusive when shown on defect web page).
"""
# Make since and until and convert to syntactially nice RunLumiType
since = RunLumiType((runMin << 32) + lbMin)
until = RunLumiType((runMax << 32) + lbMax)
# IoVs which are not defective (i.e. beamspot good)
self.iovs.add(since, until, self.defect, False)
return
def complete(self, runMin, runMax):
"""
Complete a list of IoVs to cover all LBs in a run, treating empty ones as having 'emptyState'
"""
# Create an IOV set covering the entire run(s)
run_lbs = fetch_iovs("EOR",
runs=(runMin, runMax),
what=[],
with_channel=False)
# run_lbs = IOVSet()
# lbMin = 1
# lbMax = (1 << 32) -1 # Note, lbMax is exclusive
# since = RunLumiType((runMin << 32)+lbMin)
# until = RunLumiType((runMax << 32)+lbMax)
# run_lbs.add(since, until)
if not len(run_lbs):
print "WARNING: No LBs in run according to EOR_Params - are we running before the run has finished?"
def lbsStartAtOne(iov):
"Change LBs starting at 0 to start at 1"
return iov._replace(since=RunLumi(iov.since.run, 1))
# Start LBs from 1 rather than 0 (at request of P. Onyisi) as long as run has more than one LB (else skip)
run_lbs = [lbsStartAtOne(iov) for iov in run_lbs if iov.until.lumi > 1]
# Empty IOV set for adding full list of LBs too
iovs = IOVSet()
# Ask official DB if an IoV is currently defective so can unset only those if doing reprocessing
defectsCurrentlyInDb = self.officialDb.retrieve(
(runMin, 0), (runMax + 1, 0), [self.defect])
# Order IOVs to avoid since > until
self.iovs = IOVSet(sorted(self.iovs))
#for since, until, (run_lb, iov, dbDefect) in process_iovs(run_lbs.solidify(RANGEIOV_VAL), self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb):
for since, until, (run_lb, iov, dbDefect) in process_iovs(
run_lbs, self.iovs.solidify(DEFECTIOV_VAL),
defectsCurrentlyInDb):
if not run_lb: continue # Check valid
# # Start LBs from 1 rather than 0 (at request of P. Onyisi)
# # If this makes since==until, i.e. it was a [0->1) LB range then skip
# if since.lumi==0:
# since = RunLumiType((since.run << 32)+since.lumi+1)
# if since==until: continue
# Add iovs from self.iovs treating empty ones as defective (i.e. beamspot bad/missing)
if iov:
# These are not defective
if dbDefect and dbDefect.present:
# Only store not defective IoVs if they are present on the Db so we can unset them
# (don't want to write not present defects to Db unless really chaning an existing defect)
iovs.add(since, until, self.defect, False)
else:
# These are defective
iovs.add(since, until, self.defect, True)
self.iovs = iovs
return
def writeDefects(self, tag='nominal', nonpresent=False):
"""
Write all defects to the database. If 'nonpresent' is True then write the absent ones too
"""
with self.db.storage_buffer:
for since, until, defect, present in self.iovs:
if not present and not nonpresent: continue
#print since, until, present
self._writeDefect(defect, since, until, present=present)
def _writeDefect(self,
defect,
since,
until,
tag='nominal',
description='',
comment='',
present=True,
recoverable=False):
"""
Write a single defect to the database
"""
if not defect in self.db.defect_names:
self.db.create_defect(defect, description)
self.db.insert(defect, since, until, comment, self.user, present,
recoverable)
return
def dump(self, filename=None):
"""
Dump defects to a file given by filename or stdout if no filename given
"""
from DQUtils.utils import pprint_objects
if filename is not None:
f = open(filename.replace('.db', '.txt'), "w")
# If not defects then nothing will be in the database and we write an empty file
if len(self.iovs):
pprint_objects(
self.db.retrieve(primary_only=True, nonpresent=True), f)
f.close()
else:
if len(self.iovs):
self.iovs.pprint()
else:
print '\nNo DQ defects'
# with open(filename.replace('.db', '.txt'), "w") as f:
# pprint_objects(self.db.retrieve(), f)
return
def complete(self, runMin, runMax):
"""
Complete a list of IoVs to cover all LBs in a run, treating empty ones as having 'emptyState'
"""
# Create an IOV set covering the entire run(s)
run_lbs = fetch_iovs("EOR",
runs=(runMin, runMax),
what=[],
with_channel=False)
# run_lbs = IOVSet()
# lbMin = 1
# lbMax = (1 << 32) -1 # Note, lbMax is exclusive
# since = RunLumiType((runMin << 32)+lbMin)
# until = RunLumiType((runMax << 32)+lbMax)
# run_lbs.add(since, until)
if not len(run_lbs):
print "WARNING: No LBs in run according to EOR_Params - are we running before the run has finished?"
def lbsStartAtOne(iov):
"Change LBs starting at 0 to start at 1"
return iov._replace(since=RunLumi(iov.since.run, 1))
# Start LBs from 1 rather than 0 (at request of P. Onyisi) as long as run has more than one LB (else skip)
run_lbs = [lbsStartAtOne(iov) for iov in run_lbs if iov.until.lumi > 1]
# Empty IOV set for adding full list of LBs too
iovs = IOVSet()
# Ask official DB if an IoV is currently defective so can unset only those if doing reprocessing
defectsCurrentlyInDb = self.officialDb.retrieve(
(runMin, 0), (runMax + 1, 0), [self.defect])
# Order IOVs to avoid since > until
self.iovs = IOVSet(sorted(self.iovs))
#for since, until, (run_lb, iov, dbDefect) in process_iovs(run_lbs.solidify(RANGEIOV_VAL), self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb):
for since, until, (run_lb, iov, dbDefect) in process_iovs(
run_lbs, self.iovs.solidify(DEFECTIOV_VAL),
defectsCurrentlyInDb):
if not run_lb: continue # Check valid
# # Start LBs from 1 rather than 0 (at request of P. Onyisi)
# # If this makes since==until, i.e. it was a [0->1) LB range then skip
# if since.lumi==0:
# since = RunLumiType((since.run << 32)+since.lumi+1)
# if since==until: continue
# Add iovs from self.iovs treating empty ones as defective (i.e. beamspot bad/missing)
if iov:
# These are not defective
if dbDefect and dbDefect.present:
# Only store not defective IoVs if they are present on the Db so we can unset them
# (don't want to write not present defects to Db unless really chaning an existing defect)
iovs.add(since, until, self.defect, False)
else:
# These are defective
iovs.add(since, until, self.defect, True)
self.iovs = iovs
return