def tdaq_busy_generator(self, iovs):
events = process_iovs(iovs)
counter = 0
for since, until, (state, ) in events:
if state.Run == 0: continue
#print state
if state is not None:
deadfrac = 1 - state.LiveFraction
if deadfrac < self.deadfraction_threshold:
continue
yield DefectIOV(RunLumi(state.Run, state.LumiBlock),
RunLumi(state.Run, state.LumiBlock + 1),
'GLOBAL_BUSY',
True,
comment='Average live fraction %.1f%%' %
((1 - deadfrac) * 100))
counter += 1
counter_max = counter
counter = 0
#print counter_max
events = process_iovs(iovs)
for since, until, (state, ) in events:
if state is not None:
deadfrac = 1 - state.LiveFraction
if deadfrac < self.deadfraction_threshold:
continue
#print state.Run
counter += 1
if state.Run == 0 and counter < counter_max:
print 'ERROR: Wrong run number in LumiAccounting; here is the IOV: '
print state
#print list(iovs)
continue
def diff_channel(channel, iovs_1, iovs_2):
for since, until, (iov_1, iov_2) in process_iovs(iovs_1, iovs_2):
if until.lumi == 0 or since.lumi == 0xFFFFFFFF:
# Kill silly IoV ranges
continue
if iov_1 is None and iov_2 is None:
# Both are empty
continue
if iov_1 is None or iov_2 is None:
# One or the other is empty
print " %r to %r:" % (since, until), make_pretty(
iov_1), make_pretty(iov_2)
continue
elif iov_1._contents == iov_2._contents:
# They are both equal (no difference)
continue
# There is a difference.
print " %r and %r:" % (since, until),
print "%s => %s" % (iov_1.Code, iov_2.Code)
def compute_lumi(lbs, lumis, iovs, exclude_iovsets=[], good_runs=None):
"""
Compute luminosity for an IOVSet iovs using lbs, lumis from fetch_lumi_inputs.
exclude_iovsets is a list of IOVSets for channels to be excluded from the
calculation such as not_ready, busy, etc.
good_runs is an optional set of runs to consider, such as the set of runs
which are OK for physics.
"""
# This is the old simple approach, which doesn't have a lot of features
#inputs = process_iovs(lbs, lumis, iovs)
#return sum((lb.EndTime - lb.StartTime)/1e9 * lumi.LBAvInstLumi
#for since, until, (lb, lumi, good) in inputs
#if lb and lumi and good)
inputs = process_iovs(lbs, lumis, iovs, *exclude_iovsets)
total_lumi = 0
for since, until, states in inputs:
lb, lumi, good, excludes = states[0], states[1], states[2], states[3:]
# Skip this iov if requested info not available
# or if any excluded channels are present
if not lb or not lumi or not good or any(excludes):
continue
# Optional set of runs to consider. Reject if not in set
if good_runs and since.run not in good_runs:
continue
total_lumi += (lb.EndTime - lb.StartTime) / 1e9 * lumi.LBAvInstLumi
return total_lumi
def main():
runs = 185640, 185660
lblb = fetch_iovs("LBLB", runs=runs)
lbtime = inverse_lblb(lblb)
beam_iovs = fetch_iovs("COOLOFL_DCS::/LHC/DCS/FILLSTATE", lbtime.first.since, lbtime.last.until)
stable_beam_by_lb = {}
for since_time, until_time, (lumiblock, beamstate) in process_iovs(lbtime, beam_iovs):
if not lumiblock:
# Not inside lumiblock
continue
lb = RunLumi(lumiblock.Run, lumiblock.LumiBlock)
if lb in stable_beam_by_lb:
# Require stable beam for ALL of the lumiblock (hence 'and')
# Replace with 'or' to get ANY of the lumiblock
stable_beam_by_lb[lb] = stable_beam_by_lb[lb] and beamstate.StableBeams
else:
stable_beam_by_lb[lb] = beamstate.StableBeams
@define_iov_type
def STABLEBEAM_VAL(stable):
"""
Define an IOV type which just has .stable
"""
result = (STABLEBEAM_VAL(runlb, runlb+1, state)
for runlb, state in sorted(stable_beam_by_lb.iteritems()))
result = IOVSet(connect_adjacent_iovs(result))
result.pprint()
def pix_color_to_defect_translator(flag_iovs):
from DCSCalculator2.consts import GREEN
from DCSCalculator2.variable import DefectIOV
from DQUtils import process_iovs
rv = []
defect_mapping = {
101: "PIXEL_BARREL_STANDBY",
102: "PIXEL_LAYER0_STANDBY",
103: "PIXEL_IBL_STANDBY",
104: "PIXEL_ENDCAPA_STANDBY",
105: "PIXEL_ENDCAPC_STANDBY"
}
for since, until, states in process_iovs(
*flag_iovs.by_channel.values()):
#print states
for state in states:
if state.Code != GREEN:
badfrac = 'Standby module fraction: ' + str(state.deadFrac)
rv.append(
DefectIOV(since=since,
until=until,
channel=defect_mapping[state.channel],
present=True,
comment=badfrac))
return rv
def calculate_dead_fraction_all(self, output_channel, local_variables):
log.debug("Calculating dead fractions for output: %i", output_channel)
# local_variables is a list of IOVSets (one for each input channel),
# for this output channel.
# Why would you call it local_variables?
#prev_states = []
dead_frac_iovs = IOVSet()
calc_dead_frac = self.calculate_dead_fraction
# loop over smallest IOV chunks for this output channel
for since, until, states in process_iovs(self.run_iovs, *local_variables):
run_iov = states[0]
# state_iovs is now a list of iovs, one for each input channel mapped
# to this output channel
state_iovs = states[1:]
states = [s.good for s in state_iovs]
if run_iov._is_empty:
# Ignore regions outside runs.
continue
iov_state = calc_dead_frac(since, until, output_channel,
states, state_iovs)
dead_frac_iovs.add(since, until, output_channel, *iov_state)
return dead_frac_iovs.solidify(DCSOFL_IOV)
def make_defectdiff_iovs(defects1, defects2, system_only,
ignore_not_considered):
"""
Return a list of IoVs containing the set of defects only present in
defects1 or defects2
"""
result = IOVSet()
defectset = ((since, until, (t1.defects if t1 else set(),
t2.defects if t2 else set()))
for since, until, (t1,
t2) in process_iovs(defects1, defects2))
for since, until, (t1defects, t2defects) in defectset:
if t1defects == t2defects:
continue
diff = t1defects.symmetric_difference(t2defects)
if ignore_not_considered and "GLOBAL_NOTCONSIDERED" in diff:
continue
tag1only, tag2only = t1defects & diff, t2defects & diff
if system_only:
tag1only = set(x.split("_")[0] for x in tag1only)
tag2only = set(x.split("_")[0] for x in tag2only)
result.add(since, until, tag1only, tag2only)
return result.solidify(DEFECTDIFF_VAL)
def merge_globals(self, output_channel, dead_frac_iovs, global_variables):
"""
Merge together global states to decide a final code
If the dead fraction is unavailable, writes -1.
"""
result_iovs = IOVSet()
if self.run_iovs is not None:
# run_iovs are used to constrain to what runs the calculator will
# write. If there is a hole in `run_iovs`, then no records are emitted.
state_ranges = process_iovs(self.run_iovs, dead_frac_iovs,
*global_variables)
else:
state_ranges = process_iovs(dead_frac_iovs, *global_variables)
for since, until, states in state_ranges:
if self.run_iovs:
# No run_iovs were specified, so run for all available input data
run_iov, dead_frac_iov = states[:2]
if run_iov._is_empty:
# We're outside of a run, don't write an IoV.
continue
states = states[1:]
else:
dead_frac_iov = states[0]
if not dead_frac_iov._is_empty:
dead_fraction, thrust, n_config, n_working = dead_frac_iov[4:]
else:
dead_fraction, thrust, n_config, n_working = -1., 0., -1, -1
states = states[1:]
code = self.dq_worst(states)
state = dead_fraction, thrust, n_config, n_working
if code is WHITE:
# Don't write empty regions
continue
result_iovs.add(since, until, output_channel, code, *state)
return result_iovs.solidify(DCSOFL_IOV)
def test_processing():
"""
Test that processing works
"""
iovs = fetch_iovs("SHIFTOFL", tag="DetStatusSHIFTOFL-pass1-analysis-2010B")
channels, iovsets = iovs.chans_iovsets
processed = [(since, until, states[:])
for since, until, states in process_iovs(*iovsets)]
return processed
def merge_input_information(self, channel, *inputs):
"""
Join up the information which was used to make a decision across
multiple variables.
"""
result = IOVSet()
for since, until, states in process_iovs(*inputs):
info = tuple(state._orig_iov[3:] for state in states)
result.add(since, until, channel, info)
return result.solidify(GoodIOV)
def beamspot_generator(self, iovs):
state = iovs.select_channels(0)
events = process_iovs(iovs)
for since, until, (state,) in events:
if state is not None:
#print since, until, state.status
if state.status == 7 or state.status is None:
continue
yield DefectIOV(since, until, 'TRIG_HLT_IDT_BSPOT_INVALID_STATUS', True,
comment='Bad online beamspot status')
def _defectForLB(self, lb):
"""
Get the DQ defects for the given LB from the full run info
"""
defects = []
for since, until, states in process_iovs(*self.iovsets):
if since.lumi <= lb < until.lumi:
# defects is true if set for a given run/LB
defects = [state.channel for state in states if state]
return list(set(defects))
def make_defectset_iovs(range_iovs, defect_iovs):
"""
Return a list of iovs with a set for each IoV containing the present defects
"""
chans, iovsets = defect_iovs.chans_iovsets
result = IOVSet()
for since, until, states in process_iovs(range_iovs, *iovsets):
in_range, defects = states[0], states[1:]
if in_range:
result.add(since, until, set(d.channel for d in defects if d))
return result.solidify(DEFECTSET_VAL)
def tdaq_busy_generator(self, iovs):
events = process_iovs(iovs)
counter = 0
for since, until, (state, ) in events:
if state.Run == 0 or state.Run is None: continue
if state is not None:
if state.LiveFraction is None:
deadfrac = 1
log.warning('WARNING: LiveFraction is "None" for %d %d',
state.Run, state.LumiBlock)
else:
deadfrac = 1 - state.LiveFraction
if deadfrac < self.deadfraction_threshold:
continue
yield DefectIOV(RunLumi(state.Run, state.LumiBlock),
RunLumi(state.Run, state.LumiBlock + 1),
'GLOBAL_BUSY',
True,
comment='Average live fraction %.1f%%' %
((1 - deadfrac) * 100))
counter += 1
counter_max = counter
counter = 0
events = process_iovs(iovs)
for since, until, (state, ) in events:
if state is not None and state.Run is not None:
deadfrac = 1 - state.LiveFraction
if deadfrac < self.deadfraction_threshold:
continue
counter += 1
if state.Run == 0 and counter < counter_max:
log.error(
'ERROR: Wrong run number in LumiAccounting; here is the IOV: '
)
log.error(state)
continue
def show_variable_info(lbtime, Subdetector):
subdetector = Subdetector()
for variable in subdetector.variables:
print "Processing variable:", variable.folder_name
good_iovs = variable.calculate_good_iovs(lbtime, subdetector).iovs
chans, iovs = good_iovs.chans_iovsets
if isinstance(variable, DCSC_Global_Variable):
for since, until, states in process_iovs(*iovs):
channels = [(state.channel, state.Code) for state in states
if state and state.Code < 3]
print since, until, pretty_ranges(channels)
else:
for since, until, states in process_iovs(*iovs):
channels = [
state.channel for state in states
if state and not state.good
]
print since, until, pretty_ranges(channels)
def merge_input_variables(self, inputs):
"""
Merge multiple variables together for many channels.
Takes a list of IOVSets, one for each DCSC_Variable.
"""
result = []
info_states = IOVSet(iov_type=GoodIOV)
# Reassign inputs to be a list of dictionaries with
# Key=channel_id and Val=IOVSet
inputs = [iovs.by_channel for iovs in inputs]
# set of channel ids
all_channels = sorted(set(y for x in inputs for y in x.keys()))
tally_system_states = config.opts.tally_system_states
for channel in all_channels:
# Handle one channel at a time for variable merging
c_inputs = [x[channel] for x in inputs]
# Merge "good" state across multiple variables
result.append(self.merge_inputs(channel, *c_inputs))
if tally_system_states:
# Merge Input information across multiple variables
info_state = self.merge_input_information(channel, *c_inputs)
info_states.extend(info_state)
if tally_system_states:
# Print a tally of the states for the different systems
from DQUtils.ext import tally
def pretty(state):
return "/".join(x[0] for x in state)
chans, iovs = zip(*sorted(six.iteritems(info_states.by_channel)))
for since, until, states in process_iovs(self.run_iovs, *iovs):
if states[0]._is_empty:
# Not inside a run
continue
statetally = tally(pretty(x.good) for x in states[1:])
print(since, until, statetally)
return result
def test_serializer():
"""
Try serializing a fairly complicated iovset object
"""
iovs = fetch_iovs("SHIFTOFL", runs=152166)
channels, iovsets = iovs.chans_iovsets
processed = [(since, until, states)
for since, until, states in process_iovs(*iovsets)]
serialized = dumps(processed).encode("bz2").encode("base64")
deserialized = loads(serialized.decode("base64").decode("bz2"))
assert deserialized == processed
def emittance_generator(self, iovs):
events = process_iovs(iovs)
for since, until, (state, ) in events:
#print state, state.RunLB & 0xffffffff if state.RunLB else 0
if state is not None and state.RunLB is not None:
thisrun = state.RunLB >> 32
# pseudo-LB and not to be trusted
if thisrun == 0: continue
thisLB = state.RunLB & 0xffffffff
yield DefectIOV(RunLumi(thisrun, thisLB),
RunLumi(thisrun, thisLB + 1),
'LUMI_EMITTANCESCAN',
True,
comment='Emittance scan')
def merge_inputs(self, channel, *inputs):
"""
Merge multiple variables together for one input channel.
Each 'inputs' arg is an IOVSet that corresponds to this input channel.
"""
# inputs must correspond to and be in sync with subdetector.variables...?
result = IOVSet()
# combine the IOVSets into smallest chunks using process_iovs
for since, until, states in process_iovs(*inputs):
# Get the worst state for the list of vectors
state = self.merge_variable_states(states)
result.add(since, until, channel, state)
return result.solidify(GoodIOV)
def compute_grl_diff(inputs, *iovsets):
"""
Returns a list of IoVs which differ for GRLs
"""
result = IOVSet()
for since, until, grl_states in process_iovs(*iovsets):
has_difference = any(grl_states) != all(grl_states)
if has_difference:
set_iovs = [d for d, state in zip(inputs, grl_states) if state]
unset_iovs = [
d for d, state in zip(inputs, grl_states) if not state
]
result.add(since, until, set_iovs, unset_iovs)
return result.solidify(GRLSTATE_VAL)
def make_good_iovs(self, iovs):
# Filter out channels which are permanently dead
excluded_channels = self.excluded_channels
if excluded_channels:
iovs = iovs.empty(i for i in iovs
if i.channel not in excluded_channels)
chans, iovsets = iovs.chans_iovsets
result = IOVSet()
for since, until, states in process_iovs(*iovsets):
if not all(s and s.powerStatus for s in states):
result.add(since, until, self.defect_name, True, "")
return result.solidify(DefectIOV)
def tdaq_ready_generator(self, iovs):
#state = iovs.select_channels(0)
events = process_iovs(iovs)
for since, until, (state, ) in events:
if state is not None:
if state.ReadyForPhysics == 1:
continue
yield DefectIOV(since,
until,
'GLOBAL_NOTREADY',
True,
comment='ReadyForPhysics not set')
def dcs_summary(grl_iovs, dcs_iovs, quantity="dead_fraction"):
"""
Calculate minimum, maximum and weighted average for `quantity` from
the `dcs_iovs` iov sets, for lumiblocks within the `grl_iovs`.
"""
channels, dcs_iovs = zip(*sorted(dcs_iovs.by_channel.iteritems()))
# Somewhere to store IoVs which lie within a GRL
iov_sets = dict((channel, IOVSet()) for channel in channels)
# Fetch iov information from dcs_iovs within a GRL
# process_iovs pulls out common (since, until, state) efficiently, where
# `state` is a list of IoVs which are currently active, and in the same
# order as the iov sets which are passed to it.
for since, until, iovs in process_iovs(grl_iovs, *dcs_iovs):
grl_iov, dcs_states = iovs[0], iovs[1:]
if grl_iov is None:
# `grl_iov` unset? => We are outside a GRL IoV
continue
# Copy dead fraction information for iovs lying within a GRL
for channel, dcs_iov in zip(channels, dcs_states):
properties = dcs_iov.deadFrac, dcs_iov.NConfig
iov_sets[channel].add(since, until, *properties)
@define_iov_type
def DCS_IOV(dead_fraction, n_in_config):
"Represent a dead fraction"
# 'solidify' required once an iov_set has been filled
iov_sets = dict((channel, iov_set.solidify(DCS_IOV))
for channel, iov_set in iov_sets.iteritems())
result = {}
# Consider channels, then for a given channel consider all runs.
for channel in channels:
for run, iovs in sorted(iov_sets[channel].by_run.iteritems()):
lowest, average, highest = compute_min_max_avg(iovs, quantity)
run_state = RunInfo(lowest, average, highest)
result.setdefault(channel, []).append((run, run_state))
return result
def magnet_iov_generator(self, iovs, system, measured_channel,
desired_channel, tolerance):
measured_iovs = iovs.select_channels(measured_channel)
desired_iovs = iovs.select_channels(desired_channel)
events = process_iovs(measured_iovs, desired_iovs)
for since, until, (measured, desired) in events:
# 28-05-2015: excluding empty 'desired' value, because how do we make
# a decision without an expectation? Should debug this some more, as
# the issue came up in 2015 run 253014.
if measured is not None and desired is not None and not desired._is_empty:
# NOTE: if measured is 'empty', this is always true
if measured.value <= tolerance:
# Magnet off
defect = system + '_OFF'
elif abs(measured.value - desired.value) <= tolerance:
if ((system == 'GLOBAL_SOLENOID'
and abs(desired.value - 7730.) > tolerance)
or (system == 'GLOBAL_TOROID'
and abs(desired.value - 20400.) > tolerance)):
# Magnet has non-nominal current
defect = system + '_NOTNOMINAL'
else:
defect = None
else:
# Magnet is ramping
defect = system + '_RAMPING'
if defect is None:
continue
mcurrent = '%.1f' % measured.value if measured.value is not None else 'None'
scurrent = '%.1f' % desired.value if desired.value is not None else 'None'
yield DefectIOV(
since,
until,
defect,
True,
comment='Measured current: %s, Set current: %s' %
(mcurrent, scurrent))
def calculate_good_iovs(self, lbtime, subdetector):
magnets = Magnets()
result = magnets.run(lbtime)
by_defect = result.by_channel
toroid_ramp = by_defect.get("GLOBAL_TOROID_RAMPING", IOVSet())
solenoid_ramp = by_defect.get("GLOBAL_SOLENOID_RAMPING", IOVSet())
self.input_hashes = [hash(toroid_ramp), hash(solenoid_ramp)]
result = IOVSet()
events = process_iovs(toroid_ramp, solenoid_ramp)
for since, until, (toroid, solenoid) in events:
if toroid or solenoid:
result.add(since, until, "LCD_MAGNETS_RAMPING", True, "")
self.iovs = result.solidify(DefectIOV)
return self
def sct_color_to_defect_translator(flag_iovs):
defectmap = {111: 'B', 114: 'EA', 115: 'EC'}
from DCSCalculator2.consts import GREEN
from DCSCalculator2.variable import DefectIOV
from DQUtils import process_iovs
rv = []
for since, until, states in process_iovs(
*flag_iovs.by_channel.values()):
if any(state.Code != GREEN for state in states):
# This line of code is awful!
badfrac = 'Bad fractions: ' + ', '.join(
'%s %.3f' % (defectmap[state.channel], state.deadFrac)
for state in sorted(states, key=lambda x: x.channel))
rv.append(
DefectIOV(since=since,
until=until,
channel='SCT_GLOBAL_STANDBY',
present=True,
comment=badfrac))
return rv
def compute_lumi_many_channels(lbs,
lumis,
iovs,
exclude_iovsets=[],
good_runs=None):
"""
Compute luminosity per channel for an IOVSet iovs using
lbs, lumis from fetch_lumi_inputs.
exclude_iovsets is a list of IOVSets for channels to be excluded from the
calculation such as not_ready, busy, etc.
good_runs is an optional set of runs to consider, such as the set of runs
which are OK for physics.
Return type is a dictionary of {channel name: lumi}
"""
# Split the IOVSet by channel
chans, iovsets = iovs.chans_iovsets
result = {}
# Old method, keeping it here for now
#inputs = process_iovs(lbs, lumis, *iovsets)
#for since, until, states in inputs:
#(lb, lumi), defects = states[:2], states[2:]
#if not (lb and lumi and lumi.LBAvInstLumi):
#continue
#lumi = (lb.EndTime - lb.StartTime)/1e9 * lumi.LBAvInstLumi
#for name, defect_iov in zip(chans, defects):
#if defect_iov:
#result[name] = result.get(name, 0) + lumi
# New approach, adding new functionality
num_exclude = len(exclude_iovsets)
full_iovsets = exclude_iovsets
full_iovsets.extend(iovsets)
inputs = process_iovs(lbs, lumis, *full_iovsets)
for since, until, states in inputs:
# I find the below version conceptually cleaner
#(lb, lumi), excludes, defects = (states[:2], states[2:2+num_exclude],
#states[2+num_exclude:])
(lb, lumi), defectstates = states[:2], states[2:]
excludes, defects = defectstates[:num_exclude], defectstates[
num_exclude:]
# Skip this iov if requested info not available
# or if any excluded channels are present
#if not (lb and lumi and lumi.LBAvInstLumi):
if not lb or not lumi or any(excludes):
continue
# Optional set of runs to consider. Reject if not in set
if good_runs and since.run not in good_runs:
continue
# Lumi for this lb
lumi = (lb.EndTime - lb.StartTime) / 1e9 * lumi.LBAvInstLumi
# Loop over channels present, add lumi
for name, defect_iov in zip(chans, defects):
if defect_iov:
result[name] = result.get(name, 0) + lumi
return result
def main():
f1 = "%s::%s" % (db1, options.folderBS)
f2 = "%s::%s" % (db2, options.folderLumi)
print("=" * 100)
print("Comparing: ")
print(" * ", f1, options.tagBS)
print(" * ", f2, options.tagLumi)
print("=" * 100)
requiredForNtuple = ['posX', 'posY', 'posZ', 'sigmaX', 'sigmaY', 'sigmaZ']
checkNtupleProd = all(item in varColl for item in requiredForNtuple)
if not checkNtupleProd:
print('Ntuple will not be filled missing vars')
#Open up required databases
from PyCool import cool
from CoolConvUtilities import AtlCoolLib
cooldbBS = AtlCoolLib.indirectOpen(db1, True, True, False)
cooldbLumi = AtlCoolLib.indirectOpen(db2, True, True, False)
folderBS = cooldbBS.getFolder(options.folderBS)
folderLumi = cooldbLumi.getFolder(options.folderLumi)
from InDetBeamSpotExample.COOLUtils import COOLQuery
coolQuery = COOLQuery()
if options.runMin is not None:
iov1 = options.runMin << 32
if options.runMax is not None:
iov2 = (options.runMax + 1) << 32
else:
iov2 = (options.runMin + 1) << 32
print('Plotting runs %i to %i ' % (iov1, iov2))
else:
print('No run selected -- ERROR')
return
if (iov2 > cool.ValidityKeyMax):
iov2 = cool.ValidityKeyMax
print("Reading data from database")
itrBS = folderBS.browseObjects(iov1, iov2, cool.ChannelSelection.all(),
options.tagBS)
print("...finished getting BS data")
lbDict = dict()
startRLB = 0x7FFFFFFFFFFFFFFF
endRLB = 0
outfile = ROOT.TFile("BeamspotLumi_%i.root" % (options.runMin), "recreate")
ntuple = ROOT.TNtupleD(
'BeamspotLumi', 'BeamSpotLumi',
"x:y:z:sigma_x:sigma_y:sigma_z:run:mu:lumi:year:month:day:hour:minute:epoch"
)
runs = set()
while itrBS.goToNext():
obj = itrBS.currentRef()
since = obj.since()
runBegin = since >> 32
lumiBegin = since & 0xFFFFFFFF
until = obj.until()
runUntil = until >> 32
lumiUntil = until & 0xFFFFFFFF
status = int(obj.payloadValue('status'))
if status != 59:
continue
runs.add(runBegin)
if since < startRLB:
startRLB = since
if until > endRLB:
endRLB = until
values = {}
for var in varColl:
values[var] = float(obj.payloadValue(var))
values[var + 'Err'] = float(obj.payloadValue(var + 'Err'))
values['until'] = until
lbDict[since] = values
print('Runs: ', runs)
lumi = array('d')
xd = array('d')
exd = array('d')
ydDict = {}
eydDict = {}
ydDict2 = {}
sqtrt2pi = math.sqrt(2 * math.pi)
lblb = CoolDataReader('COOLONL_TRIGGER/CONDBR2', '/TRIGGER/LUMI/LBLB')
from DQUtils.sugar import RANGEIOV_VAL, RunLumi
from DQUtils import IOVSet
grlIOVs = IOVSet.from_grl(
"data15_13TeV.periodAllYear_DetStatus-v89-pro21-02_Unknown_PHYS_StandardGRL_All_Good_25ns.xml"
)
grlIOVs += IOVSet.from_grl(
"data16_13TeV.periodAllYear_DetStatus-v89-pro21-01_DQDefects-00-02-04_PHYS_StandardGRL_All_Good_25ns.xml"
)
grlIOVs += IOVSet.from_grl(
"data17_13TeV.periodAllYear_DetStatus-v99-pro22-01_Unknown_PHYS_StandardGRL_All_Good_25ns_Triggerno17e33prim.xml"
)
grlIOVs += IOVSet.from_grl(
"data18_13TeV.periodAllYear_DetStatus-v102-pro22-04_Unknown_PHYS_StandardGRL_All_Good_25ns_Triggerno17e33prim.xml"
)
for run in runs:
iov1 = run << 32
iov2 = (run + 1) << 32
itrLumi = folderLumi.browseObjects(iov1, iov2,
cool.ChannelSelection.all(),
options.tagLumi)
print("...finished getting Lumi data for run %i" % run)
lblb.setIOVRangeFromRun(run)
lblb.readData()
if len(lblb.data) < 1:
print('No LBLB data found!')
continue
# Make time map
lblbMap = dict()
for obj in lblb.data:
lblbMap[obj.since()] = (obj.payload()['StartTime'],
obj.payload()['EndTime'])
while itrLumi.goToNext():
obj = itrLumi.currentRef()
since = obj.since()
runBegin = since >> 32
lumiBegin = since & 0xFFFFFFFF
until = obj.until()
runUntil = until >> 32
lumiUntil = until & 0xFFFFFFFF
inGRL = False
for sinceGRL, untilGRL, grl_states in process_iovs(grlIOVs):
if grl_states[0].since == None:
continue
if (sinceGRL.run <= runBegin and untilGRL.run >= runUntil
and sinceGRL.lumi <= lumiBegin
and untilGRL.lumi >= lumiUntil):
inGRL = True
break
if not inGRL:
continue
mu = float(obj.payloadValue('LBAvEvtsPerBX'))
instlumi = float(obj.payloadValue('LBAvInstLumi'))
#if( mu < 10 or mu > 65 ):
#print 'Mu: %2.1f Run : %d LB: %d - %d Lumi: %f' % (mu,runBegin,lumiBegin,lumiUntil,instlumi)
if since in lbDict:
if lbDict[since]['sigmaX'] > 0.1:
continue
startTime = lblbMap.get(obj.since(), (0., 0.))[0]
endTime = lblbMap.get(lbDict[since]['until'],
(0., 0.))[0] #[1] end of lumiblock
mylumi = (endTime - startTime) / 1e9 * instlumi / 1e9
thisTime = time.gmtime(startTime / 1.e9)
year = thisTime[0]
month = thisTime[1]
day = thisTime[2]
hour = thisTime[3]
mins = thisTime[4]
sec = thisTime[5]
lumi.append(mylumi)
# in fb^-1
xd.append(mu)
exd.append(0)
if options.plotSomething:
for var in varColl:
if not var in ydDict:
ydDict[var] = array('d')
ydDict2[var] = array('d')
eydDict[var] = array('d')
ydDict2[var].append(mu /
(lbDict[since][var] * sqtrt2pi))
ydDict[var].append(lbDict[since][var])
eydDict[var].append(lbDict[since][var + 'Err'])
if checkNtupleProd and lbDict[since]['sigmaZErr'] < 5:
ntuple.Fill(lbDict[since]['posX'], lbDict[since]['posY'],
lbDict[since]['posZ'], lbDict[since]['sigmaX'],
lbDict[since]['sigmaY'],
lbDict[since]['sigmaZ'], runBegin, mu, mylumi,
year, month, day, hour, mins, startTime / 1.e9)
runStart = startRLB >> 32
runEnd = endRLB >> 32
fillStart = fillEnd = 0
timeStart = timeEnd = 0
beamEnergy = 13
try:
timeStart = coolQuery.lbTime(int(startRLB >> 32),
int(startRLB & 0xFFFFFFFF))[0]
except:
pass
try:
timeEnd = coolQuery.lbTime(int(endRLB >> 32),
int(endRLB & 0xFFFFFFFF) - 1)[1]
except:
pass
try:
fillStart = coolQuery.getLHCInfo(timeStart).get('FillNumber', 0)
except:
pass
try:
fillEnd = coolQuery.getLHCInfo(timeEnd).get('FillNumber', 0)
except:
pass
try:
beamEnergy = coolQuery.getLHCInfo(timeStart).get('BeamEnergyGeV', 0)
beamEnergy *= 2e-3
except:
pass
ntuple.Write()
if not options.plotSomething:
return
from InDetBeamSpotExample import ROOTUtils
ROOTUtils.setStyle()
canvas = ROOT.TCanvas('BeamSpotComparison', 'BeamSpotComparison', 1600,
1200)
canvas.cd()
ROOT.gPad.SetTopMargin(0.05)
ROOT.gPad.SetLeftMargin(0.15)
ROOT.gPad.SetRightMargin(0.05)
if not options.plotGraph:
ROOT.gPad.SetRightMargin(0.15)
#Plot each variable
for var in varColl:
if var not in ydDict:
print('Missing yd: ', var)
if var not in eydDict:
print('Missing eyd: ', var)
continue
gr = ROOT.TGraphErrors(len(xd), xd, ydDict[var], exd, eydDict[var])
xmin = min(xd)
xmax = max(xd)
ymin = min(ydDict[var])
ymax = max(ydDict[var])
h = (ymax - ymin)
ymin -= 0.25 * h
ymaxSmall = ymax + 0.25 * h
ymax += 0.75 * h
ymin2 = min(ydDict2[var])
ymax2 = max(ydDict2[var])
h = (ymax2 - ymin2)
ymin2 -= 0.25 * h
ymax2 += 0.75 * h
h = (xmax - xmin)
xmin -= 0.05 * h
xmax += 0.05 * h
#This histogram is made just to make it easier to manipulate the margins
histo = ROOT.TH2D('hd' + var, 'hd' + var, 100, xmin, xmax, 100, ymin,
ymax)
histo.GetYaxis().SetTitle(varDef(var, 'atit', var))
histo.GetXaxis().SetTitle('Average interactions per bunch crossing')
histo.GetZaxis().SetTitle('Entries')
histo2 = ROOT.TH2D('hd2' + var, 'hd2' + var, 100, xmin, xmax, 100,
ymin2, ymax2)
histo2.GetYaxis().SetTitle(
"<Interaction density> @ z=0 [interactions/mm]")
histo2.GetXaxis().SetTitle('Average interactions per bunch crossing')
histo2.GetZaxis().SetTitle('Entries')
histo2W = ROOT.TH2D('hd3' + var, 'hd3' + var, 100, xmin, xmax, 100,
ymin2, ymax2)
histo2W.GetYaxis().SetTitle(
"<Interaction density> @ z=0 [interactions/mm]")
histo2W.GetXaxis().SetTitle('Average interactions per bunch crossing')
histo2W.GetZaxis().SetTitle('Integrated Luminosity (fb^{-1}/bin)')
histoW = ROOT.TH2D('hdW' + var, 'hdW' + var, 100, xmin, xmax, 100,
ymin, ymax)
histoW.GetYaxis().SetTitle(varDef(var, 'atit', var))
histoW.GetXaxis().SetTitle('Average interactions per bunch crossing')
histoW.GetZaxis().SetTitle('Integrated Luminosity (fb^{-1}/bin)')
histoW1D = ROOT.TH1D('hd1D' + var, 'hd1D' + var, 100, ymin, ymaxSmall)
histoW1D.GetXaxis().SetTitle(varDef(var, 'atit', var))
histoW1D.GetYaxis().SetTitle('Integrated Luminosity (fb^{-1}/bin)')
histo.Draw()
if options.plotGraph:
gr.Draw("p")
else:
for mu, x, l in zip(xd, ydDict[var], lumi):
histo.Fill(mu, x)
histoW.Fill(mu, x, l)
histoW1D.Fill(x, l)
for mu, x, l in zip(xd, ydDict2[var], lumi):
histo2.Fill(mu, x)
histo2W.Fill(mu, x, l)
histo.Draw("colz")
histo.Write()
histoW.Write()
histo2.Write()
histo2W.Write()
histoW1D.Write()
# Add some information to the graph
ROOTUtils.atlasLabel(0.53,
0.87,
False,
offset=0.12,
isForApproval=False,
customstring="Internal",
energy='%2.0f' % beamEnergy,
size=0.055)
ROOTUtils.drawText(0.18, 0.87, 0.055, varDef(var, 'title', var))
comments = []
if runStart == runEnd:
comments.append('Run %i' % runStart)
else:
comments.append('Runs %i - %i' % (runStart, runEnd))
if fillStart == fillEnd:
comments.append('Fill %i' % fillStart)
else:
comments.append('Fills %i - %i' % (fillStart, fillEnd))
t1 = time.strftime('%d %b %Y', time.localtime(timeStart))
t2 = time.strftime('%d %b %Y', time.localtime(timeEnd))
if t1 == t2:
comments.append(t1)
else:
comments.append('%s - %s' % (t1, t2))
ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42)
canvas.Print("Run_%d_%sVsMu.png" % (options.runMin, var))
canvas.Print("Run_%d_%sVsMu.pdf" % (options.runMin, var))
if not options.plotGraph:
canvas.SetLogz(True)
canvas.Print("Run_%d_%sVsMuLog.png" % (options.runMin, var))
canvas.Print("Run_%d_%sVsMuLog.pdf" % (options.runMin, var))
canvas.SetLogz(False)
histo2.Draw("colz")
ROOTUtils.atlasLabel(0.53,
0.87,
False,
offset=0.12,
isForApproval=False,
customstring="Internal",
energy='%2.0f' % beamEnergy,
size=0.055)
ROOTUtils.drawText(0.18, 0.87, 0.055, "Interaction density")
ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42)
canvas.Print("Run_%d_Mu%sVsMu.png" % (options.runMin, var))
canvas.Print("Run_%d_Mu%sVsMu.pdf" % (options.runMin, var))
canvas.SetLogz(True)
canvas.Print("Run_%d_Mu%sVsMuLog.png" % (options.runMin, var))
canvas.Print("Run_%d_Mu%sVsMuLog.pdf" % (options.runMin, var))
canvas.SetLogz(False)
histoW.Draw("colz")
histoW.SetMinimum(0.005)
ROOTUtils.atlasLabel(0.53,
0.87,
False,
offset=0.12,
isForApproval=False,
customstring="Internal",
energy='%2.0f' % beamEnergy,
size=0.055)
ROOTUtils.drawText(0.18, 0.87, 0.055, varDef(var, 'title', var))
ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42)
canvas.Print("Run_%d_%sVsMuW.png" % (options.runMin, var))
canvas.Print("Run_%d_%sVsMuW.pdf" % (options.runMin, var))
canvas.SetLogz(True)
canvas.Print("Run_%d_%sVsMuWLog.png" % (options.runMin, var))
canvas.Print("Run_%d_%sVsMuWLog.pdf" % (options.runMin, var))
canvas.SetLogz(False)
histo2W.Draw("colz")
histo2W.SetMinimum(0.01)
ROOTUtils.atlasLabel(0.53,
0.87,
False,
offset=0.12,
isForApproval=False,
customstring="Internal",
energy='%2.0f' % beamEnergy,
size=0.055)
ROOTUtils.drawText(0.18, 0.87, 0.055, "Interaction density")
ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42)
canvas.Print("Run_%d_Mu%sVsMuW.png" % (options.runMin, var))
canvas.Print("Run_%d_Mu%sVsMuW.pdf" % (options.runMin, var))
canvas.SetLogz(True)
canvas.Print("Run_%d_Mu%sVsMuWLog.png" % (options.runMin, var))
canvas.Print("Run_%d_Mu%sVsMuWLog.pdf" % (options.runMin, var))
canvas.SetLogz(False)
histoW1D.Draw("colz")
ROOTUtils.atlasLabel(0.53,
0.87,
False,
offset=0.12,
isForApproval=False,
customstring="Internal",
energy='%2.0f' % beamEnergy,
size=0.055)
ROOTUtils.drawText(0.18, 0.87, 0.055, varDef(var, 'title', var))
ROOTUtils.drawText(0.18,
0.81,
0.05,
"#mu=%2.4f RMS=%2.4f" %
(histoW1D.GetMean(), histoW1D.GetRMS()),
font=42)
canvas.Print("Run_%d_%s1D.png" % (options.runMin, var))
canvas.Print("Run_%d_%s1D.pdf" % (options.runMin, var))
canvas.SetLogy(True)
canvas.Print("Run_%d_%s1DLog.png" % (options.runMin, var))
canvas.Print("Run_%d_%s1DLog.pdf" % (options.runMin, var))
canvas.SetLogy(False)
def main():
f1 = "%s::%s" % (db1, options.folder)
f2 = "%s::%s" % (db2, options.folder)
print "=" * 100
print "Comparing: "
print " * ", f1, options.tag1
print " * ", f2, options.tag2
print "=" * 100
if options.runMin is not None:
runs = options.runMin
if options.runMax is not None:
runs = options.runMin, options.runMax
tag1iovs = fetch_iovs(f1, runs=runs, tag=options.tag1)
tag2iovs = fetch_iovs(f2, runs=runs, tag=options.tag2)
affected_runs = set()
ar2 = set()
#runiovs = IOVSet()
#runiovs.add(RunLumiType((runs << 32)+0), RunLumiType((runs << 32)+(1 << 32)-1))
# Loop over iovs in the two tags
# for since, until, (t1, t2, r) in process_iovs(tag1iovs, tag2iovs, runiovs.solidify(RANGEIOV_VAL)):
for since, until, (t1, t2) in process_iovs(tag1iovs, tag2iovs):
# Ignore IoVs spanning runs
if not options.span and since.run != until.run: continue
# Make sure run exists in both
#if not t1.since or not t2.since: continue
# Only consider runs in the range interested in
# if not r: continue
# Check for IOVs with different status (one of which is non-zero)
if (t1.status == 0 or t2.status == 0) and (t1.status != t2.status):
if options.statusonly and not options.summary:
print since, until, t1.status, t2.status
ar2.add(since.run)
# Check for differences in all fields
if not options.statusonly:
for i, field in enumerate(t1._fields):
if t1[i] != t2[i]:
#if field in ['posX', 'posY'] and abs(t1[i]-t2[i]) > 0.1e-3:
#if field not in ['status'] and t1[i] != 0 and abs(t1[i]-t2[i])/t1[i] > 0.001:
#if field in ['posZ'] and t1[i] != 0 and abs(t1[i]-t2[i])/t1[i] > 0.05:
#if field in ['sigmaX', 'sigmaY'] and abs(t1[i]-t2[i]) > 0.2e-3:
#if field in ['sigmaZ'] and abs(t1[i]-t2[i])/t1[i] > 0.002:
if not options.summary:
print since, until, field, t1[i], t2[i]
affected_runs.add(since.run)
# Print results
if not options.statusonly:
print "Runs with differences (%s):" % len(affected_runs), ", ".join(
map(str, sorted(affected_runs)))
print "Runs with different status (and one non-zero):", ", ".join(
map(str, sorted(ar2)))
def complete(self, runMin, runMax):
"""
Complete a list of IoVs to cover all LBs in a run, treating empty ones as having 'emptyState'
"""
# Create an IOV set covering the entire run(s)
run_lbs = fetch_iovs("EOR",
runs=(runMin, runMax),
what=[],
with_channel=False)
# run_lbs = IOVSet()
# lbMin = 1
# lbMax = (1 << 32) -1 # Note, lbMax is exclusive
# since = RunLumiType((runMin << 32)+lbMin)
# until = RunLumiType((runMax << 32)+lbMax)
# run_lbs.add(since, until)
if not len(run_lbs):
print "WARNING: No LBs in run according to EOR_Params - are we running before the run has finished?"
def lbsStartAtOne(iov):
"Change LBs starting at 0 to start at 1"
return iov._replace(since=RunLumi(iov.since.run, 1))
# Start LBs from 1 rather than 0 (at request of P. Onyisi) as long as run has more than one LB (else skip)
run_lbs = [lbsStartAtOne(iov) for iov in run_lbs if iov.until.lumi > 1]
# Empty IOV set for adding full list of LBs too
iovs = IOVSet()
# Ask official DB if an IoV is currently defective so can unset only those if doing reprocessing
defectsCurrentlyInDb = self.officialDb.retrieve(
(runMin, 0), (runMax + 1, 0), [self.defect])
# Order IOVs to avoid since > until
self.iovs = IOVSet(sorted(self.iovs))
#for since, until, (run_lb, iov, dbDefect) in process_iovs(run_lbs.solidify(RANGEIOV_VAL), self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb):
for since, until, (run_lb, iov, dbDefect) in process_iovs(
run_lbs, self.iovs.solidify(DEFECTIOV_VAL),
defectsCurrentlyInDb):
if not run_lb: continue # Check valid
# # Start LBs from 1 rather than 0 (at request of P. Onyisi)
# # If this makes since==until, i.e. it was a [0->1) LB range then skip
# if since.lumi==0:
# since = RunLumiType((since.run << 32)+since.lumi+1)
# if since==until: continue
# Add iovs from self.iovs treating empty ones as defective (i.e. beamspot bad/missing)
if iov:
# These are not defective
if dbDefect and dbDefect.present:
# Only store not defective IoVs if they are present on the Db so we can unset them
# (don't want to write not present defects to Db unless really chaning an existing defect)
iovs.add(since, until, self.defect, False)
else:
# These are defective
iovs.add(since, until, self.defect, True)
self.iovs = iovs
return
