def tdaq_busy_generator(self, iovs): events = process_iovs(iovs) counter = 0 for since, until, (state, ) in events: if state.Run == 0: continue #print state if state is not None: deadfrac = 1 - state.LiveFraction if deadfrac < self.deadfraction_threshold: continue yield DefectIOV(RunLumi(state.Run, state.LumiBlock), RunLumi(state.Run, state.LumiBlock + 1), 'GLOBAL_BUSY', True, comment='Average live fraction %.1f%%' % ((1 - deadfrac) * 100)) counter += 1 counter_max = counter counter = 0 #print counter_max events = process_iovs(iovs) for since, until, (state, ) in events: if state is not None: deadfrac = 1 - state.LiveFraction if deadfrac < self.deadfraction_threshold: continue #print state.Run counter += 1 if state.Run == 0 and counter < counter_max: print 'ERROR: Wrong run number in LumiAccounting; here is the IOV: ' print state #print list(iovs) continue
def diff_channel(channel, iovs_1, iovs_2): for since, until, (iov_1, iov_2) in process_iovs(iovs_1, iovs_2): if until.lumi == 0 or since.lumi == 0xFFFFFFFF: # Kill silly IoV ranges continue if iov_1 is None and iov_2 is None: # Both are empty continue if iov_1 is None or iov_2 is None: # One or the other is empty print " %r to %r:" % (since, until), make_pretty( iov_1), make_pretty(iov_2) continue elif iov_1._contents == iov_2._contents: # They are both equal (no difference) continue # There is a difference. print " %r and %r:" % (since, until), print "%s => %s" % (iov_1.Code, iov_2.Code)
def compute_lumi(lbs, lumis, iovs, exclude_iovsets=[], good_runs=None): """ Compute luminosity for an IOVSet iovs using lbs, lumis from fetch_lumi_inputs. exclude_iovsets is a list of IOVSets for channels to be excluded from the calculation such as not_ready, busy, etc. good_runs is an optional set of runs to consider, such as the set of runs which are OK for physics. """ # This is the old simple approach, which doesn't have a lot of features #inputs = process_iovs(lbs, lumis, iovs) #return sum((lb.EndTime - lb.StartTime)/1e9 * lumi.LBAvInstLumi #for since, until, (lb, lumi, good) in inputs #if lb and lumi and good) inputs = process_iovs(lbs, lumis, iovs, *exclude_iovsets) total_lumi = 0 for since, until, states in inputs: lb, lumi, good, excludes = states[0], states[1], states[2], states[3:] # Skip this iov if requested info not available # or if any excluded channels are present if not lb or not lumi or not good or any(excludes): continue # Optional set of runs to consider. Reject if not in set if good_runs and since.run not in good_runs: continue total_lumi += (lb.EndTime - lb.StartTime) / 1e9 * lumi.LBAvInstLumi return total_lumi
def main(): runs = 185640, 185660 lblb = fetch_iovs("LBLB", runs=runs) lbtime = inverse_lblb(lblb) beam_iovs = fetch_iovs("COOLOFL_DCS::/LHC/DCS/FILLSTATE", lbtime.first.since, lbtime.last.until) stable_beam_by_lb = {} for since_time, until_time, (lumiblock, beamstate) in process_iovs(lbtime, beam_iovs): if not lumiblock: # Not inside lumiblock continue lb = RunLumi(lumiblock.Run, lumiblock.LumiBlock) if lb in stable_beam_by_lb: # Require stable beam for ALL of the lumiblock (hence 'and') # Replace with 'or' to get ANY of the lumiblock stable_beam_by_lb[lb] = stable_beam_by_lb[lb] and beamstate.StableBeams else: stable_beam_by_lb[lb] = beamstate.StableBeams @define_iov_type def STABLEBEAM_VAL(stable): """ Define an IOV type which just has .stable """ result = (STABLEBEAM_VAL(runlb, runlb+1, state) for runlb, state in sorted(stable_beam_by_lb.iteritems())) result = IOVSet(connect_adjacent_iovs(result)) result.pprint()
def pix_color_to_defect_translator(flag_iovs): from DCSCalculator2.consts import GREEN from DCSCalculator2.variable import DefectIOV from DQUtils import process_iovs rv = [] defect_mapping = { 101: "PIXEL_BARREL_STANDBY", 102: "PIXEL_LAYER0_STANDBY", 103: "PIXEL_IBL_STANDBY", 104: "PIXEL_ENDCAPA_STANDBY", 105: "PIXEL_ENDCAPC_STANDBY" } for since, until, states in process_iovs( *flag_iovs.by_channel.values()): #print states for state in states: if state.Code != GREEN: badfrac = 'Standby module fraction: ' + str(state.deadFrac) rv.append( DefectIOV(since=since, until=until, channel=defect_mapping[state.channel], present=True, comment=badfrac)) return rv
def calculate_dead_fraction_all(self, output_channel, local_variables): log.debug("Calculating dead fractions for output: %i", output_channel) # local_variables is a list of IOVSets (one for each input channel), # for this output channel. # Why would you call it local_variables? #prev_states = [] dead_frac_iovs = IOVSet() calc_dead_frac = self.calculate_dead_fraction # loop over smallest IOV chunks for this output channel for since, until, states in process_iovs(self.run_iovs, *local_variables): run_iov = states[0] # state_iovs is now a list of iovs, one for each input channel mapped # to this output channel state_iovs = states[1:] states = [s.good for s in state_iovs] if run_iov._is_empty: # Ignore regions outside runs. continue iov_state = calc_dead_frac(since, until, output_channel, states, state_iovs) dead_frac_iovs.add(since, until, output_channel, *iov_state) return dead_frac_iovs.solidify(DCSOFL_IOV)
def make_defectdiff_iovs(defects1, defects2, system_only, ignore_not_considered): """ Return a list of IoVs containing the set of defects only present in defects1 or defects2 """ result = IOVSet() defectset = ((since, until, (t1.defects if t1 else set(), t2.defects if t2 else set())) for since, until, (t1, t2) in process_iovs(defects1, defects2)) for since, until, (t1defects, t2defects) in defectset: if t1defects == t2defects: continue diff = t1defects.symmetric_difference(t2defects) if ignore_not_considered and "GLOBAL_NOTCONSIDERED" in diff: continue tag1only, tag2only = t1defects & diff, t2defects & diff if system_only: tag1only = set(x.split("_")[0] for x in tag1only) tag2only = set(x.split("_")[0] for x in tag2only) result.add(since, until, tag1only, tag2only) return result.solidify(DEFECTDIFF_VAL)
def merge_globals(self, output_channel, dead_frac_iovs, global_variables): """ Merge together global states to decide a final code If the dead fraction is unavailable, writes -1. """ result_iovs = IOVSet() if self.run_iovs is not None: # run_iovs are used to constrain to what runs the calculator will # write. If there is a hole in `run_iovs`, then no records are emitted. state_ranges = process_iovs(self.run_iovs, dead_frac_iovs, *global_variables) else: state_ranges = process_iovs(dead_frac_iovs, *global_variables) for since, until, states in state_ranges: if self.run_iovs: # No run_iovs were specified, so run for all available input data run_iov, dead_frac_iov = states[:2] if run_iov._is_empty: # We're outside of a run, don't write an IoV. continue states = states[1:] else: dead_frac_iov = states[0] if not dead_frac_iov._is_empty: dead_fraction, thrust, n_config, n_working = dead_frac_iov[4:] else: dead_fraction, thrust, n_config, n_working = -1., 0., -1, -1 states = states[1:] code = self.dq_worst(states) state = dead_fraction, thrust, n_config, n_working if code is WHITE: # Don't write empty regions continue result_iovs.add(since, until, output_channel, code, *state) return result_iovs.solidify(DCSOFL_IOV)
def test_processing(): """ Test that processing works """ iovs = fetch_iovs("SHIFTOFL", tag="DetStatusSHIFTOFL-pass1-analysis-2010B") channels, iovsets = iovs.chans_iovsets processed = [(since, until, states[:]) for since, until, states in process_iovs(*iovsets)] return processed
def merge_input_information(self, channel, *inputs): """ Join up the information which was used to make a decision across multiple variables. """ result = IOVSet() for since, until, states in process_iovs(*inputs): info = tuple(state._orig_iov[3:] for state in states) result.add(since, until, channel, info) return result.solidify(GoodIOV)
def beamspot_generator(self, iovs): state = iovs.select_channels(0) events = process_iovs(iovs) for since, until, (state,) in events: if state is not None: #print since, until, state.status if state.status == 7 or state.status is None: continue yield DefectIOV(since, until, 'TRIG_HLT_IDT_BSPOT_INVALID_STATUS', True, comment='Bad online beamspot status')
def _defectForLB(self, lb): """ Get the DQ defects for the given LB from the full run info """ defects = [] for since, until, states in process_iovs(*self.iovsets): if since.lumi <= lb < until.lumi: # defects is true if set for a given run/LB defects = [state.channel for state in states if state] return list(set(defects))
def make_defectset_iovs(range_iovs, defect_iovs): """ Return a list of iovs with a set for each IoV containing the present defects """ chans, iovsets = defect_iovs.chans_iovsets result = IOVSet() for since, until, states in process_iovs(range_iovs, *iovsets): in_range, defects = states[0], states[1:] if in_range: result.add(since, until, set(d.channel for d in defects if d)) return result.solidify(DEFECTSET_VAL)
def tdaq_busy_generator(self, iovs): events = process_iovs(iovs) counter = 0 for since, until, (state, ) in events: if state.Run == 0 or state.Run is None: continue if state is not None: if state.LiveFraction is None: deadfrac = 1 log.warning('WARNING: LiveFraction is "None" for %d %d', state.Run, state.LumiBlock) else: deadfrac = 1 - state.LiveFraction if deadfrac < self.deadfraction_threshold: continue yield DefectIOV(RunLumi(state.Run, state.LumiBlock), RunLumi(state.Run, state.LumiBlock + 1), 'GLOBAL_BUSY', True, comment='Average live fraction %.1f%%' % ((1 - deadfrac) * 100)) counter += 1 counter_max = counter counter = 0 events = process_iovs(iovs) for since, until, (state, ) in events: if state is not None and state.Run is not None: deadfrac = 1 - state.LiveFraction if deadfrac < self.deadfraction_threshold: continue counter += 1 if state.Run == 0 and counter < counter_max: log.error( 'ERROR: Wrong run number in LumiAccounting; here is the IOV: ' ) log.error(state) continue
def show_variable_info(lbtime, Subdetector): subdetector = Subdetector() for variable in subdetector.variables: print "Processing variable:", variable.folder_name good_iovs = variable.calculate_good_iovs(lbtime, subdetector).iovs chans, iovs = good_iovs.chans_iovsets if isinstance(variable, DCSC_Global_Variable): for since, until, states in process_iovs(*iovs): channels = [(state.channel, state.Code) for state in states if state and state.Code < 3] print since, until, pretty_ranges(channels) else: for since, until, states in process_iovs(*iovs): channels = [ state.channel for state in states if state and not state.good ] print since, until, pretty_ranges(channels)
def merge_input_variables(self, inputs): """ Merge multiple variables together for many channels. Takes a list of IOVSets, one for each DCSC_Variable. """ result = [] info_states = IOVSet(iov_type=GoodIOV) # Reassign inputs to be a list of dictionaries with # Key=channel_id and Val=IOVSet inputs = [iovs.by_channel for iovs in inputs] # set of channel ids all_channels = sorted(set(y for x in inputs for y in x.keys())) tally_system_states = config.opts.tally_system_states for channel in all_channels: # Handle one channel at a time for variable merging c_inputs = [x[channel] for x in inputs] # Merge "good" state across multiple variables result.append(self.merge_inputs(channel, *c_inputs)) if tally_system_states: # Merge Input information across multiple variables info_state = self.merge_input_information(channel, *c_inputs) info_states.extend(info_state) if tally_system_states: # Print a tally of the states for the different systems from DQUtils.ext import tally def pretty(state): return "/".join(x[0] for x in state) chans, iovs = zip(*sorted(six.iteritems(info_states.by_channel))) for since, until, states in process_iovs(self.run_iovs, *iovs): if states[0]._is_empty: # Not inside a run continue statetally = tally(pretty(x.good) for x in states[1:]) print(since, until, statetally) return result
def test_serializer(): """ Try serializing a fairly complicated iovset object """ iovs = fetch_iovs("SHIFTOFL", runs=152166) channels, iovsets = iovs.chans_iovsets processed = [(since, until, states) for since, until, states in process_iovs(*iovsets)] serialized = dumps(processed).encode("bz2").encode("base64") deserialized = loads(serialized.decode("base64").decode("bz2")) assert deserialized == processed
def emittance_generator(self, iovs): events = process_iovs(iovs) for since, until, (state, ) in events: #print state, state.RunLB & 0xffffffff if state.RunLB else 0 if state is not None and state.RunLB is not None: thisrun = state.RunLB >> 32 # pseudo-LB and not to be trusted if thisrun == 0: continue thisLB = state.RunLB & 0xffffffff yield DefectIOV(RunLumi(thisrun, thisLB), RunLumi(thisrun, thisLB + 1), 'LUMI_EMITTANCESCAN', True, comment='Emittance scan')
def merge_inputs(self, channel, *inputs): """ Merge multiple variables together for one input channel. Each 'inputs' arg is an IOVSet that corresponds to this input channel. """ # inputs must correspond to and be in sync with subdetector.variables...? result = IOVSet() # combine the IOVSets into smallest chunks using process_iovs for since, until, states in process_iovs(*inputs): # Get the worst state for the list of vectors state = self.merge_variable_states(states) result.add(since, until, channel, state) return result.solidify(GoodIOV)
def compute_grl_diff(inputs, *iovsets): """ Returns a list of IoVs which differ for GRLs """ result = IOVSet() for since, until, grl_states in process_iovs(*iovsets): has_difference = any(grl_states) != all(grl_states) if has_difference: set_iovs = [d for d, state in zip(inputs, grl_states) if state] unset_iovs = [ d for d, state in zip(inputs, grl_states) if not state ] result.add(since, until, set_iovs, unset_iovs) return result.solidify(GRLSTATE_VAL)
def make_good_iovs(self, iovs): # Filter out channels which are permanently dead excluded_channels = self.excluded_channels if excluded_channels: iovs = iovs.empty(i for i in iovs if i.channel not in excluded_channels) chans, iovsets = iovs.chans_iovsets result = IOVSet() for since, until, states in process_iovs(*iovsets): if not all(s and s.powerStatus for s in states): result.add(since, until, self.defect_name, True, "") return result.solidify(DefectIOV)
def tdaq_ready_generator(self, iovs): #state = iovs.select_channels(0) events = process_iovs(iovs) for since, until, (state, ) in events: if state is not None: if state.ReadyForPhysics == 1: continue yield DefectIOV(since, until, 'GLOBAL_NOTREADY', True, comment='ReadyForPhysics not set')
def dcs_summary(grl_iovs, dcs_iovs, quantity="dead_fraction"): """ Calculate minimum, maximum and weighted average for `quantity` from the `dcs_iovs` iov sets, for lumiblocks within the `grl_iovs`. """ channels, dcs_iovs = zip(*sorted(dcs_iovs.by_channel.iteritems())) # Somewhere to store IoVs which lie within a GRL iov_sets = dict((channel, IOVSet()) for channel in channels) # Fetch iov information from dcs_iovs within a GRL # process_iovs pulls out common (since, until, state) efficiently, where # `state` is a list of IoVs which are currently active, and in the same # order as the iov sets which are passed to it. for since, until, iovs in process_iovs(grl_iovs, *dcs_iovs): grl_iov, dcs_states = iovs[0], iovs[1:] if grl_iov is None: # `grl_iov` unset? => We are outside a GRL IoV continue # Copy dead fraction information for iovs lying within a GRL for channel, dcs_iov in zip(channels, dcs_states): properties = dcs_iov.deadFrac, dcs_iov.NConfig iov_sets[channel].add(since, until, *properties) @define_iov_type def DCS_IOV(dead_fraction, n_in_config): "Represent a dead fraction" # 'solidify' required once an iov_set has been filled iov_sets = dict((channel, iov_set.solidify(DCS_IOV)) for channel, iov_set in iov_sets.iteritems()) result = {} # Consider channels, then for a given channel consider all runs. for channel in channels: for run, iovs in sorted(iov_sets[channel].by_run.iteritems()): lowest, average, highest = compute_min_max_avg(iovs, quantity) run_state = RunInfo(lowest, average, highest) result.setdefault(channel, []).append((run, run_state)) return result
def magnet_iov_generator(self, iovs, system, measured_channel, desired_channel, tolerance): measured_iovs = iovs.select_channels(measured_channel) desired_iovs = iovs.select_channels(desired_channel) events = process_iovs(measured_iovs, desired_iovs) for since, until, (measured, desired) in events: # 28-05-2015: excluding empty 'desired' value, because how do we make # a decision without an expectation? Should debug this some more, as # the issue came up in 2015 run 253014. if measured is not None and desired is not None and not desired._is_empty: # NOTE: if measured is 'empty', this is always true if measured.value <= tolerance: # Magnet off defect = system + '_OFF' elif abs(measured.value - desired.value) <= tolerance: if ((system == 'GLOBAL_SOLENOID' and abs(desired.value - 7730.) > tolerance) or (system == 'GLOBAL_TOROID' and abs(desired.value - 20400.) > tolerance)): # Magnet has non-nominal current defect = system + '_NOTNOMINAL' else: defect = None else: # Magnet is ramping defect = system + '_RAMPING' if defect is None: continue mcurrent = '%.1f' % measured.value if measured.value is not None else 'None' scurrent = '%.1f' % desired.value if desired.value is not None else 'None' yield DefectIOV( since, until, defect, True, comment='Measured current: %s, Set current: %s' % (mcurrent, scurrent))
def calculate_good_iovs(self, lbtime, subdetector): magnets = Magnets() result = magnets.run(lbtime) by_defect = result.by_channel toroid_ramp = by_defect.get("GLOBAL_TOROID_RAMPING", IOVSet()) solenoid_ramp = by_defect.get("GLOBAL_SOLENOID_RAMPING", IOVSet()) self.input_hashes = [hash(toroid_ramp), hash(solenoid_ramp)] result = IOVSet() events = process_iovs(toroid_ramp, solenoid_ramp) for since, until, (toroid, solenoid) in events: if toroid or solenoid: result.add(since, until, "LCD_MAGNETS_RAMPING", True, "") self.iovs = result.solidify(DefectIOV) return self
def sct_color_to_defect_translator(flag_iovs): defectmap = {111: 'B', 114: 'EA', 115: 'EC'} from DCSCalculator2.consts import GREEN from DCSCalculator2.variable import DefectIOV from DQUtils import process_iovs rv = [] for since, until, states in process_iovs( *flag_iovs.by_channel.values()): if any(state.Code != GREEN for state in states): # This line of code is awful! badfrac = 'Bad fractions: ' + ', '.join( '%s %.3f' % (defectmap[state.channel], state.deadFrac) for state in sorted(states, key=lambda x: x.channel)) rv.append( DefectIOV(since=since, until=until, channel='SCT_GLOBAL_STANDBY', present=True, comment=badfrac)) return rv
def compute_lumi_many_channels(lbs, lumis, iovs, exclude_iovsets=[], good_runs=None): """ Compute luminosity per channel for an IOVSet iovs using lbs, lumis from fetch_lumi_inputs. exclude_iovsets is a list of IOVSets for channels to be excluded from the calculation such as not_ready, busy, etc. good_runs is an optional set of runs to consider, such as the set of runs which are OK for physics. Return type is a dictionary of {channel name: lumi} """ # Split the IOVSet by channel chans, iovsets = iovs.chans_iovsets result = {} # Old method, keeping it here for now #inputs = process_iovs(lbs, lumis, *iovsets) #for since, until, states in inputs: #(lb, lumi), defects = states[:2], states[2:] #if not (lb and lumi and lumi.LBAvInstLumi): #continue #lumi = (lb.EndTime - lb.StartTime)/1e9 * lumi.LBAvInstLumi #for name, defect_iov in zip(chans, defects): #if defect_iov: #result[name] = result.get(name, 0) + lumi # New approach, adding new functionality num_exclude = len(exclude_iovsets) full_iovsets = exclude_iovsets full_iovsets.extend(iovsets) inputs = process_iovs(lbs, lumis, *full_iovsets) for since, until, states in inputs: # I find the below version conceptually cleaner #(lb, lumi), excludes, defects = (states[:2], states[2:2+num_exclude], #states[2+num_exclude:]) (lb, lumi), defectstates = states[:2], states[2:] excludes, defects = defectstates[:num_exclude], defectstates[ num_exclude:] # Skip this iov if requested info not available # or if any excluded channels are present #if not (lb and lumi and lumi.LBAvInstLumi): if not lb or not lumi or any(excludes): continue # Optional set of runs to consider. Reject if not in set if good_runs and since.run not in good_runs: continue # Lumi for this lb lumi = (lb.EndTime - lb.StartTime) / 1e9 * lumi.LBAvInstLumi # Loop over channels present, add lumi for name, defect_iov in zip(chans, defects): if defect_iov: result[name] = result.get(name, 0) + lumi return result
def main(): f1 = "%s::%s" % (db1, options.folderBS) f2 = "%s::%s" % (db2, options.folderLumi) print("=" * 100) print("Comparing: ") print(" * ", f1, options.tagBS) print(" * ", f2, options.tagLumi) print("=" * 100) requiredForNtuple = ['posX', 'posY', 'posZ', 'sigmaX', 'sigmaY', 'sigmaZ'] checkNtupleProd = all(item in varColl for item in requiredForNtuple) if not checkNtupleProd: print('Ntuple will not be filled missing vars') #Open up required databases from PyCool import cool from CoolConvUtilities import AtlCoolLib cooldbBS = AtlCoolLib.indirectOpen(db1, True, True, False) cooldbLumi = AtlCoolLib.indirectOpen(db2, True, True, False) folderBS = cooldbBS.getFolder(options.folderBS) folderLumi = cooldbLumi.getFolder(options.folderLumi) from InDetBeamSpotExample.COOLUtils import COOLQuery coolQuery = COOLQuery() if options.runMin is not None: iov1 = options.runMin << 32 if options.runMax is not None: iov2 = (options.runMax + 1) << 32 else: iov2 = (options.runMin + 1) << 32 print('Plotting runs %i to %i ' % (iov1, iov2)) else: print('No run selected -- ERROR') return if (iov2 > cool.ValidityKeyMax): iov2 = cool.ValidityKeyMax print("Reading data from database") itrBS = folderBS.browseObjects(iov1, iov2, cool.ChannelSelection.all(), options.tagBS) print("...finished getting BS data") lbDict = dict() startRLB = 0x7FFFFFFFFFFFFFFF endRLB = 0 outfile = ROOT.TFile("BeamspotLumi_%i.root" % (options.runMin), "recreate") ntuple = ROOT.TNtupleD( 'BeamspotLumi', 'BeamSpotLumi', "x:y:z:sigma_x:sigma_y:sigma_z:run:mu:lumi:year:month:day:hour:minute:epoch" ) runs = set() while itrBS.goToNext(): obj = itrBS.currentRef() since = obj.since() runBegin = since >> 32 lumiBegin = since & 0xFFFFFFFF until = obj.until() runUntil = until >> 32 lumiUntil = until & 0xFFFFFFFF status = int(obj.payloadValue('status')) if status != 59: continue runs.add(runBegin) if since < startRLB: startRLB = since if until > endRLB: endRLB = until values = {} for var in varColl: values[var] = float(obj.payloadValue(var)) values[var + 'Err'] = float(obj.payloadValue(var + 'Err')) values['until'] = until lbDict[since] = values print('Runs: ', runs) lumi = array('d') xd = array('d') exd = array('d') ydDict = {} eydDict = {} ydDict2 = {} sqtrt2pi = math.sqrt(2 * math.pi) lblb = CoolDataReader('COOLONL_TRIGGER/CONDBR2', '/TRIGGER/LUMI/LBLB') from DQUtils.sugar import RANGEIOV_VAL, RunLumi from DQUtils import IOVSet grlIOVs = IOVSet.from_grl( "data15_13TeV.periodAllYear_DetStatus-v89-pro21-02_Unknown_PHYS_StandardGRL_All_Good_25ns.xml" ) grlIOVs += IOVSet.from_grl( "data16_13TeV.periodAllYear_DetStatus-v89-pro21-01_DQDefects-00-02-04_PHYS_StandardGRL_All_Good_25ns.xml" ) grlIOVs += IOVSet.from_grl( "data17_13TeV.periodAllYear_DetStatus-v99-pro22-01_Unknown_PHYS_StandardGRL_All_Good_25ns_Triggerno17e33prim.xml" ) grlIOVs += IOVSet.from_grl( "data18_13TeV.periodAllYear_DetStatus-v102-pro22-04_Unknown_PHYS_StandardGRL_All_Good_25ns_Triggerno17e33prim.xml" ) for run in runs: iov1 = run << 32 iov2 = (run + 1) << 32 itrLumi = folderLumi.browseObjects(iov1, iov2, cool.ChannelSelection.all(), options.tagLumi) print("...finished getting Lumi data for run %i" % run) lblb.setIOVRangeFromRun(run) lblb.readData() if len(lblb.data) < 1: print('No LBLB data found!') continue # Make time map lblbMap = dict() for obj in lblb.data: lblbMap[obj.since()] = (obj.payload()['StartTime'], obj.payload()['EndTime']) while itrLumi.goToNext(): obj = itrLumi.currentRef() since = obj.since() runBegin = since >> 32 lumiBegin = since & 0xFFFFFFFF until = obj.until() runUntil = until >> 32 lumiUntil = until & 0xFFFFFFFF inGRL = False for sinceGRL, untilGRL, grl_states in process_iovs(grlIOVs): if grl_states[0].since == None: continue if (sinceGRL.run <= runBegin and untilGRL.run >= runUntil and sinceGRL.lumi <= lumiBegin and untilGRL.lumi >= lumiUntil): inGRL = True break if not inGRL: continue mu = float(obj.payloadValue('LBAvEvtsPerBX')) instlumi = float(obj.payloadValue('LBAvInstLumi')) #if( mu < 10 or mu > 65 ): #print 'Mu: %2.1f Run : %d LB: %d - %d Lumi: %f' % (mu,runBegin,lumiBegin,lumiUntil,instlumi) if since in lbDict: if lbDict[since]['sigmaX'] > 0.1: continue startTime = lblbMap.get(obj.since(), (0., 0.))[0] endTime = lblbMap.get(lbDict[since]['until'], (0., 0.))[0] #[1] end of lumiblock mylumi = (endTime - startTime) / 1e9 * instlumi / 1e9 thisTime = time.gmtime(startTime / 1.e9) year = thisTime[0] month = thisTime[1] day = thisTime[2] hour = thisTime[3] mins = thisTime[4] sec = thisTime[5] lumi.append(mylumi) # in fb^-1 xd.append(mu) exd.append(0) if options.plotSomething: for var in varColl: if not var in ydDict: ydDict[var] = array('d') ydDict2[var] = array('d') eydDict[var] = array('d') ydDict2[var].append(mu / (lbDict[since][var] * sqtrt2pi)) ydDict[var].append(lbDict[since][var]) eydDict[var].append(lbDict[since][var + 'Err']) if checkNtupleProd and lbDict[since]['sigmaZErr'] < 5: ntuple.Fill(lbDict[since]['posX'], lbDict[since]['posY'], lbDict[since]['posZ'], lbDict[since]['sigmaX'], lbDict[since]['sigmaY'], lbDict[since]['sigmaZ'], runBegin, mu, mylumi, year, month, day, hour, mins, startTime / 1.e9) runStart = startRLB >> 32 runEnd = endRLB >> 32 fillStart = fillEnd = 0 timeStart = timeEnd = 0 beamEnergy = 13 try: timeStart = coolQuery.lbTime(int(startRLB >> 32), int(startRLB & 0xFFFFFFFF))[0] except: pass try: timeEnd = coolQuery.lbTime(int(endRLB >> 32), int(endRLB & 0xFFFFFFFF) - 1)[1] except: pass try: fillStart = coolQuery.getLHCInfo(timeStart).get('FillNumber', 0) except: pass try: fillEnd = coolQuery.getLHCInfo(timeEnd).get('FillNumber', 0) except: pass try: beamEnergy = coolQuery.getLHCInfo(timeStart).get('BeamEnergyGeV', 0) beamEnergy *= 2e-3 except: pass ntuple.Write() if not options.plotSomething: return from InDetBeamSpotExample import ROOTUtils ROOTUtils.setStyle() canvas = ROOT.TCanvas('BeamSpotComparison', 'BeamSpotComparison', 1600, 1200) canvas.cd() ROOT.gPad.SetTopMargin(0.05) ROOT.gPad.SetLeftMargin(0.15) ROOT.gPad.SetRightMargin(0.05) if not options.plotGraph: ROOT.gPad.SetRightMargin(0.15) #Plot each variable for var in varColl: if var not in ydDict: print('Missing yd: ', var) if var not in eydDict: print('Missing eyd: ', var) continue gr = ROOT.TGraphErrors(len(xd), xd, ydDict[var], exd, eydDict[var]) xmin = min(xd) xmax = max(xd) ymin = min(ydDict[var]) ymax = max(ydDict[var]) h = (ymax - ymin) ymin -= 0.25 * h ymaxSmall = ymax + 0.25 * h ymax += 0.75 * h ymin2 = min(ydDict2[var]) ymax2 = max(ydDict2[var]) h = (ymax2 - ymin2) ymin2 -= 0.25 * h ymax2 += 0.75 * h h = (xmax - xmin) xmin -= 0.05 * h xmax += 0.05 * h #This histogram is made just to make it easier to manipulate the margins histo = ROOT.TH2D('hd' + var, 'hd' + var, 100, xmin, xmax, 100, ymin, ymax) histo.GetYaxis().SetTitle(varDef(var, 'atit', var)) histo.GetXaxis().SetTitle('Average interactions per bunch crossing') histo.GetZaxis().SetTitle('Entries') histo2 = ROOT.TH2D('hd2' + var, 'hd2' + var, 100, xmin, xmax, 100, ymin2, ymax2) histo2.GetYaxis().SetTitle( "<Interaction density> @ z=0 [interactions/mm]") histo2.GetXaxis().SetTitle('Average interactions per bunch crossing') histo2.GetZaxis().SetTitle('Entries') histo2W = ROOT.TH2D('hd3' + var, 'hd3' + var, 100, xmin, xmax, 100, ymin2, ymax2) histo2W.GetYaxis().SetTitle( "<Interaction density> @ z=0 [interactions/mm]") histo2W.GetXaxis().SetTitle('Average interactions per bunch crossing') histo2W.GetZaxis().SetTitle('Integrated Luminosity (fb^{-1}/bin)') histoW = ROOT.TH2D('hdW' + var, 'hdW' + var, 100, xmin, xmax, 100, ymin, ymax) histoW.GetYaxis().SetTitle(varDef(var, 'atit', var)) histoW.GetXaxis().SetTitle('Average interactions per bunch crossing') histoW.GetZaxis().SetTitle('Integrated Luminosity (fb^{-1}/bin)') histoW1D = ROOT.TH1D('hd1D' + var, 'hd1D' + var, 100, ymin, ymaxSmall) histoW1D.GetXaxis().SetTitle(varDef(var, 'atit', var)) histoW1D.GetYaxis().SetTitle('Integrated Luminosity (fb^{-1}/bin)') histo.Draw() if options.plotGraph: gr.Draw("p") else: for mu, x, l in zip(xd, ydDict[var], lumi): histo.Fill(mu, x) histoW.Fill(mu, x, l) histoW1D.Fill(x, l) for mu, x, l in zip(xd, ydDict2[var], lumi): histo2.Fill(mu, x) histo2W.Fill(mu, x, l) histo.Draw("colz") histo.Write() histoW.Write() histo2.Write() histo2W.Write() histoW1D.Write() # Add some information to the graph ROOTUtils.atlasLabel(0.53, 0.87, False, offset=0.12, isForApproval=False, customstring="Internal", energy='%2.0f' % beamEnergy, size=0.055) ROOTUtils.drawText(0.18, 0.87, 0.055, varDef(var, 'title', var)) comments = [] if runStart == runEnd: comments.append('Run %i' % runStart) else: comments.append('Runs %i - %i' % (runStart, runEnd)) if fillStart == fillEnd: comments.append('Fill %i' % fillStart) else: comments.append('Fills %i - %i' % (fillStart, fillEnd)) t1 = time.strftime('%d %b %Y', time.localtime(timeStart)) t2 = time.strftime('%d %b %Y', time.localtime(timeEnd)) if t1 == t2: comments.append(t1) else: comments.append('%s - %s' % (t1, t2)) ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42) canvas.Print("Run_%d_%sVsMu.png" % (options.runMin, var)) canvas.Print("Run_%d_%sVsMu.pdf" % (options.runMin, var)) if not options.plotGraph: canvas.SetLogz(True) canvas.Print("Run_%d_%sVsMuLog.png" % (options.runMin, var)) canvas.Print("Run_%d_%sVsMuLog.pdf" % (options.runMin, var)) canvas.SetLogz(False) histo2.Draw("colz") ROOTUtils.atlasLabel(0.53, 0.87, False, offset=0.12, isForApproval=False, customstring="Internal", energy='%2.0f' % beamEnergy, size=0.055) ROOTUtils.drawText(0.18, 0.87, 0.055, "Interaction density") ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42) canvas.Print("Run_%d_Mu%sVsMu.png" % (options.runMin, var)) canvas.Print("Run_%d_Mu%sVsMu.pdf" % (options.runMin, var)) canvas.SetLogz(True) canvas.Print("Run_%d_Mu%sVsMuLog.png" % (options.runMin, var)) canvas.Print("Run_%d_Mu%sVsMuLog.pdf" % (options.runMin, var)) canvas.SetLogz(False) histoW.Draw("colz") histoW.SetMinimum(0.005) ROOTUtils.atlasLabel(0.53, 0.87, False, offset=0.12, isForApproval=False, customstring="Internal", energy='%2.0f' % beamEnergy, size=0.055) ROOTUtils.drawText(0.18, 0.87, 0.055, varDef(var, 'title', var)) ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42) canvas.Print("Run_%d_%sVsMuW.png" % (options.runMin, var)) canvas.Print("Run_%d_%sVsMuW.pdf" % (options.runMin, var)) canvas.SetLogz(True) canvas.Print("Run_%d_%sVsMuWLog.png" % (options.runMin, var)) canvas.Print("Run_%d_%sVsMuWLog.pdf" % (options.runMin, var)) canvas.SetLogz(False) histo2W.Draw("colz") histo2W.SetMinimum(0.01) ROOTUtils.atlasLabel(0.53, 0.87, False, offset=0.12, isForApproval=False, customstring="Internal", energy='%2.0f' % beamEnergy, size=0.055) ROOTUtils.drawText(0.18, 0.87, 0.055, "Interaction density") ROOTUtils.drawText(0.18, 0.81, 0.05, ';'.join(comments), font=42) canvas.Print("Run_%d_Mu%sVsMuW.png" % (options.runMin, var)) canvas.Print("Run_%d_Mu%sVsMuW.pdf" % (options.runMin, var)) canvas.SetLogz(True) canvas.Print("Run_%d_Mu%sVsMuWLog.png" % (options.runMin, var)) canvas.Print("Run_%d_Mu%sVsMuWLog.pdf" % (options.runMin, var)) canvas.SetLogz(False) histoW1D.Draw("colz") ROOTUtils.atlasLabel(0.53, 0.87, False, offset=0.12, isForApproval=False, customstring="Internal", energy='%2.0f' % beamEnergy, size=0.055) ROOTUtils.drawText(0.18, 0.87, 0.055, varDef(var, 'title', var)) ROOTUtils.drawText(0.18, 0.81, 0.05, "#mu=%2.4f RMS=%2.4f" % (histoW1D.GetMean(), histoW1D.GetRMS()), font=42) canvas.Print("Run_%d_%s1D.png" % (options.runMin, var)) canvas.Print("Run_%d_%s1D.pdf" % (options.runMin, var)) canvas.SetLogy(True) canvas.Print("Run_%d_%s1DLog.png" % (options.runMin, var)) canvas.Print("Run_%d_%s1DLog.pdf" % (options.runMin, var)) canvas.SetLogy(False)
def main(): f1 = "%s::%s" % (db1, options.folder) f2 = "%s::%s" % (db2, options.folder) print "=" * 100 print "Comparing: " print " * ", f1, options.tag1 print " * ", f2, options.tag2 print "=" * 100 if options.runMin is not None: runs = options.runMin if options.runMax is not None: runs = options.runMin, options.runMax tag1iovs = fetch_iovs(f1, runs=runs, tag=options.tag1) tag2iovs = fetch_iovs(f2, runs=runs, tag=options.tag2) affected_runs = set() ar2 = set() #runiovs = IOVSet() #runiovs.add(RunLumiType((runs << 32)+0), RunLumiType((runs << 32)+(1 << 32)-1)) # Loop over iovs in the two tags # for since, until, (t1, t2, r) in process_iovs(tag1iovs, tag2iovs, runiovs.solidify(RANGEIOV_VAL)): for since, until, (t1, t2) in process_iovs(tag1iovs, tag2iovs): # Ignore IoVs spanning runs if not options.span and since.run != until.run: continue # Make sure run exists in both #if not t1.since or not t2.since: continue # Only consider runs in the range interested in # if not r: continue # Check for IOVs with different status (one of which is non-zero) if (t1.status == 0 or t2.status == 0) and (t1.status != t2.status): if options.statusonly and not options.summary: print since, until, t1.status, t2.status ar2.add(since.run) # Check for differences in all fields if not options.statusonly: for i, field in enumerate(t1._fields): if t1[i] != t2[i]: #if field in ['posX', 'posY'] and abs(t1[i]-t2[i]) > 0.1e-3: #if field not in ['status'] and t1[i] != 0 and abs(t1[i]-t2[i])/t1[i] > 0.001: #if field in ['posZ'] and t1[i] != 0 and abs(t1[i]-t2[i])/t1[i] > 0.05: #if field in ['sigmaX', 'sigmaY'] and abs(t1[i]-t2[i]) > 0.2e-3: #if field in ['sigmaZ'] and abs(t1[i]-t2[i])/t1[i] > 0.002: if not options.summary: print since, until, field, t1[i], t2[i] affected_runs.add(since.run) # Print results if not options.statusonly: print "Runs with differences (%s):" % len(affected_runs), ", ".join( map(str, sorted(affected_runs))) print "Runs with different status (and one non-zero):", ", ".join( map(str, sorted(ar2)))
def complete(self, runMin, runMax): """ Complete a list of IoVs to cover all LBs in a run, treating empty ones as having 'emptyState' """ # Create an IOV set covering the entire run(s) run_lbs = fetch_iovs("EOR", runs=(runMin, runMax), what=[], with_channel=False) # run_lbs = IOVSet() # lbMin = 1 # lbMax = (1 << 32) -1 # Note, lbMax is exclusive # since = RunLumiType((runMin << 32)+lbMin) # until = RunLumiType((runMax << 32)+lbMax) # run_lbs.add(since, until) if not len(run_lbs): print "WARNING: No LBs in run according to EOR_Params - are we running before the run has finished?" def lbsStartAtOne(iov): "Change LBs starting at 0 to start at 1" return iov._replace(since=RunLumi(iov.since.run, 1)) # Start LBs from 1 rather than 0 (at request of P. Onyisi) as long as run has more than one LB (else skip) run_lbs = [lbsStartAtOne(iov) for iov in run_lbs if iov.until.lumi > 1] # Empty IOV set for adding full list of LBs too iovs = IOVSet() # Ask official DB if an IoV is currently defective so can unset only those if doing reprocessing defectsCurrentlyInDb = self.officialDb.retrieve( (runMin, 0), (runMax + 1, 0), [self.defect]) # Order IOVs to avoid since > until self.iovs = IOVSet(sorted(self.iovs)) #for since, until, (run_lb, iov, dbDefect) in process_iovs(run_lbs.solidify(RANGEIOV_VAL), self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb): for since, until, (run_lb, iov, dbDefect) in process_iovs( run_lbs, self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb): if not run_lb: continue # Check valid # # Start LBs from 1 rather than 0 (at request of P. Onyisi) # # If this makes since==until, i.e. it was a [0->1) LB range then skip # if since.lumi==0: # since = RunLumiType((since.run << 32)+since.lumi+1) # if since==until: continue # Add iovs from self.iovs treating empty ones as defective (i.e. beamspot bad/missing) if iov: # These are not defective if dbDefect and dbDefect.present: # Only store not defective IoVs if they are present on the Db so we can unset them # (don't want to write not present defects to Db unless really chaning an existing defect) iovs.add(since, until, self.defect, False) else: # These are defective iovs.add(since, until, self.defect, True) self.iovs = iovs return