def main(): runs = 185640, 185660 lblb = fetch_iovs("LBLB", runs=runs) lbtime = inverse_lblb(lblb) beam_iovs = fetch_iovs("COOLOFL_DCS::/LHC/DCS/FILLSTATE", lbtime.first.since, lbtime.last.until) stable_beam_by_lb = {} for since_time, until_time, (lumiblock, beamstate) in process_iovs(lbtime, beam_iovs): if not lumiblock: # Not inside lumiblock continue lb = RunLumi(lumiblock.Run, lumiblock.LumiBlock) if lb in stable_beam_by_lb: # Require stable beam for ALL of the lumiblock (hence 'and') # Replace with 'or' to get ANY of the lumiblock stable_beam_by_lb[lb] = stable_beam_by_lb[lb] and beamstate.StableBeams else: stable_beam_by_lb[lb] = beamstate.StableBeams @define_iov_type def STABLEBEAM_VAL(stable): """ Define an IOV type which just has .stable """ result = (STABLEBEAM_VAL(runlb, runlb+1, state) for runlb, state in sorted(stable_beam_by_lb.iteritems())) result = IOVSet(connect_adjacent_iovs(result)) result.pprint()
def fetch_lumi_inputs(range_iov, tag="OflLumi-7TeV-002"): """ Retrieve information required to calculate the luminosity from the database """ lbs = fetch_iovs( "LBLB", with_channel=False, #loud=True, *range_iov) if tag == "ONLINE": lumis = fetch_iovs( "COOLONL_TRIGGER::/TRIGGER/LUMI/OnlPrefLumi", channels=[0], #loud=True, *range_iov) # Select valid lumis = IOVSet(l for l in lumis if (l.Valid & 0xFFFF) % 10 == 0) else: # Apparently this string changes from time to time... lumi_folder = "COOLOFL_TRIGGER::/TRIGGER/OFLLUMI/OflPrefLumi" # Run-2 #lumi_folder = "COOLOFL_TRIGGER::/TRIGGER/OFLLUMI/LBLESTOFL" # Run-1 lumis = fetch_iovs( lumi_folder, tag=tag, channels=[0], #loud=True, *range_iov) return lbs, lumis
def diff_folder(iov_range, folder, tag_1, tag_2, interesting_channels): other_args = dict(named_channels=True) iovs_1 = fetch_iovs(folder, tag=tag_1, *iov_range, **other_args).by_channel iovs_2 = fetch_iovs(folder, tag=tag_2, *iov_range, **other_args).by_channel all_channels = set(iovs_1.keys() + iovs_2.keys()) for channel in sorted(all_channels): if channel not in interesting_channels: continue print "Considering", channel iovs_1_c = iovs_1.get(channel, IOVSet()) iovs_2_c = iovs_2.get(channel, IOVSet()) diff_channel(channel, iovs_1_c, iovs_2_c)
def main(): from DCSCalculator2.subdetectors import CSC, MDT lbtime = fetch_iovs("LBTIME", (177540, 0)) print "lbtime =", lbtime show_variable_info(lbtime, MDT)
def test_processing(): """ Test that processing works """ iovs = fetch_iovs("SHIFTOFL", tag="DetStatusSHIFTOFL-pass1-analysis-2010B") channels, iovsets = iovs.chans_iovsets processed = [(since, until, states[:]) for since, until, states in process_iovs(*iovsets)] return processed
def test_read(): """ Check if retrieving the Code and Comment field from DetStatusSHIFTOFL-pass1-analysis-2010B changed Reads some data from the database which is expected not to change. A problem with this could either indicate a broken API, or a changed database """ tag = "DetStatusSHIFTOFL-pass1-analysis-2010B" result = fetch_iovs("SHIFTOFL", what=["Code", "Comment"], tag=tag) return result
def main(): init_logger(verbose=2) # Query the last four days of information start_time = datetime.now() - timedelta(days=2) with timer("Query DQMFONL"): iovs = fetch_iovs("DQMFONL", start_time) log.info("Got iovs from DQMFONL: %r", iovs) with timer("Query LBTIME"): lbiovs = fetch_iovs("LBTIME", start_time) chans, states = iovs.chans_iovsets with timer("run build_table_data"): data = build_table_data(lbiovs, *states) with timer("run table_data_to_html"): table = table_data_to_html(chans, data) print table
def _get_virtual_defect_logics(self, tag): if tag != "HEAD" and not self.defect_logic_folder.existsUserTag(tag): # The tag doesn't exist, so there is nothing to fetch. return {} logics = fetch_iovs(self.defect_logic_folder, tag=tag, named_channels=True) logics = dict((l.channel, DefectLogic(l)) for l in logics) for defect_name, defect_logic in logics.iteritems(): defect_logic._populate(logics) return logics
def test_serializer(): """ Try serializing a fairly complicated iovset object """ iovs = fetch_iovs("SHIFTOFL", runs=152166) channels, iovsets = iovs.chans_iovsets processed = [(since, until, states) for since, until, states in process_iovs(*iovsets)] serialized = dumps(processed).encode("bz2").encode("base64") deserialized = loads(serialized.decode("base64").decode("bz2")) assert deserialized == processed
def test(): path = "/afs/cern.ch/user/b/beate/public/DQAna/StableBeams-periodC1.xml" grl_iovs = grl_iovs_from_xml(path) since, until = grl_iovs.range_iov dcs_iovs = fetch_iovs("DCSOFL", since, until, named_channels=True) # Instead of "dead_fraction", "n_in_config" can be used. result = dcs_summary(grl_iovs, dcs_iovs, "dead_fraction") for channel, states in sorted(result.iteritems()): print channel for run, info in states: print "", run, "Low: % .3f Ave: % .3f Hi: % .3f" % ( info.lowest, info.average, info.highest)
def get_trigger_channel(trigger, iovrange): """ Returns the set of iovs for which the trigger is valid """ raise NotImplementedError("Implementation incomplete") # First check just the first run. iovs = fetch_iovs(LVL1_MENU, runs=iovrange.since.run, what=["ItemName"]) result = [i for i in iovs if i.ItemName == trigger] if not result: raise UnknownTrigger() assert len(result) == 1, "This should not happen." channel = result[0].channel iovs = fetch_iovs(LVL1_MENU, channels=[channel], what=["ItemName"], *iovrange) return iovs.select(ItemName == trigger)
def calculate_idbs(run_range): # List of lumiblocks to run over lbtime = inverse_lblb(fetch_iovs("LBLB", runs=run_range)) idbs = IDBS() idbs.input_db = "COOLOFL_INDET/CONDBR2" result_iovs = idbs.run(lbtime) print "Will write the following:" pprint_objects(result_iovs) for entry in result_iovs: print entry.since.run, entry.since.lumi, "->", entry.until.lumi, entry.Code write_iovs("idbs_test.db/CONDBR2::/GLOBAL/DETSTATUS/DCSOFL", result_iovs, dcsofl_cool_record())
def read(self, query_range, folder_base, folder_name): """ Read the relevant data from COOL for this variable """ if folder_name.startswith("/"): folder_path = folder_name else: # For relative folders prepend the folder_base folder_path = "/".join((folder_base, folder_name)) log.info("Querying COOL folder %s", folder_path) if config.opts.check_input_time: self.fetch_args["with_time"] = True # Massage DB access if '/' in self.input_db: newdbstring = self.input_db.rsplit('/', 1)[0] else: newdbstring = self.input_db if config.opts.input_database.startswith('sqlite'): self.fetch_args['database'] = config.opts.input_database else: self.fetch_args['database'] = ( '%s/%s' % (newdbstring, config.opts.input_database)) if self.fetch_args: log.debug("Fetching with args: %r", self.fetch_args) iovs = fetch_iovs(folder_path, *query_range, **self.fetch_args) # Prints even when not doing debug. # TODO: fix this. Might be broken in DQUtils.logger #if log.isEnabledFor(logging.DEBUG): # log.debug("Dumping input IOVs:") # for iov in iovs: # print iov if config.opts.check_input_time: self.print_time_info(iovs) if log.isEnabledFor(logging.INFO): input_hash = hash(iovs) self.input_hashes.append(input_hash) log.info(" -> Input hash: % 09x (len=%i)", input_hash, len(iovs)) return iovs
def main(): f1 = "%s::%s" % (db1, options.folder) f2 = "%s::%s" % (db2, options.folder) print "=" * 100 print "Comparing: " print " * ", f1, options.tag1 print " * ", f2, options.tag2 print "=" * 100 if options.runMin is not None: runs = options.runMin if options.runMax is not None: runs = options.runMin, options.runMax tag1iovs = fetch_iovs(f1, runs=runs, tag=options.tag1) tag2iovs = fetch_iovs(f2, runs=runs, tag=options.tag2) affected_runs = set() ar2 = set() #runiovs = IOVSet() #runiovs.add(RunLumiType((runs << 32)+0), RunLumiType((runs << 32)+(1 << 32)-1)) # Loop over iovs in the two tags # for since, until, (t1, t2, r) in process_iovs(tag1iovs, tag2iovs, runiovs.solidify(RANGEIOV_VAL)): for since, until, (t1, t2) in process_iovs(tag1iovs, tag2iovs): # Ignore IoVs spanning runs if not options.span and since.run != until.run: continue # Make sure run exists in both #if not t1.since or not t2.since: continue # Only consider runs in the range interested in # if not r: continue # Check for IOVs with different status (one of which is non-zero) if (t1.status == 0 or t2.status == 0) and (t1.status != t2.status): if options.statusonly and not options.summary: print since, until, t1.status, t2.status ar2.add(since.run) # Check for differences in all fields if not options.statusonly: for i, field in enumerate(t1._fields): if t1[i] != t2[i]: #if field in ['posX', 'posY'] and abs(t1[i]-t2[i]) > 0.1e-3: #if field not in ['status'] and t1[i] != 0 and abs(t1[i]-t2[i])/t1[i] > 0.001: #if field in ['posZ'] and t1[i] != 0 and abs(t1[i]-t2[i])/t1[i] > 0.05: #if field in ['sigmaX', 'sigmaY'] and abs(t1[i]-t2[i]) > 0.2e-3: #if field in ['sigmaZ'] and abs(t1[i]-t2[i])/t1[i] > 0.002: if not options.summary: print since, until, field, t1[i], t2[i] affected_runs.add(since.run) # Print results if not options.statusonly: print "Runs with differences (%s):" % len(affected_runs), ", ".join( map(str, sorted(affected_runs))) print "Runs with different status (and one non-zero):", ", ".join( map(str, sorted(ar2)))
lb_length = fin.Get('%s/GLOBAL/DQTGlobalWZFinder/m_lblength_lb' % runname) lbmin, lbmax = lb_length.GetXaxis().GetXmin(), lb_length.GetXaxis().GetXmax() logging.info( 'Starting livetime lookup ... (remove when we have a proper in-file implementation ...)' ) livetime = ROOT.TProfile('livetime', 'Livetime', int(lbmax - lbmin), lbmin, lbmax) official_lum = ROOT.TProfile('official_lum', 'official integrated luminosity', int(lbmax - lbmin), lbmin, lbmax) official_lum_zero = ROOT.TProfile('official_lum_zero', 'official inst luminosity', int(lbmax - lbmin), lbmin, lbmax) from DQUtils import fetch_iovs from DQUtils.iov_arrangement import inverse_lblb lblb = fetch_iovs("LBLB", runs=int(runname[4:])) lbtime = inverse_lblb(lblb) #print list(lbtime) iovs_acct = fetch_iovs('COOLOFL_TRIGGER::/TRIGGER/OFLLUMI/LumiAccounting', lbtime.first.since, lbtime.last.until, tag='OflLumiAcct-13TeV-004') #iovs_lum = fetch_iovs('COOLOFL_TRIGGER::/TRIGGER/OFLLUMI/OflPrefLumi', lblb.first.since, lblb.last.until, tag='OflLumi-13TeV-003') #print list(iovs_lum) for iov in iovs_acct: if not lbmin < iov.LumiBlock < lbmax: continue livetime.Fill(iov.LumiBlock, iov.LiveFraction) #print iov.InstLumi, iovs_lum[iov.LumiBlock-1].LBAvInstLumi official_lum_zero.Fill(iov.LumiBlock, iov.InstLumi / 1e3)
#! /usr/bin/env python from IPython.Shell import IPShellEmbed; ip = IPShellEmbed(["-pdb"]) from DCSCalculator2.subdetectors.tgc import TGC, DCSC_Variable_TGC_HV, TGC_Database, get_mapping from DQUtils import fetch_iovs from DQUtils.iov_arrangement import inverse_lblb from itertools import chain runs = 165732, 165733 iovs = inverse_lblb(fetch_iovs("LBLB", runs=runs)) start, end = iovs.first.since, iovs.last.until TGC_data = DCSC_Variable_TGC_HV(None, None) TGC_data.db = TGC_Database() mapping = get_mapping(TGC_data.db) TGC_data.all_channels = set(chain(*mapping.values())) tgc_conditions = TGC_data.read_tgc_conditions(start.date, end.date) tgc_channels_nonempty = set(iov.channel for iov in tgc_conditions if iov) print "Total IoVs for query range:", len(tgc_conditions) print "Channels with IoV data:", len(tgc_conditions.channels) print "Channels with nonempty IoV data:", len(tgc_channels_nonempty) print "Channels in mapping table:", len(TGC_data.all_channels)
def test_fetch_single_detstatus_channel(): result = fetch_iovs("SHIFTOFL", channels=["SCTEA"]) print "Result length:", len(result)
def test_fetch_single_non_detstatus_channel(): result = fetch_iovs("VIRTUALFLAGS", channels=["CP_TRACKING"]) print "Result length:", len(result)
tl.Branch('z_one', o_z_one, 'z_one/F') tl.Branch('z_two', o_z_two, 'z_two/F') tl.Branch('trigeff', o_trigeff, 'trigeff/F') tl.Branch('trigeffstat', o_trigeffstat, 'trigeffstat/F') tl.Branch('recoeff', o_recoeff, 'recoeff/F') tl.Branch('recoeffstat', o_recoeffstat, 'recoeffstat/F') tl.Branch('alleff', o_alleff, 'alleff/F') tl.Branch('alleffstat', o_alleffstat, 'alleffstat/F') tl.Branch('ae', o_ae, 'ae/F') tl.Branch('aestat', o_aestat, 'aestat/F') from DQUtils import fetch_iovs #rset=set(_[0] for _ in rlb) #print rset lblb = fetch_iovs("LBLB", runs=int(runname[4:])).by_run for lb in sorted(lbdirs): if runmode == "Zee": h = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_eltrigtp_matches' % (runname, lb)) hmo = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_ele_tight_good_os' % (runname, lb)) hms = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_ele_tight_good_ss' % (runname, lb)) hno = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_ele_tight_bad_os' % (runname, lb)) hns = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_ele_tight_bad_ss' % (runname, lb)) if runmode == "Zmumu": h = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_mutrigtp_matches' % (runname, lb)) hmo = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_muloosetp_match_os' % (runname, lb)) hms = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_muloosetp_match_ss' % (runname, lb)) hno = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_muloosetp_nomatch_os' % (runname, lb)) hns = infile.Get('%s/%s/GLOBAL/DQTGlobalWZFinder/m_muloosetp_nomatch_ss' % (runname, lb)) lbnum = int(lb[3:]) yld = (h[2], h[3])
def retrieve(self, since=None, until=None, channels=None, nonpresent=False, primary_only=False, ignore=None, with_primary_dependencies=False, intersect=False, with_time=False, evaluate_full=True): """ Retrieve defects from the database. Parameters: `since`, `until` : IoV range to query (Default: All) `channels` : A list of channels to query. Can contain a mixture of defect names and ids. (Default: None means all channels, including all virtual) `nonpresent` : Only return IoVs which are currently "present" `primary_only` : Only return primary IoVs, no virtual ones. `ignore` : Set of defects which won't be treated as bad. `with_primary_dependencies` : When querying virtual flags, also get primary flags which went into the calculation. `intersect` : Intersect the result with the query range so that no iov spans outside the query range `with_time` : Also retrieves the time the defect was inserted ~2x slower queries, doesn't work for virtual defects `evaluate_full` : If specified, also compute the `comment` and `recoverable` fields of virtual defects. Causes a ~0.6x slowdown """ if ignore is not None and not isinstance(ignore, set): assert False, "ignore parameter should be set type" desired_channels = None # Figure out the IDs of channels to query and their virtuality if channels is not None: query_channels = set(self.defect_names_as_ids(channels)) virtual_channels = query_channels - self.defect_ids primary_channels = query_channels & self.defect_ids else: # Empty channels list means query all channels # (including all virtual) if primary_only: virtual_channels = None else: virtual_channels = self.virtual_defect_ids primary_channels = sorted(self.defect_ids) query_channels = None # (all) primary_output_names = [self.defect_id_map[pid] for pid in primary_channels] virtual_output_names = [] # (optionally populated below) # Resolve virtual channels to query here, and the primary dependents. if virtual_channels: assert not primary_only, "Requested virtual channels with primary_only=True" assert not with_time, "with_time flag only works for primary defects" virtual_output_names = [self.virtual_defect_id_map[vid] for vid in virtual_channels] ordered_logics = self._resolve_evaluation_order(virtual_output_names) if channels is not None: # Since not all channels are being queried, it is necessary to # add the desired primary channels to the query primary_needed = self.resolve_primary_defects(ordered_logics) primary_channels |= set(self.defect_names_as_ids(primary_needed)) query_channels = primary_channels if with_primary_dependencies: primary_output_names.extend(sorted(primary_needed)) for logic in ordered_logics: logic.set_evaluation(evaluate_full) # Figure out if the set of channels will produce too many ranges for COOL filter_channels = None if query_channels is not None: query_channels = sorted(query_channels) query_ranges = list_to_channelselection(query_channels, None, True) if len(query_ranges) >= 50: # We're querying too many ranges. Query everything, filter later. desired_channels = set(primary_output_names + virtual_output_names) query_channels = None # (everything) # Retrieve primary IoVs primary_iovs = fetch_iovs(self.defects_folder, since, until, query_channels, self.defects_tag, named_channels=True, unicode_strings=True, with_time=with_time) # Calculate virtual defects (if necessary) if primary_only or not virtual_channels: result = primary_iovs else: if not primary_iovs: return IOVSet() args = (primary_iovs, ordered_logics, virtual_output_names, primary_output_names, since, until, ignore) result = calculate_virtual_defects(*args) # Filter out results which have their present bit removed # (unless otherwise specified) if not nonpresent: result = IOVSet(iov for iov in result if iov.present) # Filter out channels which weren't queried by the user # (to get around 50 channelselection COOL limit) if desired_channels: result = IOVSet(iov for iov in result if iov.channel in desired_channels) if intersect: result = result.intersect_range((since, until)) return result
def complete(self, runMin, runMax): """ Complete a list of IoVs to cover all LBs in a run, treating empty ones as having 'emptyState' """ # Create an IOV set covering the entire run(s) run_lbs = fetch_iovs("EOR", runs=(runMin, runMax), what=[], with_channel=False) # run_lbs = IOVSet() # lbMin = 1 # lbMax = (1 << 32) -1 # Note, lbMax is exclusive # since = RunLumiType((runMin << 32)+lbMin) # until = RunLumiType((runMax << 32)+lbMax) # run_lbs.add(since, until) if not len(run_lbs): print "WARNING: No LBs in run according to EOR_Params - are we running before the run has finished?" def lbsStartAtOne(iov): "Change LBs starting at 0 to start at 1" return iov._replace(since=RunLumi(iov.since.run, 1)) # Start LBs from 1 rather than 0 (at request of P. Onyisi) as long as run has more than one LB (else skip) run_lbs = [lbsStartAtOne(iov) for iov in run_lbs if iov.until.lumi > 1] # Empty IOV set for adding full list of LBs too iovs = IOVSet() # Ask official DB if an IoV is currently defective so can unset only those if doing reprocessing defectsCurrentlyInDb = self.officialDb.retrieve( (runMin, 0), (runMax + 1, 0), [self.defect]) # Order IOVs to avoid since > until self.iovs = IOVSet(sorted(self.iovs)) #for since, until, (run_lb, iov, dbDefect) in process_iovs(run_lbs.solidify(RANGEIOV_VAL), self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb): for since, until, (run_lb, iov, dbDefect) in process_iovs( run_lbs, self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb): if not run_lb: continue # Check valid # # Start LBs from 1 rather than 0 (at request of P. Onyisi) # # If this makes since==until, i.e. it was a [0->1) LB range then skip # if since.lumi==0: # since = RunLumiType((since.run << 32)+since.lumi+1) # if since==until: continue # Add iovs from self.iovs treating empty ones as defective (i.e. beamspot bad/missing) if iov: # These are not defective if dbDefect and dbDefect.present: # Only store not defective IoVs if they are present on the Db so we can unset them # (don't want to write not present defects to Db unless really chaning an existing defect) iovs.add(since, until, self.defect, False) else: # These are defective iovs.add(since, until, self.defect, True) self.iovs = iovs return
def test_fetch(): fetch_iovs("DCSOFL", runs=152166) fetch_iovs("COOLOFL_DCS::/LHC/DCS/FILLSTATE", runs=152166) fetch_iovs("COOLOFL_DCS/COMP200::/LHC/DCS/FILLSTATE", runs=152166)