def make_defectdiff_iovs(defects1, defects2, system_only, ignore_not_considered): """ Return a list of IoVs containing the set of defects only present in defects1 or defects2 """ result = IOVSet() defectset = ((since, until, (t1.defects if t1 else set(), t2.defects if t2 else set())) for since, until, (t1, t2) in process_iovs(defects1, defects2)) for since, until, (t1defects, t2defects) in defectset: if t1defects == t2defects: continue diff = t1defects.symmetric_difference(t2defects) if ignore_not_considered and "GLOBAL_NOTCONSIDERED" in diff: continue tag1only, tag2only = t1defects & diff, t2defects & diff if system_only: tag1only = set(x.split("_")[0] for x in tag1only) tag2only = set(x.split("_")[0] for x in tag2only) result.add(since, until, tag1only, tag2only) return result.solidify(DEFECTDIFF_VAL)
def main(): runs = 185640, 185660 lblb = fetch_iovs("LBLB", runs=runs) lbtime = inverse_lblb(lblb) beam_iovs = fetch_iovs("COOLOFL_DCS::/LHC/DCS/FILLSTATE", lbtime.first.since, lbtime.last.until) stable_beam_by_lb = {} for since_time, until_time, (lumiblock, beamstate) in process_iovs(lbtime, beam_iovs): if not lumiblock: # Not inside lumiblock continue lb = RunLumi(lumiblock.Run, lumiblock.LumiBlock) if lb in stable_beam_by_lb: # Require stable beam for ALL of the lumiblock (hence 'and') # Replace with 'or' to get ANY of the lumiblock stable_beam_by_lb[lb] = stable_beam_by_lb[lb] and beamstate.StableBeams else: stable_beam_by_lb[lb] = beamstate.StableBeams @define_iov_type def STABLEBEAM_VAL(stable): """ Define an IOV type which just has .stable """ result = (STABLEBEAM_VAL(runlb, runlb+1, state) for runlb, state in sorted(stable_beam_by_lb.iteritems())) result = IOVSet(connect_adjacent_iovs(result)) result.pprint()
def dump_grl(self, params): from DQUtils import IOVSet total = IOVSet() for f in params.files: f = R.TFile.Open(f) grls = loads(f.file_metadata.GetString().Data()) for grl in grls: grl = IOVSet.from_grl_string(grl) assert not grl & total total |= grl print total, "lumiblocks:", total.lb_counts total.to_grl(params.output)
def make_defectset_iovs(range_iovs, defect_iovs): """ Return a list of iovs with a set for each IoV containing the present defects """ chans, iovsets = defect_iovs.chans_iovsets result = IOVSet() for since, until, states in process_iovs(range_iovs, *iovsets): in_range, defects = states[0], states[1:] if in_range: result.add(since, until, set(d.channel for d in defects if d)) return result.solidify(DEFECTSET_VAL)
def compute_grl_diff(inputs, *iovsets): """ Returns a list of IoVs which differ for GRLs """ result = IOVSet() for since, until, grl_states in process_iovs(*iovsets): has_difference = any(grl_states) != all(grl_states) if has_difference: set_iovs = [d for d, state in zip(inputs, grl_states) if state] unset_iovs = [ d for d, state in zip(inputs, grl_states) if not state ] result.add(since, until, set_iovs, unset_iovs) return result.solidify(GRLSTATE_VAL)
def fetch_lumi_inputs(range_iov, tag="OflLumi-7TeV-002"): """ Retrieve information required to calculate the luminosity from the database """ lbs = fetch_iovs( "LBLB", with_channel=False, #loud=True, *range_iov) if tag == "ONLINE": lumis = fetch_iovs( "COOLONL_TRIGGER::/TRIGGER/LUMI/OnlPrefLumi", channels=[0], #loud=True, *range_iov) # Select valid lumis = IOVSet(l for l in lumis if (l.Valid & 0xFFFF) % 10 == 0) else: # Apparently this string changes from time to time... lumi_folder = "COOLOFL_TRIGGER::/TRIGGER/OFLLUMI/OflPrefLumi" # Run-2 #lumi_folder = "COOLOFL_TRIGGER::/TRIGGER/OFLLUMI/LBLESTOFL" # Run-1 lumis = fetch_iovs( lumi_folder, tag=tag, channels=[0], #loud=True, *range_iov) return lbs, lumis
def main(inputs): if not inputs: raise RuntimeError("Please specify a list of directories to compare " "on the command line") for i in inputs: exist = exists(i) if i.endswith(".xml") else isdir(i) if not exist: raise RuntimeError("Can't find input file/directory named %s" % i) # Load an iovset per GRL iovsets = [ IOVSet.from_grl(i) if i.endswith(".xml") else grl_from_dir(i) for i in inputs ] short_inputs = map(trunc_left, inputs) differences = compute_grl_diff(short_inputs, *iovsets) if differences: print_diff(differences) print "Files Checked:" for filename, iovs in zip(inputs, iovsets): print " Present LBs: %6i (%s)" % (iovs.lb_counts, filename) runs, lbs = len(differences.runs), differences.lb_counts print "Total runs (lbs) with differing states: %i (%i)" % (runs, lbs) else: print "GRLs are equal: " for filename in inputs: print filename raise SystemExit(bool(differences))
def split_grl(ptag, periods, grl_file, by_period): """ Doc """ grl = IOVSet.from_grl(grl_file) name_left, _, extension = basename(grl_file).rpartition(".") assert name_left, "No '.' in filename?" iovs_total = IOVSet() name_all_period = [] for period in periods: period_iovset = get_period_iovset_from_ami(ptag, period) iovs = grl & period_iovset iovs_total |= iovs name_all_period.append(period) if by_period: output_name = "{0}_period{1}.{2}".format(name_left, period, extension) print "{0:4s} : {1:3} runs, {2:5} lumiblocks".format( period, len(iovs.runs), iovs.lb_counts) iovs.to_grl(output_name) iovs_total = grl & iovs_total output_name = "{0}_period{1}.{2}".format(name_left, "to".join(name_all_period), extension) print "{0:4s} : {1:3} runs, {2:5} lumiblocks".format( name_all_period, len(iovs_total.runs), iovs_total.lb_counts) iovs_total.to_grl(output_name)
def new_defects_tag(self, name, description, iovranges=None): """ Clones the current DEFECTS tag (specified in the constructor) to a new one If iovranges != None, does the slower thing of copying IOVs one by one Parameters: `name` : Name of the new tag `description` : Description of the contents of this tag """ name = name.encode('ascii') description = description.encode('utf-8') if name.startswith("DetStatus"): raise RuntimeError("Only specify the last part of the defect tag") log.info("Creating new tag %s", name) name = "DetStatusDEFECTS-%s" % name if iovranges is None: self.defects_folder.cloneTagAsUserTag(self.defects_tag, name, description) return name # Fetch all primary defects relevant to `iovranges` kwargs = dict(primary_only=True, nonpresent=True, intersect=True) iovsets = (self.retrieve(*iovrange, **kwargs) for iovrange in iovranges) to_copy = IOVSet.from_iovsets(iovsets) log.info("Copying IoVs: %r", to_copy) with self.storage_buffer: for iov in to_copy: self._insert_iov(iov, name) # If there are no IOVs to copy, there is no new tag to describe if to_copy: self.defects_folder.setTagDescription(name, description) return name
def retrieve(self, since=None, until=None, channels=None, nonpresent=False, primary_only=False, ignore=None, with_primary_dependencies=False, intersect=False, with_time=False, evaluate_full=True): """ Retrieve defects from the database. Parameters: `since`, `until` : IoV range to query (Default: All) `channels` : A list of channels to query. Can contain a mixture of defect names and ids. (Default: None means all channels, including all virtual) `nonpresent` : Only return IoVs which are currently "present" `primary_only` : Only return primary IoVs, no virtual ones. `ignore` : Set of defects which won't be treated as bad. `with_primary_dependencies` : When querying virtual flags, also get primary flags which went into the calculation. `intersect` : Intersect the result with the query range so that no iov spans outside the query range `with_time` : Also retrieves the time the defect was inserted ~2x slower queries, doesn't work for virtual defects `evaluate_full` : If specified, also compute the `comment` and `recoverable` fields of virtual defects. Causes a ~0.6x slowdown """ if ignore is not None and not isinstance(ignore, set): assert False, "ignore parameter should be set type" desired_channels = None # Figure out the IDs of channels to query and their virtuality if channels is not None: query_channels = set(self.defect_names_as_ids(channels)) virtual_channels = query_channels - self.defect_ids primary_channels = query_channels & self.defect_ids else: # Empty channels list means query all channels # (including all virtual) if primary_only: virtual_channels = None else: virtual_channels = self.virtual_defect_ids primary_channels = sorted(self.defect_ids) query_channels = None # (all) primary_output_names = [self.defect_id_map[pid] for pid in primary_channels] virtual_output_names = [] # (optionally populated below) # Resolve virtual channels to query here, and the primary dependents. if virtual_channels: assert not primary_only, "Requested virtual channels with primary_only=True" assert not with_time, "with_time flag only works for primary defects" virtual_output_names = [self.virtual_defect_id_map[vid] for vid in virtual_channels] ordered_logics = self._resolve_evaluation_order(virtual_output_names) if channels is not None: # Since not all channels are being queried, it is necessary to # add the desired primary channels to the query primary_needed = self.resolve_primary_defects(ordered_logics) primary_channels |= set(self.defect_names_as_ids(primary_needed)) query_channels = primary_channels if with_primary_dependencies: primary_output_names.extend(sorted(primary_needed)) for logic in ordered_logics: logic.set_evaluation(evaluate_full) # Figure out if the set of channels will produce too many ranges for COOL filter_channels = None if query_channels is not None: query_channels = sorted(query_channels) query_ranges = list_to_channelselection(query_channels, None, True) if len(query_ranges) >= 50: # We're querying too many ranges. Query everything, filter later. desired_channels = set(primary_output_names + virtual_output_names) query_channels = None # (everything) # Retrieve primary IoVs primary_iovs = fetch_iovs(self.defects_folder, since, until, query_channels, self.defects_tag, named_channels=True, unicode_strings=True, with_time=with_time) # Calculate virtual defects (if necessary) if primary_only or not virtual_channels: result = primary_iovs else: if not primary_iovs: return IOVSet() args = (primary_iovs, ordered_logics, virtual_output_names, primary_output_names, since, until, ignore) result = calculate_virtual_defects(*args) # Filter out results which have their present bit removed # (unless otherwise specified) if not nonpresent: result = IOVSet(iov for iov in result if iov.present) # Filter out channels which weren't queried by the user # (to get around 50 channelselection COOL limit) if desired_channels: result = IOVSet(iov for iov in result if iov.channel in desired_channels) if intersect: result = result.intersect_range((since, until)) return result
def main(): import sys from argparse import ArgumentParser, FileType p = ArgumentParser(description="Compute differences in the defects " "database between tags") arg = p.add_argument arg("tags", metavar="DetStatus-Tag", nargs=2, type=str, help="Two tags to show the difference between") arg("-p", "--project", default=None, help="Limit considered runs to those in project") arg("-P", "--period", default=None, help="Data period") arg("-r", "--run", type=int, help="Run number to process") arg("-R", "--range", help="Run range: e.g. 150000-151000") arg("-d", "--defect", action="append", dest="defects", default=None, help="Defects to show the difference between. " "(If empty, show all. -d can be specified multiple times.)") arg("-i", "--ignore-not-considered", action="store_true", help="Ignore ranges in which GLOBAL_NOTCONSIDERED is set for either tag" ) arg("--output", default=sys.stdout, type=FileType("w"), help="Place to out the output (Default stdout)") arg("-s", "--by-system", action="store_true", help="If specified, show systems which differ, not defects") arg("-v", "--virtual", action="store_true", help="If specified, show virtual defect differences as well as primary" " (produces a lot of output)") args = p.parse_args() # tags tag1, tag2 = args.tags # range to process since, until = None, None #since, until = (200804, 0), (200804, 50) range_iovs = IOVSet() # if specifying run, only use that run, and it's easy if args.run: since, until = (args.run, 0), (args.run + 1, 0) range_iovs = IOVSet.from_runs([args.run]) elif args.range: run1, run2 = map(int, args.range.split("-")) since, until = (run1, 0), (run2, 0) range_iovs = IOVSet([RANGEIOV_VAL(RunLumi(since), RunLumi(until))]) # if specifying project, use that to build the range_iovs # TODO: what if period is specified but not project? # shouldn't I define a default project in that case? elif args.project: # fetch the project:period:runs dictionary project_dict = fetch_project_period_runs() period_dict = project_dict[args.project] # if specifying period, use those runs runs = set() if args.period: runs = set(period_dict[args.period]) else: #runs = set([ run for run in runs for (period, runs) in period_dict.iteritems() ]) for period, period_runs in period_dict.iteritems(): runs.update(period_runs) since, until = (min(runs), 0), (max(runs) + 1, 0) range_iovs = IOVSet.from_runs(runs) # debugging print 'range to process:' print 'since, until =', since, until #print 'range_iovs:' #range_iovs.pprint() # arguments for fetching defects kwargs = dict(primary_only=not args.virtual, channels=args.defects, since=since, until=until) def fetch_defects(tag): iovs = make_defectset_iovs(range_iovs, DefectsDB(tag=tag).retrieve(**kwargs)) return iovs print "Retrieving defects.." defects1, defects2 = fetch_defects(tag1), fetch_defects(tag2) #defects1.pprint() print "Computing diff.." diff_iovs = make_defectdiff_iovs(defects1, defects2, args.by_system, args.ignore_not_considered) for since, until, t1, t2 in diff_iovs: print since, until, "tag1:(%s)" % pretty(t1), "tag2:(%s)" % pretty(t2)
def main(): parser = ArgumentParser(description='List defect IOVs') add_arg = parser.add_argument # Specify range to process add_arg('-p', '--project', default='data15_13TeV', help='Data project (default: data15_13TeV)') add_arg('-P', '--period', default=None, nargs='*', help='Data period(s)') #add_arg('-r', '--run', default=None, nargs='*', help='Run number(s) to process') add_arg('-r', '--run', default=None, type=int, help='Run number to process') add_arg('-R', '--range', help='Inclusive run range: e.g. 150000-151000') # Specify defects to process add_arg('-d', '--defects', default=None, nargs='*', help='Defects to process. Use * for wildcard (default: None)') # Other job options add_arg('-c', '--connection-string', default=DEFAULT_CONNECTION_STRING, help='Database connection to use (default: %s)' % DEFAULT_CONNECTION_STRING) #add_arg('-l', '--lumi-tag', default='OflLumi-8TeV-001', #help='Luminosity tag (default: OflLumi-8TeV-001)') add_arg('-t', '--tag', default='HEAD', help='Tag to use (default: HEAD)') # Parse arguments args = parser.parse_args() # Ranges to query since, until = None, None range_iovs = IOVSet() # If specifying runs, use those if args.run: since, until = (args.run, 0), (args.run + 1, 0) range_iovs = IOVSet.from_runs([args.run]) elif args.range: run1, run2 = map(int, args.range.split('-')) since, until = (run1, 0), (run2, 0) range_iovs = IOVSet([RANGEIOV_VAL(RunLumi(since), RunLumi(until))]) # Otherwise, use the project and period settings elif args.project: # Fetch the project:period:runs dictionary project_dict = get_project_dict() period_dict = project_dict[args.project] run_set = set() # Use periods if specified if args.period and len(args.period) > 0: for period in args.period: run_set.update(period_dict[period]) # Otherwise use all periods in project else: for period, runs in period_dict.iteritems(): run_set.update(runs) since, until = (min(run_set), 0), (max(run_set) + 1, 0) range_iovs = IOVSet.from_runs(run_set) # debugging print 'range to process:' print 'since, until =', since, until # Instantiate the DB db = DefectsDB(args.connection_string, tag=args.tag) # Fetch the requested defect IOVs defect_iovs = db.retrieve(since, until, channels=args.defects) defect_iovset_list = defect_iovs.by_channel.values() # Get the overlap with the range_iovs # Use a dictionary of defect:iovset result_dict = defaultdict(IOVSet) for since, until, states in process_iovs(range_iovs, *defect_iovset_list): in_range, defects = states[0], states[1:] if in_range: for d in defects: if d: result_dict[d.channel].add(since, until, d.channel, d.comment) # Print the results for channel, result in result_dict.iteritems(): result.solidify(DefectIOV) print '\n' + channel + '\n' result.pprint()