def make_defectdiff_iovs(defects1, defects2, system_only,
ignore_not_considered):
"""
Return a list of IoVs containing the set of defects only present in
defects1 or defects2
"""
result = IOVSet()
defectset = ((since, until, (t1.defects if t1 else set(),
t2.defects if t2 else set()))
for since, until, (t1,
t2) in process_iovs(defects1, defects2))
for since, until, (t1defects, t2defects) in defectset:
if t1defects == t2defects:
continue
diff = t1defects.symmetric_difference(t2defects)
if ignore_not_considered and "GLOBAL_NOTCONSIDERED" in diff:
continue
tag1only, tag2only = t1defects & diff, t2defects & diff
if system_only:
tag1only = set(x.split("_")[0] for x in tag1only)
tag2only = set(x.split("_")[0] for x in tag2only)
result.add(since, until, tag1only, tag2only)
return result.solidify(DEFECTDIFF_VAL)
def main():
runs = 185640, 185660
lblb = fetch_iovs("LBLB", runs=runs)
lbtime = inverse_lblb(lblb)
beam_iovs = fetch_iovs("COOLOFL_DCS::/LHC/DCS/FILLSTATE", lbtime.first.since, lbtime.last.until)
stable_beam_by_lb = {}
for since_time, until_time, (lumiblock, beamstate) in process_iovs(lbtime, beam_iovs):
if not lumiblock:
# Not inside lumiblock
continue
lb = RunLumi(lumiblock.Run, lumiblock.LumiBlock)
if lb in stable_beam_by_lb:
# Require stable beam for ALL of the lumiblock (hence 'and')
# Replace with 'or' to get ANY of the lumiblock
stable_beam_by_lb[lb] = stable_beam_by_lb[lb] and beamstate.StableBeams
else:
stable_beam_by_lb[lb] = beamstate.StableBeams
@define_iov_type
def STABLEBEAM_VAL(stable):
"""
Define an IOV type which just has .stable
"""
result = (STABLEBEAM_VAL(runlb, runlb+1, state)
for runlb, state in sorted(stable_beam_by_lb.iteritems()))
result = IOVSet(connect_adjacent_iovs(result))
result.pprint()
def dump_grl(self, params):
from DQUtils import IOVSet
total = IOVSet()
for f in params.files:
f = R.TFile.Open(f)
grls = loads(f.file_metadata.GetString().Data())
for grl in grls:
grl = IOVSet.from_grl_string(grl)
assert not grl & total
total |= grl
print total, "lumiblocks:", total.lb_counts
total.to_grl(params.output)
def make_defectset_iovs(range_iovs, defect_iovs):
"""
Return a list of iovs with a set for each IoV containing the present defects
"""
chans, iovsets = defect_iovs.chans_iovsets
result = IOVSet()
for since, until, states in process_iovs(range_iovs, *iovsets):
in_range, defects = states[0], states[1:]
if in_range:
result.add(since, until, set(d.channel for d in defects if d))
return result.solidify(DEFECTSET_VAL)
def compute_grl_diff(inputs, *iovsets):
"""
Returns a list of IoVs which differ for GRLs
"""
result = IOVSet()
for since, until, grl_states in process_iovs(*iovsets):
has_difference = any(grl_states) != all(grl_states)
if has_difference:
set_iovs = [d for d, state in zip(inputs, grl_states) if state]
unset_iovs = [
d for d, state in zip(inputs, grl_states) if not state
]
result.add(since, until, set_iovs, unset_iovs)
return result.solidify(GRLSTATE_VAL)
def fetch_lumi_inputs(range_iov, tag="OflLumi-7TeV-002"):
"""
Retrieve information required to calculate the luminosity from the database
"""
lbs = fetch_iovs(
"LBLB",
with_channel=False, #loud=True,
*range_iov)
if tag == "ONLINE":
lumis = fetch_iovs(
"COOLONL_TRIGGER::/TRIGGER/LUMI/OnlPrefLumi",
channels=[0], #loud=True,
*range_iov)
# Select valid
lumis = IOVSet(l for l in lumis if (l.Valid & 0xFFFF) % 10 == 0)
else:
# Apparently this string changes from time to time...
lumi_folder = "COOLOFL_TRIGGER::/TRIGGER/OFLLUMI/OflPrefLumi" # Run-2
#lumi_folder = "COOLOFL_TRIGGER::/TRIGGER/OFLLUMI/LBLESTOFL" # Run-1
lumis = fetch_iovs(
lumi_folder,
tag=tag,
channels=[0], #loud=True,
*range_iov)
return lbs, lumis
def main(inputs):
if not inputs:
raise RuntimeError("Please specify a list of directories to compare "
"on the command line")
for i in inputs:
exist = exists(i) if i.endswith(".xml") else isdir(i)
if not exist:
raise RuntimeError("Can't find input file/directory named %s" % i)
# Load an iovset per GRL
iovsets = [
IOVSet.from_grl(i) if i.endswith(".xml") else grl_from_dir(i)
for i in inputs
]
short_inputs = map(trunc_left, inputs)
differences = compute_grl_diff(short_inputs, *iovsets)
if differences:
print_diff(differences)
print "Files Checked:"
for filename, iovs in zip(inputs, iovsets):
print " Present LBs: %6i (%s)" % (iovs.lb_counts, filename)
runs, lbs = len(differences.runs), differences.lb_counts
print "Total runs (lbs) with differing states: %i (%i)" % (runs, lbs)
else:
print "GRLs are equal: "
for filename in inputs:
print filename
raise SystemExit(bool(differences))
def split_grl(ptag, periods, grl_file, by_period):
"""
Doc
"""
grl = IOVSet.from_grl(grl_file)
name_left, _, extension = basename(grl_file).rpartition(".")
assert name_left, "No '.' in filename?"
iovs_total = IOVSet()
name_all_period = []
for period in periods:
period_iovset = get_period_iovset_from_ami(ptag, period)
iovs = grl & period_iovset
iovs_total |= iovs
name_all_period.append(period)
if by_period:
output_name = "{0}_period{1}.{2}".format(name_left, period,
extension)
print "{0:4s} : {1:3} runs, {2:5} lumiblocks".format(
period, len(iovs.runs), iovs.lb_counts)
iovs.to_grl(output_name)
iovs_total = grl & iovs_total
output_name = "{0}_period{1}.{2}".format(name_left,
"to".join(name_all_period),
extension)
print "{0:4s} : {1:3} runs, {2:5} lumiblocks".format(
name_all_period, len(iovs_total.runs), iovs_total.lb_counts)
iovs_total.to_grl(output_name)
def new_defects_tag(self, name, description, iovranges=None):
"""
Clones the current DEFECTS tag (specified in the constructor) to a new one
If iovranges != None, does the slower thing of copying IOVs one by one
Parameters:
`name` : Name of the new tag
`description` : Description of the contents of this tag
"""
name = name.encode('ascii')
description = description.encode('utf-8')
if name.startswith("DetStatus"):
raise RuntimeError("Only specify the last part of the defect tag")
log.info("Creating new tag %s", name)
name = "DetStatusDEFECTS-%s" % name
if iovranges is None:
self.defects_folder.cloneTagAsUserTag(self.defects_tag, name,
description)
return name
# Fetch all primary defects relevant to `iovranges`
kwargs = dict(primary_only=True, nonpresent=True, intersect=True)
iovsets = (self.retrieve(*iovrange, **kwargs)
for iovrange in iovranges)
to_copy = IOVSet.from_iovsets(iovsets)
log.info("Copying IoVs: %r", to_copy)
with self.storage_buffer:
for iov in to_copy:
self._insert_iov(iov, name)
# If there are no IOVs to copy, there is no new tag to describe
if to_copy:
self.defects_folder.setTagDescription(name, description)
return name
def retrieve(self, since=None, until=None, channels=None, nonpresent=False,
primary_only=False, ignore=None,
with_primary_dependencies=False, intersect=False,
with_time=False, evaluate_full=True):
"""
Retrieve defects from the database.
Parameters:
`since`, `until` : IoV range to query (Default: All)
`channels` : A list of channels to query. Can contain a mixture of
defect names and ids. (Default: None means all
channels, including all virtual)
`nonpresent` : Only return IoVs which are currently "present"
`primary_only` : Only return primary IoVs, no virtual ones.
`ignore` : Set of defects which won't be treated as bad.
`with_primary_dependencies` : When querying virtual flags, also get
primary flags which went into the calculation.
`intersect` : Intersect the result with the query range so that no
iov spans outside the query range
`with_time` : Also retrieves the time the defect was inserted
~2x slower queries, doesn't work for virtual defects
`evaluate_full` : If specified, also compute the `comment` and
`recoverable` fields of virtual defects.
Causes a ~0.6x slowdown
"""
if ignore is not None and not isinstance(ignore, set):
assert False, "ignore parameter should be set type"
desired_channels = None
# Figure out the IDs of channels to query and their virtuality
if channels is not None:
query_channels = set(self.defect_names_as_ids(channels))
virtual_channels = query_channels - self.defect_ids
primary_channels = query_channels & self.defect_ids
else:
# Empty channels list means query all channels
# (including all virtual)
if primary_only:
virtual_channels = None
else:
virtual_channels = self.virtual_defect_ids
primary_channels = sorted(self.defect_ids)
query_channels = None # (all)
primary_output_names = [self.defect_id_map[pid] for pid in primary_channels]
virtual_output_names = [] # (optionally populated below)
# Resolve virtual channels to query here, and the primary dependents.
if virtual_channels:
assert not primary_only, "Requested virtual channels with primary_only=True"
assert not with_time, "with_time flag only works for primary defects"
virtual_output_names = [self.virtual_defect_id_map[vid]
for vid in virtual_channels]
ordered_logics = self._resolve_evaluation_order(virtual_output_names)
if channels is not None:
# Since not all channels are being queried, it is necessary to
# add the desired primary channels to the query
primary_needed = self.resolve_primary_defects(ordered_logics)
primary_channels |= set(self.defect_names_as_ids(primary_needed))
query_channels = primary_channels
if with_primary_dependencies:
primary_output_names.extend(sorted(primary_needed))
for logic in ordered_logics:
logic.set_evaluation(evaluate_full)
# Figure out if the set of channels will produce too many ranges for COOL
filter_channels = None
if query_channels is not None:
query_channels = sorted(query_channels)
query_ranges = list_to_channelselection(query_channels, None, True)
if len(query_ranges) >= 50:
# We're querying too many ranges. Query everything, filter later.
desired_channels = set(primary_output_names + virtual_output_names)
query_channels = None # (everything)
# Retrieve primary IoVs
primary_iovs = fetch_iovs(self.defects_folder, since, until,
query_channels, self.defects_tag,
named_channels=True, unicode_strings=True,
with_time=with_time)
# Calculate virtual defects (if necessary)
if primary_only or not virtual_channels:
result = primary_iovs
else:
if not primary_iovs:
return IOVSet()
args = (primary_iovs, ordered_logics,
virtual_output_names, primary_output_names,
since, until, ignore)
result = calculate_virtual_defects(*args)
# Filter out results which have their present bit removed
# (unless otherwise specified)
if not nonpresent:
result = IOVSet(iov for iov in result if iov.present)
# Filter out channels which weren't queried by the user
# (to get around 50 channelselection COOL limit)
if desired_channels:
result = IOVSet(iov for iov in result
if iov.channel in desired_channels)
if intersect:
result = result.intersect_range((since, until))
return result
def main():
import sys
from argparse import ArgumentParser, FileType
p = ArgumentParser(description="Compute differences in the defects "
"database between tags")
arg = p.add_argument
arg("tags",
metavar="DetStatus-Tag",
nargs=2,
type=str,
help="Two tags to show the difference between")
arg("-p",
"--project",
default=None,
help="Limit considered runs to those in project")
arg("-P", "--period", default=None, help="Data period")
arg("-r", "--run", type=int, help="Run number to process")
arg("-R", "--range", help="Run range: e.g. 150000-151000")
arg("-d",
"--defect",
action="append",
dest="defects",
default=None,
help="Defects to show the difference between. "
"(If empty, show all. -d can be specified multiple times.)")
arg("-i",
"--ignore-not-considered",
action="store_true",
help="Ignore ranges in which GLOBAL_NOTCONSIDERED is set for either tag"
)
arg("--output",
default=sys.stdout,
type=FileType("w"),
help="Place to out the output (Default stdout)")
arg("-s",
"--by-system",
action="store_true",
help="If specified, show systems which differ, not defects")
arg("-v",
"--virtual",
action="store_true",
help="If specified, show virtual defect differences as well as primary"
" (produces a lot of output)")
args = p.parse_args()
# tags
tag1, tag2 = args.tags
# range to process
since, until = None, None
#since, until = (200804, 0), (200804, 50)
range_iovs = IOVSet()
# if specifying run, only use that run, and it's easy
if args.run:
since, until = (args.run, 0), (args.run + 1, 0)
range_iovs = IOVSet.from_runs([args.run])
elif args.range:
run1, run2 = map(int, args.range.split("-"))
since, until = (run1, 0), (run2, 0)
range_iovs = IOVSet([RANGEIOV_VAL(RunLumi(since), RunLumi(until))])
# if specifying project, use that to build the range_iovs
# TODO: what if period is specified but not project?
# shouldn't I define a default project in that case?
elif args.project:
# fetch the project:period:runs dictionary
project_dict = fetch_project_period_runs()
period_dict = project_dict[args.project]
# if specifying period, use those runs
runs = set()
if args.period:
runs = set(period_dict[args.period])
else:
#runs = set([ run for run in runs for (period, runs) in period_dict.iteritems() ])
for period, period_runs in period_dict.iteritems():
runs.update(period_runs)
since, until = (min(runs), 0), (max(runs) + 1, 0)
range_iovs = IOVSet.from_runs(runs)
# debugging
print 'range to process:'
print 'since, until =', since, until
#print 'range_iovs:'
#range_iovs.pprint()
# arguments for fetching defects
kwargs = dict(primary_only=not args.virtual,
channels=args.defects,
since=since,
until=until)
def fetch_defects(tag):
iovs = make_defectset_iovs(range_iovs,
DefectsDB(tag=tag).retrieve(**kwargs))
return iovs
print "Retrieving defects.."
defects1, defects2 = fetch_defects(tag1), fetch_defects(tag2)
#defects1.pprint()
print "Computing diff.."
diff_iovs = make_defectdiff_iovs(defects1, defects2, args.by_system,
args.ignore_not_considered)
for since, until, t1, t2 in diff_iovs:
print since, until, "tag1:(%s)" % pretty(t1), "tag2:(%s)" % pretty(t2)
def main():
parser = ArgumentParser(description='List defect IOVs')
add_arg = parser.add_argument
# Specify range to process
add_arg('-p',
'--project',
default='data15_13TeV',
help='Data project (default: data15_13TeV)')
add_arg('-P', '--period', default=None, nargs='*', help='Data period(s)')
#add_arg('-r', '--run', default=None, nargs='*', help='Run number(s) to process')
add_arg('-r',
'--run',
default=None,
type=int,
help='Run number to process')
add_arg('-R', '--range', help='Inclusive run range: e.g. 150000-151000')
# Specify defects to process
add_arg('-d',
'--defects',
default=None,
nargs='*',
help='Defects to process. Use * for wildcard (default: None)')
# Other job options
add_arg('-c',
'--connection-string',
default=DEFAULT_CONNECTION_STRING,
help='Database connection to use (default: %s)' %
DEFAULT_CONNECTION_STRING)
#add_arg('-l', '--lumi-tag', default='OflLumi-8TeV-001',
#help='Luminosity tag (default: OflLumi-8TeV-001)')
add_arg('-t', '--tag', default='HEAD', help='Tag to use (default: HEAD)')
# Parse arguments
args = parser.parse_args()
# Ranges to query
since, until = None, None
range_iovs = IOVSet()
# If specifying runs, use those
if args.run:
since, until = (args.run, 0), (args.run + 1, 0)
range_iovs = IOVSet.from_runs([args.run])
elif args.range:
run1, run2 = map(int, args.range.split('-'))
since, until = (run1, 0), (run2, 0)
range_iovs = IOVSet([RANGEIOV_VAL(RunLumi(since), RunLumi(until))])
# Otherwise, use the project and period settings
elif args.project:
# Fetch the project:period:runs dictionary
project_dict = get_project_dict()
period_dict = project_dict[args.project]
run_set = set()
# Use periods if specified
if args.period and len(args.period) > 0:
for period in args.period:
run_set.update(period_dict[period])
# Otherwise use all periods in project
else:
for period, runs in period_dict.iteritems():
run_set.update(runs)
since, until = (min(run_set), 0), (max(run_set) + 1, 0)
range_iovs = IOVSet.from_runs(run_set)
# debugging
print 'range to process:'
print 'since, until =', since, until
# Instantiate the DB
db = DefectsDB(args.connection_string, tag=args.tag)
# Fetch the requested defect IOVs
defect_iovs = db.retrieve(since, until, channels=args.defects)
defect_iovset_list = defect_iovs.by_channel.values()
# Get the overlap with the range_iovs
# Use a dictionary of defect:iovset
result_dict = defaultdict(IOVSet)
for since, until, states in process_iovs(range_iovs, *defect_iovset_list):
in_range, defects = states[0], states[1:]
if in_range:
for d in defects:
if d:
result_dict[d.channel].add(since, until, d.channel,
d.comment)
# Print the results
for channel, result in result_dict.iteritems():
result.solidify(DefectIOV)
print '\n' + channel + '\n'
result.pprint()