def test_get_intolerable_defects():
'''
Check that the intolerable defect listing function works
'''
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create some primary defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
create_defect_type(ddb, 2)
create_defect_type(ddb, 3)
# Make some virtual defects
ddb.new_virtual_defect("DQD_TEST_Virtual_DEFECT", "Comment",
"DQD_TEST_DEFECT_0")
ddb.new_virtual_defect("PRIMARY", "Comment",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1")
ddb.new_virtual_defect("PHYS_test", "Comment",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1 DQD_TEST_DEFECT_2")
ddb.new_virtual_defect("TIGHT", "Comment",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1 DQD_TEST_DEFECT_2 DQD_TEST_DEFECT_3")
# old_primary_only should be True here, so PRIMARY gives the set of defects
assert sorted(ddb.get_intolerable_defects()) == ["DQD_TEST_DEFECT_0", "DQD_TEST_DEFECT_1"], "default params/old_primary_only check failed"
assert sorted(ddb.get_intolerable_defects(old_primary_only=False, exclude=['TIGHT'])) == ["DQD_TEST_DEFECT_0", "DQD_TEST_DEFECT_1", "DQD_TEST_DEFECT_2"], 'old_primary_only=False check failed'
assert sorted(ddb.get_intolerable_defects(old_primary_only=False)) == ["DQD_TEST_DEFECT_0", "DQD_TEST_DEFECT_1"], 'regex handling check failed'
def main():
d = DefectsDB()
iov_range = None, None #
iov_range = 185000 << 32, 185500 << 32
#iov_range = 177682 << 32, 200000 << 32
with timer("Fetch defect info"):
all_defects = d.retrieve(*iov_range)
with timer("fetch peripheral information"):
descriptions = d.all_defect_descriptions
intolerable = d.get_intolerable_defects()
virtual = d.virtual_defect_names
with timer("Fetch lumi inputs"):
lbs, lumis = fetch_lumi_inputs(iov_range, "ONLINE")
d = build_defects(descriptions, virtual, intolerable, lbs, lumis,
all_defects)
target_dir = "/afs/cern.ch/user/p/pwaller/www/defects/test"
art_dir = pjoin(target_dir, "extern")
if not exists(art_dir):
makedirs(art_dir)
copy_art(art_dir)
content = build_table(headings=heading_titles, defects=d)
with open(pjoin(target_dir, "test.html"), "w") as fd:
fd.write(content)
def main():
parser = ArgumentParser(description="Summarize DQ Defects")
a = parser.add_argument
add_group = parser.add_argument_group
# Specify range to process
a("-p",
"--project",
default="data15_13TeV",
help="Data project (default: data15_13TeV)")
a("-P", "--period", default=None, nargs="*", help="Data period(s)")
a("-r", "--run", default=None, nargs="*", help="Run number(s) to process")
a("-R", "--range", help="Inclusive run range: e.g. 150000-151000")
# Specify defects to process
a("-d",
"--defects",
default=None,
nargs="*",
help="Defects to process. Use * for wildcard (default: None)")
# Print mode, primary or virtual tree
a("-q",
"--tree",
action="store_true",
help="Dump virtual defect tree. Set depth with -D")
# Other job options
a("-n",
"--num-primary",
default=-1,
type=int,
help=
"Max number of primary defects to display in default mode (default: all)"
)
a("-D",
"--depth",
default=-1,
type=int,
help=
"Max virtual defect depth to print in virtual tree mode (default: all)")
a("-c",
"--connection-string",
default=DEFAULT_CONNECTION_STRING,
help="Database connection to use (default: %s)" %
DEFAULT_CONNECTION_STRING)
a("-l",
"--lumi-tag",
default='OflLumi-8TeV-002',
help="Luminosity tag (default: OflLumi-8TeV-002)")
a("-t", "--tag", default="HEAD", help="Tag to use (default: HEAD)")
a("--require-ready",
default=True,
type=int,
help="Calculate luminosity with respect to ATLAS READY (default: True)")
a("--reject-busy",
default=False,
type=int,
help="Calculate luminosity with respect to not-busy (default: False)")
a("--nb",
action="store_true",
help="Show lumi in units of 1/nb instead of 1/pb")
a("-v",
"--verbose",
default=1,
type=int,
help="Set verbosity (default: 1)")
args = parser.parse_args()
init_logger(verbose=args.verbose)
# Units
units = 1e3 if args.nb else 1e6
unit_string = "(1/nb)" if args.nb else "(1/pb)"
# Instantiate the database interface
with timer("Instantiate DefectsDB"):
db = DefectsDB(args.connection_string, tag=args.tag)
# Set runs to process
# TODO: control these via run lumi iovs, rather than a set of good_runs
# similar to how it is done in dq_defect_compare_tags
good_runs = None
if args.range:
since, until = map(int, args.range.split("-"))
elif args.run:
good_runs = set(map(int, args.run))
#since, until = args.run, args.run+1
since, until = min(good_runs), max(good_runs) + 1
else:
project_dict = fetch_project_period_runs()
if args.period:
#good_runs = set( project_dict[args.project][args.period] )
good_runs = set()
for period in args.period:
good_runs.update(project_dict[args.project][period])
since, until = min(good_runs), max(good_runs) + 1
else:
good_runs = set()
for period, period_runs in project_dict[args.project].iteritems():
good_runs.update(period_runs)
since, until = min(good_runs), max(good_runs) + 1
iov_range = (since, 0), (until, 0)
log.info("Processing range: " + str(iov_range))
#log.info(good_runs)
# Fetch all defect IOVs in range
with timer("Fetch defects"):
all_defect_iovs = db.retrieve(*iov_range)
# Grab defect information
with timer("Fetch defect info"):
descriptions = db.all_defect_descriptions
intolerables = db.get_intolerable_defects()
virtuals = db.virtual_defect_names
# Grab lbs and lumi variables in range
with timer("Fetch lumi inputs"):
lbs, lumis = fetch_lumi_inputs(iov_range, args.lumi_tag)
# Compute lumi per channel
with timer("Compute luminosities"):
# Filtering defects
exclude_iovs = []
# Defect for ATLAS READY
if args.require_ready:
exclude_iovs.append(all_defect_iovs.by_channel["GLOBAL_NOTREADY"])
# Defect for detector busy
if args.reject_busy:
exclude_iovs.append(all_defect_iovs.by_channel["GLOBAL_BUSY"])
# Compute total luminosity
lumi_total = compute_lumi(
lbs, lumis, lbs, exclude_iovsets=exclude_iovs,
good_runs=good_runs) / units
# Compute luminosity for all defects
lumi_by_defect = compute_lumi_many_channels(
lbs,
lumis,
all_defect_iovs,
exclude_iovsets=exclude_iovs,
good_runs=good_runs)
my_defect_names = []
if args.defects:
all_defect_names = db.defect_names | db.virtual_defect_names
for defect_arg in args.defects:
my_defect_names.extend(
sorted(fnmatch.filter(all_defect_names, defect_arg)))
logics = db.virtual_defect_logics
# Build the defect objects
all_defects = {}
for name, description in sorted(descriptions.iteritems()):
virtual, intolerable = name in virtuals, name in intolerables
depends = []
logic = logics.get(name, None)
if logic: depends = logic.clauses
all_defects[name] = Defect(name, description, depends, virtual,
intolerable,
lumi_by_defect.get(name, 0) / units)
# My defects
my_defects = [all_defects[d] for d in my_defect_names]
my_defects.sort(key=lambda d: d.lumi, reverse=True)
print "\nTotal luminosity", unit_string, "\n"
print "{0:<.2f}".format(lumi_total)
print "\nDefect luminosity", unit_string, "\n"
#for defect_name in my_defect_names:
for defect in my_defects:
if args.tree:
# Virtual defect tree
print_virtual_tree(defect,
all_defects,
depth=0,
max_depth=args.depth)
else:
# Primary defect dump
print_primary_defects(defect,
all_defects,
max_primary=args.num_primary)
print ""
