def test_get_intolerable_defects(): ''' Check that the intolerable defect listing function works ''' ddb = DefectsDB(TEST_DATABASE, read_only=False) # Create some primary defects create_defect_type(ddb, 0) create_defect_type(ddb, 1) create_defect_type(ddb, 2) create_defect_type(ddb, 3) # Make some virtual defects ddb.new_virtual_defect("DQD_TEST_Virtual_DEFECT", "Comment", "DQD_TEST_DEFECT_0") ddb.new_virtual_defect("PRIMARY", "Comment", "DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1") ddb.new_virtual_defect("PHYS_test", "Comment", "DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1 DQD_TEST_DEFECT_2") ddb.new_virtual_defect("TIGHT", "Comment", "DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1 DQD_TEST_DEFECT_2 DQD_TEST_DEFECT_3") # old_primary_only should be True here, so PRIMARY gives the set of defects assert sorted(ddb.get_intolerable_defects()) == ["DQD_TEST_DEFECT_0", "DQD_TEST_DEFECT_1"], "default params/old_primary_only check failed" assert sorted(ddb.get_intolerable_defects(old_primary_only=False, exclude=['TIGHT'])) == ["DQD_TEST_DEFECT_0", "DQD_TEST_DEFECT_1", "DQD_TEST_DEFECT_2"], 'old_primary_only=False check failed' assert sorted(ddb.get_intolerable_defects(old_primary_only=False)) == ["DQD_TEST_DEFECT_0", "DQD_TEST_DEFECT_1"], 'regex handling check failed'
def main(): d = DefectsDB() iov_range = None, None # iov_range = 185000 << 32, 185500 << 32 #iov_range = 177682 << 32, 200000 << 32 with timer("Fetch defect info"): all_defects = d.retrieve(*iov_range) with timer("fetch peripheral information"): descriptions = d.all_defect_descriptions intolerable = d.get_intolerable_defects() virtual = d.virtual_defect_names with timer("Fetch lumi inputs"): lbs, lumis = fetch_lumi_inputs(iov_range, "ONLINE") d = build_defects(descriptions, virtual, intolerable, lbs, lumis, all_defects) target_dir = "/afs/cern.ch/user/p/pwaller/www/defects/test" art_dir = pjoin(target_dir, "extern") if not exists(art_dir): makedirs(art_dir) copy_art(art_dir) content = build_table(headings=heading_titles, defects=d) with open(pjoin(target_dir, "test.html"), "w") as fd: fd.write(content)
def main(): parser = ArgumentParser(description="Summarize DQ Defects") a = parser.add_argument add_group = parser.add_argument_group # Specify range to process a("-p", "--project", default="data15_13TeV", help="Data project (default: data15_13TeV)") a("-P", "--period", default=None, nargs="*", help="Data period(s)") a("-r", "--run", default=None, nargs="*", help="Run number(s) to process") a("-R", "--range", help="Inclusive run range: e.g. 150000-151000") # Specify defects to process a("-d", "--defects", default=None, nargs="*", help="Defects to process. Use * for wildcard (default: None)") # Print mode, primary or virtual tree a("-q", "--tree", action="store_true", help="Dump virtual defect tree. Set depth with -D") # Other job options a("-n", "--num-primary", default=-1, type=int, help= "Max number of primary defects to display in default mode (default: all)" ) a("-D", "--depth", default=-1, type=int, help= "Max virtual defect depth to print in virtual tree mode (default: all)") a("-c", "--connection-string", default=DEFAULT_CONNECTION_STRING, help="Database connection to use (default: %s)" % DEFAULT_CONNECTION_STRING) a("-l", "--lumi-tag", default='OflLumi-8TeV-002', help="Luminosity tag (default: OflLumi-8TeV-002)") a("-t", "--tag", default="HEAD", help="Tag to use (default: HEAD)") a("--require-ready", default=True, type=int, help="Calculate luminosity with respect to ATLAS READY (default: True)") a("--reject-busy", default=False, type=int, help="Calculate luminosity with respect to not-busy (default: False)") a("--nb", action="store_true", help="Show lumi in units of 1/nb instead of 1/pb") a("-v", "--verbose", default=1, type=int, help="Set verbosity (default: 1)") args = parser.parse_args() init_logger(verbose=args.verbose) # Units units = 1e3 if args.nb else 1e6 unit_string = "(1/nb)" if args.nb else "(1/pb)" # Instantiate the database interface with timer("Instantiate DefectsDB"): db = DefectsDB(args.connection_string, tag=args.tag) # Set runs to process # TODO: control these via run lumi iovs, rather than a set of good_runs # similar to how it is done in dq_defect_compare_tags good_runs = None if args.range: since, until = map(int, args.range.split("-")) elif args.run: good_runs = set(map(int, args.run)) #since, until = args.run, args.run+1 since, until = min(good_runs), max(good_runs) + 1 else: project_dict = fetch_project_period_runs() if args.period: #good_runs = set( project_dict[args.project][args.period] ) good_runs = set() for period in args.period: good_runs.update(project_dict[args.project][period]) since, until = min(good_runs), max(good_runs) + 1 else: good_runs = set() for period, period_runs in project_dict[args.project].iteritems(): good_runs.update(period_runs) since, until = min(good_runs), max(good_runs) + 1 iov_range = (since, 0), (until, 0) log.info("Processing range: " + str(iov_range)) #log.info(good_runs) # Fetch all defect IOVs in range with timer("Fetch defects"): all_defect_iovs = db.retrieve(*iov_range) # Grab defect information with timer("Fetch defect info"): descriptions = db.all_defect_descriptions intolerables = db.get_intolerable_defects() virtuals = db.virtual_defect_names # Grab lbs and lumi variables in range with timer("Fetch lumi inputs"): lbs, lumis = fetch_lumi_inputs(iov_range, args.lumi_tag) # Compute lumi per channel with timer("Compute luminosities"): # Filtering defects exclude_iovs = [] # Defect for ATLAS READY if args.require_ready: exclude_iovs.append(all_defect_iovs.by_channel["GLOBAL_NOTREADY"]) # Defect for detector busy if args.reject_busy: exclude_iovs.append(all_defect_iovs.by_channel["GLOBAL_BUSY"]) # Compute total luminosity lumi_total = compute_lumi( lbs, lumis, lbs, exclude_iovsets=exclude_iovs, good_runs=good_runs) / units # Compute luminosity for all defects lumi_by_defect = compute_lumi_many_channels( lbs, lumis, all_defect_iovs, exclude_iovsets=exclude_iovs, good_runs=good_runs) my_defect_names = [] if args.defects: all_defect_names = db.defect_names | db.virtual_defect_names for defect_arg in args.defects: my_defect_names.extend( sorted(fnmatch.filter(all_defect_names, defect_arg))) logics = db.virtual_defect_logics # Build the defect objects all_defects = {} for name, description in sorted(descriptions.iteritems()): virtual, intolerable = name in virtuals, name in intolerables depends = [] logic = logics.get(name, None) if logic: depends = logic.clauses all_defects[name] = Defect(name, description, depends, virtual, intolerable, lumi_by_defect.get(name, 0) / units) # My defects my_defects = [all_defects[d] for d in my_defect_names] my_defects.sort(key=lambda d: d.lumi, reverse=True) print "\nTotal luminosity", unit_string, "\n" print "{0:<.2f}".format(lumi_total) print "\nDefect luminosity", unit_string, "\n" #for defect_name in my_defect_names: for defect in my_defects: if args.tree: # Virtual defect tree print_virtual_tree(defect, all_defects, depth=0, max_depth=args.depth) else: # Primary defect dump print_primary_defects(defect, all_defects, max_primary=args.num_primary) print ""