def test_many_sparse_channels(): "Testing many virtual channels with sparsely placed IoVs" ddb = DefectsDB(TEST_DATABASE, read_only=False) all_defects = [] for i in range(52): all_defects.append(create_defect_type(ddb, i * 2)) with ddb.storage_buffer: for i in all_defects: ddb.insert(i, 0, 100, "Test", "DQDefects.tests") queried_defects = all_defects[:-1] result = ddb.retrieve(channels=queried_defects) queried_defects_set = set(queried_defects) result_defects_set = set(i.channel for i in result) unwanted_defects_set = result_defects_set - queried_defects_set assert not unwanted_defects_set, "Unwanted defects: %r" % unwanted_defects_set ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT", "Comment", " ".join(all_defects)) result = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"]) assert result.channels == set(["DQD_TEST_VIRTUAL_DEFECT"]) assert len(result) == 1 # This assumes evaluate_full == True assert result[0].comment == " ".join( sorted(all_defects)), result[0].comment result = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"], evaluate_full=False) assert result.channels == set(["DQD_TEST_VIRTUAL_DEFECT"]) assert len(result) == 1
def test_ignore_defects(): ddb = DefectsDB(TEST_DATABASE, read_only=False) # Create two defects create_defect_type(ddb, 0) create_defect_type(ddb, 1) create_defect_type(ddb, 2) ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "", "") ddb.insert("DQD_TEST_DEFECT_1", 100, 200, "", "") ddb.insert("DQD_TEST_DEFECT_2", 200, 300, "", "") ddb.new_virtual_defect( "DQD_TEST_VIRTUAL_DEFECT", "", "DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1 DQD_TEST_DEFECT_2") ignored_defects = set(["DQD_TEST_DEFECT_1"]) iovs = ddb.retrieve() assert iov_result_depends(iovs) & ignored_defects, "test is broken" iovs = ddb.retrieve(ignore=ignored_defects) assert not iov_result_depends( iovs) & ignored_defects, "result depends on ignored defects"
def test_query_with_primary_dependencies_with_ignore(): """ Checking with_primary_dependencies + ignore Checks that the set of channels and iovs returned contains the correct number of channels and iovs when specifying both with_primary_dependencies and ignore to DQDefects.Retrieve """ ddb = DefectsDB(TEST_DATABASE, read_only=False) # Create two defects defect_names = [create_defect_type(ddb, i) for i in range(3)] for i, defect in enumerate(defect_names): ddb.insert(defect, i * 100, (i + 1) * 100, "", "") ignored = set(defect_names[:1]) ddb.new_virtual_defect("DQD_TEST_VDEFECT", "", " ".join(defect_names)) iovs = ddb.retrieve(channels=["DQD_TEST_VDEFECT"], ignore=ignored) assert iovs.channels == set(["DQD_TEST_VDEFECT"]) assert len(iovs) == len(defect_names) - 1 iovs = ddb.retrieve(channels=["DQD_TEST_VDEFECT"], ignore=ignored, with_primary_dependencies=True) assert iovs.channels == set(["DQD_TEST_VDEFECT"] + defect_names) - ignored, ("Check failed.") assert len(iovs) == len(defect_names[1:]) * 2
def test_virtual_defects_invert(): "Testing virtual defects involving inversion" ddb = DefectsDB(TEST_DATABASE, read_only=False) # Create two defects create_defect_type(ddb, 0) create_defect_type(ddb, 1) create_defect_type(ddb, 2) # Make a virtual defect whose result is the combination of the above ddb.new_virtual_defect( "DQD_TEST_VIRTUAL_DEFECT", "Comment", "DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1 !DQD_TEST_DEFECT_2") # Insert an iov into each ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests") ddb.insert("DQD_TEST_DEFECT_1", 150, 200, "Test", "DQDefects.tests") ddb.insert("DQD_TEST_DEFECT_2", 50, 125, "Test", "DQDefects.tests") # See what the result of the virtual defect is iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"]) # Tests against the value assert len(iovs) == 4, list(iovs) first, second, third, fourth = iovs assert (first.since, first.until) == (0, 50), first assert (second.since, second.until) == (50, 100), second assert (third.since, third.until) == (125, 150), third assert (fourth.since, fourth.until) == (150, 200), fourth assert first.channel == second.channel == third.channel == fourth.channel == "DQD_TEST_VIRTUAL_DEFECT", ( first.channel, second.channel, third.channel, fourth.channel) del first, second, third, fourth, iovs # Test evaluate_full=False iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"], evaluate_full=False) assert len(iovs) == 2, list(iovs) # Now unset the present bit ddb.insert("DQD_TEST_DEFECT_2", 50, 150, "Test", "DQDefects.tests", present=False) iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"]) # here we are sensitive to details of python ordering ... assert len(iovs) == 3, list(iovs) first, second, third = iovs assert (first.since, first.until) == (0, 100), first assert (second.since, second.until) == (100, 150), second assert (third.since, third.until) == (150, 200), third assert first.channel == second.channel == third.channel == "DQD_TEST_VIRTUAL_DEFECT", ( first.channel, second.channel, third.channel) del first, second, third, iovs
def test_virtual_defects(): "Testing virtual defect basics" ddb = DefectsDB(TEST_DATABASE, read_only=False) # Create two defects create_defect_type(ddb, 0) create_defect_type(ddb, 1) # Make a virtual defect whose result is the combination of the above ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT", "Comment", "DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1") # Insert an iov into each ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests") ddb.insert("DQD_TEST_DEFECT_1", 150, 200, "Test", "DQDefects.tests") # See what the result of the virtual defect is iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"]) # Tests against the value assert len(iovs) == 2 first, second = iovs assert (first.since, first.until) == (0, 100) assert (second.since, second.until) == (150, 200) assert first.channel == second.channel == "DQD_TEST_VIRTUAL_DEFECT" del first, second, iovs # Now unset the present bit, and test again that the relevant preiod is fixed ddb.insert("DQD_TEST_DEFECT_1", 150, 200, "Test", "DQDefects.tests", present=False) iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"]) assert len(iovs) == 1 first = iovs[0] assert (first.since, first.until) == (0, 100) assert first.channel == "DQD_TEST_VIRTUAL_DEFECT" # Now extend the defect and see if we get a contiguous result ddb.insert("DQD_TEST_DEFECT_0", 100, 150, "Test", "DQDefects.tests") iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"]) assert len(iovs) == 1 first = iovs[0] assert (first.since, first.until) == (0, 150) assert first.channel == "DQD_TEST_VIRTUAL_DEFECT"
def test_defect_empty_retrieval(): ddb = DefectsDB(TEST_DATABASE, read_only=False) iovs = ddb.retrieve() assert not iovs # Create two defects create_defect_type(ddb, 0) create_defect_type(ddb, 1) # Make a virtual defect whose result is the combination of the above ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT", "Comment", "DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1") iovs = ddb.retrieve() assert not iovs
def test_nonpresent(): ddb = DefectsDB(TEST_DATABASE, read_only=False) create_defect_type(ddb, 0) ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests") ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests", present=False) iovs = ddb.retrieve() assert len(iovs) == 0 iovs = ddb.retrieve(nonpresent=True) assert len(iovs) == 1 assert not iovs[0].present
def test_defect_insertion_retrieval_unicode(): if six.PY3: import ROOT if ROOT.gROOT.GetVersionInt() < 62000: # Passing str objects using multibyte encodings is broken # with pyroot up to 6.18. Should be fixed in 6.20? return ddb = DefectsDB(TEST_DATABASE, read_only=False) TEST_ID = 0 TEST_DEFECT_NAME = "DQD_TEST_DEFECT_%i" % TEST_ID TEST_COMMENT = u"First test defect with character: ±" TEST_USER = u"DQDefécts.tests" TEST_SINCE, TEST_UNTIL = 0, 100 TEST_DEFECT_DESCRIPTION = u"À test defect (%i)" % TEST_ID create_defect_type_unicode(ddb, TEST_ID, TEST_DEFECT_DESCRIPTION) ddb.insert(TEST_DEFECT_NAME, TEST_SINCE, TEST_UNTIL, TEST_COMMENT, TEST_USER) iovs = ddb.retrieve() assert len(iovs) == 1 iov = iovs[0] assert iov.present assert not iov.recoverable assert iov.user == TEST_USER assert iov.comment == TEST_COMMENT assert ddb.all_defect_descriptions[ TEST_DEFECT_NAME] == TEST_DEFECT_DESCRIPTION
def test_defect_insertion_retrieval(): ddb = DefectsDB(TEST_DATABASE, read_only=False) TEST_ID = 0 TEST_DEFECT_NAME = "DQD_TEST_DEFECT_%i" % TEST_ID TEST_COMMENT = "First test defect" TEST_USER = "******" TEST_SINCE, TEST_UNTIL = 0, 100 create_defect_type(ddb, TEST_ID) ddb.insert(TEST_DEFECT_NAME, TEST_SINCE, TEST_UNTIL, TEST_COMMENT, TEST_USER) iovs = ddb.retrieve() assert len(iovs) == 1 iov = iovs[0] assert iov.since == TEST_SINCE assert iov.until == TEST_UNTIL assert iov.channel == TEST_DEFECT_NAME assert iov.present assert not iov.recoverable assert iov.user == TEST_USER assert iov.comment == TEST_COMMENT
def test_iov_tag_defects(): log.info('IOV Tags') ddb = DefectsDB(TEST_DATABASE, read_only=False) # Create two defects create_defect_type(ddb, 0) create_defect_type(ddb, 1) create_defect_type(ddb, 2) ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "comment1", "user1", present=False) ddb.insert("DQD_TEST_DEFECT_1", 100, 200, "", "") ddb.insert("DQD_TEST_DEFECT_2", 200, 300, "comment2", "user2", recoverable=True) ddb.new_defects_tag("dqd-test", "New iov tag", iovranges=[(0, 51), ((0, 210), (0, 306))]) ddb2 = DefectsDB(TEST_DATABASE, tag='DetStatusDEFECTS-dqd-test') iovs = ddb2.retrieve(nonpresent=True) assert len(iovs) == 2 assert ((iovs[0].channel, iovs[0].since, iovs[0].until, iovs[0].present, iovs[0].recoverable, iovs[0].user, iovs[0].comment) == ('DQD_TEST_DEFECT_0', 0, 51, False, False, 'user1', 'comment1')) assert ((iovs[1].channel, iovs[1].since, iovs[1].until, iovs[1].present, iovs[1].recoverable, iovs[1].user, iovs[1].comment) == ('DQD_TEST_DEFECT_2', 210, 300, True, True, 'user2', 'comment2'))
def main(): d = DefectsDB() iov_range = None, None # iov_range = 185000 << 32, 185500 << 32 #iov_range = 177682 << 32, 200000 << 32 with timer("Fetch defect info"): all_defects = d.retrieve(*iov_range) with timer("fetch peripheral information"): descriptions = d.all_defect_descriptions intolerable = d.get_intolerable_defects() virtual = d.virtual_defect_names with timer("Fetch lumi inputs"): lbs, lumis = fetch_lumi_inputs(iov_range, "ONLINE") d = build_defects(descriptions, virtual, intolerable, lbs, lumis, all_defects) target_dir = "/afs/cern.ch/user/p/pwaller/www/defects/test" art_dir = pjoin(target_dir, "extern") if not exists(art_dir): makedirs(art_dir) copy_art(art_dir) content = build_table(headings=heading_titles, defects=d) with open(pjoin(target_dir, "test.html"), "w") as fd: fd.write(content)
def test_virtual_defects_deep(): ddb = DefectsDB(TEST_DATABASE, read_only=False) # Create two defects create_defect_type(ddb, 0) create_defect_type(ddb, 1) # Insert an iov into each ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests") ddb.insert("DQD_TEST_DEFECT_1", 150, 200, "Test", "DQDefects.tests") # Make a virtual defect whose result is the combination of the above ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT_0", "", "DQD_TEST_DEFECT_0") ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT_1", "", "DQD_TEST_DEFECT_1") ddb = DefectsDB(TEST_DATABASE) ddb.retrieve()
def test_intersect(): ddb = DefectsDB(TEST_DATABASE, read_only=False) # Create two defects create_defect_type(ddb, 0) create_defect_type(ddb, 1) create_defect_type(ddb, 2) ddb.insert("DQD_TEST_DEFECT_0", 0, 1000, "", "") ddb.insert("DQD_TEST_DEFECT_0", 1000, 2000, "", "") iovs = ddb.retrieve(50, 100, intersect=True) assert len(iovs) == 1 assert (iovs[0].since, iovs[0].until) == (50, 100) iovs = ddb.retrieve(999, 1001, intersect=True) assert len(iovs) == 2 first, second = iovs assert (first.since, first.until) == (999, 1000) assert (second.since, second.until) == (1000, 1001)
def test_query_with_primary_dependencies(): ddb = DefectsDB(TEST_DATABASE, read_only=False) # Create two defects defect_names = [create_defect_type(ddb, i) for i in range(3)] for i, defect in enumerate(defect_names): ddb.insert(defect, i * 100, (i + 1) * 100, "", "") ddb.new_virtual_defect("DQD_TEST_VDEFECT", "", " ".join(defect_names)) iovs = ddb.retrieve(channels=["DQD_TEST_VDEFECT"]) assert iovs.channels == set(["DQD_TEST_VDEFECT"]) assert len(iovs) == len(defect_names) iovs = ddb.retrieve(channels=["DQD_TEST_VDEFECT"], with_primary_dependencies=True) assert iovs.channels == set(["DQD_TEST_VDEFECT"] + defect_names), ("Check failed.") assert len(iovs) == len(defect_names) * 2
def test_virtual_defects_stress(): ddb = DefectsDB(TEST_DATABASE, read_only=False) STRESS_COUNT = 20 for i in range(STRESS_COUNT): create_defect_type(ddb, i) with ddb.storage_buffer: for i in range(STRESS_COUNT): ddb.insert("DQD_TEST_DEFECT_%i" % i, i, i + 1, "Test", "DQDefects.tests") ALL_DEFECTS = " ".join("DQD_TEST_DEFECT_%i" % i for i in range(STRESS_COUNT)) ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT", "", ALL_DEFECTS) iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"]) assert len(iovs) == STRESS_COUNT - 1 assert (iovs[0].since, iovs[-1].until) == (1, STRESS_COUNT)
def test_defect_insertion_retrieval_unicode(): ddb = DefectsDB(TEST_DATABASE, read_only=False) TEST_ID = 0 TEST_DEFECT_NAME = "DQD_TEST_DEFECT_%i" % TEST_ID TEST_COMMENT = u"First test defect with character: ±" TEST_USER = u"DQDefécts.tests" TEST_SINCE, TEST_UNTIL = 0, 100 TEST_DEFECT_DESCRIPTION = u"À test defect (%i)" % TEST_ID create_defect_type_unicode(ddb, TEST_ID, TEST_DEFECT_DESCRIPTION) ddb.insert(TEST_DEFECT_NAME, TEST_SINCE, TEST_UNTIL, TEST_COMMENT, TEST_USER) iovs = ddb.retrieve() assert len(iovs) == 1 iov = iovs[0] assert iov.present assert not iov.recoverable assert iov.user == TEST_USER assert iov.comment == TEST_COMMENT assert ddb.all_defect_descriptions[TEST_DEFECT_NAME] == TEST_DEFECT_DESCRIPTION
pass pass except Exception, e: print "ERROR: can't get NoisyEvent from DQM server" print e del multicall del server #3.2 Get defects from Defects DB db = DefectsDB() lar_defects = [ d for d in (db.defect_names | db.virtual_defect_names) if d.startswith("LAR") ] defects = db.retrieve((runnum, minLb), (runnum, maxLb), lar_defects) for defect in defects: part = defect.channel.split("_")[1] #3.2.1 Check for HV trip if "HVTRIP" in defect.channel and defect.present: for lb in range(defect.since.lumi, defect.until.lumi): badLBs.add(lb) print "LumiBlock %i ignored because of a HV trip in partition %s" % ( lb, part) pass pass #3.2.2 Check for Noise Bursts from the defects if (not burstsFromCosmic): if not bulkProcessing: if "NOISEBURST" in defect.channel and defect.present: for lb in range(defect.since.lumi, defect.until.lumi):
def getLBsToIgnore(runnum,burstsFromCosmic=True,bulkProcessing=False, dropNonReady=True): badLBs=set() # 1. Get LB range for this run and LBs without "ATLAS-READY" nReadyLBs=0 nNotReadyLBs=0 tdaqdb=indirectOpen('COOLONL_TDAQ/CONDBR2') if (tdaqdb is None): print ("ERROR: Can't access COOLONL_TDAQ/CONDBR2") sys.exit(-1) fmode=tdaqdb.getFolder("/TDAQ/RunCtrl/DataTakingMode") since=(runnum<<32)+1 until=((1+runnum)<<32)-1 maxLb=0 minLb=1 itr=fmode.browseObjects(since,until,cool.ChannelSelection.all()) while itr.goToNext(): obj=itr.currentRef() pl=obj.payload() isReady=pl["ReadyForPhysics"] lb1=max(since,obj.since()) & 0xFFFFFFFF ts2=obj.until() if ts2<until: #ignore the IOV beyond the end of the run lb2=ts2 & 0xFFFFFFFF if lb2>maxLb: maxLb=lb2 if not isReady: if dropNonReady: print ("Ignoring LumiBlocks %i - %i not ATLAS READY" % (lb1,lb2)) badLBs.update(range(lb1,lb2)) nNotReadyLBs+=(lb2-lb1) else: nReadyLBs+=(lb2-lb1) pass pass pass pass itr.close() tdaqdb.closeDatabase() print ("Run %i goes up to LB %i" % (runnum,maxLb)) #2. Get problematic LBs #2.1 Look for collisions in empty bunches - Fetch from DQ Web Server source = 'tier0' stream = 'physics_CosmicCalo' serverstring="https://%[email protected]" % password server = xmlrpclib.ServerProxy(serverstring) multicall = xmlrpclib.MultiCall(server) # Look for the highest(latest) processing version of CosmicCalo by retrieving amitag run_spec = {'source': source, 'high_run': runnum, 'low_run': runnum} multicall.get_procpass_amitag_mapping(run_spec) results = multicall() if len(results[0])==0: print ("Nothing found about run",runnum,"on DQM server") proc = 0 try: list = results[0][str(runnum)] for item in list: if ("f" in item[2] and bulkProcessing and "CosmicCalo" in item[1] and item[0]>proc): proc = 2 if ("x" in item[2] and (not bulkProcessing) and "CosmicCalo" in item[1] and item[0]>proc): print (item) proc = 1 pass pass except Exception as e: print ("ERROR: can't retrieve the AMI Tag") print (e) if (proc == 0): print ("I haven't found any processing version for CosmicCalo. Assume express processing") proc=1 try: multicall = xmlrpclib.MultiCall(server) run_spec = {'source': source, 'high_run': runnum, 'stream': stream, 'proc_ver': proc, 'low_run': runnum} multicall.get_timestamp(run_spec) results=multicall() timestamp=results[0][str(runnum)] from time import asctime,localtime print ("DQM server timestamp:", asctime(localtime(timestamp))) print ("Now: ",asctime()) except Exception as e: print ("ERROR: can't get timestamp from DQM server") print (e) multicall = xmlrpclib.MultiCall(server) run_spec = {'source': source, 'high_run': runnum, 'stream': stream, 'proc_ver': proc, 'low_run': runnum} multicall.get_dqmf_all_results(run_spec,'LAr/LAR_GLOBAL/Collisions-Bkg/LArCollTimeLumiBlockTimeCut') results = multicall() RE = re.compile(r'\((?P<lb>\S+)\.0*\)') try: list = results[0][str(runnum)] for item in list: if 'NBins' in item: continue m = RE.search(item).groupdict() lb=int(m['lb']) ncollisions=int(results[0][str(runnum)][item]) if ncollisions > 50: badLBs.add(lb) print ("LumiBlock %i ignored because it is empty bunches are polluted with collisions" % lb) pass pass except Exception as e: print ("ERROR: can't get LArCollTimeLumiBlockTimeCut from DQM server") print (e) if (burstsFromCosmic):# CosmicCalo stream : from the DQ web histoName = {'EMBC':'BarrelC','EMBA':'BarrelA','EMECC':'EMECC','EMECA':'EMECA'} for iPart in histoName.keys(): multicall = xmlrpclib.MultiCall(server) #multicall.get_dqmf_all_results(run_spec,'LAr/%s/Noise/Partition/NoisyEvent_TimeVeto_%s'%(iPart,histoName[iPart])) multicall.get_dqmf_all_results(run_spec,'/LAr/%s/Occupancy-Noise/Noise_Burst/NoisyEvent_TimeVeto_%s'%(iPart,iPart)) results = multicall() try: resultlist = results[0][str(runnum)] #print ("Got %i items for NoisyEvent_TimeVeto_%s" % (len(list),histoName[iPart])) for item in resultlist: if 'NBins' in item: continue m = RE.search(item).groupdict() lb=int(m['lb']) yieldbursts=float(results[0][str(runnum)][item]) if yieldbursts > 0: badLBs.add(lb) print ("LumiBlock %i ignored because it contains bursts in CosmicCalo stream in %s" % (lb,iPart)) pass pass except Exception as e: print ("ERROR: can't get NoisyEvent from DQM server") print (e) del multicall del server #3.2 Get defects from Defects DB db = DefectsDB() lar_defects = [d for d in (db.defect_names | db.virtual_defect_names) if d.startswith("LAR")] defects = db.retrieve((runnum, minLb), (runnum, maxLb), lar_defects) for defect in defects: part=defect.channel.split("_")[1] #3.2.1 Check for HV trip if "HVTRIP" in defect.channel and defect.present: for lb in range(defect.since.lumi,defect.until.lumi): badLBs.add(lb) print ("LumiBlock %i ignored because of a HV trip in partition %s" % (lb,part)) pass pass #3.2.2 Check for Noise Bursts from the defects if (not burstsFromCosmic): if not bulkProcessing: if "NOISEBURST" in defect.channel and defect.present: for lb in range(defect.since.lumi,defect.until.lumi): badLBs.add(lb) print ("LumiBlock %i ignored because of a noise burst in partition %s" % (lb,part)) pass pass else: #not bulk processing if "SEVNOISEBURST" in defect.channel and defect.present: for lb in range(defect.since.lumi,defect.until.lumi): badLBs.add(lb) print ("LumiBlock %i ignored because of a severe noise burst in partition %s" % (lb,part)) pass pass del db #Close Defects DB nBadLBs=len(badLBs) if dropNonReady: nBadLBs=nBadLBs-nNotReadyLBs print ("Found %i not-ready LBs, %i atlas-ready LBs and %i bad LBs" % (nNotReadyLBs,nReadyLBs,nBadLBs)) return badLBs
class IDBSDefectWriter: """ Class for writing BS defects to an sqlite file """ def __init__(self, fileName, forceNew=False, dbName='IDBSDQ', tag='nominal', user='******'): """ Initialise database connection """ self.defect = None self.iovs = IOVSet() self.user = user #self.allowedDefects = DefectsDB('').defect_names if not fileName: fileName = 'tmp.' + str(os.getpid()) + '.db' self.connect(fileName, forceNew, dbName, tag) pass def __del__(self): """ Delete db to clear connection """ os.system('[[ -f tmp.%s.db ]] && rm tmp.%s.db' % (os.getpid(), os.getpid())) del self.db pass def connect(self, fileName, forceNew=False, dbName='IDBSDQ', tag='nominal'): """ Open connection to defect DB """ connString = 'sqlite://;schema=%s;dbname=%s' % (fileName, dbName) if forceNew and os.path.exists(fileName): os.remove(fileName) self.db = DefectsDB( connString, read_only=False, create=True, tag=(tag, 'HEAD') ) # Second tag is for virtual defects, which we are not interested in self.officialDb = DefectsDB() return def defectType(self, t): """ Set defect type """ self.defect = t def add(self, runMin=0, runMax=(1 << 31) - 1, lbMin=0, lbMax=(1 << 32) - 1): """ Add iovs which are NOT defective to the list Note, lbMax is exclusive here (and inclusive when shown on defect web page). """ # Make since and until and convert to syntactially nice RunLumiType since = RunLumiType((runMin << 32) + lbMin) until = RunLumiType((runMax << 32) + lbMax) # IoVs which are not defective (i.e. beamspot good) self.iovs.add(since, until, self.defect, False) return def complete(self, runMin, runMax): """ Complete a list of IoVs to cover all LBs in a run, treating empty ones as having 'emptyState' """ # Create an IOV set covering the entire run(s) run_lbs = fetch_iovs("EOR", runs=(runMin, runMax), what=[], with_channel=False) # run_lbs = IOVSet() # lbMin = 1 # lbMax = (1 << 32) -1 # Note, lbMax is exclusive # since = RunLumiType((runMin << 32)+lbMin) # until = RunLumiType((runMax << 32)+lbMax) # run_lbs.add(since, until) if not len(run_lbs): print "WARNING: No LBs in run according to EOR_Params - are we running before the run has finished?" def lbsStartAtOne(iov): "Change LBs starting at 0 to start at 1" return iov._replace(since=RunLumi(iov.since.run, 1)) # Start LBs from 1 rather than 0 (at request of P. Onyisi) as long as run has more than one LB (else skip) run_lbs = [lbsStartAtOne(iov) for iov in run_lbs if iov.until.lumi > 1] # Empty IOV set for adding full list of LBs too iovs = IOVSet() # Ask official DB if an IoV is currently defective so can unset only those if doing reprocessing defectsCurrentlyInDb = self.officialDb.retrieve( (runMin, 0), (runMax + 1, 0), [self.defect]) # Order IOVs to avoid since > until self.iovs = IOVSet(sorted(self.iovs)) #for since, until, (run_lb, iov, dbDefect) in process_iovs(run_lbs.solidify(RANGEIOV_VAL), self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb): for since, until, (run_lb, iov, dbDefect) in process_iovs( run_lbs, self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb): if not run_lb: continue # Check valid # # Start LBs from 1 rather than 0 (at request of P. Onyisi) # # If this makes since==until, i.e. it was a [0->1) LB range then skip # if since.lumi==0: # since = RunLumiType((since.run << 32)+since.lumi+1) # if since==until: continue # Add iovs from self.iovs treating empty ones as defective (i.e. beamspot bad/missing) if iov: # These are not defective if dbDefect and dbDefect.present: # Only store not defective IoVs if they are present on the Db so we can unset them # (don't want to write not present defects to Db unless really chaning an existing defect) iovs.add(since, until, self.defect, False) else: # These are defective iovs.add(since, until, self.defect, True) self.iovs = iovs return def writeDefects(self, tag='nominal', nonpresent=False): """ Write all defects to the database. If 'nonpresent' is True then write the absent ones too """ with self.db.storage_buffer: for since, until, defect, present in self.iovs: if not present and not nonpresent: continue #print since, until, present self._writeDefect(defect, since, until, present=present) def _writeDefect(self, defect, since, until, tag='nominal', description='', comment='', present=True, recoverable=False): """ Write a single defect to the database """ if not defect in self.db.defect_names: self.db.create_defect(defect, description) self.db.insert(defect, since, until, comment, self.user, present, recoverable) return def dump(self, filename=None): """ Dump defects to a file given by filename or stdout if no filename given """ from DQUtils.utils import pprint_objects if filename is not None: f = open(filename.replace('.db', '.txt'), "w") # If not defects then nothing will be in the database and we write an empty file if len(self.iovs): pprint_objects( self.db.retrieve(primary_only=True, nonpresent=True), f) f.close() else: if len(self.iovs): self.iovs.pprint() else: print '\nNo DQ defects' # with open(filename.replace('.db', '.txt'), "w") as f: # pprint_objects(self.db.retrieve(), f) return
'retrieveComments'] and "defect" in defVetoType[ iDefVeto]: # retrieve comments for defects print "@%d" % (runnumber) db = DefectsDB(tag=yearTagProperties["defect"][ yearTagTag[iYT]]) system_defects = [] for iPrefix in grlDef["prefix"]: system_defects += [ d for d in (db.defect_names | db.virtual_defect_names) if (d.startswith(iPrefix) and iDefVeto in d ) ] #lar_defects = [d for d in (db.defect_names | db.virtual_defect_names) if ((d.startswith("LAR") or d.startswith("CALO_ONLINEDB")) and iDefVeto in d)] defects = db.retrieve((runnumber, 1), (runnumber + 1, 0), system_defects) defectCompact = {} for defect in defects: if ( "SEVNOISEBURST" in defect.channel and ("HEC" in defect.channel or "FCAL" in defect.channel) ): # Skip the HEC/FCAL SEVNOISEBURST defect as they also appear in EMEC continue for iDef in options['defect']: if ( iDef in defect.channel ): # NB : some problem may arise from this incomplete test (if similar name for 2 defects) but there is a protection later when recaping defectSinceLumiAtlasReady = -1 defectUntilLumiAtlasReady = -1
print 'Getting virtual defect information...', ddb = DefectsDB(opts.defectdb, read_only=True) defectids, defectnames, defectdict = ddb.get_virtual_channels() print 'done' print 'Retrieving run ends...', runends = get_runends() print 'done' for key in mapping: if not isinstance(key, basestring): continue #if '_' in key: continue if key not in defectdict or key in ['IDPF', 'LCD', 'MET', 'IDBCM', 'TIGB']: print 'MISSING:', key continue print key, vfiovs = do_compact(vff.browseObjects((152166,0), (167845,0), [key], 'DetStatus-v03-repro05-01', selection=lambda x: x.Code < 3)) #print vfiovs d1 = set([((x.since.run, x.since.lumi), (x.until.run, x.until.lumi)) for x in truncate_iovs_to_runs(vfiovs, runends) if x.since.run != 152220]) diovs = do_compact(ddb.retrieve((152166,0), (167845,0), [key])) d2 = set([((x.since.run, x.since.lumi), (x.until.run, x.until.lumi)) for x in truncate_iovs_to_runs(diovs, runends) if x.since.run != 152220]) print d1 == d2 if d1 != d2: #print d1 d1a = dict([(((x.since.run, x.since.lumi), (x.until.run, x.until.lumi)), x.Comment) for x in truncate_iovs_to_runs(vfiovs, runends)]) d2a = dict([(((x.since.run, x.since.lumi), (x.until.run, x.until.lumi)), x.comment) for x in truncate_iovs_to_runs(diovs, runends)]) print 'In VF but not defects:', [(x, d1a[x]) for x in d1-d2] print 'In defects but not VF:', [(x, d2a[x]) for x in d2-d1]
class IDBSDefectData: """ Container for beamspot DQ defects from COOL """ defectBitPos = [ 'UNKNOWN', 'ID_BS_2010YELLOW', 'ID_BS_RUNAVERAGE', 'ID_BS_PARAMETERSTEP', 'ID_BS_NOBEAMSPOT', 'ID_BS_2010RED', 'LUMI_VDM' ] def __init__(self, database='COOLOFL_GLOBAL/CONDBR2', tag='HEAD', debug=False): #def __init__(self, database='dqflags.db/IDBSDQ', tag='nominal', debug=False): """ Initialise database connection """ self.database = database self.tag = tag self.lastRun = None self.iovsets = None self.debug = debug self.connect() pass def __del__(self): """ Delete db to clear connection """ del self.db pass def connect(self): """ Open connection to defect DB """ self.db = DefectsDB(self.database, read_only=True, create=False, tag=self.tag) self.idbsDefects = [ d for d in self.db.defect_names if d.startswith('ID_BS_') or d == 'LUMI_VDM' ] if self.debug: print self.idbsDefects return def defectList(self): """ List of all possible beamspot defects """ return self.idbsDefects def defect(self, run, lb, channels=None): """ Get list of DQ defects for a particular run and lb, caching the result for the latest (succesful) run e.g. from InDetBeamSpotExample.DQUtilities import IDBSDefectData idbs = IDBSDefectData() idbs.defect(167661,372) channels is the list of defects to look for (defaults to all ID_BS defects) """ if channels is None: channels = self.idbsDefects # Encode start and end of run lbMin = 0 lbMax = (1 << 32) - 1 since = (run << 32) + lbMin until = (run << 32) + lbMax # If the run is the same at the previous one return defects from cache if run == self.lastRun: defects = self._defectForLB(lb) if self.debug: print run, lb, defects return defects # Retrive info for entire run iovs = self.db.retrieve(since, until, channels=channels) # Check if run exists if not iovs: print "Unable to access folder with given parameters" return [] # If found, update last run and get list of IOVSets for each defect/channel self.lastRun = run chans, self.iovsets = iovs.chans_iovsets defects = self._defectForLB(lb) # Debug if self.debug: iovs.pprint() print run, lb, defects return defects def _defectForLB(self, lb): """ Get the DQ defects for the given LB from the full run info """ defects = [] for since, until, states in process_iovs(*self.iovsets): if since.lumi <= lb < until.lumi: # defects is true if set for a given run/LB defects = [state.channel for state in states if state] return list(set(defects)) def defectsRange(self, run, lbStart, lbEnd, channels=None): """ Return the maximal list of defects for a given range. lbEnd is exclusive """ defects = [] for lb in xrange(lbStart, lbEnd): defects.extend(self.defect(run, lb, channels=channels)) return list(set(defects)) def dumpRun(self, run, channels=None): """ Dump DQ info for a particular run (useful in reprocessing to compare new to old) """ if channels is None: channels = self.idbsDefects # Encode start and end of run lbMin = 0 lbMax = (1 << 32) - 1 since = (run << 32) + lbMin until = (run << 32) + lbMax # Retrive info for entire run iovs = self.db.retrieve(since, until, channels=channels, nonpresent=True) # Check if run exists if not iovs: print "Unable to access folder with given parameters" return [] iovs.pprint() return
opts.createdefects = True indb = DefectsDB(args[0], read_only=True, tag=opts.intag) outdb = DefectsDB(args[1], create=opts.createdb, read_only=False, tag=opts.outtag) since = parse_runlumi(opts.since) until = parse_runlumi(opts.until) channels = parse_channels(opts.defects) print 'Reading in IOVs...' iniovs = indb.retrieve(since=since, until=until, channels=channels, primary_only=True, nonpresent=(not opts.noabsent)) print '%d IOVs retrieved from input database' % len(iniovs) #inchannels = set((x.channel for x in iniovs)) inchannels = set(channels if channels is not None else indb.defect_names) outchannels = set(outdb.defect_names) missingchannels = inchannels - outchannels if len(missingchannels) != 0: if not opts.createdefects: print 'Missing defect(s) in target database:' print list(missingchannels) print 'Rerun with a restricted defect list (--defects) or with --createdefects' sys.exit(1) else: print 'Creating missing channels on output database'
def main(): parser = ArgumentParser(description="Summarize DQ Defects") a = parser.add_argument add_group = parser.add_argument_group # Specify range to process a("-p", "--project", default="data15_13TeV", help="Data project (default: data15_13TeV)") a("-P", "--period", default=None, nargs="*", help="Data period(s)") a("-r", "--run", default=None, nargs="*", help="Run number(s) to process") a("-R", "--range", help="Inclusive run range: e.g. 150000-151000") # Specify defects to process a("-d", "--defects", default=None, nargs="*", help="Defects to process. Use * for wildcard (default: None)") # Print mode, primary or virtual tree a("-q", "--tree", action="store_true", help="Dump virtual defect tree. Set depth with -D") # Other job options a("-n", "--num-primary", default=-1, type=int, help= "Max number of primary defects to display in default mode (default: all)" ) a("-D", "--depth", default=-1, type=int, help= "Max virtual defect depth to print in virtual tree mode (default: all)") a("-c", "--connection-string", default=DEFAULT_CONNECTION_STRING, help="Database connection to use (default: %s)" % DEFAULT_CONNECTION_STRING) a("-l", "--lumi-tag", default='OflLumi-8TeV-002', help="Luminosity tag (default: OflLumi-8TeV-002)") a("-t", "--tag", default="HEAD", help="Tag to use (default: HEAD)") a("--require-ready", default=True, type=int, help="Calculate luminosity with respect to ATLAS READY (default: True)") a("--reject-busy", default=False, type=int, help="Calculate luminosity with respect to not-busy (default: False)") a("--nb", action="store_true", help="Show lumi in units of 1/nb instead of 1/pb") a("-v", "--verbose", default=1, type=int, help="Set verbosity (default: 1)") args = parser.parse_args() init_logger(verbose=args.verbose) # Units units = 1e3 if args.nb else 1e6 unit_string = "(1/nb)" if args.nb else "(1/pb)" # Instantiate the database interface with timer("Instantiate DefectsDB"): db = DefectsDB(args.connection_string, tag=args.tag) # Set runs to process # TODO: control these via run lumi iovs, rather than a set of good_runs # similar to how it is done in dq_defect_compare_tags good_runs = None if args.range: since, until = map(int, args.range.split("-")) elif args.run: good_runs = set(map(int, args.run)) #since, until = args.run, args.run+1 since, until = min(good_runs), max(good_runs) + 1 else: project_dict = fetch_project_period_runs() if args.period: #good_runs = set( project_dict[args.project][args.period] ) good_runs = set() for period in args.period: good_runs.update(project_dict[args.project][period]) since, until = min(good_runs), max(good_runs) + 1 else: good_runs = set() for period, period_runs in project_dict[args.project].iteritems(): good_runs.update(period_runs) since, until = min(good_runs), max(good_runs) + 1 iov_range = (since, 0), (until, 0) log.info("Processing range: " + str(iov_range)) #log.info(good_runs) # Fetch all defect IOVs in range with timer("Fetch defects"): all_defect_iovs = db.retrieve(*iov_range) # Grab defect information with timer("Fetch defect info"): descriptions = db.all_defect_descriptions intolerables = db.get_intolerable_defects() virtuals = db.virtual_defect_names # Grab lbs and lumi variables in range with timer("Fetch lumi inputs"): lbs, lumis = fetch_lumi_inputs(iov_range, args.lumi_tag) # Compute lumi per channel with timer("Compute luminosities"): # Filtering defects exclude_iovs = [] # Defect for ATLAS READY if args.require_ready: exclude_iovs.append(all_defect_iovs.by_channel["GLOBAL_NOTREADY"]) # Defect for detector busy if args.reject_busy: exclude_iovs.append(all_defect_iovs.by_channel["GLOBAL_BUSY"]) # Compute total luminosity lumi_total = compute_lumi( lbs, lumis, lbs, exclude_iovsets=exclude_iovs, good_runs=good_runs) / units # Compute luminosity for all defects lumi_by_defect = compute_lumi_many_channels( lbs, lumis, all_defect_iovs, exclude_iovsets=exclude_iovs, good_runs=good_runs) my_defect_names = [] if args.defects: all_defect_names = db.defect_names | db.virtual_defect_names for defect_arg in args.defects: my_defect_names.extend( sorted(fnmatch.filter(all_defect_names, defect_arg))) logics = db.virtual_defect_logics # Build the defect objects all_defects = {} for name, description in sorted(descriptions.iteritems()): virtual, intolerable = name in virtuals, name in intolerables depends = [] logic = logics.get(name, None) if logic: depends = logic.clauses all_defects[name] = Defect(name, description, depends, virtual, intolerable, lumi_by_defect.get(name, 0) / units) # My defects my_defects = [all_defects[d] for d in my_defect_names] my_defects.sort(key=lambda d: d.lumi, reverse=True) print "\nTotal luminosity", unit_string, "\n" print "{0:<.2f}".format(lumi_total) print "\nDefect luminosity", unit_string, "\n" #for defect_name in my_defect_names: for defect in my_defects: if args.tree: # Virtual defect tree print_virtual_tree(defect, all_defects, depth=0, max_depth=args.depth) else: # Primary defect dump print_primary_defects(defect, all_defects, max_primary=args.num_primary) print ""
def main(): parser = ArgumentParser(description='List defect IOVs') add_arg = parser.add_argument # Specify range to process add_arg('-p', '--project', default='data15_13TeV', help='Data project (default: data15_13TeV)') add_arg('-P', '--period', default=None, nargs='*', help='Data period(s)') #add_arg('-r', '--run', default=None, nargs='*', help='Run number(s) to process') add_arg('-r', '--run', default=None, type=int, help='Run number to process') add_arg('-R', '--range', help='Inclusive run range: e.g. 150000-151000') # Specify defects to process add_arg('-d', '--defects', default=None, nargs='*', help='Defects to process. Use * for wildcard (default: None)') # Other job options add_arg('-c', '--connection-string', default=DEFAULT_CONNECTION_STRING, help='Database connection to use (default: %s)' % DEFAULT_CONNECTION_STRING) #add_arg('-l', '--lumi-tag', default='OflLumi-8TeV-001', #help='Luminosity tag (default: OflLumi-8TeV-001)') add_arg('-t', '--tag', default='HEAD', help='Tag to use (default: HEAD)') # Parse arguments args = parser.parse_args() # Ranges to query since, until = None, None range_iovs = IOVSet() # If specifying runs, use those if args.run: since, until = (args.run, 0), (args.run + 1, 0) range_iovs = IOVSet.from_runs([args.run]) elif args.range: run1, run2 = map(int, args.range.split('-')) since, until = (run1, 0), (run2, 0) range_iovs = IOVSet([RANGEIOV_VAL(RunLumi(since), RunLumi(until))]) # Otherwise, use the project and period settings elif args.project: # Fetch the project:period:runs dictionary project_dict = get_project_dict() period_dict = project_dict[args.project] run_set = set() # Use periods if specified if args.period and len(args.period) > 0: for period in args.period: run_set.update(period_dict[period]) # Otherwise use all periods in project else: for period, runs in period_dict.iteritems(): run_set.update(runs) since, until = (min(run_set), 0), (max(run_set) + 1, 0) range_iovs = IOVSet.from_runs(run_set) # debugging print 'range to process:' print 'since, until =', since, until # Instantiate the DB db = DefectsDB(args.connection_string, tag=args.tag) # Fetch the requested defect IOVs defect_iovs = db.retrieve(since, until, channels=args.defects) defect_iovset_list = defect_iovs.by_channel.values() # Get the overlap with the range_iovs # Use a dictionary of defect:iovset result_dict = defaultdict(IOVSet) for since, until, states in process_iovs(range_iovs, *defect_iovset_list): in_range, defects = states[0], states[1:] if in_range: for d in defects: if d: result_dict[d.channel].add(since, until, d.channel, d.comment) # Print the results for channel, result in result_dict.iteritems(): result.solidify(DefectIOV) print '\n' + channel + '\n' result.pprint()
def test_database_retrieval(): ddb = DefectsDB(TEST_DATABASE) defects = ddb.retrieve() assert len(defects) == 0, "Unexpected records on the database"
first = False else: h0Evol[iHisto].Draw("PSAME HIST") leg.Draw() c0.Update() if defectQuery: print "I am looking for LAr/Tile/Calo defects defined for the suspicious LB" from DQDefects import DefectsDB db = DefectsDB() defectList = [ d for d in (db.defect_names | db.virtual_defect_names) if ((d.startswith("LAR") and "SEV" in d) or (d.startswith("TILE")) or ( d.startswith("CALO"))) ] defects = db.retrieve((runNumber, 1), (runNumber + 1, 0), defectList) for iDef in defects: associatedSuspicious = False for iLB in range(iDef.since.lumi, iDef.until.lumi): if iLB in suspiciousLBlist: associatedSuspicious = True if associatedSuspicious: if (iDef.since.lumi == iDef.until.lumi - 1): print "%s: %d set by %s - %s" % (iDef.channel, iDef.since.lumi, iDef.user, iDef.comment) else: print "%s: %d->%d set by %s - %s" % ( iDef.channel, iDef.since.lumi, iDef.until.lumi - 1, iDef.user, iDef.comment) raw_input("I am done...")