def test_many_sparse_channels():
"Testing many virtual channels with sparsely placed IoVs"
ddb = DefectsDB(TEST_DATABASE, read_only=False)
all_defects = []
for i in range(52):
all_defects.append(create_defect_type(ddb, i * 2))
with ddb.storage_buffer:
for i in all_defects:
ddb.insert(i, 0, 100, "Test", "DQDefects.tests")
queried_defects = all_defects[:-1]
result = ddb.retrieve(channels=queried_defects)
queried_defects_set = set(queried_defects)
result_defects_set = set(i.channel for i in result)
unwanted_defects_set = result_defects_set - queried_defects_set
assert not unwanted_defects_set, "Unwanted defects: %r" % unwanted_defects_set
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT", "Comment",
" ".join(all_defects))
result = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert result.channels == set(["DQD_TEST_VIRTUAL_DEFECT"])
assert len(result) == 1
# This assumes evaluate_full == True
assert result[0].comment == " ".join(
sorted(all_defects)), result[0].comment
result = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"],
evaluate_full=False)
assert result.channels == set(["DQD_TEST_VIRTUAL_DEFECT"])
assert len(result) == 1
def test_ignore_defects():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
create_defect_type(ddb, 2)
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "", "")
ddb.insert("DQD_TEST_DEFECT_1", 100, 200, "", "")
ddb.insert("DQD_TEST_DEFECT_2", 200, 300, "", "")
ddb.new_virtual_defect(
"DQD_TEST_VIRTUAL_DEFECT", "",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1 DQD_TEST_DEFECT_2")
ignored_defects = set(["DQD_TEST_DEFECT_1"])
iovs = ddb.retrieve()
assert iov_result_depends(iovs) & ignored_defects, "test is broken"
iovs = ddb.retrieve(ignore=ignored_defects)
assert not iov_result_depends(
iovs) & ignored_defects, "result depends on ignored defects"
def test_query_with_primary_dependencies_with_ignore():
"""
Checking with_primary_dependencies + ignore
Checks that the set of channels and iovs returned contains the correct
number of channels and iovs when specifying both with_primary_dependencies
and ignore to DQDefects.Retrieve
"""
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
defect_names = [create_defect_type(ddb, i) for i in range(3)]
for i, defect in enumerate(defect_names):
ddb.insert(defect, i * 100, (i + 1) * 100, "", "")
ignored = set(defect_names[:1])
ddb.new_virtual_defect("DQD_TEST_VDEFECT", "", " ".join(defect_names))
iovs = ddb.retrieve(channels=["DQD_TEST_VDEFECT"], ignore=ignored)
assert iovs.channels == set(["DQD_TEST_VDEFECT"])
assert len(iovs) == len(defect_names) - 1
iovs = ddb.retrieve(channels=["DQD_TEST_VDEFECT"],
ignore=ignored,
with_primary_dependencies=True)
assert iovs.channels == set(["DQD_TEST_VDEFECT"] +
defect_names) - ignored, ("Check failed.")
assert len(iovs) == len(defect_names[1:]) * 2
def test_virtual_defects_invert():
"Testing virtual defects involving inversion"
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
create_defect_type(ddb, 2)
# Make a virtual defect whose result is the combination of the above
ddb.new_virtual_defect(
"DQD_TEST_VIRTUAL_DEFECT", "Comment",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1 !DQD_TEST_DEFECT_2")
# Insert an iov into each
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests")
ddb.insert("DQD_TEST_DEFECT_1", 150, 200, "Test", "DQDefects.tests")
ddb.insert("DQD_TEST_DEFECT_2", 50, 125, "Test", "DQDefects.tests")
# See what the result of the virtual defect is
iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
# Tests against the value
assert len(iovs) == 4, list(iovs)
first, second, third, fourth = iovs
assert (first.since, first.until) == (0, 50), first
assert (second.since, second.until) == (50, 100), second
assert (third.since, third.until) == (125, 150), third
assert (fourth.since, fourth.until) == (150, 200), fourth
assert first.channel == second.channel == third.channel == fourth.channel == "DQD_TEST_VIRTUAL_DEFECT", (
first.channel, second.channel, third.channel, fourth.channel)
del first, second, third, fourth, iovs
# Test evaluate_full=False
iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"],
evaluate_full=False)
assert len(iovs) == 2, list(iovs)
# Now unset the present bit
ddb.insert("DQD_TEST_DEFECT_2",
50,
150,
"Test",
"DQDefects.tests",
present=False)
iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
# here we are sensitive to details of python ordering ...
assert len(iovs) == 3, list(iovs)
first, second, third = iovs
assert (first.since, first.until) == (0, 100), first
assert (second.since, second.until) == (100, 150), second
assert (third.since, third.until) == (150, 200), third
assert first.channel == second.channel == third.channel == "DQD_TEST_VIRTUAL_DEFECT", (
first.channel, second.channel, third.channel)
del first, second, third, iovs
def test_virtual_defects():
"Testing virtual defect basics"
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
# Make a virtual defect whose result is the combination of the above
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT", "Comment",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1")
# Insert an iov into each
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests")
ddb.insert("DQD_TEST_DEFECT_1", 150, 200, "Test", "DQDefects.tests")
# See what the result of the virtual defect is
iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
# Tests against the value
assert len(iovs) == 2
first, second = iovs
assert (first.since, first.until) == (0, 100)
assert (second.since, second.until) == (150, 200)
assert first.channel == second.channel == "DQD_TEST_VIRTUAL_DEFECT"
del first, second, iovs
# Now unset the present bit, and test again that the relevant preiod is fixed
ddb.insert("DQD_TEST_DEFECT_1",
150,
200,
"Test",
"DQDefects.tests",
present=False)
iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert len(iovs) == 1
first = iovs[0]
assert (first.since, first.until) == (0, 100)
assert first.channel == "DQD_TEST_VIRTUAL_DEFECT"
# Now extend the defect and see if we get a contiguous result
ddb.insert("DQD_TEST_DEFECT_0", 100, 150, "Test", "DQDefects.tests")
iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert len(iovs) == 1
first = iovs[0]
assert (first.since, first.until) == (0, 150)
assert first.channel == "DQD_TEST_VIRTUAL_DEFECT"
def test_defect_empty_retrieval():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
iovs = ddb.retrieve()
assert not iovs
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
# Make a virtual defect whose result is the combination of the above
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT", "Comment",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1")
iovs = ddb.retrieve()
assert not iovs
def test_nonpresent():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
create_defect_type(ddb, 0)
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests")
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests", present=False)
iovs = ddb.retrieve()
assert len(iovs) == 0
iovs = ddb.retrieve(nonpresent=True)
assert len(iovs) == 1
assert not iovs[0].present
def test_defect_insertion_retrieval_unicode():
if six.PY3:
import ROOT
if ROOT.gROOT.GetVersionInt() < 62000:
# Passing str objects using multibyte encodings is broken
# with pyroot up to 6.18. Should be fixed in 6.20?
return
ddb = DefectsDB(TEST_DATABASE, read_only=False)
TEST_ID = 0
TEST_DEFECT_NAME = "DQD_TEST_DEFECT_%i" % TEST_ID
TEST_COMMENT = u"First test defect with character: ±"
TEST_USER = u"DQDefécts.tests"
TEST_SINCE, TEST_UNTIL = 0, 100
TEST_DEFECT_DESCRIPTION = u"À test defect (%i)" % TEST_ID
create_defect_type_unicode(ddb, TEST_ID, TEST_DEFECT_DESCRIPTION)
ddb.insert(TEST_DEFECT_NAME, TEST_SINCE, TEST_UNTIL, TEST_COMMENT,
TEST_USER)
iovs = ddb.retrieve()
assert len(iovs) == 1
iov = iovs[0]
assert iov.present
assert not iov.recoverable
assert iov.user == TEST_USER
assert iov.comment == TEST_COMMENT
assert ddb.all_defect_descriptions[
TEST_DEFECT_NAME] == TEST_DEFECT_DESCRIPTION
def test_defect_insertion_retrieval():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
TEST_ID = 0
TEST_DEFECT_NAME = "DQD_TEST_DEFECT_%i" % TEST_ID
TEST_COMMENT = "First test defect"
TEST_USER = "******"
TEST_SINCE, TEST_UNTIL = 0, 100
create_defect_type(ddb, TEST_ID)
ddb.insert(TEST_DEFECT_NAME, TEST_SINCE, TEST_UNTIL, TEST_COMMENT,
TEST_USER)
iovs = ddb.retrieve()
assert len(iovs) == 1
iov = iovs[0]
assert iov.since == TEST_SINCE
assert iov.until == TEST_UNTIL
assert iov.channel == TEST_DEFECT_NAME
assert iov.present
assert not iov.recoverable
assert iov.user == TEST_USER
assert iov.comment == TEST_COMMENT
def test_iov_tag_defects():
log.info('IOV Tags')
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
create_defect_type(ddb, 2)
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "comment1", "user1", present=False)
ddb.insert("DQD_TEST_DEFECT_1", 100, 200, "", "")
ddb.insert("DQD_TEST_DEFECT_2",
200,
300,
"comment2",
"user2",
recoverable=True)
ddb.new_defects_tag("dqd-test",
"New iov tag",
iovranges=[(0, 51), ((0, 210), (0, 306))])
ddb2 = DefectsDB(TEST_DATABASE, tag='DetStatusDEFECTS-dqd-test')
iovs = ddb2.retrieve(nonpresent=True)
assert len(iovs) == 2
assert ((iovs[0].channel, iovs[0].since, iovs[0].until, iovs[0].present,
iovs[0].recoverable, iovs[0].user,
iovs[0].comment) == ('DQD_TEST_DEFECT_0', 0, 51, False, False,
'user1', 'comment1'))
assert ((iovs[1].channel, iovs[1].since, iovs[1].until, iovs[1].present,
iovs[1].recoverable, iovs[1].user,
iovs[1].comment) == ('DQD_TEST_DEFECT_2', 210, 300, True, True,
'user2', 'comment2'))
def main():
d = DefectsDB()
iov_range = None, None #
iov_range = 185000 << 32, 185500 << 32
#iov_range = 177682 << 32, 200000 << 32
with timer("Fetch defect info"):
all_defects = d.retrieve(*iov_range)
with timer("fetch peripheral information"):
descriptions = d.all_defect_descriptions
intolerable = d.get_intolerable_defects()
virtual = d.virtual_defect_names
with timer("Fetch lumi inputs"):
lbs, lumis = fetch_lumi_inputs(iov_range, "ONLINE")
d = build_defects(descriptions, virtual, intolerable, lbs, lumis,
all_defects)
target_dir = "/afs/cern.ch/user/p/pwaller/www/defects/test"
art_dir = pjoin(target_dir, "extern")
if not exists(art_dir):
makedirs(art_dir)
copy_art(art_dir)
content = build_table(headings=heading_titles, defects=d)
with open(pjoin(target_dir, "test.html"), "w") as fd:
fd.write(content)
def test_virtual_defects_deep():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
# Insert an iov into each
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests")
ddb.insert("DQD_TEST_DEFECT_1", 150, 200, "Test", "DQDefects.tests")
# Make a virtual defect whose result is the combination of the above
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT_0", "",
"DQD_TEST_DEFECT_0")
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT_1", "",
"DQD_TEST_DEFECT_1")
ddb = DefectsDB(TEST_DATABASE)
ddb.retrieve()
def test_intersect():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
create_defect_type(ddb, 2)
ddb.insert("DQD_TEST_DEFECT_0", 0, 1000, "", "")
ddb.insert("DQD_TEST_DEFECT_0", 1000, 2000, "", "")
iovs = ddb.retrieve(50, 100, intersect=True)
assert len(iovs) == 1
assert (iovs[0].since, iovs[0].until) == (50, 100)
iovs = ddb.retrieve(999, 1001, intersect=True)
assert len(iovs) == 2
first, second = iovs
assert (first.since, first.until) == (999, 1000)
assert (second.since, second.until) == (1000, 1001)
def test_query_with_primary_dependencies():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
defect_names = [create_defect_type(ddb, i) for i in range(3)]
for i, defect in enumerate(defect_names):
ddb.insert(defect, i * 100, (i + 1) * 100, "", "")
ddb.new_virtual_defect("DQD_TEST_VDEFECT", "", " ".join(defect_names))
iovs = ddb.retrieve(channels=["DQD_TEST_VDEFECT"])
assert iovs.channels == set(["DQD_TEST_VDEFECT"])
assert len(iovs) == len(defect_names)
iovs = ddb.retrieve(channels=["DQD_TEST_VDEFECT"],
with_primary_dependencies=True)
assert iovs.channels == set(["DQD_TEST_VDEFECT"] +
defect_names), ("Check failed.")
assert len(iovs) == len(defect_names) * 2
def test_virtual_defects_stress():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
STRESS_COUNT = 20
for i in range(STRESS_COUNT):
create_defect_type(ddb, i)
with ddb.storage_buffer:
for i in range(STRESS_COUNT):
ddb.insert("DQD_TEST_DEFECT_%i" % i, i, i + 1, "Test",
"DQDefects.tests")
ALL_DEFECTS = " ".join("DQD_TEST_DEFECT_%i" % i
for i in range(STRESS_COUNT))
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT", "", ALL_DEFECTS)
iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert len(iovs) == STRESS_COUNT - 1
assert (iovs[0].since, iovs[-1].until) == (1, STRESS_COUNT)
def test_defect_insertion_retrieval_unicode():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
TEST_ID = 0
TEST_DEFECT_NAME = "DQD_TEST_DEFECT_%i" % TEST_ID
TEST_COMMENT = u"First test defect with character: ±"
TEST_USER = u"DQDefécts.tests"
TEST_SINCE, TEST_UNTIL = 0, 100
TEST_DEFECT_DESCRIPTION = u"À test defect (%i)" % TEST_ID
create_defect_type_unicode(ddb, TEST_ID, TEST_DEFECT_DESCRIPTION)
ddb.insert(TEST_DEFECT_NAME, TEST_SINCE, TEST_UNTIL, TEST_COMMENT, TEST_USER)
iovs = ddb.retrieve()
assert len(iovs) == 1
iov = iovs[0]
assert iov.present
assert not iov.recoverable
assert iov.user == TEST_USER
assert iov.comment == TEST_COMMENT
assert ddb.all_defect_descriptions[TEST_DEFECT_NAME] == TEST_DEFECT_DESCRIPTION
pass
pass
except Exception, e:
print "ERROR: can't get NoisyEvent from DQM server"
print e
del multicall
del server
#3.2 Get defects from Defects DB
db = DefectsDB()
lar_defects = [
d for d in (db.defect_names | db.virtual_defect_names)
if d.startswith("LAR")
]
defects = db.retrieve((runnum, minLb), (runnum, maxLb), lar_defects)
for defect in defects:
part = defect.channel.split("_")[1]
#3.2.1 Check for HV trip
if "HVTRIP" in defect.channel and defect.present:
for lb in range(defect.since.lumi, defect.until.lumi):
badLBs.add(lb)
print "LumiBlock %i ignored because of a HV trip in partition %s" % (
lb, part)
pass
pass
#3.2.2 Check for Noise Bursts from the defects
if (not burstsFromCosmic):
if not bulkProcessing:
if "NOISEBURST" in defect.channel and defect.present:
for lb in range(defect.since.lumi, defect.until.lumi):
def getLBsToIgnore(runnum,burstsFromCosmic=True,bulkProcessing=False, dropNonReady=True):
badLBs=set()
# 1. Get LB range for this run and LBs without "ATLAS-READY"
nReadyLBs=0
nNotReadyLBs=0
tdaqdb=indirectOpen('COOLONL_TDAQ/CONDBR2')
if (tdaqdb is None):
print ("ERROR: Can't access COOLONL_TDAQ/CONDBR2")
sys.exit(-1)
fmode=tdaqdb.getFolder("/TDAQ/RunCtrl/DataTakingMode")
since=(runnum<<32)+1
until=((1+runnum)<<32)-1
maxLb=0
minLb=1
itr=fmode.browseObjects(since,until,cool.ChannelSelection.all())
while itr.goToNext():
obj=itr.currentRef()
pl=obj.payload()
isReady=pl["ReadyForPhysics"]
lb1=max(since,obj.since()) & 0xFFFFFFFF
ts2=obj.until()
if ts2<until: #ignore the IOV beyond the end of the run
lb2=ts2 & 0xFFFFFFFF
if lb2>maxLb:
maxLb=lb2
if not isReady:
if dropNonReady:
print ("Ignoring LumiBlocks %i - %i not ATLAS READY" % (lb1,lb2))
badLBs.update(range(lb1,lb2))
nNotReadyLBs+=(lb2-lb1)
else:
nReadyLBs+=(lb2-lb1)
pass
pass
pass
pass
itr.close()
tdaqdb.closeDatabase()
print ("Run %i goes up to LB %i" % (runnum,maxLb))
#2. Get problematic LBs
#2.1 Look for collisions in empty bunches - Fetch from DQ Web Server
source = 'tier0'
stream = 'physics_CosmicCalo'
serverstring="https://%[email protected]" % password
server = xmlrpclib.ServerProxy(serverstring)
multicall = xmlrpclib.MultiCall(server)
# Look for the highest(latest) processing version of CosmicCalo by retrieving amitag
run_spec = {'source': source, 'high_run': runnum, 'low_run': runnum}
multicall.get_procpass_amitag_mapping(run_spec)
results = multicall()
if len(results[0])==0:
print ("Nothing found about run",runnum,"on DQM server")
proc = 0
try:
list = results[0][str(runnum)]
for item in list:
if ("f" in item[2] and bulkProcessing and "CosmicCalo" in item[1] and item[0]>proc):
proc = 2
if ("x" in item[2] and (not bulkProcessing) and "CosmicCalo" in item[1] and item[0]>proc):
print (item)
proc = 1
pass
pass
except Exception as e:
print ("ERROR: can't retrieve the AMI Tag")
print (e)
if (proc == 0):
print ("I haven't found any processing version for CosmicCalo. Assume express processing")
proc=1
try:
multicall = xmlrpclib.MultiCall(server)
run_spec = {'source': source, 'high_run': runnum, 'stream': stream, 'proc_ver': proc, 'low_run': runnum}
multicall.get_timestamp(run_spec)
results=multicall()
timestamp=results[0][str(runnum)]
from time import asctime,localtime
print ("DQM server timestamp:", asctime(localtime(timestamp)))
print ("Now: ",asctime())
except Exception as e:
print ("ERROR: can't get timestamp from DQM server")
print (e)
multicall = xmlrpclib.MultiCall(server)
run_spec = {'source': source, 'high_run': runnum, 'stream': stream, 'proc_ver': proc, 'low_run': runnum}
multicall.get_dqmf_all_results(run_spec,'LAr/LAR_GLOBAL/Collisions-Bkg/LArCollTimeLumiBlockTimeCut')
results = multicall()
RE = re.compile(r'\((?P<lb>\S+)\.0*\)')
try:
list = results[0][str(runnum)]
for item in list:
if 'NBins' in item:
continue
m = RE.search(item).groupdict()
lb=int(m['lb'])
ncollisions=int(results[0][str(runnum)][item])
if ncollisions > 50:
badLBs.add(lb)
print ("LumiBlock %i ignored because it is empty bunches are polluted with collisions" % lb)
pass
pass
except Exception as e:
print ("ERROR: can't get LArCollTimeLumiBlockTimeCut from DQM server")
print (e)
if (burstsFromCosmic):# CosmicCalo stream : from the DQ web
histoName = {'EMBC':'BarrelC','EMBA':'BarrelA','EMECC':'EMECC','EMECA':'EMECA'}
for iPart in histoName.keys():
multicall = xmlrpclib.MultiCall(server)
#multicall.get_dqmf_all_results(run_spec,'LAr/%s/Noise/Partition/NoisyEvent_TimeVeto_%s'%(iPart,histoName[iPart]))
multicall.get_dqmf_all_results(run_spec,'/LAr/%s/Occupancy-Noise/Noise_Burst/NoisyEvent_TimeVeto_%s'%(iPart,iPart))
results = multicall()
try:
resultlist = results[0][str(runnum)]
#print ("Got %i items for NoisyEvent_TimeVeto_%s" % (len(list),histoName[iPart]))
for item in resultlist:
if 'NBins' in item:
continue
m = RE.search(item).groupdict()
lb=int(m['lb'])
yieldbursts=float(results[0][str(runnum)][item])
if yieldbursts > 0:
badLBs.add(lb)
print ("LumiBlock %i ignored because it contains bursts in CosmicCalo stream in %s" % (lb,iPart))
pass
pass
except Exception as e:
print ("ERROR: can't get NoisyEvent from DQM server")
print (e)
del multicall
del server
#3.2 Get defects from Defects DB
db = DefectsDB()
lar_defects = [d for d in (db.defect_names | db.virtual_defect_names) if d.startswith("LAR")]
defects = db.retrieve((runnum, minLb), (runnum, maxLb), lar_defects)
for defect in defects:
part=defect.channel.split("_")[1]
#3.2.1 Check for HV trip
if "HVTRIP" in defect.channel and defect.present:
for lb in range(defect.since.lumi,defect.until.lumi):
badLBs.add(lb)
print ("LumiBlock %i ignored because of a HV trip in partition %s" % (lb,part))
pass
pass
#3.2.2 Check for Noise Bursts from the defects
if (not burstsFromCosmic):
if not bulkProcessing:
if "NOISEBURST" in defect.channel and defect.present:
for lb in range(defect.since.lumi,defect.until.lumi):
badLBs.add(lb)
print ("LumiBlock %i ignored because of a noise burst in partition %s" % (lb,part))
pass
pass
else: #not bulk processing
if "SEVNOISEBURST" in defect.channel and defect.present:
for lb in range(defect.since.lumi,defect.until.lumi):
badLBs.add(lb)
print ("LumiBlock %i ignored because of a severe noise burst in partition %s" % (lb,part))
pass
pass
del db #Close Defects DB
nBadLBs=len(badLBs)
if dropNonReady:
nBadLBs=nBadLBs-nNotReadyLBs
print ("Found %i not-ready LBs, %i atlas-ready LBs and %i bad LBs" % (nNotReadyLBs,nReadyLBs,nBadLBs))
return badLBs
class IDBSDefectWriter:
"""
Class for writing BS defects to an sqlite file
"""
def __init__(self,
fileName,
forceNew=False,
dbName='IDBSDQ',
tag='nominal',
user='******'):
"""
Initialise database connection
"""
self.defect = None
self.iovs = IOVSet()
self.user = user
#self.allowedDefects = DefectsDB('').defect_names
if not fileName: fileName = 'tmp.' + str(os.getpid()) + '.db'
self.connect(fileName, forceNew, dbName, tag)
pass
def __del__(self):
"""
Delete db to clear connection
"""
os.system('[[ -f tmp.%s.db ]] && rm tmp.%s.db' %
(os.getpid(), os.getpid()))
del self.db
pass
def connect(self,
fileName,
forceNew=False,
dbName='IDBSDQ',
tag='nominal'):
"""
Open connection to defect DB
"""
connString = 'sqlite://;schema=%s;dbname=%s' % (fileName, dbName)
if forceNew and os.path.exists(fileName):
os.remove(fileName)
self.db = DefectsDB(
connString, read_only=False, create=True, tag=(tag, 'HEAD')
) # Second tag is for virtual defects, which we are not interested in
self.officialDb = DefectsDB()
return
def defectType(self, t):
"""
Set defect type
"""
self.defect = t
def add(self,
runMin=0,
runMax=(1 << 31) - 1,
lbMin=0,
lbMax=(1 << 32) - 1):
"""
Add iovs which are NOT defective to the list
Note, lbMax is exclusive here (and inclusive when shown on defect web page).
"""
# Make since and until and convert to syntactially nice RunLumiType
since = RunLumiType((runMin << 32) + lbMin)
until = RunLumiType((runMax << 32) + lbMax)
# IoVs which are not defective (i.e. beamspot good)
self.iovs.add(since, until, self.defect, False)
return
def complete(self, runMin, runMax):
"""
Complete a list of IoVs to cover all LBs in a run, treating empty ones as having 'emptyState'
"""
# Create an IOV set covering the entire run(s)
run_lbs = fetch_iovs("EOR",
runs=(runMin, runMax),
what=[],
with_channel=False)
# run_lbs = IOVSet()
# lbMin = 1
# lbMax = (1 << 32) -1 # Note, lbMax is exclusive
# since = RunLumiType((runMin << 32)+lbMin)
# until = RunLumiType((runMax << 32)+lbMax)
# run_lbs.add(since, until)
if not len(run_lbs):
print "WARNING: No LBs in run according to EOR_Params - are we running before the run has finished?"
def lbsStartAtOne(iov):
"Change LBs starting at 0 to start at 1"
return iov._replace(since=RunLumi(iov.since.run, 1))
# Start LBs from 1 rather than 0 (at request of P. Onyisi) as long as run has more than one LB (else skip)
run_lbs = [lbsStartAtOne(iov) for iov in run_lbs if iov.until.lumi > 1]
# Empty IOV set for adding full list of LBs too
iovs = IOVSet()
# Ask official DB if an IoV is currently defective so can unset only those if doing reprocessing
defectsCurrentlyInDb = self.officialDb.retrieve(
(runMin, 0), (runMax + 1, 0), [self.defect])
# Order IOVs to avoid since > until
self.iovs = IOVSet(sorted(self.iovs))
#for since, until, (run_lb, iov, dbDefect) in process_iovs(run_lbs.solidify(RANGEIOV_VAL), self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb):
for since, until, (run_lb, iov, dbDefect) in process_iovs(
run_lbs, self.iovs.solidify(DEFECTIOV_VAL),
defectsCurrentlyInDb):
if not run_lb: continue # Check valid
# # Start LBs from 1 rather than 0 (at request of P. Onyisi)
# # If this makes since==until, i.e. it was a [0->1) LB range then skip
# if since.lumi==0:
# since = RunLumiType((since.run << 32)+since.lumi+1)
# if since==until: continue
# Add iovs from self.iovs treating empty ones as defective (i.e. beamspot bad/missing)
if iov:
# These are not defective
if dbDefect and dbDefect.present:
# Only store not defective IoVs if they are present on the Db so we can unset them
# (don't want to write not present defects to Db unless really chaning an existing defect)
iovs.add(since, until, self.defect, False)
else:
# These are defective
iovs.add(since, until, self.defect, True)
self.iovs = iovs
return
def writeDefects(self, tag='nominal', nonpresent=False):
"""
Write all defects to the database. If 'nonpresent' is True then write the absent ones too
"""
with self.db.storage_buffer:
for since, until, defect, present in self.iovs:
if not present and not nonpresent: continue
#print since, until, present
self._writeDefect(defect, since, until, present=present)
def _writeDefect(self,
defect,
since,
until,
tag='nominal',
description='',
comment='',
present=True,
recoverable=False):
"""
Write a single defect to the database
"""
if not defect in self.db.defect_names:
self.db.create_defect(defect, description)
self.db.insert(defect, since, until, comment, self.user, present,
recoverable)
return
def dump(self, filename=None):
"""
Dump defects to a file given by filename or stdout if no filename given
"""
from DQUtils.utils import pprint_objects
if filename is not None:
f = open(filename.replace('.db', '.txt'), "w")
# If not defects then nothing will be in the database and we write an empty file
if len(self.iovs):
pprint_objects(
self.db.retrieve(primary_only=True, nonpresent=True), f)
f.close()
else:
if len(self.iovs):
self.iovs.pprint()
else:
print '\nNo DQ defects'
# with open(filename.replace('.db', '.txt'), "w") as f:
# pprint_objects(self.db.retrieve(), f)
return
'retrieveComments'] and "defect" in defVetoType[
iDefVeto]: # retrieve comments for defects
print "@%d" % (runnumber)
db = DefectsDB(tag=yearTagProperties["defect"][
yearTagTag[iYT]])
system_defects = []
for iPrefix in grlDef["prefix"]:
system_defects += [
d for d in (db.defect_names
| db.virtual_defect_names)
if (d.startswith(iPrefix) and iDefVeto in d
)
]
#lar_defects = [d for d in (db.defect_names | db.virtual_defect_names) if ((d.startswith("LAR") or d.startswith("CALO_ONLINEDB")) and iDefVeto in d)]
defects = db.retrieve((runnumber, 1),
(runnumber + 1, 0),
system_defects)
defectCompact = {}
for defect in defects:
if (
"SEVNOISEBURST" in defect.channel
and ("HEC" in defect.channel
or "FCAL" in defect.channel)
): # Skip the HEC/FCAL SEVNOISEBURST defect as they also appear in EMEC
continue
for iDef in options['defect']:
if (
iDef in defect.channel
): # NB : some problem may arise from this incomplete test (if similar name for 2 defects) but there is a protection later when recaping
defectSinceLumiAtlasReady = -1
defectUntilLumiAtlasReady = -1
print 'Getting virtual defect information...',
ddb = DefectsDB(opts.defectdb, read_only=True)
defectids, defectnames, defectdict = ddb.get_virtual_channels()
print 'done'
print 'Retrieving run ends...',
runends = get_runends()
print 'done'
for key in mapping:
if not isinstance(key, basestring): continue
#if '_' in key: continue
if key not in defectdict or key in ['IDPF', 'LCD', 'MET', 'IDBCM', 'TIGB']:
print 'MISSING:', key
continue
print key,
vfiovs = do_compact(vff.browseObjects((152166,0), (167845,0), [key], 'DetStatus-v03-repro05-01', selection=lambda x: x.Code < 3))
#print vfiovs
d1 = set([((x.since.run, x.since.lumi), (x.until.run, x.until.lumi)) for x in truncate_iovs_to_runs(vfiovs, runends) if x.since.run != 152220])
diovs = do_compact(ddb.retrieve((152166,0), (167845,0), [key]))
d2 = set([((x.since.run, x.since.lumi), (x.until.run, x.until.lumi)) for x in truncate_iovs_to_runs(diovs, runends) if x.since.run != 152220])
print d1 == d2
if d1 != d2:
#print d1
d1a = dict([(((x.since.run, x.since.lumi), (x.until.run, x.until.lumi)), x.Comment) for x in truncate_iovs_to_runs(vfiovs, runends)])
d2a = dict([(((x.since.run, x.since.lumi), (x.until.run, x.until.lumi)), x.comment) for x in truncate_iovs_to_runs(diovs, runends)])
print 'In VF but not defects:', [(x, d1a[x]) for x in d1-d2]
print 'In defects but not VF:', [(x, d2a[x]) for x in d2-d1]
class IDBSDefectData:
"""
Container for beamspot DQ defects from COOL
"""
defectBitPos = [
'UNKNOWN', 'ID_BS_2010YELLOW', 'ID_BS_RUNAVERAGE',
'ID_BS_PARAMETERSTEP', 'ID_BS_NOBEAMSPOT', 'ID_BS_2010RED', 'LUMI_VDM'
]
def __init__(self,
database='COOLOFL_GLOBAL/CONDBR2',
tag='HEAD',
debug=False):
#def __init__(self, database='dqflags.db/IDBSDQ', tag='nominal', debug=False):
"""
Initialise database connection
"""
self.database = database
self.tag = tag
self.lastRun = None
self.iovsets = None
self.debug = debug
self.connect()
pass
def __del__(self):
"""
Delete db to clear connection
"""
del self.db
pass
def connect(self):
"""
Open connection to defect DB
"""
self.db = DefectsDB(self.database,
read_only=True,
create=False,
tag=self.tag)
self.idbsDefects = [
d for d in self.db.defect_names
if d.startswith('ID_BS_') or d == 'LUMI_VDM'
]
if self.debug:
print self.idbsDefects
return
def defectList(self):
"""
List of all possible beamspot defects
"""
return self.idbsDefects
def defect(self, run, lb, channels=None):
"""
Get list of DQ defects for a particular run and lb, caching the result for the latest (succesful) run
e.g.
from InDetBeamSpotExample.DQUtilities import IDBSDefectData
idbs = IDBSDefectData()
idbs.defect(167661,372)
channels is the list of defects to look for (defaults to all ID_BS defects)
"""
if channels is None:
channels = self.idbsDefects
# Encode start and end of run
lbMin = 0
lbMax = (1 << 32) - 1
since = (run << 32) + lbMin
until = (run << 32) + lbMax
# If the run is the same at the previous one return defects from cache
if run == self.lastRun:
defects = self._defectForLB(lb)
if self.debug: print run, lb, defects
return defects
# Retrive info for entire run
iovs = self.db.retrieve(since, until, channels=channels)
# Check if run exists
if not iovs:
print "Unable to access folder with given parameters"
return []
# If found, update last run and get list of IOVSets for each defect/channel
self.lastRun = run
chans, self.iovsets = iovs.chans_iovsets
defects = self._defectForLB(lb)
# Debug
if self.debug:
iovs.pprint()
print run, lb, defects
return defects
def _defectForLB(self, lb):
"""
Get the DQ defects for the given LB from the full run info
"""
defects = []
for since, until, states in process_iovs(*self.iovsets):
if since.lumi <= lb < until.lumi:
# defects is true if set for a given run/LB
defects = [state.channel for state in states if state]
return list(set(defects))
def defectsRange(self, run, lbStart, lbEnd, channels=None):
"""
Return the maximal list of defects for a given range. lbEnd is exclusive
"""
defects = []
for lb in xrange(lbStart, lbEnd):
defects.extend(self.defect(run, lb, channels=channels))
return list(set(defects))
def dumpRun(self, run, channels=None):
"""
Dump DQ info for a particular run (useful in reprocessing to compare new to old)
"""
if channels is None:
channels = self.idbsDefects
# Encode start and end of run
lbMin = 0
lbMax = (1 << 32) - 1
since = (run << 32) + lbMin
until = (run << 32) + lbMax
# Retrive info for entire run
iovs = self.db.retrieve(since,
until,
channels=channels,
nonpresent=True)
# Check if run exists
if not iovs:
print "Unable to access folder with given parameters"
return []
iovs.pprint()
return
opts.createdefects = True
indb = DefectsDB(args[0], read_only=True, tag=opts.intag)
outdb = DefectsDB(args[1],
create=opts.createdb,
read_only=False,
tag=opts.outtag)
since = parse_runlumi(opts.since)
until = parse_runlumi(opts.until)
channels = parse_channels(opts.defects)
print 'Reading in IOVs...'
iniovs = indb.retrieve(since=since,
until=until,
channels=channels,
primary_only=True,
nonpresent=(not opts.noabsent))
print '%d IOVs retrieved from input database' % len(iniovs)
#inchannels = set((x.channel for x in iniovs))
inchannels = set(channels if channels is not None else indb.defect_names)
outchannels = set(outdb.defect_names)
missingchannels = inchannels - outchannels
if len(missingchannels) != 0:
if not opts.createdefects:
print 'Missing defect(s) in target database:'
print list(missingchannels)
print 'Rerun with a restricted defect list (--defects) or with --createdefects'
sys.exit(1)
else:
print 'Creating missing channels on output database'
def main():
parser = ArgumentParser(description="Summarize DQ Defects")
a = parser.add_argument
add_group = parser.add_argument_group
# Specify range to process
a("-p",
"--project",
default="data15_13TeV",
help="Data project (default: data15_13TeV)")
a("-P", "--period", default=None, nargs="*", help="Data period(s)")
a("-r", "--run", default=None, nargs="*", help="Run number(s) to process")
a("-R", "--range", help="Inclusive run range: e.g. 150000-151000")
# Specify defects to process
a("-d",
"--defects",
default=None,
nargs="*",
help="Defects to process. Use * for wildcard (default: None)")
# Print mode, primary or virtual tree
a("-q",
"--tree",
action="store_true",
help="Dump virtual defect tree. Set depth with -D")
# Other job options
a("-n",
"--num-primary",
default=-1,
type=int,
help=
"Max number of primary defects to display in default mode (default: all)"
)
a("-D",
"--depth",
default=-1,
type=int,
help=
"Max virtual defect depth to print in virtual tree mode (default: all)")
a("-c",
"--connection-string",
default=DEFAULT_CONNECTION_STRING,
help="Database connection to use (default: %s)" %
DEFAULT_CONNECTION_STRING)
a("-l",
"--lumi-tag",
default='OflLumi-8TeV-002',
help="Luminosity tag (default: OflLumi-8TeV-002)")
a("-t", "--tag", default="HEAD", help="Tag to use (default: HEAD)")
a("--require-ready",
default=True,
type=int,
help="Calculate luminosity with respect to ATLAS READY (default: True)")
a("--reject-busy",
default=False,
type=int,
help="Calculate luminosity with respect to not-busy (default: False)")
a("--nb",
action="store_true",
help="Show lumi in units of 1/nb instead of 1/pb")
a("-v",
"--verbose",
default=1,
type=int,
help="Set verbosity (default: 1)")
args = parser.parse_args()
init_logger(verbose=args.verbose)
# Units
units = 1e3 if args.nb else 1e6
unit_string = "(1/nb)" if args.nb else "(1/pb)"
# Instantiate the database interface
with timer("Instantiate DefectsDB"):
db = DefectsDB(args.connection_string, tag=args.tag)
# Set runs to process
# TODO: control these via run lumi iovs, rather than a set of good_runs
# similar to how it is done in dq_defect_compare_tags
good_runs = None
if args.range:
since, until = map(int, args.range.split("-"))
elif args.run:
good_runs = set(map(int, args.run))
#since, until = args.run, args.run+1
since, until = min(good_runs), max(good_runs) + 1
else:
project_dict = fetch_project_period_runs()
if args.period:
#good_runs = set( project_dict[args.project][args.period] )
good_runs = set()
for period in args.period:
good_runs.update(project_dict[args.project][period])
since, until = min(good_runs), max(good_runs) + 1
else:
good_runs = set()
for period, period_runs in project_dict[args.project].iteritems():
good_runs.update(period_runs)
since, until = min(good_runs), max(good_runs) + 1
iov_range = (since, 0), (until, 0)
log.info("Processing range: " + str(iov_range))
#log.info(good_runs)
# Fetch all defect IOVs in range
with timer("Fetch defects"):
all_defect_iovs = db.retrieve(*iov_range)
# Grab defect information
with timer("Fetch defect info"):
descriptions = db.all_defect_descriptions
intolerables = db.get_intolerable_defects()
virtuals = db.virtual_defect_names
# Grab lbs and lumi variables in range
with timer("Fetch lumi inputs"):
lbs, lumis = fetch_lumi_inputs(iov_range, args.lumi_tag)
# Compute lumi per channel
with timer("Compute luminosities"):
# Filtering defects
exclude_iovs = []
# Defect for ATLAS READY
if args.require_ready:
exclude_iovs.append(all_defect_iovs.by_channel["GLOBAL_NOTREADY"])
# Defect for detector busy
if args.reject_busy:
exclude_iovs.append(all_defect_iovs.by_channel["GLOBAL_BUSY"])
# Compute total luminosity
lumi_total = compute_lumi(
lbs, lumis, lbs, exclude_iovsets=exclude_iovs,
good_runs=good_runs) / units
# Compute luminosity for all defects
lumi_by_defect = compute_lumi_many_channels(
lbs,
lumis,
all_defect_iovs,
exclude_iovsets=exclude_iovs,
good_runs=good_runs)
my_defect_names = []
if args.defects:
all_defect_names = db.defect_names | db.virtual_defect_names
for defect_arg in args.defects:
my_defect_names.extend(
sorted(fnmatch.filter(all_defect_names, defect_arg)))
logics = db.virtual_defect_logics
# Build the defect objects
all_defects = {}
for name, description in sorted(descriptions.iteritems()):
virtual, intolerable = name in virtuals, name in intolerables
depends = []
logic = logics.get(name, None)
if logic: depends = logic.clauses
all_defects[name] = Defect(name, description, depends, virtual,
intolerable,
lumi_by_defect.get(name, 0) / units)
# My defects
my_defects = [all_defects[d] for d in my_defect_names]
my_defects.sort(key=lambda d: d.lumi, reverse=True)
print "\nTotal luminosity", unit_string, "\n"
print "{0:<.2f}".format(lumi_total)
print "\nDefect luminosity", unit_string, "\n"
#for defect_name in my_defect_names:
for defect in my_defects:
if args.tree:
# Virtual defect tree
print_virtual_tree(defect,
all_defects,
depth=0,
max_depth=args.depth)
else:
# Primary defect dump
print_primary_defects(defect,
all_defects,
max_primary=args.num_primary)
print ""
def main():
parser = ArgumentParser(description='List defect IOVs')
add_arg = parser.add_argument
# Specify range to process
add_arg('-p',
'--project',
default='data15_13TeV',
help='Data project (default: data15_13TeV)')
add_arg('-P', '--period', default=None, nargs='*', help='Data period(s)')
#add_arg('-r', '--run', default=None, nargs='*', help='Run number(s) to process')
add_arg('-r',
'--run',
default=None,
type=int,
help='Run number to process')
add_arg('-R', '--range', help='Inclusive run range: e.g. 150000-151000')
# Specify defects to process
add_arg('-d',
'--defects',
default=None,
nargs='*',
help='Defects to process. Use * for wildcard (default: None)')
# Other job options
add_arg('-c',
'--connection-string',
default=DEFAULT_CONNECTION_STRING,
help='Database connection to use (default: %s)' %
DEFAULT_CONNECTION_STRING)
#add_arg('-l', '--lumi-tag', default='OflLumi-8TeV-001',
#help='Luminosity tag (default: OflLumi-8TeV-001)')
add_arg('-t', '--tag', default='HEAD', help='Tag to use (default: HEAD)')
# Parse arguments
args = parser.parse_args()
# Ranges to query
since, until = None, None
range_iovs = IOVSet()
# If specifying runs, use those
if args.run:
since, until = (args.run, 0), (args.run + 1, 0)
range_iovs = IOVSet.from_runs([args.run])
elif args.range:
run1, run2 = map(int, args.range.split('-'))
since, until = (run1, 0), (run2, 0)
range_iovs = IOVSet([RANGEIOV_VAL(RunLumi(since), RunLumi(until))])
# Otherwise, use the project and period settings
elif args.project:
# Fetch the project:period:runs dictionary
project_dict = get_project_dict()
period_dict = project_dict[args.project]
run_set = set()
# Use periods if specified
if args.period and len(args.period) > 0:
for period in args.period:
run_set.update(period_dict[period])
# Otherwise use all periods in project
else:
for period, runs in period_dict.iteritems():
run_set.update(runs)
since, until = (min(run_set), 0), (max(run_set) + 1, 0)
range_iovs = IOVSet.from_runs(run_set)
# debugging
print 'range to process:'
print 'since, until =', since, until
# Instantiate the DB
db = DefectsDB(args.connection_string, tag=args.tag)
# Fetch the requested defect IOVs
defect_iovs = db.retrieve(since, until, channels=args.defects)
defect_iovset_list = defect_iovs.by_channel.values()
# Get the overlap with the range_iovs
# Use a dictionary of defect:iovset
result_dict = defaultdict(IOVSet)
for since, until, states in process_iovs(range_iovs, *defect_iovset_list):
in_range, defects = states[0], states[1:]
if in_range:
for d in defects:
if d:
result_dict[d.channel].add(since, until, d.channel,
d.comment)
# Print the results
for channel, result in result_dict.iteritems():
result.solidify(DefectIOV)
print '\n' + channel + '\n'
result.pprint()
def test_database_retrieval():
ddb = DefectsDB(TEST_DATABASE)
defects = ddb.retrieve()
assert len(defects) == 0, "Unexpected records on the database"
first = False
else:
h0Evol[iHisto].Draw("PSAME HIST")
leg.Draw()
c0.Update()
if defectQuery:
print "I am looking for LAr/Tile/Calo defects defined for the suspicious LB"
from DQDefects import DefectsDB
db = DefectsDB()
defectList = [
d for d in (db.defect_names | db.virtual_defect_names)
if ((d.startswith("LAR") and "SEV" in d) or (d.startswith("TILE")) or (
d.startswith("CALO")))
]
defects = db.retrieve((runNumber, 1), (runNumber + 1, 0), defectList)
for iDef in defects:
associatedSuspicious = False
for iLB in range(iDef.since.lumi, iDef.until.lumi):
if iLB in suspiciousLBlist:
associatedSuspicious = True
if associatedSuspicious:
if (iDef.since.lumi == iDef.until.lumi - 1):
print "%s: %d set by %s - %s" % (iDef.channel, iDef.since.lumi,
iDef.user, iDef.comment)
else:
print "%s: %d->%d set by %s - %s" % (
iDef.channel, iDef.since.lumi, iDef.until.lumi - 1,
iDef.user, iDef.comment)
raw_input("I am done...")
