def test_many_sparse_channels():
"Testing many virtual channels with sparsely placed IoVs"
ddb = DefectsDB(TEST_DATABASE, read_only=False)
all_defects = []
for i in range(52):
all_defects.append(create_defect_type(ddb, i * 2))
with ddb.storage_buffer:
for i in all_defects:
ddb.insert(i, 0, 100, "Test", "DQDefects.tests")
queried_defects = all_defects[:-1]
result = ddb.retrieve(channels=queried_defects)
queried_defects_set = set(queried_defects)
result_defects_set = set(i.channel for i in result)
unwanted_defects_set = result_defects_set - queried_defects_set
assert not unwanted_defects_set, "Unwanted defects: %r" % unwanted_defects_set
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT", "Comment",
" ".join(all_defects))
result = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert result.channels == set(["DQD_TEST_VIRTUAL_DEFECT"])
assert len(result) == 1
# This assumes evaluate_full == True
assert result[0].comment == " ".join(
sorted(all_defects)), result[0].comment
result = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"],
evaluate_full=False)
assert result.channels == set(["DQD_TEST_VIRTUAL_DEFECT"])
assert len(result) == 1
def test_tagging_unicode():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
ddb.insert(u"DQD_TEST_DEFECT_0", 0, 100, u"", u"")
ddb.insert(u"DQD_TEST_DEFECT_1", 100, 200, u"", u"")
defects_tag = ddb.new_defects_tag("dqd-test", u"New iov tag")
# Make a virtual defect whose result is the combination of the above
ddb.new_virtual_defect(u"DQD_TEST_VIRTUAL_DEFECT", u"",
u"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1")
original_tag = ddb.new_logics_tag()
original_htag = ddb.new_hierarchical_tag(defects_tag, original_tag)
what = {"channels": [u"DQD_TEST_VIRTUAL_DEFECT"]}
orig_iovs = DefectsDB((TEST_DATABASE),
tag=six.ensure_text(original_htag)).retrieve(**what)
assert len(orig_iovs) == 2
assert (orig_iovs[0].since, orig_iovs[0].until) == (0, 100)
assert (orig_iovs[1].since, orig_iovs[1].until) == (100, 200)
def test_defect_insertion_retrieval():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
TEST_ID = 0
TEST_DEFECT_NAME = "DQD_TEST_DEFECT_%i" % TEST_ID
TEST_COMMENT = "First test defect"
TEST_USER = "******"
TEST_SINCE, TEST_UNTIL = 0, 100
create_defect_type(ddb, TEST_ID)
ddb.insert(TEST_DEFECT_NAME, TEST_SINCE, TEST_UNTIL, TEST_COMMENT,
TEST_USER)
iovs = ddb.retrieve()
assert len(iovs) == 1
iov = iovs[0]
assert iov.since == TEST_SINCE
assert iov.until == TEST_UNTIL
assert iov.channel == TEST_DEFECT_NAME
assert iov.present
assert not iov.recoverable
assert iov.user == TEST_USER
assert iov.comment == TEST_COMMENT
def test_query_with_primary_dependencies_with_ignore():
"""
Checking with_primary_dependencies + ignore
Checks that the set of channels and iovs returned contains the correct
number of channels and iovs when specifying both with_primary_dependencies
and ignore to DQDefects.Retrieve
"""
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
defect_names = [create_defect_type(ddb, i) for i in range(3)]
for i, defect in enumerate(defect_names):
ddb.insert(defect, i * 100, (i + 1) * 100, "", "")
ignored = set(defect_names[:1])
ddb.new_virtual_defect("DQD_TEST_VDEFECT", "", " ".join(defect_names))
iovs = ddb.retrieve(channels=["DQD_TEST_VDEFECT"], ignore=ignored)
assert iovs.channels == set(["DQD_TEST_VDEFECT"])
assert len(iovs) == len(defect_names) - 1
iovs = ddb.retrieve(channels=["DQD_TEST_VDEFECT"],
ignore=ignored,
with_primary_dependencies=True)
assert iovs.channels == set(["DQD_TEST_VDEFECT"] +
defect_names) - ignored, ("Check failed.")
assert len(iovs) == len(defect_names[1:]) * 2
def test_defect_insertion_retrieval_unicode():
if six.PY3:
import ROOT
if ROOT.gROOT.GetVersionInt() < 62000:
# Passing str objects using multibyte encodings is broken
# with pyroot up to 6.18. Should be fixed in 6.20?
return
ddb = DefectsDB(TEST_DATABASE, read_only=False)
TEST_ID = 0
TEST_DEFECT_NAME = "DQD_TEST_DEFECT_%i" % TEST_ID
TEST_COMMENT = u"First test defect with character: ±"
TEST_USER = u"DQDefécts.tests"
TEST_SINCE, TEST_UNTIL = 0, 100
TEST_DEFECT_DESCRIPTION = u"À test defect (%i)" % TEST_ID
create_defect_type_unicode(ddb, TEST_ID, TEST_DEFECT_DESCRIPTION)
ddb.insert(TEST_DEFECT_NAME, TEST_SINCE, TEST_UNTIL, TEST_COMMENT,
TEST_USER)
iovs = ddb.retrieve()
assert len(iovs) == 1
iov = iovs[0]
assert iov.present
assert not iov.recoverable
assert iov.user == TEST_USER
assert iov.comment == TEST_COMMENT
assert ddb.all_defect_descriptions[
TEST_DEFECT_NAME] == TEST_DEFECT_DESCRIPTION
def test_ignore_defects():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
create_defect_type(ddb, 2)
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "", "")
ddb.insert("DQD_TEST_DEFECT_1", 100, 200, "", "")
ddb.insert("DQD_TEST_DEFECT_2", 200, 300, "", "")
ddb.new_virtual_defect(
"DQD_TEST_VIRTUAL_DEFECT", "",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1 DQD_TEST_DEFECT_2")
ignored_defects = set(["DQD_TEST_DEFECT_1"])
iovs = ddb.retrieve()
assert iov_result_depends(iovs) & ignored_defects, "test is broken"
iovs = ddb.retrieve(ignore=ignored_defects)
assert not iov_result_depends(
iovs) & ignored_defects, "result depends on ignored defects"
def test_rename_defects():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "", "")
ddb.insert("DQD_TEST_DEFECT_1", 100, 200, "", "")
defects_tag = ddb.new_defects_tag("dqd-test", "New iov tag")
# Make a virtual defect whose result is the combination of the above
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT", "",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1")
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT_2", "",
"DQD_TEST_VIRTUAL_DEFECT")
# Make ones with inversion logic
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT_3", "",
"DQD_TEST_DEFECT_0 !DQD_TEST_DEFECT_1")
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT_4", "",
"!DQD_TEST_VIRTUAL_DEFECT")
original_tag = ddb.new_logics_tag()
original_htag = ddb.new_hierarchical_tag(defects_tag, original_tag)
# The following is no longer considered an error condition
## exc_raised = False
## try:
## ddb.rename_defect("DQD_TEST_DEFECT_0", "DQD_TEST_0")
## except AssertionError:
## exc_raised = True
## assert exc_raised, "Bad defect rename did not raise exception"
# first rename a primary defect
ddb.rename_defect("DQD_TEST_DEFECT_0", "DQD_TEST_DEFECT_3")
assert ddb.virtual_defect_logics['DQD_TEST_VIRTUAL_DEFECT'].clauses == ['DQD_TEST_DEFECT_3', 'DQD_TEST_DEFECT_1'], ddb.virtual_defect_logics['DQD_TEST_VIRTUAL_DEFECT'].clauses
assert ddb.virtual_defect_logics['DQD_TEST_VIRTUAL_DEFECT_3'].clauses == ['DQD_TEST_DEFECT_3', '!DQD_TEST_DEFECT_1'], ddb.virtual_defect_logics['DQD_TEST_VIRTUAL_DEFECT_3'].clauses
# also valid in other tags?
ddb2 = DefectsDB(TEST_DATABASE, tag=original_htag)
assert ddb2.virtual_defect_logics['DQD_TEST_VIRTUAL_DEFECT'].clauses == ['DQD_TEST_DEFECT_3', 'DQD_TEST_DEFECT_1'], ddb2.virtual_defect_logics['DQD_TEST_VIRTUAL_DEFECT'].clauses
# Now try to rename a virtual defect
ddb.rename_defect("DQD_TEST_VIRTUAL_DEFECT", "DQD_TEST_VIRTUAL_DEFECT_A")
assert ddb.virtual_defect_logics['DQD_TEST_VIRTUAL_DEFECT_2'].clauses == ['DQD_TEST_VIRTUAL_DEFECT_A'], ddb.virtual_defect_logics['DQD_TEST_VIRTUAL_DEFECT_2'].clauses
assert ddb.virtual_defect_logics['DQD_TEST_VIRTUAL_DEFECT_4'].clauses == ['!DQD_TEST_VIRTUAL_DEFECT_A'], ddb.virtual_defect_logics['DQD_TEST_VIRTUAL_DEFECT_4'].clauses
# also valid in other tags?
ddb2 = DefectsDB(TEST_DATABASE, tag=original_htag)
assert ddb2.virtual_defect_logics['DQD_TEST_VIRTUAL_DEFECT_2'].clauses == ['DQD_TEST_VIRTUAL_DEFECT_A'], ddb2.virtual_defect_logics['DQD_TEST_VIRTUAL_DEFECT_2'].clauses
ddb._virtual_defect_consistency_check()
def test_nonpresent():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
create_defect_type(ddb, 0)
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests")
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests", present=False)
iovs = ddb.retrieve()
assert len(iovs) == 0
iovs = ddb.retrieve(nonpresent=True)
assert len(iovs) == 1
assert not iovs[0].present
def test_virtual_defects_invert():
"Testing virtual defects involving inversion"
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
create_defect_type(ddb, 2)
# Make a virtual defect whose result is the combination of the above
ddb.new_virtual_defect(
"DQD_TEST_VIRTUAL_DEFECT", "Comment",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1 !DQD_TEST_DEFECT_2")
# Insert an iov into each
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests")
ddb.insert("DQD_TEST_DEFECT_1", 150, 200, "Test", "DQDefects.tests")
ddb.insert("DQD_TEST_DEFECT_2", 50, 125, "Test", "DQDefects.tests")
# See what the result of the virtual defect is
iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
# Tests against the value
assert len(iovs) == 4, list(iovs)
first, second, third, fourth = iovs
assert (first.since, first.until) == (0, 50), first
assert (second.since, second.until) == (50, 100), second
assert (third.since, third.until) == (125, 150), third
assert (fourth.since, fourth.until) == (150, 200), fourth
assert first.channel == second.channel == third.channel == fourth.channel == "DQD_TEST_VIRTUAL_DEFECT", (
first.channel, second.channel, third.channel, fourth.channel)
del first, second, third, fourth, iovs
# Test evaluate_full=False
iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"],
evaluate_full=False)
assert len(iovs) == 2, list(iovs)
# Now unset the present bit
ddb.insert("DQD_TEST_DEFECT_2",
50,
150,
"Test",
"DQDefects.tests",
present=False)
iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
# here we are sensitive to details of python ordering ...
assert len(iovs) == 3, list(iovs)
first, second, third = iovs
assert (first.since, first.until) == (0, 100), first
assert (second.since, second.until) == (100, 150), second
assert (third.since, third.until) == (150, 200), third
assert first.channel == second.channel == third.channel == "DQD_TEST_VIRTUAL_DEFECT", (
first.channel, second.channel, third.channel)
del first, second, third, iovs
def test_update_virtual_defect():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
create_defect_type(ddb, 2)
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "", "")
ddb.insert("DQD_TEST_DEFECT_1", 100, 200, "", "")
ddb.insert("DQD_TEST_DEFECT_2", 200, 300, "", "")
defects_tag = ddb.new_defects_tag("dqd-test", "New iov tag")
# Make a virtual defect whose result is the combination of the above
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT", "",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1")
original_tag = ddb.new_logics_tag()
original_htag = ddb.new_hierarchical_tag(defects_tag, original_tag)
ddb.update_virtual_defect("DQD_TEST_VIRTUAL_DEFECT",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_2")
new_tag = ddb.new_logics_tag()
new_htag = ddb.new_hierarchical_tag(defects_tag, new_tag)
what = {"channels": ["DQD_TEST_VIRTUAL_DEFECT"]}
orig_iovs = DefectsDB(TEST_DATABASE, tag=original_htag).retrieve(**what)
new_iovs = DefectsDB(TEST_DATABASE, tag=new_htag).retrieve(**what)
head_iovs = DefectsDB(TEST_DATABASE).retrieve(**what)
assert head_iovs == new_iovs
assert head_iovs != orig_iovs
assert len(head_iovs) == 2
assert (head_iovs[0].since, head_iovs[0].until) == (0, 100)
assert (head_iovs[1].since, head_iovs[1].until) == (200, 300)
from DQUtils.db import Databases
db3 = Databases.get_instance(TEST_DATABASE, read_only=True)
pfs = db3.getFolderSet('/GLOBAL/DETSTATUS')
assert pfs.tagDescription(
new_htag) != '', 'Hierarchical tag description not created'
def test_virtual_defects_deep():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
# Insert an iov into each
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests")
ddb.insert("DQD_TEST_DEFECT_1", 150, 200, "Test", "DQDefects.tests")
# Make a virtual defect whose result is the combination of the above
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT_0", "", "DQD_TEST_DEFECT_0")
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT_1", "", "DQD_TEST_DEFECT_1")
ddb = DefectsDB(TEST_DATABASE)
iovs = ddb.retrieve()
def test_intersect():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
create_defect_type(ddb, 2)
ddb.insert("DQD_TEST_DEFECT_0", 0, 1000, "", "")
ddb.insert("DQD_TEST_DEFECT_0", 1000, 2000, "", "")
iovs = ddb.retrieve(50, 100, intersect=True)
assert len(iovs) == 1
assert (iovs[0].since, iovs[0].until) == (50, 100)
iovs = ddb.retrieve(999, 1001, intersect=True)
assert len(iovs) == 2
first, second = iovs
assert (first.since, first.until) == (999, 1000)
assert (second.since, second.until) == (1000, 1001)
def test_virtual_defects_stress():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
STRESS_COUNT = 20
for i in range(STRESS_COUNT):
create_defect_type(ddb, i)
with ddb.storage_buffer:
for i in range(STRESS_COUNT):
ddb.insert("DQD_TEST_DEFECT_%i" % i, i, i + 1, "Test",
"DQDefects.tests")
ALL_DEFECTS = " ".join("DQD_TEST_DEFECT_%i" % i
for i in range(STRESS_COUNT))
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT", "", ALL_DEFECTS)
iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert len(iovs) == STRESS_COUNT - 1
assert (iovs[0].since, iovs[-1].until) == (1, STRESS_COUNT)
def test_independent_tags():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
create_defect_type(ddb, 2)
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "", "")
ddb.insert("DQD_TEST_DEFECT_1", 100, 200, "", "")
ddb.insert("DQD_TEST_DEFECT_2", 200, 300, "", "")
dtag1 = ddb.new_defects_tag("dqd-test", "New iov tag")
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "", "", present=False)
dtag2 = ddb.new_defects_tag("dqd-test-2", "New iov tag")
# Make a virtual defect whose result is the combination of the above
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT", "",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1")
ltag1 = ddb.new_logics_tag()
ddb.update_virtual_defect("DQD_TEST_VIRTUAL_DEFECT",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_2")
ltag2 = ddb.new_logics_tag()
iovs = DefectsDB(
TEST_DATABASE,
tag=("HEAD", ltag1)).retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert iov_ranges(iovs) == [(100, 200)], str(iov_ranges(iovs))
iovs = DefectsDB(
TEST_DATABASE,
tag=("HEAD", ltag2)).retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert iov_ranges(iovs) == [(200, 300)], str(iov_ranges(iovs))
iovs = DefectsDB(
TEST_DATABASE,
tag=(dtag1, ltag1)).retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert iov_ranges(iovs) == [(0, 100), (100, 200)], str(iov_ranges(iovs))
iovs = DefectsDB(
TEST_DATABASE,
tag=(dtag2, ltag2)).retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert iov_ranges(iovs) == [(200, 300)], str(iov_ranges(iovs))
htag = ddb.new_hierarchical_tag(dtag2, ltag2)
iovs = DefectsDB(TEST_DATABASE,
tag=(htag,
htag)).retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert iov_ranges(iovs) == [(200, 300)], str(iov_ranges(iovs))
iovs = DefectsDB(TEST_DATABASE,
tag=htag).retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert iov_ranges(iovs) == [(200, 300)], str(iov_ranges(iovs))
def test_iov_tag_defects():
log.info('IOV Tags')
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
create_defect_type(ddb, 2)
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "comment1", "user1", present=False)
ddb.insert("DQD_TEST_DEFECT_1", 100, 200, "", "")
ddb.insert("DQD_TEST_DEFECT_2",
200,
300,
"comment2",
"user2",
recoverable=True)
ddb.new_defects_tag("dqd-test",
"New iov tag",
iovranges=[(0, 51), ((0, 210), (0, 306))])
ddb2 = DefectsDB(TEST_DATABASE, tag='DetStatusDEFECTS-dqd-test')
iovs = ddb2.retrieve(nonpresent=True)
assert len(iovs) == 2
assert ((iovs[0].channel, iovs[0].since, iovs[0].until, iovs[0].present,
iovs[0].recoverable, iovs[0].user,
iovs[0].comment) == ('DQD_TEST_DEFECT_0', 0, 51, False, False,
'user1', 'comment1'))
assert ((iovs[1].channel, iovs[1].since, iovs[1].until, iovs[1].present,
iovs[1].recoverable, iovs[1].user,
iovs[1].comment) == ('DQD_TEST_DEFECT_2', 210, 300, True, True,
'user2', 'comment2'))
def test_virtual_defects():
"Testing virtual defect basics"
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
create_defect_type(ddb, 0)
create_defect_type(ddb, 1)
# Make a virtual defect whose result is the combination of the above
ddb.new_virtual_defect("DQD_TEST_VIRTUAL_DEFECT", "Comment",
"DQD_TEST_DEFECT_0 DQD_TEST_DEFECT_1")
# Insert an iov into each
ddb.insert("DQD_TEST_DEFECT_0", 0, 100, "Test", "DQDefects.tests")
ddb.insert("DQD_TEST_DEFECT_1", 150, 200, "Test", "DQDefects.tests")
# See what the result of the virtual defect is
iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
# Tests against the value
assert len(iovs) == 2
first, second = iovs
assert (first.since, first.until) == (0, 100)
assert (second.since, second.until) == (150, 200)
assert first.channel == second.channel == "DQD_TEST_VIRTUAL_DEFECT"
del first, second, iovs
# Now unset the present bit, and test again that the relevant preiod is fixed
ddb.insert("DQD_TEST_DEFECT_1",
150,
200,
"Test",
"DQDefects.tests",
present=False)
iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert len(iovs) == 1
first = iovs[0]
assert (first.since, first.until) == (0, 100)
assert first.channel == "DQD_TEST_VIRTUAL_DEFECT"
# Now extend the defect and see if we get a contiguous result
ddb.insert("DQD_TEST_DEFECT_0", 100, 150, "Test", "DQDefects.tests")
iovs = ddb.retrieve(channels=["DQD_TEST_VIRTUAL_DEFECT"])
assert len(iovs) == 1
first = iovs[0]
assert (first.since, first.until) == (0, 150)
assert first.channel == "DQD_TEST_VIRTUAL_DEFECT"
def test_query_with_primary_dependencies():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
# Create two defects
defect_names = [create_defect_type(ddb, i) for i in range(3)]
for i, defect in enumerate(defect_names):
ddb.insert(defect, i * 100, (i + 1) * 100, "", "")
ddb.new_virtual_defect("DQD_TEST_VDEFECT", "", " ".join(defect_names))
iovs = ddb.retrieve(channels=["DQD_TEST_VDEFECT"])
assert iovs.channels == set(["DQD_TEST_VDEFECT"])
assert len(iovs) == len(defect_names)
iovs = ddb.retrieve(channels=["DQD_TEST_VDEFECT"],
with_primary_dependencies=True)
assert iovs.channels == set(["DQD_TEST_VDEFECT"] +
defect_names), ("Check failed.")
assert len(iovs) == len(defect_names) * 2
def test_defect_insertion_retrieval_unicode():
ddb = DefectsDB(TEST_DATABASE, read_only=False)
TEST_ID = 0
TEST_DEFECT_NAME = "DQD_TEST_DEFECT_%i" % TEST_ID
TEST_COMMENT = u"First test defect with character: ±"
TEST_USER = u"DQDefécts.tests"
TEST_SINCE, TEST_UNTIL = 0, 100
TEST_DEFECT_DESCRIPTION = u"À test defect (%i)" % TEST_ID
create_defect_type_unicode(ddb, TEST_ID, TEST_DEFECT_DESCRIPTION)
ddb.insert(TEST_DEFECT_NAME, TEST_SINCE, TEST_UNTIL, TEST_COMMENT, TEST_USER)
iovs = ddb.retrieve()
assert len(iovs) == 1
iov = iovs[0]
assert iov.present
assert not iov.recoverable
assert iov.user == TEST_USER
assert iov.comment == TEST_COMMENT
assert ddb.all_defect_descriptions[TEST_DEFECT_NAME] == TEST_DEFECT_DESCRIPTION
class IDBSDefectWriter:
"""
Class for writing BS defects to an sqlite file
"""
def __init__(self,
fileName,
forceNew=False,
dbName='IDBSDQ',
tag='nominal',
user='******'):
"""
Initialise database connection
"""
self.defect = None
self.iovs = IOVSet()
self.user = user
#self.allowedDefects = DefectsDB('').defect_names
if not fileName: fileName = 'tmp.' + str(os.getpid()) + '.db'
self.connect(fileName, forceNew, dbName, tag)
pass
def __del__(self):
"""
Delete db to clear connection
"""
os.system('[[ -f tmp.%s.db ]] && rm tmp.%s.db' %
(os.getpid(), os.getpid()))
del self.db
pass
def connect(self,
fileName,
forceNew=False,
dbName='IDBSDQ',
tag='nominal'):
"""
Open connection to defect DB
"""
connString = 'sqlite://;schema=%s;dbname=%s' % (fileName, dbName)
if forceNew and os.path.exists(fileName):
os.remove(fileName)
self.db = DefectsDB(
connString, read_only=False, create=True, tag=(tag, 'HEAD')
) # Second tag is for virtual defects, which we are not interested in
self.officialDb = DefectsDB()
return
def defectType(self, t):
"""
Set defect type
"""
self.defect = t
def add(self,
runMin=0,
runMax=(1 << 31) - 1,
lbMin=0,
lbMax=(1 << 32) - 1):
"""
Add iovs which are NOT defective to the list
Note, lbMax is exclusive here (and inclusive when shown on defect web page).
"""
# Make since and until and convert to syntactially nice RunLumiType
since = RunLumiType((runMin << 32) + lbMin)
until = RunLumiType((runMax << 32) + lbMax)
# IoVs which are not defective (i.e. beamspot good)
self.iovs.add(since, until, self.defect, False)
return
def complete(self, runMin, runMax):
"""
Complete a list of IoVs to cover all LBs in a run, treating empty ones as having 'emptyState'
"""
# Create an IOV set covering the entire run(s)
run_lbs = fetch_iovs("EOR",
runs=(runMin, runMax),
what=[],
with_channel=False)
# run_lbs = IOVSet()
# lbMin = 1
# lbMax = (1 << 32) -1 # Note, lbMax is exclusive
# since = RunLumiType((runMin << 32)+lbMin)
# until = RunLumiType((runMax << 32)+lbMax)
# run_lbs.add(since, until)
if not len(run_lbs):
print "WARNING: No LBs in run according to EOR_Params - are we running before the run has finished?"
def lbsStartAtOne(iov):
"Change LBs starting at 0 to start at 1"
return iov._replace(since=RunLumi(iov.since.run, 1))
# Start LBs from 1 rather than 0 (at request of P. Onyisi) as long as run has more than one LB (else skip)
run_lbs = [lbsStartAtOne(iov) for iov in run_lbs if iov.until.lumi > 1]
# Empty IOV set for adding full list of LBs too
iovs = IOVSet()
# Ask official DB if an IoV is currently defective so can unset only those if doing reprocessing
defectsCurrentlyInDb = self.officialDb.retrieve(
(runMin, 0), (runMax + 1, 0), [self.defect])
# Order IOVs to avoid since > until
self.iovs = IOVSet(sorted(self.iovs))
#for since, until, (run_lb, iov, dbDefect) in process_iovs(run_lbs.solidify(RANGEIOV_VAL), self.iovs.solidify(DEFECTIOV_VAL), defectsCurrentlyInDb):
for since, until, (run_lb, iov, dbDefect) in process_iovs(
run_lbs, self.iovs.solidify(DEFECTIOV_VAL),
defectsCurrentlyInDb):
if not run_lb: continue # Check valid
# # Start LBs from 1 rather than 0 (at request of P. Onyisi)
# # If this makes since==until, i.e. it was a [0->1) LB range then skip
# if since.lumi==0:
# since = RunLumiType((since.run << 32)+since.lumi+1)
# if since==until: continue
# Add iovs from self.iovs treating empty ones as defective (i.e. beamspot bad/missing)
if iov:
# These are not defective
if dbDefect and dbDefect.present:
# Only store not defective IoVs if they are present on the Db so we can unset them
# (don't want to write not present defects to Db unless really chaning an existing defect)
iovs.add(since, until, self.defect, False)
else:
# These are defective
iovs.add(since, until, self.defect, True)
self.iovs = iovs
return
def writeDefects(self, tag='nominal', nonpresent=False):
"""
Write all defects to the database. If 'nonpresent' is True then write the absent ones too
"""
with self.db.storage_buffer:
for since, until, defect, present in self.iovs:
if not present and not nonpresent: continue
#print since, until, present
self._writeDefect(defect, since, until, present=present)
def _writeDefect(self,
defect,
since,
until,
tag='nominal',
description='',
comment='',
present=True,
recoverable=False):
"""
Write a single defect to the database
"""
if not defect in self.db.defect_names:
self.db.create_defect(defect, description)
self.db.insert(defect, since, until, comment, self.user, present,
recoverable)
return
def dump(self, filename=None):
"""
Dump defects to a file given by filename or stdout if no filename given
"""
from DQUtils.utils import pprint_objects
if filename is not None:
f = open(filename.replace('.db', '.txt'), "w")
# If not defects then nothing will be in the database and we write an empty file
if len(self.iovs):
pprint_objects(
self.db.retrieve(primary_only=True, nonpresent=True), f)
f.close()
else:
if len(self.iovs):
self.iovs.pprint()
else:
print '\nNo DQ defects'
# with open(filename.replace('.db', '.txt'), "w") as f:
# pprint_objects(self.db.retrieve(), f)
return
def go(iov, systems, db, indb, timewise=False):
"""
Run the DCS calculator for `run` on the `systems` specified, saving the
result to the `database`.
"""
since, until = iov
with timer("Read LBLB"):
# fetch lumi block info
lblb = fetch_iovs("LBLB",
since,
until,
with_channel=False,
database='COOLONL_TRIGGER/%s' % indb)
assert lblb, "No luminosity blocks for desired range. [%s, %s)" % (
since, until)
# lumi block time info
lbtime = inverse_lblb(lblb)
# run_iovs is....?
if timewise:
run_iovs = run_iovs_from_lblb(lbtime)
else:
run_iovs = run_iovs_from_lblb(lblb)
log.debug("Will process %i LBs over %i runs", len(lblb), len(run_iovs))
log.info("DCSC2 will run over range: %r %s", tuple(run_iovs.range_iov),
"(%i lbs)" % len(lblb))
# Initialize the system objects
# This changes 'systems' from a list of types to a list of subdetector objects
systems = [
system() for system in systems
if not getattr(system, "__DISABLED__", False)
]
# Run systems sequentially for this range
result_iovs = run_sequential(systems, lbtime, run_iovs)
# Run parallel is broken because of the use of closures to implement the
# defect translation.
#result_iovs = run_parallel(systems, lbtime, run_iovs, parallel)
if log.isEnabledFor(logging.DEBUG):
pprint_objects(result_iovs)
log.info("Calculation complete")
if db != "None":
with timer("write result (%i iovs)" % len(result_iovs)):
log.debug("Writing result (%i iovs)", len(result_iovs))
defect_iovs = filter(lambda iov: isinstance(iov, DefectIOV),
result_iovs)
dcsofl_iovs = filter(lambda iov: not isinstance(iov, DefectIOV),
result_iovs)
if len(defect_iovs) > 0:
ddb = DefectsDB(db, read_only=False, create=True)
defect_names = set(i.channel for i in defect_iovs)
for defect in defect_names:
if defect in ddb.defect_id_map:
continue
ddb.create_defect(defect, "Created by DCSCalculator2")
with ddb.storage_buffer:
for iov in defect_iovs:
ddb.insert(iov.channel, iov.since, iov.until,
iov.comment, 'sys:defectcalculator',
iov.present)
#disable DCSOFL
#dest = "%s::/GLOBAL/DETSTATUS/DCSOFL" % db
#write_iovs(dest, dcsofl_iovs, dcsofl_cool_record(), create=True)
args = len(result_iovs), hash(result_iovs)
log.info("Success. Calculated %i iovs. Result hash: 0x%0x8." % args)
def hancool_defects(runNumber,
filePath="./",
dbConnection="",
db_tag='HEAD',
isESn=True):
from . import pix_defect
analyzers = []
if isESn:
# CTP
analyzers += [ctp_defects]
# SCT
analyzers += [
sct_eff_defect,
sct_lowstat_defect,
sct_conf_defects,
sct_perlb_defects,
]
if (len(filePath) == 0 or filePath[-1] != '/'):
filePath += "/"
if (len(dbConnection) < 1):
dbConnection = "/afs/cern.ch/user/a/atlasdqm/dqmdisk1/cherrypy-devel/defectstest.db/COMP200"
import ROOT
# Conflict logic: shorter intervals override longer ones
defects_by_function = {}
# empty list for missing high stat, reserved for future use
fnames = ([[filePath + "run_" + str(runNumber) + "_han.root"], []] + [
glob.glob(
os.path.join(filePath, 'run_%s%s*_han.root' %
(runNumber, intervalType[i]))) for i in [2, 3]
])
for i, itype in enumerate(fnames):
ldefects_by_function = {}
for globname in itype:
filename = os.path.basename(globname)
##print "--------------------------------------"
since, until = getLimits(filename)
default_iov = defect_iov(*([0] * 5))
default_iov = default_iov._replace(since=since, until=until)
#print filename + ':', since, until
fobj = ROOT.TFile.Open(globname)
for func in analyzers:
rv = func(fobj, i, runNumber)
if rv is not None:
rvt = [default_iov._replace(**(i._asdict())) for i in rv]
if func not in ldefects_by_function:
ldefects_by_function[func] = rvt
else:
ldefects_by_function[func] += rvt
defects_by_function.update(ldefects_by_function)
defects = sum(defects_by_function.values(), [])
if isESn:
globname = fnames[0][0]
filename = os.path.basename(globname)
since, until = getLimits(filename)
try:
defects += pix_defect.execute(runNumber, globname, until - 1)
except Exception as e:
logging.warning('Unable to execute pixel hancool code')
logging.warning('--> %s: %s', type(e).__name__, e)
from DQDefects import DefectsDB
ddb = DefectsDB(dbConnection, read_only=False)
if isESn:
logging.info('Running detmask_defects')
detmask_defects(runNumber, ddb)
with ddb.storage_buffer:
for defect in iovs_merge(defects):
logger.debug('Uploading %s', defect)
ddb.insert(defect.defect,
since=(runNumber << 32 | defect.since),
until=(runNumber << 32 | defect.until),
comment=defect.comment,
recoverable=defect.recoverable,
added_by='sys:hancool')
print 'Reading in IOVs...'
iniovs = indb.retrieve(since=since,
until=until,
channels=channels,
primary_only=True,
nonpresent=(not opts.noabsent))
print '%d IOVs retrieved from input database' % len(iniovs)
#inchannels = set((x.channel for x in iniovs))
inchannels = set(channels if channels is not None else indb.defect_names)
outchannels = set(outdb.defect_names)
missingchannels = inchannels - outchannels
if len(missingchannels) != 0:
if not opts.createdefects:
print 'Missing defect(s) in target database:'
print list(missingchannels)
print 'Rerun with a restricted defect list (--defects) or with --createdefects'
sys.exit(1)
else:
print 'Creating missing channels on output database'
descriptions = indb.get_channel_descriptions(missingchannels)
for channel in missingchannels:
outdb.create_defect(channel, descriptions[channel])
with outdb.storage_buffer:
print 'Writing out IOVs...'
for iov in iniovs:
outdb.insert(iov.channel, iov.since, iov.until, iov.comment,
iov.user, iov.present, iov.recoverable)
print 'Done.'
