def test_return_good_split(self):
# {{{
self.interface.register_dataset(*self.create_dbs_dataset(
'test_good_split', lumis=20, filesize=2.2, tasksize=6))
tasks = self.interface.pop_units('test_good_split', 1)
assert (len(tasks) == 1)
(id, label, files, lumis, arg, _) = tasks[0]
task_update = TaskUpdate(host='hostname', id=id)
handler = TaskHandler(id, label, files, lumis, None, True)
file_update, unit_update = handler.get_unit_info(
False, task_update, {
'/test/0.root': (220, [(1, 1), (1, 2), (1, 3)]),
}, [], 100)
self.interface.update_units({
(label, "units_" + label):
[(task_update, file_update, unit_update)]
})
(jr, jd, er, ew) = self.interface.db.execute(
"""
select
units_running,
units_done,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""", (label, )).fetchone()
assert jr == 0
assert jd == 3
assert er == 220
assert ew == 100
(id, jr, jd, er) = self.interface.db.execute(
"select id, units_running, units_done, events_read from files_test_good_split where filename='/test/0.root'"
).fetchone()
assert jr == 0
assert jd == 3
assert er == 220
def handler(self, id_, files, lumis, taskdir, merge=False):
if merge:
return MergeTaskHandler(id_, self.label, files, lumis,
list(self.get_outputs(id_)), taskdir)
elif isinstance(self.dataset, MultiProductionDataset):
return MultiProductionTaskHandler(id_, self.label, files, lumis,
list(self.get_outputs(id_)),
taskdir)
elif isinstance(self.dataset, ProductionDataset) or isinstance(
self.dataset, EmptyDataset):
return ProductionTaskHandler(id_, self.label, lumis,
list(self.get_outputs(id_)), taskdir)
else:
return TaskHandler(id_,
self.label,
files,
lumis,
list(self.get_outputs(id_)),
taskdir,
local=self.local)
def test_return_bad(self):
# {{{
self.interface.register_dataset(*self.create_dbs_dataset('test_bad'))
(id, label, files, lumis, arg,
_) = self.interface.pop_units('test_bad', 1)[0]
task_update = TaskUpdate(exit_code=123,
host='hostname',
id=id,
submissions=1)
handler = TaskHandler(id, label, files, lumis, None, True)
file_update, unit_update = handler.get_unit_info(
True, task_update, {}, [], 0)
self.interface.update_units({
(label, "units_" + label):
[(task_update, file_update, unit_update)]
})
(jr, jd, er, ew) = self.interface.db.execute(
"""
select
units_running,
units_done,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""", (label, )).fetchone()
assert jr == 0
assert jd == 0
assert er in (0, None)
assert ew in (0, None)
(id, jr, jd, er) = self.interface.db.execute(
"select id, units_running, units_done, events_read from files_test_bad where filename='/test/0.root'"
).fetchone()
assert jr == 0
assert jd == 0
assert er in (0, None)
def test_file_return_ugly(self):
# {{{
self.interface.register_dataset(
*self.create_file_dataset(
'test_file_return_ugly', 5, 3))
(id, label, files, lumis, arg, _) = self.interface.pop_units('test_file_return_ugly', 1)[0]
task_update = TaskUpdate(host='hostname', id=id)
handler = TaskHandler(id, label, files, lumis, None, True)
file_update, unit_update = handler.get_unit_info(
False,
task_update,
{
'/test/0.root': (220, [(1, 1), (1, 2), (1, 3)]),
'/test/1.root': (220, [(1, 3), (1, 4), (1, 5)]),
},
['/test/2.root'],
100
)
self.interface.update_units(
{(label, "units_" + label): [(task_update, file_update, unit_update)]})
(jr, jd, jl, er, ew) = self.interface.db.execute("""
select
units_running,
units_done,
units_left,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""",
(label,)).fetchone()
assert jr == 0
assert jd == 2
assert jl == 3
assert er == 440
assert ew == 100
def test_handler(self):
# {{{
self.interface.register_dataset(*self.create_dbs_dataset(
'test_handler', lumis=11, filesize=2.2, tasksize=3))
(id, label, files, lumis, arg,
_) = self.interface.pop_units('test_handler', 1)[0]
handler = TaskHandler(123, 'test_handler', files, lumis, 'test', True)
files_info = {u'/test/0.root': (220, [(1, 1), (1, 2), (1, 3)])}
files_skipped = []
events_written = 123
update = TaskUpdate()
file_update, unit_update = \
handler.get_unit_info(False, update, files_info, files_skipped, events_written)
assert update.units_processed == 3
assert update.events_read == 220
assert update.events_written == 123
assert update.status == 2
assert file_update == [(220, 0, 1)]
assert unit_update == []
def test_return_ugly(self):
# {{{
self.interface.register_dataset(
*self.create_dbs_dataset(
'test_ugly', lumis=11, filesize=3, tasksize=6))
(id, label, files, lumis, arg, _) = self.interface.pop_units('test_ugly', 1)[0]
task_update = TaskUpdate(host='hostname', id=id)
handler = TaskHandler(id, label, files, lumis, None, True)
file_update, unit_update = handler.get_unit_info(
False,
task_update,
{
'/test/0.root': (120, [(1, 2), (1, 3)])
},
['/test/1.root'],
50
)
self.interface.update_units(
{(label, "units_" + label): [(task_update, file_update, unit_update)]})
skipped = list(
self.interface.db.execute(
"select skipped from files_{0}".format(label)))
assert skipped == [(0,), (1,), (0,), (0,)]
status = list(
self.interface.db.execute(
"select status from units_{0} where file=2 group by status".format(label)))
assert status == [(3,)]
(jr, jd, jl, er, ew) = self.interface.db.execute("""
select
units_running,
units_done,
units_left,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""",
(label,)).fetchone()
assert jr == 0
assert jd == 2
assert jl == 9
assert er == 120
assert ew == 50
(id, jr, jd) = self.interface.db.execute(
"select id, units_running, units_done from files_test_ugly where filename='/test/0.root'").fetchone()
assert jr == 0
assert jd == 2
(id, jr, jd) = self.interface.db.execute(
"select id, units_running, units_done from files_test_ugly where filename='/test/1.root'").fetchone()
assert jr == 0
assert jd == 0