def test_file_return_bad(self):
# {{{
self.interface.register_dataset(
*self.create_file_dataset(
'test_file_return_bad', 5, 3))
(id, label, files, lumis, arg, _) = self.interface.pop_units('test_file_return_bad', 1)[0]
task_update = TaskUpdate(exit_code=1234, host='hostname', id=id)
handler = TaskHandler(id, label, files, lumis, None, True)
file_update, unit_update = handler.get_unit_info(
True,
task_update,
{},
[],
0
)
self.interface.update_units(
{(label, "units_" + label): [(task_update, file_update, unit_update)]})
(jr, jd, er, ew) = self.interface.db.execute("""
select
units_running,
units_done,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""",
(label,)).fetchone()
assert jr == 0
assert jd == 0
assert er in (0, None)
assert ew in (0, None)
def test_handler(self):
# {{{
self.interface.register_dataset(
*self.create_dbs_dataset(
'test_handler', lumis=11, filesize=2.2, tasksize=3))
(id, label, files, lumis, arg, _) = self.interface.pop_units('test_handler', 1)[0]
handler = TaskHandler(123, 'test_handler', files, lumis, 'test', True)
files_info = {
u'/test/0.root': (220, [(1, 1), (1, 2), (1, 3)])
}
files_skipped = []
events_written = 123
update = TaskUpdate()
file_update, unit_update = \
handler.get_unit_info(False, update, files_info, files_skipped, events_written)
assert update.units_processed == 3
assert update.events_read == 220
assert update.events_written == 123
assert update.status == 2
assert file_update == [(220, 0, 1)]
assert unit_update == []
def test_file_return_good(self):
# {{{
self.interface.register_dataset(
*self.create_file_dataset('test_file_return_good', 5, 3))
(id, label, files, lumis, arg,
_) = self.interface.pop_units('test_file_return_good', 1)[0]
task_update = TaskUpdate(host='hostname', id=id)
handler = TaskHandler(id, label, files, lumis, None, True)
file_update, unit_update = handler.get_unit_info(
False, task_update, {
'/test/0.root': (220, [(1, 1), (1, 2), (1, 3)]),
'/test/1.root': (220, [(1, 3), (1, 4), (1, 5)]),
'/test/2.root': (160, [(1, 5), (1, 6)])
}, [], 100)
self.interface.update_units({
(label, "units_" + label):
[(task_update, file_update, unit_update)]
})
(jr, jd, er, ew) = self.interface.db.execute(
"""
select
units_running,
units_done,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""", (label, )).fetchone()
assert jr == 0
assert jd == 3
assert er == 600
assert ew == 100
def test_return_ugly(self):
# {{{
self.interface.register_dataset(*self.create_dbs_dataset(
'test_ugly', lumis=11, filesize=3, tasksize=6))
(id, label, files, lumis, arg,
_) = self.interface.pop_units('test_ugly', 1)[0]
task_update = TaskUpdate(host='hostname', id=id)
handler = TaskHandler(id, label, files, lumis, None, True)
file_update, unit_update = handler.get_unit_info(
False, task_update, {'/test/0.root': (120, [(1, 2), (1, 3)])},
['/test/1.root'], 50)
self.interface.update_units({
(label, "units_" + label):
[(task_update, file_update, unit_update)]
})
skipped = list(
self.interface.db.execute(
"select skipped from files_{0}".format(label)))
assert skipped == [(0, ), (1, ), (0, ), (0, )]
status = list(
self.interface.db.execute(
"select status from units_{0} where file=2 group by status".
format(label)))
assert status == [(3, )]
(jr, jd, jl, er, ew) = self.interface.db.execute(
"""
select
units_running,
units_done,
units_left,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""", (label, )).fetchone()
assert jr == 0
assert jd == 2
assert jl == 9
assert er == 120
assert ew == 50
(id, jr, jd) = self.interface.db.execute(
"select id, units_running, units_done from files_test_ugly where filename='/test/0.root'"
).fetchone()
assert jr == 0
assert jd == 2
(id, jr, jd) = self.interface.db.execute(
"select id, units_running, units_done from files_test_ugly where filename='/test/1.root'"
).fetchone()
assert jr == 0
assert jd == 0
def test_return_good(self):
# {{{
self.interface.register_dataset(
*self.create_dbs_dataset(
'test_good', lumis=20, filesize=3.0, tasksize=6))
(id, label, files, lumis, arg, _) = self.interface.pop_units('test_good', 1)[0]
task_update = TaskUpdate(host='hostname', id=id)
handler = TaskHandler(id, label, files, lumis, None, True)
file_update, unit_update = handler.get_unit_info(
False,
task_update,
{
'/test/0.root': (300, [(1, 1), (1, 2), (1, 3)]),
'/test/1.root': (60, [(1, 4), (1, 5), (1, 6)])
},
[],
100
)
self.interface.update_units({(label, "units_" + label): [(task_update, file_update, unit_update)]})
(jr, jd, er, ew) = self.interface.db.execute("""
select
units_running,
units_done,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""",
(label,)).fetchone()
assert jr == 0
assert jd == 6
assert er == 360
assert ew == 100
(id, jr, jd, er) = self.interface.db.execute(
"select id, units_running, units_done, events_read from files_test_good where filename='/test/0.root'").fetchone()
assert jr == 0
assert jd == 3
assert er == 300
(id, jr, jd, er) = self.interface.db.execute(
"select id, units_running, units_done, events_read from files_test_good where filename='/test/1.root'").fetchone()
assert jr == 0
assert jd == 3
assert er == 60
def test_output_info(self):
summary = ReleaseSummary()
handler = TaskHandler(
1, "test", [], [], [],
os.path.join(os.path.dirname(__file__), "data/handler/successful"))
failed, task_update, file_update, unit_update = handler.process(
DummyTask(), summary, defaultdict(lambda: defaultdict(Counter)))
outinfo = handler.output_info
assert outinfo.lumis == [(1, 261), (1, 262), (1, 263), (1, 264),
(1, 265), (1, 266), (1, 267), (1, 268),
(1, 269), (1, 270), (1, 271), (1, 272),
(1, 273), (1, 274), (1, 275), (1, 276),
(1, 277), (1, 278), (1, 279), (1, 280)]
assert outinfo.events == 4000
assert outinfo.size == 15037503
def test_return_bad_again(self):
# {{{
self.interface.register_dataset(*self.create_dbs_dataset(
'test_bad_again', lumis=20, filesize=2.2, tasksize=6))
(id, label, files, lumis, arg, _) = self.interface.pop_units('test_bad_again', 1)[0]
task_update = TaskUpdate(
exit_code=123,
host='hostname',
id=id,
submissions=1)
handler = TaskHandler(id, label, files, lumis, None, True)
file_update, unit_update = handler.get_unit_info(
True,
task_update,
{
'/test/0.root': (220, [(1, 1), (1, 2), (1, 3)]),
'/test/1.root': (220, [(1, 3), (1, 4), (1, 5)]),
'/test/2.root': (160, [(1, 5), (1, 6)])
},
[],
100
)
self.interface.update_units(
{(label, "units_" + label): [(task_update, file_update, unit_update)]})
(jr, jd, er, ew) = self.interface.db.execute("""
select
units_running,
units_done,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""",
(label,)).fetchone()
assert jr == 0
assert jd == 0
assert er in (0, None)
assert ew in (0, None)
(id, jr, jd, er) = self.interface.db.execute(
"select id, units_running, units_done, events_read from files_test_bad_again where filename='/test/0.root'").fetchone()
assert jr == 0
assert jd == 0
assert er in (0, None)
def test_file_obtain(self):
# {{{
self.interface.register_dataset(
*self.create_file_dataset(
'test_file_obtain', 5, 3))
(id, label, files, lumis, arg, _) = self.interface.pop_units('test_file_obtain', 1)[0]
parameters = {'mask': {'lumis': None}}
TaskHandler(id, label, files, lumis, None, True).adjust(
parameters, [], [], se.StorageConfiguration({}))
assert parameters['mask']['lumis'] is None
(jr, jd, er, ew) = self.interface.db.execute("""
select
units_running,
units_done,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""",
(label,)).fetchone()
assert jr == 3
assert jd == 0
assert er in (0, None)
assert ew in (0, None)
def handler(self, id_, files, lumis, taskdir, merge=False):
if merge:
return MergeTaskHandler(id_, self.label, files, lumis,
list(self.get_outputs(id_)), taskdir)
elif isinstance(self.dataset, MultiProductionDataset) or isinstance(
self.dataset, ParentMultiGridpackDataset):
return MultiProductionTaskHandler(id_, self.label, files, lumis,
list(self.get_outputs(id_)),
taskdir)
elif isinstance(self.dataset, ProductionDataset) or isinstance(
self.dataset, EmptyDataset):
return ProductionTaskHandler(id_, self.label, lumis,
list(self.get_outputs(id_)), taskdir)
elif isinstance(self.dataset, MultiGridpackDataset):
return MultiGridpackTaskHandler(id_, self.label, files, lumis,
list(self.get_outputs(id_)),
taskdir,
self.dataset.lumis_per_gridpack)
else:
return TaskHandler(id_,
self.label,
files,
lumis,
list(self.get_outputs(id_)),
taskdir,
local=self.local)
def test_return_good_split(self):
# {{{
self.interface.register_dataset(
*self.create_dbs_dataset('test_good_split', lumis=20, filesize=2.2, tasksize=6))
tasks = self.interface.pop_units('test_good_split', 1)
assert(len(tasks) == 1)
(id, label, files, lumis, arg, _) = tasks[0]
task_update = TaskUpdate(host='hostname', id=id)
handler = TaskHandler(id, label, files, lumis, None, True)
file_update, unit_update = handler.get_unit_info(
False,
task_update,
{
'/test/0.root': (220, [(1, 1), (1, 2), (1, 3)]),
},
[],
100
)
self.interface.update_units({(label, "units_" + label): [(task_update, file_update, unit_update)]})
(jr, jd, er, ew) = self.interface.db.execute("""
select
units_running,
units_done,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""", (label,)).fetchone()
assert jr == 0
assert jd == 3
assert er == 220
assert ew == 100
(id, jr, jd, er) = self.interface.db.execute(
"select id, units_running, units_done, events_read from files_test_good_split where filename='/test/0.root'").fetchone()
assert jr == 0
assert jd == 3
assert er == 220
def test_file_return_ugly(self):
# {{{
self.interface.register_dataset(
*self.create_file_dataset(
'test_file_return_ugly', 5, 3))
(id, label, files, lumis, arg, _) = self.interface.pop_units('test_file_return_ugly', 1)[0]
task_update = TaskUpdate(host='hostname', id=id)
handler = TaskHandler(id, label, files, lumis, None, True)
file_update, unit_update = handler.get_unit_info(
False,
task_update,
{
'/test/0.root': (220, [(1, 1), (1, 2), (1, 3)]),
'/test/1.root': (220, [(1, 3), (1, 4), (1, 5)]),
},
['/test/2.root'],
100
)
self.interface.update_units(
{(label, "units_" + label): [(task_update, file_update, unit_update)]})
(jr, jd, jl, er, ew) = self.interface.db.execute("""
select
units_running,
units_done,
units_left,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""",
(label,)).fetchone()
assert jr == 0
assert jd == 2
assert jl == 3
assert er == 440
assert ew == 100
def test_return_uglier(self):
# {{{
self.interface.register_dataset(
*self.create_dbs_dataset(
'test_uglier', lumis=15, filesize=3, tasksize=8))
(id, label, files, lumis, arg, _) = self.interface.pop_units('test_uglier', 1)[0]
task_update = TaskUpdate(host='hostname', id=id, submissions=1)
handler = TaskHandler(id, label, files, lumis, None, True)
file_update, unit_update = handler.get_unit_info(
False,
task_update,
{
'/test/0.root': (300, [(1, 1), (1, 2), (1, 3)]),
'/test/1.root': (300, [(1, 4), (1, 5), (1, 6)]),
},
['/test/2.root'],
100
)
self.interface.update_units({(label, "units_" + label): [(task_update, file_update, unit_update)]})
# grab another task
(id, label, files, lumis, arg, _) = self.interface.pop_units('test_uglier', 1)[0]
task_update.id = id
handler = TaskHandler(id, label, files, lumis, None, True)
file_update, unit_update = handler.get_unit_info(
False,
task_update,
{
'/test/2.root': (300, [(1, 7), (1, 8), (1, 9)]),
'/test/3.root': (300, [(1, 10), (1, 11), (1, 12)]),
'/test/4.root': (100, [(1, 13)]),
},
[],
100
)
self.interface.update_units({(label, "units_" + label): [(task_update, file_update, unit_update)]})
(jr, jd, jl, er, ew) = self.interface.db.execute("""
select
units_running,
units_done,
units_left,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""",
(label,)).fetchone()
assert jr == 0
assert jd == 13
assert jl == 2
assert er == 1300
assert ew == 200
def test_return_ugly(self):
# {{{
self.interface.register_dataset(
*self.create_dbs_dataset(
'test_ugly', lumis=11, filesize=3, tasksize=6))
(id, label, files, lumis, arg, _) = self.interface.pop_units('test_ugly', 1)[0]
task_update = TaskUpdate(host='hostname', id=id)
handler = TaskHandler(id, label, files, lumis, None, True)
file_update, unit_update = handler.get_unit_info(
False,
task_update,
{
'/test/0.root': (120, [(1, 2), (1, 3)])
},
['/test/1.root'],
50
)
self.interface.update_units(
{(label, "units_" + label): [(task_update, file_update, unit_update)]})
skipped = list(
self.interface.db.execute(
"select skipped from files_{0}".format(label)))
assert skipped == [(0,), (1,), (0,), (0,)]
status = list(
self.interface.db.execute(
"select status from units_{0} where file=2 group by status".format(label)))
assert status == [(3,)]
(jr, jd, jl, er, ew) = self.interface.db.execute("""
select
units_running,
units_done,
units_left,
(select sum(events_read) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id),
(select sum(events_written) from tasks where status in (2, 6, 8) and type = 0 and workflow = workflows.id)
from workflows where label=?""",
(label,)).fetchone()
assert jr == 0
assert jd == 2
assert jl == 9
assert er == 120
assert ew == 50
(id, jr, jd) = self.interface.db.execute(
"select id, units_running, units_done from files_test_ugly where filename='/test/0.root'").fetchone()
assert jr == 0
assert jd == 2
(id, jr, jd) = self.interface.db.execute(
"select id, units_running, units_done from files_test_ugly where filename='/test/1.root'").fetchone()
assert jr == 0
assert jd == 0