def test_task_manager_stop_on_fail(self):
num_tasks = 8
jobs = 4
tm = TaskManager(0, stop_on_fail=True)
results = set()
for i in range(jobs - 1):
tm.add_task(0, i, _do_append, i, results, 0.5)
tm.add_task(1, jobs - 1, _do_fail, 0.1)
for i in range(jobs, num_tasks):
tm.add_task(2, i, _do_append, i, results, 0)
tm.start(jobs)
time.sleep(1)
done_tasks = sorted(tm.get_finished_tasks(), key=lambda t: t.task_id)
print()
self.assertEqual(len(done_tasks), jobs)
items = zip(range(jobs), [None] * num_tasks, [None] * num_tasks)
expected_tasks = [TaskResult(task_id, error, result)
for task_id, error, result in items]
self.assertEqual(done_tasks[:3], expected_tasks[:3])
self.assertEqual(done_tasks[3].task_id, 3)
self.assertIsNotNone(done_tasks[3].error)
def test_task_manager_one_fail(self):
tm = TaskManager(4)
results = set()
num_tasks = 3
tm.add_task(0, 0, _do_append, 0, results, 0.3)
tm.add_task(0, 1, _do_fail, 0.1)
tm.add_task(0, 2, _do_append, 2, results, 0)
time.sleep(1)
done_tasks = sorted(tm.get_finished_tasks(), key=lambda v: v.task_id)
self.assertEqual(len(done_tasks), num_tasks)
items = zip(range(num_tasks), [None] * num_tasks, [None] * num_tasks)
expected_tasks = [TaskResult(task_id, error, result)
for task_id, error, result in items]
self.assertEqual(done_tasks[0], expected_tasks[0])
self.assertEqual(done_tasks[1].task_id, 1)
self.assertIsNotNone(done_tasks[1].error)
self.assertEqual(done_tasks[2], expected_tasks[2])
def test_task_manager(self):
jobs = 1
tm = TaskManager(0)
results = set()
num_of_tasks = 8
for i in range(num_of_tasks):
tm.add_task(num_of_tasks - i, i, _do_append, i, results)
tm.start(jobs)
time.sleep(0.5) # wait until all tasks are done
done_tasks = [result.task_id for result in tm.get_finished_tasks()]
expected_tasks = sorted(range(num_of_tasks), reverse=True)
self.assertEqual(done_tasks, expected_tasks)
self.assertEqual(results, set(range(num_of_tasks)))
def test_task_manager_stop(self):
jobs = 4
num_tasks = 8
tm = TaskManager(num_threads=jobs)
results = set()
for i in range(num_tasks):
tm.add_task(0, i, _do_append, i, results, 1)
time.sleep(0.2)
tm.stop()
tm.stop()
done_tasks = sorted(result.task_id
for result in tm.get_finished_tasks())
self.assertEqual(len(done_tasks), jobs)
self.assertEqual(results, set(done_tasks))
def test_task_manager_fail(self):
jobs = 4
tm = TaskManager(jobs)
num_of_tasks = 100
for i in range(num_of_tasks):
tm.add_task(0, i, _do_fail, 0.0)
tm.start(jobs)
time.sleep(0.5) # wait until all tasks are done
done_tasks = tm.get_finished_tasks()
self.assertEqual(len(done_tasks), num_of_tasks)
for i, t in enumerate(sorted(done_tasks)):
self.assertEqual(t.task_id, i)
self.assertIsNotNone(t.error)
class _NodesBuilder (object):
__slots__ = (
'vfiles',
'build_manager',
'task_manager',
'building_nodes',
)
# -----------------------------------------------------------
def __init__(self,
build_manager,
jobs=0,
keep_going=False,
with_backtrace=True,
use_sqlite=False,
force_lock=False
):
self.vfiles = _VFiles(use_sqlite=use_sqlite, force_lock=force_lock)
self.building_nodes = {}
self.build_manager = build_manager
self.task_manager = TaskManager(num_threads=jobs,
stop_on_fail=not keep_going,
with_backtrace=with_backtrace)
# -----------------------------------------------------------
def __enter__(self):
return self
# -----------------------------------------------------------
def __exit__(self, exc_type, exc_value, backtrace):
self.close()
# -----------------------------------------------------------
def _add_building_node(self, node):
conflicting_nodes = []
building_nodes = self.building_nodes
node_names = {}
for name, signature in node.get_names_and_signatures():
other = building_nodes.get(name, None)
if other is None:
node_names[name] = (node, signature)
continue
other_node, other_signature = other
if node is other_node:
continue
if other_signature != signature:
raise ErrorNodeSignatureDifferent(node)
conflicting_nodes.append(other_node)
if conflicting_nodes:
node.recheck_actual()
self.build_manager.depends(node, conflicting_nodes)
return False
building_nodes.update(node_names)
return True
# -----------------------------------------------------------
def _remove_building_node(self, node):
building_nodes = self.building_nodes
for name in node.get_names():
del building_nodes[name]
# -----------------------------------------------------------
def is_building(self):
return bool(self.building_nodes)
# -----------------------------------------------------------
def build(self, nodes):
build_manager = self.build_manager
explain = build_manager.explain
vfiles = self.vfiles
add_task = self.task_manager.add_task
tasks_check_period = 10
added_tasks = 0
changed = False
for node in nodes:
if not build_manager.lock_node(node):
continue
node.initiate()
vfile = vfiles[node.builder]
prebuit_nodes = node.prebuild()
if prebuit_nodes:
build_manager.depends(node, prebuit_nodes)
changed = True
continue
split_nodes = node.build_split(vfile, explain)
if split_nodes:
build_manager.depends(node, split_nodes)
# split nodes are not actual, check them for building conflicts
for split_node in split_nodes:
self._add_building_node(split_node)
changed = True
continue
actual = node.check_actual(vfile, explain)
if actual:
build_manager.actual_node(node)
changed = True
continue
if not self._add_building_node(node):
continue
add_task(-node.get_weight(), node, _build_node, node)
added_tasks += 1
if added_tasks == tasks_check_period:
changed = self._get_finished_nodes(block=False) or changed
added_tasks = 0
return self._get_finished_nodes(block=not changed)
# -----------------------------------------------------------
def _get_finished_nodes(self, block=True):
finished_tasks = self.task_manager.get_finished_tasks(block=block)
vfiles = self.vfiles
build_manager = self.build_manager
for task in finished_tasks:
node = task.task_id
error = task.error
self._remove_building_node(node)
vfile = vfiles[node.builder]
if error is None:
node.save(vfile)
build_manager.completed_node(node, task.result)
else:
node.save_failed(vfile)
build_manager.failed_node(node, error)
# return false when there are no more task processing threads
return finished_tasks or not block
# -----------------------------------------------------------
def clear(self, nodes):
vfiles = self.vfiles
build_manager = self.build_manager
remove_entities = {}
for node in nodes:
node.initiate()
prebuit_nodes = node.prebuild()
if prebuit_nodes:
build_manager.depends(node, prebuit_nodes)
continue
vfile = vfiles[node.builder]
node_entities = node.clear(vfile)
remove_entities.setdefault(vfile, []).extend(node_entities)
build_manager.removed_node(node)
for vfile, entities in remove_entities.items():
vfile.remove_node_entities(entities)
# -----------------------------------------------------------
def close(self):
try:
self.task_manager.stop()
self._get_finished_nodes(block=False)
finally:
self.vfiles.close()