def test_task_manager_abort_on_fail(self):
num_tasks = 8
jobs = 4
tm = TaskManager()
tm.disable_keep_going()
results = set()
for i in range(jobs - 1):
tm.add_task(0, i, _do_append, i, results, 0.5)
tm.add_task(1, jobs - 1, _do_fail, 0.1)
for i in range(jobs, num_tasks):
tm.add_task(2, i, _do_append, i, results, 0)
tm.start(jobs)
done_tasks = sorted(self.get_done_tasks(tm), key=lambda t: t.task_id)
self.assertEqual(len(done_tasks), jobs)
items = zip(range(jobs), [None] * num_tasks, [None] * num_tasks)
expected_tasks = [
TaskResult(task_id, error, result)
for task_id, error, result in items
]
self.assertEqual(done_tasks[:3], expected_tasks[:3])
self.assertEqual(done_tasks[3].task_id, 3)
self.assertIsNotNone(done_tasks[3].error)
def test_tm_expensive_stop(self):
expensive_event = threading.Event()
fail_event = threading.Event()
num_tasks = 4
jobs = 16
tm = TaskManager()
tm.disable_keep_going()
for i in range(num_tasks):
tm.add_task(0, i, _do_non_expensive, expensive_event)
tm.add_task(0, num_tasks, _do_fail, 1, fail_event)
tm.start(jobs)
fail_event.wait()
tm.add_expensive_task(num_tasks + 1, _do_expensive, expensive_event)
results = self.get_done_tasks(tm)
self.assertEqual(len(results), num_tasks + 1)
task_ids = sorted(result.task_id for result in results)
self.assertEqual(task_ids, list(range(num_tasks + 1)))
def test_tm_expensive_stop(self):
expensive_event = threading.Event()
fail_event = threading.Event()
num_tasks = 4
jobs = 16
tm = TaskManager()
tm.disable_keep_going()
for i in range(num_tasks):
tm.add_task(0, i, _do_non_expensive, expensive_event)
tm.add_task(0, num_tasks, _do_fail, 1, fail_event)
tm.start(jobs)
fail_event.wait()
tm.add_expensive_task(num_tasks + 1, _do_expensive, expensive_event)
results = self.get_done_tasks(tm)
self.assertEqual(len(results), num_tasks + 1)
task_ids = sorted(result.task_id for result in results)
self.assertEqual(task_ids, list(range(num_tasks + 1)))
def test_task_manager_abort_on_fail(self):
num_tasks = 8
jobs = 4
tm = TaskManager()
tm.disable_keep_going()
results = set()
for i in range(jobs - 1):
tm.add_task(0, i, _do_append, i, results, 0.5)
tm.add_task(1, jobs - 1, _do_fail, 0.1)
for i in range(jobs, num_tasks):
tm.add_task(2, i, _do_append, i, results, 0)
tm.start(jobs)
done_tasks = sorted(self.get_done_tasks(tm), key=lambda t: t.task_id)
self.assertEqual(len(done_tasks), jobs)
items = zip(range(jobs), [None] * num_tasks, [None] * num_tasks)
expected_tasks = [TaskResult(task_id, error, result)
for task_id, error, result in items]
self.assertEqual(done_tasks[:3], expected_tasks[:3])
self.assertEqual(done_tasks[3].task_id, 3)
self.assertIsNotNone(done_tasks[3].error)
def __init__(self,
build_manager,
jobs=0,
keep_going=False,
with_backtrace=True,
use_sqlite=False,
force_lock=False):
self.vfiles = _VFiles(use_sqlite=use_sqlite, force_lock=force_lock)
self.building_nodes = {}
self.expensive_nodes = set(build_manager._expensive_nodes)
self.build_manager = build_manager
tm = TaskManager()
if self.expensive_nodes:
tm.enable_expensive()
if not keep_going:
tm.disable_keep_going()
if not with_backtrace:
tm.disable_backtrace()
tm.start(jobs)
self.task_manager = tm
def __init__(self, build_manager,
jobs=0, keep_going=False, with_backtrace=True,
use_sqlite=False, force_lock=False):
self.vfiles = _VFiles(use_sqlite=use_sqlite, force_lock=force_lock)
self.building_nodes = {}
self.expensive_nodes = set(build_manager._expensive_nodes)
self.build_manager = build_manager
tm = TaskManager()
if self.expensive_nodes:
tm.enable_expensive()
if not keep_going:
tm.disable_keep_going()
if not with_backtrace:
tm.disable_backtrace()
tm.start(jobs)
self.task_manager = tm
def test_tm_expensive_success(self):
expensive_event = threading.Event()
num_tasks = 8
jobs = 16
tm = TaskManager()
tm.disable_keep_going()
for i in range(num_tasks):
tm.add_task(0, i, _do_non_expensive, expensive_event)
tm.start(jobs)
time.sleep(0.25)
expensive_task_ids = set()
for i in range(num_tasks, num_tasks * 2):
if i % 2:
expensive_task_ids.add(i)
tm.add_expensive_task(i, _do_expensive, expensive_event)
else:
tm.add_task(0, i, _do_non_expensive, expensive_event)
results = self.get_done_tasks(tm)
for result in results:
self.assertFalse(result.is_failed(), str(result))
self.assertEqual(len(results), num_tasks * 2)
task_ids = [result.task_id for result in results]
finished_expensive_tasks = set(task_ids[-len(expensive_task_ids):])
self.assertEqual(finished_expensive_tasks, expensive_task_ids)
def test_tm_expensive_success(self):
expensive_event = threading.Event()
num_tasks = 8
jobs = 16
tm = TaskManager()
tm.disable_keep_going()
for i in range(num_tasks):
tm.add_task(0, i, _do_non_expensive, expensive_event)
tm.start(jobs)
time.sleep(0.25)
expensive_task_ids = set()
for i in range(num_tasks, num_tasks * 2):
if i % 2:
expensive_task_ids.add(i)
tm.add_expensive_task(i, _do_expensive, expensive_event)
else:
tm.add_task(0, i, _do_non_expensive, expensive_event)
results = self.get_done_tasks(tm)
for result in results:
self.assertFalse(result.is_failed(), str(result))
self.assertEqual(len(results), num_tasks * 2)
task_ids = [result.task_id for result in results]
finished_expensive_tasks = set(task_ids[-len(expensive_task_ids):])
self.assertEqual(finished_expensive_tasks, expensive_task_ids)