def test_task_manager_stop_on_fail(self):
num_tasks = 8
jobs = 4
tm = TaskManager(0, stop_on_fail=True)
results = set()
for i in range(jobs - 1):
tm.add_task(0, i, _do_append, i, results, 0.5)
tm.add_task(1, jobs - 1, _do_fail, 0.1)
for i in range(jobs, num_tasks):
tm.add_task(2, i, _do_append, i, results, 0)
tm.start(jobs)
time.sleep(1)
done_tasks = sorted(tm.get_finished_tasks(), key=lambda t: t.task_id)
print()
self.assertEqual(len(done_tasks), jobs)
items = zip(range(jobs), [None] * num_tasks, [None] * num_tasks)
expected_tasks = [TaskResult(task_id, error, result)
for task_id, error, result in items]
self.assertEqual(done_tasks[:3], expected_tasks[:3])
self.assertEqual(done_tasks[3].task_id, 3)
self.assertIsNotNone(done_tasks[3].error)
def __init__(self,
build_manager,
jobs=0,
keep_going=False,
with_backtrace=True,
use_sqlite=False,
force_lock=False):
self.vfiles = _VFiles(use_sqlite=use_sqlite, force_lock=force_lock)
self.building_nodes = {}
self.expensive_nodes = set(build_manager._expensive_nodes)
self.build_manager = build_manager
tm = TaskManager()
if self.expensive_nodes:
tm.enable_expensive()
if not keep_going:
tm.disable_keep_going()
if not with_backtrace:
tm.disable_backtrace()
tm.start(jobs)
self.task_manager = tm
def test_task_manager_one_fail(self):
tm = TaskManager()
tm.start(4)
results = set()
num_tasks = 3
tm.add_task(0, 0, _do_append, 0, results, 0.3)
tm.add_task(0, 1, _do_fail, 0.1)
tm.add_task(0, 2, _do_append, 2, results, 0)
done_tasks = sorted(self.get_done_tasks(tm), key=lambda v: v.task_id)
self.assertEqual(len(done_tasks), num_tasks)
items = zip(range(num_tasks), [None] * num_tasks, [None] * num_tasks)
expected_tasks = [
TaskResult(task_id, error, result)
for task_id, error, result in items
]
self.assertEqual(done_tasks[0], expected_tasks[0])
self.assertEqual(done_tasks[1].task_id, 1)
self.assertIsNotNone(done_tasks[1].error)
self.assertEqual(done_tasks[2], expected_tasks[2])
def test_task_manager_one_fail(self):
tm = TaskManager()
tm.start(4)
results = set()
num_tasks = 3
tm.add_task(0, 0, _do_append, 0, results, 0.3)
tm.add_task(0, 1, _do_fail, 0.1)
tm.add_task(0, 2, _do_append, 2, results, 0)
done_tasks = sorted(self.get_done_tasks(tm), key=lambda v: v.task_id)
self.assertEqual(len(done_tasks), num_tasks)
items = zip(range(num_tasks), [None] * num_tasks, [None] * num_tasks)
expected_tasks = [TaskResult(task_id, error, result)
for task_id, error, result in items]
self.assertEqual(done_tasks[0], expected_tasks[0])
self.assertEqual(done_tasks[1].task_id, 1)
self.assertIsNotNone(done_tasks[1].error)
self.assertEqual(done_tasks[2], expected_tasks[2])
def test_task_manager_one_fail(self):
tm = TaskManager( 4 )
results = set()
num_tasks = 3
tm.addTask( 0, _doAppend, 0, results, 0.3 )
tm.addTask( 1, _doFail, 0.1 )
tm.addTask( 2, _doAppend, 2, results, 0 )
time.sleep(1)
done_tasks = sorted( tm.finishedTasks(), key= lambda v: v.task_id )
self.assertEqual( len(done_tasks), num_tasks )
expected_tasks = [ TaskResult( task_id, error, result ) \
for task_id, error, result in zip(range(num_tasks), [None] * num_tasks, [None] * num_tasks ) ]
self.assertEqual( done_tasks[0], expected_tasks[0] )
self.assertEqual( done_tasks[1].task_id, 1 )
self.assertIsNotNone( done_tasks[1].error )
self.assertEqual( done_tasks[2], expected_tasks[2] )
def test_task_manager_stop_on_fail(self):
tm = TaskManager( 4, stop_on_fail = True )
results = set()
num_tasks = 8
for i in range(3):
tm.addTask( i, _doAppend, i, results, 0.3 )
tm.addTask( i + 1, _doFail, 0.1 )
for i in range(i + 2,num_tasks):
tm.addTask( i, _doAppend, i, results, 0 )
time.sleep(1)
done_tasks = sorted( tm.finishedTasks(), key=lambda t: t.task_id )
self.assertEqual( len(done_tasks), 4 )
expected_tasks = [ TaskResult( task_id, error, result ) \
for task_id, error, result in zip(range(4), [None] * num_tasks, [None] * num_tasks ) ]
self.assertEqual( done_tasks[:3], expected_tasks[:3] )
self.assertEqual( done_tasks[3].task_id, 3 )
self.assertIsNotNone( done_tasks[3].error )
def test_task_manager(self):
tm = TaskManager( 4 )
results = set()
num_of_tasks = 8
for i in range(0,num_of_tasks):
tm.addTask( i, _doAppend, i, results )
time.sleep(0.5) # wait until all tasks are done
done_tasks = tm.finishedTasks()
expected_tasks = [ TaskResult( task_id ) for task_id in range(num_of_tasks) ]
self.assertEqual( sorted(expected_tasks), expected_tasks )
self.assertEqual( results, set(range(num_of_tasks)) )
def test_task_manager_fail(self):
tm = TaskManager( num_threads = 4 )
num_of_tasks = 100
for i in range(num_of_tasks):
tm.addTask( i, _doFail, 0.0 )
time.sleep(1) # wait until all tasks are done
done_tasks = tm.finishedTasks()
self.assertEqual( len(done_tasks), num_of_tasks )
for i, t in enumerate( sorted(done_tasks) ):
self.assertEqual( t.task_id, i )
self.assertIsNotNone( t.error )
def test_task_manager(self):
jobs = 1
tm = TaskManager()
results = set()
num_of_tasks = 8
for i in range(num_of_tasks):
tm.add_task(num_of_tasks - i, i, _do_append, i, results)
tm.start(jobs)
done_tasks = [result.task_id for result in self.get_done_tasks(tm)]
expected_tasks = sorted(range(num_of_tasks), reverse=True)
self.assertEqual(done_tasks, expected_tasks)
self.assertEqual(results, set(range(num_of_tasks)))
def __init__(self,
build_manager,
jobs=0,
keep_going=False,
with_backtrace=True,
use_sqlite=False,
force_lock=False
):
self.vfiles = _VFiles(use_sqlite=use_sqlite, force_lock=force_lock)
self.building_nodes = {}
self.build_manager = build_manager
self.task_manager = TaskManager(num_threads=jobs,
stop_on_fail=not keep_going,
with_backtrace=with_backtrace)
def __init__(self, build_manager,
jobs=0, keep_going=False, with_backtrace=True,
use_sqlite=False, force_lock=False):
self.vfiles = _VFiles(use_sqlite=use_sqlite, force_lock=force_lock)
self.building_nodes = {}
self.expensive_nodes = set(build_manager._expensive_nodes)
self.build_manager = build_manager
tm = TaskManager()
if self.expensive_nodes:
tm.enable_expensive()
if not keep_going:
tm.disable_keep_going()
if not with_backtrace:
tm.disable_backtrace()
tm.start(jobs)
self.task_manager = tm
def test_task_manager_fail(self):
jobs = 4
tm = TaskManager()
tm.start(jobs)
num_of_tasks = 100
for i in range(num_of_tasks):
tm.add_task(0, i, _do_fail, 0.0)
tm.start(jobs)
done_tasks = self.get_done_tasks(tm)
self.assertEqual(len(done_tasks), num_of_tasks)
for i, t in enumerate(sorted(done_tasks)):
self.assertEqual(t.task_id, i)
self.assertIsNotNone(t.error)
def test_task_manager_finish(self):
tm = TaskManager( 1 )
results = set()
num_tasks = 8
for i in range(num_tasks):
tm.addTask( i, _doAppend, i, results, 0.2 )
tm.finish()
for i in range(num_tasks, num_tasks * 2):
tm.addTask( i, _doAppend, i, results, 0.2 )
self.assertEqual( results, set(range(num_tasks)) )
done_tasks = tm.finishedTasks()
expected_tasks = [ TaskResult( task_id ) for task_id in range(num_tasks) ]
self.assertEqual( sorted(done_tasks), expected_tasks )
def test_task_manager_fail(self):
jobs = 4
tm = TaskManager()
tm.start(jobs)
num_of_tasks = 100
for i in range(num_of_tasks):
tm.add_task(0, i, _do_fail, 0.0)
tm.start(jobs)
done_tasks = self.get_done_tasks(tm)
self.assertEqual(len(done_tasks), num_of_tasks)
for i, t in enumerate(sorted(done_tasks)):
self.assertEqual(t.task_id, i)
self.assertIsNotNone(t.error)
def test_task_manager(self):
jobs = 1
tm = TaskManager()
results = set()
num_of_tasks = 8
for i in range(num_of_tasks):
tm.add_task(num_of_tasks - i, i, _do_append, i, results)
tm.start(jobs)
done_tasks = [result.task_id for result in self.get_done_tasks(tm)]
expected_tasks = sorted(range(num_of_tasks), reverse=True)
self.assertEqual(done_tasks, expected_tasks)
self.assertEqual(results, set(range(num_of_tasks)))
def test_task_manager(self):
jobs = 1
tm = TaskManager(0)
results = set()
num_of_tasks = 8
for i in range(num_of_tasks):
tm.add_task(num_of_tasks - i, i, _do_append, i, results)
tm.start(jobs)
time.sleep(0.5) # wait until all tasks are done
done_tasks = [result.task_id for result in tm.get_finished_tasks()]
expected_tasks = sorted(range(num_of_tasks), reverse=True)
self.assertEqual(done_tasks, expected_tasks)
self.assertEqual(results, set(range(num_of_tasks)))
def test_task_manager_stop(self):
jobs = 4
num_tasks = 8
tm = TaskManager(num_threads=jobs)
results = set()
for i in range(num_tasks):
tm.add_task(0, i, _do_append, i, results, 1)
time.sleep(0.2)
tm.stop()
tm.stop()
done_tasks = sorted(result.task_id
for result in tm.get_finished_tasks())
self.assertEqual(len(done_tasks), jobs)
self.assertEqual(results, set(done_tasks))
def test_task_manager_fail(self):
jobs = 4
tm = TaskManager(jobs)
num_of_tasks = 100
for i in range(num_of_tasks):
tm.add_task(0, i, _do_fail, 0.0)
tm.start(jobs)
time.sleep(0.5) # wait until all tasks are done
done_tasks = tm.get_finished_tasks()
self.assertEqual(len(done_tasks), num_of_tasks)
for i, t in enumerate(sorted(done_tasks)):
self.assertEqual(t.task_id, i)
self.assertIsNotNone(t.error)
def test_task_manager_stop(self):
jobs = 4
num_tasks = 8
tm = TaskManager( num_threads = jobs )
results = set()
for i in range(num_tasks):
tm.addTask( i, _doAppend, i, results, 1 )
time.sleep(0.2)
tm.stop()
tm.stop()
done_tasks = sorted( tm.finishedTasks(), key = lambda result: result.task_id )
expected_tasks = [ TaskResult( task_id ) for task_id in range(jobs) ]
self.assertEqual( sorted(done_tasks), expected_tasks )
self.assertEqual( results, set(range(jobs)) )
def test_tm_expensive_success(self):
expensive_event = threading.Event()
num_tasks = 8
jobs = 16
tm = TaskManager()
tm.disable_keep_going()
for i in range(num_tasks):
tm.add_task(0, i, _do_non_expensive, expensive_event)
tm.start(jobs)
time.sleep(0.25)
expensive_task_ids = set()
for i in range(num_tasks, num_tasks * 2):
if i % 2:
expensive_task_ids.add(i)
tm.add_expensive_task(i, _do_expensive, expensive_event)
else:
tm.add_task(0, i, _do_non_expensive, expensive_event)
results = self.get_done_tasks(tm)
for result in results:
self.assertFalse(result.is_failed(), str(result))
self.assertEqual(len(results), num_tasks * 2)
task_ids = [result.task_id for result in results]
finished_expensive_tasks = set(task_ids[-len(expensive_task_ids):])
self.assertEqual(finished_expensive_tasks, expensive_task_ids)
def test_tm_expensive_keep_going(self):
expensive_event = threading.Event()
num_tasks = 8
jobs = 16
tm = TaskManager()
tm.disable_backtrace()
for i in range(num_tasks):
tm.add_task(0, i, _do_non_expensive, expensive_event)
tm.add_task(0, num_tasks, _do_fail, 1)
tm.start(jobs)
time.sleep(0.25)
tm.add_expensive_task(num_tasks + 1, _do_expensive, expensive_event)
results = self.get_done_tasks(tm)
self.assertEqual(len(results), num_tasks + 2)
def test_task_manager_stop(self):
jobs = 4
num_tasks = 8
tm = TaskManager()
tm.start(jobs)
tm.stop()
tm.start(jobs)
results = set()
for i in range(num_tasks):
tm.add_task(0, i, _do_append, i, results, 0.2 * (i + 1))
time.sleep(0.1)
tm.stop()
tm.stop()
done_tasks = sorted(result.task_id
for result in self.get_done_tasks(tm))
self.assertEqual(len(done_tasks), jobs)
self.assertEqual(results, set(done_tasks))
tm.start(jobs)
done_tasks = sorted(result.task_id
for result in self.get_done_tasks(tm))
self.assertEqual(len(done_tasks), max(0, num_tasks - jobs))
tm.stop()
tm.start(jobs)
for i in range(jobs):
tm.add_task(0, i, _do_append, i, results, 0.2 * i)
self.get_done_tasks(tm)
tm.stop()
def test_task_manager_stop(self):
jobs = 4
num_tasks = 8
tm = TaskManager()
tm.start(jobs)
tm.stop()
tm.start(jobs)
results = set()
for i in range(num_tasks):
tm.add_task(0, i, _do_append, i, results, 0.2 * (i + 1))
time.sleep(0.1)
tm.stop()
tm.stop()
done_tasks = sorted(result.task_id
for result in self.get_done_tasks(tm))
self.assertEqual(len(done_tasks), jobs)
self.assertEqual(results, set(done_tasks))
tm.start(jobs)
done_tasks = sorted(result.task_id
for result in self.get_done_tasks(tm))
self.assertEqual(len(done_tasks), max(0, num_tasks - jobs))
tm.stop()
tm.start(jobs)
for i in range(jobs):
tm.add_task(0, i, _do_append, i, results, 0.2 * i)
self.get_done_tasks(tm)
tm.stop()
def test_task_manager_abort_on_fail(self):
num_tasks = 8
jobs = 4
tm = TaskManager()
tm.disable_keep_going()
results = set()
for i in range(jobs - 1):
tm.add_task(0, i, _do_append, i, results, 0.5)
tm.add_task(1, jobs - 1, _do_fail, 0.1)
for i in range(jobs, num_tasks):
tm.add_task(2, i, _do_append, i, results, 0)
tm.start(jobs)
done_tasks = sorted(self.get_done_tasks(tm), key=lambda t: t.task_id)
self.assertEqual(len(done_tasks), jobs)
items = zip(range(jobs), [None] * num_tasks, [None] * num_tasks)
expected_tasks = [TaskResult(task_id, error, result)
for task_id, error, result in items]
self.assertEqual(done_tasks[:3], expected_tasks[:3])
self.assertEqual(done_tasks[3].task_id, 3)
self.assertIsNotNone(done_tasks[3].error)
def __init__( self, build_manager, jobs = 0, keep_going = False, with_backtrace = True ):
self.vfiles = _VFiles()
self.node_states = {}
self.building_nodes = {}
self.build_manager = build_manager
self.task_manager = TaskManager( num_threads = jobs, stop_on_fail = not keep_going, with_backtrace = with_backtrace )
def test_tm_expensive_stop(self):
expensive_event = threading.Event()
fail_event = threading.Event()
num_tasks = 4
jobs = 16
tm = TaskManager()
tm.disable_keep_going()
for i in range(num_tasks):
tm.add_task(0, i, _do_non_expensive, expensive_event)
tm.add_task(0, num_tasks, _do_fail, 1, fail_event)
tm.start(jobs)
fail_event.wait()
tm.add_expensive_task(num_tasks + 1, _do_expensive, expensive_event)
results = self.get_done_tasks(tm)
self.assertEqual(len(results), num_tasks + 1)
task_ids = sorted(result.task_id for result in results)
self.assertEqual(task_ids, list(range(num_tasks + 1)))
def test_task_manager_abort_on_fail(self):
num_tasks = 8
jobs = 4
tm = TaskManager()
tm.disable_keep_going()
results = set()
for i in range(jobs - 1):
tm.add_task(0, i, _do_append, i, results, 0.5)
tm.add_task(1, jobs - 1, _do_fail, 0.1)
for i in range(jobs, num_tasks):
tm.add_task(2, i, _do_append, i, results, 0)
tm.start(jobs)
done_tasks = sorted(self.get_done_tasks(tm), key=lambda t: t.task_id)
self.assertEqual(len(done_tasks), jobs)
items = zip(range(jobs), [None] * num_tasks, [None] * num_tasks)
expected_tasks = [
TaskResult(task_id, error, result)
for task_id, error, result in items
]
self.assertEqual(done_tasks[:3], expected_tasks[:3])
self.assertEqual(done_tasks[3].task_id, 3)
self.assertIsNotNone(done_tasks[3].error)
def test_tm_expensive_success(self):
expensive_event = threading.Event()
num_tasks = 8
jobs = 16
tm = TaskManager()
tm.disable_keep_going()
for i in range(num_tasks):
tm.add_task(0, i, _do_non_expensive, expensive_event)
tm.start(jobs)
time.sleep(0.25)
expensive_task_ids = set()
for i in range(num_tasks, num_tasks * 2):
if i % 2:
expensive_task_ids.add(i)
tm.add_expensive_task(i, _do_expensive, expensive_event)
else:
tm.add_task(0, i, _do_non_expensive, expensive_event)
results = self.get_done_tasks(tm)
for result in results:
self.assertFalse(result.is_failed(), str(result))
self.assertEqual(len(results), num_tasks * 2)
task_ids = [result.task_id for result in results]
finished_expensive_tasks = set(task_ids[-len(expensive_task_ids):])
self.assertEqual(finished_expensive_tasks, expensive_task_ids)
class _NodesBuilder (object):
__slots__ = (
'vfiles',
'build_manager',
'task_manager',
'building_nodes',
)
# -----------------------------------------------------------
def __init__(self,
build_manager,
jobs=0,
keep_going=False,
with_backtrace=True,
use_sqlite=False,
force_lock=False
):
self.vfiles = _VFiles(use_sqlite=use_sqlite, force_lock=force_lock)
self.building_nodes = {}
self.build_manager = build_manager
self.task_manager = TaskManager(num_threads=jobs,
stop_on_fail=not keep_going,
with_backtrace=with_backtrace)
# -----------------------------------------------------------
def __enter__(self):
return self
# -----------------------------------------------------------
def __exit__(self, exc_type, exc_value, backtrace):
self.close()
# -----------------------------------------------------------
def _add_building_node(self, node):
conflicting_nodes = []
building_nodes = self.building_nodes
node_names = {}
for name, signature in node.get_names_and_signatures():
other = building_nodes.get(name, None)
if other is None:
node_names[name] = (node, signature)
continue
other_node, other_signature = other
if node is other_node:
continue
if other_signature != signature:
raise ErrorNodeSignatureDifferent(node)
conflicting_nodes.append(other_node)
if conflicting_nodes:
node.recheck_actual()
self.build_manager.depends(node, conflicting_nodes)
return False
building_nodes.update(node_names)
return True
# -----------------------------------------------------------
def _remove_building_node(self, node):
building_nodes = self.building_nodes
for name in node.get_names():
del building_nodes[name]
# -----------------------------------------------------------
def is_building(self):
return bool(self.building_nodes)
# -----------------------------------------------------------
def build(self, nodes):
build_manager = self.build_manager
explain = build_manager.explain
vfiles = self.vfiles
add_task = self.task_manager.add_task
tasks_check_period = 10
added_tasks = 0
changed = False
for node in nodes:
if not build_manager.lock_node(node):
continue
node.initiate()
vfile = vfiles[node.builder]
prebuit_nodes = node.prebuild()
if prebuit_nodes:
build_manager.depends(node, prebuit_nodes)
changed = True
continue
split_nodes = node.build_split(vfile, explain)
if split_nodes:
build_manager.depends(node, split_nodes)
# split nodes are not actual, check them for building conflicts
for split_node in split_nodes:
self._add_building_node(split_node)
changed = True
continue
actual = node.check_actual(vfile, explain)
if actual:
build_manager.actual_node(node)
changed = True
continue
if not self._add_building_node(node):
continue
add_task(-node.get_weight(), node, _build_node, node)
added_tasks += 1
if added_tasks == tasks_check_period:
changed = self._get_finished_nodes(block=False) or changed
added_tasks = 0
return self._get_finished_nodes(block=not changed)
# -----------------------------------------------------------
def _get_finished_nodes(self, block=True):
finished_tasks = self.task_manager.get_finished_tasks(block=block)
vfiles = self.vfiles
build_manager = self.build_manager
for task in finished_tasks:
node = task.task_id
error = task.error
self._remove_building_node(node)
vfile = vfiles[node.builder]
if error is None:
node.save(vfile)
build_manager.completed_node(node, task.result)
else:
node.save_failed(vfile)
build_manager.failed_node(node, error)
# return false when there are no more task processing threads
return finished_tasks or not block
# -----------------------------------------------------------
def clear(self, nodes):
vfiles = self.vfiles
build_manager = self.build_manager
remove_entities = {}
for node in nodes:
node.initiate()
prebuit_nodes = node.prebuild()
if prebuit_nodes:
build_manager.depends(node, prebuit_nodes)
continue
vfile = vfiles[node.builder]
node_entities = node.clear(vfile)
remove_entities.setdefault(vfile, []).extend(node_entities)
build_manager.removed_node(node)
for vfile, entities in remove_entities.items():
vfile.remove_node_entities(entities)
# -----------------------------------------------------------
def close(self):
try:
self.task_manager.stop()
self._get_finished_nodes(block=False)
finally:
self.vfiles.close()
def test_tm_expensive_stop(self):
expensive_event = threading.Event()
fail_event = threading.Event()
num_tasks = 4
jobs = 16
tm = TaskManager()
tm.disable_keep_going()
for i in range(num_tasks):
tm.add_task(0, i, _do_non_expensive, expensive_event)
tm.add_task(0, num_tasks, _do_fail, 1, fail_event)
tm.start(jobs)
fail_event.wait()
tm.add_expensive_task(num_tasks + 1, _do_expensive, expensive_event)
results = self.get_done_tasks(tm)
self.assertEqual(len(results), num_tasks + 1)
task_ids = sorted(result.task_id for result in results)
self.assertEqual(task_ids, list(range(num_tasks + 1)))
class _NodesBuilder (object):
__slots__ = \
(
'vfiles',
'build_manager',
'task_manager',
'node_states',
'building_nodes',
)
#//-------------------------------------------------------//
def __init__( self, build_manager, jobs = 0, keep_going = False, with_backtrace = True ):
self.vfiles = _VFiles()
self.node_states = {}
self.building_nodes = {}
self.build_manager = build_manager
self.task_manager = TaskManager( num_threads = jobs, stop_on_fail = not keep_going, with_backtrace = with_backtrace )
#//-------------------------------------------------------//
def __enter__(self):
return self
#//-------------------------------------------------------//
def __exit__(self, exc_type, exc_value, backtrace):
self.close()
#//-------------------------------------------------------//
def _getNodeState( self, node ):
try:
state = self.node_states[ node ]
except KeyError:
state = _NodeState()
self.node_states[ node ] = state
return state
#//-------------------------------------------------------//
def _removeNodeState( self, node ):
try:
del self.node_states[ node ]
except KeyError:
pass
#//-------------------------------------------------------//
def _addBuildingNode( self, node, state ):
conflicting_nodes = []
building_nodes = self.building_nodes
for name, signature in node.getNamesAndSignatures():
node_signature = (node, signature)
other_node, other_signature = building_nodes.setdefault( name, node_signature )
if other_node is not node:
if other_signature != signature:
raise ErrorNodeSignatureDifferent( node )
conflicting_nodes.append( other_node )
if conflicting_nodes:
state.check_actual = True
self.build_manager.depends( node, conflicting_nodes )
return True
return False
#//-------------------------------------------------------//
def _removeBuildingNode( self, node ):
building_nodes = self.building_nodes
for name in node.getNames():
del building_nodes[ name ]
#//-------------------------------------------------------//
def isBuilding(self):
return bool(self.building_nodes)
#//-------------------------------------------------------//
def _checkPrebuildDepends( self, node ):
dep_nodes = node.buildDepends()
if dep_nodes:
self.build_manager.depends( node, dep_nodes )
return True
return False
#//-------------------------------------------------------//
def _checkPrebuildReplace( self, node ):
if node.buildReplace():
new_node_sources = node.getSourceNodes()
if new_node_sources:
self.build_manager.depends( node, new_node_sources )
return True
return False
#//-------------------------------------------------------//
def _checkPrebuildSplit( self, node, state ):
build_manager = self.build_manager
if state.check_split:
state.check_split = False
check_actual = True
if node.isBatch() and state.check_actual:
# Check for changed sources of BatchNode
vfile = self.vfiles[ node.builder ]
actual = build_manager.isActualNode( node, vfile )
if actual:
self._removeNodeState( node )
build_manager.actualNode( node )
return True
check_actual = False
split_nodes = node.buildSplit()
if split_nodes:
state.split_nodes = split_nodes
for split_node in split_nodes:
split_state = self._getNodeState( split_node )
split_state.check_split = False
split_state.check_depends = False
split_state.check_replace = False
split_state.check_actual = check_actual
split_state.initialized = split_node.builder is node.builder
self.build_manager.depends( node, split_nodes )
return True
elif state.split_nodes is not None:
if node.isBatch():
node._populateTargets()
else:
targets = []
for split_node in state.split_nodes:
targets += split_node.getTargetValues()
node.target_values = targets
self._removeNodeState( node )
self.build_manager.completedSplitNode( node )
return True
return False
#//-------------------------------------------------------//
def _prebuild( self, node, state ):
# print( "node: %s, state: %s" % (node, state))
if not state.initialized:
node.initiate()
state.initialized = True
if state.check_depends:
state.check_depends = False
if self._checkPrebuildDepends( node ):
return True
if state.check_replace:
state.check_replace = False
if self._checkPrebuildReplace( node ):
return True
if self._checkPrebuildSplit( node, state ):
return True
return False
#//-------------------------------------------------------//
def build( self, nodes ):
build_manager = self.build_manager
vfiles = self.vfiles
addTask = self.task_manager.addTask
tasks_check_period = 10
added_tasks = 0
changed = False
for node in nodes:
node_state = self._getNodeState( node )
if self._prebuild( node, node_state ):
changed = True
continue
if self._addBuildingNode( node, node_state ):
continue
if node_state.check_actual:
vfile = vfiles[ node.builder ]
actual = build_manager.isActualNode( node, vfile )
if actual:
self._removeNodeState( node )
self._removeBuildingNode( node )
build_manager.actualNode( node )
changed = True
continue
addTask( node, _buildNode, node )
added_tasks += 1
if added_tasks == tasks_check_period:
changed = self._getFinishedNodes( block = False ) or changed
added_tasks = 0
self._getFinishedNodes( block = not changed )
#//-------------------------------------------------------//
def _getFinishedNodes( self, block = True ):
# print("tasks: %s, finished_tasks: %s" % (self.task_manager.unfinished_tasks, self.task_manager.finished_tasks.qsize()))
finished_tasks = self.task_manager.finishedTasks( block = block )
vfiles = self.vfiles
build_manager = self.build_manager
for task in finished_tasks:
node = task.task_id
error = task.error
self._removeNodeState( node )
self._removeBuildingNode( node )
vfile = vfiles[ node.builder ]
if error is None:
node.save( vfile )
build_manager.completedNode( node, task.result )
else:
if node.isBatch():
node.save( vfile )
build_manager.failedNode( node, error )
return bool(finished_tasks)
#//-------------------------------------------------------//
def clear( self, nodes ):
vfiles = self.vfiles
build_manager = self.build_manager
for node in nodes:
node_state = self._getNodeState( node )
node_state.check_actual = False
if self._prebuild( node, node_state ):
continue
vfile = vfiles[ node.builder ]
node.clear( vfile )
build_manager.removedNode( node )
#//-------------------------------------------------------//
def status( self, nodes ):
vfiles = self.vfiles
build_manager = self.build_manager
for node in nodes:
node_state = self._getNodeState( node )
node_state.check_actual = False
if self._prebuild( node, node_state ):
continue
vfile = vfiles[ node.builder ]
if build_manager.isActualNode( node, vfile ):
build_manager.actualNodeStatus( node )
else:
build_manager.outdatedNodeStatus( node )
#//-------------------------------------------------------//
def close( self ):
try:
self.task_manager.stop()
self._getFinishedNodes( block = False )
finally:
self.vfiles.close()
def test_tm_expensive_keep_going(self):
expensive_event = threading.Event()
num_tasks = 8
jobs = 16
tm = TaskManager()
tm.disable_backtrace()
for i in range(num_tasks):
tm.add_task(0, i, _do_non_expensive, expensive_event)
tm.add_task(0, num_tasks, _do_fail, 1)
tm.start(jobs)
time.sleep(0.25)
tm.add_expensive_task(num_tasks + 1, _do_expensive, expensive_event)
results = self.get_done_tasks(tm)
self.assertEqual(len(results), num_tasks + 2)