def test_task_manager_stop_on_fail(self):
tm = TaskManager( 4, stop_on_fail = True )
results = set()
num_tasks = 8
for i in range(3):
tm.addTask( i, _doAppend, i, results, 0.3 )
tm.addTask( i + 1, _doFail, 0.1 )
for i in range(i + 2,num_tasks):
tm.addTask( i, _doAppend, i, results, 0 )
time.sleep(1)
done_tasks = sorted( tm.finishedTasks(), key=lambda t: t.task_id )
self.assertEqual( len(done_tasks), 4 )
expected_tasks = [ TaskResult( task_id, error, result ) \
for task_id, error, result in zip(range(4), [None] * num_tasks, [None] * num_tasks ) ]
self.assertEqual( done_tasks[:3], expected_tasks[:3] )
self.assertEqual( done_tasks[3].task_id, 3 )
self.assertIsNotNone( done_tasks[3].error )
def test_task_manager_one_fail(self):
tm = TaskManager( 4 )
results = set()
num_tasks = 3
tm.addTask( 0, _doAppend, 0, results, 0.3 )
tm.addTask( 1, _doFail, 0.1 )
tm.addTask( 2, _doAppend, 2, results, 0 )
time.sleep(1)
done_tasks = sorted( tm.finishedTasks(), key= lambda v: v.task_id )
self.assertEqual( len(done_tasks), num_tasks )
expected_tasks = [ TaskResult( task_id, error, result ) \
for task_id, error, result in zip(range(num_tasks), [None] * num_tasks, [None] * num_tasks ) ]
self.assertEqual( done_tasks[0], expected_tasks[0] )
self.assertEqual( done_tasks[1].task_id, 1 )
self.assertIsNotNone( done_tasks[1].error )
self.assertEqual( done_tasks[2], expected_tasks[2] )
def test_task_manager_fail(self):
tm = TaskManager( num_threads = 4 )
num_of_tasks = 100
for i in range(num_of_tasks):
tm.addTask( i, _doFail, 0.0 )
time.sleep(1) # wait until all tasks are done
done_tasks = tm.finishedTasks()
self.assertEqual( len(done_tasks), num_of_tasks )
for i, t in enumerate( sorted(done_tasks) ):
self.assertEqual( t.task_id, i )
self.assertIsNotNone( t.error )
def test_task_manager(self):
tm = TaskManager( 4 )
results = set()
num_of_tasks = 8
for i in range(0,num_of_tasks):
tm.addTask( i, _doAppend, i, results )
time.sleep(0.5) # wait until all tasks are done
done_tasks = tm.finishedTasks()
expected_tasks = [ TaskResult( task_id ) for task_id in range(num_of_tasks) ]
self.assertEqual( sorted(expected_tasks), expected_tasks )
self.assertEqual( results, set(range(num_of_tasks)) )
def test_task_manager_finish(self):
tm = TaskManager( 1 )
results = set()
num_tasks = 8
for i in range(num_tasks):
tm.addTask( i, _doAppend, i, results, 0.2 )
tm.finish()
for i in range(num_tasks, num_tasks * 2):
tm.addTask( i, _doAppend, i, results, 0.2 )
self.assertEqual( results, set(range(num_tasks)) )
done_tasks = tm.finishedTasks()
expected_tasks = [ TaskResult( task_id ) for task_id in range(num_tasks) ]
self.assertEqual( sorted(done_tasks), expected_tasks )
def test_task_manager_stop(self):
jobs = 4
num_tasks = 8
tm = TaskManager( num_threads = jobs )
results = set()
for i in range(num_tasks):
tm.addTask( i, _doAppend, i, results, 1 )
time.sleep(0.2)
tm.stop()
tm.stop()
done_tasks = sorted( tm.finishedTasks(), key = lambda result: result.task_id )
expected_tasks = [ TaskResult( task_id ) for task_id in range(jobs) ]
self.assertEqual( sorted(done_tasks), expected_tasks )
self.assertEqual( results, set(range(jobs)) )
class _NodesBuilder (object):
__slots__ = \
(
'vfiles',
'build_manager',
'task_manager',
'node_states',
'building_nodes',
)
#//-------------------------------------------------------//
def __init__( self, build_manager, jobs = 0, keep_going = False, with_backtrace = True ):
self.vfiles = _VFiles()
self.node_states = {}
self.building_nodes = {}
self.build_manager = build_manager
self.task_manager = TaskManager( num_threads = jobs, stop_on_fail = not keep_going, with_backtrace = with_backtrace )
#//-------------------------------------------------------//
def __enter__(self):
return self
#//-------------------------------------------------------//
def __exit__(self, exc_type, exc_value, backtrace):
self.close()
#//-------------------------------------------------------//
def _getNodeState( self, node ):
try:
state = self.node_states[ node ]
except KeyError:
state = _NodeState()
self.node_states[ node ] = state
return state
#//-------------------------------------------------------//
def _removeNodeState( self, node ):
try:
del self.node_states[ node ]
except KeyError:
pass
#//-------------------------------------------------------//
def _addBuildingNode( self, node, state ):
conflicting_nodes = []
building_nodes = self.building_nodes
for name, signature in node.getNamesAndSignatures():
node_signature = (node, signature)
other_node, other_signature = building_nodes.setdefault( name, node_signature )
if other_node is not node:
if other_signature != signature:
raise ErrorNodeSignatureDifferent( node )
conflicting_nodes.append( other_node )
if conflicting_nodes:
state.check_actual = True
self.build_manager.depends( node, conflicting_nodes )
return True
return False
#//-------------------------------------------------------//
def _removeBuildingNode( self, node ):
building_nodes = self.building_nodes
for name in node.getNames():
del building_nodes[ name ]
#//-------------------------------------------------------//
def isBuilding(self):
return bool(self.building_nodes)
#//-------------------------------------------------------//
def _checkPrebuildDepends( self, node ):
dep_nodes = node.buildDepends()
if dep_nodes:
self.build_manager.depends( node, dep_nodes )
return True
return False
#//-------------------------------------------------------//
def _checkPrebuildReplace( self, node ):
if node.buildReplace():
new_node_sources = node.getSourceNodes()
if new_node_sources:
self.build_manager.depends( node, new_node_sources )
return True
return False
#//-------------------------------------------------------//
def _checkPrebuildSplit( self, node, state ):
build_manager = self.build_manager
if state.check_split:
state.check_split = False
check_actual = True
if node.isBatch() and state.check_actual:
# Check for changed sources of BatchNode
vfile = self.vfiles[ node.builder ]
actual = build_manager.isActualNode( node, vfile )
if actual:
self._removeNodeState( node )
build_manager.actualNode( node )
return True
check_actual = False
split_nodes = node.buildSplit()
if split_nodes:
state.split_nodes = split_nodes
for split_node in split_nodes:
split_state = self._getNodeState( split_node )
split_state.check_split = False
split_state.check_depends = False
split_state.check_replace = False
split_state.check_actual = check_actual
split_state.initialized = split_node.builder is node.builder
self.build_manager.depends( node, split_nodes )
return True
elif state.split_nodes is not None:
if node.isBatch():
node._populateTargets()
else:
targets = []
for split_node in state.split_nodes:
targets += split_node.getTargetValues()
node.target_values = targets
self._removeNodeState( node )
self.build_manager.completedSplitNode( node )
return True
return False
#//-------------------------------------------------------//
def _prebuild( self, node, state ):
# print( "node: %s, state: %s" % (node, state))
if not state.initialized:
node.initiate()
state.initialized = True
if state.check_depends:
state.check_depends = False
if self._checkPrebuildDepends( node ):
return True
if state.check_replace:
state.check_replace = False
if self._checkPrebuildReplace( node ):
return True
if self._checkPrebuildSplit( node, state ):
return True
return False
#//-------------------------------------------------------//
def build( self, nodes ):
build_manager = self.build_manager
vfiles = self.vfiles
addTask = self.task_manager.addTask
tasks_check_period = 10
added_tasks = 0
changed = False
for node in nodes:
node_state = self._getNodeState( node )
if self._prebuild( node, node_state ):
changed = True
continue
if self._addBuildingNode( node, node_state ):
continue
if node_state.check_actual:
vfile = vfiles[ node.builder ]
actual = build_manager.isActualNode( node, vfile )
if actual:
self._removeNodeState( node )
self._removeBuildingNode( node )
build_manager.actualNode( node )
changed = True
continue
addTask( node, _buildNode, node )
added_tasks += 1
if added_tasks == tasks_check_period:
changed = self._getFinishedNodes( block = False ) or changed
added_tasks = 0
self._getFinishedNodes( block = not changed )
#//-------------------------------------------------------//
def _getFinishedNodes( self, block = True ):
# print("tasks: %s, finished_tasks: %s" % (self.task_manager.unfinished_tasks, self.task_manager.finished_tasks.qsize()))
finished_tasks = self.task_manager.finishedTasks( block = block )
vfiles = self.vfiles
build_manager = self.build_manager
for task in finished_tasks:
node = task.task_id
error = task.error
self._removeNodeState( node )
self._removeBuildingNode( node )
vfile = vfiles[ node.builder ]
if error is None:
node.save( vfile )
build_manager.completedNode( node, task.result )
else:
if node.isBatch():
node.save( vfile )
build_manager.failedNode( node, error )
return bool(finished_tasks)
#//-------------------------------------------------------//
def clear( self, nodes ):
vfiles = self.vfiles
build_manager = self.build_manager
for node in nodes:
node_state = self._getNodeState( node )
node_state.check_actual = False
if self._prebuild( node, node_state ):
continue
vfile = vfiles[ node.builder ]
node.clear( vfile )
build_manager.removedNode( node )
#//-------------------------------------------------------//
def status( self, nodes ):
vfiles = self.vfiles
build_manager = self.build_manager
for node in nodes:
node_state = self._getNodeState( node )
node_state.check_actual = False
if self._prebuild( node, node_state ):
continue
vfile = vfiles[ node.builder ]
if build_manager.isActualNode( node, vfile ):
build_manager.actualNodeStatus( node )
else:
build_manager.outdatedNodeStatus( node )
#//-------------------------------------------------------//
def close( self ):
try:
self.task_manager.stop()
self._getFinishedNodes( block = False )
finally:
self.vfiles.close()