def spawn_and_publish(self, spawns, refs, producer=None, taskset=None):
producer_task = None
if producer is not None:
producer_task = self.get_task(producer["task_id"])
taskset = producer_task.taskset
upd = TaskGraphUpdate()
for spawn in spawns:
task_object = build_taskpool_task_from_descriptor(spawn, producer_task, taskset)
upd.spawn(task_object)
for ref in refs:
upd.publish(ref, producer_task)
upd.commit(self)
def _report_tasks(self, report, toplevel_task, worker):
# XXX SOS22 This does a damn sight more than just reporting the tasks!
with self._lock:
tx = TaskGraphUpdate()
root_task = self.task_graph.get_task(report[0][0])
ciel.log('Received report from task %s with %d entries' % (root_task.task_id, len(report)), 'SCHED', logging.DEBUG)
try:
self.workers[worker].deassign_task(root_task)
except KeyError:
# This can happen if we recieve the report after the worker is deemed to have failed. In this case, we should
# accept the report and ignore the failed worker.
pass
for (parent_id, success, payload) in report:
ciel.log('Processing report record from task %s' % (parent_id), 'SCHED', logging.DEBUG)
parent_task = self.task_graph.get_task(parent_id)
if success:
ciel.log('Task %s was successful' % (parent_id), 'SCHED', logging.DEBUG)
(spawned, published, profiling) = payload
parent_task.set_profiling(profiling)
parent_task.set_state(TASK_COMMITTED)
self.record_task_stats(parent_task, worker)
for child in spawned:
child_task = build_taskpool_task_from_descriptor(child, parent_task)
ciel.log('Task %s spawned task %s' % (parent_id, child_task.task_id), 'SCHED', logging.DEBUG)
tx.spawn(child_task)
#parent_task.children.append(child_task)
for ref in published:
ciel.log('Task %s published reference %s' % (parent_id, str(ref)), 'SCHED', logging.DEBUG)
tx.publish(ref, parent_task)
else:
ciel.log('Task %s failed' % (parent_id), 'SCHED', logging.WARN)
# Only one failed task per-report, at the moment.
self.investigate_task_failure(parent_task, payload)
self.schedule()
return
tx.commit(self.task_graph)
self.task_graph.reduce_graph_for_references(toplevel_task.expected_outputs)
# XXX: Need to remove assigned task from worker(s).
self.schedule()
def spawn_and_publish(self, spawns, refs, producer=None, taskset=None):
producer_task = None
if producer is not None:
producer_task = self.get_task(producer["task_id"])
taskset = producer_task.taskset
upd = TaskGraphUpdate()
for spawn in spawns:
task_object = build_taskpool_task_from_descriptor(
spawn, producer_task, taskset)
upd.spawn(task_object)
for ref in refs:
upd.publish(ref, producer_task)
upd.commit(self)
def _report_tasks(self, report, toplevel_task, worker):
with self._lock:
tx = TaskGraphUpdate()
root_task = self.task_graph.get_task(report[0][0])
ciel.log(
'Received report from task %s with %d entries' %
(root_task.task_id, len(report)), 'SCHED', logging.DEBUG)
try:
self.workers[worker].deassign_task(root_task)
except KeyError:
# This can happen if we recieve the report after the worker is deemed to have failed. In this case, we should
# accept the report and ignore the failed worker.
pass
for (parent_id, success, payload) in report:
ciel.log('Processing report record from task %s' % (parent_id),
'SCHED', logging.DEBUG)
parent_task = self.task_graph.get_task(parent_id)
if success:
ciel.log('Task %s was successful' % (parent_id), 'SCHED',
logging.DEBUG)
(spawned, published, profiling) = payload
parent_task.set_profiling(profiling)
parent_task.set_state(TASK_COMMITTED)
self.record_task_stats(parent_task, worker)
for child in spawned:
child_task = build_taskpool_task_from_descriptor(
child, parent_task)
ciel.log(
'Task %s spawned task %s' %
(parent_id, child_task.task_id), 'SCHED',
logging.DEBUG)
tx.spawn(child_task)
#parent_task.children.append(child_task)
for ref in published:
ciel.log(
'Task %s published reference %s' %
(parent_id, str(ref)), 'SCHED', logging.DEBUG)
tx.publish(ref, parent_task)
else:
ciel.log('Task %s failed' % (parent_id), 'SCHED',
logging.WARN)
# Only one failed task per-report, at the moment.
self.investigate_task_failure(parent_task, payload)
self.schedule()
return
tx.commit(self.task_graph)
self.task_graph.reduce_graph_for_references(
toplevel_task.expected_outputs)
# XXX: Need to remove assigned task from worker(s).
self.schedule()
