def run_as_batch_worker(task_list, cli_args, kwargs):
found_task = False
for root_task in task_list:
for task in task_iterator(root_task):
if task.task_id != cli_args.task_id:
continue
found_task = True
set_setting("_dispatch_local_execution", True)
# TODO: We do not process the information if (a) we have a new dependency and (b) why the task has failed.
# TODO: Would be also nice to run the event handlers
try:
create_output_dirs(task)
task.run()
task.on_success()
except BaseException as ex:
task.on_failure(ex)
raise ex
return
if not found_task:
raise ValueError(f"The task id {task.task_id} to be executed by this batch worker "
f"does not exist in the locally reproduced task graph.")
def dry_run(task_list):
nonfinished_task_list = collections.defaultdict(set)
for root_task in task_list:
for task in task_iterator(root_task, only_non_complete=True):
nonfinished_task_list[task.__class__.__name__].add(task)
non_completed_tasks = 0
for task_class in sorted(nonfinished_task_list):
print(task_class)
for task in nonfinished_task_list[task_class]:
print("\tWould run", task)
# execute the dry_run method of the task if it is implemented
if hasattr(task, 'dry_run'):
print("\tcall: dry_run()")
task.dry_run()
print()
non_completed_tasks += 1
if non_completed_tasks:
print("In total", non_completed_tasks)
exit(1)
print("All tasks are finished!")
exit(0)
def dry_run(task_list):
nonfinished_task_list = collections.defaultdict(set)
for root_task in task_list:
for task in task_iterator(root_task, only_non_complete=True):
nonfinished_task_list[task.__class__.__name__].add(task)
non_completed_tasks = 0
for task_class in sorted(nonfinished_task_list):
print(task_class)
for task in nonfinished_task_list[task_class]:
print("\tWould run", task)
print()
non_completed_tasks += 1
if non_completed_tasks:
print("In total", non_completed_tasks)
exit(1)
else:
print("All tasks are finished!")
exit(0)