def run_as_batch_worker(task_list, cli_args, kwargs):
found_task = False
for root_task in task_list:
for task in task_iterator(root_task):
if task.task_id != cli_args.task_id:
continue
found_task = True
set_setting("_dispatch_local_execution", True)
# TODO: We do not process the information if (a) we have a new dependency and (b) why the task has failed.
# TODO: Would be also nice to run the event handlers
try:
create_output_dirs(task)
task.run()
task.on_success()
except BaseException as ex:
task.on_failure(ex)
raise ex
return
if not found_task:
raise ValueError(f"The task id {task.task_id} to be executed by this batch worker "
f"does not exist in the locally reproduced task graph.")
def run_test_mode(task_list, cli_args, kwargs):
set_setting("_dispatch_local_execution", True)
luigi.build(task_list, log_level="DEBUG", local_scheduler=True, **kwargs)
def run_local(task_list, cli_args, kwargs):
set_setting("batch_system", "local")
run_luigi(task_list, cli_args, kwargs)