def remove_stoppable_task_targets(tasks_creator):
stoppable_task = StoppableTask()
if stoppable_task.failed_target.exists():
stoppable_task.failed_target.remove()
if stoppable_task.timers_dir.exists():
shutil.rmtree(str(stoppable_task.timers_dir))
if stoppable_task.dependencies_dir.exists():
shutil.rmtree(str(stoppable_task.dependencies_dir))
tasks = tasks_creator()
return tasks
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._prepare_outputs()
stoppable_task = StoppableTask()
if stoppable_task.failed_target.exists():
print("removed failed target")
stoppable_task.failed_target.remove()
self.actual_release_types = [
ReleaseType[release_type] for release_type in self.release_types
]
def collect_dependencies() -> Set[TaskDependency]:
stoppable_task = StoppableTask()
dependencies = set()
for root, directories, files in os.walk(stoppable_task.dependencies_dir):
for file in files:
file_path = Path(root).joinpath(file)
with open(file_path) as f:
for line in f.readlines():
task_dependency = TaskDependency.from_json(line)
if task_dependency.state == DependencyState.requested.name:
dependencies.add(task_dependency)
return dependencies
def run_tasks(tasks_creator: Callable[[], List[luigi.Task]],
workers: int,
task_dependencies_dot_file: str,
on_success: Callable[[], None] = None,
on_failure: Callable[[], None] = None):
setup_worker()
start_time = datetime.now()
tasks = remove_stoppable_task_targets(tasks_creator)
no_scheduling_errors = luigi.build(tasks,
workers=workers,
local_scheduler=True,
log_level="INFO")
if StoppableTask().failed_target.exists() or not no_scheduling_errors:
handle_failure(on_failure)
else:
handle_success(on_success, task_dependencies_dot_file, start_time)