def remove_stoppable_task_targets(tasks_creator): stoppable_task = StoppableTask() if stoppable_task.failed_target.exists(): stoppable_task.failed_target.remove() if stoppable_task.timers_dir.exists(): shutil.rmtree(str(stoppable_task.timers_dir)) if stoppable_task.dependencies_dir.exists(): shutil.rmtree(str(stoppable_task.dependencies_dir)) tasks = tasks_creator() return tasks
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._prepare_outputs() stoppable_task = StoppableTask() if stoppable_task.failed_target.exists(): print("removed failed target") stoppable_task.failed_target.remove() self.actual_release_types = [ ReleaseType[release_type] for release_type in self.release_types ]
def collect_dependencies() -> Set[TaskDependency]: stoppable_task = StoppableTask() dependencies = set() for root, directories, files in os.walk(stoppable_task.dependencies_dir): for file in files: file_path = Path(root).joinpath(file) with open(file_path) as f: for line in f.readlines(): task_dependency = TaskDependency.from_json(line) if task_dependency.state == DependencyState.requested.name: dependencies.add(task_dependency) return dependencies
def run_tasks(tasks_creator: Callable[[], List[luigi.Task]], workers: int, task_dependencies_dot_file: str, on_success: Callable[[], None] = None, on_failure: Callable[[], None] = None): setup_worker() start_time = datetime.now() tasks = remove_stoppable_task_targets(tasks_creator) no_scheduling_errors = luigi.build(tasks, workers=workers, local_scheduler=True, log_level="INFO") if StoppableTask().failed_target.exists() or not no_scheduling_errors: handle_failure(on_failure) else: handle_success(on_success, task_dependencies_dot_file, start_time)