def pipelines_start(self: 'celery_app.task', pipeline_run_id: int) -> None: pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id) if not pipeline_run: _logger.info('Pipeline `%s` does not exist any more.', pipeline_run_id) pipeline_run.on_schedule() dag, op_runs = pipeline_run.dag sorted_ops = dags.sort_topologically(dag=dag) op_runs_to_start = [ op_runs[op_run_id] for op_run_id in sorted_ops if op_runs[op_run_id].last_status == OperationStatuses.CREATED ] concurrency = pipeline_run.pipeline.n_operation_runs_to_start future_check = False while op_runs_to_start and concurrency > 0: op_run = op_runs_to_start.pop() if op_run.schedule_start(): # If we end up here it means that the task future_check = True else: concurrency -= 1 if op_runs_to_start or future_check: # Schedule another task self.retry(countdown=Intervals.PIPELINES_SCHEDULER)
def pipelines_stop_operations(pipeline_run_id: int, message: str = None) -> None: pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id) if not pipeline_run: _logger.info('Pipeline `%s` does not exist any more.', pipeline_run_id) stop_operation_runs_for_pipeline_run(pipeline_run, message=message)
def stop_pipeline_operation_runs(pipeline_run_id, message=None): pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id) if not pipeline_run: logger.info( 'Pipeline `{}` does not exist any more.'.format(pipeline_run_id)) stop_operation_runs_for_pipeline_run(pipeline_run, message=message)
def pipelines_check_statuses(pipeline_run_id, status, message=None): pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id) if not pipeline_run: _logger.info('Pipeline `%s` does not exist any more.', pipeline_run_id) if status in OperationStatuses.DONE_STATUS: status = PipelineStatuses.FINISHED pipeline_run.set_status(status=status, message=message)
def pipelines_skip_operations(pipeline_run_id, message=None): pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id) if not pipeline_run: _logger.info('Pipeline `%s` does not exist any more.', pipeline_run_id) # We stop all op runs first stop_operation_runs_for_pipeline_run(pipeline_run, message=message) # Then we marked them as skipped skip_operation_runs_for_pipeline_run(pipeline_run, message=message)
def start_pipeline(self, pipeline_run_id): pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id) if not pipeline_run: logger.info('Pipeline `{}` does not exist any more.'.format(pipeline_run_id)) pipeline_run.on_schedule() dag, op_runs = pipeline_run.dag independent_op_run_ids = dags.get_independent_nodes(dag=dag) op_runs_to_start = [op_runs[op_run_id] for op_run_id in independent_op_run_ids if op_runs[op_run_id].last_status == OperationStatuses.CREATED] concurrency = pipeline_run.pipeline.concurrency concurrency = concurrency or len(op_runs_to_start) while op_runs_to_start: op_run = op_runs_to_start.pop() op_run.schedule_start() concurrency -= 1 if concurrency == 0: break if op_runs_to_start: # Schedule another task self.retry(countdown=Intervals.PIPELINES_SCHEDULER)
def pipelines_start(self, pipeline_run_id): pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id) if not pipeline_run: _logger.info('Pipeline `%s` does not exist any more.', pipeline_run_id) pipeline_run.on_schedule() dag, op_runs = pipeline_run.dag sorted_ops = dags.sort_topologically(dag=dag) op_runs_to_start = [op_runs[op_run_id] for op_run_id in sorted_ops if op_runs[op_run_id].last_status == OperationStatuses.CREATED] concurrency = pipeline_run.pipeline.n_operation_runs_to_start future_check = False while op_runs_to_start and concurrency > 0: op_run = op_runs_to_start.pop() if op_run.schedule_start(): # If we end up here it means that the task future_check = True else: concurrency -= 1 if op_runs_to_start or future_check: # Schedule another task self.retry(countdown=Intervals.PIPELINES_SCHEDULER)
def pipelines_stop_operations(pipeline_run_id, message=None): pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id) if not pipeline_run: _logger.info('Pipeline `%s` does not exist any more.', pipeline_run_id) stop_operation_runs_for_pipeline_run(pipeline_run, message=message)
def check_pipeline_run_status(pipeline_run_id, status, message=None): pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id) if not pipeline_run: logger.info('Pipeline `{}` does not exist any more.'.format(pipeline_run_id)) pipeline_run.set_status(status=status, message=message)