def pipelines_start(self: 'celery_app.task', pipeline_run_id: int) -> None:
pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id)
if not pipeline_run:
_logger.info('Pipeline `%s` does not exist any more.', pipeline_run_id)
pipeline_run.on_schedule()
dag, op_runs = pipeline_run.dag
sorted_ops = dags.sort_topologically(dag=dag)
op_runs_to_start = [
op_runs[op_run_id] for op_run_id in sorted_ops
if op_runs[op_run_id].last_status == OperationStatuses.CREATED
]
concurrency = pipeline_run.pipeline.n_operation_runs_to_start
future_check = False
while op_runs_to_start and concurrency > 0:
op_run = op_runs_to_start.pop()
if op_run.schedule_start():
# If we end up here it means that the task
future_check = True
else:
concurrency -= 1
if op_runs_to_start or future_check:
# Schedule another task
self.retry(countdown=Intervals.PIPELINES_SCHEDULER)
def pipelines_stop_operations(pipeline_run_id: int,
message: str = None) -> None:
pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id)
if not pipeline_run:
_logger.info('Pipeline `%s` does not exist any more.', pipeline_run_id)
stop_operation_runs_for_pipeline_run(pipeline_run, message=message)
def stop_pipeline_operation_runs(pipeline_run_id, message=None):
pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id)
if not pipeline_run:
logger.info(
'Pipeline `{}` does not exist any more.'.format(pipeline_run_id))
stop_operation_runs_for_pipeline_run(pipeline_run, message=message)
def pipelines_check_statuses(pipeline_run_id, status, message=None):
pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id)
if not pipeline_run:
_logger.info('Pipeline `%s` does not exist any more.', pipeline_run_id)
if status in OperationStatuses.DONE_STATUS:
status = PipelineStatuses.FINISHED
pipeline_run.set_status(status=status, message=message)
def pipelines_check_statuses(pipeline_run_id, status, message=None):
pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id)
if not pipeline_run:
_logger.info('Pipeline `%s` does not exist any more.', pipeline_run_id)
if status in OperationStatuses.DONE_STATUS:
status = PipelineStatuses.FINISHED
pipeline_run.set_status(status=status, message=message)
def pipelines_skip_operations(pipeline_run_id, message=None):
pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id)
if not pipeline_run:
_logger.info('Pipeline `%s` does not exist any more.', pipeline_run_id)
# We stop all op runs first
stop_operation_runs_for_pipeline_run(pipeline_run, message=message)
# Then we marked them as skipped
skip_operation_runs_for_pipeline_run(pipeline_run, message=message)
def pipelines_skip_operations(pipeline_run_id, message=None):
pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id)
if not pipeline_run:
_logger.info('Pipeline `%s` does not exist any more.', pipeline_run_id)
# We stop all op runs first
stop_operation_runs_for_pipeline_run(pipeline_run, message=message)
# Then we marked them as skipped
skip_operation_runs_for_pipeline_run(pipeline_run, message=message)
def start_pipeline(self, pipeline_run_id):
pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id)
if not pipeline_run:
logger.info('Pipeline `{}` does not exist any more.'.format(pipeline_run_id))
pipeline_run.on_schedule()
dag, op_runs = pipeline_run.dag
independent_op_run_ids = dags.get_independent_nodes(dag=dag)
op_runs_to_start = [op_runs[op_run_id] for op_run_id in independent_op_run_ids
if op_runs[op_run_id].last_status == OperationStatuses.CREATED]
concurrency = pipeline_run.pipeline.concurrency
concurrency = concurrency or len(op_runs_to_start)
while op_runs_to_start:
op_run = op_runs_to_start.pop()
op_run.schedule_start()
concurrency -= 1
if concurrency == 0:
break
if op_runs_to_start:
# Schedule another task
self.retry(countdown=Intervals.PIPELINES_SCHEDULER)
def pipelines_start(self, pipeline_run_id):
pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id)
if not pipeline_run:
_logger.info('Pipeline `%s` does not exist any more.', pipeline_run_id)
pipeline_run.on_schedule()
dag, op_runs = pipeline_run.dag
sorted_ops = dags.sort_topologically(dag=dag)
op_runs_to_start = [op_runs[op_run_id] for op_run_id in sorted_ops
if op_runs[op_run_id].last_status == OperationStatuses.CREATED]
concurrency = pipeline_run.pipeline.n_operation_runs_to_start
future_check = False
while op_runs_to_start and concurrency > 0:
op_run = op_runs_to_start.pop()
if op_run.schedule_start():
# If we end up here it means that the task
future_check = True
else:
concurrency -= 1
if op_runs_to_start or future_check:
# Schedule another task
self.retry(countdown=Intervals.PIPELINES_SCHEDULER)
def pipelines_stop_operations(pipeline_run_id, message=None):
pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id)
if not pipeline_run:
_logger.info('Pipeline `%s` does not exist any more.', pipeline_run_id)
stop_operation_runs_for_pipeline_run(pipeline_run, message=message)
def check_pipeline_run_status(pipeline_run_id, status, message=None):
pipeline_run = get_pipeline_run(pipeline_run_id=pipeline_run_id)
if not pipeline_run:
logger.info('Pipeline `{}` does not exist any more.'.format(pipeline_run_id))
pipeline_run.set_status(status=status, message=message)
