def new_operation_run_status(sender, **kwargs): instance = kwargs['instance'] operation_run = instance.operation_run pipeline_run = operation_run.pipeline_run # Update job last_status operation_run.status = instance set_started_at(instance=operation_run, status=instance.status, starting_statuses=[PipelineStatuses.RUNNING]) set_finished_at(instance=operation_run, status=instance.status, is_done=PipelineStatuses.is_done) operation_run.save() # No need to check if it is just created if instance.status == OperationStatuses.CREATED: return # Check if we need to update the pipeline_run's status celery_app.send_task(PipelinesCeleryTasks.PIPELINES_CHECK_STATUSES, kwargs={ 'pipeline_run_id': pipeline_run.id, 'status': instance.status, 'message': instance.message }) if operation_run.is_done: # Notify downstream that instance is done, and that its dependency can start. downstream_runs = operation_run.downstream_runs.filter( status__status=OperationStatuses.CREATED) for op_run in downstream_runs: celery_app.send_task( PipelinesCeleryTasks.PIPELINES_START_OPERATION, kwargs={'operation_run_id': op_run.id})
def experiment_group_status_post_save(sender, **kwargs): instance = kwargs['instance'] experiment_group = instance.experiment_group previous_status = experiment_group.last_status # update experiment last_status experiment_group.status = instance if instance.status == ExperimentGroupLifeCycle.RUNNING: experiment_group.started_at = now() set_started_at(instance=experiment_group, status=instance.status, starting_statuses=[ExperimentGroupLifeCycle.RUNNING]) set_finished_at(instance=experiment_group, status=instance.status, is_done=ExperimentGroupLifeCycle.is_done) experiment_group.save(update_fields=['status', 'started_at', 'updated_at', 'finished_at']) auditor.record(event_type=EXPERIMENT_GROUP_NEW_STATUS, instance=experiment_group, previous_status=previous_status) if instance.status == ExperimentGroupLifeCycle.CREATED: auditor.record(event_type=EXPERIMENT_GROUP_CREATED, instance=experiment_group) elif instance.status == ExperimentGroupLifeCycle.STOPPED: auditor.record(event_type=EXPERIMENT_GROUP_STOPPED, instance=experiment_group, previous_status=previous_status) if ExperimentGroupLifeCycle.is_done(instance.status): auditor.record(event_type=EXPERIMENT_GROUP_DONE, instance=experiment_group, previous_status=previous_status) new_operation_run_status(entity_type=content_types.EXPERIMENT_GROUP, entity=experiment_group, status=instance.status)
def new_pipeline_run_status(sender, **kwargs): instance = kwargs['instance'] pipeline_run = instance.pipeline_run # Update job last_status pipeline_run.status = instance set_started_at(instance=pipeline_run, status=instance.status, starting_statuses=[PipelineStatuses.RUNNING]) set_finished_at(instance=pipeline_run, status=instance.status, is_done=PipelineStatuses.is_done) pipeline_run.save() # Notify operations with status change. This is necessary if we skip or stop the dag run. if pipeline_run.stopped: celery_app.send_task(PipelinesCeleryTasks.PIPELINES_STOP_OPERATIONS, kwargs={ 'pipeline_run_id': pipeline_run.id, 'message': 'Pipeline run was stopped' }) if pipeline_run.skipped: celery_app.send_task(PipelinesCeleryTasks.PIPELINES_SKIP_OPERATIONS, kwargs={ 'pipeline_run_id': pipeline_run.id, 'message': 'Pipeline run was skipped' })
def experiment_group_status_post_save(sender, **kwargs): instance = kwargs['instance'] experiment_group = instance.experiment_group previous_status = experiment_group.last_status # update experiment last_status experiment_group.status = instance if instance.status == ExperimentGroupLifeCycle.RUNNING: experiment_group.started_at = now() set_started_at(instance=experiment_group, status=instance.status, starting_statuses=[ExperimentGroupLifeCycle.RUNNING]) set_finished_at(instance=experiment_group, status=instance.status, is_done=ExperimentGroupLifeCycle.is_done) experiment_group.save() auditor.record(event_type=EXPERIMENT_GROUP_NEW_STATUS, instance=experiment_group, previous_status=previous_status) if instance.status == ExperimentGroupLifeCycle.STOPPED: auditor.record(event_type=EXPERIMENT_GROUP_STOPPED, instance=experiment_group, previous_status=previous_status) if ExperimentGroupLifeCycle.is_done(instance.status): auditor.record(event_type=EXPERIMENT_GROUP_DONE, instance=experiment_group, previous_status=previous_status)
def experiment_status_post_save(sender, **kwargs): instance = kwargs['instance'] experiment = instance.experiment previous_status = experiment.last_status # update experiment last_status experiment.status = instance set_started_at(instance=experiment, status=instance.status, starting_statuses=[ ExperimentLifeCycle.STARTING, ExperimentLifeCycle.RUNNING ], running_status=ExperimentLifeCycle.RUNNING) set_finished_at(instance=experiment, status=instance.status, is_done=ExperimentLifeCycle.is_done) experiment.save(update_fields=['status', 'started_at', 'finished_at']) auditor.record(event_type=EXPERIMENT_NEW_STATUS, instance=experiment, previous_status=previous_status) if instance.status == ExperimentLifeCycle.SUCCEEDED: # update all workers with succeeded status, since we will trigger a stop mechanism for job in experiment.jobs.all(): if not job.is_done: job.set_status(JobLifeCycle.SUCCEEDED, message='Master is done.') auditor.record(event_type=EXPERIMENT_SUCCEEDED, instance=experiment, previous_status=previous_status) if instance.status == ExperimentLifeCycle.FAILED: auditor.record(event_type=EXPERIMENT_FAILED, instance=experiment, previous_status=previous_status) if instance.status == ExperimentLifeCycle.STOPPED: auditor.record(event_type=EXPERIMENT_STOPPED, instance=experiment, previous_status=previous_status) if ExperimentLifeCycle.is_done(instance.status): auditor.record(event_type=EXPERIMENT_DONE, instance=experiment, previous_status=previous_status) # Check if it's part of an experiment group, and start following tasks if not experiment.is_independent: celery_app.send_task( HPCeleryTasks.HP_START, kwargs={'experiment_group_id': experiment.experiment_group.id}, countdown=1)
def experiment_status_post_save(sender, **kwargs): instance = kwargs['instance'] experiment = instance.experiment previous_status = experiment.last_status # update experiment last_status experiment.status = instance set_started_at(instance=experiment, status=instance.status, starting_statuses=[ ExperimentLifeCycle.STARTING, ExperimentLifeCycle.RUNNING ], running_status=ExperimentLifeCycle.RUNNING) set_finished_at(instance=experiment, status=instance.status, is_done=ExperimentLifeCycle.is_done) experiment.save( update_fields=['status', 'started_at', 'updated_at', 'finished_at']) auditor.record(event_type=EXPERIMENT_NEW_STATUS, instance=experiment, previous_status=previous_status) if instance.status == ExperimentLifeCycle.CREATED: auditor.record(event_type=EXPERIMENT_CREATED, instance=experiment) elif instance.status == ExperimentLifeCycle.SUCCEEDED: # update all workers with succeeded status, since we will trigger a stop mechanism for job in experiment.jobs.all(): if not job.is_done: job.set_status(JobLifeCycle.SUCCEEDED, message='Master is done.') auditor.record(event_type=EXPERIMENT_SUCCEEDED, instance=experiment, previous_status=previous_status) elif instance.status == ExperimentLifeCycle.FAILED: auditor.record(event_type=EXPERIMENT_FAILED, instance=experiment, previous_status=previous_status) elif instance.status == ExperimentLifeCycle.STOPPED: auditor.record(event_type=EXPERIMENT_STOPPED, instance=experiment, previous_status=previous_status) if ExperimentLifeCycle.is_done(instance.status): auditor.record(event_type=EXPERIMENT_DONE, instance=experiment, previous_status=previous_status) new_operation_run_status(entity_type=content_types.EXPERIMENT, entity=experiment, status=instance.status)
def experiment_status_post_save(sender, **kwargs): instance = kwargs['instance'] experiment = instance.experiment previous_status = experiment.last_status # update experiment last_status experiment.status = instance set_started_at(instance=experiment, status=instance.status, starting_statuses=[ExperimentLifeCycle.STARTING]) set_finished_at(instance=experiment, status=instance.status, is_done=ExperimentLifeCycle.is_done) experiment.save() auditor.record(event_type=EXPERIMENT_NEW_STATUS, instance=experiment, previous_status=previous_status) if instance.status == ExperimentLifeCycle.SUCCEEDED: # update all workers with succeeded status, since we will trigger a stop mechanism for job in experiment.jobs.all(): if not job.is_done: job.set_status(JobLifeCycle.SUCCEEDED, message='Master is done.') auditor.record(event_type=EXPERIMENT_SUCCEEDED, instance=experiment, previous_status=previous_status) if instance.status == ExperimentLifeCycle.FAILED: auditor.record(event_type=EXPERIMENT_FAILED, instance=experiment, previous_status=previous_status) if instance.status == ExperimentLifeCycle.STOPPED: auditor.record(event_type=EXPERIMENT_STOPPED, instance=experiment, previous_status=previous_status) if ExperimentLifeCycle.is_done(instance.status): auditor.record(event_type=EXPERIMENT_DONE, instance=experiment, previous_status=previous_status)
def new_pipeline_run_status_post_save(sender, **kwargs): instance = kwargs['instance'] pipeline_run = instance.pipeline_run previous_status = pipeline_run.last_status # Update job last_status pipeline_run.status = instance set_started_at(instance=pipeline_run, status=instance.status, starting_statuses=[PipelineLifeCycle.RUNNING]) set_finished_at(instance=pipeline_run, status=instance.status, is_done=PipelineLifeCycle.is_done) pipeline_run.save(update_fields=['status', 'started_at', 'finished_at']) # Notify operations with status change. This is necessary if we skip or stop the dag run. if pipeline_run.stopped: auditor.record(event_type=PIPELINE_RUN_STOPPED, instance=pipeline_run, previous_status=previous_status) if pipeline_run.skipped: auditor.record(event_type=PIPELINE_RUN_SKIPPED, instance=pipeline_run, previous_status=previous_status)