def new_operation_run_status(sender, **kwargs):
instance = kwargs['instance']
operation_run = instance.operation_run
pipeline_run = operation_run.pipeline_run
# Update job last_status
operation_run.status = instance
set_started_at(instance=operation_run,
status=instance.status,
starting_statuses=[PipelineStatuses.RUNNING])
set_finished_at(instance=operation_run,
status=instance.status,
is_done=PipelineStatuses.is_done)
operation_run.save()
# No need to check if it is just created
if instance.status == OperationStatuses.CREATED:
return
# Check if we need to update the pipeline_run's status
celery_app.send_task(PipelinesCeleryTasks.PIPELINES_CHECK_STATUSES,
kwargs={
'pipeline_run_id': pipeline_run.id,
'status': instance.status,
'message': instance.message
})
if operation_run.is_done:
# Notify downstream that instance is done, and that its dependency can start.
downstream_runs = operation_run.downstream_runs.filter(
status__status=OperationStatuses.CREATED)
for op_run in downstream_runs:
celery_app.send_task(
PipelinesCeleryTasks.PIPELINES_START_OPERATION,
kwargs={'operation_run_id': op_run.id})
def experiment_group_status_post_save(sender, **kwargs):
instance = kwargs['instance']
experiment_group = instance.experiment_group
previous_status = experiment_group.last_status
# update experiment last_status
experiment_group.status = instance
if instance.status == ExperimentGroupLifeCycle.RUNNING:
experiment_group.started_at = now()
set_started_at(instance=experiment_group,
status=instance.status,
starting_statuses=[ExperimentGroupLifeCycle.RUNNING])
set_finished_at(instance=experiment_group,
status=instance.status,
is_done=ExperimentGroupLifeCycle.is_done)
experiment_group.save(update_fields=['status', 'started_at', 'updated_at', 'finished_at'])
auditor.record(event_type=EXPERIMENT_GROUP_NEW_STATUS,
instance=experiment_group,
previous_status=previous_status)
if instance.status == ExperimentGroupLifeCycle.CREATED:
auditor.record(event_type=EXPERIMENT_GROUP_CREATED, instance=experiment_group)
elif instance.status == ExperimentGroupLifeCycle.STOPPED:
auditor.record(event_type=EXPERIMENT_GROUP_STOPPED,
instance=experiment_group,
previous_status=previous_status)
if ExperimentGroupLifeCycle.is_done(instance.status):
auditor.record(event_type=EXPERIMENT_GROUP_DONE,
instance=experiment_group,
previous_status=previous_status)
new_operation_run_status(entity_type=content_types.EXPERIMENT_GROUP,
entity=experiment_group,
status=instance.status)
def new_pipeline_run_status(sender, **kwargs):
instance = kwargs['instance']
pipeline_run = instance.pipeline_run
# Update job last_status
pipeline_run.status = instance
set_started_at(instance=pipeline_run,
status=instance.status,
starting_statuses=[PipelineStatuses.RUNNING])
set_finished_at(instance=pipeline_run,
status=instance.status,
is_done=PipelineStatuses.is_done)
pipeline_run.save()
# Notify operations with status change. This is necessary if we skip or stop the dag run.
if pipeline_run.stopped:
celery_app.send_task(PipelinesCeleryTasks.PIPELINES_STOP_OPERATIONS,
kwargs={
'pipeline_run_id': pipeline_run.id,
'message': 'Pipeline run was stopped'
})
if pipeline_run.skipped:
celery_app.send_task(PipelinesCeleryTasks.PIPELINES_SKIP_OPERATIONS,
kwargs={
'pipeline_run_id': pipeline_run.id,
'message': 'Pipeline run was skipped'
})
def experiment_group_status_post_save(sender, **kwargs):
instance = kwargs['instance']
experiment_group = instance.experiment_group
previous_status = experiment_group.last_status
# update experiment last_status
experiment_group.status = instance
if instance.status == ExperimentGroupLifeCycle.RUNNING:
experiment_group.started_at = now()
set_started_at(instance=experiment_group,
status=instance.status,
starting_statuses=[ExperimentGroupLifeCycle.RUNNING])
set_finished_at(instance=experiment_group,
status=instance.status,
is_done=ExperimentGroupLifeCycle.is_done)
experiment_group.save()
auditor.record(event_type=EXPERIMENT_GROUP_NEW_STATUS,
instance=experiment_group,
previous_status=previous_status)
if instance.status == ExperimentGroupLifeCycle.STOPPED:
auditor.record(event_type=EXPERIMENT_GROUP_STOPPED,
instance=experiment_group,
previous_status=previous_status)
if ExperimentGroupLifeCycle.is_done(instance.status):
auditor.record(event_type=EXPERIMENT_GROUP_DONE,
instance=experiment_group,
previous_status=previous_status)
def experiment_status_post_save(sender, **kwargs):
instance = kwargs['instance']
experiment = instance.experiment
previous_status = experiment.last_status
# update experiment last_status
experiment.status = instance
set_started_at(instance=experiment,
status=instance.status,
starting_statuses=[
ExperimentLifeCycle.STARTING,
ExperimentLifeCycle.RUNNING
],
running_status=ExperimentLifeCycle.RUNNING)
set_finished_at(instance=experiment,
status=instance.status,
is_done=ExperimentLifeCycle.is_done)
experiment.save(update_fields=['status', 'started_at', 'finished_at'])
auditor.record(event_type=EXPERIMENT_NEW_STATUS,
instance=experiment,
previous_status=previous_status)
if instance.status == ExperimentLifeCycle.SUCCEEDED:
# update all workers with succeeded status, since we will trigger a stop mechanism
for job in experiment.jobs.all():
if not job.is_done:
job.set_status(JobLifeCycle.SUCCEEDED,
message='Master is done.')
auditor.record(event_type=EXPERIMENT_SUCCEEDED,
instance=experiment,
previous_status=previous_status)
if instance.status == ExperimentLifeCycle.FAILED:
auditor.record(event_type=EXPERIMENT_FAILED,
instance=experiment,
previous_status=previous_status)
if instance.status == ExperimentLifeCycle.STOPPED:
auditor.record(event_type=EXPERIMENT_STOPPED,
instance=experiment,
previous_status=previous_status)
if ExperimentLifeCycle.is_done(instance.status):
auditor.record(event_type=EXPERIMENT_DONE,
instance=experiment,
previous_status=previous_status)
# Check if it's part of an experiment group, and start following tasks
if not experiment.is_independent:
celery_app.send_task(
HPCeleryTasks.HP_START,
kwargs={'experiment_group_id': experiment.experiment_group.id},
countdown=1)
def experiment_status_post_save(sender, **kwargs):
instance = kwargs['instance']
experiment = instance.experiment
previous_status = experiment.last_status
# update experiment last_status
experiment.status = instance
set_started_at(instance=experiment,
status=instance.status,
starting_statuses=[
ExperimentLifeCycle.STARTING,
ExperimentLifeCycle.RUNNING
],
running_status=ExperimentLifeCycle.RUNNING)
set_finished_at(instance=experiment,
status=instance.status,
is_done=ExperimentLifeCycle.is_done)
experiment.save(
update_fields=['status', 'started_at', 'updated_at', 'finished_at'])
auditor.record(event_type=EXPERIMENT_NEW_STATUS,
instance=experiment,
previous_status=previous_status)
if instance.status == ExperimentLifeCycle.CREATED:
auditor.record(event_type=EXPERIMENT_CREATED, instance=experiment)
elif instance.status == ExperimentLifeCycle.SUCCEEDED:
# update all workers with succeeded status, since we will trigger a stop mechanism
for job in experiment.jobs.all():
if not job.is_done:
job.set_status(JobLifeCycle.SUCCEEDED,
message='Master is done.')
auditor.record(event_type=EXPERIMENT_SUCCEEDED,
instance=experiment,
previous_status=previous_status)
elif instance.status == ExperimentLifeCycle.FAILED:
auditor.record(event_type=EXPERIMENT_FAILED,
instance=experiment,
previous_status=previous_status)
elif instance.status == ExperimentLifeCycle.STOPPED:
auditor.record(event_type=EXPERIMENT_STOPPED,
instance=experiment,
previous_status=previous_status)
if ExperimentLifeCycle.is_done(instance.status):
auditor.record(event_type=EXPERIMENT_DONE,
instance=experiment,
previous_status=previous_status)
new_operation_run_status(entity_type=content_types.EXPERIMENT,
entity=experiment,
status=instance.status)
def experiment_status_post_save(sender, **kwargs):
instance = kwargs['instance']
experiment = instance.experiment
previous_status = experiment.last_status
# update experiment last_status
experiment.status = instance
set_started_at(instance=experiment,
status=instance.status,
starting_statuses=[ExperimentLifeCycle.STARTING])
set_finished_at(instance=experiment,
status=instance.status,
is_done=ExperimentLifeCycle.is_done)
experiment.save()
auditor.record(event_type=EXPERIMENT_NEW_STATUS,
instance=experiment,
previous_status=previous_status)
if instance.status == ExperimentLifeCycle.SUCCEEDED:
# update all workers with succeeded status, since we will trigger a stop mechanism
for job in experiment.jobs.all():
if not job.is_done:
job.set_status(JobLifeCycle.SUCCEEDED,
message='Master is done.')
auditor.record(event_type=EXPERIMENT_SUCCEEDED,
instance=experiment,
previous_status=previous_status)
if instance.status == ExperimentLifeCycle.FAILED:
auditor.record(event_type=EXPERIMENT_FAILED,
instance=experiment,
previous_status=previous_status)
if instance.status == ExperimentLifeCycle.STOPPED:
auditor.record(event_type=EXPERIMENT_STOPPED,
instance=experiment,
previous_status=previous_status)
if ExperimentLifeCycle.is_done(instance.status):
auditor.record(event_type=EXPERIMENT_DONE,
instance=experiment,
previous_status=previous_status)
def new_pipeline_run_status_post_save(sender, **kwargs):
instance = kwargs['instance']
pipeline_run = instance.pipeline_run
previous_status = pipeline_run.last_status
# Update job last_status
pipeline_run.status = instance
set_started_at(instance=pipeline_run,
status=instance.status,
starting_statuses=[PipelineLifeCycle.RUNNING])
set_finished_at(instance=pipeline_run,
status=instance.status,
is_done=PipelineLifeCycle.is_done)
pipeline_run.save(update_fields=['status', 'started_at', 'finished_at'])
# Notify operations with status change. This is necessary if we skip or stop the dag run.
if pipeline_run.stopped:
auditor.record(event_type=PIPELINE_RUN_STOPPED,
instance=pipeline_run,
previous_status=previous_status)
if pipeline_run.skipped:
auditor.record(event_type=PIPELINE_RUN_SKIPPED,
instance=pipeline_run,
previous_status=previous_status)
