def experiment_group_status_post_save(sender, **kwargs):
instance = kwargs['instance']
experiment_group = instance.experiment_group
previous_status = experiment_group.last_status
# update experiment last_status
experiment_group.status = instance
if instance.status == ExperimentGroupLifeCycle.RUNNING:
experiment_group.started_at = now()
set_started_at(instance=experiment_group,
status=instance.status,
starting_statuses=[ExperimentGroupLifeCycle.RUNNING])
set_finished_at(instance=experiment_group,
status=instance.status,
is_done=ExperimentGroupLifeCycle.is_done)
experiment_group.save(update_fields=['status', 'started_at', 'updated_at', 'finished_at'])
auditor.record(event_type=EXPERIMENT_GROUP_NEW_STATUS,
instance=experiment_group,
previous_status=previous_status)
if instance.status == ExperimentGroupLifeCycle.CREATED:
auditor.record(event_type=EXPERIMENT_GROUP_CREATED, instance=experiment_group)
elif instance.status == ExperimentGroupLifeCycle.STOPPED:
auditor.record(event_type=EXPERIMENT_GROUP_STOPPED,
instance=experiment_group,
previous_status=previous_status)
if ExperimentGroupLifeCycle.is_done(instance.status):
auditor.record(event_type=EXPERIMENT_GROUP_DONE,
instance=experiment_group,
previous_status=previous_status)
new_operation_run_status(entity_type=content_types.EXPERIMENT_GROUP,
entity=experiment_group,
status=instance.status)
def tensorboard_job_status_post_save(sender, **kwargs):
instance = kwargs['instance']
job = instance.job
previous_status = job.last_status
# Update job last_status
job.status = instance
set_job_started_at(instance=job, status=instance.status)
set_job_finished_at(instance=job, status=instance.status)
job.save(update_fields=['status', 'started_at', 'updated_at', 'finished_at'])
auditor.record(event_type=TENSORBOARD_NEW_STATUS,
instance=job,
previous_status=previous_status,
target='project')
if instance.status == JobLifeCycle.STOPPED:
auditor.record(event_type=TENSORBOARD_STOPPED,
instance=job,
previous_status=previous_status,
target='project')
elif instance.status == JobLifeCycle.FAILED:
auditor.record(event_type=TENSORBOARD_FAILED,
instance=job,
previous_status=previous_status,
target='project')
elif instance.status == JobLifeCycle.STOPPED:
auditor.record(event_type=TENSORBOARD_SUCCEEDED,
instance=job,
previous_status=previous_status,
target='project')
if JobLifeCycle.is_done(instance.status):
RedisStatuses.delete_status(job.uuid.hex)
new_operation_run_status(entity_type=content_types.TENSORBOARD_JOB,
entity=job,
status=instance.status)
def job_status_post_save(sender, **kwargs):
instance = kwargs['instance']
job = instance.job
previous_status = job.last_status
# Update job last_status
job.status = instance
set_job_started_at(instance=job, status=instance.status)
set_job_finished_at(instance=job, status=instance.status)
job.save(update_fields=['status', 'started_at', 'updated_at', 'finished_at'])
auditor.record(event_type=JOB_NEW_STATUS,
instance=job,
previous_status=previous_status)
if instance.status == JobLifeCycle.CREATED:
auditor.record(event_type=JOB_CREATED, instance=job)
elif instance.status == JobLifeCycle.STOPPED:
auditor.record(event_type=JOB_STOPPED,
instance=job,
previous_status=previous_status)
elif instance.status == JobLifeCycle.FAILED:
auditor.record(event_type=JOB_FAILED,
instance=job,
previous_status=previous_status)
elif instance.status == JobLifeCycle.SUCCEEDED:
auditor.record(event_type=JOB_SUCCEEDED,
instance=job,
previous_status=previous_status)
if JobLifeCycle.is_done(instance.status):
auditor.record(event_type=JOB_DONE,
instance=job,
previous_status=previous_status)
RedisStatuses.delete_status(job.uuid.hex)
new_operation_run_status(entity_type=content_types.JOB,
entity=job,
status=instance.status)
def notebook_job_status_post_save(sender, **kwargs):
instance = kwargs['instance']
job = instance.job
previous_status = job.last_status
# Update job last_status
job.status = instance
set_job_started_at(instance=job, status=instance.status)
set_job_finished_at(instance=job, status=instance.status)
job.save(
update_fields=['status', 'started_at', 'updated_at', 'finished_at'])
auditor.record(event_type=NOTEBOOK_NEW_STATUS,
instance=job,
previous_status=previous_status,
target='project')
if instance.status == JobLifeCycle.STOPPED:
auditor.record(event_type=NOTEBOOK_STOPPED,
instance=job,
previous_status=previous_status,
target='project')
elif instance.status == JobLifeCycle.FAILED:
auditor.record(event_type=NOTEBOOK_FAILED,
instance=job,
previous_status=previous_status,
target='project')
elif instance.status == JobLifeCycle.STOPPED:
auditor.record(event_type=NOTEBOOK_SUCCEEDED,
instance=job,
previous_status=previous_status,
target='project')
new_operation_run_status(entity_type=content_types.NOTEBOOK_JOB,
entity=job,
status=instance.status)
def experiment_status_post_save(sender, **kwargs):
instance = kwargs['instance']
experiment = instance.experiment
previous_status = experiment.last_status
# update experiment last_status
experiment.status = instance
set_started_at(instance=experiment,
status=instance.status,
starting_statuses=[
ExperimentLifeCycle.STARTING,
ExperimentLifeCycle.RUNNING
],
running_status=ExperimentLifeCycle.RUNNING)
set_finished_at(instance=experiment,
status=instance.status,
is_done=ExperimentLifeCycle.is_done)
experiment.save(
update_fields=['status', 'started_at', 'updated_at', 'finished_at'])
auditor.record(event_type=EXPERIMENT_NEW_STATUS,
instance=experiment,
previous_status=previous_status)
if instance.status == ExperimentLifeCycle.CREATED:
auditor.record(event_type=EXPERIMENT_CREATED, instance=experiment)
elif instance.status == ExperimentLifeCycle.SUCCEEDED:
# update all workers with succeeded status, since we will trigger a stop mechanism
for job in experiment.jobs.all():
if not job.is_done:
job.set_status(JobLifeCycle.SUCCEEDED,
message='Master is done.')
auditor.record(event_type=EXPERIMENT_SUCCEEDED,
instance=experiment,
previous_status=previous_status)
elif instance.status == ExperimentLifeCycle.FAILED:
auditor.record(event_type=EXPERIMENT_FAILED,
instance=experiment,
previous_status=previous_status)
elif instance.status == ExperimentLifeCycle.STOPPED:
auditor.record(event_type=EXPERIMENT_STOPPED,
instance=experiment,
previous_status=previous_status)
if ExperimentLifeCycle.is_done(instance.status):
auditor.record(event_type=EXPERIMENT_DONE,
instance=experiment,
previous_status=previous_status)
new_operation_run_status(entity_type=content_types.EXPERIMENT,
entity=experiment,
status=instance.status)
