def experiment_group_status_post_save(sender, **kwargs): instance = kwargs['instance'] experiment_group = instance.experiment_group previous_status = experiment_group.last_status # update experiment last_status experiment_group.status = instance if instance.status == ExperimentGroupLifeCycle.RUNNING: experiment_group.started_at = now() set_started_at(instance=experiment_group, status=instance.status, starting_statuses=[ExperimentGroupLifeCycle.RUNNING]) set_finished_at(instance=experiment_group, status=instance.status, is_done=ExperimentGroupLifeCycle.is_done) experiment_group.save(update_fields=['status', 'started_at', 'updated_at', 'finished_at']) auditor.record(event_type=EXPERIMENT_GROUP_NEW_STATUS, instance=experiment_group, previous_status=previous_status) if instance.status == ExperimentGroupLifeCycle.CREATED: auditor.record(event_type=EXPERIMENT_GROUP_CREATED, instance=experiment_group) elif instance.status == ExperimentGroupLifeCycle.STOPPED: auditor.record(event_type=EXPERIMENT_GROUP_STOPPED, instance=experiment_group, previous_status=previous_status) if ExperimentGroupLifeCycle.is_done(instance.status): auditor.record(event_type=EXPERIMENT_GROUP_DONE, instance=experiment_group, previous_status=previous_status) new_operation_run_status(entity_type=content_types.EXPERIMENT_GROUP, entity=experiment_group, status=instance.status)
def tensorboard_job_status_post_save(sender, **kwargs): instance = kwargs['instance'] job = instance.job previous_status = job.last_status # Update job last_status job.status = instance set_job_started_at(instance=job, status=instance.status) set_job_finished_at(instance=job, status=instance.status) job.save(update_fields=['status', 'started_at', 'updated_at', 'finished_at']) auditor.record(event_type=TENSORBOARD_NEW_STATUS, instance=job, previous_status=previous_status, target='project') if instance.status == JobLifeCycle.STOPPED: auditor.record(event_type=TENSORBOARD_STOPPED, instance=job, previous_status=previous_status, target='project') elif instance.status == JobLifeCycle.FAILED: auditor.record(event_type=TENSORBOARD_FAILED, instance=job, previous_status=previous_status, target='project') elif instance.status == JobLifeCycle.STOPPED: auditor.record(event_type=TENSORBOARD_SUCCEEDED, instance=job, previous_status=previous_status, target='project') if JobLifeCycle.is_done(instance.status): RedisStatuses.delete_status(job.uuid.hex) new_operation_run_status(entity_type=content_types.TENSORBOARD_JOB, entity=job, status=instance.status)
def job_status_post_save(sender, **kwargs): instance = kwargs['instance'] job = instance.job previous_status = job.last_status # Update job last_status job.status = instance set_job_started_at(instance=job, status=instance.status) set_job_finished_at(instance=job, status=instance.status) job.save(update_fields=['status', 'started_at', 'updated_at', 'finished_at']) auditor.record(event_type=JOB_NEW_STATUS, instance=job, previous_status=previous_status) if instance.status == JobLifeCycle.CREATED: auditor.record(event_type=JOB_CREATED, instance=job) elif instance.status == JobLifeCycle.STOPPED: auditor.record(event_type=JOB_STOPPED, instance=job, previous_status=previous_status) elif instance.status == JobLifeCycle.FAILED: auditor.record(event_type=JOB_FAILED, instance=job, previous_status=previous_status) elif instance.status == JobLifeCycle.SUCCEEDED: auditor.record(event_type=JOB_SUCCEEDED, instance=job, previous_status=previous_status) if JobLifeCycle.is_done(instance.status): auditor.record(event_type=JOB_DONE, instance=job, previous_status=previous_status) RedisStatuses.delete_status(job.uuid.hex) new_operation_run_status(entity_type=content_types.JOB, entity=job, status=instance.status)
def notebook_job_status_post_save(sender, **kwargs): instance = kwargs['instance'] job = instance.job previous_status = job.last_status # Update job last_status job.status = instance set_job_started_at(instance=job, status=instance.status) set_job_finished_at(instance=job, status=instance.status) job.save( update_fields=['status', 'started_at', 'updated_at', 'finished_at']) auditor.record(event_type=NOTEBOOK_NEW_STATUS, instance=job, previous_status=previous_status, target='project') if instance.status == JobLifeCycle.STOPPED: auditor.record(event_type=NOTEBOOK_STOPPED, instance=job, previous_status=previous_status, target='project') elif instance.status == JobLifeCycle.FAILED: auditor.record(event_type=NOTEBOOK_FAILED, instance=job, previous_status=previous_status, target='project') elif instance.status == JobLifeCycle.STOPPED: auditor.record(event_type=NOTEBOOK_SUCCEEDED, instance=job, previous_status=previous_status, target='project') new_operation_run_status(entity_type=content_types.NOTEBOOK_JOB, entity=job, status=instance.status)
def experiment_status_post_save(sender, **kwargs): instance = kwargs['instance'] experiment = instance.experiment previous_status = experiment.last_status # update experiment last_status experiment.status = instance set_started_at(instance=experiment, status=instance.status, starting_statuses=[ ExperimentLifeCycle.STARTING, ExperimentLifeCycle.RUNNING ], running_status=ExperimentLifeCycle.RUNNING) set_finished_at(instance=experiment, status=instance.status, is_done=ExperimentLifeCycle.is_done) experiment.save( update_fields=['status', 'started_at', 'updated_at', 'finished_at']) auditor.record(event_type=EXPERIMENT_NEW_STATUS, instance=experiment, previous_status=previous_status) if instance.status == ExperimentLifeCycle.CREATED: auditor.record(event_type=EXPERIMENT_CREATED, instance=experiment) elif instance.status == ExperimentLifeCycle.SUCCEEDED: # update all workers with succeeded status, since we will trigger a stop mechanism for job in experiment.jobs.all(): if not job.is_done: job.set_status(JobLifeCycle.SUCCEEDED, message='Master is done.') auditor.record(event_type=EXPERIMENT_SUCCEEDED, instance=experiment, previous_status=previous_status) elif instance.status == ExperimentLifeCycle.FAILED: auditor.record(event_type=EXPERIMENT_FAILED, instance=experiment, previous_status=previous_status) elif instance.status == ExperimentLifeCycle.STOPPED: auditor.record(event_type=EXPERIMENT_STOPPED, instance=experiment, previous_status=previous_status) if ExperimentLifeCycle.is_done(instance.status): auditor.record(event_type=EXPERIMENT_DONE, instance=experiment, previous_status=previous_status) new_operation_run_status(entity_type=content_types.EXPERIMENT, entity=experiment, status=instance.status)