def experiment_group_status_post_save(sender, **kwargs): instance = kwargs['instance'] experiment_group = instance.experiment_group previous_status = experiment_group.last_status # update experiment last_status experiment_group.status = instance if instance.status == ExperimentGroupLifeCycle.RUNNING: experiment_group.started_at = now() set_started_at(instance=experiment_group, status=instance.status, starting_statuses=[ExperimentGroupLifeCycle.RUNNING]) set_finished_at(instance=experiment_group, status=instance.status, is_done=ExperimentGroupLifeCycle.is_done) experiment_group.save(update_fields=['status', 'started_at', 'finished_at']) auditor.record(event_type=EXPERIMENT_GROUP_NEW_STATUS, instance=experiment_group, previous_status=previous_status) if instance.status == ExperimentGroupLifeCycle.CREATED: auditor.record(event_type=EXPERIMENT_GROUP_CREATED, instance=experiment_group) elif instance.status == ExperimentGroupLifeCycle.STOPPED: auditor.record(event_type=EXPERIMENT_GROUP_STOPPED, instance=experiment_group, previous_status=previous_status) if ExperimentGroupLifeCycle.is_done(instance.status): auditor.record(event_type=EXPERIMENT_GROUP_DONE, instance=experiment_group, previous_status=previous_status)
def experiments_group_create(self, experiment_group_id): experiment_group = _get_group_or_retry( experiment_group_id=experiment_group_id, task=self) if not experiment_group: return last_status = experiment_group.last_status if not ExperimentGroupLifeCycle.can_transition( status_from=last_status, status_to=ExperimentGroupLifeCycle.RUNNING): _logger.info( 'Experiment group id `%s` cannot transition from `%s` to `%s`.', experiment_group_id, last_status, ExperimentGroupLifeCycle.RUNNING) return def hp_create(): experiment_group.set_status(ExperimentGroupLifeCycle.RUNNING) celery_app.send_task( HPCeleryTasks.HP_CREATE, kwargs={'experiment_group_id': experiment_group_id}, countdown=1) # We start first by creating a build if necessary # No need to build the image, start the experiment directly if not experiment_group.specification.build: hp_create() return _, image_exists, build_status = dockerizer_scheduler.create_build_job( user=experiment_group.user, project=experiment_group.project, config=experiment_group.specification.build, configmap_refs=experiment_group.specification.configmap_refs, secret_refs=experiment_group.specification.secret_refs, code_reference=experiment_group.code_reference) if image_exists: # The image already exists, so we can start the experiment right away hp_create() return if not build_status: experiment_group.set_status(ExperimentGroupLifeCycle.FAILED, message='Could not start build process.') return # We start the group process hp_create()
def new_experiment_status(sender, **kwargs): instance = kwargs['instance'] created = kwargs.get('created', False) experiment_group = instance.experiment_group previous_status = experiment_group.last_status if created: # update experiment last_status experiment_group.status = instance experiment_group.save() auditor.record(event_type=EXPERIMENT_GROUP_NEW_STATUS, instance=experiment_group, previous_status=previous_status) if instance.status == ExperimentGroupLifeCycle.STOPPED: auditor.record(event_type=EXPERIMENT_GROUP_STOPPED, instance=experiment_group, previous_status=previous_status) if ExperimentGroupLifeCycle.is_done(instance.status): auditor.record(event_type=EXPERIMENT_GROUP_FINISHED, instance=experiment_group, previous_status=previous_status)
def new_experiment_status(sender, **kwargs): instance = kwargs['instance'] created = kwargs.get('created', False) experiment_group = instance.experiment_group previous_status = experiment_group.last_status if created: # update experiment last_status experiment_group.status = instance experiment_group.save() auditor.record(event_type=EXPERIMENT_GROUP_NEW_STATUS, instance=experiment_group, previous_status=previous_status) if instance.status == ExperimentGroupLifeCycle.STOPPED: auditor.record(event_type=EXPERIMENT_GROUP_STOPPED, instance=experiment_group, previous_status=previous_status) if ExperimentGroupLifeCycle.is_done(instance.status): auditor.record(event_type=EXPERIMENT_GROUP_FINISHED, instance=experiment_group, previous_status=previous_status)