def experiment_group_status_post_save(sender, **kwargs):
instance = kwargs['instance']
experiment_group = instance.experiment_group
previous_status = experiment_group.last_status
# update experiment last_status
experiment_group.status = instance
if instance.status == ExperimentGroupLifeCycle.RUNNING:
experiment_group.started_at = now()
set_started_at(instance=experiment_group,
status=instance.status,
starting_statuses=[ExperimentGroupLifeCycle.RUNNING])
set_finished_at(instance=experiment_group,
status=instance.status,
is_done=ExperimentGroupLifeCycle.is_done)
experiment_group.save(update_fields=['status', 'started_at', 'finished_at'])
auditor.record(event_type=EXPERIMENT_GROUP_NEW_STATUS,
instance=experiment_group,
previous_status=previous_status)
if instance.status == ExperimentGroupLifeCycle.CREATED:
auditor.record(event_type=EXPERIMENT_GROUP_CREATED, instance=experiment_group)
elif instance.status == ExperimentGroupLifeCycle.STOPPED:
auditor.record(event_type=EXPERIMENT_GROUP_STOPPED,
instance=experiment_group,
previous_status=previous_status)
if ExperimentGroupLifeCycle.is_done(instance.status):
auditor.record(event_type=EXPERIMENT_GROUP_DONE,
instance=experiment_group,
previous_status=previous_status)
def experiments_group_create(self, experiment_group_id):
experiment_group = _get_group_or_retry(
experiment_group_id=experiment_group_id, task=self)
if not experiment_group:
return
last_status = experiment_group.last_status
if not ExperimentGroupLifeCycle.can_transition(
status_from=last_status,
status_to=ExperimentGroupLifeCycle.RUNNING):
_logger.info(
'Experiment group id `%s` cannot transition from `%s` to `%s`.',
experiment_group_id, last_status, ExperimentGroupLifeCycle.RUNNING)
return
def hp_create():
experiment_group.set_status(ExperimentGroupLifeCycle.RUNNING)
celery_app.send_task(
HPCeleryTasks.HP_CREATE,
kwargs={'experiment_group_id': experiment_group_id},
countdown=1)
# We start first by creating a build if necessary
# No need to build the image, start the experiment directly
if not experiment_group.specification.build:
hp_create()
return
_, image_exists, build_status = dockerizer_scheduler.create_build_job(
user=experiment_group.user,
project=experiment_group.project,
config=experiment_group.specification.build,
configmap_refs=experiment_group.specification.configmap_refs,
secret_refs=experiment_group.specification.secret_refs,
code_reference=experiment_group.code_reference)
if image_exists:
# The image already exists, so we can start the experiment right away
hp_create()
return
if not build_status:
experiment_group.set_status(ExperimentGroupLifeCycle.FAILED,
message='Could not start build process.')
return
# We start the group process
hp_create()
def new_experiment_status(sender, **kwargs):
instance = kwargs['instance']
created = kwargs.get('created', False)
experiment_group = instance.experiment_group
previous_status = experiment_group.last_status
if created:
# update experiment last_status
experiment_group.status = instance
experiment_group.save()
auditor.record(event_type=EXPERIMENT_GROUP_NEW_STATUS,
instance=experiment_group,
previous_status=previous_status)
if instance.status == ExperimentGroupLifeCycle.STOPPED:
auditor.record(event_type=EXPERIMENT_GROUP_STOPPED,
instance=experiment_group,
previous_status=previous_status)
if ExperimentGroupLifeCycle.is_done(instance.status):
auditor.record(event_type=EXPERIMENT_GROUP_FINISHED,
instance=experiment_group,
previous_status=previous_status)
def new_experiment_status(sender, **kwargs):
instance = kwargs['instance']
created = kwargs.get('created', False)
experiment_group = instance.experiment_group
previous_status = experiment_group.last_status
if created:
# update experiment last_status
experiment_group.status = instance
experiment_group.save()
auditor.record(event_type=EXPERIMENT_GROUP_NEW_STATUS,
instance=experiment_group,
previous_status=previous_status)
if instance.status == ExperimentGroupLifeCycle.STOPPED:
auditor.record(event_type=EXPERIMENT_GROUP_STOPPED,
instance=experiment_group,
previous_status=previous_status)
if ExperimentGroupLifeCycle.is_done(instance.status):
auditor.record(event_type=EXPERIMENT_GROUP_FINISHED,
instance=experiment_group,
previous_status=previous_status)
