def hp_bo_create(experiment_group_id):
experiment_group = get_valid_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
create(experiment_group)
def hp_random_search_create(experiment_group_id):
experiment_group = get_valid_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
create(experiment_group)
def check_group_finished(self, experiment_group_id):
experiment_group = get_valid_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group.non_done_experiments.exists():
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
experiment_group.set_status(status=ExperimentGroupLifeCycle.SUCCEEDED)
def hp_hyperband_start(self, experiment_group_id):
experiment_group = get_valid_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
should_retry = base.start_group_experiments(experiment_group=experiment_group)
if should_retry:
# Schedule another task
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
hp_hyperband_iterate.delay(experiment_group_id=experiment_group_id)
def _get_group_or_retry(experiment_group_id, task):
experiment_group = get_valid_experiment_group(experiment_group_id=experiment_group_id)
if experiment_group:
return experiment_group
# We retry if experiment group does not exist
if task.request.retries < 2:
logger.info('Trying again for ExperimentGroup `%s`.', experiment_group_id)
task.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
logger.info('Something went wrong, '
'the ExperimentGroup `%s` does not exist anymore.', experiment_group_id)
return None
def stop_group_experiments(experiment_group_id, pending, message=None):
experiment_group = get_valid_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
if pending:
for experiment in experiment_group.pending_experiments:
# Update experiment status to show that its stopped
experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message)
else:
experiments = experiment_group.experiments.exclude(
experiment_status__status__in=ExperimentLifeCycle.DONE_STATUS).distinct()
for experiment in experiments:
if experiment.is_running:
stop_experiment.delay(experiment_id=experiment.id)
else:
# Update experiment status to show that its stopped
experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message)
def hp_bo_iterate(self, experiment_group_id):
experiment_group = get_valid_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
if experiment_group.non_done_experiments.count() > 0:
# Schedule another task, because all experiment must be done
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
iteration_config = experiment_group.iteration_config
iteration_manager = experiment_group.iteration_manager
search_manager = experiment_group.search_manager
iteration_manager.update_iteration()
if search_manager.should_reschedule(iteration=iteration_config.iteration):
hp_bo_create.delay(experiment_group_id=experiment_group_id)
return