def hp_hyperband_iterate(self, experiment_group_id):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
if experiment_group.non_done_experiments.count() > 0:
# Schedule another task, because all experiment must be done
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
iteration_config = experiment_group.iteration_config
iteration_manager = experiment_group.iteration_manager
search_manager = experiment_group.search_manager
iteration_manager.update_iteration()
if search_manager.should_reschedule(iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration):
hp_hyperband_create.delay(experiment_group_id=experiment_group_id)
return
if search_manager.should_reduce_configs(iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration):
iteration_manager.reduce_configs()
hp_hyperband_start.delay(experiment_group_id=experiment_group_id)
return
experiment_group.set_status(status=ExperimentGroupLifeCycle.SUCCEEDED)
def hp_bo_create(experiment_group_id):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
create(experiment_group)
def hp_random_search_create(experiment_group_id):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
create(experiment_group)
def hp_bo_start(self, experiment_group_id):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
should_retry = base.start_group_experiments(experiment_group=experiment_group)
if should_retry:
# Schedule another task
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
hp_bo_iterate.delay(experiment_group_id=experiment_group_id)
def hp_random_search_start(self, experiment_group_id):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
should_retry = base.start_group_experiments(
experiment_group=experiment_group)
if should_retry:
# Schedule another task
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
experiment_group.set_status(status=ExperimentGroupLifeCycle.SUCCEEDED)
def hp_grid_search_start(self, experiment_group_id):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
should_retry = base.start_group_experiments(
experiment_group=experiment_group)
if should_retry:
# Schedule another task
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
base.check_group_experiments_finished(experiment_group_id)
def hp_bo_iterate(self, experiment_group_id):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
if experiment_group.non_done_experiments.count() > 0:
# Schedule another task, because all experiment must be done
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
iteration_config = experiment_group.iteration_config
iteration_manager = experiment_group.iteration_manager
search_manager = experiment_group.search_manager
iteration_manager.update_iteration()
if search_manager.should_reschedule(iteration=iteration_config.iteration):
hp_bo_create.delay(experiment_group_id=experiment_group_id)
return
base.check_group_experiments_finished(experiment_group_id)
def stop_group_experiments(experiment_group_id, pending, message=None):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
if pending:
for experiment in experiment_group.pending_experiments:
# Update experiment status to show that its stopped
experiment.set_status(status=ExperimentLifeCycle.STOPPED,
message=message)
else:
experiments = experiment_group.experiments.exclude(
experiment_status__status__in=ExperimentLifeCycle.DONE_STATUS
).distinct()
for experiment in experiments:
if experiment.is_running:
stop_experiment.delay(experiment_id=experiment.id)
else:
# Update experiment status to show that its stopped
experiment.set_status(status=ExperimentLifeCycle.STOPPED,
message=message)
experiment_group.set_status(ExperimentGroupLifeCycle.STOPPED)