def hp_hyperband_iterate(self, experiment_group_id):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
if experiment_group.non_done_experiments.count() > 0:
# Schedule another task, because all experiment must be done
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
iteration_config = experiment_group.iteration_config
iteration_manager = experiment_group.iteration_manager
search_manager = experiment_group.search_manager
iteration_manager.update_iteration()
if search_manager.should_reschedule(iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration):
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_CREATE,
kwargs={'experiment_group_id': experiment_group_id})
return
if search_manager.should_reduce_configs(iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration):
iteration_manager.reduce_configs()
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_START,
kwargs={'experiment_group_id': experiment_group_id})
return
base.check_group_experiments_finished(experiment_group_id)
def hp_bo_iterate(self, experiment_group_id, auto_retry=False):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
if experiment_group.non_done_experiments.count() > 0:
if auto_retry:
# Schedule another task, because all experiment must be done
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
iteration_config = experiment_group.iteration_config
iteration_manager = experiment_group.iteration_manager
search_manager = experiment_group.search_manager
iteration_manager.update_iteration()
if search_manager.should_reschedule(iteration=iteration_config.iteration):
celery_app.send_task(
HPCeleryTasks.HP_BO_CREATE,
kwargs={'experiment_group_id': experiment_group_id})
return
base.check_group_experiments_finished(experiment_group_id,
auto_retry=auto_retry)
def hp_hyperband_iterate(self, experiment_group_id):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
if experiment_group.non_done_experiments.count() > 0:
# Schedule another task, because all experiment must be done
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
iteration_config = experiment_group.iteration_config
iteration_manager = experiment_group.iteration_manager
search_manager = experiment_group.search_manager
iteration_manager.update_iteration()
if search_manager.should_reschedule(
iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration):
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_CREATE,
kwargs={'experiment_group_id': experiment_group_id})
return
if search_manager.should_reduce_configs(
iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration):
iteration_manager.reduce_configs()
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_START,
kwargs={'experiment_group_id': experiment_group_id})
return
base.check_group_experiments_finished(experiment_group_id)
def hp_grid_search_start(self, experiment_group_id):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
should_retry = base.start_group_experiments(experiment_group=experiment_group)
if should_retry:
# Schedule another task
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
base.check_group_experiments_finished(experiment_group_id)
def hp_random_search_start(self, experiment_group_id):
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
should_retry = base.start_group_experiments(experiment_group=experiment_group)
if should_retry:
# Schedule another task
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
base.check_group_experiments_finished(experiment_group_id)
def hp_grid_search_start(self, experiment_group_id, auto_retry=False):
if not base.should_group_start(experiment_group_id=experiment_group_id,
task=HPCeleryTasks.HP_GRID_SEARCH_START,
auto_retry=auto_retry):
return
experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id)
if not experiment_group:
return
should_retry = base.start_group_experiments(experiment_group=experiment_group)
if should_retry:
if auto_retry:
# Schedule another task
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
base.check_group_experiments_finished(experiment_group_id, auto_retry=auto_retry)
def hp_hyperband_iterate(self, experiment_group_id, auto_retry=False):
experiment_group = get_running_experiment_group(
experiment_group_id=experiment_group_id)
if not experiment_group:
return
if experiment_group.non_done_experiments.count() > 0:
if auto_retry:
# Schedule another task, because all experiment must be done
self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)
return
iteration_config = experiment_group.iteration_config
iteration_manager = experiment_group.iteration_manager
search_manager = experiment_group.search_manager
iteration_manager.update_iteration()
if search_manager.should_reschedule(
iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration):
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_CREATE,
kwargs={'experiment_group_id': experiment_group_id})
return
if search_manager.should_reduce_configs(
iteration=iteration_config.iteration,
bracket_iteration=iteration_config.bracket_iteration):
try:
iteration_manager.reduce_configs()
except ExperimentGroupException:
experiment_group.set_status(
ExperimentGroupLifeCycle.FAILED,
message='Experiment group could not create new iteration.')
return
celery_app.send_task(
HPCeleryTasks.HP_HYPERBAND_START,
kwargs={'experiment_group_id': experiment_group_id})
return
base.check_group_experiments_finished(experiment_group_id,
auto_retry=auto_retry)
