def hp_hyperband_iterate(self, experiment_group_id): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return if experiment_group.non_done_experiments.count() > 0: # Schedule another task, because all experiment must be done self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return iteration_config = experiment_group.iteration_config iteration_manager = experiment_group.iteration_manager search_manager = experiment_group.search_manager iteration_manager.update_iteration() if search_manager.should_reschedule(iteration=iteration_config.iteration, bracket_iteration=iteration_config.bracket_iteration): celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_CREATE, kwargs={'experiment_group_id': experiment_group_id}) return if search_manager.should_reduce_configs(iteration=iteration_config.iteration, bracket_iteration=iteration_config.bracket_iteration): iteration_manager.reduce_configs() celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_START, kwargs={'experiment_group_id': experiment_group_id}) return base.check_group_experiments_finished(experiment_group_id)
def hp_bo_iterate(self, experiment_group_id, auto_retry=False): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return if experiment_group.non_done_experiments.count() > 0: if auto_retry: # Schedule another task, because all experiment must be done self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return iteration_config = experiment_group.iteration_config iteration_manager = experiment_group.iteration_manager search_manager = experiment_group.search_manager iteration_manager.update_iteration() if search_manager.should_reschedule(iteration=iteration_config.iteration): celery_app.send_task( HPCeleryTasks.HP_BO_CREATE, kwargs={'experiment_group_id': experiment_group_id}) return base.check_group_experiments_finished(experiment_group_id, auto_retry=auto_retry)
def hp_hyperband_iterate(self, experiment_group_id): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return if experiment_group.non_done_experiments.count() > 0: # Schedule another task, because all experiment must be done self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return iteration_config = experiment_group.iteration_config iteration_manager = experiment_group.iteration_manager search_manager = experiment_group.search_manager iteration_manager.update_iteration() if search_manager.should_reschedule( iteration=iteration_config.iteration, bracket_iteration=iteration_config.bracket_iteration): celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_CREATE, kwargs={'experiment_group_id': experiment_group_id}) return if search_manager.should_reduce_configs( iteration=iteration_config.iteration, bracket_iteration=iteration_config.bracket_iteration): iteration_manager.reduce_configs() celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_START, kwargs={'experiment_group_id': experiment_group_id}) return base.check_group_experiments_finished(experiment_group_id)
def hp_grid_search_start(self, experiment_group_id): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return should_retry = base.start_group_experiments(experiment_group=experiment_group) if should_retry: # Schedule another task self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return base.check_group_experiments_finished(experiment_group_id)
def hp_random_search_start(self, experiment_group_id): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return should_retry = base.start_group_experiments(experiment_group=experiment_group) if should_retry: # Schedule another task self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return base.check_group_experiments_finished(experiment_group_id)
def hp_grid_search_start(self, experiment_group_id, auto_retry=False): if not base.should_group_start(experiment_group_id=experiment_group_id, task=HPCeleryTasks.HP_GRID_SEARCH_START, auto_retry=auto_retry): return experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return should_retry = base.start_group_experiments(experiment_group=experiment_group) if should_retry: if auto_retry: # Schedule another task self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return base.check_group_experiments_finished(experiment_group_id, auto_retry=auto_retry)
def hp_hyperband_iterate(self, experiment_group_id, auto_retry=False): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return if experiment_group.non_done_experiments.count() > 0: if auto_retry: # Schedule another task, because all experiment must be done self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return iteration_config = experiment_group.iteration_config iteration_manager = experiment_group.iteration_manager search_manager = experiment_group.search_manager iteration_manager.update_iteration() if search_manager.should_reschedule( iteration=iteration_config.iteration, bracket_iteration=iteration_config.bracket_iteration): celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_CREATE, kwargs={'experiment_group_id': experiment_group_id}) return if search_manager.should_reduce_configs( iteration=iteration_config.iteration, bracket_iteration=iteration_config.bracket_iteration): try: iteration_manager.reduce_configs() except ExperimentGroupException: experiment_group.set_status( ExperimentGroupLifeCycle.FAILED, message='Experiment group could not create new iteration.') return celery_app.send_task( HPCeleryTasks.HP_HYPERBAND_START, kwargs={'experiment_group_id': experiment_group_id}) return base.check_group_experiments_finished(experiment_group_id, auto_retry=auto_retry)