def hp_hyperband_iterate(self, experiment_group_id): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return if experiment_group.non_done_experiments.count() > 0: # Schedule another task, because all experiment must be done self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return iteration_config = experiment_group.iteration_config iteration_manager = experiment_group.iteration_manager search_manager = experiment_group.search_manager iteration_manager.update_iteration() if search_manager.should_reschedule(iteration=iteration_config.iteration, bracket_iteration=iteration_config.bracket_iteration): hp_hyperband_create.delay(experiment_group_id=experiment_group_id) return if search_manager.should_reduce_configs(iteration=iteration_config.iteration, bracket_iteration=iteration_config.bracket_iteration): iteration_manager.reduce_configs() hp_hyperband_start.delay(experiment_group_id=experiment_group_id) return experiment_group.set_status(status=ExperimentGroupLifeCycle.SUCCEEDED)
def hp_bo_create(experiment_group_id): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return create(experiment_group)
def hp_random_search_create(experiment_group_id): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return create(experiment_group)
def hp_bo_start(self, experiment_group_id): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return should_retry = base.start_group_experiments(experiment_group=experiment_group) if should_retry: # Schedule another task self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return hp_bo_iterate.delay(experiment_group_id=experiment_group_id)
def hp_random_search_start(self, experiment_group_id): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return should_retry = base.start_group_experiments( experiment_group=experiment_group) if should_retry: # Schedule another task self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return experiment_group.set_status(status=ExperimentGroupLifeCycle.SUCCEEDED)
def hp_grid_search_start(self, experiment_group_id): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return should_retry = base.start_group_experiments( experiment_group=experiment_group) if should_retry: # Schedule another task self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return base.check_group_experiments_finished(experiment_group_id)
def hp_bo_iterate(self, experiment_group_id): experiment_group = get_running_experiment_group(experiment_group_id=experiment_group_id) if not experiment_group: return if experiment_group.non_done_experiments.count() > 0: # Schedule another task, because all experiment must be done self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return iteration_config = experiment_group.iteration_config iteration_manager = experiment_group.iteration_manager search_manager = experiment_group.search_manager iteration_manager.update_iteration() if search_manager.should_reschedule(iteration=iteration_config.iteration): hp_bo_create.delay(experiment_group_id=experiment_group_id) return base.check_group_experiments_finished(experiment_group_id)
def stop_group_experiments(experiment_group_id, pending, message=None): experiment_group = get_running_experiment_group( experiment_group_id=experiment_group_id) if not experiment_group: return if pending: for experiment in experiment_group.pending_experiments: # Update experiment status to show that its stopped experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message) else: experiments = experiment_group.experiments.exclude( experiment_status__status__in=ExperimentLifeCycle.DONE_STATUS ).distinct() for experiment in experiments: if experiment.is_running: stop_experiment.delay(experiment_id=experiment.id) else: # Update experiment status to show that its stopped experiment.set_status(status=ExperimentLifeCycle.STOPPED, message=message) experiment_group.set_status(ExperimentGroupLifeCycle.STOPPED)