def __init__(self, config_space: ConfigurationSpace, source_hpo_data: List, seed: int, history_dataset_features: List = None, num_src_hpo_trial: int = 50, surrogate_type='rf'): self.method_id = None self.config_space = config_space self.random_seed = seed self.num_src_hpo_trial = num_src_hpo_trial self.source_hpo_data = source_hpo_data self.source_surrogates = None self.target_surrogate = None self.history_dataset_features = history_dataset_features # The number of source problems. if source_hpo_data is not None: self.K = len(source_hpo_data) if history_dataset_features is not None: assert len(history_dataset_features) == self.K self.surrogate_type = surrogate_type self.types, self.bounds = get_types(config_space) self.instance_features = None self.var_threshold = VERY_SMALL_NUMBER self.w = None self.eta_list = list() # meta features. self.meta_feature_scaler = None self.meta_feature_imputer = None self.target_weight = list() self.logger = get_logger(self.__class__.__name__)
def evaluate(mth, run_i, seed): print(mth, run_i, seed, '===== start =====', flush=True) def objective_function(config): y = problem.evaluate_config(config) return y from cma import CMAEvolutionStrategy from litebo.utils.util_funcs import get_types from litebo.utils.config_space import Configuration types, bounds = get_types(cs) assert all(types == 0) # Check Constant Hyperparameter const_idx = list() for i, bound in enumerate(bounds): if np.isnan(bound[1]): const_idx.append(i) hp_num = len(bounds) - len(const_idx) es = CMAEvolutionStrategy(hp_num * [0], 0.99, inopts={'bounds': [0, 1], 'seed': seed}) global_start_time = time.time() global_trial_counter = 0 config_list = [] perf_list = [] time_list = [] eval_num = 0 while eval_num < max_runs: X = es.ask(number=es.popsize) _X = X.copy() for i in range(len(_X)): for index in const_idx: _X[i] = np.insert(_X[i], index, 0) # np.insert returns a copy # _X = np.asarray(_X) values = [] for xi in _X: # convert array to Configuration config = Configuration(cs, vector=xi) perf = objective_function(config) global_time = time.time() - global_start_time global_trial_counter += 1 values.append(perf) print('=== CMAES Trial %d: %s perf=%f global_time=%f' % (global_trial_counter, config, perf, global_time)) config_list.append(config) perf_list.append(perf) time_list.append(global_time) values = np.reshape(values, (-1,)) es.tell(X, values) eval_num += es.popsize print('===== Total evaluation times=%d. Truncate to max_runs=%d.' % (eval_num, max_runs)) config_list = config_list[:max_runs] perf_list = perf_list[:max_runs] time_list = time_list[:max_runs] return config_list, perf_list, time_list
def build_surrogate(func_str='prf', config_space=None, rng=None, history_hpo_data=None): assert config_space is not None func_str = func_str.lower() types, bounds = get_types(config_space) seed = rng.randint(MAXINT) if func_str == 'prf': return RandomForestWithInstances(types=types, bounds=bounds, seed=seed) elif func_str.startswith('gp'): return create_gp_model(model_type=func_str, config_space=config_space, types=types, bounds=bounds, rng=rng) elif func_str.startswith('mfgpe'): from litebo.surrogate.tlbo.mfgpe import MFGPE inner_surrogate_type = 'prf' return MFGPE(config_space, history_hpo_data, seed, surrogate_type=inner_surrogate_type, num_src_hpo_trial=-1) elif func_str.startswith('tlbo'): print('the current surrogate is', func_str) if 'rgpe' in func_str: from litebo.surrogate.tlbo.rgpe import RGPE inner_surrogate_type = func_str.split('_')[-1] return RGPE(config_space, history_hpo_data, seed, surrogate_type=inner_surrogate_type, num_src_hpo_trial=-1) elif 'sgpr' in func_str: from litebo.surrogate.tlbo.stacking_gpr import SGPR inner_surrogate_type = func_str.split('_')[-1] return SGPR(config_space, history_hpo_data, seed, surrogate_type=inner_surrogate_type, num_src_hpo_trial=-1) elif 'topov3' in func_str: from litebo.surrogate.tlbo.topo_variant3 import TOPO_V3 inner_surrogate_type = func_str.split('_')[-1] return TOPO_V3(config_space, history_hpo_data, seed, surrogate_type=inner_surrogate_type, num_src_hpo_trial=-1) else: raise ValueError('Invalid string %s for tlbo surrogate!' % func_str) else: raise ValueError('Invalid string %s for surrogate!' % func_str)
def __init__(self, objective_function, config_space, sample_strategy='bo', time_limit_per_trial=180, max_runs=200, logging_dir='logs', initial_configurations=None, initial_batch=1, batch_size=3, task_id=None, rng=None): super().__init__(config_space, task_id, output_dir=logging_dir) self.logger = super()._get_logger(self.__class__.__name__) if rng is None: run_id, rng = get_rng() self.batch_size = batch_size self.init_batch = initial_batch self.max_iterations = max_runs self.iteration_id = 0 self.sls_max_steps = None self.n_sls_iterations = 5 self.sls_n_steps_plateau_walk = 10 self.time_limit_per_trial = time_limit_per_trial self.default_obj_value = MAXINT self.sample_strategy = sample_strategy self.configurations = list() self.failed_configurations = list() self.perfs = list() # Initialize the basic component in BO. self.config_space.seed(rng.randint(MAXINT)) self.objective_function = objective_function types, bounds = get_types(config_space) # TODO: what is the feature array. self.model = RandomForestWithInstances(types=types, bounds=bounds, seed=rng.randint(MAXINT)) if self.sample_strategy == 'local_penalization': self.acquisition_function = LPEI(self.model) else: self.acquisition_function = EI(self.model) self.optimizer = InterleavedLocalAndRandomSearch( acquisition_function=self.acquisition_function, config_space=self.config_space, rng=np.random.RandomState(seed=rng.randint(MAXINT)), max_steps=self.sls_max_steps, n_steps_plateau_walk=self.sls_n_steps_plateau_walk, n_sls_iterations=self.n_sls_iterations) self._random_search = RandomSearch(self.acquisition_function, self.config_space, rng) self.random_configuration_chooser = ChooserProb(prob=0.25, rng=rng)
def __init__( self, acquisition_function: AbstractAcquisitionFunction, config_space: ConfigurationSpace, rand_prob: float = 0.0, rng: Union[bool, np.random.RandomState] = None, ): super().__init__(acquisition_function, config_space, rng) self.random_chooser = ChooserProb(prob=rand_prob, rng=rng) types, bounds = get_types(self.config_space) assert all(types == 0) self.bounds = bounds
def __init__(self, acquisition_function: AbstractAcquisitionFunction, config_space: ConfigurationSpace, rng: Union[bool, np.random.RandomState] = None, batch_size=None, rand_prob=0.0): super().__init__(acquisition_function, config_space, rng) self.random_chooser = ChooserProb(prob=rand_prob, rng=rng) if batch_size is None: types, bounds = get_types(self.config_space) dim = np.sum(types == 0) self.batch_size = min(5000, max(2000, 200 * dim)) else: self.batch_size = batch_size
def maximize(self, runhistory: HistoryContainer, num_points: int, **kwargs) -> Iterable[Tuple[float, Configuration]]: try: from cma import CMAEvolutionStrategy except ImportError: raise ImportError("Package cma is not installed!") types, bounds = get_types(self.config_space) assert all(types == 0) # Check Constant Hyperparameter const_idx = list() for i, bound in enumerate(bounds): if np.isnan(bound[1]): const_idx.append(i) hp_num = len(bounds) - len(const_idx) es = CMAEvolutionStrategy(hp_num * [0], 0.99, inopts={'bounds': [0, 1]}) eval_num = 0 next_configs_by_acq_value = list() while eval_num < num_points: X = es.ask(number=es.popsize) _X = X.copy() for i in range(len(_X)): for index in const_idx: _X[i] = np.insert(_X[i], index, 0) _X = np.asarray(_X) values = self.acquisition_function._compute(_X) values = np.reshape(values, (-1, )) es.tell(X, values) next_configs_by_acq_value.extend([(values[i], _X[i]) for i in range(es.popsize)]) eval_num += es.popsize next_configs_by_acq_value.sort(reverse=True, key=lambda x: x[0]) next_configs_by_acq_value = [_[1] for _ in next_configs_by_acq_value] next_configs_by_acq_value = [ Configuration(self.config_space, vector=array) for array in next_configs_by_acq_value ] challengers = ChallengerList(next_configs_by_acq_value, self.config_space, self.random_chooser) self.random_chooser.next_smbo_iteration() return challengers
def __init__( self, acquisition_function: AbstractAcquisitionFunction, config_space: ConfigurationSpace, rand_prob: float = 0.0, rng: Union[bool, np.random.RandomState] = None, ): super().__init__(acquisition_function, config_space, rng) self.random_chooser = ChooserProb(prob=rand_prob, rng=rng) types, bounds = get_types(self.config_space) assert all(types == 0) self.bounds = bounds options = dict(disp=False, maxiter=1000) self.scipy_config = dict(tol=None, method='L-BFGS-B', options=options)
def setup_bo_basics(self): if self.num_objs == 1 or self.acq_type == 'parego': self.surrogate_model = build_surrogate( func_str=self.surrogate_type, config_space=self.config_space, rng=self.rng, history_hpo_data=self.history_bo_data) else: # multi-objectives self.surrogate_model = [ build_surrogate(func_str=self.surrogate_type, config_space=self.config_space, rng=self.rng, history_hpo_data=self.history_bo_data) for _ in range(self.num_objs) ] if self.num_constraints > 0: self.constraint_models = [ build_surrogate(func_str=self.constraint_surrogate_type, config_space=self.config_space, rng=self.rng) for _ in range(self.num_constraints) ] if self.acq_type in ['mesmo', 'mesmoc', 'mesmoc2', 'usemo']: types, bounds = get_types(self.config_space) self.acquisition_function = build_acq_func( func_str=self.acq_type, model=self.surrogate_model, constraint_models=self.constraint_models, types=types, bounds=bounds) else: self.acquisition_function = build_acq_func( func_str=self.acq_type, model=self.surrogate_model, constraint_models=self.constraint_models, ref_point=self.ref_point) if self.acq_type == 'usemo': self.acq_optimizer_type = 'usemo_optimizer' elif self.acq_type.startswith('mesmo'): self.acq_optimizer_type = 'mesmo_optimizer' self.optimizer = build_optimizer(func_str=self.acq_optimizer_type, acq_func=self.acquisition_function, config_space=self.config_space, rng=self.rng)
def setup_bo_basics(self): if self.num_objs == 1: self.surrogate_model = build_surrogate( func_str=self.surrogate_type, config_space=self.config_space, rng=self.rng, history_hpo_data=self.history_bo_data) else: # multi-objectives self.surrogate_model = [ build_surrogate(func_str=self.surrogate_type, config_space=self.config_space, rng=self.rng, history_hpo_data=self.history_bo_data) for _ in range(self.num_objs) ] if self.num_constraints > 0: self.constraint_models = [ build_surrogate(func_str=self.constraint_surrogate_type, config_space=self.config_space, rng=self.rng) for _ in range(self.num_constraints) ] self.acquisition_function = build_acq_func( func_str=self.acq_type, model=self.surrogate_model, constraint_models=self.constraint_models, mc_times=self.mc_times, ref_point=self.ref_point) self.optimizer = build_optimizer(func_str=self.acq_optimizer_type, acq_func=self.acquisition_function, config_space=self.config_space, rng=self.rng) if self.use_trust_region: types, bounds = get_types(self.config_space) cont_dim = np.sum(types == 0) self.turbo_state = TurboState(cont_dim) else: self.turbo_state = None
def __init__(self, config_space: ConfigurationSpace, size, lower_bounds=None, upper_bounds=None, random_state=None): """ Parameters ---------- config_space : ConfigurationSpace ConfigurationSpace to do sampling. size : int N Number of samples. lower_bounds : lower bounds in [0, 1] for continuous dimensions (optional) upper_bounds : upper bounds in [0, 1] for continuous dimensions (optional) """ self.config_space = config_space types, bounds = get_types(config_space) self.search_dims = [] for i in range(len(types)): if types[i] == 0 and bounds[i][1] == 1.0: # Integer and float self.search_dims.append((0.0, 1.0)) elif types[i] > 0: # Categorical self.search_dims.append(list(range(types[i]))) else: raise NotImplementedError() self.size = size default_lb, default_ub = zip(*bounds) self.lower_bounds = np.array( default_lb) if lower_bounds is None else np.clip( lower_bounds, default_lb, default_ub) self.upper_bounds = np.array( default_ub) if upper_bounds is None else np.clip( upper_bounds, default_lb, default_ub) self.rng = check_random_state(random_state)