def get_mixed_gp(cat_dims, cont_dims, rs, noise=1e-3, normalize_y=True): from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel, HammingKernel cat_dims = np.array(cat_dims, dtype=np.int) cont_dims = np.array(cont_dims, dtype=np.int) n_dimensions = len(cat_dims) + len(cont_dims) cov_amp = ConstantKernel( 2.0, constant_value_bounds=(1e-10, 2), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rs), ) exp_kernel = Matern( np.ones([len(cont_dims)]), [(np.exp(-10), np.exp(2)) for _ in range(len(cont_dims))], nu=2.5, operate_on=cont_dims, ) ham_kernel = HammingKernel( np.ones([len(cat_dims)]), [(np.exp(-10), np.exp(2)) for _ in range(len(cat_dims))], operate_on=cat_dims, ) noise_kernel = WhiteKernel( noise_level=noise, noise_level_bounds=(1e-10, 2), prior=HorseshoePrior(scale=0.1, rng=rs), ) kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel bounds = [0] * n_dimensions types = np.zeros(n_dimensions) for c in cont_dims: bounds[c] = (0., 1.) for c in cat_dims: types[c] = 3 bounds[c] = (3, np.nan) cs = ConfigurationSpace() for c in cont_dims: cs.add_hyperparameter(UniformFloatHyperparameter('X%d' % c, 0, 1)) for c in cat_dims: cs.add_hyperparameter( CategoricalHyperparameter('X%d' % c, [0, 1, 2, 3])) model = GaussianProcess( configspace=cs, bounds=bounds, types=types, kernel=kernel, seed=rs.randint(low=1, high=10000), normalize_y=normalize_y, ) return model
def get_gp(n_dimensions, rs, noise=1e-3, normalize_y=True, average_samples=False, n_iter=50): from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel cov_amp = ConstantKernel( 2.0, constant_value_bounds=(1e-10, 2), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rs), ) exp_kernel = Matern( np.ones([n_dimensions]), [(np.exp(-10), np.exp(2)) for _ in range(n_dimensions)], nu=2.5, prior=None, ) noise_kernel = WhiteKernel( noise_level=noise, noise_level_bounds=(1e-10, 2), prior=HorseshoePrior(scale=0.1, rng=rs), ) kernel = cov_amp * exp_kernel + noise_kernel n_mcmc_walkers = 3 * len(kernel.theta) if n_mcmc_walkers % 2 == 1: n_mcmc_walkers += 1 bounds = [(0., 1.) for _ in range(n_dimensions)] types = np.zeros(n_dimensions) configspace = ConfigurationSpace() for i in range(n_dimensions): configspace.add_hyperparameter( UniformFloatHyperparameter('x%d' % i, 0, 1)) model = GaussianProcessMCMC( configspace=configspace, types=types, bounds=bounds, kernel=kernel, n_mcmc_walkers=n_mcmc_walkers, chain_length=n_iter, burnin_steps=n_iter, normalize_y=normalize_y, seed=rs.randint(low=1, high=10000), mcmc_sampler='emcee', average_samples=average_samples, ) return model
def _construct_model(configspace, rng): types, bounds = _configspace_to_types_and_bounds(configspace) cont_dims = np.nonzero(types == 0)[0] cat_dims = np.nonzero(types != 0)[0] cov_amp = ConstantKernel( 2.0, constant_value_bounds=(np.exp(-10), np.exp(2)), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng), ) if len(cont_dims) > 0: exp_kernel = Matern( np.ones([len(cont_dims)]), [(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cont_dims))], nu=2.5, operate_on=cont_dims, ) if len(cat_dims) > 0: ham_kernel = HammingKernel( np.ones([len(cat_dims)]), [(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cat_dims))], operate_on=cat_dims, ) noise_kernel = WhiteKernel( noise_level=1e-8, noise_level_bounds=(np.exp(-25), np.exp(2)), prior=HorseshoePrior(scale=0.1, rng=rng), ) if len(cont_dims) > 0 and len(cat_dims) > 0: # both kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel elif len(cont_dims) > 0 and len(cat_dims) == 0: # only cont kernel = cov_amp * exp_kernel + noise_kernel elif len(cont_dims) == 0 and len(cat_dims) > 0: # only cont kernel = cov_amp * ham_kernel + noise_kernel else: raise ValueError() def _impute_inactive(self, X): X = X.copy() return _impute_conditional_data(X, self.configspace) seed = random.randint(0, 100) GaussianProcess._impute_inactive = _impute_inactive return GaussianProcess( configspace=configspace, types=types, bounds=bounds, seed=seed, kernel=kernel )
def test_gradient(self): for sigma in (0.5, 1., 2.): prior = LognormalPrior(mean=0, sigma=sigma, rng=np.random.RandomState(1)) # The function appears to be unstable above 15 for theta in range(0, 15): # Gradient approximation becomes unstable when going closer to zero theta += 1e-2 error = scipy.optimize.check_grad( lambda x: prior.lnprob(x[0]), lambda x: prior.gradient(x[0]), np.array([theta]), epsilon=1e-5, ) self.assertAlmostEqual(error, 0, delta=5, msg=theta)
def get_gp(n_dimensions, rs, noise=1e-3, normalize_y=True) -> GaussianProcess: from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel cov_amp = ConstantKernel( 2.0, constant_value_bounds=(1e-10, 2), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rs), ) exp_kernel = Matern( np.ones([n_dimensions]), [(np.exp(-10), np.exp(2)) for _ in range(n_dimensions)], nu=2.5, ) noise_kernel = WhiteKernel( noise_level=noise, noise_level_bounds=(1e-10, 2), prior=HorseshoePrior(scale=0.1, rng=rs), ) kernel = cov_amp * exp_kernel + noise_kernel bounds = [(0., 1.) for _ in range(n_dimensions)] types = np.zeros(n_dimensions) configspace = ConfigurationSpace() for i in range(n_dimensions): configspace.add_hyperparameter( UniformFloatHyperparameter('x%d' % i, 0, 1)) model = GaussianProcess( configspace=configspace, bounds=bounds, types=types, kernel=kernel, seed=rs.randint(low=1, high=10000), normalize_y=normalize_y, n_opt_restarts=2, ) return model
def __init__(self, n_dims: int, rng: np.random.RandomState = None): """ This class is a verbatim copy of the implementation of RoBO: Klein, A. and Falkner, S. and Mansur, N. and Hutter, F. RoBO: A Flexible and Robust Bayesian Optimization Framework in Python In: NIPS 2017 Bayesian Optimization Workshop """ if rng is None: self.rng = np.random.RandomState(np.random.randint(0, 10000)) else: self.rng = rng # The number of hyperparameters self.n_dims = n_dims # Prior for the Matern52 lengthscales self.tophat = TophatPrior(-10, 2, rng=self.rng) # Prior for the covariance amplitude self.ln_prior = LognormalPrior(mean=0.0, sigma=1.0, rng=self.rng) # Prior for the noise self.horseshoe = HorseshoePrior(scale=0.1, rng=self.rng)
def _train(self, X: np.ndarray, y: np.ndarray): """Trains the random forest on X and y. Parameters ---------- X : np.ndarray [n_samples, n_features (config + instance features)] Input data points. Y : np.ndarray [n_samples, ] The corresponding target values. Returns ------- self """ self.X = X self.y = y.flatten() from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel, HammingKernel from smac.epm.gp_base_prior import HorseshoePrior, LognormalPrior self.rf = sklearn.ensemble.RandomForestRegressor( max_features=0.5, bootstrap=True, max_depth=3, min_samples_leaf=10, n_estimators=N_EST, ) # self.rf.fit(X, np.log(y - np.min(y) + 1e-7).ravel()) self.rf.fit(X, y.ravel()) indicators = np.array(self.rf.apply(X)) all_datasets = [] all_targets = [] all_mappings = [] for est in range(N_EST): unique = np.unique(indicators[:, est]) mapping = {j: i for i, j in enumerate(unique)} datasets = [[] for _ in unique] targets = [[] for _ in indicators] for indicator, x, y_ in zip(indicators[:, est], X, y): index = mapping[indicator] datasets[index].append(x) targets[index].append(y_) all_mappings.append(mapping) all_datasets.append(datasets) all_targets.append(targets) # print('Before') # for est in range(N_EST): # for dataset in all_datasets[est]: # print(len(dataset)) for est in range(N_EST): n_nodes = self.rf.estimators_[est].tree_.node_count children_left = self.rf.estimators_[est].tree_.children_left children_right = self.rf.estimators_[est].tree_.children_right feature = self.rf.estimators_[est].tree_.feature threshold = self.rf.estimators_[est].tree_.threshold # The tree structure can be traversed to compute various properties such # as the depth of each node and whether or not it is a leaf. node_depth = np.zeros(shape=n_nodes, dtype=np.int64) is_leaves = np.zeros(shape=n_nodes, dtype=bool) stack = [(0, -1)] # seed is the root node id and its parent depth while len(stack) > 0: node_id, parent_depth = stack.pop() node_depth[node_id] = parent_depth + 1 # If we have a test node if (children_left[node_id] != children_right[node_id]): stack.append((children_left[node_id], parent_depth + 1)) stack.append((children_right[node_id], parent_depth + 1)) else: is_leaves[node_id] = True rules = {} import copy def extend(rule, idx): if is_leaves[idx]: rules[idx] = rule else: rule_left = copy.deepcopy(rule) rule_left.append((threshold[idx], '<=', feature[idx])) extend(rule_left, children_left[idx]) rule_right = copy.deepcopy(rule) rule_right.append((threshold[idx], '>', feature[idx])) extend(rule_right, children_right[idx]) extend([], 0) #print(rules) for key, rule in rules.items(): lower = -np.ones((X.shape[1], )) * np.inf upper = np.ones((X.shape[1], )) * np.inf for element in rule: if element[1] == '<=': if element[0] < upper[element[2]]: upper[element[2]] = element[0] else: if element[0] > lower[element[2]]: lower[element[2]] = element[0] for feature_idx in range(X.shape[1]): closest_lower = -np.inf closes_lower_idx = None closest_upper = np.inf closest_upper_idx = None for x in X: if x[feature_idx] > lower[feature_idx] and x[ feature_idx] < upper[feature_idx]: continue if x[feature_idx] <= lower[feature_idx]: if x[feature_idx] > closest_lower: closest_lower = x[feature_idx] closes_lower_idx = feature_idx if x[feature_idx] >= upper[feature_idx]: if x[feature_idx] < closest_upper: closest_upper = x[feature_idx] closest_upper_idx = feature_idx if closest_upper_idx is not None: all_datasets[est][all_mappings[est][key]].append( X[closest_upper_idx]) all_targets[est][all_mappings[est][key]].append( y[closest_upper_idx]) if closes_lower_idx is not None: all_datasets[est][all_mappings[est][key]].append( X[closes_lower_idx]) all_targets[est][all_mappings[est][key]].append( y[closes_lower_idx]) # print('After') # for est in range(N_EST): # for dataset in all_datasets[est]: # print(len(dataset)) self.all_mappings = all_mappings self.models = [] for est in range(N_EST): models = [] for dataset, targets_ in zip(all_datasets[est], all_targets[est]): cov_amp = ConstantKernel( 2.0, constant_value_bounds=(np.exp(-10), np.exp(2)), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=self.rng), ) cont_dims = np.nonzero(self.types == 0)[0] cat_dims = np.nonzero(self.types != 0)[0] if len(cont_dims) > 0: exp_kernel = Matern( np.ones([len(cont_dims)]), [(np.exp(-10), np.exp(2)) for _ in range(len(cont_dims))], nu=2.5, operate_on=cont_dims, ) if len(cat_dims) > 0: ham_kernel = HammingKernel( np.ones([len(cat_dims)]), [(np.exp(-10), np.exp(2)) for _ in range(len(cat_dims))], operate_on=cat_dims, ) noise_kernel = WhiteKernel( noise_level=1e-8, noise_level_bounds=(np.exp(-25), np.exp(2)), prior=HorseshoePrior(scale=0.1, rng=self.rng), ) if len(cont_dims) > 0 and len(cat_dims) > 0: # both kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel elif len(cont_dims) > 0 and len(cat_dims) == 0: # only cont kernel = cov_amp * exp_kernel + noise_kernel elif len(cont_dims) == 0 and len(cat_dims) > 0: # only cont kernel = cov_amp * ham_kernel + noise_kernel else: raise ValueError() gp = GaussianProcess( configspace=self.configspace, types=self.types, bounds=self.bounds, kernel=kernel, normalize_y=True, seed=self.rng.randint(low=0, high=10000), ) gp.train(np.array(dataset), np.array(targets_)) gp._train(X, y, do_optimize=False) models.append(gp) self.models.append(models) return self
def __init__(self, api_config, config_space, parallel_setting="LS"): super(SMAC4EPMOpimizer, self).__init__(api_config) self.cs = config_space self.num_hps = len(self.cs.get_hyperparameters()) if parallel_setting not in ["CL_min", "CL_max", "CL_mean", "KB", "LS"]: raise ValueError( "parallel_setting can only be one of the following: " "CL_min, CL_max, CL_mean, KB, LS") self.parallel_setting = parallel_setting rng = np.random.RandomState(seed=0) scenario = Scenario({ "run_obj": "quality", # we optimize quality (alt. to runtime) "runcount-limit": 128, "cs": self.cs, # configuration space "deterministic": True, "limit_resources": False, }) self.stats = Stats(scenario) # traj = TrajLogger(output_dir=None, stats=self.stats) self.runhistory = RunHistory() r2e_def_kwargs = { "scenario": scenario, "num_params": self.num_hps, "success_states": [ StatusType.SUCCESS, ], "impute_censored_data": False, "scale_perc": 5, } self.random_chooser = ChooserProb(rng=rng, prob=0.0) types, bounds = get_types(self.cs, instance_features=None) model_kwargs = { "configspace": self.cs, "types": types, "bounds": bounds, "seed": rng.randint(MAXINT), } models = [] cov_amp = ConstantKernel( 2.0, constant_value_bounds=(np.exp(-10), np.exp(2)), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng), ) cont_dims = np.array(np.where(np.array(types) == 0)[0], dtype=np.int) cat_dims = np.where(np.array(types) != 0)[0] if len(cont_dims) > 0: exp_kernel = Matern( np.ones([len(cont_dims)]), [(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cont_dims))], nu=2.5, operate_on=cont_dims, ) if len(cat_dims) > 0: ham_kernel = HammingKernel( np.ones([len(cat_dims)]), [(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cat_dims))], operate_on=cat_dims, ) assert len(cont_dims) + len(cat_dims) == len( scenario.cs.get_hyperparameters()) noise_kernel = WhiteKernel( noise_level=1e-8, noise_level_bounds=(np.exp(-25), np.exp(2)), prior=HorseshoePrior(scale=0.1, rng=rng), ) if len(cont_dims) > 0 and len(cat_dims) > 0: # both kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel elif len(cont_dims) > 0 and len(cat_dims) == 0: # only cont kernel = cov_amp * exp_kernel + noise_kernel elif len(cont_dims) == 0 and len(cat_dims) > 0: # only cont kernel = cov_amp * ham_kernel + noise_kernel else: raise ValueError() gp_kwargs = {"kernel": kernel} rf_kwargs = {} rf_kwargs["num_trees"] = model_kwargs.get("num_trees", 10) rf_kwargs["do_bootstrapping"] = model_kwargs.get( "do_bootstrapping", True) rf_kwargs["ratio_features"] = model_kwargs.get("ratio_features", 1.0) rf_kwargs["min_samples_split"] = model_kwargs.get( "min_samples_split", 2) rf_kwargs["min_samples_leaf"] = model_kwargs.get("min_samples_leaf", 1) rf_kwargs["log_y"] = model_kwargs.get("log_y", True) rf_log = RandomForestWithInstances(**model_kwargs, **rf_kwargs) rf_kwargs = copy.deepcopy(rf_kwargs) rf_kwargs["log_y"] = False rf_no_log = RandomForestWithInstances(**model_kwargs, **rf_kwargs) rh2epm_cost = RunHistory2EPM4Cost(**r2e_def_kwargs) rh2epm_log_cost = RunHistory2EPM4LogScaledCost(**r2e_def_kwargs) rh2epm_copula = RunHistory2EPM4GaussianCopulaCorrect(**r2e_def_kwargs) self.combinations = [] # 2 models * 4 acquisition functions acq_funcs = [EI, PI, LogEI, LCB] acq_func_instances = [] # acq_func_maximizer_instances = [] n_sls_iterations = { 1: 10, 2: 10, 3: 10, 4: 10, 5: 10, 6: 10, 7: 8, 8: 6, }.get(len(self.cs.get_hyperparameters()), 5) acq_func_maximizer_kwargs = { "config_space": self.cs, "rng": rng, "max_steps": 5, "n_steps_plateau_walk": 5, "n_sls_iterations": n_sls_iterations, } self.idx_ei = 0 self.num_models = len(models) self.num_acq_funcs = len(acq_funcs) no_transform_gp = GaussianProcess(**copy.deepcopy(model_kwargs), **copy.deepcopy(gp_kwargs)) ei = EI(model=no_transform_gp) acq_func_maximizer_kwargs["acquisition_function"] = ei ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((no_transform_gp, ei, ei_opt, rh2epm_cost)) pi = PI(model=no_transform_gp) acq_func_maximizer_kwargs["acquisition_function"] = pi pi_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((no_transform_gp, pi, pi_opt, rh2epm_cost)) lcb = LCB(model=no_transform_gp) acq_func_maximizer_kwargs["acquisition_function"] = lcb lcb_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((no_transform_gp, lcb, lcb_opt, rh2epm_cost)) gp = GaussianProcess(**copy.deepcopy(model_kwargs), **copy.deepcopy(gp_kwargs)) ei = EI(model=gp) acq_func_maximizer_kwargs["acquisition_function"] = ei ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((gp, ei, ei_opt, rh2epm_copula)) gp = GaussianProcess(**copy.deepcopy(model_kwargs), **copy.deepcopy(gp_kwargs)) ei = LogEI(model=gp) acq_func_maximizer_kwargs["acquisition_function"] = ei ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((gp, ei, ei_opt, rh2epm_log_cost)) ei = EI(model=rf_no_log) acq_func_maximizer_kwargs["acquisition_function"] = ei ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((rf_no_log, ei, ei_opt, rh2epm_cost)) ei = LogEI(model=rf_log) acq_func_maximizer_kwargs["acquisition_function"] = ei ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((rf_log, ei, ei_opt, rh2epm_log_cost)) ei = EI(model=rf_no_log) acq_func_maximizer_kwargs["acquisition_function"] = ei ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs) self.combinations.append((rf_no_log, ei, ei_opt, rh2epm_copula)) self.num_acq_instances = len(acq_func_instances) self.best_observation = np.inf self.next_evaluations = []
def __init__(self, model_type: str = "gp_mcmc", **kwargs: typing.Any): scenario = kwargs["scenario"] if len(scenario.cs.get_hyperparameters()) <= 21201: kwargs["initial_design"] = kwargs.get("initial_design", SobolDesign) else: raise ValueError( 'The default initial design "Sobol sequence" can only handle up to 21201 dimensions. ' 'Please use a different initial design, such as "the Latin Hypercube design".', ) kwargs["runhistory2epm"] = kwargs.get("runhistory2epm", RunHistory2EPM4Cost) init_kwargs = kwargs.get("initial_design_kwargs", dict()) or dict() init_kwargs["n_configs_x_params"] = init_kwargs.get("n_configs_x_params", 8) init_kwargs["max_config_fracs"] = init_kwargs.get("max_config_fracs", 0.25) kwargs["initial_design_kwargs"] = init_kwargs if kwargs.get("model") is None: model_kwargs = kwargs.get("model_kwargs", dict()) or dict() _, rng = get_rng( rng=kwargs.get("rng", None), run_id=kwargs.get("run_id", None), logger=None, ) types, bounds = get_types(kwargs["scenario"].cs, instance_features=None) cov_amp = ConstantKernel( 2.0, constant_value_bounds=(np.exp(-10), np.exp(2)), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng), ) cont_dims = np.where(np.array(types) == 0)[0] cat_dims = np.where(np.array(types) != 0)[0] if len(cont_dims) > 0: exp_kernel = Matern( np.ones([len(cont_dims)]), [ (np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cont_dims)) ], nu=2.5, operate_on=cont_dims, ) if len(cat_dims) > 0: ham_kernel = HammingKernel( np.ones([len(cat_dims)]), [ (np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cat_dims)) ], operate_on=cat_dims, ) assert (len(cont_dims) + len(cat_dims)) == len( scenario.cs.get_hyperparameters() ) noise_kernel = WhiteKernel( noise_level=1e-8, noise_level_bounds=(np.exp(-25), np.exp(2)), prior=HorseshoePrior(scale=0.1, rng=rng), ) if len(cont_dims) > 0 and len(cat_dims) > 0: # both kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel elif len(cont_dims) > 0 and len(cat_dims) == 0: # only cont kernel = cov_amp * exp_kernel + noise_kernel elif len(cont_dims) == 0 and len(cat_dims) > 0: # only cont kernel = cov_amp * ham_kernel + noise_kernel else: raise ValueError() if model_type == "gp": model_class = GaussianProcess # type: typing.Type[BaseModel] kwargs["model"] = model_class model_kwargs["kernel"] = kernel model_kwargs["normalize_y"] = True model_kwargs["seed"] = rng.randint(0, 2**20) elif model_type == "gp_mcmc": model_class = GaussianProcessMCMC kwargs["model"] = model_class kwargs["integrate_acquisition_function"] = True model_kwargs["kernel"] = kernel n_mcmc_walkers = 3 * len(kernel.theta) if n_mcmc_walkers % 2 == 1: n_mcmc_walkers += 1 model_kwargs["n_mcmc_walkers"] = n_mcmc_walkers model_kwargs["chain_length"] = 250 model_kwargs["burnin_steps"] = 250 model_kwargs["normalize_y"] = True model_kwargs["seed"] = rng.randint(0, 2**20) else: raise ValueError("Unknown model type %s" % model_type) kwargs["model_kwargs"] = model_kwargs if kwargs.get("random_configuration_chooser") is None: random_config_chooser_kwargs = ( kwargs.get( "random_configuration_chooser_kwargs", dict(), ) or dict() ) random_config_chooser_kwargs["prob"] = random_config_chooser_kwargs.get( "prob", 0.08447232371720552 ) kwargs["random_configuration_chooser_kwargs"] = random_config_chooser_kwargs if kwargs.get("acquisition_function_optimizer") is None: acquisition_function_optimizer_kwargs = ( kwargs.get( "acquisition_function_optimizer_kwargs", dict(), ) or dict() ) acquisition_function_optimizer_kwargs["n_sls_iterations"] = 10 kwargs[ "acquisition_function_optimizer_kwargs" ] = acquisition_function_optimizer_kwargs # only 1 configuration per SMBO iteration intensifier_kwargs = kwargs.get("intensifier_kwargs", dict()) or dict() intensifier_kwargs["min_chall"] = 1 kwargs["intensifier_kwargs"] = intensifier_kwargs scenario.intensification_percentage = 1e-10 super().__init__(**kwargs) if self.solver.scenario.n_features > 0: raise NotImplementedError("BOGP cannot handle instances") self.logger.info(self.__class__) self.solver.scenario.acq_opt_challengers = 1000 # type: ignore[attr-defined] # noqa F821 # activate predict incumbent self.solver.epm_chooser.predict_x_best = True
def _component_builder(self, conf:typing.Union[Configuration, dict]) \ -> typing.Tuple[AbstractAcquisitionFunction, AbstractEPM]: """ builds new Acquisition function object and EPM object and returns these Parameters ---------- conf: typing.Union[Configuration, dict] configuration specificing "model" and "acq_func" Returns ------- typing.Tuple[AbstractAcquisitionFunction, AbstractEPM] """ types, bounds = get_types( self.config_space, instance_features=self.scenario.feature_array) if conf["model"] == "RF": model = RandomForestWithInstances( configspace=self.config_space, types=types, bounds=bounds, instance_features=self.scenario.feature_array, seed=self.rng.randint(MAXINT), pca_components=conf.get("pca_dim", self.scenario.PCA_DIM), log_y=conf.get("log_y", self.scenario.transform_y in ["LOG", "LOGS"]), num_trees=conf.get("num_trees", self.scenario.rf_num_trees), do_bootstrapping=conf.get("do_bootstrapping", self.scenario.rf_do_bootstrapping), ratio_features=conf.get("ratio_features", self.scenario.rf_ratio_features), min_samples_split=conf.get("min_samples_split", self.scenario.rf_min_samples_split), min_samples_leaf=conf.get("min_samples_leaf", self.scenario.rf_min_samples_leaf), max_depth=conf.get("max_depth", self.scenario.rf_max_depth), ) elif conf["model"] == "GP": from smac.epm.gp_kernels import ConstantKernel, HammingKernel, WhiteKernel, Matern cov_amp = ConstantKernel( 2.0, constant_value_bounds=(np.exp(-10), np.exp(2)), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=self.rng), ) cont_dims = np.nonzero(types == 0)[0] cat_dims = np.nonzero(types != 0)[0] if len(cont_dims) > 0: exp_kernel = Matern( np.ones([len(cont_dims)]), [(np.exp(-10), np.exp(2)) for _ in range(len(cont_dims))], nu=2.5, operate_on=cont_dims, ) if len(cat_dims) > 0: ham_kernel = HammingKernel( np.ones([len(cat_dims)]), [(np.exp(-10), np.exp(2)) for _ in range(len(cat_dims))], operate_on=cat_dims, ) noise_kernel = WhiteKernel( noise_level=1e-8, noise_level_bounds=(np.exp(-25), np.exp(2)), prior=HorseshoePrior(scale=0.1, rng=self.rng), ) if len(cont_dims) > 0 and len(cat_dims) > 0: # both kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel elif len(cont_dims) > 0 and len(cat_dims) == 0: # only cont kernel = cov_amp * exp_kernel + noise_kernel elif len(cont_dims) == 0 and len(cat_dims) > 0: # only cont kernel = cov_amp * ham_kernel + noise_kernel else: raise ValueError() n_mcmc_walkers = 3 * len(kernel.theta) if n_mcmc_walkers % 2 == 1: n_mcmc_walkers += 1 model = GaussianProcessMCMC( self.config_space, types=types, bounds=bounds, kernel=kernel, n_mcmc_walkers=n_mcmc_walkers, chain_length=250, burnin_steps=250, normalize_y=True, seed=self.rng.randint(low=0, high=10000), ) if conf["acq_func"] == "EI": acq = EI(model=model, par=conf.get("par_ei", 0)) elif conf["acq_func"] == "LCB": acq = LCB(model=model, par=conf.get("par_lcb", 0)) elif conf["acq_func"] == "PI": acq = PI(model=model, par=conf.get("par_pi", 0)) elif conf["acq_func"] == "LogEI": # par value should be in log-space acq = LogEI(model=model, par=conf.get("par_logei", 0)) return acq, model
def get_kernel(self, config_space): """ Code is very similar to those from :class:`smac.SMAC4BO`. Length scale were chosen from SMAC where they were obtained by hyperparameter optimization made in https://arxiv.org/abs/1908.06674. :params config_space: SMAC configuration space with parameters. :type config_space: :class:`smac.configspace.ConfigurationSpace` """ # First of all get type of kernel kernel_cls, nu = self._get_kernel_class_and_nu() # get types and bounds for config_space types, bounds = get_types(config_space, instance_features=None) # get random state for priors rng = self._get_random_state() # create kernel to hold variance cov_amp = smac.epm.gp_kernels.ConstantKernel( 2.0, constant_value_bounds=(np.exp(-10), np.exp(2)), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng), ) # Understand information about parameters cont_dims = np.where(np.array(types) == 0)[0] cat_dims = np.where(np.array(types) != 0)[0] # bounds for length scale from https://arxiv.org/abs/1908.06674 lslims = (np.exp(-6.754111155189306), np.exp(0.0858637988771976)) # create kernel for continuous parameters if len(cont_dims) > 0: exp_kwargs = { "length_scale": np.ones([len(cont_dims)]), "length_scale_bounds": [lslims for _ in range(len(cont_dims))], "operate_on": cont_dims, "prior": None, "has_conditions": False, } if nu is not None: exp_kwargs["nu"] = nu exp_kernel = kernel_cls(**exp_kwargs) # kernel for categorical parameters if len(cat_dims) > 0: ham_kernel = smac.epm.gp_kernels.HammingKernel( length_scale=np.ones([len(cat_dims)]), length_scale_bounds=[lslims for _ in range(len(cat_dims))], operate_on=cat_dims, ) # create noise kernel noise_kernel = smac.epm.gp_kernels.WhiteKernel( noise_level=1e-8, noise_level_bounds=(np.exp(-25), np.exp(2)), prior=HorseshoePrior(scale=0.1, rng=rng), ) # create final kernel as combination if len(cont_dims) > 0 and len(cat_dims) > 0: # both kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel elif len(cont_dims) > 0 and len(cat_dims) == 0: # only continuous parameters kernel = cov_amp * exp_kernel + noise_kernel elif len(cont_dims) == 0 and len(cat_dims) > 0: # only categorical parameters kernel = cov_amp * ham_kernel + noise_kernel return kernel
def __init__(self, model_type='gp_mcmc', **kwargs): """ Constructor see ~smac.facade.smac_facade for documentation """ scenario = kwargs['scenario'] kwargs['initial_design'] = kwargs.get('initial_design', SobolDesign) kwargs['runhistory2epm'] = kwargs.get('runhistory2epm', RunHistory2EPM4Cost) init_kwargs = kwargs.get('initial_design_kwargs', dict()) init_kwargs['n_configs_x_params'] = init_kwargs.get( 'n_configs_x_params', 8) init_kwargs['max_config_fracs'] = init_kwargs.get( 'max_config_fracs', 0.25) kwargs['initial_design_kwargs'] = init_kwargs if kwargs.get('model') is None: from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel, HammingKernel model_kwargs = kwargs.get('model_kwargs', dict()) _, rng = get_rng(rng=kwargs.get("rng", None), run_id=kwargs.get("run_id", None), logger=None) types, bounds = get_types(kwargs['scenario'].cs, instance_features=None) cov_amp = ConstantKernel( 2.0, constant_value_bounds=(np.exp(-10), np.exp(2)), prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng), ) cont_dims = np.nonzero(types == 0)[0] cat_dims = np.nonzero(types != 0)[0] if len(cont_dims) > 0: exp_kernel = Matern( np.ones([len(cont_dims)]), [(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cont_dims))], nu=2.5, operate_on=cont_dims, ) if len(cat_dims) > 0: ham_kernel = HammingKernel( np.ones([len(cat_dims)]), [(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cat_dims))], operate_on=cat_dims, ) noise_kernel = WhiteKernel( noise_level=1e-8, noise_level_bounds=(np.exp(-25), np.exp(2)), prior=HorseshoePrior(scale=0.1, rng=rng), ) if len(cont_dims) > 0 and len(cat_dims) > 0: # both kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel elif len(cont_dims) > 0 and len(cat_dims) == 0: # only cont kernel = cov_amp * exp_kernel + noise_kernel elif len(cont_dims) == 0 and len(cat_dims) > 0: # only cont kernel = cov_amp * ham_kernel + noise_kernel else: raise ValueError() if model_type == "gp": model_class = GaussianProcess kwargs['model'] = model_class model_kwargs['kernel'] = kernel model_kwargs['normalize_y'] = True model_kwargs['seed'] = rng.randint(0, 2**20) elif model_type == "gp_mcmc": model_class = GaussianProcessMCMC kwargs['model'] = model_class kwargs['integrate_acquisition_function'] = True model_kwargs['kernel'] = kernel n_mcmc_walkers = 3 * len(kernel.theta) if n_mcmc_walkers % 2 == 1: n_mcmc_walkers += 1 model_kwargs['n_mcmc_walkers'] = n_mcmc_walkers model_kwargs['chain_length'] = 250 model_kwargs['burnin_steps'] = 250 model_kwargs['normalize_y'] = True model_kwargs['seed'] = rng.randint(0, 2**20) else: raise ValueError('Unknown model type %s' % model_type) kwargs['model_kwargs'] = model_kwargs if kwargs.get('random_configuration_chooser') is None: random_config_chooser_kwargs = kwargs.get( 'random_configuration_chooser_kwargs', dict()) random_config_chooser_kwargs[ 'prob'] = random_config_chooser_kwargs.get( 'prob', 0.08447232371720552) kwargs[ 'random_configuration_chooser_kwargs'] = random_config_chooser_kwargs if kwargs.get('acquisition_function_optimizer') is None: acquisition_function_optimizer_kwargs = kwargs.get( 'acquisition_function_optimizer_kwargs', dict()) acquisition_function_optimizer_kwargs['n_sls_iterations'] = 10 kwargs[ 'acquisition_function_optimizer_kwargs'] = acquisition_function_optimizer_kwargs # only 1 configuration per SMBO iteration intensifier_kwargs = kwargs.get('intensifier_kwargs', dict()) intensifier_kwargs['min_chall'] = 1 kwargs['intensifier_kwargs'] = intensifier_kwargs scenario.intensification_percentage = 1e-10 super().__init__(**kwargs) if self.solver.scenario.n_features > 0: raise NotImplementedError("BOGP cannot handle instances") self.logger.info(self.__class__) self.solver.scenario.acq_opt_challengers = 1000 # activate predict incumbent self.solver.predict_incumbent = True