class DefaultPrior(BasePrior):
def __init__(self, n_dims: int, rng: np.random.RandomState = None):
"""
This class is a verbatim copy of the implementation of RoBO:
Klein, A. and Falkner, S. and Mansur, N. and Hutter, F.
RoBO: A Flexible and Robust Bayesian Optimization Framework in Python
In: NIPS 2017 Bayesian Optimization Workshop
"""
if rng is None:
self.rng = np.random.RandomState(np.random.randint(0, 10000))
else:
self.rng = rng
# The number of hyperparameters
self.n_dims = n_dims
# Prior for the Matern52 lengthscales
self.tophat = TophatPrior(-10, 2, rng=self.rng)
# Prior for the covariance amplitude
self.ln_prior = LognormalPrior(mean=0.0, sigma=1.0, rng=self.rng)
# Prior for the noise
self.horseshoe = HorseshoePrior(scale=0.1, rng=self.rng)
def lnprob(self, theta: np.ndarray):
lp = 0
# Covariance amplitude
lp += self.ln_prior.lnprob(theta[0])
# Lengthscales
lp += self.tophat.lnprob(theta[1:-1])
# Noise
lp += self.horseshoe.lnprob(theta[-1])
return lp
def sample_from_prior(self, n_samples: int):
p0 = np.zeros([n_samples, self.n_dims])
# Covariance amplitude
p0[:, 0] = self.ln_prior.sample_from_prior(n_samples)[:, 0]
# Lengthscales
ls_sample = np.array([
self.tophat.sample_from_prior(n_samples)[:, 0]
for _ in range(1, (self.n_dims - 1))
]).T
p0[:, 1:(self.n_dims - 1)] = ls_sample
# Noise
p0[:, -1] = self.horseshoe.sample_from_prior(n_samples)[:, 0]
return p0
def gradient(self, theta: np.ndarray):
# TODO: Implement real gradient here
return np.zeros([theta.shape[0]])
def test_gradient(self):
for scale in (0.1, 0.5, 1., 2.):
prior = HorseshoePrior(scale=scale, rng=np.random.RandomState(1))
# The function appears to be unstable above 15
for theta in range(-20, 15):
if theta == 0:
continue
error = scipy.optimize.check_grad(
lambda x: prior.lnprob(x[0]),
lambda x: prior.gradient(x[0]),
np.array([theta]),
epsilon=1e-5,
)
self.assertAlmostEqual(error, 0, delta=5)
def get_mixed_gp(cat_dims, cont_dims, rs, noise=1e-3, normalize_y=True):
from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel, HammingKernel
cat_dims = np.array(cat_dims, dtype=np.int)
cont_dims = np.array(cont_dims, dtype=np.int)
n_dimensions = len(cat_dims) + len(cont_dims)
cov_amp = ConstantKernel(
2.0,
constant_value_bounds=(1e-10, 2),
prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rs),
)
exp_kernel = Matern(
np.ones([len(cont_dims)]),
[(np.exp(-10), np.exp(2)) for _ in range(len(cont_dims))],
nu=2.5,
operate_on=cont_dims,
)
ham_kernel = HammingKernel(
np.ones([len(cat_dims)]),
[(np.exp(-10), np.exp(2)) for _ in range(len(cat_dims))],
operate_on=cat_dims,
)
noise_kernel = WhiteKernel(
noise_level=noise,
noise_level_bounds=(1e-10, 2),
prior=HorseshoePrior(scale=0.1, rng=rs),
)
kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel
bounds = [0] * n_dimensions
types = np.zeros(n_dimensions)
for c in cont_dims:
bounds[c] = (0., 1.)
for c in cat_dims:
types[c] = 3
bounds[c] = (3, np.nan)
cs = ConfigurationSpace()
for c in cont_dims:
cs.add_hyperparameter(UniformFloatHyperparameter('X%d' % c, 0, 1))
for c in cat_dims:
cs.add_hyperparameter(
CategoricalHyperparameter('X%d' % c, [0, 1, 2, 3]))
model = GaussianProcess(
configspace=cs,
bounds=bounds,
types=types,
kernel=kernel,
seed=rs.randint(low=1, high=10000),
normalize_y=normalize_y,
)
return model
def _construct_model(configspace, rng):
types, bounds = _configspace_to_types_and_bounds(configspace)
cont_dims = np.nonzero(types == 0)[0]
cat_dims = np.nonzero(types != 0)[0]
cov_amp = ConstantKernel(
2.0,
constant_value_bounds=(np.exp(-10), np.exp(2)),
prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng),
)
if len(cont_dims) > 0:
exp_kernel = Matern(
np.ones([len(cont_dims)]),
[(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in
range(len(cont_dims))],
nu=2.5,
operate_on=cont_dims,
)
if len(cat_dims) > 0:
ham_kernel = HammingKernel(
np.ones([len(cat_dims)]),
[(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in
range(len(cat_dims))],
operate_on=cat_dims,
)
noise_kernel = WhiteKernel(
noise_level=1e-8,
noise_level_bounds=(np.exp(-25), np.exp(2)),
prior=HorseshoePrior(scale=0.1, rng=rng),
)
if len(cont_dims) > 0 and len(cat_dims) > 0:
# both
kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel
elif len(cont_dims) > 0 and len(cat_dims) == 0:
# only cont
kernel = cov_amp * exp_kernel + noise_kernel
elif len(cont_dims) == 0 and len(cat_dims) > 0:
# only cont
kernel = cov_amp * ham_kernel + noise_kernel
else:
raise ValueError()
def _impute_inactive(self, X):
X = X.copy()
return _impute_conditional_data(X, self.configspace)
seed = random.randint(0, 100)
GaussianProcess._impute_inactive = _impute_inactive
return GaussianProcess(
configspace=configspace, types=types, bounds=bounds, seed=seed, kernel=kernel
)
def get_gp(n_dimensions,
rs,
noise=1e-3,
normalize_y=True,
average_samples=False,
n_iter=50):
from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel
cov_amp = ConstantKernel(
2.0,
constant_value_bounds=(1e-10, 2),
prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rs),
)
exp_kernel = Matern(
np.ones([n_dimensions]),
[(np.exp(-10), np.exp(2)) for _ in range(n_dimensions)],
nu=2.5,
prior=None,
)
noise_kernel = WhiteKernel(
noise_level=noise,
noise_level_bounds=(1e-10, 2),
prior=HorseshoePrior(scale=0.1, rng=rs),
)
kernel = cov_amp * exp_kernel + noise_kernel
n_mcmc_walkers = 3 * len(kernel.theta)
if n_mcmc_walkers % 2 == 1:
n_mcmc_walkers += 1
bounds = [(0., 1.) for _ in range(n_dimensions)]
types = np.zeros(n_dimensions)
configspace = ConfigurationSpace()
for i in range(n_dimensions):
configspace.add_hyperparameter(
UniformFloatHyperparameter('x%d' % i, 0, 1))
model = GaussianProcessMCMC(
configspace=configspace,
types=types,
bounds=bounds,
kernel=kernel,
n_mcmc_walkers=n_mcmc_walkers,
chain_length=n_iter,
burnin_steps=n_iter,
normalize_y=normalize_y,
seed=rs.randint(low=1, high=10000),
mcmc_sampler='emcee',
average_samples=average_samples,
)
return model
def __init__(self, n_dims: int, rng: np.random.RandomState = None):
"""
This class is a verbatim copy of the implementation of RoBO:
Klein, A. and Falkner, S. and Mansur, N. and Hutter, F.
RoBO: A Flexible and Robust Bayesian Optimization Framework in Python
In: NIPS 2017 Bayesian Optimization Workshop
"""
if rng is None:
self.rng = np.random.RandomState(np.random.randint(0, 10000))
else:
self.rng = rng
# The number of hyperparameters
self.n_dims = n_dims
# Prior for the Matern52 lengthscales
self.tophat = TophatPrior(-10, 2, rng=self.rng)
# Prior for the covariance amplitude
self.ln_prior = LognormalPrior(mean=0.0, sigma=1.0, rng=self.rng)
# Prior for the noise
self.horseshoe = HorseshoePrior(scale=0.1, rng=self.rng)
def test_lnprob_and_grad_scalar(self):
prior = HorseshoePrior(scale=1, rng=np.random.RandomState(1))
# Legal scalar
self.assertEqual(prior.lnprob(-1), 1.1450937952919953)
self.assertEqual(prior.gradient(-1), -0.6089187456211098)
# Boundary
self.assertTrue(np.isinf(prior._lnprob(0)))
self.assertTrue(np.isinf(prior._gradient(0)))
def get_gp(n_dimensions, rs, noise=1e-3, normalize_y=True) -> GaussianProcess:
from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel
cov_amp = ConstantKernel(
2.0,
constant_value_bounds=(1e-10, 2),
prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rs),
)
exp_kernel = Matern(
np.ones([n_dimensions]),
[(np.exp(-10), np.exp(2)) for _ in range(n_dimensions)],
nu=2.5,
)
noise_kernel = WhiteKernel(
noise_level=noise,
noise_level_bounds=(1e-10, 2),
prior=HorseshoePrior(scale=0.1, rng=rs),
)
kernel = cov_amp * exp_kernel + noise_kernel
bounds = [(0., 1.) for _ in range(n_dimensions)]
types = np.zeros(n_dimensions)
configspace = ConfigurationSpace()
for i in range(n_dimensions):
configspace.add_hyperparameter(
UniformFloatHyperparameter('x%d' % i, 0, 1))
model = GaussianProcess(
configspace=configspace,
bounds=bounds,
types=types,
kernel=kernel,
seed=rs.randint(low=1, high=10000),
normalize_y=normalize_y,
n_opt_restarts=2,
)
return model
def _train(self, X: np.ndarray, y: np.ndarray):
"""Trains the random forest on X and y.
Parameters
----------
X : np.ndarray [n_samples, n_features (config + instance features)]
Input data points.
Y : np.ndarray [n_samples, ]
The corresponding target values.
Returns
-------
self
"""
self.X = X
self.y = y.flatten()
from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel, HammingKernel
from smac.epm.gp_base_prior import HorseshoePrior, LognormalPrior
self.rf = sklearn.ensemble.RandomForestRegressor(
max_features=0.5,
bootstrap=True,
max_depth=3,
min_samples_leaf=10,
n_estimators=N_EST,
)
# self.rf.fit(X, np.log(y - np.min(y) + 1e-7).ravel())
self.rf.fit(X, y.ravel())
indicators = np.array(self.rf.apply(X))
all_datasets = []
all_targets = []
all_mappings = []
for est in range(N_EST):
unique = np.unique(indicators[:, est])
mapping = {j: i for i, j in enumerate(unique)}
datasets = [[] for _ in unique]
targets = [[] for _ in indicators]
for indicator, x, y_ in zip(indicators[:, est], X, y):
index = mapping[indicator]
datasets[index].append(x)
targets[index].append(y_)
all_mappings.append(mapping)
all_datasets.append(datasets)
all_targets.append(targets)
# print('Before')
# for est in range(N_EST):
# for dataset in all_datasets[est]:
# print(len(dataset))
for est in range(N_EST):
n_nodes = self.rf.estimators_[est].tree_.node_count
children_left = self.rf.estimators_[est].tree_.children_left
children_right = self.rf.estimators_[est].tree_.children_right
feature = self.rf.estimators_[est].tree_.feature
threshold = self.rf.estimators_[est].tree_.threshold
# The tree structure can be traversed to compute various properties such
# as the depth of each node and whether or not it is a leaf.
node_depth = np.zeros(shape=n_nodes, dtype=np.int64)
is_leaves = np.zeros(shape=n_nodes, dtype=bool)
stack = [(0, -1)] # seed is the root node id and its parent depth
while len(stack) > 0:
node_id, parent_depth = stack.pop()
node_depth[node_id] = parent_depth + 1
# If we have a test node
if (children_left[node_id] != children_right[node_id]):
stack.append((children_left[node_id], parent_depth + 1))
stack.append((children_right[node_id], parent_depth + 1))
else:
is_leaves[node_id] = True
rules = {}
import copy
def extend(rule, idx):
if is_leaves[idx]:
rules[idx] = rule
else:
rule_left = copy.deepcopy(rule)
rule_left.append((threshold[idx], '<=', feature[idx]))
extend(rule_left, children_left[idx])
rule_right = copy.deepcopy(rule)
rule_right.append((threshold[idx], '>', feature[idx]))
extend(rule_right, children_right[idx])
extend([], 0)
#print(rules)
for key, rule in rules.items():
lower = -np.ones((X.shape[1], )) * np.inf
upper = np.ones((X.shape[1], )) * np.inf
for element in rule:
if element[1] == '<=':
if element[0] < upper[element[2]]:
upper[element[2]] = element[0]
else:
if element[0] > lower[element[2]]:
lower[element[2]] = element[0]
for feature_idx in range(X.shape[1]):
closest_lower = -np.inf
closes_lower_idx = None
closest_upper = np.inf
closest_upper_idx = None
for x in X:
if x[feature_idx] > lower[feature_idx] and x[
feature_idx] < upper[feature_idx]:
continue
if x[feature_idx] <= lower[feature_idx]:
if x[feature_idx] > closest_lower:
closest_lower = x[feature_idx]
closes_lower_idx = feature_idx
if x[feature_idx] >= upper[feature_idx]:
if x[feature_idx] < closest_upper:
closest_upper = x[feature_idx]
closest_upper_idx = feature_idx
if closest_upper_idx is not None:
all_datasets[est][all_mappings[est][key]].append(
X[closest_upper_idx])
all_targets[est][all_mappings[est][key]].append(
y[closest_upper_idx])
if closes_lower_idx is not None:
all_datasets[est][all_mappings[est][key]].append(
X[closes_lower_idx])
all_targets[est][all_mappings[est][key]].append(
y[closes_lower_idx])
# print('After')
# for est in range(N_EST):
# for dataset in all_datasets[est]:
# print(len(dataset))
self.all_mappings = all_mappings
self.models = []
for est in range(N_EST):
models = []
for dataset, targets_ in zip(all_datasets[est], all_targets[est]):
cov_amp = ConstantKernel(
2.0,
constant_value_bounds=(np.exp(-10), np.exp(2)),
prior=LognormalPrior(mean=0.0, sigma=1.0, rng=self.rng),
)
cont_dims = np.nonzero(self.types == 0)[0]
cat_dims = np.nonzero(self.types != 0)[0]
if len(cont_dims) > 0:
exp_kernel = Matern(
np.ones([len(cont_dims)]),
[(np.exp(-10), np.exp(2))
for _ in range(len(cont_dims))],
nu=2.5,
operate_on=cont_dims,
)
if len(cat_dims) > 0:
ham_kernel = HammingKernel(
np.ones([len(cat_dims)]),
[(np.exp(-10), np.exp(2))
for _ in range(len(cat_dims))],
operate_on=cat_dims,
)
noise_kernel = WhiteKernel(
noise_level=1e-8,
noise_level_bounds=(np.exp(-25), np.exp(2)),
prior=HorseshoePrior(scale=0.1, rng=self.rng),
)
if len(cont_dims) > 0 and len(cat_dims) > 0:
# both
kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel
elif len(cont_dims) > 0 and len(cat_dims) == 0:
# only cont
kernel = cov_amp * exp_kernel + noise_kernel
elif len(cont_dims) == 0 and len(cat_dims) > 0:
# only cont
kernel = cov_amp * ham_kernel + noise_kernel
else:
raise ValueError()
gp = GaussianProcess(
configspace=self.configspace,
types=self.types,
bounds=self.bounds,
kernel=kernel,
normalize_y=True,
seed=self.rng.randint(low=0, high=10000),
)
gp.train(np.array(dataset), np.array(targets_))
gp._train(X, y, do_optimize=False)
models.append(gp)
self.models.append(models)
return self
def __init__(self, api_config, config_space, parallel_setting="LS"):
super(SMAC4EPMOpimizer, self).__init__(api_config)
self.cs = config_space
self.num_hps = len(self.cs.get_hyperparameters())
if parallel_setting not in ["CL_min", "CL_max", "CL_mean", "KB", "LS"]:
raise ValueError(
"parallel_setting can only be one of the following: "
"CL_min, CL_max, CL_mean, KB, LS")
self.parallel_setting = parallel_setting
rng = np.random.RandomState(seed=0)
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alt. to runtime)
"runcount-limit": 128,
"cs": self.cs, # configuration space
"deterministic": True,
"limit_resources": False,
})
self.stats = Stats(scenario)
# traj = TrajLogger(output_dir=None, stats=self.stats)
self.runhistory = RunHistory()
r2e_def_kwargs = {
"scenario": scenario,
"num_params": self.num_hps,
"success_states": [
StatusType.SUCCESS,
],
"impute_censored_data": False,
"scale_perc": 5,
}
self.random_chooser = ChooserProb(rng=rng, prob=0.0)
types, bounds = get_types(self.cs, instance_features=None)
model_kwargs = {
"configspace": self.cs,
"types": types,
"bounds": bounds,
"seed": rng.randint(MAXINT),
}
models = []
cov_amp = ConstantKernel(
2.0,
constant_value_bounds=(np.exp(-10), np.exp(2)),
prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng),
)
cont_dims = np.array(np.where(np.array(types) == 0)[0], dtype=np.int)
cat_dims = np.where(np.array(types) != 0)[0]
if len(cont_dims) > 0:
exp_kernel = Matern(
np.ones([len(cont_dims)]),
[(np.exp(-6.754111155189306), np.exp(0.0858637988771976))
for _ in range(len(cont_dims))],
nu=2.5,
operate_on=cont_dims,
)
if len(cat_dims) > 0:
ham_kernel = HammingKernel(
np.ones([len(cat_dims)]),
[(np.exp(-6.754111155189306), np.exp(0.0858637988771976))
for _ in range(len(cat_dims))],
operate_on=cat_dims,
)
assert len(cont_dims) + len(cat_dims) == len(
scenario.cs.get_hyperparameters())
noise_kernel = WhiteKernel(
noise_level=1e-8,
noise_level_bounds=(np.exp(-25), np.exp(2)),
prior=HorseshoePrior(scale=0.1, rng=rng),
)
if len(cont_dims) > 0 and len(cat_dims) > 0:
# both
kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel
elif len(cont_dims) > 0 and len(cat_dims) == 0:
# only cont
kernel = cov_amp * exp_kernel + noise_kernel
elif len(cont_dims) == 0 and len(cat_dims) > 0:
# only cont
kernel = cov_amp * ham_kernel + noise_kernel
else:
raise ValueError()
gp_kwargs = {"kernel": kernel}
rf_kwargs = {}
rf_kwargs["num_trees"] = model_kwargs.get("num_trees", 10)
rf_kwargs["do_bootstrapping"] = model_kwargs.get(
"do_bootstrapping", True)
rf_kwargs["ratio_features"] = model_kwargs.get("ratio_features", 1.0)
rf_kwargs["min_samples_split"] = model_kwargs.get(
"min_samples_split", 2)
rf_kwargs["min_samples_leaf"] = model_kwargs.get("min_samples_leaf", 1)
rf_kwargs["log_y"] = model_kwargs.get("log_y", True)
rf_log = RandomForestWithInstances(**model_kwargs, **rf_kwargs)
rf_kwargs = copy.deepcopy(rf_kwargs)
rf_kwargs["log_y"] = False
rf_no_log = RandomForestWithInstances(**model_kwargs, **rf_kwargs)
rh2epm_cost = RunHistory2EPM4Cost(**r2e_def_kwargs)
rh2epm_log_cost = RunHistory2EPM4LogScaledCost(**r2e_def_kwargs)
rh2epm_copula = RunHistory2EPM4GaussianCopulaCorrect(**r2e_def_kwargs)
self.combinations = []
# 2 models * 4 acquisition functions
acq_funcs = [EI, PI, LogEI, LCB]
acq_func_instances = []
# acq_func_maximizer_instances = []
n_sls_iterations = {
1: 10,
2: 10,
3: 10,
4: 10,
5: 10,
6: 10,
7: 8,
8: 6,
}.get(len(self.cs.get_hyperparameters()), 5)
acq_func_maximizer_kwargs = {
"config_space": self.cs,
"rng": rng,
"max_steps": 5,
"n_steps_plateau_walk": 5,
"n_sls_iterations": n_sls_iterations,
}
self.idx_ei = 0
self.num_models = len(models)
self.num_acq_funcs = len(acq_funcs)
no_transform_gp = GaussianProcess(**copy.deepcopy(model_kwargs),
**copy.deepcopy(gp_kwargs))
ei = EI(model=no_transform_gp)
acq_func_maximizer_kwargs["acquisition_function"] = ei
ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((no_transform_gp, ei, ei_opt, rh2epm_cost))
pi = PI(model=no_transform_gp)
acq_func_maximizer_kwargs["acquisition_function"] = pi
pi_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((no_transform_gp, pi, pi_opt, rh2epm_cost))
lcb = LCB(model=no_transform_gp)
acq_func_maximizer_kwargs["acquisition_function"] = lcb
lcb_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((no_transform_gp, lcb, lcb_opt, rh2epm_cost))
gp = GaussianProcess(**copy.deepcopy(model_kwargs),
**copy.deepcopy(gp_kwargs))
ei = EI(model=gp)
acq_func_maximizer_kwargs["acquisition_function"] = ei
ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((gp, ei, ei_opt, rh2epm_copula))
gp = GaussianProcess(**copy.deepcopy(model_kwargs),
**copy.deepcopy(gp_kwargs))
ei = LogEI(model=gp)
acq_func_maximizer_kwargs["acquisition_function"] = ei
ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((gp, ei, ei_opt, rh2epm_log_cost))
ei = EI(model=rf_no_log)
acq_func_maximizer_kwargs["acquisition_function"] = ei
ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((rf_no_log, ei, ei_opt, rh2epm_cost))
ei = LogEI(model=rf_log)
acq_func_maximizer_kwargs["acquisition_function"] = ei
ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((rf_log, ei, ei_opt, rh2epm_log_cost))
ei = EI(model=rf_no_log)
acq_func_maximizer_kwargs["acquisition_function"] = ei
ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((rf_no_log, ei, ei_opt, rh2epm_copula))
self.num_acq_instances = len(acq_func_instances)
self.best_observation = np.inf
self.next_evaluations = []
def __init__(self, model_type: str = "gp_mcmc", **kwargs: typing.Any):
scenario = kwargs["scenario"]
if len(scenario.cs.get_hyperparameters()) <= 21201:
kwargs["initial_design"] = kwargs.get("initial_design", SobolDesign)
else:
raise ValueError(
'The default initial design "Sobol sequence" can only handle up to 21201 dimensions. '
'Please use a different initial design, such as "the Latin Hypercube design".',
)
kwargs["runhistory2epm"] = kwargs.get("runhistory2epm", RunHistory2EPM4Cost)
init_kwargs = kwargs.get("initial_design_kwargs", dict()) or dict()
init_kwargs["n_configs_x_params"] = init_kwargs.get("n_configs_x_params", 8)
init_kwargs["max_config_fracs"] = init_kwargs.get("max_config_fracs", 0.25)
kwargs["initial_design_kwargs"] = init_kwargs
if kwargs.get("model") is None:
model_kwargs = kwargs.get("model_kwargs", dict()) or dict()
_, rng = get_rng(
rng=kwargs.get("rng", None),
run_id=kwargs.get("run_id", None),
logger=None,
)
types, bounds = get_types(kwargs["scenario"].cs, instance_features=None)
cov_amp = ConstantKernel(
2.0,
constant_value_bounds=(np.exp(-10), np.exp(2)),
prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng),
)
cont_dims = np.where(np.array(types) == 0)[0]
cat_dims = np.where(np.array(types) != 0)[0]
if len(cont_dims) > 0:
exp_kernel = Matern(
np.ones([len(cont_dims)]),
[
(np.exp(-6.754111155189306), np.exp(0.0858637988771976))
for _ in range(len(cont_dims))
],
nu=2.5,
operate_on=cont_dims,
)
if len(cat_dims) > 0:
ham_kernel = HammingKernel(
np.ones([len(cat_dims)]),
[
(np.exp(-6.754111155189306), np.exp(0.0858637988771976))
for _ in range(len(cat_dims))
],
operate_on=cat_dims,
)
assert (len(cont_dims) + len(cat_dims)) == len(
scenario.cs.get_hyperparameters()
)
noise_kernel = WhiteKernel(
noise_level=1e-8,
noise_level_bounds=(np.exp(-25), np.exp(2)),
prior=HorseshoePrior(scale=0.1, rng=rng),
)
if len(cont_dims) > 0 and len(cat_dims) > 0:
# both
kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel
elif len(cont_dims) > 0 and len(cat_dims) == 0:
# only cont
kernel = cov_amp * exp_kernel + noise_kernel
elif len(cont_dims) == 0 and len(cat_dims) > 0:
# only cont
kernel = cov_amp * ham_kernel + noise_kernel
else:
raise ValueError()
if model_type == "gp":
model_class = GaussianProcess # type: typing.Type[BaseModel]
kwargs["model"] = model_class
model_kwargs["kernel"] = kernel
model_kwargs["normalize_y"] = True
model_kwargs["seed"] = rng.randint(0, 2**20)
elif model_type == "gp_mcmc":
model_class = GaussianProcessMCMC
kwargs["model"] = model_class
kwargs["integrate_acquisition_function"] = True
model_kwargs["kernel"] = kernel
n_mcmc_walkers = 3 * len(kernel.theta)
if n_mcmc_walkers % 2 == 1:
n_mcmc_walkers += 1
model_kwargs["n_mcmc_walkers"] = n_mcmc_walkers
model_kwargs["chain_length"] = 250
model_kwargs["burnin_steps"] = 250
model_kwargs["normalize_y"] = True
model_kwargs["seed"] = rng.randint(0, 2**20)
else:
raise ValueError("Unknown model type %s" % model_type)
kwargs["model_kwargs"] = model_kwargs
if kwargs.get("random_configuration_chooser") is None:
random_config_chooser_kwargs = (
kwargs.get(
"random_configuration_chooser_kwargs",
dict(),
)
or dict()
)
random_config_chooser_kwargs["prob"] = random_config_chooser_kwargs.get(
"prob", 0.08447232371720552
)
kwargs["random_configuration_chooser_kwargs"] = random_config_chooser_kwargs
if kwargs.get("acquisition_function_optimizer") is None:
acquisition_function_optimizer_kwargs = (
kwargs.get(
"acquisition_function_optimizer_kwargs",
dict(),
)
or dict()
)
acquisition_function_optimizer_kwargs["n_sls_iterations"] = 10
kwargs[
"acquisition_function_optimizer_kwargs"
] = acquisition_function_optimizer_kwargs
# only 1 configuration per SMBO iteration
intensifier_kwargs = kwargs.get("intensifier_kwargs", dict()) or dict()
intensifier_kwargs["min_chall"] = 1
kwargs["intensifier_kwargs"] = intensifier_kwargs
scenario.intensification_percentage = 1e-10
super().__init__(**kwargs)
if self.solver.scenario.n_features > 0:
raise NotImplementedError("BOGP cannot handle instances")
self.logger.info(self.__class__)
self.solver.scenario.acq_opt_challengers = 1000 # type: ignore[attr-defined] # noqa F821
# activate predict incumbent
self.solver.epm_chooser.predict_x_best = True
def _component_builder(self, conf:typing.Union[Configuration, dict]) \
-> typing.Tuple[AbstractAcquisitionFunction, AbstractEPM]:
"""
builds new Acquisition function object
and EPM object and returns these
Parameters
----------
conf: typing.Union[Configuration, dict]
configuration specificing "model" and "acq_func"
Returns
-------
typing.Tuple[AbstractAcquisitionFunction, AbstractEPM]
"""
types, bounds = get_types(
self.config_space, instance_features=self.scenario.feature_array)
if conf["model"] == "RF":
model = RandomForestWithInstances(
configspace=self.config_space,
types=types,
bounds=bounds,
instance_features=self.scenario.feature_array,
seed=self.rng.randint(MAXINT),
pca_components=conf.get("pca_dim", self.scenario.PCA_DIM),
log_y=conf.get("log_y", self.scenario.transform_y
in ["LOG", "LOGS"]),
num_trees=conf.get("num_trees", self.scenario.rf_num_trees),
do_bootstrapping=conf.get("do_bootstrapping",
self.scenario.rf_do_bootstrapping),
ratio_features=conf.get("ratio_features",
self.scenario.rf_ratio_features),
min_samples_split=conf.get("min_samples_split",
self.scenario.rf_min_samples_split),
min_samples_leaf=conf.get("min_samples_leaf",
self.scenario.rf_min_samples_leaf),
max_depth=conf.get("max_depth", self.scenario.rf_max_depth),
)
elif conf["model"] == "GP":
from smac.epm.gp_kernels import ConstantKernel, HammingKernel, WhiteKernel, Matern
cov_amp = ConstantKernel(
2.0,
constant_value_bounds=(np.exp(-10), np.exp(2)),
prior=LognormalPrior(mean=0.0, sigma=1.0, rng=self.rng),
)
cont_dims = np.nonzero(types == 0)[0]
cat_dims = np.nonzero(types != 0)[0]
if len(cont_dims) > 0:
exp_kernel = Matern(
np.ones([len(cont_dims)]),
[(np.exp(-10), np.exp(2)) for _ in range(len(cont_dims))],
nu=2.5,
operate_on=cont_dims,
)
if len(cat_dims) > 0:
ham_kernel = HammingKernel(
np.ones([len(cat_dims)]),
[(np.exp(-10), np.exp(2)) for _ in range(len(cat_dims))],
operate_on=cat_dims,
)
noise_kernel = WhiteKernel(
noise_level=1e-8,
noise_level_bounds=(np.exp(-25), np.exp(2)),
prior=HorseshoePrior(scale=0.1, rng=self.rng),
)
if len(cont_dims) > 0 and len(cat_dims) > 0:
# both
kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel
elif len(cont_dims) > 0 and len(cat_dims) == 0:
# only cont
kernel = cov_amp * exp_kernel + noise_kernel
elif len(cont_dims) == 0 and len(cat_dims) > 0:
# only cont
kernel = cov_amp * ham_kernel + noise_kernel
else:
raise ValueError()
n_mcmc_walkers = 3 * len(kernel.theta)
if n_mcmc_walkers % 2 == 1:
n_mcmc_walkers += 1
model = GaussianProcessMCMC(
self.config_space,
types=types,
bounds=bounds,
kernel=kernel,
n_mcmc_walkers=n_mcmc_walkers,
chain_length=250,
burnin_steps=250,
normalize_y=True,
seed=self.rng.randint(low=0, high=10000),
)
if conf["acq_func"] == "EI":
acq = EI(model=model, par=conf.get("par_ei", 0))
elif conf["acq_func"] == "LCB":
acq = LCB(model=model, par=conf.get("par_lcb", 0))
elif conf["acq_func"] == "PI":
acq = PI(model=model, par=conf.get("par_pi", 0))
elif conf["acq_func"] == "LogEI":
# par value should be in log-space
acq = LogEI(model=model, par=conf.get("par_logei", 0))
return acq, model
def get_kernel(self, config_space):
"""
Code is very similar to those from :class:`smac.SMAC4BO`.
Length scale were chosen from SMAC where they were obtained by
hyperparameter optimization made in https://arxiv.org/abs/1908.06674.
:params config_space: SMAC configuration space with parameters.
:type config_space: :class:`smac.configspace.ConfigurationSpace`
"""
# First of all get type of kernel
kernel_cls, nu = self._get_kernel_class_and_nu()
# get types and bounds for config_space
types, bounds = get_types(config_space, instance_features=None)
# get random state for priors
rng = self._get_random_state()
# create kernel to hold variance
cov_amp = smac.epm.gp_kernels.ConstantKernel(
2.0,
constant_value_bounds=(np.exp(-10), np.exp(2)),
prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng),
)
# Understand information about parameters
cont_dims = np.where(np.array(types) == 0)[0]
cat_dims = np.where(np.array(types) != 0)[0]
# bounds for length scale from https://arxiv.org/abs/1908.06674
lslims = (np.exp(-6.754111155189306), np.exp(0.0858637988771976))
# create kernel for continuous parameters
if len(cont_dims) > 0:
exp_kwargs = {
"length_scale": np.ones([len(cont_dims)]),
"length_scale_bounds": [lslims for _ in range(len(cont_dims))],
"operate_on": cont_dims,
"prior": None,
"has_conditions": False,
}
if nu is not None:
exp_kwargs["nu"] = nu
exp_kernel = kernel_cls(**exp_kwargs)
# kernel for categorical parameters
if len(cat_dims) > 0:
ham_kernel = smac.epm.gp_kernels.HammingKernel(
length_scale=np.ones([len(cat_dims)]),
length_scale_bounds=[lslims for _ in range(len(cat_dims))],
operate_on=cat_dims,
)
# create noise kernel
noise_kernel = smac.epm.gp_kernels.WhiteKernel(
noise_level=1e-8,
noise_level_bounds=(np.exp(-25), np.exp(2)),
prior=HorseshoePrior(scale=0.1, rng=rng),
)
# create final kernel as combination
if len(cont_dims) > 0 and len(cat_dims) > 0:
# both
kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel
elif len(cont_dims) > 0 and len(cat_dims) == 0:
# only continuous parameters
kernel = cov_amp * exp_kernel + noise_kernel
elif len(cont_dims) == 0 and len(cat_dims) > 0:
# only categorical parameters
kernel = cov_amp * ham_kernel + noise_kernel
return kernel
def __init__(self, model_type='gp_mcmc', **kwargs):
"""
Constructor
see ~smac.facade.smac_facade for documentation
"""
scenario = kwargs['scenario']
kwargs['initial_design'] = kwargs.get('initial_design', SobolDesign)
kwargs['runhistory2epm'] = kwargs.get('runhistory2epm',
RunHistory2EPM4Cost)
init_kwargs = kwargs.get('initial_design_kwargs', dict())
init_kwargs['n_configs_x_params'] = init_kwargs.get(
'n_configs_x_params', 8)
init_kwargs['max_config_fracs'] = init_kwargs.get(
'max_config_fracs', 0.25)
kwargs['initial_design_kwargs'] = init_kwargs
if kwargs.get('model') is None:
from smac.epm.gp_kernels import ConstantKernel, Matern, WhiteKernel, HammingKernel
model_kwargs = kwargs.get('model_kwargs', dict())
_, rng = get_rng(rng=kwargs.get("rng", None),
run_id=kwargs.get("run_id", None),
logger=None)
types, bounds = get_types(kwargs['scenario'].cs,
instance_features=None)
cov_amp = ConstantKernel(
2.0,
constant_value_bounds=(np.exp(-10), np.exp(2)),
prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng),
)
cont_dims = np.nonzero(types == 0)[0]
cat_dims = np.nonzero(types != 0)[0]
if len(cont_dims) > 0:
exp_kernel = Matern(
np.ones([len(cont_dims)]),
[(np.exp(-6.754111155189306), np.exp(0.0858637988771976))
for _ in range(len(cont_dims))],
nu=2.5,
operate_on=cont_dims,
)
if len(cat_dims) > 0:
ham_kernel = HammingKernel(
np.ones([len(cat_dims)]),
[(np.exp(-6.754111155189306), np.exp(0.0858637988771976))
for _ in range(len(cat_dims))],
operate_on=cat_dims,
)
noise_kernel = WhiteKernel(
noise_level=1e-8,
noise_level_bounds=(np.exp(-25), np.exp(2)),
prior=HorseshoePrior(scale=0.1, rng=rng),
)
if len(cont_dims) > 0 and len(cat_dims) > 0:
# both
kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel
elif len(cont_dims) > 0 and len(cat_dims) == 0:
# only cont
kernel = cov_amp * exp_kernel + noise_kernel
elif len(cont_dims) == 0 and len(cat_dims) > 0:
# only cont
kernel = cov_amp * ham_kernel + noise_kernel
else:
raise ValueError()
if model_type == "gp":
model_class = GaussianProcess
kwargs['model'] = model_class
model_kwargs['kernel'] = kernel
model_kwargs['normalize_y'] = True
model_kwargs['seed'] = rng.randint(0, 2**20)
elif model_type == "gp_mcmc":
model_class = GaussianProcessMCMC
kwargs['model'] = model_class
kwargs['integrate_acquisition_function'] = True
model_kwargs['kernel'] = kernel
n_mcmc_walkers = 3 * len(kernel.theta)
if n_mcmc_walkers % 2 == 1:
n_mcmc_walkers += 1
model_kwargs['n_mcmc_walkers'] = n_mcmc_walkers
model_kwargs['chain_length'] = 250
model_kwargs['burnin_steps'] = 250
model_kwargs['normalize_y'] = True
model_kwargs['seed'] = rng.randint(0, 2**20)
else:
raise ValueError('Unknown model type %s' % model_type)
kwargs['model_kwargs'] = model_kwargs
if kwargs.get('random_configuration_chooser') is None:
random_config_chooser_kwargs = kwargs.get(
'random_configuration_chooser_kwargs', dict())
random_config_chooser_kwargs[
'prob'] = random_config_chooser_kwargs.get(
'prob', 0.08447232371720552)
kwargs[
'random_configuration_chooser_kwargs'] = random_config_chooser_kwargs
if kwargs.get('acquisition_function_optimizer') is None:
acquisition_function_optimizer_kwargs = kwargs.get(
'acquisition_function_optimizer_kwargs', dict())
acquisition_function_optimizer_kwargs['n_sls_iterations'] = 10
kwargs[
'acquisition_function_optimizer_kwargs'] = acquisition_function_optimizer_kwargs
# only 1 configuration per SMBO iteration
intensifier_kwargs = kwargs.get('intensifier_kwargs', dict())
intensifier_kwargs['min_chall'] = 1
kwargs['intensifier_kwargs'] = intensifier_kwargs
scenario.intensification_percentage = 1e-10
super().__init__(**kwargs)
if self.solver.scenario.n_features > 0:
raise NotImplementedError("BOGP cannot handle instances")
self.logger.info(self.__class__)
self.solver.scenario.acq_opt_challengers = 1000
# activate predict incumbent
self.solver.predict_incumbent = True
def test_sample_from_prior(self):
prior = HorseshoePrior(scale=1, rng=np.random.RandomState(1))
samples = prior.sample_from_prior(10)
# Test that the rng is set
self.assertAlmostEqual(samples[0], 1.0723988839129437)
