def test_compute(self):
lcb = LCB(self.model)
X_test = np.random.rand(5, 2)
a = lcb.compute(X_test, derivative=False)
assert a.shape[0] == X_test.shape[0]
assert len(a.shape) == 1
np.testing.assert_almost_equal(a, np.ones(X_test.shape[0]) * (- np.mean(self.y) + np.std(self.y)), decimal=3)
def test_compute(self):
lcb = LCB(self.model)
X_test = np.random.rand(5, 2)
a = lcb.compute(X_test, derivative=False)
assert a.shape[0] == X_test.shape[0]
assert len(a.shape) == 1
np.testing.assert_almost_equal(a,
np.ones(X_test.shape[0]) *
(-np.mean(self.y) + np.std(self.y)),
decimal=3)
def build_acquisition_func(acquisition_func, model):
"""
Build acquisition function
Parameters
----------
acquisition_func: str
Name of the acquisition function. Can be one of ``['ei', 'log_ei', 'pi', 'lcb']``.
model: ``robo.models.base_model.BaseModel``
Model used for the Bayesian optimization.
"""
if acquisition_func == "ei":
acquisition_func = EI(model)
elif acquisition_func == "log_ei":
acquisition_func = LogEI(model)
elif acquisition_func == "pi":
acquisition_func = PI(model)
elif acquisition_func == "lcb":
acquisition_func = LCB(model)
else:
raise ValueError("'{}' is not a valid acquisition function".format(
acquisition_func))
return acquisition_func
def test_lcb(self):
lcb = LCB(self.model)
acq = MarginalizationGPMCMC(lcb)
X_test = np.random.rand(5, 2)
a = acq.compute(X_test, derivative=False)
assert a.shape[0] == X_test.shape[0]
assert len(a.shape) == 1
def setUp(self):
lower = np.zeros([1])
upper = np.ones([1])
kernel = george.kernels.Matern52Kernel(np.array([1]), dim=1, ndim=1)
model = GaussianProcess(kernel)
lcb = LCB(model)
maximizer = RandomSampling(lcb, lower, upper)
self.solver = BayesianOptimization(objective_func, lower, upper, lcb,
model, maximizer)
def bohamiann(objective_function,
lower,
upper,
num_iterations=30,
maximizer="random",
acquisition_func="log_ei",
n_init=3,
output_path=None,
rng=None):
"""
Bohamiann uses Bayesian neural networks to model the objective function [1] inside Bayesian optimization.
Bayesian neural networks usually scale better with the number of function evaluations and the number of dimensions
than Gaussian processes.
[1] Bayesian optimization with robust Bayesian neural networks
J. T. Springenberg and A. Klein and S. Falkner and F. Hutter
Advances in Neural Information Processing Systems 29
Parameters
----------
objective_function: function
The objective function that is minimized. This function gets a numpy array (D,) as input and returns
the function value (scalar)
lower: np.ndarray (D,)
The lower bound of the search space
upper: np.ndarray (D,)
The upper bound of the search space
num_iterations: int
The number of iterations (initial design + BO)
acquisition_func: {"ei", "log_ei", "lcb", "pi"}
The acquisition function
maximizer: {"direct", "cmaes", "random", "scipy"}
The optimizer for the acquisition function. NOTE: "cmaes" only works in D > 1 dimensions
n_init: int
Number of points for the initial design. Make sure that it is <= num_iterations.
output_path: string
Specifies the path where the intermediate output after each iteration will be saved.
If None no output will be saved to disk.
rng: numpy.random.RandomState
Random number generator
Returns
-------
dict with all results
"""
assert upper.shape[0] == lower.shape[0]
assert n_init <= num_iterations, "Number of initial design point has to be <= than the number of iterations"
if rng is None:
rng = np.random.RandomState(np.random.randint(0, 10000))
model = BayesianNeuralNetwork(sampling_method="sghmc",
l_rate=np.sqrt(1e-4),
mdecay=0.05,
burn_in=3000,
n_iters=50000,
precondition=True,
normalize_input=True,
normalize_output=True)
if acquisition_func == "ei":
a = EI(model)
elif acquisition_func == "log_ei":
a = LogEI(model)
elif acquisition_func == "pi":
a = PI(model)
elif acquisition_func == "lcb":
a = LCB(model)
else:
print("ERROR: %s is not a valid acquisition function!" %
acquisition_func)
return
if maximizer == "cmaes":
max_func = CMAES(a, lower, upper, verbose=True, rng=rng)
elif maximizer == "direct":
max_func = Direct(a, lower, upper, verbose=True)
elif maximizer == "random":
max_func = RandomSampling(a, lower, upper, rng=rng)
elif maximizer == "scipy":
max_func = SciPyOptimizer(a, lower, upper, rng=rng)
bo = BayesianOptimization(objective_function,
lower,
upper,
a,
model,
max_func,
initial_points=n_init,
output_path=output_path,
rng=rng)
x_best, f_min = bo.run(num_iterations)
results = dict()
results["x_opt"] = x_best
results["f_opt"] = f_min
results["incumbents"] = [inc for inc in bo.incumbents]
results["incumbent_values"] = [val for val in bo.incumbents_values]
results["runtime"] = bo.runtime
results["overhead"] = bo.time_overhead
results["X"] = [x.tolist() for x in bo.X]
results["y"] = [y for y in bo.y]
return results
def bayesian_optimization(objective_function,
lower,
upper,
num_iterations=30,
maximizer="random",
acquisition_func="log_ei",
model_type="gp_mcmc",
n_init=3,
rng=None,
output_path=None):
"""
General interface for Bayesian optimization for global black box
optimization problems.
Parameters
----------
objective_function: function
The objective function that is minimized. This function gets a numpy
array (D,) as input and returns the function value (scalar)
lower: np.ndarray (D,)
The lower bound of the search space
upper: np.ndarray (D,)
The upper bound of the search space
num_iterations: int
The number of iterations (initial design + BO)
maximizer: {"direct", "cmaes", "random", "scipy"}
The optimizer for the acquisition function. NOTE: "cmaes" only works in D > 1 dimensions
acquisition_func: {"ei", "log_ei", "lcb", "pi"}
The acquisition function
model_type: {"gp", "gp_mcmc", "rf"}
The model for the objective function.
n_init: int
Number of points for the initial design. Make sure that it
is <= num_iterations.
output_path: string
Specifies the path where the intermediate output after each iteration will be saved.
If None no output will be saved to disk.
rng: numpy.random.RandomState
Random number generator
Returns
-------
dict with all results
"""
assert upper.shape[0] == lower.shape[0], "Dimension miss match"
assert np.all(lower < upper), "Lower bound >= upper bound"
assert n_init <= num_iterations, "Number of initial design point has to be <= than the number of iterations"
if rng is None:
rng = np.random.RandomState(np.random.randint(0, 10000))
cov_amp = 2
n_dims = lower.shape[0]
initial_ls = np.ones([n_dims])
exp_kernel = george.kernels.Matern52Kernel(initial_ls, ndim=n_dims)
kernel = cov_amp * exp_kernel
prior = DefaultPrior(len(kernel) + 1)
n_hypers = 3 * len(kernel)
if n_hypers % 2 == 1:
n_hypers += 1
if model_type == "gp":
model = GaussianProcess(kernel,
prior=prior,
rng=rng,
normalize_output=False,
normalize_input=True,
lower=lower,
upper=upper)
elif model_type == "gp_mcmc":
model = GaussianProcessMCMC(kernel,
prior=prior,
n_hypers=n_hypers,
chain_length=200,
burnin_steps=100,
normalize_input=True,
normalize_output=True,
rng=rng,
lower=lower,
upper=upper)
elif model_type == "rf":
model = RandomForest(rng=rng)
else:
raise ValueError("'{}' is not a valid model".format(model_type))
if acquisition_func == "ei":
a = EI(model)
elif acquisition_func == "log_ei":
a = LogEI(model)
elif acquisition_func == "pi":
a = PI(model)
elif acquisition_func == "lcb":
a = LCB(model)
else:
raise ValueError("'{}' is not a valid acquisition function".format(
acquisition_func))
if model_type == "gp_mcmc":
acquisition_func = MarginalizationGPMCMC(a)
else:
acquisition_func = a
if maximizer == "cmaes":
max_func = CMAES(acquisition_func,
lower,
upper,
verbose=False,
rng=rng)
elif maximizer == "direct":
max_func = Direct(acquisition_func, lower, upper, verbose=True)
elif maximizer == "random":
max_func = RandomSampling(acquisition_func, lower, upper, rng=rng)
elif maximizer == "scipy":
max_func = SciPyOptimizer(acquisition_func, lower, upper, rng=rng)
else:
raise ValueError("'{}' is not a valid function to maximize the "
"acquisition function".format(maximizer))
bo = BayesianOptimization(objective_function,
lower,
upper,
acquisition_func,
model,
max_func,
initial_points=n_init,
rng=rng,
output_path=output_path)
x_best, f_min = bo.run(num_iterations)
results = dict()
results["x_opt"] = x_best
results["f_opt"] = f_min
results["incumbents"] = [inc for inc in bo.incumbents]
results["incumbent_values"] = [val for val in bo.incumbents_values]
results["runtime"] = bo.runtime
results["overhead"] = bo.time_overhead
results["X"] = [x.tolist() for x in bo.X]
results["y"] = [y for y in bo.y]
return results
def bayesian_optimization(objective_function,
lower,
upper,
num_iterations=30,
X_init=None,
Y_init=None,
maximizer="random",
acquisition_func="log_ei",
model_type="gp_mcmc",
n_init=3,
rng=None,
output_path=None,
kernel=None,
sampling_method="origin",
distance="cosine",
replacement=True,
pool=None,
best=None):
"""
General interface for Bayesian optimization for global black box
optimization problems.
Parameters
----------
objective_function: function
The objective function that is minimized. This function gets a numpy
array (D,) as input and returns the function value (scalar)
lower: np.ndarray (D,)
The lower bound of the search space
upper: np.ndarray (D,)
The upper bound of the search space
num_iterations: int
The number of iterations (initial design + BO)
X_init: np.ndarray(N,D)
Initial points to warmstart BO
Y_init: np.ndarray(N,1)
Function values of the already initial points
maximizer: {"random", "scipy", "differential_evolution"}
The optimizer for the acquisition function.
acquisition_func: {"ei", "log_ei", "lcb", "pi"}
The acquisition function
model_type: {"gp", "gp_mcmc", "rf", "bohamiann", "dngo"}
The model for the objective function.
n_init: int
Number of points for the initial design. Make sure that it
is <= num_iterations.
output_path: string
Specifies the path where the intermediate output after each iteration will be saved.
If None no output will be saved to disk.
rng: numpy.random.RandomState
Random number generator
kernel: george.kernels.ConstantKernel
{"constant", "polynomial", "linear", "dotproduct",
"exp", "expsquared", "matern32", "matern52", "rationalquadratic",
"cosine", "expsine2", "heuristic"}
Specify the kernel for Gaussian process.
sampling_method: {"origin", "approx", "exact"}
Specify the method to choose next sample to update model.
approx: choose the sample in the candidate pool that is closest (measured by distance
arg) to the one returned from maximizing acquisition function.
exact: evaluate all samples in the candidate pool on acquisition function
and choose the one with maximum output.
distance: {"cosine", "euclidean"}
The distance measurement for approximation sampling.
replacement: boolean
Whether to sample from pool with replacement.
pool: np.ndarray(N,D)
Candidate pool containing possible x
best: float
Stop training when the best point is sampled.
Returns
-------
dict with all results
"""
assert upper.shape[0] == lower.shape[0], "Dimension miss match"
assert np.all(lower < upper), "Lower bound >= upper bound"
assert n_init <= num_iterations, "Number of initial design point has to be <= than the number of iterations"
if rng is None:
rng = np.random.RandomState(np.random.randint(0, 10000))
cov_amp = 2
#n_dims = lower.shape[0]
#initial_ls = np.ones([n_dims])
# if kernel == "constant":
# exp_kernel = george.kernels.ConstantKernel(1, ndim=n_dims)
# elif kernel == "polynomial":
# exp_kernel = george.kernels.PolynomialKernel(log_sigma2=1, order=3, ndim=n_dims)
# elif kernel == "linear":
# exp_kernel = george.kernels.LinearKernel(log_gamma2=1, order=3, ndim=n_dims)
# elif kernel == "dotproduct":
# exp_kernel = george.kernels.DotProductKernel(ndim=n_dims)
# elif kernel == "exp":
# exp_kernel = george.kernels.ExpKernel(initial_ls, ndim=n_dims)
# elif kernel == "expsquared":
# exp_kernel = george.kernels.ExpSquaredKernel(initial_ls, ndim=n_dims)
# elif kernel == "matern32":
# exp_kernel = george.kernels.Matern32Kernel(initial_ls, ndim=n_dims)
# elif kernel == "matern52":
# exp_kernel = george.kernels.Matern52Kernel(initial_ls, ndim=n_dims)
# elif kernel == "rationalquadratic":
# exp_kernel = george.kernels.RationalQuadraticKernel(log_alpha=1, metric=initial_ls, ndim=n_dims)
# elif kernel == "cosine":
# exp_kernel = george.kernels.CosineKernel(4, ndim=n_dims)
# elif kernel == "expsine2":
# exp_kernel = george.kerngels.ExpSine2Kernel(1, 2, ndim=n_dims)
# elif kernel == "heuristic":
# exp_kernel = george.kernels.PythonKernel(heuristic_kernel_function, ndim=n_dims)
# else:
# raise ValueError("'{}' is not a valid kernel".format(kernel))
kernel = cov_amp * kernel
prior = DefaultPrior(len(kernel) + 1)
n_hypers = 3 * len(kernel)
if n_hypers % 2 == 1:
n_hypers += 1
if model_type == "gp":
model = GaussianProcess(kernel,
prior=prior,
rng=rng,
normalize_output=False,
normalize_input=True,
lower=lower,
upper=upper)
elif model_type == "gp_mcmc":
model = GaussianProcessMCMC(kernel,
prior=prior,
n_hypers=n_hypers,
chain_length=200,
burnin_steps=100,
normalize_input=True,
normalize_output=False,
rng=rng,
lower=lower,
upper=upper)
elif model_type == "rf":
model = RandomForest(rng=rng)
elif model_type == "bohamiann":
model = WrapperBohamiann()
elif model_type == "dngo":
model = DNGO()
else:
raise ValueError("'{}' is not a valid model".format(model_type))
if acquisition_func == "ei":
a = EI(model)
elif acquisition_func == "log_ei":
a = LogEI(model)
elif acquisition_func == "pi":
a = PI(model)
elif acquisition_func == "lcb":
a = LCB(model)
else:
raise ValueError("'{}' is not a valid acquisition function".format(
acquisition_func))
if model_type == "gp_mcmc":
acquisition_func = MarginalizationGPMCMC(a)
else:
acquisition_func = a
if maximizer == "random":
max_func = RandomSampling(acquisition_func, lower, upper, rng=rng)
elif maximizer == "scipy":
max_func = SciPyOptimizer(acquisition_func, lower, upper, rng=rng)
elif maximizer == "differential_evolution":
max_func = DifferentialEvolution(acquisition_func,
lower,
upper,
rng=rng)
else:
raise ValueError("'{}' is not a valid function to maximize the "
"acquisition function".format(maximizer))
if sampling_method == "exact":
max_func = ExactSampling(acquisition_func,
lower,
upper,
pool,
replacement,
rng=rng)
init_design = init_exact_random
elif sampling_method == "approx":
max_func = ApproxSampling(acquisition_func,
lower,
upper,
pool,
replacement,
distance,
rng=rng)
init_design = init_exact_random
else:
init_design = init_latin_hypercube_sampling
bo = BayesianOptimization(objective_function,
lower,
upper,
acquisition_func,
model,
max_func,
pool,
best,
sampling_method,
distance,
replacement,
initial_points=n_init,
rng=rng,
initial_design=init_design,
output_path=output_path)
x_best, f_min = bo.run(num_iterations, X=X_init, y=Y_init)
results = dict()
results["x_opt"] = x_best
results["f_opt"] = f_min
results["incumbents"] = [inc for inc in bo.incumbents]
results["incumbent_values"] = [val for val in bo.incumbents_values]
results["runtime"] = bo.runtime
results["overhead"] = bo.time_overhead
results["X"] = [x.tolist() for x in bo.X]
results["y"] = [y for y in bo.y]
return results
def bohamiann(objective_function,
lower,
upper,
num_iterations=30,
acquisition_func="log_ei",
n_init=3,
rng=None):
"""
General interface for Bayesian optimization for global black box optimization problems.
Parameters
----------
objective_function: function
The objective function that is minimized. This function gets a numpy array (D,) as input and returns
the function value (scalar)
lower: np.ndarray (D,)
The lower bound of the search space
upper: np.ndarray (D,)
The upper bound of the search space
num_iterations: int
The number of iterations (initial design + BO)
acquisition_func: {"ei", "log_ei", "lcb", "pi"}
The acquisition function
n_init: int
Number of points for the initial design. Make sure that it is <= num_iterations.
rng: numpy.random.RandomState
Random number generator
Returns
-------
dict with all results
"""
assert upper.shape[0] == lower.shape[0]
assert n_init <= num_iterations, "Number of initial design point has to be <= than the number of iterations"
if rng is None:
rng = np.random.RandomState(np.random.randint(0, 10000))
model = BayesianNeuralNetwork(sampling_method="sghmc",
l_rate=np.sqrt(1e-4),
mdecay=0.05,
burn_in=3000,
n_iters=50000,
precondition=True,
normalize_input=True,
normalize_output=True)
if acquisition_func == "ei":
a = EI(model)
elif acquisition_func == "log_ei":
a = LogEI(model)
elif acquisition_func == "pi":
a = PI(model)
elif acquisition_func == "lcb":
a = LCB(model)
else:
print("ERROR: %s is not a valid acquisition function!" %
acquisition_func)
return
max_func = Direct(a, lower, upper, verbose=False)
bo = BayesianOptimization(objective_function,
lower,
upper,
a,
model,
max_func,
initial_points=n_init,
rng=rng)
x_best, f_min = bo.run(num_iterations)
results = dict()
results["x_opt"] = x_best
results["f_opt"] = f_min
results["incumbents"] = [inc for inc in bo.incumbents]
results["incumbent_values"] = [val for val in bo.incumbents_values]
results["runtime"] = bo.runtime
results["overhead"] = bo.time_overhead
return results
def bayesian_optimization(objective_function,
lower,
upper,
num_iterations=30,
maximizer="direct",
acquisition_func="log_ei",
model="gp_mcmc",
n_init=3,
rng=None):
"""
General interface for Bayesian optimization for global black box optimization problems.
Parameters
----------
objective_function: function
The objective function that is minimized. This function gets a numpy array (D,) as input and returns
the function value (scalar)
lower: np.ndarray (D,)
The lower bound of the search space
upper: np.ndarray (D,)
The upper bound of the search space
num_iterations: int
The number of iterations (initial design + BO)
maximizer: {"direct", "cmaes"}
Defines how the acquisition function is maximized. NOTE: "cmaes" only works in D > 1 dimensions
acquisition_func: {"ei", "log_ei", "lcb", "pi"}
The acquisition function
model: {"gp", "gp_mcmc"}
The model for the objective function.
n_init: int
Number of points for the initial design. Make sure that it is <= num_iterations.
rng: numpy.random.RandomState
Random number generator
Returns
-------
dict with all results
"""
assert upper.shape[0] == lower.shape[0]
assert n_init <= num_iterations, "Number of initial design point has to be <= than the number of iterations"
if rng is None:
rng = np.random.RandomState(np.random.randint(0, 10000))
cov_amp = 2
n_dims = lower.shape[0]
initial_ls = np.ones([n_dims])
exp_kernel = george.kernels.Matern52Kernel(initial_ls, ndim=n_dims)
kernel = cov_amp * exp_kernel
prior = DefaultPrior(len(kernel) + 1)
n_hypers = 3 * len(kernel)
if n_hypers % 2 == 1:
n_hypers += 1
if model == "gp":
gp = GaussianProcess(kernel,
prior=prior,
rng=rng,
normalize_output=True,
normalize_input=True,
lower=lower,
upper=upper)
elif model == "gp_mcmc":
gp = GaussianProcessMCMC(kernel,
prior=prior,
n_hypers=n_hypers,
chain_length=200,
burnin_steps=100,
normalize_input=True,
normalize_output=True,
rng=rng,
lower=lower,
upper=upper)
else:
print("ERROR: %s is not a valid model!" % model)
return
if acquisition_func == "ei":
a = EI(gp)
elif acquisition_func == "log_ei":
a = LogEI(gp)
elif acquisition_func == "pi":
a = PI(gp)
elif acquisition_func == "lcb":
a = LCB(gp)
else:
print("ERROR: %s is not a valid acquisition function!" %
acquisition_func)
return
if model == "gp":
acquisition_func = a
elif model == "gp_mcmc":
acquisition_func = MarginalizationGPMCMC(a)
if maximizer == "cmaes":
max_func = CMAES(acquisition_func,
lower,
upper,
verbose=False,
rng=rng)
elif maximizer == "direct":
max_func = Direct(acquisition_func, lower, upper, verbose=False)
else:
print(
"ERROR: %s is not a valid function to maximize the acquisition function!"
% maximizer)
return
bo = BayesianOptimization(objective_function,
lower,
upper,
acquisition_func,
gp,
max_func,
initial_points=n_init,
rng=rng)
x_best, f_min = bo.run(num_iterations)
results = dict()
results["x_opt"] = x_best
results["f_opt"] = f_min
results["incumbents"] = [inc for inc in bo.incumbents]
results["incumbent_values"] = [val for val in bo.incumbents_values]
results["runtime"] = bo.runtime
results["overhead"] = bo.time_overhead
return results
def build_optimizer(model,
maximizer="random",
acquisition_func="log_ei",
maximizer_seed=1):
"""
General interface for Bayesian optimization for global black box
optimization problems.
Parameters
----------
maximizer: {"random", "scipy", "differential_evolution"}
The optimizer for the acquisition function.
acquisition_func: {"ei", "log_ei", "lcb", "pi"}
The acquisition function
maximizer_seed: int
Seed for random number generator of the acquisition function maximizer
Returns
-------
Optimizer
"""
if acquisition_func == "ei":
a = EI(model)
elif acquisition_func == "log_ei":
a = LogEI(model)
elif acquisition_func == "pi":
a = PI(model)
elif acquisition_func == "lcb":
a = LCB(model)
else:
raise ValueError("'{}' is not a valid acquisition function".format(
acquisition_func))
if isinstance(model, GaussianProcessMCMC):
acquisition_func = MarginalizationGPMCMC(a)
else:
acquisition_func = a
maximizer_rng = numpy.random.RandomState(maximizer_seed)
if maximizer == "random":
max_func = RandomSampling(acquisition_func,
model.lower,
model.upper,
rng=maximizer_rng)
elif maximizer == "scipy":
max_func = SciPyOptimizer(acquisition_func,
model.lower,
model.upper,
rng=maximizer_rng)
elif maximizer == "differential_evolution":
max_func = DifferentialEvolution(acquisition_func,
model.lower,
model.upper,
rng=maximizer_rng)
else:
raise ValueError("'{}' is not a valid function to maximize the "
"acquisition function".format(maximizer))
# NOTE: Internal RNG of BO won't be used.
# NOTE: Nb of initial points won't be used within BO, but rather outside
bo = BayesianOptimization(lambda: None,
model.lower,
model.upper,
acquisition_func,
model,
max_func,
initial_points=None,
rng=None,
initial_design=init_latin_hypercube_sampling,
output_path=None)
return bo