def bay_opt(self):
# Initialize sample
for i in range(self.niter):
m52_1 = ConstantKernel(1) * RBF(np.array([100] * 12))
self.gpr_obj = GaussianProcessRegressor(kernel=m52_1,
alpha=10,
noise="gaussian")
m52_2 = ConstantKernel(1) * RBF(np.array([100] * 12))
self.gpr_constraint = GaussianProcessRegressor(kernel=m52_2,
alpha=10,
noise="gaussian")
# Update Gaussian process with existing samples
#print(self.x.shape,self.y_obj.shape )
self.gpr_obj.fit(self.x, self.y_obj)
#print(self.gpr_obj.predict(self.x))
self.gpr_constraint.fit(self.x, self.y_constraint)
# Obtain next sampling point from the acquisition function (expected_improvement)
X_next = self.propose_location()
# Obtain next noisy sample from the objective function
Y_next1 = np.array([self.obj_func(X_next)]).reshape(-1, 1)
Y_next2 = np.array([self.constraint_func(X_next)]).reshape(-1, 1)
#print(Y_next1, Y_next1.shape, Y_next2,Y_next2.shape)
# Add sample to previous samples
self.x = np.vstack((self.x, X_next))
self.y_obj = np.vstack((self.y_obj, Y_next1))
self.y_constraint = np.vstack((self.y_constraint, Y_next2))
idx = np.where(self.y_constraint > 0)[0]
t = idx[np.argmin(self.y_obj[idx])]
self.f_best = self.y_obj[t]
self.min_x = self.x[t]
return self.f_best, self.min_x
def minimal_gp(request):
kernel = ConstantKernel(
constant_value=1 ** 2, constant_value_bounds=(0.01 ** 2, 1 ** 2)
) * RBF(length_scale=1.0, length_scale_bounds=(0.5, 1.5))
gp = BayesGPR(
random_state=1, normalize_y=False, kernel=kernel, warp_inputs=request.param
)
return gp
def kriging_surrogates():
surrogate_model = GaussianProcessRegressor(normalize_y=True)
kernels = [
1.0 * Matern(nu=5 / 2),
1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-1, 10.0))
]
params = [kernels]
return surrogate_model, params
def test_mean_gradient():
length_scale = np.arange(1, 6)
X = rng.randn(10, 5)
y = rng.randn(10)
X_new = rng.randn(5)
rbf = RBF(length_scale=length_scale, length_scale_bounds="fixed")
gpr = GaussianProcessRegressor(rbf, random_state=0).fit(X, y)
mean, std, mean_grad = gpr.predict(
np.expand_dims(X_new, axis=0),
return_std=True, return_cov=False, return_mean_grad=True)
num_grad = optimize.approx_fprime(
X_new, lambda x: predict_wrapper(x, gpr)[0], 1e-4)
assert_array_almost_equal(mean_grad, num_grad, decimal=3)
def optimize(x0, y0, n_calls):
estimator = GaussianProcessRegressor(alpha=1e-4,
normalize_y=True,
noise='gaussian',
n_restarts_optimizer=10,
kernel=RBF())
w = gp_minimize(black_box, [(low, high)] * dim,
base_estimator=estimator,
acq_func="EI",
n_calls=n_calls,
verbose=False,
x0=x0,
y0=y0,
n_random_starts=1,
n_jobs=-1)
return w.x_iters, w.func_vals.tolist()
def test_guess_priors():
"""Construct a complicated kernel and check if priors are constructed
correctly."""
kernel = Exponentiation(
ConstantKernel(constant_value_bounds="fixed") * Matern() +
WhiteKernel() + CompoundKernel([RBF(), Matern()]),
2.0,
)
priors = guess_priors(kernel)
assert len(priors) == 4
expected = [
-1.737085713764618,
-4.107091211892862,
-1.737085713764618,
-1.737085713764618,
]
for p, v in zip(priors, expected):
assert_almost_equal(p(0.0), v)
def test_guess_priors():
"""Construct a complicated kernel and check if priors are constructed
correctly."""
kernel = Exponentiation(
ConstantKernel(constant_value_bounds="fixed") * Matern() +
WhiteKernel() + RBF(length_scale=(1.0, 1.0)),
2.0,
)
priors = guess_priors(kernel)
assert len(priors) == 4
expected = [
-0.02116327824572739,
-2.112906921232193,
-0.02116327824572739,
-0.02116327824572739,
]
for p, v in zip(priors, expected):
assert_almost_equal(p(-0.9), v)
import numpy as np from skopt.learning import GaussianProcessRegressor from skopt.learning.gaussian_process.kernels import RBF all_x = np.reshape(np.linspace(0, 6, 100), (-1, 1)) all_f = [black_box(xi) for xi in all_x] # Plot all points. plt.plot(all_x, all_f) # Train only one third of the training data. X = np.reshape(np.linspace(4, 6, 10), (-1, 1)) y = [black_box(xi) for xi in X] # Use RBF kernel. rbf = RBF(length_scale=1.0) gpr = GaussianProcessRegressor(kernel=rbf, alpha=1e-12) gpr.fit(X, y) plt.plot(np.ravel(X), y, "ro", label="Fit points") # Predict on all data. y_pred, y_std = gpr.predict(all_x, return_std=True) all_x_plot = np.ravel(all_x) upper_bound = y_pred + 1.96 * y_std lower_bound = y_pred - 1.96 * y_std plt.plot(all_x_plot, y_pred, "r--", label="Predictions") plt.plot(all_x_plot, lower_bound, color="red") plt.plot(all_x_plot, upper_bound, color="red") plt.fill_between(all_x_plot, lower_bound, upper_bound, facecolor="lightcoral") plt.legend()
from skopt.learning.gaussian_process import GaussianProcessRegressor
from skopt.learning.gaussian_process.kernels import ConstantKernel
from skopt.learning.gaussian_process.kernels import DotProduct
from skopt.learning.gaussian_process.kernels import ExpSineSquared
from skopt.learning.gaussian_process.kernels import HammingKernel
from skopt.learning.gaussian_process.kernels import Matern
from skopt.learning.gaussian_process.kernels import RationalQuadratic
from skopt.learning.gaussian_process.kernels import RBF
from skopt.learning.gaussian_process.kernels import WhiteKernel
KERNELS = []
for length_scale in [np.arange(1, 6), [0.2, 0.3, 0.5, 0.6, 0.1]]:
KERNELS.extend([
RBF(length_scale=length_scale),
Matern(length_scale=length_scale, nu=0.5),
Matern(length_scale=length_scale, nu=1.5),
Matern(length_scale=length_scale, nu=2.5),
RationalQuadratic(alpha=2.0, length_scale=2.0),
ExpSineSquared(length_scale=2.0, periodicity=3.0),
ConstantKernel(constant_value=1.0),
WhiteKernel(noise_level=2.0),
Matern(length_scale=length_scale, nu=2.5)**3.0,
RBF(length_scale=length_scale) +
Matern(length_scale=length_scale, nu=1.5),
RBF(length_scale=length_scale) *
Matern(length_scale=length_scale, nu=1.5),
DotProduct(sigma_0=2.0)
])
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_array_almost_equal
from sklearn.utils.testing import assert_array_equal
import pytest
from skopt.learning import GaussianProcessRegressor
from skopt.learning.gaussian_process.kernels import RBF
from skopt.learning.gaussian_process.kernels import Matern
from skopt.learning.gaussian_process.kernels import WhiteKernel
from skopt.learning.gaussian_process.gpr import _param_for_white_kernel_in_Sum
rng = np.random.RandomState(0)
X = rng.randn(5, 5)
y = rng.randn(5)
rbf = RBF()
wk = WhiteKernel()
mat = Matern()
kernel1 = rbf
kernel2 = mat + rbf
kernel3 = mat * rbf
kernel4 = wk * rbf
kernel5 = mat + rbf * wk
def predict_wrapper(X, gpr):
"""Predict that can handle 1-D input"""
X = np.expand_dims(X, axis=0)
return gpr.predict(X, return_std=True)
first_loop_legal_upper_bounds = [i for i in range(3, 301, 3)]
#second_loop_legal_upper_bounds = [i // 3 * 4 for i in first_loop_legal_upper_bounds]
#space_size = len(first_loop_legal_upper_bounds)
#total_amount_of_inner_part = [first_loop_legal_upper_bounds[i] * second_loop_legal_upper_bounds[i] \
# for i in range(space_size)]
def crop_number(n):
return min(first_loop_legal_upper_bounds, key=lambda t: abs(t - n))
###
# In[ ]:
kernel = Product(ConstantKernel(1), RBF(1)) + ConstantKernel(1)
model = GaussianProcessRegressor(alpha=0,
normalize_y=True,
noise='gaussian',
n_restarts_optimizer=10,
kernel=kernel)
optimizer = Optimizer([[low_constraint, high_constraint]] * dim,
model,
n_initial_points=n_initial_points,
acq_func='EI',
acq_optimizer='lbfgs',
random_state=None)
# In[ ]:
import numpy as np
from scipy import optimize
from sklearn.utils.testing import assert_array_almost_equal
from skopt.learning.gaussian_process.kernels import ConstantKernel
from skopt.learning.gaussian_process.kernels import DotProduct
from skopt.learning.gaussian_process.kernels import Exponentiation
from skopt.learning.gaussian_process.kernels import ExpSineSquared
from skopt.learning.gaussian_process.kernels import Matern
from skopt.learning.gaussian_process.kernels import RationalQuadratic
from skopt.learning.gaussian_process.kernels import RBF
from skopt.learning.gaussian_process.kernels import WhiteKernel
length_scale = np.arange(1, 6)
KERNELS = [
RBF(length_scale=length_scale),
Matern(length_scale=length_scale, nu=0.5),
Matern(length_scale=length_scale, nu=1.5),
Matern(length_scale=length_scale, nu=2.5),
RationalQuadratic(alpha=2.0, length_scale=2.0),
ExpSineSquared(length_scale=2.0, periodicity=3.0),
ConstantKernel(constant_value=1.0),
WhiteKernel(noise_level=2.0),
Matern(length_scale=length_scale, nu=2.5)**3.0,
RBF(length_scale=length_scale) + Matern(length_scale=length_scale, nu=1.5),
RBF(length_scale=length_scale) * Matern(length_scale=length_scale, nu=1.5),
DotProduct(sigma_0=2.0)
]
rng = np.random.RandomState(0)
X = rng.randn(5)
def cook_estimator(base_estimator, space=None, **kwargs):
"""
Cook a default estimator.
For the special base_estimator called "DUMMY" the return value is None.
This corresponds to sampling points at random, hence there is no need
for an estimator.
Parameters
----------
* `base_estimator` ["GP", "RF", "ET", "GBRT", "DUMMY"
or sklearn regressor, default="GP"]:
Should inherit from `sklearn.base.RegressorMixin`.
In addition the `predict` method should have an optional `return_std`
argument, which returns `std(Y | x)`` along with `E[Y | x]`.
If base_estimator is one of ["GP", "RF", "ET", "GBRT", "DUMMY"], a
surrogate model corresponding to the relevant `X_minimize` function
is created.
* `space` [Space instance]:
Has to be provided if the base_estimator is a gaussian process.
Ignored otherwise.
* `kwargs` [dict]:
Extra parameters provided to the base_estimator at init time.
"""
if isinstance(base_estimator, str):
base_estimator = base_estimator.upper()
if base_estimator not in ["GP", "ET", "RF", "GBRT", "DUMMY", "GPM32", "GPM1", "RBF", "RQ"]:
raise ValueError("Valid strings for the base_estimator parameter "
" are: 'RF', 'ET', 'GP', 'GBRT' or 'DUMMY' not "
"%s." % base_estimator)
elif not is_regressor(base_estimator):
raise ValueError("base_estimator has to be a regressor.")
if base_estimator == "GP":
if space is not None:
space = Space(space)
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
is_cat = space.is_categorical
else:
raise ValueError("Expected a Space instance, not None.")
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
# only special if *all* dimensions are categorical
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(n_dims))
else:
other_kernel = Matern(
length_scale=np.ones(n_dims),
length_scale_bounds=[(0.01, 100)] * n_dims, nu=2.5)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True, noise="gaussian",
n_restarts_optimizer=2)
elif base_estimator == "GPM32":
if space is not None:
space = Space(space)
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
is_cat = space.is_categorical
else:
raise ValueError("Expected a Space instance, not None.")
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
# only special if *all* dimensions are categorical
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(n_dims))
else:
other_kernel = Matern(
length_scale=np.ones(n_dims),
length_scale_bounds=[(0.01, 100)] * n_dims, nu=1.5)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True, noise="gaussian",
n_restarts_optimizer=2)
elif base_estimator == "GPM1":
if space is not None:
space = Space(space)
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
is_cat = space.is_categorical
else:
raise ValueError("Expected a Space instance, not None.")
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
# only special if *all* dimensions are categorical
if is_cat:
other_kernel = HammingKernel(length_scale=np.ones(n_dims))
else:
other_kernel = Matern(
length_scale=np.ones(n_dims),
length_scale_bounds=[(0.01, 100)] * n_dims, nu=1.5)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True, noise="gaussian",
n_restarts_optimizer=2)
elif base_estimator == "RBF":
if space is not None:
space = Space(space)
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
other_kernel = RBF(length_scale=np.ones(n_dims))
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True, noise="gaussian",
n_restarts_optimizer=2)
elif base_estimator == "RQ":
if space is not None:
space = Space(space)
space = Space(normalize_dimensions(space.dimensions))
n_dims = space.transformed_n_dims
cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0))
other_kernel = RationalQuadratic(length_scale=np.ones(n_dims), alpha=0.1)
base_estimator = GaussianProcessRegressor(
kernel=cov_amplitude * other_kernel,
normalize_y=True, noise="gaussian",
n_restarts_optimizer=2)
elif base_estimator == "RF":
base_estimator = RandomForestRegressor(n_estimators=100,
min_samples_leaf=3)
elif base_estimator == "ET":
base_estimator = ExtraTreesRegressor(n_estimators=100,
min_samples_leaf=3)
elif base_estimator == "GBRT":
gbrt = GradientBoostingRegressor(n_estimators=30, loss="quantile")
base_estimator = GradientBoostingQuantileRegressor(base_estimator=gbrt)
elif base_estimator == "DUMMY":
return None
base_estimator.set_params(**kwargs)
return base_estimator
def minimal_gp():
kernel = (ConstantKernel(constant_value=1**2,
constant_value_bounds=(0.01**2, 1**2)) *
RBF(length_scale=1.0, length_scale_bounds=(0.5, 1.5)))
gp = BayesGPR(random_state=1, normalize_y=False, kernel=kernel)
return gp
from utils import black_box
from utils import plot_space
import numpy as np
from skopt.optimizer import Optimizer
from skopt.learning import GaussianProcessRegressor
from skopt.learning.gaussian_process.kernels import RBF
from skopt.benchmarks import branin
# Search from 0.0 to 6.0
dimensions = ((0.0, 6.0),)
# Initialize estimator.
gpr = GaussianProcessRegressor(kernel=RBF(), noise=0.0)
optimizer = Optimizer(
dimensions=dimensions,
base_estimator=gpr,
n_random_starts=0,
acq_func="LCB",
random_state=0)
# Tell some points to the optimizer and ask for the next point.
X = np.reshape(np.linspace(4, 6, 10), (-1, 1)).tolist()
y = [black_box(xi) for xi in X]
optimizer.tell(X, y)
x_cand = optimizer.ask()
y_cand = black_box(x_cand)
plot = plot_space(X, y, optimizer.models[-1], x_cand)
plot.show()
# Tell and ask again.
def fit(self, X, y):
"""Fit Gaussian process regression model.
Parameters
----------
X : array-like, shape = (n_samples, n_features)
Training data
y : array-like, shape = (n_samples, [n_output_dims])
Target values
Returns
-------
self
Returns an instance of self.
"""
if self.kernel is None:
self.kernel = ConstantKernel(1.0, constant_value_bounds="fixed") \
* RBF(1.0, length_scale_bounds="fixed")
if self.noise and not _param_for_white_kernel_in_Sum(self.kernel)[0]:
if self.noise == "gaussian":
self.kernel = self.kernel + WhiteKernel()
else:
self.kernel = self.kernel + WhiteKernel(
noise_level=self.noise, noise_level_bounds="fixed")
super(GaussianProcessRegressor, self).fit(X, y)
self.noise_ = None
if self.noise:
# The noise component of this kernel should be set to zero
# while estimating K(X_test, X_test)
# Note that the term K(X, X) should include the noise but
# this (K(X, X))^-1y is precomputed as the attribute `alpha_`.
# (Notice the underscore).
# This has been described in Eq 2.24 of
# http://www.gaussianprocess.org/gpml/chapters/RW2.pdf
# Hence this hack
if isinstance(self.kernel_, WhiteKernel):
self.kernel_.set_params(noise_level=0.0)
else:
white_present, white_param = _param_for_white_kernel_in_Sum(
self.kernel_)
# This should always be true. Just in case.
if white_present:
noise_kernel = self.kernel_.get_params()[white_param]
self.noise_ = noise_kernel.noise_level
self.kernel_.set_params(
**{white_param: WhiteKernel(noise_level=0.0)})
# Precompute arrays needed at prediction
L_inv = solve_triangular(self.L_.T, np.eye(self.L_.shape[0]))
self.K_inv_ = L_inv.dot(L_inv.T)
# Fix deprecation warning #462
if int(sklearn.__version__[2:4]) >= 23:
self.y_train_std_ = self._y_train_std
self.y_train_mean_ = self._y_train_mean
elif int(sklearn.__version__[2:4]) >= 19:
self.y_train_mean_ = self._y_train_mean
self.y_train_std_ = 1
else:
self.y_train_mean_ = self.y_train_mean
self.y_train_std_ = 1
return self
