def interpolate(thetas,
z_thetas,
xx,
yy,
method='linear',
z_uncertainties_thetas=None,
matern_exponent=0.5,
length_scale_min=0.001,
length_scale_default=1.,
length_scale_max=1000.,
noise_level=0.001,
subtract_min=False):
if method == 'cubic':
interpolator = CloughTocher2DInterpolator(thetas[:], z_thetas)
zz = interpolator(np.dstack((xx.flatten(), yy.flatten())))
zi = zz.reshape(xx.shape)
elif method == 'gp':
if z_uncertainties_thetas is not None:
gp = GaussianProcessRegressor(
normalize_y=True,
kernel=ConstantKernel(1.0, (1.e-9, 1.e9)) * Matern(
length_scale=[length_scale_default],
length_scale_bounds=[(length_scale_min, length_scale_max)],
nu=matern_exponent) + WhiteKernel(noise_level),
n_restarts_optimizer=10,
alpha=z_uncertainties_thetas)
else:
gp = GaussianProcessRegressor(
normalize_y=True,
kernel=ConstantKernel(1.0, (1.e-9, 1.e9)) * Matern(
length_scale=length_scale_default,
length_scale_bounds=(length_scale_min, length_scale_max),
nu=matern_exponent) + WhiteKernel(noise_level),
n_restarts_optimizer=10)
gp.fit(thetas[:], z_thetas[:])
zz, _ = gp.predict(np.c_[xx.ravel(), yy.ravel()], return_std=True)
zi = zz.reshape(xx.shape)
elif method == 'linear':
interpolator = LinearNDInterpolator(thetas[:], z_thetas)
zz = interpolator(np.dstack((xx.flatten(), yy.flatten())))
zi = zz.reshape(xx.shape)
else:
raise ValueError
mle = np.unravel_index(zi.argmin(), zi.shape)
if subtract_min:
zi -= zi[mle]
return zi, mle
def test_gpr_uses_noise():
""" Test that gpr is using WhiteKernel by default"""
X = np.random.normal(size=[100, 2])
Y = np.random.normal(size=[100])
g_gaussian = GaussianProcessRegressor()
g_gaussian.fit(X, Y)
m, sigma = g_gaussian.predict(X[0:1], return_cov=True)
assert sigma > 0
def test_gpr_uses_noise():
""" Test that gpr is using WhiteKernel"""
X = np.random.normal(size=[100, 2])
Y = np.random.normal(size=[100])
g_gaussian = GaussianProcessRegressor(noise='gaussian')
g_gaussian.fit(X, Y)
m, sigma = g_gaussian.predict(X[0:1], return_cov=True)
assert sigma > 0
def test_acquisition_api():
rng = np.random.RandomState(0)
X = rng.randn(10, 2)
y = rng.randn(10)
gpr = GaussianProcessRegressor()
gpr.fit(X, y)
for method in [gaussian_ei, gaussian_lcb, gaussian_pi]:
assert_array_equal(method(X, gpr).shape, 10)
assert_raises(ValueError, method, rng.rand(10), gpr)
def test_acquisition_api():
rng = np.random.RandomState(0)
X = rng.randn(10, 2)
y = rng.randn(10)
gpr = GaussianProcessRegressor()
gpr.fit(X, y)
for method in [gaussian_ei, gaussian_lcb, gaussian_pi]:
assert_array_equal(method(X, gpr).shape, 10)
assert_raises(ValueError, method, rng.rand(10), gpr)
def gpbo_cycle(ndim,
space,
target_f,
n_iters=10,
acq_function=ei,
model=None,
n_multi_start=100,
show_progress=True):
xrange = (lambda title, n: tqdm_notebook(range(n), postfix=title)
) if show_progress else (lambda title, n: range(n))
space = np.array(space)
if model is None:
kernel = WhiteKernel(0.001, noise_level_bounds=[1.0e-5, 1.0e-3]) + \
Matern(1.0, nu=1.5, length_scale_bounds=[1.0e-3, 1.0e+3])
model = GaussianProcessRegressor(kernel=kernel,
normalize_y=False,
noise=None,
n_restarts_optimizer=2)
known_points = []
known_values = []
cost = []
for i in xrange('BO iteration', n_iters):
acq = acq_function(model, known_points, known_values)
candidates = []
for _ in xrange('acquisition', n_multi_start):
x0 = np.random.uniform(size=(ndim, ))
x, f, _ = fmin_l_bfgs_b(maxiter=1000,
func=acq,
x0=x0,
approx_grad=False,
bounds=[(0, 1)] * ndim)
candidates.append((x, f))
best = np.argmin([f for x, f in candidates])
suggestion, _ = candidates[best]
suggestion = reverse_transform(suggestion.reshape(1, -1), space)[0, :]
point_cost, observed = target_f(suggestion)
known_points.append(suggestion)
known_values.append(observed)
cost.append(point_cost)
model.fit(transform(np.array(known_points), space),
np.array(known_values))
yield model, acq, space, known_points, known_values, cost
def test_acquisition_gradient():
rng = np.random.RandomState(0)
X = rng.randn(20, 5)
y = rng.randn(20)
X_new = rng.randn(5)
mat = Matern()
wk = WhiteKernel()
gpr = GaussianProcessRegressor(kernel=mat + wk)
gpr.fit(X, y)
for acq_func in ["LCB", "PI", "EI"]:
check_gradient_correctness(X_new, gpr, acq_func, np.max(y))
def test_acquisition_gradient():
rng = np.random.RandomState(0)
X = rng.randn(20, 5)
y = rng.randn(20)
X_new = rng.randn(5)
mat = Matern()
wk = WhiteKernel()
gpr = GaussianProcessRegressor(kernel=mat + wk)
gpr.fit(X, y)
for acq_func in ["LCB", "PI", "EI"]:
check_gradient_correctness(X_new, gpr, acq_func, np.max(y))
def optimize(self,
num_vars,
objective_function,
gradient_function=None,
variable_bounds=None,
initial_point=None):
callbacks = []
def alt_obj_fn(pars):
fn = objective_function(pars)
callbacks.append({'point': pars, 'fn': fn})
return fn
result = super().optimize(num_vars, alt_obj_fn, gradient_function,
variable_bounds, initial_point)
if self._num_restarts is not None:
for i in range(self._num_restarts - 1):
if variable_bounds is not None:
init_pt = [
np.random.uniform(dn, up) for dn, up in variable_bounds
]
else:
init_pt = [
np.random.uniform(-np.pi, +np.pi)
for _ in range(num_vars)
]
result_new = super().optimize(num_vars, alt_obj_fn,
gradient_function,
variable_bounds, init_pt)
if result_new[1] < result[1]:
result = result_new
X = [step['point'] for step in callbacks]
y = [step['fn'] for step in callbacks]
if self._make_model and (len(callbacks) < self._max_model_points):
model = GaussianProcessRegressor()
model.fit(X, y)
else:
model = None
if variable_bounds is not None:
space = Space([Real(low, high) for low, high in variable_bounds])
else:
space = None
self.optimization_result = create_result(
X, y, space=space, models=[model] if model is not None else None)
return result
def __init__(self, seq_len, embedder, Xinit, yinit, noise_std=None):
self.seq_len = seq_len
self.noise_std = noise_std
self.current_best_seq = None
self.current_best_val = np.inf
self.X_sample = []
self.y_sample = []
self.nqueries = 0
m52 = ConstantKernel(1.0) * StringEmbedKernel(seq_len=seq_len,
embedder=embedder)
if noise_std is None:
noise_std = np.std(yinit)
gpr = GaussianProcessRegressor(kernel=m52, alpha=noise_std**2)
gpr.fit(Xinit, yinit)
self.gpr = gpr
def test_white_kernel_as_noise():
# first .fit()
gpr1 = GaussianProcessRegressor(rbf + wk).fit(X, y)
gpr2 = GaussianProcessRegressor(rbf, noise="gaussian").fit(X, y)
mean1, std1 = gpr1.predict(X, return_std=True)
mean2, std2 = gpr2.predict(X, return_std=True)
assert_almost_equal(gpr1.kernel_.k2.noise_level, gpr2.noise_, 4)
assert not np.any(std1 == std2)
assert _param_for_white_kernel_in_Sum(gpr1.kernel_)[1] == 'k2'
assert _param_for_white_kernel_in_Sum(gpr2.kernel_)[1] == 'k2'
# second .fit()
gpr1 = gpr1.fit(X, y)
gpr2 = gpr2.fit(X, y)
mean1, std1 = gpr1.predict(X, return_std=True)
mean2, std2 = gpr2.predict(X, return_std=True)
assert_almost_equal(gpr1.kernel_.k2.noise_level, gpr2.noise_, 4)
assert _param_for_white_kernel_in_Sum(gpr1.kernel_)[1] == 'k2'
assert _param_for_white_kernel_in_Sum(gpr2.kernel_)[1] == 'k2'
assert not np.any(std1 == std2)
def test_gpr_handles_similar_points():
"""
This tests whether our implementation of GPR
does not crash when the covariance matrix whose
inverse is calculated during fitting of the
regressor is singular.
Singular covariance matrix often indicates
that same or very close points are explored
during the optimization procedure.
Essentially checks that the default value of `alpha` is non zero.
"""
X = np.random.rand(8, 3)
y = np.random.rand(8)
X[:3, :] = 0.0
y[:3] = 1.0
model = GaussianProcessRegressor()
# this fails if singular matrix is not handled
model.fit(X, y)
def test_gpr_handles_similar_points():
"""
This tests whether our implementation of GPR
does not crash when the covariance matrix whose
inverse is calculated during fitting of the
regressor is singular.
Singular covariance matrix often indicates
that same or very close points are explored
during the optimization procedure.
Essentially checks that the default value of `alpha` is non zero.
"""
X = np.random.rand(8, 3)
y = np.random.rand(8)
X[:3, :] = 0.0
y[:3] = 1.0
model = GaussianProcessRegressor()
# this fails if singular matrix is not handled
model.fit(X, y)
class BayeOpt():
def __init__(self,
obj_func,
bounds,
gpr_obj,
init_x,
niter = 100,
n_restarts = 20): # replace Gaussian with Matern52 later
"""
obj_func: function to minimize
constraint_func: constraint function
constraint_value: the minimum of allowable value
bounds: the bounds of the value np.array(d * 2)
init_x: initial data np.array(n * d)
gpr: gaussian process for object function
gpr_constraint: gaussian process for constraint
niter: the iter times
n_restarts: initial points to find minimums
"""
# data parameters
self.obj_func = obj_func
self.bounds = bounds
self.gpr_obj = gpr_obj
self.init_x = init_x
self.niter = niter
self.n_restarts = n_restarts
self.x = self.init_x
self.y_obj = np.array([self.obj_func(t) for t in self.x]).reshape(-1, 1)
# GP parameters
self.f_best = self.y_obj.min()
print(self.y_obj.shape, self.y_obj)
def expected_improvement(self, X, xi=0.01):
'''
Computes the EI at points X based on existing samples X_sample
and Y_sample using a Gaussian process surrogate model.
Args:
X: Points at which EI shall be computed (m x d).
X_sample: Sample locations (n x d).
Y_sample: Sample values (n x 1).
gpr: A GaussianProcessRegressor fitted to samples.
xi: Exploitation-exploration trade-off parameter.
Returns:
Expected improvements at points X.
'''
#print(X,self.gpr_obj.predict(X))
mu_obj, sigma_obj = self.gpr_obj.predict(X, return_std=True)
mu_sample = self.gpr_obj.predict(self.x)
mu_obj = mu_obj.reshape(-1, 1)
sigma_obj = sigma_obj.reshape(-1, 1)
# Needed for noise-based model,
# otherwise use np.max(Y_sample).
# See also section 2.4 in [...]
# compute ei
mu_sample_opt = np.min(mu_sample)
with np.errstate(divide='warn'):
imp = mu_sample_opt - mu_obj - xi
Z = imp / sigma_obj
ei = imp * norm.cdf(Z) + sigma_obj * norm.pdf(Z)
ei[sigma_obj == 0.0] = 0.0
return ei
def propose_location(self):
'''
Proposes the next sampling point by optimizing the acquisition function.
Args:
acquisition: Acquisition function.
X_sample: Sample locations (n x d).
Y_sample: Sample values (n x 1).
gpr: A GaussianProcessRegressor fitted to samples.
Returns:
Location of the acquisition function maximum.
'''
dim = self.x.shape[1]
min_val = 1
min_x = None
def min_obj(X):
# Minimization objective is the negative acquisition function
return -1 * self.expected_improvement(X.reshape(-1, dim))
# return -acquisition(X, X_sample, Y_sample, gpr)
# Find the best optimum by starting from n_restart different random points.
for x0 in np.random.uniform(self.bounds[:, 0], self.bounds[:, 1], size=(self.n_restarts, dim)):
#print("------------------------")
res = minimize(min_obj, x0=x0, bounds=self.bounds, method='L-BFGS-B') #’Nelder-Mead’ ,'L-BFGS-B'
if res.fun < min_val:
min_val = res.fun[0]
min_x = res.x
return min_x
def bay_opt(self):
# Initialize sample
for _ in range(self.niter):
m52_1 = ConstantKernel(1) * RBF(np.array([100]*12))
self.gpr_obj = GaussianProcessRegressor(kernel=m52_1, alpha=10, noise = "gaussian")
# Update Gaussian process with existing samples
#print(self.x.shape,self.y_obj.shape )
self.gpr_obj.fit(self.x, self.y_obj)
#print(self.gpr_obj.predict(self.x))
# Obtain next sampling point from the acquisition function (expected_improvement)
X_next = self.propose_location()
# Obtain next noisy sample from the objective function
Y_next1 = np.array([self.obj_func(X_next)]).reshape(-1,1)
#print(Y_next1, Y_next1.shape, Y_next2,Y_next2.shape)
# Add sample to previous samples
self.x = np.vstack((self.x, X_next))
self.y_obj = np.vstack((self.y_obj, Y_next1))
self.f_best = np.min(self.y_obj)
self.min_x = self.x[np.argmin(self.y_obj)]
return self.f_best, self.min_x
from skopt.learning.gaussian_process.kernels import RBF all_x = np.reshape(np.linspace(0, 6, 100), (-1, 1)) all_f = [black_box(xi) for xi in all_x] # Plot all points. plt.plot(all_x, all_f) # Train only one third of the training data. X = np.reshape(np.linspace(4, 6, 10), (-1, 1)) y = [black_box(xi) for xi in X] # Use RBF kernel. rbf = RBF(length_scale=1.0) gpr = GaussianProcessRegressor(kernel=rbf, alpha=1e-12) gpr.fit(X, y) plt.plot(np.ravel(X), y, "ro", label="Fit points") # Predict on all data. y_pred, y_std = gpr.predict(all_x, return_std=True) all_x_plot = np.ravel(all_x) upper_bound = y_pred + 1.96 * y_std lower_bound = y_pred - 1.96 * y_std plt.plot(all_x_plot, y_pred, "r--", label="Predictions") plt.plot(all_x_plot, lower_bound, color="red") plt.plot(all_x_plot, upper_bound, color="red") plt.fill_between(all_x_plot, lower_bound, upper_bound, facecolor="lightcoral") plt.legend() plt.show()
class BayeOpt():
def __init__(self,
obj_func,
constraint_func,
constraint_value,
bounds,
gpr_obj,
gpr_constraint,
init_x,
niter=100,
n_restarts=20): # replace Gaussian with Matern52 later
"""
obj_func: function to minimize
constraint_func: constraint function
constraint_value: the minimum of allowable value
bounds: the bounds of the value np.array(d * 2)
init_x: initial data np.array(n * d)
gpr: gaussian process for object function
gpr_constraint: gaussian process for constraint
niter: the iter times
n_restarts: initial points to find minimums
"""
# data parameters
self.obj_func = obj_func
self.constraint_func = constraint_func
self.constraint_value = constraint_value
self.bounds = bounds
self.gpr_obj = gpr_obj
self.gpr_constraint = gpr_constraint
self.init_x = init_x
self.niter = niter
self.n_restarts = n_restarts
self.x = self.init_x
self.y_obj = np.array([self.obj_func(t)
for t in self.x]).reshape(-1, 1)
self.y_constraint = np.array([self.constraint_func(t)
for t in self.x]).reshape(-1, 1)
# GP parameters
self.f_best = self.y_obj.min()
print(self.y_obj.shape, self.y_obj)
print(self.y_constraint.shape, self.y_constraint)
def expected_improvement(self, X, xi=0.01):
#print(X,self.gpr_obj.predict(X))
mu_obj, sigma_obj = self.gpr_obj.predict(X, return_std=True)
mu_constraint, sigma_constraint = self.gpr_constraint.predict(
X, return_std=True)
mu_sample = self.gpr_obj.predict(self.x)
mu_obj = mu_obj.reshape(-1, 1)
sigma_obj = sigma_obj.reshape(-1, 1)
sigma_constraint = sigma_constraint.reshape(-1, 1)
mu_constraint = mu_constraint.reshape(-1, 1)
# Needed for noise-based model,
# otherwise use np.max(Y_sample).
# See also section 2.4 in [...]
# compute ei
mu_sample_opt = np.min(mu_sample)
with np.errstate(divide='warn'):
imp = mu_sample_opt - mu_obj - xi
Z = imp / sigma_obj
ei = imp * norm.cdf(Z) + sigma_obj * norm.pdf(Z)
ei[sigma_obj == 0.0] = 0.0
# compute PF(x):
Z_constraint = (self.constraint_value -
mu_constraint) / sigma_constraint
FP = 1 - norm.cdf(Z_constraint)
# print(ei * FP)
return ei * FP
def propose_location(self):
dim = self.x.shape[1]
min_val = 1
min_x = None
def min_obj(X):
# Minimization objective is the negative acquisition function
return -1 * self.expected_improvement(X.reshape(-1, dim))
# Find the best optimum by starting from n_restart different random points.
for x0 in np.random.uniform(self.bounds[:, 0],
self.bounds[:, 1],
size=(self.n_restarts, dim)):
#print("------------------------")
res = minimize(min_obj,
x0=x0,
bounds=self.bounds,
method='L-BFGS-B') #’Nelder-Mead’ ,'L-BFGS-B'
if res.fun < min_val:
min_val = res.fun[0]
min_x = res.x
return min_x
def bay_opt(self):
# Initialize sample
for i in range(self.niter):
m52_1 = ConstantKernel(1) * RBF(np.array([100] * 12))
self.gpr_obj = GaussianProcessRegressor(kernel=m52_1,
alpha=10,
noise="gaussian")
m52_2 = ConstantKernel(1) * RBF(np.array([100] * 12))
self.gpr_constraint = GaussianProcessRegressor(kernel=m52_2,
alpha=10,
noise="gaussian")
# Update Gaussian process with existing samples
#print(self.x.shape,self.y_obj.shape )
self.gpr_obj.fit(self.x, self.y_obj)
#print(self.gpr_obj.predict(self.x))
self.gpr_constraint.fit(self.x, self.y_constraint)
# Obtain next sampling point from the acquisition function (expected_improvement)
X_next = self.propose_location()
# Obtain next noisy sample from the objective function
Y_next1 = np.array([self.obj_func(X_next)]).reshape(-1, 1)
Y_next2 = np.array([self.constraint_func(X_next)]).reshape(-1, 1)
#print(Y_next1, Y_next1.shape, Y_next2,Y_next2.shape)
# Add sample to previous samples
self.x = np.vstack((self.x, X_next))
self.y_obj = np.vstack((self.y_obj, Y_next1))
self.y_constraint = np.vstack((self.y_constraint, Y_next2))
idx = np.where(self.y_constraint > 0)[0]
t = idx[np.argmin(self.y_obj[idx])]
self.f_best = self.y_obj[t]
self.min_x = self.x[t]
return self.f_best, self.min_x
noise = 0.2
bounds = np.array([[-1.0, 2.0]])
def f(X, noise=noise):
return -np.sin(3*X) - X**2 + 0.7*X + noise * np.random.randn(*X.shape)
X = np.arange(bounds[:, 0], bounds[:, 1], 0.01).reshape(-1, 1)
# Noise-free objective function values at X
Y = f(X,0)
# Use custom kernel and estimator to match previous example
m52 = ConstantKernel(1.0) * Matern(length_scale=1.0, nu=2.5)
gpr = GaussianProcessRegressor(kernel=m52, alpha=noise**2)
X_init = np.array([[-0.9], [1.1]])
Y_init = f(X_init)
r = gp_minimize(lambda x: -f(np.array(x))[0],
bounds.tolist(),
base_estimator=gpr,
acq_func='EI', # expected improvement
xi=0.01, # exploitation-exploration trade-off
n_calls=10, # number of iterations
n_random_starts=0, # initial samples are provided
x0=X_init.tolist(), # initial samples
y0=-Y_init.ravel())
# Fit GP model to samples for plotting results
gpr.fit(r.x_iters, -r.func_vals)
print("So far")
# Plot the fitted model and the noisy samples
plot_approximation(gpr, X, Y, r.x_iters, -r.func_vals, show_legend=True)
plt.show()
def dictListToNpArray(dictList):
return np.array([list(dict.values()) for dict in dictList])
XSobol = dictListToNpArray(XSobolJson)
YSobol = dictListToNpArray(YSobolJson)
XOSS = dictListToNpArray(XOSSJson)
YOSS = dictListToNpArray(YOSSJson)
# Reception and connection check
assert(XSobol.shape[0] == YSobol.shape[0])
# Compute the Kriging surrogate
print("Fitting the Kriging Model")
krigingModel = GaussianProcessRegressor(random_state=0)
krigingModel.fit(XSobol, YSobol)
# Compute the XGBoost surrogate
print("Fitting the XGBoost Model")
XGBoostModel = fitXGBoost(XSobol, YSobol)
# At this point, we have the XGBoost surrogate model. What we need next is the bit which returns the parameterisations
# for positive calibrations.
nbModels = 2
predictions = [None] * nbModels
# This is not nice at all
print("Predicting on the two models")
predictions[0] = krigingModel.predict(XOSS).flatten()
predictions[1] = XGBoostModel.predict(XOSS).flatten()
