def interpolate(thetas, z_thetas, xx, yy, method='linear', z_uncertainties_thetas=None, matern_exponent=0.5, length_scale_min=0.001, length_scale_default=1., length_scale_max=1000., noise_level=0.001, subtract_min=False): if method == 'cubic': interpolator = CloughTocher2DInterpolator(thetas[:], z_thetas) zz = interpolator(np.dstack((xx.flatten(), yy.flatten()))) zi = zz.reshape(xx.shape) elif method == 'gp': if z_uncertainties_thetas is not None: gp = GaussianProcessRegressor( normalize_y=True, kernel=ConstantKernel(1.0, (1.e-9, 1.e9)) * Matern( length_scale=[length_scale_default], length_scale_bounds=[(length_scale_min, length_scale_max)], nu=matern_exponent) + WhiteKernel(noise_level), n_restarts_optimizer=10, alpha=z_uncertainties_thetas) else: gp = GaussianProcessRegressor( normalize_y=True, kernel=ConstantKernel(1.0, (1.e-9, 1.e9)) * Matern( length_scale=length_scale_default, length_scale_bounds=(length_scale_min, length_scale_max), nu=matern_exponent) + WhiteKernel(noise_level), n_restarts_optimizer=10) gp.fit(thetas[:], z_thetas[:]) zz, _ = gp.predict(np.c_[xx.ravel(), yy.ravel()], return_std=True) zi = zz.reshape(xx.shape) elif method == 'linear': interpolator = LinearNDInterpolator(thetas[:], z_thetas) zz = interpolator(np.dstack((xx.flatten(), yy.flatten()))) zi = zz.reshape(xx.shape) else: raise ValueError mle = np.unravel_index(zi.argmin(), zi.shape) if subtract_min: zi -= zi[mle] return zi, mle
def test_gpr_uses_noise(): """ Test that gpr is using WhiteKernel by default""" X = np.random.normal(size=[100, 2]) Y = np.random.normal(size=[100]) g_gaussian = GaussianProcessRegressor() g_gaussian.fit(X, Y) m, sigma = g_gaussian.predict(X[0:1], return_cov=True) assert sigma > 0
def test_gpr_uses_noise(): """ Test that gpr is using WhiteKernel""" X = np.random.normal(size=[100, 2]) Y = np.random.normal(size=[100]) g_gaussian = GaussianProcessRegressor(noise='gaussian') g_gaussian.fit(X, Y) m, sigma = g_gaussian.predict(X[0:1], return_cov=True) assert sigma > 0
def test_acquisition_api(): rng = np.random.RandomState(0) X = rng.randn(10, 2) y = rng.randn(10) gpr = GaussianProcessRegressor() gpr.fit(X, y) for method in [gaussian_ei, gaussian_lcb, gaussian_pi]: assert_array_equal(method(X, gpr).shape, 10) assert_raises(ValueError, method, rng.rand(10), gpr)
def test_acquisition_api(): rng = np.random.RandomState(0) X = rng.randn(10, 2) y = rng.randn(10) gpr = GaussianProcessRegressor() gpr.fit(X, y) for method in [gaussian_ei, gaussian_lcb, gaussian_pi]: assert_array_equal(method(X, gpr).shape, 10) assert_raises(ValueError, method, rng.rand(10), gpr)
def gpbo_cycle(ndim, space, target_f, n_iters=10, acq_function=ei, model=None, n_multi_start=100, show_progress=True): xrange = (lambda title, n: tqdm_notebook(range(n), postfix=title) ) if show_progress else (lambda title, n: range(n)) space = np.array(space) if model is None: kernel = WhiteKernel(0.001, noise_level_bounds=[1.0e-5, 1.0e-3]) + \ Matern(1.0, nu=1.5, length_scale_bounds=[1.0e-3, 1.0e+3]) model = GaussianProcessRegressor(kernel=kernel, normalize_y=False, noise=None, n_restarts_optimizer=2) known_points = [] known_values = [] cost = [] for i in xrange('BO iteration', n_iters): acq = acq_function(model, known_points, known_values) candidates = [] for _ in xrange('acquisition', n_multi_start): x0 = np.random.uniform(size=(ndim, )) x, f, _ = fmin_l_bfgs_b(maxiter=1000, func=acq, x0=x0, approx_grad=False, bounds=[(0, 1)] * ndim) candidates.append((x, f)) best = np.argmin([f for x, f in candidates]) suggestion, _ = candidates[best] suggestion = reverse_transform(suggestion.reshape(1, -1), space)[0, :] point_cost, observed = target_f(suggestion) known_points.append(suggestion) known_values.append(observed) cost.append(point_cost) model.fit(transform(np.array(known_points), space), np.array(known_values)) yield model, acq, space, known_points, known_values, cost
def test_acquisition_gradient(): rng = np.random.RandomState(0) X = rng.randn(20, 5) y = rng.randn(20) X_new = rng.randn(5) mat = Matern() wk = WhiteKernel() gpr = GaussianProcessRegressor(kernel=mat + wk) gpr.fit(X, y) for acq_func in ["LCB", "PI", "EI"]: check_gradient_correctness(X_new, gpr, acq_func, np.max(y))
def test_acquisition_gradient(): rng = np.random.RandomState(0) X = rng.randn(20, 5) y = rng.randn(20) X_new = rng.randn(5) mat = Matern() wk = WhiteKernel() gpr = GaussianProcessRegressor(kernel=mat + wk) gpr.fit(X, y) for acq_func in ["LCB", "PI", "EI"]: check_gradient_correctness(X_new, gpr, acq_func, np.max(y))
def optimize(self, num_vars, objective_function, gradient_function=None, variable_bounds=None, initial_point=None): callbacks = [] def alt_obj_fn(pars): fn = objective_function(pars) callbacks.append({'point': pars, 'fn': fn}) return fn result = super().optimize(num_vars, alt_obj_fn, gradient_function, variable_bounds, initial_point) if self._num_restarts is not None: for i in range(self._num_restarts - 1): if variable_bounds is not None: init_pt = [ np.random.uniform(dn, up) for dn, up in variable_bounds ] else: init_pt = [ np.random.uniform(-np.pi, +np.pi) for _ in range(num_vars) ] result_new = super().optimize(num_vars, alt_obj_fn, gradient_function, variable_bounds, init_pt) if result_new[1] < result[1]: result = result_new X = [step['point'] for step in callbacks] y = [step['fn'] for step in callbacks] if self._make_model and (len(callbacks) < self._max_model_points): model = GaussianProcessRegressor() model.fit(X, y) else: model = None if variable_bounds is not None: space = Space([Real(low, high) for low, high in variable_bounds]) else: space = None self.optimization_result = create_result( X, y, space=space, models=[model] if model is not None else None) return result
def __init__(self, seq_len, embedder, Xinit, yinit, noise_std=None): self.seq_len = seq_len self.noise_std = noise_std self.current_best_seq = None self.current_best_val = np.inf self.X_sample = [] self.y_sample = [] self.nqueries = 0 m52 = ConstantKernel(1.0) * StringEmbedKernel(seq_len=seq_len, embedder=embedder) if noise_std is None: noise_std = np.std(yinit) gpr = GaussianProcessRegressor(kernel=m52, alpha=noise_std**2) gpr.fit(Xinit, yinit) self.gpr = gpr
def test_white_kernel_as_noise(): # first .fit() gpr1 = GaussianProcessRegressor(rbf + wk).fit(X, y) gpr2 = GaussianProcessRegressor(rbf, noise="gaussian").fit(X, y) mean1, std1 = gpr1.predict(X, return_std=True) mean2, std2 = gpr2.predict(X, return_std=True) assert_almost_equal(gpr1.kernel_.k2.noise_level, gpr2.noise_, 4) assert not np.any(std1 == std2) assert _param_for_white_kernel_in_Sum(gpr1.kernel_)[1] == 'k2' assert _param_for_white_kernel_in_Sum(gpr2.kernel_)[1] == 'k2' # second .fit() gpr1 = gpr1.fit(X, y) gpr2 = gpr2.fit(X, y) mean1, std1 = gpr1.predict(X, return_std=True) mean2, std2 = gpr2.predict(X, return_std=True) assert_almost_equal(gpr1.kernel_.k2.noise_level, gpr2.noise_, 4) assert _param_for_white_kernel_in_Sum(gpr1.kernel_)[1] == 'k2' assert _param_for_white_kernel_in_Sum(gpr2.kernel_)[1] == 'k2' assert not np.any(std1 == std2)
def test_gpr_handles_similar_points(): """ This tests whether our implementation of GPR does not crash when the covariance matrix whose inverse is calculated during fitting of the regressor is singular. Singular covariance matrix often indicates that same or very close points are explored during the optimization procedure. Essentially checks that the default value of `alpha` is non zero. """ X = np.random.rand(8, 3) y = np.random.rand(8) X[:3, :] = 0.0 y[:3] = 1.0 model = GaussianProcessRegressor() # this fails if singular matrix is not handled model.fit(X, y)
def test_gpr_handles_similar_points(): """ This tests whether our implementation of GPR does not crash when the covariance matrix whose inverse is calculated during fitting of the regressor is singular. Singular covariance matrix often indicates that same or very close points are explored during the optimization procedure. Essentially checks that the default value of `alpha` is non zero. """ X = np.random.rand(8, 3) y = np.random.rand(8) X[:3, :] = 0.0 y[:3] = 1.0 model = GaussianProcessRegressor() # this fails if singular matrix is not handled model.fit(X, y)
class BayeOpt(): def __init__(self, obj_func, bounds, gpr_obj, init_x, niter = 100, n_restarts = 20): # replace Gaussian with Matern52 later """ obj_func: function to minimize constraint_func: constraint function constraint_value: the minimum of allowable value bounds: the bounds of the value np.array(d * 2) init_x: initial data np.array(n * d) gpr: gaussian process for object function gpr_constraint: gaussian process for constraint niter: the iter times n_restarts: initial points to find minimums """ # data parameters self.obj_func = obj_func self.bounds = bounds self.gpr_obj = gpr_obj self.init_x = init_x self.niter = niter self.n_restarts = n_restarts self.x = self.init_x self.y_obj = np.array([self.obj_func(t) for t in self.x]).reshape(-1, 1) # GP parameters self.f_best = self.y_obj.min() print(self.y_obj.shape, self.y_obj) def expected_improvement(self, X, xi=0.01): ''' Computes the EI at points X based on existing samples X_sample and Y_sample using a Gaussian process surrogate model. Args: X: Points at which EI shall be computed (m x d). X_sample: Sample locations (n x d). Y_sample: Sample values (n x 1). gpr: A GaussianProcessRegressor fitted to samples. xi: Exploitation-exploration trade-off parameter. Returns: Expected improvements at points X. ''' #print(X,self.gpr_obj.predict(X)) mu_obj, sigma_obj = self.gpr_obj.predict(X, return_std=True) mu_sample = self.gpr_obj.predict(self.x) mu_obj = mu_obj.reshape(-1, 1) sigma_obj = sigma_obj.reshape(-1, 1) # Needed for noise-based model, # otherwise use np.max(Y_sample). # See also section 2.4 in [...] # compute ei mu_sample_opt = np.min(mu_sample) with np.errstate(divide='warn'): imp = mu_sample_opt - mu_obj - xi Z = imp / sigma_obj ei = imp * norm.cdf(Z) + sigma_obj * norm.pdf(Z) ei[sigma_obj == 0.0] = 0.0 return ei def propose_location(self): ''' Proposes the next sampling point by optimizing the acquisition function. Args: acquisition: Acquisition function. X_sample: Sample locations (n x d). Y_sample: Sample values (n x 1). gpr: A GaussianProcessRegressor fitted to samples. Returns: Location of the acquisition function maximum. ''' dim = self.x.shape[1] min_val = 1 min_x = None def min_obj(X): # Minimization objective is the negative acquisition function return -1 * self.expected_improvement(X.reshape(-1, dim)) # return -acquisition(X, X_sample, Y_sample, gpr) # Find the best optimum by starting from n_restart different random points. for x0 in np.random.uniform(self.bounds[:, 0], self.bounds[:, 1], size=(self.n_restarts, dim)): #print("------------------------") res = minimize(min_obj, x0=x0, bounds=self.bounds, method='L-BFGS-B') #’Nelder-Mead’ ,'L-BFGS-B' if res.fun < min_val: min_val = res.fun[0] min_x = res.x return min_x def bay_opt(self): # Initialize sample for _ in range(self.niter): m52_1 = ConstantKernel(1) * RBF(np.array([100]*12)) self.gpr_obj = GaussianProcessRegressor(kernel=m52_1, alpha=10, noise = "gaussian") # Update Gaussian process with existing samples #print(self.x.shape,self.y_obj.shape ) self.gpr_obj.fit(self.x, self.y_obj) #print(self.gpr_obj.predict(self.x)) # Obtain next sampling point from the acquisition function (expected_improvement) X_next = self.propose_location() # Obtain next noisy sample from the objective function Y_next1 = np.array([self.obj_func(X_next)]).reshape(-1,1) #print(Y_next1, Y_next1.shape, Y_next2,Y_next2.shape) # Add sample to previous samples self.x = np.vstack((self.x, X_next)) self.y_obj = np.vstack((self.y_obj, Y_next1)) self.f_best = np.min(self.y_obj) self.min_x = self.x[np.argmin(self.y_obj)] return self.f_best, self.min_x
from skopt.learning.gaussian_process.kernels import RBF all_x = np.reshape(np.linspace(0, 6, 100), (-1, 1)) all_f = [black_box(xi) for xi in all_x] # Plot all points. plt.plot(all_x, all_f) # Train only one third of the training data. X = np.reshape(np.linspace(4, 6, 10), (-1, 1)) y = [black_box(xi) for xi in X] # Use RBF kernel. rbf = RBF(length_scale=1.0) gpr = GaussianProcessRegressor(kernel=rbf, alpha=1e-12) gpr.fit(X, y) plt.plot(np.ravel(X), y, "ro", label="Fit points") # Predict on all data. y_pred, y_std = gpr.predict(all_x, return_std=True) all_x_plot = np.ravel(all_x) upper_bound = y_pred + 1.96 * y_std lower_bound = y_pred - 1.96 * y_std plt.plot(all_x_plot, y_pred, "r--", label="Predictions") plt.plot(all_x_plot, lower_bound, color="red") plt.plot(all_x_plot, upper_bound, color="red") plt.fill_between(all_x_plot, lower_bound, upper_bound, facecolor="lightcoral") plt.legend() plt.show()
class BayeOpt(): def __init__(self, obj_func, constraint_func, constraint_value, bounds, gpr_obj, gpr_constraint, init_x, niter=100, n_restarts=20): # replace Gaussian with Matern52 later """ obj_func: function to minimize constraint_func: constraint function constraint_value: the minimum of allowable value bounds: the bounds of the value np.array(d * 2) init_x: initial data np.array(n * d) gpr: gaussian process for object function gpr_constraint: gaussian process for constraint niter: the iter times n_restarts: initial points to find minimums """ # data parameters self.obj_func = obj_func self.constraint_func = constraint_func self.constraint_value = constraint_value self.bounds = bounds self.gpr_obj = gpr_obj self.gpr_constraint = gpr_constraint self.init_x = init_x self.niter = niter self.n_restarts = n_restarts self.x = self.init_x self.y_obj = np.array([self.obj_func(t) for t in self.x]).reshape(-1, 1) self.y_constraint = np.array([self.constraint_func(t) for t in self.x]).reshape(-1, 1) # GP parameters self.f_best = self.y_obj.min() print(self.y_obj.shape, self.y_obj) print(self.y_constraint.shape, self.y_constraint) def expected_improvement(self, X, xi=0.01): #print(X,self.gpr_obj.predict(X)) mu_obj, sigma_obj = self.gpr_obj.predict(X, return_std=True) mu_constraint, sigma_constraint = self.gpr_constraint.predict( X, return_std=True) mu_sample = self.gpr_obj.predict(self.x) mu_obj = mu_obj.reshape(-1, 1) sigma_obj = sigma_obj.reshape(-1, 1) sigma_constraint = sigma_constraint.reshape(-1, 1) mu_constraint = mu_constraint.reshape(-1, 1) # Needed for noise-based model, # otherwise use np.max(Y_sample). # See also section 2.4 in [...] # compute ei mu_sample_opt = np.min(mu_sample) with np.errstate(divide='warn'): imp = mu_sample_opt - mu_obj - xi Z = imp / sigma_obj ei = imp * norm.cdf(Z) + sigma_obj * norm.pdf(Z) ei[sigma_obj == 0.0] = 0.0 # compute PF(x): Z_constraint = (self.constraint_value - mu_constraint) / sigma_constraint FP = 1 - norm.cdf(Z_constraint) # print(ei * FP) return ei * FP def propose_location(self): dim = self.x.shape[1] min_val = 1 min_x = None def min_obj(X): # Minimization objective is the negative acquisition function return -1 * self.expected_improvement(X.reshape(-1, dim)) # Find the best optimum by starting from n_restart different random points. for x0 in np.random.uniform(self.bounds[:, 0], self.bounds[:, 1], size=(self.n_restarts, dim)): #print("------------------------") res = minimize(min_obj, x0=x0, bounds=self.bounds, method='L-BFGS-B') #’Nelder-Mead’ ,'L-BFGS-B' if res.fun < min_val: min_val = res.fun[0] min_x = res.x return min_x def bay_opt(self): # Initialize sample for i in range(self.niter): m52_1 = ConstantKernel(1) * RBF(np.array([100] * 12)) self.gpr_obj = GaussianProcessRegressor(kernel=m52_1, alpha=10, noise="gaussian") m52_2 = ConstantKernel(1) * RBF(np.array([100] * 12)) self.gpr_constraint = GaussianProcessRegressor(kernel=m52_2, alpha=10, noise="gaussian") # Update Gaussian process with existing samples #print(self.x.shape,self.y_obj.shape ) self.gpr_obj.fit(self.x, self.y_obj) #print(self.gpr_obj.predict(self.x)) self.gpr_constraint.fit(self.x, self.y_constraint) # Obtain next sampling point from the acquisition function (expected_improvement) X_next = self.propose_location() # Obtain next noisy sample from the objective function Y_next1 = np.array([self.obj_func(X_next)]).reshape(-1, 1) Y_next2 = np.array([self.constraint_func(X_next)]).reshape(-1, 1) #print(Y_next1, Y_next1.shape, Y_next2,Y_next2.shape) # Add sample to previous samples self.x = np.vstack((self.x, X_next)) self.y_obj = np.vstack((self.y_obj, Y_next1)) self.y_constraint = np.vstack((self.y_constraint, Y_next2)) idx = np.where(self.y_constraint > 0)[0] t = idx[np.argmin(self.y_obj[idx])] self.f_best = self.y_obj[t] self.min_x = self.x[t] return self.f_best, self.min_x
noise = 0.2 bounds = np.array([[-1.0, 2.0]]) def f(X, noise=noise): return -np.sin(3*X) - X**2 + 0.7*X + noise * np.random.randn(*X.shape) X = np.arange(bounds[:, 0], bounds[:, 1], 0.01).reshape(-1, 1) # Noise-free objective function values at X Y = f(X,0) # Use custom kernel and estimator to match previous example m52 = ConstantKernel(1.0) * Matern(length_scale=1.0, nu=2.5) gpr = GaussianProcessRegressor(kernel=m52, alpha=noise**2) X_init = np.array([[-0.9], [1.1]]) Y_init = f(X_init) r = gp_minimize(lambda x: -f(np.array(x))[0], bounds.tolist(), base_estimator=gpr, acq_func='EI', # expected improvement xi=0.01, # exploitation-exploration trade-off n_calls=10, # number of iterations n_random_starts=0, # initial samples are provided x0=X_init.tolist(), # initial samples y0=-Y_init.ravel()) # Fit GP model to samples for plotting results gpr.fit(r.x_iters, -r.func_vals) print("So far") # Plot the fitted model and the noisy samples plot_approximation(gpr, X, Y, r.x_iters, -r.func_vals, show_legend=True) plt.show()
def dictListToNpArray(dictList): return np.array([list(dict.values()) for dict in dictList]) XSobol = dictListToNpArray(XSobolJson) YSobol = dictListToNpArray(YSobolJson) XOSS = dictListToNpArray(XOSSJson) YOSS = dictListToNpArray(YOSSJson) # Reception and connection check assert(XSobol.shape[0] == YSobol.shape[0]) # Compute the Kriging surrogate print("Fitting the Kriging Model") krigingModel = GaussianProcessRegressor(random_state=0) krigingModel.fit(XSobol, YSobol) # Compute the XGBoost surrogate print("Fitting the XGBoost Model") XGBoostModel = fitXGBoost(XSobol, YSobol) # At this point, we have the XGBoost surrogate model. What we need next is the bit which returns the parameterisations # for positive calibrations. nbModels = 2 predictions = [None] * nbModels # This is not nice at all print("Predicting on the two models") predictions[0] = krigingModel.predict(XOSS).flatten() predictions[1] = XGBoostModel.predict(XOSS).flatten()