def GP_stacking(y_train, X_train, X_test): m = SparseGPRegression(X_train, y_train, num_inducing=20, kernel=RBF(X_train.shape[1])) m.optimize("bfgs", messages=0) y_pred = m.predict(X_test)[0] return y_pred.flatten()
def _update(self, new_X, new_y, eps=EPSILON): X, y = self.data X = deepcopy(X) y = deepcopy(y) if len(X) >= self._r_min: X_vec = np.array([self.params2vec(x) for x in X]) y = np.array(y) if self._is_normalize: sig = np.sqrt(np.var(y)) sig = max(sig, eps) mu = np.mean(y) y = (y - mu) / sig if self._backend == "sklearn": if self._kernel is None: self._kernel = Matern(nu=2.5) self.model = GaussianProcessRegressor( kernel=self._kernel, n_restarts_optimizer=self._n_restarts_optimizer, random_state=self._random_state, normalize_y=False ) self.model.fit(X_vec, y) elif self._backend.lower() == "gpy": y = np.array(y)[:, None] if self.model is None: self._create_model(X_vec, y) else: self.model.set_XY(X_vec, y) self.model.optimize_restarts(self._n_restarts_optimizer, optimizer=self._optimizer, max_iters=self._max_iters, messages=False, verbose=False, ipython_notebook=False)
def model_builder(X, Y, kernel): if X.shape[0] < 1000 and not sparse: tmp = GPRegression(X, Y, kernel=kernel) else: tmp = SparseGPRegression(X, Y, num_inducing=num_inducing, kernel=kernel) if name is not None: tmp.name = name return tmp
def given_resample(self, data=[], dt=1): """ Resample the dynamics function given a list of inferred voltage and state trajectories """ # import pdb; pdb.set_trace() uu = self.Z[:, 0] V = self.Z[:, 1] # Evaluate dz/dt using true Kdr dynamics g = lambda x: x**4 ginv = lambda u: u**(1. / 4) dg_dx = lambda x: 4 * x**3 logistic = sigma logit = sigma_inv dlogit = lambda x: 1. / (x * (1.0 - x)) u_to_x = lambda u: ginv(logistic(u)) x_to_u = lambda x: logit(g(x)) # uu = x_to_u(xx) # Compute dynamics du/dt alpha = lambda V: 0.01 * (V + 55.) / (1 - np.exp(-(V + 55.) / 10.)) beta = lambda V: 0.125 * np.exp(-(V + 65.) / 80.) dx_dt = lambda x, V: alpha(V) * (1 - x) - beta(V) * x du_dt = lambda u, V: dlogit(g(u_to_x(u))) * dg_dx(u_to_x(u)) * dx_dt( u_to_x(u), V) X = self.Z Y = du_dt(uu, V)[:, None] # Set up the sparse GP regression model with the sampled inputs and outputs # gpr = SparseGPRegression(X, Y, self.kernel, Z=self.Z) # gpr.likelihood.variance = 0.01 gpr = GPRegression(X, Y, self.kernel) # HACK: Optimize the hyperparameters # contrain all parameters to be positive # gpr.constrain_positive('') # optimize and plot # gpr.ensure_default_constraints() # gpr.optimize_restarts(num_restarts=10) # gpr.plot() # import pdb; pdb.set_trace() # HACK: Rather than using a truly nonparametric approach, just sample # the GP at the grid of inducing points and interpolate at the GP mean self.h = gpr.posterior_samples_f(self.Z, size=1) # HACK: Recreate the GP with the sampled function h self.gp = SparseGPRegression(self.Z, self.h, self.kernel, num_inducing=25)
def _create_model(self, X, y): """ Creates the GPy model given some input data X and Y. """ # Define kernel input_dim = X.shape[1] if self._kernel is None: kern = GPy.kern.Matern52(input_dim, variance=1., ARD=self._ARD) else: kern = self._kernel self._kernel = None # Define model noise_var = y.var() * 0.01 if not self._sparse: self.model = GPRegression(X, y, kernel=kern, noise_var=noise_var) else: self.model = SparseGPRegression(X, y, kernel=kern, num_inducing=self._num_inducing) self.model.Gaussian_noise.constrain_bounded(1e-9, 1e6, warning=False)
def compute_runtimes(dimensions, n, num_inducing, kernel_class, simulator_class, tag: str): """Compute runtimes at different dimensionalities for sample_size iterations at every dimension.""" sample_size = 10 kernel_creation = [] gp_creation = [] gp_optimization = [] for i in trange(len(dimensions)): dim = dimensions[i] kernel_creation_array = np.empty(sample_size) gp_creation_array = np.empty(sample_size) gp_optimization_array = np.empty(sample_size) for j in range(sample_size): simulator = simulator_class(n, random_state=j) data = simulator.simulate(dim, n_informative=n) a = time.time() kernel = kernel_class(dim) b = time.time() m = SparseGPRegression(data.X_train, data.y_train, num_inducing=num_inducing, kernel=kernel) c = time.time() m.optimize() d = time.time() kernel_creation_array[j] = b - a gp_creation_array[j] = c - b gp_optimization_array[j] = d - c kernel_creation.append(kernel_creation_array.mean()) gp_creation.append(gp_creation_array.mean()) gp_optimization.append(gp_optimization_array.mean()) return RuntimeResult(kernel_creation, gp_creation, gp_optimization, dimensions, tag)
def gp_surrogate(self, Z=None, Y=None, kern_x=None, kern_p=None, num_inducing=None): self.set_training_data(Z, Y) assert self.Z is not None and self.Y is not None self.set_kernels(kern_x, kern_p) assert self.kern_x is not None and self.kern_p is not None R, J = binary_dimensions(self.Z, self.binary_variables) gps = [] for e in range(self.num_outputs): gps.append([]) for r in R: Jr = (J == r) if not np.any(Jr): gps[e].append(None) continue dim_xb = self.dim_x - self.dim_b dim = self.dim_x + self.dim_p kernx = self.kern_x(dim_xb, self.non_binary_variables, 'kernx') kernp = self.kern_p(self.dim_p, range(self.dim_x, dim), 'kernp') #Zr = self.Z[ np.ix_(Jr, I ) ] Zr = self.Z[Jr] Yr = self.Y[np.ix_(Jr, [e])] numi = self.max_num_inducing if num_inducing is None \ else num_inducing gp = SparseGPRegression(Zr, Yr, kernx * kernp, num_inducing=numi) gps[e].append(gp) self.gps = gps
def _gp_regression(self, X, Y, kern, **kwargs): num_inducing = kwargs.get('num_inducing', 100) return SparseGPRegression(X, Y, kern, num_inducing=num_inducing)
# plt.scatter(X, y) # plt.show() gp_regression = GPRegression(X, y) gp_regression.optimize(messages=True) log_likelihood1 = gp_regression.log_likelihood() model_output(gp_regression, title="GP Regression with loglikelihood: " + str(log_likelihood1)) ################################# # inducing variables, u. Each inducing variable has its own associated input index, Z, which lives in the same space as X. Z = np.hstack((np.linspace(2.5, 4., 3), np.linspace(7, 8.5, 3)))[:, None] sparse_regression = SparseGPRegression(X, y, kernel=rbf, Z=Z) sparse_regression.noise_var = noise_var sparse_regression.inducing_inputs.constrain_fixed() sparse_regression.optimize(messages=True) log_likelihood2 = sparse_regression.log_likelihood() #inducing variables fixed model_output( sparse_regression, title="Inducing variables fixed parameters optimized with loglikelihood: " + str(log_likelihood2[0][0])) #inducing variables optimized sparse_regression.inducing_inputs.unconstrain()
class BayesSampler(BaseSampler): """Bayesian optimization sampler Sample next location based on gaussian process Parameters ---------- space: list(dict) Define search space. Each element has to the following key values: 'name', 'type', and 'domain' (,'num_grid' is optional). init_X: array-like(float), shape=(n_samples, n_dim) The list of parameters to initizlie sampler init_y: array-like(float), shape(n_samples,) The list of score of init_X r_min: int The number of random samples before starting using gaussian process method: str The name of acquisition functions kernel: kernel object, optional is_normalize: bool If ture, normalized score values are used for optimization n_restarts_optimizer: int The number of trial to opimize GP hyperparameters backend: str (default 'gpy') Determine which GP package is used. That has to be either of 'gpy' or 'sklearn'. optimizer: str The name of optimizers of hyperparameters of GP, which is valid when backend='gpy'. max_iters: int The maximum number of iteration to optimize hyperparamters of GP, which is valid when backend='gpy'. ARD: bool Wheather to use ARD for kernel, which is valid when backend='gpy'. sparse: bool If true, use sparse GP, which is valid when backend='gpy'. num_inducing: int The number of inducing inputs for sparse GP, which is valid when backend='gpy' and sparse is True. random_state: int """ sampler_name = "bayes" def __init__(self, space, init_X=None, init_y=None, r_min=3, method="EI", kernel=None, is_normalize=True, n_restarts_optimizer=10, backend="gpy", optimizer="bfgs", max_iters=1000, ARD=False, sparse=False, num_inducing=10, random_state=RANDOM_STATE): super(BayesSampler, self).__init__(space, init_X, init_y) self._r_min = r_min self.model = None self.acquisition_func = self._get_acquisition_func(method) self._is_normalize = is_normalize self._ARD = ARD self._kernel = kernel self._sparse = sparse self._num_inducing = num_inducing self._optimizer = optimizer self._max_iters = max_iters self._backend = backend.lower() self._n_restarts_optimizer = n_restarts_optimizer self._random_state = random_state def _update(self, new_X, new_y, eps=EPSILON): X, y = self.data X = deepcopy(X) y = deepcopy(y) if len(X) >= self._r_min: X_vec = np.array([self.params2vec(x) for x in X]) y = np.array(y) if self._is_normalize: sig = np.sqrt(np.var(y)) sig = max(sig, eps) mu = np.mean(y) y = (y - mu) / sig if self._backend == "sklearn": if self._kernel is None: self._kernel = Matern(nu=2.5) self.model = GaussianProcessRegressor( kernel=self._kernel, n_restarts_optimizer=self._n_restarts_optimizer, random_state=self._random_state, normalize_y=False ) self.model.fit(X_vec, y) elif self._backend.lower() == "gpy": y = np.array(y)[:, None] if self.model is None: self._create_model(X_vec, y) else: self.model.set_XY(X_vec, y) self.model.optimize_restarts(self._n_restarts_optimizer, optimizer=self._optimizer, max_iters=self._max_iters, messages=False, verbose=False, ipython_notebook=False) def _create_model(self, X, y): """ Creates the GPy model given some input data X and Y. """ # Define kernel input_dim = X.shape[1] if self._kernel is None: kern = GPy.kern.Matern52(input_dim, variance=1., ARD=self._ARD) else: kern = self._kernel self._kernel = None # Define model noise_var = y.var() * 0.01 if not self._sparse: self.model = GPRegression(X, y, kernel=kern, noise_var=noise_var) else: self.model = SparseGPRegression(X, y, kernel=kern, num_inducing=self._num_inducing) self.model.Gaussian_noise.constrain_bounded(1e-9, 1e6, warning=False) def sample(self, num_samples=1, *args, **kwargs): """Sample next location to evaluate based on GP Parameters --------- num_samples: int The number of samples Returns ------- Xs: list(dict), length is num_samples """ _num_data = self.num_data if _num_data < self._r_min: Xs = self._random_sample(num_samples) else: Xs = self._bayes_sample(num_samples) return Xs def _bayes_sample(self, num_samples, num_restarts=25): num_restarts = max(num_samples, num_restarts) init_params = self._random_sample(num_restarts) init_xs = [self.params2vec(param) for param in init_params] bounds = self.design_space.get_bounds() if self._backend == "sklearn": evaluated_loss = np.array(self.model.y_train_) else: evaluated_loss = np.array(self.model.Y)[:, 0] ys = [] xs = [] def minus_ac(x): return -self.acquisition_func(x, self.model, evaluated_loss, mode=self._backend) for x0 in init_xs: res = minimize(fun=minus_ac, x0=x0, bounds=bounds, method='L-BFGS-B') ys.append(-res.fun) xs.append(res.x) idx = np.argsort(ys)[::-1][:num_samples] best_x = np.array(xs)[idx] best_params = [self.vec2params(x) for x in best_x] return best_params def _random_sample(self, num_samples): Xs = [] for i in range(num_samples): x = random_sample(self.params_conf) Xs.append(x) return list(Xs) def _get_acquisition_func(self, method): if method == "EI": return expected_improvement else: raise NotImplementedError(method) def params2vec(self, params): # Not include fixed params vec = [] for conf in self.params_conf: val = params[conf['name']] vec.append(val) vec = self.design_space.objective_to_model([vec]) return np.array(vec) def vec2params(self, vec): # Not include fixed params params = {} vec = self.design_space.model_to_objective([vec]) for i, val in enumerate(vec): conf = self.params_conf[i] params[conf["name"]] = val return params