def optimize_s_sn_l(self, sn, s, l): assert (isinstance(sn, float)) assert (l.shape == (self.fix_W.shape[1],)) # Create a GP self.kernel.update_params(W=self.fix_W, s=s, l=l) gp_reg = GPRegression(self.X, self.Y.reshape(-1, 1), self.kernel, noise_var=sn) try: gp_reg.optimize(optimizer="lbfgs", max_iters=config['max_iter_parameter_optimization']) except Exception as e: print(e) print(gp_reg.kern.K(gp_reg.X)) print("Error above!") # TODO: does this optimization work in the correct direction? new_variance = gp_reg.kern.inner_kernel.variance new_lengthscale = gp_reg.kern.inner_kernel.lengthscale new_sn = gp_reg['Gaussian_noise.variance'] assert gp_reg.kern.inner_kernel.lengthscale is not None assert gp_reg.kern.inner_kernel.variance is not None # assert not np.isclose(np.asarray(new_lengthscale), np.zeros_like(new_lengthscale) ).all(), new_lengthscale return float(new_variance), new_lengthscale.copy(), float(new_sn)
def permute(self, varb=None): # get model input x = self.x.copy() y = self.y.copy() # shuffle targeet variable col = np.where(varb == self.x_keys)[0] # which one? shuffled = np.random.choice(x[:, col].ravel(), size=x.shape[0], replace=False) x[:, col] = shuffled[:, np.newaxis] # replace # same steps as fit, below can be done more concisely, but I worry about minor differenes x_dim = x.shape[1] y_dim = y.shape[1] kern = buildKernel(x_dim, ARD=self.ARD) mcopy = GPRegression(x, y, kern) if self.heteroscedastic: kern = addFixedKernel(kern, y_dim, self.error) mcopy = GPRegression(x, y, kern) mcopy.optimize() return mcopy.log_likelihood()
def maximize(self, iterations): self.start_time2 = time() for n in np.arange(iterations): X = np.array(self.x).reshape(-1, 1) if self.dim == 1 else np.array( self.x) Y = np.array(self.y).reshape(-1, 1) gpreg = GPRegression(X, Y, kernel=self.kernel, noise_var=self.noise_var) gpreg.optimize() # MAP for kernel hyper-parameters vals, par = self.minimize_negative_acquisition(gpreg) if self.dim == 1: self.give_new_point(vals, par) else: self.x.append(par[vals.argmin()]) self.y.append(self.objective(self.x[-1])) if self.log_info: print( "%s step in BO; objective value: %.4f at %.4f; time: %.2f s." % (len(self.x), self.y[-1], self.x[-1], time() - self.start_time2 + self.end_time1 - self.start_time1)) if n % 10 == 0: self.a += 1 self.aquis_par.append(self.sample_aquis_param(self.a)) self.end_time2 = time()
def fit(self): if self.model: self.noise = self.model.Gaussian_noise.variance[0] return None x_dim = self.x.shape[1] # number of input dimensions, 1 if only time y_dim = self.y.shape[ 1] # number of ouptut dimensions, typically only 1 for log OD kern = buildKernel(x_dim, ARD=self.ARD) m = GPRegression(self.x, self.y, kern) if self.heteroscedastic: kern = addFixedKernel(kern, y_dim, self.error) m = GPRegression(self.x, self.y, kern) m.optimize() self.noise = m.Gaussian_noise.variance[ 0] # should be negligible (<1e-10) for full model if self.heteroscedastic: m.kern = m.kern.parts[ 0] # cannot predict with fixed kernel, so remove it self.model = m
def _model_chooser(self): """ Initialize the model used for the optimization """ kernel = Matern52(len(self.variables_list), variance=1., ARD=False) gpmodel = GPRegression(self.X, self.Y, kernel) gpmodel.optimize() self.model = GPyModelWrapper(gpmodel) if self.noiseless: gpmodel.Gaussian_noise.constrain_fixed(0.001) self.model = GPyModelWrapper(gpmodel)
def predict_y2(self): x = self.x_new y = self.y1 m = GPRegression(x, y) m.optimize() mu, cov = (m.predictive_gradients(x)[0], None) self.y2 = mu[:, 0] self.cov2 = cov
def learn_flow(X, y, lengthscales, variance=1.0): dimensions = X.shape[1] # lengthscales = [ l_scale for d in range(dimensions)] kernel = GPy.kern.rbf(dimensions, ARD=True, lengthscale=lengthscales, variance=variance) m = GPRegression(X,y,kernel) m.optimize('bfgs', max_iters=1000) return m
def run_optimisation(current_range, freq_range, power_range): parameter_space = ParameterSpace([\ ContinuousParameter('current', current_range[0], current_range[1]), \ ContinuousParameter('freq', freq_range[0], freq_range[1]), \ ContinuousParameter('power', power_range[0], power_range[1]) ]) def function(X): current = X[:, 0] freq = X[:, 1] power = X[:, 2] out = np.zeros((len(current), 1)) for g in range(len(current)): ''' Set JPA Current, Frequency & Power ''' out[g, 0] = -get_SNR( plot=False)[-1] #Negative as want to maximise SNR return out num_data_points = 10 design = RandomDesign(parameter_space) X = design.get_samples(num_data_points) Y = function(X) model_gpy = GPRegression(X, Y) model_gpy.optimize() model_emukit = GPyModelWrapper(model_gpy) exp_imprv = ExpectedImprovement(model=model_emukit) optimizer = GradientAcquisitionOptimizer(space=parameter_space) point_calc = SequentialPointCalculator(exp_imprv, optimizer) coords = [] min = [] bayesopt_loop = BayesianOptimizationLoop(model=model_emukit, space=parameter_space, acquisition=exp_imprv, batch_size=1) stopping_condition = FixedIterationsStoppingCondition(i_max=100) bayesopt_loop.run_loop(q, stopping_condition) coord_results = bayesopt_loop.get_results().minimum_location min_value = bayesopt_loop.get_results().minimum_value step_results = bayesopt_loop.get_results().best_found_value_per_iteration print(coord_results) print(min_value) return coord_results, abs(min_value)
def fit(self, X_train, Y_train): num_samples = len(X_train) if num_samples > self.max_sample: print('num_samples > {}, splitting training; {} steps'.format( self.max_sample, max(10, num_samples // self.max_sample))) i = 0 while i < max(10, num_samples // self.max_sample): #if self.verbose: print('step {}/{}'.format( i + 1, max(10, num_samples // self.max_sample))) train_idx = np.random.choice(num_samples, self.max_sample, replace=False) X = X_train[train_idx] Y = Y_train[train_idx] kernels = [] for dim in range(X_train.shape[1]): kernels.append(self.kernel(1, active_dims=[dim])) kern_multi = ft.reduce(lambda a, b: a + b, kernels) try: gpr = GPRegression(X, Y, kernel=kern_multi, normalizer=self.normalizer) gpr.optimize(messages=self.verbose, max_iters=200) self.models.append(gpr) i += 1 except (np.linalg.LinAlgError): continue else: Repeat = True print('num_samples <= 300, single training') while Repeat: kernels = [] for dim in range(X_train.shape[1]): kernels.append(self.kernel(1, active_dims=[dim])) kern_multi = ft.reduce(lambda a, b: a + b, kernels) try: gpr = GPRegression(X_train, Y_train, kernel=kern_multi, normalizer=self.normalizer) gpr.optimize(messages=self.verbose, max_iters=200) self.models.append(gpr) Repeat = False except (np.linalg.LinAlgError): continue return self
def rmse_rbf(x_train: np.ndarray, y_train: np.ndarray, x_test: np.ndarray, y_test: np.ndarray) -> float: """RMSE of a GPy RBF kernel. :param x_train: :param y_train: :param x_test: :param y_test: :return: """ model = GPRegression(x_train, y_train, kernel=RBF(input_dim=x_train.shape[1])) model.optimize() return compute_gpy_model_rmse(model, x_test, y_test)
def maximize(self, iterations): self.start_time2 = time() for n in np.arange(iterations): X = np.array(self.x).reshape(-1, 1) if self.dim == 1 else np.array(self.x) Y = np.array(self.y).reshape(-1, 1) gpreg = GPRegression(X, Y, kernel=self.kernel, noise_var=self.noise_var) gpreg.optimize() # MAP for kernel hyper-parameters vals, par = self.minimize_negative_acquisition(gpreg) if self.dim == 1: self.give_new_point(vals, par) else: self.x.append(par[vals.argmin()]) self.y.append(self.objective(self.x[-1])) if self.log_info: print("%s step in BO; objective value: %.4f at %.4f; time: %.2f s." % (len(self.x), self.y[-1], self.x[-1], time() - self.start_time2 + self.end_time1 - self.start_time1)) if n % 10 == 0: self.a += 1 self.aquis_par.append(self.sample_aquis_param(self.a)) self.end_time2 = time()
class Kernel(object): def __init__(self, x0, y0, cons=None, alpha=opt.ke_alpha, beta=opt.ke_beta, input_size=opt.ke_input_size, hidden_size=opt.ke_hidden_size, num_layers=opt.ke_num_layers, bidirectional=opt.ke_bidirectional, lr=opt.ke_lr, weight_decay=opt.ke_weight_decay): super(Kernel, self).__init__() self.alpha = alpha self.beta = beta self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers, bidirectional=bidirectional) self.lstm = self.lstm.to(opt.device) self.input_size = input_size self.hidden_size = hidden_size self.num_layers = num_layers self.bidirectional = bidirectional self.bi = 2 if bidirectional else 1 self.x = [x0] self.y = torch.tensor([y0], dtype=torch.float, device=opt.device, requires_grad=False) self.cons = [cons] inp, out = clean_x(self.x, self.cons) self.model = GPRegression(inp, out) self.model.Gaussian_noise.constrain_fixed(1e-6, warning=False) self.model.optimize() self.x_best = x0 self.y_best = y0 self.i_best = 0 self.n = 1 self.E = self.embedding(x0).view(1, -1) self.K = self.kernel(self.E[0], self.E[0]).view(1, 1) self.K_inv = torch.inverse(self.K + self.beta * torch.eye(self.n, device=opt.device)) self.optimizer = optim.Adam(self.lstm.parameters(), lr=lr, weight_decay=weight_decay) def embedding(self, xi): inputs = xi.view(-1, 1, self.input_size) outputs, (hn, cn) = self.lstm(inputs) outputs = torch.mean(outputs.squeeze(1), dim=0) outputs = outputs / torch.norm(outputs) return outputs def kernel(self, ei, ej): d = ei - ej d = torch.sum(d * d) k = torch.exp(-d / (2 * self.alpha)) return k def kernel_batch(self, en): n = self.n k = torch.zeros(n, device=opt.device) for i in range(n): k[i] = self.kernel(self.E[i], en) return k def predict(self, xn): n = self.n en = self.embedding(xn) k = self.kernel_batch(en) kn = self.kernel(en, en) t = torch.mm(k.view(1, n), self.K_inv) mu = torch.mm(t, self.y.view(n, 1)) sigma = kn - torch.mm(t, k.view(n, 1)) sigma = torch.sqrt(sigma + self.beta) return mu, sigma def acquisition_cons(self, xn): with torch.no_grad(): xn_ = np.array([xn.cpu().numpy().flatten()]) mu_cons, sigma_cons = self.model.predict(xn_) sigma_cons = sqrt(sigma_cons) PoF = norm.cdf(0, mu_cons, sigma_cons) mu, sigma = self.predict(xn) mu = mu.item() sigma = sigma.item() y_best = self.y_best z = (mu - y_best) / sigma ei = (mu - y_best) * norm.cdf(z) + sigma * norm.pdf(z) return ei * PoF def acquisition(self, xn): with torch.no_grad(): mu, sigma = self.predict(xn) mu = mu.item() sigma = sigma.item() y_best = self.y_best z = (mu - y_best) / sigma ei = (mu - y_best) * norm.cdf(z) + sigma * norm.pdf(z) return ei def kernel_batch_ex(self, t): n = self.n k = torch.zeros(n - 1, device=opt.device) for i in range(t): k[i] = self.kernel(self.E[i], self.E[t]) for i in range(t + 1, n): k[i - 1] = self.kernel(self.E[t], self.E[i]) return k def predict_ex(self, t): n = self.n k = self.kernel_batch_ex(t) kt = self.kernel(self.E[t], self.E[t]) indices = list(range(t)) + list(range(t + 1, n)) indices = torch.tensor(indices, dtype=torch.long, device=opt.device) K = self.K K = torch.index_select(K, 0, indices) K = torch.index_select(K, 1, indices) K_inv = torch.inverse(K + self.beta * torch.eye(n - 1, device=opt.device)) y = torch.index_select(self.y, 0, indices) t = torch.mm(k.view(1, n - 1), K_inv) mu = torch.mm(t, y.view(n - 1, 1)) sigma = kt - torch.mm(t, k.view(n - 1, 1)) sigma = torch.sqrt(sigma + self.beta) return mu, sigma def add_sample(self, xn, yn, consn): self.x.append(xn) self.y = torch.cat((self.y, torch.tensor([yn], dtype=torch.float, device=opt.device, requires_grad=False))) self.cons.append(consn) inp, out = clean_x(self.x, self.cons) self.model.set_XY(inp, out) self.model.optimize() n = self.n if consn > 0: if yn > self.y_best: self.x_best = xn self.y_best = yn self.i_best = n en = self.embedding(xn) k = self.kernel_batch(en) kn = self.kernel(en, en) self.E = torch.cat((self.E, en.view(1, -1)), 0) self.K = torch.cat( (torch.cat((self.K, k.view(n, 1)), 1), torch.cat((k.view(1, n), kn.view(1, 1)), 1)), 0) self.n += 1 self.K_inv = torch.inverse(self.K + self.beta * torch.eye(self.n, device=opt.device)) def add_batch(self, x, y, cons): self.x.extend(x) self.y = torch.cat((self.y, y)) self.cons.extend(cons) inp, out = clean_x(self.x, self.cons) self.model.set_XY(inp, out) self.model.optimize() m = len(x) for i in range(m): n = self.n if self.cons[i] > 0: if y[i].item() > self.y_best: self.x_best = x[i] self.y_best = y[i].item() self.i_best = n en = self.embedding(x[i]) k = self.kernel_batch(en) kn = self.kernel(en, en) self.E = torch.cat((self.E, en.view(1, -1)), 0) self.K = torch.cat( (torch.cat((self.K, k.view(n, 1)), 1), torch.cat((k.view(1, n), kn.view(1, 1)), 1)), 0) self.n += 1 self.K_inv = torch.inverse(self.K + self.beta * torch.eye(self.n, device=opt.device)) def update_EK(self): n = self.n E_ = torch.zeros((n, self.E.size(1)), device=opt.device) for i in range(n): E_[i] = self.embedding(self.x[i]) self.E = E_ K_ = torch.zeros((n, n), device=opt.device) for i in range(n): for j in range(i, n): k = self.kernel(self.E[i], self.E[j]) K_[i, j] = k K_[j, i] = k self.K = K_ self.K_inv = torch.inverse(self.K + self.beta * torch.eye(self.n, device=opt.device)) def loss(self): n = self.n l = torch.zeros(n, device=opt.device) for i in range(n): mu, sigma = self.predict_ex(i) d = self.y[i] - mu l[i] = -(0.918939 + torch.log(sigma) + d * d / (2 * sigma * sigma)) l = -torch.mean(l) return l def opt_step(self): if self.n < 2: return 0.0 self.optimizer.zero_grad() l = self.loss() ll = -l.item() l.backward() self.optimizer.step() self.update_EK() return ll def save(self, save_path): path = os.path.dirname(save_path) if not os.path.exists(path): os.makedirs(path) torch.save(self, save_path)
def _fit_model(self, X, Y): model = GPRegression(X, Y, self.kernel) model.optimize(messages=False, max_f_eval=self.max_feval) self.model = model
sum1 = 0 for i in range(0, len(x) - 1): sum1 += (100 * (x[i + 1] - x[i]**2)**2 + (x[i] - 1)**2) return sum1 if __name__ == "__main__": np.random.seed(1) dim = 50 f = sphere X = np.random.uniform(-5, 5, (10, dim)) y = np.array([f(xi) for xi in X]).reshape(-1, 1) gpy_kern = RBFg(input_dim=dim, ARD=False) gpy_model = GPRegression(X, y, kernel=gpy_kern) gpy_model.optimize() mobo_kern = RBF(ARD=False) mobo_model = moboGP(mobo_kern, X, y) mobo_likelihood = Likelihood(mobo_model) mobo_likelihood.evaluate() optimizer = lbfgsb(mobo_model) optimizer.opt() print("gpy:", gpy_model.log_likelihood()) print("mobo:", mobo_model.log_likelihood) Xtest = np.random.uniform(-5, 5, (1000, dim)) ytest = np.array([f(xi) for xi in Xtest]).reshape(-1, 1)
mu, _ = model.predict(x_new) return mu[0, 0] list_mse_gpr = [] list_mse_lgb = [] list_mse_svr = [] for i in range(1): print(i) x_train_m, y_train_m = a_gpr.sample_point(x_train, y_train, iter=80) k_rbf = RBF(input_dim=n, variance=0.5, lengthscale=1) gp_model = GPRegression(x_train_m, np.reshape(y_train_m, (-1, 1)), kernel=k_rbf) gp_model.optimize(messages=False) model_lgb = lgb.LGBMRegressor() model_lgb.fit(x_train_m, y_train_m) model_svr = SVR() model_svr.fit(x_train_m, y_train_m) y_pre_con = [ pre_gp_mu_var(np.reshape(x_test[i], (1, -1)), gp_model) for i in range(np.shape(x_test)[0]) ] y_pre_lgb = [ model_lgb.predict(np.reshape(x_test[i], (1, -1))) for i in range(np.shape(x_test)[0]) ]
init_design = RandomDesign(space) X_init = init_design.get_samples(2) Y_init = np.array([b.objective_function(xi)["function_value"] for xi in X_init])[:, None] if args.model_type == "bnn": model = Bohamiann(X_init=X_init, Y_init=Y_init, verbose=True) elif args.model_type == "rf": model = RandomForest(X_init=X_init, Y_init=Y_init) with_gradients = False elif args.model_type == "gp": kernel = Matern52(len(list_params), variance=1., ARD=True) gpmodel = GPRegression(X_init, Y_init, kernel) gpmodel.optimize() model = GPyModelWrapper(gpmodel) acquisition = ExpectedImprovement(model) acquisition_optimizer = DirectOptimizer(space) candidate_point_calculator = Sequential(acquisition, acquisition_optimizer) bo = BayesianOptimizationLoop(model=model, space=space, X_init=X_init, Y_init=Y_init, acquisition=acquisition, candidate_point_calculator=candidate_point_calculator) overhead = [] st = time.time() for i in range(args.num_iterations): t = time.time() bo.run_loop(user_function=obj, stopping_condition=FixedIterationsStoppingCondition(i + X_init.shape[0]))
X[:x_half, :] = np.linspace(0, 2, x_half)[:, None] # First cluster of inputs/covariates X[x_half:, :] = np.linspace( 8, 10, x_half)[:, None] # Second cluster of inputs/covariates rbf = RBF(input_dim=1) mu = np.zeros(N) cov = rbf.K(X) + np.eye(N) * np.sqrt(noise_var) y = np.random.multivariate_normal(mu, cov).reshape(-1, 1) # plt.scatter(X, y) # plt.show() gp_regression = GPRegression(X, y) gp_regression.optimize(messages=True) log_likelihood1 = gp_regression.log_likelihood() model_output(gp_regression, title="GP Regression with loglikelihood: " + str(log_likelihood1)) ################################# # inducing variables, u. Each inducing variable has its own associated input index, Z, which lives in the same space as X. Z = np.hstack((np.linspace(2.5, 4., 3), np.linspace(7, 8.5, 3)))[:, None] sparse_regression = SparseGPRegression(X, y, kernel=rbf, Z=Z) sparse_regression.noise_var = noise_var sparse_regression.inducing_inputs.constrain_fixed() sparse_regression.optimize(messages=True)
def main(): print("######################") global target, X0, Y0, values, frac_M, frac_X, bo_flag #target_params = np.array([[0.14,0.4],[1.4,0.03]]) #target = LiX_wrapper(True,'LiF','Rocksalt','JC', # target_params,False,False,eng) target = np.array([[-764.5, 6.012 * 0.99, 6.012 * 0.99, 6.012 * 0.99]]) if focus == 'energy': target_comp = target[0, 0].reshape(1, -1) if focus == 'constant': target_comp = target[0, 1].reshape(1, -1) else: target_comp = target[0, :4].reshape(1, -1) print('Target initialized!') latin_design = LatinDesign(parameter_space=parameter_space) X0 = latin_design.get_samples(INIT_POINTS) Y0 = np.array([]) for x in X0: x = np.array([x]) Y0 = np.append(Y0, f.evaluate(x)) values = [] for y in Y0: values.append(y.Y) values = np.asarray(values, dtype=float) ### Redundancy check if (values[:, 7:-1] == values[0, 7]).all(): values = values[:, :7] frac_X = False if (values[:, 4:7] == values[0, 4]).all(): values = values[:, :4] frac_M = False values = values.reshape(-1, np.max(np.shape(target))) bo_flag = True if focus == 'energy': values = values[:, 0].reshape(-1, 1) if focus == 'constant': values = values[:, 1:4].reshape(-1, 3) ### BO Loop kern = Matern52(X0.shape[1], variance=1) model = GPRegression(X0, values, kernel=kern, normalizer=True, noise_var=NOISE) # Kernel = None: RBF default model.optimize(optimizer='lbfgsb') model.optimize_restarts(num_restarts=50, verbose=False) model_wrapped = GPyModelWrapper(model) acq = L2_LCB(model=model_wrapped, target=target_comp, beta=np.float64(1.)) # beta is the exploration constant bayesopt_loop = BayesianOptimizationLoop(model=model_wrapped, space=parameter_space, acquisition=acq) bayesopt_loop.run_loop(f, BO_ITER) return save(bayesopt_loop)
class DCKE(RegressorMixin): """ Dynamically Controlled Kernel Estimation Computes the conditional expectation $E[Y \mid X=x]$ from a training set $X_i$, $y_i$, $i=1, \ldots, N$ of joint realizations of $X$ and $Y$ for an arbitrary prediction set of $x$'s. The DCKE regressor first uses local regression on a mesh grid to solve the problem on the mesh grid and then uses GPR to evaluate in between the points on the mesh grid. Optionally, a control variate $Z$ can be supplied together with $\mu_Z = E[Z \mid X=x_k]$ for the points $x_k$ on the mesh grid. In that case, the expectation $E[Y +\beta (Z-\mu_Z) \mid X=x_k]$ is computed on the mesh grid with variance reduced by the correlation between $Y$ and $Z$. """ def __init__(self, locreg, gpr_kernel): """ Initializes the DCKE object. :param locreg: an instance of LocalRegression :param gpr_kernel: an instance of GPy.kern """ self.locreg = locreg self.gpr_kernel = gpr_kernel self.gpr_ = None self.X_train_ = None self.y_train_ = None self.x_mesh_ = None self.y_mesh_ = None self.Z_ = None self.mz_ = None self.cov_ = None self.var_ = None self.beta_ = None def fit(self, X, y, x_mesh, Z=None, mz=None, bandwidth=None): """ Fits the DCKE to training data. :param X: a numpy array of shape (num_samples, num_dimensions) :param y: a numpy array of shape (num_samples,) :param Z: a numpy array of shape (num_samples,) :param x_mesh: a numpy array of shape (num_meshes, num_dimensions) :param mz: a numpy array of shape (num_meshes,) any any mz[k] satisties mz[k] = E[Z \mid X=x_k]$ where x_k are the points in x_mesh :param bandwidth: bandwidth parameter for the local regression :return: """ self.X_train_ = X self.y_train_ = y self.x_mesh_ = x_mesh if Z is None and mz is None: self.Z_ = np.zeros_like(self.y_train_) self.mz_ = np.zeros(self.x_mesh_.shape[0]) elif (Z is None and mz is not None) or (Z is not None and mz is None): raise ValueError( 'Parameter Z and mz have to be either both None or both not None.' ) else: self.Z_ = Z self.mz_ = mz self.locreg.warm_start = True self.locreg.fit(X, y, bandwidth) def _calculate_locregs(self): """ Uses the approximate conditional expectation operator $\tilde E[_ \mid X=x]$ defined by the local regression in self.locreg to compute the approximate optimal beta for the control variate $Z$ defined by $\beta_x = - \tfrac{\Cov[Y, Z \mid X=x]}{\Var[Z \mid X=x]}$ for all $x$ in self.x_mesh. :return: beta, a numpy array of shape (num_mesh_points, ) """ h = self.locreg.bandwidth n = self.x_mesh_.shape[0] self.cov_ = np.zeros(n) self.var_ = np.zeros(n) self.y_mesh_ = np.zeros(n) self.beta_ = np.zeros(n) m_y = np.zeros(n) m_z = np.zeros(n) for i in range(n): m_y[i] = self.locreg.predict(np.atleast_2d( self.x_mesh_[i]).T).squeeze() self.locreg.fit_partial(np.atleast_2d(self.Z_).T, h) m_z[i] = self.locreg.predict_partial().squeeze() self.locreg.fit_partial( (self.y_train_ - m_y[i]) * (self.Z_ - m_z[i]), h) self.cov_[i] = self.locreg.predict_partial().squeeze() self.locreg.fit_partial((self.Z_ - m_z[i])**2, h) self.var_[i] = self.locreg.predict_partial().squeeze() self.beta_[i] = -self.cov_[i] / self.var_[i] self.locreg.fit_partial( self.y_train_ + self.beta_[i] * (self.Z_ - self.mz_[i]), h) self.y_mesh_[i] = self.locreg.predict_partial() def predict(self, X): """ Predicts the conditional expectation $E[Y \mid X=x]$ for all x in $X$. :param X: a numpy array of shape (num_predictions, num_dimensions) :return: a numpy array of shape (num_predictions,) """ self._calculate_locregs() self.gpr_ = GPRegression(self.x_mesh_, np.atleast_2d(self.y_mesh_).T, self.gpr_kernel) self.gpr_.optimize(messages=False) #self.gpr_.optimize_restarts(num_restarts = 10) y_pred, self.gp_var_ = self.gpr_.predict(X) self.gp_var_ = self.gp_var_.squeeze() return y_pred.squeeze()
class EmpiricalStickBreakingGPModel(Model): """ Compute the empirical probability given the counts, convert the empirical probability into a real valued vector that can be modeled with a GP. """ def __init__(self, K, kernel, D=1, alpha=1): self.alpha = alpha self.K = K self.D = D self.kernel = kernel def add_data(self, Z, X, optimize_hypers=True): assert Z.ndim == 2 and Z.shape[1] == self.D M = Z.shape[0] assert X.shape == (M, self.K) # Get the empirical probabilities (offset by 1 to ensure nonzero) pi_emp_train = (self.alpha+X).astype(np.float) / \ (self.alpha + X).sum(axis=1)[:,None] # Convert these to psi's self.Z = Z self.psi = np.array([pi_to_psi(pi) for pi in pi_emp_train]) # Compute the mean value of psi self.mu = self.psi.mean(axis=0) self.psi -= self.mu # Create the GP Regression model from GPy.models import GPRegression self.model = GPRegression(Z, self.psi, self.kernel) # Optimize the kernel parameters if optimize_hypers: self.model.optimize(messages=True) def initialize_from_data(self, initialize_to_mle=False): "For consistency" pass def generate(self, keep=True, **kwargs): raise NotImplementedError def collapsed_predict(self, Z_test): psi_pred, psi_pred_var = self.model.predict(Z_test, full_cov=False) psi_pred += self.mu pi_pred = np.array([psi_to_pi(psi) for psi in psi_pred]) return pi_pred, psi_pred, psi_pred_var def predict(self, Z_test): return self.collapsed_predict(Z_test) def predictive_log_likelihood(self, Z_test, X_test): pi_pred, _, _ = self.predict(Z_test) pll = 0 pll += gammaln(X_test.sum(axis=1)+1).sum() - gammaln(X_test+1).sum() pll += np.nansum(X_test * np.log(pi_pred)) return pll, pi_pred
X_train = np.array([-4, -3, -2, -1, 3]).reshape(-1, 1) Y_train = np.sin(X_train) rbf = RBF(input_dim=1, variance=1.0, lengthscale=1.0) brownian = Brownian(input_dim=1, variance=1.0) periodic = PeriodicExponential(input_dim=1, variance=2.0, n_freq=100) cosine = Cosine(input_dim=1, variance=2) exponential = Exponential(input_dim=1, variance=2.0) integral = Integral(input_dim=1, variances=2.0) matern = Matern32(input_dim=1, variance=2.0) gpr = GPRegression(X_train, Y_train, matern) # Fix the noise variance to known value gpr.Gaussian_noise.variance = noise**2 gpr.Gaussian_noise.variance.fix() # Run optimization ret = gpr.optimize() print(ret) # Display optimized parameter values print(gpr) # Obtain optimized kernel parameters #l = gpr.rbf.lengthscale.values[0] #sigma_f = np.sqrt(gpr.rbf.variance.values[0]) # Plot the results with the built-in plot function gpr.plot() plt.show()
from deepgp import DeepGP from visualization import plot_gp, model_output, pred_range, visualize_pinball data = pods.datasets.olympic_marathon_men() x = data['X'] y = data['Y'] offset = np.mean(y) scale = np.sqrt(np.var(y)) xlim = (1875, 2030) ylim = (2.5, 6.5) yhat = (y - offset) / scale gp_regression = GPRegression(x, yhat) gp_regression.optimize() model_output(gp_regression) ###################deep gp hidden = 1 dgp = DeepGP( [y.shape[1], hidden, x.shape[1]], Y=yhat, X=x, inits=['PCA', 'PCA'], kernels=[RBF(hidden, ARD=True), RBF(x.shape[1], ARD=True)], # the kernels for each layer num_inducing=50, back_constraint=False)
data = pods.datasets.della_gatta_TRP63_gene_expression(data_set='della_gatta', gene_number=937) x = data['X'] y = data['Y'] offset = np.mean(y) scale = np.sqrt(np.var(y)) yhat = (y - offset) / scale #kernel = RBF(input_dim=1, variance=100) #kernel = Matern32(input_dim=1, variance=2.0, lengthscale=200) model = GPRegression(x, yhat) model.kern.lengthscale = 20 #this will widen with 100, 200 #gp_regression.likelihood.variance = 0.001 print(model.log_likelihood()) model.optimize() print(model.log_likelihood()) xt = np.linspace(-20, 260, 100)[:, np.newaxis] yt_mean, yt_var = model.predict(xt) plot_gp(yt_mean, yt_var, xt, X_train=model.X.flatten(), Y_train=model.Y.flatten())
# Check that resonator geometry is sensible: if l_ind[g] > l_cap[g]/2: out[g,0] = 10e20 #Large cost to bad geometry else: out[g,0] = -simulation_wrapper(host, COMSOL_model, paramfile, w[g], t, l_ind[g], pen, omega, gap_cap[g], w_cap[g], l_cap[g], w_mesa, h_mesa, gap_ind[g])[0] return out # Set up random seeding of parameter space num_data_points = no_random_seeds design = RandomDesign(parameter_space) X = design.get_samples(num_data_points) Y = q(X) # Set up emukit model model_gpy = GPRegression(X,Y) model_gpy.optimize() model_emukit = GPyModelWrapper(model_gpy) # Set up Bayesian optimisation routine exp_imprv = ExpectedImprovement(model = model_emukit) optimizer = GradientAcquisitionOptimizer(space = parameter_space) point_calc = SequentialPointCalculator(exp_imprv,optimizer) # Bayesian optimisation routine bayesopt_loop = BayesianOptimizationLoop(model = model_emukit, space = parameter_space, acquisition=exp_imprv, batch_size=1) stopping_condition = FixedIterationsStoppingCondition(i_max = no_BO_sims) bayesopt_loop.run_loop(q, stopping_condition)