def sklearn_fit_optimize(index, matrix): raise NotImplementedError from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import RBF, ConstantKernel, WhiteKernel y = matrix.loc[index, :].values x = np.log2(1 + matrix.columns.get_level_values('timepoint').str.replace( "d", "").astype(int).values.reshape((y.shape[0], 1))) kernel = 1 * (RBF(length_scale=1.0, length_scale_bounds=(1e-05, 100000.0))) # + # WhiteKernel(noise_level=1.0, noise_level_bounds=(1e-05, 100000.0)))# + # ConstantKernel(constant_value=1.0, constant_value_bounds=(1e-05, 100000.0))) white_kernel = 1 * ( # WhiteKernel(noise_level=1.0, noise_level_bounds=(1e-05, 100000.0)) )# + ConstantKernel(constant_value=0., constant_value_bounds=(1e-05, 100000.0))) # white_kernel = ConstantKernel(constant_value=1.0, constant_value_bounds=(1e-05, 100000.0)) m = GaussianProcessRegressor(kernel, normalize_y=False) try: m.fit(x, y) except: return np.nan, np.nan w_m = GaussianProcessRegressor(white_kernel, normalize_y=False) try: w_m.fit(x, y) except: return np.nan, np.nan return m.log_marginal_likelihood(), w_m.log_marginal_likelihood()
def test_gpr_lml_error(): """Check that we raise the proper error in the LML method.""" gpr = GaussianProcessRegressor(kernel=RBF()).fit(X, y) err_msg = "Gradient can only be evaluated for theta!=None" with pytest.raises(ValueError, match=err_msg): gpr.log_marginal_likelihood(eval_gradient=True)
def test_lml_without_cloning_kernel(kernel): # Test that lml of optimized kernel is stored correctly. gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) input_theta = np.ones(gpr.kernel_.theta.shape, dtype=np.float64) gpr.log_marginal_likelihood(input_theta, clone_kernel=False) assert_almost_equal(gpr.kernel_.theta, input_theta, 7)
def test_lml_improving(): """ Test that hyperparameter-tuning improves log-marginal likelihood. """ for kernel in kernels: if kernel == fixed_kernel: continue gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) assert_greater(gpr.log_marginal_likelihood(gpr.kernel_.theta), gpr.log_marginal_likelihood(kernel.theta))
def test_lml_improving(kernel): if sys.maxsize <= 2**32: pytest.xfail("This test may fail on 32 bit Python") # Test that hyperparameter-tuning improves log-marginal likelihood. gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) assert gpr.log_marginal_likelihood( gpr.kernel_.theta) > gpr.log_marginal_likelihood(kernel.theta)
def test_lml_gradient(kernel): # Compare analytic and numeric gradient of log marginal likelihood. gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) lml, lml_gradient = gpr.log_marginal_likelihood(kernel.theta, True) lml_gradient_approx = approx_fprime( kernel.theta, lambda theta: gpr.log_marginal_likelihood(theta, False), 1e-10) assert_almost_equal(lml_gradient, lml_gradient_approx, 3)
def test_lml_gradient(): """ Compare analytic and numeric gradient of log marginal likelihood. """ for kernel in kernels: gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) lml, lml_gradient = gpr.log_marginal_likelihood(kernel.theta, True) lml_gradient_approx = approx_fprime( kernel.theta, lambda theta: gpr.log_marginal_likelihood(theta, False), 1e-10 ) assert_almost_equal(lml_gradient, lml_gradient_approx, 3)
def sk_train(self, X_train, y_train, kernels=['rbf', 'matern32', 'matern52'], offset=False, verboseQ=False, saveQ=True): for kern in (kernels): if verboseQ: print('********* \n', kern) sk_kernel = self.SK_kernel(kern, X_train.shape[1]) sk_kernel += WhiteKernel(noise_level=0.1**2, noise_level_bounds=(1e-8, 10)) # noise if offset: #add bias kernel sk_kernel += Ck(1) t0 = time.time() gpr = GaussianProcessRegressor(kernel=sk_kernel, n_restarts_optimizer=5) gpr.fit(X_train, y_train) self.sk_t = time.time() - t0 if verboseQ: print('took ', self.sk_t, ' seconds') print("Inital kernel: %s" % gpr.kernel) print("Learned kernel: %s" % gpr.kernel_) print("Log-marginal-likelihood: %.3f" % gpr.log_marginal_likelihood(gpr.kernel_.theta)) if saveQ: with open('SK_gpr_' + str(kern) + '.pickle', 'wb') as handle: pickle.dump(gpr, handle, protocol=pickle.HIGHEST_PROTOCOL) self.results['ll'] = gpr.log_marginal_likelihood(gpr.kernel_.theta) try: self.results['amp_param'] = gpr.kernel_.get_params( )['k1__k1__k1__constant_value'] self.results['noise_param_variance'] = gpr.kernel_.get_params( )['k1__k2__noise_level'] self.results['length_scale_param'] = gpr.kernel_.get_params( )['k1__k1__k2__length_scale'].tolist() self.results['offset_param'] = gpr.kernel_.get_params( )['k2__constant_value'] except: self.results['length_scale_param'] = gpr.kernel_.get_params( )['k1__k2__length_scale'].tolist() self.results['noise_param_variance'] = gpr.kernel_.get_params( )['k2__noise_level'] self.results['amp_param'] = gpr.kernel_.get_params( )['k1__k1__constant_value'] self.results['offset_param'] = None
def gene_expression_classifier(expr_data, profile_cols=None): """ The logic below is based on Kalaitzis & Lawrence 2011. The implementation seems to work and the results that I'm getting are close to what their R script is reporting, though there are small discrepancies in the numbers. :param expr_data: The expression data as a pandas dataframe :param profile_cols: The profile column names (see example below). :return: Return updated dataframe with likelihoods computed and compared. """ if profile_cols is None: profile_cols = np.array( [0, 20, 40, 60, 80, 100, 120, 140, 160, 180, 200, 220, 240], dtype=np.float64) profile_cols.shape = (len(profile_cols), 1) for index, row in expr_data.iterrows(): profile = np.array(row, dtype=np.float64) profile.shape = (len(profile), 1) # Calculate the variance in the profile profile_var = np.var(profile) # Create noisy GP noisy_kern = .001 * profile_var * RBF( length_scale=1. / 1000) + WhiteKernel(.999 * profile_var) noisy_m = GaussianProcessRegressor(kernel=noisy_kern, n_restarts_optimizer=10) # Fit GP to data noisy_m.fit(profile_cols, profile) # Compute log marginal likelihood of fitted GP expr_data.loc[index, 'opt_likelihood'] = noisy_m.log_marginal_likelihood() # Compute log marginal likelihood of hyperparameters that correspond to noise. expr_data.loc[index, 'noisy_likelihood'] = noisy_m.log_marginal_likelihood( theta=[ np.log(.999 * profile_var), np.log(.05), np.log(.001 * profile_var) ]) expr_data.loc[index, 'likelihood_ratio'] = expr_data.loc[index, 'opt_likelihood'] - \ expr_data.loc[index, 'noisy_likelihood'] return expr_data
def sk_kernel(self, hypers_dict): amp = hypers_dict['amplitude_covar'] lengthscales = np.diag(hypers_dict['precisionMatrix'])**-0.5 noise_var = hypers_dict['noise_variance'] se_ard = Ck(amp) * RBF(length_scale=lengthscales, length_scale_bounds=(1e-6, 10)) noise = WhiteKernel(noise_level=noise_var, noise_level_bounds=(1e-9, 1)) # noise terms sk_kernel = se_ard if self.noiseQ: sk_kernel += noise t0 = time.time() gpr = GaussianProcessRegressor(kernel=sk_kernel, n_restarts_optimizer=5) print("Initial kernel: %s" % gpr.kernel) # self.ytrain = [y[0][0] for y in self.Y_obs] gpr.fit(self.X_obs, np.array(self.Y_obs).flatten()) print('SK fit time is ', time.time() - t0) print("Learned kernel: %s" % gpr.kernel_) print("Log-marginal-likelihood: %.3f" % gpr.log_marginal_likelihood(gpr.kernel_.theta)) #print(gpr.kernel_.get_params()) if self.noiseQ: # RBF w/ noise sk_ls = gpr.kernel_.get_params()['k1__k2__length_scale'] sk_amp = gpr.kernel_.get_params()['k1__k1__constant_value'] sk_loklik = gpr.log_marginal_likelihood(gpr.kernel_.theta) sk_noise = gpr.kernel_.get_params()['k2__noise_level'] else: #RBF w/o noise sk_ls = gpr.kernel_.get_params()['k2__length_scale'] sk_amp = gpr.kernel_.get_params()['k1__constant_value'] sk_loklik = gpr.log_marginal_likelihood(gpr.kernel_.theta) sk_noise = 0 # make dict sk_hypers = {} sk_hypers['precisionMatrix'] = np.diag(1. / (sk_ls**2)) sk_hypers['noise_variance'] = sk_noise sk_hypers['amplitude_covar'] = sk_amp return sk_loklik, sk_hypers
def test_gp_regression(): while True: alpha = np.random.rand() N = np.random.randint(2, 100) M = np.random.randint(2, 100) K = np.random.randint(1, N) J = np.random.randint(1, 3) X = np.random.rand(N, M) y = np.random.rand(N, J) X_test = np.random.rand(K, M) gp = GPRegression(kernel="RBFKernel(sigma=1)", alpha=alpha) gold = GaussianProcessRegressor(kernel=None, alpha=alpha, optimizer=None, normalize_y=False) gp.fit(X, y) gold.fit(X, y) preds, _ = gp.predict(X_test) gold_preds = gold.predict(X_test) np.testing.assert_almost_equal(preds, gold_preds) mll = gp.marginal_log_likelihood() gold_mll = gold.log_marginal_likelihood() np.testing.assert_almost_equal(mll, gold_mll) print("PASSED")
def fit_and_test_n_dim(samples, rewards, n, alpha=0.1, length_scale=5, use_nn = False): if use_nn: nn = Sequential() nn.add(Dense(16, input_shape=(8,), activation="relu")) nn.add(Dense(8, activation="relu")) nn.add(Dense(1, activation="linear")) opt = Adam(lr=0.01) nn.compile(loss="mse", optimizer=opt) else: gp_kernel = C(12.0, (1e-3, 2e3)) * RBF(length_scale) gp = GaussianProcessRegressor(kernel=gp_kernel, n_restarts_optimizer=8, alpha=alpha) num_training = 300 relevant_samples_training = samples[:num_training,:n ] relevant_rewards_training = rewards[:num_training] relevant_samples_test = samples[num_training:,:n ] relevant_rewards_test = rewards[num_training:] if use_nn: nn.fit(relevant_samples_training, relevant_rewards_training, epochs=10000, batch_size=800) ll = 0 predictions = nn.predict_on_batch(relevant_samples_test) else: print("before fitting") gp.fit(relevant_samples_training, relevant_rewards_training) print("after fitting") ll = gp.log_marginal_likelihood() predictions = gp.predict(relevant_samples_test) se = (predictions-relevant_rewards_test)**2 mse = np.mean(se) plt.scatter(relevant_rewards_test, se) plt.show() return mse, ll
def gp(self,true_graph, training_idx): training_X = [] training_Y = [] for idx in training_idx: training_Y.append(true_graph.vertex_[idx].ig_) nbs_p = self.adjacent_p(idx,true_graph) print("The vert {}".format(idx)) print("The training X is {}".format(nbs_p)) print("The training Y is {}".format(training_Y[-1])) print("===============================================") training_X.append(nbs_p) rbf = 1.0 * RBF(length_scale=1.0) matern = 1.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0), nu=1.5) gp_opt = GaussianProcessRegressor(kernel=rbf) gp_opt.fit(training_X,training_Y) print("The trained hyperparameter are {}".format((gp_opt.kernel_.theta))) print("Log Marginal Likelihood (optimized): %.3f" % gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta)) # Contour 3d x1_ = [i for i in range(self.rows_+1)] x2_ = [j for j in range(self.cols_+1)] X1_, X2_ = np.meshgrid(x1_,x2_) y_mean = np.empty([self.rows_+1,self.cols_+1]) y_true = np.empty([self.rows_+1,self.cols_+1]) y_std = np.empty([self.rows_+1,self.cols_+1]) y_mean_u = np.empty([self.rows_+1,self.cols_+1]) y_mean_d = np.empty([self.rows_+1,self.cols_+1]) for i in range(self.rows_+1): for j in range(self.cols_+1): cur_idx = (self.rows_-X1_[i][j]-1)*self.cols_+X2_[i][j] print("X: {}, Y: {}, idx: {}".format(X2_[i][j], X1_[i][j],cur_idx)) if X1_[i][j]<self.rows_ and X1_[i][j]>=0 and X2_[i][j]<self.cols_ and X2_[i][j]>=0 and cur_idx in nz_ig: neigh_p = self.adjacent_p(cur_idx,true_graph) y_mean[X2_[i][j],X1_[i][j]], y_std[X2_[i][j],X1_[i][j]] = gp_opt.predict([neigh_p], return_std=True) print("Prediction ========================") print("Vertex {}, X:{}, Y:{}".format(cur_idx,X2_[i][j],X1_[i][j])) print("Testing data is {}".format(neigh_p)) print("Predicted IG {}".format(y_mean[X2_[i][j],X1_[i][j]])) else: y_mean[X2_[i][j] ,X1_[i][j]] = 0 y_std[X2_[i][j],X1_[i][j]] = 0.0 y_mean_u[X2_[i][j],X1_[i][j]] = y_mean[X2_[i][j],X1_[i][j]] + y_std[X2_[i][j],X1_[i][j]] y_mean_d[X2_[i][j],X1_[i][j]] = y_mean[X2_[i][j],X1_[i][j]] - y_std[X2_[i][j],X1_[i][j]] if X2_[i][j]<self.cols_ and X1_[i][j]<self.rows_ and X1_[i][j]>=0 and X2_[i][j]>=0: idx_ = (self.rows_-X1_[i][j]-1)*self.cols_+X2_[i][j] y_true[X2_[i][j],X1_[i][j]] = true_graph.vertex_[idx_].ig_ self.vertex_[idx_].ig_ = y_mean[X2_[i][j],X1_[i][j]] true_graph.vertex_[idx_].L2_error_ = (true_graph.vertex_[idx_].ig_ - self.vertex_[idx_].ig_)**2 self.vertex_[idx_].ig_ub_ = y_mean[X2_[i][j],X1_[i][j]] + y_std[X2_[i][j],X1_[i][j]] self.vertex_[idx_].ig_lb_ = y_mean[X2_[i][j],X1_[i][j]] - y_std[X2_[i][j],X1_[i][j]] if round(true_graph.vertex_[idx_].ig_,3) <= round(self.vertex_[idx_].ig_ub_,3) and round(true_graph.vertex_[idx_].ig_,3) >= round(self.vertex_[idx_].ig_lb_,3): self.vertex_[idx_].ig_pred_ = True
def test_random_starts(): """ Test that an increasing number of random-starts of GP fitting only increases the log marginal likelihood of the chosen theta. """ n_samples, n_features = 25, 2 np.random.seed(0) rng = np.random.RandomState(0) X = rng.randn(n_samples, n_features) * 2 - 1 y = np.sin(X).sum(axis=1) + np.sin(3 * X).sum(axis=1) \ + rng.normal(scale=0.1, size=n_samples) kernel = C(1.0, (1e-2, 1e2)) \ * RBF(length_scale=[1.0] * n_features, length_scale_bounds=[(1e-4, 1e+2)] * n_features) \ + WhiteKernel(noise_level=1e-5, noise_level_bounds=(1e-5, 1e1)) last_lml = -np.inf for n_restarts_optimizer in range(5): gp = GaussianProcessRegressor( kernel=kernel, n_restarts_optimizer=n_restarts_optimizer, random_state=0, ).fit(X, y) lml = gp.log_marginal_likelihood(gp.kernel_.theta) assert_greater(lml, last_lml - np.finfo(np.float32).eps) last_lml = lml
def fit_model(self, train_x, train_y): """ TODO: enter your code here """ """ Task 1 : Model selection""" length_scales = [1, 2, 3, 4] kernels = [Matern(), RationalQuadratic()] # kernels = [ExpSineSquared()] # Grid search: # for kernel in kernels: # gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9) # gp.X_train_ = train_x # gp.y_train_ = train_y # gp.kernel_ = kernel # for length_scale in length_scales: # log_like_GS = gp.log_marginal_likelihood([length_scale]) # print(log_like_GS) # Optimization method: P_likelihood = [] Hyper_params = [None] * len(kernels) for i, kernel in enumerate(kernels): gp = GaussianProcessRegressor(kernel=kernel, alpha=1e-30) gp.fit(train_x, train_y) # print(gp._get_param_names()) # print(gp.get_params()) print("Kernel type:", kernel) Hyper_params[i] = gp.kernel_.theta print(f"Hyperparameters {i}:", Hyper_params[i]) P_likelihood.append(gp.log_marginal_likelihood()) print(f"kernel {i}:", P_likelihood[i]) pass
def main(n_samples): np.random.seed(0) print('player 0') data0 = pd.read_csv("./saves/GP0.csv", index_col=0) print('data\n%s' % (data0.tail(2))) #pre = {'B':[0,0,0,0],'C':[1,0,1,0],'D':[2,0,1,0],'F':[1,2,1,1]} #post = {'0': [1,0,0],'1': [0,1,0], '2':[0,0,1]} raw = data0[-n_samples:]['name'] dic = { 'B': [0], 'C': [1], 'D': [2], 'F': [3], '0': [0], '1': [1], '2': [2] } cols = ['pre0', 'post0'] translation = translate(0, raw, dic, cols) print('translation\n%s' % (translation.tail(2))) # kernel = W() + C(10, (1e-3, 1e1)) + C(1, (1e-3, 1e1)) * RBF(10, (1e-3, 1e1)) + C(1, (1e-3, 1e1)) * Matern(length_scale=1.0, length_scale_bounds=(1e-3, 1e1), nu=1.5) + C(1, (1e-3, 1e1)) * RationalQuadratic(length_scale=1.0, alpha=0.1) # kernel = W() + C(1, (1e-3, 1e1)) + C(1, (1e-3, 1e1)) * RBF(1, (1e-3, 1e1)) + C(1, (1e-3, 1e1)) * ESS(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) + C(1, (1e-3, 1e1)) * DP() + C(1, (1e-3, 1e1)) * ESS(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) * RBF(1, (1e-3, 1e1)) + C(1, (1e-3, 1e1)) * ESS(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) * DP() + C(1, (1e-3, 1e1)) * RBF(1, (1e-3, 1e1)) * DP() + C(1, (1e-3, 1e1)) * ESS(1.0, 5.0, periodicity_bounds=(1e-2, 1e1)) * RBF(1, (1e-3, 1e1)) * DP() # kernel = W() + C(1, (1e-3, 1e1)) + C(1, (1e-3, 1e1)) * RationalQuadratic(length_scale=1.0, alpha=0.1) + ESS(1.0, 2.0, periodicity_bounds=(1e-2, 1e2)) * DP() + C(1, (1e-3, 1e1)) * ESS(1.0, 2.0, periodicity_bounds=(1e-2, 1e2)) kernel = W() + C(1, (1e-3, 1e1)) + C(1, (1e-3, 1e1)) * RBF( 1, (1e-3, 1e1)) + ESS(1.0, 2.0, periodicity_bounds=(1e-2, 1e1)) * DP() X0_train, X0_test, y0_train, y0_test = train_test_split( translation, data0[-n_samples:][['va0', 'va1', 'va2']]) gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9) stime = time.time() gp.fit(X0_train, y0_train) print("\nTime for GPR fitting: %.3f" % (time.time() - stime)) print("Posterior (kernel: %s)\n Log-Likelihood: %.3f" % (gp.kernel_, gp.log_marginal_likelihood(gp.kernel_.theta))) print('training result: %f' % r2_score(y0_train, gp.predict(X0_train))) print('test result: %f' % r2_score(y0_test, gp.predict(X0_test))) test = raw.loc[y0_test.index] dic = { 'B': [[0, 0], [0, 0]], 'C': [[1, 0], [1, 0]], 'D': [[2, 0], [1, 0]], 'F': [[1, 2], [1, 1]], '0': [[0]], '1': [[1]], '2': [[2]] } cols = ['action0', 'action1', 'hand'] original = translate(0, test, dic, cols) M = model('p0') M.restore() stime = time.time() prediction = [] for i in range(test.size): I = original[['hand', 'action0', 'action1']].iloc[i] prediction.append(M.predict(I)[0, :]) print("\nNeural Net: %.3f" % (time.time() - stime)) print('test result: %f' % r2_score(y0_test, prediction))
def getgpr(X, y): kernel = ConstantKernel() + 1.0**2 * Matern( length_scale=2.0, nu=1.5) + WhiteKernel(noise_level=1) gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=10) gp.fit(X, y) print(gp.kernel_) print(gp.log_marginal_likelihood(gp.kernel_.theta)) return gp
def test_converged_to_local_maximum(kernel): # Test that we are in local maximum after hyperparameter-optimization. gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) lml, lml_gradient = gpr.log_marginal_likelihood(gpr.kernel_.theta, True) assert np.all((np.abs(lml_gradient) < 1e-4) | (gpr.kernel_.theta == gpr.kernel_.bounds[:, 0]) | (gpr.kernel_.theta == gpr.kernel_.bounds[:, 1]))
def test_converged_to_local_maximum(kernel): # Test that we are in local maximum after hyperparameter-optimization. gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) lml, lml_gradient = \ gpr.log_marginal_likelihood(gpr.kernel_.theta, True) assert_true(np.all((np.abs(lml_gradient) < 1e-4) | (gpr.kernel_.theta == gpr.kernel_.bounds[:, 0]) | (gpr.kernel_.theta == gpr.kernel_.bounds[:, 1])))
def test_custom_optimizer(): """ Test that GPR can use externally defined optimizers. """ # Define a dummy optimizer that simply tests 50 random hyperparameters def optimizer(obj_func, initial_theta, bounds): rng = np.random.RandomState(0) theta_opt, func_min = initial_theta, obj_func(initial_theta, eval_gradient=False) for _ in range(50): theta = np.atleast_1d(rng.uniform(np.maximum(-2, bounds[:, 0]), np.minimum(1, bounds[:, 1]))) f = obj_func(theta, eval_gradient=False) if f < func_min: theta_opt, func_min = theta, f return theta_opt, func_min for kernel in kernels: if kernel == fixed_kernel: continue gpr = GaussianProcessRegressor(kernel=kernel, optimizer=optimizer) gpr.fit(X, y) # Checks that optimizer improved marginal likelihood assert_greater(gpr.log_marginal_likelihood(gpr.kernel_.theta), gpr.log_marginal_likelihood(gpr.kernel.theta))
def fitness(self, candidate): data = self.gym.state['data'] kernel = candidate.epigenesis() gp = GaussianProcessRegressor(kernel=kernel, alpha=0.001, normalize_y=False) try: gp = gp.fit(*data) except np.linalg.linalg.LinAlgError: return -1e10 return gp.log_marginal_likelihood()
def test_custom_optimizer(kernel): # Test that GPR can use externally defined optimizers. # Define a dummy optimizer that simply tests 50 random hyperparameters def optimizer(obj_func, initial_theta, bounds): rng = np.random.RandomState(0) theta_opt, func_min = \ initial_theta, obj_func(initial_theta, eval_gradient=False) for _ in range(50): theta = np.atleast_1d(rng.uniform(np.maximum(-2, bounds[:, 0]), np.minimum(1, bounds[:, 1]))) f = obj_func(theta, eval_gradient=False) if f < func_min: theta_opt, func_min = theta, f return theta_opt, func_min gpr = GaussianProcessRegressor(kernel=kernel, optimizer=optimizer) gpr.fit(X, y) # Checks that optimizer improved marginal likelihood assert (gpr.log_marginal_likelihood(gpr.kernel_.theta) > gpr.log_marginal_likelihood(gpr.kernel.theta))
def gpr(self): rbf = 1.0 * RBF(length_scale=1.0) matern = 1.0 * Matern( length_scale=1.0, length_scale_bounds=(1e-8, 5.0), nu=1.5) gp_opt = GaussianProcessRegressor(kernel=matern) gp_opt.fit(self.training_x_, self.training_y_) self.gpr_ = gp_opt print("The trained hyperparameter are {}".format( (gp_opt.kernel_.theta))) print("Log Marginal Likelihood (optimized): %.3f" % gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta))
def test_converged_to_local_maximum(): """ Test that we are in local maximum after hyperparameter-optimization.""" for kernel in kernels: if kernel == fixed_kernel: continue gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) lml, lml_gradient = \ gpr.log_marginal_likelihood(gpr.kernel_.theta, True) assert_true(np.all((np.abs(lml_gradient) < 1e-4) | (gpr.kernel_.theta == gpr.kernel_.bounds[:, 0]) | (gpr.kernel_.theta == gpr.kernel_.bounds[:, 1])))
def sk_kernel(self, amp, noise_var, lengthscales): se_ard = Ck(amp) * RBF(length_scale=lengthscales, length_scale_bounds=(0.000001, 20)) noise = WhiteKernel(noise_level=noise_var, noise_level_bounds=(1e-5, 100)) # noise terms # sk_kernel = se_ard + noise + Ck(0.4) #with bias sk_kernel = se_ard + noise gpr = GaussianProcessRegressor(kernel=sk_kernel, n_restarts_optimizer=5) gpr.fit(self.X_obs, self.Y_obs) print("Learned kernel: %s" % gpr.kernel_) print("Log-marginal-likelihood: %.3f" % gpr.log_marginal_likelihood(gpr.kernel_.theta)) sk_ls = gpr.kernel_.get_params()['k1__k1__k2__length_scale'] sk_noise = gpr.kernel_.get_params()['k1__k2__noise_level'] sk_amp = gpr.kernel_.get_params()['k1__k1__k1'] sk_loklik = gpr.log_marginal_likelihood(gpr.kernel_.theta) return sk_loklik, sk_amp, sk_noise, sk_ls
class Gaussian_Process(): def __init__(self, kernel, mode): self.kernel = kernel if mode == 'OPT': self.gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=10, normalize_y=True) elif mode == 'MAP': self.gp = GaussianProcessRegressor(kernel=kernel, optimizer = None, n_restarts_optimizer=10, normalize_y=True) else: raise Exception('Wrong GP initial mode!!!') ''' def optimise__with_splice_sampling(self, initial_theta, num_iters, sigma, burn_in, X, y): self.fit(X, y) slice_sampler = Slice_sampler(num_iters = num_iters, sigma = sigma, burn_in = burn_in, gp = self) samples = slice_sampler.sample(init = initial_theta) theta_opt = [np.mean(samples[0]),np.mean(samples[1])] self.gp.set_params(**{"kernel__k1__length_scale": np.mean(samples[0]), "kernel__k2__noise_level": np.power(np.mean(samples[0]),2)}) ''' def set_params(self, theta_set): self.gp.set_params(**{"kernel__k1__noise_level": np.abs(theta_set[0]), "kernel__k2__k1__constant_value": np.abs(theta_set[1]), "kernel__k2__k2__length_scale": theta_set[2:]}) def log_marginal_likelihood(self,theta): return self.gp.log_marginal_likelihood(theta) def log_prior_parameters(self, theta): return np.sum(np.log([ss.norm(0, 1).pdf(theta_k) for theta_k in theta])) def log_joint_unnorm(self, theta): return self.log_marginal_likelihood(theta) + self.log_prior_parameters(theta) def fit(self, X, y): self.gp.fit(X, y) def predict(self, X): mu, sigma = self.gp.predict(X, return_std =True) return mu, sigma
def train_standard_gp(pch, snr, pch_pred): pch = pch.reshape(-1, 1) # needed for SK learn input snr = snr.reshape(-1, 1) pch_pred = pch_pred.reshape(-1, 1) kernel_sk = C(1, (1e-5, 1e5)) * RBF(1, (1e-5, 1e5)) + W(1, (1e-8, 1e5)) gpr = GaussianProcessRegressor(kernel=kernel_sk, n_restarts_optimizer=20, normalize_y=True) gpr.fit(pch, snr) mu_sk, std_sk = gpr.predict(pch_pred, return_std=True) std_sk = np.reshape(std_sk, (np.size(std_sk), 1)) theta = gpr.kernel_.theta lml = gpr.log_marginal_likelihood() return mu_sk, std_sk, theta, lml
def regressor(X_train, Y_train): kernel = 1.0 * RBF(length_scale=0.01, length_scale_bounds=(1e-1, 1e2)) + ( DotProduct()**3) * WhiteKernel(noise_level=2.e-8, noise_level_bounds=(1e-10, 1e-1)) gp = GaussianProcessRegressor(kernel=kernel, alpha=0., n_restarts_optimizer=15).fit( X_train, Y_train) print "kernel init: ", kernel print "kernel init params: ", kernel.theta print "kenel optimum: ", gp.kernel_ print "opt kernel params: ", gp.kernel_.theta print "LML (opt): ", gp.log_marginal_likelihood() return gp
def bic(self, parents, target, tau_max): train_x, train_y = self._get_train_data(parents, target, tau_max) kernel = RBF(length_scale=1, length_scale_bounds=(1e-2, 1e3)) \ + WhiteKernel(noise_level=1e0, noise_level_bounds=(1e-10, 1e+1)) gp = GaussianProcessRegressor(kernel=kernel, alpha=0.0, normalize_y=True, n_restarts_optimizer=2) gp.fit(train_x[idx], train_y[idx]) eps = 1e-8 theta_opt = gp.kernel_.theta n_params = theta_opt.shape[0] H = np.zeros((n_params, n_params)) log_mll, mll_grad = gp.log_marginal_likelihood(theta_opt, eval_gradient=True) for i in range(n_params): e_ii = np.zeros(n_params) e_ii[i] = 1 log_mll_2, mll_grad_2 = gp.log_marginal_likelihood( np.log(np.exp(theta_opt) + eps * e_ii), eval_gradient=True) H[i, i] = (mll_grad_2[i] - mll_grad[i]) / eps for i in range(n_params): for j in range(i, n_params): e_ij = np.zeros(n_params) e_ij[i], e_ij[j] = 1, 1 log_mll_2, mll_grad_2 = gp.log_marginal_likelihood( np.log(np.exp(theta_opt) + eps * e_ij), eval_gradient=True) H[i, j] = 0.5 * ((np.dot(e_ij, mll_grad_2 - mll_grad)) / eps - H[i, i] - H[j, j]) H[j, i] = H[i, j].copy() detH = np.abs(np.linalg.det(H)) + 1e-2 print(H, detH, mll_grad, mll_grad_2, theta_opt) return log_mll, log_mll - 0.5 * np.log(detH)
def gpregression(Xtrain, Ytrain, Nfeature): cmean = [1.0] * Nfeature cbound = [[1e-3, 10000]] * Nfeature # kernel = C(1.0, [1e-3, 1e3]) * RBF(cmean, cbound) kernel = C(1.0, (1e-3, 1e3)) * matk(cmean, cbound, 2.5) + Wht(1.0, (1e-3, 1e3)) gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=50, normalize_y=False) gp.fit(Xtrain, Ytrain) print("initial parameters:", kernel) print("optimal parameters A:", gp.kernel_, "likelihood:", gp.log_marginal_likelihood(gp.kernel_.theta)) # print("likelihood:", gp.log_marginal_likelihood(gp.kernel_.theta)) return gp
def train_phys_gp(pch, pch_phys, snr, snr_phys, pch_pred): pch = pch.reshape(-1, 1) # needed for SK learn input pch_phys = pch_phys.reshape(-1, 1) pch_pred = pch_pred.reshape(-1, 1) snr = snr.reshape(-1, 1) snr_phys = snr_phys.reshape(-1, 1) kernel_sk = C(1, (1e-5, 1e5)) * RBF(1, (1e-8, 1e5)) + W(1, (1e-5, 1e5)) gpr_phys = GaussianProcessRegressor(kernel=kernel_sk, n_restarts_optimizer=20, normalize_y=True) gpr_phys.fit_phys(pch, pch_phys, snr, snr_phys) mu_sk_phys, std_sk_phys = gpr_phys.predict(pch_pred, return_std=True) std_sk_phys = np.reshape(std_sk_phys, (np.size(std_sk_phys), 1)) theta_phys = gpr_phys.kernel_.theta lml_phys = gpr_phys.log_marginal_likelihood() return mu_sk_phys, std_sk_phys, theta_phys, lml_phys
def makeGaussianProcess(): global y_t_pred, result prefix = "%s_GP_FULL" % (name) #kernel = RBF(1e1,(1e-5,1e7)) kernel = RationalQuadratic() #(1e1,(1e-5,1e7)) #kernel = ExpSineSquared()#(1e1,(1e-5,1e7)) model = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9) x1 = x[:, 3:6:2] x_t1 = x_t[:, 3:6:2] y_t_pred = model.fit(x1, y).predict(x_t1) r = model.score(x1, y) print("score r = %s" % r) print "Coefficients: %s" % model.get_params() #print "Highest Coefficients: %s" % str(sorted(model.get_params(),key=lambda x:-x)) print str( (model.kernel_, model.log_marginal_likelihood(model.kernel_.theta))) return prefix, model
def fit_gp(x, y, sqrt=True, in_lengthscale=1000, in_noise=2.0, std_band=1.96, lgth_s=[500, 2000], err_s=[2, 4]): """Fit Gaussian Process. If sqrt: Take the square root for fit. Return: x, y, y_max, y_min""" if sqrt: y = np.sqrt(y) ### Sort Values idx = np.argsort(x) x = x[idx] y = y[idx] X = x[:, np.newaxis] kernel = 1.0 * RBF(length_scale=in_lengthscale, length_scale_bounds=lgth_s) \ + WhiteKernel(noise_level=in_noise, noise_level_bounds=err_s) gp = GaussianProcessRegressor(kernel=kernel, alpha=0.0).fit(X, y) X_ = np.linspace(np.min(x), np.max(x), 1000) y_mean, y_cov = gp.predict(X_[:, np.newaxis], return_cov=True) title = "Initial: %s\nOptimum: %s\nLog-Marginal-Likelihood: %s" % ( kernel, gp.kernel_, gp.log_marginal_likelihood(gp.kernel_.theta)) print(title) y_up = y_mean + std_band * np.sqrt(np.diag(y_cov)) y_down = y_mean - std_band * np.sqrt(np.diag(y_cov)) error_var = gp.kernel_.get_params()["k2__noise_level"] stds = np.sqrt(np.diag(y_cov) - error_var) assert (np.min(stds) >= 0) # Sanity Check that Error in Mean is >0: y_up_mean = y_mean + std_band * stds y_down_mean = y_mean - std_band * stds if sqrt: y_mean, y_up, y_down = np.maximum(y_mean, 0)**2, y_up**2, np.maximum( y_down, 0)**2 y_up_mean, y_down_mean = y_up_mean**2, np.maximum(y_down_mean, 0)**2 return X_, y_mean, y_up, y_down, y_up_mean, y_down_mean
def test_random_starts(): # Test that an increasing number of random-starts of GP fitting only # increases the log marginal likelihood of the chosen theta. n_samples, n_features = 25, 2 rng = np.random.RandomState(0) X = rng.randn(n_samples, n_features) * 2 - 1 y = np.sin(X).sum(axis=1) + np.sin(3 * X).sum(axis=1) \ + rng.normal(scale=0.1, size=n_samples) kernel = C(1.0, (1e-2, 1e2)) \ * RBF(length_scale=[1.0] * n_features, length_scale_bounds=[(1e-4, 1e+2)] * n_features) \ + WhiteKernel(noise_level=1e-5, noise_level_bounds=(1e-5, 1e1)) last_lml = -np.inf for n_restarts_optimizer in range(5): gp = GaussianProcessRegressor( kernel=kernel, n_restarts_optimizer=n_restarts_optimizer, random_state=0,).fit(X, y) lml = gp.log_marginal_likelihood(gp.kernel_.theta) assert_greater(lml, last_lml - np.finfo(np.float32).eps) last_lml = lml
def plot(df, options): UNIQ_GROUPS = df.group.unique() UNIQ_GROUPS.sort() sns.set_style("white") grppal = sns.color_palette("Set2", len(UNIQ_GROUPS)) print '# UNIQ GROUPS', UNIQ_GROUPS cent_stats = df.groupby(['position', 'group']).apply(stats_per_group) cent_stats.reset_index(inplace=True) print cent_stats import time from sklearn import preprocessing from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import Matern, WhiteKernel, ExpSineSquared, ConstantKernel, RBF mean = cent_stats['values'].mean() # kernel = ConstantKernel() + Matern(length_scale=mean, nu=3 / 2) + \ # WhiteKernel(noise_level=1e-10) kernel = 1**2 * Matern(length_scale=1, nu=1.5) + \ WhiteKernel(noise_level=0.1) figure = plt.figure(figsize=(10, 6)) palette = sns.color_palette('muted') for i,GRP in enumerate(UNIQ_GROUPS): groupDf = cent_stats[cent_stats['group'] == GRP] X = groupDf['position'].values.reshape((-1, 1)) y = groupDf['values'].values.reshape((-1, 1)) y = preprocessing.scale(y) N = groupDf['subj_count'].values.max() # sns.lmplot(x="position", y="values", row="group", # fit_reg=False, data=groupDf) stime = time.time() gp = GaussianProcessRegressor(kernel=kernel, normalize_y=True) gp.fit(X, y) print gp.kernel_ print gp.log_marginal_likelihood() print("Time for GPR fitting: %.3f" % (time.time() - stime)) stime = time.time() pred_x = np.linspace(0, 30, 100) y_mean, y_std = gp.predict(pred_x.reshape((-1, 1)), return_std=True) y_mean = y_mean[:, 0] print("Time for GPR prediction: %.3f" % (time.time() - stime)) group_color = palette[i] ci = y_std / math.sqrt(N) * 1.96 plt.scatter(X, y, color=group_color, alpha=0.1) plt.plot(pred_x, y_mean, color=group_color) plt.fill_between(pred_x, y_mean - ci, y_mean + ci, color=group_color, alpha=0.3) if options.title: plt.suptitle(options.title) if options.output: plt.savefig(options.output, dpi=150) if options.is_show: plt.show()
def test_lml_precomputed(): """ Test that lml of optimized kernel is stored correctly. """ for kernel in kernels: gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) assert_equal(gpr.log_marginal_likelihood(gpr.kernel_.theta), gpr.log_marginal_likelihood())
print "y: ", y # initialize the kernel kernel_test = RBF(length_scale=[2]) print "after kernel init" # train the model gp_test = GaussianProcessRegressor(kernel=kernel_test, alpha=0, normalize_y=True) print "after GP regressor" gp_test.fit(XT, y) print ("GPML kernel: %s" % gp_test.kernel_) print ("Log-marginal-likelihood: %.3f" % gp_test.log_marginal_likelihood(gp_test.kernel_.theta)) X_ = [] for i in range(15): X_.append([i + 0.5]) XT_ = scaler.transform(X_) print "X_ ", X_ y_pred, y_std = gp_test.predict(XT_, return_std=True) # Plot the predict result X = np.array(X) y = np.array(y) X_ = np.array(X_) plt.scatter(X[:, 0], y, c="k") plt.plot(X_[:, 0], y_pred)
plt.figure(0) kernel = 1.0 * RBF(length_scale=100.0, length_scale_bounds=(1e-2, 1e3)) \ + WhiteKernel(noise_level=1, noise_level_bounds=(1e-10, 1e+1)) gp = GaussianProcessRegressor(kernel=kernel, alpha=0.0).fit(X, y) X_ = np.linspace(0, 5, 100) y_mean, y_cov = gp.predict(X_[:, np.newaxis], return_cov=True) plt.plot(X_, y_mean, 'k', lw=3, zorder=9) plt.fill_between(X_, y_mean - np.sqrt(np.diag(y_cov)), y_mean + np.sqrt(np.diag(y_cov)), alpha=0.5, color='k') plt.plot(X_, 0.5*np.sin(3*X_), 'r', lw=3, zorder=9) plt.scatter(X[:, 0], y, c='r', s=50, zorder=10) plt.title("Initial: %s\nOptimum: %s\nLog-Marginal-Likelihood: %s" % (kernel, gp.kernel_, gp.log_marginal_likelihood(gp.kernel_.theta))) plt.tight_layout() # Second run plt.figure(1) kernel = 1.0 * RBF(length_scale=1.0, length_scale_bounds=(1e-2, 1e3)) \ + WhiteKernel(noise_level=1e-5, noise_level_bounds=(1e-10, 1e+1)) gp = GaussianProcessRegressor(kernel=kernel, alpha=0.0).fit(X, y) X_ = np.linspace(0, 5, 100) y_mean, y_cov = gp.predict(X_[:, np.newaxis], return_cov=True) plt.plot(X_, y_mean, 'k', lw=3, zorder=9) plt.fill_between(X_, y_mean - np.sqrt(np.diag(y_cov)), y_mean + np.sqrt(np.diag(y_cov)), alpha=0.5, color='k') plt.plot(X_, 0.5*np.sin(3*X_), 'r', lw=3, zorder=9)
# Specify stationary and non-stationary kernel kernel_matern = C(1.0, (1e-10, 1000)) \ * Matern(length_scale_bounds=(1e-1, 1e3), nu=1.5) gp_matern = GaussianProcessRegressor(kernel=kernel_matern) kernel_lls = C(1.0, (1e-10, 1000)) \ * LocalLengthScalesKernel.construct(X, l_L=0.1, l_U=2.0, l_samples=5) gp_lls = GaussianProcessRegressor(kernel=kernel_lls, optimizer=de_optimizer) # Fit GPs gp_matern.fit(X, y) gp_lls.fit(X, y) print "Learned kernel Matern: %s" % gp_matern.kernel_ print "Log-marginal-likelihood Matern: %s" \ % gp_matern.log_marginal_likelihood(gp_matern.kernel_.theta) print "Learned kernel LLS: %s" % gp_lls.kernel_ print "Log-marginal-likelihood LLS: %s" \ % gp_lls.log_marginal_likelihood(gp_lls.kernel_.theta) # Compute GP mean and standard deviation on test data X_ = np.linspace(-1, 1, 500) y_mean_lls, y_std_lls = gp_lls.predict(X_[:, np.newaxis], return_std=True) y_mean_matern, y_std_matern = \ gp_matern.predict(X_[:, np.newaxis], return_std=True) plt.figure(figsize=(7, 7)) plt.subplot(2, 1, 1)
print 'after kernel init' # train the model gp_test = GaussianProcessRegressor(kernel=kernel_test, alpha=8, normalize_y=True) print 'after GP regressor' # print("GPML kernel: %s" % gp_test.kernel_) # print("Log-marginal-likelihood: %.3f" # % gp_test.log_marginal_likelihood(gp_test.kernel_.theta)) gp_test.fit(XT, y) print("GPML kernel: %s" % gp_test.kernel_) print("Log-marginal-likelihood: %.3f" % gp_test.log_marginal_likelihood(gp_test.kernel_.theta)) X_ = [] for i in range(15): X_.append([i+0.5, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1]) XT_ = scaler.transform(X_) print 'X_ ', XT_ y_pred, y_std = gp_test.predict(XT_, return_std=True) # Plot the predict result X = np.array(X) y = np.array(y) X_ = np.array(X_) plt.scatter(X[:, 0], y, c='k') plt.plot(X_[:, 0], y_pred)
plt.plot(X_, y_samples, lw=1) plt.xlim(0, 5) plt.ylim(-3, 3) plt.title("Prior (kernel: %s)" % kernel, fontsize=12) # Generate data and fit GP rng = np.random.RandomState(4) X = rng.uniform(0, 5, 10)[:, np.newaxis] y = np.sin((X[:, 0] - 2.5) ** 2) gp.fit(X, y) # Plot posterior plt.subplot(2, 1, 2) X_ = np.linspace(0, 5, 100) y_mean, y_std = gp.predict(X_[:, np.newaxis], return_std=True) plt.plot(X_, y_mean, "k", lw=3, zorder=9) plt.fill_between(X_, y_mean - y_std, y_mean + y_std, alpha=0.5, color="k") y_samples = gp.sample_y(X_[:, np.newaxis], 10) plt.plot(X_, y_samples, lw=1) plt.scatter(X[:, 0], y, c="r", s=50, zorder=10) plt.xlim(0, 5) plt.ylim(-3, 3) plt.title( "Posterior (kernel: %s)\n Log-Likelihood: %.3f" % (gp.kernel_, gp.log_marginal_likelihood(gp.kernel_.theta)), fontsize=12, ) plt.tight_layout() plt.show()
plt.figure() plt.scatter(inputs_x_array[:,0][index_y1::len_x2],y_pred[index_y1::len_x2]) #from x1_min to x1_max plt.xlabel('X Label (radius)') plt.ylabel('Z Label (density)') plt.show() index_y2 = 7 #only valid until len_x1 print("Radius is "+str(inputs_x_array[:,0][index_y2*len_x2:index_y2*len_x2 + len_x2][0])+"m") plt.figure() plt.scatter(inputs_x_array[:,1][index_y2*len_x2:index_y2*len_x2 + len_x2],y_pred[index_y2*len_x2:index_y2*len_x2 + len_x2]) #from x2_min to x2_max plt.xlabel('Y Label (time)') plt.ylabel('Z Label (density)') plt.show() print(gp.kernel_) #gives optimized hyperparameters print(gp.log_marginal_likelihood(gp.kernel_.theta)) #log-likelihood alpha=1e-10 input_prediction = gp.predict(X,return_std=True) K, K_gradient = gp.kernel_(X, eval_gradient=True) K[np.diag_indices_from(K)] += alpha L = cholesky(K, lower=True) # Line 2 # Support multi-dimensional output of self.y_train_ if y.ndim == 1: y = y[:, np.newaxis] alpha = cho_solve((L, True), y) log_likelihood_dims = -0.5 * np.einsum("ik,ik->k", y, alpha) log_likelihood_dims -= np.log(np.diag(L)).sum()
def f(X): # target function is just a linear relationship + heteroscadastic noise return X + 0.5*np.random.multivariate_normal(np.zeros(X.shape[0]), np.diag(X**2), 1)[0] X = np.random.uniform(-7.5, 7.5, n_samples) # input data y = f(X) # Generate target values by applying function to manifold # Gaussian Process with RBF kernel and homoscedastic noise level kernel_homo = C(1.0, (1e-10, 1000)) * RBF(1, (0.01, 100.0)) \ + WhiteKernel(1e-3, (1e-10, 50.0)) gp_homoscedastic = GaussianProcessRegressor(kernel=kernel_homo, alpha=0) gp_homoscedastic.fit(X[:, np.newaxis], y) print "Homoscedastic kernel: %s" % gp_homoscedastic.kernel_ print "Homoscedastic LML: %.3f" \ % gp_homoscedastic.log_marginal_likelihood(gp_homoscedastic.kernel_.theta) print # Gaussian Process with RBF kernel and heteroscedastic noise level prototypes = KMeans(n_clusters=10).fit(X[:, np.newaxis]).cluster_centers_ kernel_hetero = C(1.0, (1e-10, 1000)) * RBF(1, (0.01, 100.0)) \ + HeteroscedasticKernel.construct(prototypes, 1e-3, (1e-10, 50.0), gamma=5.0, gamma_bounds="fixed") gp_heteroscedastic = GaussianProcessRegressor(kernel=kernel_hetero, alpha=0) gp_heteroscedastic.fit(X[:, np.newaxis], y) print "Heteroscedastic kernel: %s" % gp_heteroscedastic.kernel_ print "Heteroscedastic LML: %.3f" \ % gp_heteroscedastic.log_marginal_likelihood(gp_heteroscedastic.kernel_.theta) # Plot result
def test_lml_improving(kernel): # Test that hyperparameter-tuning improves log-marginal likelihood. gpr = GaussianProcessRegressor(kernel=kernel).fit(X, y) assert_greater(gpr.log_marginal_likelihood(gpr.kernel_.theta), gpr.log_marginal_likelihood(kernel.theta))
architecture=((n_features, n_hidden, n_dim_manifold),) kernel_nn = C(1.0, (1e-10, 100)) \ * ManifoldKernel.construct(base_kernel=RBF(0.1, (1.0, 100.0)), architecture=architecture, transfer_fct="tanh", max_nn_weight=1.0) \ + WhiteKernel(1e-3, (1e-10, 1e-1)) gp_nn = GaussianProcessRegressor(kernel=kernel_nn, alpha=0, n_restarts_optimizer=3) # Fit GPs and create scatter plot on test data gp.fit(X, y) gp_nn.fit(X, y) print "Initial kernel: %s" % gp_nn.kernel print "Log-marginal-likelihood: %s" \ % gp_nn.log_marginal_likelihood(gp_nn.kernel.theta) print "Learned kernel: %s" % gp_nn.kernel_ print "Log-marginal-likelihood: %s" \ % gp_nn.log_marginal_likelihood(gp_nn.kernel_.theta) X_test_ = np.random.uniform(-5, 5, (1000, n_dim_manifold)) X_nn_test = X_test_.dot(A) y_test = f(X_test_) plt.figure(figsize=(8, 6)) plt.subplot(1, 2, 1) plt.scatter(y_test, gp.predict(X_nn_test), c='b', label="GP RBF") plt.scatter(y_test, gp_nn.predict(X_nn_test), c='r', label="GP NN") plt.xlabel("True") plt.ylabel("Predicted") plt.legend(loc=0)
k2 = 2.4**2 * RBF(length_scale=90.0) \ * ExpSineSquared(length_scale=1.3, periodicity=1.0) # seasonal component # medium term irregularity k3 = 0.66**2 \ * RationalQuadratic(length_scale=1.2, alpha=0.78) k4 = 0.18**2 * RBF(length_scale=0.134) \ + WhiteKernel(noise_level=0.19**2) # noise terms kernel_gpml = k1 + k2 + k3 + k4 gp = GaussianProcessRegressor(kernel=kernel_gpml, alpha=0, optimizer=None, normalize_y=True) gp.fit(X, y) print("GPML kernel: %s" % gp.kernel_) print("Log-marginal-likelihood: %.3f" % gp.log_marginal_likelihood(gp.kernel_.theta)) # Kernel with optimized parameters k1 = 50.0**2 * RBF(length_scale=50.0) # long term smooth rising trend k2 = 2.0**2 * RBF(length_scale=100.0) \ * ExpSineSquared(length_scale=1.0, periodicity=1.0, periodicity_bounds="fixed") # seasonal component # medium term irregularities k3 = 0.5**2 * RationalQuadratic(length_scale=1.0, alpha=1.0) k4 = 0.1**2 * RBF(length_scale=0.1) \ + WhiteKernel(noise_level=0.1**2, noise_level_bounds=(1e-3, np.inf)) # noise terms kernel = k1 + k2 + k3 + k4 gp = GaussianProcessRegressor(kernel=kernel, alpha=0, normalize_y=True)