def run_methods(train_points, train_targets, test_points, test_targets, model_parameters, m_list, file_name, title, show=False, full=True, vi=True): method = 'means' optimizer = 'L-BFGS-B' max_iter = 50 options = {'maxiter': max_iter, 'disp': False, 'mydisp': True} means_r2 = [] vi_r2 = [] for m in m_list: print('m:', m) print('Finding means...') means = KMeans(n_clusters=m, n_init=1, max_iter=20) means.fit(train_points.T) inputs = means.cluster_centers_.T print('...found') model_covariance_obj = SquaredExponential(np.copy(model_parameters)) new_gp = GPR(model_covariance_obj, method='means', optimizer=optimizer) res = new_gp.fit(train_points, train_targets, num_inputs=m, optimizer_options=options, inputs=inputs) predicted_y_test, _, _ = new_gp.predict(test_points) means_r2.append(r2_score(test_targets, predicted_y_test)) if vi: model_covariance_obj = SquaredExponential(np.copy(model_parameters)) new_gp = GPR(model_covariance_obj, method='vi', optimizer=optimizer) res = new_gp.fit(train_points, train_targets, num_inputs=m, optimizer_options=options, inputs=inputs) predicted_y_test, _, _ = new_gp.predict(test_points) vi_r2.append(r2_score(test_targets, predicted_y_test)) if full: model_covariance_obj = SquaredExponential(np.copy(model_parameters)) new_gp = GPR(model_covariance_obj, method='brute') res = new_gp.fit(train_points, train_targets, max_iter=max_iter) predicted_y_test, _, _ = new_gp.predict(test_points, train_points, train_targets) brute_r2 = r2_score(test_targets, predicted_y_test) plt.plot(range(len(m_list)), means_r2, '-kx', label='vi-means') if vi: plt.plot(range(len(m_list)), vi_r2, '-rx', label='vi') if full: plt.plot(range(len(m_list)), len(m_list) * [brute_r2], '--g', label='full GP') plt.xticks(range(len(m_list)), m_list) plt.xlabel('m') plt.ylabel('$R^2$-score on test data') # plt.ylim(0.5, 1) plt.legend(loc=4) plt.title(title) plt.savefig('../Plots/inducing_inputs/'+file_name + '.pgf') if show: plt.show()
def run_methods(train_points, train_targets, test_points, test_targets, model_parameters, optimizer_options, file_name, ind_num, title, show=False): method = 'means' print('Finding means...') means = KMeans(n_clusters=ind_num, n_init=1, max_iter=20) means.fit(train_points.T) inputs = means.cluster_centers_.T print('...found') for optimizer, color, opts in zip(['L-BFGS-B', 'Projected Newton'], ['-kx', '-mx'], optimizer_options): print('Optimizer', optimizer) model_covariance_obj = SquaredExponential(np.copy(model_parameters)) new_gp = GPR(model_covariance_obj, method=method, optimizer=optimizer) res = new_gp.fit(train_points, train_targets, num_inputs=ind_num, optimizer_options=opts, inputs=inputs) name = optimizer metric = lambda w: new_gp.get_prediction_quality(w, train_points, train_targets, test_points, test_targets) x_lst, y_lst = res.plot_performance(metric, 'i', freq=1) plt.plot(x_lst, y_lst, color, label=name) plt.xlabel('Epoch') plt.ylabel('$R^2$-score on test data') plt.legend() plt.title(title) plt.savefig('../Plots/vi_variations/'+file_name + '.pgf') if show: plt.show()
def run_methods(train_points, train_targets, test_points, test_targets, model_parameters, optimizer_options, file_name, ind_num, title, show=False): print('Finding means...') means = KMeans(n_clusters=ind_num, n_init=1, max_iter=40) means.fit(train_points.T) inputs = means.cluster_centers_.T print('...found') # method = 'svi' # parametrization = 'natural' # # optimizer = 'L-BFGS-B' # color = '-yo' # opts = optimizer_options[0] # print('svi') # model_covariance_obj = SquaredExponential(np.copy(model_parameters)) # new_gp = GPR(model_covariance_obj, method=method, parametrization=parametrization) # res = new_gp.fit(train_points, train_targets, num_inputs=ind_num, optimizer_options=opts, inputs=inputs) # name = 'svi-natural' # metric = lambda w: new_gp.get_prediction_quality(w, test_points, test_targets) # x_lst, y_lst = res.plot_performance(metric, 'i', freq=1) # plt.plot(x_lst, y_lst, color, label=name) print('vi-means') method = 'means' opt_options = optimizer_options[1] model_covariance_obj = SquaredExponential(np.copy(model_parameters)) new_gp = GPR(model_covariance_obj, method=method) res = new_gp.fit(train_points, train_targets, num_inputs=ind_num, optimizer_options=opt_options, inputs=inputs) name = 'vi-means' metric = lambda w: new_gp.get_prediction_quality(w, train_points, train_targets, test_points, test_targets) x_lst, y_lst = res.plot_performance(metric, 'i', freq=1) plt.plot(x_lst, y_lst, '-kx', label=name) print(x_lst[-1]) plt.ylabel('$R^2$-score on test data') plt.legend() plt.title(title) # plt.savefig('../Plots/vi_vs_svi/'+file_name + '.pgf') if show: plt.show()
def run_methods(train_points, train_targets, test_points, test_targets, model_parameters, optimizer_options, file_name, ind_num, title, show=False): method = 'svi' parametrization = 'cholesky' means = KMeans(n_clusters=ind_num, n_init=3, max_iter=100, random_state=241) means.fit(train_points.T) inputs = means.cluster_centers_.T # for optimizer, color, opts in zip(['SAG', 'FG', 'L-BFGS-B'], ['-ro', '-bo', '-go'], # optimizer_options[:-1]): # print('Optimizer', optimizer) # model_covariance_obj = SquaredExponential(np.copy(model_parameters)) # new_gp = GPR(model_covariance_obj, method=method, parametrization=parametrization, optimizer=optimizer) # res = new_gp.fit(train_points, train_targets, num_inputs=ind_num, optimizer_options=opts, inputs=inputs) # name = 'svi-' + optimizer # metric = lambda w: new_gp.get_prediction_quality(w, test_points, test_targets) # x_lst, y_lst = res.plot_performance(metric, 'i', freq=5) # plt.plot(x_lst, y_lst, color, label=name) parametrization = 'natural' print('Natural parametrization') opt_options = optimizer_options[-1] model_covariance_obj = SquaredExponential(np.copy(model_parameters)) new_gp = GPR(model_covariance_obj, method=method, parametrization=parametrization) res = new_gp.fit(train_points, train_targets, num_inputs=ind_num, optimizer_options=opt_options, inputs=inputs) name = 'svi-natural' metric = lambda w: new_gp.get_prediction_quality(w, test_points, test_targets) x_lst, y_lst = res.plot_performance(metric, 'i', freq=5) print(y_lst) plt.plot(x_lst, y_lst, '-yo', label=name) plt.xlabel('Epoch') plt.ylabel('$R^2$-score on test data') plt.legend() plt.title(title) # plt.savefig('../Plots/svi_variations/'+file_name + '.pgf') if show: plt.show()
import numpy as np from matplotlib import pyplot as plt from GP.covariance_functions import SquaredExponential from GP.gaussian_process_regression import GPR from GP.plotting import gp_plot_reg_data data_params = np.array([1.1, 0.3, 0.1]) data_covariance_obj = SquaredExponential(data_params) # model_params = np.array([10.6, 5.2, 0.1]) model_params = np.array([1.5, 0.15, 0.1]) model_covariance_obj = SquaredExponential(model_params) gp = GPR(data_covariance_obj) num = 700 test_num = 100 dim = 1 seed = 22 method = 'brute' # possible methods: 'brute', 'vi', 'means', 'svi' parametrization = 'natural' # possible parametrizations for svi method: cholesky, natural ind_inputs_num = 5 max_iter = 100 lbfgsb_options = {'maxiter': max_iter, 'disp': False} np.random.seed(seed) x_tr = np.random.rand(dim, num) if dim == 1: x_test = np.linspace(0, 1, test_num) x_test = x_test.reshape(1, test_num) else: x_test = np.random.rand(dim, test_num) y_tr, y_test = gp.generate_data(x_tr, x_test, seed=seed)
mu, sigma1 = 0, 10 x_tr, y_tr = load_svmlight_file('Data/Classification/australian.txt') x_tr = x_tr.T x_tr = x_tr.toarray() # y_g = y_g.toarray() data_name = 'Australian' x_tr = (x_tr + 1) / 2 y_tr = y_tr.reshape((y_tr.size, 1)) x_test = x_tr[:, int(x_tr.shape[1] * 0.8):] y_test = y_tr[int(x_tr.shape[1] * 0.8):, :] y_tr = y_tr[:int(x_tr.shape[1] * 0.8), :] x_tr = x_tr[:, :int(x_tr.shape[1] * 0.8)] print("Number of data points: ", x_tr.shape[1]) print("Number of test points: ", x_test.shape[1]) print("Number of features: ", x_tr.shape[0]) # #Generating the starting point # np.random.seed(random_seed_w0) # w0 = np.random.rand(3) model_params = np.array([10., 0.7, 3.]) model_covariance_obj = SquaredExponential(model_params) new_gp = GaussianProcess(model_covariance_obj, lambda x: 0, 'class') new_gp.find_hyper_parameters(x_tr, y_tr, max_iter=100) print(new_gp.covariance_obj.get_params()) predicted_y_test = new_gp.predict(x_test, x_tr, y_tr) print("Mistakes: ", np.sum(predicted_y_test != y_test))