def run_methods(train_points, train_targets, test_points, test_targets,
model_parameters, m_list, file_name, title, show=False, full=True, vi=True):
method = 'means'
optimizer = 'L-BFGS-B'
max_iter = 50
options = {'maxiter': max_iter, 'disp': False, 'mydisp': True}
means_r2 = []
vi_r2 = []
for m in m_list:
print('m:', m)
print('Finding means...')
means = KMeans(n_clusters=m, n_init=1, max_iter=20)
means.fit(train_points.T)
inputs = means.cluster_centers_.T
print('...found')
model_covariance_obj = SquaredExponential(np.copy(model_parameters))
new_gp = GPR(model_covariance_obj, method='means', optimizer=optimizer)
res = new_gp.fit(train_points, train_targets, num_inputs=m, optimizer_options=options, inputs=inputs)
predicted_y_test, _, _ = new_gp.predict(test_points)
means_r2.append(r2_score(test_targets, predicted_y_test))
if vi:
model_covariance_obj = SquaredExponential(np.copy(model_parameters))
new_gp = GPR(model_covariance_obj, method='vi', optimizer=optimizer)
res = new_gp.fit(train_points, train_targets, num_inputs=m, optimizer_options=options, inputs=inputs)
predicted_y_test, _, _ = new_gp.predict(test_points)
vi_r2.append(r2_score(test_targets, predicted_y_test))
if full:
model_covariance_obj = SquaredExponential(np.copy(model_parameters))
new_gp = GPR(model_covariance_obj, method='brute')
res = new_gp.fit(train_points, train_targets, max_iter=max_iter)
predicted_y_test, _, _ = new_gp.predict(test_points, train_points, train_targets)
brute_r2 = r2_score(test_targets, predicted_y_test)
plt.plot(range(len(m_list)), means_r2, '-kx', label='vi-means')
if vi:
plt.plot(range(len(m_list)), vi_r2, '-rx', label='vi')
if full:
plt.plot(range(len(m_list)), len(m_list) * [brute_r2], '--g', label='full GP')
plt.xticks(range(len(m_list)), m_list)
plt.xlabel('m')
plt.ylabel('$R^2$-score on test data')
# plt.ylim(0.5, 1)
plt.legend(loc=4)
plt.title(title)
plt.savefig('../Plots/inducing_inputs/'+file_name + '.pgf')
if show:
plt.show()
def run_methods(train_points, train_targets, test_points, test_targets,
model_parameters, optimizer_options, file_name, ind_num, title, show=False):
method = 'means'
print('Finding means...')
means = KMeans(n_clusters=ind_num, n_init=1, max_iter=20)
means.fit(train_points.T)
inputs = means.cluster_centers_.T
print('...found')
for optimizer, color, opts in zip(['L-BFGS-B', 'Projected Newton'], ['-kx', '-mx'],
optimizer_options):
print('Optimizer', optimizer)
model_covariance_obj = SquaredExponential(np.copy(model_parameters))
new_gp = GPR(model_covariance_obj, method=method, optimizer=optimizer)
res = new_gp.fit(train_points, train_targets, num_inputs=ind_num, optimizer_options=opts, inputs=inputs)
name = optimizer
metric = lambda w: new_gp.get_prediction_quality(w, train_points, train_targets, test_points, test_targets)
x_lst, y_lst = res.plot_performance(metric, 'i', freq=1)
plt.plot(x_lst, y_lst, color, label=name)
plt.xlabel('Epoch')
plt.ylabel('$R^2$-score on test data')
plt.legend()
plt.title(title)
plt.savefig('../Plots/vi_variations/'+file_name + '.pgf')
if show:
plt.show()
def run_methods(train_points, train_targets, test_points, test_targets,
model_parameters, optimizer_options, file_name, ind_num, title, show=False):
print('Finding means...')
means = KMeans(n_clusters=ind_num, n_init=1, max_iter=40)
means.fit(train_points.T)
inputs = means.cluster_centers_.T
print('...found')
# method = 'svi'
# parametrization = 'natural'
# # optimizer = 'L-BFGS-B'
# color = '-yo'
# opts = optimizer_options[0]
# print('svi')
# model_covariance_obj = SquaredExponential(np.copy(model_parameters))
# new_gp = GPR(model_covariance_obj, method=method, parametrization=parametrization)
# res = new_gp.fit(train_points, train_targets, num_inputs=ind_num, optimizer_options=opts, inputs=inputs)
# name = 'svi-natural'
# metric = lambda w: new_gp.get_prediction_quality(w, test_points, test_targets)
# x_lst, y_lst = res.plot_performance(metric, 'i', freq=1)
# plt.plot(x_lst, y_lst, color, label=name)
print('vi-means')
method = 'means'
opt_options = optimizer_options[1]
model_covariance_obj = SquaredExponential(np.copy(model_parameters))
new_gp = GPR(model_covariance_obj, method=method)
res = new_gp.fit(train_points, train_targets, num_inputs=ind_num, optimizer_options=opt_options, inputs=inputs)
name = 'vi-means'
metric = lambda w: new_gp.get_prediction_quality(w, train_points, train_targets, test_points, test_targets)
x_lst, y_lst = res.plot_performance(metric, 'i', freq=1)
plt.plot(x_lst, y_lst, '-kx', label=name)
print(x_lst[-1])
plt.ylabel('$R^2$-score on test data')
plt.legend()
plt.title(title)
# plt.savefig('../Plots/vi_vs_svi/'+file_name + '.pgf')
if show:
plt.show()
def run_methods(train_points, train_targets, test_points, test_targets,
model_parameters, optimizer_options, file_name, ind_num, title, show=False):
method = 'svi'
parametrization = 'cholesky'
means = KMeans(n_clusters=ind_num, n_init=3, max_iter=100, random_state=241)
means.fit(train_points.T)
inputs = means.cluster_centers_.T
# for optimizer, color, opts in zip(['SAG', 'FG', 'L-BFGS-B'], ['-ro', '-bo', '-go'],
# optimizer_options[:-1]):
# print('Optimizer', optimizer)
# model_covariance_obj = SquaredExponential(np.copy(model_parameters))
# new_gp = GPR(model_covariance_obj, method=method, parametrization=parametrization, optimizer=optimizer)
# res = new_gp.fit(train_points, train_targets, num_inputs=ind_num, optimizer_options=opts, inputs=inputs)
# name = 'svi-' + optimizer
# metric = lambda w: new_gp.get_prediction_quality(w, test_points, test_targets)
# x_lst, y_lst = res.plot_performance(metric, 'i', freq=5)
# plt.plot(x_lst, y_lst, color, label=name)
parametrization = 'natural'
print('Natural parametrization')
opt_options = optimizer_options[-1]
model_covariance_obj = SquaredExponential(np.copy(model_parameters))
new_gp = GPR(model_covariance_obj, method=method, parametrization=parametrization)
res = new_gp.fit(train_points, train_targets, num_inputs=ind_num, optimizer_options=opt_options, inputs=inputs)
name = 'svi-natural'
metric = lambda w: new_gp.get_prediction_quality(w, test_points, test_targets)
x_lst, y_lst = res.plot_performance(metric, 'i', freq=5)
print(y_lst)
plt.plot(x_lst, y_lst, '-yo', label=name)
plt.xlabel('Epoch')
plt.ylabel('$R^2$-score on test data')
plt.legend()
plt.title(title)
# plt.savefig('../Plots/svi_variations/'+file_name + '.pgf')
if show:
plt.show()
import numpy as np
from matplotlib import pyplot as plt
from GP.covariance_functions import SquaredExponential
from GP.gaussian_process_regression import GPR
from GP.plotting import gp_plot_reg_data
data_params = np.array([1.1, 0.3, 0.1])
data_covariance_obj = SquaredExponential(data_params)
# model_params = np.array([10.6, 5.2, 0.1])
model_params = np.array([1.5, 0.15, 0.1])
model_covariance_obj = SquaredExponential(model_params)
gp = GPR(data_covariance_obj)
num = 700
test_num = 100
dim = 1
seed = 22
method = 'brute' # possible methods: 'brute', 'vi', 'means', 'svi'
parametrization = 'natural' # possible parametrizations for svi method: cholesky, natural
ind_inputs_num = 5
max_iter = 100
lbfgsb_options = {'maxiter': max_iter, 'disp': False}
np.random.seed(seed)
x_tr = np.random.rand(dim, num)
if dim == 1:
x_test = np.linspace(0, 1, test_num)
x_test = x_test.reshape(1, test_num)
else:
x_test = np.random.rand(dim, test_num)
y_tr, y_test = gp.generate_data(x_tr, x_test, seed=seed)
mu, sigma1 = 0, 10
x_tr, y_tr = load_svmlight_file('Data/Classification/australian.txt')
x_tr = x_tr.T
x_tr = x_tr.toarray()
# y_g = y_g.toarray()
data_name = 'Australian'
x_tr = (x_tr + 1) / 2
y_tr = y_tr.reshape((y_tr.size, 1))
x_test = x_tr[:, int(x_tr.shape[1] * 0.8):]
y_test = y_tr[int(x_tr.shape[1] * 0.8):, :]
y_tr = y_tr[:int(x_tr.shape[1] * 0.8), :]
x_tr = x_tr[:, :int(x_tr.shape[1] * 0.8)]
print("Number of data points: ", x_tr.shape[1])
print("Number of test points: ", x_test.shape[1])
print("Number of features: ", x_tr.shape[0])
# #Generating the starting point
# np.random.seed(random_seed_w0)
# w0 = np.random.rand(3)
model_params = np.array([10., 0.7, 3.])
model_covariance_obj = SquaredExponential(model_params)
new_gp = GaussianProcess(model_covariance_obj, lambda x: 0, 'class')
new_gp.find_hyper_parameters(x_tr, y_tr, max_iter=100)
print(new_gp.covariance_obj.get_params())
predicted_y_test = new_gp.predict(x_test, x_tr, y_tr)
print("Mistakes: ", np.sum(predicted_y_test != y_test))