class TestKernelSematics(object):
# To calculate each element by hand, type in the following into wolfram alpha
# f(x)=(1+sqrt(3)*x)exp(−sqrt(3)*x)
# And calculate each r individually
#
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
def test_kernel_identity_W_zero_inp(self):
self.init()
X = np.asarray([[0, 0, 0], [0, 0, 0]])
W = np.asarray([[1, 0], [0, 1], [0, 0]])
self.kernel.update_params(W=W,
l=np.asarray(
[1. for i in range(self.active_dim)]),
s=1.)
self.real_kernel = Matern32(
self.active_dim,
ARD=True,
lengthscale=self.kernel.inner_kernel.lengthscale)
y_hat = self.kernel.K(X)
y = np.asarray([[1, 1], [1, 1]])
assert np.isclose(y, y_hat, rtol=1e-16).all()
def test_kernel_reverted_W(self):
self.init()
X = np.asarray([[0, 0.5, 0], [0, 0, 0.5]])
W = np.asarray([[0, 1], [0, 0], [1, 0]])
# After projection, we get
# X_new = [
# [0, 0],
# [0.5, 0]
# ]
# r = 0.25
self.kernel.update_params(W=W,
l=np.asarray(
[1. for i in range(self.active_dim)]),
s=1.)
y_hat = self.kernel.K(X)
y = np.asarray([[1, 0.784888], [0.784888, 1]])
assert np.isclose(y, y_hat, rtol=1e-4).all()
def test_kernel_some_random_W(self):
self.init()
for i in range(100):
X = np.random.rand(5, self.real_dim)
# Sample and re-assign
# TODO: just let the kernel resample all parameters
W = self.kernel.sample_W()
s = self.kernel.sample_variance()
l = self.kernel.sample_lengthscale()
self.kernel.update_params(W=W, l=l, s=s)
y_hat = self.kernel.K(X)
y = self.kernel.inner_kernel.K(np.dot(X, W))
assert np.isclose(y, y_hat).all()
def test_kernel_some_random_W_independent_inner_kernel(self):
self.init()
for i in range(100):
X = np.random.rand(5, self.real_dim)
# Sample and re-assign
# TODO: change this by just resampling using a function within the kernel
W = self.kernel.sample_W()
s = self.kernel.sample_variance()
l = self.kernel.sample_lengthscale()
self.kernel.update_params(W=W, l=l, s=s)
y_hat = self.kernel.K(X)
# Define the new kernel
real_kernel = Matern32(self.active_dim,
variance=s,
ARD=True,
lengthscale=l)
y = real_kernel.K(np.dot(X, W))
assert np.isclose(y, y_hat).all()
class TestMatrixRecoveryNaive(object):
def __init__(self):
pass
# This skips the test
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 20 # 50
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 0.1
self.W = self.kernel.sample_W()
self.function = Camelback()
self.real_W = np.asarray([
[0, 1],
[1, 0],
[0, 0]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
# [[0.9486833]
# [0.31622777]]
self.X = np.random.rand(self.no_samples, self.real_dim)
print(self.X.shape)
Z = np.dot(self.X, self.real_W)
print(Z.shape)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.no_tries = 2
def test_visualize_augmented_sinusoidal_function(self):
self.init()
import os
if not os.path.exists("./pics/camelback/"):
os.makedirs("./pics/")
os.makedirs("./pics/camelback/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
print("Real hidden matrix is: ", self.real_W)
for j in range(self.no_tries):
print("Try number : ", j)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.run_two_step_optimization(self.kernel, self.sn, self.X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn)
y_hat = gp_reg.predict(self.X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
np.savetxt("./pics/camelback/Iter_" + str(j) + "__" + "Loss_" + str(l) + ".txt", W_hat)
class TestKernel(object):
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
def test_parameters_are_set_successfully(self):
"""
Check if parameters are set successfully / setters work correctly
:return:
"""
self.init()
W1, l1, s1 = self.kernel.W, self.kernel.inner_kernel.lengthscale, self.kernel.inner_kernel.variance
W1 = W1.copy()
l1 = l1.copy()
s1 = s1.copy()
new_W = np.zeros((self.real_dim, self.active_dim), dtype=np.float64)
for i in range(self.real_dim):
for j in range(self.active_dim):
new_W[i, j] = np.random.normal(0, 1)
Q, R = np.linalg.qr(new_W)
# Set new parameters
self.kernel.update_params(W=Q,
l=np.random.rand(self.active_dim, ),
s=5.22)
assert not np.isclose(np.asarray(self.kernel.inner_kernel.lengthscale),
np.asarray(l1)).all()
assert not np.isclose(np.asarray(self.kernel.inner_kernel.variance),
np.asarray(s1))
assert not np.isclose(self.kernel.W, W1).all()
def test_kernel_returns_gram_matrix_correct_shape(self):
"""
Check
:return:
"""
self.init()
A = np.random.rand(self.no_samples, self.real_dim)
B = np.random.rand(self.no_samples, self.real_dim)
# print("Before we go into the function: ")
# print(A)
# print(B)
Cov = self.kernel.K(A, B)
assert Cov.shape == (self.no_samples, self.no_samples)
def test_kernel_returns_diag_correct_shape(self):
self.init()
A = np.random.rand(self.no_samples, self.real_dim)
# print("Before we go into the function Kdiag: ")
# print(A)
Kdiag = self.kernel.Kdiag(A)
assert Kdiag.shape == (self.no_samples, ), (Kdiag.shape, )
def test_kernel_K_of_r_words_for_vectors(self):
self.init()
x = np.random.rand(self.no_samples)
# print("Before we go into the function Kdiag: ")
# print(x)
kr = self.kernel.K_of_r(x)
assert kr.shape == (self.no_samples, ), (kr.shape, )
class TestMatrixRecovery(object):
"""
We hide a function depending on a matrix A within a higher dimension.
We then test if our algorithm can successfully approximate/find this matrix
(A_hat is the approximated one).
More specifically, check if:
f(A x) = f(A_hat x)
"""
def init(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 75
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Hide the matrix over here!
if self.real_dim == 3 and self.active_dim == 2:
self.function = Camelback()
self.real_W = np.asarray([
[0, 1],
[1, 0],
[0, 0]
])
elif self.real_dim == 2 and self.active_dim == 1:
self.function = Parabola()
self.real_W = np.asarray([
[1],
[1],
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
else:
assert False, "W was not set!"
self.sn = 0.1
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.w_optimizer = t_WOptimizer(
self.kernel,
self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale,
self.X, self.Y
)
# We create the following kernel just to have access to the sample_W function!
# TripathyMaternKernel(self.real_dim)
self.tries = 10
self.max_iter = 1 # 150
assert False
self.metrics = Metrics(self.no_samples)
def test_if_function_is_found(self):
"""
Replace these tests by the actual optimizer function!
:return:
"""
self.init()
print("Real matrix is: ", self.real_W)
all_tries = []
for i in range(self.tries):
# Initialize random guess
W_hat = self.kernel.sample_W()
# Find a good W!
for i in range(self.max_iter):
W_hat = self.w_optimizer.optimize_stiefel_manifold(W_hat)
print("Difference to real W is: ", (W_hat - self.real_W))
assert W_hat.shape == self.real_W.shape
self.kernel.update_params(
W=W_hat,
l=self.kernel.inner_kernel.lengthscale,
s=self.kernel.inner_kernel.variance
)
# TODO: update the gaussian process with the new kernels parameters! (i.e. W_hat)
# Create the gp_regression function and pass in the predictor function as f_hat
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=self.sn)
res = self.metrics.mean_difference_points(
fnc=self.function._f,
fnc_hat=gp_reg.predict,
A=self.real_W,
A_hat=W_hat,
X=self.X
)
all_tries.append(res)
print(all_tries)
assert np.asarray(all_tries).any()
def test_if_hidden_matrix_is_found_multiple_initializations(self):
self.init()
print("Real matrix is: ", self.real_W)
all_tries = []
for i in range(self.tries):
# Initialize random guess
W_hat = self.kernel.sample_W()
# Find a good W!
for i in range(self.max_iter):
W_hat = self.w_optimizer.optimize_stiefel_manifold(W_hat)
print("Difference to real (AA.T) W is: ", (W_hat - self.real_W))
assert W_hat.shape == self.real_W.shape
assert not (W_hat == self.real_W).all()
res = self.metrics.projects_into_same_original_point(self.real_W, W_hat)
all_tries.append(res)
assert True in all_tries
class TestMatrixRecoveryHighDegreePolynomial:
def __init__(self):
self.real_dim = 5
self.active_dim = 1
self.no_samples = 20 # This should be proportional to the number of dimensions
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
self.function = PolynomialKernel() # Change the function
# Generate the positive semi-definite matrix
self.real_W = np.random.random((self.real_dim, self.active_dim))
# Take the real W to be completely random
# assert np.isclose( np.dot(self.real_W.T, self.real_W), np.eye(self.active_dim) )
self.X = np.random.rand(self.no_samples, self.real_dim)
#self.X = np.random.rand(self.no_samples, self.real_dim)
print(self.X.shape)
Z = np.dot(self.X, self.real_W).reshape(-1, self.active_dim)
print("The projected input matrix: ", Z.shape)
print(Z.shape)
self.Y = self.function.f(Z.T).reshape(-1, 1)
print("Shapes of X and Y: ", (self.X.shape, self.Y.shape) )
self.optimizer = TripathyOptimizer()
def check_if_matrix_is_found(self):
import os
if not os.path.exists("./highD/"):
os.makedirs("./highD/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
# print("Real hidden matrix is: ", self.real_W)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts(self.kernel, self.X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
np.savetxt("./highD/" + str(l) + "_BestLoss.txt", W_hat)
np.savetxt("./highD/" + str(l) + "_realMatr.txt", self.real_W)
class VisualizeTwoStepOptimizationWithRestarts:
def __init__(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 100
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 0.1 # 1e-7 # 0.1
self.W = self.kernel.sample_W()
self.function = AugmentedSinusoidal()
self.real_W = np.asarray([
[3],
[1]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
# [[0.9486833]
# [0.31622777]]
x_range = np.linspace(0., 1., int(np.sqrt(self.no_samples)))
y_range = np.linspace(0., 1., int(np.sqrt(self.no_samples)))
self.X = cartesian([x_range, y_range])
#self.X = np.random.rand(self.no_samples, self.real_dim)
print(self.X.shape)
Z = np.dot(self.X, self.real_W).reshape(-1, 1)
print(Z.shape)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.optimizer = TripathyOptimizer()
# self.w_optimizer = t_WOptimizer(
# self.kernel, # TODO: does the kernel take over the W?
# self.sn,
# np.asscalar(self.kernel.inner_kernel.variance),
# self.kernel.inner_kernel.lengthscale,
# self.X, self.Y
# )
self.PLOT_MEAN = True
def visualize_augmented_sinusoidal_function(self):
x_range = np.linspace(0., 1., 80)
y_range = np.linspace(0., 1., 80)
X = cartesian([x_range, y_range])
import os
if not os.path.exists("./pics-twostep/"):
os.makedirs("./pics-twostep/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
print("Real hidden matrix is: ", self.real_W)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts(self.kernel, self.X, self.Y)
# TODO: Check if these values are attained over multiple iterations (check if assert otherwise fails)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn)
y = self.function.f( np.dot(X, self.real_W).T )
if self.PLOT_MEAN:
y_hat = gp_reg.predict(X)[0].squeeze()
else:
y_hat = gp_reg.predict(self.X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
fig = plt.figure()
ax = Axes3D(fig)
# First plot the real function
ax.scatter(X[:,0], X[:, 1], y, s=1)
if self.PLOT_MEAN:
ax.scatter(X[:, 0], X[:, 1], y_hat, cmap=plt.cm.jet)
else:
ax.scatter(self.X[:,0], self.X[:, 1], y_hat, cmap=plt.cm.jet)
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
fig.savefig('./pics-twostep/BestLoss_' + str(l) + '.png', )
plt.show()
plt.close(fig)
np.savetxt("./pics-twostep/BestLoss_" + str(l) + ".txt", W_hat)
class VisualizedTestingWParabola:
def __init__(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 50
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 0.1
self.W = self.kernel.sample_W()
self.function = Parabola()
self.real_W = np.asarray([
[1],
[1]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.w_optimizer = t_WOptimizer(
self.kernel, # TODO: does the kernel take over the W?
self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale,
self.X, self.Y
)
self.no_tries = 1000
def visualize_quadratic_function(self):
x_range = np.linspace(0., 1., 80)
y_range = np.linspace(0., 1., 80)
X = cartesian([x_range, y_range])
import os
if not os.path.exists("./pics/"):
os.makedirs("./pics/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
for j in range(self.no_tries):
print("Try number : ", j)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.run_two_step_optimization(self.kernel, self.sn, self.X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn)
y = self.function.f( np.dot(X, self.real_W).T )
y_hat = gp_reg.predict(self.X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
fig = plt.figure()
ax = Axes3D(fig)
# First plot the real function
ax.scatter(X[:,0], X[:, 1], y, s=1)
ax.scatter(self.X[:,0], self.X[:, 1], y_hat, cmap=plt.cm.jet)
fig.savefig('./pics/Iter_' + str(j) + '.png', )
# plt.show()
plt.close(fig)
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
np.savetxt("./pics/Iter_" + str(j) + "__" + "Loss_" + str(l) + ".txt", W_hat)
class VisualizeFeatureSelection:
# F dependent functions
def setup_environment(self):
# Environment dimensions
self.f_input_dim = 2
# Environment coefficients
self.a1 = -0.1
self.a2 = 0.1
# Setup the projection matrix
# self.real_W = self.kernel.sample_W()
# self.real_W = np.random.random((self.f_input_dim, self.active_dim))
# self.real_W = self.real_W / np.linalg.norm(self.real_W)
self.real_W = np.asarray(
[[5.889490030086947936e-01, 8.081701998063678394e-01]]).T
def f_env(self, x, disp=False):
real_phi_x = np.concatenate([
np.square(x[:, 0] - self.a1).reshape(x.shape[0], -1),
np.square(x[:, 1] - self.a2).reshape(x.shape[0], -1)
],
axis=1)
Z = np.dot(real_phi_x, self.real_W).reshape(-1, self.active_dim)
if not disp:
assert Z.shape == (self.no_samples,
self.active_dim), (Z.shape, (self.no_samples,
self.active_dim))
# print("The projected input matrix: ", Z.shape)
# print(Z.shape)
# self.Y = self.function.f(Z.T).reshape(-1, 1)
out = self.function.f(Z.T).reshape(-1, 1)
if not disp:
assert out.shape == (self.no_samples, 1)
return out
# G dependent function
def setup_approximation(self):
self.g_input_dim = 5
self.kernel = TripathyMaternKernel(self.g_input_dim, self.active_dim)
def phi(self, x):
assert x.shape[1] == 2
# We could just do x, but at this point it doesn't matter
phi_x = np.concatenate(
[np.square(x), x,
np.ones((x.shape[0], )).reshape(-1, 1)], axis=1)
# print("Phi x is: ", phi_x)
assert phi_x.shape == (self.no_samples, self.g_input_dim)
return phi_x
# Part where we generate the data
def generate_data(self):
self.X = (np.random.rand(self.no_samples, self.init_dimension) -
0.5) * 2.
# Generate the real Y
self.Y = self.f_env(self.X).reshape(-1, 1)
assert self.Y.shape == (self.no_samples, 1)
# Generate the kernelized X to use to figure it out
self.phi_X = self.phi(self.X)
def __init__(self):
self.function = Parabola()
self.init_dimension = 2 # All the input at the beginning is always 1D!
self.active_dim = self.function.d
self.no_samples = 50 # 200
self.setup_environment()
self.setup_approximation()
self.generate_data()
print("Data shape is: ")
print(self.X.shape)
print(self.phi_X.shape)
def plot_3d(self, y_hat, title):
# Plot real function
x1 = np.linspace(-1, 1, 100)
x2 = np.linspace(-1, 1, 100)
x_real = cartesian([x1, x2])
y_real = self.f_env(x_real, disp=True)
# Create the plot
fig = plt.figure()
ax = Axes3D(fig)
ax.view_init(azim=30)
# First plot the real function
ax.scatter(x_real[:, 0], x_real[:, 1], y_real, 'k.', alpha=.3, s=1)
ax.scatter(self.X[:, 0], self.X[:, 1], y_hat, cmap=plt.cm.jet)
fig.savefig('./featureSelection/' + title + '.png')
plt.show()
# plt.close(fig)
def check_if_matrix_is_found(self):
print("Starting to optimize stuf...")
import os
if not os.path.exists("./featureSelection/"):
os.makedirs("./featureSelection/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
print("Real hidden matrix is: ", self.real_W)
# Start with the approximation of the real matrix
W_hat = self.kernel.sample_W()
self.kernel.update_params(W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale)
W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts(
self.kernel, self.phi_X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.phi_X, self.Y, self.kernel, noise_var=sn)
# Maybe predict even more values ? (plot the entire surface?)
y_hat = gp_reg.predict(self.phi_X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
# Save the W just in case
l = loss(self.kernel, W_hat, sn, s, l, self.phi_X, self.Y)
np.savetxt(
config['basepath'] + "/featureSelection/" + str(l) +
"_BestLoss.txt", W_hat)
np.savetxt(
config['basepath'] + "/featureSelection/" + str(l) +
"_realMatr.txt", self.real_W)
# Create the gp_regression function and pass in the predictor function as f_hat
self.plot_3d(y_hat, title=str(l) + "_BestLoss")