class TestAProcesses(object):
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
self.W = self.kernel.sample_W()
self.sn = 2.
self.function = Rosenbrock()
self.real_W = np.asarray([[0, 0], [0, 1], [1, 0]])
self.function = Rosenbrock()
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T)
self.w_optimizer = t_WOptimizer(
self.kernel, self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale, self.X, self.Y)
def test__A_returns_correct_values(self):
"""
Calculate this example by hand
:return:
"""
pass
def test__gamma_returns_correct_values(self):
"""
Calculate this example by hand
:return:
"""
pass
class TestIndividualFunctions(object):
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
self.sn = 2.
self.W = self.kernel.sample_W()
self.function = Rosenbrock()
self.real_W = np.asarray([[0, 0], [0, 1], [1, 0]])
self.function = Rosenbrock()
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T)
self.w_optimizer = t_WOptimizer(
self.kernel, self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale, self.X, self.Y)
def test__A_fnc(self):
self.init()
res = self.w_optimizer._A(self.W)
# TODO: check what dimension this is supposed to have!
assert res.shape == (self.real_dim, self.real_dim)
def test_gamma_returns_correct_dimensions(self):
self.init()
res = self.w_optimizer._gamma(1e-3, self.W)
assert res.shape == (self.W.shape[0], self.W.shape[1])
class TestKernelSematics(object):
# To calculate each element by hand, type in the following into wolfram alpha
# f(x)=(1+sqrt(3)*x)exp(−sqrt(3)*x)
# And calculate each r individually
#
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
def test_kernel_identity_W_zero_inp(self):
self.init()
X = np.asarray([[0, 0, 0], [0, 0, 0]])
W = np.asarray([[1, 0], [0, 1], [0, 0]])
self.kernel.update_params(W=W,
l=np.asarray(
[1. for i in range(self.active_dim)]),
s=1.)
self.real_kernel = Matern32(
self.active_dim,
ARD=True,
lengthscale=self.kernel.inner_kernel.lengthscale)
y_hat = self.kernel.K(X)
y = np.asarray([[1, 1], [1, 1]])
assert np.isclose(y, y_hat, rtol=1e-16).all()
def test_kernel_reverted_W(self):
self.init()
X = np.asarray([[0, 0.5, 0], [0, 0, 0.5]])
W = np.asarray([[0, 1], [0, 0], [1, 0]])
# After projection, we get
# X_new = [
# [0, 0],
# [0.5, 0]
# ]
# r = 0.25
self.kernel.update_params(W=W,
l=np.asarray(
[1. for i in range(self.active_dim)]),
s=1.)
y_hat = self.kernel.K(X)
y = np.asarray([[1, 0.784888], [0.784888, 1]])
assert np.isclose(y, y_hat, rtol=1e-4).all()
def test_kernel_some_random_W(self):
self.init()
for i in range(100):
X = np.random.rand(5, self.real_dim)
# Sample and re-assign
# TODO: just let the kernel resample all parameters
W = self.kernel.sample_W()
s = self.kernel.sample_variance()
l = self.kernel.sample_lengthscale()
self.kernel.update_params(W=W, l=l, s=s)
y_hat = self.kernel.K(X)
y = self.kernel.inner_kernel.K(np.dot(X, W))
assert np.isclose(y, y_hat).all()
def test_kernel_some_random_W_independent_inner_kernel(self):
self.init()
for i in range(100):
X = np.random.rand(5, self.real_dim)
# Sample and re-assign
# TODO: change this by just resampling using a function within the kernel
W = self.kernel.sample_W()
s = self.kernel.sample_variance()
l = self.kernel.sample_lengthscale()
self.kernel.update_params(W=W, l=l, s=s)
y_hat = self.kernel.K(X)
# Define the new kernel
real_kernel = Matern32(self.active_dim,
variance=s,
ARD=True,
lengthscale=l)
y = real_kernel.K(np.dot(X, W))
assert np.isclose(y, y_hat).all()
class TestParameterOptimization(object):
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
self.real_sn = 0.01
self.real_s = 0.1
self.real_l = 0.1 * np.ones(self.active_dim)
self.fix_W = self.kernel.sample_W()
self.function = Rosenbrock()
self.fix_W = np.asarray([[0, 0], [0, 1], [1, 0]])
self.function = Rosenbrock()
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.fix_W)
self.Y = self.function.f(Z.T).reshape(-1, 1)
print("Shape are: ", (self.X.shape, self.Y.shape))
self.parameter_optimizer = t_ParameterOptimizer(
self.fix_W, self.kernel, self.X, self.Y.T)
def test_parameter_opt_does_not_err(self):
self.init()
# Have some initial guesses for sn, s, l
s = float(np.random.rand(1))
sn = float(np.random.rand(1))
l = np.random.rand(self.active_dim)
new_s, new_l, new_sn = self.parameter_optimizer.optimize_s_sn_l(
sn, s, l)
print(s, new_s)
print(sn, new_sn)
print(l, new_l)
assert not np.isclose(new_s, s), (new_s, s)
assert not np.isclose(new_l, l).all(), (new_l, l)
assert not np.isclose(new_sn, sn), (new_sn, sn)
def test_parameter_optimizes_loss(self):
self.init()
s_init = float(np.random.rand(1))
sn_init = float(np.random.rand(1))
l_init = np.random.rand(self.active_dim)
old_loss = loss(self.kernel, self.fix_W, sn_init, s_init, l_init,
self.X, self.Y)
new_s, new_l, new_sn = self.parameter_optimizer.optimize_s_sn_l(
sn_init, s_init, l_init)
new_loss = loss(self.kernel, self.fix_W, new_sn, new_s, new_l, self.X,
self.Y)
# print("Old loss, new loss ", (old_loss, new_loss))
# TODO: Should this be a new smaller value, or a value toward zero, or a new bigger value?
# assert new_loss <= old_loss
assert new_loss != old_loss
def test_parameters_change(self):
self.init()
s_init = float(np.random.rand(1))
sn_init = float(np.random.rand(1))
l_init = np.random.rand(self.active_dim)
old_loss = loss(self.kernel, self.fix_W, sn_init, s_init, l_init,
self.X, self.Y)
new_s, new_l, new_sn = self.parameter_optimizer.optimize_s_sn_l(
sn_init, s_init, l_init)
assert s_init != new_s, (s_init, new_s)
assert not np.isclose(l_init, new_l).all(), (l_init, new_l)
assert sn_init != new_sn, (sn_init, new_sn)
class TestMatrixRecoveryNaive(object):
def __init__(self):
pass
# This skips the test
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 20 # 50
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 0.1
self.W = self.kernel.sample_W()
self.function = Camelback()
self.real_W = np.asarray([
[0, 1],
[1, 0],
[0, 0]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
# [[0.9486833]
# [0.31622777]]
self.X = np.random.rand(self.no_samples, self.real_dim)
print(self.X.shape)
Z = np.dot(self.X, self.real_W)
print(Z.shape)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.no_tries = 2
def test_visualize_augmented_sinusoidal_function(self):
self.init()
import os
if not os.path.exists("./pics/camelback/"):
os.makedirs("./pics/")
os.makedirs("./pics/camelback/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
print("Real hidden matrix is: ", self.real_W)
for j in range(self.no_tries):
print("Try number : ", j)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.run_two_step_optimization(self.kernel, self.sn, self.X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn)
y_hat = gp_reg.predict(self.X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
np.savetxt("./pics/camelback/Iter_" + str(j) + "__" + "Loss_" + str(l) + ".txt", W_hat)
class TestMatrixRecovery(object):
"""
We hide a function depending on a matrix A within a higher dimension.
We then test if our algorithm can successfully approximate/find this matrix
(A_hat is the approximated one).
More specifically, check if:
f(A x) = f(A_hat x)
"""
def init(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 75
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Hide the matrix over here!
if self.real_dim == 3 and self.active_dim == 2:
self.function = Camelback()
self.real_W = np.asarray([
[0, 1],
[1, 0],
[0, 0]
])
elif self.real_dim == 2 and self.active_dim == 1:
self.function = Parabola()
self.real_W = np.asarray([
[1],
[1],
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
else:
assert False, "W was not set!"
self.sn = 0.1
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.w_optimizer = t_WOptimizer(
self.kernel,
self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale,
self.X, self.Y
)
# We create the following kernel just to have access to the sample_W function!
# TripathyMaternKernel(self.real_dim)
self.tries = 10
self.max_iter = 1 # 150
assert False
self.metrics = Metrics(self.no_samples)
def test_if_function_is_found(self):
"""
Replace these tests by the actual optimizer function!
:return:
"""
self.init()
print("Real matrix is: ", self.real_W)
all_tries = []
for i in range(self.tries):
# Initialize random guess
W_hat = self.kernel.sample_W()
# Find a good W!
for i in range(self.max_iter):
W_hat = self.w_optimizer.optimize_stiefel_manifold(W_hat)
print("Difference to real W is: ", (W_hat - self.real_W))
assert W_hat.shape == self.real_W.shape
self.kernel.update_params(
W=W_hat,
l=self.kernel.inner_kernel.lengthscale,
s=self.kernel.inner_kernel.variance
)
# TODO: update the gaussian process with the new kernels parameters! (i.e. W_hat)
# Create the gp_regression function and pass in the predictor function as f_hat
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=self.sn)
res = self.metrics.mean_difference_points(
fnc=self.function._f,
fnc_hat=gp_reg.predict,
A=self.real_W,
A_hat=W_hat,
X=self.X
)
all_tries.append(res)
print(all_tries)
assert np.asarray(all_tries).any()
def test_if_hidden_matrix_is_found_multiple_initializations(self):
self.init()
print("Real matrix is: ", self.real_W)
all_tries = []
for i in range(self.tries):
# Initialize random guess
W_hat = self.kernel.sample_W()
# Find a good W!
for i in range(self.max_iter):
W_hat = self.w_optimizer.optimize_stiefel_manifold(W_hat)
print("Difference to real (AA.T) W is: ", (W_hat - self.real_W))
assert W_hat.shape == self.real_W.shape
assert not (W_hat == self.real_W).all()
res = self.metrics.projects_into_same_original_point(self.real_W, W_hat)
all_tries.append(res)
assert True in all_tries
class TestMatrixRecoveryHighDegreePolynomial:
def __init__(self):
self.real_dim = 5
self.active_dim = 1
self.no_samples = 20 # This should be proportional to the number of dimensions
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
self.function = PolynomialKernel() # Change the function
# Generate the positive semi-definite matrix
self.real_W = np.random.random((self.real_dim, self.active_dim))
# Take the real W to be completely random
# assert np.isclose( np.dot(self.real_W.T, self.real_W), np.eye(self.active_dim) )
self.X = np.random.rand(self.no_samples, self.real_dim)
#self.X = np.random.rand(self.no_samples, self.real_dim)
print(self.X.shape)
Z = np.dot(self.X, self.real_W).reshape(-1, self.active_dim)
print("The projected input matrix: ", Z.shape)
print(Z.shape)
self.Y = self.function.f(Z.T).reshape(-1, 1)
print("Shapes of X and Y: ", (self.X.shape, self.Y.shape) )
self.optimizer = TripathyOptimizer()
def check_if_matrix_is_found(self):
import os
if not os.path.exists("./highD/"):
os.makedirs("./highD/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
# print("Real hidden matrix is: ", self.real_W)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts(self.kernel, self.X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
np.savetxt("./highD/" + str(l) + "_BestLoss.txt", W_hat)
np.savetxt("./highD/" + str(l) + "_realMatr.txt", self.real_W)
class VisualizeTwoStepOptimizationWithRestarts:
def __init__(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 100
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 0.1 # 1e-7 # 0.1
self.W = self.kernel.sample_W()
self.function = AugmentedSinusoidal()
self.real_W = np.asarray([
[3],
[1]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
# [[0.9486833]
# [0.31622777]]
x_range = np.linspace(0., 1., int(np.sqrt(self.no_samples)))
y_range = np.linspace(0., 1., int(np.sqrt(self.no_samples)))
self.X = cartesian([x_range, y_range])
#self.X = np.random.rand(self.no_samples, self.real_dim)
print(self.X.shape)
Z = np.dot(self.X, self.real_W).reshape(-1, 1)
print(Z.shape)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.optimizer = TripathyOptimizer()
# self.w_optimizer = t_WOptimizer(
# self.kernel, # TODO: does the kernel take over the W?
# self.sn,
# np.asscalar(self.kernel.inner_kernel.variance),
# self.kernel.inner_kernel.lengthscale,
# self.X, self.Y
# )
self.PLOT_MEAN = True
def visualize_augmented_sinusoidal_function(self):
x_range = np.linspace(0., 1., 80)
y_range = np.linspace(0., 1., 80)
X = cartesian([x_range, y_range])
import os
if not os.path.exists("./pics-twostep/"):
os.makedirs("./pics-twostep/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
print("Real hidden matrix is: ", self.real_W)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts(self.kernel, self.X, self.Y)
# TODO: Check if these values are attained over multiple iterations (check if assert otherwise fails)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn)
y = self.function.f( np.dot(X, self.real_W).T )
if self.PLOT_MEAN:
y_hat = gp_reg.predict(X)[0].squeeze()
else:
y_hat = gp_reg.predict(self.X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
fig = plt.figure()
ax = Axes3D(fig)
# First plot the real function
ax.scatter(X[:,0], X[:, 1], y, s=1)
if self.PLOT_MEAN:
ax.scatter(X[:, 0], X[:, 1], y_hat, cmap=plt.cm.jet)
else:
ax.scatter(self.X[:,0], self.X[:, 1], y_hat, cmap=plt.cm.jet)
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
fig.savefig('./pics-twostep/BestLoss_' + str(l) + '.png', )
plt.show()
plt.close(fig)
np.savetxt("./pics-twostep/BestLoss_" + str(l) + ".txt", W_hat)
class VisualizedTestingTau:
def __init__(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 2.
self.W = self.kernel.sample_W()
self.function = Parabola()
self.real_W = np.asarray([
[1],
[1]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W).reshape((-1, 1))
self.Y = self.function.f(Z.T).squeeze()
self.w_optimizer = t_WOptimizer(
self.kernel, # TODO: does the kernel take over the W?
self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale,
self.X, self.Y
)
# Define the plotting variables
self.tau_arr = np.linspace(0., self.w_optimizer.tau_max, 100)
def visualize_tau_trajectory_for_random_W(self):
"""
Visualize the trajectory of the gamma
function against the loss function
we have f(tau) = F( gamma(tau, W) )
:return:
"""
loss_arr = []
# Sample a random W
W_init = self.kernel.sample_W()
for tau in self.tau_arr:
print("New tau is: ", tau)
W = self.w_optimizer._gamma(tau, W_init)
loss_val = loss(
self.kernel,
W,
self.sn,
self.kernel.inner_kernel.variance,
self.kernel.inner_kernel.lengthscale,
self.X,
self.Y.squeeze()
)
loss_arr.append(loss_val)
print(loss_arr)
plt.title("Tau vs Loss - Randomly sampled W")
plt.scatter(self.tau_arr, loss_arr)
plt.axis([min(self.tau_arr), max(self.tau_arr), min(loss_arr), max(loss_arr)])
plt.show()
def visualize_tau_trajectory_for_identity_W(self):
"""
Visualize the trajectory of the gamma
function against the loss function
we have f(tau) = F( gamma(tau, W) )
:return:
"""
loss_arr = []
# Sample a random W
W_init = self.real_W
for tau in self.tau_arr:
print("New tau is: ", tau)
W = self.w_optimizer._gamma(tau, W_init)
loss_val = loss(
self.kernel,
W,
self.sn,
self.kernel.inner_kernel.variance,
self.kernel.inner_kernel.lengthscale,
self.X,
self.Y.squeeze()
)
loss_arr.append(loss_val)
print(loss_arr)
plt.title("Tau vs Loss - Identity-similarsampled W")
plt.scatter(self.tau_arr, loss_arr)
plt.axis([min(self.tau_arr), max(self.tau_arr), min(loss_arr), max(loss_arr)])
plt.show()
class VisualizedTestingWParabola:
def __init__(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 50
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 0.1
self.W = self.kernel.sample_W()
self.function = Parabola()
self.real_W = np.asarray([
[1],
[1]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.w_optimizer = t_WOptimizer(
self.kernel, # TODO: does the kernel take over the W?
self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale,
self.X, self.Y
)
self.no_tries = 1000
def visualize_quadratic_function(self):
x_range = np.linspace(0., 1., 80)
y_range = np.linspace(0., 1., 80)
X = cartesian([x_range, y_range])
import os
if not os.path.exists("./pics/"):
os.makedirs("./pics/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
for j in range(self.no_tries):
print("Try number : ", j)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.run_two_step_optimization(self.kernel, self.sn, self.X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn)
y = self.function.f( np.dot(X, self.real_W).T )
y_hat = gp_reg.predict(self.X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
fig = plt.figure()
ax = Axes3D(fig)
# First plot the real function
ax.scatter(X[:,0], X[:, 1], y, s=1)
ax.scatter(self.X[:,0], self.X[:, 1], y_hat, cmap=plt.cm.jet)
fig.savefig('./pics/Iter_' + str(j) + '.png', )
# plt.show()
plt.close(fig)
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
np.savetxt("./pics/Iter_" + str(j) + "__" + "Loss_" + str(l) + ".txt", W_hat)
class TestTauProcesses(object):
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
self.W = self.kernel.sample_W()
self.sn = 2.
self.function = Rosenbrock()
self.real_W = np.asarray([[0, 0], [0, 1], [1, 0]])
self.function = Rosenbrock()
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T)
self.w_optimizer = t_WOptimizer(
self.kernel, self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale, self.X, self.Y)
def test_tau_trajectory_determines_W(self):
self.init()
no_samples = 20
W_init = self.kernel.sample_W()
all_Ws = []
tau_delta = 1e-5 #1e-4 is optimal!
tau_0 = 0
for i in range(no_samples):
inp_tau = min(tau_0 + i * tau_delta, self.w_optimizer.tau_max)
new_W = self.w_optimizer._gamma(inp_tau, W_init)
all_Ws.append(new_W)
for i in range(no_samples - 1):
assert ((all_Ws[i] - all_Ws[i + 1])**2).mean() >= 1e-16, str(
(i, all_Ws[i], all_Ws[i + 1]))
def test_tau_trajectory_determines_W_static(self):
"""
Not changing tau keeps W the same
:return:
"""
self.init()
no_samples = 20
tau_0 = np.random.rand(1) * self.w_optimizer.tau_max
for i in range(no_samples):
W_init = self.kernel.sample_W()
new_W1 = self.w_optimizer._gamma(tau_0, W_init)
new_W2 = self.w_optimizer._gamma(tau_0, W_init)
assert (new_W1 == new_W2).all()
def test__find_best_tau_finds_a_better_tau(self):
# TODO: semi-flaky test!
self.init()
W_init = self.kernel.sample_W()
init_tau = 5e-5
W_init = self.w_optimizer._gamma(init_tau, W_init)
new_tau = self.w_optimizer._find_best_tau(np.copy(W_init))
new_W = self.w_optimizer._gamma(new_tau, W_init)
old_loss = loss(self.kernel, W_init, self.sn,
self.kernel.inner_kernel.variance,
self.kernel.inner_kernel.lengthscale, self.X, self.Y)
new_loss = loss(self.kernel, new_W, self.sn,
self.kernel.inner_kernel.variance,
self.kernel.inner_kernel.lengthscale, self.X, self.Y)
if old_loss == new_loss:
warnings.warn("Old loss equals new loss! The W has not improved!")
print("WARNING: Old loss equals new loss! The W has not improved!")
print("Changes in W!")
print(W_init)
print(new_W)
print("Losses")
print(old_loss - new_loss)
# assert abs(new_loss - old_loss) > 1e-12
# TODO: This is a flaky test!
assert new_loss >= old_loss # TODO: is this ok? It looks like it heavily depends on how W is initialized!
def test_optimize_stiefel_manifold_doesnt_err(self):
self.init()
# For the sake of testing, do 10 iterations
self.w_optimizer.optimize_stiefel_manifold(self.W)
class VisualizeFeatureSelection:
# F dependent functions
def setup_environment(self):
# Environment dimensions
self.f_input_dim = 2
# Environment coefficients
self.a1 = -0.1
self.a2 = 0.1
# Setup the projection matrix
# self.real_W = self.kernel.sample_W()
# self.real_W = np.random.random((self.f_input_dim, self.active_dim))
# self.real_W = self.real_W / np.linalg.norm(self.real_W)
self.real_W = np.asarray(
[[5.889490030086947936e-01, 8.081701998063678394e-01]]).T
def f_env(self, x, disp=False):
real_phi_x = np.concatenate([
np.square(x[:, 0] - self.a1).reshape(x.shape[0], -1),
np.square(x[:, 1] - self.a2).reshape(x.shape[0], -1)
],
axis=1)
Z = np.dot(real_phi_x, self.real_W).reshape(-1, self.active_dim)
if not disp:
assert Z.shape == (self.no_samples,
self.active_dim), (Z.shape, (self.no_samples,
self.active_dim))
# print("The projected input matrix: ", Z.shape)
# print(Z.shape)
# self.Y = self.function.f(Z.T).reshape(-1, 1)
out = self.function.f(Z.T).reshape(-1, 1)
if not disp:
assert out.shape == (self.no_samples, 1)
return out
# G dependent function
def setup_approximation(self):
self.g_input_dim = 5
self.kernel = TripathyMaternKernel(self.g_input_dim, self.active_dim)
def phi(self, x):
assert x.shape[1] == 2
# We could just do x, but at this point it doesn't matter
phi_x = np.concatenate(
[np.square(x), x,
np.ones((x.shape[0], )).reshape(-1, 1)], axis=1)
# print("Phi x is: ", phi_x)
assert phi_x.shape == (self.no_samples, self.g_input_dim)
return phi_x
# Part where we generate the data
def generate_data(self):
self.X = (np.random.rand(self.no_samples, self.init_dimension) -
0.5) * 2.
# Generate the real Y
self.Y = self.f_env(self.X).reshape(-1, 1)
assert self.Y.shape == (self.no_samples, 1)
# Generate the kernelized X to use to figure it out
self.phi_X = self.phi(self.X)
def __init__(self):
self.function = Parabola()
self.init_dimension = 2 # All the input at the beginning is always 1D!
self.active_dim = self.function.d
self.no_samples = 50 # 200
self.setup_environment()
self.setup_approximation()
self.generate_data()
print("Data shape is: ")
print(self.X.shape)
print(self.phi_X.shape)
def plot_3d(self, y_hat, title):
# Plot real function
x1 = np.linspace(-1, 1, 100)
x2 = np.linspace(-1, 1, 100)
x_real = cartesian([x1, x2])
y_real = self.f_env(x_real, disp=True)
# Create the plot
fig = plt.figure()
ax = Axes3D(fig)
ax.view_init(azim=30)
# First plot the real function
ax.scatter(x_real[:, 0], x_real[:, 1], y_real, 'k.', alpha=.3, s=1)
ax.scatter(self.X[:, 0], self.X[:, 1], y_hat, cmap=plt.cm.jet)
fig.savefig('./featureSelection/' + title + '.png')
plt.show()
# plt.close(fig)
def check_if_matrix_is_found(self):
print("Starting to optimize stuf...")
import os
if not os.path.exists("./featureSelection/"):
os.makedirs("./featureSelection/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
print("Real hidden matrix is: ", self.real_W)
# Start with the approximation of the real matrix
W_hat = self.kernel.sample_W()
self.kernel.update_params(W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale)
W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts(
self.kernel, self.phi_X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.phi_X, self.Y, self.kernel, noise_var=sn)
# Maybe predict even more values ? (plot the entire surface?)
y_hat = gp_reg.predict(self.phi_X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
# Save the W just in case
l = loss(self.kernel, W_hat, sn, s, l, self.phi_X, self.Y)
np.savetxt(
config['basepath'] + "/featureSelection/" + str(l) +
"_BestLoss.txt", W_hat)
np.savetxt(
config['basepath'] + "/featureSelection/" + str(l) +
"_realMatr.txt", self.real_W)
# Create the gp_regression function and pass in the predictor function as f_hat
self.plot_3d(y_hat, title=str(l) + "_BestLoss")