def __init__(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 2.
self.W = self.kernel.sample_W()
self.function = Parabola()
self.real_W = np.asarray([
[1],
[1]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W).reshape((-1, 1))
self.Y = self.function.f(Z.T).squeeze()
self.w_optimizer = t_WOptimizer(
self.kernel, # TODO: does the kernel take over the W?
self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale,
self.X, self.Y
)
# Define the plotting variables
self.tau_arr = np.linspace(0., self.w_optimizer.tau_max, 100)
def __init__(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 100
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 0.1 # 1e-7 # 0.1
self.W = self.kernel.sample_W()
self.function = AugmentedSinusoidal()
self.real_W = np.asarray([
[3],
[1]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
# [[0.9486833]
# [0.31622777]]
x_range = np.linspace(0., 1., int(np.sqrt(self.no_samples)))
y_range = np.linspace(0., 1., int(np.sqrt(self.no_samples)))
self.X = cartesian([x_range, y_range])
#self.X = np.random.rand(self.no_samples, self.real_dim)
print(self.X.shape)
Z = np.dot(self.X, self.real_W).reshape(-1, 1)
print(Z.shape)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.optimizer = TripathyOptimizer()
self.no_tries = 1
self.PLOT_MEAN = True
def __init__(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 50
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 0.1
self.W = self.kernel.sample_W()
self.function = Parabola()
self.real_W = np.asarray([
[1],
[1]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.w_optimizer = t_WOptimizer(
self.kernel, # TODO: does the kernel take over the W?
self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale,
self.X, self.Y
)
self.no_tries = 1000
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 20 # 50
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 0.1
self.W = self.kernel.sample_W()
self.function = Camelback()
self.real_W = np.asarray([
[0, 1],
[1, 0],
[0, 0]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
# [[0.9486833]
# [0.31622777]]
self.X = np.random.rand(self.no_samples, self.real_dim)
print(self.X.shape)
Z = np.dot(self.X, self.real_W)
print(Z.shape)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.no_tries = 2
def __init__(self):
self.real_dim = 5
self.active_dim = 1
self.no_samples = 20 # This should be proportional to the number of dimensions
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
self.function = PolynomialKernel() # Change the function
# Generate the positive semi-definite matrix
self.real_W = np.random.random((self.real_dim, self.active_dim))
# Take the real W to be completely random
# assert np.isclose( np.dot(self.real_W.T, self.real_W), np.eye(self.active_dim) )
self.X = np.random.rand(self.no_samples, self.real_dim)
#self.X = np.random.rand(self.no_samples, self.real_dim)
print(self.X.shape)
Z = np.dot(self.X, self.real_W).reshape(-1, self.active_dim)
print("The projected input matrix: ", Z.shape)
print(Z.shape)
self.Y = self.function.f(Z.T).reshape(-1, 1)
print("Shapes of X and Y: ", (self.X.shape, self.Y.shape) )
self.optimizer = TripathyOptimizer()
def set_new_kernel(self, d, W=None, variance=None, lengthscale=None):
self.kernel = TripathyMaternKernel(
real_dim=self.domain.d,
active_dim=d,
W=W,
variance=variance,
lengthscale=lengthscale
)
def init(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 75
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Hide the matrix over here!
if self.real_dim == 3 and self.active_dim == 2:
self.function = Camelback()
self.real_W = np.asarray([
[0, 1],
[1, 0],
[0, 0]
])
elif self.real_dim == 2 and self.active_dim == 1:
self.function = Parabola()
self.real_W = np.asarray([
[1],
[1],
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
else:
assert False, "W was not set!"
self.sn = 0.1
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.w_optimizer = t_WOptimizer(
self.kernel,
self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale,
self.X, self.Y
)
# We create the following kernel just to have access to the sample_W function!
# TripathyMaternKernel(self.real_dim)
self.tries = 10
self.max_iter = 1 # 150
assert False
self.metrics = Metrics(self.no_samples)
class TestAProcesses(object):
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
self.W = self.kernel.sample_W()
self.sn = 2.
self.function = Rosenbrock()
self.real_W = np.asarray([[0, 0], [0, 1], [1, 0]])
self.function = Rosenbrock()
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T)
self.w_optimizer = t_WOptimizer(
self.kernel, self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale, self.X, self.Y)
def test__A_returns_correct_values(self):
"""
Calculate this example by hand
:return:
"""
pass
def test__gamma_returns_correct_values(self):
"""
Calculate this example by hand
:return:
"""
pass
def run_optimization(X, Y, Y_noise, input_d):
# TODO: Y_noise is notuuse!
assert X.shape[1] == input_d, (X.shape, input_d)
# Run the optimizer
optimizer = TripathyOptimizer()
W_hat, sn, l, s, d = optimizer.find_active_subspace(X, Y)
# Spawn new kernel
kernel = TripathyMaternKernel(
real_dim=input_d,
active_dim=d,
W=W_hat,
variance=s,
lengthscale=l
)
# Spawn new GPRegression object
gpreg = GPRegression(
input_dim=input_d,
kernel=kernel,
noise_var=sn,
calculate_gradients=True
)
# Return the optimized gpreg object
return gpreg
def init(self):
self.no_samples = 20
self.function = Parabola()
self.real_dim = 2
self.active_dim = 1
self.function._set_dimension(self.active_dim)
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Hide the matrix over here!
# self.real_W = np.asarray([
# [0, 0],
# [0, 1],
# [1, 0]
# ])
self.real_W = np.asarray([[1], [1]])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
assert Z.shape == (self.no_samples, self.active_dim)
self.Y = self.function.f(Z.T).reshape(self.no_samples, 1)
#assert self.Y.shape == (self.no_samples,)
self.sn = 0.8
def visualize_angle_loss():
# Training parameters
NUM_TRAINING_POINTS = 100
fnc_config = {"noise_var": 0.01}
# Optimizer parameters
local_config = {"M_l": 100, "m": 300, "n": 300, "losses": [], "leps": 1e-6}
# local_config = {
# "M_l": 10,
# "m": 10,
# "n": 10,
# "losses": [],
# "leps": 1e-6
# }
class dotdict(dict):
"""dot.notation access to dictionary attributes"""
__getattr__ = dict.get
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
local_config = dotdict(local_config)
for name, fnc, active_d in FNC_TUPLES:
# Generate training points
X_train = (np.random.rand(NUM_TRAINING_POINTS, fnc.d) *
fnc._domain.range) + ((fnc._domain.u + fnc._domain.l) / 2.)
Y_train = fnc.f(X_train.T).reshape(-1, 1)
# Generate the kernel
t_kernel = TripathyMaternKernel(real_dim=fnc.d,
active_dim=active_d,
W=None,
variance=None,
lengthscale=None)
# Run the two-step-optimization on the respective function
# Retrieve intermediate matrices from there
all_Ws = run_two_step_optimization(local_config,
t_kernel=t_kernel,
sn=fnc_config['noise_var'],
X=X_train,
Y=Y_train,
save_Ws=True)
print(all_Ws)
losses = [
loss(t_kernel, W, sn, s, l * np.ones((W.shape[1], )), X_train,
Y_train) for W, sn, s, l in all_Ws
]
all_Ws = [x[0] for x in all_Ws]
print("All losses are: ", local_config.losses)
print("All are: ", losses)
# Check if the loss decreases after we receive the individual parameters
visualize_angle_given_W_array(fnc.W.T, all_Ws, name)
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
self.W = self.kernel.sample_W()
self.sn = 2.
self.function = Rosenbrock()
self.real_W = np.asarray([[0, 0], [0, 1], [1, 0]])
self.function = Rosenbrock()
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T)
self.w_optimizer = t_WOptimizer(
self.kernel, self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale, self.X, self.Y)
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
self.real_sn = 0.01
self.real_s = 0.1
self.real_l = 0.1 * np.ones(self.active_dim)
self.fix_W = self.kernel.sample_W()
self.function = Rosenbrock()
self.fix_W = np.asarray([[0, 0], [0, 1], [1, 0]])
self.function = Rosenbrock()
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.fix_W)
self.Y = self.function.f(Z.T).reshape(-1, 1)
print("Shape are: ", (self.X.shape, self.Y.shape))
self.parameter_optimizer = t_ParameterOptimizer(
self.fix_W, self.kernel, self.X, self.Y.T)
class TestIndividualFunctions(object):
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
self.sn = 2.
self.W = self.kernel.sample_W()
self.function = Rosenbrock()
self.real_W = np.asarray([[0, 0], [0, 1], [1, 0]])
self.function = Rosenbrock()
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T)
self.w_optimizer = t_WOptimizer(
self.kernel, self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale, self.X, self.Y)
def test__A_fnc(self):
self.init()
res = self.w_optimizer._A(self.W)
# TODO: check what dimension this is supposed to have!
assert res.shape == (self.real_dim, self.real_dim)
def test_gamma_returns_correct_dimensions(self):
self.init()
res = self.w_optimizer._gamma(1e-3, self.W)
assert res.shape == (self.W.shape[0], self.W.shape[1])
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
class TestKernelSematics(object):
# To calculate each element by hand, type in the following into wolfram alpha
# f(x)=(1+sqrt(3)*x)exp(−sqrt(3)*x)
# And calculate each r individually
#
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
def test_kernel_identity_W_zero_inp(self):
self.init()
X = np.asarray([[0, 0, 0], [0, 0, 0]])
W = np.asarray([[1, 0], [0, 1], [0, 0]])
self.kernel.update_params(W=W,
l=np.asarray(
[1. for i in range(self.active_dim)]),
s=1.)
self.real_kernel = Matern32(
self.active_dim,
ARD=True,
lengthscale=self.kernel.inner_kernel.lengthscale)
y_hat = self.kernel.K(X)
y = np.asarray([[1, 1], [1, 1]])
assert np.isclose(y, y_hat, rtol=1e-16).all()
def test_kernel_reverted_W(self):
self.init()
X = np.asarray([[0, 0.5, 0], [0, 0, 0.5]])
W = np.asarray([[0, 1], [0, 0], [1, 0]])
# After projection, we get
# X_new = [
# [0, 0],
# [0.5, 0]
# ]
# r = 0.25
self.kernel.update_params(W=W,
l=np.asarray(
[1. for i in range(self.active_dim)]),
s=1.)
y_hat = self.kernel.K(X)
y = np.asarray([[1, 0.784888], [0.784888, 1]])
assert np.isclose(y, y_hat, rtol=1e-4).all()
def test_kernel_some_random_W(self):
self.init()
for i in range(100):
X = np.random.rand(5, self.real_dim)
# Sample and re-assign
# TODO: just let the kernel resample all parameters
W = self.kernel.sample_W()
s = self.kernel.sample_variance()
l = self.kernel.sample_lengthscale()
self.kernel.update_params(W=W, l=l, s=s)
y_hat = self.kernel.K(X)
y = self.kernel.inner_kernel.K(np.dot(X, W))
assert np.isclose(y, y_hat).all()
def test_kernel_some_random_W_independent_inner_kernel(self):
self.init()
for i in range(100):
X = np.random.rand(5, self.real_dim)
# Sample and re-assign
# TODO: change this by just resampling using a function within the kernel
W = self.kernel.sample_W()
s = self.kernel.sample_variance()
l = self.kernel.sample_lengthscale()
self.kernel.update_params(W=W, l=l, s=s)
y_hat = self.kernel.K(X)
# Define the new kernel
real_kernel = Matern32(self.active_dim,
variance=s,
ARD=True,
lengthscale=l)
y = real_kernel.K(np.dot(X, W))
assert np.isclose(y, y_hat).all()
class TestKernel(object):
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
def test_parameters_are_set_successfully(self):
"""
Check if parameters are set successfully / setters work correctly
:return:
"""
self.init()
W1, l1, s1 = self.kernel.W, self.kernel.inner_kernel.lengthscale, self.kernel.inner_kernel.variance
W1 = W1.copy()
l1 = l1.copy()
s1 = s1.copy()
new_W = np.zeros((self.real_dim, self.active_dim), dtype=np.float64)
for i in range(self.real_dim):
for j in range(self.active_dim):
new_W[i, j] = np.random.normal(0, 1)
Q, R = np.linalg.qr(new_W)
# Set new parameters
self.kernel.update_params(W=Q,
l=np.random.rand(self.active_dim, ),
s=5.22)
assert not np.isclose(np.asarray(self.kernel.inner_kernel.lengthscale),
np.asarray(l1)).all()
assert not np.isclose(np.asarray(self.kernel.inner_kernel.variance),
np.asarray(s1))
assert not np.isclose(self.kernel.W, W1).all()
def test_kernel_returns_gram_matrix_correct_shape(self):
"""
Check
:return:
"""
self.init()
A = np.random.rand(self.no_samples, self.real_dim)
B = np.random.rand(self.no_samples, self.real_dim)
# print("Before we go into the function: ")
# print(A)
# print(B)
Cov = self.kernel.K(A, B)
assert Cov.shape == (self.no_samples, self.no_samples)
def test_kernel_returns_diag_correct_shape(self):
self.init()
A = np.random.rand(self.no_samples, self.real_dim)
# print("Before we go into the function Kdiag: ")
# print(A)
Kdiag = self.kernel.Kdiag(A)
assert Kdiag.shape == (self.no_samples, ), (Kdiag.shape, )
def test_kernel_K_of_r_words_for_vectors(self):
self.init()
x = np.random.rand(self.no_samples)
# print("Before we go into the function Kdiag: ")
# print(x)
kr = self.kernel.K_of_r(x)
assert kr.shape == (self.no_samples, ), (kr.shape, )
class TestParameterOptimization(object):
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
self.real_sn = 0.01
self.real_s = 0.1
self.real_l = 0.1 * np.ones(self.active_dim)
self.fix_W = self.kernel.sample_W()
self.function = Rosenbrock()
self.fix_W = np.asarray([[0, 0], [0, 1], [1, 0]])
self.function = Rosenbrock()
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.fix_W)
self.Y = self.function.f(Z.T).reshape(-1, 1)
print("Shape are: ", (self.X.shape, self.Y.shape))
self.parameter_optimizer = t_ParameterOptimizer(
self.fix_W, self.kernel, self.X, self.Y.T)
def test_parameter_opt_does_not_err(self):
self.init()
# Have some initial guesses for sn, s, l
s = float(np.random.rand(1))
sn = float(np.random.rand(1))
l = np.random.rand(self.active_dim)
new_s, new_l, new_sn = self.parameter_optimizer.optimize_s_sn_l(
sn, s, l)
print(s, new_s)
print(sn, new_sn)
print(l, new_l)
assert not np.isclose(new_s, s), (new_s, s)
assert not np.isclose(new_l, l).all(), (new_l, l)
assert not np.isclose(new_sn, sn), (new_sn, sn)
def test_parameter_optimizes_loss(self):
self.init()
s_init = float(np.random.rand(1))
sn_init = float(np.random.rand(1))
l_init = np.random.rand(self.active_dim)
old_loss = loss(self.kernel, self.fix_W, sn_init, s_init, l_init,
self.X, self.Y)
new_s, new_l, new_sn = self.parameter_optimizer.optimize_s_sn_l(
sn_init, s_init, l_init)
new_loss = loss(self.kernel, self.fix_W, new_sn, new_s, new_l, self.X,
self.Y)
# print("Old loss, new loss ", (old_loss, new_loss))
# TODO: Should this be a new smaller value, or a value toward zero, or a new bigger value?
# assert new_loss <= old_loss
assert new_loss != old_loss
def test_parameters_change(self):
self.init()
s_init = float(np.random.rand(1))
sn_init = float(np.random.rand(1))
l_init = np.random.rand(self.active_dim)
old_loss = loss(self.kernel, self.fix_W, sn_init, s_init, l_init,
self.X, self.Y)
new_s, new_l, new_sn = self.parameter_optimizer.optimize_s_sn_l(
sn_init, s_init, l_init)
assert s_init != new_s, (s_init, new_s)
assert not np.isclose(l_init, new_l).all(), (l_init, new_l)
assert sn_init != new_sn, (sn_init, new_sn)
def visualize_angle_loss():
NUM_TRIES = 15 # 50
# Training parameters
NUM_TRAINING_POINTS = 100
fnc_config = {"noise_var": 0.005}
# Optimizer parameters
# Following is ok for parabola
# local_config = {
# "M_l": 100,
# "m": 1,
# "n": 1,
# "losses": [],
# "leps": 1e-16
# }
# Following is ok for camelback
local_config = {"M_l": 100, "m": 1, "n": 1, "losses": [], "leps": 1e-13}
# local_config = {
# "M_l": 100,
# "m": 300,
# "n": 300,
# "losses": [],
# "leps": 1e-12
# }
# local_config = {
# "M_l": 10,
# "m": 10,
# "n": 10,
# "losses": [],
# "leps": 1e-6
# }
class dotdict(dict):
"""dot.notation access to dictionary attributes"""
__getattr__ = dict.get
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
local_config = dotdict(local_config)
for name, fnc, active_d in FNC_TUPLES:
if name == "Parabola-2D->1D":
config['active_dimension'] = 1
print("Active dimensions are at: ", config['active_dimension'])
else:
config['active_dimension'] = 2
all_angles = []
all_losses = []
title = name + "_" + str(NUM_TRIES) + "_" + str(random.randint(
1, 1000))
# Run for a few times
for n in range(NUM_TRIES):
print("Now at step: ", n)
# Generate training points
X_train = (np.random.rand(NUM_TRAINING_POINTS, fnc.d) *
fnc._domain.range) + (
(fnc._domain.u + fnc._domain.l) / 2.)
Y_train = fnc.f(X_train.T).reshape(-1, 1)
# Generate the kernel
t_kernel = TripathyMaternKernel(real_dim=fnc.d,
active_dim=active_d,
W=None,
variance=None,
lengthscale=None)
# Run the two-step-optimization on the respective function
# Retrieve intermediate matrices from there
all_Ws, best_config = run_two_step_optimization(
local_config,
t_kernel=t_kernel,
sn=fnc_config['noise_var'],
X=X_train,
Y=Y_train,
save_Ws=True,
save_best_config=True)
print(all_Ws)
losses = [
loss(t_kernel, W, sn, s, l * np.ones((W.shape[1], )), X_train,
Y_train) for W, sn, l, s in all_Ws
]
all_Ws = [x[0] for x in all_Ws]
print("All losses are: ", local_config.losses)
print("All are: ", losses)
# Calculate the angles
angles = list(
map(lambda x: calculate_angle_between_two_matrices(fnc.W.T, x),
all_Ws))
all_angles.append(angles) # Creates a 2d array
all_losses.append(losses) # Creates a 2d array
# Check if the loss decreases after we receive the individual parameters
# Pad the losses and arrays to the maximum size of the runs
all_angles = pad_2d_list(all_angles)
all_losses = pad_2d_list(all_losses)
print(all_angles.shape)
print(all_losses.shape)
visualize_angle_array_stddev(all_angles, title=title)
visualize_loss_array_stddev(all_losses, title=title)
visualize_loss_array_stddev(all_losses,
title=title,
subtract_mean=True)
# TODO: take max index for log-likelihood, and visualize angle and log-likelihood
print("Retrieving array from: ", all_losses[:, -1].reshape(-1))
max_index = np.argmax(all_losses[:, -1].reshape(-1))
print("Best index is: ", max_index)
print("Best found loss is: ")
visualize_angle_array_stddev(all_angles,
title=title,
max_index=max_index)
visualize_loss_array_stddev(all_losses,
title=title,
max_index=max_index)
print("Done...")
class TestMatrixRecoveryHighDegreePolynomial:
def __init__(self):
self.real_dim = 5
self.active_dim = 1
self.no_samples = 20 # This should be proportional to the number of dimensions
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
self.function = PolynomialKernel() # Change the function
# Generate the positive semi-definite matrix
self.real_W = np.random.random((self.real_dim, self.active_dim))
# Take the real W to be completely random
# assert np.isclose( np.dot(self.real_W.T, self.real_W), np.eye(self.active_dim) )
self.X = np.random.rand(self.no_samples, self.real_dim)
#self.X = np.random.rand(self.no_samples, self.real_dim)
print(self.X.shape)
Z = np.dot(self.X, self.real_W).reshape(-1, self.active_dim)
print("The projected input matrix: ", Z.shape)
print(Z.shape)
self.Y = self.function.f(Z.T).reshape(-1, 1)
print("Shapes of X and Y: ", (self.X.shape, self.Y.shape) )
self.optimizer = TripathyOptimizer()
def check_if_matrix_is_found(self):
import os
if not os.path.exists("./highD/"):
os.makedirs("./highD/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
# print("Real hidden matrix is: ", self.real_W)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts(self.kernel, self.X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
np.savetxt("./highD/" + str(l) + "_BestLoss.txt", W_hat)
np.savetxt("./highD/" + str(l) + "_realMatr.txt", self.real_W)
class VisualizedTestingTau:
def __init__(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 2.
self.W = self.kernel.sample_W()
self.function = Parabola()
self.real_W = np.asarray([
[1],
[1]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W).reshape((-1, 1))
self.Y = self.function.f(Z.T).squeeze()
self.w_optimizer = t_WOptimizer(
self.kernel, # TODO: does the kernel take over the W?
self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale,
self.X, self.Y
)
# Define the plotting variables
self.tau_arr = np.linspace(0., self.w_optimizer.tau_max, 100)
def visualize_tau_trajectory_for_random_W(self):
"""
Visualize the trajectory of the gamma
function against the loss function
we have f(tau) = F( gamma(tau, W) )
:return:
"""
loss_arr = []
# Sample a random W
W_init = self.kernel.sample_W()
for tau in self.tau_arr:
print("New tau is: ", tau)
W = self.w_optimizer._gamma(tau, W_init)
loss_val = loss(
self.kernel,
W,
self.sn,
self.kernel.inner_kernel.variance,
self.kernel.inner_kernel.lengthscale,
self.X,
self.Y.squeeze()
)
loss_arr.append(loss_val)
print(loss_arr)
plt.title("Tau vs Loss - Randomly sampled W")
plt.scatter(self.tau_arr, loss_arr)
plt.axis([min(self.tau_arr), max(self.tau_arr), min(loss_arr), max(loss_arr)])
plt.show()
def visualize_tau_trajectory_for_identity_W(self):
"""
Visualize the trajectory of the gamma
function against the loss function
we have f(tau) = F( gamma(tau, W) )
:return:
"""
loss_arr = []
# Sample a random W
W_init = self.real_W
for tau in self.tau_arr:
print("New tau is: ", tau)
W = self.w_optimizer._gamma(tau, W_init)
loss_val = loss(
self.kernel,
W,
self.sn,
self.kernel.inner_kernel.variance,
self.kernel.inner_kernel.lengthscale,
self.X,
self.Y.squeeze()
)
loss_arr.append(loss_val)
print(loss_arr)
plt.title("Tau vs Loss - Identity-similarsampled W")
plt.scatter(self.tau_arr, loss_arr)
plt.axis([min(self.tau_arr), max(self.tau_arr), min(loss_arr), max(loss_arr)])
plt.show()
class TestMatrixRecoveryNaive(object):
def __init__(self):
pass
# This skips the test
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 20 # 50
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 0.1
self.W = self.kernel.sample_W()
self.function = Camelback()
self.real_W = np.asarray([
[0, 1],
[1, 0],
[0, 0]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
# [[0.9486833]
# [0.31622777]]
self.X = np.random.rand(self.no_samples, self.real_dim)
print(self.X.shape)
Z = np.dot(self.X, self.real_W)
print(Z.shape)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.no_tries = 2
def test_visualize_augmented_sinusoidal_function(self):
self.init()
import os
if not os.path.exists("./pics/camelback/"):
os.makedirs("./pics/")
os.makedirs("./pics/camelback/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
print("Real hidden matrix is: ", self.real_W)
for j in range(self.no_tries):
print("Try number : ", j)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.run_two_step_optimization(self.kernel, self.sn, self.X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn)
y_hat = gp_reg.predict(self.X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
np.savetxt("./pics/camelback/Iter_" + str(j) + "__" + "Loss_" + str(l) + ".txt", W_hat)
class VisualizedTestingWParabola:
def __init__(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 50
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 0.1
self.W = self.kernel.sample_W()
self.function = Parabola()
self.real_W = np.asarray([
[1],
[1]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.w_optimizer = t_WOptimizer(
self.kernel, # TODO: does the kernel take over the W?
self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale,
self.X, self.Y
)
self.no_tries = 1000
def visualize_quadratic_function(self):
x_range = np.linspace(0., 1., 80)
y_range = np.linspace(0., 1., 80)
X = cartesian([x_range, y_range])
import os
if not os.path.exists("./pics/"):
os.makedirs("./pics/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
for j in range(self.no_tries):
print("Try number : ", j)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.run_two_step_optimization(self.kernel, self.sn, self.X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn)
y = self.function.f( np.dot(X, self.real_W).T )
y_hat = gp_reg.predict(self.X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
fig = plt.figure()
ax = Axes3D(fig)
# First plot the real function
ax.scatter(X[:,0], X[:, 1], y, s=1)
ax.scatter(self.X[:,0], self.X[:, 1], y_hat, cmap=plt.cm.jet)
fig.savefig('./pics/Iter_' + str(j) + '.png', )
# plt.show()
plt.close(fig)
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
np.savetxt("./pics/Iter_" + str(j) + "__" + "Loss_" + str(l) + ".txt", W_hat)
class TestMatrixRecovery(object):
"""
We hide a function depending on a matrix A within a higher dimension.
We then test if our algorithm can successfully approximate/find this matrix
(A_hat is the approximated one).
More specifically, check if:
f(A x) = f(A_hat x)
"""
def init(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 75
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Hide the matrix over here!
if self.real_dim == 3 and self.active_dim == 2:
self.function = Camelback()
self.real_W = np.asarray([
[0, 1],
[1, 0],
[0, 0]
])
elif self.real_dim == 2 and self.active_dim == 1:
self.function = Parabola()
self.real_W = np.asarray([
[1],
[1],
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
else:
assert False, "W was not set!"
self.sn = 0.1
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.w_optimizer = t_WOptimizer(
self.kernel,
self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale,
self.X, self.Y
)
# We create the following kernel just to have access to the sample_W function!
# TripathyMaternKernel(self.real_dim)
self.tries = 10
self.max_iter = 1 # 150
assert False
self.metrics = Metrics(self.no_samples)
def test_if_function_is_found(self):
"""
Replace these tests by the actual optimizer function!
:return:
"""
self.init()
print("Real matrix is: ", self.real_W)
all_tries = []
for i in range(self.tries):
# Initialize random guess
W_hat = self.kernel.sample_W()
# Find a good W!
for i in range(self.max_iter):
W_hat = self.w_optimizer.optimize_stiefel_manifold(W_hat)
print("Difference to real W is: ", (W_hat - self.real_W))
assert W_hat.shape == self.real_W.shape
self.kernel.update_params(
W=W_hat,
l=self.kernel.inner_kernel.lengthscale,
s=self.kernel.inner_kernel.variance
)
# TODO: update the gaussian process with the new kernels parameters! (i.e. W_hat)
# Create the gp_regression function and pass in the predictor function as f_hat
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=self.sn)
res = self.metrics.mean_difference_points(
fnc=self.function._f,
fnc_hat=gp_reg.predict,
A=self.real_W,
A_hat=W_hat,
X=self.X
)
all_tries.append(res)
print(all_tries)
assert np.asarray(all_tries).any()
def test_if_hidden_matrix_is_found_multiple_initializations(self):
self.init()
print("Real matrix is: ", self.real_W)
all_tries = []
for i in range(self.tries):
# Initialize random guess
W_hat = self.kernel.sample_W()
# Find a good W!
for i in range(self.max_iter):
W_hat = self.w_optimizer.optimize_stiefel_manifold(W_hat)
print("Difference to real (AA.T) W is: ", (W_hat - self.real_W))
assert W_hat.shape == self.real_W.shape
assert not (W_hat == self.real_W).all()
res = self.metrics.projects_into_same_original_point(self.real_W, W_hat)
all_tries.append(res)
assert True in all_tries
def setup_approximation(self):
self.g_input_dim = 5
self.kernel = TripathyMaternKernel(self.g_input_dim, self.active_dim)
class TripathyGP(ConfidenceBoundModel):
"""
Base class for GP optimization.
Handles common functionality.
"""
def set_new_kernel(self, d, W=None, variance=None, lengthscale=None):
self.kernel = TripathyMaternKernel(
real_dim=self.domain.d,
active_dim=d,
W=W,
variance=variance,
lengthscale=lengthscale
)
def set_new_gp(self, noise_var=None):
self.gp = GPRegression(
input_dim=self.domain.d,
kernel=self.kernel,
noise_var=noise_var if noise_var else 2., # self.config.noise_var,
calculate_gradients=self.config.calculate_gradients
)
def set_new_gp_and_kernel(self, d, W, variance, lengthscale, noise_var):
self.set_new_kernel(d, W, variance, lengthscale)
self.set_new_gp(noise_var)
def __init__(self, domain, calculate_always=False):
super(TripathyGP, self).__init__(domain)
self.optimizer = TripathyOptimizer()
# TODO: d is chosen to be an arbitrary value rn!
self.set_new_gp_and_kernel(2, None, None, None, None)
# number of data points
self.t = 0
self.kernel = self.kernel.copy()
self._woodbury_chol = np.asfortranarray(self.gp.posterior._woodbury_chol) # we create a copy of the matrix in fortranarray, such that we can directly pass it to lapack dtrtrs without doing another copy
self._woodbury_vector = self.gp.posterior._woodbury_vector.copy()
self._X = self.gp.X.copy()
self._Y = np.empty(shape=(0,1))
self._beta = 2
self._bias = self.config.bias
self.calculate_always = calculate_always
@property
def beta(self):
return self._beta
@property
def scale(self):
if self.gp.kern.name == 'sum':
return sum([part.variance for part in self.gp.kern.parts])
else:
return np.sqrt(self.gp.kern.variance)
@property
def bias(self):
return self._bias
def _get_gp(self):
return GPRegression(self.domain.d, self.kernel, noise_var=self.config.noise_var, calculate_gradients=self.config.calculate_gradients)
def add_data(self, x, y):
"""
Add a new function observation to the GPs.
Parameters
----------
x: 2d-array
y: 2d-array
"""
self.i = 1 if not ("i" in dir(self)) else self.i + 1
# print("Add data ", self.i)
x = np.atleast_2d(x)
y = np.atleast_2d(y)
self.set_data(x, y, append=True)
# TODO: check if this is called anyhow!
def optimize(self):
self._update_beta()
def _update_cache(self):
# if not self.config.calculate_gradients:
self._woodbury_chol = np.asfortranarray(self.gp.posterior._woodbury_chol)
self._woodbury_vector = self.gp.posterior._woodbury_vector.copy()
self._X = self.gp.X.copy()
self._update_beta()
def _optimize_bias(self):
self._bias = minimize(self._bias_loss, self._bias, method='L-BFGS-B')['x'].copy()
self._set_bias(self._bias)
logger.info(f"Updated bias to {self._bias}")
def _bias_loss(self, c):
# calculate mean and norm for new bias via a new woodbury_vector
new_woodbury_vector,_= dpotrs(self._woodbury_chol, self._Y - c, lower=1)
K = self.gp.kern.K(self.gp.X)
mean = np.dot(K, new_woodbury_vector)
norm = new_woodbury_vector.T.dot(mean)
# loss is least_squares_error + norm
return np.asscalar(np.sum(np.square(mean + c - self._Y)) + norm)
def _set_bias(self, c):
self._bias = c
self.gp.set_Y(self._Y - c)
self._woodbury_vector = self.gp.posterior._woodbury_vector.copy()
def _update_beta(self):
logdet = self._get_logdet()
logdet_priornoise = self._get_logdet_prior_noise()
self._beta = np.sqrt(2 * np.log(1/self.delta) + (logdet - logdet_priornoise)) + self._norm()
def _optimize_var(self):
# fix all parameters
for p in self.gp.parameters:
p.fix()
if self.gp.kern.name == 'sum':
for part in self.gp.kern.parts:
part.variance.unfix()
else:
self.gp.kern.variance.unfix()
self.gp.optimize()
if self.gp.kern.name == 'sum':
values = []
for part in self.gp.kern.parts:
values.append(np.asscalar(part.variance.values))
else:
values = np.asscalar(self.gp.kern.variance.values)
logger.info(f"Updated prior variance to {values}")
# unfix all parameters
for p in self.gp.parameters:
p.unfix()
def _get_logdet(self):
return 2.*np.sum(np.log(np.diag(self.gp.posterior._woodbury_chol)))
def _get_logdet_prior_noise(self):
return self.t * np.log(self.gp.likelihood.variance.values)
def mean_var(self, x):
"""Recompute the confidence intervals form the GP.
Parameters
----------
context: ndarray
Array that contains the context used to compute the sets
"""
x = np.atleast_2d(x)
if self.config.calculate_gradients or True:
mean,var = self.gp.predict_noiseless(x)
else:
mean,var = self._raw_predict(x)
return mean + self._bias, var
def mean_var_grad(self, x):
return self.gp.predictive_gradients(x)
def var(self, x):
return self.mean_var(x)[1]
def mean(self, x):
return self.mean_var(x)[0]
def set_data(self, X, Y, append=True):
if append:
X = np.concatenate((self.gp.X, X))
Y = np.concatenate((self.gp.Y, Y))
# Do our optimization now
if self.i % 100 == 99 or self.calculate_always:
import time
start_time = time.time()
print("Adding data: ", self.i)
# TODO: UNCOMMENT THE FOLLOWING LINE AGAIN!
# This is just to check if tripathy conforms with the other version
# W_hat, sn, l, s, d = self.optimizer.find_active_subspace(X, Y)
d = 2
W_hat = np.asarray([
[-0.31894555, 0.78400512, 0.38970008, 0.06119476, 0.35776912],
[-0.27150973, 0.066002, 0.42761931, -0.32079484, -0.79759551]
]).T
s = 1.
l = 1.5
sn = 2. # 0.01 #self.config.noise_var
print("--- %s seconds ---" % (time.time() - start_time))
# Overwrite GP and kernel values
self.set_new_gp_and_kernel(d=d, W=W_hat, variance=s, lengthscale=l, noise_var=sn)
# TODO: Should the following not come before the optimization?
self.gp.set_XY(X, Y)
self.t = X.shape[0]
self._update_cache()
def sample(self, X=None):
class GPSampler:
def __init__(self, X, Y, kernel, var):
self.X = X
self.Y = Y
self.N = var * np.ones(shape=Y.shape)
self.kernel = kernel
self.m = GPy.models.GPHeteroscedasticRegression(self.X, self.Y, self.kernel)
self.m['.*het_Gauss.variance'] = self.N
def __call__(self, X):
X = np.atleast_2d(X)
sample = np.empty(shape=(X.shape[0], 1))
# iteratively generate sample values for all x in x_test
for i, x in enumerate(X):
sample[i] = self.m.posterior_samples_f(x.reshape((1, -1)), size=1)
# add observation as without noise
self.X = np.vstack((self.X, x))
self.Y = np.vstack((self.Y, sample[i]))
self.N = np.vstack((self.N, 0))
# recalculate model
self.m = GPy.models.GPHeteroscedasticRegression(self.X, self.Y, self.kernel)
self.m['.*het_Gauss.variance'] = self.N # Set the noise parameters to the error in Y
return sample
return GPSampler(self.gp.X.copy(), self.gp.Y.copy(), self.kernel, self.gp.likelihood.variance)
def _raw_predict(self, Xnew):
Kx = self.kernel.K(self._X, Xnew)
mu = np.dot(Kx.T, self._woodbury_vector)
if len(mu.shape)==1:
mu = mu.reshape(-1,1)
Kxx = self.kernel.Kdiag(Xnew)
tmp = lapack.dtrtrs(self._woodbury_chol, Kx, lower=1, trans=0, unitdiag=0)[0]
var = (Kxx - np.square(tmp).sum(0))[:,None]
return mu, var
def _raw_predict_covar(self, Xnew, Xcond):
Kx = self.kernel.K(self._X, np.vstack((Xnew,Xcond)))
tmp = lapack.dtrtrs(self._woodbury_chol, Kx, lower=1, trans=0, unitdiag=0)[0]
n = Xnew.shape[0]
tmp1 = tmp[:,:n]
tmp2 = tmp[:,n:]
Kxx = self.kernel.K(Xnew, Xcond)
var = Kxx - (tmp1.T).dot(tmp2)
Kxx_new = self.kernel.Kdiag(Xnew)
var_Xnew = (Kxx_new - np.square(tmp1).sum(0))[:,None]
return var_Xnew, var
def _norm(self):
norm = self._woodbury_vector.T.dot(self.gp.kern.K(self.gp.X)).dot(self._woodbury_vector)
return np.asscalar(np.sqrt(norm))
def __getstate__(self):
self_dict = self.__dict__.copy()
del self_dict['gp'] # remove the gp from state dict to allow pickling. calculations are done via the cache woodbury/cholesky
return self_dict
class TestTauProcesses(object):
def init(self):
self.real_dim = 3
self.active_dim = 2
self.no_samples = 5
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
self.W = self.kernel.sample_W()
self.sn = 2.
self.function = Rosenbrock()
self.real_W = np.asarray([[0, 0], [0, 1], [1, 0]])
self.function = Rosenbrock()
self.X = np.random.rand(self.no_samples, self.real_dim)
Z = np.dot(self.X, self.real_W)
self.Y = self.function.f(Z.T)
self.w_optimizer = t_WOptimizer(
self.kernel, self.sn,
np.asscalar(self.kernel.inner_kernel.variance),
self.kernel.inner_kernel.lengthscale, self.X, self.Y)
def test_tau_trajectory_determines_W(self):
self.init()
no_samples = 20
W_init = self.kernel.sample_W()
all_Ws = []
tau_delta = 1e-5 #1e-4 is optimal!
tau_0 = 0
for i in range(no_samples):
inp_tau = min(tau_0 + i * tau_delta, self.w_optimizer.tau_max)
new_W = self.w_optimizer._gamma(inp_tau, W_init)
all_Ws.append(new_W)
for i in range(no_samples - 1):
assert ((all_Ws[i] - all_Ws[i + 1])**2).mean() >= 1e-16, str(
(i, all_Ws[i], all_Ws[i + 1]))
def test_tau_trajectory_determines_W_static(self):
"""
Not changing tau keeps W the same
:return:
"""
self.init()
no_samples = 20
tau_0 = np.random.rand(1) * self.w_optimizer.tau_max
for i in range(no_samples):
W_init = self.kernel.sample_W()
new_W1 = self.w_optimizer._gamma(tau_0, W_init)
new_W2 = self.w_optimizer._gamma(tau_0, W_init)
assert (new_W1 == new_W2).all()
def test__find_best_tau_finds_a_better_tau(self):
# TODO: semi-flaky test!
self.init()
W_init = self.kernel.sample_W()
init_tau = 5e-5
W_init = self.w_optimizer._gamma(init_tau, W_init)
new_tau = self.w_optimizer._find_best_tau(np.copy(W_init))
new_W = self.w_optimizer._gamma(new_tau, W_init)
old_loss = loss(self.kernel, W_init, self.sn,
self.kernel.inner_kernel.variance,
self.kernel.inner_kernel.lengthscale, self.X, self.Y)
new_loss = loss(self.kernel, new_W, self.sn,
self.kernel.inner_kernel.variance,
self.kernel.inner_kernel.lengthscale, self.X, self.Y)
if old_loss == new_loss:
warnings.warn("Old loss equals new loss! The W has not improved!")
print("WARNING: Old loss equals new loss! The W has not improved!")
print("Changes in W!")
print(W_init)
print(new_W)
print("Losses")
print(old_loss - new_loss)
# assert abs(new_loss - old_loss) > 1e-12
# TODO: This is a flaky test!
assert new_loss >= old_loss # TODO: is this ok? It looks like it heavily depends on how W is initialized!
def test_optimize_stiefel_manifold_doesnt_err(self):
self.init()
# For the sake of testing, do 10 iterations
self.w_optimizer.optimize_stiefel_manifold(self.W)
class VisualizeFeatureSelection:
# F dependent functions
def setup_environment(self):
# Environment dimensions
self.f_input_dim = 2
# Environment coefficients
self.a1 = -0.1
self.a2 = 0.1
# Setup the projection matrix
# self.real_W = self.kernel.sample_W()
# self.real_W = np.random.random((self.f_input_dim, self.active_dim))
# self.real_W = self.real_W / np.linalg.norm(self.real_W)
self.real_W = np.asarray(
[[5.889490030086947936e-01, 8.081701998063678394e-01]]).T
def f_env(self, x, disp=False):
real_phi_x = np.concatenate([
np.square(x[:, 0] - self.a1).reshape(x.shape[0], -1),
np.square(x[:, 1] - self.a2).reshape(x.shape[0], -1)
],
axis=1)
Z = np.dot(real_phi_x, self.real_W).reshape(-1, self.active_dim)
if not disp:
assert Z.shape == (self.no_samples,
self.active_dim), (Z.shape, (self.no_samples,
self.active_dim))
# print("The projected input matrix: ", Z.shape)
# print(Z.shape)
# self.Y = self.function.f(Z.T).reshape(-1, 1)
out = self.function.f(Z.T).reshape(-1, 1)
if not disp:
assert out.shape == (self.no_samples, 1)
return out
# G dependent function
def setup_approximation(self):
self.g_input_dim = 5
self.kernel = TripathyMaternKernel(self.g_input_dim, self.active_dim)
def phi(self, x):
assert x.shape[1] == 2
# We could just do x, but at this point it doesn't matter
phi_x = np.concatenate(
[np.square(x), x,
np.ones((x.shape[0], )).reshape(-1, 1)], axis=1)
# print("Phi x is: ", phi_x)
assert phi_x.shape == (self.no_samples, self.g_input_dim)
return phi_x
# Part where we generate the data
def generate_data(self):
self.X = (np.random.rand(self.no_samples, self.init_dimension) -
0.5) * 2.
# Generate the real Y
self.Y = self.f_env(self.X).reshape(-1, 1)
assert self.Y.shape == (self.no_samples, 1)
# Generate the kernelized X to use to figure it out
self.phi_X = self.phi(self.X)
def __init__(self):
self.function = Parabola()
self.init_dimension = 2 # All the input at the beginning is always 1D!
self.active_dim = self.function.d
self.no_samples = 50 # 200
self.setup_environment()
self.setup_approximation()
self.generate_data()
print("Data shape is: ")
print(self.X.shape)
print(self.phi_X.shape)
def plot_3d(self, y_hat, title):
# Plot real function
x1 = np.linspace(-1, 1, 100)
x2 = np.linspace(-1, 1, 100)
x_real = cartesian([x1, x2])
y_real = self.f_env(x_real, disp=True)
# Create the plot
fig = plt.figure()
ax = Axes3D(fig)
ax.view_init(azim=30)
# First plot the real function
ax.scatter(x_real[:, 0], x_real[:, 1], y_real, 'k.', alpha=.3, s=1)
ax.scatter(self.X[:, 0], self.X[:, 1], y_hat, cmap=plt.cm.jet)
fig.savefig('./featureSelection/' + title + '.png')
plt.show()
# plt.close(fig)
def check_if_matrix_is_found(self):
print("Starting to optimize stuf...")
import os
if not os.path.exists("./featureSelection/"):
os.makedirs("./featureSelection/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
print("Real hidden matrix is: ", self.real_W)
# Start with the approximation of the real matrix
W_hat = self.kernel.sample_W()
self.kernel.update_params(W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale)
W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts(
self.kernel, self.phi_X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.phi_X, self.Y, self.kernel, noise_var=sn)
# Maybe predict even more values ? (plot the entire surface?)
y_hat = gp_reg.predict(self.phi_X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
# Save the W just in case
l = loss(self.kernel, W_hat, sn, s, l, self.phi_X, self.Y)
np.savetxt(
config['basepath'] + "/featureSelection/" + str(l) +
"_BestLoss.txt", W_hat)
np.savetxt(
config['basepath'] + "/featureSelection/" + str(l) +
"_realMatr.txt", self.real_W)
# Create the gp_regression function and pass in the predictor function as f_hat
self.plot_3d(y_hat, title=str(l) + "_BestLoss")
class VisualizeTwoStepOptimizationWithRestarts:
def __init__(self):
self.real_dim = 2
self.active_dim = 1
self.no_samples = 100
self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
# Parameters
self.sn = 0.1 # 1e-7 # 0.1
self.W = self.kernel.sample_W()
self.function = AugmentedSinusoidal()
self.real_W = np.asarray([
[3],
[1]
])
self.real_W = self.real_W / np.linalg.norm(self.real_W)
# [[0.9486833]
# [0.31622777]]
x_range = np.linspace(0., 1., int(np.sqrt(self.no_samples)))
y_range = np.linspace(0., 1., int(np.sqrt(self.no_samples)))
self.X = cartesian([x_range, y_range])
#self.X = np.random.rand(self.no_samples, self.real_dim)
print(self.X.shape)
Z = np.dot(self.X, self.real_W).reshape(-1, 1)
print(Z.shape)
self.Y = self.function.f(Z.T).reshape(-1, 1)
self.optimizer = TripathyOptimizer()
# self.w_optimizer = t_WOptimizer(
# self.kernel, # TODO: does the kernel take over the W?
# self.sn,
# np.asscalar(self.kernel.inner_kernel.variance),
# self.kernel.inner_kernel.lengthscale,
# self.X, self.Y
# )
self.PLOT_MEAN = True
def visualize_augmented_sinusoidal_function(self):
x_range = np.linspace(0., 1., 80)
y_range = np.linspace(0., 1., 80)
X = cartesian([x_range, y_range])
import os
if not os.path.exists("./pics-twostep/"):
os.makedirs("./pics-twostep/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
print("Real hidden matrix is: ", self.real_W)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts(self.kernel, self.X, self.Y)
# TODO: Check if these values are attained over multiple iterations (check if assert otherwise fails)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn)
y = self.function.f( np.dot(X, self.real_W).T )
if self.PLOT_MEAN:
y_hat = gp_reg.predict(X)[0].squeeze()
else:
y_hat = gp_reg.predict(self.X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
fig = plt.figure()
ax = Axes3D(fig)
# First plot the real function
ax.scatter(X[:,0], X[:, 1], y, s=1)
if self.PLOT_MEAN:
ax.scatter(X[:, 0], X[:, 1], y_hat, cmap=plt.cm.jet)
else:
ax.scatter(self.X[:,0], self.X[:, 1], y_hat, cmap=plt.cm.jet)
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
fig.savefig('./pics-twostep/BestLoss_' + str(l) + '.png', )
plt.show()
plt.close(fig)
np.savetxt("./pics-twostep/BestLoss_" + str(l) + ".txt", W_hat)