def test__find_best_tau_finds_a_better_tau(self):
# TODO: semi-flaky test!
self.init()
W_init = self.kernel.sample_W()
init_tau = 5e-5
W_init = self.w_optimizer._gamma(init_tau, W_init)
new_tau = self.w_optimizer._find_best_tau(np.copy(W_init))
new_W = self.w_optimizer._gamma(new_tau, W_init)
old_loss = loss(self.kernel, W_init, self.sn,
self.kernel.inner_kernel.variance,
self.kernel.inner_kernel.lengthscale, self.X, self.Y)
new_loss = loss(self.kernel, new_W, self.sn,
self.kernel.inner_kernel.variance,
self.kernel.inner_kernel.lengthscale, self.X, self.Y)
if old_loss == new_loss:
warnings.warn("Old loss equals new loss! The W has not improved!")
print("WARNING: Old loss equals new loss! The W has not improved!")
print("Changes in W!")
print(W_init)
print(new_W)
print("Losses")
print(old_loss - new_loss)
# assert abs(new_loss - old_loss) > 1e-12
# TODO: This is a flaky test!
assert new_loss >= old_loss # TODO: is this ok? It looks like it heavily depends on how W is initialized!
def visualize_tau_trajectory_for_identity_W(self):
"""
Visualize the trajectory of the gamma
function against the loss function
we have f(tau) = F( gamma(tau, W) )
:return:
"""
loss_arr = []
# Sample a random W
W_init = self.real_W
for tau in self.tau_arr:
print("New tau is: ", tau)
W = self.w_optimizer._gamma(tau, W_init)
loss_val = loss(
self.kernel,
W,
self.sn,
self.kernel.inner_kernel.variance,
self.kernel.inner_kernel.lengthscale,
self.X,
self.Y.squeeze()
)
loss_arr.append(loss_val)
print(loss_arr)
plt.title("Tau vs Loss - Identity-similarsampled W")
plt.scatter(self.tau_arr, loss_arr)
plt.axis([min(self.tau_arr), max(self.tau_arr), min(loss_arr), max(loss_arr)])
plt.show()
def visualize_angle_loss():
# Training parameters
NUM_TRAINING_POINTS = 100
fnc_config = {"noise_var": 0.01}
# Optimizer parameters
local_config = {"M_l": 100, "m": 300, "n": 300, "losses": [], "leps": 1e-6}
# local_config = {
# "M_l": 10,
# "m": 10,
# "n": 10,
# "losses": [],
# "leps": 1e-6
# }
class dotdict(dict):
"""dot.notation access to dictionary attributes"""
__getattr__ = dict.get
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
local_config = dotdict(local_config)
for name, fnc, active_d in FNC_TUPLES:
# Generate training points
X_train = (np.random.rand(NUM_TRAINING_POINTS, fnc.d) *
fnc._domain.range) + ((fnc._domain.u + fnc._domain.l) / 2.)
Y_train = fnc.f(X_train.T).reshape(-1, 1)
# Generate the kernel
t_kernel = TripathyMaternKernel(real_dim=fnc.d,
active_dim=active_d,
W=None,
variance=None,
lengthscale=None)
# Run the two-step-optimization on the respective function
# Retrieve intermediate matrices from there
all_Ws = run_two_step_optimization(local_config,
t_kernel=t_kernel,
sn=fnc_config['noise_var'],
X=X_train,
Y=Y_train,
save_Ws=True)
print(all_Ws)
losses = [
loss(t_kernel, W, sn, s, l * np.ones((W.shape[1], )), X_train,
Y_train) for W, sn, s, l in all_Ws
]
all_Ws = [x[0] for x in all_Ws]
print("All losses are: ", local_config.losses)
print("All are: ", losses)
# Check if the loss decreases after we receive the individual parameters
visualize_angle_given_W_array(fnc.W.T, all_Ws, name)
def visualize_quadratic_function(self):
x_range = np.linspace(0., 1., 80)
y_range = np.linspace(0., 1., 80)
X = cartesian([x_range, y_range])
import os
if not os.path.exists("./pics/"):
os.makedirs("./pics/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
for j in range(self.no_tries):
print("Try number : ", j)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.run_two_step_optimization(self.kernel, self.sn, self.X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn)
y = self.function.f( np.dot(X, self.real_W).T )
y_hat = gp_reg.predict(self.X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
fig = plt.figure()
ax = Axes3D(fig)
# First plot the real function
ax.scatter(X[:,0], X[:, 1], y, s=1)
ax.scatter(self.X[:,0], self.X[:, 1], y_hat, cmap=plt.cm.jet)
fig.savefig('./pics/Iter_' + str(j) + '.png', )
# plt.show()
plt.close(fig)
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
np.savetxt("./pics/Iter_" + str(j) + "__" + "Loss_" + str(l) + ".txt", W_hat)
def test_parameter_optimizes_loss(self):
self.init()
s_init = float(np.random.rand(1))
sn_init = float(np.random.rand(1))
l_init = np.random.rand(self.active_dim)
old_loss = loss(self.kernel, self.fix_W, sn_init, s_init, l_init,
self.X, self.Y)
new_s, new_l, new_sn = self.parameter_optimizer.optimize_s_sn_l(
sn_init, s_init, l_init)
new_loss = loss(self.kernel, self.fix_W, new_sn, new_s, new_l, self.X,
self.Y)
# print("Old loss, new loss ", (old_loss, new_loss))
# TODO: Should this be a new smaller value, or a value toward zero, or a new bigger value?
# assert new_loss <= old_loss
assert new_loss != old_loss
def test_visualize_augmented_sinusoidal_function(self):
self.init()
import os
if not os.path.exists("./pics/camelback/"):
os.makedirs("./pics/")
os.makedirs("./pics/camelback/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
print("Real hidden matrix is: ", self.real_W)
for j in range(self.no_tries):
print("Try number : ", j)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.run_two_step_optimization(self.kernel, self.sn, self.X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn)
y_hat = gp_reg.predict(self.X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
np.savetxt("./pics/camelback/Iter_" + str(j) + "__" + "Loss_" + str(l) + ".txt", W_hat)
def check_if_matrix_is_found(self):
print("Starting to optimize stuf...")
import os
if not os.path.exists("./featureSelection/"):
os.makedirs("./featureSelection/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
print("Real hidden matrix is: ", self.real_W)
# Start with the approximation of the real matrix
W_hat = self.kernel.sample_W()
self.kernel.update_params(W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale)
W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts(
self.kernel, self.phi_X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.phi_X, self.Y, self.kernel, noise_var=sn)
# Maybe predict even more values ? (plot the entire surface?)
y_hat = gp_reg.predict(self.phi_X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
# Save the W just in case
l = loss(self.kernel, W_hat, sn, s, l, self.phi_X, self.Y)
np.savetxt(
config['basepath'] + "/featureSelection/" + str(l) +
"_BestLoss.txt", W_hat)
np.savetxt(
config['basepath'] + "/featureSelection/" + str(l) +
"_realMatr.txt", self.real_W)
# Create the gp_regression function and pass in the predictor function as f_hat
self.plot_3d(y_hat, title=str(l) + "_BestLoss")
def test_parameters_change(self):
self.init()
s_init = float(np.random.rand(1))
sn_init = float(np.random.rand(1))
l_init = np.random.rand(self.active_dim)
old_loss = loss(self.kernel, self.fix_W, sn_init, s_init, l_init,
self.X, self.Y)
new_s, new_l, new_sn = self.parameter_optimizer.optimize_s_sn_l(
sn_init, s_init, l_init)
assert s_init != new_s, (s_init, new_s)
assert not np.isclose(l_init, new_l).all(), (l_init, new_l)
assert sn_init != new_sn, (sn_init, new_sn)
def check_if_matrix_is_found(self):
import os
if not os.path.exists("./highD/"):
os.makedirs("./highD/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
# print("Real hidden matrix is: ", self.real_W)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts(self.kernel, self.X, self.Y)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
np.savetxt("./highD/" + str(l) + "_BestLoss.txt", W_hat)
np.savetxt("./highD/" + str(l) + "_realMatr.txt", self.real_W)
def visualize_angle_loss():
NUM_TRIES = 15 # 50
# Training parameters
NUM_TRAINING_POINTS = 100
fnc_config = {"noise_var": 0.005}
# Optimizer parameters
# Following is ok for parabola
# local_config = {
# "M_l": 100,
# "m": 1,
# "n": 1,
# "losses": [],
# "leps": 1e-16
# }
# Following is ok for camelback
local_config = {"M_l": 100, "m": 1, "n": 1, "losses": [], "leps": 1e-13}
# local_config = {
# "M_l": 100,
# "m": 300,
# "n": 300,
# "losses": [],
# "leps": 1e-12
# }
# local_config = {
# "M_l": 10,
# "m": 10,
# "n": 10,
# "losses": [],
# "leps": 1e-6
# }
class dotdict(dict):
"""dot.notation access to dictionary attributes"""
__getattr__ = dict.get
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
local_config = dotdict(local_config)
for name, fnc, active_d in FNC_TUPLES:
if name == "Parabola-2D->1D":
config['active_dimension'] = 1
print("Active dimensions are at: ", config['active_dimension'])
else:
config['active_dimension'] = 2
all_angles = []
all_losses = []
title = name + "_" + str(NUM_TRIES) + "_" + str(random.randint(
1, 1000))
# Run for a few times
for n in range(NUM_TRIES):
print("Now at step: ", n)
# Generate training points
X_train = (np.random.rand(NUM_TRAINING_POINTS, fnc.d) *
fnc._domain.range) + (
(fnc._domain.u + fnc._domain.l) / 2.)
Y_train = fnc.f(X_train.T).reshape(-1, 1)
# Generate the kernel
t_kernel = TripathyMaternKernel(real_dim=fnc.d,
active_dim=active_d,
W=None,
variance=None,
lengthscale=None)
# Run the two-step-optimization on the respective function
# Retrieve intermediate matrices from there
all_Ws, best_config = run_two_step_optimization(
local_config,
t_kernel=t_kernel,
sn=fnc_config['noise_var'],
X=X_train,
Y=Y_train,
save_Ws=True,
save_best_config=True)
print(all_Ws)
losses = [
loss(t_kernel, W, sn, s, l * np.ones((W.shape[1], )), X_train,
Y_train) for W, sn, l, s in all_Ws
]
all_Ws = [x[0] for x in all_Ws]
print("All losses are: ", local_config.losses)
print("All are: ", losses)
# Calculate the angles
angles = list(
map(lambda x: calculate_angle_between_two_matrices(fnc.W.T, x),
all_Ws))
all_angles.append(angles) # Creates a 2d array
all_losses.append(losses) # Creates a 2d array
# Check if the loss decreases after we receive the individual parameters
# Pad the losses and arrays to the maximum size of the runs
all_angles = pad_2d_list(all_angles)
all_losses = pad_2d_list(all_losses)
print(all_angles.shape)
print(all_losses.shape)
visualize_angle_array_stddev(all_angles, title=title)
visualize_loss_array_stddev(all_losses, title=title)
visualize_loss_array_stddev(all_losses,
title=title,
subtract_mean=True)
# TODO: take max index for log-likelihood, and visualize angle and log-likelihood
print("Retrieving array from: ", all_losses[:, -1].reshape(-1))
max_index = np.argmax(all_losses[:, -1].reshape(-1))
print("Best index is: ", max_index)
print("Best found loss is: ")
visualize_angle_array_stddev(all_angles,
title=title,
max_index=max_index)
visualize_loss_array_stddev(all_losses,
title=title,
max_index=max_index)
print("Done...")
def visualize_augmented_sinusoidal_function(self):
x_range = np.linspace(0., 1., 80)
y_range = np.linspace(0., 1., 80)
X = cartesian([x_range, y_range])
import os
if not os.path.exists("./pics-twostep/"):
os.makedirs("./pics-twostep/")
#################################
# TRAIN THE W_OPTIMIZER #
#################################
Opt = TripathyOptimizer()
print("Real hidden matrix is: ", self.real_W)
W_hat = self.kernel.sample_W()
self.kernel.update_params(
W=W_hat,
s=self.kernel.inner_kernel.variance,
l=self.kernel.inner_kernel.lengthscale
)
W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts(self.kernel, self.X, self.Y)
# TODO: Check if these values are attained over multiple iterations (check if assert otherwise fails)
# Create the gp_regression function and pass in the predictor function as f_hat
self.kernel.update_params(W=W_hat, l=l, s=s)
gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn)
y = self.function.f( np.dot(X, self.real_W).T )
if self.PLOT_MEAN:
y_hat = gp_reg.predict(X)[0].squeeze()
else:
y_hat = gp_reg.predict(self.X)[0].squeeze()
#################################
# END TRAIN THE W_OPTIMIZER #
#################################
fig = plt.figure()
ax = Axes3D(fig)
# First plot the real function
ax.scatter(X[:,0], X[:, 1], y, s=1)
if self.PLOT_MEAN:
ax.scatter(X[:, 0], X[:, 1], y_hat, cmap=plt.cm.jet)
else:
ax.scatter(self.X[:,0], self.X[:, 1], y_hat, cmap=plt.cm.jet)
# Save the W just in case
l = loss(
self.kernel,
W_hat,
sn,
s,
l,
self.X,
self.Y
)
fig.savefig('./pics-twostep/BestLoss_' + str(l) + '.png', )
plt.show()
plt.close(fig)
np.savetxt("./pics-twostep/BestLoss_" + str(l) + ".txt", W_hat)