def __init__(self): self.real_dim = 2 self.active_dim = 1 self.no_samples = 5 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) # Parameters self.sn = 2. self.W = self.kernel.sample_W() self.function = Parabola() self.real_W = np.asarray([ [1], [1] ]) self.real_W = self.real_W / np.linalg.norm(self.real_W) self.X = np.random.rand(self.no_samples, self.real_dim) Z = np.dot(self.X, self.real_W).reshape((-1, 1)) self.Y = self.function.f(Z.T).squeeze() self.w_optimizer = t_WOptimizer( self.kernel, # TODO: does the kernel take over the W? self.sn, np.asscalar(self.kernel.inner_kernel.variance), self.kernel.inner_kernel.lengthscale, self.X, self.Y ) # Define the plotting variables self.tau_arr = np.linspace(0., self.w_optimizer.tau_max, 100)
def __init__(self): self.real_dim = 2 self.active_dim = 1 self.no_samples = 100 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) # Parameters self.sn = 0.1 # 1e-7 # 0.1 self.W = self.kernel.sample_W() self.function = AugmentedSinusoidal() self.real_W = np.asarray([ [3], [1] ]) self.real_W = self.real_W / np.linalg.norm(self.real_W) # [[0.9486833] # [0.31622777]] x_range = np.linspace(0., 1., int(np.sqrt(self.no_samples))) y_range = np.linspace(0., 1., int(np.sqrt(self.no_samples))) self.X = cartesian([x_range, y_range]) #self.X = np.random.rand(self.no_samples, self.real_dim) print(self.X.shape) Z = np.dot(self.X, self.real_W).reshape(-1, 1) print(Z.shape) self.Y = self.function.f(Z.T).reshape(-1, 1) self.optimizer = TripathyOptimizer() self.no_tries = 1 self.PLOT_MEAN = True
def __init__(self): self.real_dim = 2 self.active_dim = 1 self.no_samples = 50 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) # Parameters self.sn = 0.1 self.W = self.kernel.sample_W() self.function = Parabola() self.real_W = np.asarray([ [1], [1] ]) self.real_W = self.real_W / np.linalg.norm(self.real_W) self.X = np.random.rand(self.no_samples, self.real_dim) Z = np.dot(self.X, self.real_W) self.Y = self.function.f(Z.T).reshape(-1, 1) self.w_optimizer = t_WOptimizer( self.kernel, # TODO: does the kernel take over the W? self.sn, np.asscalar(self.kernel.inner_kernel.variance), self.kernel.inner_kernel.lengthscale, self.X, self.Y ) self.no_tries = 1000
def init(self): self.real_dim = 3 self.active_dim = 2 self.no_samples = 20 # 50 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) # Parameters self.sn = 0.1 self.W = self.kernel.sample_W() self.function = Camelback() self.real_W = np.asarray([ [0, 1], [1, 0], [0, 0] ]) self.real_W = self.real_W / np.linalg.norm(self.real_W) # [[0.9486833] # [0.31622777]] self.X = np.random.rand(self.no_samples, self.real_dim) print(self.X.shape) Z = np.dot(self.X, self.real_W) print(Z.shape) self.Y = self.function.f(Z.T).reshape(-1, 1) self.no_tries = 2
def __init__(self): self.real_dim = 5 self.active_dim = 1 self.no_samples = 20 # This should be proportional to the number of dimensions self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) self.function = PolynomialKernel() # Change the function # Generate the positive semi-definite matrix self.real_W = np.random.random((self.real_dim, self.active_dim)) # Take the real W to be completely random # assert np.isclose( np.dot(self.real_W.T, self.real_W), np.eye(self.active_dim) ) self.X = np.random.rand(self.no_samples, self.real_dim) #self.X = np.random.rand(self.no_samples, self.real_dim) print(self.X.shape) Z = np.dot(self.X, self.real_W).reshape(-1, self.active_dim) print("The projected input matrix: ", Z.shape) print(Z.shape) self.Y = self.function.f(Z.T).reshape(-1, 1) print("Shapes of X and Y: ", (self.X.shape, self.Y.shape) ) self.optimizer = TripathyOptimizer()
def set_new_kernel(self, d, W=None, variance=None, lengthscale=None): self.kernel = TripathyMaternKernel( real_dim=self.domain.d, active_dim=d, W=W, variance=variance, lengthscale=lengthscale )
def init(self): self.real_dim = 2 self.active_dim = 1 self.no_samples = 75 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) # Hide the matrix over here! if self.real_dim == 3 and self.active_dim == 2: self.function = Camelback() self.real_W = np.asarray([ [0, 1], [1, 0], [0, 0] ]) elif self.real_dim == 2 and self.active_dim == 1: self.function = Parabola() self.real_W = np.asarray([ [1], [1], ]) self.real_W = self.real_W / np.linalg.norm(self.real_W) else: assert False, "W was not set!" self.sn = 0.1 self.X = np.random.rand(self.no_samples, self.real_dim) Z = np.dot(self.X, self.real_W) self.Y = self.function.f(Z.T).reshape(-1, 1) self.w_optimizer = t_WOptimizer( self.kernel, self.sn, np.asscalar(self.kernel.inner_kernel.variance), self.kernel.inner_kernel.lengthscale, self.X, self.Y ) # We create the following kernel just to have access to the sample_W function! # TripathyMaternKernel(self.real_dim) self.tries = 10 self.max_iter = 1 # 150 assert False self.metrics = Metrics(self.no_samples)
class TestAProcesses(object): def init(self): self.real_dim = 3 self.active_dim = 2 self.no_samples = 5 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) self.W = self.kernel.sample_W() self.sn = 2. self.function = Rosenbrock() self.real_W = np.asarray([[0, 0], [0, 1], [1, 0]]) self.function = Rosenbrock() self.X = np.random.rand(self.no_samples, self.real_dim) Z = np.dot(self.X, self.real_W) self.Y = self.function.f(Z.T) self.w_optimizer = t_WOptimizer( self.kernel, self.sn, np.asscalar(self.kernel.inner_kernel.variance), self.kernel.inner_kernel.lengthscale, self.X, self.Y) def test__A_returns_correct_values(self): """ Calculate this example by hand :return: """ pass def test__gamma_returns_correct_values(self): """ Calculate this example by hand :return: """ pass
def run_optimization(X, Y, Y_noise, input_d): # TODO: Y_noise is notuuse! assert X.shape[1] == input_d, (X.shape, input_d) # Run the optimizer optimizer = TripathyOptimizer() W_hat, sn, l, s, d = optimizer.find_active_subspace(X, Y) # Spawn new kernel kernel = TripathyMaternKernel( real_dim=input_d, active_dim=d, W=W_hat, variance=s, lengthscale=l ) # Spawn new GPRegression object gpreg = GPRegression( input_dim=input_d, kernel=kernel, noise_var=sn, calculate_gradients=True ) # Return the optimized gpreg object return gpreg
def init(self): self.no_samples = 20 self.function = Parabola() self.real_dim = 2 self.active_dim = 1 self.function._set_dimension(self.active_dim) self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) # Hide the matrix over here! # self.real_W = np.asarray([ # [0, 0], # [0, 1], # [1, 0] # ]) self.real_W = np.asarray([[1], [1]]) self.real_W = self.real_W / np.linalg.norm(self.real_W) self.X = np.random.rand(self.no_samples, self.real_dim) Z = np.dot(self.X, self.real_W) assert Z.shape == (self.no_samples, self.active_dim) self.Y = self.function.f(Z.T).reshape(self.no_samples, 1) #assert self.Y.shape == (self.no_samples,) self.sn = 0.8
def visualize_angle_loss(): # Training parameters NUM_TRAINING_POINTS = 100 fnc_config = {"noise_var": 0.01} # Optimizer parameters local_config = {"M_l": 100, "m": 300, "n": 300, "losses": [], "leps": 1e-6} # local_config = { # "M_l": 10, # "m": 10, # "n": 10, # "losses": [], # "leps": 1e-6 # } class dotdict(dict): """dot.notation access to dictionary attributes""" __getattr__ = dict.get __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ local_config = dotdict(local_config) for name, fnc, active_d in FNC_TUPLES: # Generate training points X_train = (np.random.rand(NUM_TRAINING_POINTS, fnc.d) * fnc._domain.range) + ((fnc._domain.u + fnc._domain.l) / 2.) Y_train = fnc.f(X_train.T).reshape(-1, 1) # Generate the kernel t_kernel = TripathyMaternKernel(real_dim=fnc.d, active_dim=active_d, W=None, variance=None, lengthscale=None) # Run the two-step-optimization on the respective function # Retrieve intermediate matrices from there all_Ws = run_two_step_optimization(local_config, t_kernel=t_kernel, sn=fnc_config['noise_var'], X=X_train, Y=Y_train, save_Ws=True) print(all_Ws) losses = [ loss(t_kernel, W, sn, s, l * np.ones((W.shape[1], )), X_train, Y_train) for W, sn, s, l in all_Ws ] all_Ws = [x[0] for x in all_Ws] print("All losses are: ", local_config.losses) print("All are: ", losses) # Check if the loss decreases after we receive the individual parameters visualize_angle_given_W_array(fnc.W.T, all_Ws, name)
def init(self): self.real_dim = 3 self.active_dim = 2 self.no_samples = 5 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) self.W = self.kernel.sample_W() self.sn = 2. self.function = Rosenbrock() self.real_W = np.asarray([[0, 0], [0, 1], [1, 0]]) self.function = Rosenbrock() self.X = np.random.rand(self.no_samples, self.real_dim) Z = np.dot(self.X, self.real_W) self.Y = self.function.f(Z.T) self.w_optimizer = t_WOptimizer( self.kernel, self.sn, np.asscalar(self.kernel.inner_kernel.variance), self.kernel.inner_kernel.lengthscale, self.X, self.Y)
def init(self): self.real_dim = 3 self.active_dim = 2 self.no_samples = 5 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) self.real_sn = 0.01 self.real_s = 0.1 self.real_l = 0.1 * np.ones(self.active_dim) self.fix_W = self.kernel.sample_W() self.function = Rosenbrock() self.fix_W = np.asarray([[0, 0], [0, 1], [1, 0]]) self.function = Rosenbrock() self.X = np.random.rand(self.no_samples, self.real_dim) Z = np.dot(self.X, self.fix_W) self.Y = self.function.f(Z.T).reshape(-1, 1) print("Shape are: ", (self.X.shape, self.Y.shape)) self.parameter_optimizer = t_ParameterOptimizer( self.fix_W, self.kernel, self.X, self.Y.T)
class TestIndividualFunctions(object): def init(self): self.real_dim = 3 self.active_dim = 2 self.no_samples = 5 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) self.sn = 2. self.W = self.kernel.sample_W() self.function = Rosenbrock() self.real_W = np.asarray([[0, 0], [0, 1], [1, 0]]) self.function = Rosenbrock() self.X = np.random.rand(self.no_samples, self.real_dim) Z = np.dot(self.X, self.real_W) self.Y = self.function.f(Z.T) self.w_optimizer = t_WOptimizer( self.kernel, self.sn, np.asscalar(self.kernel.inner_kernel.variance), self.kernel.inner_kernel.lengthscale, self.X, self.Y) def test__A_fnc(self): self.init() res = self.w_optimizer._A(self.W) # TODO: check what dimension this is supposed to have! assert res.shape == (self.real_dim, self.real_dim) def test_gamma_returns_correct_dimensions(self): self.init() res = self.w_optimizer._gamma(1e-3, self.W) assert res.shape == (self.W.shape[0], self.W.shape[1])
def init(self): self.real_dim = 3 self.active_dim = 2 self.no_samples = 5 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim)
class TestKernelSematics(object): # To calculate each element by hand, type in the following into wolfram alpha # f(x)=(1+sqrt(3)*x)exp(−sqrt(3)*x) # And calculate each r individually # def init(self): self.real_dim = 3 self.active_dim = 2 self.no_samples = 5 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) def test_kernel_identity_W_zero_inp(self): self.init() X = np.asarray([[0, 0, 0], [0, 0, 0]]) W = np.asarray([[1, 0], [0, 1], [0, 0]]) self.kernel.update_params(W=W, l=np.asarray( [1. for i in range(self.active_dim)]), s=1.) self.real_kernel = Matern32( self.active_dim, ARD=True, lengthscale=self.kernel.inner_kernel.lengthscale) y_hat = self.kernel.K(X) y = np.asarray([[1, 1], [1, 1]]) assert np.isclose(y, y_hat, rtol=1e-16).all() def test_kernel_reverted_W(self): self.init() X = np.asarray([[0, 0.5, 0], [0, 0, 0.5]]) W = np.asarray([[0, 1], [0, 0], [1, 0]]) # After projection, we get # X_new = [ # [0, 0], # [0.5, 0] # ] # r = 0.25 self.kernel.update_params(W=W, l=np.asarray( [1. for i in range(self.active_dim)]), s=1.) y_hat = self.kernel.K(X) y = np.asarray([[1, 0.784888], [0.784888, 1]]) assert np.isclose(y, y_hat, rtol=1e-4).all() def test_kernel_some_random_W(self): self.init() for i in range(100): X = np.random.rand(5, self.real_dim) # Sample and re-assign # TODO: just let the kernel resample all parameters W = self.kernel.sample_W() s = self.kernel.sample_variance() l = self.kernel.sample_lengthscale() self.kernel.update_params(W=W, l=l, s=s) y_hat = self.kernel.K(X) y = self.kernel.inner_kernel.K(np.dot(X, W)) assert np.isclose(y, y_hat).all() def test_kernel_some_random_W_independent_inner_kernel(self): self.init() for i in range(100): X = np.random.rand(5, self.real_dim) # Sample and re-assign # TODO: change this by just resampling using a function within the kernel W = self.kernel.sample_W() s = self.kernel.sample_variance() l = self.kernel.sample_lengthscale() self.kernel.update_params(W=W, l=l, s=s) y_hat = self.kernel.K(X) # Define the new kernel real_kernel = Matern32(self.active_dim, variance=s, ARD=True, lengthscale=l) y = real_kernel.K(np.dot(X, W)) assert np.isclose(y, y_hat).all()
class TestKernel(object): def init(self): self.real_dim = 3 self.active_dim = 2 self.no_samples = 5 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) def test_parameters_are_set_successfully(self): """ Check if parameters are set successfully / setters work correctly :return: """ self.init() W1, l1, s1 = self.kernel.W, self.kernel.inner_kernel.lengthscale, self.kernel.inner_kernel.variance W1 = W1.copy() l1 = l1.copy() s1 = s1.copy() new_W = np.zeros((self.real_dim, self.active_dim), dtype=np.float64) for i in range(self.real_dim): for j in range(self.active_dim): new_W[i, j] = np.random.normal(0, 1) Q, R = np.linalg.qr(new_W) # Set new parameters self.kernel.update_params(W=Q, l=np.random.rand(self.active_dim, ), s=5.22) assert not np.isclose(np.asarray(self.kernel.inner_kernel.lengthscale), np.asarray(l1)).all() assert not np.isclose(np.asarray(self.kernel.inner_kernel.variance), np.asarray(s1)) assert not np.isclose(self.kernel.W, W1).all() def test_kernel_returns_gram_matrix_correct_shape(self): """ Check :return: """ self.init() A = np.random.rand(self.no_samples, self.real_dim) B = np.random.rand(self.no_samples, self.real_dim) # print("Before we go into the function: ") # print(A) # print(B) Cov = self.kernel.K(A, B) assert Cov.shape == (self.no_samples, self.no_samples) def test_kernel_returns_diag_correct_shape(self): self.init() A = np.random.rand(self.no_samples, self.real_dim) # print("Before we go into the function Kdiag: ") # print(A) Kdiag = self.kernel.Kdiag(A) assert Kdiag.shape == (self.no_samples, ), (Kdiag.shape, ) def test_kernel_K_of_r_words_for_vectors(self): self.init() x = np.random.rand(self.no_samples) # print("Before we go into the function Kdiag: ") # print(x) kr = self.kernel.K_of_r(x) assert kr.shape == (self.no_samples, ), (kr.shape, )
class TestParameterOptimization(object): def init(self): self.real_dim = 3 self.active_dim = 2 self.no_samples = 5 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) self.real_sn = 0.01 self.real_s = 0.1 self.real_l = 0.1 * np.ones(self.active_dim) self.fix_W = self.kernel.sample_W() self.function = Rosenbrock() self.fix_W = np.asarray([[0, 0], [0, 1], [1, 0]]) self.function = Rosenbrock() self.X = np.random.rand(self.no_samples, self.real_dim) Z = np.dot(self.X, self.fix_W) self.Y = self.function.f(Z.T).reshape(-1, 1) print("Shape are: ", (self.X.shape, self.Y.shape)) self.parameter_optimizer = t_ParameterOptimizer( self.fix_W, self.kernel, self.X, self.Y.T) def test_parameter_opt_does_not_err(self): self.init() # Have some initial guesses for sn, s, l s = float(np.random.rand(1)) sn = float(np.random.rand(1)) l = np.random.rand(self.active_dim) new_s, new_l, new_sn = self.parameter_optimizer.optimize_s_sn_l( sn, s, l) print(s, new_s) print(sn, new_sn) print(l, new_l) assert not np.isclose(new_s, s), (new_s, s) assert not np.isclose(new_l, l).all(), (new_l, l) assert not np.isclose(new_sn, sn), (new_sn, sn) def test_parameter_optimizes_loss(self): self.init() s_init = float(np.random.rand(1)) sn_init = float(np.random.rand(1)) l_init = np.random.rand(self.active_dim) old_loss = loss(self.kernel, self.fix_W, sn_init, s_init, l_init, self.X, self.Y) new_s, new_l, new_sn = self.parameter_optimizer.optimize_s_sn_l( sn_init, s_init, l_init) new_loss = loss(self.kernel, self.fix_W, new_sn, new_s, new_l, self.X, self.Y) # print("Old loss, new loss ", (old_loss, new_loss)) # TODO: Should this be a new smaller value, or a value toward zero, or a new bigger value? # assert new_loss <= old_loss assert new_loss != old_loss def test_parameters_change(self): self.init() s_init = float(np.random.rand(1)) sn_init = float(np.random.rand(1)) l_init = np.random.rand(self.active_dim) old_loss = loss(self.kernel, self.fix_W, sn_init, s_init, l_init, self.X, self.Y) new_s, new_l, new_sn = self.parameter_optimizer.optimize_s_sn_l( sn_init, s_init, l_init) assert s_init != new_s, (s_init, new_s) assert not np.isclose(l_init, new_l).all(), (l_init, new_l) assert sn_init != new_sn, (sn_init, new_sn)
def visualize_angle_loss(): NUM_TRIES = 15 # 50 # Training parameters NUM_TRAINING_POINTS = 100 fnc_config = {"noise_var": 0.005} # Optimizer parameters # Following is ok for parabola # local_config = { # "M_l": 100, # "m": 1, # "n": 1, # "losses": [], # "leps": 1e-16 # } # Following is ok for camelback local_config = {"M_l": 100, "m": 1, "n": 1, "losses": [], "leps": 1e-13} # local_config = { # "M_l": 100, # "m": 300, # "n": 300, # "losses": [], # "leps": 1e-12 # } # local_config = { # "M_l": 10, # "m": 10, # "n": 10, # "losses": [], # "leps": 1e-6 # } class dotdict(dict): """dot.notation access to dictionary attributes""" __getattr__ = dict.get __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ local_config = dotdict(local_config) for name, fnc, active_d in FNC_TUPLES: if name == "Parabola-2D->1D": config['active_dimension'] = 1 print("Active dimensions are at: ", config['active_dimension']) else: config['active_dimension'] = 2 all_angles = [] all_losses = [] title = name + "_" + str(NUM_TRIES) + "_" + str(random.randint( 1, 1000)) # Run for a few times for n in range(NUM_TRIES): print("Now at step: ", n) # Generate training points X_train = (np.random.rand(NUM_TRAINING_POINTS, fnc.d) * fnc._domain.range) + ( (fnc._domain.u + fnc._domain.l) / 2.) Y_train = fnc.f(X_train.T).reshape(-1, 1) # Generate the kernel t_kernel = TripathyMaternKernel(real_dim=fnc.d, active_dim=active_d, W=None, variance=None, lengthscale=None) # Run the two-step-optimization on the respective function # Retrieve intermediate matrices from there all_Ws, best_config = run_two_step_optimization( local_config, t_kernel=t_kernel, sn=fnc_config['noise_var'], X=X_train, Y=Y_train, save_Ws=True, save_best_config=True) print(all_Ws) losses = [ loss(t_kernel, W, sn, s, l * np.ones((W.shape[1], )), X_train, Y_train) for W, sn, l, s in all_Ws ] all_Ws = [x[0] for x in all_Ws] print("All losses are: ", local_config.losses) print("All are: ", losses) # Calculate the angles angles = list( map(lambda x: calculate_angle_between_two_matrices(fnc.W.T, x), all_Ws)) all_angles.append(angles) # Creates a 2d array all_losses.append(losses) # Creates a 2d array # Check if the loss decreases after we receive the individual parameters # Pad the losses and arrays to the maximum size of the runs all_angles = pad_2d_list(all_angles) all_losses = pad_2d_list(all_losses) print(all_angles.shape) print(all_losses.shape) visualize_angle_array_stddev(all_angles, title=title) visualize_loss_array_stddev(all_losses, title=title) visualize_loss_array_stddev(all_losses, title=title, subtract_mean=True) # TODO: take max index for log-likelihood, and visualize angle and log-likelihood print("Retrieving array from: ", all_losses[:, -1].reshape(-1)) max_index = np.argmax(all_losses[:, -1].reshape(-1)) print("Best index is: ", max_index) print("Best found loss is: ") visualize_angle_array_stddev(all_angles, title=title, max_index=max_index) visualize_loss_array_stddev(all_losses, title=title, max_index=max_index) print("Done...")
class TestMatrixRecoveryHighDegreePolynomial: def __init__(self): self.real_dim = 5 self.active_dim = 1 self.no_samples = 20 # This should be proportional to the number of dimensions self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) self.function = PolynomialKernel() # Change the function # Generate the positive semi-definite matrix self.real_W = np.random.random((self.real_dim, self.active_dim)) # Take the real W to be completely random # assert np.isclose( np.dot(self.real_W.T, self.real_W), np.eye(self.active_dim) ) self.X = np.random.rand(self.no_samples, self.real_dim) #self.X = np.random.rand(self.no_samples, self.real_dim) print(self.X.shape) Z = np.dot(self.X, self.real_W).reshape(-1, self.active_dim) print("The projected input matrix: ", Z.shape) print(Z.shape) self.Y = self.function.f(Z.T).reshape(-1, 1) print("Shapes of X and Y: ", (self.X.shape, self.Y.shape) ) self.optimizer = TripathyOptimizer() def check_if_matrix_is_found(self): import os if not os.path.exists("./highD/"): os.makedirs("./highD/") ################################# # TRAIN THE W_OPTIMIZER # ################################# Opt = TripathyOptimizer() # print("Real hidden matrix is: ", self.real_W) W_hat = self.kernel.sample_W() self.kernel.update_params( W=W_hat, s=self.kernel.inner_kernel.variance, l=self.kernel.inner_kernel.lengthscale ) W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts(self.kernel, self.X, self.Y) # Create the gp_regression function and pass in the predictor function as f_hat self.kernel.update_params(W=W_hat, l=l, s=s) ################################# # END TRAIN THE W_OPTIMIZER # ################################# # Save the W just in case l = loss( self.kernel, W_hat, sn, s, l, self.X, self.Y ) np.savetxt("./highD/" + str(l) + "_BestLoss.txt", W_hat) np.savetxt("./highD/" + str(l) + "_realMatr.txt", self.real_W)
class VisualizedTestingTau: def __init__(self): self.real_dim = 2 self.active_dim = 1 self.no_samples = 5 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) # Parameters self.sn = 2. self.W = self.kernel.sample_W() self.function = Parabola() self.real_W = np.asarray([ [1], [1] ]) self.real_W = self.real_W / np.linalg.norm(self.real_W) self.X = np.random.rand(self.no_samples, self.real_dim) Z = np.dot(self.X, self.real_W).reshape((-1, 1)) self.Y = self.function.f(Z.T).squeeze() self.w_optimizer = t_WOptimizer( self.kernel, # TODO: does the kernel take over the W? self.sn, np.asscalar(self.kernel.inner_kernel.variance), self.kernel.inner_kernel.lengthscale, self.X, self.Y ) # Define the plotting variables self.tau_arr = np.linspace(0., self.w_optimizer.tau_max, 100) def visualize_tau_trajectory_for_random_W(self): """ Visualize the trajectory of the gamma function against the loss function we have f(tau) = F( gamma(tau, W) ) :return: """ loss_arr = [] # Sample a random W W_init = self.kernel.sample_W() for tau in self.tau_arr: print("New tau is: ", tau) W = self.w_optimizer._gamma(tau, W_init) loss_val = loss( self.kernel, W, self.sn, self.kernel.inner_kernel.variance, self.kernel.inner_kernel.lengthscale, self.X, self.Y.squeeze() ) loss_arr.append(loss_val) print(loss_arr) plt.title("Tau vs Loss - Randomly sampled W") plt.scatter(self.tau_arr, loss_arr) plt.axis([min(self.tau_arr), max(self.tau_arr), min(loss_arr), max(loss_arr)]) plt.show() def visualize_tau_trajectory_for_identity_W(self): """ Visualize the trajectory of the gamma function against the loss function we have f(tau) = F( gamma(tau, W) ) :return: """ loss_arr = [] # Sample a random W W_init = self.real_W for tau in self.tau_arr: print("New tau is: ", tau) W = self.w_optimizer._gamma(tau, W_init) loss_val = loss( self.kernel, W, self.sn, self.kernel.inner_kernel.variance, self.kernel.inner_kernel.lengthscale, self.X, self.Y.squeeze() ) loss_arr.append(loss_val) print(loss_arr) plt.title("Tau vs Loss - Identity-similarsampled W") plt.scatter(self.tau_arr, loss_arr) plt.axis([min(self.tau_arr), max(self.tau_arr), min(loss_arr), max(loss_arr)]) plt.show()
class TestMatrixRecoveryNaive(object): def __init__(self): pass # This skips the test def init(self): self.real_dim = 3 self.active_dim = 2 self.no_samples = 20 # 50 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) # Parameters self.sn = 0.1 self.W = self.kernel.sample_W() self.function = Camelback() self.real_W = np.asarray([ [0, 1], [1, 0], [0, 0] ]) self.real_W = self.real_W / np.linalg.norm(self.real_W) # [[0.9486833] # [0.31622777]] self.X = np.random.rand(self.no_samples, self.real_dim) print(self.X.shape) Z = np.dot(self.X, self.real_W) print(Z.shape) self.Y = self.function.f(Z.T).reshape(-1, 1) self.no_tries = 2 def test_visualize_augmented_sinusoidal_function(self): self.init() import os if not os.path.exists("./pics/camelback/"): os.makedirs("./pics/") os.makedirs("./pics/camelback/") ################################# # TRAIN THE W_OPTIMIZER # ################################# Opt = TripathyOptimizer() print("Real hidden matrix is: ", self.real_W) for j in range(self.no_tries): print("Try number : ", j) W_hat = self.kernel.sample_W() self.kernel.update_params( W=W_hat, s=self.kernel.inner_kernel.variance, l=self.kernel.inner_kernel.lengthscale ) W_hat, sn, l, s = Opt.run_two_step_optimization(self.kernel, self.sn, self.X, self.Y) # Create the gp_regression function and pass in the predictor function as f_hat self.kernel.update_params(W=W_hat, l=l, s=s) gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn) y_hat = gp_reg.predict(self.X)[0].squeeze() ################################# # END TRAIN THE W_OPTIMIZER # ################################# # Save the W just in case l = loss( self.kernel, W_hat, sn, s, l, self.X, self.Y ) np.savetxt("./pics/camelback/Iter_" + str(j) + "__" + "Loss_" + str(l) + ".txt", W_hat)
class VisualizedTestingWParabola: def __init__(self): self.real_dim = 2 self.active_dim = 1 self.no_samples = 50 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) # Parameters self.sn = 0.1 self.W = self.kernel.sample_W() self.function = Parabola() self.real_W = np.asarray([ [1], [1] ]) self.real_W = self.real_W / np.linalg.norm(self.real_W) self.X = np.random.rand(self.no_samples, self.real_dim) Z = np.dot(self.X, self.real_W) self.Y = self.function.f(Z.T).reshape(-1, 1) self.w_optimizer = t_WOptimizer( self.kernel, # TODO: does the kernel take over the W? self.sn, np.asscalar(self.kernel.inner_kernel.variance), self.kernel.inner_kernel.lengthscale, self.X, self.Y ) self.no_tries = 1000 def visualize_quadratic_function(self): x_range = np.linspace(0., 1., 80) y_range = np.linspace(0., 1., 80) X = cartesian([x_range, y_range]) import os if not os.path.exists("./pics/"): os.makedirs("./pics/") ################################# # TRAIN THE W_OPTIMIZER # ################################# Opt = TripathyOptimizer() for j in range(self.no_tries): print("Try number : ", j) W_hat = self.kernel.sample_W() self.kernel.update_params( W=W_hat, s=self.kernel.inner_kernel.variance, l=self.kernel.inner_kernel.lengthscale ) W_hat, sn, l, s = Opt.run_two_step_optimization(self.kernel, self.sn, self.X, self.Y) # Create the gp_regression function and pass in the predictor function as f_hat self.kernel.update_params(W=W_hat, l=l, s=s) gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn) y = self.function.f( np.dot(X, self.real_W).T ) y_hat = gp_reg.predict(self.X)[0].squeeze() ################################# # END TRAIN THE W_OPTIMIZER # ################################# fig = plt.figure() ax = Axes3D(fig) # First plot the real function ax.scatter(X[:,0], X[:, 1], y, s=1) ax.scatter(self.X[:,0], self.X[:, 1], y_hat, cmap=plt.cm.jet) fig.savefig('./pics/Iter_' + str(j) + '.png', ) # plt.show() plt.close(fig) # Save the W just in case l = loss( self.kernel, W_hat, sn, s, l, self.X, self.Y ) np.savetxt("./pics/Iter_" + str(j) + "__" + "Loss_" + str(l) + ".txt", W_hat)
class TestMatrixRecovery(object): """ We hide a function depending on a matrix A within a higher dimension. We then test if our algorithm can successfully approximate/find this matrix (A_hat is the approximated one). More specifically, check if: f(A x) = f(A_hat x) """ def init(self): self.real_dim = 2 self.active_dim = 1 self.no_samples = 75 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) # Hide the matrix over here! if self.real_dim == 3 and self.active_dim == 2: self.function = Camelback() self.real_W = np.asarray([ [0, 1], [1, 0], [0, 0] ]) elif self.real_dim == 2 and self.active_dim == 1: self.function = Parabola() self.real_W = np.asarray([ [1], [1], ]) self.real_W = self.real_W / np.linalg.norm(self.real_W) else: assert False, "W was not set!" self.sn = 0.1 self.X = np.random.rand(self.no_samples, self.real_dim) Z = np.dot(self.X, self.real_W) self.Y = self.function.f(Z.T).reshape(-1, 1) self.w_optimizer = t_WOptimizer( self.kernel, self.sn, np.asscalar(self.kernel.inner_kernel.variance), self.kernel.inner_kernel.lengthscale, self.X, self.Y ) # We create the following kernel just to have access to the sample_W function! # TripathyMaternKernel(self.real_dim) self.tries = 10 self.max_iter = 1 # 150 assert False self.metrics = Metrics(self.no_samples) def test_if_function_is_found(self): """ Replace these tests by the actual optimizer function! :return: """ self.init() print("Real matrix is: ", self.real_W) all_tries = [] for i in range(self.tries): # Initialize random guess W_hat = self.kernel.sample_W() # Find a good W! for i in range(self.max_iter): W_hat = self.w_optimizer.optimize_stiefel_manifold(W_hat) print("Difference to real W is: ", (W_hat - self.real_W)) assert W_hat.shape == self.real_W.shape self.kernel.update_params( W=W_hat, l=self.kernel.inner_kernel.lengthscale, s=self.kernel.inner_kernel.variance ) # TODO: update the gaussian process with the new kernels parameters! (i.e. W_hat) # Create the gp_regression function and pass in the predictor function as f_hat gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=self.sn) res = self.metrics.mean_difference_points( fnc=self.function._f, fnc_hat=gp_reg.predict, A=self.real_W, A_hat=W_hat, X=self.X ) all_tries.append(res) print(all_tries) assert np.asarray(all_tries).any() def test_if_hidden_matrix_is_found_multiple_initializations(self): self.init() print("Real matrix is: ", self.real_W) all_tries = [] for i in range(self.tries): # Initialize random guess W_hat = self.kernel.sample_W() # Find a good W! for i in range(self.max_iter): W_hat = self.w_optimizer.optimize_stiefel_manifold(W_hat) print("Difference to real (AA.T) W is: ", (W_hat - self.real_W)) assert W_hat.shape == self.real_W.shape assert not (W_hat == self.real_W).all() res = self.metrics.projects_into_same_original_point(self.real_W, W_hat) all_tries.append(res) assert True in all_tries
def setup_approximation(self): self.g_input_dim = 5 self.kernel = TripathyMaternKernel(self.g_input_dim, self.active_dim)
class TripathyGP(ConfidenceBoundModel): """ Base class for GP optimization. Handles common functionality. """ def set_new_kernel(self, d, W=None, variance=None, lengthscale=None): self.kernel = TripathyMaternKernel( real_dim=self.domain.d, active_dim=d, W=W, variance=variance, lengthscale=lengthscale ) def set_new_gp(self, noise_var=None): self.gp = GPRegression( input_dim=self.domain.d, kernel=self.kernel, noise_var=noise_var if noise_var else 2., # self.config.noise_var, calculate_gradients=self.config.calculate_gradients ) def set_new_gp_and_kernel(self, d, W, variance, lengthscale, noise_var): self.set_new_kernel(d, W, variance, lengthscale) self.set_new_gp(noise_var) def __init__(self, domain, calculate_always=False): super(TripathyGP, self).__init__(domain) self.optimizer = TripathyOptimizer() # TODO: d is chosen to be an arbitrary value rn! self.set_new_gp_and_kernel(2, None, None, None, None) # number of data points self.t = 0 self.kernel = self.kernel.copy() self._woodbury_chol = np.asfortranarray(self.gp.posterior._woodbury_chol) # we create a copy of the matrix in fortranarray, such that we can directly pass it to lapack dtrtrs without doing another copy self._woodbury_vector = self.gp.posterior._woodbury_vector.copy() self._X = self.gp.X.copy() self._Y = np.empty(shape=(0,1)) self._beta = 2 self._bias = self.config.bias self.calculate_always = calculate_always @property def beta(self): return self._beta @property def scale(self): if self.gp.kern.name == 'sum': return sum([part.variance for part in self.gp.kern.parts]) else: return np.sqrt(self.gp.kern.variance) @property def bias(self): return self._bias def _get_gp(self): return GPRegression(self.domain.d, self.kernel, noise_var=self.config.noise_var, calculate_gradients=self.config.calculate_gradients) def add_data(self, x, y): """ Add a new function observation to the GPs. Parameters ---------- x: 2d-array y: 2d-array """ self.i = 1 if not ("i" in dir(self)) else self.i + 1 # print("Add data ", self.i) x = np.atleast_2d(x) y = np.atleast_2d(y) self.set_data(x, y, append=True) # TODO: check if this is called anyhow! def optimize(self): self._update_beta() def _update_cache(self): # if not self.config.calculate_gradients: self._woodbury_chol = np.asfortranarray(self.gp.posterior._woodbury_chol) self._woodbury_vector = self.gp.posterior._woodbury_vector.copy() self._X = self.gp.X.copy() self._update_beta() def _optimize_bias(self): self._bias = minimize(self._bias_loss, self._bias, method='L-BFGS-B')['x'].copy() self._set_bias(self._bias) logger.info(f"Updated bias to {self._bias}") def _bias_loss(self, c): # calculate mean and norm for new bias via a new woodbury_vector new_woodbury_vector,_= dpotrs(self._woodbury_chol, self._Y - c, lower=1) K = self.gp.kern.K(self.gp.X) mean = np.dot(K, new_woodbury_vector) norm = new_woodbury_vector.T.dot(mean) # loss is least_squares_error + norm return np.asscalar(np.sum(np.square(mean + c - self._Y)) + norm) def _set_bias(self, c): self._bias = c self.gp.set_Y(self._Y - c) self._woodbury_vector = self.gp.posterior._woodbury_vector.copy() def _update_beta(self): logdet = self._get_logdet() logdet_priornoise = self._get_logdet_prior_noise() self._beta = np.sqrt(2 * np.log(1/self.delta) + (logdet - logdet_priornoise)) + self._norm() def _optimize_var(self): # fix all parameters for p in self.gp.parameters: p.fix() if self.gp.kern.name == 'sum': for part in self.gp.kern.parts: part.variance.unfix() else: self.gp.kern.variance.unfix() self.gp.optimize() if self.gp.kern.name == 'sum': values = [] for part in self.gp.kern.parts: values.append(np.asscalar(part.variance.values)) else: values = np.asscalar(self.gp.kern.variance.values) logger.info(f"Updated prior variance to {values}") # unfix all parameters for p in self.gp.parameters: p.unfix() def _get_logdet(self): return 2.*np.sum(np.log(np.diag(self.gp.posterior._woodbury_chol))) def _get_logdet_prior_noise(self): return self.t * np.log(self.gp.likelihood.variance.values) def mean_var(self, x): """Recompute the confidence intervals form the GP. Parameters ---------- context: ndarray Array that contains the context used to compute the sets """ x = np.atleast_2d(x) if self.config.calculate_gradients or True: mean,var = self.gp.predict_noiseless(x) else: mean,var = self._raw_predict(x) return mean + self._bias, var def mean_var_grad(self, x): return self.gp.predictive_gradients(x) def var(self, x): return self.mean_var(x)[1] def mean(self, x): return self.mean_var(x)[0] def set_data(self, X, Y, append=True): if append: X = np.concatenate((self.gp.X, X)) Y = np.concatenate((self.gp.Y, Y)) # Do our optimization now if self.i % 100 == 99 or self.calculate_always: import time start_time = time.time() print("Adding data: ", self.i) # TODO: UNCOMMENT THE FOLLOWING LINE AGAIN! # This is just to check if tripathy conforms with the other version # W_hat, sn, l, s, d = self.optimizer.find_active_subspace(X, Y) d = 2 W_hat = np.asarray([ [-0.31894555, 0.78400512, 0.38970008, 0.06119476, 0.35776912], [-0.27150973, 0.066002, 0.42761931, -0.32079484, -0.79759551] ]).T s = 1. l = 1.5 sn = 2. # 0.01 #self.config.noise_var print("--- %s seconds ---" % (time.time() - start_time)) # Overwrite GP and kernel values self.set_new_gp_and_kernel(d=d, W=W_hat, variance=s, lengthscale=l, noise_var=sn) # TODO: Should the following not come before the optimization? self.gp.set_XY(X, Y) self.t = X.shape[0] self._update_cache() def sample(self, X=None): class GPSampler: def __init__(self, X, Y, kernel, var): self.X = X self.Y = Y self.N = var * np.ones(shape=Y.shape) self.kernel = kernel self.m = GPy.models.GPHeteroscedasticRegression(self.X, self.Y, self.kernel) self.m['.*het_Gauss.variance'] = self.N def __call__(self, X): X = np.atleast_2d(X) sample = np.empty(shape=(X.shape[0], 1)) # iteratively generate sample values for all x in x_test for i, x in enumerate(X): sample[i] = self.m.posterior_samples_f(x.reshape((1, -1)), size=1) # add observation as without noise self.X = np.vstack((self.X, x)) self.Y = np.vstack((self.Y, sample[i])) self.N = np.vstack((self.N, 0)) # recalculate model self.m = GPy.models.GPHeteroscedasticRegression(self.X, self.Y, self.kernel) self.m['.*het_Gauss.variance'] = self.N # Set the noise parameters to the error in Y return sample return GPSampler(self.gp.X.copy(), self.gp.Y.copy(), self.kernel, self.gp.likelihood.variance) def _raw_predict(self, Xnew): Kx = self.kernel.K(self._X, Xnew) mu = np.dot(Kx.T, self._woodbury_vector) if len(mu.shape)==1: mu = mu.reshape(-1,1) Kxx = self.kernel.Kdiag(Xnew) tmp = lapack.dtrtrs(self._woodbury_chol, Kx, lower=1, trans=0, unitdiag=0)[0] var = (Kxx - np.square(tmp).sum(0))[:,None] return mu, var def _raw_predict_covar(self, Xnew, Xcond): Kx = self.kernel.K(self._X, np.vstack((Xnew,Xcond))) tmp = lapack.dtrtrs(self._woodbury_chol, Kx, lower=1, trans=0, unitdiag=0)[0] n = Xnew.shape[0] tmp1 = tmp[:,:n] tmp2 = tmp[:,n:] Kxx = self.kernel.K(Xnew, Xcond) var = Kxx - (tmp1.T).dot(tmp2) Kxx_new = self.kernel.Kdiag(Xnew) var_Xnew = (Kxx_new - np.square(tmp1).sum(0))[:,None] return var_Xnew, var def _norm(self): norm = self._woodbury_vector.T.dot(self.gp.kern.K(self.gp.X)).dot(self._woodbury_vector) return np.asscalar(np.sqrt(norm)) def __getstate__(self): self_dict = self.__dict__.copy() del self_dict['gp'] # remove the gp from state dict to allow pickling. calculations are done via the cache woodbury/cholesky return self_dict
class TestTauProcesses(object): def init(self): self.real_dim = 3 self.active_dim = 2 self.no_samples = 5 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) self.W = self.kernel.sample_W() self.sn = 2. self.function = Rosenbrock() self.real_W = np.asarray([[0, 0], [0, 1], [1, 0]]) self.function = Rosenbrock() self.X = np.random.rand(self.no_samples, self.real_dim) Z = np.dot(self.X, self.real_W) self.Y = self.function.f(Z.T) self.w_optimizer = t_WOptimizer( self.kernel, self.sn, np.asscalar(self.kernel.inner_kernel.variance), self.kernel.inner_kernel.lengthscale, self.X, self.Y) def test_tau_trajectory_determines_W(self): self.init() no_samples = 20 W_init = self.kernel.sample_W() all_Ws = [] tau_delta = 1e-5 #1e-4 is optimal! tau_0 = 0 for i in range(no_samples): inp_tau = min(tau_0 + i * tau_delta, self.w_optimizer.tau_max) new_W = self.w_optimizer._gamma(inp_tau, W_init) all_Ws.append(new_W) for i in range(no_samples - 1): assert ((all_Ws[i] - all_Ws[i + 1])**2).mean() >= 1e-16, str( (i, all_Ws[i], all_Ws[i + 1])) def test_tau_trajectory_determines_W_static(self): """ Not changing tau keeps W the same :return: """ self.init() no_samples = 20 tau_0 = np.random.rand(1) * self.w_optimizer.tau_max for i in range(no_samples): W_init = self.kernel.sample_W() new_W1 = self.w_optimizer._gamma(tau_0, W_init) new_W2 = self.w_optimizer._gamma(tau_0, W_init) assert (new_W1 == new_W2).all() def test__find_best_tau_finds_a_better_tau(self): # TODO: semi-flaky test! self.init() W_init = self.kernel.sample_W() init_tau = 5e-5 W_init = self.w_optimizer._gamma(init_tau, W_init) new_tau = self.w_optimizer._find_best_tau(np.copy(W_init)) new_W = self.w_optimizer._gamma(new_tau, W_init) old_loss = loss(self.kernel, W_init, self.sn, self.kernel.inner_kernel.variance, self.kernel.inner_kernel.lengthscale, self.X, self.Y) new_loss = loss(self.kernel, new_W, self.sn, self.kernel.inner_kernel.variance, self.kernel.inner_kernel.lengthscale, self.X, self.Y) if old_loss == new_loss: warnings.warn("Old loss equals new loss! The W has not improved!") print("WARNING: Old loss equals new loss! The W has not improved!") print("Changes in W!") print(W_init) print(new_W) print("Losses") print(old_loss - new_loss) # assert abs(new_loss - old_loss) > 1e-12 # TODO: This is a flaky test! assert new_loss >= old_loss # TODO: is this ok? It looks like it heavily depends on how W is initialized! def test_optimize_stiefel_manifold_doesnt_err(self): self.init() # For the sake of testing, do 10 iterations self.w_optimizer.optimize_stiefel_manifold(self.W)
class VisualizeFeatureSelection: # F dependent functions def setup_environment(self): # Environment dimensions self.f_input_dim = 2 # Environment coefficients self.a1 = -0.1 self.a2 = 0.1 # Setup the projection matrix # self.real_W = self.kernel.sample_W() # self.real_W = np.random.random((self.f_input_dim, self.active_dim)) # self.real_W = self.real_W / np.linalg.norm(self.real_W) self.real_W = np.asarray( [[5.889490030086947936e-01, 8.081701998063678394e-01]]).T def f_env(self, x, disp=False): real_phi_x = np.concatenate([ np.square(x[:, 0] - self.a1).reshape(x.shape[0], -1), np.square(x[:, 1] - self.a2).reshape(x.shape[0], -1) ], axis=1) Z = np.dot(real_phi_x, self.real_W).reshape(-1, self.active_dim) if not disp: assert Z.shape == (self.no_samples, self.active_dim), (Z.shape, (self.no_samples, self.active_dim)) # print("The projected input matrix: ", Z.shape) # print(Z.shape) # self.Y = self.function.f(Z.T).reshape(-1, 1) out = self.function.f(Z.T).reshape(-1, 1) if not disp: assert out.shape == (self.no_samples, 1) return out # G dependent function def setup_approximation(self): self.g_input_dim = 5 self.kernel = TripathyMaternKernel(self.g_input_dim, self.active_dim) def phi(self, x): assert x.shape[1] == 2 # We could just do x, but at this point it doesn't matter phi_x = np.concatenate( [np.square(x), x, np.ones((x.shape[0], )).reshape(-1, 1)], axis=1) # print("Phi x is: ", phi_x) assert phi_x.shape == (self.no_samples, self.g_input_dim) return phi_x # Part where we generate the data def generate_data(self): self.X = (np.random.rand(self.no_samples, self.init_dimension) - 0.5) * 2. # Generate the real Y self.Y = self.f_env(self.X).reshape(-1, 1) assert self.Y.shape == (self.no_samples, 1) # Generate the kernelized X to use to figure it out self.phi_X = self.phi(self.X) def __init__(self): self.function = Parabola() self.init_dimension = 2 # All the input at the beginning is always 1D! self.active_dim = self.function.d self.no_samples = 50 # 200 self.setup_environment() self.setup_approximation() self.generate_data() print("Data shape is: ") print(self.X.shape) print(self.phi_X.shape) def plot_3d(self, y_hat, title): # Plot real function x1 = np.linspace(-1, 1, 100) x2 = np.linspace(-1, 1, 100) x_real = cartesian([x1, x2]) y_real = self.f_env(x_real, disp=True) # Create the plot fig = plt.figure() ax = Axes3D(fig) ax.view_init(azim=30) # First plot the real function ax.scatter(x_real[:, 0], x_real[:, 1], y_real, 'k.', alpha=.3, s=1) ax.scatter(self.X[:, 0], self.X[:, 1], y_hat, cmap=plt.cm.jet) fig.savefig('./featureSelection/' + title + '.png') plt.show() # plt.close(fig) def check_if_matrix_is_found(self): print("Starting to optimize stuf...") import os if not os.path.exists("./featureSelection/"): os.makedirs("./featureSelection/") ################################# # TRAIN THE W_OPTIMIZER # ################################# Opt = TripathyOptimizer() print("Real hidden matrix is: ", self.real_W) # Start with the approximation of the real matrix W_hat = self.kernel.sample_W() self.kernel.update_params(W=W_hat, s=self.kernel.inner_kernel.variance, l=self.kernel.inner_kernel.lengthscale) W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts( self.kernel, self.phi_X, self.Y) # Create the gp_regression function and pass in the predictor function as f_hat self.kernel.update_params(W=W_hat, l=l, s=s) gp_reg = GPRegression(self.phi_X, self.Y, self.kernel, noise_var=sn) # Maybe predict even more values ? (plot the entire surface?) y_hat = gp_reg.predict(self.phi_X)[0].squeeze() ################################# # END TRAIN THE W_OPTIMIZER # ################################# # Save the W just in case l = loss(self.kernel, W_hat, sn, s, l, self.phi_X, self.Y) np.savetxt( config['basepath'] + "/featureSelection/" + str(l) + "_BestLoss.txt", W_hat) np.savetxt( config['basepath'] + "/featureSelection/" + str(l) + "_realMatr.txt", self.real_W) # Create the gp_regression function and pass in the predictor function as f_hat self.plot_3d(y_hat, title=str(l) + "_BestLoss")
class VisualizeTwoStepOptimizationWithRestarts: def __init__(self): self.real_dim = 2 self.active_dim = 1 self.no_samples = 100 self.kernel = TripathyMaternKernel(self.real_dim, self.active_dim) # Parameters self.sn = 0.1 # 1e-7 # 0.1 self.W = self.kernel.sample_W() self.function = AugmentedSinusoidal() self.real_W = np.asarray([ [3], [1] ]) self.real_W = self.real_W / np.linalg.norm(self.real_W) # [[0.9486833] # [0.31622777]] x_range = np.linspace(0., 1., int(np.sqrt(self.no_samples))) y_range = np.linspace(0., 1., int(np.sqrt(self.no_samples))) self.X = cartesian([x_range, y_range]) #self.X = np.random.rand(self.no_samples, self.real_dim) print(self.X.shape) Z = np.dot(self.X, self.real_W).reshape(-1, 1) print(Z.shape) self.Y = self.function.f(Z.T).reshape(-1, 1) self.optimizer = TripathyOptimizer() # self.w_optimizer = t_WOptimizer( # self.kernel, # TODO: does the kernel take over the W? # self.sn, # np.asscalar(self.kernel.inner_kernel.variance), # self.kernel.inner_kernel.lengthscale, # self.X, self.Y # ) self.PLOT_MEAN = True def visualize_augmented_sinusoidal_function(self): x_range = np.linspace(0., 1., 80) y_range = np.linspace(0., 1., 80) X = cartesian([x_range, y_range]) import os if not os.path.exists("./pics-twostep/"): os.makedirs("./pics-twostep/") ################################# # TRAIN THE W_OPTIMIZER # ################################# Opt = TripathyOptimizer() print("Real hidden matrix is: ", self.real_W) W_hat = self.kernel.sample_W() self.kernel.update_params( W=W_hat, s=self.kernel.inner_kernel.variance, l=self.kernel.inner_kernel.lengthscale ) W_hat, sn, l, s = Opt.try_two_step_optimization_with_restarts(self.kernel, self.X, self.Y) # TODO: Check if these values are attained over multiple iterations (check if assert otherwise fails) # Create the gp_regression function and pass in the predictor function as f_hat self.kernel.update_params(W=W_hat, l=l, s=s) gp_reg = GPRegression(self.X, self.Y, self.kernel, noise_var=sn) y = self.function.f( np.dot(X, self.real_W).T ) if self.PLOT_MEAN: y_hat = gp_reg.predict(X)[0].squeeze() else: y_hat = gp_reg.predict(self.X)[0].squeeze() ################################# # END TRAIN THE W_OPTIMIZER # ################################# fig = plt.figure() ax = Axes3D(fig) # First plot the real function ax.scatter(X[:,0], X[:, 1], y, s=1) if self.PLOT_MEAN: ax.scatter(X[:, 0], X[:, 1], y_hat, cmap=plt.cm.jet) else: ax.scatter(self.X[:,0], self.X[:, 1], y_hat, cmap=plt.cm.jet) # Save the W just in case l = loss( self.kernel, W_hat, sn, s, l, self.X, self.Y ) fig.savefig('./pics-twostep/BestLoss_' + str(l) + '.png', ) plt.show() plt.close(fig) np.savetxt("./pics-twostep/BestLoss_" + str(l) + ".txt", W_hat)