def test_different_params_similar_approximation(self, initializer, scale): random_seed.set_random_seed(12345) rff_layer1 = kernel_layers.RandomFourierFeatures( output_dim=3000, kernel_initializer=initializer, scale=scale, name='rff1') rff_layer2 = kernel_layers.RandomFourierFeatures( output_dim=2000, kernel_initializer=initializer, scale=scale, name='rff2') # Two distinct inputs. x = constant_op.constant([[1.0, -1.0, 0.5]]) y = constant_op.constant([[-1.0, 1.0, 1.0]]) # Apply both layers to both inputs. output_x1 = math.sqrt(2.0 / 3000.0) * rff_layer1(x) output_y1 = math.sqrt(2.0 / 3000.0) * rff_layer1(y) output_x2 = math.sqrt(2.0 / 2000.0) * rff_layer2(x) output_y2 = math.sqrt(2.0 / 2000.0) * rff_layer2(y) # Compute the inner products of the outputs (on inputs x and y) for both # layers. For any fixed random features layer rff_layer, and inputs x, y, # rff_layer(x)^T * rff_layer(y) ~= K(x,y) up to a normalization factor. approx_kernel1 = kernelized_utils.inner_product(output_x1, output_y1) approx_kernel2 = kernelized_utils.inner_product(output_x2, output_y2) self._assert_all_close(approx_kernel1, approx_kernel2, atol=0.08)
def test_different_params_similar_approximation(self, initializer, scale): # Layers initialized using different randomness (seed). rff_layer1 = kernel_layers.RandomFourierFeatures( output_dim=3000, kernel_initializer=initializer, scale=scale, name='rff1') rff_layer2 = kernel_layers.RandomFourierFeatures( output_dim=2000, kernel_initializer=initializer, scale=scale, name='rff2') # Two distinct inputs. x = constant_op.constant([[1.0, -1.0, 0.5]]) y = constant_op.constant([[-1.0, 1.0, 1.0]]) # Apply both layers to both inputs. output_x1 = math.sqrt(2.0 / 3000.0) * rff_layer1.apply(x) output_y1 = math.sqrt(2.0 / 3000.0) * rff_layer1.apply(y) output_x2 = math.sqrt(2.0 / 2000.0) * rff_layer2.apply(x) output_y2 = math.sqrt(2.0 / 2000.0) * rff_layer2.apply(y) # Compute the inner products of the outputs (on inputs x and y) for both # layers. For any fixed random features layer rff_layer, and inputs x, y, # rff_layer(x)^T * rff_layer(y) ~= K(x,y) up to a normalization factor. approx_kernel1 = kernelized_utils.inner_product(output_x1, output_y1) approx_kernel2 = kernelized_utils.inner_product(output_x2, output_y2) self._assert_all_close(approx_kernel1, approx_kernel2, atol=0.05)
def test_good_kernel_approximation_multiple_inputs(self, initializer, scale, exact_kernel_fn): # Parameters. input_dim = 5 output_dim = 2000 x_rows = 20 y_rows = 30 x = constant_op.constant(np.random.uniform(size=(x_rows, input_dim)), dtype=dtypes.float32) y = constant_op.constant(np.random.uniform(size=(y_rows, input_dim)), dtype=dtypes.float32) random_seed.set_random_seed(1234) rff_layer = kernel_layers.RandomFourierFeatures( output_dim=output_dim, kernel_initializer=initializer, scale=scale, name='random_fourier_features') # The shapes of output_x and output_y are (x_rows, output_dim) and # (y_rows, output_dim) respectively. output_x = math.sqrt(2.0 / output_dim) * rff_layer(x) output_y = math.sqrt(2.0 / output_dim) * rff_layer(y) approx_kernel_matrix = kernelized_utils.inner_product( output_x, output_y) exact_kernel_matrix = exact_kernel_fn(x, y) self._assert_all_close(approx_kernel_matrix, exact_kernel_matrix, atol=0.05)
def test_bad_kernel_approximation(self, initializer, scale, exact_kernel_fn): """Approximation is bad when output dimension is small.""" # Two distinct inputs. x = constant_op.constant([[1.0, -1.0, 0.5]]) y = constant_op.constant([[-1.0, 1.0, 1.0]]) small_output_dim = 10 random_seed.set_random_seed(1234) # Initialize layer. rff_layer = kernel_layers.RandomFourierFeatures( output_dim=small_output_dim, kernel_initializer=initializer, scale=scale, name='random_fourier_features') # Apply layer to both inputs. output_x = math.sqrt(2.0 / small_output_dim) * rff_layer(x) output_y = math.sqrt(2.0 / small_output_dim) * rff_layer(y) # The inner products of the outputs (on inputs x and y) approximates the # real value of the RBF kernel but poorly since the output dimension of the # layer is small. exact_kernel_value = exact_kernel_fn(x, y) approx_kernel_value = kernelized_utils.inner_product(output_x, output_y) abs_error = math_ops.abs(exact_kernel_value - approx_kernel_value) if not context.executing_eagerly(): with self.cached_session() as sess: keras_backend._initialize_variables(sess) abs_error_eval = sess.run([abs_error]) self.assertGreater(abs_error_eval[0][0], 0.05) self.assertLess(abs_error_eval[0][0], 0.5) else: self.assertGreater(abs_error, 0.05) self.assertLess(abs_error, 0.5)
def test_good_kernel_approximation_multiple_inputs(self, initializer, scale, exact_kernel_fn): # Parameters. input_dim = 5 output_dim = 5000 x_rows = 20 y_rows = 30 random_seed.set_random_seed(1234) x = random_ops.random_uniform(shape=(x_rows, input_dim), maxval=1.0) y = random_ops.random_uniform(shape=(y_rows, input_dim), maxval=1.0) rff_layer = kernel_layers.RandomFourierFeatures( output_dim=output_dim, kernel_initializer=initializer, scale=scale, name='random_fourier_features') # The shapes of output_x and output_y are (x_rows, output_dim) and # (y_rows, output_dim) respectively. output_x = math.sqrt(2.0 / output_dim) * rff_layer.apply(x) output_y = math.sqrt(2.0 / output_dim) * rff_layer.apply(y) approx_kernel_matrix = kernelized_utils.inner_product( output_x, output_y) exact_kernel_matrix = exact_kernel_fn(x, y) self._assert_all_close(approx_kernel_matrix, exact_kernel_matrix, atol=0.1)
def test_good_kernel_approximation_multiple_inputs(self, initializer, scale, exact_kernel_fn): # Parameters. input_dim = 5 output_dim = 5000 x_rows = 20 y_rows = 30 random_seed.set_random_seed(1234) x = random_ops.random_uniform(shape=(x_rows, input_dim), maxval=1.0) y = random_ops.random_uniform(shape=(y_rows, input_dim), maxval=1.0) rff_layer = kernel_layers.RandomFourierFeatures( output_dim=output_dim, kernel_initializer=initializer, scale=scale, name='random_fourier_features') # The shapes of output_x and output_y are (x_rows, output_dim) and # (y_rows, output_dim) respectively. output_x = math.sqrt(2.0 / output_dim) * rff_layer.apply(x) output_y = math.sqrt(2.0 / output_dim) * rff_layer.apply(y) approx_kernel_matrix = kernelized_utils.inner_product(output_x, output_y) exact_kernel_matrix = exact_kernel_fn(x, y) self._assert_all_close(approx_kernel_matrix, exact_kernel_matrix, atol=0.1)
def test_bad_kernel_approximation(self, initializer, scale, exact_kernel_fn): """Approximation is bad when output dimension is small.""" # Two distinct inputs. x = constant_op.constant([[1.0, -1.0, 0.5]]) y = constant_op.constant([[-1.0, 1.0, 1.0]]) small_output_dim = 10 random_seed.set_random_seed(1234) # Initialize layer. rff_layer = kernel_layers.RandomFourierFeatures( output_dim=small_output_dim, kernel_initializer=initializer, scale=scale, name='random_fourier_features') # Apply layer to both inputs. output_x = math.sqrt(2.0 / small_output_dim) * rff_layer.apply(x) output_y = math.sqrt(2.0 / small_output_dim) * rff_layer.apply(y) # The inner products of the outputs (on inputs x and y) approximates the # real value of the RBF kernel but poorly since the output dimension of the # layer is small. exact_kernel_value = exact_kernel_fn(x, y) approx_kernel_value = kernelized_utils.inner_product(output_x, output_y) abs_error = math_ops.abs(exact_kernel_value - approx_kernel_value) if not context.executing_eagerly(): with self.cached_session() as sess: keras_backend._initialize_variables(sess) abs_error_eval = sess.run([abs_error]) self.assertGreater(abs_error_eval[0][0], 0.05) self.assertLess(abs_error_eval[0][0], 0.5) else: self.assertGreater(abs_error, 0.05) self.assertLess(abs_error, 0.5)
def test_fourier_feature_layer_compute_covariance_of_inducing_variables(batch_size): """ Ensure that the random fourier feature map can be used to approximate the covariance matrix between the inducing point vectors of a sparse GP, with the condition that the number of latent GP models is greater than one. """ n_features = 10000 kernel = gpflow.kernels.SquaredExponential() fourier_features = RandomFourierFeatures(kernel, n_features, dtype=tf.float64) x_new = tf.ones(shape=(2 * batch_size + 1, 1), dtype=tf.float64) u = fourier_features(x_new) approx_kernel_matrix = inner_product(u, u) actual_kernel_matrix = kernel.K(x_new, x_new) np.testing.assert_allclose(approx_kernel_matrix, actual_kernel_matrix, atol=0.05)
def test_fourier_features_can_approximate_kernel_multidim(kernel_class, lengthscale, n_dims): n_features = 10000 x_rows = 20 y_rows = 30 # ARD lengthscales = np.random.rand((n_dims)) * lengthscale kernel = kernel_class(lengthscales=lengthscales) fourier_features = RandomFourierFeatures(kernel, n_features, dtype=tf.float64) x = tf.random.uniform((x_rows, n_dims), dtype=tf.float64) y = tf.random.uniform((y_rows, n_dims), dtype=tf.float64) u = fourier_features(x) v = fourier_features(y) approx_kernel_matrix = inner_product(u, v) actual_kernel_matrix = kernel.K(x, y) np.testing.assert_allclose(approx_kernel_matrix, actual_kernel_matrix, atol=0.05)