def _last_layer(self, H_X, M, filter_size, stride, layer_params=None, pad = 0): if layer_params is None: layer_params = {} NHWC = H_X.shape conv_output_count = np.prod(NHWC[1:]) Z = layer_params.get('Z') q_mu = layer_params.get('q_mu') q_sqrt = layer_params.get('q_sqrt') if Z is not None: saved_filter_size = int(np.sqrt(Z.shape[1] / NHWC[3])) if filter_size != saved_filter_size: print("filter_size {} != {} for last layer. Resetting parameters.".format(filter_size, saved_filter_size)) Z = None q_mu = None q_sqrt = None if self.flags.last_kernel == 'rbf': H_X = H_X.reshape(H_X.shape[0], -1) lengthscales = layer_params.get('lengthscales', 5.0) variance = layer_params.get('variance', 5.0) kernel = gpflow.kernels.RBF(conv_output_count, lengthscales=lengthscales, variance=variance, ARD=True) if Z is None: Z = select_initial_inducing_points(H_X, M) inducing = features.InducingPoints(Z) else: lengthscales = layer_params.get('base_kernel/lengthscales', 5.0) variance = layer_params.get('base_kernel/variance', 5.0) input_dim = filter_size**2 * NHWC[3] view = FullView(input_size=NHWC[1:], filter_size=filter_size, feature_maps=NHWC[3], stride=stride, pad = pad) if Z is None: inducing = PatchInducingFeatures.from_images(H_X, M, filter_size) else: inducing = PatchInducingFeatures(Z) patch_weights = layer_params.get('patch_weights') if self.flags.last_kernel == 'conv': kernel = ConvKernel( base_kernel=gpflow.kernels.RBF(input_dim, variance=variance, lengthscales=lengthscales), view=view, patch_weights=patch_weights) elif self.flags.last_kernel == 'add': kernel = AdditivePatchKernel( base_kernel=gpflow.kernels.RBF(input_dim, variance=variance, lengthscales=lengthscales), view=view, patch_weights=patch_weights) else: raise ValueError("Invalid last layer kernel") return SVGP_Layer(kern=kernel, num_outputs=10, feature=inducing, mean_function=gpflow.mean_functions.Zero(output_dim=10), white=self.flags.white, q_mu=q_mu, q_sqrt=q_sqrt)
def test_inducing_points_equivalence(self): # Inducing features must be the same as the kernel evaluations with self.test_context() as session: Z = np.random.randn(101, 3) f = features.InducingPoints(Z) kernels = [ gpflow.kernels.RBF(3, 0.46, lengthscales=np.array([0.143, 1.84, 2.0]), ARD=True), gpflow.kernels.Periodic(3, 0.4, 1.8) ] for k in kernels: self.assertTrue(np.allclose(session.run(features.Kuu(f, k)), k.compute_K_symm(Z)))
def feature(session_tf): return features.InducingPoints(Data.Z)
def feat2(): return features.InducingPoints(Data.Z2)
def __init__(self, Y, T, kern, M=10, num_latent=3, dynamic_kern=None, Z=None, KL_weight=None): """ Initialise VGPDS. This only works with Gaussian Likelihood for now :param Y: data matrix, size T (number of time points) x D (dimensions) :param T: time vector, positive real value, size 1 x T :param kern: Mapping kernel X -> Y specification, by default RBF :param M: Number of inducing points :param num_latent: Number of latent dimension. This is automatically found unless user force latent dimension :param force_latent_dim: Specify whether strict latent dimension is enforced :param dynamic_kern: temporal dynamics kernel specification, by default RBF :param Z: matrix of inducing points :param KL_weight: Weight of KL . weight of bound = 1 - w(KL) """ X_mean = large_PCA(Y, num_latent) GPModel.__init__(self, X_mean, Y, kern, likelihood=likelihoods.Gaussian(), mean_function=Zero()) del self.X # This is a params self.T = np.transpose(T[np.newaxis]) self.num_latent = num_latent if KL_weight is None: self.KL_weight = 0.5 else: assert KL_weight <= 1 assert KL_weight >= 0 self.KL_weight = KL_weight #This is only one way to initialize mu_bar_q mu_bar_q = X_mean lambda_q = np.ones((self.T.shape[0], self.num_latent)) if dynamic_kern is None: self.dynamic_kern = kernels.RBF(1) + kernels.Bias( 1) + kernels.White(1) else: self.dynamic_kern = dynamic_kern self.mu_bar_q = Parameter(mu_bar_q) self.lambda_q = Parameter(lambda_q) self.num_time, self.num_latent = X_mean.shape self.output_dim = Y.shape[1] # inducing points if Z is None: # By default we initialize by subset of initial latent points Z = np.random.permutation(X_mean.copy())[:M] self.feature = features.InducingPoints(Z) assert len(self.feature) == M
class Data: rng = np.random.RandomState(1) num_data = 5 num_ind = 4 D_in = 2 D_out = 2 Xmu = rng.randn(num_data, D_in) L = gen_L(rng, num_data, D_in, D_in) Xvar = np.array([l @ l.T for l in L]) Z = rng.randn(num_ind, D_in) # distributions don't need to be compiled (No Parameter objects) # but the members should be Tensors created in the same graph graph = tf.Graph() with test_util.session_context(graph) as sess: gauss = Gaussian(tf.constant(Xmu), tf.constant(Xvar)) dirac = Gaussian(tf.constant(Xmu), tf.constant(np.zeros((num_data, D_in, D_in)))) gauss_diag = DiagonalGaussian(tf.constant(Xmu), tf.constant(rng.rand(num_data, D_in))) dirac_diag = DiagonalGaussian(tf.constant(Xmu), tf.constant(np.zeros((num_data, D_in)))) dirac_markov_gauss = MarkovGaussian( tf.constant(Xmu), tf.constant(np.zeros((2, num_data, D_in, D_in)))) # create the covariance for the pairwise markov-gaussian dummy_gen = lambda rng, n, *shape: np.array( [rng.randn(*shape) for _ in range(n)]) L_mg = dummy_gen(rng, num_data, D_in, 2 * D_in) # N+1 x D x 2D LL = np.concatenate((L_mg[:-1], L_mg[1:]), 1) # N x 2D x 2D Xcov = LL @ np.transpose(LL, (0, 2, 1)) Xc = np.concatenate((Xcov[:, :D_in, :D_in], Xcov[-1:, D_in:, D_in:]), 0) # N+1 x D x D Xcross = np.concatenate( (Xcov[:, :D_in, D_in:], np.zeros( (1, D_in, D_in))), 0) # N+1 x D x D Xcc = np.stack([Xc, Xcross]) # 2 x N+1 x D x D markov_gauss = MarkovGaussian(Xmu, Xcc) with gpflow.decors.defer_build(): # features ip = features.InducingPoints(Z) # kernels rbf_prod_seperate_dims = kernels.Product([ kernels.RBF(1, variance=rng.rand(), lengthscales=rng.rand(), active_dims=[0]), kernels.RBF(1, variance=rng.rand(), lengthscales=rng.rand(), active_dims=[1]) ]) rbf_lin_sum = kernels.Sum([ kernels.RBF(D_in, variance=rng.rand(), lengthscales=rng.rand()), kernels.RBF(D_in, variance=rng.rand(), lengthscales=rng.rand()), kernels.Linear(D_in, variance=rng.rand()) ]) rbf = kernels.RBF(D_in, variance=rng.rand(), lengthscales=rng.rand()) lin_kern = kernels.Linear(D_in, variance=rng.rand()) # mean functions lin = mean_functions.Linear(rng.rand(D_in, D_out), rng.rand(D_out)) iden = mean_functions.Identity( D_in) # Note: Identity can only be used if Din == Dout zero = mean_functions.Zero(output_dim=D_out) const = mean_functions.Constant(rng.rand(D_out))