def test_multioutput_with_diag_q_sqrt(session_tf): data = DataMixedKernel q_sqrt_diag = np.ones((data.M, data.L)) * 2 q_sqrt = np.repeat(np.eye(data.M)[None, ...], data.L, axis=0) * 2 # L x M x M kern_list = [RBF(data.D) for _ in range(data.L)] k1 = mk.SeparateMixedMok(kern_list, W=data.W) f1 = mf.SharedIndependentMof(InducingPoints(data.X[:data.M, ...].copy())) m1 = SVGP(data.X, data.Y, k1, Gaussian(), feat=f1, q_mu=data.mu_data, q_sqrt=q_sqrt_diag, q_diag=True) kern_list = [RBF(data.D) for _ in range(data.L)] k2 = mk.SeparateMixedMok(kern_list, W=data.W) f2 = mf.SharedIndependentMof(InducingPoints(data.X[:data.M, ...].copy())) m2 = SVGP(data.X, data.Y, k2, Gaussian(), feat=f2, q_mu=data.mu_data, q_sqrt=q_sqrt, q_diag=False) check_equality_predictions(session_tf, [m1, m2])
def test_compare_mixed_kernel(session_tf): data = DataMixedKernel kern_list = [RBF(data.D) for _ in range(data.L)] k1 = mk.SeparateMixedMok(kern_list, W=data.W) f1 = mf.SharedIndependentMof(InducingPoints(data.X[:data.M, ...].copy())) m1 = SVGP(data.X, data.Y, k1, Gaussian(), feat=f1, q_mu=data.mu_data, q_sqrt=data.sqrt_data) kern_list = [RBF(data.D) for _ in range(data.L)] k2 = mk.SeparateMixedMok(kern_list, W=data.W) f2 = mf.MixedKernelSharedMof(InducingPoints(data.X[:data.M, ...].copy())) m2 = SVGP(data.X, data.Y, k2, Gaussian(), feat=f2, q_mu=data.mu_data, q_sqrt=data.sqrt_data) check_equality_predictions(session_tf, [m1, m2])
def __init__(self, dim, input_dim=0, kern=None, Z=None, n_ind_pts=100, mean_fn=None, Q_diag=None, Umu=None, Ucov_chol=None, jitter=gps.numerics.jitter_level, name=None): super().__init__(name=name) self.OBSERVATIONS_AS_INPUT = False self.dim = dim self.input_dim = input_dim self.jitter = jitter self.Q_sqrt = Param(np.ones(self.dim) if Q_diag is None else Q_diag ** 0.5, transform=gtf.positive) self.n_ind_pts = n_ind_pts if Z is None else (Z[0].shape[-2] if isinstance(Z, list) else Z.shape[-2]) if isinstance(Z, np.ndarray) and Z.ndim == 2: self.Z = mf.SharedIndependentMof(gp.features.InducingPoints(Z)) else: Z_list = [np.random.randn(self.n_ind_pts, self.dim + self.input_dim) for _ in range(self.dim)] if Z is None else [z for z in Z] self.Z = mf.SeparateIndependentMof([gp.features.InducingPoints(z) for z in Z_list]) if isinstance(kern, gp.kernels.Kernel): self.kern = mk.SharedIndependentMok(kern, self.dim) else: kern_list = kern or [gp.kernels.Matern32(self.dim + self.input_dim, ARD=True) for _ in range(self.dim)] self.kern = mk.SeparateIndependentMok(kern_list) self.mean_fn = mean_fn or mean_fns.Identity(self.dim) self.Umu = Param(np.zeros((self.dim, self.n_ind_pts)) if Umu is None else Umu) # Lm^-1(Umu - m(Z)) transform = gtf.LowerTriangular(self.n_ind_pts, num_matrices=self.dim, squeeze=False) self.Ucov_chol = Param(np.tile(np.eye(self.n_ind_pts)[None, ...], [self.dim, 1, 1]) if Ucov_chol is None else Ucov_chol, transform=transform) # Lm^-1(Ucov_chol) self._Kzz = None
def test_separate_independent_mok(session_tf): """ We use different independent kernels for each of the output dimensions. We can achieve this in two ways: 1) efficient: SeparateIndependentMok with Shared/SeparateIndependentMof 2) inefficient: SeparateIndependentMok with InducingPoints However, both methods should return the same conditional, and after optimization return the same log likelihood. """ # Model 1 (INefficient) q_mu_1 = np.random.randn(Data.M * Data.P, 1) q_sqrt_1 = np.tril(np.random.randn(Data.M * Data.P, Data.M * Data.P))[None, ...] # 1 x MP x MP kern_list_1 = [ RBF(Data.D, variance=0.5, lengthscales=1.2) for _ in range(Data.P) ] kernel_1 = mk.SeparateIndependentMok(kern_list_1) feature_1 = InducingPoints(Data.X[:Data.M, ...].copy()) m1 = SVGP(Data.X, Data.Y, kernel_1, Gaussian(), feature_1, q_mu=q_mu_1, q_sqrt=q_sqrt_1) m1.set_trainable(False) m1.q_sqrt.set_trainable(True) m1.q_mu.set_trainable(True) gpflow.training.ScipyOptimizer().minimize(m1, maxiter=Data.MAXITER) # Model 2 (efficient) q_mu_2 = np.random.randn(Data.M, Data.P) q_sqrt_2 = np.array([ np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P) ]) # P x M x M kern_list_2 = [ RBF(Data.D, variance=0.5, lengthscales=1.2) for _ in range(Data.P) ] kernel_2 = mk.SeparateIndependentMok(kern_list_2) feature_2 = mf.SharedIndependentMof( InducingPoints(Data.X[:Data.M, ...].copy())) m2 = SVGP(Data.X, Data.Y, kernel_2, Gaussian(), feature_2, q_mu=q_mu_2, q_sqrt=q_sqrt_2) m2.set_trainable(False) m2.q_sqrt.set_trainable(True) m2.q_mu.set_trainable(True) gpflow.training.ScipyOptimizer().minimize(m2, maxiter=Data.MAXITER) check_equality_predictions(session_tf, [m1, m2])
def init(): feature = mf.SharedIndependentMof(gpflow.features.InducingPoints(Z.copy())) #Define POI kernels poi_list = [0, 1, 2, 3, 4] kern_list = [] for i in range(len(poi_list)): kern = POI(effects=0.5, lengthscale=5, input_dim=D, locs_poi=locs_poi, typeIndicator=typeIndicator, typIdx=poi_list[i], locs=Z[:, 0:2], mindist=typeMinDist[i], name=str(poi_list[i]), kernel_type="Linear") kern_list.append(kern) #Add spatial kernel kern_spatial = gpflow.kernels.Matern32(input_dim=D, lengthscales=100) #Define kernel list kern_list.append(kern_spatial) L = len(kern_list) W = np.ones((L, 1)) W_t = np.transpose(W) kernel = mk.SeparateMixedMok(kern_list, W=W_t) #Define linear mean function #mean_fct = SlicedLinear(A = theta, p=p) mean_fct = SlicedNN(p=p) q_mu = np.random.normal(0.0, 1, (M, L)) q_sqrt = np.repeat(np.eye(M)[None, ...], L, axis=0) * 1.0 m = gpflow.models.SVGP( X=X_train, Y=y_train, kern=kernel, likelihood=gpflow.likelihoods.Gaussian(), feat=feature, whiten=True, #minibatch_size=len(X_train), mean_function=mean_fct, # + mean_poi, q_mu=q_mu, q_sqrt=q_sqrt, name='svgp') m.likelihood.variance = 0.01 #Initialze params m.feature.trainable = False m.kern.W.trainable = False return m
def test_sample_conditional_mixedkernel(session_tf): q_mu = np.random.randn(Data.M, Data.L) # M x L q_sqrt = np.array([ np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.L) ]) # L x M x M Z = Data.X[:Data.M, ...] # M x D N = int(10e5) Xs = np.ones((N, Data.D), dtype=float_type) values = {"Xnew": Xs, "q_mu": q_mu, "q_sqrt": q_sqrt} placeholders = _create_placeholder_dict(values) feed_dict = _create_feed_dict(placeholders, values) # Path 1: mixed kernel: most efficient route W = np.random.randn(Data.P, Data.L) mixed_kernel = mk.SeparateMixedMok([RBF(Data.D) for _ in range(Data.L)], W) mixed_feature = mf.MixedKernelSharedMof(InducingPoints(Z.copy())) sample = sample_conditional(placeholders["Xnew"], mixed_feature, mixed_kernel, placeholders["q_mu"], q_sqrt=placeholders["q_sqrt"], white=True) value = session_tf.run(sample, feed_dict=feed_dict) # Path 2: independent kernels, mixed later separate_kernel = mk.SeparateIndependentMok( [RBF(Data.D) for _ in range(Data.L)]) shared_feature = mf.SharedIndependentMof(InducingPoints(Z.copy())) sample2 = sample_conditional(placeholders["Xnew"], shared_feature, separate_kernel, placeholders["q_mu"], q_sqrt=placeholders["q_sqrt"], white=True) value2 = session_tf.run(sample2, feed_dict=feed_dict) value2 = np.matmul(value2, W.T) # check if mean and covariance of samples are similar np.testing.assert_array_almost_equal(np.mean(value, axis=0), np.mean(value2, axis=0), decimal=1) np.testing.assert_array_almost_equal(np.cov(value, rowvar=False), np.cov(value2, rowvar=False), decimal=1)
def main(): X = np.loadtxt("../data/neur.X.txt") Y = np.loadtxt("../data/neur.Y.txt") gpflow.reset_default_graph_and_session() name = 'test' minibatch_size = 500 W1_init = normalize(np.random.random(size=(C, K1))) W2_init = normalize(np.random.random(size=(G, K2))) with gpflow.defer_build(): kernel = mk.SharedIndependentMok( gpflow.kernels.RBF(1, active_dims=[0]), K1 * K2) Z = np.linspace(0, 1, T)[:, None].astype(np.float64) feature = gpflow.features.InducingPoints(Z) feature = mf.SharedIndependentMof(feature) model = SplitGPM(X, Y, np.log(W1_init + 1e-5), np.log(W2_init + 1e-5), kernel, gpflow.likelihoods.Gaussian(), feat=feature, minibatch_size=minibatch_size, name=name) model.compile() model.W1.set_trainable(True) # learn cell assignments model.W2.set_trainable(True) # learn gene assignments model.feature.set_trainable(True) # move inducing points model.kern.set_trainable(True) # learn kernel parameters model.likelihood.set_trainable(True) # lear likelihood parameters adam = gpflow.train.AdamOptimizer(0.005) adam.minimize(model, maxiter=10000) save_model(model)
def train(self, verbose=True, maxiter=1000): with gpflow.settings.temp_settings(self.gpflow_config): # Default parameters if self.kern is None: self.kern = gpflow.kernels.SquaredExponential( input_dim=self.n_in_dims, variance=self.dtype(0.2), lengthscales=self.dtype(1.0)) if self.lh is None: self.lh = gpflow.likelihoods.Gaussian( variance=self.dtype(0.02)) if self.feature is None: self.feature = gpf.features.InducingPoints(self.Z) if self.multi_output: self.kern = mok.SharedIndependentMok( self.kern, output_dimensionality=self.n_out_dims) self.feature = mof.SharedIndependentMof(self.feature) self.m = gpflow.models.SVGP(self.X, self.Y, self.kern, likelihood=self.lh, feat=self.feature, mean_function=self.mf, minibatch_size=self.batch_size) opt = gpflow.train.tensorflow_optimizer.AdamOptimizer( learning_rate=0.1, beta1=0.9, beta2=0.999, epsilon=1e-8) opt.minimize(self.m, maxiter=maxiter) if verbose: pd.set_option('display.max_rows', 20) pd.set_option('display.max_columns', 10) print(self.m.as_pandas_table()) print('Log likelihood: ', self.m.compute_log_likelihood())
def test_shared_independent_mok(session_tf): """ In this test we use the same kernel and the same inducing features for each of the outputs. The outputs are considered to be uncorrelated. This is how GPflow handled multiple outputs before the multioutput framework was added. We compare three models here: 1) an ineffient one, where we use a SharedIndepedentMok with InducingPoints. This combination will uses a Kff of size N x P x N x P, Kfu if size N x P x M x P which is extremely inefficient as most of the elements are zero. 2) efficient: SharedIndependentMok and SharedIndependentMof This combinations uses the most efficient form of matrices 3) the old way, efficient way: using Kernel and InducingPoints Model 2) and 3) follow more or less the same code path. """ # Model 1 q_mu_1 = np.random.randn(Data.M * Data.P, 1) # MP x 1 q_sqrt_1 = np.tril(np.random.randn(Data.M * Data.P, Data.M * Data.P))[None, ...] # 1 x MP x MP kernel_1 = mk.SharedIndependentMok( RBF(Data.D, variance=0.5, lengthscales=1.2), Data.P) feature_1 = InducingPoints(Data.X[:Data.M, ...].copy()) m1 = SVGP(Data.X, Data.Y, kernel_1, Gaussian(), feature_1, q_mu=q_mu_1, q_sqrt=q_sqrt_1) m1.set_trainable(False) m1.q_sqrt.set_trainable(True) gpflow.training.ScipyOptimizer().minimize(m1, maxiter=Data.MAXITER) # Model 2 q_mu_2 = np.reshape(q_mu_1, [Data.M, Data.P]) # M x P q_sqrt_2 = np.array([ np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P) ]) # P x M x M kernel_2 = RBF(Data.D, variance=0.5, lengthscales=1.2) feature_2 = InducingPoints(Data.X[:Data.M, ...].copy()) m2 = SVGP(Data.X, Data.Y, kernel_2, Gaussian(), feature_2, q_mu=q_mu_2, q_sqrt=q_sqrt_2) m2.set_trainable(False) m2.q_sqrt.set_trainable(True) gpflow.training.ScipyOptimizer().minimize(m2, maxiter=Data.MAXITER) # Model 3 q_mu_3 = np.reshape(q_mu_1, [Data.M, Data.P]) # M x P q_sqrt_3 = np.array([ np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P) ]) # P x M x M kernel_3 = mk.SharedIndependentMok( RBF(Data.D, variance=0.5, lengthscales=1.2), Data.P) feature_3 = mf.SharedIndependentMof( InducingPoints(Data.X[:Data.M, ...].copy())) m3 = SVGP(Data.X, Data.Y, kernel_3, Gaussian(), feature_3, q_mu=q_mu_3, q_sqrt=q_sqrt_3) m3.set_trainable(False) m3.q_sqrt.set_trainable(True) gpflow.training.ScipyOptimizer().minimize(m3, maxiter=Data.MAXITER) check_equality_predictions(session_tf, [m1, m2, m3])
y = y[:, :, :G] mask = ~np.isnan(y) Y = y[mask][:, None] X = x[mask][:, None] weight_idx = np.tile(np.arange(N * G).reshape(N, G)[None], (T, 1, 1))[mask] # gp objects if global_trajectories: num_clusters = K * L + L else: num_clusters = K * L kernel = mk.SharedIndependentMok(gpflow.kernels.RBF(1), num_clusters) feature = mf.SharedIndependentMof( gpflow.features.InducingPoints( np.arange(T).astype(np.float64).reshape(-1, 1))) likelihood = gpflow.likelihoods.Gaussian() # model -- for hyperparameter learning with gpflow.defer_build(): m = MixtureSVGP(X, Y, weight_idx, kern=kernel, num_latent=num_clusters, num_data=X.shape[0], likelihood=likelihood, feat=feature, minibatch_size=minibatch_size)
def __init__(self, latent_dim, Y, inputs=None, emissions=None, px1_mu=None, px1_cov=None, kern=None, Z=None, n_ind_pts=100, mean_fn=None, Q_diag=None, Umu=None, Ucov_chol=None, qx1_mu=None, qx1_cov=None, As=None, bs=None, Ss=None, n_samples=100, seed=None, parallel_iterations=10, jitter=gps.numerics.jitter_level, name=None): super().__init__(name=name) self.latent_dim = latent_dim self.T, self.obs_dim = Y.shape self.Y = Param(Y, trainable=False) self.inputs = None if inputs is None else Param(inputs, trainable=False) self.input_dim = 0 if self.inputs is None else self.inputs.shape[1] self.qx1_mu = Param( np.zeros(self.latent_dim) if qx1_mu is None else qx1_mu) self.qx1_cov_chol = Param( np.eye(self.latent_dim) if qx1_cov is None else np.linalg.cholesky(qx1_cov), transform=gtf.LowerTriangular(self.latent_dim, squeeze=True)) self.As = Param( np.ones((self.T - 1, self.latent_dim)) if As is None else As) self.bs = Param( np.zeros((self.T - 1, self.latent_dim)) if bs is None else bs) self.Q_sqrt = Param( np.ones(self.latent_dim) if Q_diag is None else Q_diag**0.5, transform=gtf.positive) if Ss is False: self._S_chols = None else: self.S_chols = Param( np.tile(self.Q_sqrt.value.copy()[None, ...], [self.T - 1, 1]) if Ss is None else (np.sqrt(Ss) if Ss.ndim == 2 else np.linalg.cholesky(Ss)), transform=gtf.positive if (Ss is None or Ss.ndim == 2) else gtf.LowerTriangular( self.latent_dim, num_matrices=self.T - 1, squeeze=False)) self.emissions = emissions or GaussianEmissions( latent_dim=self.latent_dim, obs_dim=self.obs_dim) self.px1_mu = Param( np.zeros(self.latent_dim) if px1_mu is None else px1_mu, trainable=False) self.px1_cov_chol = None if px1_cov is None else \ Param(np.sqrt(px1_cov) if px1_cov.ndim == 1 else np.linalg.cholesky(px1_cov), trainable=False, transform=gtf.positive if px1_cov.ndim == 1 else gtf.LowerTriangular(self.latent_dim, squeeze=True)) self.n_samples = n_samples self.seed = seed self.parallel_iterations = parallel_iterations self.jitter = jitter # Inference-specific attributes (see gpssm_models.py for appropriate choices): nans = tf.constant(np.zeros( (self.T, self.n_samples, self.latent_dim)) * np.nan, dtype=gps.float_type) self.sample_fn = lambda **kwargs: (nans, None) self.sample_kwargs = {} self.KL_fn = lambda *fs: tf.constant(np.nan, dtype=gps.float_type) # GP Transitions: self.n_ind_pts = n_ind_pts if Z is None else ( Z[0].shape[-2] if isinstance(Z, list) else Z.shape[-2]) if isinstance(Z, np.ndarray) and Z.ndim == 2: self.Z = mf.SharedIndependentMof(gp.features.InducingPoints(Z)) else: Z_list = [ np.random.randn(self.n_ind_pts, self.latent_dim + self.input_dim) for _ in range(self.latent_dim) ] if Z is None else [z for z in Z] self.Z = mf.SeparateIndependentMof( [gp.features.InducingPoints(z) for z in Z_list]) if isinstance(kern, gp.kernels.Kernel): self.kern = mk.SharedIndependentMok(kern, self.latent_dim) else: kern_list = kern or [ gp.kernels.Matern32(self.latent_dim + self.input_dim, ARD=True) for _ in range(self.latent_dim) ] self.kern = mk.SeparateIndependentMok(kern_list) self.mean_fn = mean_fn or mean_fns.Identity(self.latent_dim) self.Umu = Param( np.zeros((self.latent_dim, self.n_ind_pts)) if Umu is None else Umu) # (Lm^-1)(Umu - m(Z)) LT_transform = gtf.LowerTriangular(self.n_ind_pts, num_matrices=self.latent_dim, squeeze=False) self.Ucov_chol = Param(np.tile( np.eye(self.n_ind_pts)[None, ...], [self.latent_dim, 1, 1]) if Ucov_chol is None else Ucov_chol, transform=LT_transform) # (Lm^-1)Lu self._Kzz = None
def shared_independent(self): return mf.SharedIndependentMof(make_ip())