def test_mixed_mok_with_Id_vs_independent_mok():
data = DataMixedKernelWithEye
# Independent model
k1 = mk.SharedIndependent(SquaredExponential(variance=0.5, lengthscales=1.2), data.L)
f1 = InducingPoints(data.X[: data.M, ...])
model_1 = SVGP(k1, Gaussian(), f1, q_mu=data.mu_data_full, q_sqrt=data.sqrt_data_full)
set_trainable(model_1, False)
set_trainable(model_1.q_sqrt, True)
gpflow.optimizers.Scipy().minimize(
model_1.training_loss_closure(Data.data),
variables=model_1.trainable_variables,
method="BFGS",
compile=True,
)
# Mixed Model
kern_list = [SquaredExponential(variance=0.5, lengthscales=1.2) for _ in range(data.L)]
k2 = mk.LinearCoregionalization(kern_list, data.W)
f2 = InducingPoints(data.X[: data.M, ...])
model_2 = SVGP(k2, Gaussian(), f2, q_mu=data.mu_data_full, q_sqrt=data.sqrt_data_full)
set_trainable(model_2, False)
set_trainable(model_2.q_sqrt, True)
gpflow.optimizers.Scipy().minimize(
model_2.training_loss_closure(Data.data),
variables=model_2.trainable_variables,
method="BFGS",
compile=True,
)
check_equality_predictions(Data.data, [model_1, model_2])
def test_separate_independent_mok():
"""
We use different independent kernels for each of the output dimensions.
We can achieve this in two ways:
1) efficient: SeparateIndependentMok with Shared/SeparateIndependentMof
2) inefficient: SeparateIndependentMok with InducingPoints
However, both methods should return the same conditional,
and after optimization return the same log likelihood.
"""
# Model 1 (Inefficient)
q_mu_1 = np.random.randn(Data.M * Data.P, 1)
q_sqrt_1 = np.tril(np.random.randn(Data.M * Data.P, Data.M * Data.P))[None, ...] # 1 x MP x MP
kern_list_1 = [SquaredExponential(variance=0.5, lengthscales=1.2) for _ in range(Data.P)]
kernel_1 = mk.SeparateIndependent(kern_list_1)
inducing_variable_1 = InducingPoints(Data.X[: Data.M, ...])
model_1 = SVGP(
kernel_1, Gaussian(), inducing_variable_1, num_latent_gps=1, q_mu=q_mu_1, q_sqrt=q_sqrt_1,
)
set_trainable(model_1, False)
set_trainable(model_1.q_sqrt, True)
set_trainable(model_1.q_mu, True)
gpflow.optimizers.Scipy().minimize(
model_1.training_loss_closure(Data.data),
variables=model_1.trainable_variables,
method="BFGS",
compile=True,
)
# Model 2 (efficient)
q_mu_2 = np.random.randn(Data.M, Data.P)
q_sqrt_2 = np.array(
[np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)]
) # P x M x M
kern_list_2 = [SquaredExponential(variance=0.5, lengthscales=1.2) for _ in range(Data.P)]
kernel_2 = mk.SeparateIndependent(kern_list_2)
inducing_variable_2 = mf.SharedIndependentInducingVariables(
InducingPoints(Data.X[: Data.M, ...])
)
model_2 = SVGP(
kernel_2,
Gaussian(),
inducing_variable_2,
num_latent_gps=Data.P,
q_mu=q_mu_2,
q_sqrt=q_sqrt_2,
)
set_trainable(model_2, False)
set_trainable(model_2.q_sqrt, True)
set_trainable(model_2.q_mu, True)
gpflow.optimizers.Scipy().minimize(
model_2.training_loss_closure(Data.data),
variables=model_2.trainable_variables,
method="BFGS",
compile=True,
)
check_equality_predictions(Data.data, [model_1, model_2])
def test_multioutput_with_diag_q_sqrt():
data = DataMixedKernel
q_sqrt_diag = np.ones((data.M, data.L)) * 2
q_sqrt = np.repeat(np.eye(data.M)[None, ...], data.L,
axis=0) * 2 # L x M x M
kern_list = [SquaredExponential() for _ in range(data.L)]
k1 = mk.LinearCoregionalization(kern_list, W=data.W)
f1 = mf.SharedIndependentInducingVariables(
InducingPoints(data.X[:data.M, ...]))
model_1 = SVGP(k1,
Gaussian(),
inducing_variable=f1,
q_mu=data.mu_data,
q_sqrt=q_sqrt_diag,
q_diag=True)
kern_list = [SquaredExponential() for _ in range(data.L)]
k2 = mk.LinearCoregionalization(kern_list, W=data.W)
f2 = mf.SharedIndependentInducingVariables(
InducingPoints(data.X[:data.M, ...]))
model_2 = SVGP(k2,
Gaussian(),
inducing_variable=f2,
q_mu=data.mu_data,
q_sqrt=q_sqrt,
q_diag=False)
check_equality_predictions(Data.X, Data.Y, [model_1, model_2])
def test_MixedKernelSeparateMof():
data = DataMixedKernel
kern_list = [SquaredExponential() for _ in range(data.L)]
inducing_variable_list = [
InducingPoints(data.X[:data.M, ...]) for _ in range(data.L)
]
k1 = mk.LinearCoregionalization(kern_list, W=data.W)
f1 = mf.SeparateIndependentInducingVariables(inducing_variable_list)
model_1 = SVGP(k1,
Gaussian(),
inducing_variable=f1,
q_mu=data.mu_data,
q_sqrt=data.sqrt_data)
kern_list = [SquaredExponential() for _ in range(data.L)]
inducing_variable_list = [
InducingPoints(data.X[:data.M, ...]) for _ in range(data.L)
]
k2 = mk.LinearCoregionalization(kern_list, W=data.W)
f2 = mf.SeparateIndependentInducingVariables(inducing_variable_list)
model_2 = SVGP(k2,
Gaussian(),
inducing_variable=f2,
q_mu=data.mu_data,
q_sqrt=data.sqrt_data)
check_equality_predictions(Data.X, Data.Y, [model_1, model_2])
def test_sample_conditional_mixedkernel():
q_mu = tf.random.uniform((Data.M, Data.L), dtype=tf.float64) # M x L
q_sqrt = tf.convert_to_tensor(
[np.tril(tf.random.uniform((Data.M, Data.M), dtype=tf.float64)) for _ in range(Data.L)]
) # L x M x M
Z = Data.X[: Data.M, ...] # M x D
N = int(10e5)
Xs = np.ones((N, Data.D), dtype=float_type)
# Path 1: mixed kernel: most efficient route
W = np.random.randn(Data.P, Data.L)
mixed_kernel = mk.LinearCoregionalization([SquaredExponential() for _ in range(Data.L)], W)
optimal_inducing_variable = mf.SharedIndependentInducingVariables(InducingPoints(Z))
value, mean, var = sample_conditional(
Xs, optimal_inducing_variable, mixed_kernel, q_mu, q_sqrt=q_sqrt, white=True
)
# Path 2: independent kernels, mixed later
separate_kernel = mk.SeparateIndependent([SquaredExponential() for _ in range(Data.L)])
fallback_inducing_variable = mf.SharedIndependentInducingVariables(InducingPoints(Z))
value2, mean2, var2 = sample_conditional(
Xs, fallback_inducing_variable, separate_kernel, q_mu, q_sqrt=q_sqrt, white=True
)
value2 = np.matmul(value2, W.T)
# check if mean and covariance of samples are similar
np.testing.assert_array_almost_equal(np.mean(value, axis=0), np.mean(value2, axis=0), decimal=1)
np.testing.assert_array_almost_equal(
np.cov(value, rowvar=False), np.cov(value2, rowvar=False), decimal=1
)
def test_multi_scale_inducing_equivalence_inducing_points(N, M, D):
# Multiscale must be equivalent to inducing points when variance is zero
Xnew, Z = np.random.randn(N, D), np.random.randn(M, D)
rbf = gpflow.kernels.SquaredExponential(1.3441,
lengthscale=np.random.uniform(
0.5, 3., D))
inducing_variable_zero_lengthscale = Multiscale(Z,
scales=np.zeros(Z.shape))
inducing_variable_inducing_point = InducingPoints(Z)
multi_scale_Kuf = Kuf(inducing_variable_zero_lengthscale, rbf, Xnew)
inducing_point_Kuf = Kuf(inducing_variable_inducing_point, rbf, Xnew)
deviation_percent_Kuf = np.max(
np.abs(multi_scale_Kuf - inducing_point_Kuf) / inducing_point_Kuf *
100)
assert deviation_percent_Kuf < 0.1
multi_scale_Kuu = Kuu(inducing_variable_zero_lengthscale, rbf)
inducing_point_Kuu = Kuu(inducing_variable_inducing_point, rbf)
deviation_percent_Kuu = np.max(
np.abs(multi_scale_Kuu - inducing_point_Kuu) / inducing_point_Kuu *
100)
assert deviation_percent_Kuu < 0.1
def main(config):
assert config is not None, ValueError
tf.random.set_seed(config.seed)
gpflow_config.set_default_float(config.floatx)
gpflow_config.set_default_jitter(config.jitter)
X = tf.random.uniform([config.num_test, config.input_dims],
dtype=floatx())
Z_shape = config.num_cond, config.input_dims
for cls in SupportedBaseKernels:
minval = config.rel_lengthscales_min * (config.input_dims**0.5)
maxval = config.rel_lengthscales_max * (config.input_dims**0.5)
lenscales = tf.random.uniform(shape=[config.input_dims],
minval=minval,
maxval=maxval,
dtype=floatx())
q_sqrt = tf.zeros([1] + 2 * [config.num_cond], dtype=floatx())
kern = cls(lengthscales=lenscales,
variance=config.kernel_variance)
Z = InducingPoints(tf.random.uniform(Z_shape, dtype=floatx()))
const = tf.random.normal([1], dtype=floatx())
model = SVGP(kernel=kern,
likelihood=None,
inducing_variable=Z,
mean_function=mean_functions.Constant(c=const),
q_sqrt=q_sqrt)
mf, Sff = subroutine(config, model, X)
mg, Sgg = model.predict_f(X, full_cov=True)
tol = config.error_tol
assert allclose(mf, mg, tol, tol)
assert allclose(Sff, Sgg, tol, tol)
def test_switched_likelihood_regression_valid_num_latent_gps(num_latent_gps):
"""
A Regression test when using Switched likelihood: the number of latent
functions in a GP model must be equal to the number of columns in Y minus
one. The final column of Y is used to index the switch. If the number of
latent functions does not match, an exception will be raised.
"""
x = np.random.randn(100, 1)
y = np.hstack((np.random.randn(100, 1), np.random.randint(0, 3, (100, 1))))
data = x, y
Z = InducingPoints(np.random.randn(num_latent_gps, 1))
likelihoods = [StudentT()] * 3
switched_likelihood = SwitchedLikelihood(likelihoods)
m = gpflow.models.SVGP(
kernel=gpflow.kernels.Matern12(),
inducing_variable=Z,
likelihood=switched_likelihood,
num_latent_gps=num_latent_gps,
)
if num_latent_gps == 1:
_ = m.training_loss(data)
else:
with pytest.raises(tf.errors.InvalidArgumentError):
_ = m.training_loss(data)
def test_dense_separate(config: ConfigFourierDense = None):
if config is None:
config = ConfigFourierDense()
tf.random.set_seed(config.seed)
gpflow_config.set_default_float(config.floatx)
gpflow_config.set_default_jitter(config.jitter)
allZ = []
allK = []
for cls in SupportedBaseKernels:
lenscales = tf.random.uniform(shape=[config.input_dims],
minval=config.lengthscales_min,
maxval=config.lengthscales_max,
dtype=floatx())
rel_variance = tf.random.uniform(shape=[],
minval=0.9,
maxval=1.1,
dtype=floatx())
allK.append(
cls(lengthscales=lenscales,
variance=config.kernel_variance * rel_variance))
allZ.append(
InducingPoints(
tf.random.uniform([config.num_cond, config.input_dims],
dtype=floatx())))
kern = kernels.SeparateIndependent(allK)
Z = SeparateIndependentInducingVariables(allZ)
X = tf.random.uniform([config.num_test, config.input_dims], dtype=floatx())
_test_fourier_dense_common(config, kern, X, Z)
def test_separate_independent_conditional_with_q_sqrt_none():
"""
In response to bug #1523, this test checks that separate_independent_condtional
does not fail when q_sqrt=None.
"""
q_sqrt = None
data = DataMixedKernel
kern_list = [SquaredExponential() for _ in range(data.L)]
kernel = gpflow.kernels.SeparateIndependent(kern_list)
inducing_variable_list = [
InducingPoints(data.X[:data.M, ...]) for _ in range(data.L)
]
inducing_variable = mf.SeparateIndependentInducingVariables(
inducing_variable_list)
mu_1, var_1 = gpflow.conditionals.conditional(
data.X,
inducing_variable,
kernel,
data.mu_data,
full_cov=False,
full_output_cov=False,
q_sqrt=q_sqrt,
white=True,
)
def test_mixed_mok_with_Id_vs_independent_mok():
data = DataMixedKernelWithEye
# Independent model
k1 = mk.SharedIndependent(
SquaredExponential(variance=0.5, lengthscale=1.2), data.L)
f1 = InducingPoints(data.X[:data.M, ...])
model_1 = SVGP(k1,
Gaussian(),
f1,
q_mu=data.mu_data_full,
q_sqrt=data.sqrt_data_full)
set_trainable(model_1, False)
model_1.q_sqrt.trainable = True
@tf.function(autograph=False)
def closure1():
return -model_1.log_marginal_likelihood(Data.X, Data.Y)
gpflow.optimizers.Scipy().minimize(closure1,
variables=model_1.trainable_variables,
method='BFGS')
# Mixed Model
kern_list = [
SquaredExponential(variance=0.5, lengthscale=1.2)
for _ in range(data.L)
]
k2 = mk.LinearCoregionalization(kern_list, data.W)
f2 = InducingPoints(data.X[:data.M, ...])
model_2 = SVGP(k2,
Gaussian(),
f2,
q_mu=data.mu_data_full,
q_sqrt=data.sqrt_data_full)
set_trainable(model_2, False)
model_2.q_sqrt.trainable = True
@tf.function(autograph=False)
def closure2():
return -model_2.log_marginal_likelihood(Data.X, Data.Y)
gpflow.optimizers.Scipy().minimize(closure2,
variables=model_2.trainable_variables,
method='BFGS')
check_equality_predictions(Data.X, Data.Y, [model_1, model_2])
def test_sample_conditional(whiten, full_cov, full_output_cov):
if full_cov and full_output_cov:
return
q_mu = tf.random.uniform((Data.M, Data.P), dtype=tf.float64) # [M, P]
q_sqrt = tf.convert_to_tensor([
np.tril(tf.random.uniform((Data.M, Data.M), dtype=tf.float64))
for _ in range(Data.P)
]) # [P, M, M]
Z = Data.X[:Data.M, ...] # [M, D]
Xs = np.ones((Data.N, Data.D), dtype=float_type)
inducing_variable = InducingPoints(Z)
kernel = SquaredExponential()
# Path 1
value_f, mean_f, var_f = sample_conditional(
Xs,
inducing_variable,
kernel,
q_mu,
q_sqrt=q_sqrt,
white=whiten,
full_cov=full_cov,
full_output_cov=full_output_cov,
num_samples=int(1e5),
)
value_f = value_f.numpy().reshape((-1, ) + value_f.numpy().shape[2:])
# Path 2
if full_output_cov:
pytest.skip(
"sample_conditional with X instead of inducing_variable does not support full_output_cov"
)
value_x, mean_x, var_x = sample_conditional(
Xs,
Z,
kernel,
q_mu,
q_sqrt=q_sqrt,
white=whiten,
full_cov=full_cov,
full_output_cov=full_output_cov,
num_samples=int(1e5),
)
value_x = value_x.numpy().reshape((-1, ) + value_x.numpy().shape[2:])
# check if mean and covariance of samples are similar
np.testing.assert_array_almost_equal(np.mean(value_x, axis=0),
np.mean(value_f, axis=0),
decimal=1)
np.testing.assert_array_almost_equal(np.cov(value_x, rowvar=False),
np.cov(value_f, rowvar=False),
decimal=1)
np.testing.assert_allclose(mean_x, mean_f)
np.testing.assert_allclose(var_x, var_f)
def main(config):
assert config is not None, ValueError
tf.random.set_seed(config.seed)
gpflow_config.set_default_float(config.floatx)
gpflow_config.set_default_jitter(config.jitter)
X = tf.random.uniform([config.num_test, config.input_dims],
dtype=floatx())
allK = []
allZ = []
Z_shape = config.num_cond, config.input_dims
for cls in SupportedBaseKernels:
minval = config.rel_lengthscales_min * (config.input_dims**0.5)
maxval = config.rel_lengthscales_max * (config.input_dims**0.5)
lenscales = tf.random.uniform(shape=[config.input_dims],
minval=minval,
maxval=maxval,
dtype=floatx())
rel_variance = tf.random.uniform(shape=[],
minval=0.9,
maxval=1.1,
dtype=floatx())
allK.append(
cls(lengthscales=lenscales,
variance=config.kernel_variance * rel_variance))
allZ.append(
InducingPoints(tf.random.uniform(Z_shape, dtype=floatx())))
kern = kernels.SeparateIndependent(allK)
Z = SeparateIndependentInducingVariables(allZ)
Kuu = covariances.Kuu(Z,
kern,
jitter=gpflow_config.default_jitter())
q_sqrt = tf.linalg.cholesky(Kuu)\
* tf.random.uniform(shape=[kern.num_latent_gps, 1, 1],
minval=0.0,
maxval=0.5,
dtype=floatx())
const = tf.random.normal([len(kern.kernels)], dtype=floatx())
model = SVGP(kernel=kern,
likelihood=None,
inducing_variable=Z,
mean_function=mean_functions.Constant(c=const),
q_sqrt=q_sqrt,
whiten=False,
num_latent_gps=len(allK))
mf, Sff = subroutine(config, model, X)
mg, Sgg = model.predict_f(X, full_cov=True)
tol = config.error_tol
assert allclose(mf, mg, tol, tol)
assert allclose(Sff, Sgg, tol, tol)
def __init__(self,
kern,
Z,
num_outputs,
mean_function,
white=False,
input_prop_dim=None,
**kwargs):
"""
A sparse variational GP layer in whitened representation. This layer holds the kernel,
variational parameters, inducing points and mean function.
The underlying model at inputs X is
f = Lv + mean_function(X), where v \sim N(0, I) and LL^T = kern.K(X)
The variational distribution over the inducing points is
q(v) = N(q_mu, q_sqrt q_sqrt^T)
The layer holds D_out independent GPs with the same kernel and inducing points.
:param kern: The kernel for the layer (input_dim = D_in)
:param Z: Inducing points (M, D_in)
:param num_outputs: The number of GP outputs (q_mu is shape (M, num_outputs))
:param mean_function: The mean function
:return:
"""
super().__init__(input_prop_dim=input_prop_dim, **kwargs)
self.num_inducing = Z.shape[0]
# Inducing points prior mean
q_mu = np.zeros((self.num_inducing, num_outputs))
self.q_mu = Parameter(q_mu, name="q_mu")
# Square-root of inducing points prior covariance
q_sqrt = np.tile(
np.eye(self.num_inducing)[None, :, :], [num_outputs, 1, 1])
self.q_sqrt = Parameter(q_sqrt, transform=triangular(), name="q_sqrt")
self.feature = InducingPoints(Z)
self.kern = kern
self.mean_function = mean_function
self.num_outputs = num_outputs
self.white = white
if not self.white: # initialize to prior
Ku = self.kern.K(Z)
Lu = np.linalg.cholesky(Ku + np.eye(Z.shape[0]) *
gpflow.default_jitter())
self.q_sqrt = Parameter(np.tile(Lu[None, :, :],
[num_outputs, 1, 1]),
transform=triangular(),
name="q_sqrt")
self.Ku, self.Lu, self.Ku_tiled, self.Lu_tiled = None, None, None, None
self.needs_build_cholesky = True
def test_other_models_full_cov(model_setup, input_dim, output_dim, N, Ntest,
M):
covar_shape = (output_dim, Ntest, Ntest)
X, Y = rng.randn(N, input_dim), rng.randn(N, output_dim)
Z = InducingPoints(rng.randn(M, input_dim))
Xtest = rng.randn(Ntest, input_dim)
model_gp = model_setup.get_model(Z, num_latent=output_dim, data=(X, Y))
mu1, var = model_gp.predict_f(Xtest, full_cov=False)
mu2, covar = model_gp.predict_f(Xtest, full_cov=True)
assert np.allclose(mu1, mu2, atol=1.e-10)
assert covar.shape == covar_shape
assert var.shape == (Ntest, output_dim)
for i in range(output_dim):
assert np.allclose(var[:, i], np.diag(covar[i, :, :]))
def test_smoke():
domain = np.array([[0., 10.]])
kernel = SquaredExponential()
events = rng.uniform(0, 10, size=20)[:, None]
feature = InducingPoints(np.linspace(0, 10, 20)[:, None])
M = len(feature)
m = VBPP(feature, kernel, domain, np.zeros(M), np.eye(M))
Kuu = m.compute_Kuu()
m.q_sqrt.assign(np.linalg.cholesky(Kuu))
assert np.allclose(m.prior_kl(tf.identity(Kuu)).numpy(), 0.0)
def objective_closure():
return - m.elbo(events)
opt = gpflow.optimizers.Scipy()
opt.minimize(objective_closure, m.trainable_variables, options=dict(maxiter=2))
def main(config):
assert config is not None, ValueError
tf.random.set_seed(config.seed)
gpflow_config.set_default_float(config.floatx)
gpflow_config.set_default_jitter(config.jitter)
X = tf.random.uniform([config.num_test, config.input_dims],
dtype=floatx())
Z_shape = config.num_cond, config.input_dims
for cls in SupportedBaseKernels:
minval = config.rel_lengthscales_min * (config.input_dims**0.5)
maxval = config.rel_lengthscales_max * (config.input_dims**0.5)
lenscales = tf.random.uniform(shape=[config.input_dims],
minval=minval,
maxval=maxval,
dtype=floatx())
base = cls(lengthscales=lenscales,
variance=config.kernel_variance)
kern = kernels.SharedIndependent(base, output_dim=2)
Z = SharedIndependentInducingVariables(
InducingPoints(tf.random.uniform(Z_shape, dtype=floatx())))
Kuu = covariances.Kuu(Z,
kern,
jitter=gpflow_config.default_jitter())
q_sqrt = tf.stack([
tf.zeros(2 * [config.num_cond], dtype=floatx()),
tf.linalg.cholesky(Kuu)
])
const = tf.random.normal([2], dtype=floatx())
model = SVGP(kernel=kern,
likelihood=None,
inducing_variable=Z,
mean_function=mean_functions.Constant(c=const),
q_sqrt=q_sqrt,
whiten=False,
num_latent_gps=2)
mf, Sff = subroutine(config, model, X)
mg, Sgg = model.predict_f(X, full_cov=True)
tol = config.error_tol
assert allclose(mf, mg, tol, tol)
assert allclose(Sff, Sgg, tol, tol)
def test_dense(config: ConfigFourierDense = None):
if config is None:
config = ConfigFourierDense()
tf.random.set_seed(config.seed)
gpflow_config.set_default_float(config.floatx)
gpflow_config.set_default_jitter(config.jitter)
X = tf.random.uniform([config.num_test, config.input_dims], dtype=floatx())
Z = tf.random.uniform([config.num_cond, config.input_dims], dtype=floatx())
Z = InducingPoints(Z)
for cls in SupportedBaseKernels:
lenscales = tf.random.uniform(shape=[config.input_dims],
minval=config.lengthscales_min,
maxval=config.lengthscales_max,
dtype=floatx())
kern = cls(lengthscales=lenscales, variance=config.kernel_variance)
_test_fourier_dense_common(config, kern, X, Z)
def test_verify_compatibility_type_errors():
valid_inducing_variable = construct_basic_inducing_variables([35],
input_dim=40)
valid_kernel = construct_basic_kernel([Matern52()])
valid_mean_function = Zero(
) # all gpflow mean functions are currently valid
with pytest.raises(GPLayerIncompatibilityException
): # gpflow kernels must be MultioutputKernels
verify_compatibility(Matern52(), valid_mean_function,
valid_inducing_variable)
Z = valid_inducing_variable.inducing_variable_list[0].Z
inducing_variable = InducingPoints(Z)
with pytest.raises(
GPLayerIncompatibilityException
): # gpflow inducing_variables must be MultioutputInducingVariables
verify_compatibility(valid_kernel, valid_mean_function,
inducing_variable)
def test_switched_likelihood_regression_valid_num_latent(X, Y, num_latent):
"""
A Regression test when using Switched likelihood: the number of latent
functions in a GP model must be equal to the number of columns in Y minus
one. The final column of Y is used to index the switch. If the number of
latent functions does not match, an exception will be raised.
"""
Z = InducingPoints(np.random.randn(num_latent, 1))
likelihoods = [StudentT()] * 3
switched_likelihood = SwitchedLikelihood(likelihoods)
m = gpflow.models.SVGP(kernel=gpflow.kernels.Matern12(),
inducing_variable=Z,
likelihood=switched_likelihood,
num_latent=num_latent)
if num_latent == 1:
m.log_likelihood(X, Y)
else:
with pytest.raises(tf.errors.InvalidArgumentError):
m.log_likelihood(X, Y)
def test_sgpr_posterior_update_cache_with_variables_no_precompute(
register_posterior_test, q_sqrt_factory, whiten,
precompute_cache_type):
kernel = gpflow.kernels.SquaredExponential()
X = np.random.randn(NUM_INDUCING_POINTS, INPUT_DIMS)
Y = np.random.randn(NUM_INDUCING_POINTS, 1)
Z = np.random.randn(NUM_INDUCING_POINTS, INPUT_DIMS)
posterior = SGPRPosterior(
kernel=kernel,
data=(X, Y),
inducing_variable=InducingPoints(Z),
likelihood_variance=gpflow.Parameter(0.1),
num_latent_gps=1,
precompute_cache=precompute_cache_type,
mean_function=Zero(),
)
posterior.update_cache(PrecomputeCacheType.VARIABLE)
register_posterior_test(posterior, SGPRPosterior)
assert isinstance(posterior.alpha, tf.Variable)
assert isinstance(posterior.Qinv, tf.Variable)
def test_multi_scale_inducing_equivalence_inducing_points(N, M, D):
# Multiscale must be equivalent to inducing points when variance is zero
Xnew, Z = np.random.randn(N, D), np.random.randn(M, D)
rbf = gpflow.kernels.SquaredExponential(1.3441,
lengthscales=np.random.uniform(
0.5, 3.0, D))
inducing_variable_zero_lengthscales = Multiscale(Z,
scales=np.zeros(Z.shape) +
1e-10)
inducing_variable_inducing_point = InducingPoints(Z)
multi_scale_Kuf = Kuf(inducing_variable_zero_lengthscales, rbf, Xnew)
inducing_point_Kuf = Kuf(inducing_variable_inducing_point, rbf, Xnew)
relative_error_Kuf = np.abs(multi_scale_Kuf -
inducing_point_Kuf) / inducing_point_Kuf
assert np.max(relative_error_Kuf) < 0.1e-2 # 0.1 %
multi_scale_Kuu = Kuu(inducing_variable_zero_lengthscales, rbf)
inducing_point_Kuu = Kuu(inducing_variable_inducing_point, rbf)
relative_error_Kuu = np.abs(multi_scale_Kuu -
inducing_point_Kuu) / inducing_point_Kuu
assert np.max(relative_error_Kuu) < 0.1e-2 # 0.1 %
def test_separate_independent_mof():
"""
Same test as above but we use different (i.e. separate) inducing inducing
for each of the output dimensions.
"""
np.random.seed(0)
# Model 1 (INefficient)
q_mu_1 = np.random.randn(Data.M * Data.P, 1)
q_sqrt_1 = np.tril(np.random.randn(Data.M * Data.P,
Data.M * Data.P))[None,
...] # 1 x MP x MP
kernel_1 = mk.SharedIndependent(
SquaredExponential(variance=0.5, lengthscale=1.2), Data.P)
inducing_variable_1 = InducingPoints(Data.X[:Data.M, ...])
model_1 = SVGP(kernel_1,
Gaussian(),
inducing_variable_1,
q_mu=q_mu_1,
q_sqrt=q_sqrt_1)
set_trainable(model_1, False)
model_1.q_sqrt.trainable = True
model_1.q_mu.trainable = True
@tf.function(autograph=False)
def closure1():
return -model_1.log_marginal_likelihood(Data.X, Data.Y)
gpflow.optimizers.Scipy().minimize(closure1,
variables=model_1.trainable_variables,
method='BFGS')
# Model 2 (efficient)
q_mu_2 = np.random.randn(Data.M, Data.P)
q_sqrt_2 = np.array([
np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)
]) # P x M x M
kernel_2 = mk.SharedIndependent(
SquaredExponential(variance=0.5, lengthscale=1.2), Data.P)
inducing_variable_list_2 = [
InducingPoints(Data.X[:Data.M, ...]) for _ in range(Data.P)
]
inducing_variable_2 = mf.SeparateIndependentInducingVariables(
inducing_variable_list_2)
model_2 = SVGP(kernel_2,
Gaussian(),
inducing_variable_2,
q_mu=q_mu_2,
q_sqrt=q_sqrt_2)
set_trainable(model_2, False)
model_2.q_sqrt.trainable = True
model_2.q_mu.trainable = True
@tf.function(autograph=False)
def closure2():
return -model_2.log_marginal_likelihood(Data.X, Data.Y)
gpflow.optimizers.Scipy().minimize(closure2,
variables=model_2.trainable_variables,
method='BFGS')
# Model 3 (Inefficient): an idenitical inducing variable is used P times,
# and treated as a separate one.
q_mu_3 = np.random.randn(Data.M, Data.P)
q_sqrt_3 = np.array([
np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)
]) # P x M x M
kern_list = [
SquaredExponential(variance=0.5, lengthscale=1.2)
for _ in range(Data.P)
]
kernel_3 = mk.SeparateIndependent(kern_list)
inducing_variable_list_3 = [
InducingPoints(Data.X[:Data.M, ...]) for _ in range(Data.P)
]
inducing_variable_3 = mf.SeparateIndependentInducingVariables(
inducing_variable_list_3)
model_3 = SVGP(kernel_3,
Gaussian(),
inducing_variable_3,
q_mu=q_mu_3,
q_sqrt=q_sqrt_3)
set_trainable(model_3, False)
model_3.q_sqrt.trainable = True
model_3.q_mu.trainable = True
@tf.function(autograph=False)
def closure3():
return -model_3.log_marginal_likelihood(Data.X, Data.Y)
gpflow.optimizers.Scipy().minimize(closure3,
variables=model_3.trainable_variables,
method='BFGS')
check_equality_predictions(Data.X, Data.Y, [model_1, model_2, model_3])
def test_shared_independent_mok():
"""
In this test we use the same kernel and the same inducing inducing
for each of the outputs. The outputs are considered to be uncorrelated.
This is how GPflow handled multiple outputs before the multioutput framework was added.
We compare three models here:
1) an ineffient one, where we use a SharedIndepedentMok with InducingPoints.
This combination will uses a Kff of size N x P x N x P, Kfu if size N x P x M x P
which is extremely inefficient as most of the elements are zero.
2) efficient: SharedIndependentMok and SharedIndependentMof
This combinations uses the most efficient form of matrices
3) the old way, efficient way: using Kernel and InducingPoints
Model 2) and 3) follow more or less the same code path.
"""
np.random.seed(0)
# Model 1
q_mu_1 = np.random.randn(Data.M * Data.P, 1) # MP x 1
q_sqrt_1 = np.tril(np.random.randn(Data.M * Data.P,
Data.M * Data.P))[None,
...] # 1 x MP x MP
kernel_1 = mk.SharedIndependent(
SquaredExponential(variance=0.5, lengthscale=1.2), Data.P)
inducing_variable = InducingPoints(Data.X[:Data.M, ...])
model_1 = SVGP(kernel_1,
Gaussian(),
inducing_variable,
q_mu=q_mu_1,
q_sqrt=q_sqrt_1,
num_latent=Data.Y.shape[-1])
set_trainable(model_1, False)
model_1.q_sqrt.trainable = True
@tf.function(autograph=False)
def closure1():
return -model_1.log_marginal_likelihood(Data.X, Data.Y)
gpflow.optimizers.Scipy().minimize(closure1,
variables=model_1.trainable_variables,
options=dict(maxiter=500),
method='BFGS')
# Model 2
q_mu_2 = np.reshape(q_mu_1, [Data.M, Data.P]) # M x P
q_sqrt_2 = np.array([
np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)
]) # P x M x M
kernel_2 = SquaredExponential(variance=0.5, lengthscale=1.2)
inducing_variable_2 = InducingPoints(Data.X[:Data.M, ...])
model_2 = SVGP(kernel_2,
Gaussian(),
inducing_variable_2,
num_latent=Data.P,
q_mu=q_mu_2,
q_sqrt=q_sqrt_2)
set_trainable(model_2, False)
model_2.q_sqrt.trainable = True
@tf.function(autograph=False)
def closure2():
return -model_2.log_marginal_likelihood(Data.X, Data.Y)
gpflow.optimizers.Scipy().minimize(closure2,
variables=model_2.trainable_variables,
options=dict(maxiter=500),
method='BFGS')
# Model 3
q_mu_3 = np.reshape(q_mu_1, [Data.M, Data.P]) # M x P
q_sqrt_3 = np.array([
np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)
]) # P x M x M
kernel_3 = mk.SharedIndependent(
SquaredExponential(variance=0.5, lengthscale=1.2), Data.P)
inducing_variable_3 = mf.SharedIndependentInducingVariables(
InducingPoints(Data.X[:Data.M, ...]))
model_3 = SVGP(kernel_3,
Gaussian(),
inducing_variable_3,
num_latent=Data.P,
q_mu=q_mu_3,
q_sqrt=q_sqrt_3)
set_trainable(model_3, False)
model_3.q_sqrt.trainable = True
@tf.function(autograph=False)
def closure3():
return -model_3.log_marginal_likelihood(Data.X, Data.Y)
gpflow.optimizers.Scipy().minimize(closure3,
variables=model_3.trainable_variables,
options=dict(maxiter=500),
method='BFGS')
check_equality_predictions(Data.X, Data.Y, [model_1, model_2, model_3])
def test_inducing_equivalence(N, kernel):
# Inducing inducing must be the same as the kernel evaluations
Z = np.random.randn(N, 5)
inducing_variable = InducingPoints(Z)
assert_allclose(Kuu(inducing_variable, kernel), kernel(Z))
def test_inducing_points_inducing_variable_len(N, D):
Z = np.random.randn(N, D)
inducing_variable = InducingPoints(Z)
assert_equal(len(inducing_variable), N)
relative_error_Kuf = np.abs(multi_scale_Kuf -
inducing_point_Kuf) / inducing_point_Kuf
assert np.max(relative_error_Kuf) < 0.1e-2 # 0.1 %
multi_scale_Kuu = Kuu(inducing_variable_zero_lengthscales, rbf)
inducing_point_Kuu = Kuu(inducing_variable_inducing_point, rbf)
relative_error_Kuu = np.abs(multi_scale_Kuu -
inducing_point_Kuu) / inducing_point_Kuu
assert np.max(relative_error_Kuu) < 0.1e-2 # 0.1 %
_inducing_variables_and_kernels = [
[
2,
InducingPoints(np.random.randn(71, 2)),
gpflow.kernels.SquaredExponential(variance=1.84,
lengthscales=np.random.uniform(
0.5, 3.0, 2)),
],
[
2,
InducingPoints(np.random.randn(71, 2)),
gpflow.kernels.Matern12(variance=1.84,
lengthscales=np.random.uniform(0.5, 3.0, 2)),
],
[
2,
Multiscale(np.random.randn(71, 2),
np.random.uniform(0.5, 3, size=(71, 2))),
gpflow.kernels.SquaredExponential(variance=1.84,
def test_models_with_mean_functions_changes(model_class):
"""
Simply check that all models have a higher prediction with a constant mean
function than with a zero mean function.
For compositions of mean functions check that multiplication/ addition of
a constant results in a higher prediction, whereas addition of zero/
mutliplication with one does not.
"""
data = rng.randn(Datum.N, Datum.input_dim), rng.randn(Datum.N, 1)
predict_at = rng.randn(Datum.Ntest, Datum.input_dim)
inducing_variable = InducingPoints(Z=rng.randn(Datum.M, Datum.input_dim))
kernel = gpflow.kernels.Matern32()
likelihood = gpflow.likelihoods.Gaussian()
zero_mean = Zero()
non_zero_mean = Constant(c=np.ones(1) * 10)
if model_class in [gpflow.models.GPR]:
model_zero_mean = model_class(data, kernel=kernel, mean_function=zero_mean)
model_non_zero_mean = model_class(data, kernel=kernel, mean_function=non_zero_mean)
elif model_class in [gpflow.models.VGP]:
model_zero_mean = model_class(data, likelihood=likelihood, kernel=kernel, mean_function=zero_mean)
model_non_zero_mean = model_class(data, likelihood=likelihood, kernel=kernel, mean_function=non_zero_mean)
elif model_class in [gpflow.models.SVGP]:
model_zero_mean = model_class(kernel=kernel,
likelihood=likelihood,
inducing_variable=inducing_variable,
mean_function=zero_mean)
model_non_zero_mean = model_class(kernel=kernel,
likelihood=likelihood,
inducing_variable=inducing_variable,
mean_function=non_zero_mean)
elif model_class in [gpflow.models.SGPR, gpflow.models.GPRFITC]:
model_zero_mean = model_class(data,
kernel=kernel,
inducing_variable=inducing_variable,
mean_function=zero_mean)
model_non_zero_mean = model_class(data,
kernel=kernel,
inducing_variable=inducing_variable,
mean_function=non_zero_mean)
elif model_class in [gpflow.models.SGPMC]:
model_zero_mean = model_class(data,
kernel=kernel,
likelihood=likelihood,
inducing_variable=inducing_variable,
mean_function=zero_mean)
model_non_zero_mean = model_class(data,
kernel=kernel,
likelihood=likelihood,
inducing_variable=inducing_variable,
mean_function=non_zero_mean)
elif model_class in [gpflow.models.GPMC]:
model_zero_mean = model_class(data,
kernel=kernel,
likelihood=likelihood,
mean_function=zero_mean)
model_non_zero_mean = model_class(data,
kernel=kernel,
likelihood=likelihood,
mean_function=non_zero_mean)
else:
raise (NotImplementedError)
mu_zero, var_zero = model_zero_mean.predict_f(predict_at)
mu_non_zero, var_non_zero = model_non_zero_mean.predict_f(predict_at)
# predictive variance remains unchanged after modifying mean function
assert np.all(var_zero.numpy() == var_non_zero.numpy())
# predictive mean changes after modifying mean function
assert not np.all(mu_zero.numpy() == mu_non_zero.numpy())
def inducing_points():
return InducingPoints(rng.randn(Nn, 1))
# 1.07, 1.38, 1.94, 2.05, 2.36
# ]).astype(default_float())
# q = np.array([
# -2.50, -0.78, -2.50, 1.81, -0.47,
# -0.47, -2.55, 1.81, 1.10, -0.57,
# -2.60, 1.56, -0.62, 1.86, 0.80
# ]).astype(default_float())
# Z_aug = np.random.rand(3 * 5,).astype(default_float())
# num_inducing_points = Z.shape[0]
# axs[0].plot(Z, q, 'o', color='black')
# Shallow sparse GP
Z1 = np.linspace(min(X), max(X),
num=num_inducing_points)[..., None].astype(default_float())
# Z1 = Z[..., None]
feat1 = SharedIndependentInducingVariables(InducingPoints(Z1))
kern1 = SharedIndependent(Kernel(lengthscales=lengthscale,
variance=outer_variance),
output_dim=1)
layer1 = GPLayer(kern1, feat1, X.shape[0], mean_function=Zero(), white=False)
# layer1.q_mu.assign(value=q[..., None])
lik_layer = LikelihoodLayer(Gaussian(variance=likelihood_variance))
model = DeepGP([layer1], lik_layer, input_dim=1, output_dim=1)
model.compile(tf.optimizers.Adam(learning_rate=learning_rate))
callbacks = [
tf.keras.callbacks.ReduceLROnPlateau(
monitor="loss",
patience=patience,
factor=factor,
