def test_monitor(session_tf):
np.random.seed(0)
X = np.random.rand(10000, 1) * 10
Y = np.sin(X) + np.random.randn(*X.shape)
with gpflow.defer_build():
m = gpflow.models.SVGP(X, Y, gpflow.kernels.RBF(1), gpflow.likelihoods.Gaussian(),
Z=np.linspace(0, 10, 5)[:, None],
minibatch_size=100, name="SVGP")
m.likelihood.variance = 0.01
m.compile()
global_step = tf.Variable(0, trainable=False, name="global_step")
session_tf.run(global_step.initializer)
adam = gpflow.train.AdamOptimizer(0.01).make_optimize_action(m, global_step=global_step)
# create a filewriter for summaries
fw = tf.summary.FileWriter('./model_tensorboard', m.graph)
print_lml = mon.PrintTimings(itertools.count(), mon.Trigger.ITER, single_line=True, global_step=global_step)
sleep = mon.SleepAction(itertools.count(), mon.Trigger.ITER, 0.0)
saver = mon.StoreSession(itertools.count(step=3), mon.Trigger.ITER, session_tf,
hist_path="./monitor-saves/checkpoint", global_step=global_step)
tensorboard = mon.ModelTensorBoard(itertools.count(step=3), mon.Trigger.ITER, m, fw, global_step=global_step)
lml_tensorboard = mon.LmlTensorBoard(itertools.count(step=5), mon.Trigger.ITER, m, fw, global_step=global_step)
callback = mon.CallbackAction(mon.seq_exp_lin(2.0, np.inf, 1e-3), mon.Trigger.TOTAL_TIME, lambda x, b: x, m)
actions = [adam, print_lml, tensorboard, lml_tensorboard, saver, sleep, callback]
gpflow.actions.Loop(actions, stop=11)()
def test_sample_conditional_mixedkernel(session_tf):
q_mu = np.random.randn(Data.M , Data.L) # M x L
q_sqrt = np.array([np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.L)]) # L x M x M
Z = Data.X[:Data.M,...] # M x D
N = int(10e5)
Xs = np.ones((N, Data.D), dtype=float_type)
values = {"Xnew": Xs, "q_mu": q_mu, "q_sqrt": q_sqrt}
placeholders = _create_placeholder_dict(values)
feed_dict = _create_feed_dict(placeholders, values)
# Path 1: mixed kernel: most efficient route
W = np.random.randn(Data.P, Data.L)
mixed_kernel = mk.SeparateMixedMok([RBF(Data.D) for _ in range(Data.L)], W)
mixed_feature = mf.MixedKernelSharedMof(InducingPoints(Z.copy()))
sample = sample_conditional(placeholders["Xnew"], mixed_feature, mixed_kernel,
placeholders["q_mu"], q_sqrt=placeholders["q_sqrt"], white=True)
value, mean, var = session_tf.run(sample, feed_dict=feed_dict)
# Path 2: independent kernels, mixed later
separate_kernel = mk.SeparateIndependentMok([RBF(Data.D) for _ in range(Data.L)])
shared_feature = mf.SharedIndependentMof(InducingPoints(Z.copy()))
sample2 = sample_conditional(placeholders["Xnew"], shared_feature, separate_kernel,
placeholders["q_mu"], q_sqrt=placeholders["q_sqrt"], white=True)
value2, mean2, var2 = session_tf.run(sample2, feed_dict=feed_dict)
value2 = np.matmul(value2, W.T)
# check if mean and covariance of samples are similar
np.testing.assert_array_almost_equal(np.mean(value, axis=0),
np.mean(value2, axis=0), decimal=1)
np.testing.assert_array_almost_equal(np.cov(value, rowvar=False),
np.cov(value2, rowvar=False), decimal=1)
def test_rollaxis_idempotent(session_tf, rolls):
A = np.random.randn(10, 5, 3, 20, 1)
A_tf = tf.convert_to_tensor(A)
A_left_right = session_tf.run(_rollaxis_left(_rollaxis_right(A_tf, 2), 2))
A_right_left = session_tf.run(_rollaxis_right(_rollaxis_left(A_tf, 2), 2))
assert_allclose(A, A_left_right)
assert_allclose(A, A_right_left)
def test_mixed_shared(session_tf, fun):
f = Mofs().mixed_shared()
k = Moks().separate_mixed()
if fun is mf.Kuu:
t = tf.cholesky(fun(f, k, jitter=1e-9))
else:
t = fun(f, k, Datum.Xnew)
print(t.shape)
session_tf.run(t)
def test_sample_conditional(session_tf, whiten, full_cov, full_output_cov):
q_mu = np.random.randn(Data.M, Data.P) # M x P
q_sqrt = np.array([
np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)
]) # P x M x M
Z = Data.X[:Data.M, ...] # M x D
Xs = np.ones((Data.N, Data.D), dtype=float_type)
feature = InducingPoints(Z.copy())
kernel = RBF(Data.D)
values = {"Z": Z, "Xnew": Xs, "q_mu": q_mu, "q_sqrt": q_sqrt}
placeholders = _create_placeholder_dict(values)
feed_dict = _create_feed_dict(placeholders, values)
# Path 1
sample_f = sample_conditional(placeholders["Xnew"],
feature,
kernel,
placeholders["q_mu"],
q_sqrt=placeholders["q_sqrt"],
white=whiten,
full_cov=full_cov,
full_output_cov=full_output_cov,
num_samples=int(1e5))
value_f, mean_f, var_f = session_tf.run(sample_f, feed_dict=feed_dict)
value_f = value_f.reshape((-1, ) + value_f.shape[2:])
# Path 2
if full_output_cov:
pytest.skip(
"sample_conditional with X instead of feature does not support full_output_cov"
)
sample_x = sample_conditional(placeholders["Xnew"],
placeholders["Z"],
kernel,
placeholders["q_mu"],
q_sqrt=placeholders["q_sqrt"],
white=whiten,
full_cov=full_cov,
full_output_cov=full_output_cov,
num_samples=int(1e5))
value_x, mean_x, var_x = session_tf.run(sample_x, feed_dict=feed_dict)
value_x = value_x.reshape((-1, ) + value_x.shape[2:])
# check if mean and covariance of samples are similar
np.testing.assert_array_almost_equal(np.mean(value_x, axis=0),
np.mean(value_f, axis=0),
decimal=1)
np.testing.assert_array_almost_equal(np.cov(value_x, rowvar=False),
np.cov(value_f, rowvar=False),
decimal=1)
np.testing.assert_allclose(mean_x, mean_f)
np.testing.assert_allclose(var_x, var_f)
def test_softmax_y_shape_assert(session_tf):
"""
SoftMax assumes the class is given as a label (not, e.g., one-hot
encoded), and hence just uses the first column of Y. To prevent
silent errors, there is a tf assertion that ensures Y only has one
dimension. This test checks that this assert works as intended.
"""
F, Y, feed = _prepare(dimF=5, dimY=2)
l = gpflow.likelihoods.SoftMax(5)
l.compile()
try:
session_tf.run(l.logp(F, Y), feed_dict=feed)
except tf.errors.InvalidArgumentError as e:
assert "assertion failed" in e.message
def test_eKzxKxz_no_uncertainty(session_tf, kernel, feature):
kern = kernel()
eKzxKxz = expectation(dirac_diag(), (kern, feature), (kern, feature))
Kxz = kern.K(Data.Xmu, Data.Z)
eKzxKxz, Kxz = session_tf.run([eKzxKxz, Kxz])
KzxKxz = Kxz[:, :, None] * Kxz[:, None, :]
assert_allclose(eKzxKxz, KzxKxz, rtol=RTOL)
def test_multivariate_normal(session_tf, x, mu, cov_sqrt):
cov = np.dot(cov_sqrt, cov_sqrt.T)
L = np.linalg.cholesky(cov)
if len(x.shape) != 2 or len(mu.shape) != 2:
with pytest.raises(Exception) as e_info:
gp_result = logdensities.multivariate_normal(
tf.convert_to_tensor(x), tf.convert_to_tensor(mu),
tf.convert_to_tensor(L))
else:
x_tf = tf.placeholder(settings.float_type)
mu_tf = tf.placeholder(settings.float_type)
gp_result = logdensities.multivariate_normal(x_tf, mu_tf,
tf.convert_to_tensor(L))
gp_result = session_tf.run(gp_result, feed_dict={x_tf: x, mu_tf: mu})
if mu.shape[1] > 1:
if x.shape[1] > 1:
sp_result = [
mvn.logpdf(x[:, i], mu[:, i], cov)
for i in range(mu.shape[1])
]
else:
sp_result = [
mvn.logpdf(x.ravel(), mu[:, i], cov)
for i in range(mu.shape[1])
]
else:
sp_result = mvn.logpdf(x.T, mu.ravel(), cov)
assert_allclose(gp_result, sp_result)
def test_exKxz_markov_no_uncertainty(session_tf, kernel, feature):
exKxz = expectation(dirac_markov_gauss(), (kernel(), feature),
identity_mean())
exKxz = session_tf.run(exKxz)
Kzx = kernel().compute_K(Data.Xmu_markov[:-1, :], Data.Z) # NxM
xKxz = Kzx[..., None] * Data.Xmu_markov[1:, None, :] # NxMxD
assert_allclose(exKxz, xKxz, rtol=RTOL)
def test_multivariate_normal(session_tf, x, mu, cov_sqrt):
cov = np.dot(cov_sqrt, cov_sqrt.T)
L = np.linalg.cholesky(cov)
if len(x.shape) != 2 or len(mu.shape) != 2:
with pytest.raises(Exception) as e_info:
gp_result = logdensities.multivariate_normal(
tf.convert_to_tensor(x),
tf.convert_to_tensor(mu),
tf.convert_to_tensor(L))
else:
x_tf = tf.placeholder(settings.float_type)
mu_tf = tf.placeholder(settings.float_type)
gp_result = logdensities.multivariate_normal(
x_tf, mu_tf, tf.convert_to_tensor(L))
gp_result = session_tf.run(gp_result, feed_dict={x_tf: x, mu_tf: mu})
if mu.shape[1] > 1:
if x.shape[1] > 1:
sp_result = [mvn.logpdf(x[:,i], mu[:,i], cov) for i in range(mu.shape[1])]
else:
sp_result = [mvn.logpdf(x.ravel(), mu[:, i], cov) for i in range(mu.shape[1])]
else:
sp_result = mvn.logpdf(x.T, mu.ravel(), cov)
assert_allclose(gp_result, sp_result)
def test_sample_mvn(session_tf, cov_structure, num_samples):
"""
Draws 10,000 samples from a distribution
with known mean and covariance. The test checks
if the mean and covariance of the samples is
close to the true mean and covariance.
"""
N, D = 10000, 2
means = tf.ones((N, D), dtype=float_type)
if cov_structure == "full":
covs = tf.eye(D, batch_shape=[N], dtype=float_type)
elif cov_structure == "diag":
covs = tf.ones((N, D), dtype=float_type)
samples = _sample_mvn(means, covs, cov_structure, num_samples=num_samples)
value = session_tf.run(samples)
if num_samples is None:
assert value.shape == (N, D)
else:
assert value.shape == (num_samples, N, D)
value = value.reshape(-1, D)
samples_mean = np.mean(value, axis=0)
samples_cov = np.cov(value, rowvar=False)
np.testing.assert_array_almost_equal(samples_mean, [1., 1.], decimal=1)
np.testing.assert_array_almost_equal(samples_cov, [[1., 0.], [0., 1.]], decimal=1)
def test_sample_mvn(session_tf, cov_structure, num_samples):
"""
Draws 10,000 samples from a distribution
with known mean and covariance. The test checks
if the mean and covariance of the samples is
close to the true mean and covariance.
"""
N, D = 10000, 2
means = tf.ones((N, D), dtype=float_type)
if cov_structure == "full":
covs = tf.eye(D, batch_shape=[N], dtype=float_type)
elif cov_structure == "diag":
covs = tf.ones((N, D), dtype=float_type)
samples = _sample_mvn(means, covs, cov_structure, num_samples=num_samples)
value = session_tf.run(samples)
if num_samples is None:
assert value.shape == (N, D)
else:
assert value.shape == (num_samples, N, D)
value = value.reshape(-1, D)
samples_mean = np.mean(value, axis=0)
samples_cov = np.cov(value, rowvar=False)
np.testing.assert_array_almost_equal(samples_mean, [1., 1.], decimal=1)
np.testing.assert_array_almost_equal(samples_cov, [[1., 0.], [0., 1.]],
decimal=1)
def test_eKzxKxz_no_uncertainty(session_tf, kernel, feature):
kern = kernel()
eKzxKxz = expectation(dirac_diag(), (kern, feature), (kern, feature))
Kxz = kern.K(Data.Xmu, Data.Z)
eKzxKxz, Kxz = session_tf.run([eKzxKxz, Kxz])
KzxKxz = Kxz[:, :, None] * Kxz[:, None, :]
assert_allclose(eKzxKxz, KzxKxz, rtol=RTOL)
def test_broadcasting_mix_latent_gps(session_tf, full_cov, full_output_cov):
S, N = 7, 20 # batch size, num data points
P, L = 10, 5 # observation dimensionality, num latent GPs
W = np.random.randn(P, L) # mixing matrix
g_mu = np.random.randn(S, N, L) # mean of the L latent GPs
g_sqrt_diag = np.tril(np.random.randn(S * L, N, N), -1) # [L*S, N, N]
g_sqrt_diag = np.reshape(g_sqrt_diag, [L, S, N, N])
g_var_diag = g_sqrt_diag @ np.transpose(g_sqrt_diag,
[0, 1, 3, 2]) # [L, S, N, N]
g_var = np.zeros([S, N, L, N, L])
for l in range(L):
g_var[:, :, l, :, l] = g_var_diag[
l, :, :, :] # replace diagonal elements by g_var_diag
# reference numpy implementation for mean
f_mu_ref = g_mu @ W.T # [S, N, P]
# reference numpy implementation for variance
g_var_tmp = np.transpose(g_var, [0, 1, 3, 2, 4]) # [S, N, N, L, L]
f_var_ref = W @ g_var_tmp @ W.T # [S, N, N, P, P]
f_var_ref = np.transpose(f_var_ref, [0, 1, 3, 2, 4]) # [S, N, P, N, P]
if not full_cov:
g_var_diag = np.array([g_var_diag[:, :, n, n]
for n in range(N)]) # [N, L, S]
g_var_diag = np.transpose(g_var_diag, [2, 0, 1]) # [S, N, L]
# run gpflow's implementation
f_mu, f_var = session_tf.run(
_mix_latent_gp(tf.convert_to_tensor(W), tf.convert_to_tensor(g_mu),
tf.convert_to_tensor(g_var_diag), full_cov,
full_output_cov))
# we strip down f_var_ref to the elements we need
if not full_output_cov and not full_cov:
f_var_ref = np.array([f_var_ref[:, :, p, :, p]
for p in range(P)]) # [P, S, N, N]
f_var_ref = np.array([f_var_ref[:, :, n, n]
for n in range(N)]) # [N, P, S]
f_var_ref = np.transpose(f_var_ref, [2, 0, 1]) # [S, N, P]
elif not full_output_cov and full_cov:
f_var_ref = np.array([f_var_ref[:, :, p, :, p]
for p in range(P)]) # [P, S, N, N]
f_var_ref = np.transpose(f_var_ref, [1, 0, 2, 3]) # [S, P, N, N]
elif full_output_cov and not full_cov:
f_var_ref = np.array([f_var_ref[:, n, :, n, :]
for n in range(N)]) # [N, S, P, P]
f_var_ref = np.transpose(f_var_ref, [1, 0, 2, 3]) # [S, N, P, P]
else:
pass # f_var_ref has shape [..., N, P, N, P] as expected
# check equality for mean and variance of f
assert_allclose(f_mu_ref, f_mu)
assert_allclose(f_var_ref, f_var)
def test_sample_conditional_mixedkernel(session_tf):
q_mu = np.random.randn(Data.M, Data.L) # M x L
q_sqrt = np.array([
np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.L)
]) # L x M x M
Z = Data.X[:Data.M, ...] # M x D
N = int(10e5)
Xs = np.ones((N, Data.D), dtype=float_type)
values = {"Xnew": Xs, "q_mu": q_mu, "q_sqrt": q_sqrt}
placeholders = _create_placeholder_dict(values)
feed_dict = _create_feed_dict(placeholders, values)
# Path 1: mixed kernel: most efficient route
W = np.random.randn(Data.P, Data.L)
mixed_kernel = mk.SeparateMixedMok([RBF(Data.D) for _ in range(Data.L)], W)
mixed_feature = mf.MixedKernelSharedMof(InducingPoints(Z.copy()))
sample = sample_conditional(placeholders["Xnew"],
mixed_feature,
mixed_kernel,
placeholders["q_mu"],
q_sqrt=placeholders["q_sqrt"],
white=True)
value = session_tf.run(sample, feed_dict=feed_dict)
# Path 2: independent kernels, mixed later
separate_kernel = mk.SeparateIndependentMok(
[RBF(Data.D) for _ in range(Data.L)])
shared_feature = mf.SharedIndependentMof(InducingPoints(Z.copy()))
sample2 = sample_conditional(placeholders["Xnew"],
shared_feature,
separate_kernel,
placeholders["q_mu"],
q_sqrt=placeholders["q_sqrt"],
white=True)
value2 = session_tf.run(sample2, feed_dict=feed_dict)
value2 = np.matmul(value2, W.T)
# check if mean and covariance of samples are similar
np.testing.assert_array_almost_equal(np.mean(value, axis=0),
np.mean(value2, axis=0),
decimal=1)
np.testing.assert_array_almost_equal(np.cov(value, rowvar=False),
np.cov(value2, rowvar=False),
decimal=1)
def test_diagquad_logspace(session_tf, mu1, var1, mu2, var2):
alpha = 2.5
quad = gpflow.quadrature.ndiagquad(lambda *X: (X[0] + alpha * X[1]),
25, [cast(mu1), cast(mu2)],
[cast(var1), cast(var2)],
logspace=True)
res = session_tf.run(quad)
expected = mu1 + var1 / 2 + alpha * mu2 + alpha**2 * var2 / 2
assert_allclose(res, expected)
def test_unknown_size_inputs(session_tf):
"""
Test for #725 and #734. When the shape of the Gaussian's mean had at least
one unknown parameter, `gauss_kl` would blow up. This happened because
`tf.size` can only output types `tf.int32` or `tf.int64`.
"""
mu_ph = tf.placeholder(settings.float_type, [None, None])
sqrt_ph = tf.placeholder(settings.float_type, [None, None, None])
mu = np.ones([1, 4], dtype=settings.float_type)
sqrt = np.ones([4, 1, 1], dtype=settings.float_type)
feed_dict = {mu_ph: mu, sqrt_ph: sqrt}
known_shape_tf = gauss_kl(*map(tf.constant, [mu, sqrt]))
unknown_shape_tf = gauss_kl(mu_ph, sqrt_ph)
known_shape = session_tf.run(known_shape_tf)
unknown_shape = session_tf.run(unknown_shape_tf, feed_dict=feed_dict)
np.testing.assert_allclose(known_shape, unknown_shape)
def test_diagquad_with_kwarg(session_tf, mu2, var2):
alpha = np.array([2.5, -1.3])
quad = gpflow.quadrature.ndiagquad(lambda X, Y: tf.exp(X * Y),
25,
cast(mu2),
cast(var2),
Y=alpha)
res = session_tf.run(quad)
expected = np.exp(alpha * mu2 + alpha**2 * var2 / 2)
assert_allclose(res, expected)
def test_unknown_size_inputs(session_tf):
"""
Test for #725 and #734. When the shape of the Gaussian's mean had at least
one unknown parameter, `gauss_kl` would blow up. This happened because
`tf.size` can only output types `tf.int32` or `tf.int64`.
"""
mu_ph = tf.placeholder(settings.float_type, [None, None])
sqrt_ph = tf.placeholder(settings.float_type, [None, None, None])
mu = np.ones([1, 4], dtype=settings.float_type)
sqrt = np.ones([4, 1, 1], dtype=settings.float_type)
feed_dict = {mu_ph: mu, sqrt_ph: sqrt}
known_shape_tf = gauss_kl(*map(tf.constant, [mu, sqrt]))
unknown_shape_tf = gauss_kl(mu_ph, sqrt_ph)
known_shape = session_tf.run(known_shape_tf)
unknown_shape = session_tf.run(unknown_shape_tf, feed_dict=feed_dict)
np.testing.assert_allclose(known_shape, unknown_shape)
def test_diagquad_2d(session_tf, mu1, var1, mu2, var2):
alpha = 2.5
quad = gpflow.quadrature.ndiagquad(
lambda *X: tf.exp(X[0] + alpha * X[1]),
35, # using logspace=True we can reduce this, see test_diagquad_logspace
[cast(mu1), cast(mu2)],
[cast(var1), cast(var2)])
res = session_tf.run(quad)
expected = np.exp(mu1 + var1 / 2 + alpha * mu2 + alpha**2 * var2 / 2)
assert_allclose(res, expected)
def test_shape_asserts(session_tf):
A = np.random.randn(5)
B = np.random.randn(5)
L = np.tril(np.random.randn(5, 5))
# Static shape check:
with pytest.raises(ValueError):
tA = tf.identity(A)
tB = tf.identity(B)
tL = tf.identity(L)
res = logdensities.multivariate_normal(tA, tB, tL)
# Dynamic shape check:
# the following results in a segfault before PR#964
with pytest.raises(tf.errors.InvalidArgumentError):
vA = tf.placeholder(tf.float64)
vB = tf.placeholder(tf.float64)
vL = tf.placeholder(tf.float64)
res = logdensities.multivariate_normal(vA, vB, vL)
session_tf.run(res, {vA: A, vB: B, vL: L})
def test_sample_conditional(session_tf, whiten):
q_mu = np.random.randn(Data.M, Data.P) # M x P
q_sqrt = np.array([
np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)
]) # P x M x M
Z = Data.X[:Data.M, ...] # M x D
Xs = np.ones((int(10e5), Data.D), dtype=float_type)
feature = InducingPoints(Z.copy())
kernel = RBF(Data.D)
values = {"Z": Z, "Xnew": Xs, "q_mu": q_mu, "q_sqrt": q_sqrt}
placeholders = _create_placeholder_dict(values)
feed_dict = _create_feed_dict(placeholders, values)
# Path 1
sample = sample_conditional(placeholders["Xnew"],
placeholders["Z"],
kernel,
placeholders["q_mu"],
q_sqrt=placeholders["q_sqrt"],
white=whiten)
value = session_tf.run(sample, feed_dict=feed_dict)
# Path 2
sample2 = sample_conditional(placeholders["Xnew"],
feature,
kernel,
placeholders["q_mu"],
q_sqrt=placeholders["q_sqrt"],
white=whiten)
value2 = session_tf.run(sample2, feed_dict=feed_dict)
# check if mean and covariance of samples are similar
np.testing.assert_array_almost_equal(np.mean(value, axis=0),
np.mean(value2, axis=0),
decimal=1)
np.testing.assert_array_almost_equal(np.cov(value, rowvar=False),
np.cov(value2, rowvar=False),
decimal=1)
def test_hypers_SVGP_vs_SGPR_tensors(session_tf, svgp, sgpr):
"""
Test SVGP vs SGPR. Running optimization as tensors w/o GPflow wrapper.
"""
anchor = False
variationals = [(svgp.q_mu, svgp.q_sqrt)]
svgp.q_mu.trainable = False
svgp.q_sqrt.trainable = False
o1 = NatGradOptimizer(Datum.gamma)
o1_tensor = o1.make_optimize_tensor(svgp, var_list=variationals)
o2 = GradientDescentOptimizer(Datum.learning_rate)
o2_tensor = o2.make_optimize_tensor(svgp)
o3 = NatGradOptimizer(Datum.gamma)
o3_tensor = o3.make_optimize_tensor(svgp, var_list=variationals)
session_tf.run(o1_tensor)
sgpr_likelihood = sgpr.compute_log_likelihood()
svgp_likelihood = svgp.compute_log_likelihood()
assert_allclose(sgpr_likelihood, svgp_likelihood, atol=1e-5)
session_tf.run(o2_tensor)
session_tf.run(o3_tensor)
GradientDescentOptimizer(Datum.learning_rate).minimize(sgpr, maxiter=1, anchor=anchor)
sgpr_likelihood = sgpr.compute_log_likelihood()
svgp_likelihood = svgp.compute_log_likelihood()
assert_allclose(sgpr_likelihood, svgp_likelihood, atol=1e-5)
def test_base_conditional_vs_ref(session_tf, full_cov,
features_inducing_points):
"""
Test that conditionals agree with a slow-but-clear numpy implementation
"""
Dy, N, M, Dx = 5, 4, 3, 2
X = np.random.randn(N, Dx)
Z = np.random.randn(M, Dx)
kern = gpflow.kernels.Matern52(Dx, lengthscales=0.5)
q_mu = np.random.randn(M, Dy)
q_sqrt = np.tril(np.random.randn(Dy, M, M), -1)
def numpy_conditional(X, Z, kern, q_mu, q_sqrt):
Kmm = kern.compute_K_symm(
Z) + np.eye(M) * settings.numerics.jitter_level
Kmn = kern.compute_K(Z, X)
Knn = kern.compute_K_symm(X)
Kmm, Kmn, Knm, Knn = [
np.tile(k[None, :, :], [Dy, 1, 1]) for k in [Kmm, Kmn, Kmn.T, Knn]
]
S = q_sqrt @ np.transpose(q_sqrt, [0, 2, 1])
Kmm_inv = np.linalg.inv(Kmm)
mean = np.einsum('dmn,dmM,Md->nd', Kmn, Kmm_inv, q_mu)
cov = Knn + Knm @ Kmm_inv @ (S - Kmm) @ Kmm_inv @ Kmn
return mean, cov
mean_np, cov_np = numpy_conditional(X, Z, kern, q_mu, q_sqrt)
if features_inducing_points:
Z = gpflow.features.InducingPoints(Z)
mean_tf, cov_tf = gpflow.conditionals.conditional(
X,
Z,
kern,
q_mu,
q_sqrt=tf.identity(q_sqrt),
white=False,
full_cov=full_cov)
mean_tf, cov_tf = session_tf.run([mean_tf, cov_tf])
if not full_cov:
cov_np = np.diagonal(cov_np, axis1=-1, axis2=-2).T
assert_allclose(mean_np, mean_tf)
assert_allclose(cov_np, cov_tf)
def test_sample_conditional(session_tf, whiten, full_cov, full_output_cov):
q_mu = np.random.randn(Data.M , Data.P) # M x P
q_sqrt = np.array([np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)]) # P x M x M
Z = Data.X[:Data.M, ...] # M x D
Xs = np.ones((Data.N, Data.D), dtype=float_type)
feature = InducingPoints(Z.copy())
kernel = RBF(Data.D)
values = {"Z": Z, "Xnew": Xs, "q_mu": q_mu, "q_sqrt": q_sqrt}
placeholders = _create_placeholder_dict(values)
feed_dict = _create_feed_dict(placeholders, values)
# Path 1
sample_f = sample_conditional(placeholders["Xnew"], feature, kernel,
placeholders["q_mu"], q_sqrt=placeholders["q_sqrt"], white=whiten,
full_cov=full_cov, full_output_cov=full_output_cov, num_samples=int(1e5))
value_f, mean_f, var_f = session_tf.run(sample_f, feed_dict=feed_dict)
value_f = value_f.reshape((-1,) + value_f.shape[2:])
# Path 2
if full_output_cov:
pytest.skip("sample_conditional with X instead of feature does not support full_output_cov")
sample_x = sample_conditional(placeholders["Xnew"], placeholders["Z"], kernel,
placeholders["q_mu"], q_sqrt=placeholders["q_sqrt"], white=whiten,
full_cov=full_cov, full_output_cov=full_output_cov, num_samples=int(1e5))
value_x, mean_x, var_x = session_tf.run(sample_x, feed_dict=feed_dict)
value_x = value_x.reshape((-1,) + value_x.shape[2:])
# check if mean and covariance of samples are similar
np.testing.assert_array_almost_equal(np.mean(value_x, axis=0),
np.mean(value_f, axis=0), decimal=1)
np.testing.assert_array_almost_equal(np.cov(value_x, rowvar=False),
np.cov(value_f, rowvar=False), decimal=1)
np.testing.assert_allclose(mean_x, mean_f)
np.testing.assert_allclose(var_x, var_f)
def test_RBF_eKzxKxz_gradient_not_NaN(session_tf):
"""
Ensure that <K_{Z, x} K_{x, Z}>_p(x) is not NaN and correct, when
K_{Z, Z} is zero with finite precision. See pull request #595.
"""
kern = gpflow.kernels.RBF(1, lengthscales=0.1)
kern.variance = 2.
p = gpflow.probability_distributions.Gaussian(
tf.constant([[10]], dtype=gpflow.settings.tf_float),
tf.constant([[[0.1]]], dtype=gpflow.settings.tf_float))
z = gpflow.features.InducingPoints([[-10.], [10.]])
ekz = expectation(p, (kern, z), (kern, z))
g, = tf.gradients(ekz, kern.lengthscales._unconstrained_tensor)
grad = session_tf.run(g)
assert grad is not None and not np.isnan(grad)
def test_rollaxis(session_tf, rolls, direction):
A = np.random.randn(10, 5, 3)
A_tf = tf.convert_to_tensor(A)
if direction == "left":
perm = [1, 2, 0] if rolls == 1 else [2, 0, 1]
elif direction == "right":
perm = [2, 0, 1] if rolls == 1 else [1, 2, 0]
A_rolled_ref = np.transpose(A, perm)
if direction == "left":
A_rolled_tf = _rollaxis_left(A_tf, rolls)
elif direction == "right":
A_rolled_tf = _rollaxis_right(A_tf, rolls)
A_rolled_tf = session_tf.run(A_rolled_tf)
assert_allclose(A_rolled_ref, A_rolled_tf)
def test_multivariate_normal(session_tf, x, mu, cov):
cov = np.dot(cov, cov.T)
L = np.linalg.cholesky(cov)
gp_result = densities.multivariate_normal(x, mu, L)
gp_result = session_tf.run(gp_result)
if len(mu.shape) > 1 and mu.shape[1] > 1:
if len(x.shape) > 1 and x.shape[1] > 1:
sp_result = [
mvn.logpdf(x[:, i], mu[:, i], cov) for i in range(mu.shape[1])
]
else:
sp_result = [
mvn.logpdf(x.ravel(), mu[:, i], cov)
for i in range(mu.shape[1])
]
else:
sp_result = mvn.logpdf(x.T, mu.ravel(), cov)
assert_allclose(gp_result, sp_result)
def test_RBF_eKzxKxz_gradient_notNaN(session_tf):
"""
Ensure that <K_{Z, x} K_{x, Z}>_p(x) is not NaN and correct, when
K_{Z, Z} is zero with finite precision. See pull request #595.
"""
kern = gpflow.kernels.RBF(1, lengthscales=0.1)
kern.variance = 2.
p = gpflow.probability_distributions.Gaussian(
tf.constant([[10]], dtype=gpflow.settings.tf_float),
tf.constant([[[0.1]]], dtype=gpflow.settings.tf_float))
z = gpflow.features.InducingPoints([[-10.], [10.]])
ekz = expectation(p, (kern, z), (kern, z))
g, = tf.gradients(ekz, kern.lengthscales._unconstrained_tensor)
grad = session_tf.run(g)
assert grad is not None and not np.isnan(grad)
def test_kernel_euclidean_distance(session_tf, kernel):
'''
Tests output & gradients of kernels that are a function of the (scaled) euclidean distance
of the points. We test on a high dimensional space, which can generate very small distances
causing the scaled_square_dist to generate some negative values.
'''
k = kernel(Datum.D)
K = k.compute_K_symm(Datum.X)
assert not np.isnan(K).any(
), 'There are NaNs in the output of the ' + kernel.__name__ + ' kernel.'
assert np.isfinite(K).all(
), 'There are Infs in the output of the ' + kernel.__name__ + ' kernel.'
X = tf.placeholder(settings.float_type)
dK = session_tf.run(tf.gradients(k.K(X, X), X)[0], feed_dict={X: Datum.X})
assert not np.isnan(dK).any(
), 'There are NaNs in the gradient of the ' + kernel.__name__ + ' kernel.'
assert np.isfinite(dK).all(
), 'There are Infs in the output of the ' + kernel.__name__ + ' kernel.'
def test_hypers_SVGP_vs_SGPR_tensors(session_tf, svgp, sgpr):
"""
Test SVGP vs SGPR. Running optimization as tensors w/o GPflow wrapper.
"""
anchor = False
variationals = [(svgp.q_mu, svgp.q_sqrt)]
svgp.q_mu.trainable = False
svgp.q_sqrt.trainable = False
o1 = NatGradOptimizer(Datum.gamma)
o1.minimize(svgp, var_list=variationals, maxiter=0, anchor=anchor)
o1_tensor = o1.minimize_operation
o2 = GradientDescentOptimizer(Datum.learning_rate)
o2.minimize(svgp, maxiter=0, anchor=anchor)
o2_tensor = o2.minimize_operation
o3 = NatGradOptimizer(Datum.gamma)
o3.minimize(svgp, var_list=variationals, maxiter=0, anchor=anchor)
o3_tensor = o3.minimize_operation
session_tf.run(o1_tensor)
sgpr_likelihood = sgpr.compute_log_likelihood()
svgp_likelihood = svgp.compute_log_likelihood()
assert_allclose(sgpr_likelihood, svgp_likelihood, atol=1e-5)
session_tf.run(o2_tensor)
session_tf.run(o3_tensor)
GradientDescentOptimizer(Datum.learning_rate).minimize(sgpr,
maxiter=1,
anchor=anchor)
sgpr_likelihood = sgpr.compute_log_likelihood()
svgp_likelihood = svgp.compute_log_likelihood()
assert_allclose(sgpr_likelihood, svgp_likelihood, atol=1e-5)
def test_eKdiag_no_uncertainty(session_tf, kernel):
eKdiag = expectation(dirac_diag(), kernel())
Kdiag = kernel().Kdiag(Data.Xmu)
eKdiag, Kdiag = session_tf.run([eKdiag, Kdiag])
assert_allclose(eKdiag, Kdiag, rtol=RTOL)
def test_exKxz_markov_no_uncertainty(session_tf, kernel, feature):
exKxz = expectation(dirac_markov_gauss(), (kernel(), feature), identity_mean())
exKxz = session_tf.run(exKxz)
Kzx = kernel().compute_K(Data.Xmu_markov[:-1, :], Data.Z) # NxM
xKxz = Kzx[..., None] * Data.Xmu_markov[1:, None, :] # NxMxD
assert_allclose(exKxz, xKxz, rtol=RTOL)
def test_diagquad_1d(session_tf, mu1, var1):
quad = gpflow.quadrature.ndiagquad([lambda *X: tf.exp(X[0])], 25,
[cast(mu1)], [cast(var1)])
res, = session_tf.run(quad)
expected = np.exp(mu1 + var1 / 2)
assert_allclose(res, expected)
def test_eMxKxz_no_uncertainty(session_tf, kernel, feature, mean):
exKxz = expectation(dirac_diag(), mean(), (kernel(), feature))
Kxz = kernel().K(Data.Xmu, Data.Z)
xKxz = expectation(dirac_gauss(), mean())[:, :, None] * Kxz[:, None, :]
exKxz, xKxz = session_tf.run([exKxz, xKxz])
assert_allclose(exKxz, xKxz, rtol=RTOL)
def test_conditional_broadcasting(session_tf, full_cov, white,
conditional_type):
"""
Test that the `conditional` and `sample_conditional` broadcasts correctly
over leading dimensions of Xnew. Xnew can be shape [..., N, D],
and conditional should broadcast over the [...].
"""
X_ = tf.placeholder(tf.float64, [None, None])
q_mu = np.random.randn(Data.M, Data.Dy)
q_sqrt = np.tril(np.random.randn(Data.Dy, Data.M, Data.M), -1)
if conditional_type == "Z":
feat = Data.Z
kern = gpflow.kernels.Matern52(Data.Dx, lengthscales=0.5)
elif conditional_type == "inducing_points":
feat = gpflow.features.InducingPoints(Data.Z)
kern = gpflow.kernels.Matern52(Data.Dx, lengthscales=0.5)
elif conditional_type == "mixing":
# variational params have different output dim in this case
q_mu = np.random.randn(Data.M, Data.L)
q_sqrt = np.tril(np.random.randn(Data.L, Data.M, Data.M), -1)
feat = mf.MixedKernelSharedMof(gpflow.features.InducingPoints(Data.Z))
kern = mk.SeparateMixedMok(kernels=[
gpflow.kernels.Matern52(Data.Dx, lengthscales=0.5)
for _ in range(Data.L)
],
W=Data.W)
if conditional_type == "mixing" and full_cov:
pytest.skip("combination is not implemented")
num_samples = 5
sample_tf, mean_tf, cov_tf = sample_conditional(
X_,
feat,
kern,
tf.convert_to_tensor(q_mu),
q_sqrt=tf.convert_to_tensor(q_sqrt),
white=white,
full_cov=full_cov,
num_samples=num_samples)
ss, ms, vs = [], [], []
for X in Data.SX:
s, m, v = session_tf.run([sample_tf, mean_tf, cov_tf], {X_: X})
ms.append(m)
vs.append(v)
ss.append(s)
ms = np.array(ms)
vs = np.array(vs)
ss = np.array(ss)
ss_S12, ms_S12, vs_S12 = session_tf.run(
sample_conditional(Data.SX,
feat,
kern,
tf.convert_to_tensor(q_mu),
q_sqrt=tf.convert_to_tensor(q_sqrt),
white=white,
full_cov=full_cov,
num_samples=num_samples))
ss_S1_S2, ms_S1_S2, vs_S1_S2 = session_tf.run(
sample_conditional(Data.S1_S2_X,
feat,
kern,
tf.convert_to_tensor(q_mu),
q_sqrt=tf.convert_to_tensor(q_sqrt),
white=white,
full_cov=full_cov,
num_samples=num_samples))
assert_allclose(ss_S12.shape, ss.shape)
assert_allclose(ms_S12, ms)
assert_allclose(vs_S12, vs)
assert_allclose(ms_S1_S2.reshape(Data.S1 * Data.S2, Data.N, Data.Dy), ms)
assert_allclose(ss_S1_S2.shape,
[Data.S1, Data.S2, num_samples, Data.N, Data.Dy])
if full_cov:
assert_allclose(
vs_S1_S2.reshape(Data.S1 * Data.S2, Data.Dy, Data.N, Data.N), vs)
else:
assert_allclose(vs_S1_S2.reshape(Data.S1 * Data.S2, Data.N, Data.Dy),
vs)
def test_eMxKxz_no_uncertainty(session_tf, kernel, feature, mean):
exKxz = expectation(dirac_diag(), mean(), (kernel(), feature))
Kxz = kernel().K(Data.Xmu, Data.Z)
xKxz = expectation(dirac_gauss(), mean())[:, :, None] * Kxz[:, None, :]
exKxz, xKxz = session_tf.run([exKxz, xKxz])
assert_allclose(exKxz, xKxz, rtol=RTOL)
def test_eKxz_no_uncertainty(session_tf, kernel, feature):
eKxz = expectation(dirac_diag(), (kernel(), feature))
Kxz = kernel().K(Data.Xmu, Data.Z)
eKxz, Kxz = session_tf.run([eKxz, Kxz])
assert_allclose(eKxz, Kxz, rtol=RTOL)
def test_exKxz_pairwise_no_uncertainty(session_tf, kernel, feature):
exKxz_pairwise = expectation(dirac_markov_gauss(), (feature, kernel()), identity())
exKxz_pairwise = session_tf.run(exKxz_pairwise)
Kxz = kernel().compute_K(Data.Xmu[:-1, :], Data.Z) # NxM
xKxz_pairwise = np.einsum('nm,nd->nmd', Kxz, Data.Xmu[1:, :])
assert_almost_equal(exKxz_pairwise, xKxz_pairwise)
def test_eKxzzx_no_uncertainty(session_tf, kernel, feature):
eKxzzx = expectation(dirac(), (feature, kernel()), (feature, kernel()))
Kxz = kernel().K(Data.Xmu, Data.Z)
eKxzzx, Kxz = session_tf.run([eKxzzx, Kxz])
Kxzzx = Kxz[:, :, None] * Kxz[:, None, :]
assert_almost_equal(eKxzzx, Kxzzx)
def test_eKdiag_no_uncertainty(session_tf, kernel):
eKdiag = expectation(dirac_diag(), kernel())
Kdiag = kernel().Kdiag(Data.Xmu)
eKdiag, Kdiag = session_tf.run([eKdiag, Kdiag])
assert_allclose(eKdiag, Kdiag, rtol=RTOL)
def test_eKxz_no_uncertainty(session_tf, kernel, feature):
eKxz = expectation(dirac_diag(), (kernel(), feature))
Kxz = kernel().K(Data.Xmu, Data.Z)
eKxz, Kxz = session_tf.run([eKxz, Kxz])
assert_allclose(eKxz, Kxz, rtol=RTOL)
