def test_separate_independent_mok(session_tf):
"""
We use different independent kernels for each of the output dimensions.
We can achieve this in two ways:
1) efficient: SeparateIndependentMok with Shared/SeparateIndependentMof
2) inefficient: SeparateIndependentMok with InducingPoints
However, both methods should return the same conditional,
and after optimization return the same log likelihood.
"""
# Model 1 (INefficient)
q_mu_1 = np.random.randn(Data.M * Data.P, 1)
q_sqrt_1 = np.tril(np.random.randn(Data.M * Data.P, Data.M * Data.P))[None, ...] # 1 x MP x MP
kern_list_1 = [RBF(Data.D, variance=0.5, lengthscales=1.2) for _ in range(Data.P)]
kernel_1 = mk.SeparateIndependentMok(kern_list_1)
feature_1 = InducingPoints(Data.X[:Data.M,...].copy())
m1 = SVGP(Data.X, Data.Y, kernel_1, Gaussian(), feature_1, q_mu=q_mu_1, q_sqrt=q_sqrt_1)
m1.set_trainable(False)
m1.q_sqrt.set_trainable(True)
m1.q_mu.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m1, maxiter=Data.MAXITER)
# Model 2 (efficient)
q_mu_2 = np.random.randn(Data.M, Data.P)
q_sqrt_2 = np.array([np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)]) # P x M x M
kern_list_2 = [RBF(Data.D, variance=0.5, lengthscales=1.2) for _ in range(Data.P)]
kernel_2 = mk.SeparateIndependentMok(kern_list_2)
feature_2 = mf.SharedIndependentMof(InducingPoints(Data.X[:Data.M, ...].copy()))
m2 = SVGP(Data.X, Data.Y, kernel_2, Gaussian(), feature_2, q_mu=q_mu_2, q_sqrt=q_sqrt_2)
m2.set_trainable(False)
m2.q_sqrt.set_trainable(True)
m2.q_mu.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m2, maxiter=Data.MAXITER)
check_equality_predictions(session_tf, [m1, m2])
def test_mixed_mok_with_Id_vs_independent_mok(session_tf):
data = DataMixedKernelWithEye
# Independent model
k1 = mk.SharedIndependentMok(RBF(data.D, variance=0.5, lengthscales=1.2),
data.L)
f1 = InducingPoints(data.X[:data.M, ...].copy())
m1 = SVGP(data.X,
data.Y,
k1,
Gaussian(),
f1,
q_mu=data.mu_data_full,
q_sqrt=data.sqrt_data_full)
m1.set_trainable(False)
m1.q_sqrt.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m1, maxiter=data.MAXITER)
# Mixed Model
kern_list = [
RBF(data.D, variance=0.5, lengthscales=1.2) for _ in range(data.L)
]
k2 = mk.SeparateMixedMok(kern_list, data.W)
f2 = InducingPoints(data.X[:data.M, ...].copy())
m2 = SVGP(data.X,
data.Y,
k2,
Gaussian(),
f2,
q_mu=data.mu_data_full,
q_sqrt=data.sqrt_data_full)
m2.set_trainable(False)
m2.q_sqrt.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m2, maxiter=data.MAXITER)
check_equality_predictions(session_tf, [m1, m2])
def test_separate_independent_mok(session_tf):
"""
We use different independent kernels for each of the output dimensions.
We can achieve this in two ways:
1) efficient: SeparateIndependentMok with Shared/SeparateIndependentMof
2) inefficient: SeparateIndependentMok with InducingPoints
However, both methods should return the same conditional,
and after optimization return the same log likelihood.
"""
# Model 1 (INefficient)
q_mu_1 = np.random.randn(Data.M * Data.P, 1)
q_sqrt_1 = np.tril(np.random.randn(Data.M * Data.P,
Data.M * Data.P))[None,
...] # 1 x MP x MP
kern_list_1 = [
RBF(Data.D, variance=0.5, lengthscales=1.2) for _ in range(Data.P)
]
kernel_1 = mk.SeparateIndependentMok(kern_list_1)
feature_1 = InducingPoints(Data.X[:Data.M, ...].copy())
m1 = SVGP(Data.X,
Data.Y,
kernel_1,
Gaussian(),
feature_1,
q_mu=q_mu_1,
q_sqrt=q_sqrt_1)
m1.set_trainable(False)
m1.q_sqrt.set_trainable(True)
m1.q_mu.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m1, maxiter=Data.MAXITER)
# Model 2 (efficient)
q_mu_2 = np.random.randn(Data.M, Data.P)
q_sqrt_2 = np.array([
np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)
]) # P x M x M
kern_list_2 = [
RBF(Data.D, variance=0.5, lengthscales=1.2) for _ in range(Data.P)
]
kernel_2 = mk.SeparateIndependentMok(kern_list_2)
feature_2 = mf.SharedIndependentMof(
InducingPoints(Data.X[:Data.M, ...].copy()))
m2 = SVGP(Data.X,
Data.Y,
kernel_2,
Gaussian(),
feature_2,
q_mu=q_mu_2,
q_sqrt=q_sqrt_2)
m2.set_trainable(False)
m2.q_sqrt.set_trainable(True)
m2.q_mu.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m2, maxiter=Data.MAXITER)
check_equality_predictions(session_tf, [m1, m2])
def test_mixed_mok_with_Id_vs_independent_mok(session_tf):
data = DataMixedKernelWithEye
# Independent model
k1 = mk.SharedIndependentMok(RBF(data.D, variance=0.5, lengthscales=1.2), data.L)
f1 = InducingPoints(data.X[:data.M, ...].copy())
m1 = SVGP(data.X, data.Y, k1, Gaussian(), f1,
q_mu=data.mu_data_full, q_sqrt=data.sqrt_data_full)
m1.set_trainable(False)
m1.q_sqrt.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m1, maxiter=data.MAXITER)
# Mixed Model
kern_list = [RBF(data.D, variance=0.5, lengthscales=1.2) for _ in range(data.L)]
k2 = mk.SeparateMixedMok(kern_list, data.W)
f2 = InducingPoints(data.X[:data.M, ...].copy())
m2 = SVGP(data.X, data.Y, k2, Gaussian(), f2,
q_mu=data.mu_data_full, q_sqrt=data.sqrt_data_full)
m2.set_trainable(False)
m2.q_sqrt.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m2, maxiter=data.MAXITER)
check_equality_predictions(session_tf, [m1, m2])
def test_separate_independent_mof(session_tf):
"""
Same test as above but we use different (i.e. separate) inducing features
for each of the output dimensions.
"""
np.random.seed(0)
# Model 1 (INefficient)
q_mu_1 = np.random.randn(Data.M * Data.P, 1)
q_sqrt_1 = np.tril(np.random.randn(Data.M * Data.P, Data.M * Data.P))[None, ...] # 1 x MP x MP
kernel_1 = mk.SharedIndependentMok(RBF(Data.D, variance=0.5, lengthscales=1.2), Data.P)
feature_1 = InducingPoints(Data.X[:Data.M,...].copy())
m1 = SVGP(Data.X, Data.Y, kernel_1, Gaussian(), feature_1, q_mu=q_mu_1, q_sqrt=q_sqrt_1)
m1.set_trainable(False)
m1.q_sqrt.set_trainable(True)
m1.q_mu.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m1, maxiter=Data.MAXITER)
# Model 2 (efficient)
q_mu_2 = np.random.randn(Data.M, Data.P)
q_sqrt_2 = np.array([np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)]) # P x M x M
kernel_2 = mk.SharedIndependentMok(RBF(Data.D, variance=0.5, lengthscales=1.2), Data.P)
feat_list_2 = [InducingPoints(Data.X[:Data.M, ...].copy()) for _ in range(Data.P)]
feature_2 = mf.SeparateIndependentMof(feat_list_2)
m2 = SVGP(Data.X, Data.Y, kernel_2, Gaussian(), feature_2, q_mu=q_mu_2, q_sqrt=q_sqrt_2)
m2.set_trainable(False)
m2.q_sqrt.set_trainable(True)
m2.q_mu.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m2, maxiter=Data.MAXITER)
# Model 3 (Inefficient): an idenitical feature is used P times,
# and treated as a separate feature.
q_mu_3 = np.random.randn(Data.M, Data.P)
q_sqrt_3 = np.array([np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)]) # P x M x M
kern_list = [RBF(Data.D, variance=0.5, lengthscales=1.2) for _ in range(Data.P)]
kernel_3 = mk.SeparateIndependentMok(kern_list)
feat_list_3 = [InducingPoints(Data.X[:Data.M, ...].copy()) for _ in range(Data.P)]
feature_3 = mf.SeparateIndependentMof(feat_list_3)
m3 = SVGP(Data.X, Data.Y, kernel_3, Gaussian(), feature_3, q_mu=q_mu_3, q_sqrt=q_sqrt_3)
m3.set_trainable(False)
m3.q_sqrt.set_trainable(True)
m3.q_mu.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m3, maxiter=Data.MAXITER)
check_equality_predictions(session_tf, [m1, m2, m3])
def test_shared_independent_mok(session_tf):
"""
In this test we use the same kernel and the same inducing features
for each of the outputs. The outputs are considered to be uncorrelated.
This is how GPflow handled multiple outputs before the multioutput framework was added.
We compare three models here:
1) an ineffient one, where we use a SharedIndepedentMok with InducingPoints.
This combination will uses a Kff of size N x P x N x P, Kfu if size N x P x M x P
which is extremely inefficient as most of the elements are zero.
2) efficient: SharedIndependentMok and SharedIndependentMof
This combinations uses the most efficient form of matrices
3) the old way, efficient way: using Kernel and InducingPoints
Model 2) and 3) follow more or less the same code path.
"""
# Model 1
q_mu_1 = np.random.randn(Data.M * Data.P, 1) # MP x 1
q_sqrt_1 = np.tril(np.random.randn(Data.M * Data.P, Data.M * Data.P))[None, ...] # 1 x MP x MP
kernel_1 = mk.SharedIndependentMok(RBF(Data.D, variance=0.5, lengthscales=1.2), Data.P)
feature_1 = InducingPoints(Data.X[:Data.M,...].copy())
m1 = SVGP(Data.X, Data.Y, kernel_1, Gaussian(), feature_1, q_mu=q_mu_1, q_sqrt=q_sqrt_1)
m1.set_trainable(False)
m1.q_sqrt.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m1, maxiter=Data.MAXITER)
# Model 2
q_mu_2 = np.reshape(q_mu_1, [Data.M, Data.P]) # M x P
q_sqrt_2 = np.array([np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)]) # P x M x M
kernel_2 = RBF(Data.D, variance=0.5, lengthscales=1.2)
feature_2 = InducingPoints(Data.X[:Data.M, ...].copy())
m2 = SVGP(Data.X, Data.Y, kernel_2, Gaussian(), feature_2, q_mu=q_mu_2, q_sqrt=q_sqrt_2)
m2.set_trainable(False)
m2.q_sqrt.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m2, maxiter=Data.MAXITER)
# Model 3
q_mu_3 = np.reshape(q_mu_1, [Data.M, Data.P]) # M x P
q_sqrt_3 = np.array([np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)]) # P x M x M
kernel_3 = mk.SharedIndependentMok(RBF(Data.D, variance=0.5, lengthscales=1.2), Data.P)
feature_3 = mf.SharedIndependentMof(InducingPoints(Data.X[:Data.M, ...].copy()))
m3 = SVGP(Data.X, Data.Y, kernel_3, Gaussian(), feature_3, q_mu=q_mu_3, q_sqrt=q_sqrt_3)
m3.set_trainable(False)
m3.q_sqrt.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m3, maxiter=Data.MAXITER)
check_equality_predictions(session_tf, [m1, m2, m3])
def test_separate_independent_mof(session_tf):
"""
Same test as above but we use different (i.e. separate) inducing features
for each of the output dimensions.
"""
np.random.seed(0)
# Model 1 (INefficient)
q_mu_1 = np.random.randn(Data.M * Data.P, 1)
q_sqrt_1 = np.tril(np.random.randn(Data.M * Data.P,
Data.M * Data.P))[None,
...] # 1 x MP x MP
kernel_1 = mk.SharedIndependentMok(
RBF(Data.D, variance=0.5, lengthscales=1.2), Data.P)
feature_1 = InducingPoints(Data.X[:Data.M, ...].copy())
m1 = SVGP(Data.X,
Data.Y,
kernel_1,
Gaussian(),
feature_1,
q_mu=q_mu_1,
q_sqrt=q_sqrt_1)
m1.set_trainable(False)
m1.q_sqrt.set_trainable(True)
m1.q_mu.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m1, maxiter=Data.MAXITER)
# Model 2 (efficient)
q_mu_2 = np.random.randn(Data.M, Data.P)
q_sqrt_2 = np.array([
np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)
]) # P x M x M
kernel_2 = mk.SharedIndependentMok(
RBF(Data.D, variance=0.5, lengthscales=1.2), Data.P)
feat_list_2 = [
InducingPoints(Data.X[:Data.M, ...].copy()) for _ in range(Data.P)
]
feature_2 = mf.SeparateIndependentMof(feat_list_2)
m2 = SVGP(Data.X,
Data.Y,
kernel_2,
Gaussian(),
feature_2,
q_mu=q_mu_2,
q_sqrt=q_sqrt_2)
m2.set_trainable(False)
m2.q_sqrt.set_trainable(True)
m2.q_mu.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m2, maxiter=Data.MAXITER)
# Model 3 (Inefficient): an idenitical feature is used P times,
# and treated as a separate feature.
q_mu_3 = np.random.randn(Data.M, Data.P)
q_sqrt_3 = np.array([
np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)
]) # P x M x M
kern_list = [
RBF(Data.D, variance=0.5, lengthscales=1.2) for _ in range(Data.P)
]
kernel_3 = mk.SeparateIndependentMok(kern_list)
feat_list_3 = [
InducingPoints(Data.X[:Data.M, ...].copy()) for _ in range(Data.P)
]
feature_3 = mf.SeparateIndependentMof(feat_list_3)
m3 = SVGP(Data.X,
Data.Y,
kernel_3,
Gaussian(),
feature_3,
q_mu=q_mu_3,
q_sqrt=q_sqrt_3)
m3.set_trainable(False)
m3.q_sqrt.set_trainable(True)
m3.q_mu.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m3, maxiter=Data.MAXITER)
check_equality_predictions(session_tf, [m1, m2, m3])
def test_shared_independent_mok(session_tf):
"""
In this test we use the same kernel and the same inducing features
for each of the outputs. The outputs are considered to be uncorrelated.
This is how GPflow handled multiple outputs before the multioutput framework was added.
We compare three models here:
1) an ineffient one, where we use a SharedIndepedentMok with InducingPoints.
This combination will uses a Kff of size N x P x N x P, Kfu if size N x P x M x P
which is extremely inefficient as most of the elements are zero.
2) efficient: SharedIndependentMok and SharedIndependentMof
This combinations uses the most efficient form of matrices
3) the old way, efficient way: using Kernel and InducingPoints
Model 2) and 3) follow more or less the same code path.
"""
# Model 1
q_mu_1 = np.random.randn(Data.M * Data.P, 1) # MP x 1
q_sqrt_1 = np.tril(np.random.randn(Data.M * Data.P,
Data.M * Data.P))[None,
...] # 1 x MP x MP
kernel_1 = mk.SharedIndependentMok(
RBF(Data.D, variance=0.5, lengthscales=1.2), Data.P)
feature_1 = InducingPoints(Data.X[:Data.M, ...].copy())
m1 = SVGP(Data.X,
Data.Y,
kernel_1,
Gaussian(),
feature_1,
q_mu=q_mu_1,
q_sqrt=q_sqrt_1)
m1.set_trainable(False)
m1.q_sqrt.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m1, maxiter=Data.MAXITER)
# Model 2
q_mu_2 = np.reshape(q_mu_1, [Data.M, Data.P]) # M x P
q_sqrt_2 = np.array([
np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)
]) # P x M x M
kernel_2 = RBF(Data.D, variance=0.5, lengthscales=1.2)
feature_2 = InducingPoints(Data.X[:Data.M, ...].copy())
m2 = SVGP(Data.X,
Data.Y,
kernel_2,
Gaussian(),
feature_2,
q_mu=q_mu_2,
q_sqrt=q_sqrt_2)
m2.set_trainable(False)
m2.q_sqrt.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m2, maxiter=Data.MAXITER)
# Model 3
q_mu_3 = np.reshape(q_mu_1, [Data.M, Data.P]) # M x P
q_sqrt_3 = np.array([
np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)
]) # P x M x M
kernel_3 = mk.SharedIndependentMok(
RBF(Data.D, variance=0.5, lengthscales=1.2), Data.P)
feature_3 = mf.SharedIndependentMof(
InducingPoints(Data.X[:Data.M, ...].copy()))
m3 = SVGP(Data.X,
Data.Y,
kernel_3,
Gaussian(),
feature_3,
q_mu=q_mu_3,
q_sqrt=q_sqrt_3)
m3.set_trainable(False)
m3.q_sqrt.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m3, maxiter=Data.MAXITER)
check_equality_predictions(session_tf, [m1, m2, m3])
