def test_optimize(self):
with defer_build():
input_layer = InputLayer(input_dim=1,
output_dim=1,
num_inducing=self.M,
kernel=RBF(1) + White(1),
multitask=True)
output_layer = OutputLayer(input_dim=1,
output_dim=1,
num_inducing=self.M,
kernel=RBF(1) + White(1),
multitask=True)
seq = MultitaskSequential([input_layer, output_layer])
model = MultitaskDSDGP(X=self.X,
Y=self.Y,
Z=self.Z,
layers=seq,
likelihood=SwitchedLikelihood(
[Gaussian(), Gaussian()]),
num_latent=1)
model.compile()
before = model.compute_log_likelihood()
opt = gpflow.train.AdamOptimizer(0.01)
opt.minimize(model, maxiter=100)
after = model.compute_log_likelihood()
self.assertGreaterEqual(after, before)
def test_mixed_mok_with_Id_vs_independent_mok(session_tf):
data = DataMixedKernelWithEye
# Independent model
k1 = mk.SharedIndependentMok(RBF(data.D, variance=0.5, lengthscales=1.2),
data.L)
f1 = InducingPoints(data.X[:data.M, ...].copy())
m1 = SVGP(data.X,
data.Y,
k1,
Gaussian(),
f1,
q_mu=data.mu_data_full,
q_sqrt=data.sqrt_data_full)
m1.set_trainable(False)
m1.q_sqrt.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m1, maxiter=data.MAXITER)
# Mixed Model
kern_list = [
RBF(data.D, variance=0.5, lengthscales=1.2) for _ in range(data.L)
]
k2 = mk.SeparateMixedMok(kern_list, data.W)
f2 = InducingPoints(data.X[:data.M, ...].copy())
m2 = SVGP(data.X,
data.Y,
k2,
Gaussian(),
f2,
q_mu=data.mu_data_full,
q_sqrt=data.sqrt_data_full)
m2.set_trainable(False)
m2.q_sqrt.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m2, maxiter=data.MAXITER)
check_equality_predictions(session_tf, [m1, m2])
def test_mixed_mok_with_Id_vs_independent_mok():
data = DataMixedKernelWithEye
# Independent model
k1 = mk.SharedIndependent(SquaredExponential(variance=0.5, lengthscales=1.2), data.L)
f1 = InducingPoints(data.X[: data.M, ...])
model_1 = SVGP(k1, Gaussian(), f1, q_mu=data.mu_data_full, q_sqrt=data.sqrt_data_full)
set_trainable(model_1, False)
set_trainable(model_1.q_sqrt, True)
gpflow.optimizers.Scipy().minimize(
model_1.training_loss_closure(Data.data),
variables=model_1.trainable_variables,
method="BFGS",
compile=True,
)
# Mixed Model
kern_list = [SquaredExponential(variance=0.5, lengthscales=1.2) for _ in range(data.L)]
k2 = mk.LinearCoregionalization(kern_list, data.W)
f2 = InducingPoints(data.X[: data.M, ...])
model_2 = SVGP(k2, Gaussian(), f2, q_mu=data.mu_data_full, q_sqrt=data.sqrt_data_full)
set_trainable(model_2, False)
set_trainable(model_2.q_sqrt, True)
gpflow.optimizers.Scipy().minimize(
model_2.training_loss_closure(Data.data),
variables=model_2.trainable_variables,
method="BFGS",
compile=True,
)
check_equality_predictions(Data.data, [model_1, model_2])
def test_compare_mixed_kernel(session_tf):
data = DataMixedKernel
kern_list = [RBF(data.D) for _ in range(data.L)]
k1 = mk.SeparateMixedMok(kern_list, W=data.W)
f1 = mf.SharedIndependentMof(InducingPoints(data.X[:data.M, ...].copy()))
m1 = SVGP(data.X,
data.Y,
k1,
Gaussian(),
feat=f1,
q_mu=data.mu_data,
q_sqrt=data.sqrt_data)
kern_list = [RBF(data.D) for _ in range(data.L)]
k2 = mk.SeparateMixedMok(kern_list, W=data.W)
f2 = mf.MixedKernelSharedMof(InducingPoints(data.X[:data.M, ...].copy()))
m2 = SVGP(data.X,
data.Y,
k2,
Gaussian(),
feat=f2,
q_mu=data.mu_data,
q_sqrt=data.sqrt_data)
check_equality_predictions(session_tf, [m1, m2])
def test_separate_independent_mok():
"""
We use different independent kernels for each of the output dimensions.
We can achieve this in two ways:
1) efficient: SeparateIndependentMok with Shared/SeparateIndependentMof
2) inefficient: SeparateIndependentMok with InducingPoints
However, both methods should return the same conditional,
and after optimization return the same log likelihood.
"""
# Model 1 (Inefficient)
q_mu_1 = np.random.randn(Data.M * Data.P, 1)
q_sqrt_1 = np.tril(np.random.randn(Data.M * Data.P, Data.M * Data.P))[None, ...] # 1 x MP x MP
kern_list_1 = [SquaredExponential(variance=0.5, lengthscales=1.2) for _ in range(Data.P)]
kernel_1 = mk.SeparateIndependent(kern_list_1)
inducing_variable_1 = InducingPoints(Data.X[: Data.M, ...])
model_1 = SVGP(
kernel_1, Gaussian(), inducing_variable_1, num_latent_gps=1, q_mu=q_mu_1, q_sqrt=q_sqrt_1,
)
set_trainable(model_1, False)
set_trainable(model_1.q_sqrt, True)
set_trainable(model_1.q_mu, True)
gpflow.optimizers.Scipy().minimize(
model_1.training_loss_closure(Data.data),
variables=model_1.trainable_variables,
method="BFGS",
compile=True,
)
# Model 2 (efficient)
q_mu_2 = np.random.randn(Data.M, Data.P)
q_sqrt_2 = np.array(
[np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)]
) # P x M x M
kern_list_2 = [SquaredExponential(variance=0.5, lengthscales=1.2) for _ in range(Data.P)]
kernel_2 = mk.SeparateIndependent(kern_list_2)
inducing_variable_2 = mf.SharedIndependentInducingVariables(
InducingPoints(Data.X[: Data.M, ...])
)
model_2 = SVGP(
kernel_2,
Gaussian(),
inducing_variable_2,
num_latent_gps=Data.P,
q_mu=q_mu_2,
q_sqrt=q_sqrt_2,
)
set_trainable(model_2, False)
set_trainable(model_2.q_sqrt, True)
set_trainable(model_2.q_mu, True)
gpflow.optimizers.Scipy().minimize(
model_2.training_loss_closure(Data.data),
variables=model_2.trainable_variables,
method="BFGS",
compile=True,
)
check_equality_predictions(Data.data, [model_1, model_2])
def init_model(self, Model, X, Y):
"""
Initialize the model
TODO: Currently I'm coding in the choice of having purely a combo
of Matern and Linear. We can make this flexible.
"""
Dx = X.shape[1]
kern = Matern52(input_dim=len(self.matern_dims),
active_dims=self.matern_dims,
lengthscales=SETTINGS.lengthscales * Dx ** 0.5) + \
Linear(input_dim=len(self.linear_dims),
active_dims=self.linear_dims)
lik = Gaussian()
lik.variance = SETTINGS.likelihood_variance
gamma = kmeans2(X, self.M_gamma, minit='points')[
0] if self.M_gamma > 0 else np.empty((0, Dx))
beta = kmeans2(X, self.M_beta, minit='points')[0]
if self.M_gamma > 0:
gamma_minibatch_size = SETTINGS.gamma_minibatch_size
else
gamma_minibatch_size = None
self.model = Model(X, Y, kern, lik, gamma, beta,
minibatch_size=SETTINGS.minibatch_size,
gamma_minibatch_size=gamma_minibatch_size)
self.sess = self.model.enquire_session()
def get_ELBOs(self):
X = np.array([[1.,2.,3.],[1.4,2.1,3.],[1.1,2.,3.],[1.,2.,3.1]])
Y = np.array([[0.],[2.],[1.2],[3.5]])
Z = np.array([[1.,2.,3.5],[1.3,2.2,3.1]])
sm_sqrt = np.tril(np.random.rand(8,8))
mu_M = np.array([[1.,2.8,3.,4.,5.7,6.,3.,3.2]]).T
kernels = [gpflow.kernels.RBF(3),gpflow.kernels.RBF(2,lengthscales=4.0),gpflow.kernels.RBF(1,lengthscales=2.0)]
mydgp = my_FullDGP(X.copy(),Y.copy(),Z.copy(),kernels,Gaussian(),mu_M=mu_M,S_M=sm_sqrt,whitened_prior=False)
zs =[[[0.1,0.5],[-0.3,0.2],[1.,-1.3],[2.,0.2]],[[.1],[.2],[1.2],[0.1]],[[1.],[.5],[.25],[0.5]]]
ELBO_normal = mydgp._build_likelihood(zs)
sess = mydgp.enquire_session()
ELBO_normal = sess.run(ELBO_normal)
KL_normal = mydgp.get_KL()
LMMs = mydgp.get_LM()
L_K_inv = np.linalg.inv(block_diag(LMMs[0],LMMs[0],LMMs[1],LMMs[2]))
mu_M2, sm_sqrt2 = L_K_inv @ mu_M, L_K_inv @ sm_sqrt
kernels2 = [gpflow.kernels.RBF(3),gpflow.kernels.RBF(2,lengthscales=4.0),gpflow.kernels.RBF(1,lengthscales=2.0)]
mydgp2 = my_FullDGP(X,Y,Z,kernels2,Gaussian(),mu_M=mu_M2,S_M=sm_sqrt2,whitened_prior=True)
ELBO_white = mydgp2._build_likelihood(zs)
sess2 = mydgp2.enquire_session()
ELBO_white = sess2.run(ELBO_white)
KL_white = mydgp2.get_KL()
return ELBO_normal, KL_normal, ELBO_white, KL_white
def test_multioutput_with_diag_q_sqrt():
data = DataMixedKernel
q_sqrt_diag = np.ones((data.M, data.L)) * 2
q_sqrt = np.repeat(np.eye(data.M)[None, ...], data.L,
axis=0) * 2 # L x M x M
kern_list = [SquaredExponential() for _ in range(data.L)]
k1 = mk.LinearCoregionalization(kern_list, W=data.W)
f1 = mf.SharedIndependentInducingVariables(
InducingPoints(data.X[:data.M, ...]))
model_1 = SVGP(k1,
Gaussian(),
inducing_variable=f1,
q_mu=data.mu_data,
q_sqrt=q_sqrt_diag,
q_diag=True)
kern_list = [SquaredExponential() for _ in range(data.L)]
k2 = mk.LinearCoregionalization(kern_list, W=data.W)
f2 = mf.SharedIndependentInducingVariables(
InducingPoints(data.X[:data.M, ...]))
model_2 = SVGP(k2,
Gaussian(),
inducing_variable=f2,
q_mu=data.mu_data,
q_sqrt=q_sqrt,
q_diag=False)
check_equality_predictions(Data.X, Data.Y, [model_1, model_2])
def test_multioutput_with_diag_q_sqrt(session_tf):
data = DataMixedKernel
q_sqrt_diag = np.ones((data.M, data.L)) * 2
q_sqrt = np.repeat(np.eye(data.M)[None, ...], data.L,
axis=0) * 2 # L x M x M
kern_list = [RBF(data.D) for _ in range(data.L)]
k1 = mk.SeparateMixedMok(kern_list, W=data.W)
f1 = mf.SharedIndependentMof(InducingPoints(data.X[:data.M, ...].copy()))
m1 = SVGP(data.X,
data.Y,
k1,
Gaussian(),
feat=f1,
q_mu=data.mu_data,
q_sqrt=q_sqrt_diag,
q_diag=True)
kern_list = [RBF(data.D) for _ in range(data.L)]
k2 = mk.SeparateMixedMok(kern_list, W=data.W)
f2 = mf.SharedIndependentMof(InducingPoints(data.X[:data.M, ...].copy()))
m2 = SVGP(data.X,
data.Y,
k2,
Gaussian(),
feat=f2,
q_mu=data.mu_data,
q_sqrt=q_sqrt,
q_diag=False)
check_equality_predictions(session_tf, [m1, m2])
def test_MixedKernelSeparateMof():
data = DataMixedKernel
kern_list = [SquaredExponential() for _ in range(data.L)]
inducing_variable_list = [
InducingPoints(data.X[:data.M, ...]) for _ in range(data.L)
]
k1 = mk.LinearCoregionalization(kern_list, W=data.W)
f1 = mf.SeparateIndependentInducingVariables(inducing_variable_list)
model_1 = SVGP(k1,
Gaussian(),
inducing_variable=f1,
q_mu=data.mu_data,
q_sqrt=data.sqrt_data)
kern_list = [SquaredExponential() for _ in range(data.L)]
inducing_variable_list = [
InducingPoints(data.X[:data.M, ...]) for _ in range(data.L)
]
k2 = mk.LinearCoregionalization(kern_list, W=data.W)
f2 = mf.SeparateIndependentInducingVariables(inducing_variable_list)
model_2 = SVGP(k2,
Gaussian(),
inducing_variable=f2,
q_mu=data.mu_data,
q_sqrt=data.sqrt_data)
check_equality_predictions(Data.X, Data.Y, [model_1, model_2])
def test_gaussian(self):
lik = Gaussian()
lik.variance = 0.01
N, Ns, D_Y = self.X.shape[0], self.Xs.shape[0], self.D_Y
Y = np.random.randn(N, D_Y)
Ys = np.random.randn(Ns, D_Y)
for L in [1, 2]:
self.compare_to_single_layer(Y, Ys, lik, L)
def __init__(self,
X,
Y,
inducing_points,
final_inducing_points,
hidden_units,
units,
share_inducing_inputs=True):
Model.__init__(self)
assert X.shape[0] == Y.shape[0]
self.num_data, D_X = X.shape
self.D_Y = 1
self.num_samples = 100
kernels = []
for l in range(hidden_units + 1):
ks = []
if (l > 0):
D = units
else:
D = D_X
if (l < hidden_units):
for w in range(units):
ks.append(
RBF(D, lengthscales=1., variance=1.) +
White(D, variance=1e-5))
else:
ks.append(RBF(D, lengthscales=1., variance=1.))
kernels.append(ks)
self.dims_in = [D_X] + [units] * hidden_units
self.dims_out = [units] * hidden_units + [1]
q_mus, q_sqrts, Zs, mean_functions = init_layers(
X, self.dims_in, self.dims_out, inducing_points,
final_inducing_points, share_inducing_inputs)
layers = []
for q_mu, q_sqrt, Z, mean_function, kernel in zip(
q_mus, q_sqrts, Zs, mean_functions, kernels):
layers.append(Layer(kernel, q_mu, q_sqrt, Z, mean_function))
self.layers = ParamList(layers)
for layer in self.layers[:-1]: # fix the inner layer mean functions
layer.mean_function.fixed = True
self.likelihood = Gaussian()
minibatch_size = 10000 if X.shape[0] > 10000 else None
if minibatch_size is not None:
self.X = MinibatchData(X, minibatch_size)
self.Y = MinibatchData(Y, minibatch_size)
else:
self.X = DataHolder(X)
self.Y = DataHolder(Y)
def test_separate_independent_mok(session_tf):
"""
We use different independent kernels for each of the output dimensions.
We can achieve this in two ways:
1) efficient: SeparateIndependentMok with Shared/SeparateIndependentMof
2) inefficient: SeparateIndependentMok with InducingPoints
However, both methods should return the same conditional,
and after optimization return the same log likelihood.
"""
# Model 1 (INefficient)
q_mu_1 = np.random.randn(Data.M * Data.P, 1)
q_sqrt_1 = np.tril(np.random.randn(Data.M * Data.P,
Data.M * Data.P))[None,
...] # 1 x MP x MP
kern_list_1 = [
RBF(Data.D, variance=0.5, lengthscales=1.2) for _ in range(Data.P)
]
kernel_1 = mk.SeparateIndependentMok(kern_list_1)
feature_1 = InducingPoints(Data.X[:Data.M, ...].copy())
m1 = SVGP(Data.X,
Data.Y,
kernel_1,
Gaussian(),
feature_1,
q_mu=q_mu_1,
q_sqrt=q_sqrt_1)
m1.set_trainable(False)
m1.q_sqrt.set_trainable(True)
m1.q_mu.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m1, maxiter=Data.MAXITER)
# Model 2 (efficient)
q_mu_2 = np.random.randn(Data.M, Data.P)
q_sqrt_2 = np.array([
np.tril(np.random.randn(Data.M, Data.M)) for _ in range(Data.P)
]) # P x M x M
kern_list_2 = [
RBF(Data.D, variance=0.5, lengthscales=1.2) for _ in range(Data.P)
]
kernel_2 = mk.SeparateIndependentMok(kern_list_2)
feature_2 = mf.SharedIndependentMof(
InducingPoints(Data.X[:Data.M, ...].copy()))
m2 = SVGP(Data.X,
Data.Y,
kernel_2,
Gaussian(),
feature_2,
q_mu=q_mu_2,
q_sqrt=q_sqrt_2)
m2.set_trainable(False)
m2.q_sqrt.set_trainable(True)
m2.q_mu.set_trainable(True)
gpflow.training.ScipyOptimizer().minimize(m2, maxiter=Data.MAXITER)
check_equality_predictions(session_tf, [m1, m2])
def get_Full_and_MF_ELBOs(self, white):
X = np.array([[1., 2., 3.], [1., 2.1, 3.], [1.1, 2., 3.],
[1., 2., 3.1]])
Y = np.array([[1.], [2.], [1.2], [3.]])
Z = np.array([[1., 2., 3.], [1.3, 2.2, 3.1]])
sm_sqrt2 = np.array([[1., 0., 0., 0., 0., 0., 0., 0.],
[0.5, 1., 0., 0., 0., 0., 0., 0.],
[0., 0., 1., 0., 0., 0., 0., 0.],
[0., 0., 0.95, 1., 0., 0., 0., 0.],
[0., 0., 0., 0., 1., 0., 0., 0.],
[0., 0., 0., 0., 0.1, 1., 0., 0.],
[0., 0., 0., 0., 0., 0., 1., 0.],
[0., 0., 0., 0., 0., 0., 0.25, 1.]])
mu_M = np.array([[1., 2., 3., 4., 5., 6., 3., 3.]]).T
kernels = [
gpflow.kernels.RBF(3),
gpflow.kernels.RBF(2, lengthscales=4.0),
gpflow.kernels.RBF(1, lengthscales=2.0)
]
#all_Zs, all_mean_funcs = init_linear(X,Z,kernels)
#mylayers = Fully_Coupled_Layers(X,Y,Z,kernels,all_mean_funcs,all_Zs,S_M = sm_sqrt2, mu_M = mu_M)
mydgp = my_FullDGP(X,
Y,
Z,
kernels,
Gaussian(),
mu_M=mu_M,
S_M=sm_sqrt2,
whitened_prior=white) #,mylayers)
zs = [[[0.1, 0.5], [-0.3, 0.2], [1., -1.3], [2., 0.]],
[[.1], [.2], [.2], [0.1]], [[1.], [.5], [.2], [0.5]]]
#f, muNtilde, SNtilde, K1NM, mean, var = mydgp.propagate(X,zs=zs)
ELBO_diag = mydgp._build_likelihood(zs=zs)
session = gpflow.get_default_session()
ELBO_diag = session.run(ELBO_diag)
z1 = [[0.1, 0.5], [-0.3, 0.2], [1., -1.3], [2., 0.]]
z2 = [[.1], [.2], [.2], [0.1]]
z3 = [[1.], [.5], [.2], [0.5]]
Saldgp = sal0dgp(X, Y, Z, kernels, Gaussian(), white=white)
myqsqrt = np.array([[[[1., 0.], [0.5, 1.]], [[1., 0.], [0.95, 1.]]],
[[[1., 0.], [0.1, 1.]]], [[[1., 0.], [0.25, 1.]]]])
myqmu = [[[1., 3.], [2., 4.]], [[5.], [6.]], [[3.], [3.]]]
Saldgp.set_qsqrt(myqsqrt, myqmu)
ELBO_sal = Saldgp.my_ELBO(z1, z2, z3)
return ELBO_diag, ELBO_sal, mydgp.get_KL(), Saldgp.get_KL()
def make_single_layer_models(X, Y, Z):
D = X.shape[1]
Y_mean, Y_std = np.average(Y), np.std(Y)
m_sgpr = SGPR(X,
Y,
RBF(D, variance=Y_std**2),
Z.copy(),
mean_function=Constant(Y_mean))
m_svgp = SVGP(X,
Y,
RBF(D, variance=Y_std**2),
Gaussian(),
Z.copy(),
mean_function=Constant(Y_mean))
m_fitc = GPRFITC(X,
Y,
RBF(D, variance=Y_std**2),
Z.copy(),
mean_function=Constant(Y_mean))
for m in [m_sgpr, m_svgp, m_fitc]:
m.mean_function.fixed = True
m.likelihood.variance = 0.1 * Y_std
return m_sgpr, m_svgp, m_fitc
def test_variational_univariate_conditionals(diag, whiten):
q_mu = np.ones((1, Datum.num_latent)) * Datum.posterior_mean
ones = np.ones((1, Datum.num_latent)) if diag else np.ones(
(1, 1, Datum.num_latent))
q_sqrt = ones * Datum.posterior_std
model = gpflow.models.SVGP(kernel=SquaredExponential(variance=Datum.K),
likelihood=Gaussian(),
inducing_variable=Datum.Z,
num_latent=Datum.num_latent,
q_diag=diag,
whiten=whiten,
q_mu=q_mu,
q_sqrt=q_sqrt)
fmean_func, fvar_func = gpflow.conditionals.conditional(
Datum.X,
Datum.Z,
model.kernel,
model.q_mu,
q_sqrt=model.q_sqrt,
white=whiten)
mean_value, var_value = fmean_func[0, 0], fvar_func[0, 0]
assert_allclose(mean_value - Datum.posterior_mean, 0, atol=4)
assert_allclose(var_value - Datum.posterior_var, 0, atol=4)
def build_gpflux_deep_gp(layer_sizes, num_data):
gp_layers = build_gp_layers(layer_sizes, num_data)
likelihood = Gaussian()
likelihood_layer = LikelihoodLayer(likelihood)
model = DeepGP(gp_layers, likelihood_layer).as_training_model()
return model, None
def make_dgp(X, Y, Z, L):
D = X.shape[1]
# the layer shapes are defined by the kernel dims, so here all hidden layers are D dimensional
kernels = []
#for l in range(L):
kernels.append(RBF(5))
kernels.append(RBF(2))
kernels.append(RBF(9))
# between layer noise (doesn't actually make much difference but we include it anyway)
#for kernel in kernels[:-1]:
# kernel += White(D, variance=1e-5)
mb = 1000 if X.shape[0] > 1000 else None
model = DGP(X,
Y,
Z,
kernels,
Gaussian(),
num_samples=10,
minibatch_size=mb)
# start the inner layers almost deterministically
for layer in model.layers[:-1]:
layer.q_sqrt = layer.q_sqrt.value * 1e-5
return model
def __init__(self,
data: RegressionData,
kernel: Kernel,
mu_old: Optional[tf.Tensor],
Su_old: Optional[tf.Tensor],
Kaa_old: Optional[tf.Tensor],
Z_old: Optional[tf.Tensor],
inducing_variable: Union[InducingPoints, np.ndarray],
mean_function=Zero()):
"""
Z is a matrix of pseudo inputs, size M x D
kern, mean_function are appropriate gpflow objects
mu_old, Su_old are mean and covariance of old q(u)
Z_old is the old inducing inputs
This method only works with a Gaussian likelihood.
"""
X, Y = data
self.X = X
self.Y = Y
likelihood = Gaussian()
self.inducing_variable = gpflow.models.util.inducingpoint_wrapper(inducing_variable)
GPModel.__init__(self, kernel, likelihood, mean_function, inducing_variable.size)
self.num_data = X.shape[0]
self.num_latent = Y.shape[1]
self.mu_old = gpflow.Parameter(mu_old, trainable=False)
self.M_old = Z_old.shape[0]
self.Su_old = gpflow.Parameter(Su_old, trainable=False)
self.Kaa_old = gpflow.Parameter(Kaa_old, trainable=False)
self.Z_old = gpflow.Parameter(Z_old, trainable=False)
def initialize_modelM(self):
M = np.minimum(int(100), self.Xnorm.shape[0])
Z_M = kmeans2(np.copy(self.Xnorm), M, minit='points')[0]
# X -> z -> X : GP0(X -> z), GP1(z -> XY) Deep GP autoencoder
dims = [self.Xnorm.shape[1], int(2)]
# define a list of kernels for the deep GPs
kernels = []
for dim_i in dims:
kernels.append(gpflow.kernels.RBF(input_dim=dim_i, ARD=True, lengthscales=np.ones(shape=[dim_i]) * 0.2)) # not relevant any more
# + gpflow.kernels.Matern32(input_dim=dim_i, ARD=True, lengthscales=np.ones(shape=[dim_i]) * 0.2))
# add white noise
for kernel_i, dim_i in zip(kernels[:-1], dims[:-1]):
kernel_i += gpflow.kernels.White(input_dim=dim_i, variance=1e-05)
# define the deep GP model
XYnorm = np.concatenate([self.Xnorm, self.Ynorm], axis=1)
dgp_model = DGP(X=np.copy(self.Xnorm), Y=np.copy(XYnorm),
Z=Z_M, kernels=kernels, likelihood=Gaussian(),
num_samples=5,
minibatch_size=None)
# start the inner layers almost deterministically
for layer in dgp_model.layers[:-1]:
layer.q_sqrt = layer.q_sqrt.value * 1e-5
# # start with small lengthscales
# for layer_i, dim_i in zip(dgp_model.layers, dims):
# layer.kern.lengthscales = np.ones(shape=[dim_i]) * 0.2
dgp_model.likelihood.likelihood.variance = 0.01
return kernels, dgp_model
def make_dgp(X, Y, Z, L):
D = X.shape[1]
Y_mean, Y_std = np.average(Y), np.std(Y)
# the layer shapes are defined by the kernel dims, so here all hidden layers are D dimensional
kernels = []
for l in range(L):
kernels.append(RBF(D, lengthscales=1., variance=1.))
# between layer noise (doesn't actually make much difference but we include it anyway)
for kernel in kernels[:-1]:
kernel += White(D, variance=1e-5)
mb = 10000 if X.shape[0] > 10000 else None
model = DGP(X, Y, Z, kernels, Gaussian(), num_samples=1, minibatch_size=mb)
# same final layer inits we used for the single layer model
model.layers[-1].kern.variance = Y_std**2
model.likelihood.variance = Y_std * 0.1
model.layers[-1].mean_function = Constant(Y_mean)
model.layers[-1].mean_function.fixed = True
# start the inner layers almost deterministically
for layer in model.layers[:-1]:
layer.q_sqrt = layer.q_sqrt.value * 1e-5
return model
def build_deep_gp(input_dim, num_data):
layers = [input_dim, 2, 2, 1]
# Below are different ways to build layers
# 1. Pass in Lists:
kernel_list = [RBF(), Matern12()]
num_inducing = [25, 25]
l1_kernel = construct_basic_kernel(kernels=kernel_list)
l1_inducing = construct_basic_inducing_variables(num_inducing=num_inducing, input_dim=layers[0])
# 2. Pass in kernels, specificy output dims (shared hyperparams/variables)
l2_kernel = construct_basic_kernel(kernels=RBF(), output_dim=layers[2], share_hyperparams=True)
l2_inducing = construct_basic_inducing_variables(
num_inducing=25, input_dim=layers[1], share_variables=True
)
# 3. Pass in kernels, specificy output dims (independent hyperparams/vars)
# By default and the constructor will make indep. copies
l3_kernel = construct_basic_kernel(kernels=RBF(), output_dim=layers[3])
l3_inducing = construct_basic_inducing_variables(
num_inducing=25, input_dim=layers[2], output_dim=layers[3]
)
# Assemble at the end
gp_layers = [
GPLayer(l1_kernel, l1_inducing, num_data),
GPLayer(l2_kernel, l2_inducing, num_data),
GPLayer(l3_kernel, l3_inducing, num_data, mean_function=Zero()),
]
return DeepGP(gp_layers, Gaussian(0.1))
def test(self):
kern1 = RBF(1)
kern2 = RBF(2)
lik = Gaussian()
X = np.zeros((1, 1))
model = DGP(X, X, X, [kern1, kern2], lik)
model.compute_log_likelihood()
def test_cde_direct_parametrization(test_data, w_dim, use_keras_compile):
"""Test a directly parameterized CDE, using functional API, both eager or compiled.
Test that the losses decrease."""
tf.random.set_seed(0)
np.random.seed(0)
# 1. Set up data
x_data, y_data = test_data
num_data, x_dim = x_data.shape
# 2. Set up layers
prior_means = np.zeros(w_dim)
prior_std = np.ones(w_dim)
encoder = DirectlyParameterizedNormalDiag(num_data, w_dim)
prior = tfp.distributions.MultivariateNormalDiag(prior_means, prior_std)
lv = LatentVariableLayer(prior, encoder)
[gp] = build_gp_layers([x_dim + w_dim, 1], num_data)
likelihood_layer = LikelihoodLayer(Gaussian())
# 3. Build the model
dgp = DeepGP([lv, gp], likelihood_layer)
model = dgp.as_training_model()
# 4. Train the model and check 2nd half of loss is lower than first
loss_history = train_model(x_data, y_data, model, use_keras_compile)
epochs = len(loss_history)
assert np.all(loss_history[:(epochs // 2)] > loss_history[(epochs // 2):])
def test_mixed_mok_with_Id_vs_independent_mok():
data = DataMixedKernelWithEye
# Independent model
k1 = mk.SharedIndependent(
SquaredExponential(variance=0.5, lengthscale=1.2), data.L)
f1 = InducingPoints(data.X[:data.M, ...])
model_1 = SVGP(k1,
Gaussian(),
f1,
q_mu=data.mu_data_full,
q_sqrt=data.sqrt_data_full)
set_trainable(model_1, False)
model_1.q_sqrt.trainable = True
@tf.function(autograph=False)
def closure1():
return -model_1.log_marginal_likelihood(Data.X, Data.Y)
gpflow.optimizers.Scipy().minimize(closure1,
variables=model_1.trainable_variables,
method='BFGS')
# Mixed Model
kern_list = [
SquaredExponential(variance=0.5, lengthscale=1.2)
for _ in range(data.L)
]
k2 = mk.LinearCoregionalization(kern_list, data.W)
f2 = InducingPoints(data.X[:data.M, ...])
model_2 = SVGP(k2,
Gaussian(),
f2,
q_mu=data.mu_data_full,
q_sqrt=data.sqrt_data_full)
set_trainable(model_2, False)
model_2.q_sqrt.trainable = True
@tf.function(autograph=False)
def closure2():
return -model_2.log_marginal_likelihood(Data.X, Data.Y)
gpflow.optimizers.Scipy().minimize(closure2,
variables=model_2.trainable_variables,
method='BFGS')
check_equality_predictions(Data.X, Data.Y, [model_1, model_2])
def test_vs_single_layer(self):
lik = Gaussian()
lik_var = 0.01
lik.variance = lik_var
N, Ns, D_Y, D_X = self.X.shape[0], self.Xs.shape[
0], self.D_Y, self.X.shape[1]
Y = np.random.randn(N, D_Y)
Ys = np.random.randn(Ns, D_Y)
kern = Matern52(self.X.shape[1], lengthscales=0.5)
# mf = Linear(A=np.random.randn(D_X, D_Y), b=np.random.randn(D_Y))
mf = Zero()
m_gpr = GPR(self.X, Y, kern, mean_function=mf)
m_gpr.likelihood.variance = lik_var
mean_gpr, var_gpr = m_gpr.predict_y(self.Xs)
test_lik_gpr = m_gpr.predict_density(self.Xs, Ys)
pred_m_gpr, pred_v_gpr = m_gpr.predict_f(self.Xs)
pred_mfull_gpr, pred_vfull_gpr = m_gpr.predict_f_full_cov(self.Xs)
kerns = []
kerns.append(
Matern52(self.X.shape[1], lengthscales=0.5, variance=1e-1))
kerns.append(kern)
layer0 = GPMC_Layer(kerns[0], self.X.copy(), D_X, Identity())
layer1 = GPR_Layer(kerns[1], mf, D_Y)
m_dgp = DGP_Heinonen(self.X, Y, lik, [layer0, layer1])
mean_dgp, var_dgp = m_dgp.predict_y(self.Xs, 1)
test_lik_dgp = m_dgp.predict_density(self.Xs, Ys, 1)
pred_m_dgp, pred_v_dgp = m_dgp.predict_f(self.Xs, 1)
pred_mfull_dgp, pred_vfull_dgp = m_dgp.predict_f_full_cov(
self.Xs, 1)
tol = 1e-4
assert_allclose(mean_dgp[0], mean_gpr, atol=tol, rtol=tol)
assert_allclose(test_lik_dgp, test_lik_gpr, atol=tol, rtol=tol)
assert_allclose(pred_m_dgp[0], pred_m_gpr, atol=tol, rtol=tol)
assert_allclose(pred_mfull_dgp[0],
pred_mfull_gpr,
atol=tol,
rtol=tol)
assert_allclose(pred_vfull_dgp[0],
pred_vfull_gpr,
atol=tol,
rtol=tol)
def prepare(self):
N = 100
M = 10
rng = np.random.RandomState(42)
X = rng.randn(N, 2)
Y = rng.randn(N, 1)
Z = rng.randn(M, 2)
X_ind = rng.randint(0, 2, (N, 1))
Z_ind = rng.randint(0, 2, (M, 1))
X = np.hstack([X, X_ind])
Y = np.hstack([Y, X_ind])
Z = np.hstack([Z, Z_ind])
Xs = rng.randn(M, 2)
Xs_ind = rng.randint(0, 2, (M, 1))
Xs = np.hstack([Xs, Xs_ind])
with defer_build():
lik = SwitchedLikelihood([Gaussian(), Gaussian()])
input_layer = InputLayer(input_dim=2,
output_dim=1,
num_inducing=M,
kernel=RBF(2) + White(2),
mean_function=Linear(A=np.ones((3, 1))),
multitask=True)
output_layer = OutputLayer(input_dim=1,
output_dim=1,
num_inducing=M,
kernel=RBF(1) + White(1),
multitask=True)
seq = MultitaskSequential([input_layer, output_layer])
model = MultitaskDSDGP(X=X,
Y=Y,
Z=Z,
layers=seq,
likelihood=lik,
num_latent=1)
model.compile()
return model, Xs
def make_mf_dgp(cls, X, Y, Z, add_linear=True, minibatch_size=None):
"""
Constructor for convenience. Constructs a mf-dgp model from training data and inducing point locations
:param X: List of target
:param Y:
:param Z:
:param add_linear:
:return:
"""
n_fidelities = len(X)
Din = X[0].shape[1]
Dout = Y[0].shape[1]
kernels = [RBF(Din, active_dims=list(range(Din)), variance=1., lengthscales=1, ARD=True)]
for l in range(1, n_fidelities):
D = Din + Dout
D_range = list(range(D))
k_corr = RBF(Din, active_dims=D_range[:Din], lengthscales=1, variance=1.0, ARD=True)
k_prev = RBF(Dout, active_dims=D_range[Din:], variance=1., lengthscales=1.0)
k_in = RBF(Din, active_dims=D_range[:Din], variance=1., lengthscales=1, ARD=True)
if add_linear:
k_l = k_corr * (k_prev + Linear(Dout, active_dims=D_range[Din:], variance=1.)) + k_in
else:
k_l = k_corr * k_prev + k_in
kernels.append(k_l)
"""
A White noise kernel is currently expected by Mf-DGP at all layers except the last.
In cases where no noise is desired, this should be set to 0 and fixed, as follows:
white = White(1, variance=0.)
white.variance.trainable = False
kernels[i] += white
"""
for i, kernel in enumerate(kernels[:-1]):
kernels[i] += White(1, variance=1e-6)
num_data = 0
for i in range(len(X)):
_log.info('\nData at Fidelity {}'.format(i + 1))
_log.info('X - {}'.format(X[i].shape))
_log.info('Y - {}'.format(Y[i].shape))
_log.info('Z - {}'.format(Z[i].shape))
num_data += X[i].shape[0]
layers = init_layers_mf(Y, Z, kernels, num_outputs=Dout)
model = DGP_Base(X, Y, Gaussian(), layers, num_samples=10, minibatch_size=minibatch_size)
return model
def diag_SM(self):
X = np.array([[1., 2., 3.], [1., 2.1, 3.], [1.1, 2., 3.],
[1., 2., 3.1]])
Y = np.array([[1.], [2.], [.2], [3.]])
Z = np.array([[1., 2., 3.], [1.3, 2.2, 3.1]])
kernels = [
gpflow.kernels.RBF(3),
gpflow.kernels.RBF(2, lengthscales=4.0),
gpflow.kernels.RBF(1, lengthscales=2.0)
]
sm_sqrt2 = np.array([[1., 0., 0., 0., 0., 0., 0., 0.],
[0.5, 1., 0., 0., 0., 0., 0., 0.],
[0., 0., 1., 0., 0., 0., 0., 0.],
[0., 0., 0.95, 1., 0., 0., 0., 0.],
[0., 0., 0., 0., 1., 0., 0., 0.],
[0., 0., 0., 0., 0.1, 1., 0., 0.],
[0., 0., 0., 0., 0., 0., 1., 0.],
[0., 0., 0., 0., 0., 0., 0.25, 1.]])
mu_M = np.array([[1., 2., 3., 4., 5., 6., 3., 3.]]).T
mydgp = Full_DGP(X, Y, Z, kernels, Gaussian(), S_M=sm_sqrt2, mu_M=mu_M)
zs = [[[0.1, 0.5], [-0.3, 0.2], [1., -1.3], [2., 0.]],
[[.1], [.2], [.2], [0.1]], [[1.], [.5], [.2], [0.5]]]
diag_f, _, _, _, _, _ = mydgp.propagate(X, zs=zs)
session = gpflow.get_default_session()
diag_f = session.run(diag_f)
z1 = [[0.1, 0.5], [-0.3, 0.2], [1., -1.3], [2., 0.]]
z2 = [[.1], [.2], [.2], [0.1]]
z3 = [[1.], [.5], [.2], [0.5]]
Saldgp = sal0dgp(X, Y, Z, kernels, Gaussian())
myqsqrt = np.array([[[[1., 0.], [0.5, 1.]], [[1., 0.], [0.95, 1.]]],
[[[1., 0.], [0.1, 1.]]], [[[1., 0.], [0.25, 1.]]]])
myqmu = [[[1., 3.], [2., 4.]], [[5.], [6.]], [[3.], [3.]]]
Saldgp.set_qsqrt(myqsqrt, myqmu)
temp, _, _ = Saldgp.my_propagate(X, z1, z2, z3)
sal_f = temp[0][0]
sal_f = np.append(sal_f, temp[1][0], axis=1)
sal_f = np.append(sal_f, temp[2][0], axis=1)
return sal_f, diag_f
def __init__(self,
data: Tuple,
kernel: Kernel,
inducing_variables,
partitions: Union[List, Tuple],
likelihood: Likelihood = Gaussian()):
super().__init__(data=data,
kernel=kernel,
likelihood=likelihood,
inducing_variable=inducing_variables,
mean_function=Zero(),
num_latent_gps=1)
self.partitions = partitions