def _init_X(self, wins, Ys, Us, X_variance, nDims, init='Y'):
#import pdb; pdb.set_trace()
Xs = []
for i_layer in range(1,self.nLayers):
X = []
for i_seq in range(len(Ys)):
Y = Ys[i_seq]
U_win, U_dim = wins[i_layer], nDims[i_layer]
if init=='Y':
mean = np.zeros((U_win+Y.shape[0], U_dim))
mean[U_win:] = Y[:,:U_dim]
var = np.zeros((U_win+Y.shape[0],U_dim))+X_variance
X.append(NormalPosterior(mean,var,name='qX_'+str(i_seq)))
elif init=='rand' :
mean = np.zeros((U_win+Y.shape[0], U_dim))
mean[:] = np.random.randn(*mean.shape)
var = np.zeros((U_win+Y.shape[0],U_dim))+X_variance
X.append(NormalPosterior(mean,var,name='qX_'+str(i_seq)))
elif init=='zero':
mean = np.zeros((U_win+Y.shape[0], U_dim))
mean[:] = np.random.randn(*mean.shape)*0.01
var = np.zeros((U_win+Y.shape[0],U_dim))+X_variance
X.append(NormalPosterior(mean,var,name='qX_'+str(i_seq)))
Xs.append(X)
return Xs
def gen_pred_model(self, Y=None, init_X='encoder', binObserved=False):
from GPy.core.parameterization.variational import NormalPosterior
from deepgp.models.pred_model import PredDeepGP
if Y is not None:
Xs = [Y]
if init_X=='nn':
xy = self.collect_all_XY()
for i in range(self.nLayers):
x_mean, x_var = PredDeepGP._init_Xs_NN(Y,i,xy)
Xs.append(NormalPosterior(x_mean, x_var))
elif init_X=='encoder':
x_mean = Y
x_mean[np.isnan(x_mean)] = 0.
for layer in self.layers:
x_mean = layer.encoder.predict(x_mean)
Xs.append(NormalPosterior(x_mean, np.ones(x_mean.shape)*layer.X_var))
layers = []
layer_lower = None
for i in range(self.nLayers):
layers.append(self.layers[i].gen_pred_layer(layer_lower=layer_lower,Y=Xs[i], X=Xs[i+1], binObserved=(i==0 and binObserved)))
layer_lower = layers[-1]
pmodel = PredDeepGP(layers)
if init_X=='nn': pmodel.set_train_data(xy)
return pmodel
def predict(self,
Xnew,
full_cov=False,
Y_metadata=None,
kern=None,
view=0):
"""Make a prediction from the deep Gaussian process model for a given input"""
from GPy.core.parameterization.variational import NormalPosterior
if self.repeatX:
assert self.nLayers == 2
mean, var = self.layers[-1].predict(Xnew)
Xnew_norm = (Xnew - self.repeatX_Xmean) / self.repeatX_Xstd
Xmean = np.hstack([mean, Xnew_norm])
Xvar = np.empty_like(Xmean)
Xvar[:] = 1e-6
Xvar[:, :self.nDimsOrig[1]] = var
x = NormalPosterior(Xmean, Xvar)
else:
x = Xnew
for l in self.layers[:0:-1]:
mean, var = l.predict(x)
var = np.clip(var, 1e-8, np.inf)
if var.shape[1] == 1:
var = np.tile(var, mean.shape[1])
x = NormalPosterior(mean, var)
mrd_flag = hasattr(self.layers[0], 'views')
if mrd_flag:
return self.layers[0].views[view].predict(x)
else:
return self.layers[0].predict(x)
def setUp(self):
from GPy.core.parameterization.variational import NormalPosterior
N, M, Q = 100, 20, 3
X = np.random.randn(N, Q)
X_var = np.random.rand(N, Q) + 0.01
self.Z = np.random.randn(M, Q)
self.qX = NormalPosterior(X, X_var)
self.w1 = np.random.randn(N)
self.w2 = np.random.randn(N, M)
self.w3 = np.random.randn(M, M)
self.w3 = self.w3 #+self.w3.T
self.w3n = np.random.randn(N, M, M)
self.w3n = self.w3n + np.swapaxes(self.w3n, 1, 2)
def collect_all_XY(self, root=0):
if self.mpi_comm is None:
XY = [self.obslayer.Y.copy()]
for l in self.layers: XY.append(l.X.copy())
return XY
else:
from mpi4py import MPI
from GPy.core.parameterization.variational import NormalPosterior
N,D = self.Y.shape
N_list = np.array(self.mpi_comm.allgather(N))
N_all = np.sum(N_list)
Y_all = np.empty((N_all,D)) if self.mpi_comm.rank==root else None
self.mpi_comm.Gatherv([self.Y, MPI.DOUBLE], [Y_all, (N_list*D, None), MPI.DOUBLE], root=root)
if self.mpi_comm.rank==root:
XY = [Y_all]
for l in self.layers:
Q = l.X.shape[1]
X_mean_all = np.empty((N_all,Q)) if self.mpi_comm.rank==root else None
self.mpi_comm.Gatherv([l.X.mean.values, MPI.DOUBLE], [X_mean_all, (N_list*Q, None), MPI.DOUBLE], root=root)
X_var_all = np.empty((N_all,Q)) if self.mpi_comm.rank==root else None
self.mpi_comm.Gatherv([l.X.variance.values, MPI.DOUBLE], [X_var_all, (N_list*Q, None), MPI.DOUBLE], root=root)
if self.mpi_comm.rank==root:
XY.append(NormalPosterior(X_mean_all, X_var_all))
if self.mpi_comm.rank==root: return XY
else: return None
def __init__(self, dim_down, dim_up, likelihood, MLP_dims=None, X=None, X_variance=None, init='rand', Z=None, num_inducing=10, kernel=None, inference_method=None, uncertain_inputs=True,mpi_comm=None, mpi_root=0, back_constraint=True, name='mrdlayer'):
#assert back_constraint
self.uncertain_inputs = uncertain_inputs
Y = self.Y if self.layer_lower is None else self.layer_lower.X
assert isinstance(dim_down, list) or isinstance(dim_down, tuple)
assert isinstance(kernel, list) and len(kernel)==len(dim_down), "The number of kernels has to be equal to the number of input modalities!"
super(MRDLayer, self).__init__(name=name)
self.mpi_comm, self.mpi_root = mpi_comm, mpi_root
self.back_constraint = True if back_constraint else False
self.views = []
for i in range(len(dim_down)):
view = MRDView(Y[i], dim_down[i],dim_up,likelihood=None if likelihood is None else likelihood[i], MLP_dims=None if MLP_dims is None else MLP_dims[i],
X=X, X_variance=X_variance, Z=None if Z is None else Z[i], num_inducing=num_inducing if isinstance(num_inducing,int) else num_inducing[i],
kernel= None if kernel is None else kernel[i], inference_method=None if inference_method is None else inference_method[i], uncertain_inputs=uncertain_inputs,
mpi_comm=mpi_comm, mpi_root=mpi_root, back_constraint=back_constraint, name='view_'+str(i))
self.views.append(view)
if self.back_constraint:
self.X = None
self._aggregate_qX()
else:
self.X = NormalPosterior(X,X_variance)
self.link_parameters(*self.views)
for v in self.views: v.X = self.X
self.link_parameters(self.X)
def __init__(self, X, Y, kernel=None, Z=None, num_inducing=10, X_variance=None, mean_function=None, normalizer=None, mpi_comm=None, name='sparse_gp'):
num_data, input_dim = X.shape
# kern defaults to rbf (plus white for stability)
if kernel is None:
kernel = kern.RBF(input_dim)# + kern.white(input_dim, variance=1e-3)
# Z defaults to a subset of the data
if Z is None:
i = np.random.permutation(num_data)[:min(num_inducing, num_data)]
Z = X.view(np.ndarray)[i].copy()
else:
assert Z.shape[1] == input_dim
likelihood = likelihoods.Gaussian()
if not (X_variance is None):
X = NormalPosterior(X,X_variance)
if mpi_comm is not None:
from ..inference.latent_function_inference.var_dtc_parallel import VarDTC_minibatch
infr = VarDTC_minibatch(mpi_comm=mpi_comm)
else:
infr = VarDTC()
SparseGP_MPI.__init__(self, X, Y, Z, kernel, likelihood, mean_function=mean_function,
inference_method=infr, normalizer=normalizer, mpi_comm=mpi_comm, name=name)
def __init__(self,
X,
X_variance,
Y,
kernel=None,
Z=None,
num_inducing=10,
Y_metadata=None,
normalizer=None):
from GPy.core.parameterization.variational import NormalPosterior
if kernel is None:
kernel = kern.RBF(X.shape[1])
likelihood = likelihoods.Bernoulli()
if Z is None:
i = np.random.permutation(X.shape[0])[:num_inducing]
Z = X[i].copy()
else:
assert Z.shape[1] == X.shape[1]
X = NormalPosterior(X, X_variance)
SparseGP.__init__(self,
X,
Y,
Z,
kernel,
likelihood,
inference_method=EPDTC(),
name='SparseGPClassification',
Y_metadata=Y_metadata,
normalizer=normalizer)
def _next_minibatch(self):
#import pdb; pdb.set_trace()
batch_idx, prev_sample_ix = self.data_streamer.get_cur_index()
batch_idx, samples_idxes, Ys_n, Us_n = self.data_streamer.next_minibatch()
# make U_pre_step shift ->
Ys = []; Us = []; Ys_all = []; Us_all = [];
for i in range(len(Ys_n)):
Y, U = Ys_n[i], Us_n[i]
if self.U_pre_step:
U_t = U[:-1].copy()
Y_t = Y[self.U_win:].copy()
else:
Y_t = Y[self.U_win-1:].copy()
Us_all.append(U)
Ys_all.append(Y)
Us.append( NormalPosterior(U_t,np.ones(U_t.shape)*1e-10) )
Ys.append(Y_t)
# make U_pre_step shift <-
#import pdb; pdb.set_trace()
self.Us = Us
self.Ys = Ys
self.Ys_all = Ys_all
self.Us_all = Us_all
self.Us = Us
self.Ys = Ys
self.Ys_all = Ys_all
self.mb_inf_sample_idxes = samples_idxes[:]
self.minibatch_size = len(self.mb_inf_sample_idxes)
def test_predict_uncertain_inputs(self):
""" Projection of Gaussian through a linear function is still gaussian, and moments are analytical to compute, so we can check this case for predictions easily """
X = np.linspace(-5, 5, 10)[:, None]
Y = 2 * X + np.random.randn(*X.shape) * 1e-3
m = GPy.models.BayesianGPLVM(Y,
1,
X=X,
kernel=GPy.kern.Linear(1),
num_inducing=1)
m.Gaussian_noise[:] = 1e-4
m.X.mean[:] = X[:]
m.X.variance[:] = 1e-5
m.X.fix()
m.optimize()
X_pred_mu = np.random.randn(5, 1)
X_pred_var = np.random.rand(5, 1) + 1e-5
from GPy.core.parameterization.variational import NormalPosterior
X_pred = NormalPosterior(X_pred_mu, X_pred_var)
# mu = \int f(x)q(x|mu,S) dx = \int 2x.q(x|mu,S) dx = 2.mu
# S = \int (f(x) - m)^2q(x|mu,S) dx = \int f(x)^2 q(x) dx - mu**2 = 4(mu^2 + S) - (2.mu)^2 = 4S
Y_mu_true = 2 * X_pred_mu
Y_var_true = 4 * X_pred_var
Y_mu_pred, Y_var_pred = m.predict_noiseless(X_pred)
np.testing.assert_allclose(Y_mu_true, Y_mu_pred, rtol=1e-3)
np.testing.assert_allclose(Y_var_true, Y_var_pred, rtol=1e-3)
def append_XY(self,Y, X):
from GPy import ObsAr
assert self.X_observed
self.update_model(False)
self.layers[-1].set_newX(X,append=True)
for l in self.layers[:0:-1]:
y_mean, y_var = l.predict(l.X[-X.shape[0]:])
l.layer_lower.set_newX(NormalPosterior(y_mean, np.ones_like(y_mean)*y_var), append=True)
self.layers[0].Y = ObsAr(np.vstack([self.layers[0].Y, Y]))
self.update_model(True)
def create_posterior_object(m, samples):
"""Create a NormalPosterior object.
:param m: GP with which to create posterior
:param samples: function inputs to create the posterior at
:return:
"""
mu, covar = m.predict_noiseless(samples, full_cov=True)
variances = covar.diagonal()
variances = np.reshape(variances, (len(samples), 1))
return NormalPosterior(means=mu, variances=variances)
def __init__(self,
X,
Y,
indexD,
kernel=None,
Z=None,
num_inducing=10,
X_variance=None,
normalizer=None,
mpi_comm=None,
individual_Y_noise=False,
name='sparse_gp'):
assert len(Y.shape) == 1 or Y.shape[1] == 1
self.individual_Y_noise = individual_Y_noise
self.indexD = indexD
output_dim = int(np.max(indexD)) + 1
num_data, input_dim = X.shape
# kern defaults to rbf (plus white for stability)
if kernel is None:
kernel = kern.RBF(
input_dim) # + kern.white(input_dim, variance=1e-3)
# Z defaults to a subset of the data
if Z is None:
i = np.random.permutation(num_data)[:min(num_inducing, num_data)]
Z = X.view(np.ndarray)[i].copy()
else:
assert Z.shape[1] == input_dim
if individual_Y_noise:
likelihood = likelihoods.Gaussian(variance=np.array(
[np.var(Y[indexD == d]) for d in range(output_dim)]) * 0.01)
else:
likelihood = likelihoods.Gaussian(variance=np.var(Y) * 0.01)
if not (X_variance is None):
X = NormalPosterior(X, X_variance)
infr = VarDTC_MD()
SparseGP_MPI.__init__(self,
X,
Y,
Z,
kernel,
likelihood,
inference_method=infr,
normalizer=normalizer,
mpi_comm=mpi_comm,
name=name)
self.output_dim = output_dim
def test_posterior(self):
X = np.random.randn(3,5)
Xv = np.random.rand(*X.shape)
par = NormalPosterior(X,Xv)
par.gradient = 10
pcopy = par.copy()
pcopy.gradient = 10
self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
self.assertListEqual(par.gradient_full.tolist(), pcopy.gradient_full.tolist())
self.assertSequenceEqual(str(par), str(pcopy))
self.assertIsNot(par.param_array, pcopy.param_array)
self.assertIsNot(par.gradient_full, pcopy.gradient_full)
with tempfile.TemporaryFile('w+b') as f:
par.pickle(f)
f.seek(0)
pcopy = pickle.load(f)
self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
pcopy.gradient = 10
np.testing.assert_allclose(par.gradient_full, pcopy.gradient_full)
np.testing.assert_allclose(pcopy.mean.gradient_full, 10)
self.assertSequenceEqual(str(par), str(pcopy))
def test_setxy_bgplvm(self):
k = GPy.kern.RBF(1)
m = GPy.models.BayesianGPLVM(self.Y, 2, kernel=k)
mu, var = m.predict(m.X)
X = m.X.copy()
Xnew = NormalPosterior(m.X.mean[:10].copy(), m.X.variance[:10].copy())
m.set_XY(Xnew, m.Y[:10])
assert (m.checkgrad())
m.set_XY(X, self.Y)
mu2, var2 = m.predict(m.X)
np.testing.assert_allclose(mu, mu2)
np.testing.assert_allclose(var, var2)
def predict(self, Xnew, full_cov=False, Y_metadata=None, kern=None):
from GPy.core.parameterization.variational import NormalPosterior
if self.repeatX:
assert self.nLayers==2
mean,var = self.layers[-1].predict(Xnew)
Xnew_norm = (Xnew - self.repeatX_Xmean)/self.repeatX_Xstd
Xmean = np.hstack([mean,Xnew_norm])
Xvar = np.empty_like(Xmean)
Xvar[:] = 1e-6
Xvar[:,:self.nDimsOrig[1]] = var
x = NormalPosterior(Xmean,Xvar)
else:
x = Xnew
for l in self.layers[:0:-1]:
mean, var = l.predict(x)
var = np.clip(var,1e-8, np.inf)
if var.shape[1]==1:
var = np.tile(var,mean.shape[1])
x = NormalPosterior(mean, var)
return self.layers[0].predict(x)
def _next_minibatch(self):
batch_idx, prev_sample_ix = self.data_streamer.get_cur_index()
batch_idx, samples_idxes, Ys_n, Us_n = self.data_streamer.next_minibatch()
# make U_pre_step shift ->
Ys = []; Us = [];
for i in range(len(Ys_n)):
Y, U = Ys_n[i], Us_n[i]
if self.U_pre_step:
U = U[:-1].copy()
Y = Y[self.U_win:].copy()
else:
Y = Y[self.U_win-1:].copy()
Us.append(NormalPosterior(U,np.ones(U.shape)*1e-10))
Ys.append(Y)
# make U_pre_step shift <-
#import pdb; pdb.set_trace()
Xs = self._init_X(self.wins, Ys, None, None, nDims=self.nDims, init='nan')
self.Us = Us
self.Ys = Ys
self.mb_inf_sample_idxes = samples_idxes[:]
previous_layer = None
for i in range(self.nLayers-1,-1,-1): # start from top layer
layer = self.layers[self.nLayers - i - 1]
layer_Us = None
layer_Xs = None
# if i==self.nLayers-1: # Top layer
# layer_Us = Us if layer.withControl else None
# elif i==0: # Observed layer
# layer_Xs = Ys
# #else: # Other layers
# # layer_Us = Xs[i]
# # layer_Xs = Xs[i-1]
#
# layer.set_inputs_and_outputs(len(Ys), Xs=layer_Xs, Us=layer_Us, samples_idxes=samples_idxes)
if i==self.nLayers-1: # Top layer
layer_Us = Us if layer.withControl else None
layer_Xs = Xs[i-1]
elif i==0: # Observed layer
layer_Us = previous_layer.Xs_flat if (previous_layer is not None) else None
layer_Xs = Ys
else: # Other layers
layer_Us = previous_layer.Xs_flat if (previous_layer is not None) else None
layer_Xs = Xs[i-1]
#import pdb; pdb.set_trace()
layer.set_inputs_and_outputs(len(Ys), Xs=layer_Xs, Us=layer_Us, samples_idxes=samples_idxes)
previous_layer = layer
def __init__(self, wins, Y, U=None, U_win=1, nDims=None, X_variance=0.01, num_inducing=10,
likelihood = None, name='autoreg', kernels=None, U_pre_step=True, init='Y',
inducing_init='kmeans', back_cstr=False, MLP_dims=None):
super(DeepAutoreg, self).__init__(name=name)
#import pdb; pdb.set_trace() # Alex
Ys, Us = Y,U
if isinstance(Ys, np.ndarray): Ys = [Ys]
if Us is not None and isinstance(Us, np.ndarray): Us = [Us]
self.nLayers = len(wins)
self.back_cstr = back_cstr
self.wins = wins
#self.input_dim = 1
#self.output_dim = 1
self._log_marginal_likelihood = np.nan
self.U_pre_step = U_pre_step
self.nDims = nDims if nDims is not None else [Ys[0].shape[1]]+[1]*(self.nLayers-1)
if Us is not None:
assert len(Ys)==len(Us)
self.Us = []
self.Ys = []
for i in range(len(Ys)):
Y, U = Ys[i], Us[i]
# assert Y.shape[0]==U.shape[0], "the signal and control should be aligned."
if U_pre_step:
U = U[:-1].copy()
Y = Y[U_win:].copy()
else:
Y = Y[U_win-1:].copy()
self.Us.append(NormalPosterior(U.copy(),np.ones(U.shape)*1e-10))
self.Ys.append(Y)
else:
self.Us = Us
self.Ys = Ys
self.Xs = self._init_X(wins, self.Ys, self.Us, X_variance, init=init, nDims=self.nDims)
# Parameters which exist differently per layer but specified as single componenents are here expanded to each layer
if not isinstance(num_inducing, list or tuple): num_inducing = [num_inducing]*self.nLayers
# Initialize Layers
self.layers = []
for i in range(self.nLayers-1,-1,-1):
if i==self.nLayers-1:
self.layers.append(Layer(None, self.Xs[i-1], X_win=wins[i], Us=self.Us, U_win=U_win, num_inducing=num_inducing[i], kernel=kernels[i] if kernels is not None else None, noise_var=0.01, name='layer_'+str(i), back_cstr=back_cstr, MLP_dims=MLP_dims, inducing_init=inducing_init))
elif i==0:
self.layers.append(Layer(self.layers[-1], self.Ys, X_win=wins[i], Us=self.Xs[i], U_win=wins[i+1], num_inducing=num_inducing[i], kernel=kernels[i] if kernels is not None else None, likelihood=likelihood, noise_var=1., back_cstr=back_cstr, name='layer_'+str(i), inducing_init=inducing_init))
else:
self.layers.append(Layer(self.layers[-1], self.Xs[i-1], X_win=wins[i], Us=self.Xs[i], U_win=wins[i+1], num_inducing=num_inducing[i], kernel=kernels[i] if kernels is not None else None, noise_var=0.01, name='layer_'+str(i), back_cstr=back_cstr, MLP_dims=MLP_dims, inducing_init=inducing_init))
self.link_parameters(*self.layers)
def test_fixed_inputs_uncertain(self):
from GPy.plotting.matplot_dep.util import fixed_inputs
import GPy
from GPy.core.parameterization.variational import NormalPosterior
X_mu = np.random.randn(10, 3)
X_var = np.random.randn(10, 3)
X = NormalPosterior(X_mu, X_var)
Y = np.sin(X_mu) + np.random.randn(10, 3)*1e-3
m = GPy.models.BayesianGPLVM(Y, X=X_mu, X_variance=X_var, input_dim=3)
fixed = fixed_inputs(m, [1], fix_routine='median', as_list=True, X_all=False)
self.assertTrue((0, np.median(X.mean.values[:,0])) in fixed)
self.assertTrue((2, np.median(X.mean.values[:,2])) in fixed)
self.assertTrue(len([t for t in fixed if t[0] == 1]) == 0) # Unfixed input should not be in fixed
def set_newX(self, X, append=False):
from GPy import ObsAr
if not self.uncertain_inputs:
if append:
self.X = ObsAr(np.vstack([self.X, X]))
else:
self.X = X if isinstance(X,ObsAr) else ObsAr(X)
else:
self.unlink_parameter(self.X)
if append:
self.X = NormalPosterior(np.vstack([self.X.mean.values, X.mean.values]),np.vstack([self.X.variance.values, X.variance.values]))
else:
self.X = X
self.link_parameter(self.X)
def __init__(self, which, X, X_variance, Z, num_inducing, kernel):
super(PsiStatModel, self).__init__(name='psi stat test')
self.which = which
self.X = Param("X", X)
self.X_variance = Param('X_variance', X_variance, Logexp())
self.q = NormalPosterior(self.X, self.X_variance)
self.Z = Param("Z", Z)
self.N, self.input_dim = X.shape
self.num_inducing, input_dim = Z.shape
assert self.input_dim == input_dim, "shape missmatch: Z:{!s} X:{!s}".format(
Z.shape, X.shape)
self.kern = kernel
self.psi_ = self.kern.__getattribute__(self.which)(self.Z, self.q)
self.add_parameters(self.q, self.Z, self.kern)
def __init__(self, Y, input_dim, X=None, X_variance=None, init='PCA', num_inducing=10,
Z=None, kernel=None, inference_method=None, likelihood=None,
name='bayesian gplvm', normalizer=None,
missing_data=False, stochastic=False, batchsize=1):
self.logger = logging.getLogger(self.__class__.__name__)
if X is None:
from ..util.initialization import initialize_latent
self.logger.info("initializing latent space X with method {}".format(init))
X, fracs = initialize_latent(init, input_dim, Y)
else:
fracs = np.ones(input_dim)
self.init = init
if Z is None:
self.logger.info("initializing inducing inputs")
Z = np.random.permutation(X.copy())[:num_inducing]
assert Z.shape[1] == X.shape[1]
if X_variance is False:
self.logger.info('no variance on X, activating sparse GPLVM')
X = Param("latent space", X)
else:
if X_variance is None:
self.logger.info("initializing latent space variance ~ uniform(0,.1)")
X_variance = np.random.uniform(0,.1,X.shape)
self.variational_prior = NormalPrior()
X = NormalPosterior(X, X_variance)
if kernel is None:
self.logger.info("initializing kernel RBF")
kernel = kern.RBF(input_dim, lengthscale=1./fracs, ARD=True) #+ kern.Bias(input_dim) + kern.White(input_dim)
if likelihood is None:
likelihood = Gaussian()
self.kl_factr = 1.
if inference_method is None:
from ..inference.latent_function_inference.var_dtc import VarDTC
self.logger.debug("creating inference_method var_dtc")
inference_method = VarDTC(limit=3 if not missing_data else Y.shape[1])
super(BayesianGPLVMMiniBatch,self).__init__(X, Y, Z, kernel, likelihood=likelihood,
name=name, inference_method=inference_method,
normalizer=normalizer,
missing_data=missing_data, stochastic=stochastic,
batchsize=batchsize)
self.X = X
self.link_parameter(self.X, 0)
def _init_XY(self):
X_win, X_dim, U_win, U_dim = self.X_win, self.X_dim, self.U_win, self.U_dim
self._update_conv()
if X_win > 0:
X_mean_conv, X_var_conv = np.vstack(self.X_mean_conv), np.vstack(
self.X_var_conv)
if self.withControl:
U_mean_conv, U_var_conv = np.vstack(self.U_mean_conv), np.vstack(
self.U_var_conv)
if not self.withControl:
self.X = NormalPosterior(X_mean_conv, X_var_conv)
elif X_win == 0:
self.X = NormalPosterior(U_mean_conv, U_var_conv)
else:
self.X = NormalPosterior(np.hstack([X_mean_conv, U_mean_conv]),
np.hstack([X_var_conv, U_var_conv]))
if self.X_observed:
self.Y = np.vstack([x[X_win:] for x in self.Xs_flat])
else:
self.Y = NormalPosterior(
np.vstack([x.mean.values[X_win:] for x in self.Xs_flat]),
np.vstack([x.variance.values[X_win:] for x in self.Xs_flat]))
def setUp(self):
from GPy.core.parameterization.variational import NormalPosterior
N, M, Q = 100, 20, 3
X = np.random.randn(N, Q)
X_var = np.random.rand(N, Q) + 0.01
self.Z = np.random.randn(M, Q)
self.qX = NormalPosterior(X, X_var)
self.w1 = np.random.randn(N)
self.w2 = np.random.randn(N, M)
self.w3 = np.random.randn(M, M)
self.w3 = self.w3 + self.w3.T
self.w3n = np.random.randn(N, M, M)
self.w3n = self.w3n + np.swapaxes(self.w3n, 1, 2)
def _aggregate_qX(self):
if self.back_constraint:
if self.X is None:
self.X = NormalPosterior(np.zeros_like(self.views[0].Xv.mean.values), np.zeros_like(self.views[0].Xv.variance.values))
else:
self.X.mean[:] = 0
self.X.variance[:] = 0
self.prec_denom = np.zeros_like(self.X.variance.values)
for v in self.views:
self.prec_denom += 1./v.Xv.variance.values
self.X.mean += v.Xv.mean.values/v.Xv.variance.values
self.X.mean /= self.prec_denom
self.X.variance[:] = 1./self.prec_denom
else:
for v in self.views:
v.X = self.X
def setUp(self):
self.kerns = (
#GPy.kern.RBF([0,1,2], ARD=True)+GPy.kern.Bias(self.input_dim)+GPy.kern.White(self.input_dim),
#GPy.kern.RBF(self.input_dim)+GPy.kern.Bias(self.input_dim)+GPy.kern.White(self.input_dim),
#GPy.kern.Linear(self.input_dim) + GPy.kern.Bias(self.input_dim) + GPy.kern.White(self.input_dim),
#GPy.kern.Linear(self.input_dim, ARD=True) + GPy.kern.Bias(self.input_dim) + GPy.kern.White(self.input_dim),
GPy.kern.Linear([1, 3, 6, 7], ARD=True) +
GPy.kern.RBF([0, 5, 8], ARD=True) +
GPy.kern.White(self.input_dim), )
self.q_x_mean = np.random.randn(self.input_dim)[None]
self.q_x_variance = np.exp(.5 * np.random.randn(self.input_dim))[None]
self.q_x_samples = np.random.randn(
self.Nsamples, self.input_dim) * np.sqrt(
self.q_x_variance) + self.q_x_mean
self.q_x = NormalPosterior(self.q_x_mean, self.q_x_variance)
self.Z = np.random.randn(self.num_inducing, self.input_dim)
self.q_x_mean.shape = (1, self.input_dim)
self.q_x_variance.shape = (1, self.input_dim)
def _run_encoder_forward(self,):
"""
Runs the encoder and generate layer's q_X.
"""
# stack batches over axis 1. This is required by pytorch.
if not self.encode_include_input:
out_means, out_vars = self.encoder.forward_computation( np.stack(self.Ys_all, axis=1) )
else:
out_means, out_vars = self.encoder.forward_computation( np.stack(self.Ys_all, axis=1), np.stack(self.Us_all, axis=1) )
Xs = []
#import pdb; pdb.set_trace()
# iteration over layers (start from lower, observed is 0), inside each layer over i_seq
for layer_idx in range(len(out_means)): # iteration over layers
Xs_layer = []
for sample_idx in range(len(out_means[layer_idx])): # iteration over layers
Xs_layer.append( NormalPosterior( out_means[layer_idx][sample_idx], out_vars[layer_idx][sample_idx], name='qX_'+str( layer_idx ) ) )
Xs.append( Xs_layer )
return Xs
def _init_encoder(self, MLP_dims):
from .mlp import MLP
from copy import deepcopy
from GPy.core.parameterization.transformations import Logexp
X_win, X_dim, U_win, U_dim = self.X_win, self.X_dim, self.U_win, self.U_dim
assert X_win > 0, "Neural Network constraints only applies autoregressive structure!"
Q = X_win * X_dim + U_win * U_dim if self.withControl else X_win * X_dim
self.init_Xs = [
NormalPosterior(self.Xs_flat[i].mean.values[:X_win],
self.Xs_flat[i].variance.values[:X_win],
name='init_Xs_' + str(i)) for i in range(self.nSeq)
]
for init_X in self.init_Xs:
init_X.mean[:] = np.random.randn(*init_X.shape) * 1e-2
self.encoder = MLP([Q, Q * 2, Q +
X_dim / 2, X_dim] if MLP_dims is None else [Q] +
deepcopy(MLP_dims) + [X_dim])
self.Xs_var = [
Param('X_var_' + str(i),
self.Xs_flat[i].variance.values[X_win:].copy(), Logexp())
for i in range(self.nSeq)
]
def __init__(self, Y, dim_down, dim_up, likelihood, MLP_dims=None, X=None, X_variance=None, init='rand', Z=None, num_inducing=10, kernel=None, inference_method=None, uncertain_inputs=True,mpi_comm=None, mpi_root=0, back_constraint=True, name='mrd-view'):
self.uncertain_inputs = uncertain_inputs
self.layer_lower = None
self.scale = 1.
if back_constraint:
from .mlp import MLP
from copy import deepcopy
self.encoder = MLP([dim_down, int((dim_down+dim_up)*2./3.), int((dim_down+dim_up)/3.), dim_up] if MLP_dims is None else [dim_down]+deepcopy(MLP_dims)+[dim_up])
X = self.encoder.predict(Y.mean.values if isinstance(Y, VariationalPosterior) else Y)
X_variance = 0.0001*np.ones(X.shape)
self.back_constraint = True
else:
self.back_constraint = False
if Z is None:
Z = np.random.rand(num_inducing, dim_up)*2-1. #np.random.permutation(X.copy())[:num_inducing]
assert Z.shape[1] == X.shape[1]
if likelihood is None: likelihood = likelihoods.Gaussian(variance=Y.var()*0.01)
if uncertain_inputs: X = NormalPosterior(X, X_variance)
if kernel is None: kernel = kern.RBF(dim_up, ARD = True)
# The command below will also give the field self.X to the view.
super(MRDView, self).__init__(X, Y, Z, kernel, likelihood, inference_method=inference_method, mpi_comm=mpi_comm, mpi_root=mpi_root, name=name)
if back_constraint: self.link_parameter(self.encoder)
if self.uncertain_inputs and self.back_constraint:
from GPy import Param
from GPy.core.parameterization.transformations import Logexp
self.X_var_common = Param('X_var',X_variance[0].copy(),Logexp())
self.link_parameters(self.X_var_common)
# There's some redundancy in the self.Xv and self.X. Currently we use self.X for the likelihood part and all calculations part,
# self.Xv is only used for the self.Xv.gradient part.
# This is redundant but it's there in case we want to do the product of experts MRD model.
self.Xv = self.X
def __init__(self, layer_lower, dim_down, dim_up, likelihood, X=None, X_variance=None, init='PCA', Z=None, num_inducing=10, kernel=None, inference_method=None, uncertain_inputs=True,mpi_comm=None, mpi_root=0, back_constraint=True, encoder=None, auto_update=True, name='layer'):
self.uncertain_inputs = uncertain_inputs
self.layer_lower = layer_lower
Y = self.Y if self.layer_lower is None else self.layer_lower.X
self.back_constraint = back_constraint
from deepgp.util.util import initialize_latent
if X is None: X, _ = initialize_latent(init, Y.shape[0], dim_up, Y.mean.values if isinstance(Y, VariationalPosterior) else Y)
if X_variance is None: X_variance = 0.01*np.ones(X.shape) + 0.01*np.random.rand(*X.shape)
if Z is None:
if self.back_constraint: Z = np.random.rand(num_inducing, dim_up)*2-1.
else:
if num_inducing<=X.shape[0]:
Z = X[np.random.permutation(X.shape[0])[:num_inducing]].copy()
else:
Z_more = np.random.rand(num_inducing-X.shape[0],X.shape[1])*(X.max(0)-X.min(0))+X.min(0)
Z = np.vstack([X.copy(),Z_more])
assert Z.shape[1] == X.shape[1]
if mpi_comm is not None:
from ..util.parallel import broadcastArrays
broadcastArrays([Z], mpi_comm, mpi_root)
if uncertain_inputs: X = NormalPosterior(X, X_variance)
if kernel is None: kernel = kern.RBF(dim_up, ARD = True)
assert kernel.input_dim==X.shape[1], "The dimensionality of input has to be equal to the input dimensionality of kernel!"
self.Kuu_sigma = Param('Kuu_var', np.zeros(num_inducing)+1e-3, Logexp())
super(Layer, self).__init__(X, Y, Z, kernel, likelihood, inference_method=inference_method, mpi_comm=mpi_comm, mpi_root=mpi_root, auto_update=auto_update, name=name)
self.link_parameter(self.Kuu_sigma)
if back_constraint: self.encoder = encoder
if self.uncertain_inputs and not self.back_constraint:
self.link_parameter(self.X)
def test_posterior(self):
X = np.random.randn(3,5)
Xv = np.random.rand(*X.shape)
par = NormalPosterior(X,Xv)
par.gradient = 10
pcopy = par.copy()
pcopy.gradient = 10
self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
self.assertListEqual(par.gradient_full.tolist(), pcopy.gradient_full.tolist())
self.assertSequenceEqual(str(par), str(pcopy))
self.assertIsNot(par.param_array, pcopy.param_array)
self.assertIsNot(par.gradient_full, pcopy.gradient_full)
with tempfile.TemporaryFile('w+b') as f:
par.pickle(f)
f.seek(0)
pcopy = pickle.load(f)
self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist())
pcopy.gradient = 10
np.testing.assert_allclose(par.gradient_full, pcopy.gradient_full)
np.testing.assert_allclose(pcopy.mean.gradient_full, 10)
self.assertSequenceEqual(str(par), str(pcopy))
def __init__(self,
X,
Y,
Xr_dim,
kernel=None,
kernel_row=None,
Z=None,
Z_row=None,
X_row=None,
Xvariance_row=None,
num_inducing=(10, 10),
qU_var_r_W_dim=None,
qU_var_c_W_dim=None,
init='GP',
name='GPMR'):
#Kernel
if kernel is None:
kernel = kern.RBF(X.shape[1])
if kernel_row is None:
kernel_row = kern.RBF(Xr_dim, name='kern_row')
if init == 'GP':
from . import SparseGPRegression, BayesianGPLVM
from ..util.linalg import jitchol
Mc, Mr = num_inducing
print('Intializing with GP...')
print('Fit Sparse GP...')
m_sgp = SparseGPRegression(X,
Y,
kernel=kernel.copy(),
num_inducing=Mc)
m_sgp.likelihood.variance[:] = Y.var() * 0.01
m_sgp.optimize(max_iters=1000)
print('Fit BGPLVM...')
m_lvm = BayesianGPLVM(m_sgp.posterior.mean.copy().T,
Xr_dim,
kernel=kernel_row.copy(),
num_inducing=Mr)
m_lvm.likelihood.variance[:] = m_lvm.Y.var() * 0.01
m_lvm.optimize(max_iters=10000)
kernel[:] = m_sgp.kern.param_array.copy()
kernel.variance[:] = np.sqrt(kernel.variance)
Z = m_sgp.Z.values.copy()
kernel_row[:] = m_lvm.kern.param_array.copy()
kernel_row.variance[:] = np.sqrt(kernel_row.variance)
Z_row = m_lvm.Z.values.copy()
X_row = m_lvm.X.mean.values.copy()
Xvariance_row = m_lvm.X.variance.values
qU_mean = m_lvm.posterior.mean.T.copy()
qU_var_col_W = jitchol(m_sgp.posterior.covariance)
qU_var_col_diag = np.full(Mc, 1e-5)
qU_var_row_W = jitchol(m_lvm.posterior.covariance)
qU_var_row_diag = np.full(Mr, 1e-5)
print('Done.')
else:
qU_mean = np.zeros(num_inducing)
qU_var_col_W = np.random.randn(
num_inducing[0], num_inducing[0]
if qU_var_c_W_dim is None else qU_var_c_W_dim) * 0.01
qU_var_col_diag = np.full(num_inducing[0], 1e-5)
qU_var_row_W = np.random.randn(
num_inducing[1], num_inducing[1]
if qU_var_r_W_dim is None else qU_var_r_W_dim) * 0.01
qU_var_row_diag = np.full(num_inducing[1], 1e-5)
if X_row is None:
u, s, v = np.linalg.svd(Y)
X_row = Y.T.dot(u[:, :Xr_dim]) #*np.sqrt(s)[:Xr_dim])
X_row = X_row / X_row.std(0)
if Xvariance_row is None:
Xvariance_row = np.ones((Y.shape[1], Xr_dim)) * 0.0001
if Z is None:
Z = X[np.random.permutation(X.shape[0])[:num_inducing[0]]].copy()
if Z_row is None:
Z_row = X_row[np.random.permutation(
X_row.shape[0])[:num_inducing[1]]].copy()
self.kern_row = kernel_row
self.X_row = NormalPosterior(X_row, Xvariance_row, name='Xr')
self.Z_row = Param('Zr', Z_row)
self.variational_prior_row = NormalPrior()
self.qU_mean = Param('qU_mean', qU_mean)
self.qU_var_c_W = Param('qU_var_col_W', qU_var_col_W)
self.qU_var_c_diag = Param('qU_var_col_diag', qU_var_col_diag,
Logexp())
self.qU_var_r_W = Param('qU_var_row_W', qU_var_row_W)
self.qU_var_r_diag = Param('qU_var_row_diag', qU_var_row_diag,
Logexp())
#Likelihood
likelihood = likelihoods.Gaussian(variance=np.var(Y) * 0.01)
from ..inference.latent_function_inference import VarDTC_SVI_Multiout
inference_method = VarDTC_SVI_Multiout()
super(GPMultioutRegression,
self).__init__(X,
Y,
Z,
kernel,
likelihood=likelihood,
name=name,
inference_method=inference_method)
self.link_parameters(self.kern_row, self.X_row, self.Z_row,
self.qU_mean, self.qU_var_c_W, self.qU_var_c_diag,
self.qU_var_r_W, self.qU_var_r_diag)
self._log_marginal_likelihood = np.nan
class Kernel_Psi_statistics_GradientTests(unittest.TestCase):
def setUp(self):
from GPy.core.parameterization.variational import NormalPosterior
N, M, Q = 100, 20, 3
X = np.random.randn(N, Q)
X_var = np.random.rand(N, Q) + 0.01
self.Z = np.random.randn(M, Q)
self.qX = NormalPosterior(X, X_var)
self.w1 = np.random.randn(N)
self.w2 = np.random.randn(N, M)
self.w3 = np.random.randn(M, M)
self.w3 = self.w3 + self.w3.T
self.w3n = np.random.randn(N, M, M)
self.w3n = self.w3n + np.swapaxes(self.w3n, 1, 2)
def test_kernels(self):
from GPy.kern import RBF, Linear, MLP
Q = self.Z.shape[1]
kernels = [RBF(Q, ARD=True), Linear(Q, ARD=True)]
for k in kernels:
k.randomize()
self._test_kernel_param(k)
self._test_Z(k)
self._test_qX(k)
self._test_kernel_param(k, psi2n=True)
self._test_Z(k, psi2n=True)
self._test_qX(k, psi2n=True)
def _test_kernel_param(self, kernel, psi2n=False):
def f(p):
kernel.param_array[:] = p
psi0 = kernel.psi0(self.Z, self.qX)
psi1 = kernel.psi1(self.Z, self.qX)
if not psi2n:
psi2 = kernel.psi2(self.Z, self.qX)
return (self.w1 * psi0).sum() + (self.w2 * psi1).sum() + (self.w3 * psi2).sum()
else:
psi2 = kernel.psi2n(self.Z, self.qX)
return (self.w1 * psi0).sum() + (self.w2 * psi1).sum() + (self.w3n * psi2).sum()
def df(p):
kernel.param_array[:] = p
kernel.update_gradients_expectations(self.w1, self.w2, self.w3 if not psi2n else self.w3n, self.Z, self.qX)
return kernel.gradient.copy()
from GPy.models import GradientChecker
m = GradientChecker(f, df, kernel.param_array.copy())
self.assertTrue(m.checkgrad())
def _test_Z(self, kernel, psi2n=False):
def f(p):
psi0 = kernel.psi0(p, self.qX)
psi1 = kernel.psi1(p, self.qX)
psi2 = kernel.psi2(p, self.qX)
if not psi2n:
psi2 = kernel.psi2(p, self.qX)
return (self.w1 * psi0).sum() + (self.w2 * psi1).sum() + (self.w3 * psi2).sum()
else:
psi2 = kernel.psi2n(p, self.qX)
return (self.w1 * psi0).sum() + (self.w2 * psi1).sum() + (self.w3n * psi2).sum()
def df(p):
return kernel.gradients_Z_expectations(self.w1, self.w2, self.w3 if not psi2n else self.w3n, p, self.qX)
from GPy.models import GradientChecker
m = GradientChecker(f, df, self.Z.copy())
self.assertTrue(m.checkgrad())
def _test_qX(self, kernel, psi2n=False):
def f(p):
self.qX.param_array[:] = p
self.qX._trigger_params_changed()
psi0 = kernel.psi0(self.Z, self.qX)
psi1 = kernel.psi1(self.Z, self.qX)
if not psi2n:
psi2 = kernel.psi2(self.Z, self.qX)
return (self.w1 * psi0).sum() + (self.w2 * psi1).sum() + (self.w3 * psi2).sum()
else:
psi2 = kernel.psi2n(self.Z, self.qX)
return (self.w1 * psi0).sum() + (self.w2 * psi1).sum() + (self.w3n * psi2).sum()
def df(p):
self.qX.param_array[:] = p
self.qX._trigger_params_changed()
grad = kernel.gradients_qX_expectations(
self.w1, self.w2, self.w3 if not psi2n else self.w3n, self.Z, self.qX
)
self.qX.set_gradients(grad)
return self.qX.gradient.copy()
from GPy.models import GradientChecker
m = GradientChecker(f, df, self.qX.param_array.copy())
self.assertTrue(m.checkgrad())
