def _init_X(self, wins, Ys, Us, X_variance, nDims, init='Y'): #import pdb; pdb.set_trace() Xs = [] for i_layer in range(1,self.nLayers): X = [] for i_seq in range(len(Ys)): Y = Ys[i_seq] U_win, U_dim = wins[i_layer], nDims[i_layer] if init=='Y': mean = np.zeros((U_win+Y.shape[0], U_dim)) mean[U_win:] = Y[:,:U_dim] var = np.zeros((U_win+Y.shape[0],U_dim))+X_variance X.append(NormalPosterior(mean,var,name='qX_'+str(i_seq))) elif init=='rand' : mean = np.zeros((U_win+Y.shape[0], U_dim)) mean[:] = np.random.randn(*mean.shape) var = np.zeros((U_win+Y.shape[0],U_dim))+X_variance X.append(NormalPosterior(mean,var,name='qX_'+str(i_seq))) elif init=='zero': mean = np.zeros((U_win+Y.shape[0], U_dim)) mean[:] = np.random.randn(*mean.shape)*0.01 var = np.zeros((U_win+Y.shape[0],U_dim))+X_variance X.append(NormalPosterior(mean,var,name='qX_'+str(i_seq))) Xs.append(X) return Xs
def gen_pred_model(self, Y=None, init_X='encoder', binObserved=False): from GPy.core.parameterization.variational import NormalPosterior from deepgp.models.pred_model import PredDeepGP if Y is not None: Xs = [Y] if init_X=='nn': xy = self.collect_all_XY() for i in range(self.nLayers): x_mean, x_var = PredDeepGP._init_Xs_NN(Y,i,xy) Xs.append(NormalPosterior(x_mean, x_var)) elif init_X=='encoder': x_mean = Y x_mean[np.isnan(x_mean)] = 0. for layer in self.layers: x_mean = layer.encoder.predict(x_mean) Xs.append(NormalPosterior(x_mean, np.ones(x_mean.shape)*layer.X_var)) layers = [] layer_lower = None for i in range(self.nLayers): layers.append(self.layers[i].gen_pred_layer(layer_lower=layer_lower,Y=Xs[i], X=Xs[i+1], binObserved=(i==0 and binObserved))) layer_lower = layers[-1] pmodel = PredDeepGP(layers) if init_X=='nn': pmodel.set_train_data(xy) return pmodel
def predict(self, Xnew, full_cov=False, Y_metadata=None, kern=None, view=0): """Make a prediction from the deep Gaussian process model for a given input""" from GPy.core.parameterization.variational import NormalPosterior if self.repeatX: assert self.nLayers == 2 mean, var = self.layers[-1].predict(Xnew) Xnew_norm = (Xnew - self.repeatX_Xmean) / self.repeatX_Xstd Xmean = np.hstack([mean, Xnew_norm]) Xvar = np.empty_like(Xmean) Xvar[:] = 1e-6 Xvar[:, :self.nDimsOrig[1]] = var x = NormalPosterior(Xmean, Xvar) else: x = Xnew for l in self.layers[:0:-1]: mean, var = l.predict(x) var = np.clip(var, 1e-8, np.inf) if var.shape[1] == 1: var = np.tile(var, mean.shape[1]) x = NormalPosterior(mean, var) mrd_flag = hasattr(self.layers[0], 'views') if mrd_flag: return self.layers[0].views[view].predict(x) else: return self.layers[0].predict(x)
def setUp(self): from GPy.core.parameterization.variational import NormalPosterior N, M, Q = 100, 20, 3 X = np.random.randn(N, Q) X_var = np.random.rand(N, Q) + 0.01 self.Z = np.random.randn(M, Q) self.qX = NormalPosterior(X, X_var) self.w1 = np.random.randn(N) self.w2 = np.random.randn(N, M) self.w3 = np.random.randn(M, M) self.w3 = self.w3 #+self.w3.T self.w3n = np.random.randn(N, M, M) self.w3n = self.w3n + np.swapaxes(self.w3n, 1, 2)
def collect_all_XY(self, root=0): if self.mpi_comm is None: XY = [self.obslayer.Y.copy()] for l in self.layers: XY.append(l.X.copy()) return XY else: from mpi4py import MPI from GPy.core.parameterization.variational import NormalPosterior N,D = self.Y.shape N_list = np.array(self.mpi_comm.allgather(N)) N_all = np.sum(N_list) Y_all = np.empty((N_all,D)) if self.mpi_comm.rank==root else None self.mpi_comm.Gatherv([self.Y, MPI.DOUBLE], [Y_all, (N_list*D, None), MPI.DOUBLE], root=root) if self.mpi_comm.rank==root: XY = [Y_all] for l in self.layers: Q = l.X.shape[1] X_mean_all = np.empty((N_all,Q)) if self.mpi_comm.rank==root else None self.mpi_comm.Gatherv([l.X.mean.values, MPI.DOUBLE], [X_mean_all, (N_list*Q, None), MPI.DOUBLE], root=root) X_var_all = np.empty((N_all,Q)) if self.mpi_comm.rank==root else None self.mpi_comm.Gatherv([l.X.variance.values, MPI.DOUBLE], [X_var_all, (N_list*Q, None), MPI.DOUBLE], root=root) if self.mpi_comm.rank==root: XY.append(NormalPosterior(X_mean_all, X_var_all)) if self.mpi_comm.rank==root: return XY else: return None
def __init__(self, dim_down, dim_up, likelihood, MLP_dims=None, X=None, X_variance=None, init='rand', Z=None, num_inducing=10, kernel=None, inference_method=None, uncertain_inputs=True,mpi_comm=None, mpi_root=0, back_constraint=True, name='mrdlayer'): #assert back_constraint self.uncertain_inputs = uncertain_inputs Y = self.Y if self.layer_lower is None else self.layer_lower.X assert isinstance(dim_down, list) or isinstance(dim_down, tuple) assert isinstance(kernel, list) and len(kernel)==len(dim_down), "The number of kernels has to be equal to the number of input modalities!" super(MRDLayer, self).__init__(name=name) self.mpi_comm, self.mpi_root = mpi_comm, mpi_root self.back_constraint = True if back_constraint else False self.views = [] for i in range(len(dim_down)): view = MRDView(Y[i], dim_down[i],dim_up,likelihood=None if likelihood is None else likelihood[i], MLP_dims=None if MLP_dims is None else MLP_dims[i], X=X, X_variance=X_variance, Z=None if Z is None else Z[i], num_inducing=num_inducing if isinstance(num_inducing,int) else num_inducing[i], kernel= None if kernel is None else kernel[i], inference_method=None if inference_method is None else inference_method[i], uncertain_inputs=uncertain_inputs, mpi_comm=mpi_comm, mpi_root=mpi_root, back_constraint=back_constraint, name='view_'+str(i)) self.views.append(view) if self.back_constraint: self.X = None self._aggregate_qX() else: self.X = NormalPosterior(X,X_variance) self.link_parameters(*self.views) for v in self.views: v.X = self.X self.link_parameters(self.X)
def __init__(self, X, Y, kernel=None, Z=None, num_inducing=10, X_variance=None, mean_function=None, normalizer=None, mpi_comm=None, name='sparse_gp'): num_data, input_dim = X.shape # kern defaults to rbf (plus white for stability) if kernel is None: kernel = kern.RBF(input_dim)# + kern.white(input_dim, variance=1e-3) # Z defaults to a subset of the data if Z is None: i = np.random.permutation(num_data)[:min(num_inducing, num_data)] Z = X.view(np.ndarray)[i].copy() else: assert Z.shape[1] == input_dim likelihood = likelihoods.Gaussian() if not (X_variance is None): X = NormalPosterior(X,X_variance) if mpi_comm is not None: from ..inference.latent_function_inference.var_dtc_parallel import VarDTC_minibatch infr = VarDTC_minibatch(mpi_comm=mpi_comm) else: infr = VarDTC() SparseGP_MPI.__init__(self, X, Y, Z, kernel, likelihood, mean_function=mean_function, inference_method=infr, normalizer=normalizer, mpi_comm=mpi_comm, name=name)
def __init__(self, X, X_variance, Y, kernel=None, Z=None, num_inducing=10, Y_metadata=None, normalizer=None): from GPy.core.parameterization.variational import NormalPosterior if kernel is None: kernel = kern.RBF(X.shape[1]) likelihood = likelihoods.Bernoulli() if Z is None: i = np.random.permutation(X.shape[0])[:num_inducing] Z = X[i].copy() else: assert Z.shape[1] == X.shape[1] X = NormalPosterior(X, X_variance) SparseGP.__init__(self, X, Y, Z, kernel, likelihood, inference_method=EPDTC(), name='SparseGPClassification', Y_metadata=Y_metadata, normalizer=normalizer)
def _next_minibatch(self): #import pdb; pdb.set_trace() batch_idx, prev_sample_ix = self.data_streamer.get_cur_index() batch_idx, samples_idxes, Ys_n, Us_n = self.data_streamer.next_minibatch() # make U_pre_step shift -> Ys = []; Us = []; Ys_all = []; Us_all = []; for i in range(len(Ys_n)): Y, U = Ys_n[i], Us_n[i] if self.U_pre_step: U_t = U[:-1].copy() Y_t = Y[self.U_win:].copy() else: Y_t = Y[self.U_win-1:].copy() Us_all.append(U) Ys_all.append(Y) Us.append( NormalPosterior(U_t,np.ones(U_t.shape)*1e-10) ) Ys.append(Y_t) # make U_pre_step shift <- #import pdb; pdb.set_trace() self.Us = Us self.Ys = Ys self.Ys_all = Ys_all self.Us_all = Us_all self.Us = Us self.Ys = Ys self.Ys_all = Ys_all self.mb_inf_sample_idxes = samples_idxes[:] self.minibatch_size = len(self.mb_inf_sample_idxes)
def test_predict_uncertain_inputs(self): """ Projection of Gaussian through a linear function is still gaussian, and moments are analytical to compute, so we can check this case for predictions easily """ X = np.linspace(-5, 5, 10)[:, None] Y = 2 * X + np.random.randn(*X.shape) * 1e-3 m = GPy.models.BayesianGPLVM(Y, 1, X=X, kernel=GPy.kern.Linear(1), num_inducing=1) m.Gaussian_noise[:] = 1e-4 m.X.mean[:] = X[:] m.X.variance[:] = 1e-5 m.X.fix() m.optimize() X_pred_mu = np.random.randn(5, 1) X_pred_var = np.random.rand(5, 1) + 1e-5 from GPy.core.parameterization.variational import NormalPosterior X_pred = NormalPosterior(X_pred_mu, X_pred_var) # mu = \int f(x)q(x|mu,S) dx = \int 2x.q(x|mu,S) dx = 2.mu # S = \int (f(x) - m)^2q(x|mu,S) dx = \int f(x)^2 q(x) dx - mu**2 = 4(mu^2 + S) - (2.mu)^2 = 4S Y_mu_true = 2 * X_pred_mu Y_var_true = 4 * X_pred_var Y_mu_pred, Y_var_pred = m.predict_noiseless(X_pred) np.testing.assert_allclose(Y_mu_true, Y_mu_pred, rtol=1e-3) np.testing.assert_allclose(Y_var_true, Y_var_pred, rtol=1e-3)
def append_XY(self,Y, X): from GPy import ObsAr assert self.X_observed self.update_model(False) self.layers[-1].set_newX(X,append=True) for l in self.layers[:0:-1]: y_mean, y_var = l.predict(l.X[-X.shape[0]:]) l.layer_lower.set_newX(NormalPosterior(y_mean, np.ones_like(y_mean)*y_var), append=True) self.layers[0].Y = ObsAr(np.vstack([self.layers[0].Y, Y])) self.update_model(True)
def create_posterior_object(m, samples): """Create a NormalPosterior object. :param m: GP with which to create posterior :param samples: function inputs to create the posterior at :return: """ mu, covar = m.predict_noiseless(samples, full_cov=True) variances = covar.diagonal() variances = np.reshape(variances, (len(samples), 1)) return NormalPosterior(means=mu, variances=variances)
def __init__(self, X, Y, indexD, kernel=None, Z=None, num_inducing=10, X_variance=None, normalizer=None, mpi_comm=None, individual_Y_noise=False, name='sparse_gp'): assert len(Y.shape) == 1 or Y.shape[1] == 1 self.individual_Y_noise = individual_Y_noise self.indexD = indexD output_dim = int(np.max(indexD)) + 1 num_data, input_dim = X.shape # kern defaults to rbf (plus white for stability) if kernel is None: kernel = kern.RBF( input_dim) # + kern.white(input_dim, variance=1e-3) # Z defaults to a subset of the data if Z is None: i = np.random.permutation(num_data)[:min(num_inducing, num_data)] Z = X.view(np.ndarray)[i].copy() else: assert Z.shape[1] == input_dim if individual_Y_noise: likelihood = likelihoods.Gaussian(variance=np.array( [np.var(Y[indexD == d]) for d in range(output_dim)]) * 0.01) else: likelihood = likelihoods.Gaussian(variance=np.var(Y) * 0.01) if not (X_variance is None): X = NormalPosterior(X, X_variance) infr = VarDTC_MD() SparseGP_MPI.__init__(self, X, Y, Z, kernel, likelihood, inference_method=infr, normalizer=normalizer, mpi_comm=mpi_comm, name=name) self.output_dim = output_dim
def test_posterior(self): X = np.random.randn(3,5) Xv = np.random.rand(*X.shape) par = NormalPosterior(X,Xv) par.gradient = 10 pcopy = par.copy() pcopy.gradient = 10 self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist()) self.assertListEqual(par.gradient_full.tolist(), pcopy.gradient_full.tolist()) self.assertSequenceEqual(str(par), str(pcopy)) self.assertIsNot(par.param_array, pcopy.param_array) self.assertIsNot(par.gradient_full, pcopy.gradient_full) with tempfile.TemporaryFile('w+b') as f: par.pickle(f) f.seek(0) pcopy = pickle.load(f) self.assertListEqual(par.param_array.tolist(), pcopy.param_array.tolist()) pcopy.gradient = 10 np.testing.assert_allclose(par.gradient_full, pcopy.gradient_full) np.testing.assert_allclose(pcopy.mean.gradient_full, 10) self.assertSequenceEqual(str(par), str(pcopy))
def test_setxy_bgplvm(self): k = GPy.kern.RBF(1) m = GPy.models.BayesianGPLVM(self.Y, 2, kernel=k) mu, var = m.predict(m.X) X = m.X.copy() Xnew = NormalPosterior(m.X.mean[:10].copy(), m.X.variance[:10].copy()) m.set_XY(Xnew, m.Y[:10]) assert (m.checkgrad()) m.set_XY(X, self.Y) mu2, var2 = m.predict(m.X) np.testing.assert_allclose(mu, mu2) np.testing.assert_allclose(var, var2)
def predict(self, Xnew, full_cov=False, Y_metadata=None, kern=None): from GPy.core.parameterization.variational import NormalPosterior if self.repeatX: assert self.nLayers==2 mean,var = self.layers[-1].predict(Xnew) Xnew_norm = (Xnew - self.repeatX_Xmean)/self.repeatX_Xstd Xmean = np.hstack([mean,Xnew_norm]) Xvar = np.empty_like(Xmean) Xvar[:] = 1e-6 Xvar[:,:self.nDimsOrig[1]] = var x = NormalPosterior(Xmean,Xvar) else: x = Xnew for l in self.layers[:0:-1]: mean, var = l.predict(x) var = np.clip(var,1e-8, np.inf) if var.shape[1]==1: var = np.tile(var,mean.shape[1]) x = NormalPosterior(mean, var) return self.layers[0].predict(x)
def _next_minibatch(self): batch_idx, prev_sample_ix = self.data_streamer.get_cur_index() batch_idx, samples_idxes, Ys_n, Us_n = self.data_streamer.next_minibatch() # make U_pre_step shift -> Ys = []; Us = []; for i in range(len(Ys_n)): Y, U = Ys_n[i], Us_n[i] if self.U_pre_step: U = U[:-1].copy() Y = Y[self.U_win:].copy() else: Y = Y[self.U_win-1:].copy() Us.append(NormalPosterior(U,np.ones(U.shape)*1e-10)) Ys.append(Y) # make U_pre_step shift <- #import pdb; pdb.set_trace() Xs = self._init_X(self.wins, Ys, None, None, nDims=self.nDims, init='nan') self.Us = Us self.Ys = Ys self.mb_inf_sample_idxes = samples_idxes[:] previous_layer = None for i in range(self.nLayers-1,-1,-1): # start from top layer layer = self.layers[self.nLayers - i - 1] layer_Us = None layer_Xs = None # if i==self.nLayers-1: # Top layer # layer_Us = Us if layer.withControl else None # elif i==0: # Observed layer # layer_Xs = Ys # #else: # Other layers # # layer_Us = Xs[i] # # layer_Xs = Xs[i-1] # # layer.set_inputs_and_outputs(len(Ys), Xs=layer_Xs, Us=layer_Us, samples_idxes=samples_idxes) if i==self.nLayers-1: # Top layer layer_Us = Us if layer.withControl else None layer_Xs = Xs[i-1] elif i==0: # Observed layer layer_Us = previous_layer.Xs_flat if (previous_layer is not None) else None layer_Xs = Ys else: # Other layers layer_Us = previous_layer.Xs_flat if (previous_layer is not None) else None layer_Xs = Xs[i-1] #import pdb; pdb.set_trace() layer.set_inputs_and_outputs(len(Ys), Xs=layer_Xs, Us=layer_Us, samples_idxes=samples_idxes) previous_layer = layer
def __init__(self, wins, Y, U=None, U_win=1, nDims=None, X_variance=0.01, num_inducing=10, likelihood = None, name='autoreg', kernels=None, U_pre_step=True, init='Y', inducing_init='kmeans', back_cstr=False, MLP_dims=None): super(DeepAutoreg, self).__init__(name=name) #import pdb; pdb.set_trace() # Alex Ys, Us = Y,U if isinstance(Ys, np.ndarray): Ys = [Ys] if Us is not None and isinstance(Us, np.ndarray): Us = [Us] self.nLayers = len(wins) self.back_cstr = back_cstr self.wins = wins #self.input_dim = 1 #self.output_dim = 1 self._log_marginal_likelihood = np.nan self.U_pre_step = U_pre_step self.nDims = nDims if nDims is not None else [Ys[0].shape[1]]+[1]*(self.nLayers-1) if Us is not None: assert len(Ys)==len(Us) self.Us = [] self.Ys = [] for i in range(len(Ys)): Y, U = Ys[i], Us[i] # assert Y.shape[0]==U.shape[0], "the signal and control should be aligned." if U_pre_step: U = U[:-1].copy() Y = Y[U_win:].copy() else: Y = Y[U_win-1:].copy() self.Us.append(NormalPosterior(U.copy(),np.ones(U.shape)*1e-10)) self.Ys.append(Y) else: self.Us = Us self.Ys = Ys self.Xs = self._init_X(wins, self.Ys, self.Us, X_variance, init=init, nDims=self.nDims) # Parameters which exist differently per layer but specified as single componenents are here expanded to each layer if not isinstance(num_inducing, list or tuple): num_inducing = [num_inducing]*self.nLayers # Initialize Layers self.layers = [] for i in range(self.nLayers-1,-1,-1): if i==self.nLayers-1: self.layers.append(Layer(None, self.Xs[i-1], X_win=wins[i], Us=self.Us, U_win=U_win, num_inducing=num_inducing[i], kernel=kernels[i] if kernels is not None else None, noise_var=0.01, name='layer_'+str(i), back_cstr=back_cstr, MLP_dims=MLP_dims, inducing_init=inducing_init)) elif i==0: self.layers.append(Layer(self.layers[-1], self.Ys, X_win=wins[i], Us=self.Xs[i], U_win=wins[i+1], num_inducing=num_inducing[i], kernel=kernels[i] if kernels is not None else None, likelihood=likelihood, noise_var=1., back_cstr=back_cstr, name='layer_'+str(i), inducing_init=inducing_init)) else: self.layers.append(Layer(self.layers[-1], self.Xs[i-1], X_win=wins[i], Us=self.Xs[i], U_win=wins[i+1], num_inducing=num_inducing[i], kernel=kernels[i] if kernels is not None else None, noise_var=0.01, name='layer_'+str(i), back_cstr=back_cstr, MLP_dims=MLP_dims, inducing_init=inducing_init)) self.link_parameters(*self.layers)
def test_fixed_inputs_uncertain(self): from GPy.plotting.matplot_dep.util import fixed_inputs import GPy from GPy.core.parameterization.variational import NormalPosterior X_mu = np.random.randn(10, 3) X_var = np.random.randn(10, 3) X = NormalPosterior(X_mu, X_var) Y = np.sin(X_mu) + np.random.randn(10, 3)*1e-3 m = GPy.models.BayesianGPLVM(Y, X=X_mu, X_variance=X_var, input_dim=3) fixed = fixed_inputs(m, [1], fix_routine='median', as_list=True, X_all=False) self.assertTrue((0, np.median(X.mean.values[:,0])) in fixed) self.assertTrue((2, np.median(X.mean.values[:,2])) in fixed) self.assertTrue(len([t for t in fixed if t[0] == 1]) == 0) # Unfixed input should not be in fixed
def set_newX(self, X, append=False): from GPy import ObsAr if not self.uncertain_inputs: if append: self.X = ObsAr(np.vstack([self.X, X])) else: self.X = X if isinstance(X,ObsAr) else ObsAr(X) else: self.unlink_parameter(self.X) if append: self.X = NormalPosterior(np.vstack([self.X.mean.values, X.mean.values]),np.vstack([self.X.variance.values, X.variance.values])) else: self.X = X self.link_parameter(self.X)
def __init__(self, which, X, X_variance, Z, num_inducing, kernel): super(PsiStatModel, self).__init__(name='psi stat test') self.which = which self.X = Param("X", X) self.X_variance = Param('X_variance', X_variance, Logexp()) self.q = NormalPosterior(self.X, self.X_variance) self.Z = Param("Z", Z) self.N, self.input_dim = X.shape self.num_inducing, input_dim = Z.shape assert self.input_dim == input_dim, "shape missmatch: Z:{!s} X:{!s}".format( Z.shape, X.shape) self.kern = kernel self.psi_ = self.kern.__getattribute__(self.which)(self.Z, self.q) self.add_parameters(self.q, self.Z, self.kern)
def __init__(self, Y, input_dim, X=None, X_variance=None, init='PCA', num_inducing=10, Z=None, kernel=None, inference_method=None, likelihood=None, name='bayesian gplvm', normalizer=None, missing_data=False, stochastic=False, batchsize=1): self.logger = logging.getLogger(self.__class__.__name__) if X is None: from ..util.initialization import initialize_latent self.logger.info("initializing latent space X with method {}".format(init)) X, fracs = initialize_latent(init, input_dim, Y) else: fracs = np.ones(input_dim) self.init = init if Z is None: self.logger.info("initializing inducing inputs") Z = np.random.permutation(X.copy())[:num_inducing] assert Z.shape[1] == X.shape[1] if X_variance is False: self.logger.info('no variance on X, activating sparse GPLVM') X = Param("latent space", X) else: if X_variance is None: self.logger.info("initializing latent space variance ~ uniform(0,.1)") X_variance = np.random.uniform(0,.1,X.shape) self.variational_prior = NormalPrior() X = NormalPosterior(X, X_variance) if kernel is None: self.logger.info("initializing kernel RBF") kernel = kern.RBF(input_dim, lengthscale=1./fracs, ARD=True) #+ kern.Bias(input_dim) + kern.White(input_dim) if likelihood is None: likelihood = Gaussian() self.kl_factr = 1. if inference_method is None: from ..inference.latent_function_inference.var_dtc import VarDTC self.logger.debug("creating inference_method var_dtc") inference_method = VarDTC(limit=3 if not missing_data else Y.shape[1]) super(BayesianGPLVMMiniBatch,self).__init__(X, Y, Z, kernel, likelihood=likelihood, name=name, inference_method=inference_method, normalizer=normalizer, missing_data=missing_data, stochastic=stochastic, batchsize=batchsize) self.X = X self.link_parameter(self.X, 0)
def _init_XY(self): X_win, X_dim, U_win, U_dim = self.X_win, self.X_dim, self.U_win, self.U_dim self._update_conv() if X_win > 0: X_mean_conv, X_var_conv = np.vstack(self.X_mean_conv), np.vstack( self.X_var_conv) if self.withControl: U_mean_conv, U_var_conv = np.vstack(self.U_mean_conv), np.vstack( self.U_var_conv) if not self.withControl: self.X = NormalPosterior(X_mean_conv, X_var_conv) elif X_win == 0: self.X = NormalPosterior(U_mean_conv, U_var_conv) else: self.X = NormalPosterior(np.hstack([X_mean_conv, U_mean_conv]), np.hstack([X_var_conv, U_var_conv])) if self.X_observed: self.Y = np.vstack([x[X_win:] for x in self.Xs_flat]) else: self.Y = NormalPosterior( np.vstack([x.mean.values[X_win:] for x in self.Xs_flat]), np.vstack([x.variance.values[X_win:] for x in self.Xs_flat]))
def setUp(self): from GPy.core.parameterization.variational import NormalPosterior N, M, Q = 100, 20, 3 X = np.random.randn(N, Q) X_var = np.random.rand(N, Q) + 0.01 self.Z = np.random.randn(M, Q) self.qX = NormalPosterior(X, X_var) self.w1 = np.random.randn(N) self.w2 = np.random.randn(N, M) self.w3 = np.random.randn(M, M) self.w3 = self.w3 + self.w3.T self.w3n = np.random.randn(N, M, M) self.w3n = self.w3n + np.swapaxes(self.w3n, 1, 2)
def _aggregate_qX(self): if self.back_constraint: if self.X is None: self.X = NormalPosterior(np.zeros_like(self.views[0].Xv.mean.values), np.zeros_like(self.views[0].Xv.variance.values)) else: self.X.mean[:] = 0 self.X.variance[:] = 0 self.prec_denom = np.zeros_like(self.X.variance.values) for v in self.views: self.prec_denom += 1./v.Xv.variance.values self.X.mean += v.Xv.mean.values/v.Xv.variance.values self.X.mean /= self.prec_denom self.X.variance[:] = 1./self.prec_denom else: for v in self.views: v.X = self.X
def setUp(self): self.kerns = ( #GPy.kern.RBF([0,1,2], ARD=True)+GPy.kern.Bias(self.input_dim)+GPy.kern.White(self.input_dim), #GPy.kern.RBF(self.input_dim)+GPy.kern.Bias(self.input_dim)+GPy.kern.White(self.input_dim), #GPy.kern.Linear(self.input_dim) + GPy.kern.Bias(self.input_dim) + GPy.kern.White(self.input_dim), #GPy.kern.Linear(self.input_dim, ARD=True) + GPy.kern.Bias(self.input_dim) + GPy.kern.White(self.input_dim), GPy.kern.Linear([1, 3, 6, 7], ARD=True) + GPy.kern.RBF([0, 5, 8], ARD=True) + GPy.kern.White(self.input_dim), ) self.q_x_mean = np.random.randn(self.input_dim)[None] self.q_x_variance = np.exp(.5 * np.random.randn(self.input_dim))[None] self.q_x_samples = np.random.randn( self.Nsamples, self.input_dim) * np.sqrt( self.q_x_variance) + self.q_x_mean self.q_x = NormalPosterior(self.q_x_mean, self.q_x_variance) self.Z = np.random.randn(self.num_inducing, self.input_dim) self.q_x_mean.shape = (1, self.input_dim) self.q_x_variance.shape = (1, self.input_dim)
def _run_encoder_forward(self,): """ Runs the encoder and generate layer's q_X. """ # stack batches over axis 1. This is required by pytorch. if not self.encode_include_input: out_means, out_vars = self.encoder.forward_computation( np.stack(self.Ys_all, axis=1) ) else: out_means, out_vars = self.encoder.forward_computation( np.stack(self.Ys_all, axis=1), np.stack(self.Us_all, axis=1) ) Xs = [] #import pdb; pdb.set_trace() # iteration over layers (start from lower, observed is 0), inside each layer over i_seq for layer_idx in range(len(out_means)): # iteration over layers Xs_layer = [] for sample_idx in range(len(out_means[layer_idx])): # iteration over layers Xs_layer.append( NormalPosterior( out_means[layer_idx][sample_idx], out_vars[layer_idx][sample_idx], name='qX_'+str( layer_idx ) ) ) Xs.append( Xs_layer ) return Xs
def _init_encoder(self, MLP_dims): from .mlp import MLP from copy import deepcopy from GPy.core.parameterization.transformations import Logexp X_win, X_dim, U_win, U_dim = self.X_win, self.X_dim, self.U_win, self.U_dim assert X_win > 0, "Neural Network constraints only applies autoregressive structure!" Q = X_win * X_dim + U_win * U_dim if self.withControl else X_win * X_dim self.init_Xs = [ NormalPosterior(self.Xs_flat[i].mean.values[:X_win], self.Xs_flat[i].variance.values[:X_win], name='init_Xs_' + str(i)) for i in range(self.nSeq) ] for init_X in self.init_Xs: init_X.mean[:] = np.random.randn(*init_X.shape) * 1e-2 self.encoder = MLP([Q, Q * 2, Q + X_dim / 2, X_dim] if MLP_dims is None else [Q] + deepcopy(MLP_dims) + [X_dim]) self.Xs_var = [ Param('X_var_' + str(i), self.Xs_flat[i].variance.values[X_win:].copy(), Logexp()) for i in range(self.nSeq) ]
def __init__(self, Y, dim_down, dim_up, likelihood, MLP_dims=None, X=None, X_variance=None, init='rand', Z=None, num_inducing=10, kernel=None, inference_method=None, uncertain_inputs=True,mpi_comm=None, mpi_root=0, back_constraint=True, name='mrd-view'): self.uncertain_inputs = uncertain_inputs self.layer_lower = None self.scale = 1. if back_constraint: from .mlp import MLP from copy import deepcopy self.encoder = MLP([dim_down, int((dim_down+dim_up)*2./3.), int((dim_down+dim_up)/3.), dim_up] if MLP_dims is None else [dim_down]+deepcopy(MLP_dims)+[dim_up]) X = self.encoder.predict(Y.mean.values if isinstance(Y, VariationalPosterior) else Y) X_variance = 0.0001*np.ones(X.shape) self.back_constraint = True else: self.back_constraint = False if Z is None: Z = np.random.rand(num_inducing, dim_up)*2-1. #np.random.permutation(X.copy())[:num_inducing] assert Z.shape[1] == X.shape[1] if likelihood is None: likelihood = likelihoods.Gaussian(variance=Y.var()*0.01) if uncertain_inputs: X = NormalPosterior(X, X_variance) if kernel is None: kernel = kern.RBF(dim_up, ARD = True) # The command below will also give the field self.X to the view. super(MRDView, self).__init__(X, Y, Z, kernel, likelihood, inference_method=inference_method, mpi_comm=mpi_comm, mpi_root=mpi_root, name=name) if back_constraint: self.link_parameter(self.encoder) if self.uncertain_inputs and self.back_constraint: from GPy import Param from GPy.core.parameterization.transformations import Logexp self.X_var_common = Param('X_var',X_variance[0].copy(),Logexp()) self.link_parameters(self.X_var_common) # There's some redundancy in the self.Xv and self.X. Currently we use self.X for the likelihood part and all calculations part, # self.Xv is only used for the self.Xv.gradient part. # This is redundant but it's there in case we want to do the product of experts MRD model. self.Xv = self.X
def __init__(self, layer_lower, dim_down, dim_up, likelihood, X=None, X_variance=None, init='PCA', Z=None, num_inducing=10, kernel=None, inference_method=None, uncertain_inputs=True,mpi_comm=None, mpi_root=0, back_constraint=True, encoder=None, auto_update=True, name='layer'): self.uncertain_inputs = uncertain_inputs self.layer_lower = layer_lower Y = self.Y if self.layer_lower is None else self.layer_lower.X self.back_constraint = back_constraint from deepgp.util.util import initialize_latent if X is None: X, _ = initialize_latent(init, Y.shape[0], dim_up, Y.mean.values if isinstance(Y, VariationalPosterior) else Y) if X_variance is None: X_variance = 0.01*np.ones(X.shape) + 0.01*np.random.rand(*X.shape) if Z is None: if self.back_constraint: Z = np.random.rand(num_inducing, dim_up)*2-1. else: if num_inducing<=X.shape[0]: Z = X[np.random.permutation(X.shape[0])[:num_inducing]].copy() else: Z_more = np.random.rand(num_inducing-X.shape[0],X.shape[1])*(X.max(0)-X.min(0))+X.min(0) Z = np.vstack([X.copy(),Z_more]) assert Z.shape[1] == X.shape[1] if mpi_comm is not None: from ..util.parallel import broadcastArrays broadcastArrays([Z], mpi_comm, mpi_root) if uncertain_inputs: X = NormalPosterior(X, X_variance) if kernel is None: kernel = kern.RBF(dim_up, ARD = True) assert kernel.input_dim==X.shape[1], "The dimensionality of input has to be equal to the input dimensionality of kernel!" self.Kuu_sigma = Param('Kuu_var', np.zeros(num_inducing)+1e-3, Logexp()) super(Layer, self).__init__(X, Y, Z, kernel, likelihood, inference_method=inference_method, mpi_comm=mpi_comm, mpi_root=mpi_root, auto_update=auto_update, name=name) self.link_parameter(self.Kuu_sigma) if back_constraint: self.encoder = encoder if self.uncertain_inputs and not self.back_constraint: self.link_parameter(self.X)
def __init__(self, X, Y, Xr_dim, kernel=None, kernel_row=None, Z=None, Z_row=None, X_row=None, Xvariance_row=None, num_inducing=(10, 10), qU_var_r_W_dim=None, qU_var_c_W_dim=None, init='GP', name='GPMR'): #Kernel if kernel is None: kernel = kern.RBF(X.shape[1]) if kernel_row is None: kernel_row = kern.RBF(Xr_dim, name='kern_row') if init == 'GP': from . import SparseGPRegression, BayesianGPLVM from ..util.linalg import jitchol Mc, Mr = num_inducing print('Intializing with GP...') print('Fit Sparse GP...') m_sgp = SparseGPRegression(X, Y, kernel=kernel.copy(), num_inducing=Mc) m_sgp.likelihood.variance[:] = Y.var() * 0.01 m_sgp.optimize(max_iters=1000) print('Fit BGPLVM...') m_lvm = BayesianGPLVM(m_sgp.posterior.mean.copy().T, Xr_dim, kernel=kernel_row.copy(), num_inducing=Mr) m_lvm.likelihood.variance[:] = m_lvm.Y.var() * 0.01 m_lvm.optimize(max_iters=10000) kernel[:] = m_sgp.kern.param_array.copy() kernel.variance[:] = np.sqrt(kernel.variance) Z = m_sgp.Z.values.copy() kernel_row[:] = m_lvm.kern.param_array.copy() kernel_row.variance[:] = np.sqrt(kernel_row.variance) Z_row = m_lvm.Z.values.copy() X_row = m_lvm.X.mean.values.copy() Xvariance_row = m_lvm.X.variance.values qU_mean = m_lvm.posterior.mean.T.copy() qU_var_col_W = jitchol(m_sgp.posterior.covariance) qU_var_col_diag = np.full(Mc, 1e-5) qU_var_row_W = jitchol(m_lvm.posterior.covariance) qU_var_row_diag = np.full(Mr, 1e-5) print('Done.') else: qU_mean = np.zeros(num_inducing) qU_var_col_W = np.random.randn( num_inducing[0], num_inducing[0] if qU_var_c_W_dim is None else qU_var_c_W_dim) * 0.01 qU_var_col_diag = np.full(num_inducing[0], 1e-5) qU_var_row_W = np.random.randn( num_inducing[1], num_inducing[1] if qU_var_r_W_dim is None else qU_var_r_W_dim) * 0.01 qU_var_row_diag = np.full(num_inducing[1], 1e-5) if X_row is None: u, s, v = np.linalg.svd(Y) X_row = Y.T.dot(u[:, :Xr_dim]) #*np.sqrt(s)[:Xr_dim]) X_row = X_row / X_row.std(0) if Xvariance_row is None: Xvariance_row = np.ones((Y.shape[1], Xr_dim)) * 0.0001 if Z is None: Z = X[np.random.permutation(X.shape[0])[:num_inducing[0]]].copy() if Z_row is None: Z_row = X_row[np.random.permutation( X_row.shape[0])[:num_inducing[1]]].copy() self.kern_row = kernel_row self.X_row = NormalPosterior(X_row, Xvariance_row, name='Xr') self.Z_row = Param('Zr', Z_row) self.variational_prior_row = NormalPrior() self.qU_mean = Param('qU_mean', qU_mean) self.qU_var_c_W = Param('qU_var_col_W', qU_var_col_W) self.qU_var_c_diag = Param('qU_var_col_diag', qU_var_col_diag, Logexp()) self.qU_var_r_W = Param('qU_var_row_W', qU_var_row_W) self.qU_var_r_diag = Param('qU_var_row_diag', qU_var_row_diag, Logexp()) #Likelihood likelihood = likelihoods.Gaussian(variance=np.var(Y) * 0.01) from ..inference.latent_function_inference import VarDTC_SVI_Multiout inference_method = VarDTC_SVI_Multiout() super(GPMultioutRegression, self).__init__(X, Y, Z, kernel, likelihood=likelihood, name=name, inference_method=inference_method) self.link_parameters(self.kern_row, self.X_row, self.Z_row, self.qU_mean, self.qU_var_c_W, self.qU_var_c_diag, self.qU_var_r_W, self.qU_var_r_diag) self._log_marginal_likelihood = np.nan
class Kernel_Psi_statistics_GradientTests(unittest.TestCase): def setUp(self): from GPy.core.parameterization.variational import NormalPosterior N, M, Q = 100, 20, 3 X = np.random.randn(N, Q) X_var = np.random.rand(N, Q) + 0.01 self.Z = np.random.randn(M, Q) self.qX = NormalPosterior(X, X_var) self.w1 = np.random.randn(N) self.w2 = np.random.randn(N, M) self.w3 = np.random.randn(M, M) self.w3 = self.w3 + self.w3.T self.w3n = np.random.randn(N, M, M) self.w3n = self.w3n + np.swapaxes(self.w3n, 1, 2) def test_kernels(self): from GPy.kern import RBF, Linear, MLP Q = self.Z.shape[1] kernels = [RBF(Q, ARD=True), Linear(Q, ARD=True)] for k in kernels: k.randomize() self._test_kernel_param(k) self._test_Z(k) self._test_qX(k) self._test_kernel_param(k, psi2n=True) self._test_Z(k, psi2n=True) self._test_qX(k, psi2n=True) def _test_kernel_param(self, kernel, psi2n=False): def f(p): kernel.param_array[:] = p psi0 = kernel.psi0(self.Z, self.qX) psi1 = kernel.psi1(self.Z, self.qX) if not psi2n: psi2 = kernel.psi2(self.Z, self.qX) return (self.w1 * psi0).sum() + (self.w2 * psi1).sum() + (self.w3 * psi2).sum() else: psi2 = kernel.psi2n(self.Z, self.qX) return (self.w1 * psi0).sum() + (self.w2 * psi1).sum() + (self.w3n * psi2).sum() def df(p): kernel.param_array[:] = p kernel.update_gradients_expectations(self.w1, self.w2, self.w3 if not psi2n else self.w3n, self.Z, self.qX) return kernel.gradient.copy() from GPy.models import GradientChecker m = GradientChecker(f, df, kernel.param_array.copy()) self.assertTrue(m.checkgrad()) def _test_Z(self, kernel, psi2n=False): def f(p): psi0 = kernel.psi0(p, self.qX) psi1 = kernel.psi1(p, self.qX) psi2 = kernel.psi2(p, self.qX) if not psi2n: psi2 = kernel.psi2(p, self.qX) return (self.w1 * psi0).sum() + (self.w2 * psi1).sum() + (self.w3 * psi2).sum() else: psi2 = kernel.psi2n(p, self.qX) return (self.w1 * psi0).sum() + (self.w2 * psi1).sum() + (self.w3n * psi2).sum() def df(p): return kernel.gradients_Z_expectations(self.w1, self.w2, self.w3 if not psi2n else self.w3n, p, self.qX) from GPy.models import GradientChecker m = GradientChecker(f, df, self.Z.copy()) self.assertTrue(m.checkgrad()) def _test_qX(self, kernel, psi2n=False): def f(p): self.qX.param_array[:] = p self.qX._trigger_params_changed() psi0 = kernel.psi0(self.Z, self.qX) psi1 = kernel.psi1(self.Z, self.qX) if not psi2n: psi2 = kernel.psi2(self.Z, self.qX) return (self.w1 * psi0).sum() + (self.w2 * psi1).sum() + (self.w3 * psi2).sum() else: psi2 = kernel.psi2n(self.Z, self.qX) return (self.w1 * psi0).sum() + (self.w2 * psi1).sum() + (self.w3n * psi2).sum() def df(p): self.qX.param_array[:] = p self.qX._trigger_params_changed() grad = kernel.gradients_qX_expectations( self.w1, self.w2, self.w3 if not psi2n else self.w3n, self.Z, self.qX ) self.qX.set_gradients(grad) return self.qX.gradient.copy() from GPy.models import GradientChecker m = GradientChecker(f, df, self.qX.param_array.copy()) self.assertTrue(m.checkgrad())