def natural_grad_qu(model, n_iter=1, step_size=step_rate, momentum=0.0): global mk_ant, mk_aux, mk, V_i, Vk, Lk, Vk, Vki_ant """"Initialize the step-sizes""" "" beta2_k = step_size #use step_size*0.1 for Convolutional MOGP gamma2_k = momentum alpha2_k = step_size N_posteriors = model.q_u_means.shape[1] if n_iter == 1: V_i = choleskies.multiple_dpotri( choleskies.flat_to_triang(model.q_u_chols.values)).copy() Vk = np.zeros_like(V_i) for i in range(N_posteriors): Vk[i, :, :] = 0.5 * (model.posteriors[i].covariance.copy() + model.posteriors[i].covariance.T.copy()) Lk = np.zeros_like(Vk) mk = model.q_u_means.values.copy() Vki_ant = V_i.copy() mk_aux = mk.copy() dL_dm, dL_dV = compute_stoch_grads_for_qu_HetMOGP(model=model) mk_ant = mk_aux.copy() mk_aux = mk.copy() if not model.q_u_means.is_fixed and not model.q_u_chols.is_fixed: mk_ant = mk_aux.copy() mk_aux = mk.copy() for i in range(N_posteriors): try: V_i[i, :, :] = V_i[i, :, :] + 2 * beta2_k * dL_dV[ i] #+ 1.0e-6*np.eye(*Vk[i,:,:].shape) Vk[i, :, :] = np.linalg.inv(V_i[i, :, :]) Vk[i, :, :] = 0.5 * (np.array(Vk[i, :, :]) + np.array(Vk[i, :, :].T)) Lk[i, :, :] = np.linalg.cholesky(Vk[i, :, :]) mk[:, i] = mk[:, i] - alpha2_k * np.dot( Vk[i, :, :], dL_dm[i]) + gamma2_k * np.dot( np.dot(Vk[i, :, :], Vki_ant[i, :, :]), (mk[:, i] - mk_ant[:, i])) except LinAlgError: print("Overflow") Vk[i, :, :] = np.linalg.inv(V_i[i, :, :]) Vk[i, :, :] = 1.0e-1 * np.eye( *Vk[i, :, :].shape ) #nearestPD(Vk[i,:,:]) # + 1.0e-3*np.eye(*Vk[i,:,:].shape) Lk[i, :, :] = linalg.jitchol(Vk[i, :, :]) V_i[i, :, :] = np.linalg.inv(Vk[i, :, :]) mk[:, i] = mk[:, i] * 0.0 Vki_ant = V_i.copy() model.L_u.setfield(choleskies.triang_to_flat(Lk.copy()), np.float64) model.m_u.setfield(mk.copy(), np.float64)
def __init__(self, X, Y, Z, kern_list_uq, kern_list_Gx, kern_list_Tq, likelihood, Y_metadata, name='ConvHetMOGP_VIK', batch_size=None): self.batch_size = batch_size self.kern_list = kern_list_uq self.likelihood = likelihood self.Y_metadata = Y_metadata self.kern_list_Gdj = kern_list_Gx self.kern_list_Tq = kern_list_Tq self.num_inducing = Z.shape[0] # M self.num_latent_funcs = len(kern_list_uq) # Q self.num_output_funcs = likelihood.num_output_functions( self.Y_metadata) #This is the number J in the paper self.W_list, self.kappa_list = util.random_W_kappas( self.num_latent_funcs, self.num_output_funcs, rank=1) check_ARD_uq = [kern.lengthscale.shape[0] > 1 for kern in kern_list_uq] check_ARD_Gx = [ kern.lengthscale.shape[0] > 1 for kern in kern_list_Gx ] # This is just to verify Automatic Relevance Determination check_ARD_Tq = [kern.lengthscale.shape[0] > 1 for kern in kern_list_Tq] if (sum(check_ARD_uq) == 0) and (sum(check_ARD_Gx) == 0) and (sum(check_ARD_Tq) == 0): isARD = False elif (sum(check_ARD_uq) == check_ARD_uq.__len__()) and ( sum(check_ARD_Gx) == check_ARD_Gx.__len__()) and (sum(check_ARD_Tq) == check_ARD_Tq.__len__()): isARD = True else: print( '\nAll kernel_lists for Uq, Gx and Tx have to coincide in Automatic Relevance Determination,' ) print('All kernel_lists have to coincide: ARD=True or ARD=False\n') assert (sum(check_ARD_uq) == check_ARD_uq.__len__()) and ( sum(check_ARD_Gx) == check_ARD_Gx.__len__()) and (sum(check_ARD_Tq) == check_ARD_Tq.__len__()) self.kern_aux = GPy.kern.RBF( input_dim=Z.shape[1], lengthscale=1.0, variance=1.0, name='rbf_aux', ARD=isARD) + GPy.kern.White(input_dim=Z.shape[1]) self.kern_aux.white.variance = 1e-6 self.Xmulti = X self.Ymulti = Y # Batch the data self.Xmulti_all, self.Ymulti_all = X, Y if batch_size is None: #self.stochastic = False Xmulti_batch, Ymulti_batch = X, Y else: # Makes a climin slicer to make drawing minibatches much quicker #self.stochastic = False #"This was True as Pablo had it" self.slicer_list = [] [ self.slicer_list.append( draw_mini_slices(Xmulti_task.shape[0], self.batch_size)) for Xmulti_task in self.Xmulti ] Xmulti_batch, Ymulti_batch = self.new_batch() self.Xmulti, self.Ymulti = Xmulti_batch, Ymulti_batch # Initialize inducing points Z self.Xdim = Z.shape[1] Z = np.tile(Z, (1, self.num_latent_funcs)) inference_method = SVMOGPInf() super(ConvHetMOGP_VIK, self).__init__(X=Xmulti_batch[0][1:10], Y=Ymulti_batch[0][1:10], Z=Z, kernel=kern_list_uq[0], likelihood=likelihood, mean_function=None, X_variance=None, inference_method=inference_method, Y_metadata=Y_metadata, name=name, normalizer=False) self.unlink_parameter( self.kern) # Unlink SparseGP default param kernel _, self.B_list = util.LCM(input_dim=self.Xdim, output_dim=self.num_output_funcs, rank=1, kernels_list=self.kern_list, W_list=self.W_list, kappa_list=self.kappa_list) # Set-up optimization parameters: [Z, m_u, L_u] self.q_u_means = [ Param( 'm_u' + str(dj), 10.0 * np.random.randn(self.num_inducing, self.num_latent_funcs) + 10.0 * np.tile(np.random.randn(1, self.num_latent_funcs), (self.num_inducing, 1))) for dj in range(self.num_output_funcs) ] chols = choleskies.triang_to_flat( np.tile(3 * np.eye(self.num_inducing)[None, :, :], (self.num_latent_funcs, 1, 1))) self.q_u_chols = Param('L_u', chols) self.link_parameter(self.Z, index=0) [self.link_parameter(q_u_means) for q_u_means in self.q_u_means] self.link_parameters(self.q_u_chols) [self.link_parameter(kern_q) for kern_q in kern_list_uq] # link all kernels [self.link_parameter(B_q) for B_q in self.B_list] [self.link_parameter(kern_list_Gjd) for kern_list_Gjd in kern_list_Gx] [self.link_parameter(kern_list_Tq) for kern_list_Tq in kern_list_Tq] #self.link_parameter(self.kern_aux.white.variance) self.vem_step = True # [True=VE-step, False=VM-step] self.ve_count = 0 self.elbo = np.zeros((1, 1)) self.index_VEM = 0 #this is a variable to index correctly the self.elbo when using VEM self.Gauss_Newton = False #This is a flag for using the Gauss-Newton approximation when dL_dV is needed for kern_q in self.kern_list: kern_q.variance = 1.0 kern_q.variance.fix() for kern_Gjd in self.kern_list_Gdj: kern_Gjd.variance = 1.0 kern_Gjd.variance.fix() #print('IN fix Gdj') for kern_Tq in self.kern_list_Tq: kern_Tq.variance = 1.0 kern_Tq.variance.fix()
def __init__(self, X, Y, Z, kern_list, likelihood, Y_metadata, name='SVMOGP', batch_size=None): self.batch_size = batch_size self.kern_list = kern_list self.likelihood = likelihood self.Y_metadata = Y_metadata self.num_inducing = Z.shape[0] # M self.num_latent_funcs = len(kern_list) # Q self.num_output_funcs = likelihood.num_output_functions( self.Y_metadata) self.W_list, self.kappa_list = util.random_W_kappas( self.num_latent_funcs, self.num_output_funcs, rank=1) self.Xmulti = X self.Ymulti = Y # Batch the data self.Xmulti_all, self.Ymulti_all = X, Y if batch_size is None: self.stochastic = False Xmulti_batch, Ymulti_batch = X, Y else: # Makes a climin slicer to make drawing minibatches much quicker self.stochastic = True self.slicer_list = [] [ self.slicer_list.append( draw_mini_slices(Xmulti_task.shape[0], self.batch_size)) for Xmulti_task in self.Xmulti ] Xmulti_batch, Ymulti_batch = self.new_batch() self.Xmulti, self.Ymulti = Xmulti_batch, Ymulti_batch # Initialize inducing points Z #Z = kmm_init(self.X_all, self.num_inducing) self.Xdim = Z.shape[1] Z = np.tile(Z, (1, self.num_latent_funcs)) inference_method = SVMOGPInf() super(SVMOGP, self).__init__(X=Xmulti_batch[0][1:10], Y=Ymulti_batch[0][1:10], Z=Z, kernel=kern_list[0], likelihood=likelihood, mean_function=None, X_variance=None, inference_method=inference_method, Y_metadata=Y_metadata, name=name, normalizer=False) self.unlink_parameter( self.kern) # Unlink SparseGP default param kernel _, self.B_list = util.LCM(input_dim=self.Xdim, output_dim=self.num_output_funcs, rank=1, kernels_list=self.kern_list, W_list=self.W_list, kappa_list=self.kappa_list) # Set-up optimization parameters: [Z, m_u, L_u] self.q_u_means = Param( 'm_u', 5 * np.random.randn(self.num_inducing, self.num_latent_funcs) + np.tile(np.random.randn(1, self.num_latent_funcs), (self.num_inducing, 1))) chols = choleskies.triang_to_flat( np.tile( np.eye(self.num_inducing)[None, :, :], (self.num_latent_funcs, 1, 1))) self.q_u_chols = Param('L_u', chols) self.link_parameter(self.Z, index=0) self.link_parameter(self.q_u_means) self.link_parameters(self.q_u_chols) [self.link_parameter(kern_q) for kern_q in kern_list] # link all kernels [self.link_parameter(B_q) for B_q in self.B_list] self.vem_step = True # [True=VE-step, False=VM-step] self.ve_count = 0 self.elbo = np.zeros((1, 1))
def calculate_gradients(self, q_U, S_u, Su_add_Kuu_chol, p_U, q_F, VE_dm, VE_dv, Ntask, M, Q, D, f_index, d_index, q): """ Calculates gradients of the Log-marginal distribution p(Y) wrt variational parameters mu_q, S_q """ # Algebra for q(u) and p(u): m_u = q_U.mu_u.copy() #L_u = choleskies.flat_to_triang(q_U.chols_u.copy()) #S_u = np.empty((Q, M, M)) #[np.dot(L_u[q, :, :], L_u[q, :, :].T, S_u[q, :, :]) for q in range(Q)] Kuu = p_U.Kuu.copy() Luu = p_U.Luu.copy() Kuui = p_U.Kuui.copy() S_qi, _ = linalg.dpotri(np.asfortranarray(Su_add_Kuu_chol[q, :, :])) if np.any(np.isinf(S_qi)): raise ValueError("Sqi: Cholesky representation unstable") # KL Terms dKL_dmu_q = [] dKL_dKqq = 0 for d in range(D): dKL_dmu_q.append(np.dot(Kuui[q, :, :], m_u[d][:, q, None])) #same dKL_dKqq += -0.5 * S_qi + 0.5 * Kuui[q, :, :] - 0.5 * Kuui[q, :, :].dot(S_u[q, :, :]).dot(Kuui[q, :, :]) \ - 0.5 * np.dot(Kuui[q, :, :], np.dot(m_u[d][:, q, None], m_u[d][:, q, None].T)).dot(Kuui[q, :, :].T) # same #dKL_dS_q = 0.5 * (Kuui[q,:,:] - S_qi) #old dKL_dS_q = 0.5 * (Kuui[q, :, :] - S_qi) * D # VE Terms #dVE_dmu_q = np.zeros((M, 1)) dVE_dmu_q = [] dVE_dS_q = np.zeros((M, M)) dVE_dKqq = np.zeros((M, M)) dVE_dKqd = [] dVE_dKdiag = [] dL_dmu_q = [] for d, q_fd in enumerate(q_F): Nt = Ntask[f_index[d]] dVE_dmu_q.append( np.dot(q_fd.Afdu[q, :, :].T, VE_dm[f_index[d]][:, d_index[d]])[:, None]) dL_dmu_q.append(dVE_dmu_q[d] - dKL_dmu_q[d]) Adv = q_fd.Afdu[q, :, :].T * VE_dv[f_index[d]][:, d_index[d], None].T Adv = np.ascontiguousarray(Adv) AdvA = np.dot(Adv.reshape(-1, Nt), q_fd.Afdu[q, :, :]).reshape(M, M) dVE_dS_q += AdvA # Derivatives dKuquq tmp_dv = np.dot(AdvA, S_u[q, :, :]).dot(Kuui[q, :, :]) dVE_dKqq += -tmp_dv - tmp_dv.T #+ AdvA last term not included in the derivative Adm = np.dot(q_fd.Afdu[q, :, :].T, VE_dm[f_index[d]][:, d_index[d], None]) dVE_dKqq += -np.dot(Adm, np.dot(Kuui[q, :, :], m_u[d][:, q, None]).T) # Derivatives dKuqfd tmp = np.dot(S_u[q, :, :], Kuui[q, :, :]) tmp = 2. * tmp #2. * (tmp - np.eye(M)) # the term -2Adv not included dve_kqd = np.dot(np.dot(Kuui[q, :, :], m_u[d][:, q, None]), VE_dm[f_index[d]][:, d_index[d], None].T) dve_kqd += np.dot(tmp.T, Adv) dVE_dKqd.append(dve_kqd) # Derivatives dKdiag dVE_dKdiag.append(VE_dv[f_index[d]][:, d_index[d]]) dVE_dKqq = 0.5 * (dVE_dKqq + dVE_dKqq.T) # Sum of VE and KL terms #dL_dmu_q = dVE_dmu_q - dKL_dmu_q dL_dS_q = dVE_dS_q - dKL_dS_q dL_dKqq = dVE_dKqq - dKL_dKqq dL_dKdq = dVE_dKqd dL_dKdiag = dVE_dKdiag # Pass S_q gradients to its low-triangular representation L_q chol_u = q_U.chols_u.copy() L_q = choleskies.flat_to_triang(chol_u[:, q:q + 1]) dL_dL_q = 2. * np.array( [np.dot(a, b) for a, b in zip(dL_dS_q[None, :, :], L_q)]) dL_dL_q = choleskies.triang_to_flat(dL_dL_q) # Posterior posterior_q = [] for d in range(D): posterior_q.append( Posterior(mean=m_u[d][:, q, None], cov=S_u[q, :, :] + Kuu[q, :, :], K=Kuu[q, :, :], prior_mean=np.zeros(m_u[d][:, q, None].shape))) return dL_dmu_q, dL_dL_q, dL_dS_q, posterior_q, dL_dKqq, dL_dKdq, dL_dKdiag
def calculate_gradients(self, q_U, p_U, q_F, VE_dm, VE_dv, Ntask, M, Q, D, f_index, d_index, j): """ Calculates gradients of the Log-marginal distribution p(Y) wrt variational parameters mu_q, S_q """ # Algebra for q(u) and p(u): m_u = q_U.mu_u.copy() L_u = choleskies.flat_to_triang(q_U.chols_u.copy()) #S_u = np.empty((Q, M, M)) S_u = np.dot( L_u[j, :, :], L_u[j, :, :].T ) #This could be done outside and recieve it to reduce computation #[np.dot(L_u[q, :, :], L_u[q, :, :].T, S_u[q, :, :]) for q in range(Q)] Kuu = p_U.Kuu.copy() Luu = p_U.Luu.copy() Kuui = p_U.Kuui.copy() S_qi, _ = linalg.dpotri(np.asfortranarray(L_u[j, :, :])) if np.any(np.isinf(S_qi)): raise ValueError("Sqi: Cholesky representation unstable") # KL Terms dKL_dmu_j = np.dot(Kuui[j, :, :], m_u[:, j, None]) dKL_dS_j = 0.5 * (Kuui[j, :, :] - S_qi) dKL_dKjj = 0.5 * Kuui[j,:,:] - 0.5 * Kuui[j,:,:].dot(S_u).dot(Kuui[j,:,:]) \ - 0.5 * np.dot(Kuui[j,:,:],np.dot(m_u[:, j, None],m_u[:, j, None].T)).dot(Kuui[j,:,:].T) # VE Terms dVE_dmu_j = np.zeros((M, 1)) dVE_dS_j = np.zeros((M, M)) dVE_dKjj = np.zeros((M, M)) dVE_dKjd = [] dVE_dKdiag = [] Nt = Ntask[f_index[j]] dVE_dmu_j += np.dot(q_F[j].Afdu.T, VE_dm[f_index[j]][:, d_index[j]])[:, None] Adv = q_F[j].Afdu.T * VE_dv[f_index[j]][:, d_index[j], None].T Adv = np.ascontiguousarray(Adv) AdvA = np.dot(Adv.reshape(-1, Nt), q_F[j].Afdu).reshape(M, M) dVE_dS_j += AdvA # Derivatives dKuquq tmp_dv = np.dot(AdvA, S_u).dot(Kuui[j, :, :]) dVE_dKjj += AdvA - tmp_dv - tmp_dv.T Adm = np.dot(q_F[j].Afdu.T, VE_dm[f_index[j]][:, d_index[j], None]) dVE_dKjj += -np.dot(Adm, np.dot(Kuui[j, :, :], m_u[:, j, None]).T) # Derivatives dKuqfd tmp = np.dot(S_u, Kuui[j, :, :]) tmp = 2. * (tmp - np.eye(M)) dve_kjd = np.dot(np.dot(Kuui[j, :, :], m_u[:, j, None]), VE_dm[f_index[j]][:, d_index[j], None].T) dve_kjd += np.dot(tmp.T, Adv) dVE_dKjd.append(dve_kjd) # Derivatives dKdiag dVE_dKdiag.append(VE_dv[f_index[j]][:, d_index[j]]) dVE_dKjj = 0.5 * (dVE_dKjj + dVE_dKjj.T) # Sum of VE and KL terms dL_dmu_j = dVE_dmu_j - dKL_dmu_j dL_dS_j = dVE_dS_j - dKL_dS_j dL_dKjj = dVE_dKjj - dKL_dKjj dL_dKdj = dVE_dKjd[0].copy() #Here we just pass the unique position dL_dKdiag = dVE_dKdiag[0].copy( ) #Here we just pass the unique position # Pass S_q gradients to its low-triangular representation L_q chol_u = q_U.chols_u.copy() L_j = choleskies.flat_to_triang(chol_u[:, j:j + 1]) dL_dL_j = 2. * np.array( [np.dot(a, b) for a, b in zip(dL_dS_j[None, :, :], L_j)]) dL_dL_j = choleskies.triang_to_flat(dL_dL_j) # Posterior posterior_j = Posterior(mean=m_u[:, j, None], cov=S_u, K=Kuu[j, :, :], prior_mean=np.zeros(m_u[:, j, None].shape)) return dL_dmu_j, dL_dL_j, dL_dS_j, posterior_j, dL_dKjj, dL_dKdj, dL_dKdiag
def __init__(self, X, Y, Z, kern_list, likelihood, mean_functions=None, name='SVGPMulti', Y_metadata=None, batchsize=None): """ Extension to the SVGP to allow multiple latent function, where the latent functions are assumed independant (have one kernel per latent function) """ # super(SVGPMulti, self).__init__(name) # Parameterized.__init__(self) assert X.ndim == 2 self.Y_metadata = Y_metadata _, self.output_dim = Y.shape # self.Z = Param('inducing inputs', Z) # self.num_inducing = Z.shape[0] # self.likelihood = likelihood self.kern_list = kern_list self.batchsize = batchsize #Batch the data self.X_all, self.Y_all = X, Y if batchsize is None: X_batch, Y_batch = X, Y else: import climin.util #Make a climin slicer to make drawing minibatches much quicker self.slicer = climin.util.draw_mini_slices(self.X_all.shape[0], self.batchsize) X_batch, Y_batch = self.new_batch() # if isinstance(X_batch, (ObsAr, VariationalPosterior)): # self.X = X_batch.copy() # else: # self.X = ObsAr(X_batch) # self.Y = Y_batch #create the SVI inference method # self.inference_method = svgp_inf() inference_method = svgp_inf() #Initialize base model super(SVGPMulti, self).__init__(X=X_batch, Y=Y_batch, Z=Z, kernel=kern_list[0], likelihood=likelihood, mean_function=None, X_variance=None, inference_method=inference_method, name=name, Y_metadata=Y_metadata, normalizer=False) self.unlink_parameter(self.kern) # We don't want a single kern # self.num_data, self.input_dim = self.X.shape self.num_outputs = self.Y.shape[1] self.num_latent_funcs = self.likelihood.request_num_latent_functions( self.Y_all) #Make a latent function per dimension self.q_u_means = Param( 'q_u_means', np.zeros((self.num_inducing, self.num_latent_funcs))) chols = choleskies.triang_to_flat( np.tile( np.eye(self.num_inducing)[None, :, :], (self.num_latent_funcs, 1, 1))) self.q_u_chols = Param('qf_u_chols', chols) self.link_parameter(self.Z, index=0) self.link_parameter(self.q_u_means) self.link_parameter(self.q_u_chols) # self.link_parameter(self.likelihood) #Must pass a list of kernels that work on each latent function for now assert len(kern_list) == self.num_latent_funcs #Add the rest of the kernels, one kernel per latent function [self.link_parameter(kern) for kern in kern_list] #self.latent_f_list = [self.mf, self.mg] #self.latent_fchol_list = [self.cholf, self.cholg] if mean_functions is None: self.mean_functions = [None] * self.num_latent_funcs elif len(mean_functions) != len(kern_list): raise ValueError("Must provide a mean function for all latent\n\ functions as a list, provide None if no latent\n\ function is needed for a specific latent function" ) else: self.mean_functions = [] for m_f in mean_functions: if m_f is not None: self.link_parameter(m_f) self.mean_functions.append(m_f)
def calculate_gradients(self, log_marginal, latent_info, dF_dmu, dF_dv, num_inducing, num_outputs, num_data): """ Given a named tuple for lots of parameters of the latent function, calculate the gradients wrt to its latent functions and kernel """ l = latent_info #derivatives of expected likelihood, assuming zero mean function #Adv = l.A.T[:, :, None]*dF_dv[None, :, :] # As if dF_Dv is diagonal Adv = l.A[None,:,:]*dF_dv.T[:,None,:] # As if dF_Dv is diagonal, D, M, N #Admu = l.A.T.dot(dF_dmu) Admu = l.A.dot(dF_dmu) #AdvA = np.dstack([np.dot(l.A.T, Adv[:,:,i].T) for i in range(num_outputs)]) Adv = np.ascontiguousarray(Adv) # makes for faster operations later...(inc dsymm) AdvA = np.dot(Adv.reshape(-1, num_data),l.A.T).reshape(num_outputs, num_inducing, num_inducing ) #tmp = linalg.ijk_jlk_to_il(AdvA, l.S).dot(l.Kmmi) tmp = np.sum([np.dot(a,s) for a, s in zip(AdvA, l.S)],0).dot(l.Kmmi) #dF_dKmm = -Admu.dot(l.Kmmim.T) + AdvA.sum(-1) - tmp - tmp.T dF_dKmm = -Admu.dot(l.Kmmim.T) + AdvA.sum(0) - tmp - tmp.T dF_dKmm = 0.5*(dF_dKmm + dF_dKmm.T) # necessary? GPy bug? #tmp = 2.*(linalg.ij_jlk_to_ilk(l.Kmmi, l.S) - np.eye(num_inducing)[:,:,None]) tmp = l.S.reshape(-1, num_inducing).dot(l.Kmmi).reshape(num_outputs, num_inducing , num_inducing ) tmp = 2.*(tmp - np.eye(num_inducing)[None, :,:]) #dF_dKmn = linalg.ijk_jlk_to_il(tmp, Adv) + l.Kmmim.dot(dF_dmu.T) dF_dKmn = l.Kmmim.dot(dF_dmu.T) for a,b in zip(tmp, Adv): dF_dKmn += np.dot(a.T, b) dF_dm = Admu dF_dS = AdvA #gradient of the KL term (assuming zero mean function) #Si,_ = linalg.dpotri(np.asfortranarray(L), lower=1) Si = choleskies.multiple_dpotri(l.L) if np.any(np.isinf(Si)): raise ValueError("Cholesky representation unstable") #S = S + np.eye(S.shape[0])*1e-5*np.max(np.max(S)) #Si, Lnew, _,_ = linalg.pdinv(S) dKL_dm = l.Kmmim.copy() #dKL_dS = 0.5*(l.Kmmi[:,:,None] - Si) dKL_dS = 0.5*(l.Kmmi[None,:,:] - Si) #dKL_dKmm = 0.5*num_outputs*l.Kmmi - 0.5*l.Kmmi.dot(l.S.sum(-1)).dot(l.Kmmi) - 0.5*l.Kmmim.dot(l.Kmmim.T) dKL_dKmm = 0.5*num_outputs*l.Kmmi - 0.5*l.Kmmi.dot(l.S.sum(0)).dot(l.Kmmi) - 0.5*l.Kmmim.dot(l.Kmmim.T) #adjust gradient to account for mean function dL_dmfZ = None dL_dmfX = None KL = l.KL if l.mean_function is not None: #adjust KL term for mean function Kmmi_mfZ = np.dot(l.Kmmi, l.prior_mean_u) KL += -np.sum(l.q_u_mean*Kmmi_mfZ) KL += 0.5*np.sum(Kmmi_mfZ*l.prior_mean_u) #adjust gradient for mean fucntion dKL_dm -= Kmmi_mfZ dKL_dKmm += l.Kmmim.dot(Kmmi_mfZ.T) dKL_dKmm -= 0.5*Kmmi_mfZ.dot(Kmmi_mfZ.T) #compute gradients for mean_function dKL_dmfZ = Kmmi_mfZ - l.Kmmim dF_dmfX = dF_dmu.copy() dF_dmfZ = -Admu dF_dKmn -= np.dot(Kmmi_mfZ, dF_dmu.T) dF_dKmm += Admu.dot(Kmmi_mfZ.T) dL_dmfZ = dF_dmfZ - dKL_dmfZ dL_dmfX = dF_dmfX dL_dm, dL_dS, dL_dKmm, dL_dKmn = dF_dm - dKL_dm, dF_dS - dKL_dS, dF_dKmm - dKL_dKmm, dF_dKmn #dL_dchol = np.dstack([2.*np.dot(dL_dS[:, :, i], l.L[: , :,i]) for i in range(num_outputs)]) dL_dchol = 2.*np.array([np.dot(a,b) for a, b in zip(dL_dS, l.L) ]) dL_dchol = choleskies.triang_to_flat(dL_dchol) log_marginal -= KL dL_dKdiag = dF_dv.sum(1) return log_marginal, dL_dKmm, dL_dKmn, dL_dKdiag, dL_dm, dL_dchol, dL_dmfZ, dL_dmfX
def elbo_derivatives(self, q_U, p_U, q_F, VE_dm, VE_dv, Ntask, dims, f_index, d_index, q): """ Description: Returns ELBO derivatives w.r.t. variational parameters and hyperparameters Equation: gradients = {dL/dmu, dL/dS, dL/dKmm, dL/Kmn, dL/dKdiag} Paper: In Appendix 4 and 5 Extra_Info: Gradients w.r.t. hyperparameters use chain-rule and GPy. Note that Kmm, Kmn, Kdiag are matrices """ Q = dims['Q'] M = dims['M'] #------------------------------------# ALGEBRA FOR DERIVATIVES #------------------------------------# ####### Algebra for q(u) and p(u) ####### m_u = q_U.mu_u.copy() L_u = choleskies.flat_to_triang(q_U.chols_u.copy()) S_u = np.empty((Q, M, M)) [np.dot(L_u[q, :, :], L_u[q, :, :].T, S_u[q, :, :]) for q in range(Q)] Kuu = p_U.Kuu.copy() Kuui = p_U.Kuui.copy() S_qi, _ = linalg.dpotri(np.asfortranarray(L_u[q, :, :])) if np.any(np.isinf(S_qi)): raise ValueError("Sqi: Cholesky representation unstable") #-------------------------------------# DERIVATIVES OF ELBO TERMS #----------------------------------# ####### KL Terms ####### dKL_dmu_q = np.dot(Kuui[q, :, :], m_u[:, q, None]) dKL_dS_q = 0.5 * (Kuui[q, :, :] - S_qi) dKL_dKqq = 0.5 * Kuui[q, :, :] - 0.5 * Kuui[q, :, :].dot(S_u[q, :, :]).dot(Kuui[q, :, :]) \ - 0.5 * np.dot(Kuui[q, :, :], np.dot(m_u[:, q, None], m_u[:, q, None].T)).dot(Kuui[q, :, :].T) ####### Variational Expectation (VE) Terms ####### dVE_dmu_q = np.zeros((M, 1)) dVE_dS_q = np.zeros((M, M)) dVE_dKqq = np.zeros((M, M)) dVE_dKqd = [] dVE_dKdiag = [] for d, q_fd in enumerate(q_F): Nt = Ntask[f_index[d]] dVE_dmu_q += np.dot(q_fd.Afdu[q, :, :].T, VE_dm[f_index[d]][:, d_index[d]])[:, None] Adv = q_fd.Afdu[q, :, :].T * VE_dv[f_index[d]][:, d_index[d], None].T Adv = np.ascontiguousarray(Adv) AdvA = np.dot(Adv.reshape(-1, Nt), q_fd.Afdu[q, :, :]).reshape(M, M) dVE_dS_q += AdvA ####### Derivatives dKuquq ####### tmp_dv = np.dot(AdvA, S_u[q, :, :]).dot(Kuui[q, :, :]) dVE_dKqq += AdvA - tmp_dv - tmp_dv.T Adm = np.dot(q_fd.Afdu[q, :, :].T, VE_dm[f_index[d]][:, d_index[d], None]) dVE_dKqq += -np.dot(Adm, np.dot(Kuui[q, :, :], m_u[:, q, None]).T) ####### Derivatives dKuqfd ####### tmp = np.dot(S_u[q, :, :], Kuui[q, :, :]) tmp = 2. * (tmp - np.eye(M)) dve_kqd = np.dot(np.dot(Kuui[q, :, :], m_u[:, q, None]), VE_dm[f_index[d]][:, d_index[d], None].T) dve_kqd += np.dot(tmp.T, Adv) dVE_dKqd.append(dve_kqd) ####### Derivatives dKdiag ####### dVE_dKdiag.append(VE_dv[f_index[d]][:, d_index[d]]) dVE_dKqq = 0.5 * (dVE_dKqq + dVE_dKqq.T) #--------------------------------------# FINAL ELBO DERIVATIVES #------------------------------------# ####### ELBO derivatives ---> sum of VE and KL terms ####### dL_dmu_q = dVE_dmu_q - dKL_dmu_q dL_dS_q = dVE_dS_q - dKL_dS_q dL_dKqq = dVE_dKqq - dKL_dKqq dL_dKdq = dVE_dKqd dL_dKdiag = dVE_dKdiag ####### Pass S_q gradients to its low-triangular representation L_q ####### chol_u = q_U.chols_u.copy() L_q = choleskies.flat_to_triang(chol_u[:, q:q + 1]) dL_dL_q = 2. * np.array( [np.dot(a, b) for a, b in zip(dL_dS_q[None, :, :], L_q)]) dL_dL_q = choleskies.triang_to_flat(dL_dL_q) return dL_dmu_q, dL_dL_q, dL_dS_q, dL_dKqq, dL_dKdq, dL_dKdiag
def __init__(self, X, Y, Z, kern_list, likelihood, mean_functions=None, name='SVGPMulti', Y_metadata=None, batchsize=None): """ Extension to the SVGP to allow multiple latent function, where the latent functions are assumed independant (have one kernel per latent function) """ # super(SVGPMulti, self).__init__(name) # Parameterized.__init__(self) assert X.ndim == 2 self.Y_metadata = Y_metadata _, self.output_dim = Y.shape # self.Z = Param('inducing inputs', Z) # self.num_inducing = Z.shape[0] # self.likelihood = likelihood self.kern_list = kern_list self.batchsize = batchsize #Batch the data self.X_all, self.Y_all = X, Y if batchsize is None: X_batch, Y_batch = X, Y else: import climin.util #Make a climin slicer to make drawing minibatches much quicker self.slicer = climin.util.draw_mini_slices(self.X_all.shape[0], self.batchsize) X_batch, Y_batch = self.new_batch() # if isinstance(X_batch, (ObsAr, VariationalPosterior)): # self.X = X_batch.copy() # else: # self.X = ObsAr(X_batch) # self.Y = Y_batch #create the SVI inference method # self.inference_method = svgp_inf() inference_method = svgp_inf() #Initialize base model super(SVGPMulti, self).__init__(X=X_batch, Y=Y_batch, Z=Z, kernel=kern_list[0], likelihood=likelihood, mean_function=None, X_variance=None, inference_method=inference_method, name=name, Y_metadata=Y_metadata, normalizer=False) self.unlink_parameter(self.kern) # We don't want a single kern # self.num_data, self.input_dim = self.X.shape self.num_outputs = self.Y.shape[1] self.num_latent_funcs = self.likelihood.request_num_latent_functions(self.Y_all) #Make a latent function per dimension self.q_u_means = Param('q_u_means', np.zeros((self.num_inducing, self.num_latent_funcs))) chols = choleskies.triang_to_flat(np.tile(np.eye(self.num_inducing)[None,:,:], (self.num_latent_funcs,1,1))) self.q_u_chols = Param('qf_u_chols', chols) self.link_parameter(self.Z, index=0) self.link_parameter(self.q_u_means) self.link_parameter(self.q_u_chols) # self.link_parameter(self.likelihood) #Must pass a list of kernels that work on each latent function for now assert len(kern_list) == self.num_latent_funcs #Add the rest of the kernels, one kernel per latent function [self.link_parameter(kern) for kern in kern_list] #self.latent_f_list = [self.mf, self.mg] #self.latent_fchol_list = [self.cholf, self.cholg] if mean_functions is None: self.mean_functions = [None]*self.num_latent_funcs elif len(mean_functions) != len(kern_list): raise ValueError("Must provide a mean function for all latent\n\ functions as a list, provide None if no latent\n\ function is needed for a specific latent function") else: self.mean_functions = [] for m_f in mean_functions: if m_f is not None: self.link_parameter(m_f) self.mean_functions.append(m_f)
def __init__(self, X, Y, Z, kern_list, likelihood, Y_metadata, name='HetMOGP', batch_size=None): """ :param X: Input data :param Y: (Heterogeneous) Output data :param Z: Inducing inputs :param kern_list: Kernel functions of GP priors :param likelihood: (Heterogeneous) Likelihoods :param Y_metadata: Linking info between F->likelihoods :param name: Model name :param batch_size: Size of batch for stochastic optimization Description: Initialization method for the model class """ #---------------------------------------# INITIALIZATIONS #--------------------------------------------# ####### Initialization of class variables ####### self.batch_size = batch_size self.kern_list = kern_list self.likelihood = likelihood self.Y_metadata = Y_metadata ####### Heterogeneous Data ####### self.Xmulti = X self.Ymulti = Y ####### Batches of Data for Stochastic Mode ####### self.Xmulti_all, self.Ymulti_all = X, Y if batch_size is None: self.stochastic = False Xmulti_batch, Ymulti_batch = X, Y else: ####### Makes a climin slicer to make drawing minibatches much quicker ####### self.stochastic = True self.slicer_list = [] [ self.slicer_list.append( draw_mini_slices(Xmulti_task.shape[0], self.batch_size)) for Xmulti_task in self.Xmulti ] Xmulti_batch, Ymulti_batch = self.new_batch() self.Xmulti, self.Ymulti = Xmulti_batch, Ymulti_batch ####### Model dimensions {M, Q, D} ####### self.num_inducing = Z.shape[0] # M self.num_latent_funcs = len(kern_list) # Q self.num_output_funcs = likelihood.num_output_functions( self.Y_metadata) ####### Inducing points Z ####### self.Xdim = Z.shape[1] Z = np.tile(Z, (1, self.num_latent_funcs)) ####### Inference ####### inference_method = Inference() ####### Model class (and inherited classes) super-initialization ####### super(HetMOGP, self).__init__(X=Xmulti_batch[0][1:10], Y=Ymulti_batch[0][1:10], Z=Z, kernel=kern_list[0], likelihood=likelihood, mean_function=None, X_variance=None, inference_method=inference_method, Y_metadata=Y_metadata, name=name, normalizer=False) ####### Initialization of the Multi-output GP mixing ####### self.W_list, self.kappa_list = multi_output.random_W_kappas( self.num_latent_funcs, self.num_output_funcs, rank=1) _, self.B_list = multi_output.LCM(input_dim=self.Xdim, output_dim=self.num_output_funcs, rank=1, kernels_list=self.kern_list, W_list=self.W_list, kappa_list=self.kappa_list) ####### Initialization of Variational Parameters (q_u_means = \mu, q_u_chols = lower_triang(S)) ####### self.q_u_means = Param( 'm_u', 0 * np.random.randn(self.num_inducing, self.num_latent_funcs) + 0 * np.tile(np.random.randn(1, self.num_latent_funcs), (self.num_inducing, 1))) chols = choleskies.triang_to_flat( np.tile( np.eye(self.num_inducing)[None, :, :], (self.num_latent_funcs, 1, 1))) self.q_u_chols = Param('L_u', chols) #-----------------------------# LINKS FOR OPTIMIZABLE PARAMETERS #---------------------------------------# ####### Linking and Un-linking of parameters and hyperaparameters (for ParamZ optimizer) ####### self.unlink_parameter( self.kern) # Unlink SparseGP default param kernel self.link_parameter(self.Z, index=0) self.link_parameter(self.q_u_means) self.link_parameters(self.q_u_chols) [self.link_parameter(kern_q) for kern_q in kern_list] # link all kernels [self.link_parameter(B_q) for B_q in self.B_list] ####### EXTRA. Auxiliary variables ####### self.vem_step = True # [True=VE-step, False=VM-step] self.ve_count = 0 self.elbo = np.zeros((1, 1))
def __init__(self, X, Y, Z, kern_list, likelihood, Y_metadata, name='SVMOGP', batch_size=None, non_chained=True): self.batch_size = batch_size self.kern_list = kern_list self.likelihood = likelihood self.Y_metadata = Y_metadata self.num_inducing = Z.shape[0] # M self.num_latent_funcs = len(kern_list) # Q self.num_output_funcs = likelihood.num_output_functions(Y_metadata) if (not non_chained): assert self.num_output_funcs == self.num_latent_funcs, "we need a latent function per likelihood parameter" if non_chained: self.W_list, self.kappa_list = util.random_W_kappas( self.num_latent_funcs, self.num_output_funcs, rank=1) else: self.W_list, self.kappa_list = util.Chained_W_kappas( self.num_latent_funcs, self.num_output_funcs, rank=1) self.Xmulti = X self.Ymulti = Y self.iAnnMulti = Y_metadata['iAnn'] # Batch the data self.Xmulti_all, self.Ymulti_all, self.iAnn_all = X, Y, Y_metadata[ 'iAnn'] if batch_size is None: #self.stochastic = False Xmulti_batch, Ymulti_batch, iAnnmulti_batch = X, Y, Y_metadata[ 'iAnn'] else: # Makes a climin slicer to make drawing minibatches much quicker #self.stochastic = False #"This was True as Pablo had it" self.slicer_list = [] [ self.slicer_list.append( draw_mini_slices(Xmulti_task.shape[0], self.batch_size)) for Xmulti_task in self.Xmulti ] Xmulti_batch, Ymulti_batch, iAnnmulti_batch = self.new_batch() self.Xmulti, self.Ymulti, self.iAnnMulti = Xmulti_batch, Ymulti_batch, iAnnmulti_batch self.Y_metadata.update(iAnn=iAnnmulti_batch) # Initialize inducing points Z #Z = kmm_init(self.X_all, self.num_inducing) self.Xdim = Z.shape[1] Z = np.tile(Z, (1, self.num_latent_funcs)) inference_method = SVMOGPInf() super(SVMOGP, self).__init__(X=Xmulti_batch[0][1:10], Y=Ymulti_batch[0][1:10], Z=Z, kernel=kern_list[0], likelihood=likelihood, mean_function=None, X_variance=None, inference_method=inference_method, Y_metadata=Y_metadata, name=name, normalizer=False) self.unlink_parameter( self.kern) # Unlink SparseGP default param kernel _, self.B_list = util.LCM(input_dim=self.Xdim, output_dim=self.num_output_funcs, rank=1, kernels_list=self.kern_list, W_list=self.W_list, kappa_list=self.kappa_list) # Set-up optimization parameters: [Z, m_u, L_u] self.q_u_means = Param( 'm_u', 0.0 * np.random.randn(self.num_inducing, self.num_latent_funcs) + 0.0 * np.tile(np.random.randn(1, self.num_latent_funcs), (self.num_inducing, 1))) chols = choleskies.triang_to_flat( np.tile( np.eye(self.num_inducing)[None, :, :], (self.num_latent_funcs, 1, 1))) self.q_u_chols = Param('L_u', chols) self.link_parameter(self.Z, index=0) self.link_parameter(self.q_u_means) self.link_parameters(self.q_u_chols) [self.link_parameter(kern_q) for kern_q in kern_list] # link all kernels [self.link_parameter(B_q) for B_q in self.B_list] self.vem_step = True # [True=VE-step, False=VM-step] self.ve_count = 0 self.elbo = np.zeros((1, 1)) self.index_VEM = 0 #this is a variable to index correctly the self.elbo when using VEM self.Gauss_Newton = False #This is a flag for using the Gauss-Newton approximation when dL_dV is needed
def calculate_gradients(self, q_U, p_U_new, p_U_old, p_U_var, q_F, VE_dm, VE_dv, Ntask, M, Q, D, f_index, d_index,q): """ Calculates gradients of the Log-marginal distribution p(Y) wrt variational parameters mu_q, S_q """ # Algebra for q(u): m_u = q_U.mu_u.copy() L_u = choleskies.flat_to_triang(q_U.chols_u.copy()) S_u = np.empty((Q, M, M)) [np.dot(L_u[q, :, :], L_u[q, :, :].T, S_u[q, :, :]) for q in range(Q)] S_qi, _ = linalg.dpotri(np.asfortranarray(L_u[q, :, :])) if np.any(np.isinf(S_qi)): raise ValueError("Sqi: Cholesky representation unstable") # Algebra for p(u) Kuu_new = p_U_new.Kuu.copy() Luu_new = p_U_new.Luu.copy() Kuui_new = p_U_new.Kuui.copy() Kuu_old = p_U_old.Kuu.copy() Luu_old = p_U_old.Luu.copy() Kuui_old = p_U_old.Kuui.copy() Mu_var = p_U_var.Mu.copy() Kuu_var = p_U_var.Kuu.copy() Luu_var = p_U_var.Luu.copy() Kuui_var = p_U_var.Kuui.copy() # KL Terms dKLnew_dmu_q = np.dot(Kuui_new[q,:,:], m_u[:, q, None]) dKLnew_dS_q = 0.5 * (Kuui_new[q,:,:] - S_qi) dKLold_dmu_q = np.dot(Kuui_old[q,:,:], m_u[:, q, None]) dKLold_dS_q = 0.5 * (Kuui_old[q,:,:] - S_qi) dKLvar_dmu_q = np.dot(Kuui_var[q,:,:], (m_u[:, q, None] - Mu_var[q, :, :])) # important!! (Eq. 69 MCB) dKLvar_dS_q = 0.5 * (Kuui_var[q,:,:] - S_qi) dKLnew_dKqq = 0.5 * Kuui_new[q,:,:] - 0.5 * Kuui_new[q,:,:].dot(S_u[q, :, :]).dot(Kuui_new[q,:,:]) \ - 0.5 * np.dot(Kuui_new[q,:,:],np.dot(m_u[:, q, None],m_u[:, q, None].T)).dot(Kuui_new[q,:,:].T) dKLold_dKqq = 0.5 * Kuui_old[q,:,:] - 0.5 * Kuui_old[q,:,:].dot(S_u[q, :, :]).dot(Kuui_old[q,:,:]) \ - 0.5 * np.dot(Kuui_old[q,:,:],np.dot(m_u[:, q, None],m_u[:, q, None].T)).dot(Kuui_old[q,:,:].T) #dKLvar_dKqq = 0.5 * Kuui_var[q,:,:] - 0.5 * Kuui_var[q,:,:].dot(S_u[q, :, :]).dot(Kuui_var[q,:,:]) \ # - 0.5 * np.dot(Kuui_var[q,:,:],np.dot(m_u[:, q, None],m_u[:, q, None].T)).dot(Kuui_var[q,:,:].T) \ # + 0.5 * np.dot(Kuui_var[q,:,:], np.dot(m_u[:,q,None], Mu_var[q,:,:].T)).dot(Kuui_var[q,:,:].T) \ # + 0.5 * np.dot(Kuui_var[q,:,:], np.dot(Mu_var[q,:,:], m_u[:,q,None].T)).dot(Kuui_var[q,:,:].T) \ # - 0.5 * np.dot(Kuui_var[q,:,:],np.dot(Mu_var[q,:,:], Mu_var[q,:,:].T)).dot(Kuui_var[q,:,:].T) #KLvar += 0.5 * np.sum(Kuui_var[q, :, :] * S_u[q, :, :]) \ # + 0.5 * np.dot((Mu_var[q, :, :] - m_u[:, q, None]).T, np.dot(Kuui_var[q, :, :], (Mu_var[q, :, :] - m_u[:, q, None]))) \ # - 0.5 * M \ # + 0.5 * 2. * np.sum(np.log(np.abs(np.diag(Luu_var[q, :, :])))) \ # - 0.5 * 2. * np.sum(np.log(np.abs(np.diag(L_u[q, :, :])))) # # VE Terms dVE_dmu_q = np.zeros((M, 1)) dVE_dS_q = np.zeros((M, M)) dVE_dKqq = np.zeros((M, M)) dVE_dKqd = [] dVE_dKdiag = [] for d, q_fd in enumerate(q_F): Nt = Ntask[f_index[d]] dVE_dmu_q += np.dot(q_fd.Afdu[q, :, :].T, VE_dm[f_index[d]][:,d_index[d]])[:, None] Adv = q_fd.Afdu[q,:,:].T * VE_dv[f_index[d]][:,d_index[d],None].T Adv = np.ascontiguousarray(Adv) AdvA = np.dot(Adv.reshape(-1, Nt), q_fd.Afdu[q, :, :]).reshape(M, M) dVE_dS_q += AdvA # Derivatives dKuquq tmp_dv = np.dot(AdvA, S_u[q, :, :]).dot(Kuui_new[q,:,:]) dVE_dKqq += AdvA - tmp_dv - tmp_dv.T Adm = np.dot(q_fd.Afdu[q, :, :].T, VE_dm[f_index[d]][:,d_index[d],None]) dVE_dKqq += - np.dot(Adm, np.dot(Kuui_new[q,:,:], m_u[:, q, None]).T) # Derivatives dKuqfd tmp = np.dot(S_u[q, :, :], Kuui_new[q,:,:]) tmp = 2. * (tmp - np.eye(M)) dve_kqd = np.dot(np.dot(Kuui_new[q,:,:], m_u[:, q, None]), VE_dm[f_index[d]][:,d_index[d],None].T) dve_kqd += np.dot(tmp.T, Adv) dVE_dKqd.append(dve_kqd) # Derivatives dKdiag dVE_dKdiag.append(VE_dv[f_index[d]][:,d_index[d]]) dVE_dKqq = 0.5 * (dVE_dKqq + dVE_dKqq.T) # Derivatives of variational parameters dL_dmu_q = dVE_dmu_q - dKLnew_dmu_q + dKLold_dmu_q - dKLvar_dmu_q dL_dS_q = dVE_dS_q - dKLnew_dS_q + dKLold_dS_q - dKLvar_dS_q # Derivatives of prior hyperparameters # if using Zgrad, dL_dKqq = dVE_dKqq - dKLnew_dKqq + dKLold_dKqq - dKLvar_dKqq # otherwise for hyperparameters: dL_dKqq = dVE_dKqq - dKLnew_dKqq dL_dKqq = dVE_dKqq - dKLnew_dKqq #+ dKLold_dKqq - dKLvar_dKqq # dKLold_dKqq sólo para Zgrad, dKLvar_dKqq to be done (for Zgrad) dL_dKdq = dVE_dKqd dL_dKdiag = dVE_dKdiag # Pass S_q gradients to its low-triangular representation L_q chol_u = q_U.chols_u.copy() L_q = choleskies.flat_to_triang(chol_u[:,q:q+1]) dL_dL_q = 2. * np.array([np.dot(a, b) for a, b in zip(dL_dS_q[None,:,:], L_q)]) dL_dL_q = choleskies.triang_to_flat(dL_dL_q) # Posterior posterior_q = Posterior(mean=m_u[:, q, None], cov=S_u[q, :, :], K=Kuu_new[q,:,:], prior_mean=np.zeros(m_u[:, q, None].shape)) return dL_dmu_q, dL_dL_q, dL_dS_q, posterior_q, dL_dKqq, dL_dKdq, dL_dKdiag