def parameters_changed(self): K = self.kern.K(self.X) self.L = L = GPy.util.linalg.jitchol(K) F = self.F = blas.dtrmm(1.0,L, self.V, lower=1, trans_a=0) #compute the log likelihood self._loglik = self.likelihood.logpdf(F, self.Y).sum() dL_dF = self.likelihood.dlogpdf_df(F, self.Y) self.likelihood.gradient = self.likelihood.dlogpdf_dtheta(F, self.Y).sum(1).sum(1) #here's the prior for V self._loglik += -0.5*self.num_data*self.output_dim*np.log(2*np.pi) - 0.5*np.sum(np.square(self.V)) #set all gradients to zero, then only compute necessary gradients. self.gradient = 0. #compute dL_dV if not self.V.is_fixed: self.V.gradient = -self.V + blas.dtrmm(1.0, L, dL_dF, trans_a=1, lower=1) #compute kernel gradients if not self.kern.is_fixed: dL_dL = np.dot(dL_dF, self.V.T) # where the first L is the likelihood, the second L is the triangular matrix #nasty reverse Cholesky to get dL_dK dL_dK = GPy.util.choleskies.backprop_gradient(dL_dL, L) self.kern.update_gradients_full(dL_dK, self.X)
def parameters_changed(self): K = self.kern.K(self.X) self.L = L = GPy.util.linalg.jitchol(K) F = self.F = blas.dtrmm(1.0, L, self.V, lower=1, trans_a=0) #compute the log likelihood self._loglik = self.likelihood.logpdf(F, self.Y).sum() dL_dF = self.likelihood.dlogpdf_df(F, self.Y) self.likelihood.gradient = self.likelihood.dlogpdf_dtheta( F, self.Y).sum(1).sum(1) #here's the prior for V self._loglik += -0.5 * self.num_data * self.output_dim * np.log( 2 * np.pi) - 0.5 * np.sum(np.square(self.V)) #set all gradients to zero, then only compute necessary gradients. self.gradient = 0. #compute dL_dV if not self.V.is_fixed: self.V.gradient = -self.V + blas.dtrmm( 1.0, L, dL_dF, trans_a=1, lower=1) #compute kernel gradients if not self.kern.is_fixed: dL_dL = np.dot( dL_dF, self.V.T ) # where the first L is the likelihood, the second L is the triangular matrix #nasty reverse Cholesky to get dL_dK dL_dK = GPy.util.choleskies.backprop_gradient(dL_dL, L) self.kern.update_gradients_full(dL_dK, self.X)
def calculate_mu_var(self, X, Y, Z, q_u_mean, q_u_chol, kern, mean_function, num_inducing, num_data, num_outputs): """ Calculate posterior mean and variance for the latent function values for use in the expectation over the likelihood """ #expand cholesky representation L = choleskies.flat_to_triang(q_u_chol) #S = linalg.ijk_ljk_to_ilk(L, L) #L.dot(L.T) S = np.empty((num_outputs, num_inducing, num_inducing)) [np.dot(L[i,:,:], L[i,:,:].T, S[i,:,:]) for i in range(num_outputs)] #logdetS = np.array([2.*np.sum(np.log(np.abs(np.diag(L[:,:,i])))) for i in range(L.shape[-1])]) logdetS = np.array([2.*np.sum(np.log(np.abs(np.diag(L[i,:,:])))) for i in range(L.shape[0])]) #compute mean function stuff if mean_function is not None: prior_mean_u = mean_function.f(Z) prior_mean_f = mean_function.f(X) else: prior_mean_u = np.zeros((num_inducing, num_outputs)) prior_mean_f = np.zeros((num_data, num_outputs)) #compute kernel related stuff Kmm = kern.K(Z) #Knm = kern.K(X, Z) Kmn = kern.K(Z, X) Knn_diag = kern.Kdiag(X) #Kmmi, Lm, Lmi, logdetKmm = linalg.pdinv(Kmm) Lm = linalg.jitchol(Kmm) logdetKmm = 2.*np.sum(np.log(np.diag(Lm))) Kmmi, _ = linalg.dpotri(Lm) #compute the marginal means and variances of q(f) #A = np.dot(Knm, Kmmi) A, _ = linalg.dpotrs(Lm, Kmn) #mu = prior_mean_f + np.dot(A, q_u_mean - prior_mean_u) mu = prior_mean_f + np.dot(A.T, q_u_mean - prior_mean_u) #v = Knn_diag[:,None] - np.sum(A*Knm,1)[:,None] + np.sum(A[:,:,None] * linalg.ij_jlk_to_ilk(A, S), 1) v = np.empty((num_data, num_outputs)) for i in range(num_outputs): tmp = dtrmm(1.0,L[i].T, A, lower=0, trans_a=0) v[:,i] = np.sum(np.square(tmp),0) v += (Knn_diag - np.sum(A*Kmn,0))[:,None] #compute the KL term Kmmim = np.dot(Kmmi, q_u_mean) #KLs = -0.5*logdetS -0.5*num_inducing + 0.5*logdetKmm + 0.5*np.einsum('ij,ijk->k', Kmmi, S) + 0.5*np.sum(q_u_mean*Kmmim,0) KLs = -0.5*logdetS -0.5*num_inducing + 0.5*logdetKmm + 0.5*np.sum(Kmmi[None,:,:]*S,1).sum(1) + 0.5*np.sum(q_u_mean*Kmmim,0) KL = KLs.sum() latent_detail = LatentFunctionDetails(q_u_mean=q_u_mean, q_u_chol=q_u_chol, mean_function=mean_function, mu=mu, v=v, prior_mean_u=prior_mean_u, L=L, A=A, S=S, Kmm=Kmm, Kmmi=Kmmi, Kmmim=Kmmim, KL=KL) return latent_detail
def variational_q_fd(self, X, Z, q_U, p_U, kern_list, B, N, dims, d): """ Description: Returns the posterior approximation q(f) for the latent output functions (LOFs) Equation: q(f) = \int p(f|u)q(u)du Paper: In Section 2.2.2 / Variational Bounds """ Q = dims['Q'] M = dims['M'] #-----------------------------------------# POSTERIOR ALGEBRA #-------------------------------------# ####### Algebra for q(u) ####### m_u = q_U.mu_u.copy() L_u = choleskies.flat_to_triang(q_U.chols_u.copy()) S_u = np.empty((Q, M, M)) [np.dot(L_u[q, :, :], L_u[q, :, :].T, S_u[q, :, :]) for q in range(Q)] ####### Algebra for p(f_d|u) ####### Kfdu = multi_output.cross_covariance(X, Z, B, kern_list, d) Luu = p_U.Luu.copy() Kff = multi_output.function_covariance(X, B, kern_list, d) Kff_diag = np.diag(Kff) ####### Algebra for q(f_d) = E_{q(u)}[p(f_d|u)] ####### Afdu = np.empty((Q, N, M)) # Afdu = K_{fduq}Ki_{uquq} m_fd = np.zeros((N, 1)) v_fd = np.zeros((N, 1)) S_fd = np.zeros((N, N)) v_fd += Kff_diag[:, None] S_fd += Kff for q in range(Q): ####### Expectation w.r.t. u_q part ####### R, _ = linalg.dpotrs(np.asfortranarray(Luu[q, :, :]), Kfdu[:, q * M:(q * M) + M].T) Afdu[q, :, :] = R.T m_fd += np.dot(Afdu[q, :, :], m_u[:, q, None]) # exp tmp = dtrmm(alpha=1.0, a=L_u[q, :, :].T, b=R, lower=0, trans_a=0) v_fd += np.sum(np.square(tmp), 0)[:, None] - np.sum( R * Kfdu[:, q * M:(q * M) + M].T, 0)[:, None] # exp S_fd += np.dot(np.dot(R.T, S_u[q, :, :]), R) - np.dot( Kfdu[:, q * M:(q * M) + M], R) if (v_fd < 0).any(): print('v negative!') #--------------------------------------# VARIATIONAL POSTERIOR (LOFs) #-----------------------------------# ####### Variational output distribution q_fd() ####### q_fd = qfd(m_fd=m_fd, v_fd=v_fd, Kfdu=Kfdu, Afdu=Afdu, S_fd=S_fd) return q_fd
def calculate_q_f(self, X, Z, q_U, p_U, kern_list, B, M, N, Q, D, d): """ Calculates the mean and variance of q(f_d) as Equation: E_q(U)\{p(f_d|U)\} """ # Algebra for q(u): m_u = q_U.mu_u.copy() L_u = choleskies.flat_to_triang(q_U.chols_u.copy()) S_u = np.empty((Q, M, M)) [np.dot(L_u[q, :, :], L_u[q, :, :].T, S_u[q, :, :]) for q in range(Q)] # Algebra for p(f_d|u): Kfdu = util.cross_covariance(X, Z, B, kern_list, d) Kuu = p_U.Kuu.copy() Luu = p_U.Luu.copy() Kuui = p_U.Kuui.copy() Kff = util.function_covariance(X, B, kern_list, d) Kff_diag = np.diag(Kff) # Algebra for q(f_d) = E_{q(u)}[p(f_d|u)] Afdu = np.empty((Q, N, M)) #Afdu = K_{fduq}Ki_{uquq} m_fd = np.zeros((N, 1)) v_fd = np.zeros((N, 1)) S_fd = np.zeros((N, N)) v_fd += Kff_diag[:, None] S_fd += Kff for q in range(Q): # Expectation part R, _ = linalg.dpotrs(np.asfortranarray(Luu[q, :, :]), Kfdu[:, q * M:(q * M) + M].T) Afdu[q, :, :] = R.T m_fd += np.dot(Afdu[q, :, :], m_u[:, q, None]) #exp tmp = dtrmm(alpha=1.0, a=L_u[q, :, :].T, b=R, lower=0, trans_a=0) v_fd += np.sum(np.square(tmp), 0)[:, None] - np.sum( R * Kfdu[:, q * M:(q * M) + M].T, 0)[:, None] #exp S_fd += np.dot(np.dot(R.T, S_u[q, :, :]), R) - np.dot( Kfdu[:, q * M:(q * M) + M], R) if (v_fd < 0).any(): print('v negative!') q_fd = qfd(m_fd=m_fd, v_fd=v_fd, Kfdu=Kfdu, Afdu=Afdu, S_fd=S_fd) return q_fd
def inference(self, q_u_mean, q_u_chol, kern, X, Z, likelihood, Y, mean_function=None, Y_metadata=None, KL_scale=1.0, batch_scale=1.0): num_data, _ = Y.shape num_inducing, num_outputs = q_u_mean.shape #expand cholesky representation L = choleskies.flat_to_triang(q_u_chol) S = np.empty((num_outputs, num_inducing, num_inducing)) [np.dot(L[i,:,:], L[i,:,:].T, S[i,:,:]) for i in range(num_outputs)] #Si,_ = linalg.dpotri(np.asfortranarray(L), lower=1) Si = choleskies.multiple_dpotri(L) logdetS = np.array([2.*np.sum(np.log(np.abs(np.diag(L[i,:,:])))) for i in range(L.shape[0])]) if np.any(np.isinf(Si)): raise ValueError("Cholesky representation unstable") #compute mean function stuff if mean_function is not None: prior_mean_u = mean_function.f(Z) prior_mean_f = mean_function.f(X) else: prior_mean_u = np.zeros((num_inducing, num_outputs)) prior_mean_f = np.zeros((num_data, num_outputs)) #compute kernel related stuff Kmm = kern.K(Z) Kmn = kern.K(Z, X) Knn_diag = kern.Kdiag(X) Lm = linalg.jitchol(Kmm) logdetKmm = 2.*np.sum(np.log(np.diag(Lm))) Kmmi, _ = linalg.dpotri(Lm) #compute the marginal means and variances of q(f) A, _ = linalg.dpotrs(Lm, Kmn) mu = prior_mean_f + np.dot(A.T, q_u_mean - prior_mean_u) v = np.empty((num_data, num_outputs)) for i in range(num_outputs): tmp = dtrmm(1.0,L[i].T, A, lower=0, trans_a=0) v[:,i] = np.sum(np.square(tmp),0) v += (Knn_diag - np.sum(A*Kmn,0))[:,None] #compute the KL term Kmmim = np.dot(Kmmi, q_u_mean) KLs = -0.5*logdetS -0.5*num_inducing + 0.5*logdetKmm + 0.5*np.sum(Kmmi[None,:,:]*S,1).sum(1) + 0.5*np.sum(q_u_mean*Kmmim,0) KL = KLs.sum() #gradient of the KL term (assuming zero mean function) dKL_dm = Kmmim.copy() dKL_dS = 0.5*(Kmmi[None,:,:] - Si) dKL_dKmm = 0.5*num_outputs*Kmmi - 0.5*Kmmi.dot(S.sum(0)).dot(Kmmi) - 0.5*Kmmim.dot(Kmmim.T) if mean_function is not None: #adjust KL term for mean function Kmmi_mfZ = np.dot(Kmmi, prior_mean_u) KL += -np.sum(q_u_mean*Kmmi_mfZ) KL += 0.5*np.sum(Kmmi_mfZ*prior_mean_u) #adjust gradient for mean fucntion dKL_dm -= Kmmi_mfZ dKL_dKmm += Kmmim.dot(Kmmi_mfZ.T) dKL_dKmm -= 0.5*Kmmi_mfZ.dot(Kmmi_mfZ.T) #compute gradients for mean_function dKL_dmfZ = Kmmi_mfZ - Kmmim #quadrature for the likelihood F, dF_dmu, dF_dv, dF_dthetaL = likelihood.variational_expectations(Y, mu, v, Y_metadata=Y_metadata) #rescale the F term if working on a batch F, dF_dmu, dF_dv = F*batch_scale, dF_dmu*batch_scale, dF_dv*batch_scale if dF_dthetaL is not None: dF_dthetaL = dF_dthetaL.sum(1).sum(1)*batch_scale #derivatives of expected likelihood, assuming zero mean function Adv = A[None,:,:]*dF_dv.T[:,None,:] # As if dF_Dv is diagonal, D, M, N Admu = A.dot(dF_dmu) Adv = np.ascontiguousarray(Adv) # makes for faster operations later...(inc dsymm) AdvA = np.dot(Adv.reshape(-1, num_data),A.T).reshape(num_outputs, num_inducing, num_inducing ) tmp = np.sum([np.dot(a,s) for a, s in zip(AdvA, S)],0).dot(Kmmi) dF_dKmm = -Admu.dot(Kmmim.T) + AdvA.sum(0) - tmp - tmp.T dF_dKmm = 0.5*(dF_dKmm + dF_dKmm.T) # necessary? GPy bug? tmp = S.reshape(-1, num_inducing).dot(Kmmi).reshape(num_outputs, num_inducing , num_inducing ) tmp = 2.*(tmp - np.eye(num_inducing)[None, :,:]) dF_dKmn = Kmmim.dot(dF_dmu.T) for a,b in zip(tmp, Adv): dF_dKmn += np.dot(a.T, b) dF_dm = Admu dF_dS = AdvA #adjust gradient to account for mean function if mean_function is not None: dF_dmfX = dF_dmu.copy() dF_dmfZ = -Admu dF_dKmn -= np.dot(Kmmi_mfZ, dF_dmu.T) dF_dKmm += Admu.dot(Kmmi_mfZ.T) #sum (gradients of) expected likelihood and KL part log_marginal = F.sum() - KL dL_dm, dL_dS, dL_dKmm, dL_dKmn = dF_dm - dKL_dm, dF_dS- dKL_dS, dF_dKmm- dKL_dKmm, dF_dKmn dL_dchol = 2.*np.array([np.dot(a,b) for a, b in zip(dL_dS, L) ]) dL_dchol = choleskies.triang_to_flat(dL_dchol) grad_dict = {'dL_dKmm':dL_dKmm, 'dL_dKmn':dL_dKmn, 'dL_dKdiag': dF_dv.sum(1), 'dL_dm':dL_dm, 'dL_dchol':dL_dchol, 'dL_dthetaL':dF_dthetaL} if mean_function is not None: grad_dict['dL_dmfZ'] = dF_dmfZ - dKL_dmfZ grad_dict['dL_dmfX'] = dF_dmfX return Posterior(mean=q_u_mean, cov=S.T, K=Kmm, prior_mean=prior_mean_u), log_marginal, grad_dict
def inference( self, q_u_mean, q_u_chol, kern, X, Z, likelihood, Y, mean_function=None, Y_metadata=None, KL_scale=1.0, batch_scale=1.0, ): num_data, _ = Y.shape num_inducing, num_outputs = q_u_mean.shape # expand cholesky representation L = choleskies.flat_to_triang(q_u_chol) S = np.empty((num_outputs, num_inducing, num_inducing)) [np.dot(L[i, :, :], L[i, :, :].T, S[i, :, :]) for i in range(num_outputs)] # Si,_ = linalg.dpotri(np.asfortranarray(L), lower=1) Si = choleskies.multiple_dpotri(L) logdetS = np.array([2.0 * np.sum(np.log(np.abs(np.diag(L[i, :, :])))) for i in range(L.shape[0])]) if np.any(np.isinf(Si)): raise ValueError("Cholesky representation unstable") # compute mean function stuff if mean_function is not None: prior_mean_u = mean_function.f(Z) prior_mean_f = mean_function.f(X) else: prior_mean_u = np.zeros((num_inducing, num_outputs)) prior_mean_f = np.zeros((num_data, num_outputs)) # compute kernel related stuff Kmm = kern.K(Z) Kmn = kern.K(Z, X) Knn_diag = kern.Kdiag(X) Lm = linalg.jitchol(Kmm) logdetKmm = 2.0 * np.sum(np.log(np.diag(Lm))) Kmmi, _ = linalg.dpotri(Lm) # compute the marginal means and variances of q(f) A, _ = linalg.dpotrs(Lm, Kmn) mu = prior_mean_f + np.dot(A.T, q_u_mean - prior_mean_u) v = np.empty((num_data, num_outputs)) for i in range(num_outputs): tmp = dtrmm(1.0, L[i].T, A, lower=0, trans_a=0) v[:, i] = np.sum(np.square(tmp), 0) v += (Knn_diag - np.sum(A * Kmn, 0))[:, None] # compute the KL term Kmmim = np.dot(Kmmi, q_u_mean) KLs = ( -0.5 * logdetS - 0.5 * num_inducing + 0.5 * logdetKmm + 0.5 * np.sum(Kmmi[None, :, :] * S, 1).sum(1) + 0.5 * np.sum(q_u_mean * Kmmim, 0) ) KL = KLs.sum() # gradient of the KL term (assuming zero mean function) dKL_dm = Kmmim.copy() dKL_dS = 0.5 * (Kmmi[None, :, :] - Si) dKL_dKmm = 0.5 * num_outputs * Kmmi - 0.5 * Kmmi.dot(S.sum(0)).dot(Kmmi) - 0.5 * Kmmim.dot(Kmmim.T) if mean_function is not None: # adjust KL term for mean function Kmmi_mfZ = np.dot(Kmmi, prior_mean_u) KL += -np.sum(q_u_mean * Kmmi_mfZ) KL += 0.5 * np.sum(Kmmi_mfZ * prior_mean_u) # adjust gradient for mean fucntion dKL_dm -= Kmmi_mfZ dKL_dKmm += Kmmim.dot(Kmmi_mfZ.T) dKL_dKmm -= 0.5 * Kmmi_mfZ.dot(Kmmi_mfZ.T) # compute gradients for mean_function dKL_dmfZ = Kmmi_mfZ - Kmmim # quadrature for the likelihood F, dF_dmu, dF_dv, dF_dthetaL = likelihood.variational_expectations(Y, mu, v, Y_metadata=Y_metadata) # rescale the F term if working on a batch F, dF_dmu, dF_dv = F * batch_scale, dF_dmu * batch_scale, dF_dv * batch_scale if dF_dthetaL is not None: dF_dthetaL = dF_dthetaL.sum(1).sum(1) * batch_scale # derivatives of expected likelihood, assuming zero mean function Adv = A[None, :, :] * dF_dv.T[:, None, :] # As if dF_Dv is diagonal, D, M, N Admu = A.dot(dF_dmu) Adv = np.ascontiguousarray(Adv) # makes for faster operations later...(inc dsymm) AdvA = np.dot(Adv.reshape(-1, num_data), A.T).reshape(num_outputs, num_inducing, num_inducing) tmp = np.sum([np.dot(a, s) for a, s in zip(AdvA, S)], 0).dot(Kmmi) dF_dKmm = -Admu.dot(Kmmim.T) + AdvA.sum(0) - tmp - tmp.T dF_dKmm = 0.5 * (dF_dKmm + dF_dKmm.T) # necessary? GPy bug? tmp = S.reshape(-1, num_inducing).dot(Kmmi).reshape(num_outputs, num_inducing, num_inducing) tmp = 2.0 * (tmp - np.eye(num_inducing)[None, :, :]) dF_dKmn = Kmmim.dot(dF_dmu.T) for a, b in zip(tmp, Adv): dF_dKmn += np.dot(a.T, b) dF_dm = Admu dF_dS = AdvA # adjust gradient to account for mean function if mean_function is not None: dF_dmfX = dF_dmu.copy() dF_dmfZ = -Admu dF_dKmn -= np.dot(Kmmi_mfZ, dF_dmu.T) dF_dKmm += Admu.dot(Kmmi_mfZ.T) # sum (gradients of) expected likelihood and KL part log_marginal = F.sum() - KL dL_dm, dL_dS, dL_dKmm, dL_dKmn = dF_dm - dKL_dm, dF_dS - dKL_dS, dF_dKmm - dKL_dKmm, dF_dKmn dL_dchol = 2.0 * np.array([np.dot(a, b) for a, b in zip(dL_dS, L)]) dL_dchol = choleskies.triang_to_flat(dL_dchol) grad_dict = { "dL_dKmm": dL_dKmm, "dL_dKmn": dL_dKmn, "dL_dKdiag": dF_dv.sum(1), "dL_dm": dL_dm, "dL_dchol": dL_dchol, "dL_dthetaL": dF_dthetaL, } if mean_function is not None: grad_dict["dL_dmfZ"] = dF_dmfZ - dKL_dmfZ grad_dict["dL_dmfX"] = dF_dmfX return Posterior(mean=q_u_mean, cov=S.T, K=Kmm, prior_mean=prior_mean_u), log_marginal, grad_dict
def local_likelihood(self, X, Y, l_svgps, g_q_u_mean, g_q_u_chol, g_kern, g_Z, g_mean_function=None): num_data, _ = Y.shape F = np.zeros([num_data, len(l_svgps)]) #global g_num_inducing, num_outputs = g_q_u_mean.shape #expand cholesky representation g_L = choleskies.flat_to_triang(g_q_u_chol) g_S = np.empty((num_outputs, g_num_inducing, g_num_inducing)) [ np.dot(g_L[i, :, :], g_L[i, :, :].T, g_S[i, :, :]) for i in range(num_outputs) ] if g_mean_function is not None: g_prior_mean_u = g_mean_function.f(g_Z) g_prior_mean_f = g_mean_function.f(X) else: g_prior_mean_u = np.zeros((g_num_inducing, num_outputs)) g_prior_mean_f = np.zeros((num_data, num_outputs)) g_Kmm = g_kern.K(g_Z) g_Kmn = g_kern.K(g_Z, X) #g_Knn_diag = g_kern.Kdiag(X) g_Lm = linalg.jitchol(g_Kmm) g_A, _ = linalg.dpotrs(g_Lm, g_Kmn) g_mu = g_prior_mean_f + np.dot(g_A.T, g_q_u_mean - g_prior_mean_u) g_v = np.empty((num_data, num_outputs)) for i in range(num_outputs): tmp = dtrmm(1.0, g_L[i].T, g_A, lower=0, trans_a=0) g_v[:, i] = np.sum(np.square(tmp), 0) #Anh v += (Knn_diag - np.sum(A*Kmn,0))[:,None] for i in range(len(l_svgps)): num_inducing, num_outputs = l_svgps[i].q_u_mean.shape #expand cholesky representation L = choleskies.flat_to_triang(l_svgps[i].q_u_chol) S = np.empty((num_outputs, num_inducing, num_inducing)) [ np.dot(L[i, :, :], L[i, :, :].T, S[i, :, :]) for i in range(num_outputs) ] prior_mean_u = np.zeros((num_inducing, num_outputs)) prior_mean_f = np.zeros((num_data, num_outputs)) #compute kernel related stuff Kmm = l_svgps[i].kern.K(l_svgps[i].Z) Kmn = l_svgps[i].kern.K(l_svgps[i].Z, X) Knn_diag = l_svgps[i].kern.Kdiag(X) Lm = linalg.jitchol(Kmm) #Kmmi, _ = linalg.dpotri(Lm) #compute the marginal means and variances of q(f) A, _ = linalg.dpotrs(Lm, Kmn) mu = prior_mean_f + np.dot(A.T, l_svgps[i].q_u_mean - prior_mean_u) v = np.empty((num_data, num_outputs)) for i in range(num_outputs): tmp = dtrmm(1.0, L[i].T, A, lower=0, trans_a=0) v[:, i] = np.sum(np.square(tmp), 0) v += (Knn_diag - np.sum(A * Kmn, 0))[:, None] #final marginal means and variances of q(f) mu += g_mu v += g_v #quadrature for the likelihood F[:, i, None], dF_dmu, dF_dv, dF_dthetaL = l_svgps[ i].likelihood.variational_expectations( Y, mu, v, Y_metadata=l_svgps[i].Y_metadata) return F
def local_inference_multithread(self, X_full, Y_full, rho_full, lsvgp, g_mu_full, g_v_full, KL_scale=1.0, batch_scale=1.0): index = np.where(rho_full > 0)[0] X = X_full[index, :] Y = Y_full[index, :] rho = rho_full[index, :] g_mu = g_mu_full[index, :] g_v = g_v_full[index, :] num_data, _ = Y.shape num_data_full, _ = Y_full.shape q_u_mean = lsvgp.q_u_mean num_inducing, num_outputs = q_u_mean.shape #expand cholesky representation L = choleskies.flat_to_triang(lsvgp.q_u_chol) S = np.empty((num_outputs, num_inducing, num_inducing)) [ np.dot(L[i, :, :], L[i, :, :].T, S[i, :, :]) for i in range(num_outputs) ] #Si,_ = linalg.dpotri(np.asfortranarray(L), lower=1) Si = choleskies.multiple_dpotri(L) logdetS = np.array([ 2. * np.sum(np.log(np.abs(np.diag(L[i, :, :])))) for i in range(L.shape[0]) ]) if np.any(np.isinf(Si)): raise ValueError("Cholesky representation unstable") prior_mean_u = np.zeros((num_inducing, num_outputs)) prior_mean_f = np.zeros((num_data, num_outputs)) #compute kernel related stuff Kmm = lsvgp.kern.K(lsvgp.Z) Kmn = lsvgp.kern.K(lsvgp.Z, X) Knn_diag = lsvgp.kern.Kdiag1(X) Lm = linalg.jitchol(Kmm) logdetKmm = 2. * np.sum(np.log(np.diag(Lm))) Kmmi, _ = linalg.dpotri(Lm) #compute the marginal means and variances of q(f) A, _ = linalg.dpotrs(Lm, Kmn) mu = prior_mean_f + np.dot(A.T, q_u_mean - prior_mean_u) v = np.empty((num_data, num_outputs)) for i in range(num_outputs): tmp = dtrmm(1.0, L[i].T, A, lower=0, trans_a=0) v[:, i] = np.sum(np.square(tmp), 0) v += (Knn_diag - np.sum(A * Kmn, 0))[:, None] #final marginal means and variances of q(f) mu += g_mu v += g_v #compute the KL term Kmmim = np.dot(Kmmi, q_u_mean) KLs = -0.5 * logdetS - 0.5 * num_inducing + 0.5 * logdetKmm + 0.5 * np.sum( Kmmi[None, :, :] * S, 1).sum(1) + 0.5 * np.sum( q_u_mean * Kmmim, 0) KL = KLs.sum() #gradient of the KL term (assuming zero mean function) dKL_dm = Kmmim.copy() dKL_dS = 0.5 * (Kmmi[None, :, :] - Si) dKL_dKmm = 0.5 * num_outputs * Kmmi - 0.5 * Kmmi.dot( S.sum(0)).dot(Kmmi) - 0.5 * Kmmim.dot(Kmmim.T) #quadrature for the likelihood F, dF_dmu, dF_dv, dF_dthetaL = lsvgp.likelihood.variational_expectations( Y, mu, v, Y_metadata=lsvgp.Y_metadata) #multiply with rho F, dF_dmu, dF_dv = F * rho, dF_dmu * rho, dF_dv * rho #rescale the F term if working on a batch F, dF_dmu, dF_dv = F * batch_scale, dF_dmu * batch_scale, dF_dv * batch_scale if dF_dthetaL is not None: dF_dthetaL = dF_dthetaL.sum(1).sum(1) * batch_scale #derivatives of expected likelihood w.r.t. local parameters, assuming zero mean function Adv = A[ None, :, :] * dF_dv.T[:, None, :] # As if dF_Dv is diagonal, D, M, N Admu = A.dot(dF_dmu) #Adv = np.ascontiguousarray(Adv) # makes for faster operations later...(inc dsymm) AdvA = np.dot(Adv.reshape(-1, num_data), A.T).reshape(num_outputs, num_inducing, num_inducing) tmp = np.sum([np.dot(a, s) for a, s in zip(AdvA, S)], 0).dot(Kmmi) dF_dKmm = -Admu.dot(Kmmim.T) + AdvA.sum(0) - tmp - tmp.T dF_dKmm = 0.5 * (dF_dKmm + dF_dKmm.T) # necessary? GPy bug? tmp = S.reshape(-1, num_inducing).dot(Kmmi).reshape( num_outputs, num_inducing, num_inducing) tmp = 2. * (tmp - np.eye(num_inducing)[None, :, :]) dF_dKmn = Kmmim.dot(dF_dmu.T) for a, b in zip(tmp, Adv): dF_dKmn += np.dot(a.T, b) dF_dm = Admu dF_dS = AdvA #sum (gradients of) expected likelihood and KL part w.r.t local parameters log_marginal = F.sum() - KL dL_dm, dL_dS, dL_dKmm, dL_dKmn = dF_dm - dKL_dm, dF_dS - dKL_dS, dF_dKmm - dKL_dKmm, dF_dKmn dL_dchol = 2. * np.array([np.dot(a, b) for a, b in zip(dL_dS, L)]) dL_dchol = choleskies.triang_to_flat(dL_dchol) grad_dict = { 'dL_dKmm': dL_dKmm, 'dL_dKmn': dL_dKmn, 'dL_dKdiag': dF_dv.sum(1), 'dL_dm': dL_dm, 'dL_dchol': dL_dchol, 'dL_dthetaL': dF_dthetaL } return Posterior(mean=q_u_mean, cov=S.T, K=Kmm, prior_mean=prior_mean_u ), log_marginal, grad_dict, index, dF_dmu, dF_dv