def slda_update_log_phi(text, log_phi, log_gamma, log_beta, y_d, eta, sigma_squared): """ Same as update_phi_lda_E_step but in log probability space. """ (N, K) = log_phi.shape log_phi_sum = logsumexp(log_phi, axis=0) Ns = (N * sigma_squared) ElogTheta = graphlib.dirichlet_expectation(np.exp(log_gamma)) front = (-1.0 / (2 * N * Ns)) pC = (1.0 * y_d / Ns * eta) eta_dot_eta = front * (eta * eta) log_const = np.log(ElogTheta + pC + eta_dot_eta) log_right_eta_times_const = np.log(front * 2 * eta) ensure(isinstance(text, np.ndarray)) # if text is in array form, do an approximate fast matrix update log_phi_minus_n = -1 + (logsumexp([log_phi, (-1 + log_phi_sum)])) log_phi[:,:] = logsumexp([log_beta[:,text].T, logdotexp(np.matrix(logdotexp(log_phi_minus_n, np.log(eta))).T, np.matrix(log_right_eta_times_const)), log_const,], axis=0) graphlib.log_row_normalize(log_phi) return log_phi
def lda_update_log_gamma(log_alpha, log_phi, log_gamma): """ Same as lda_update_gamma, but in log probability space. """ ensure(log_phi.shape[1] == len(log_gamma)) log_gamma[:] = logsumexp([log_alpha, logsumexp(log_phi, axis=0)], axis=0) return log_gamma
def test_log_row_normalize(): m = np.log(np.array([[2,2,4], [3,2,1]])) answer = np.log(np.array([[0.25, 0.25, 0.5], [0.5, 0.333333333333333, 0.166666666666667]])) assert abs(graphlib.logsumexp(m[0,:]) - np.log(8)) < .0000000001 assert abs(graphlib.logsumexp(m[1,:]) - np.log(6)) < .0000000001 assert abs(graphlib.logsumexp(answer[0,:])) < .0000000001 out = graphlib.log_row_normalize(m) assert same(out, answer)
def calculate_EZ_from_small_log_phis(log_phi1, log_phi2): """ Accepts a two small phi matrices (like (NdxK) and (NcxJ)) Calculates E[Zd]. Returns the final vector (K+J). E[Z] = φ := (1/N)ΣNφn """ Ndc = log_phi1.shape[0] + log_phi2.shape[0] ez = np.concatenate((logsumexp(log_phi1, axis=0), logsumexp(log_phi2, axis=0)), axis=1) return ez - np.log(Ndc)
def calculate_EZ_from_big_log_phi(big_log_phi): """ Accepts a big phi matrix (like ((Nd+Nc) x (K+J)) Calculates E[Zd]. Returns the final vector (K+J). E[Z] = φ := (1/N)ΣNφn """ Ndc,KJ = big_log_phi.shape return logsumexp(big_log_phi, axis=0) - np.log(Ndc)
def _unoptimized_slda_update_phi(text, phi, gamma, beta, y_d, eta, sigma_squared): """ Update phi in LDA. phi is N x K matrix. gamma is a K-size vector update phid: φd,n ∝ exp{ E[log θ|γ] + E[log p(wn|β1:K)] + (y / Nσ2) η — [2(ηTφd,-n)η + (η∘η)] / (2N2σ2) } Note that E[log p(wn|β1:K)] = log βTwn """ (N, K) = phi.shape #assert len(eta) == K #assert len(gamma) == K #assert beta.shape[0] == K phi_sum = np.sum(phi, axis=0) Ns = (N * sigma_squared) ElogTheta = graphlib.dirichlet_expectation(gamma) ensure(len(ElogTheta) == K) pC = (1.0 * y_d / Ns * eta) eta_dot_eta = (eta * eta) front = (-1.0 / (2 * N * Ns)) for n,word,count in iterwords(text): phi_sum -= phi[n] ensure(len(phi_sum) == K) pB = np.log(beta[:,word]) pD = (front * (((2 * np.dot(eta, phi_sum) * eta) + eta_dot_eta)) ) ensure(len(pB) == K) ensure(len(pC) == K) ensure(len(pD) == K) # must exponentiate and sum immediately! #phi[n,:] = np.exp(ElogTheta + pB + pC + pD) #phi[n,:] /= np.sum(phi[n,:]) # log normalize before exp for numerical stability phi[n,:] = ElogTheta + pB + pC + pD phi[n,:] -= graphlib.logsumexp(phi[n,:]) phi[n,:] = np.exp(phi[n,:]) # add this back into the sum # unlike in LDA, this cannot be computed in parallel phi_sum += phi[n] return phi
def partial_slda_update_phi(text, phi, gamma, beta, y_d, eta, sigma_squared): """Same as slda update phi, but eta may be smaller than total number of topics. So only some of the topics contribute to y. """ (N, K) = phi.shape Ks = len(eta) phi_sum = np.sum(phi[:,:Ks], axis=0) Ns = (N * sigma_squared) ElogTheta = graphlib.dirichlet_expectation(gamma) front = (-1.0 / (2 * N * Ns)) eta_dot_eta = front * (eta * eta) pC = ((1.0 * y_d / Ns) * eta) + eta_dot_eta right_eta_times_const = (front * 2 * eta) if isinstance(text, np.ndarray): # if text is in array form, do an approximate fast matrix update phi_minus_n = -(phi[:,:Ks] - phi_sum) phi[:,:] = ElogTheta + np.log(beta[:,text].T) phi[:,:Ks] += pC phi[:,:Ks] += np.dot(np.matrix(np.dot(phi_minus_n, eta)).T, np.matrix(right_eta_times_const)) graphlib.log_row_normalize(phi) phi[:,:] = np.exp(phi[:,:]) else: # otherwise, iterate through each word for n,word,count in iterwords(text): phi_sum -= phi[n,:Ks] pB = np.log(beta[:,word]) pD = (np.dot(eta, phi_sum) * right_eta_times_const) # must exponentiate and normalize immediately! phi[n,:] = ElogTheta + pB phi[n,:] += pC + pD phi[n,:] -= graphlib.logsumexp(phi[n,:]) # normalize in logspace phi[n,:] = np.exp(phi[n,:]) # add this back into the sum # unlike in LDA, this cannot be computed in parallel phi_sum += phi[n,:Ks] return phi
def calculate_EZZT_from_small_log_phis(phi1, phi2): """ Accepts a big phi matrix (like ((Nd+Nc) x (K+J)) Calculates E[ZdZdT]. Returns the final matrix ((K+J) x (K+J)). (Also, E[ZdZdT] = (1/N2)(ΣNΣm!=nφd,nφd,mT + ΣNdiag{φd,n}) """ Nd,K = phi1.shape Nc,J = phi2.shape (Ndc, KJ) = (Nd+Nc, K+J) inner_sum = np.zeros((KJ, KJ)) p1 = np.matrix(phi1) p2 = np.matrix(phi2) for i in xrange(K): for j in xrange(K): m = logdotexp(np.matrix(p1[:,i]), np.matrix(p1[:,j]).T) m += np.diagonal(np.ones(Nd) * -1000) inner_sum[i,j] = logsumexp(m.flatten()) for i in xrange(J): for j in xrange(J): m = logdotexp(np.matrix(p2[:,i]), np.matrix(p2[:,j]).T) m += np.diagonal(np.ones(Nc) * -1000) inner_sum[K+i,K+j] = logsumexp(m.flatten()) for i in xrange(K): for j in xrange(J): m = logdotexp(np.matrix(p1[:,i]), np.matrix(p2[:,j]).T) inner_sum[i,K+j] = logsumexp(m.flatten()) for i in xrange(J): for j in xrange(K): m = logdotexp(np.matrix(p2[:,i]), np.matrix(p1[:,j]).T) inner_sum[K+i,j] = logsumexp(m.flatten()) big_phi_sum = np.concatenate((logsumexp(phi1, axis=0), logsumexp(phi2, axis=0)), axis=1) ensure(big_phi_sum.shape == (KJ,)) for i in xrange(KJ): inner_sum[i,i] = logsumexp([inner_sum[i,i], big_phi_sum[i]]) inner_sum -= np.log(Ndc * Ndc) return inner_sum
def test_initialize_random(): original = np.ones((4,7)) out = original.copy() graphlib.initialize_random(out) assert original.shape == out.shape assert not same(out, original) sumrows = np.sum(out, axis=1) assert same(sumrows, np.ones(out.shape[0])) # now test log of the same original = np.ones((4,7)) out = original.copy() graphlib.initialize_log_random(out) assert original.shape == out.shape assert not same(out, original) sumrows = graphlib.logsumexp(out, axis=1) assert same(np.exp(sumrows), np.ones(out.shape[0]))