def doc_e_step(self, doc, ss, Elogsticks_1st, \ word_list, unique_words, var_converge, \ max_iter=100): """ e step for a single doc """ batchids = [unique_words[id] for id in doc.words] Elogbeta_doc = self.m_Elogbeta[:, doc.words] ## very similar to the hdp equations v = np.zeros((2, self.m_K-1)) v[0] = 1.0 v[1] = self.m_alpha # The following line is of no use. Elogsticks_2nd = expect_log_sticks(v) # back to the uniform phi = np.ones((len(doc.words), self.m_K)) * 1.0/self.m_K likelihood = 0.0 old_likelihood = -1e100 converge = 1.0 eps = 1e-100 iter = 0 # not yet support second level optimization yet, to be done in the future while iter < max_iter and (converge < 0.0 or converge > var_converge): ### update variational parameters # var_phi if iter < 3: var_phi = np.dot(phi.T, (Elogbeta_doc * doc.counts).T) (log_var_phi, log_norm) = utils.log_normalize(var_phi) var_phi = np.exp(log_var_phi) else: var_phi = np.dot(phi.T, (Elogbeta_doc * doc.counts).T) + Elogsticks_1st (log_var_phi, log_norm) = utils.log_normalize(var_phi) var_phi = np.exp(log_var_phi) # phi if iter < 3: phi = np.dot(var_phi, Elogbeta_doc).T (log_phi, log_norm) = utils.log_normalize(phi) phi = np.exp(log_phi) else: phi = np.dot(var_phi, Elogbeta_doc).T + Elogsticks_2nd (log_phi, log_norm) = utils.log_normalize(phi) phi = np.exp(log_phi) # v phi_all = phi * np.array(doc.counts)[:,np.newaxis] v[0] = 1.0 + np.sum(phi_all[:,:self.m_K-1], 0) phi_cum = np.flipud(np.sum(phi_all[:,1:], 0)) v[1] = self.m_alpha + np.flipud(np.cumsum(phi_cum)) Elogsticks_2nd = expect_log_sticks(v) likelihood = 0.0 # compute likelihood # var_phi part/ C in john's notation likelihood += np.sum((Elogsticks_1st - log_var_phi) * var_phi) # v part/ v in john's notation, john's beta is alpha here log_alpha = np.log(self.m_alpha) likelihood += (self.m_K-1) * log_alpha dig_sum = sp.psi(np.sum(v, 0)) likelihood += np.sum((np.array([1.0, self.m_alpha])[:,np.newaxis]-v) * (sp.psi(v)-dig_sum)) likelihood -= np.sum(sp.gammaln(np.sum(v, 0))) - np.sum(sp.gammaln(v)) # Z part likelihood += np.sum((Elogsticks_2nd - log_phi) * phi) # X part, the data part likelihood += np.sum(phi.T * np.dot(var_phi, Elogbeta_doc * doc.counts)) converge = (likelihood - old_likelihood)/abs(old_likelihood) old_likelihood = likelihood if converge < -0.000001: print "warning, likelihood is decreasing!" iter += 1 # update the suff_stat ss # this time it only contains information from one doc ss.m_var_sticks_ss += np.sum(var_phi, 0) ss.m_var_beta_ss[:, batchids] += np.dot(var_phi.T, phi.T * doc.counts) return(likelihood)
def doc_e_step(self, doc, ss, Elogbeta, Elogsticks_1st, var_converge, fresh=False): Elogbeta_doc = Elogbeta[:, doc.words] v = np.zeros((2, self.m_K-1)) phi = np.ones((doc.length, self.m_K)) * 1.0/self.m_K # the following line is of no use Elogsticks_2nd = expect_log_sticks(v) likelihood = 0.0 old_likelihood = -1e1000 converge = 1.0 eps = 1e-100 iter = 0 max_iter = 100 #(TODO): support second level optimization in the future while iter < max_iter and (converge < 0.0 or converge > var_converge): ### update variational parameters # var_phi if iter < 3 and fresh: var_phi = np.dot(phi.T, (Elogbeta_doc * doc.counts).T) (log_var_phi, log_norm) = utils.log_normalize(var_phi) var_phi = np.exp(log_var_phi) else: var_phi = np.dot(phi.T, (Elogbeta_doc * doc.counts).T) + Elogsticks_1st (log_var_phi, log_norm) = utils.log_normalize(var_phi) var_phi = np.exp(log_var_phi) # phi if iter < 3: phi = np.dot(var_phi, Elogbeta_doc).T (log_phi, log_norm) = utils.log_normalize(phi) phi = np.exp(log_phi) else: phi = np.dot(var_phi, Elogbeta_doc).T + Elogsticks_2nd (log_phi, log_norm) = utils.log_normalize(phi) phi = np.exp(log_phi) # v phi_all = phi * np.array(doc.counts)[:,np.newaxis] v[0] = 1.0 + np.sum(phi_all[:,:self.m_K-1], 0) phi_cum = np.flipud(np.sum(phi_all[:,1:], 0)) v[1] = self.m_alpha + np.flipud(np.cumsum(phi_cum)) Elogsticks_2nd = expect_log_sticks(v) likelihood = 0.0 # compute likelihood # var_phi part/ C in john's notation likelihood += np.sum((Elogsticks_1st - log_var_phi) * var_phi) # v part/ v in john's notation, john's beta is alpha here log_alpha = np.log(self.m_alpha) likelihood += (self.m_K-1) * log_alpha dig_sum = sp.psi(np.sum(v, 0)) likelihood += np.sum((np.array([1.0, self.m_alpha])[:,np.newaxis]-v) * (sp.psi(v)-dig_sum)) likelihood -= np.sum(sp.gammaln(np.sum(v, 0))) - np.sum(sp.gammaln(v)) # Z part likelihood += np.sum((Elogsticks_2nd - log_phi) * phi) # X part, the data part likelihood += np.sum(phi.T * np.dot(var_phi, Elogbeta_doc * doc.counts)) converge = (likelihood - old_likelihood)/abs(old_likelihood) old_likelihood = likelihood if converge < 0: print "warning, likelihood is decreasing!" iter += 1 # update the suff_stat ss ss.m_var_sticks_ss += np.sum(var_phi, 0) ss.m_var_beta_ss[:, doc.words] += np.dot(var_phi.T, phi.T * doc.counts) return(likelihood)
def doc_e_step(self, doc, ss, Elogbeta, Elogsticks_1st, var_converge, fresh=False): Elogbeta_doc = Elogbeta[:, doc.words] v = np.zeros((2, self.m_K - 1)) phi = np.ones((doc.length, self.m_K)) * 1.0 / self.m_K # the following line is of no use Elogsticks_2nd = expect_log_sticks(v) likelihood = 0.0 old_likelihood = -1e1000 converge = 1.0 eps = 1e-100 iter = 0 max_iter = 100 #(TODO): support second level optimization in the future while iter < max_iter and (converge < 0.0 or converge > var_converge): ### update variational parameters # var_phi if iter < 3 and fresh: var_phi = np.dot(phi.T, (Elogbeta_doc * doc.counts).T) (log_var_phi, log_norm) = utils.log_normalize(var_phi) var_phi = np.exp(log_var_phi) else: var_phi = np.dot( phi.T, (Elogbeta_doc * doc.counts).T) + Elogsticks_1st (log_var_phi, log_norm) = utils.log_normalize(var_phi) var_phi = np.exp(log_var_phi) # phi if iter < 3: phi = np.dot(var_phi, Elogbeta_doc).T (log_phi, log_norm) = utils.log_normalize(phi) phi = np.exp(log_phi) else: phi = np.dot(var_phi, Elogbeta_doc).T + Elogsticks_2nd (log_phi, log_norm) = utils.log_normalize(phi) phi = np.exp(log_phi) # v phi_all = phi * np.array(doc.counts)[:, np.newaxis] v[0] = 1.0 + np.sum(phi_all[:, :self.m_K - 1], 0) phi_cum = np.flipud(np.sum(phi_all[:, 1:], 0)) v[1] = self.m_alpha + np.flipud(np.cumsum(phi_cum)) Elogsticks_2nd = expect_log_sticks(v) likelihood = 0.0 # compute likelihood # var_phi part/ C in john's notation likelihood += np.sum((Elogsticks_1st - log_var_phi) * var_phi) # v part/ v in john's notation, john's beta is alpha here log_alpha = np.log(self.m_alpha) likelihood += (self.m_K - 1) * log_alpha dig_sum = sp.psi(np.sum(v, 0)) likelihood += np.sum( (np.array([1.0, self.m_alpha])[:, np.newaxis] - v) * (sp.psi(v) - dig_sum)) likelihood -= np.sum(sp.gammaln(np.sum(v, 0))) - np.sum( sp.gammaln(v)) # Z part likelihood += np.sum((Elogsticks_2nd - log_phi) * phi) # X part, the data part likelihood += np.sum(phi.T * np.dot(var_phi, Elogbeta_doc * doc.counts)) converge = (likelihood - old_likelihood) / abs(old_likelihood) old_likelihood = likelihood if converge < 0: print "warning, likelihood is decreasing!" iter += 1 # update the suff_stat ss ss.m_var_sticks_ss += np.sum(var_phi, 0) ss.m_var_beta_ss[:, doc.words] += np.dot(var_phi.T, phi.T * doc.counts) return (likelihood)