def expectation_step(self):
expected_pi_sample_cluster, expected_e_sample_cluster, likelihood_cluster_sample = self.pre_expectation_step()
likelihood_cluster_sample += self.w[:, np.newaxis]
tmp = logsumexp(likelihood_cluster_sample, 0, keepdims=True)
log_likelihood = np.sum(tmp)
likelihood_cluster_sample -= tmp
expected_pi_sample_cluster += likelihood_cluster_sample[:, :, np.newaxis]
expected_e_sample_cluster += likelihood_cluster_sample[:, :, np.newaxis, np.newaxis]
log_expected_pi = logsumexp(expected_pi_sample_cluster, 1)
log_expected_e = logsumexp(expected_e_sample_cluster, (0, 1))
expected_w = logsumexp(likelihood_cluster_sample, 1)
return expected_w, log_expected_pi, log_expected_e, log_likelihood
def maximization_step(self, log_expected_w, log_expected_pi, log_expected_e, params):
if 'w' in params:
w = log_expected_w - logsumexp(log_expected_w)
else:
w = self.w
if 'pi' in params:
pi = log_expected_pi - logsumexp(log_expected_pi, axis=1, keepdims=True)
else:
pi = self.pi
if 'e' in params:
e = log_expected_e - logsumexp(log_expected_e, axis=1, keepdims=True)
else:
e = self.e
return w, pi, e
def expected_topics(self, data):
self.set_data(data)
real_m, good_indices = self.remove_zero_prob_mutations()
self.pi = np.log(self.pi)
self.w = np.log(self.w)
self.e = np.log(self.e)
expected_pi_sample_cluster, expected_e_sample_cluster, likelihood_sample_cluster = self.pre_expectation_step()
likelihood_sample_cluster += self.w[:, np.newaxis]
likelihood_sample_cluster -= logsumexp(likelihood_sample_cluster, 0, keepdims=True)
expected_pi_sample_cluster += likelihood_sample_cluster[:, :, np.newaxis]
log_expected_topics = logsumexp(expected_pi_sample_cluster, 0)
self.pi = np.exp(self.pi)
self.e = np.exp(self.e)
self.w = np.exp(self.w)
self.insert_zero_prob_mutations(data, real_m, good_indices)
return np.exp(log_expected_topics)
def pre_expectation_step(self):
num_samples = self.data.shape[0]
num_clusters, num_topics, num_words = self.num_clusters, self.num_topics, self.num_words
log_likelihood = np.zeros((num_clusters, num_samples))
log_expected_e = np.zeros((num_clusters, num_samples, num_topics, num_words))
log_expected_pi = np.zeros((num_clusters, num_samples, num_topics))
log_e = self.e
for l, log_pi in enumerate(self.pi):
log_prob_topic_word = (log_e.T + log_pi).T
log_prob_word = logsumexp(log_prob_topic_word, axis=0)
log_likelihood[l] = np.dot(self.data, log_prob_word)
tmp = self.log_data - log_prob_word[np.newaxis, :]
log_expected_e[l] = log_prob_topic_word[np.newaxis, :, :] + tmp[:, np.newaxis, :]
log_expected_pi[l] = logsumexp(log_expected_e[l], axis=-1)
return log_expected_pi, log_expected_e, log_likelihood
def slow_pre_expectation_step(self):
num_samples = self.data.shape[0]
num_clusters, num_topics, num_words = self.num_clusters, self.num_topics, self.num_words
log_likelihood = np.zeros((num_clusters, num_samples))
log_expected_e = np.zeros((num_clusters, num_samples, num_topics, num_words))
log_expected_pi = np.zeros((num_clusters, num_samples, num_topics))
log_e = self.e
for l, log_pi in enumerate(self.pi):
log_prob_topic_word = (log_e.T + log_pi).T
for n in range(self.data.shape[0]):
curr_log_b = self.log_data[n]
curr_b = self.data[n]
log_prob_word = logsumexp(log_prob_topic_word, axis=0)
log_likelihood[l, n] = np.inner(log_prob_word, curr_b)
log_expected_e[l, n] = log_prob_topic_word + curr_log_b - log_prob_word
log_expected_pi[l, n] = logsumexp(log_expected_e[l, n], axis=1)
return log_expected_pi, log_expected_e, log_likelihood
def expectation_step(self, value_list):
log_likelihood = 0 if 'log_likelihood' in value_list else None
log_expected_e = np.log(np.zeros((self.k, self.m))) if 'e' in value_list else None
log_expected_pi = np.empty((self.n, self.k)) if 'pi' in value_list else None
log_e = self.e
for i in range(self.n):
log_data = self.log_data[i]
log_pi = self.pi[i]
log_prob_topic_word = (log_e.T + log_pi).T
log_prob_word = logsumexp(log_prob_topic_word, axis=0)
curr_log_expected_e = log_prob_topic_word + log_data - log_prob_word
if 'log_likelihood' in value_list:
log_likelihood += np.inner(log_prob_word, self.data[i])
if 'e' in value_list:
np.logaddexp(curr_log_expected_e, log_expected_e, log_expected_e)
if 'pi' in value_list:
log_expected_pi[i] = logsumexp(curr_log_expected_e, axis=1)
return log_expected_pi, log_expected_e, log_likelihood
def maximization_step(self, log_expected_pi=None, log_expected_e=None):
pi = log_expected_pi - logsumexp(log_expected_pi, axis=1, keepdims=True) if log_expected_pi is not None else self.pi
e = log_expected_e - logsumexp(log_expected_e, axis=1, keepdims=True) if log_expected_e is not None else self.e
return pi, e