def resample_psi(self): Lmbda = np.linalg.inv(self.theta_prior.sigma) h = Lmbda.dot(self.theta_prior.mu) randvec = np.random.randn(self.D, self.T - 1) # pre-generate randomness for d, c in enumerate(self.doc_topic_counts): self.psi[d] = sample_infogaussian(Lmbda + np.diag(self.omega[d]), h + kappa_vec(c), randvec[d])
def _info_form_heldout_log_likelihood(self, X, M=10): """ We can analytically integrate out z (latent states) given omega. To estimate the heldout log likelihood of a data sequence, we Monte Carlo integrate over omega, where omega is drawn from the prior. :param data: :param M: number of Monte Carlo samples for integrating out omega :return: """ # assert len(self.data_list) == 1, "TODO: Support more than 1 data set" T, K = X.shape assert K == self.K kappa = kappa_vec(X) N = N_vec(X) # Compute the data-specific normalization constant from the # augmented multinomial distribution Z_mul = (gammaln(N + 1) - gammaln(X[:, :-1] + 1) - gammaln(N - X[:, :-1] + 1)).sum() Z_mul += (-N * np.log(2.)).sum() # Monte carlo integrate wrt omega ~ PG(N, 0) import pypolyagamma as ppg hlls = np.zeros(M) for m in range(M): # Sample omega using the emission distributions samplers omega = np.zeros(N.size) ppg.pgdrawvpar(self.emission_distn.ppgs, N.ravel(), np.zeros(N.size), omega) omega = omega.reshape((T, K - 1)) # Exactly integrate out the latent states z using message passing # The "data" is the normal potential from the states = MultinomialLDSStates(model=self, data=X) conditional_mean = kappa / np.clip( omega, 1e-64, np.inf) - self.emission_distn.mu[None, :] conditional_prec = np.zeros((T, K - 1, K - 1)) for t in range(T): conditional_prec[t, :, :] = np.diag(omega[t, :]) Z_lds = states.info_log_likelihood(conditional_mean, conditional_prec) # Sum them up to get the heldout log likelihood for this omega hlls[m] = Z_mul + Z_lds # Now take the log of the average to get the log likelihood hll = logsumexp(hlls) - np.log(M) # Use bootstrap to compute error bars samples = np.random.choice(hlls, size=(100, M), replace=True) hll_samples = logsumexp(samples, axis=1) - np.log(M) std_hll = hll_samples.std() return hll, std_hll
def _resample_X(): pis = model.pi(data) X = np.array([np.random.multinomial(N_max, pis[m]) for m in xrange(M)]) N = N_vec(X).astype(np.float) kappa = kappa_vec(X) data["X"] = X data["N"] = N data["kappa"] = kappa
def _info_form_heldout_log_likelihood(self, X, M=10): """ We can analytically integrate out z (latent states) given omega. To estimate the heldout log likelihood of a data sequence, we Monte Carlo integrate over omega, where omega is drawn from the prior. :param data: :param M: number of Monte Carlo samples for integrating out omega :return: """ # assert len(self.data_list) == 1, "TODO: Support more than 1 data set" T, K = X.shape assert K == self.K kappa = kappa_vec(X) N = N_vec(X) # Compute the data-specific normalization constant from the # augmented multinomial distribution Z_mul = (gammaln(N + 1) - gammaln(X[:,:-1]+1) - gammaln(N-X[:,:-1]+1)).sum() Z_mul += (-N * np.log(2.)).sum() # Monte carlo integrate wrt omega ~ PG(N, 0) import pypolyagamma as ppg hlls = np.zeros(M) for m in range(M): # Sample omega using the emission distributions samplers omega = np.zeros(N.size) ppg.pgdrawvpar(self.emission_distn.ppgs, N.ravel(), np.zeros(N.size), omega) omega = omega.reshape((T, K-1)) # Exactly integrate out the latent states z using message passing # The "data" is the normal potential from the states = MultinomialLDSStates(model=self, data=X) conditional_mean = kappa / np.clip(omega, 1e-64,np.inf) - self.emission_distn.mu[None, :] conditional_prec = np.zeros((T, K-1, K-1)) for t in range(T): conditional_prec[t,:,:] = np.diag(omega[t,:]) Z_lds = states.info_log_likelihood(conditional_mean, conditional_prec) # Sum them up to get the heldout log likelihood for this omega hlls[m] = Z_mul + Z_lds # Now take the log of the average to get the log likelihood hll = logsumexp(hlls) - np.log(M) # Use bootstrap to compute error bars samples = np.random.choice(hlls, size=(100, M), replace=True) hll_samples = logsumexp(samples, axis=1) - np.log(M) std_hll = hll_samples.std() return hll, std_hll
def _get_lds_effective_params(self): mu_uniform, sigma_uniform = compute_uniform_mean_psi(self.V) mu_init = np.tile(mu_uniform, self.K) sigma_init = np.tile(np.diag(sigma_uniform), self.K) sigma_states = np.repeat(self.sigmasq_states, (self.V - 1) * self.K) sigma_obs = 1.0 / self.omega y = kappa_vec(self.time_word_topic_counts, axis=1) / self.omega return mu_init, sigma_init, sigma_states, sigma_obs.reshape(y.shape[0], -1), y.reshape(y.shape[0], -1)
def _get_lds_effective_params(self): mu_uniform, sigma_uniform = compute_uniform_mean_psi(self.V) mu_init = np.tile(mu_uniform, self.K) sigma_init = np.tile(np.diag(sigma_uniform), self.K) sigma_states = np.repeat(self.sigmasq_states, (self.V - 1) * self.K) sigma_obs = 1. / self.omega y = kappa_vec(self.time_word_topic_counts, axis=1) / self.omega return mu_init, sigma_init, sigma_states, \ sigma_obs.reshape(y.shape[0], -1), y.reshape(y.shape[0], -1)
def augment_data(self, augmented_data): """ Augment the data with auxiliary variables :param augmented_data: :return: """ x = augmented_data["x"] T, K = x.shape assert K == self.K augmented_data["kappa"] = kappa_vec(x) augmented_data["omega"] = np.ones((T,K-1)) self.resample_omega([augmented_data]) return augmented_data
def conditional_psi(self, x): """ Compute the conditional distribution over psi given observation x and omega :param x: :return: """ assert x.ndim == 2 Omega = np.diag(self.omega) Sigma_cond = inv(Omega + inv(self.Sigma)) # kappa is the mean dot precision, i.e. the sufficient statistic of a Gaussian # therefore we can sum over datapoints kappa = kappa_vec(x).sum(0) mu_cond = Sigma_cond.dot(kappa + solve(self.Sigma, self.mu)) return mu_cond, Sigma_cond
def add_data(self, Z, X, fixed_kernel=True): # Z is the array of points where multinomial vectors are observed # X is the corresponding set of multinomial vectors assert Z.ndim == 2 and Z.shape[1] == self.D M = Z.shape[0] assert X.shape == (M, self.K), "X must be MxK" # Compute kappa and N for each of the m inputs N = N_vec(X).astype(np.float) kappa = kappa_vec(X) # Initialize the auxiliary variables omega = np.ones((M, self.K-1)) # Initialize a "sample" from psi psi = np.zeros((M, self.K-1)) # Precompute the kernel for the case where it is fixed if fixed_kernel: C = self.kernel.K(Z) C += 1e-6 * np.eye(M) C_inv = np.linalg.inv(C) else: C = None C_inv = None # Pack all this up into a dict augmented_data = \ { "X": X, "Z": Z, "M": M, "N": N, "C": C, "C_inv": C_inv, "kappa": kappa, "omega": omega, "psi": psi } self.data_list.append(augmented_data) return augmented_data