def _expected_log_likelihood(self, x): # Natural parameter of marginal log-distirbution # are the expected statsitics of the posterior nat_param = self.expected_statistics() # Data statistics under a Gaussian likelihood # log-parition is subsumed into nat*stats liklihood = GaussianWithDiagonalPrecision( mu=np.empty_like(nat_param[0])) stats = liklihood.statistics(x, vectorize=True) log_base = liklihood.log_base() return log_base + np.einsum('k,nk->n', nat_param[0], stats[0])\ + np.einsum('k,nk->n', nat_param[1], stats[1])\ + np.einsum('k,nk->n', nat_param[2], stats[2])\ + np.einsum('k,nk->n', nat_param[3], stats[3])
class GaussianWithNormalGamma: """ Multivariate Diagonal Gaussian distribution class. Uses a Normal-Gamma prior and posterior Parameters are mean and precision matrix: mu, lmbdas """ def __init__(self, prior, likelihood=None): # Normal-Gamma conjugate self.prior = prior # Normal-Gamma posterior self.posterior = copy.deepcopy(prior) # Gaussian likelihood if likelihood is not None: self.likelihood = likelihood else: mu, lmbdas = self.prior.rvs() self.likelihood = GaussianWithDiagonalPrecision(mu=mu, lmbdas=lmbdas) def empirical_bayes(self, data): self.prior.nat_param = self.likelihood.statistics(data) self.likelihood.params = self.prior.rvs() return self # Max a posteriori def max_aposteriori(self, data, weights=None): stats = self.likelihood.statistics(data) if weights is None\ else self.likelihood.weighted_statistics(data, weights) self.posterior.nat_param = self.prior.nat_param + stats self.likelihood.params = self.posterior.mode( ) # mode of gamma might not exist return self # Gibbs sampling def resample(self, data=[]): stats = self.likelihood.statistics(data) self.posterior.nat_param = self.prior.nat_param + stats self.likelihood.params = self.posterior.rvs() return self # Mean field def meanfield_update(self, data, weights=None): stats = self.likelihood.statistics(data) if weights is None\ else self.likelihood.weighted_statistics(data, weights) self.posterior.nat_param = self.prior.nat_param + stats self.likelihood.params = self.posterior.rvs() return self def meanfield_sgdstep(self, data, weights, prob, stepsize): stats = self.likelihood.statistics(data) if weights is None\ else self.likelihood.weighted_statistics(data, weights) self.posterior.nat_param = (1. - stepsize) * self.posterior.nat_param\ + stepsize * (self.prior.nat_param + 1. / prob * stats) self.likelihood.params = self.posterior.rvs() return self def variational_lowerbound(self): q_entropy = self.posterior.entropy() qp_cross_entropy = self.prior.cross_entropy(self.posterior) return q_entropy - qp_cross_entropy def log_marginal_likelihood(self): log_partition_prior = self.prior.log_partition() log_partition_posterior = self.posterior.log_partition() return log_partition_posterior - log_partition_prior def posterior_predictive_gaussian(self): mu, kappas, alphas, betas = self.posterior.params c = 1. + 1. / kappas lmbdas = (alphas / betas) * 1. / c return mu, lmbdas def log_posterior_predictive_gaussian(self, x): mu, lmbdas = self.posterior_predictive_gaussian() return GaussianWithDiagonalPrecision(mu=mu, lmbdas=lmbdas).log_likelihood(x) def posterior_predictive_studentt(self): mu, kappas, alphas, betas = self.posterior.params dfs = 2. * alphas c = 1. + 1. / kappas lmbdas = (alphas / betas) * 1. / c return mu, lmbdas, dfs def log_posterior_predictive_studentt(self, x): mu, lmbdas, dfs = self.posterior_predictive_studentt() log_posterior = 0. for _x, _mu, _lmbda, _df in zip(x, mu, lmbdas, dfs): log_posterior += mvt_logpdf(_x.reshape(-1, 1), _mu.reshape(-1, 1), _lmbda.reshape(-1, 1, 1), _df) return log_posterior