def _expected_log_likelihood(self, x):
# Natural parameter of marginal log-distirbution
# are the expected statsitics of the posterior
nat_param = self.expected_statistics()
# Data statistics under a Gaussian likelihood
# log-parition is subsumed into nat*stats
liklihood = GaussianWithDiagonalPrecision(
mu=np.empty_like(nat_param[0]))
stats = liklihood.statistics(x, vectorize=True)
log_base = liklihood.log_base()
return log_base + np.einsum('k,nk->n', nat_param[0], stats[0])\
+ np.einsum('k,nk->n', nat_param[1], stats[1])\
+ np.einsum('k,nk->n', nat_param[2], stats[2])\
+ np.einsum('k,nk->n', nat_param[3], stats[3])
class GaussianWithNormalGamma:
"""
Multivariate Diagonal Gaussian distribution class.
Uses a Normal-Gamma prior and posterior
Parameters are mean and precision matrix:
mu, lmbdas
"""
def __init__(self, prior, likelihood=None):
# Normal-Gamma conjugate
self.prior = prior
# Normal-Gamma posterior
self.posterior = copy.deepcopy(prior)
# Gaussian likelihood
if likelihood is not None:
self.likelihood = likelihood
else:
mu, lmbdas = self.prior.rvs()
self.likelihood = GaussianWithDiagonalPrecision(mu=mu,
lmbdas=lmbdas)
def empirical_bayes(self, data):
self.prior.nat_param = self.likelihood.statistics(data)
self.likelihood.params = self.prior.rvs()
return self
# Max a posteriori
def max_aposteriori(self, data, weights=None):
stats = self.likelihood.statistics(data) if weights is None\
else self.likelihood.weighted_statistics(data, weights)
self.posterior.nat_param = self.prior.nat_param + stats
self.likelihood.params = self.posterior.mode(
) # mode of gamma might not exist
return self
# Gibbs sampling
def resample(self, data=[]):
stats = self.likelihood.statistics(data)
self.posterior.nat_param = self.prior.nat_param + stats
self.likelihood.params = self.posterior.rvs()
return self
# Mean field
def meanfield_update(self, data, weights=None):
stats = self.likelihood.statistics(data) if weights is None\
else self.likelihood.weighted_statistics(data, weights)
self.posterior.nat_param = self.prior.nat_param + stats
self.likelihood.params = self.posterior.rvs()
return self
def meanfield_sgdstep(self, data, weights, prob, stepsize):
stats = self.likelihood.statistics(data) if weights is None\
else self.likelihood.weighted_statistics(data, weights)
self.posterior.nat_param = (1. - stepsize) * self.posterior.nat_param\
+ stepsize * (self.prior.nat_param + 1. / prob * stats)
self.likelihood.params = self.posterior.rvs()
return self
def variational_lowerbound(self):
q_entropy = self.posterior.entropy()
qp_cross_entropy = self.prior.cross_entropy(self.posterior)
return q_entropy - qp_cross_entropy
def log_marginal_likelihood(self):
log_partition_prior = self.prior.log_partition()
log_partition_posterior = self.posterior.log_partition()
return log_partition_posterior - log_partition_prior
def posterior_predictive_gaussian(self):
mu, kappas, alphas, betas = self.posterior.params
c = 1. + 1. / kappas
lmbdas = (alphas / betas) * 1. / c
return mu, lmbdas
def log_posterior_predictive_gaussian(self, x):
mu, lmbdas = self.posterior_predictive_gaussian()
return GaussianWithDiagonalPrecision(mu=mu,
lmbdas=lmbdas).log_likelihood(x)
def posterior_predictive_studentt(self):
mu, kappas, alphas, betas = self.posterior.params
dfs = 2. * alphas
c = 1. + 1. / kappas
lmbdas = (alphas / betas) * 1. / c
return mu, lmbdas, dfs
def log_posterior_predictive_studentt(self, x):
mu, lmbdas, dfs = self.posterior_predictive_studentt()
log_posterior = 0.
for _x, _mu, _lmbda, _df in zip(x, mu, lmbdas, dfs):
log_posterior += mvt_logpdf(_x.reshape(-1, 1), _mu.reshape(-1, 1),
_lmbda.reshape(-1, 1, 1), _df)
return log_posterior