class TestGaussianClass(unittest.TestCase):
def setUp(self):
self.gaussian = Gaussian('numbers_gaussian.txt')
def test_readdata(self):
self.assertEqual(self.gaussian.data[0:7], [36, 37, 38, 38, 39, 39, 40],
'data not read in correctly')
def test_meancalculation(self):
self.assertEqual(round(self.gaussian.calculate_mean(),2),
round(sum(self.gaussian.data) / float(len(self.gaussian.data)),2),\
'calculated mean not as expected')
def test_stdevcalculation(self):
self.assertEqual(round(self.gaussian.calculate_stdev(), 2), 3.92,\
'sample standard deviation incorrect')
self.assertEqual(round(self.gaussian.calculate_stdev(0), 2), 3.90,\
'population standard deviation incorrect')
def test_pdf(self):
self.assertEqual(round(self.gaussian.pdf(40), 5), 0.03803,\
'propability density function does not give expected result')
def test_def(self):
self.assertEqual(
str(self.gaussian), "mean {}, variance {}".format(45.5, 15.36),
'parameters of the Gaussian\
not as expected')
def test_add(self):
gaussian_one = Gaussian(25, 3)
gaussian_two = Gaussian(30, 4)
gaussian_sum = gaussian_one + gaussian_two
self.assertEqual(gaussian_sum.mean, 55)
self.assertEqual(gaussian_sum.stdev, 5)
def set_params(self):
self.params = OrderedDict()
if self.prior is None:
self.prior = Gaussian(self.dim_h)
if self.posterior is None:
self.posterior = MLP(self.dim_in,
self.dim_h,
dim_hs=[],
rng=self.rng,
trng=self.trng,
h_act='T.nnet.sigmoid',
distribution='gaussian')
if self.conditional is None:
self.conditional = MLP(self.dim_h,
self.dim_in,
dim_hs=[],
rng=self.rng,
trng=self.trng,
h_act='T.nnet.sigmoid',
distribution='binomial')
self.posterior.name = self.name + '_posterior'
self.conditional.name = self.name + '_conditional'
def test_pdf(self):
self.gaussian = Gaussian(25, 2)
self.assertEqual(round(self.gaussian.pdf(25), 5), 0.19947,\
'pdf function does not give expected result')
self.gaussian.read_data_file('numbers.txt')
self.assertEqual(round(self.gaussian.pdf(75), 5), 0.00429,\
'pdf function after calculating mean and stdev does not give expected result')
def __init__(self,img_params,model_params,latent_params):
super(CatVAE, self).__init__()
image_dim = img_params['image_dim']
image_size = img_params['image_size']
n_downsample = model_params['n_downsample']
dim = model_params['dim']
n_res = model_params['n_res']
norm = model_params['norm']
activ = model_params['activ']
pad_type = model_params['pad_type']
n_mlp = model_params['n_mlp']
mlp_dim = model_params['mlp_dim']
self.continious_dim = latent_params['continious']
self.prior_cont = Gaussian(self.continious_dim)
self.categorical_dim = latent_params['categorical']
self.prior_catg = Categorical(self.categorical_dim)
self.gumbel = Gumbel(self.categorical_dim)
self.encoder = CatEncoder(n_downsample,n_res,n_mlp,image_size,image_dim,dim,mlp_dim,
latent_params,norm,activ,pad_type)
conv_inp_size = image_size // (2**n_downsample)
decoder_inp_dim = self.continious_dim + self.categorical_dim
self.decoder = Decoder(n_downsample,n_res,n_mlp,decoder_inp_dim,mlp_dim,conv_inp_size,
dim,image_dim,norm,activ,pad_type)
def elbo(self, input, target, samples=20):
outputs = []
log_priors = torch.zeros(samples)
log_variational_posteriors = torch.zeros(samples)
# draw n_samples from the posterior (run n_samples forward passes)
for i in range(samples):
outputs.append(self(input, sample=True))
log_priors[i] = self.log_prior()
log_variational_posteriors[i] = self.log_variational_posterior()
log_prior = log_priors.sum()
log_variational_posterior = log_variational_posteriors.sum()
outputs = torch.stack(outputs)
y_dist = Gaussian(outputs.mean(0), self.noise)
# negative_log_likelihood = self.loss_function(
# outputs.mean(0), target, reduction='sum')
negative_log_likelihood = -y_dist.log_prob(target) / len(input)
# loss = nll + kl
loss = negative_log_likelihood
kl = (log_variational_posterior - log_prior) / self.n_training
loss += kl
return loss, log_prior, log_variational_posterior, negative_log_likelihood
def propose_merge(self, i, j, component_weights):
target = j if np.argmax([component_weights[i], component_weights[j]
]) else i
source = i if target == j else j
proposal_means = self.currentmeans.copy()
proposal_means[target] = (
component_weights[target] * proposal_means[target] +
component_weights[source] * proposal_means[source]) / (
component_weights[target] + component_weights[source])
proposal_means = np.delete(proposal_means, source)
proposal_auxiliary = self.auxiliary.copy()
proposal_auxiliary[:, target] += proposal_auxiliary[:, source]
proposal_auxiliary = np.delete(proposal_auxiliary, source, axis=1)
log_MH = 0
for seg in range(self.n_segments):
accept_top = Gaussian(
np.dot(proposal_auxiliary[seg], proposal_means),
np.sum(proposal_auxiliary[seg]) / self.currentprecision).pdf(
self.jumps[seg])
accept_bot = Gaussian(
np.dot(self.auxiliary[seg], self.currentmeans),
np.sum(self.auxiliary[seg]) / self.currentprecision).pdf(
self.jumps[seg])
log_MH += np.log(accept_top) - np.log(accept_bot)
print(log_MH)
if np.random.rand() < np.exp(log_MH):
self.merge(target, source, proposal_means)
class VAE(nn.Module):
def __init__(self,img_params,model_params,latent_params):
super(VAE, self).__init__()
image_dim = img_params['image_dim']
image_size = img_params['image_size']
n_downsample = model_params['n_downsample']
dim = model_params['dim']
n_res = model_params['n_res']
norm = model_params['norm']
activ = model_params['activ']
pad_type = model_params['pad_type']
n_mlp = model_params['n_mlp']
mlp_dim = model_params['mlp_dim']
self.latent_dim = latent_params['latent_dim']
self.prior = Gaussian(self.latent_dim)
self.encoder = Encoder(n_downsample,n_res,n_mlp,image_size,image_dim,dim,mlp_dim,
self.latent_dim,norm,activ,pad_type)
conv_inp_size = image_size // (2**n_downsample)
self.decoder = Decoder(n_downsample,n_res,n_mlp,self.latent_dim,mlp_dim,conv_inp_size,
dim,image_dim,norm,activ,pad_type)
def forward(self,x):
latent_distr = self.encoder(x)
latent_distr = self.prior.activate(latent_distr)
samples = self.prior.sample(latent_distr)
return self.decoder(samples),latent_distr,samples
def test_params_init(self):
np.random.seed(1)
m0 = np.random.multivariate_normal(np.zeros(2), 4*np.eye(2))
prm0f = np.concatenate((m0, np.array([1,0,0,1])))
np.random.seed(2)
m0 = np.random.multivariate_normal(np.zeros(2), 4*np.eye(2))
prm0t = np.concatenate((m0, np.zeros(2)))
np.random.seed(4)
mu = np.array([[0,1,2,3],[0,-1,-2,-3]])
k = np.random.choice(2, p=np.array([0.5,0.5]))
lsig = np.array([[4,5,6,7],[-4,-5,-6,-7]])
mu0 = mu[k]+np.random.randn(4)*np.sqrt(10)*np.exp(lsig[k,:])
LSig = np.random.randn(4)+lsig[k]
prm4t = np.hstack((mu0, LSig))
g2t = Gaussian(2, True)
np.random.seed(1)
par0f = self.g2f.params_init(np.empty((0,6)), np.empty((0,1)), 4)
np.random.seed(2)
par0t = g2t.params_init(np.empty((0,6)), np.empty((0,1)), 4)
np.random.seed(4)
par4t = self.g4t.params_init(self.param4t, np.array([0.1, 0.9]), 10)
self.assertTrue(np.all(np.round(par0f,3)==np.round(prm0f,3)))
self.assertTrue(np.all(np.round(par0t,3)==np.round(prm0t,3)))
self.assertTrue(np.all(np.round(par4t,3)==np.round(prm4t,3)))
def log_likelihood(self):
# evaluate the likelihood of the labelling (which is,
# conveniently, just the likelihood of the current mixture
# model)
# FIXME: This should really be cached for the last invocation
score = Counter()
for c_idx, cluster in self._cluster_to_datum.iteritems():
if not cluster: continue
# Evaluate the likelihood of each individual cluster
cluster_size = len(cluster)
# The mean of the data points belonging to this cluster
cluster_datum_mean = sum(cluster) / cluster_size
# p(c)
score += Gaussian.log_prob(cluster_datum_mean, self._prior_mean,
self._prior_precision)
# p(x|c)
score += sum(
Gaussian.log_prob(datum, cluster_datum_mean,
self._cluster_precision)
for datum in cluster)
# score => p(x, c)
# for the gaussian the dimensions are independent so we should
# just be able to combine them directly
return score.total_count()
def update_segments(self, it):
"""
Samples the auxiliary variable using a Metropolis-Hastings step
conditional on the updated parameter values.
:param it: theh MCMC iteration number.
:return: the average number of proposal acceptances over each segment.
"""
accept_count = 0
zerotrun_pois = (ZeroTruncatedPoission(
rate=self.rate * self.delta).sample(self.n_segments))
for seg in range(self.n_segments):
prop = zerotrun_pois[seg]
prop_components = np.random.multinomial(
prop,
self.currentmixing_coeffs / np.sum(self.currentmixing_coeffs))
accept_top = Gaussian(np.dot(prop_components, self.currentmeans),
prop / self.currentprecision).pdf(
self.jumps[seg])
accept_bot = Gaussian(
np.dot(self.auxiliary[seg], self.currentmeans),
np.sum(self.auxiliary[seg]) / self.currentprecision).pdf(
self.jumps[seg])
if np.random.rand() < accept_top / accept_bot:
accept_count += 1
self.auxiliary[seg] = prop_components
return accept_count / self.n_segments
def __init__(self, in_features, out_features):
super().__init__()
self.in_features = in_features
self.out_features = out_features
# Weight parameters
self.weight_mu = nn.Parameter(
torch.Tensor(out_features, in_features).uniform_(-0.2, 0.2))
self.weight_rho = nn.Parameter(
torch.Tensor(out_features, in_features).uniform_(1e-1, 2))
# variational posterior for the weights
self.weight = Gaussian(self.weight_mu, self.weight_rho)
# Bias parameters
self.bias_mu = nn.Parameter(
torch.Tensor(out_features).uniform_(-0.2, 0.2))
self.bias_rho = nn.Parameter(
torch.Tensor(out_features).uniform_(1e-1, 2))
# variational posterior for the bias
self.bias = Gaussian(self.bias_mu, self.bias_rho)
# Prior distributions
self.weight_prior = Gaussian(torch.Tensor([0.]), torch.Tensor([1.]))
self.bias_prior = Gaussian(torch.Tensor([0.]), torch.Tensor([1.]))
# initialize log_prior and log_posterior as 0
self.log_prior = 0
self.log_variational_posterior = 0
class TestGaussianClass(unittest.TestCase):
def setUp(self):
self.gaussian = Gaussian(25, 2)
def test_initialization(self):
self.assertEqual(self.gaussian.mean, 25, 'incorrect mean')
self.assertEqual(self.gaussian.stdev, 2,
'incorrect standard deviation')
def test_pdf(self):
self.assertEqual(round(self.gaussian.pdf(25), 5), 0.19947,
'pdf function does not give expected result')
def test_meancalculation(self):
self.gaussian.read_data_file('numbers.txt', True)
self.assertEqual(
self.gaussian.calculate_mean(),
sum(self.gaussian.data) / float(len(self.gaussian.data)),
'calculated mean not as expected')
def test_stdevcalculation(self):
self.gaussian.read_data_file('numbers.txt', True)
self.assertEqual(round(self.gaussian.stdev, 2), 92.87,
'sample standard deviation incorrect')
self.gaussian.read_data_file('numbers.txt', False)
self.assertEqual(round(self.gaussian.stdev, 2), 88.55,
'population standard deviation incorrect')
def unpack(dim_in=None,
dim_h=None,
prior=None,
recognition_net=None,
generation_net=None,
extra_args=dict(),
distributions=dict(),
dims=dict(),
dataset_args=dict(),
center_input=None,
**model_args):
print 'Unpacking model with parameters %s' % model_args.keys()
print 'Forming Gaussian prior model'
prior_model = Gaussian(dim_h)
models = []
print 'Forming MLPs'
kwargs = GBN.mlp_factory(dim_h,
dims,
distributions,
recognition_net=recognition_net,
generation_net=generation_net)
kwargs['prior'] = prior_model
models.append(prior_model)
print 'Forming GBN'
model = GBN(dim_in, dim_h, **kwargs)
models.append(model)
models += [model.posterior, model.conditional]
return models, model_args, extra_args
def __init__(
self,
ensemble_size,
in_dim,
out_dim,
encoder_hidden_dim,
decoder_hidden_dim,
latent_dim,
n_hidden,
learn_rate=0.0001,
):
super(DynamicsEnsemble, self).__init__()
self.ensemble_size = ensemble_size
self.encoder = Encoder(in_dim,
encoder_hidden_dim,
latent_dim,
n_hidden=n_hidden)
self.decoders = self.build_decoder_ensemble(out_dim,
decoder_hidden_dim,
latent_dim, n_hidden)
self.opt = optim.Adam(self.parameters(), lr=learn_rate)
self.gaussian = Gaussian(latent_dim)
self.next_obs = None
self.reward = None
def _cluster_log_probs(self, cluster, cluster_size, cluster_mean, cluster_covariance, new_point): """ Return the posterior, prior, and likelihood of new_point being in the cluster of size cluster_size centered at cluster_mean """ # the updated mean new_mean = (cluster_mean * cluster_size + new_point) / (cluster_size + 1) posterior_precision = self._prior_precision + self._cluster_precision # convex combination for mean posterior_mean = self._prior_mean * self._prior_precision posterior_mean += cluster_mean * self._cluster_precision posterior_mean /= posterior_precision posterior = Gaussian.log_prob(new_mean, posterior_mean, posterior_precision) # prior is keyed on the (potentially) updated params prior = Gaussian.log_prob(new_mean, self._prior_mean, self._prior_precision) likelihood = Gaussian.log_prob(new_point, new_mean, self._cluster_precision) return posterior, prior, likelihood
def setUp(self):
self.g1f = Gaussian(1, False)
self.g2f = Gaussian(2, False)
self.g4t = Gaussian(4, True)
self.param1f = np.array([0,2])
self.param2f = np.array([np.arange(6), 2*np.arange(6)])
self.param4t = np.array([np.arange(8), -np.arange(8)])
def __init__(self,img_params,model_params,latent_params):
super(VAE, self).__init__()
image_dim = img_params['image_dim']
image_size = img_params['image_size']
n_downsample = model_params['n_downsample']
dim = model_params['dim']
n_res = model_params['n_res']
norm = model_params['norm']
activ = model_params['activ']
pad_type = model_params['pad_type']
n_mlp = model_params['n_mlp']
mlp_dim = model_params['mlp_dim']
self.latent_dim = latent_params['latent_dim']
self.prior = Gaussian(self.latent_dim)
self.encoder = Encoder(n_downsample,n_res,n_mlp,image_size,image_dim,dim,mlp_dim,
self.latent_dim,norm,activ,pad_type)
conv_inp_size = image_size // (2**n_downsample)
self.decoder = Decoder(n_downsample,n_res,n_mlp,self.latent_dim,mlp_dim,conv_inp_size,
dim,image_dim,norm,activ,pad_type)
def CollapsedGibbsSampling(model, num_iterations):
"""Performs a specified number of iterations of the Collapsed Gibbs Sampling algorithm using the model specified."""
# For each iteration
for iter_id in range(num_iterations):
print model.cluster_count
print model.cluster_popn
print model.data
print model.membership
# For each data point in the model
for data_id in range(len(model.data)):
# Remove data point from its current cluster
data_point = model.data[data_id]
cluster_id = model.membership[data_id]
model.cluster_popn[cluster_id] -= 1
model.clusters[cluster_id].rem_data(data_point)
# If the cluster is now empty delete it
if model.cluster_popn[cluster_id] == 0:
model.cluster_count -= 1
del model.clusters[cluster_id]
del model.cluster_popn[cluster_id]
tmp_idx = np.where(model.membership > cluster_id)
model.membership[tmp_idx] -= 1
# Generate numpy array of relative probabilities based on prior for membership and current datapoints in each cluster
p = np.asarray(model.conditional_prob(data_point))
# Check that p is normalised
p = p / np.sum(p)
# Generate cumulative probability mass function
p = np.cumsum(p)
rand_val = np.random.random()
# Select new cluster based on probabilities
new_cluster_id = np.sum(np.greater(rand_val, p))
# If a new cluster was chosen then create it. For convenience an empty cluster will always be at the end of the list. Thus we only have to create a copy of this cluster for future iterations, without storing the prior.
if new_cluster_id == model.cluster_count:
model.cluster_count += 1
model.clusters.append(Gaussian(np.asarray([]), model.prior))
model.cluster_popn.append(0)
# Add data point to its new cluster
model.membership[data_id] = new_cluster_id
model.cluster_popn[new_cluster_id] += 1
model.clusters[new_cluster_id].add_data(data_point)
def log_likelihood(self): # evaluate the likelihood of the labelling (which is, # conveniently, just the likelihood of the current mixture # model) # FIXME: This should really be cached for the last invocation score = Counter() for c_idx, cluster in self._cluster_to_datum.iteritems(): if not cluster: continue # Evaluate the likelihood of each individual cluster cluster_size = len(cluster) # The mean of the data points belonging to this cluster cluster_datum_mean = sum(cluster) / cluster_size # p(c) score += Gaussian.log_prob(cluster_datum_mean, self._prior_mean, self._prior_precision) # p(x|c) score += sum(Gaussian.log_prob(datum, cluster_datum_mean, self._cluster_precision) for datum in cluster) # score => p(x, c) # for the gaussian the dimensions are independent so we should # just be able to combine them directly return score.total_count()
def _cluster_log_probs(self, cluster, cluster_size, cluster_mean,
cluster_covariance, new_point):
""" Return the posterior, prior, and likelihood of new_point
being in the cluster of size cluster_size centered at cluster_mean
"""
# the updated mean
new_mean = (cluster_mean * cluster_size + new_point) / (cluster_size +
1)
posterior_precision = self._prior_precision + self._cluster_precision
# convex combination for mean
posterior_mean = self._prior_mean * self._prior_precision
posterior_mean += cluster_mean * self._cluster_precision
posterior_mean /= posterior_precision
posterior = Gaussian.log_prob(new_mean, posterior_mean,
posterior_precision)
# prior is keyed on the (potentially) updated params
prior = Gaussian.log_prob(new_mean, self._prior_mean,
self._prior_precision)
likelihood = Gaussian.log_prob(new_point, new_mean,
self._cluster_precision)
return posterior, prior, likelihood
class TestGaussianClass(unittest.TestCase):
def setUp(self):
self.gaussian = Gaussian(25, 2)
self.gaussian.read_file('numbers.txt')
def test_initialization(self):
self.assertEqual(self.gaussian.mean, 25, 'incorrect mean')
self.assertEqual(self.gaussian.stdev, 2,
'incorrect standard deviation')
def test_readdata(self):
self.assertEqual(self.gaussian.data,\
[1, 3, 99, 100, 120, 32, 330, 23, 76, 44, 31], 'data not read in correctly')
def test_meancalculation(self):
self.assertEqual(self.gaussian.calculate_mean(), 78.0909090909091,
'calculated mean not as expected')
def test_stdevcalculation(self):
self.assertEqual(round(self.gaussian.calculate_stdev(True), 2), 92.87,
'sample standard deviation incorrect')
self.assertEqual(round(self.gaussian.calculate_stdev(0), 2), 88.55,
'population standard deviation incorrect')
def test_pdf(self):
self.assertEqual(round(self.gaussian.density(25), 5), 0.19947,\
'pdf function does not give expected result')
self.gaussian.replace_stats_with_data()
self.assertEqual(round(self.gaussian.density(75), 5), 0.00429,\
'pdf function after calculating mean and stdev does not give expected result')
def test_add(self):
gaussian_one = Gaussian(25, 3)
gaussian_two = Gaussian(30, 4)
gaussian_sum = gaussian_one + gaussian_two
self.assertEqual(gaussian_sum.mean, 55)
self.assertEqual(gaussian_sum.stdev, 5)
class BLinear(nn.Module):
"""Bayesian Linear layer, default prior is Gaussian for weights and bias"""
def __init__(self, in_features, out_features):
super().__init__()
self.in_features = in_features
self.out_features = out_features
# Weight parameters
self.weight_mu = nn.Parameter(
torch.Tensor(out_features, in_features).uniform_(-0.2, 0.2))
self.weight_rho = nn.Parameter(
torch.Tensor(out_features, in_features).uniform_(1e-1, 2))
# variational posterior for the weights
self.weight = Gaussian(self.weight_mu, self.weight_rho)
# Bias parameters
self.bias_mu = nn.Parameter(
torch.Tensor(out_features).uniform_(-0.2, 0.2))
self.bias_rho = nn.Parameter(
torch.Tensor(out_features).uniform_(1e-1, 2))
# variational posterior for the bias
self.bias = Gaussian(self.bias_mu, self.bias_rho)
# Prior distributions
self.weight_prior = Gaussian(torch.Tensor([0.]), torch.Tensor([1.]))
self.bias_prior = Gaussian(torch.Tensor([0.]), torch.Tensor([1.]))
# initialize log_prior and log_posterior as 0
self.log_prior = 0
self.log_variational_posterior = 0
def forward(self, input, sample=False, calculate_log_probs=False):
# 1. Sample weights and bias from variational posterior
if self.training or sample:
weight = self.weight.sample()
bias = self.bias.sample()
else:
weight = self.weight.mu
bias = self.bias.mu
# 2. Update log_prior and log_posterior according to current approximation
if self.training or calculate_log_probs:
self.log_prior = self.weight_prior.log_prob(
weight) + self.bias_prior.log_prob(bias)
self.log_variational_posterior = self.weight.log_prob(
weight) + self.bias.log_prob(bias)
else:
self.log_prior, self.log_variational_posterior = 0, 0
# 3. Do a forward pass through the layer
return F.linear(input, weight, bias)
import numpy as np
import tensorflow as tf
sess = tf.Session()
def random_softmax(ndim):
x = np.random.uniform(size=(ndim, ))
x = x - np.max(x)
x = np.exp(x) / np.sum(np.exp(x))
return np.cast['float32'](x)
with such.A("Product Distribution") as it:
dist1 = Product([Categorical(5), Categorical(3)])
dist2 = Product([Gaussian(5), dist1])
@it.should
def test_dist_info_keys():
it.assertEqual(set(dist1.dist_info_keys), {"id_0_prob", "id_1_prob"})
it.assertEqual(
set(dist2.dist_info_keys),
{"id_0_mean", "id_0_stddev", "id_1_id_0_prob", "id_1_id_1_prob"})
@it.should
def test_kl_sym():
old_id_0_prob = np.array([random_softmax(5)])
old_id_1_prob = np.array([random_softmax(3)])
new_id_0_prob = np.array([random_softmax(5)])
new_id_1_prob = np.array([random_softmax(3)])
old_dist_info_vars = dict(id_0_prob=tf.constant(old_id_0_prob),
def setUp(self):
self.gaussian = Gaussian(25, 2)
self.gaussian.read_data_file('numbers.txt')
def setUp(self):
self.gaussian = Gaussian(25, 2)
class TestGaussian(unittest.TestCase):
def setUp(self):
self.g1f = Gaussian(1, False)
self.g2f = Gaussian(2, False)
self.g4t = Gaussian(4, True)
self.param1f = np.array([0,2])
self.param2f = np.array([np.arange(6), 2*np.arange(6)])
self.param4t = np.array([np.arange(8), -np.arange(8)])
def test_reparam(self):
t1f = {"g_mu":np.array([0]), "g_Sig":np.array([4]), "g_Siginv":np.array([0.25])}
theta1f = self.g1f.reparam(self.param1f)
self.assertDictEqual(t1f, theta1f)
t2f = {"g_mu":np.array([[0,1],[0,2]]), "g_Sig":np.array([[[13,23],[23,41]],[[52,92],[92,164]]]), "g_Siginv":np.array([[[10.25,-5.75],[-5.75,3.25]],[[2.5625,-1.4375],[-1.4375,0.8125]]])}
theta2f = self.g2f.reparam(self.param2f)
for key in t2f.keys():
with self.subTest(key=key):
self.assertTrue(np.all(np.round(t2f[key],2)==np.round(theta2f[key],2)))
t4t = {"g_mu":np.array([[0,1,2,3],[0,-1,-2,-3]]), "g_Sig":np.array([np.exp([4,5,6,7]),np.exp([-4,-5,-6,-7])])}
theta4t = self.g4t.reparam(self.param4t)
for key in t4t.keys():
with self.subTest(key=key):
self.assertTrue(np.all(t4t[key]==theta4t[key]))
def test_logpdf(self):
X1f = np.random.randn(5)
p1f = norm.logpdf(X1f, 0, 2)
logp1f = self.g1f.logpdf(self.param1f, X1f)
self.assertTrue(np.all(np.round(p1f[:,np.newaxis],5)==np.round(logp1f,5)))
X2f = np.random.randn(2)
p2f1 = multivariate_normal.logpdf(X2f, np.array([0,1]), np.array([[13,23],[23,41]]))
p2f2 = multivariate_normal.logpdf(X2f, np.array([0,2]), np.array([[52,92],[92,164]]))
p2f = np.array([p2f1, p2f2])
logp2f = self.g2f.logpdf(self.param2f, X2f)
self.assertTrue(np.all(np.round(p2f[np.newaxis,:],5)==np.round(logp2f,5)))
X4t = np.random.randn(8).reshape((2,4))
p4t1 = multivariate_normal.logpdf(X4t, np.array([0,1,2,3]),np.diag(np.exp(np.array([4,5,6,7]))))
p4t2 = multivariate_normal.logpdf(X4t, np.array([0,-1,-2,-3]),np.diag(np.exp(np.array([-4,-5,-6,-7]))))
p4t = np.hstack((p4t1[:,np.newaxis], p4t2[:,np.newaxis]))
logp4t = self.g4t.logpdf(self.param4t, X4t)
self.assertTrue(np.all(np.around(p4t,5)==np.around(logp4t,5)))
def test_sample(self):
np.random.seed(1)
x1 = 0 + 2* np.random.randn(5)
x2 = np.array([0,1]) + np.dot(np.random.randn(1,2), np.array([[2,3],[4,5]]))
x4 = np.array([0,1,2,3]) + np.dot(np.random.randn(10,4), np.diag(np.exp(np.array([4,5,6,7])/2)))
np.random.seed(1)
samp1 = self.g1f.sample(self.param1f, 5)
samp2 = self.g2f.sample(self.param2f[0,:], 1)
samp4 = self.g4t.sample(self.param4t[0,:], 10)
self.assertTrue(np.all(np.round(x1[:,np.newaxis],3)==np.round(samp1,3)))
self.assertTrue(np.all(np.round(x2,3)==np.round(samp2,3)))
self.assertTrue(np.all(np.round(x4,3)==np.round(samp4,3)))
def test_cross_sample(self):
var1f = 2 / np.array([1/4+1/25])
mu1f = np.array([0])
cov2f = 2 * np.linalg.inv(np.array([[10.25,-5.75],[-5.75,3.25]])+np.array([[2.5625,-1.4375],[-1.4375,0.8125]]))
mu2f = 0.5*np.dot(cov2f, np.dot(np.array([[10.25,-5.75],[-5.75,3.25]]),np.array([0,1])) + np.dot(np.array([[2.5625,-1.4375],[-1.4375,0.8125]]),np.array([0,2])))
cov4t = 2 /(np.exp(np.arange(-4,-8,-1)) + np.exp(np.arange(4,8)))
mu4t = 0.5*(cov4t*(np.arange(4)*np.exp(np.arange(-4,-8,-1))+np.arange(0,-4,-1)*np.exp(np.arange(4,8))))
np.random.seed(1)
x1 = np.random.multivariate_normal(mu1f, np.atleast_2d(var1f), 5)
np.random.seed(2)
x2 = np.random.multivariate_normal(mu2f, cov2f, 1)
np.random.seed(0)
x4 = mu4t + np.sqrt(cov4t)*np.random.randn(10, 4)
np.random.seed(1)
samp1 = self.g1f.cross_sample(self.param1f, -2.5*self.param1f, 5)
np.random.seed(2)
samp2 = self.g2f.cross_sample(self.param2f[0,:], self.param2f[1,:], 1)
np.random.seed(0)
samp4 = self.g4t.cross_sample(self.param4t[0,:], self.param4t[1,:], 10)
self.assertTrue(np.all(np.round(x1,3)==np.round(samp1,3)))
self.assertTrue(np.all(np.round(x2,3)==np.round(samp2,3)))
self.assertTrue(np.all(np.round(x4,3)==np.round(samp4,3)))
def test_log_sqrt_pair_integral(self):
l1 = -0.5*np.log(10/(2*4))
difmu2 = np.array([0,1])
s = np.array([[13,23],[23,41]])
S = np.array([[52,92],[92,164]])
S2 = np.array([[32.5,57.5],[57.5,102.5]])
l21 = -0.125*(difmu2*np.linalg.solve(S2,difmu2)).sum()- 0.5*np.linalg.slogdet(S2)[1] + 0.25*np.linalg.slogdet(s)[1] + 0.25*np.linalg.slogdet(S)[1]
l2 = np.array([0,l21])
difmu4 = np.array([[0,-0.5,-1,-1.5],[0,2.5,5,7.5]])
lsig = self.param4t[0, 4:]
Lsig = self.param4t[:,4:]*1.5
lSig2 = np.log(0.5)+np.logaddexp(lsig, Lsig)
l4 = -0.125*np.sum(np.exp(-lSig2)*difmu4**2, axis=1) - 0.5*np.sum(lSig2, axis=1) + 0.25*np.sum(lsig) + 0.25*np.sum(Lsig, axis=1)
la1f = self.g1f.log_sqrt_pair_integral(self.param1f, -2*self.param1f)
la2f = self.g2f.log_sqrt_pair_integral(self.param2f[0,:], self.param2f)
la4t = self.g4t.log_sqrt_pair_integral(self.param4t[0,:], 1.5*self.param4t)
self.assertTrue(np.all(np.round(l1,5)==np.round(la1f,5)))
self.assertTrue(np.all(np.round(l2,5)==np.round(la2f,5)))
self.assertTrue(np.all(np.round(l4,5)==np.round(la4t,5)))
def test_params_init(self):
np.random.seed(1)
m0 = np.random.multivariate_normal(np.zeros(2), 4*np.eye(2))
prm0f = np.concatenate((m0, np.array([1,0,0,1])))
np.random.seed(2)
m0 = np.random.multivariate_normal(np.zeros(2), 4*np.eye(2))
prm0t = np.concatenate((m0, np.zeros(2)))
np.random.seed(4)
mu = np.array([[0,1,2,3],[0,-1,-2,-3]])
k = np.random.choice(2, p=np.array([0.5,0.5]))
lsig = np.array([[4,5,6,7],[-4,-5,-6,-7]])
mu0 = mu[k]+np.random.randn(4)*np.sqrt(10)*np.exp(lsig[k,:])
LSig = np.random.randn(4)+lsig[k]
prm4t = np.hstack((mu0, LSig))
g2t = Gaussian(2, True)
np.random.seed(1)
par0f = self.g2f.params_init(np.empty((0,6)), np.empty((0,1)), 4)
np.random.seed(2)
par0t = g2t.params_init(np.empty((0,6)), np.empty((0,1)), 4)
np.random.seed(4)
par4t = self.g4t.params_init(self.param4t, np.array([0.1, 0.9]), 10)
self.assertTrue(np.all(np.round(par0f,3)==np.round(prm0f,3)))
self.assertTrue(np.all(np.round(par0t,3)==np.round(prm0t,3)))
self.assertTrue(np.all(np.round(par4t,3)==np.round(prm4t,3)))
class TestGaussianClass(unittest.TestCase):
def setUp(self):
self.gaussian = Gaussian(25, 2)
self.gaussian.read_data_file('numbers.txt')
def test_initialization(self):
self.assertEqual(self.gaussian.mean, 25, 'incorrect mean')
self.assertEqual(self.gaussian.stdev, 2,
'incorrect standard deviation')
def test_readdata(self):
self.assertEqual(self.gaussian.data, \
[1, 3, 99, 100, 120, 32, 330, 23, 76, 44, 31], 'data not read in correctly')
def test_meancalculation(self):
self.assertEqual(
self.gaussian.calculate_mean(),
sum(self.gaussian.data) / float(len(self.gaussian.data)),
'calculated mean not as expected')
def test_stdevcalculation(self):
self.assertEqual(round(self.gaussian.calculate_stdev(), 2), 92.87,
'sample standard deviation incorrect')
self.assertEqual(round(self.gaussian.calculate_stdev(0), 2), 88.55,
'population standard deviation incorrect')
def test_pdf(self):
self.assertEqual(round(self.gaussian.pdf(25), 5), 0.19947,
'pdf function does not give expected results')
self.gaussian.calculate_mean()
self.gaussian.calculate_stdev()
self.assertEqual(
round(self.gaussian.pdf(75), 5), 0.00429,
'pdf function after calculating mean and stdev '
'does not give expected result')
def test_add(self):
gaussian_one = Gaussian(25, 3)
gaussian_two = Gaussian(30, 4)
gaussian_sum = gaussian_one + gaussian_two
self.assertEqual(gaussian_sum.mean, 55)
self.assertEqual(gaussian_sum.stdev, 5)
def test_confidenceintervals(self):
self.assertEqual(
self.gaussian.calculate_confidence_intervals(0.95),
[23.205529189316458, 132.97628899250174],
'calculate_confidence_intervals does not give'
'expected result')
self.assertEqual(
self.gaussian.calculate_confidence_intervals(0.99),
[6.123854832188144, 150.05796334963003],
'calculate_confidence_intervals does not give'
'expected result')
class CatVAE(nn.Module):
# Auto-encoder architecture
def __init__(self,img_params,model_params,latent_params):
super(CatVAE, self).__init__()
image_dim = img_params['image_dim']
image_size = img_params['image_size']
n_downsample = model_params['n_downsample']
dim = model_params['dim']
n_res = model_params['n_res']
norm = model_params['norm']
activ = model_params['activ']
pad_type = model_params['pad_type']
n_mlp = model_params['n_mlp']
mlp_dim = model_params['mlp_dim']
self.continious_dim = latent_params['continious']
self.prior_cont = Gaussian(self.continious_dim)
self.categorical_dim = latent_params['categorical']
self.prior_catg = Categorical(self.categorical_dim)
self.gumbel = Gumbel(self.categorical_dim)
self.encoder = CatEncoder(n_downsample,n_res,n_mlp,image_size,image_dim,dim,mlp_dim,
latent_params,norm,activ,pad_type)
conv_inp_size = image_size // (2**n_downsample)
decoder_inp_dim = self.continious_dim + self.categorical_dim
self.decoder = Decoder(n_downsample,n_res,n_mlp,decoder_inp_dim,mlp_dim,conv_inp_size,
dim,image_dim,norm,activ,pad_type)
def forward(self, x, tempr):
latent_distr = self.encoder(x)
#categorical distr
categorical_distr = latent_distr[:,-self.categorical_dim:]
categorical_distr_act = self.prior_catg.activate(categorical_distr)# need for KL
catg_samples = self.gumbel.gumbel_softmax_sample(categorical_distr,tempr) # categotical sampling, reconstruction
#continious distr
continious_distr = latent_distr[:,:-self.categorical_dim]
continious_distr_act = self.prior_cont.activate(continious_distr)
cont_samples = self.prior_cont.sample(continious_distr_act)
#create full latent code
full_samples = torch.cat([cont_samples,catg_samples],1)
recons = self.decoder(full_samples)
return recons, full_samples, categorical_distr_act, continious_distr_act
def encode_decode(self, x, tempr=0.4, hard_catg=True):
latent_distr = self.encoder(x)
#categorical distr stuff
categorical_distr = latent_distr[:,-self.categorical_dim:]
if hard_catg:
#just make one hot vector
catg_samples = self.prior_catg.logits_to_onehot(categorical_distr)
else:
#make smoothed one hot by softmax
catg_samples = self.prior_catg.activate(categorical_distr)['prob']
#continious distr stuff
continious_distr = latent_distr[:,:-self.categorical_dim]
continious_distr_act = self.prior_cont.activate(continious_distr)
cont_samples = continious_distr_act['mean']
#create full latent code
full_samples = torch.cat([cont_samples,catg_samples],1)
recons = self.decoder(full_samples)
return recons, full_samples#, categorical_distr_act, continious_distr_act
def sample_full_prior(self, batch_size, device='cuda:0'):
cont_samples = self.prior_cont.sample_prior(batch_size, device=device)
catg_samples = self.prior_catg.sample_prior(batch_size, device=device)
full_samples = torch.cat([cont_samples,catg_samples],1)
return full_samples
from distributions import Gaussian
g1 = Gaussian(25, 2)
g1.read_data_file('numbers.txt')
g1.plot_histogram()
from distributions import Gaussian
# initialize two gaussian distributions
gaussian_one = Gaussian(25, 3)
gaussian_two = Gaussian(30, 2)
# initialize a third gaussian distribution reading in a data efile
gaussian_three = Gaussian()
gaussian_three.read_data_file('numbers.txt')
gaussian_three.calculate_mean()
gaussian_three.calculate_stdev()
print(gaussian_one.mean)
print(gaussian_two.mean)
print(gaussian_one.stdev)
print(gaussian_two.stdev)
print(gaussian_three.mean)
print(gaussian_three.stdev)
# plot histogram of gaussian three
gaussian_three.plot_histogram_pdf()
# add gaussian_one and gaussian_two together
gaussian_one + gaussian_two
def setUp(self):
self.gaussian = Gaussian('numbers_gaussian.txt')