def __init__(self, in_features, out_features):
super().__init__()
self.in_features = in_features
self.out_features = out_features
# Weight parameters
self.weight_mu = nn.Parameter(
torch.Tensor(out_features, in_features).uniform_(-0.2, 0.2))
self.weight_rho = nn.Parameter(
torch.Tensor(out_features, in_features).uniform_(1e-1, 2))
# variational posterior for the weights
self.weight = Gaussian(self.weight_mu, self.weight_rho)
# Bias parameters
self.bias_mu = nn.Parameter(
torch.Tensor(out_features).uniform_(-0.2, 0.2))
self.bias_rho = nn.Parameter(
torch.Tensor(out_features).uniform_(1e-1, 2))
# variational posterior for the bias
self.bias = Gaussian(self.bias_mu, self.bias_rho)
# Prior distributions
self.weight_prior = Gaussian(torch.Tensor([0.]), torch.Tensor([1.]))
self.bias_prior = Gaussian(torch.Tensor([0.]), torch.Tensor([1.]))
# initialize log_prior and log_posterior as 0
self.log_prior = 0
self.log_variational_posterior = 0
def setUp(self):
self.g1f = Gaussian(1, False)
self.g2f = Gaussian(2, False)
self.g4t = Gaussian(4, True)
self.param1f = np.array([0,2])
self.param2f = np.array([np.arange(6), 2*np.arange(6)])
self.param4t = np.array([np.arange(8), -np.arange(8)])
def propose_merge(self, i, j, component_weights):
target = j if np.argmax([component_weights[i], component_weights[j]
]) else i
source = i if target == j else j
proposal_means = self.currentmeans.copy()
proposal_means[target] = (
component_weights[target] * proposal_means[target] +
component_weights[source] * proposal_means[source]) / (
component_weights[target] + component_weights[source])
proposal_means = np.delete(proposal_means, source)
proposal_auxiliary = self.auxiliary.copy()
proposal_auxiliary[:, target] += proposal_auxiliary[:, source]
proposal_auxiliary = np.delete(proposal_auxiliary, source, axis=1)
log_MH = 0
for seg in range(self.n_segments):
accept_top = Gaussian(
np.dot(proposal_auxiliary[seg], proposal_means),
np.sum(proposal_auxiliary[seg]) / self.currentprecision).pdf(
self.jumps[seg])
accept_bot = Gaussian(
np.dot(self.auxiliary[seg], self.currentmeans),
np.sum(self.auxiliary[seg]) / self.currentprecision).pdf(
self.jumps[seg])
log_MH += np.log(accept_top) - np.log(accept_bot)
print(log_MH)
if np.random.rand() < np.exp(log_MH):
self.merge(target, source, proposal_means)
def update_segments(self, it):
"""
Samples the auxiliary variable using a Metropolis-Hastings step
conditional on the updated parameter values.
:param it: theh MCMC iteration number.
:return: the average number of proposal acceptances over each segment.
"""
accept_count = 0
zerotrun_pois = (ZeroTruncatedPoission(
rate=self.rate * self.delta).sample(self.n_segments))
for seg in range(self.n_segments):
prop = zerotrun_pois[seg]
prop_components = np.random.multinomial(
prop,
self.currentmixing_coeffs / np.sum(self.currentmixing_coeffs))
accept_top = Gaussian(np.dot(prop_components, self.currentmeans),
prop / self.currentprecision).pdf(
self.jumps[seg])
accept_bot = Gaussian(
np.dot(self.auxiliary[seg], self.currentmeans),
np.sum(self.auxiliary[seg]) / self.currentprecision).pdf(
self.jumps[seg])
if np.random.rand() < accept_top / accept_bot:
accept_count += 1
self.auxiliary[seg] = prop_components
return accept_count / self.n_segments
def test_add(self):
gaussian_one = Gaussian(25, 3)
gaussian_two = Gaussian(30, 4)
gaussian_sum = gaussian_one + gaussian_two
self.assertEqual(gaussian_sum.mean, 55)
self.assertEqual(gaussian_sum.stdev, 5)
def unpack(dim_in=None,
dim_h=None,
prior=None,
recognition_net=None,
generation_net=None,
extra_args=dict(),
distributions=dict(),
dims=dict(),
dataset_args=dict(),
center_input=None,
**model_args):
print 'Unpacking model with parameters %s' % model_args.keys()
print 'Forming Gaussian prior model'
prior_model = Gaussian(dim_h)
models = []
print 'Forming MLPs'
kwargs = GBN.mlp_factory(dim_h,
dims,
distributions,
recognition_net=recognition_net,
generation_net=generation_net)
kwargs['prior'] = prior_model
models.append(prior_model)
print 'Forming GBN'
model = GBN(dim_in, dim_h, **kwargs)
models.append(model)
models += [model.posterior, model.conditional]
return models, model_args, extra_args
def set_params(self):
self.params = OrderedDict()
if self.prior is None:
self.prior = Gaussian(self.dim_h)
if self.posterior is None:
self.posterior = MLP(self.dim_in,
self.dim_h,
dim_hs=[],
rng=self.rng,
trng=self.trng,
h_act='T.nnet.sigmoid',
distribution='gaussian')
if self.conditional is None:
self.conditional = MLP(self.dim_h,
self.dim_in,
dim_hs=[],
rng=self.rng,
trng=self.trng,
h_act='T.nnet.sigmoid',
distribution='binomial')
self.posterior.name = self.name + '_posterior'
self.conditional.name = self.name + '_conditional'
def test_pdf(self):
self.gaussian = Gaussian(25, 2)
self.assertEqual(round(self.gaussian.pdf(25), 5), 0.19947,\
'pdf function does not give expected result')
self.gaussian.read_data_file('numbers.txt')
self.assertEqual(round(self.gaussian.pdf(75), 5), 0.00429,\
'pdf function after calculating mean and stdev does not give expected result')
def __init__(
self,
ensemble_size,
in_dim,
out_dim,
encoder_hidden_dim,
decoder_hidden_dim,
latent_dim,
n_hidden,
learn_rate=0.0001,
):
super(DynamicsEnsemble, self).__init__()
self.ensemble_size = ensemble_size
self.encoder = Encoder(in_dim,
encoder_hidden_dim,
latent_dim,
n_hidden=n_hidden)
self.decoders = self.build_decoder_ensemble(out_dim,
decoder_hidden_dim,
latent_dim, n_hidden)
self.opt = optim.Adam(self.parameters(), lr=learn_rate)
self.gaussian = Gaussian(latent_dim)
self.next_obs = None
self.reward = None
def __init__(self,img_params,model_params,latent_params):
super(CatVAE, self).__init__()
image_dim = img_params['image_dim']
image_size = img_params['image_size']
n_downsample = model_params['n_downsample']
dim = model_params['dim']
n_res = model_params['n_res']
norm = model_params['norm']
activ = model_params['activ']
pad_type = model_params['pad_type']
n_mlp = model_params['n_mlp']
mlp_dim = model_params['mlp_dim']
self.continious_dim = latent_params['continious']
self.prior_cont = Gaussian(self.continious_dim)
self.categorical_dim = latent_params['categorical']
self.prior_catg = Categorical(self.categorical_dim)
self.gumbel = Gumbel(self.categorical_dim)
self.encoder = CatEncoder(n_downsample,n_res,n_mlp,image_size,image_dim,dim,mlp_dim,
latent_params,norm,activ,pad_type)
conv_inp_size = image_size // (2**n_downsample)
decoder_inp_dim = self.continious_dim + self.categorical_dim
self.decoder = Decoder(n_downsample,n_res,n_mlp,decoder_inp_dim,mlp_dim,conv_inp_size,
dim,image_dim,norm,activ,pad_type)
def test_params_init(self):
np.random.seed(1)
m0 = np.random.multivariate_normal(np.zeros(2), 4*np.eye(2))
prm0f = np.concatenate((m0, np.array([1,0,0,1])))
np.random.seed(2)
m0 = np.random.multivariate_normal(np.zeros(2), 4*np.eye(2))
prm0t = np.concatenate((m0, np.zeros(2)))
np.random.seed(4)
mu = np.array([[0,1,2,3],[0,-1,-2,-3]])
k = np.random.choice(2, p=np.array([0.5,0.5]))
lsig = np.array([[4,5,6,7],[-4,-5,-6,-7]])
mu0 = mu[k]+np.random.randn(4)*np.sqrt(10)*np.exp(lsig[k,:])
LSig = np.random.randn(4)+lsig[k]
prm4t = np.hstack((mu0, LSig))
g2t = Gaussian(2, True)
np.random.seed(1)
par0f = self.g2f.params_init(np.empty((0,6)), np.empty((0,1)), 4)
np.random.seed(2)
par0t = g2t.params_init(np.empty((0,6)), np.empty((0,1)), 4)
np.random.seed(4)
par4t = self.g4t.params_init(self.param4t, np.array([0.1, 0.9]), 10)
self.assertTrue(np.all(np.round(par0f,3)==np.round(prm0f,3)))
self.assertTrue(np.all(np.round(par0t,3)==np.round(prm0t,3)))
self.assertTrue(np.all(np.round(par4t,3)==np.round(prm4t,3)))
def elbo(self, input, target, samples=20):
outputs = []
log_priors = torch.zeros(samples)
log_variational_posteriors = torch.zeros(samples)
# draw n_samples from the posterior (run n_samples forward passes)
for i in range(samples):
outputs.append(self(input, sample=True))
log_priors[i] = self.log_prior()
log_variational_posteriors[i] = self.log_variational_posterior()
log_prior = log_priors.sum()
log_variational_posterior = log_variational_posteriors.sum()
outputs = torch.stack(outputs)
y_dist = Gaussian(outputs.mean(0), self.noise)
# negative_log_likelihood = self.loss_function(
# outputs.mean(0), target, reduction='sum')
negative_log_likelihood = -y_dist.log_prob(target) / len(input)
# loss = nll + kl
loss = negative_log_likelihood
kl = (log_variational_posterior - log_prior) / self.n_training
loss += kl
return loss, log_prior, log_variational_posterior, negative_log_likelihood
def CollapsedGibbsSampling(model, num_iterations):
"""Performs a specified number of iterations of the Collapsed Gibbs Sampling algorithm using the model specified."""
# For each iteration
for iter_id in range(num_iterations):
print model.cluster_count
print model.cluster_popn
print model.data
print model.membership
# For each data point in the model
for data_id in range(len(model.data)):
# Remove data point from its current cluster
data_point = model.data[data_id]
cluster_id = model.membership[data_id]
model.cluster_popn[cluster_id] -= 1
model.clusters[cluster_id].rem_data(data_point)
# If the cluster is now empty delete it
if model.cluster_popn[cluster_id] == 0:
model.cluster_count -= 1
del model.clusters[cluster_id]
del model.cluster_popn[cluster_id]
tmp_idx = np.where(model.membership > cluster_id)
model.membership[tmp_idx] -= 1
# Generate numpy array of relative probabilities based on prior for membership and current datapoints in each cluster
p = np.asarray(model.conditional_prob(data_point))
# Check that p is normalised
p = p / np.sum(p)
# Generate cumulative probability mass function
p = np.cumsum(p)
rand_val = np.random.random()
# Select new cluster based on probabilities
new_cluster_id = np.sum(np.greater(rand_val, p))
# If a new cluster was chosen then create it. For convenience an empty cluster will always be at the end of the list. Thus we only have to create a copy of this cluster for future iterations, without storing the prior.
if new_cluster_id == model.cluster_count:
model.cluster_count += 1
model.clusters.append(Gaussian(np.asarray([]), model.prior))
model.cluster_popn.append(0)
# Add data point to its new cluster
model.membership[data_id] = new_cluster_id
model.cluster_popn[new_cluster_id] += 1
model.clusters[new_cluster_id].add_data(data_point)
def __init__(self,img_params,model_params,latent_params):
super(VAE, self).__init__()
image_dim = img_params['image_dim']
image_size = img_params['image_size']
n_downsample = model_params['n_downsample']
dim = model_params['dim']
n_res = model_params['n_res']
norm = model_params['norm']
activ = model_params['activ']
pad_type = model_params['pad_type']
n_mlp = model_params['n_mlp']
mlp_dim = model_params['mlp_dim']
self.latent_dim = latent_params['latent_dim']
self.prior = Gaussian(self.latent_dim)
self.encoder = Encoder(n_downsample,n_res,n_mlp,image_size,image_dim,dim,mlp_dim,
self.latent_dim,norm,activ,pad_type)
conv_inp_size = image_size // (2**n_downsample)
self.decoder = Decoder(n_downsample,n_res,n_mlp,self.latent_dim,mlp_dim,conv_inp_size,
dim,image_dim,norm,activ,pad_type)
from distributions import Gaussian
g1 = Gaussian(22, 2)
g1.read_data_file('numbers.txt')
print(g1.mean)
g1.plot_histogram()
g1.plot_histogram_pdf()
#
# Exposure weighting in unbinned mode
#
exposures = ra.random(len(seq))
bb.setCellSizes(exposures)
win1 = plot.Window(1)
for ncpPrior in range(1, 10):
xx, yy = bb.lightCurve(ncpPrior)
plot.curve(xx, num.array(yy)*binsize, color='r', linewidth=5)
#
# Binned mode
#
func0 = Gaussian(1, 10, 3)
class Histogram(object):
def __init__(self, xmin, xmax, nx):
self.bins = num.zeros(nx, dtype=num.float)
self.xstep = (xmax - xmin)/float(nx-1)
self.xvals = num.linspace(xmin, xmax, nx) + self.xstep/2.
def add(self, x, wt=1):
i = int((x - self.xvals[0])/self.xstep)
if i < len(self.bins):
self.bins[i] += wt
hist = Histogram(0, 20, 30)
for i in range(100):
hist.add(func0.draw())
bb = BayesianBlocks.BayesianBlocks(hist.xvals[0] - hist.xstep/2.,
import numpy as np
import tensorflow as tf
sess = tf.Session()
def random_softmax(ndim):
x = np.random.uniform(size=(ndim, ))
x = x - np.max(x)
x = np.exp(x) / np.sum(np.exp(x))
return np.cast['float32'](x)
with such.A("Product Distribution") as it:
dist1 = Product([Categorical(5), Categorical(3)])
dist2 = Product([Gaussian(5), dist1])
@it.should
def test_dist_info_keys():
it.assertEqual(set(dist1.dist_info_keys), {"id_0_prob", "id_1_prob"})
it.assertEqual(
set(dist2.dist_info_keys),
{"id_0_mean", "id_0_stddev", "id_1_id_0_prob", "id_1_id_1_prob"})
@it.should
def test_kl_sym():
old_id_0_prob = np.array([random_softmax(5)])
old_id_1_prob = np.array([random_softmax(3)])
new_id_0_prob = np.array([random_softmax(5)])
new_id_1_prob = np.array([random_softmax(3)])
old_dist_info_vars = dict(id_0_prob=tf.constant(old_id_0_prob),
# # if torch.cuda.is_available():
# # flex_model.cuda()
# # vae, iwae = flex_model.train_and_eval(logposterior=logposterior, model=model, x=x)
# vae, iwae = optimize_local_expressive(logposterior, model, x)
# print (vae.data.cpu().numpy(),iwae.data.cpu().numpy(),'flex')
# vaes_flex.append(vae.data.cpu().numpy())
# iwaes_flex.append(iwae.data.cpu().numpy())
# q_local = Gaussian(hyper_config) #, mean, logvar)
# q_local = Flow(hyper_config).cuda()#, mean, logvar)
hnf = 0
if q_name == 'Gaus':
q_local = Gaussian(hyper_config)
elif q_name == 'Flow':
q_local = Flow(hyper_config).cuda()
elif q_name == 'Flow1':
q_local = Flow1(hyper_config).cuda()
elif q_name == 'HNF':
q_local = HNF(hyper_config)
hnf = 1
else:
dfadfas
# print (q_local)
# vae, iwae = optimize_local_gaussian(logposterior, model, x)
vae, iwae = optimize_local_q_dist(logposterior, hyper_config, x, q_local)
print (vae.data.cpu().numpy(),iwae.data.cpu().numpy(),'reg')
def setUp(self):
self.gaussian = Gaussian(25, 2)
self.gaussian.read_data_file('numbers.txt')
num_days = 100 # Number of days the experiment is run
env_classes = []
phase_lengths = []
configuration = get_test_multiphase_configuration()
for phase_data in configuration['phase_data']:
phase_lengths.append(phase_data['duration'])
phase_env_classes = []
for (id, ldata) in enumerate(phase_data['left_classes']):
phase_env_classes.append(
Class_Env(
id, True,
Gaussian(ldata['new_node_rate_mean'],
ldata['new_node_rate_variance']),
Uniform_Discrete(ldata['time_to_stay_min'],
ldata['time_to_stay_max'])))
num_left_classes = len(phase_data['left_classes'])
for (id, ldata) in enumerate(phase_data['right_classes']):
phase_env_classes.append(
Class_Env(
id + num_left_classes, False,
Gaussian(ldata['new_node_rate_mean'],
ldata['new_node_rate_variance']),
Uniform_Discrete(ldata['time_to_stay_min'],
ldata['time_to_stay_max'])))
for (ids, edge_data) in phase_data['edge_data'].items():
class_edge = Class_Env_Edge(Bernoulli(edge_data['mean']),
edge_data['weight'])
l_class = [c for c in phase_env_classes if c.id == ids[0]][0]
def _pack_gaussian(self, mus, sigmas):
"""Update"""
components = []
for i in range(len(mus)):
components.append(Gaussian(mean=mus[i], cov=sigmas[i]))
return components
# 'decoder_arch': [[z_size,x_size]],
# 'q_dist': standard, #FFG_LN#,#hnf,#aux_nf,#flow1,#,
# 'cuda': 1
# }
# hyper_config = {
# 'x_size': x_size,
# 'z_size': z_size,
# 'act_func': F.tanh,# F.relu,
# 'encoder_arch': [[x_size,200],[200,200],[200,z_size*2]],
# 'decoder_arch': [[z_size,200],[200,200],[200,200],[200,200],[200,x_size]],
# 'q_dist': standard, #FFG_LN#,#hnf,#aux_nf,#flow1,#,
# 'cuda': 1
# }
q = Gaussian(hyper_config)
# q = Flow(hyper_config)
hyper_config['q'] = q
print('Init model')
model = VAE(hyper_config)
if torch.cuda.is_available():
model.cuda()
print('\nModel:', hyper_config, '\n')
# path_to_load_variables=''
path_to_save_variables = home + '/Documents/tmp/inference_suboptimality/fashion_params/LE_binarized_fashion' #.pt'
# path_to_save_variables=home+'/Documents/tmp/inference_suboptimality/fashion_params/binarized_fashion_' #.pt'
# path_to_save_variables=home+'/Documents/tmp/pytorch_vae'+str(epochs)+'.pt'
# # 4 hidden decoder
# hyper_config = {
# 'x_size': x_size,
# 'z_size': z_size,
# 'act_func': F.tanh,# F.relu,
# 'encoder_arch': [[x_size,200],[200,200],[200,z_size*2]],
# 'decoder_arch': [[z_size,200],[200,200],[200,200],[200,200],[200,x_size]],
# 'q_dist': standard, #FFG_LN#,#hnf,#aux_nf,#flow1,#,
# 'cuda': 1
# }
q = Gaussian(hyper_config)
# q = Flow(hyper_config)
hyper_config['q'] = q
# Which gpu
# os.environ['CUDA_VISIBLE_DEVICES'] = gpu_to_use
print ('Init model')
model = VAE(hyper_config)
if torch.cuda.is_available():
model.cuda()
print ('using cuda')
else:
from distributions import Gaussian gaussian_one = Gaussian(22, 2) print(gaussian_one.mean)
def setUp(self):
self.gaussian = Gaussian(25, 2)
from distributions import Gaussian
g1 = Gaussian(25, 2)
g1.read_data_file('numbers.txt')
g1.plot_histogram()
def setUp(self):
self.gaussian = Gaussian('numbers_gaussian.txt')
from distributions import Gaussian gaussian_one = Gaussian(25, 2) gaussian_one.mean print(gaussian_one + gaussian_one)
from distributions import Gaussian
# initialize two gaussian distributions
gaussian_one = Gaussian(25, 3)
gaussian_two = Gaussian(30, 2)
# initialize a third gaussian distribution reading in a data efile
gaussian_three = Gaussian()
gaussian_three.read_data_file('numbers.txt')
gaussian_three.calculate_mean()
gaussian_three.calculate_stdev()
print(gaussian_one.mean)
print(gaussian_two.mean)
print(gaussian_one.stdev)
print(gaussian_two.stdev)
print(gaussian_three.mean)
print(gaussian_three.stdev)
# plot histogram of gaussian three
gaussian_three.plot_histogram_pdf()
# add gaussian_one and gaussian_two together
gaussian_one + gaussian_two
from distributions import Gaussian
gaussian_one = Gaussian(10, 5)
print('Mean:', gaussian_one.mean)
print('Std. deviation:', gaussian_one.stdev)
import distributions
print(distributions.__file__)
