def negative_iwae_bound(self, x, iw):
"""
Computes the Importance Weighted Autoencoder Bound
Additionally, we also compute the ELBO KL and reconstruction terms
Args:
x: tensor: (batch, dim): Observations
iw: int: (): Number of importance weighted samples
Returns:
niwae: tensor: (): Negative IWAE bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
################################################################################
# TODO: Modify/complete the code here
# Compute niwae (negative IWAE) with iw importance samples, and the KL
# and Rec decomposition of the Evidence Lower Bound
#
# Outputs should all be scalar
################################################################################
batch = x.size()[0]
"""
sample iw z's
for z_i in sample:
find p(z_i, all x)
find q(z_i, x)
average
"""
phi_m, phi_v = self.enc.encode(x) # (batch, z_dim)
phi_m, phi_v = ut.duplicate(phi_m,
iw), ut.duplicate(phi_v,
iw) # (batch*iw, z_dim)
x_iw = ut.duplicate(x, iw)
z_hat = ut.sample_gaussian(phi_m, phi_v) # (batch*iw, z_dim)
log_q_zx = ut.log_normal(z_hat, phi_m, phi_v) # (batch*iw)
log_p_z = ut.log_normal(z_hat, *self.z_prior) # (batch*iw)
log_p_xz = ut.log_bernoulli_with_logits(
x_iw, self.dec.decode(z_hat)) # (batch*iw)
f = lambda x: x.reshape(iw, batch).transpose(1, 0)
log_p_xz, log_q_zx, log_p_z = f(log_p_xz), f(log_q_zx), f(log_p_z)
iwae = ut.log_mean_exp(log_p_xz - log_q_zx + log_p_z, -1)
iwae = iwae.mean(0)
niwae = -iwae
kl = ut.log_mean_exp(log_q_zx - log_p_z, -1)
kl = kl.mean(0)
rec = ut.log_mean_exp(log_p_xz, -1)
rec = -rec.mean(0)
################################################################################
# End of code modification
################################################################################
return niwae, kl, rec
def negative_iwae_bound(self, x, iw):
"""
Computes the Importance Weighted Autoencoder Bound
Additionally, we also compute the ELBO KL and reconstruction terms
Args:
x: tensor: (batch, dim): Observations
iw: int: (): Number of importance weighted samples
Returns:
niwae: tensor: (): Negative IWAE bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
################################################################################
# TODO: Modify/complete the code here
# Compute niwae (negative IWAE) with iw importance samples, and the KL
# and Rec decomposition of the Evidence Lower Bound
#
# Outputs should all be scalar
################################################################################
q_m, q_v = self.enc.encode(x)
q_m_, q_v_ = ut.duplicate(q_m, rep=iw), ut.duplicate(q_v, rep=iw)
z_given_x = ut.sample_gaussian(q_m_, q_v_)
decoded_bernoulli_logits = self.dec.decode(z_given_x)
#duplicate x
x_dup = ut.duplicate(x, rep=iw)
rec = ut.log_bernoulli_with_logits(x_dup, decoded_bernoulli_logits)
#compute kl
p_m, p_v = torch.zeros(q_m.shape), torch.ones(q_v.shape)
p_m_, p_v_ = ut.duplicate(p_m, iw), ut.duplicate(p_v, iw)
#print("p_m", p_m.shape)
log_q_phi = ut.log_normal(z_given_x, q_m_, q_v_) #encoded distribution
log_p = ut.log_normal(z_given_x, p_m_, p_v_) #prior distribution
kl = log_q_phi - log_p
niwae = rec - kl
#reshape to size (iw, bs) and then sum
niwae = ut.log_mean_exp(niwae.reshape(iw, -1), dim=0)
rec = ut.log_mean_exp(rec, dim=0)
kl = ut.log_mean_exp(kl, dim=0)
niwae = -torch.mean(niwae)
kl = torch.mean(kl)
rec = torch.mean(kl)
################################################################################
# End of code modification
################################################################################
return niwae, kl, rec
def negative_iwae_bound(self, x, iw):
"""
Computes the Importance Weighted Autoencoder Bound
Additionally, we also compute the ELBO KL and reconstruction terms
Args:
x: tensor: (batch, dim): Observations
iw: int: (): Number of importance weighted samples
Returns:
niwae: tensor: (): Negative IWAE bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
m, v = self.enc.encode(x)
batch_size, dim = m.shape
# Duplicate
m = ut.duplicate(m, iw)
v = ut.duplicate(v, iw)
x = ut.duplicate(x, iw)
z = ut.sample_gaussian(m, v)
logits = self.dec.decode(z)
km = self.km.repeat(batch_size, 1, 1)
kv = self.kv.repeat(batch_size, 1, 1)
km = ut.duplicate(km, iw)
kv = ut.duplicate(kv, iw)
kl_vec = ut.log_normal(z, m, v) - ut.log_normal_mixture(z, km, kv)
kl = torch.mean(kl_vec)
# TODO: compute the values below
rec_vec = ut.log_bernoulli_with_logits(x, logits)
rec = torch.neg(torch.mean(rec_vec))
if iw > 1:
iwtensor = torch.zeros(iw)
j = 0
while j < iw:
i = 0
sum = 0
while i < batch_size:
sum += rec_vec[j * batch_size + i]
i += 1
iwtensor[j] = sum / batch_size - kl
j += 1
niwae = torch.neg(ut.log_mean_exp(iwtensor, 0))
else:
niwae = rec + kl
return niwae, kl, rec
def negative_iwae_bound(self, x, iw):
"""
Computes the Importance Weighted Autoencoder Bound
Additionally, we also compute the ELBO KL and reconstruction terms
Args:
x: tensor: (batch, dim): Observations
iw: int: (): Number of importance weighted samples
Returns:
niwae: tensor: (): Negative IWAE bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
################################################################################
# TODO: Modify/complete the code here
# Compute niwae (negative IWAE) with iw importance samples, and the KL
# and Rec decomposition of the Evidence Lower Bound
#
# Outputs should all be scalar
################################################################################
m, v = self.enc.encode(x)
# m, v -> (batch, dim)
# (batch, dim) -> (batch*iw, dim)
m = ut.duplicate(m, iw)
# (batch, dim) -> (batch*iw, dim)
v = ut.duplicate(v, iw)
# (batch, dim) -> (batch*iw, dim)
x = ut.duplicate(x, iw)
# z -> (batch*iw, dim)
z = ut.sample_gaussian(m, v)
logits = self.dec.decode(z)
kl = ut.log_normal(z, m, v) - ut.log_normal(z, self.z_prior_m,
self.z_prior_v)
rec = -ut.log_bernoulli_with_logits(x, logits)
nelbo = kl + rec
niwae = -ut.log_mean_exp(-nelbo.reshape(iw, -1), dim=0)
niwae, kl, rec = niwae.mean(), kl.mean(), rec.mean()
################################################################################
# End of code modification
################################################################################
return niwae, kl, rec
def negative_iwae_bound(self, x, iw):
"""
Computes the Importance Weighted Autoencoder Bound
Additionally, we also compute the ELBO KL and reconstruction terms
Args:
x: tensor: (batch, dim): Observations
iw: int: (): Number of importance weighted samples
Returns:
niwae: tensor: (): Negative IWAE bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
################################################################################
# TODO: Modify/complete the code here
# Compute niwae (negative IWAE) with iw importance samples, and the KL
# and Rec decomposition of the Evidence Lower Bound
#
# Outputs should all be scalar
################################################################################
# Compute the mixture of Gaussian prior
prior = ut.gaussian_parameters(self.z_pre, dim=1)
q_m, q_v = self.enc.encode(x)
q_m_, q_v_ = ut.duplicate(q_m, rep=iw), ut.duplicate(q_v, rep=iw)
z_given_x = ut.sample_gaussian(q_m_, q_v_)
decoded_bernoulli_logits = self.dec.decode(z_given_x)
#duplicate x
x_dup = ut.duplicate(x, rep=iw)
rec = ut.log_bernoulli_with_logits(x_dup, decoded_bernoulli_logits)
log_p_theta = ut.log_normal_mixture(z_given_x, prior[0], prior[1])
log_q_phi = ut.log_normal(z_given_x, q_m_, q_v_)
kl = log_q_phi - log_p_theta
niwae = rec - kl
niwae = ut.log_mean_exp(niwae.reshape(iw, -1), dim=0)
niwae = -torch.mean(niwae)
#yay!
################################################################################
# End of code modification
################################################################################
return niwae, kl, rec
def sample_clipped_x(self, batch):
z = self.sample_z(batch)
z = ut.duplicate(z, 10)
y = np.arange(10)
y = z.new(np.repeat(np.eye(10)[y], batch, 0))
x_m = self.compute_mean_given(z, y)
return x_m
def negative_elbo_bound(self, x):
"""
Computes the Evidence Lower Bound, KL and, Reconstruction costs
Args:
x: tensor: (batch, dim): Observations
Returns:
nelbo: tensor: (): Negative evidence lower bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
y_logits = self.cls.classify(x)
# Duplicate y based on x's batch size. Then duplicate x
y = np.repeat(np.arange(self.y_dim), x.size(0))
y = x.new(np.eye(self.y_dim)[y])
x = ut.duplicate(x, self.y_dim)
m, v = self.enc.encode(x, y)
z = ut.sample_gaussian(m, v)
x_logits = self.dec.decode(z, y)
# TODO: compute the values below
nelbo, kl_z, kl_y, rec = 0, 0, 0, 0
return nelbo, kl_z, kl_y, rec
def negative_elbo_bound(self, x):
"""
Computes the Evidence Lower Bound, KL and, Reconstruction costs
Args:
x: tensor: (batch, dim): Observations
Returns:
nelbo: tensor: (): Negative evidence lower bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
################################################################################
# TODO: Modify/complete the code here
# Compute negative Evidence Lower Bound and its KL_Z, KL_Y and Rec decomposition
#
# To assist you in the vectorization of the summation over y, we have
# the computation of q(y | x) and some tensor tiling code for you.
#
# Note that nelbo = kl_z + kl_y + rec
#
# Outputs should all be scalar
################################################################################
y_logits = self.cls.classify(x)
y_logprob = F.log_softmax(y_logits, dim=1)
y_prob = torch.softmax(y_logits, dim=1) # (batch, y_dim)
p_y = 1 / 10 * torch.ones_like(y_prob)
kl_y = torch.mean(ut.kl_cat(y_prob, y_logprob, torch.log(p_y)), dim=0)
batch_size = x.shape[0]
# Duplicate y based on x's batch size. Then duplicate x
# This enumerates all possible combination of x with labels (0, 1, ..., 9)
y = np.repeat(np.arange(self.y_dim), x.size(0))
y = x.new(np.eye(self.y_dim)[y])
x = ut.duplicate(x, self.y_dim)
qm, qv = self.enc.encode(x, y)
z = ut.sample_gaussian(qm, qv)
recon_logits = self.dec.decode(z, y)
p_x_given_yz = ut.log_bernoulli_with_logits(x, recon_logits)
p_x_given_yz = p_x_given_yz.reshape(self.y_dim, batch_size).transpose(
0, 1) #[batch, 10]
rec = -torch.mean(torch.sum(p_x_given_yz * y_prob, dim=1), dim=0)
kl_z_over_xy = ut.kl_normal(qm, qv, self.z_prior_m, self.z_prior_v)
kl_z_over_xy = kl_z_over_xy.reshape(self.y_dim,
batch_size).transpose(0, 1)
kl_z = torch.mean(torch.sum(kl_z_over_xy * y_prob, dim=1), dim=0)
nelbo = rec + kl_y + kl_z
################################################################################
# End of code modification
################################################################################
return nelbo, kl_z, kl_y, rec
def negative_elbo_bound(self, x):
"""
Computes the Evidence Lower Bound, KL and, Reconstruction costs
Args:
x: tensor: (batch, dim): Observations
Returns:
nelbo: tensor: (): Negative evidence lower bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
################################################################################
# TODO: Modify/complete the code here
# Compute negative Evidence Lower Bound and its KL_Z, KL_Y and Rec decomposition
#
# To assist you in the vectorization of the summation over y, we have
# the computation of q(y | x) and some tensor tiling code for you.
#
# Note that nelbo = kl_z + kl_y + rec
#
# Outputs should all be scalar
################################################################################
y_logits = self.cls.classify(x)
y_logprob = F.log_softmax(y_logits, dim=1)
y_prob = torch.softmax(y_logits, dim=1)
# Duplicate y based on x's batch size. Then duplicate x
# This enumerates all possible combination of x with labels (0, 1, ..., 9)
y = np.repeat(np.arange(self.y_dim), x.size(0))
y = x.new(np.eye(self.y_dim)[y])
x = ut.duplicate(x, self.y_dim)
# sample z from x and y
qm, qv = self.enc.encode(x, y)
z = ut.sample_gaussian(qm, qv)
# compute kl
x_logits = self.dec.decode(z, y)
kl_y = ut.kl_cat(y_prob, y_logprob, np.log(1.0 / self.y_dim))
kl_z = ut.kl_normal(qm, qv, self.z_prior[0], self.z_prior[1])
rec_loss = -ut.log_bernoulli_with_logits(x, x_logits)
# (y_dim * batch)
# Compute the expected reconstruction and kl base on the distribution q(y|x), q(y,z|x)
rec_loss_y = (y_prob.t() * rec_loss.reshape(self.y_dim, -1)).sum(0)
kl_z_y = (y_prob.t() * kl_z.reshape(self.y_dim, -1)).sum(0)
# Reduce to means
kl_y, kl_z, rec = kl_y.mean(), kl_z_y.mean(), rec_loss_y.mean()
nelbo = rec + kl_z + kl_y
################################################################################
# End of code modification
################################################################################
return nelbo, kl_z, kl_y, rec
def negative_elbo_bound(self, x):
"""
Computes the Evidence Lower Bound, KL and, Reconstruction costs
Args:
x: tensor: (batch, dim): Observations
Returns:
nelbo: tensor: (): Negative evidence lower bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
################################################################################
# TODO: Modify/complete the code here
# Compute negative Evidence Lower Bound and its KL_Z, KL_Y and Rec decomposition
#
# To assist you in the vectorization of the summation over y, we have
# the computation of q(y | x) and some tensor tiling code for you.
#
# Note that nelbo = kl_z + kl_y + rec
#
# Outputs should all be scalar
################################################################################
y_logits = self.cls.classify(x)
y_logprob = F.log_softmax(y_logits, dim=1)
y_prob = torch.softmax(y_logprob, dim=1) # (batch, y_dim)
# Duplicate y based on x's batch size. Then duplicate x
# This enumerates all possible combination of x with labels (0, 1, ..., 9)
y = np.repeat(np.arange(self.y_dim), x.size(0))
y = x.new(np.eye(self.y_dim)[y])
x = ut.duplicate(x, self.y_dim)
#
# Generate samples.
qm, qv = self.enc.encode(x, y)
z_sample = ut.sample_gaussian(qm, qv)
xprime = self.dec.decode(z_sample, y)
#
# Compute loss.
y_prior = torch.ones_like(y_logprob) / self.y_dim
kl_y = ut.kl_cat(y_prob, y_logprob, y_prior)
#
# Data is duplicated in a way to make the batch dimension second.
kl_z = ut.kl_normal(qm, qv, self.z_prior_m,
self.z_prior_v).view(self.y_dim, -1)
rec = -ut.log_bernoulli_with_logits(x, xprime).view(self.y_dim, -1)
#
# Swap axis where the probabilitiees are to match the new batch dimensions.
nelbo = kl_y + (y_prob.t() * (kl_z + rec)).sum(0)
nelbo = nelbo.mean()
# Test set classification accuracy: 0.8104000091552734
################################################################################
# End of code modification
################################################################################
return nelbo, kl_z, kl_y, rec
def negative_iwae_bound(self, x, iw):
"""
Computes the Importance Weighted Autoencoder Bound
Additionally, we also compute the ELBO KL and reconstruction terms
Args:
x: tensor: (batch, dim): Observations
iw: int: (): Number of importance weighted samples
Returns:
niwae: tensor: (): Negative IWAE bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
################################################################################
# TODO: Modify/complete the code here
# Compute niwae (negative IWAE) with iw importance samples, and the KL
# and Rec decomposition of the Evidence Lower Bound
#
# Outputs should all be scalar
################################################################################
X_dupl = ut.duplicate(x, iw) # Input "x" is duplicated "iw" times
(m, v) = self.enc.encode(X_dupl) # compute the encoder outut
z = ut.sample_gaussian(
m, v) # sample a point from the multivariate Gaussian
logits = self.dec.decode(z) # pass the sampled "Z" through the decoder
# Calculate log Prob of the output x_hat given latent z
ln_P_x_z = ut.log_bernoulli_with_logits(X_dupl, logits)
# Calculate log(P(z))
#ln_P_z = -torch.sum(z*z, -1)/2.0
ln_P_z = ut.log_normal(z, self.z_prior_m, self.z_prior_v)
# Calculate log(Q(z | x)), Conditional Prob of Latent given x
#ln_q_z_x = -torch.sum((z-m)*(z-m)/(2.0*v) + torch.log(v), -1)
ln_q_z_x = ut.log_normal(z, m, v)
exponent = ln_P_x_z + ln_P_z - ln_q_z_x
exponent = exponent.reshape(iw, -1)
L_m_x = ut.log_mean_exp(exponent, 0)
niwae = -torch.mean(L_m_x)
kl = torch.tensor(0)
rec = torch.tensor(0)
################################################################################
# End of code modification
################################################################################
return niwae, kl, rec
def negative_iwae_bound(self, x, iw):
"""
Computes the Importance Weighted Autoencoder Bound
Additionally, we also compute the ELBO KL and reconstruction terms
Args:
x: tensor: (batch, dim): Observations
iw: int: (): Number of importance weighted samples
Returns:
niwae: tensor: (): Negative IWAE bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
m, v = self.enc.encode(x)
# Duplicate
m = ut.duplicate(m, iw)
v = ut.duplicate(v, iw)
x = ut.duplicate(x, iw)
z = ut.sample_gaussian(m, v)
logits = self.dec.decode(z)
# TODO: compute the values below
# Get KL and Rec of elbo again
pm = torch.zeros((m.shape))
pv = torch.ones((v.shape))
kl = ut.kl_normal(m, v, pm, pv)
rec = ut.log_bernoulli_with_logits(x, logits)
# Now get the log mean of the exp of the KL divergence and subtact the
# reconstuction from all of the weighted samples
niwae = ut.log_mean_exp(ut.kl_normal(m, v, pm, pv),
dim=0) - torch.mean(
ut.log_bernoulli_with_logits(x, logits))
return niwae, kl, rec
def negative_iwae_bound(self, x, iw):
"""
Computes the Importance Weighted Autoencoder Bound
Additionally, we also compute the ELBO KL and reconstruction terms
Args:
x: tensor: (batch, dim): Observations
iw: int: (): Number of importance weighted samples
Returns:
niwae: tensor: (): Negative IWAE bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
################################################################################
# TODO: Modify/complete the code here
# Compute niwae (negative IWAE) with iw importance samples, and the KL
# and Rec decomposition of the Evidence Lower Bound
#
# Outputs should all be scalar
################################################################################
N_batches, dims = x.size()
x = ut.duplicate(x, iw)
q_mu, q_var = self.enc.encode(x)
z_samp = ut.sample_gaussian(q_mu, q_var)
logits = self.dec.decode(z_samp)
probs = ut.log_bernoulli_with_logits(x, logits)
log_vals = -ut.kl_normal(q_mu, q_var, torch.zeros_like(q_mu), torch.ones_like(q_var))
# log_vals = ut.log_normal(z_samp, torch.zeros_like(q_mu), torch.ones_like(q_var)) - ut.log_normal(z_samp, q_mu, q_var)
probs = probs + log_vals
niwae = torch.mean(-ut.log_mean_exp(probs.reshape(N_batches, iw), 1))
kl = torch.tensor(0)
rec = torch.tensor(0)
# niwae = kl + rec
################################################################################
# End of code modification
################################################################################
return niwae, kl, rec
def negative_iwae_bound(self, x, iw):
"""
Computes the Importance Weighted Autoencoder Bound
Additionally, we also compute the ELBO KL and reconstruction terms
Args:
x: tensor: (batch, dim): Observations
iw: int: (): Number of importance weighted samples
Returns:
niwae: tensor: (): Negative IWAE bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
################################################################################
# TODO: Modify/complete the code here
# Compute niwae (negative IWAE) with iw importance samples, and the KL
# and Rec decomposition of the Evidence Lower Bound
#
# Outputs should all be scalar
################################################################################
niwae = 0
for i in range(x.size()[0]):
x_i = x[i][:].view(1, x.size()[1])
x_i = ut.duplicate(x_i, iw)
m, v = self.enc.encode(x_i)
z = ut.sample_gaussian(m, v)
x_hat = self.dec.decode(z)
exponent = ut.log_bernoulli_with_logits(x_i, x_hat) + \
ut.log_normal(z, self.z_prior_m.expand(m.size()), self.z_prior_v.expand(v.size())) \
- ut.log_normal(z, m, v)
niwae += -ut.log_mean_exp(exponent, 0).squeeze()
#print(np.std(exponent.data.cpu().numpy()))
#print(exponent.data.cpu().numpy().shape)
niwae = niwae / x.size()[0]
kl = rec = torch.tensor(0)
################################################################################
# End of code modification
################################################################################
return niwae, kl, rec
def negative_elbo_bound(self, x):
"""
Computes the Evidence Lower Bound, KL and, Reconstruction costs
Args:
x: tensor: (batch, dim): Observations
Returns:
nelbo: tensor: (): Negative evidence lower bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
################################################################################
# TODO: Modify/complete the code here
# Compute negative Evidence Lower Bound and its KL_Z, KL_Y and Rec decomposition
#
# To assist you in the vectorization of the summation over y, we have
# the computation of q(y | x) and some tensor tiling code for you.
#
# Note that nelbo = kl_z + kl_y + rec
#
# Outputs should all be scalar
################################################################################
y_logits = self.cls.classify(x)
y_logprob = F.log_softmax(y_logits, dim=1)
y_prob = torch.softmax(y_logprob, dim=1) # (batch, y_dim)
# Duplicate y based on x's batch size. Then duplicate x
# This enumerates all possible combination of x with labels (0, 1, ..., 9)
y = np.repeat(np.arange(self.y_dim), x.size(0))
y = x.new(np.eye(self.y_dim)[y])
x = ut.duplicate(x, self.y_dim)
################################################################################
# End of code modification
################################################################################
return nelbo, kl_z, kl_y, rec
def negative_elbo_bound(self, x):
"""
Computes the Evidence Lower Bound, KL and, Reconstruction costs
Args:
x: tensor: (batch, dim): Observations
Returns:
nelbo: tensor: (): Negative evidence lower bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
################################################################################
# TODO: Modify/complete the code here
# Compute negative Evidence Lower Bound and its KL_Z, KL_Y and Rec decomposition
#
# To assist you in the vectorization of the summation over y, we have
# the computation of q(y | x) and some tensor tiling code for you.
#
# Note that nelbo = kl_z + kl_y + rec
#
# Outputs should all be scalar
################################################################################
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
y_logits = self.cls.classify(x)
y_logprob = F.log_softmax(y_logits, dim=1)
y_prob = torch.softmax(y_logprob, dim=1) # (batch, y_dim)
# Duplicate y based on x's batch size. Then duplicate x
# This enumerates all possible combination of x with labels (0, 1, ..., 9)
y = np.repeat(np.arange(self.y_dim), x.size(0))
y = x.new(np.eye(self.y_dim)[y]) #1000,10. 0,100,200 dupe
x = ut.duplicate(x, self.y_dim) #1000,784. 0,100,200 dupe
#100x10
y_prior = torch.tensor([0.1]).expand_as(y_prob).to(device)
y_logprior = torch.log(y_prior)
#(batch size,)
kl_ys = ut.kl_cat(y_prob, y_logprob, y_logprior)
kl_y = torch.mean(kl_ys)
#1000 x 64. Still 0,100,200 corresponding...
zqm, zqv = self.enc.encode(x, y)
zpm = self.z_prior_m.expand_as(zqm)
zpv = self.z_prior_v.expand_as(zqv)
#so the zpm, zpv go as x quickly, y slowly
#equivalent to y being the 0th dimension
#(batch_size * y_dim,)
kl_zs_flat = ut.kl_normal(zqm, zqv, zpm, zpv)
kl_zs = kl_zs_flat.reshape(10,100).t()
kl_zs_weighted = kl_zs * y_prob
batch_kl_zs = kl_zs_weighted.sum(1)
kl_z = batch_kl_zs.mean()
#1000 x 64
z = ut.sample_gaussian(zqm, zqv)
#1000 x 784
probs = self.dec.decode(z, y)
#(batch_size * y_dim,)
recs_flat = -1.0 * ut.log_bernoulli_with_logits(x, probs)
recs = recs_flat.reshape(10,100).t()
recs_weighted = recs * y_prob
batch_recs = recs_weighted.sum(1)
rec = batch_recs.mean()
nelbos = kl_ys + batch_kl_zs + batch_recs
nelbo = torch.mean(nelbos)
################################################################################
# End of code modification
################################################################################
return nelbo, kl_z, kl_y, rec
def negative_elbo_bound(self, x):
"""
Computes the Evidence Lower Bound, KL and, Reconstruction costs
Args:
x: tensor: (batch, dim): Observations
Returns:
nelbo: tensor: (): Negative evidence lower bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
################################################################################
# TODO: Modify/complete the code here
# Compute negative Evidence Lower Bound and its KL_Z, KL_Y and Rec decomposition
#
# To assist you in the vectorization of the summation over y, we have
# the computation of q(y | x) and some tensor tiling code for you.
#
# Note that nelbo = kl_z + kl_y + rec
#
# Outputs should all be scalar
################################################################################
N_batches, dims = x.size()
y_logits = self.cls.classify(x)
y_logprob = F.log_softmax(y_logits, dim=1)
y_prob = torch.softmax(y_logprob, dim=1) # (batch, y_dim)
# Duplicate y based on x's batch size. Then duplicate x
# This enumerates all possible combination of x with labels (0, 1, ..., 9)
y = np.repeat(np.arange(self.y_dim), x.size(0))
y = x.new(np.eye(self.y_dim)[y])
x = ut.duplicate(x, self.y_dim)
q_mu, q_var = self.enc.encode(x, y)
z_samp = ut.sample_gaussian(q_mu, q_var)
logits = self.dec.decode(z_samp, y)
rec_ls = -ut.log_bernoulli_with_logits(x, logits)
rec = torch.mean(
torch.sum(y_prob * rec_ls.reshape(N_batches, -1), dim=1))
kl_y = torch.mean(
ut.kl_cat(y_prob, y_logprob,
torch.log(torch.ones_like(y_prob) / self.y_dim)))
kl_z_ls = ut.kl_normal(q_mu, q_var, torch.zeros_like(q_mu),
torch.ones_like(q_var))
kl_z = torch.mean(
torch.sum(y_prob * kl_z_ls.reshape(N_batches, -1), dim=1))
nelbo = kl_z + kl_y + rec
################################################################################
# End of code modification
################################################################################
return nelbo, kl_z, kl_y, rec
def negative_iwae_bound(self, x, iw):
"""
Computes the Importance Weighted Autoencoder Bound
Additionally, we also compute the ELBO KL and reconstruction terms
Args:
x: tensor: (batch, dim): Observations
iw: int: (): Number of importance weighted samples
Returns:
niwae: tensor: (): Negative IWAE bound
kl: tensor: (): ELBO KL divergence to prior
rec: tensor: (): ELBO Reconstruction term
"""
################################################################################
# TODO: Modify/complete the code here
# Compute niwae (negative IWAE) with iw importance samples, and the KL
# and Rec decomposition of the Evidence Lower Bound
#
# Outputs should all be scalar
################################################################################
# Compute the mixture of Gaussian prior
prior = ut.gaussian_parameters(self.z_pre, dim=1)
prior_m, prior_v = prior
batch = x.shape[0]
multi_x = ut.duplicate(x, iw)
qm, qv = self.enc.encode(x)
multi_qm = ut.duplicate(qm, iw)
multi_qv = ut.duplicate(qv, iw)
# z will be (batch*iw x z_dim)
# with sampled z's for a given x non-contiguous!
z = ut.sample_gaussian(multi_qm, multi_qv)
probs = self.dec.decode(z)
recs = ut.log_bernoulli_with_logits(multi_x, probs)
rec = -1.0 * torch.mean(recs)
multi_m = prior_m.expand(batch * iw, *prior_m.shape[1:])
multi_v = prior_v.expand(batch * iw, *prior_v.shape[1:])
z_priors = ut.log_normal_mixture(z, multi_m, multi_v)
x_posteriors = recs
z_posteriors = ut.log_normal(z, multi_qm, multi_qv)
kls = z_posteriors - z_priors
kl = torch.mean(kls)
log_ratios = z_priors + x_posteriors - z_posteriors
# Should be (batch*iw, z_dim), batch ratios non contiguous
unflat_log_ratios = log_ratios.reshape(iw, batch)
niwaes = ut.log_mean_exp(unflat_log_ratios, 0)
niwae = -1.0 * torch.mean(niwaes)
################################################################################
# End of code modification
################################################################################
return niwae, kl, rec
