def lower_bound_and_log_likelihood(relaxed=False):
def log_joint(observed):
model = vae(observed, n, n_x, n_z, n_k, tau_p, n_particles,
relaxed)
log_pz, log_px_z = model.local_log_prob(['z', 'x'])
return log_pz + log_px_z
variational = q_net({}, x, n_z, n_k, tau_q, n_particles, relaxed)
qz_samples, log_qz = variational.query('z',
outputs=True,
local_log_prob=True)
lower_bound = zs.variational.elbo(log_joint,
observed={'x': x_obs},
latent={'z': [qz_samples, log_qz]},
axis=0)
cost = tf.reduce_mean(lower_bound.sgvb())
lower_bound = tf.reduce_mean(lower_bound)
# Importance sampling estimates of marginal log likelihood
is_log_likelihood = tf.reduce_mean(
zs.is_loglikelihood(log_joint, {'x': x_obs},
{'z': [qz_samples, log_qz]},
axis=0))
return cost, lower_bound, is_log_likelihood
variational = q_net({}, x, n_z, n_particles, is_training)
qz_samples, log_qz = variational.query('z', outputs=True,
local_log_prob=True)
# TODO: add tests for repeated calls of flows
qz_samples, log_qz = zs.planar_normalizing_flow(qz_samples, log_qz,
n_iters=n_planar_flows)
qz_samples, log_qz = zs.planar_normalizing_flow(qz_samples, log_qz,
n_iters=n_planar_flows)
lower_bound = tf.reduce_mean(
zs.sgvb(log_joint, {'x': x_obs}, {'z': [qz_samples, log_qz]}, axis=0))
# Importance sampling estimates of log likelihood:
# Fast, used for evaluation during training
is_log_likelihood = tf.reduce_mean(
zs.is_loglikelihood(log_joint, {'x': x_obs},
{'z': [qz_samples, log_qz]}, axis=0))
learning_rate_ph = tf.placeholder(tf.float32, shape=[], name='lr')
optimizer = tf.train.AdamOptimizer(learning_rate_ph, epsilon=1e-4)
grads = optimizer.compute_gradients(-lower_bound)
infer = optimizer.apply_gradients(grads)
params = tf.trainable_variables()
for i in params:
print(i.name, i.get_shape())
saver = tf.train.Saver(max_to_keep=10)
# Run the inference
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
def main():
tf.set_random_seed(1234)
np.random.seed(1234)
# Load MNIST
data_path = os.path.join(conf.data_dir, 'mnist.pkl.gz')
x_train, t_train, x_valid, t_valid, x_test, t_test = \
dataset.load_mnist_realval(data_path)
x_train = np.vstack([x_train, x_valid])
x_test = np.random.binomial(1, x_test, size=x_test.shape)
x_dim = x_train.shape[1]
# Define model/inference parameters
z_dim = 40
n_planar_flows = 10
# Build the computation graph
n_particles = tf.placeholder(tf.int32, shape=[], name="n_particles")
x_input = tf.placeholder(tf.float32, shape=[None, x_dim], name="x")
x = tf.cast(tf.less(tf.random_uniform(tf.shape(x_input)), x_input),
tf.int32)
n = tf.placeholder(tf.int32, shape=[], name="n")
model = build_gen(n, x_dim, z_dim, n_particles)
q_net = build_q_net(x, z_dim, n_particles)
qz_samples, log_qz = q_net.query('z', outputs=True, local_log_prob=True)
# TODO: add tests for repeated calls of flows
qz_samples, log_qz = zs.planar_normalizing_flow(qz_samples,
log_qz,
n_iters=n_planar_flows)
qz_samples, log_qz = zs.planar_normalizing_flow(qz_samples,
log_qz,
n_iters=n_planar_flows)
lower_bound = zs.variational.elbo(model,
observed={"x": x},
latent={"z": [qz_samples, log_qz]},
axis=0)
cost = tf.reduce_mean(lower_bound.sgvb())
lower_bound = tf.reduce_mean(lower_bound)
# Importance sampling estimates of marginal log likelihood
is_log_likelihood = tf.reduce_mean(
zs.is_loglikelihood(model, {'x': x}, {'z': [qz_samples, log_qz]},
axis=0))
optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
infer_op = optimizer.minimize(cost)
# Define training/evaluation parameters
epochs = 3000
batch_size = 128
iters = x_train.shape[0] // batch_size
test_freq = 10
test_batch_size = 400
test_iters = x_test.shape[0] // test_batch_size
# Run the inference
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(1, epochs + 1):
time_epoch = -time.time()
np.random.shuffle(x_train)
lbs = []
for t in range(iters):
x_batch = x_train[t * batch_size:(t + 1) * batch_size]
_, lb = sess.run([infer_op, lower_bound],
feed_dict={
x_input: x_batch,
n_particles: 1,
n: batch_size
})
lbs.append(lb)
time_epoch += time.time()
print('Epoch {} ({:.1f}s): Lower bound = {}'.format(
epoch, time_epoch, np.mean(lbs)))
if epoch % test_freq == 0:
time_test = -time.time()
test_lbs = []
test_lls = []
for t in range(test_iters):
test_x_batch = x_test[t * test_batch_size:(t + 1) *
test_batch_size]
test_lb = sess.run(lower_bound,
feed_dict={
x: test_x_batch,
n_particles: 1,
n: test_batch_size
})
test_ll = sess.run(is_log_likelihood,
feed_dict={
x: test_x_batch,
n_particles: 1000,
n: test_batch_size
})
test_lbs.append(test_lb)
test_lls.append(test_ll)
time_test += time.time()
print('>>> TEST ({:.1f}s)'.format(time_test))
print('>> Test lower bound = {}'.format(np.mean(test_lbs)))
print('>> Test log likelihood (IS) = {}'.format(
np.mean(test_lls)))
def main():
data_path = "/home/zhikun/PycharmProjects/data/MNIST/mnist.pkl.gz"
# the file comes from http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz
x_train, t_train, x_valid, t_valid, x_test, t_test = load_mnist(data_path)
x_train = np.vstack([x_train, x_valid])
y_train = np.vstack([t_train, t_valid])
x_test = np.random.binomial(1, x_test, size=x_test.shape)
x_dim = x_train.shape[1]
y_dim = y_train.shape[1]
# Define model parameters
z_dim = 40
n_particles = tf.placeholder(tf.int32, shape=[], name="n_particles")
x_input = tf.placeholder(tf.float32, shape=[None, x_dim], name="x")
x = tf.cast(tf.less(tf.random_uniform(tf.shape(x_input)), x_input),
tf.int32)
y = tf.placeholder(tf.float32, shape=[None, 10], name="y")
n = tf.placeholder(tf.int32, shape=[], name="n")
model = build_gen(x_dim, z_dim, y, n, n_particles)
variational = build_q_net(x, z_dim, y, n_particles)
lower_bound = zs.variational.elbo(model, {"x": x},
variational=variational,
axis=0)
cost = tf.reduce_mean(lower_bound.sgvb())
lower_bound = tf.reduce_mean(lower_bound)
# # Importance sampling estimates of marginal log likelihood
is_log_likelihood = tf.reduce_mean(
zs.is_loglikelihood(model, {"x": x}, proposal=variational, axis=0))
optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
infer_op = optimizer.minimize(cost)
# Random generation
y_observe = tf.placeholder(tf.float32, shape=[None, 10], name="y_observe")
x_gen = tf.reshape(model.observe(y=y_observe)["x_mean"], [-1, 28, 28, 1])
# Define training/evaluation parameters
epochs = 3000
batch_size = 128
iters = x_train.shape[0] // batch_size
save_freq = 10
test_freq = 10
test_batch_size = 50
test_iters = x_test.shape[0] // test_batch_size
result_path = "results/c-vae"
# Run the inference
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(1, epochs + 1):
time_epoch = -time.time()
lbs = []
for t in range(iters):
x_batch = x_train[t * batch_size:(t + 1) * batch_size]
y_batch = y_train[t * batch_size:(t + 1) * batch_size]
_, lb = sess.run([infer_op, lower_bound],
feed_dict={
x_input: x_batch,
y: y_batch,
n_particles: 1,
n: batch_size
})
lbs.append(lb)
time_epoch += time.time()
print("Epoch {} ({:.1f}s): Lower bound = {}".format(
epoch, time_epoch, np.mean(lbs)))
if epoch % test_freq == 0:
time_test = -time.time()
test_lbs, test_lls = [], []
for t in range(test_iters):
test_x_batch = x_test[t * test_batch_size:(t + 1) *
test_batch_size]
test_y_batch = t_test[t * test_batch_size:(t + 1) *
test_batch_size]
test_lb = sess.run(lower_bound,
feed_dict={
x: test_x_batch,
y: test_y_batch,
n_particles: 1,
n: test_batch_size
})
test_ll = sess.run(is_log_likelihood,
feed_dict={
x: test_x_batch,
y: test_y_batch,
n_particles: 1000,
n: test_batch_size
})
test_lbs.append(test_lb)
test_lls.append(test_ll)
time_test += time.time()
print(">>> TEST ({:.1f}s)".format(time_test))
print(">> Test lower bound = {}".format(np.mean(test_lbs)))
print('>> Test log likelihood (IS) = {}'.format(
np.mean(test_lls)))
if epoch % save_freq == 0:
y_index = np.repeat(np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
10)
y_index = np.eye(10)[y_index.reshape(-1)]
images = sess.run(x_gen,
feed_dict={
y_observe: y_index,
y: y_index,
n: 100,
n_particles: 1
})
name = os.path.join(result_path,
"c-vae.epoch.{}.png".format(epoch))
save_image_collections(images, name)
def main():
# Load MNIST
data_path = os.path.join(conf.data_dir, 'mnist.pkl.gz')
x_train, t_train, x_valid, t_valid, x_test, t_test = \
dataset.load_mnist_realval(data_path)
x_train = np.vstack([x_train, x_valid])
x_test = np.random.binomial(1, x_test, size=x_test.shape)
x_dim = x_train.shape[1]
# Define model parameters
z_dim = 40
# Build the computation graph
n_particles = tf.placeholder(tf.int32, shape=[], name='n_particles')
x_input = tf.placeholder(tf.float32, shape=[None, x_dim], name='x')
x = tf.to_int32(tf.less(tf.random_uniform(tf.shape(x_input)), x_input))
n = tf.shape(x)[0]
def log_joint(observed):
model = vae(observed, x_dim, z_dim, n, n_particles)
log_pz, log_px_z = model.local_log_prob(['z', 'x'])
return log_pz + log_px_z
variational = q_net({'x': x}, x_dim, z_dim, n_particles)
qz_samples, log_qz = variational.query('z', outputs=True,
local_log_prob=True)
lower_bound = zs.variational.elbo(log_joint,
observed={'x': x},
latent={'z': [qz_samples, log_qz]},
axis=0)
cost = tf.reduce_mean(lower_bound.sgvb())
lower_bound = tf.reduce_mean(lower_bound)
# Importance sampling estimates of marginal log likelihood
is_log_likelihood = tf.reduce_mean(
zs.is_loglikelihood(log_joint, {'x': x},
{'z': [qz_samples, log_qz]}, axis=0))
optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
infer_op = optimizer.minimize(cost)
# Generate images
n_gen = 100
x_mean = vae({}, x_dim, z_dim, n_gen).outputs('x_mean')
x_gen = tf.reshape(x_mean, [-1, 28, 28, 1])
# Define training/evaluation parameters
epochs = 3000
batch_size = 128
iters = x_train.shape[0] // batch_size
save_freq = 10
test_freq = 10
test_batch_size = 400
test_iters = x_test.shape[0] // test_batch_size
result_path = "results/vae"
# Run the inference
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(1, epochs + 1):
time_epoch = -time.time()
np.random.shuffle(x_train)
lbs = []
for t in range(iters):
x_batch = x_train[t * batch_size:(t + 1) * batch_size]
_, lb = sess.run([infer_op, lower_bound],
feed_dict={x_input: x_batch,
n_particles: 1})
lbs.append(lb)
time_epoch += time.time()
print('Epoch {} ({:.1f}s): Lower bound = {}'.format(
epoch, time_epoch, np.mean(lbs)))
if epoch % test_freq == 0:
time_test = -time.time()
test_lbs = []
test_lls = []
for t in range(test_iters):
test_x_batch = x_test[t * test_batch_size:
(t + 1) * test_batch_size]
test_lb = sess.run(lower_bound,
feed_dict={x: test_x_batch,
n_particles: 1})
test_ll = sess.run(is_log_likelihood,
feed_dict={x: test_x_batch,
n_particles: 1000})
test_lbs.append(test_lb)
test_lls.append(test_ll)
time_test += time.time()
print('>>> TEST ({:.1f}s)'.format(time_test))
print('>> Test lower bound = {}'.format(np.mean(test_lbs)))
print('>> Test log likelihood (IS) = {}'.format(
np.mean(test_lls)))
if epoch % save_freq == 0:
images = sess.run(x_gen)
name = os.path.join(result_path,
"vae.epoch.{}.png".format(epoch))
save_image_collections(images, name)
def main():
# Load MNIST
data_path = os.path.join(conf.data_dir, "mnist.pkl.gz")
x_train, t_train, x_valid, t_valid, x_test, t_test = \
dataset.load_mnist_realval(data_path)
x_train = np.vstack([x_train, x_valid])
x_test = np.random.binomial(1, x_test, size=x_test.shape)
x_dim = x_train.shape[1]
# Define model parameters
z_dim = 40
# Build the computation graph
n_particles = tf.placeholder(tf.int32, shape=[], name="n_particles")
x_input = tf.placeholder(tf.float32, shape=[None, x_dim], name="x")
x = tf.cast(tf.less(tf.random_uniform(tf.shape(x_input)), x_input),
tf.int32)
n = tf.placeholder(tf.int32, shape=[], name="n")
model = build_gen(x_dim, z_dim, n, n_particles)
variational = build_q_net(x, z_dim, n_particles)
lower_bound = zs.variational.elbo(model, {"x": x},
variational=variational,
axis=0)
cost = tf.reduce_mean(lower_bound.sgvb())
lower_bound = tf.reduce_mean(lower_bound)
# # Importance sampling estimates of marginal log likelihood
is_log_likelihood = tf.reduce_mean(
zs.is_loglikelihood(model, {"x": x}, proposal=variational, axis=0))
optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
infer_op = optimizer.minimize(cost)
# Random generation
x_gen = tf.reshape(model.observe()["x_mean"], [-1, 28, 28, 1])
# Define training/evaluation parameters
epochs = 3000
batch_size = 128
iters = x_train.shape[0] // batch_size
save_freq = 10
test_freq = 10
test_batch_size = 400
test_iters = x_test.shape[0] // test_batch_size
result_path = "results/vae"
# Run the inference
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(1, epochs + 1):
time_epoch = -time.time()
np.random.shuffle(x_train)
lbs = []
for t in range(iters):
x_batch = x_train[t * batch_size:(t + 1) * batch_size]
_, lb = sess.run([infer_op, lower_bound],
feed_dict={
x_input: x_batch,
n_particles: 1,
n: batch_size
})
lbs.append(lb)
time_epoch += time.time()
print("Epoch {} ({:.1f}s): Lower bound = {}".format(
epoch, time_epoch, np.mean(lbs)))
if epoch % test_freq == 0:
time_test = -time.time()
test_lbs, test_lls = [], []
for t in range(test_iters):
test_x_batch = x_test[t * test_batch_size:(t + 1) *
test_batch_size]
test_lb = sess.run(lower_bound,
feed_dict={
x: test_x_batch,
n_particles: 1,
n: test_batch_size
})
test_ll = sess.run(is_log_likelihood,
feed_dict={
x: test_x_batch,
n_particles: 1000,
n: test_batch_size
})
test_lbs.append(test_lb)
test_lls.append(test_ll)
time_test += time.time()
print(">>> TEST ({:.1f}s)".format(time_test))
print(">> Test lower bound = {}".format(np.mean(test_lbs)))
print('>> Test log likelihood (IS) = {}'.format(
np.mean(test_lls)))
if epoch % save_freq == 0:
images = sess.run(x_gen, feed_dict={n: 100, n_particles: 1})
name = os.path.join(result_path,
"vae.epoch.{}.png".format(epoch))
save_image_collections(images, name)
def main():
# Load MNIST
data_path = os.path.join(conf.data_dir, 'mnist.pkl.gz')
x_train, t_train, x_valid, t_valid, x_test, t_test = \
dataset.load_mnist_realval(data_path)
x_train = np.vstack([x_train, x_valid])
x_test = np.random.binomial(1, x_test, size=x_test.shape)
x_dim = x_train.shape[1]
# Define model parameters
z_dim = 40
# Build the computation graph
is_training = tf.placeholder(tf.bool, shape=[], name='is_training')
n_particles = tf.placeholder(tf.int32, shape=[], name='n_particles')
x_input = tf.placeholder(tf.float32, shape=[None, x_dim], name='x')
x = tf.to_int32(tf.less(tf.random_uniform(tf.shape(x_input)), x_input))
n = tf.shape(x)[0]
def log_joint(observed):
model = vae(observed, n, x_dim, z_dim, n_particles, is_training)
log_pz, log_px_z = model.local_log_prob(['z', 'x'])
return log_pz + log_px_z
variational = q_net(x, z_dim, n_particles, is_training)
qz_samples, log_qz = variational.query('z',
outputs=True,
local_log_prob=True)
cx = tf.expand_dims(baseline_net(x), 0)
lower_bound = zs.variational.elbo(log_joint,
observed={'x': x},
latent={'z': [qz_samples, log_qz]},
axis=0)
cost, baseline_cost = lower_bound.reinforce(baseline=cx)
cost = tf.reduce_mean(cost + baseline_cost)
lower_bound = tf.reduce_mean(lower_bound)
# Importance sampling estimates of marginal log likelihood
is_log_likelihood = tf.reduce_mean(
zs.is_loglikelihood(log_joint, {'x': x}, {'z': [qz_samples, log_qz]},
axis=0))
optimizer = tf.train.AdamOptimizer(0.001)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
infer_op = optimizer.minimize(cost)
# Define training/evaluation parameters
epochs = 3000
batch_size = 128
iters = x_train.shape[0] // batch_size
test_freq = 10
test_batch_size = 400
test_iters = x_test.shape[0] // test_batch_size
# Run the inference
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(1, epochs + 1):
time_epoch = -time.time()
np.random.shuffle(x_train)
lbs = []
for t in range(iters):
x_batch = x_train[t * batch_size:(t + 1) * batch_size]
_, lb = sess.run([infer_op, lower_bound],
feed_dict={
x_input: x_batch,
is_training: True,
n_particles: 1
})
lbs.append(lb)
time_epoch += time.time()
print('Epoch {} ({:.1f}s): Lower bound = {}'.format(
epoch, time_epoch, np.mean(lbs)))
if epoch % test_freq == 0:
time_test = -time.time()
test_lbs = []
test_lls = []
for t in range(test_iters):
test_x_batch = x_test[t * test_batch_size:(t + 1) *
test_batch_size]
test_lb = sess.run(lower_bound,
feed_dict={
x: test_x_batch,
is_training: False,
n_particles: 1
})
test_ll = sess.run(is_log_likelihood,
feed_dict={
x: test_x_batch,
is_training: False,
n_particles: 1000
})
test_lbs.append(test_lb)
test_lls.append(test_ll)
time_test += time.time()
print('>>> TEST ({:.1f}s)'.format(time_test))
print('>> Test lower bound = {}'.format(np.mean(test_lbs)))
print('>> Test log likelihood (IS) = {}'.format(
np.mean(test_lls)))
local_log_prob=True)
qh1_samples, log_qh1 = variational.query('h1',
outputs=True,
local_log_prob=True)
cost, lower_bound = zs.rws(log_joint, {'x': x_obs}, {
'h3': [qh3_samples, log_qh3],
'h2': [qh2_samples, log_qh2],
'h1': [qh1_samples, log_qh1]
},
axis=0)
lower_bound = tf.reduce_mean(lower_bound)
cost = tf.reduce_mean(cost)
log_likelihood = tf.reduce_mean(
zs.is_loglikelihood(log_joint, {'x': x_obs}, {
'h3': [qh3_samples, log_qh3],
'h2': [qh2_samples, log_qh2],
'h1': [qh1_samples, log_qh1]
},
axis=0))
learning_rate_ph = tf.placeholder(tf.float32, shape=[], name='lr')
optimizer = tf.train.AdamOptimizer(learning_rate_ph, epsilon=1e-4)
grads = optimizer.compute_gradients(cost)
infer = optimizer.apply_gradients(grads)
params = tf.trainable_variables()
for i in params:
print(i.name, i.get_shape())
# Run the inference
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# 'z_1': [qz_samples1, log_qz1],'z_2': [qz_samples2, log_qz2]}, axis=0))
_, x_recon = VLAE(
{
'z_0': qz_samples0,
'z_1': qz_samples1,
'z_2': qz_samples2
}, n, n_x, n_z_0, n_z_1, n_z_2, n_particles, is_training)
lower_bound = tf.reduce_mean(
tf.square(tf.reshape(x_recon, [-1, n_x]) - tf.cast(x, tf.float32)))
# Importance sampling estimates of marginal log likelihood
is_log_likelihood = tf.reduce_mean(
zs.is_loglikelihood(log_joint, {'x': x_obs}, {
'z_0': [qz_samples0, log_qz0],
'z_1': [qz_samples1, log_qz1],
'z_2': [qz_samples2, log_qz2]
},
axis=0))
learning_rate_ph = tf.placeholder(tf.float32, shape=[], name='lr')
optimizer = tf.train.AdamOptimizer(learning_rate_ph, epsilon=1e-4)
grads = optimizer.compute_gradients(lower_bound)
infer = optimizer.apply_gradients(grads)
params = tf.trainable_variables()
for i in params:
print(i.name, i.get_shape())
saver = tf.train.Saver(max_to_keep=10)
# Run the inference
def main():
# Load MNIST
data_path = os.path.join(conf.data_dir, "mnist.pkl.gz")
x_train, t_train, x_valid, t_valid, x_test, t_test = \
dataset.load_mnist_realval(data_path)
x_train = np.vstack([x_train, x_valid])
x_test = np.random.binomial(1, x_test, size=x_test.shape)
# Define model parameters
x_dim = x_train.shape[1]
z_dim = 40
# Build the computation graph
# how many samples to draw from the distribution, more samples, more accuracy
n_particles = tf.placeholder(tf.int32, shape=[], name="n_particles")
# input data to feed the variational
x_input = tf.placeholder(tf.float32, shape=[None, x_dim], name="x")
x = tf.cast(tf.less(tf.random_uniform(tf.shape(x_input)), x_input),
tf.int32)
# batch size
n = tf.placeholder(tf.int32, shape=[], name="n")
# add random noise to the variance of the q_model so to
# get more various samples when generating new digits
std_noise = tf.placeholder_with_default(0., shape=[], name="std_noise")
# build the model (encoder) and the q_model (variational or decoder)
model = build_gen(x_dim, z_dim, n, n_particles)
q_model = build_q_net(x, z_dim, n_particles, std_noise)
variational = q_model.observe()
# calculate ELBO
lower_bound = zs.variational.elbo(model, {"x": x},
variational=variational,
axis=0)
cost = tf.reduce_mean(lower_bound.sgvb())
lower_bound = tf.reduce_mean(lower_bound)
# calculate marginal log likelihood
is_log_likelihood = tf.reduce_mean(
zs.is_loglikelihood(model, {"x": x}, proposal=variational, axis=0))
# optimizer
optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
infer_op = optimizer.minimize(cost)
# define training/evaluation parameters
epochs = 1000
batch_size = 128
iters = x_train.shape[0] // batch_size
test_freq = 100
test_batch_size = 400
test_iters = x_test.shape[0] // test_batch_size
result_path = "results/vae_digits"
checkpoints_path = "checkpoints/vae_digits"
# used to save checkpoints during training
saver = tf.train.Saver(max_to_keep=10)
save_model_freq = 100
# run the inference
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# restore the model parameters from the latest checkpoint
ckpt_file = tf.train.latest_checkpoint(checkpoints_path)
begin_epoch = 1
if ckpt_file is not None:
print('Restoring model from {}...'.format(ckpt_file))
begin_epoch = int(ckpt_file.split('.')[-2]) + 1
saver.restore(sess, ckpt_file)
# begin training
for epoch in range(begin_epoch, epochs + 1):
time_epoch = -time.time()
np.random.shuffle(x_train)
lbs = []
for t in range(iters):
x_batch = x_train[t * batch_size:(t + 1) * batch_size]
_, lb = sess.run([infer_op, lower_bound],
feed_dict={
x_input: x_batch,
n_particles: 1,
n: batch_size
})
lbs.append(lb)
time_epoch += time.time()
print("Epoch {} ({:.1f}s): Lower bound = {}".format(
epoch, time_epoch, np.mean(lbs)))
# test marginal log likelihood
if epoch % test_freq == 0:
time_test = -time.time()
test_lbs, test_lls = [], []
for t in range(test_iters):
test_x_batch = x_test[t * test_batch_size:(t + 1) *
test_batch_size]
test_lb = sess.run(lower_bound,
feed_dict={
x: test_x_batch,
n_particles: 1,
n: test_batch_size
})
test_ll = sess.run(is_log_likelihood,
feed_dict={
x: test_x_batch,
n_particles: 1000,
n: test_batch_size
})
test_lbs.append(test_lb)
test_lls.append(test_ll)
time_test += time.time()
print(">>> TEST ({:.1f}s)".format(time_test))
print(">> Test lower bound = {}".format(np.mean(test_lbs)))
print('>> Test log likelihood (IS) = {}'.format(
np.mean(test_lls)))
# save model parameters
if epoch % save_model_freq == 0:
print('Saving model...')
save_path = os.path.join(checkpoints_path,
"vae.epoch.{}.ckpt".format(epoch))
if not os.path.exists(os.path.dirname(save_path)):
os.makedirs(os.path.dirname(save_path))
saver.save(sess, save_path)
print('Done')
# random generation of images from latent distribution
x_gen = tf.reshape(model.observe()["x_mean"], [-1, 28, 28, 1])
images = sess.run(x_gen, feed_dict={n: 100, n_particles: 1})
name = os.path.join(result_path, "random_samples.png")
save_image_collections(images, name)
# the following code generates 100 samples for each number
test_n = [3, 2, 1, 90, 95, 23, 11, 0, 84, 7]
# map each digit to a corresponding sample from the test set so we can generate similar digits
for i in range(len(test_n)):
# get latent distribution from the variational giving as input a fixed sample from the dataset
z = q_model.observe(x=np.expand_dims(x_test[test_n[i]], 0))['z']
# run the computation graph adding noise to computed variance to get different output samples
latent = sess.run(z,
feed_dict={
x_input:
np.expand_dims(x_test[test_n[i]], 0),
n: 1,
n_particles: 100,
std_noise: 0.7
})
# get the image from the model giving as input the latent distribution z
x_gen = tf.reshape(
model.observe(z=latent)["x_mean"], [-1, 28, 28, 1])
images = sess.run(x_gen, feed_dict={})
name = os.path.join(result_path, "{}.png".format(i))
save_image_collections(images, name)
