def main(_):
ed.set_seed(42)
# DATA. MNIST batches are fed at training time.
(x_train, _), (x_test, _) = mnist(FLAGS.data_dir)
x_train_generator = generator(x_train, FLAGS.M)
x_ph = tf.placeholder(tf.float32, [FLAGS.M, 784])
z_ph = tf.placeholder(tf.float32, [FLAGS.M, FLAGS.d])
# MODEL
with tf.variable_scope("Gen"):
xf = gen_data(z_ph, FLAGS.hidden_units)
zf = gen_latent(x_ph, FLAGS.hidden_units)
# INFERENCE:
optimizer = tf.train.AdamOptimizer()
optimizer_d = tf.train.AdamOptimizer()
inference = ed.BiGANInference(
latent_vars={zf: z_ph}, data={xf: x_ph},
discriminator=discriminative_network)
inference.initialize(
optimizer=optimizer, optimizer_d=optimizer_d, n_iter=100000, n_print=3000)
sess = ed.get_session()
init_op = tf.global_variables_initializer()
sess.run(init_op)
idx = np.random.randint(FLAGS.M, size=16)
i = 0
for t in range(inference.n_iter):
if t % inference.n_print == 1:
samples = sess.run(xf, feed_dict={z_ph: z_batch})
samples = samples[idx, ]
fig = plot(samples)
plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format(
'Generated', str(i).zfill(3)), bbox_inches='tight')
plt.close(fig)
fig = plot(x_batch[idx, ])
plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format(
'Base', str(i).zfill(3)), bbox_inches='tight')
plt.close(fig)
zsam = sess.run(zf, feed_dict={x_ph: x_batch})
reconstructions = sess.run(xf, feed_dict={z_ph: zsam})
reconstructions = reconstructions[idx, ]
fig = plot(reconstructions)
plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format(
'Reconstruct', str(i).zfill(3)), bbox_inches='tight')
plt.close(fig)
i += 1
x_batch = next(x_train_generator)
z_batch = np.random.normal(0, 1, [FLAGS.M, FLAGS.d])
info_dict = inference.update(feed_dict={x_ph: x_batch, z_ph: z_batch})
inference.print_progress(info_dict)
def main(_):
ed.set_seed(42)
# DATA. MNIST batches are fed at training time.
(x_train, _), (x_test, _) = mnist(FLAGS.data_dir)
x_train_generator = generator(x_train, FLAGS.M)
# MODEL
# Define a subgraph of the full model, corresponding to a minibatch of
# size M.
z = Normal(loc=tf.zeros([FLAGS.M, FLAGS.d]),
scale=tf.ones([FLAGS.M, FLAGS.d]))
hidden = tf.layers.dense(z, 256, activation=tf.nn.relu)
x = Bernoulli(logits=tf.layers.dense(hidden, 28 * 28))
# INFERENCE
# Define a subgraph of the variational model, corresponding to a
# minibatch of size M.
x_ph = tf.placeholder(tf.int32, [FLAGS.M, 28 * 28])
hidden = tf.layers.dense(tf.cast(x_ph, tf.float32),
256,
activation=tf.nn.relu)
qz = Normal(loc=tf.layers.dense(hidden, FLAGS.d),
scale=tf.layers.dense(hidden,
FLAGS.d,
activation=tf.nn.softplus))
# Bind p(x, z) and q(z | x) to the same TensorFlow placeholder for x.
inference = ed.KLqp({z: qz}, data={x: x_ph})
optimizer = tf.train.RMSPropOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer)
tf.global_variables_initializer().run()
n_iter_per_epoch = x_train.shape[0] // FLAGS.M
for epoch in range(1, FLAGS.n_epoch + 1):
print("Epoch: {0}".format(epoch))
avg_loss = 0.0
pbar = Progbar(n_iter_per_epoch)
for t in range(1, n_iter_per_epoch + 1):
pbar.update(t)
x_batch = next(x_train_generator)
info_dict = inference.update(feed_dict={x_ph: x_batch})
avg_loss += info_dict['loss']
# Print a lower bound to the average marginal likelihood for an
# image.
avg_loss /= n_iter_per_epoch
avg_loss /= FLAGS.M
print("-log p(x) <= {:0.3f}".format(avg_loss))
# Prior predictive check.
images = x.eval()
for m in range(FLAGS.M):
imsave(
os.path.join(FLAGS.out_dir, '%d.png') % m,
images[m].reshape(28, 28))
def __init__(self, data_path, flatten=True):
""" Take in a path to (down)load data."""
(train_x, train_y), (test_x, test_y) = mnist(data_path)
if not flatten:
train_x = train_x.reshape(-1, 28, 28, 1)
test_x = test_x.reshape(-1, 28, 28, 1)
self.train = (binarize(train_x), one_hot(train_y))
self.test = (binarize(test_x), one_hot(test_y))
self.batch_size = 0
self.epoch_size = train_y.shape[0]
def main(_):
ed.set_seed(42)
# DATA. MNIST batches are fed at training time.
(x_train, _), (x_test, _) = mnist(FLAGS.data_dir)
x_train_generator = generator(x_train, FLAGS.M)
# MODEL
# Define a subgraph of the full model, corresponding to a minibatch of
# size M.
z = Normal(loc=tf.zeros([FLAGS.M, FLAGS.d]),
scale=tf.ones([FLAGS.M, FLAGS.d]))
hidden = tf.layers.dense(z, 256, activation=tf.nn.relu)
x = Bernoulli(logits=tf.layers.dense(hidden, 28 * 28))
# INFERENCE
# Define a subgraph of the variational model, corresponding to a
# minibatch of size M.
x_ph = tf.placeholder(tf.int32, [FLAGS.M, 28 * 28])
hidden = tf.layers.dense(tf.cast(x_ph, tf.float32), 256,
activation=tf.nn.relu)
qz = Normal(loc=tf.layers.dense(hidden, FLAGS.d),
scale=tf.layers.dense(
hidden, FLAGS.d, activation=tf.nn.softplus))
# Bind p(x, z) and q(z | x) to the same TensorFlow placeholder for x.
inference = ed.KLqp({z: qz}, data={x: x_ph})
optimizer = tf.train.RMSPropOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer)
tf.global_variables_initializer().run()
n_iter_per_epoch = x_train.shape[0] // FLAGS.M
for epoch in range(1, FLAGS.n_epoch + 1):
print("Epoch: {0}".format(epoch))
avg_loss = 0.0
pbar = Progbar(n_iter_per_epoch)
for t in range(1, n_iter_per_epoch + 1):
pbar.update(t)
x_batch = next(x_train_generator)
info_dict = inference.update(feed_dict={x_ph: x_batch})
avg_loss += info_dict['loss']
# Print a lower bound to the average marginal likelihood for an
# image.
avg_loss /= n_iter_per_epoch
avg_loss /= FLAGS.M
print("-log p(x) <= {:0.3f}".format(avg_loss))
# Prior predictive check.
images = x.eval()
for m in range(FLAGS.M):
imsave(os.path.join(FLAGS.out_dir, '%d.png') % m,
images[m].reshape(28, 28))
def generative_adversarial_network_example():
ed.set_seed(42)
data_dir = '/tmp/data'
out_dir = '/tmp/out'
if not os.path.exists(out_dir):
os.makedirs(out_dir)
M = 128 # Batch size during training.
d = 100 # Latent dimension.
(x_train, _), (x_test, _) = mnist(data_dir)
x_train_generator = generator(x_train, M)
x_ph = tf.placeholder(tf.float32, [M, 784])
#--------------------
# GANs posit generative models using an implicit mechanism.
# Given some random noise, the data is assumed to be generated by a deterministic function of that noise.
with tf.variable_scope('Gen'):
eps = Uniform(tf.zeros([M, d]) - 1.0, tf.ones([M, d]))
x = generative_network(eps)
#--------------------
# In Edward, the GAN algorithm (GANInference) simply takes the implicit density model on x as input, binded to its realizations x_ph.
# In addition, a parameterized function discriminator is provided to distinguish their samples.
inference = ed.GANInference(data={x: x_ph}, discriminator=discriminative_network)
# We'll use ADAM as optimizers for both the generator and discriminator.
# We'll run the algorithm for 15,000 iterations and print progress every 1,000 iterations.
optimizer = tf.train.AdamOptimizer()
optimizer_d = tf.train.AdamOptimizer()
inference = ed.GANInference(data={x: x_ph}, discriminator=discriminative_network)
inference.initialize(optimizer=optimizer, optimizer_d=optimizer_d, n_iter=15000, n_print=1000)
# We now form the main loop which trains the GAN.
# At each iteration, it takes a minibatch and updates the parameters according to the algorithm.
sess = ed.get_session()
tf.global_variables_initializer().run()
idx = np.random.randint(M, size=16)
i = 0
for t in range(inference.n_iter):
if t % inference.n_print == 0:
samples = sess.run(x)
samples = samples[idx, ]
fig = plot(samples)
plt.savefig(os.path.join(out_dir, '{}.png').format(str(i).zfill(3)), bbox_inches='tight')
plt.close(fig)
i += 1
x_batch = next(x_train_generator)
info_dict = inference.update(feed_dict={x_ph: x_batch})
inference.print_progress(info_dict)
def load_mnist():
(train_im, train_lab), (test_im, test_lab) = mnist("data/mnist")
train_im_mean = np.mean(train_im, 0)
train_im_std = np.std(train_im, 0)
std_eps = 1e-7
train_im = (train_im - train_im_mean) / (train_im_std + std_eps)
test_im = (test_im - train_im_mean) / (train_im_std + std_eps)
# train_im /= 255.0
# test_im /= 255.0
return (train_im, train_lab), (test_im, test_lab)
def main(_):
ed.set_seed(42)
# DATA. MNIST batches are fed at training time.
(x_train, _), (x_test, _) = mnist(FLAGS.data_dir)
x_train_generator = generator(x_train, FLAGS.M)
# MODEL
z = Normal(loc=tf.zeros([FLAGS.M, FLAGS.d]),
scale=tf.ones([FLAGS.M, FLAGS.d]))
logits = generative_network(z)
x = Bernoulli(logits=logits)
# INFERENCE
x_ph = tf.placeholder(tf.int32, [FLAGS.M, 28 * 28])
loc, scale = inference_network(tf.cast(x_ph, tf.float32))
qz = Normal(loc=loc, scale=scale)
# Bind p(x, z) and q(z | x) to the same placeholder for x.
inference = ed.KLqp({z: qz}, data={x: x_ph})
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer)
hidden_rep = tf.sigmoid(logits)
tf.global_variables_initializer().run()
n_iter_per_epoch = x_train.shape[0] // FLAGS.M
for epoch in range(1, FLAGS.n_epoch + 1):
print("Epoch: {0}".format(epoch))
avg_loss = 0.0
pbar = Progbar(n_iter_per_epoch)
for t in range(1, n_iter_per_epoch + 1):
pbar.update(t)
x_batch = next(x_train_generator)
info_dict = inference.update(feed_dict={x_ph: x_batch})
avg_loss += info_dict['loss']
# Print a lower bound to the average marginal likelihood for an
# image.
avg_loss /= n_iter_per_epoch
avg_loss /= FLAGS.M
print("-log p(x) <= {:0.3f}".format(avg_loss))
# Visualize hidden representations.
images = hidden_rep.eval()
for m in range(FLAGS.M):
imsave(
os.path.join(FLAGS.out_dir, '%d.png') % m,
images[m].reshape(28, 28))
def main(_):
ed.set_seed(42)
# DATA. MNIST batches are fed at training time.
(x_train, _), (x_test, _) = mnist(FLAGS.data_dir)
x_train_generator = generator(x_train, FLAGS.M)
# MODEL
z = Normal(loc=tf.zeros([FLAGS.M, FLAGS.d]),
scale=tf.ones([FLAGS.M, FLAGS.d]))
logits = generative_network(z)
x = Bernoulli(logits=logits)
# INFERENCE
x_ph = tf.placeholder(tf.int32, [FLAGS.M, 28 * 28])
loc, scale = inference_network(tf.cast(x_ph, tf.float32))
qz = Normal(loc=loc, scale=scale)
# Bind p(x, z) and q(z | x) to the same placeholder for x.
inference = ed.KLqp({z: qz}, data={x: x_ph})
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer)
hidden_rep = tf.sigmoid(logits)
tf.global_variables_initializer().run()
n_iter_per_epoch = x_train.shape[0] // FLAGS.M
for epoch in range(1, FLAGS.n_epoch + 1):
print("Epoch: {0}".format(epoch))
avg_loss = 0.0
pbar = Progbar(n_iter_per_epoch)
for t in range(1, n_iter_per_epoch + 1):
pbar.update(t)
x_batch = next(x_train_generator)
info_dict = inference.update(feed_dict={x_ph: x_batch})
avg_loss += info_dict['loss']
# Print a lower bound to the average marginal likelihood for an
# image.
avg_loss /= n_iter_per_epoch
avg_loss /= FLAGS.M
print("-log p(x) <= {:0.3f}".format(avg_loss))
# Visualize hidden representations.
images = hidden_rep.eval()
for m in range(FLAGS.M):
imsave(os.path.join(FLAGS.out_dir, '%d.png') % m,
images[m].reshape(28, 28))
def main(_):
ed.set_seed(42)
# DATA. MNIST batches are fed at training time.
(x_train, _), (x_test, _) = mnist(FLAGS.data_dir)
x_train_generator = generator(x_train, FLAGS.M)
x_ph = tf.placeholder(tf.float32, [FLAGS.M, 784])
# MODEL
with tf.variable_scope("Gen"):
eps = Uniform(low=tf.zeros([FLAGS.M, FLAGS.d]) - 1.0,
high=tf.ones([FLAGS.M, FLAGS.d]))
x = generative_network(eps)
# INFERENCE
optimizer = tf.train.RMSPropOptimizer(learning_rate=5e-5)
optimizer_d = tf.train.RMSPropOptimizer(learning_rate=5e-5)
inference = ed.WGANInference(
data={x: x_ph}, discriminator=discriminative_network)
inference.initialize(
optimizer=optimizer, optimizer_d=optimizer_d,
n_iter=15000, n_print=1000, clip=0.01, penalty=None)
sess = ed.get_session()
tf.global_variables_initializer().run()
idx = np.random.randint(FLAGS.M, size=16)
i = 0
for t in range(inference.n_iter):
if t % inference.n_print == 0:
samples = sess.run(x)
samples = samples[idx, ]
fig = plot(samples)
plt.savefig(os.path.join(FLAGS.out_dir, '{}.png').format(
str(i).zfill(3)), bbox_inches='tight')
plt.close(fig)
i += 1
x_batch = next(x_train_generator)
for _ in range(5):
inference.update(feed_dict={x_ph: x_batch}, variables="Disc")
info_dict = inference.update(feed_dict={x_ph: x_batch}, variables="Gen")
# note: not printing discriminative objective; `info_dict` above
# does not store it since updating only "Gen"
info_dict['t'] = info_dict['t'] // 6 # say set of 6 updates is 1 iteration
inference.print_progress(info_dict)
def load_data(dataset,
train_pct=1.0,
root_dir: str = '/home/mirgahney/Projects/datasets'):
if dataset == "cifar":
(Xtrain, Ytrain), (Xtest,
Ytest) = observations.cifar10(f'{root_dir}/cifar')
Xtrain = np.transpose(Xtrain, [0, 2, 3, 1])
Xtest = np.transpose(Xtest, [0, 2, 3, 1])
mean = Xtrain.mean((0, 1, 2))
std = Xtrain.std((0, 1, 2))
Xtrain = (Xtrain - mean) / std
Xtest = (Xtest - mean) / std
if train_pct < 1.0:
Xvalid, Xtrain, Yvalid, Ytrain = train_test_split(
Xtrain, Ytrain, stratify=Ytrain, test_size=train_pct)
print(Xtrain.shape)
elif dataset == "fashion_mnist":
(Xtrain, Ytrain), (
Xtest,
Ytest) = observations.fashion_mnist(f'{root_dir}/fashion_mnist')
mean = Xtrain.mean(axis=0)
std = Xtrain.std()
Xtrain = (Xtrain - mean) / std
Xtest = (Xtest - mean) / std
Xtrain = Xtrain.reshape(-1, 28, 28, 1)
Xtest = Xtest.reshape(-1, 28, 28, 1)
if train_pct < 1.0:
Xvalid, Xtrain, Yvalid, Ytrain = train_test_split(
Xtrain, Ytrain, stratify=Ytrain, test_size=train_pct)
print(Xtrain.shape)
else:
(Xtrain, Ytrain), (Xtest,
Ytest) = observations.mnist(f'{root_dir}/mnist')
mean = Xtrain.mean(axis=0)
std = Xtrain.std()
Xtrain = (Xtrain - mean) / std
Xtest = (Xtest - mean) / std
Xtrain = Xtrain.reshape(-1, 28, 28, 1)
Xtest = Xtest.reshape(-1, 28, 28, 1)
Xvalid, Xtrain, Ytrain_2, Yvalid = train_test_split(
Xtrain, Ytrain, stratify=Ytrain, test_size=train_pct)
print(Xtrain.shape)
return (Xtrain, Ytrain), (Xvalid, Yvalid), (Xtest, Ytest)
def load_mnist(fashion=False):
if fashion:
(train_im, train_lab), (test_im,
test_lab) = fashion_mnist("data/fashion")
else:
(train_im, train_lab), (test_im, test_lab) = mnist("data/mnist")
train_im_mean = np.mean(train_im, 0)
train_im_std = np.std(train_im, 0)
std_eps = 1e-7
# train_im = (train_im - train_im_mean) / (train_im_std + std_eps)
# test_im = (test_im - train_im_mean) / (train_im_std + std_eps)
train_im /= 255.0
test_im /= 255.0
train_im = np.reshape(train_im, (-1, 28, 28, 1))
test_im = np.reshape(test_im, (-1, 28, 28, 1))
return (train_im, train_lab), (test_im, test_lab)
def load_data():
if flags.dataset == "cifar":
(Xtrain, Ytrain), (Xtest, Ytest) = observations.cifar10(
'/home/mirgahney/Projects/datasets/cifar')
Xtrain = np.transpose(Xtrain, [0, 2, 3, 1])
Xtest = np.transpose(Xtest, [0, 2, 3, 1])
mean = Xtrain.mean((0, 1, 2))
std = Xtrain.std((0, 1, 2))
Xtrain = (Xtrain - mean) / std
Xtest = (Xtest - mean) / std
Xtrain_2, Xtrain, Ytrain_2, Ytrain = train_test_split(
Xtrain, Ytrain, stratify=Ytrain, test_size=flags.train_pct)
print(Xtrain.shape)
elif flags.dataset == "fashion_mnist":
(Xtrain, Ytrain), (Xtest, Ytest) = observations.fashion_mnist(
'/home/mirgahney/Projects/datasets/fashion_mnist')
mean = Xtrain.mean(axis=0)
std = Xtrain.std()
Xtrain = (Xtrain - mean) / std
Xtest = (Xtest - mean) / std
Xtrain = Xtrain.reshape(-1, 28, 28, 1)
Xtest = Xtest.reshape(-1, 28, 28, 1)
if flags.train_pct < 1.0:
Xtrain_2, Xtrain, Ytrain_2, Ytrain = train_test_split(
Xtrain, Ytrain, stratify=Ytrain, test_size=flags.train_pct)
print(Xtrain.shape)
else:
(Xtrain, Ytrain), (Xtest, Ytest) = observations.mnist(
'/home/mirgahney/Projects/datasets/mnist')
mean = Xtrain.mean(axis=0)
std = Xtrain.std()
Xtrain = (Xtrain - mean) / std
Xtest = (Xtest - mean) / std
Xtrain = Xtrain.reshape(-1, 28, 28, 1)
Xtest = Xtest.reshape(-1, 28, 28, 1)
Xtrain_2, Xtrain, Ytrain_2, Ytrain = train_test_split(
Xtrain, Ytrain, stratify=Ytrain, test_size=flags.train_pct)
print(Xtrain.shape)
return (Xtrain, Ytrain), (Xtest, Ytest)
def load_data(dataset, data_dir):
x_train = None
x_test = None
x_train_generator = None
image_width = None
image_height = None
if dataset == "mnist":
(x_train, _), (x_test, _) = observations.mnist(data_dir)
return {
"image_height": 28,
"image_width": 28,
"x_train": x_train,
"x_test": x_test
}
elif dataset == "fashion-mnist":
(x_train, _), (x_test, _) = observations.fashion_mnist(data_dir)
return {
"image_height": 28,
"image_width": 28,
"x_train": x_train,
"x_test": x_test
}
else:
raise Exception("dataset not supported: {}".format(dataset))
def main(_):
ed.set_seed(42)
# DATA. MNIST batches are fed at training time.
(x_train, _), (x_test, _) = mnist(FLAGS.data_dir)
x_train_generator = generator(x_train, FLAGS.M)
x_ph = tf.placeholder(tf.float32, [FLAGS.M, 784])
z_ph = tf.placeholder(tf.float32, [FLAGS.M, FLAGS.d])
# MODEL
with tf.variable_scope("Gen"):
xf = gen_data(z_ph, FLAGS.hidden_units)
zf = gen_latent(x_ph, FLAGS.hidden_units)
# INFERENCE:
optimizer = tf.train.AdamOptimizer()
optimizer_d = tf.train.AdamOptimizer()
inference = ed.BiGANInference(latent_vars={zf: z_ph},
data={xf: x_ph},
discriminator=discriminative_network)
inference.initialize(optimizer=optimizer,
optimizer_d=optimizer_d,
n_iter=100000,
n_print=3000)
sess = ed.get_session()
init_op = tf.global_variables_initializer()
sess.run(init_op)
idx = np.random.randint(FLAGS.M, size=16)
i = 0
for t in range(inference.n_iter):
if t % inference.n_print == 1:
samples = sess.run(xf, feed_dict={z_ph: z_batch})
samples = samples[idx, ]
fig = plot(samples)
plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format(
'Generated',
str(i).zfill(3)),
bbox_inches='tight')
plt.close(fig)
fig = plot(x_batch[idx, ])
plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format(
'Base',
str(i).zfill(3)),
bbox_inches='tight')
plt.close(fig)
zsam = sess.run(zf, feed_dict={x_ph: x_batch})
reconstructions = sess.run(xf, feed_dict={z_ph: zsam})
reconstructions = reconstructions[idx, ]
fig = plot(reconstructions)
plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format(
'Reconstruct',
str(i).zfill(3)),
bbox_inches='tight')
plt.close(fig)
i += 1
x_batch = next(x_train_generator)
z_batch = np.random.normal(0, 1, [FLAGS.M, FLAGS.d])
info_dict = inference.update(feed_dict={x_ph: x_batch, z_ph: z_batch})
inference.print_progress(info_dict)
loc = params[:, :d]
scale = tf.nn.softplus(params[:, d:])
return loc, scale
ed.set_seed(42)
data_dir = "/tmp/data"
out_dir = "/tmp/out"
if not os.path.exists(out_dir):
os.makedirs(out_dir)
M = 128 # batch size during training
d = 10 # latent dimension
# DATA. MNIST batches are fed at training time.
(x_train, _), (x_test, _) = mnist(data_dir)
x_train_generator = generator(x_train, M)
# MODEL
z = Normal(loc=tf.zeros([M, d]), scale=tf.ones([M, d]))
logits = generative_network(z)
x = Bernoulli(logits=logits)
# INFERENCE
x_ph = tf.placeholder(tf.int32, [M, 28 * 28])
loc, scale = inference_network(tf.cast(x_ph, tf.float32))
qz = Normal(loc=loc, scale=scale)
# Bind p(x, z) and q(z | x) to the same placeholder for x.
data = {x: x_ph}
inference = ed.ReparameterizationKLKLqp({z: qz}, data)
# -*- coding: utf-8 -*-
"""
Created on Wed Sep 27 10:46:34 2017
@author: ryanhausen
"""
import Edward as ed
from Edward.models import Normal, Categorical
from observations import mnist
import Tensorflow as tf
import numpy as np
batch_size = 128
(x_train, y_train), (x_test, y_test) = mnist("~/data")
normal = lambda shp, name: Normal(loc=tf.zeros(shp), scale=tf.ones(shp), name=name)
rand_norm = lambda shp, name: tf.Variable(tf.random_normal(shp), name=name)
qnormal = lambda shp: Normal(loc=rand_norm(shp,'loc'), scale=tf.nn.softplus(rand_norm(shp, 'scale')))
def bnn(x):
stride1 = [1, 1, 1, 1]
stride2 = [1, 2, 2, 1]
valid, same = 'VALID', 'SAME'
# conv1
x = tf.nn.conv2d(x, W0, s=stride1, pad=valid) + B0
x = tf.nn.relu(x)
# conv2
num_classes = 10
vector_list, tmp_root = Make_SPN_from_RegionGraph(
rg_layers, np.random.RandomState(100), num_classes=num_classes, num_gauss=5, num_sums=5
)
args = SpnArgs()
args.num_gauss = 5
args.num_sums = 5
spns = vector_list[-1][0]
tensor_spn = RatSpn(10, vector_list=vector_list, args=args, name="tensor-spn-from-vectorlist")
input_ph = tf.placeholder(tf.float32, (1000, 28 * 28))
output = tensor_spn.forward(input_ph)
(train_im, train_lab), (test_im, test_lab) = mnist("data/mnist")
scalar = StandardScaler().fit(train_im)
train_im = scalar.transform(train_im)
test_im = scalar.transform(test_im)
add_parametric_inference_support()
for spn in spns:
valid, err = is_valid(spn)
print(valid, err)
print(get_structure_stats(spn))
print("starting")
tfstart = time.perf_counter()
log_likelihood(spn, train_im[0:1000, :])
end = time.perf_counter()
optimizable_observation.data += optimizable_observation.grad.data * self.learning_rate
optimizable_observation.grad.data.zero_()
return optimizable_observation
if __name__ == '__main__':
import numpy as np
from observations import mnist
from matplotlib import pyplot as plt
from sklearn.metrics.classification import classification_report, accuracy_score
predictor = Predictor(layer_sizes=[10, 100, 300, 784])
(x_train, y_train), (x_test, y_test) = mnist('../data/mnist')
EPOCHS = 100
for i in range(EPOCHS):
for idx in range(0, x_train.shape[0]):
print('Started to train picture no.' + str(idx) + ' in epoch no.' +
str(i))
predictor.train_parameters(observation=torch.from_numpy(
x_train[idx].reshape(1, 784) / 255.0).float(),
target=torch.from_numpy(
np.eye(10)[y_train[idx]].reshape(
1, 10)).float())
num_classes=num_classes,
num_gauss=5,
num_sums=5)
args = SpnArgs()
args.num_gauss = 5
args.num_sums = 5
spns = vector_list[-1][0]
tensor_spn = RatSpn(10,
vector_list=vector_list,
args=args,
name='tensor-spn-from-vectorlist')
input_ph = tf.placeholder(tf.float32, (1000, 28 * 28))
output = tensor_spn.forward(input_ph)
(train_im, train_lab), (test_im, test_lab) = mnist('data/mnist')
scalar = StandardScaler().fit(train_im)
train_im = scalar.transform(train_im)
test_im = scalar.transform(test_im)
add_parametric_inference_support()
for spn in spns:
valid, err = is_valid(spn)
print(valid, err)
print(get_structure_stats(spn))
print("starting")
tfstart = time.perf_counter()
log_likelihood(spn, train_im[0:1000, :])
end = time.perf_counter()
def __init__(self, datadir, batch_size):
data = obs.mnist(join(datadir, "MNIST"))
self._load(data, batch_size)
def run():
(x_train, y_train), (x_test, y_test) = observations.mnist('~/data/MNIST')
dataset_size = x_train.shape[0]
inputs = tf.placeholder(tf.float32, [None, 28 * 28], 'inputs')
labels = tf.placeholder(tf.int32, [None], 'labels')
data_indices = tf.placeholder(tf.int32, [None], 'data_indices')
def network(context: modular.ModularContext):
modules = modular.create_dense_modules(inputs, module_count=10, units=32)
hidden = modular.modular_layer(inputs, modules, parallel_count=1, context=context)
hidden = tf.nn.relu(hidden)
modules = modular.create_dense_modules(hidden, module_count=8, units=10)
logits = modular.modular_layer(hidden, modules, parallel_count=1, context=context)
target = modular.modularize_target(labels, context)
loglikelihood = tf.distributions.Categorical(logits).log_prob(target)
predicted = tf.argmax(logits, axis=-1, output_type=tf.int32)
accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, target), tf.float32))
selection_entropy = context.selection_entropy()
batch_selection_entropy = context.batch_selection_entropy()
return loglikelihood, logits, accuracy, selection_entropy, batch_selection_entropy
template = tf.make_template('network', network)
optimizer = tf.train.AdamOptimizer()
e_step, m_step, eval = modular.modularize(template, optimizer, dataset_size,
data_indices, sample_size=10)
ll, logits, accuracy, s_entropy, bs_entropy = eval
tf.summary.scalar('loglikelihood', tf.reduce_mean(ll))
tf.summary.scalar('accuracy', accuracy)
tf.summary.scalar('entropy/exp_selection', tf.exp(s_entropy))
tf.summary.scalar('entropy/exp_batch_selection', tf.exp(bs_entropy))
with tf.Session() as sess:
time = '{:%Y-%m-%d_%H:%M:%S}'.format(datetime.datetime.now())
writer = tf.summary.FileWriter('logs/train_{}'.format(time))
test_writer = tf.summary.FileWriter('logs/test_{}'.format(time))
general_summaries = tf.summary.merge_all()
m_step_summaries = tf.summary.merge([create_m_step_summaries(), general_summaries])
sess.run(tf.global_variables_initializer())
batches = generator([x_train, y_train, np.arange(dataset_size)], 32)
for i, (batch_x, batch_y, indices) in enumerate(batches):
feed_dict = {
inputs: batch_x,
labels: batch_y,
data_indices: indices
}
step = e_step if i % 10 == 0 else m_step
summaries = m_step_summaries if step == m_step else general_summaries
_, summary_data = sess.run([step, summaries], feed_dict)
writer.add_summary(summary_data, global_step=i)
if i % 100 == 0:
test_feed_dict = {inputs: x_test, labels: y_test}
summary_data = sess.run(general_summaries, test_feed_dict)
test_writer.add_summary(summary_data, global_step=i)
writer.close()
test_writer.close()
'''
Tensorflow experimental implementation of the paper
Deep Predictive Coding Network for Object Recognition
https://arxiv.org/pdf/1802.04762.pdf
'''
from observations import mnist
import tensorflow as tf
import numpy as np
from sklearn.metrics import classification_report, accuracy_score
import datetime
(x_train, y_train), (x_test, y_test) = mnist('./data')
graph = tf.Graph()
convolutional_layer_specs = [{
'filters': [2, 2, 1, 1],
'strides': [1, 1, 1, 1]
}, {
'filters': [2, 2, 1, 1],
'strides': [1, 1, 1, 1]
}, {
'filters': [2, 2, 1, 1],
'strides': [1, 1, 1, 1]
}, {
'filters': [2, 2, 1, 1],
'strides': [1, 1, 1, 1]
}, {
def run():
(x_train, y_train), (x_test, y_test) = observations.mnist('~/data/MNIST')
dataset_size = x_train.shape[0]
inputs = tf.placeholder(tf.float32, [None, 28 * 28], 'inputs')
labels = tf.placeholder(tf.int32, [None], 'labels')
data_indices = tf.placeholder(tf.int32, [None], 'data_indices')
def network(context: modular.ModularContext):
modules = modular.create_dense_modules(inputs,
module_count=10,
units=32)
hidden = modular.modular_layer(inputs,
modules,
parallel_count=1,
context=context)
hidden = tf.nn.relu(hidden)
modules = modular.create_dense_modules(hidden,
module_count=8,
units=10)
logits = modular.modular_layer(hidden,
modules,
parallel_count=1,
context=context)
target = modular.modularize_target(labels, context)
loglikelihood = tf.distributions.Categorical(logits).log_prob(target)
predicted = tf.argmax(logits, axis=-1, output_type=tf.int32)
accuracy = tf.reduce_mean(
tf.cast(tf.equal(predicted, target), tf.float32))
selection_entropy = context.selection_entropy()
batch_selection_entropy = context.batch_selection_entropy()
return loglikelihood, logits, accuracy, selection_entropy, batch_selection_entropy
template = tf.make_template('network', network)
optimizer = tf.train.AdamOptimizer()
e_step, m_step, eval = modular.modularize(template,
optimizer,
dataset_size,
data_indices,
sample_size=10)
ll, logits, accuracy, s_entropy, bs_entropy = eval
tf.summary.scalar('loglikelihood', tf.reduce_mean(ll))
tf.summary.scalar('accuracy', accuracy)
tf.summary.scalar('entropy/exp_selection', tf.exp(s_entropy))
tf.summary.scalar('entropy/exp_batch_selection', tf.exp(bs_entropy))
with tf.Session() as sess:
time = '{:%Y-%m-%d_%H:%M:%S}'.format(datetime.datetime.now())
writer = tf.summary.FileWriter('logs/train_{}'.format(time))
test_writer = tf.summary.FileWriter('logs/test_{}'.format(time))
general_summaries = tf.summary.merge_all()
m_step_summaries = tf.summary.merge(
[create_m_step_summaries(), general_summaries])
sess.run(tf.global_variables_initializer())
batches = generator([x_train, y_train, np.arange(dataset_size)], 32)
for i, (batch_x, batch_y, indices) in enumerate(batches):
feed_dict = {
inputs: batch_x,
labels: batch_y,
data_indices: indices
}
step = e_step if i % 10 == 0 else m_step
summaries = m_step_summaries if step == m_step else general_summaries
_, summary_data = sess.run([step, summaries], feed_dict)
writer.add_summary(summary_data, global_step=i)
if i % 100 == 0:
test_feed_dict = {inputs: x_test, labels: y_test}
summary_data = sess.run(general_summaries, test_feed_dict)
test_writer.add_summary(summary_data, global_step=i)
writer.close()
test_writer.close()
def load_multi_mnist(path, max_digits=2, canvas_size=50, seed=42):
"""
Code pulled from observations library and customized to
collect bounding box information.
Load the multiple MNIST data set [@eslami2016attend]. It modifies
the original MNIST such that each image contains a number of
non-overlapping random MNIST digits with equal probability.
Args:
path: str.
Path to directory which either stores file or otherwise file will
be downloaded and extracted there. Filename is
`'multi_mnist_{}_{}_{}.npz'.format(max_digits, canvas_size, seed)`.
max_digits: int, optional.
Maximum number of non-overlapping MNIST digits per image to
generate if not cached.
canvas_size: list of two int, optional.
Width x height pixel size of generated images if not cached.
seed: int, optional.
Random seed to generate the data set from MNIST if not cached.
Returns:
Tuple of (np.ndarray of dtype uint8, list)
`(x_train, y_train), (x_test, y_test)`. Each element in the y's is a
np.ndarray of labels, one label for each digit in the image.
"""
def _sample_one(canvas_size, x_data, y_data):
i = np.random.randint(x_data.shape[0])
digit = x_data[i]
label = y_data[i]
scale = 0.1 * np.random.randn() + 1.3
resized = scipy.misc.imresize(digit, 1.0 / scale)
width = resized.shape[0]
padding = canvas_size - width
pad_l = np.random.randint(0, padding)
pad_r = np.random.randint(0, padding)
pad_width = ((pad_l, padding - pad_l), (pad_r, padding - pad_r))
positioned = np.pad(resized, pad_width, 'constant', constant_values=0)
bbox = (pad_l, pad_r, pad_l + width, pad_r + width)
return positioned, label, bbox
def _sample_multi(num_digits, canvas_size, x_data, y_data):
canvas = np.zeros((canvas_size, canvas_size))
labels = []
bboxes = []
for _ in range(num_digits):
positioned_digit, label, bbox = _sample_one(
canvas_size, x_data, y_data)
canvas += positioned_digit
labels.append(label)
bboxes.append(bbox)
labels = np.array(labels, dtype=np.uint8)
if np.max(canvas) > 255: # crude check for overlapping digits
return _sample_multi(num_digits, canvas_size, x_data, y_data)
else:
return canvas, labels, bboxes
def _build_dataset(x_data, y_data, max_digits, canvas_size):
x = []
y = []
data_size = x_data.shape[0]
data_num_digits = np.random.randint(max_digits + 1, size=data_size)
x_data = np.reshape(x_data, [data_size, 28, 28])
bboxes_arr = np.zeros((data_size, max_digits, 4))
for i, num_digits in enumerate(data_num_digits):
canvas, labels, bboxes = _sample_multi(num_digits, canvas_size,
x_data, y_data)
x.append(canvas)
y.append(labels)
for j, bbox in enumerate(bboxes):
bboxes_arr[i, j] = bbox
x = np.array(x, dtype=np.uint8)
return x, y, bboxes_arr
path = os.path.expanduser(path)
cache_filename = 'multi_mnist_{}_{}_{}.npz'.format(max_digits, canvas_size,
seed)
if os.path.exists(os.path.join(path, cache_filename)):
data = np.load(os.path.join(path, cache_filename), allow_pickle=True)
return (data['x_train'], data['y_train'], data['x_bbox']),\
(data['x_test'], data['y_test'], data['y_bbox'])
np.random.seed(seed)
(x_train, y_train), (x_test, y_test) = mnist(path)
x_train, y_train, x_bbox = _build_dataset(x_train, y_train, max_digits,
canvas_size)
x_test, y_test, y_bbox = _build_dataset(x_test, y_test, max_digits,
canvas_size)
with open(os.path.join(path, cache_filename), 'wb') as f:
np.savez_compressed(f,
x_train=x_train,
y_train=y_train,
x_test=x_test,
y_test=y_test,
x_bbox=x_bbox,
y_bbox=y_bbox)
return (x_train, y_train, x_bbox), (x_test, y_test, y_bbox)
