def main(_): ed.set_seed(42) # DATA. MNIST batches are fed at training time. (x_train, _), (x_test, _) = mnist(FLAGS.data_dir) x_train_generator = generator(x_train, FLAGS.M) x_ph = tf.placeholder(tf.float32, [FLAGS.M, 784]) z_ph = tf.placeholder(tf.float32, [FLAGS.M, FLAGS.d]) # MODEL with tf.variable_scope("Gen"): xf = gen_data(z_ph, FLAGS.hidden_units) zf = gen_latent(x_ph, FLAGS.hidden_units) # INFERENCE: optimizer = tf.train.AdamOptimizer() optimizer_d = tf.train.AdamOptimizer() inference = ed.BiGANInference( latent_vars={zf: z_ph}, data={xf: x_ph}, discriminator=discriminative_network) inference.initialize( optimizer=optimizer, optimizer_d=optimizer_d, n_iter=100000, n_print=3000) sess = ed.get_session() init_op = tf.global_variables_initializer() sess.run(init_op) idx = np.random.randint(FLAGS.M, size=16) i = 0 for t in range(inference.n_iter): if t % inference.n_print == 1: samples = sess.run(xf, feed_dict={z_ph: z_batch}) samples = samples[idx, ] fig = plot(samples) plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format( 'Generated', str(i).zfill(3)), bbox_inches='tight') plt.close(fig) fig = plot(x_batch[idx, ]) plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format( 'Base', str(i).zfill(3)), bbox_inches='tight') plt.close(fig) zsam = sess.run(zf, feed_dict={x_ph: x_batch}) reconstructions = sess.run(xf, feed_dict={z_ph: zsam}) reconstructions = reconstructions[idx, ] fig = plot(reconstructions) plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format( 'Reconstruct', str(i).zfill(3)), bbox_inches='tight') plt.close(fig) i += 1 x_batch = next(x_train_generator) z_batch = np.random.normal(0, 1, [FLAGS.M, FLAGS.d]) info_dict = inference.update(feed_dict={x_ph: x_batch, z_ph: z_batch}) inference.print_progress(info_dict)
def main(_): ed.set_seed(42) # DATA. MNIST batches are fed at training time. (x_train, _), (x_test, _) = mnist(FLAGS.data_dir) x_train_generator = generator(x_train, FLAGS.M) # MODEL # Define a subgraph of the full model, corresponding to a minibatch of # size M. z = Normal(loc=tf.zeros([FLAGS.M, FLAGS.d]), scale=tf.ones([FLAGS.M, FLAGS.d])) hidden = tf.layers.dense(z, 256, activation=tf.nn.relu) x = Bernoulli(logits=tf.layers.dense(hidden, 28 * 28)) # INFERENCE # Define a subgraph of the variational model, corresponding to a # minibatch of size M. x_ph = tf.placeholder(tf.int32, [FLAGS.M, 28 * 28]) hidden = tf.layers.dense(tf.cast(x_ph, tf.float32), 256, activation=tf.nn.relu) qz = Normal(loc=tf.layers.dense(hidden, FLAGS.d), scale=tf.layers.dense(hidden, FLAGS.d, activation=tf.nn.softplus)) # Bind p(x, z) and q(z | x) to the same TensorFlow placeholder for x. inference = ed.KLqp({z: qz}, data={x: x_ph}) optimizer = tf.train.RMSPropOptimizer(0.01, epsilon=1.0) inference.initialize(optimizer=optimizer) tf.global_variables_initializer().run() n_iter_per_epoch = x_train.shape[0] // FLAGS.M for epoch in range(1, FLAGS.n_epoch + 1): print("Epoch: {0}".format(epoch)) avg_loss = 0.0 pbar = Progbar(n_iter_per_epoch) for t in range(1, n_iter_per_epoch + 1): pbar.update(t) x_batch = next(x_train_generator) info_dict = inference.update(feed_dict={x_ph: x_batch}) avg_loss += info_dict['loss'] # Print a lower bound to the average marginal likelihood for an # image. avg_loss /= n_iter_per_epoch avg_loss /= FLAGS.M print("-log p(x) <= {:0.3f}".format(avg_loss)) # Prior predictive check. images = x.eval() for m in range(FLAGS.M): imsave( os.path.join(FLAGS.out_dir, '%d.png') % m, images[m].reshape(28, 28))
def __init__(self, data_path, flatten=True): """ Take in a path to (down)load data.""" (train_x, train_y), (test_x, test_y) = mnist(data_path) if not flatten: train_x = train_x.reshape(-1, 28, 28, 1) test_x = test_x.reshape(-1, 28, 28, 1) self.train = (binarize(train_x), one_hot(train_y)) self.test = (binarize(test_x), one_hot(test_y)) self.batch_size = 0 self.epoch_size = train_y.shape[0]
def main(_): ed.set_seed(42) # DATA. MNIST batches are fed at training time. (x_train, _), (x_test, _) = mnist(FLAGS.data_dir) x_train_generator = generator(x_train, FLAGS.M) # MODEL # Define a subgraph of the full model, corresponding to a minibatch of # size M. z = Normal(loc=tf.zeros([FLAGS.M, FLAGS.d]), scale=tf.ones([FLAGS.M, FLAGS.d])) hidden = tf.layers.dense(z, 256, activation=tf.nn.relu) x = Bernoulli(logits=tf.layers.dense(hidden, 28 * 28)) # INFERENCE # Define a subgraph of the variational model, corresponding to a # minibatch of size M. x_ph = tf.placeholder(tf.int32, [FLAGS.M, 28 * 28]) hidden = tf.layers.dense(tf.cast(x_ph, tf.float32), 256, activation=tf.nn.relu) qz = Normal(loc=tf.layers.dense(hidden, FLAGS.d), scale=tf.layers.dense( hidden, FLAGS.d, activation=tf.nn.softplus)) # Bind p(x, z) and q(z | x) to the same TensorFlow placeholder for x. inference = ed.KLqp({z: qz}, data={x: x_ph}) optimizer = tf.train.RMSPropOptimizer(0.01, epsilon=1.0) inference.initialize(optimizer=optimizer) tf.global_variables_initializer().run() n_iter_per_epoch = x_train.shape[0] // FLAGS.M for epoch in range(1, FLAGS.n_epoch + 1): print("Epoch: {0}".format(epoch)) avg_loss = 0.0 pbar = Progbar(n_iter_per_epoch) for t in range(1, n_iter_per_epoch + 1): pbar.update(t) x_batch = next(x_train_generator) info_dict = inference.update(feed_dict={x_ph: x_batch}) avg_loss += info_dict['loss'] # Print a lower bound to the average marginal likelihood for an # image. avg_loss /= n_iter_per_epoch avg_loss /= FLAGS.M print("-log p(x) <= {:0.3f}".format(avg_loss)) # Prior predictive check. images = x.eval() for m in range(FLAGS.M): imsave(os.path.join(FLAGS.out_dir, '%d.png') % m, images[m].reshape(28, 28))
def generative_adversarial_network_example(): ed.set_seed(42) data_dir = '/tmp/data' out_dir = '/tmp/out' if not os.path.exists(out_dir): os.makedirs(out_dir) M = 128 # Batch size during training. d = 100 # Latent dimension. (x_train, _), (x_test, _) = mnist(data_dir) x_train_generator = generator(x_train, M) x_ph = tf.placeholder(tf.float32, [M, 784]) #-------------------- # GANs posit generative models using an implicit mechanism. # Given some random noise, the data is assumed to be generated by a deterministic function of that noise. with tf.variable_scope('Gen'): eps = Uniform(tf.zeros([M, d]) - 1.0, tf.ones([M, d])) x = generative_network(eps) #-------------------- # In Edward, the GAN algorithm (GANInference) simply takes the implicit density model on x as input, binded to its realizations x_ph. # In addition, a parameterized function discriminator is provided to distinguish their samples. inference = ed.GANInference(data={x: x_ph}, discriminator=discriminative_network) # We'll use ADAM as optimizers for both the generator and discriminator. # We'll run the algorithm for 15,000 iterations and print progress every 1,000 iterations. optimizer = tf.train.AdamOptimizer() optimizer_d = tf.train.AdamOptimizer() inference = ed.GANInference(data={x: x_ph}, discriminator=discriminative_network) inference.initialize(optimizer=optimizer, optimizer_d=optimizer_d, n_iter=15000, n_print=1000) # We now form the main loop which trains the GAN. # At each iteration, it takes a minibatch and updates the parameters according to the algorithm. sess = ed.get_session() tf.global_variables_initializer().run() idx = np.random.randint(M, size=16) i = 0 for t in range(inference.n_iter): if t % inference.n_print == 0: samples = sess.run(x) samples = samples[idx, ] fig = plot(samples) plt.savefig(os.path.join(out_dir, '{}.png').format(str(i).zfill(3)), bbox_inches='tight') plt.close(fig) i += 1 x_batch = next(x_train_generator) info_dict = inference.update(feed_dict={x_ph: x_batch}) inference.print_progress(info_dict)
def load_mnist(): (train_im, train_lab), (test_im, test_lab) = mnist("data/mnist") train_im_mean = np.mean(train_im, 0) train_im_std = np.std(train_im, 0) std_eps = 1e-7 train_im = (train_im - train_im_mean) / (train_im_std + std_eps) test_im = (test_im - train_im_mean) / (train_im_std + std_eps) # train_im /= 255.0 # test_im /= 255.0 return (train_im, train_lab), (test_im, test_lab)
def main(_): ed.set_seed(42) # DATA. MNIST batches are fed at training time. (x_train, _), (x_test, _) = mnist(FLAGS.data_dir) x_train_generator = generator(x_train, FLAGS.M) # MODEL z = Normal(loc=tf.zeros([FLAGS.M, FLAGS.d]), scale=tf.ones([FLAGS.M, FLAGS.d])) logits = generative_network(z) x = Bernoulli(logits=logits) # INFERENCE x_ph = tf.placeholder(tf.int32, [FLAGS.M, 28 * 28]) loc, scale = inference_network(tf.cast(x_ph, tf.float32)) qz = Normal(loc=loc, scale=scale) # Bind p(x, z) and q(z | x) to the same placeholder for x. inference = ed.KLqp({z: qz}, data={x: x_ph}) optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0) inference.initialize(optimizer=optimizer) hidden_rep = tf.sigmoid(logits) tf.global_variables_initializer().run() n_iter_per_epoch = x_train.shape[0] // FLAGS.M for epoch in range(1, FLAGS.n_epoch + 1): print("Epoch: {0}".format(epoch)) avg_loss = 0.0 pbar = Progbar(n_iter_per_epoch) for t in range(1, n_iter_per_epoch + 1): pbar.update(t) x_batch = next(x_train_generator) info_dict = inference.update(feed_dict={x_ph: x_batch}) avg_loss += info_dict['loss'] # Print a lower bound to the average marginal likelihood for an # image. avg_loss /= n_iter_per_epoch avg_loss /= FLAGS.M print("-log p(x) <= {:0.3f}".format(avg_loss)) # Visualize hidden representations. images = hidden_rep.eval() for m in range(FLAGS.M): imsave( os.path.join(FLAGS.out_dir, '%d.png') % m, images[m].reshape(28, 28))
def main(_): ed.set_seed(42) # DATA. MNIST batches are fed at training time. (x_train, _), (x_test, _) = mnist(FLAGS.data_dir) x_train_generator = generator(x_train, FLAGS.M) # MODEL z = Normal(loc=tf.zeros([FLAGS.M, FLAGS.d]), scale=tf.ones([FLAGS.M, FLAGS.d])) logits = generative_network(z) x = Bernoulli(logits=logits) # INFERENCE x_ph = tf.placeholder(tf.int32, [FLAGS.M, 28 * 28]) loc, scale = inference_network(tf.cast(x_ph, tf.float32)) qz = Normal(loc=loc, scale=scale) # Bind p(x, z) and q(z | x) to the same placeholder for x. inference = ed.KLqp({z: qz}, data={x: x_ph}) optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0) inference.initialize(optimizer=optimizer) hidden_rep = tf.sigmoid(logits) tf.global_variables_initializer().run() n_iter_per_epoch = x_train.shape[0] // FLAGS.M for epoch in range(1, FLAGS.n_epoch + 1): print("Epoch: {0}".format(epoch)) avg_loss = 0.0 pbar = Progbar(n_iter_per_epoch) for t in range(1, n_iter_per_epoch + 1): pbar.update(t) x_batch = next(x_train_generator) info_dict = inference.update(feed_dict={x_ph: x_batch}) avg_loss += info_dict['loss'] # Print a lower bound to the average marginal likelihood for an # image. avg_loss /= n_iter_per_epoch avg_loss /= FLAGS.M print("-log p(x) <= {:0.3f}".format(avg_loss)) # Visualize hidden representations. images = hidden_rep.eval() for m in range(FLAGS.M): imsave(os.path.join(FLAGS.out_dir, '%d.png') % m, images[m].reshape(28, 28))
def main(_): ed.set_seed(42) # DATA. MNIST batches are fed at training time. (x_train, _), (x_test, _) = mnist(FLAGS.data_dir) x_train_generator = generator(x_train, FLAGS.M) x_ph = tf.placeholder(tf.float32, [FLAGS.M, 784]) # MODEL with tf.variable_scope("Gen"): eps = Uniform(low=tf.zeros([FLAGS.M, FLAGS.d]) - 1.0, high=tf.ones([FLAGS.M, FLAGS.d])) x = generative_network(eps) # INFERENCE optimizer = tf.train.RMSPropOptimizer(learning_rate=5e-5) optimizer_d = tf.train.RMSPropOptimizer(learning_rate=5e-5) inference = ed.WGANInference( data={x: x_ph}, discriminator=discriminative_network) inference.initialize( optimizer=optimizer, optimizer_d=optimizer_d, n_iter=15000, n_print=1000, clip=0.01, penalty=None) sess = ed.get_session() tf.global_variables_initializer().run() idx = np.random.randint(FLAGS.M, size=16) i = 0 for t in range(inference.n_iter): if t % inference.n_print == 0: samples = sess.run(x) samples = samples[idx, ] fig = plot(samples) plt.savefig(os.path.join(FLAGS.out_dir, '{}.png').format( str(i).zfill(3)), bbox_inches='tight') plt.close(fig) i += 1 x_batch = next(x_train_generator) for _ in range(5): inference.update(feed_dict={x_ph: x_batch}, variables="Disc") info_dict = inference.update(feed_dict={x_ph: x_batch}, variables="Gen") # note: not printing discriminative objective; `info_dict` above # does not store it since updating only "Gen" info_dict['t'] = info_dict['t'] // 6 # say set of 6 updates is 1 iteration inference.print_progress(info_dict)
def load_data(dataset, train_pct=1.0, root_dir: str = '/home/mirgahney/Projects/datasets'): if dataset == "cifar": (Xtrain, Ytrain), (Xtest, Ytest) = observations.cifar10(f'{root_dir}/cifar') Xtrain = np.transpose(Xtrain, [0, 2, 3, 1]) Xtest = np.transpose(Xtest, [0, 2, 3, 1]) mean = Xtrain.mean((0, 1, 2)) std = Xtrain.std((0, 1, 2)) Xtrain = (Xtrain - mean) / std Xtest = (Xtest - mean) / std if train_pct < 1.0: Xvalid, Xtrain, Yvalid, Ytrain = train_test_split( Xtrain, Ytrain, stratify=Ytrain, test_size=train_pct) print(Xtrain.shape) elif dataset == "fashion_mnist": (Xtrain, Ytrain), ( Xtest, Ytest) = observations.fashion_mnist(f'{root_dir}/fashion_mnist') mean = Xtrain.mean(axis=0) std = Xtrain.std() Xtrain = (Xtrain - mean) / std Xtest = (Xtest - mean) / std Xtrain = Xtrain.reshape(-1, 28, 28, 1) Xtest = Xtest.reshape(-1, 28, 28, 1) if train_pct < 1.0: Xvalid, Xtrain, Yvalid, Ytrain = train_test_split( Xtrain, Ytrain, stratify=Ytrain, test_size=train_pct) print(Xtrain.shape) else: (Xtrain, Ytrain), (Xtest, Ytest) = observations.mnist(f'{root_dir}/mnist') mean = Xtrain.mean(axis=0) std = Xtrain.std() Xtrain = (Xtrain - mean) / std Xtest = (Xtest - mean) / std Xtrain = Xtrain.reshape(-1, 28, 28, 1) Xtest = Xtest.reshape(-1, 28, 28, 1) Xvalid, Xtrain, Ytrain_2, Yvalid = train_test_split( Xtrain, Ytrain, stratify=Ytrain, test_size=train_pct) print(Xtrain.shape) return (Xtrain, Ytrain), (Xvalid, Yvalid), (Xtest, Ytest)
def load_mnist(fashion=False): if fashion: (train_im, train_lab), (test_im, test_lab) = fashion_mnist("data/fashion") else: (train_im, train_lab), (test_im, test_lab) = mnist("data/mnist") train_im_mean = np.mean(train_im, 0) train_im_std = np.std(train_im, 0) std_eps = 1e-7 # train_im = (train_im - train_im_mean) / (train_im_std + std_eps) # test_im = (test_im - train_im_mean) / (train_im_std + std_eps) train_im /= 255.0 test_im /= 255.0 train_im = np.reshape(train_im, (-1, 28, 28, 1)) test_im = np.reshape(test_im, (-1, 28, 28, 1)) return (train_im, train_lab), (test_im, test_lab)
def load_data(): if flags.dataset == "cifar": (Xtrain, Ytrain), (Xtest, Ytest) = observations.cifar10( '/home/mirgahney/Projects/datasets/cifar') Xtrain = np.transpose(Xtrain, [0, 2, 3, 1]) Xtest = np.transpose(Xtest, [0, 2, 3, 1]) mean = Xtrain.mean((0, 1, 2)) std = Xtrain.std((0, 1, 2)) Xtrain = (Xtrain - mean) / std Xtest = (Xtest - mean) / std Xtrain_2, Xtrain, Ytrain_2, Ytrain = train_test_split( Xtrain, Ytrain, stratify=Ytrain, test_size=flags.train_pct) print(Xtrain.shape) elif flags.dataset == "fashion_mnist": (Xtrain, Ytrain), (Xtest, Ytest) = observations.fashion_mnist( '/home/mirgahney/Projects/datasets/fashion_mnist') mean = Xtrain.mean(axis=0) std = Xtrain.std() Xtrain = (Xtrain - mean) / std Xtest = (Xtest - mean) / std Xtrain = Xtrain.reshape(-1, 28, 28, 1) Xtest = Xtest.reshape(-1, 28, 28, 1) if flags.train_pct < 1.0: Xtrain_2, Xtrain, Ytrain_2, Ytrain = train_test_split( Xtrain, Ytrain, stratify=Ytrain, test_size=flags.train_pct) print(Xtrain.shape) else: (Xtrain, Ytrain), (Xtest, Ytest) = observations.mnist( '/home/mirgahney/Projects/datasets/mnist') mean = Xtrain.mean(axis=0) std = Xtrain.std() Xtrain = (Xtrain - mean) / std Xtest = (Xtest - mean) / std Xtrain = Xtrain.reshape(-1, 28, 28, 1) Xtest = Xtest.reshape(-1, 28, 28, 1) Xtrain_2, Xtrain, Ytrain_2, Ytrain = train_test_split( Xtrain, Ytrain, stratify=Ytrain, test_size=flags.train_pct) print(Xtrain.shape) return (Xtrain, Ytrain), (Xtest, Ytest)
def load_data(dataset, data_dir): x_train = None x_test = None x_train_generator = None image_width = None image_height = None if dataset == "mnist": (x_train, _), (x_test, _) = observations.mnist(data_dir) return { "image_height": 28, "image_width": 28, "x_train": x_train, "x_test": x_test } elif dataset == "fashion-mnist": (x_train, _), (x_test, _) = observations.fashion_mnist(data_dir) return { "image_height": 28, "image_width": 28, "x_train": x_train, "x_test": x_test } else: raise Exception("dataset not supported: {}".format(dataset))
def main(_): ed.set_seed(42) # DATA. MNIST batches are fed at training time. (x_train, _), (x_test, _) = mnist(FLAGS.data_dir) x_train_generator = generator(x_train, FLAGS.M) x_ph = tf.placeholder(tf.float32, [FLAGS.M, 784]) z_ph = tf.placeholder(tf.float32, [FLAGS.M, FLAGS.d]) # MODEL with tf.variable_scope("Gen"): xf = gen_data(z_ph, FLAGS.hidden_units) zf = gen_latent(x_ph, FLAGS.hidden_units) # INFERENCE: optimizer = tf.train.AdamOptimizer() optimizer_d = tf.train.AdamOptimizer() inference = ed.BiGANInference(latent_vars={zf: z_ph}, data={xf: x_ph}, discriminator=discriminative_network) inference.initialize(optimizer=optimizer, optimizer_d=optimizer_d, n_iter=100000, n_print=3000) sess = ed.get_session() init_op = tf.global_variables_initializer() sess.run(init_op) idx = np.random.randint(FLAGS.M, size=16) i = 0 for t in range(inference.n_iter): if t % inference.n_print == 1: samples = sess.run(xf, feed_dict={z_ph: z_batch}) samples = samples[idx, ] fig = plot(samples) plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format( 'Generated', str(i).zfill(3)), bbox_inches='tight') plt.close(fig) fig = plot(x_batch[idx, ]) plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format( 'Base', str(i).zfill(3)), bbox_inches='tight') plt.close(fig) zsam = sess.run(zf, feed_dict={x_ph: x_batch}) reconstructions = sess.run(xf, feed_dict={z_ph: zsam}) reconstructions = reconstructions[idx, ] fig = plot(reconstructions) plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format( 'Reconstruct', str(i).zfill(3)), bbox_inches='tight') plt.close(fig) i += 1 x_batch = next(x_train_generator) z_batch = np.random.normal(0, 1, [FLAGS.M, FLAGS.d]) info_dict = inference.update(feed_dict={x_ph: x_batch, z_ph: z_batch}) inference.print_progress(info_dict)
loc = params[:, :d] scale = tf.nn.softplus(params[:, d:]) return loc, scale ed.set_seed(42) data_dir = "/tmp/data" out_dir = "/tmp/out" if not os.path.exists(out_dir): os.makedirs(out_dir) M = 128 # batch size during training d = 10 # latent dimension # DATA. MNIST batches are fed at training time. (x_train, _), (x_test, _) = mnist(data_dir) x_train_generator = generator(x_train, M) # MODEL z = Normal(loc=tf.zeros([M, d]), scale=tf.ones([M, d])) logits = generative_network(z) x = Bernoulli(logits=logits) # INFERENCE x_ph = tf.placeholder(tf.int32, [M, 28 * 28]) loc, scale = inference_network(tf.cast(x_ph, tf.float32)) qz = Normal(loc=loc, scale=scale) # Bind p(x, z) and q(z | x) to the same placeholder for x. data = {x: x_ph} inference = ed.ReparameterizationKLKLqp({z: qz}, data)
# -*- coding: utf-8 -*- """ Created on Wed Sep 27 10:46:34 2017 @author: ryanhausen """ import Edward as ed from Edward.models import Normal, Categorical from observations import mnist import Tensorflow as tf import numpy as np batch_size = 128 (x_train, y_train), (x_test, y_test) = mnist("~/data") normal = lambda shp, name: Normal(loc=tf.zeros(shp), scale=tf.ones(shp), name=name) rand_norm = lambda shp, name: tf.Variable(tf.random_normal(shp), name=name) qnormal = lambda shp: Normal(loc=rand_norm(shp,'loc'), scale=tf.nn.softplus(rand_norm(shp, 'scale'))) def bnn(x): stride1 = [1, 1, 1, 1] stride2 = [1, 2, 2, 1] valid, same = 'VALID', 'SAME' # conv1 x = tf.nn.conv2d(x, W0, s=stride1, pad=valid) + B0 x = tf.nn.relu(x) # conv2
num_classes = 10 vector_list, tmp_root = Make_SPN_from_RegionGraph( rg_layers, np.random.RandomState(100), num_classes=num_classes, num_gauss=5, num_sums=5 ) args = SpnArgs() args.num_gauss = 5 args.num_sums = 5 spns = vector_list[-1][0] tensor_spn = RatSpn(10, vector_list=vector_list, args=args, name="tensor-spn-from-vectorlist") input_ph = tf.placeholder(tf.float32, (1000, 28 * 28)) output = tensor_spn.forward(input_ph) (train_im, train_lab), (test_im, test_lab) = mnist("data/mnist") scalar = StandardScaler().fit(train_im) train_im = scalar.transform(train_im) test_im = scalar.transform(test_im) add_parametric_inference_support() for spn in spns: valid, err = is_valid(spn) print(valid, err) print(get_structure_stats(spn)) print("starting") tfstart = time.perf_counter() log_likelihood(spn, train_im[0:1000, :]) end = time.perf_counter()
optimizable_observation.data += optimizable_observation.grad.data * self.learning_rate optimizable_observation.grad.data.zero_() return optimizable_observation if __name__ == '__main__': import numpy as np from observations import mnist from matplotlib import pyplot as plt from sklearn.metrics.classification import classification_report, accuracy_score predictor = Predictor(layer_sizes=[10, 100, 300, 784]) (x_train, y_train), (x_test, y_test) = mnist('../data/mnist') EPOCHS = 100 for i in range(EPOCHS): for idx in range(0, x_train.shape[0]): print('Started to train picture no.' + str(idx) + ' in epoch no.' + str(i)) predictor.train_parameters(observation=torch.from_numpy( x_train[idx].reshape(1, 784) / 255.0).float(), target=torch.from_numpy( np.eye(10)[y_train[idx]].reshape( 1, 10)).float())
num_classes=num_classes, num_gauss=5, num_sums=5) args = SpnArgs() args.num_gauss = 5 args.num_sums = 5 spns = vector_list[-1][0] tensor_spn = RatSpn(10, vector_list=vector_list, args=args, name='tensor-spn-from-vectorlist') input_ph = tf.placeholder(tf.float32, (1000, 28 * 28)) output = tensor_spn.forward(input_ph) (train_im, train_lab), (test_im, test_lab) = mnist('data/mnist') scalar = StandardScaler().fit(train_im) train_im = scalar.transform(train_im) test_im = scalar.transform(test_im) add_parametric_inference_support() for spn in spns: valid, err = is_valid(spn) print(valid, err) print(get_structure_stats(spn)) print("starting") tfstart = time.perf_counter() log_likelihood(spn, train_im[0:1000, :]) end = time.perf_counter()
def __init__(self, datadir, batch_size): data = obs.mnist(join(datadir, "MNIST")) self._load(data, batch_size)
def run(): (x_train, y_train), (x_test, y_test) = observations.mnist('~/data/MNIST') dataset_size = x_train.shape[0] inputs = tf.placeholder(tf.float32, [None, 28 * 28], 'inputs') labels = tf.placeholder(tf.int32, [None], 'labels') data_indices = tf.placeholder(tf.int32, [None], 'data_indices') def network(context: modular.ModularContext): modules = modular.create_dense_modules(inputs, module_count=10, units=32) hidden = modular.modular_layer(inputs, modules, parallel_count=1, context=context) hidden = tf.nn.relu(hidden) modules = modular.create_dense_modules(hidden, module_count=8, units=10) logits = modular.modular_layer(hidden, modules, parallel_count=1, context=context) target = modular.modularize_target(labels, context) loglikelihood = tf.distributions.Categorical(logits).log_prob(target) predicted = tf.argmax(logits, axis=-1, output_type=tf.int32) accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, target), tf.float32)) selection_entropy = context.selection_entropy() batch_selection_entropy = context.batch_selection_entropy() return loglikelihood, logits, accuracy, selection_entropy, batch_selection_entropy template = tf.make_template('network', network) optimizer = tf.train.AdamOptimizer() e_step, m_step, eval = modular.modularize(template, optimizer, dataset_size, data_indices, sample_size=10) ll, logits, accuracy, s_entropy, bs_entropy = eval tf.summary.scalar('loglikelihood', tf.reduce_mean(ll)) tf.summary.scalar('accuracy', accuracy) tf.summary.scalar('entropy/exp_selection', tf.exp(s_entropy)) tf.summary.scalar('entropy/exp_batch_selection', tf.exp(bs_entropy)) with tf.Session() as sess: time = '{:%Y-%m-%d_%H:%M:%S}'.format(datetime.datetime.now()) writer = tf.summary.FileWriter('logs/train_{}'.format(time)) test_writer = tf.summary.FileWriter('logs/test_{}'.format(time)) general_summaries = tf.summary.merge_all() m_step_summaries = tf.summary.merge([create_m_step_summaries(), general_summaries]) sess.run(tf.global_variables_initializer()) batches = generator([x_train, y_train, np.arange(dataset_size)], 32) for i, (batch_x, batch_y, indices) in enumerate(batches): feed_dict = { inputs: batch_x, labels: batch_y, data_indices: indices } step = e_step if i % 10 == 0 else m_step summaries = m_step_summaries if step == m_step else general_summaries _, summary_data = sess.run([step, summaries], feed_dict) writer.add_summary(summary_data, global_step=i) if i % 100 == 0: test_feed_dict = {inputs: x_test, labels: y_test} summary_data = sess.run(general_summaries, test_feed_dict) test_writer.add_summary(summary_data, global_step=i) writer.close() test_writer.close()
''' Tensorflow experimental implementation of the paper Deep Predictive Coding Network for Object Recognition https://arxiv.org/pdf/1802.04762.pdf ''' from observations import mnist import tensorflow as tf import numpy as np from sklearn.metrics import classification_report, accuracy_score import datetime (x_train, y_train), (x_test, y_test) = mnist('./data') graph = tf.Graph() convolutional_layer_specs = [{ 'filters': [2, 2, 1, 1], 'strides': [1, 1, 1, 1] }, { 'filters': [2, 2, 1, 1], 'strides': [1, 1, 1, 1] }, { 'filters': [2, 2, 1, 1], 'strides': [1, 1, 1, 1] }, { 'filters': [2, 2, 1, 1], 'strides': [1, 1, 1, 1] }, {
def run(): (x_train, y_train), (x_test, y_test) = observations.mnist('~/data/MNIST') dataset_size = x_train.shape[0] inputs = tf.placeholder(tf.float32, [None, 28 * 28], 'inputs') labels = tf.placeholder(tf.int32, [None], 'labels') data_indices = tf.placeholder(tf.int32, [None], 'data_indices') def network(context: modular.ModularContext): modules = modular.create_dense_modules(inputs, module_count=10, units=32) hidden = modular.modular_layer(inputs, modules, parallel_count=1, context=context) hidden = tf.nn.relu(hidden) modules = modular.create_dense_modules(hidden, module_count=8, units=10) logits = modular.modular_layer(hidden, modules, parallel_count=1, context=context) target = modular.modularize_target(labels, context) loglikelihood = tf.distributions.Categorical(logits).log_prob(target) predicted = tf.argmax(logits, axis=-1, output_type=tf.int32) accuracy = tf.reduce_mean( tf.cast(tf.equal(predicted, target), tf.float32)) selection_entropy = context.selection_entropy() batch_selection_entropy = context.batch_selection_entropy() return loglikelihood, logits, accuracy, selection_entropy, batch_selection_entropy template = tf.make_template('network', network) optimizer = tf.train.AdamOptimizer() e_step, m_step, eval = modular.modularize(template, optimizer, dataset_size, data_indices, sample_size=10) ll, logits, accuracy, s_entropy, bs_entropy = eval tf.summary.scalar('loglikelihood', tf.reduce_mean(ll)) tf.summary.scalar('accuracy', accuracy) tf.summary.scalar('entropy/exp_selection', tf.exp(s_entropy)) tf.summary.scalar('entropy/exp_batch_selection', tf.exp(bs_entropy)) with tf.Session() as sess: time = '{:%Y-%m-%d_%H:%M:%S}'.format(datetime.datetime.now()) writer = tf.summary.FileWriter('logs/train_{}'.format(time)) test_writer = tf.summary.FileWriter('logs/test_{}'.format(time)) general_summaries = tf.summary.merge_all() m_step_summaries = tf.summary.merge( [create_m_step_summaries(), general_summaries]) sess.run(tf.global_variables_initializer()) batches = generator([x_train, y_train, np.arange(dataset_size)], 32) for i, (batch_x, batch_y, indices) in enumerate(batches): feed_dict = { inputs: batch_x, labels: batch_y, data_indices: indices } step = e_step if i % 10 == 0 else m_step summaries = m_step_summaries if step == m_step else general_summaries _, summary_data = sess.run([step, summaries], feed_dict) writer.add_summary(summary_data, global_step=i) if i % 100 == 0: test_feed_dict = {inputs: x_test, labels: y_test} summary_data = sess.run(general_summaries, test_feed_dict) test_writer.add_summary(summary_data, global_step=i) writer.close() test_writer.close()
def load_multi_mnist(path, max_digits=2, canvas_size=50, seed=42): """ Code pulled from observations library and customized to collect bounding box information. Load the multiple MNIST data set [@eslami2016attend]. It modifies the original MNIST such that each image contains a number of non-overlapping random MNIST digits with equal probability. Args: path: str. Path to directory which either stores file or otherwise file will be downloaded and extracted there. Filename is `'multi_mnist_{}_{}_{}.npz'.format(max_digits, canvas_size, seed)`. max_digits: int, optional. Maximum number of non-overlapping MNIST digits per image to generate if not cached. canvas_size: list of two int, optional. Width x height pixel size of generated images if not cached. seed: int, optional. Random seed to generate the data set from MNIST if not cached. Returns: Tuple of (np.ndarray of dtype uint8, list) `(x_train, y_train), (x_test, y_test)`. Each element in the y's is a np.ndarray of labels, one label for each digit in the image. """ def _sample_one(canvas_size, x_data, y_data): i = np.random.randint(x_data.shape[0]) digit = x_data[i] label = y_data[i] scale = 0.1 * np.random.randn() + 1.3 resized = scipy.misc.imresize(digit, 1.0 / scale) width = resized.shape[0] padding = canvas_size - width pad_l = np.random.randint(0, padding) pad_r = np.random.randint(0, padding) pad_width = ((pad_l, padding - pad_l), (pad_r, padding - pad_r)) positioned = np.pad(resized, pad_width, 'constant', constant_values=0) bbox = (pad_l, pad_r, pad_l + width, pad_r + width) return positioned, label, bbox def _sample_multi(num_digits, canvas_size, x_data, y_data): canvas = np.zeros((canvas_size, canvas_size)) labels = [] bboxes = [] for _ in range(num_digits): positioned_digit, label, bbox = _sample_one( canvas_size, x_data, y_data) canvas += positioned_digit labels.append(label) bboxes.append(bbox) labels = np.array(labels, dtype=np.uint8) if np.max(canvas) > 255: # crude check for overlapping digits return _sample_multi(num_digits, canvas_size, x_data, y_data) else: return canvas, labels, bboxes def _build_dataset(x_data, y_data, max_digits, canvas_size): x = [] y = [] data_size = x_data.shape[0] data_num_digits = np.random.randint(max_digits + 1, size=data_size) x_data = np.reshape(x_data, [data_size, 28, 28]) bboxes_arr = np.zeros((data_size, max_digits, 4)) for i, num_digits in enumerate(data_num_digits): canvas, labels, bboxes = _sample_multi(num_digits, canvas_size, x_data, y_data) x.append(canvas) y.append(labels) for j, bbox in enumerate(bboxes): bboxes_arr[i, j] = bbox x = np.array(x, dtype=np.uint8) return x, y, bboxes_arr path = os.path.expanduser(path) cache_filename = 'multi_mnist_{}_{}_{}.npz'.format(max_digits, canvas_size, seed) if os.path.exists(os.path.join(path, cache_filename)): data = np.load(os.path.join(path, cache_filename), allow_pickle=True) return (data['x_train'], data['y_train'], data['x_bbox']),\ (data['x_test'], data['y_test'], data['y_bbox']) np.random.seed(seed) (x_train, y_train), (x_test, y_test) = mnist(path) x_train, y_train, x_bbox = _build_dataset(x_train, y_train, max_digits, canvas_size) x_test, y_test, y_bbox = _build_dataset(x_test, y_test, max_digits, canvas_size) with open(os.path.join(path, cache_filename), 'wb') as f: np.savez_compressed(f, x_train=x_train, y_train=y_train, x_test=x_test, y_test=y_test, x_bbox=x_bbox, y_bbox=y_bbox) return (x_train, y_train, x_bbox), (x_test, y_test, y_bbox)