def main(_): ed.set_seed(42) # DATA. MNIST batches are fed at training time. (x_train, _), (x_test, _) = mnist(FLAGS.data_dir) x_train_generator = generator(x_train, FLAGS.M) x_ph = tf.placeholder(tf.float32, [FLAGS.M, 784]) z_ph = tf.placeholder(tf.float32, [FLAGS.M, FLAGS.d]) # MODEL with tf.variable_scope("Gen"): xf = gen_data(z_ph, FLAGS.hidden_units) zf = gen_latent(x_ph, FLAGS.hidden_units) # INFERENCE: optimizer = tf.train.AdamOptimizer() optimizer_d = tf.train.AdamOptimizer() inference = ed.BiGANInference( latent_vars={zf: z_ph}, data={xf: x_ph}, discriminator=discriminative_network) inference.initialize( optimizer=optimizer, optimizer_d=optimizer_d, n_iter=100000, n_print=3000) sess = ed.get_session() init_op = tf.global_variables_initializer() sess.run(init_op) idx = np.random.randint(FLAGS.M, size=16) i = 0 for t in range(inference.n_iter): if t % inference.n_print == 1: samples = sess.run(xf, feed_dict={z_ph: z_batch}) samples = samples[idx, ] fig = plot(samples) plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format( 'Generated', str(i).zfill(3)), bbox_inches='tight') plt.close(fig) fig = plot(x_batch[idx, ]) plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format( 'Base', str(i).zfill(3)), bbox_inches='tight') plt.close(fig) zsam = sess.run(zf, feed_dict={x_ph: x_batch}) reconstructions = sess.run(xf, feed_dict={z_ph: zsam}) reconstructions = reconstructions[idx, ] fig = plot(reconstructions) plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format( 'Reconstruct', str(i).zfill(3)), bbox_inches='tight') plt.close(fig) i += 1 x_batch = next(x_train_generator) z_batch = np.random.normal(0, 1, [FLAGS.M, FLAGS.d]) info_dict = inference.update(feed_dict={x_ph: x_batch, z_ph: z_batch}) inference.print_progress(info_dict)
def main(_): ed.set_seed(42) # DATA x_data = np.array([0, 1, 0, 0, 0, 0, 0, 0, 0, 1]) # MODEL p = Beta(1.0, 1.0) x = Bernoulli(probs=p, sample_shape=10) # INFERENCE qp = Empirical(params=tf.get_variable( "qp/params", [1000], initializer=tf.constant_initializer(0.5))) proposal_p = Beta(3.0, 9.0) inference = ed.MetropolisHastings({p: qp}, {p: proposal_p}, data={x: x_data}) inference.run() # CRITICISM # exact posterior has mean 0.25 and std 0.12 sess = ed.get_session() mean, stddev = sess.run([qp.mean(), qp.stddev()]) print("Inferred posterior mean:") print(mean) print("Inferred posterior stddev:") print(stddev) x_post = ed.copy(x, {p: qp}) tx_rep, tx = ed.ppc( lambda xs, zs: tf.reduce_mean(tf.cast(xs[x_post], tf.float32)), data={x_post: x_data}) ed.ppc_stat_hist_plot( tx[0], tx_rep, stat_name=r'$T \equiv$mean', bins=10) plt.show()
def main(_): ed.set_seed(42) # MODEL z = MultivariateNormalTriL( loc=tf.ones(2), scale_tril=tf.cholesky(tf.constant([[1.0, 0.8], [0.8, 1.0]]))) # INFERENCE qz = Empirical(params=tf.get_variable("qz/params", [1000, 2])) inference = ed.HMC({z: qz}) inference.run() # CRITICISM sess = ed.get_session() mean, stddev = sess.run([qz.mean(), qz.stddev()]) print("Inferred posterior mean:") print(mean) print("Inferred posterior stddev:") print(stddev) fig, ax = plt.subplots() trace = sess.run(qz.params) ax.scatter(trace[:, 0], trace[:, 1], marker=".") mvn_plot_contours(z, ax=ax) plt.show()
def test_1d(self): ed.set_seed(98765) with self.test_session(): _test(tf.zeros([1]) + 0.5, tf.zeros([1]) + 0.5, [1]) _test(tf.zeros([1]) + 0.5, tf.zeros([1]) + 0.5, [5]) _test(tf.zeros([5]) + 0.5, tf.zeros([5]) + 0.5, [1]) _test(tf.zeros([5]) + 0.5, tf.zeros([5]) + 0.5, [5])
def test_2d(self): ed.set_seed(12142) with self.test_session(): _test(tf.constant([[0.5, 0.5], [0.6, 0.4]]), [1]) _test(tf.constant([[0.5, 0.5], [0.6, 0.4]]), [2]) _test(tf.constant([[0.3, 0.2, 0.5], [0.6, 0.1, 0.3]]), [1]) _test(tf.constant([[0.3, 0.2, 0.5], [0.6, 0.1, 0.3]]), [2])
def main(_): ed.set_seed(42) N = 5000 # number of data points D = 10 # number of features # DATA w_true = np.random.randn(D) X_data = np.random.randn(N, D) p = expit(np.dot(X_data, w_true)) y_data = np.array([np.random.binomial(1, i) for i in p]) # MODEL X = tf.placeholder(tf.float32, [N, D]) w = Normal(loc=tf.zeros(D), scale=tf.ones(D)) y = Bernoulli(logits=ed.dot(X, w)) # INFERENCE qw = Normal(loc=tf.get_variable("qw/loc", [D]), scale=tf.nn.softplus(tf.get_variable("qw/scale", [D]))) inference = IWVI({w: qw}, data={X: X_data, y: y_data}) inference.run(K=5, n_iter=1000) # CRITICISM print("Mean squared error in true values to inferred posterior mean:") print(tf.reduce_mean(tf.square(w_true - qw.mean())).eval())
def main(_): ed.set_seed(42) # DATA x_data = build_toy_dataset(FLAGS.N) # MODEL pi = Dirichlet(concentration=tf.ones(FLAGS.K)) mu = Normal(0.0, 1.0, sample_shape=[FLAGS.K, FLAGS.D]) sigma = InverseGamma(concentration=1.0, rate=1.0, sample_shape=[FLAGS.K, FLAGS.D]) c = Categorical(logits=tf.log(pi) - tf.log(1.0 - pi), sample_shape=FLAGS.N) x = Normal(loc=tf.gather(mu, c), scale=tf.gather(sigma, c)) # INFERENCE qpi = Empirical(params=tf.get_variable( "qpi/params", [FLAGS.T, FLAGS.K], initializer=tf.constant_initializer(1.0 / FLAGS.K))) qmu = Empirical(params=tf.get_variable("qmu/params", [FLAGS.T, FLAGS.K, FLAGS.D], initializer=tf.zeros_initializer())) qsigma = Empirical(params=tf.get_variable("qsigma/params", [FLAGS.T, FLAGS.K, FLAGS.D], initializer=tf.ones_initializer())) qc = Empirical(params=tf.get_variable("qc/params", [FLAGS.T, FLAGS.N], initializer=tf.zeros_initializer(), dtype=tf.int32)) gpi = Dirichlet(concentration=tf.constant([1.4, 1.6])) gmu = Normal(loc=tf.constant([[1.0, 1.0], [-1.0, -1.0]]), scale=tf.constant([[0.5, 0.5], [0.5, 0.5]])) gsigma = InverseGamma(concentration=tf.constant([[1.1, 1.1], [1.1, 1.1]]), rate=tf.constant([[1.0, 1.0], [1.0, 1.0]])) gc = Categorical(logits=tf.zeros([FLAGS.N, FLAGS.K])) inference = ed.MetropolisHastings( latent_vars={pi: qpi, mu: qmu, sigma: qsigma, c: qc}, proposal_vars={pi: gpi, mu: gmu, sigma: gsigma, c: gc}, data={x: x_data}) inference.initialize() sess = ed.get_session() tf.global_variables_initializer().run() for _ in range(inference.n_iter): info_dict = inference.update() inference.print_progress(info_dict) t = info_dict['t'] if t == 1 or t % inference.n_print == 0: qpi_mean, qmu_mean = sess.run([qpi.mean(), qmu.mean()]) print("") print("Inferred membership probabilities:") print(qpi_mean) print("Inferred cluster means:") print(qmu_mean)
def build_toy_dataset(N=40, noise_std=0.1): ed.set_seed(0) x = np.concatenate([np.linspace(0, 2, num=N/2), np.linspace(6, 8, num=N/2)]) y = 0.075*x + norm.rvs(0, noise_std, size=N) x = (x - 4.0) / 4.0 x = x.reshape((N, 1)) return {'x': x, 'y': y}
def test_random(self): with self.test_session() as sess: ed.set_seed(3742) x = tf.random_normal([]) x_copy = ed.copy(x) result_copy, result = sess.run([x_copy, x]) self.assertNotAlmostEquals(result_copy, result)
def build_toy_dataset(N=40, noise_std=0.1): ed.set_seed(0) D = 1 x = np.concatenate([np.linspace(0, 2, num=N / 2), np.linspace(6, 8, num=N / 2)]) y = np.cos(x) + norm.rvs(0, noise_std, size=N) x = (x - 4.0) / 4.0 x = x.reshape((N, D)) return {"x": x, "y": y}
def build_toy_dataset(n_data=40, noise_std=0.1): ed.set_seed(0) D = 1 x = np.concatenate([np.linspace(0, 2, num=n_data/2), np.linspace(6, 8, num=n_data/2)]) y = np.cos(x) + norm.rvs(0, noise_std, size=n_data) x = (x - 4.0) / 4.0 x = x.reshape((n_data, D)) return {'x': x, 'y': y}
def test_swap_tensor_rv(self): with self.test_session(): ed.set_seed(95258) x = Normal(0.0, 0.1) y = tf.constant(1.0) z = x * y qx = Normal(10.0, 0.1) z_new = ed.copy(z, {x.value(): qx}) self.assertGreater(z_new.eval(), 5.0)
def test_scan(self): with self.test_session() as sess: ed.set_seed(42) op = tf.scan(lambda a, x: a + x, tf.constant([2.0, 3.0, 1.0])) copy_op = ed.copy(op) result_copy, result = sess.run([copy_op, op]) self.assertAllClose(result_copy, [2.0, 5.0, 6.0]) self.assertAllClose(result, [2.0, 5.0, 6.0])
def test_swap_rv_rv(self): with self.test_session(): ed.set_seed(325135) x = Normal(0.0, 0.1) y = tf.constant(1.0) z = x * y qx = Normal(10.0, 0.1) z_new = ed.copy(z, {x: qx}) self.assertGreater(z_new.eval(), 5.0)
def build_toy_dataset(n_data=40, noise_std=0.1): ed.set_seed(0) D = 1 x = np.linspace(-3, 3, num=n_data) y = np.tanh(x) + norm.rvs(0, noise_std, size=n_data) y[y < 0.5] = 0 y[y >= 0.5] = 1 x = (x - 4.0) / 4.0 x = x.reshape((n_data, D)) return {'x': x, 'y': y}
def main(_): ed.set_seed(42) # DATA. MNIST batches are fed at training time. (x_train, _), (x_test, _) = mnist(FLAGS.data_dir) x_train_generator = generator(x_train, FLAGS.M) # MODEL # Define a subgraph of the full model, corresponding to a minibatch of # size M. z = Normal(loc=tf.zeros([FLAGS.M, FLAGS.d]), scale=tf.ones([FLAGS.M, FLAGS.d])) hidden = tf.layers.dense(z, 256, activation=tf.nn.relu) x = Bernoulli(logits=tf.layers.dense(hidden, 28 * 28)) # INFERENCE # Define a subgraph of the variational model, corresponding to a # minibatch of size M. x_ph = tf.placeholder(tf.int32, [FLAGS.M, 28 * 28]) hidden = tf.layers.dense(tf.cast(x_ph, tf.float32), 256, activation=tf.nn.relu) qz = Normal(loc=tf.layers.dense(hidden, FLAGS.d), scale=tf.layers.dense( hidden, FLAGS.d, activation=tf.nn.softplus)) # Bind p(x, z) and q(z | x) to the same TensorFlow placeholder for x. inference = ed.KLqp({z: qz}, data={x: x_ph}) optimizer = tf.train.RMSPropOptimizer(0.01, epsilon=1.0) inference.initialize(optimizer=optimizer) tf.global_variables_initializer().run() n_iter_per_epoch = x_train.shape[0] // FLAGS.M for epoch in range(1, FLAGS.n_epoch + 1): print("Epoch: {0}".format(epoch)) avg_loss = 0.0 pbar = Progbar(n_iter_per_epoch) for t in range(1, n_iter_per_epoch + 1): pbar.update(t) x_batch = next(x_train_generator) info_dict = inference.update(feed_dict={x_ph: x_batch}) avg_loss += info_dict['loss'] # Print a lower bound to the average marginal likelihood for an # image. avg_loss /= n_iter_per_epoch avg_loss /= FLAGS.M print("-log p(x) <= {:0.3f}".format(avg_loss)) # Prior predictive check. images = x.eval() for m in range(FLAGS.M): imsave(os.path.join(FLAGS.out_dir, '%d.png') % m, images[m].reshape(28, 28))
def test_ordering_rv_tensor(self): # Check that random variables are copied correctly in dependency # structure. with self.test_session() as sess: ed.set_seed(12432) x = Bernoulli(logits=0.0) y = tf.cast(x, tf.float32) y_new = ed.copy(y) x_new = ed.copy(x) x_new_val, y_new_val = sess.run([x_new, y_new]) self.assertEqual(x_new_val, y_new_val)
def build_toy_dataset(n_data=40, noise_std=0.1): ed.set_seed(0) x = np.concatenate([np.linspace(0, 2, num=n_data/2), np.linspace(6, 8, num=n_data/2)]) y = 0.075*x + norm.rvs(0, noise_std, size=n_data) x = (x - 4.0) / 4.0 x = x.reshape((n_data, 1)) y = y.reshape((n_data, 1)) data = np.concatenate((y, x), axis=1) # n_data x 2 data = tf.constant(data, dtype=tf.float32) return ed.Data(data)
def test_ordering_rv_rv(self): # Check that random variables are copied correctly in dependency # structure. with self.test_session() as sess: ed.set_seed(21782) x = Normal(loc=0.0, scale=10.0) x_abs = tf.abs(x) y = Normal(loc=x_abs, scale=1e-8) y_new = ed.copy(y) x_new = ed.copy(x) x_new_val, y_new_val = sess.run([x_new, y_new]) self.assertAllClose(abs(x_new_val), y_new_val)
def build_toy_dataset(n_data=40, noise_std=0.1): ed.set_seed(0) D = 1 x = np.linspace(-3, 3, num=n_data) y = np.tanh(x) + norm.rvs(0, noise_std, size=n_data) y[y < 0.5] = 0 y[y >= 0.5] = 1 x = (x - 4.0) / 4.0 x = x.reshape((n_data, D)) y = y.reshape((n_data, 1)) data = np.concatenate((y, x), axis=1) # n_data x (D+1) data = tf.constant(data, dtype=tf.float32) return ed.Data(data)
def build_toy_dataset(coeff, n_data=40, n_data_test=20, noise_std=0.1): ed.set_seed(0) n_dim = len(coeff) x = np.random.randn(n_data+n_data_test, n_dim) y = np.dot(x, coeff) + norm.rvs(0, noise_std, size=(n_data+n_data_test)) y = y.reshape((n_data+n_data_test, 1)) data = np.concatenate((y[:n_data,:], x[:n_data,:]), axis=1) data = tf.constant(data, dtype=tf.float32) data_test = np.concatenate((y[n_data:,:], x[n_data:,:]), axis=1) data_test = tf.constant(data_test, dtype=tf.float32) return ed.Data(data), ed.Data(data_test)
def main(_): ed.set_seed(42) # DATA. MNIST batches are fed at training time. (x_train, _), (x_test, _) = mnist(FLAGS.data_dir) x_train_generator = generator(x_train, FLAGS.M) # MODEL z = Normal(loc=tf.zeros([FLAGS.M, FLAGS.d]), scale=tf.ones([FLAGS.M, FLAGS.d])) logits = generative_network(z) x = Bernoulli(logits=logits) # INFERENCE x_ph = tf.placeholder(tf.int32, [FLAGS.M, 28 * 28]) loc, scale = inference_network(tf.cast(x_ph, tf.float32)) qz = Normal(loc=loc, scale=scale) # Bind p(x, z) and q(z | x) to the same placeholder for x. inference = ed.KLqp({z: qz}, data={x: x_ph}) optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0) inference.initialize(optimizer=optimizer) hidden_rep = tf.sigmoid(logits) tf.global_variables_initializer().run() n_iter_per_epoch = x_train.shape[0] // FLAGS.M for epoch in range(1, FLAGS.n_epoch + 1): print("Epoch: {0}".format(epoch)) avg_loss = 0.0 pbar = Progbar(n_iter_per_epoch) for t in range(1, n_iter_per_epoch + 1): pbar.update(t) x_batch = next(x_train_generator) info_dict = inference.update(feed_dict={x_ph: x_batch}) avg_loss += info_dict['loss'] # Print a lower bound to the average marginal likelihood for an # image. avg_loss /= n_iter_per_epoch avg_loss /= FLAGS.M print("-log p(x) <= {:0.3f}".format(avg_loss)) # Visualize hidden representations. images = hidden_rep.eval() for m in range(FLAGS.M): imsave(os.path.join(FLAGS.out_dir, '%d.png') % m, images[m].reshape(28, 28))
def main(_): sns.set(color_codes=True) ed.set_seed(42) # DATA. We use a placeholder to represent a minibatch. During # inference, we generate data on the fly and feed `x_ph`. x_ph = tf.placeholder(tf.float32, [FLAGS.M, 1]) # MODEL with tf.variable_scope("Gen"): eps = tf.linspace(-8.0, 8.0, FLAGS.M) + 0.01 * tf.random_normal([FLAGS.M]) eps = tf.reshape(eps, [FLAGS.M, 1]) x = generative_network(eps) # INFERENCE optimizer = tf.train.GradientDescentOptimizer(0.03) optimizer_d = tf.train.GradientDescentOptimizer(0.03) inference = ed.WGANInference( data={x: x_ph}, discriminator=discriminative_network) inference.initialize( optimizer=optimizer, optimizer_d=optimizer_d, penalty=0.1, n_iter=1000) tf.global_variables_initializer().run() for _ in range(inference.n_iter): x_data = next_batch(FLAGS.M).reshape([FLAGS.M, 1]) for _ in range(5): info_dict_d = inference.update(feed_dict={x_ph: x_data}, variables="Disc") info_dict = inference.update(feed_dict={x_ph: x_data}, variables="Gen") info_dict['t'] = info_dict['t'] // 6 # say set of 6 updates is 1 iteration info_dict['loss_d'] = info_dict_d['loss_d'] # get disc loss from update inference.print_progress(info_dict) # CRITICISM db, pd, pg = get_samples(x_ph) db_x = np.linspace(-8, 8, len(db)) p_x = np.linspace(-8, 8, len(pd)) f, ax = plt.subplots(1) ax.plot(db_x, db, label="Decision boundary") ax.set_ylim(0, 1) plt.plot(p_x, pd, label="Real data") plt.plot(p_x, pg, label="Generated data") plt.title("1D Generative Adversarial Network") plt.xlabel("Data values") plt.ylabel("Probability density") plt.legend() plt.show()
def main(_): ed.set_seed(142) # DATA x_train = build_toy_dataset(FLAGS.N, FLAGS.D, FLAGS.K) # MODEL w = Normal(loc=0.0, scale=10.0, sample_shape=[FLAGS.D, FLAGS.K]) z = Normal(loc=0.0, scale=1.0, sample_shape=[FLAGS.M, FLAGS.K]) x = Normal(loc=tf.matmul(w, z, transpose_b=True), scale=tf.ones([FLAGS.D, FLAGS.M])) # INFERENCE qw_variables = [tf.get_variable("qw/loc", [FLAGS.D, FLAGS.K]), tf.get_variable("qw/scale", [FLAGS.D, FLAGS.K])] qw = Normal(loc=qw_variables[0], scale=tf.nn.softplus(qw_variables[1])) qz_variables = [tf.get_variable("qz/loc", [FLAGS.N, FLAGS.K]), tf.get_variable("qz/scale", [FLAGS.N, FLAGS.K])] idx_ph = tf.placeholder(tf.int32, FLAGS.M) qz = Normal(loc=tf.gather(qz_variables[0], idx_ph), scale=tf.nn.softplus(tf.gather(qz_variables[1], idx_ph))) x_ph = tf.placeholder(tf.float32, [FLAGS.D, FLAGS.M]) inference_w = ed.KLqp({w: qw}, data={x: x_ph, z: qz}) inference_z = ed.KLqp({z: qz}, data={x: x_ph, w: qw}) scale_factor = float(FLAGS.N) / FLAGS.M inference_w.initialize(scale={x: scale_factor, z: scale_factor}, var_list=qz_variables, n_samples=5) inference_z.initialize(scale={x: scale_factor, z: scale_factor}, var_list=qw_variables, n_samples=5) sess = ed.get_session() tf.global_variables_initializer().run() for _ in range(inference_w.n_iter): x_batch, idx_batch = next_batch(x_train, FLAGS.M) for _ in range(5): inference_z.update(feed_dict={x_ph: x_batch, idx_ph: idx_batch}) info_dict = inference_w.update(feed_dict={x_ph: x_batch, idx_ph: idx_batch}) inference_w.print_progress(info_dict) t = info_dict['t'] if t % 100 == 0: print("\nInferred principal axes:") print(sess.run(qw.mean()))
def main(_): ed.set_seed(42) # DATA. MNIST batches are fed at training time. (x_train, _), (x_test, _) = mnist(FLAGS.data_dir) x_train_generator = generator(x_train, FLAGS.M) x_ph = tf.placeholder(tf.float32, [FLAGS.M, 784]) # MODEL with tf.variable_scope("Gen"): eps = Uniform(low=tf.zeros([FLAGS.M, FLAGS.d]) - 1.0, high=tf.ones([FLAGS.M, FLAGS.d])) x = generative_network(eps) # INFERENCE optimizer = tf.train.RMSPropOptimizer(learning_rate=5e-5) optimizer_d = tf.train.RMSPropOptimizer(learning_rate=5e-5) inference = ed.WGANInference( data={x: x_ph}, discriminator=discriminative_network) inference.initialize( optimizer=optimizer, optimizer_d=optimizer_d, n_iter=15000, n_print=1000, clip=0.01, penalty=None) sess = ed.get_session() tf.global_variables_initializer().run() idx = np.random.randint(FLAGS.M, size=16) i = 0 for t in range(inference.n_iter): if t % inference.n_print == 0: samples = sess.run(x) samples = samples[idx, ] fig = plot(samples) plt.savefig(os.path.join(FLAGS.out_dir, '{}.png').format( str(i).zfill(3)), bbox_inches='tight') plt.close(fig) i += 1 x_batch = next(x_train_generator) for _ in range(5): inference.update(feed_dict={x_ph: x_batch}, variables="Disc") info_dict = inference.update(feed_dict={x_ph: x_batch}, variables="Gen") # note: not printing discriminative objective; `info_dict` above # does not store it since updating only "Gen" info_dict['t'] = info_dict['t'] // 6 # say set of 6 updates is 1 iteration inference.print_progress(info_dict)
def main(_): ed.set_seed(42) # Prior on scalar hyperparameter to Dirichlet. alpha = Gamma(1.0, 1.0) # Prior on size of Dirichlet. n = 1 + tf.cast(Exponential(0.5), tf.int32) # Build a vector of ones whose size is n; multiply it by alpha. p = Dirichlet(tf.ones([n]) * alpha) sess = ed.get_session() print(sess.run(p)) # [ 0.01012419 0.02939712 0.05036638 0.51287931 0.31020424 0.0485355 # 0.0384932 ] print(sess.run(p))
def main(_): ed.set_seed(42) # MODEL z = MultivariateNormalTriL( loc=tf.ones(2), scale_tril=tf.cholesky(tf.constant([[1.0, 0.8], [0.8, 1.0]]))) # INFERENCE qz = Empirical(params=tf.get_variable("qz/params", [2000, 2])) inference = ed.SGLD({z: qz}) inference.run(step_size=5.0) # CRITICISM sess = ed.get_session() mean, stddev = sess.run([qz.mean(), qz.stddev()]) print("Inferred posterior mean:") print(mean) print("Inferred posterior stddev:") print(stddev)
def main(_): ed.set_seed(42) # DATA x_data = np.array([0, 1, 0, 0, 0, 0, 0, 0, 0, 1]) # MODEL p = Beta(1.0, 1.0) x = Bernoulli(probs=p, sample_shape=10) # COMPLETE CONDITIONAL p_cond = ed.complete_conditional(p) sess = ed.get_session() print('p(probs | x) type:', p_cond.parameters['name']) param_vals = sess.run({key: val for key, val in six.iteritems(p_cond.parameters) if isinstance(val, tf.Tensor)}, {x: x_data}) print('parameters:') for key, val in six.iteritems(param_vals): print('%s:\t%.3f' % (key, val))
def main(_): ed.set_seed(42) # DATA x_data = build_toy_dataset(FLAGS.N, FLAGS.V) # MODEL x_ph = tf.placeholder(tf.float32, [FLAGS.N, FLAGS.V]) # Form (N, V, V) covariance, one matrix per data point. K = tf.stack([rbf(tf.reshape(xn, [FLAGS.V, 1])) + tf.diag([1e-6, 1e-6]) for xn in tf.unstack(x_ph)]) f = MultivariateNormalTriL(loc=tf.zeros([FLAGS.N, FLAGS.V]), scale_tril=tf.cholesky(K)) x = Poisson(rate=tf.exp(f)) # INFERENCE qf = Normal( loc=tf.get_variable("qf/loc", [FLAGS.N, FLAGS.V]), scale=tf.nn.softplus(tf.get_variable("qf/scale", [FLAGS.N, FLAGS.V]))) inference = ed.KLqp({f: qf}, data={x: x_data, x_ph: x_data}) inference.run(n_iter=5000)
def main(_): ed.set_seed(42) # DATA X_data, Z_true = karate("~/data") N = X_data.shape[0] # number of vertices K = 2 # number of clusters # MODEL gamma = Dirichlet(concentration=tf.ones([K])) Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K])) Z = Multinomial(total_count=1.0, probs=gamma, sample_shape=N) X = Bernoulli(probs=tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z)))) # INFERENCE (EM algorithm) qgamma = PointMass(tf.nn.softmax(tf.get_variable("qgamma/params", [K]))) qPi = PointMass(tf.nn.sigmoid(tf.get_variable("qPi/params", [K, K]))) qZ = PointMass(tf.nn.softmax(tf.get_variable("qZ/params", [N, K]))) inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: X_data}) inference.initialize(n_iter=250) tf.global_variables_initializer().run() for _ in range(inference.n_iter): info_dict = inference.update() inference.print_progress(info_dict) # CRITICISM Z_pred = qZ.mean().eval().argmax(axis=1) print("Result (label flip can happen):") print("Predicted") print(Z_pred) print("True") print(Z_true) print("Adjusted Rand Index =", adjusted_rand_score(Z_pred, Z_true))
with self.test_session() as sess: x = TransformedDistribution( distribution=Normal(1.0, 1.0), bijector=tf.contrib.distributions.bijectors.Softplus()) x.support = 'nonnegative' inference = ed.HMC([x]) inference.initialize(auto_transform=True, step_size=0.8) tf.global_variables_initializer().run() for _ in range(inference.n_iter): info_dict = inference.update() inference.print_progress(info_dict) # Check approximation on constrained space has same moments as # target distribution. n_samples = 10000 x_unconstrained = inference.transformations[x] qx = inference.latent_vars[x_unconstrained] qx_constrained = Empirical( x_unconstrained.bijector.inverse(qx.params)) x_mean, x_var = tf.nn.moments(x.sample(n_samples), 0) qx_mean, qx_var = tf.nn.moments(qx_constrained.params[500:], 0) stats = sess.run([x_mean, qx_mean, x_var, qx_var]) self.assertAllClose(stats[0], stats[1], rtol=1e-1, atol=1e-1) self.assertAllClose(stats[2], stats[3], rtol=1e-1, atol=1e-1) if __name__ == '__main__': ed.set_seed(124125) tf.test.main()
class test_random_variable_session_class(tf.test.TestCase): def test_eval(self): with self.test_session() as sess: x = Normal(mu=0.0, sigma=0.1) x_ph = tf.placeholder(tf.float32, []) y = Normal(mu=x_ph, sigma=0.1) self.assertLess(x.eval(), 5.0) self.assertLess(x.eval(sess), 5.0) self.assertLess(x.eval(feed_dict={x_ph: 100.0}), 5.0) self.assertGreater(y.eval(feed_dict={x_ph: 100.0}), 5.0) self.assertGreater(y.eval(sess, feed_dict={x_ph: 100.0}), 5.0) self.assertRaises(tf.errors.InvalidArgumentError, y.eval) self.assertRaises(tf.errors.InvalidArgumentError, y.eval, sess) def test_run(self): with self.test_session() as sess: x = Normal(mu=0.0, sigma=0.1) x_ph = tf.placeholder(tf.float32, []) y = Normal(mu=x_ph, sigma=0.1) self.assertLess(sess.run(x), 5.0) self.assertLess(sess.run(x, feed_dict={x_ph: 100.0}), 5.0) self.assertGreater(sess.run(y, feed_dict={x_ph: 100.0}), 5.0) self.assertRaises(tf.errors.InvalidArgumentError, sess.run, y) if __name__ == '__main__': ed.set_seed(82341) tf.test.main()
def set_random_seeds(x): tf.set_random_seed(x) ed.set_seed(x) np.random.seed(x)
# describing format of the objects to be returned. features = tf.parse_single_example( serialized_example, features={'outcome': tf.FixedLenFeature([], tf.float32)}) x_batch = features['outcome'] mu = Normal(loc=0.0, scale=1.0) x = Normal(loc=tf.ones([]) * mu, scale=tf.ones([])) qmu = Normal(loc=tf.Variable(0.0), scale=tf.constant(1.0)) inference = ed.KLqp({mu: qmu}, data={x: x_batch}) inference.initialize(scale={x: 10.0}) tf.global_variables_initializer().run() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) # Check data varies by session run. val = sess.run(inference.data[x]) val_1 = sess.run(inference.data[x]) self.assertNotEqual(val, val_1) coord.request_stop() coord.join(threads) if __name__ == '__main__': ed.set_seed(1512351) tf.test.main()
def main(_): ed.set_seed(42) # DATA. MNIST batches are fed at training time. (x_train, _), (x_test, _) = mnist(FLAGS.data_dir) x_train_generator = generator(x_train, FLAGS.M) x_ph = tf.placeholder(tf.float32, [FLAGS.M, 784]) z_ph = tf.placeholder(tf.float32, [FLAGS.M, FLAGS.d]) # MODEL with tf.variable_scope("Gen"): xf = gen_data(z_ph, FLAGS.hidden_units) zf = gen_latent(x_ph, FLAGS.hidden_units) # INFERENCE: optimizer = tf.train.AdamOptimizer() optimizer_d = tf.train.AdamOptimizer() inference = ed.BiGANInference(latent_vars={zf: z_ph}, data={xf: x_ph}, discriminator=discriminative_network) inference.initialize(optimizer=optimizer, optimizer_d=optimizer_d, n_iter=100000, n_print=3000) sess = ed.get_session() init_op = tf.global_variables_initializer() sess.run(init_op) idx = np.random.randint(FLAGS.M, size=16) i = 0 for t in range(inference.n_iter): if t % inference.n_print == 1: samples = sess.run(xf, feed_dict={z_ph: z_batch}) samples = samples[idx, ] fig = plot(samples) plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format( 'Generated', str(i).zfill(3)), bbox_inches='tight') plt.close(fig) fig = plot(x_batch[idx, ]) plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format( 'Base', str(i).zfill(3)), bbox_inches='tight') plt.close(fig) zsam = sess.run(zf, feed_dict={x_ph: x_batch}) reconstructions = sess.run(xf, feed_dict={z_ph: zsam}) reconstructions = reconstructions[idx, ] fig = plot(reconstructions) plt.savefig(os.path.join(FLAGS.out_dir, '{}{}.png').format( 'Reconstruct', str(i).zfill(3)), bbox_inches='tight') plt.close(fig) i += 1 x_batch = next(x_train_generator) z_batch = np.random.normal(0, 1, [FLAGS.M, FLAGS.d]) info_dict = inference.update(feed_dict={x_ph: x_batch, z_ph: z_batch}) inference.print_progress(info_dict)
def main(_): def neural_network(X): h = tf.nn.sigmoid(tf.matmul(X, W_0)) h = tf.nn.sigmoid(tf.matmul(h, W_1)) h = tf.matmul(h, W_2) return tf.reshape(h, [-1]) ed.set_seed(42) # DATA X_train = np.loadtxt('X1.txt', delimiter=",") y_train = np.loadtxt('Y1.txt', delimiter=",") X_train = X_train.reshape((50, 2)) y_train = y_train.reshape((50, )) # MODEL with tf.name_scope("model"): W_0 = Normal(loc=tf.zeros([2, 2]), scale=tf.ones([2, 2]), name="W_0") W_1 = Normal(loc=tf.zeros([2, 2]), scale=tf.ones([2, 2]), name="W_1") W_2 = Normal(loc=tf.zeros([2, 1]), scale=tf.ones([2, 1]), name="W_2") X = tf.placeholder(tf.float32, [50, 2], name="X") y = Normal(loc=neural_network(X), scale=0.1 * tf.ones(50), name="y") # INFERENCE with tf.variable_scope("posterior"): with tf.variable_scope("qW_0"): loc0 = tf.get_variable("loc", [2, 2]) scale0 = tf.nn.softplus(tf.get_variable("scale", [2, 2])) qW_0 = Normal(loc=loc0, scale=scale0) with tf.variable_scope("qW_1"): loc1 = tf.get_variable("loc", [2, 2]) scale1 = tf.nn.softplus(tf.get_variable("scale", [2, 2])) qW_1 = Normal(loc=loc1, scale=scale1) with tf.variable_scope("qW_2"): loc2 = tf.get_variable("loc", [2, 1]) scale2 = tf.nn.softplus(tf.get_variable("scale", [2, 1])) qW_2 = Normal(loc=loc2, scale=scale2) inference = ed.KLqp({ W_0: qW_0, W_1: qW_1, W_2: qW_2 }, data={ X: X_train, y: y_train }) inference.run(n_samples=5, n_iter=10000) y_post = ed.copy(y, {W_0: qW_0, W_1: qW_1, W_2: qW_2}) print(loc0, scale0) print(loc0.eval(), scale0.eval(), loc1.eval(), scale1.eval(), loc2.eval(), scale2.eval()) print("Mean squared error on test data:") print(ed.evaluate('mean_squared_error', data={ X: X_train, y_post: y_train })) print("Mean absolute error on test data:") print( ed.evaluate('mean_absolute_error', data={ X: X_train, y_post: y_train }))
def main(_): ed.set_seed(FLAGS.seed) ((Xtrain, ytrain), (Xtest, ytest)) = blr_utils.get_data() N, D = Xtrain.shape N_test, D_test = Xtest.shape weights, q_components = [], [] g = tf.Graph() with g.as_default(): tf.set_random_seed(FLAGS.seed) sess = tf.InteractiveSession() with sess.as_default(): # MODEL w = Normal(loc=tf.zeros(D), scale=1.0 * tf.ones(D)) X = tf.placeholder(tf.float32, [N, D]) y = Bernoulli(logits=ed.dot(X, w)) X_test = tf.placeholder( tf.float32, [N_test, D_test ]) # TODO why are these test variables necessary? y_test = Bernoulli(logits=ed.dot(X_test, w)) iter = 42 # TODO qw = construct_multivariatenormaldiag([D], iter, 'qw') inference = ed.KLqp({w: qw}, data={X: Xtrain, y: ytrain}) tf.global_variables_initializer().run() inference.run(n_iter=FLAGS.LMO_iter) x_post = ed.copy(y, {w: qw}) x_post_t = ed.copy(y_test, {w: qw}) print( 'log-likelihood train ', ed.evaluate('log_likelihood', data={ x_post: ytrain, X: Xtrain })) print( 'log-likelihood test ', ed.evaluate('log_likelihood', data={ x_post_t: ytest, X_test: Xtest })) print( 'binary_accuracy train ', ed.evaluate('binary_accuracy', data={ x_post: ytrain, X: Xtrain })) print( 'binary_accuracy test ', ed.evaluate('binary_accuracy', data={ x_post_t: ytest, X_test: Xtest }))
from __future__ import absolute_import from __future__ import division from __future__ import print_function import edward as ed import numpy as np import tensorflow as tf from edward.models import Bernoulli from scipy import stats ed.set_seed(98765) def _test(shape, n): rv = Bernoulli(shape, p=tf.zeros(shape) + 0.5) rv_sample = rv.sample(n) x = rv_sample.eval() x_tf = tf.constant(x, dtype=tf.float32) p = rv.p.eval() for idx in range(shape[0]): assert np.allclose( rv.log_prob_idx((idx, ), x_tf).eval(), stats.bernoulli.logpmf(x[:, idx], p[idx])) class test_bernoulli_log_prob_idx_class(tf.test.TestCase): def test_1d(self): with self.test_session(): _test((1, ), 1) _test((1, ), 5)
def getting_started_example(): # Simulate a toy dataset of 50 observations with a cosine relationship. ed.set_seed(42) N = 50 # Number of data points. D = 1 # Number of features. x_train, y_train = build_toy_dataset(N) #-------------------- # Define a two-layer Bayesian neural network. W_0 = Normal(loc=tf.zeros([D, 2]), scale=tf.ones([D, 2])) W_1 = Normal(loc=tf.zeros([2, 1]), scale=tf.ones([2, 1])) b_0 = Normal(loc=tf.zeros(2), scale=tf.ones(2)) b_1 = Normal(loc=tf.zeros(1), scale=tf.ones(1)) x = x_train y = Normal(loc=neural_network(x, W_0, W_1, b_0, b_1), scale=0.1 * tf.ones(N)) #-------------------- # Make inferences about the model from data. # We will use variational inference. # Specify a normal approximation over the weights and biases. # Defining tf.get_variable allows the variational factors' parameters to vary. They are initialized randomly. # The standard deviation parameters are constrained to be greater than zero according to a softplus transformation. qW_0 = Normal(loc=tf.get_variable('qW_0/loc', [D, 2]), scale=tf.nn.softplus(tf.get_variable('qW_0/scale', [D, 2]))) qW_1 = Normal(loc=tf.get_variable('qW_1/loc', [2, 1]), scale=tf.nn.softplus(tf.get_variable('qW_1/scale', [2, 1]))) qb_0 = Normal(loc=tf.get_variable('qb_0/loc', [2]), scale=tf.nn.softplus(tf.get_variable('qb_0/scale', [2]))) qb_1 = Normal(loc=tf.get_variable('qb_1/loc', [1]), scale=tf.nn.softplus(tf.get_variable('qb_1/scale', [1]))) # Sample functions from variational model to visualize fits. rs = np.random.RandomState(0) inputs = np.linspace(-5, 5, num=400, dtype=np.float32) x = tf.expand_dims(inputs, 1) mus = tf.stack([neural_network(x, qW_0.sample(), qW_1.sample(), qb_0.sample(), qb_1.sample()) for _ in range(10)]) # First Visualization (prior). sess = ed.get_session() tf.global_variables_initializer().run() outputs = mus.eval() fig = plt.figure(figsize=(10, 6)) ax = fig.add_subplot(111) ax.set_title('Iteration: 0') ax.plot(x_train, y_train, 'ks', alpha=0.5, label='(x, y)') ax.plot(inputs, outputs[0].T, 'r', lw=2, alpha=0.5, label='prior draws') ax.plot(inputs, outputs[1:].T, 'r', lw=2, alpha=0.5) ax.set_xlim([-5, 5]) ax.set_ylim([-2, 2]) ax.legend() plt.show() #-------------------- # Run variational inference with the Kullback-Leibler divergence in order to infer the model's latent variables with the given data. # We specify 1000 iterations. inference = ed.KLqp({W_0: qW_0, b_0: qb_0, W_1: qW_1, b_1: qb_1}, data={y: y_train}) inference.run(n_iter=1000, n_samples=5) #-------------------- # Criticize the model fit. # Bayesian neural networks define a distribution over neural networks, so we can perform a graphical check. # Draw neural networks from the inferred model and visualize how well it fits the data. # SECOND VISUALIZATION (posterior) outputs = mus.eval() fig = plt.figure(figsize=(10, 6)) ax = fig.add_subplot(111) ax.set_title('Iteration: 1000') ax.plot(x_train, y_train, 'ks', alpha=0.5, label='(x, y)') ax.plot(inputs, outputs[0].T, 'r', lw=2, alpha=0.5, label='posterior draws') ax.plot(inputs, outputs[1:].T, 'r', lw=2, alpha=0.5) ax.set_xlim([-5, 5]) ax.set_ylim([-2, 2]) ax.legend() plt.show()
axis=0) evaluator_train = Evaluator(yalltr, talltr, y_cf=np.concatenate([y_cftr, y_cfva], axis=0), mu0=np.concatenate([mu0tr, mu0va], axis=0), mu1=np.concatenate([mu1tr, mu1va], axis=0)) # zero mean, unit variance for y during training ym, ys = np.mean(ytr), np.std(ytr) ytr, yva = (ytr - ym) / ys, (yva - ym) / ys best_logpvalid = -np.inf with tf.Graph().as_default(): sess = tf.InteractiveSession() ed.set_seed(1) np.random.seed(1) tf.set_random_seed(1) x_ph_bin = tf.placeholder(tf.float32, [M, len(binfeats)], name='x_bin') # binary inputs x_ph_cont = tf.placeholder(tf.float32, [M, len(contfeats)], name='x_cont') # continuous inputs t_ph = tf.placeholder(tf.float32, [M, 1]) y_ph = tf.placeholder(tf.float32, [M, 1]) x_ph = tf.concat([x_ph_bin, x_ph_cont], 1) activation = tf.nn.elu # CEVAE model (decoder) # p(z)
training_data_list = [] test_data_list = [] for task in range(0, 10, 2): train = TrainingSet(one_hot=False) test = TestSet(one_hot=False) train.take_subset([task, task + 1]) test.take_subset([task, task + 1]) train.labels = train.labels - task test.labels = test.labels - task training_data_list.append(train) test_data_list.append(test) ##### SETTING UP THE NEURAL NETWORK ###### ed.set_seed(314159) N = 100 # number of images in a minibatch. D = 784 # number of features. K = 10 # number of classes. n_heads = 5 head_size = int(K / n_heads) # define the feedforward neural network function def neural_network(x, W_0, W_1, b_0, b_1): h = tf.tanh(tf.matmul(x, W_0) + b_0) h = tf.matmul(h, W_1) + b_1 return h
"""Return a prediction for each data point, averaging over each set of latent variables z in zs.""" x_test = xs['x'] b = zs[:, 0] W = tf.transpose(zs[:, 1:]) y_pred = tf.reduce_mean(tf.matmul(x_test, W) + b, 1) return y_pred def build_toy_dataset(N=40, coeff=np.random.randn(10), noise_std=0.1): n_dim = len(coeff) x = np.random.randn(N, n_dim).astype(np.float32) y = np.dot(x, coeff) + norm.rvs(0, noise_std, size=N) return {'x': x, 'y': y} ed.set_seed(42) model = LinearModel() variational = Variational() variational.add(Normal(model.n_vars)) coeff = np.random.randn(10) data = build_toy_dataset(coeff=coeff) inference = ed.MFVI(model, variational, data) inference.run(n_iter=250, n_samples=5, n_print=10) data_test = build_toy_dataset(coeff=coeff) x_test, y_test = data_test['x'], data_test['y'] print(ed.evaluate('mse', model, variational, {'x': x_test}, y_test))
from scipy.stats import poisson, norm import bottleneck as bn import argparse from utils import binarize_rating, exp_to_imp, binarize_spmat, \ next_batch, create_argparser, set_params, load_prefit_pfcau, \ create_metric_holders, wg_eval_acc_metrics_update_i, \ sg_eval_acc_metrics_update_i, save_eval_metrics randseed = int(time.time()) print("random seed: ", randseed) random.seed(randseed) npr.seed(randseed) ed.set_seed(randseed) tf.set_random_seed(randseed) if __name__ == '__main__': parser = create_argparser() args = parser.parse_args() all_params = set_params(args) DATA_DIR, CAUSEFIT_DIR, OUT_DATA_DIR, \ outdim, caudim, thold, M, n_iter, binary, \ pri_U, pri_V, alpha = all_params print("setting params....") print("data/cause/out directories", DATA_DIR, CAUSEFIT_DIR, OUT_DATA_DIR) print("relevance thold", thold)
def main(_): def neural_network(X): h = tf.tanh(tf.matmul(X, W_0) + b_0) h = tf.matmul(h, W_1) + b_1 return tf.reshape(h, [-1]) ed.set_seed(42) # MODEL with tf.name_scope("model"): W_0 = Normal(loc=tf.zeros([1, 32]), scale=tf.ones([1, 32]), name="W_0") W_1 = Normal(loc=tf.zeros([32, 1]), scale=tf.ones([32, 1]), name="W_1") b_0 = Normal(loc=tf.zeros(32), scale=tf.ones(32), name="b_0") b_1 = Normal(loc=tf.zeros(1), scale=tf.ones(1), name="b_1") X = tf.placeholder(tf.float32, [80, 1], name="X") y = Normal(loc=neural_network(X), scale=0.1 * tf.ones(80), name="y") # INFERENCE with tf.variable_scope("posterior"): with tf.variable_scope("qW_0"): loc = tf.get_variable("loc", [1, 32]) scale = tf.nn.softplus(tf.get_variable("scale", [1, 32])) qW_0 = Normal(loc=loc, scale=scale) with tf.variable_scope("qW_1"): loc = tf.get_variable("loc", [32, 1]) scale = tf.nn.softplus(tf.get_variable("scale", [32, 1])) qW_1 = Normal(loc=loc, scale=scale) with tf.variable_scope("qb_0"): loc = tf.get_variable("loc", [32]) scale = tf.nn.softplus(tf.get_variable("scale", [32])) qb_0 = Normal(loc=loc, scale=scale) with tf.variable_scope("qb_1"): loc = tf.get_variable("loc", [1]) scale = tf.nn.softplus(tf.get_variable("scale", [1])) qb_1 = Normal(loc=loc, scale=scale) inference = ed.KLqp({ W_0: qW_0, b_0: qb_0, W_1: qW_1, b_1: qb_1 }, data={ X: X_train, y: y_train }) inference.run(logdir='log', n_samples=5, n_iter=10000) y_post = ed.copy(y, ({W_0: qW_0, b_0: qb_0, W_1: qW_1, b_1: qb_1})) print("Mean squared error:") print(ed.evaluate('mean_squared_error', data={ X: X_train, y_post: y_train })) print("Mean absolute error:") print( ed.evaluate('mean_absolute_error', data={ X: X_train, y_post: y_train }))
from scipy.special import expit as sigmoid from scipy.misc import logsumexp from sklearn.metrics import roc_auc_score from boosting_bbvi.core.mvn import mvn # TODO add option to switch between mvn & lpl import boosting_bbvi.core.utils import blr_utils FLAGS = tf.flags.FLAGS tf.flags.DEFINE_integer("LMO_iter", 600, '') tf.flags.DEFINE_integer('n_fw_iter', 100, '') tf.flags.DEFINE_string("outdir", '/tmp', '') tf.flags.DEFINE_string("fw_variant", 'fixed', '') tf.flags.DEFINE_integer('seed', 0, 'The random seed to use for everything.') ed.set_seed(FLAGS.seed) np.random.seed(FLAGS.seed) def construct_multivariatenormaldiag(dims, iter, name=''): loc = tf.get_variable(name + "_loc%d" % iter, initializer=tf.random_normal(dims) + np.random.normal()) scale = tf.nn.softplus(tf.get_variable(name + "_scale%d" % iter, dims)) rez = mvn(loc=loc, scale=scale) return rez def main(_): ed.set_seed(FLAGS.seed)
def main(): parser = argparse.ArgumentParser( description= 'Trains a conditional density estimation model on the MNIST dataset.') parser.add_argument( '--inputdir', default='experiments/mnist/data', help='The directory where the input data files will be stored.') parser.add_argument( '--dist_type', choices=['gmm', 'discontinuous_gmm', 'edge_biased'], default='gmm', help= 'The type of underlying distribution that the labels are drawn from.') parser.add_argument( '--train_id', default='0', help= 'A trial ID. All models trained with the same trial ID will use the same train/validation datasets.' ) parser.add_argument('--train_samples', type=int, default=60000, help='The number of training examples to use.') parser.add_argument( '--validation_samples', type=float, default=0.2, help= 'The number of samples to hold out for a validation set. This is a percentage of the training samples.' ) parser.add_argument('--batchsize', type=int, default=50, help='The number of training samples per mini-batch.') parser.add_argument('--max_steps', type=int, default=100000, help='The maximum number of training steps.') parser.add_argument('--num_components', type=int, default=3, help='The number of components for the GMM method.') parser.add_argument( '--nbins', type=int, default=128, help='The number of bins in the discrete distribution.') args = parser.parse_args() dargs = vars(args) dargs['model'] = 'gmm_{num_components}'.format(**dargs) dargs[ 'outfile'] = 'experiments/mnist/results/{model}_{dist_type}_{train_samples}_{train_id}'.format( **dargs) dargs[ 'variable_scope'] = 'mnist-{model}-{dist_type}-{train_samples}-{train_id}'.format( **dargs) mnist = read_data_sets(args.inputdir, **dargs) X_validate, y_validate = mnist.validation.features, np.argmax( mnist.validation.labels, axis=-1)[:, np.newaxis] / args.nbins X_test, y_test = mnist.test.features, np.argmax( mnist.test.labels, axis=-1)[:, np.newaxis] / args.nbins ed.set_seed(42) X = tf.placeholder(tf.float32, [None, IMAGE_ROWS * IMAGE_COLS], name='X') y = tf.placeholder(tf.float32, [None, 1], name='y') data = {'X': X, 'y': y} # Create the model model = MixtureDensityNetwork(args.num_components) inference = ed.MAP([], data, model) optimizer = tf.train.AdamOptimizer(0.0001, epsilon=10.0) sess = ed.get_session() # Start TF session K.set_session(sess) # Pass session info to Keras inference.initialize(optimizer=optimizer) init = tf.global_variables_initializer() init.run() saver = tf.train.Saver() # Other models are run for 100K SGD iterations with minibatch size 50 n_epoch = args.max_steps train_loss = np.zeros(n_epoch) test_loss = np.zeros(n_epoch) best_loss = None for i in range(n_epoch): X_train, y_train = mnist.train.next_batch(args.batchsize) y_train = np.argmax(y_train, axis=-1)[:, np.newaxis] / args.nbins info_dict = inference.update(feed_dict={X: X_train, y: y_train}) if i % 100 == 0: train_loss[i] = info_dict['loss'] test_loss[i] = validate(sess, inference, X, y, X_validate, y_validate) print(i, train_loss[i], test_loss[i]) if i == 0 or test_loss[i] < best_loss: best_loss = test_loss[i] saver.save(sess, dargs['outfile']) saver.restore(sess, dargs['outfile']) print('Finished training. Scoring model...') sys.stdout.flush() # Get the resulting GMM outputs on the test set fits = fit_to_test(sess, model, X, X_test, args.nbins) save_scores_and_fits('{outfile}_score.csv'.format(**dargs), '{outfile}_fits.csv'.format(**dargs), mnist, best_loss, fits) print('Best model saved to {outfile}'.format(**dargs)) sys.stdout.flush()
def main(_): ed.set_seed(42) # DATA X_train, y_train = build_toy_dataset(FLAGS.N) X_test, y_test = build_toy_dataset(FLAGS.N) # MODEL X = tf.placeholder(tf.float32, [FLAGS.N, FLAGS.D]) w = Normal(loc=tf.zeros(FLAGS.D), scale=tf.ones(FLAGS.D)) b = Normal(loc=tf.zeros(1), scale=tf.ones(1)) y = Normal(loc=ed.dot(X, w) + b, scale=tf.ones(FLAGS.N)) # INFERENCE qw = Empirical(params=tf.get_variable("qw/params", [FLAGS.T, FLAGS.D])) qb = Empirical(params=tf.get_variable("qb/params", [FLAGS.T, 1])) inference = ed.SGHMC({w: qw, b: qb}, data={X: X_train, y: y_train}) inference.run(step_size=1e-3) # CRITICISM # Plot posterior samples. sns.jointplot(qb.params.eval()[FLAGS.nburn:FLAGS.T:FLAGS.stride], qw.params.eval()[FLAGS.nburn:FLAGS.T:FLAGS.stride]) plt.show() # Posterior predictive checks. y_post = ed.copy(y, {w: qw, b: qb}) # This is equivalent to # y_post = Normal(loc=ed.dot(X, qw) + qb, scale=tf.ones(FLAGS.N)) print("Mean squared error on test data:") print(ed.evaluate('mean_squared_error', data={X: X_test, y_post: y_test})) print("Displaying prior predictive samples.") n_prior_samples = 10 w_prior = w.sample(n_prior_samples).eval() b_prior = b.sample(n_prior_samples).eval() plt.scatter(X_train, y_train) inputs = np.linspace(-1, 10, num=400) for ns in range(n_prior_samples): output = inputs * w_prior[ns] + b_prior[ns] plt.plot(inputs, output) plt.show() print("Displaying posterior predictive samples.") n_posterior_samples = 10 w_post = qw.sample(n_posterior_samples).eval() b_post = qb.sample(n_posterior_samples).eval() plt.scatter(X_train, y_train) inputs = np.linspace(-1, 10, num=400) for ns in range(n_posterior_samples): output = inputs * w_post[ns] + b_post[ns] plt.plot(inputs, output) plt.show()
inference.run() true_posterior = Beta(a=3.0, b=9.0) val_est, val_true = sess.run([qp.mean(), true_posterior.mean()]) self.assertAllClose(val_est, val_true, rtol=1e-2, atol=1e-2) val_est, val_true = sess.run([qp.variance(), true_posterior.variance()]) self.assertAllClose(val_est, val_true, rtol=1e-2, atol=1e-2) def test_normal_normal(self): with self.test_session() as sess: x_data = np.array([0.0] * 50, dtype=np.float32) mu = Normal(mu=0.0, sigma=1.0) x = Normal(mu=mu, sigma=1.0, sample_shape=50) qmu = Empirical(params=tf.Variable(tf.ones(1000))) # analytic solution: N(mu=0.0, sigma=\sqrt{1/51}=0.140) inference = ed.Gibbs({mu: qmu}, data={x: x_data}) inference.run() self.assertAllClose(qmu.mean().eval(), 0, rtol=1e-2, atol=1e-2) self.assertAllClose(qmu.std().eval(), np.sqrt(1 / 51), rtol=1e-2, atol=1e-2) if __name__ == '__main__': ed.set_seed(127832) tf.test.main()
mean = np.dot(w, z) for d in range(D): for n in range(N): x_train[d, n] = np.random.normal(mean[d, n], sigma) print("True principal axes:") print(w) return x_train def next_batch(M): idx_batch = np.random.choice(N, M) return x_train[:, idx_batch], idx_batch ed.set_seed(142) N = 5000 # number of data points M = 100 # minibatch size D = 2 # data dimensionality K = 1 # latent dimensionality # DATA x_train = build_toy_dataset(N, D, K) # MODEL w = Normal(mu=tf.zeros([D, K]), sigma=10.0 * tf.ones([D, K])) z = Normal(mu=tf.zeros([M, K]), sigma=tf.ones([M, K])) x = Normal(mu=tf.matmul(w, z, transpose_b=True), sigma=tf.ones([D, M]))
import tensorflow as tf from edward.models import Normal class test_implicit_klqp_class(tf.test.TestCase): def test_normal_run(self): def ratio_estimator(data, local_vars, global_vars): """Use the optimal ratio estimator, r(z) = log p(z). We add a TensorFlow variable as the algorithm assumes that the function has parameters to optimize.""" w = tf.get_variable("w", []) return z.log_prob(local_vars[z]) + w with self.test_session() as sess: z = Normal(mu=5.0, sigma=1.0) qz = Normal(mu=tf.Variable(tf.random_normal([])), sigma=tf.nn.softplus(tf.Variable(tf.random_normal( [])))) inference = ed.ImplicitKLqp({z: qz}, discriminator=ratio_estimator) inference.run(n_iter=200) self.assertAllClose(qz.mean().eval(), 5.0, atol=1.0) if __name__ == '__main__': ed.set_seed(47324) tf.test.main()
return log_lik + log_prior def build_toy_dataset(n_data=40, noise_std=0.1): ed.set_seed(0) D = 1 x = np.concatenate([np.linspace(0, 2, num=n_data/2), np.linspace(6, 8, num=n_data/2)]) y = np.cos(x) + norm.rvs(0, noise_std, size=n_data) x = (x - 4.0) / 4.0 x = x.reshape((n_data, D)) y = y.reshape((n_data, 1)) data = np.concatenate((y, x), axis=1) # n_data x (D+1) data = tf.constant(data, dtype=tf.float32) return ed.Data(data) ed.set_seed(43) # TODO This converges to the zero line. I think this is an # initialization issue. model = BayesianNN(layer_sizes=[1, 10, 10, 1], nonlinearity=rbf) variational = Variational() variational.add(Normal(model.num_vars)) data = build_toy_dataset() # Set up figure fig = plt.figure(figsize=(8,8), facecolor='white') ax = fig.add_subplot(111, frameon=False) plt.ion() plt.show(block=False) def print_progress(self, t, losses, sess): if t % self.n_print == 0: