def test_categorical_likelihood(make_data): """Test aboleth with a tf.distributions.Categorical likelihood. Since it is a bit of an odd half-multivariate case. """ x, y, _, = make_data N, K = x.shape # Make two classes (K = 2) Y = np.zeros(len(y), dtype=np.int32) Y[y[:, 0] > 0] = 1 layers = ab.stack( ab.InputLayer(name='X', n_samples=10), lambda X: (X, 0.0) # Mock a sampling layer, with 2-class output ) nn, reg = layers(X=x.astype(np.float32)) like = tf.distributions.Categorical(logits=nn) ELBO = ab.elbo(like, Y, N, reg) MAP = ab.max_posterior(like, Y, reg) tc = tf.test.TestCase() with tc.test_session(): tf.global_variables_initializer().run() assert like.probs.eval().shape == (10, N, K) assert like.prob(Y).eval().shape == (10, N) L = ELBO.eval() assert np.isscalar(L) L = MAP.eval() assert np.isscalar(L)
def nnet_dropout(X, Y): """Neural net with dropout.""" reg = 0.001 # Weight prior noise = .5 # Likelihood st. dev. net = ( ab.InputLayer(name="X", n_samples=n_samples) >> ab.DenseMAP(output_dim=30, l2_reg=reg, l1_reg=0.) >> ab.Activation(tf.tanh) >> ab.DropOut(keep_prob=0.95) >> ab.DenseMAP(output_dim=20, l2_reg=reg, l1_reg=0.) >> ab.Activation(tf.tanh) >> ab.DropOut(keep_prob=0.95) >> ab.DenseMAP(output_dim=10, l2_reg=reg, l1_reg=0.) >> ab.Activation(tf.tanh) >> ab.DropOut(keep_prob=0.95) >> ab.DenseMAP(output_dim=5, l2_reg=reg, l1_reg=0.) >> ab.Activation(tf.tanh) >> ab.DenseMAP(output_dim=1, l2_reg=reg, l1_reg=0.) ) phi, reg = net(X=X) lkhood = tf.distributions.Normal(loc=phi, scale=noise) loss = ab.max_posterior(lkhood, Y, reg) return phi, loss
def main(): # Dataset mnist_data = tf.contrib.learn.datasets.mnist.read_data_sets('./mnist_demo', reshape=False) N = mnist_data.train.images.shape[0] X, Y = tf.data.Dataset.from_tensor_slices( (np.asarray(mnist_data.train.images, dtype=np.float32), np.asarray(mnist_data.train.labels, dtype=np.int64)) ).repeat(n_epochs).shuffle(N).batch(batch_size) \ .make_one_shot_iterator().get_next() # Xs, Ys = tf.data.Dataset.from_tensor_slices( # (np.asarray(mnist_data.test.images, dtype=np.float32), # np.asarray(mnist_data.test.labels, dtype=np.int64)) # ).repeat(n_epochs).make_one_shot_iterator().get_next() Xs = np.asarray(mnist_data.test.images, dtype=np.float32) Ys = np.asarray(mnist_data.test.labels, dtype=np.int64) # Model specification with tf.name_scope("model"): logits, reg = net(X=X) llh = tf.distributions.Categorical(logits=logits) loss = ab.max_posterior(llh.log_prob(Y), reg) probs = ab.sample_mean(llh.probs) accuracy = tf.reduce_mean( tf.cast(tf.equal(tf.argmax(probs, axis=1), Y), dtype=tf.float32)) # Training graph building with tf.name_scope("train"): optimizer = tf.train.AdamOptimizer(learning_rate=1e-3) global_step = tf.train.get_or_create_global_step() train = optimizer.minimize(loss, global_step=global_step) logger = tf.train.LoggingTensorHook(dict(step=global_step, loss=loss, accuracy=accuracy), every_n_secs=5) with tf.train.MonitoredTrainingSession(config=config, hooks=[logger]) as sess: while not sess.should_stop(): step, _ = sess.run([global_step, train]) # this part is a bit superfluous. Could get it working with # Datasets too but just wanted to sanity test for now. if not step % 100: val_acc = accuracy.eval(feed_dict={X: Xs, Y: Ys}, session=sess) tf.logging.info("step = %d, validation accuracy: %f", step, val_acc)
def test_map_likelihood(make_graph): """Test for expected output dimensions from deepnet.""" x, y, _, _, _, _, layers = make_graph nn, reg = layers(X=x.astype(np.float32)) like = tf.distributions.Normal(nn, scale=1.) loss = ab.max_posterior(like, y.astype(np.float32), reg) tc = tf.test.TestCase() with tc.test_session(): tf.global_variables_initializer().run() L = loss.eval() assert np.isscalar(L)
def linear(X, Y): """Linear regression with l2 regularization.""" lambda_ = 1e-4 # Weight regularizer noise = 1. # Likelihood st. dev. net = (ab.InputLayer(name="X") >> ab.DenseMAP( output_dim=1, l2_reg=lambda_, l1_reg=0.)) Xw, reg = net(X=X) lkhood = tf.distributions.Normal(loc=Xw, scale=noise) loss = ab.max_posterior(lkhood, Y, reg) # loss = 0.5 * tf.reduce_mean((Y - Xw)**2) + reg return Xw, loss
def test_map_likelihood(make_graph): """Test for expected output dimensions from deepnet.""" x, y, N, X_, Y_, N_, layers = make_graph nn, reg = layers(X=X_) log_like = tf.distributions.Normal(nn, scale=1.).log_prob(Y_) loss = ab.max_posterior(log_like, reg) tc = tf.test.TestCase() with tc.test_session(): tf.global_variables_initializer().run() L = loss.eval(feed_dict={X_: x, Y_: y}) assert np.isscalar(L)
def linear(X, Y): """Linear regression with l2 regularization.""" reg = .01 # Weight prior noise = .5 # Likelihood st. dev. net = ( ab.InputLayer(name="X", n_samples=1) >> ab.DenseMAP(output_dim=1, l2_reg=reg, l1_reg=0.) ) phi, reg = net(X=X) lkhood = tf.distributions.Normal(loc=phi, scale=noise) loss = ab.max_posterior(lkhood, Y, reg) return phi, loss
def nnet(X, Y): """Neural net with regularization.""" lambda_ = 1e-4 # Weight regularizer noise = .5 # Likelihood st. dev. net = ( ab.InputLayer(name="X", n_samples=1) >> ab.DenseMAP( output_dim=40, l2_reg=lambda_, l1_reg=0.) >> ab.Activation(tf.tanh) >> ab.DenseMAP(output_dim=20, l2_reg=lambda_, l1_reg=0.) >> ab.Activation(tf.tanh) >> ab.DenseMAP(output_dim=10, l2_reg=lambda_, l1_reg=0.) >> ab.Activation( tf.tanh) >> ab.DenseMAP(output_dim=1, l2_reg=lambda_, l1_reg=0.)) f, reg = net(X=X) lkhood = tf.distributions.Normal(loc=f, scale=noise) loss = ab.max_posterior(lkhood, Y, reg) return f, loss
def test_categorical_likelihood(make_data, likelihood): """Test aboleth with discrete likelihoods. Since these are kind of corner cases... """ x, y, _, = make_data like, K = likelihood N, _ = x.shape # Make two classes (K = 2) Y = np.zeros(len(y), dtype=np.int32) Y[y[:, 0] > 0] = 1 if K == 1: Y = Y[:, np.newaxis] X_ = tf.placeholder(tf.float32, x.shape) Y_ = tf.placeholder(tf.int32, Y.shape) n_samples_ = tf.placeholder(tf.int32) layers = ab.stack( ab.InputLayer(name='X', n_samples=n_samples_), ab.Dense(output_dim=K) ) nn, reg = layers(X=X_) like = like(logits=nn) log_like = like.log_prob(Y_) prob = like.prob(Y_) ELBO = ab.elbo(log_like, reg, N) MAP = ab.max_posterior(log_like, reg) fd = {X_: x, Y_: Y, n_samples_: 10} tc = tf.test.TestCase() with tc.test_session(): tf.global_variables_initializer().run() assert like.probs.eval(feed_dict=fd).shape == (10, N, K) assert prob.eval(feed_dict=fd).shape == (10,) + Y.shape L = ELBO.eval(feed_dict=fd) L = MAP.eval(feed_dict=fd) assert np.isscalar(L)
def nnet_dropout(X, Y): """Neural net with dropout.""" lambda_ = 1e-3 # Weight prior noise = .5 # Likelihood st. dev. net = ( ab.InputLayer(name="X", n_samples=n_samples_) >> ab.Dense(output_dim=32, l2_reg=lambda_) >> ab.Activation(tf.nn.selu) >> ab.DropOut(keep_prob=0.9, independent=True) >> ab.Dense(output_dim=16, l2_reg=lambda_) >> ab.Activation(tf.nn.selu) >> ab.DropOut(keep_prob=0.95, independent=True) >> ab.Dense(output_dim=8, l2_reg=lambda_) >> ab.Activation(tf.nn.selu) >> ab.Dense(output_dim=1, l2_reg=lambda_) ) f, reg = net(X=X) lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y) loss = ab.max_posterior(lkhood, reg) return f, loss
def main(): """Run the demo.""" n_iters = int(round(n_epochs * N / batch_size)) print("Iterations = {}".format(n_iters)) # Get training and testing data Xr, Yr, Xs, Ys = gp_draws(N, Ns, kern=kernel, noise=true_noise) # Prediction points Xq = np.linspace(-20, 20, Ns).astype(np.float32)[:, np.newaxis] Yq = np.linspace(-4, 4, Ns).astype(np.float32)[:, np.newaxis] # Set up the probability image query points Xi, Yi = np.meshgrid(Xq, Yq) Xi = Xi.astype(np.float32).reshape(-1, 1) Yi = Yi.astype(np.float32).reshape(-1, 1) _, D = Xr.shape # Name the "data" parts of the graph with tf.name_scope("Input"): # This function will make a TensorFlow queue for shuffling and batching # the data, and will run through n_epochs of the data. Xb, Yb = batch_training(Xr, Yr, n_epochs=n_epochs, batch_size=batch_size) X_ = tf.placeholder_with_default(Xb, shape=(None, D)) Y_ = tf.placeholder_with_default(Yb, shape=(None, 1)) # This is where we build the actual GP model with tf.name_scope("Deepnet"): phi, reg = net(X=X_) noise = ab.pos_variable(NOISE) ll = tf.distributions.Normal(loc=phi, scale=noise).log_prob(Y_) loss = ab.max_posterior(ll, reg) # Set up the trainig graph with tf.name_scope("Train"): optimizer = tf.train.AdamOptimizer() global_step = tf.train.create_global_step() train = optimizer.minimize(loss, global_step=global_step) # This is used for building the predictive density image with tf.name_scope("Predict"): logprob = tf.reduce_mean(ll, axis=0) # Logging learning progress log = tf.train.LoggingTensorHook({ 'step': global_step, 'loss': loss }, every_n_iter=1000) # This is the main training "loop" with tf.train.MonitoredTrainingSession(config=config, save_summaries_steps=None, save_checkpoint_secs=None, hooks=[log]) as sess: try: while not sess.should_stop(): sess.run( train, feed_dict={ n_samples_: n_samples, # tf.keras.backend.learning_phase(): 1 }) except tf.errors.OutOfRangeError: print('Input queues have been exhausted!') pass # Prediction, the [[None]] is to stop the default placeholder queue # we keep learning phase flag on even in prediction phase to draw # samples from predictive distribution Ey = sess.run(phi, feed_dict={ X_: Xq, Y_: [[None]], n_samples_: p_samples }) logPY = sess.run(logprob, feed_dict={ Y_: Yi, X_: Xi, n_samples_: p_samples }) Eymean = Ey.mean(axis=0) # Average samples to get mean predicted funtion Py = np.exp(logPY.reshape(Ns, Ns)) # Turn log-prob into prob # Plot im_min = np.amin(Py) im_size = np.amax(Py) - im_min img = (Py - im_min) / im_size f = bk.figure(tools='pan,box_zoom,reset', sizing_mode='stretch_both') f.image(image=[img], x=-20., y=-4., dw=40., dh=8, palette=bp.Plasma256) f.circle(Xr.flatten(), Yr.flatten(), fill_color='blue', legend='Training') f.line(Xs.flatten(), Ys.flatten(), line_color='blue', legend='Truth') for y in Ey: f.line(Xq.flatten(), y.flatten(), line_color='red', legend='Samples', alpha=0.2) f.line(Xq.flatten(), Eymean.flatten(), line_color='green', legend='Mean') bk.show(f)
def main(): """Run the demo.""" data = load_breast_cancer() X = data.data.astype(np.float32) y = data.target.astype(np.int32)[:, np.newaxis] X = StandardScaler().fit_transform(X).astype(np.float32) N, D = X.shape # Benchmark classifier bcl = RandomForestClassifier(random_state=RSEED) # Data with tf.name_scope("Input"): X_ = tf.placeholder(dtype=tf.float32, shape=(None, D)) Y_ = tf.placeholder(dtype=tf.float32, shape=(None, 1)) with tf.name_scope("Deepnet"): nn, reg = net(X=X_) lkhood = tf.distributions.Bernoulli(logits=nn) loss = ab.max_posterior(lkhood.log_prob(Y_), reg) prob = ab.sample_mean(lkhood.probs) with tf.name_scope("Train"): optimizer = tf.train.AdamOptimizer(learning_rate=0.001) train = optimizer.minimize(loss) kfold = KFold(n_splits=FOLDS, shuffle=True, random_state=RSEED) # Launch the graph. acc, acc_o, ll, ll_o = [], [], [], [] init = tf.global_variables_initializer() with tf.Session(config=CONFIG): for k, (r_ind, s_ind) in enumerate(kfold.split(X)): init.run() Xr, Yr = X[r_ind], y[r_ind] Xs, Ys = X[s_ind], y[s_ind] batches = ab.batch( {X_: Xr, Y_: Yr}, batch_size=BSIZE, n_iter=NITER) for i, data in enumerate(batches): train.run(feed_dict=data) if i % 1000 == 0: loss_val = loss.eval(feed_dict=data) print("Iteration {}, loss = {}".format(i, loss_val)) # Predict, NOTE: we use the mean of the likelihood to get the # probabilies ps = prob.eval(feed_dict={X_: Xs, n_samples_: PSAMPLES}) print("Fold {}:".format(k)) Ep = np.hstack((1. - ps, ps)) print_k_result(Ys, Ep, ll, acc, "BNN") bcl.fit(Xr, Yr.flatten()) Ep_o = bcl.predict_proba(Xs) print_k_result(Ys, Ep_o, ll_o, acc_o, "RF") print("-----") print_final_result(acc, ll, "BNN") print_final_result(acc_o, ll_o, "RF")