def my_model(features, labels, mode, params): N = params["N"] n_samples = NSAMPLES if mode == tf.estimator.ModeKeys.TRAIN \ else NPREDICTSAMPLES X = tf.feature_column.input_layer(features, params['feature_columns']) kernel = ab.RBF(LENSCALE, learn_lenscale=True) net = ( ab.InputLayer(name="X", n_samples=n_samples) >> ab.RandomFourier(n_features=NFEATURES, kernel=kernel) >> ab.Dense(output_dim=64, init_fn="autonorm") >> ab.Activation(tf.nn.selu) >> ab.DenseVariational(output_dim=1, full=False, prior_std=1.0, learn_prior=True) ) phi, kl = net(X=X) std = ab.pos_variable(NOISE, name="noise") ll_f = tf.distributions.Normal(loc=phi, scale=std) predict_mean = ab.sample_mean(phi) # Compute predictions. if mode == tf.estimator.ModeKeys.PREDICT: predictions = { 'predictions': predict_mean, 'samples': phi } return tf.estimator.EstimatorSpec(mode, predictions=predictions) ll = ll_f.log_prob(labels) loss = ab.elbo(ll, kl, N) tf.summary.scalar('loss', loss) # Compute evaluation metrics. mse = tf.metrics.mean_squared_error(labels=labels, predictions=predict_mean, name='mse_op') r2 = r2_metric(labels, predict_mean) metrics = {'mse': mse, 'r2': r2} if mode == tf.estimator.ModeKeys.EVAL: return tf.estimator.EstimatorSpec( mode, loss=loss, eval_metric_ops=metrics) # Create training op. assert mode == tf.estimator.ModeKeys.TRAIN optimizer = tf.train.AdamOptimizer() train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step()) return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
def bayesian_linear(X, Y): """Bayesian Linear Regression.""" noise = ab.pos_variable(1.0) net = ( ab.InputLayer(name="X", n_samples=n_samples_) >> ab.DenseVariational(output_dim=1, full=True) ) f, kl = net(X=X) lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y) loss = ab.elbo(lkhood, kl, N) return f, loss
def gaussian_process(X, Y): """Gaussian Process Regression.""" noise = ab.pos_variable(.5) kern = ab.RBF(learn_lenscale=False) # learn lengthscale net = ( ab.InputLayer(name="X", n_samples=n_samples_) >> ab.RandomFourier(n_features=50, kernel=kern) >> ab.DenseVariational(output_dim=1, full=True, learn_prior=True) ) f, kl = net(X=X) lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y) loss = ab.elbo(lkhood, kl, N) return f, loss
def deep_gaussian_process(X, Y): """Deep Gaussian Process Regression.""" noise = ab.pos_variable(.1) net = ( ab.InputLayer(name="X", n_samples=n_samples_) >> ab.RandomFourier(n_features=20, kernel=ab.RBF(learn_lenscale=True)) >> ab.DenseVariational(output_dim=5, full=False) >> ab.RandomFourier(n_features=10, kernel=ab.RBF(1., seed=1)) >> ab.DenseVariational(output_dim=1, full=False, learn_prior=True) ) f, kl = net(X=X) lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y) loss = ab.elbo(lkhood, kl, N) return f, loss
def nnet_ncp(X, Y): """Noise contrastive prior network.""" noise = ab.pos_variable(.5) lstd = 1. perturb_noise = 10. net = ( ab.InputLayer(name="X", n_samples=n_samples_) >> ab.NCPContinuousPerturb(input_noise=perturb_noise) >> ab.Dense(output_dim=32) >> ab.Activation(tf.nn.selu) >> ab.Dense(output_dim=16) >> ab.Activation(tf.nn.selu) >> ab.Dense(output_dim=8) >> ab.Activation(tf.nn.selu) >> ab.DenseNCP(output_dim=1, prior_std=.1, latent_std=lstd) ) f, kl = net(X=X) lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y) loss = ab.elbo(lkhood, kl, N) return f, loss
def main(): """Run the demo.""" n_iters = int(round(n_epochs * N / batch_size)) print("Iterations = {}".format(n_iters)) # Get training and testing data Xr, Yr, Xs, Ys = gp_draws(N, Ns, kern=kernel, noise=true_noise) # Prediction points Xq = np.linspace(-20, 20, Ns).astype(np.float32)[:, np.newaxis] Yq = np.linspace(-4, 4, Ns).astype(np.float32)[:, np.newaxis] # Set up the probability image query points Xi, Yi = np.meshgrid(Xq, Yq) Xi = Xi.astype(np.float32).reshape(-1, 1) Yi = Yi.astype(np.float32).reshape(-1, 1) _, D = Xr.shape # Name the "data" parts of the graph with tf.name_scope("Input"): # This function will make a TensorFlow queue for shuffling and batching # the data, and will run through n_epochs of the data. Xb, Yb = batch_training(Xr, Yr, n_epochs=n_epochs, batch_size=batch_size) X_ = tf.placeholder_with_default(Xb, shape=(None, D)) Y_ = tf.placeholder_with_default(Yb, shape=(None, 1)) # This is where we build the actual GP model with tf.name_scope("Deepnet"): phi, reg = net(X=X_) noise = ab.pos_variable(NOISE) ll = tf.distributions.Normal(loc=phi, scale=noise).log_prob(Y_) loss = ab.max_posterior(ll, reg) # Set up the trainig graph with tf.name_scope("Train"): optimizer = tf.train.AdamOptimizer() global_step = tf.train.create_global_step() train = optimizer.minimize(loss, global_step=global_step) # This is used for building the predictive density image with tf.name_scope("Predict"): logprob = tf.reduce_mean(ll, axis=0) # Logging learning progress log = tf.train.LoggingTensorHook({ 'step': global_step, 'loss': loss }, every_n_iter=1000) # This is the main training "loop" with tf.train.MonitoredTrainingSession(config=config, save_summaries_steps=None, save_checkpoint_secs=None, hooks=[log]) as sess: try: while not sess.should_stop(): sess.run( train, feed_dict={ n_samples_: n_samples, # tf.keras.backend.learning_phase(): 1 }) except tf.errors.OutOfRangeError: print('Input queues have been exhausted!') pass # Prediction, the [[None]] is to stop the default placeholder queue # we keep learning phase flag on even in prediction phase to draw # samples from predictive distribution Ey = sess.run(phi, feed_dict={ X_: Xq, Y_: [[None]], n_samples_: p_samples }) logPY = sess.run(logprob, feed_dict={ Y_: Yi, X_: Xi, n_samples_: p_samples }) Eymean = Ey.mean(axis=0) # Average samples to get mean predicted funtion Py = np.exp(logPY.reshape(Ns, Ns)) # Turn log-prob into prob # Plot im_min = np.amin(Py) im_size = np.amax(Py) - im_min img = (Py - im_min) / im_size f = bk.figure(tools='pan,box_zoom,reset', sizing_mode='stretch_both') f.image(image=[img], x=-20., y=-4., dw=40., dh=8, palette=bp.Plasma256) f.circle(Xr.flatten(), Yr.flatten(), fill_color='blue', legend='Training') f.line(Xs.flatten(), Ys.flatten(), line_color='blue', legend='Truth') for y in Ey: f.line(Xq.flatten(), y.flatten(), line_color='red', legend='Samples', alpha=0.2) f.line(Xq.flatten(), Eymean.flatten(), line_color='green', legend='Mean') bk.show(f)
def main(): """Run the imputation demo.""" # Fetch data, one-hot targets and standardise data data = fetch_covtype() Xo = data.data[:, :10] Xc = data.data[:, 10:] Y = (data.target - 1) Xo[:, :10] = StandardScaler().fit_transform(Xo[:, :10]) # Network construction n_samples_ = tf.placeholder_with_default(LSAMPLES, []) data_input = ab.InputLayer(name='Xo', n_samples=n_samples_) # Data input # Run this with imputation if METHOD is not None: print("Imputation method {}.".format(METHOD)) # Fake some missing data rnd = np.random.RandomState(RSEED) mask = rnd.rand(*Xo.shape) < FRAC_MISSING Xo[mask] = MISSING_VAL # Use Aboleth to imputate mask_input = ab.MaskInputLayer(name='M') # Missing data mask input xm = np.ma.array(Xo, mask=mask) if METHOD == "LearnedNormalImpute": mean = tf.Variable(np.ma.mean(xm, axis=0).data.astype(np.float32)) std = ab.pos_variable(np.ma.std(xm, axis=0) .data.astype(np.float32)) input_layer = ab.NormalImpute(data_input, mask_input, mean, std) elif METHOD == "LearnedScalarImpute": scalar = tf.Variable(tf.zeros(Xo.shape[-1])) input_layer = ab.ScalarImpute(data_input, mask_input, scalar) elif METHOD == "FixedNormalImpute": mean = np.ma.mean(xm, axis=0).data.astype(np.float32) std = np.ma.std(xm, axis=0).data.astype(np.float32) input_layer = ab.NormalImpute(data_input, mask_input, mean, std) elif METHOD == "FixedScalarImpute": mean = np.ma.mean(xm, axis=0).data.astype(np.float32) input_layer = ab.ScalarImpute(data_input, mask_input, mean) elif METHOD == "MeanImpute": input_layer = ab.MeanImpute(data_input, mask_input) else: raise ValueError("Invalid method!") # Run this without imputation else: print("No missing data") input_layer = data_input mask = np.zeros_like(Xo) cat_layers = ( ab.InputLayer(name='Xc', n_samples=n_samples_) >> ab.DenseVariational(output_dim=8) ) con_layers = ( input_layer >> ab.DenseVariational(output_dim=8) ) net = ( ab.Concat(cat_layers, con_layers) >> ab.Activation(tf.nn.selu) >> ab.DenseVariational(output_dim=NCLASSES) ) # Split the training and testing data Xo_tr, Xo_ts, Xc_tr, Xc_ts, Y_tr, Y_ts, M_tr, M_ts = train_test_split( Xo.astype(np.float32), Xc.astype(np.float32), Y.astype(np.int32), mask, test_size=FRAC_TEST, random_state=RSEED ) N_tr, Do = Xo_tr.shape _, Dc = Xc_tr.shape # Data with tf.name_scope("Input"): Xob, Xcb, Yb, Mb = batch_training(Xo_tr, Xc_tr, Y_tr, M_tr, n_epochs=NEPOCHS, batch_size=BSIZE) Xo_ = tf.placeholder_with_default(Xob, shape=(None, Do)) Xc_ = tf.placeholder_with_default(Xcb, shape=(None, Dc)) # Y_ has to be this dimension for compatability with Categorical Y_ = tf.placeholder_with_default(Yb, shape=(None,)) if METHOD is not None: M_ = tf.placeholder_with_default(Mb, shape=(None, Do)) with tf.name_scope("Deepnet"): if METHOD is not None: nn, kl = net(Xo=Xo_, Xc=Xc_, M=M_) else: nn, kl = net(Xo=Xo_, Xc=Xc_) lkhood = tf.distributions.Categorical(logits=nn) loss = ab.elbo(lkhood.log_prob(Y_), kl, N_tr) prob = ab.sample_mean(lkhood.probs) with tf.name_scope("Train"): optimizer = tf.train.AdamOptimizer() global_step = tf.train.create_global_step() train = optimizer.minimize(loss, global_step=global_step) # Logging learning progress log = tf.train.LoggingTensorHook( {'step': global_step, 'loss': loss}, every_n_iter=1000 ) # This is the main training "loop" with tf.train.MonitoredTrainingSession( config=CONFIG, save_summaries_steps=None, save_checkpoint_secs=None, hooks=[log] ) as sess: try: while not sess.should_stop(): sess.run(train) except tf.errors.OutOfRangeError: print('Input queues have been exhausted!') pass # Prediction feed_dict = {Xo_: Xo_ts, Xc_: Xc_ts, Y_: [0], n_samples_: PSAMPLES} if METHOD is not None: feed_dict[M_] = M_ts p = sess.run(prob, feed_dict=feed_dict) # Get mean of samples for prediction, and max probability assignments Ey = p.argmax(axis=1) # Score results acc = accuracy_score(Y_ts, Ey) ll = log_loss(Y_ts, p) conf = confusion_matrix(Y_ts, Ey) print("Final scores: {}".format(METHOD)) print("\tAccuracy = {}\n\tLog loss = {}\n\tConfusion =\n{}". format(acc, ll, conf))