def my_model(features, labels, mode, params):
N = params["N"]
n_samples = NSAMPLES if mode == tf.estimator.ModeKeys.TRAIN \
else NPREDICTSAMPLES
X = tf.feature_column.input_layer(features, params['feature_columns'])
kernel = ab.RBF(LENSCALE, learn_lenscale=True)
net = (
ab.InputLayer(name="X", n_samples=n_samples) >>
ab.RandomFourier(n_features=NFEATURES, kernel=kernel) >>
ab.Dense(output_dim=64, init_fn="autonorm") >>
ab.Activation(tf.nn.selu) >>
ab.DenseVariational(output_dim=1, full=False, prior_std=1.0,
learn_prior=True)
)
phi, kl = net(X=X)
std = ab.pos_variable(NOISE, name="noise")
ll_f = tf.distributions.Normal(loc=phi, scale=std)
predict_mean = ab.sample_mean(phi)
# Compute predictions.
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'predictions': predict_mean,
'samples': phi
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
ll = ll_f.log_prob(labels)
loss = ab.elbo(ll, kl, N)
tf.summary.scalar('loss', loss)
# Compute evaluation metrics.
mse = tf.metrics.mean_squared_error(labels=labels,
predictions=predict_mean,
name='mse_op')
r2 = r2_metric(labels, predict_mean)
metrics = {'mse': mse,
'r2': r2}
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(
mode, loss=loss, eval_metric_ops=metrics)
# Create training op.
assert mode == tf.estimator.ModeKeys.TRAIN
optimizer = tf.train.AdamOptimizer()
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
def bayesian_linear(X, Y):
"""Bayesian Linear Regression."""
noise = ab.pos_variable(1.0)
net = (
ab.InputLayer(name="X", n_samples=n_samples_) >>
ab.DenseVariational(output_dim=1, full=True)
)
f, kl = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y)
loss = ab.elbo(lkhood, kl, N)
return f, loss
def gaussian_process(X, Y):
"""Gaussian Process Regression."""
noise = ab.pos_variable(.5)
kern = ab.RBF(learn_lenscale=False) # learn lengthscale
net = (
ab.InputLayer(name="X", n_samples=n_samples_) >>
ab.RandomFourier(n_features=50, kernel=kern) >>
ab.DenseVariational(output_dim=1, full=True, learn_prior=True)
)
f, kl = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y)
loss = ab.elbo(lkhood, kl, N)
return f, loss
def deep_gaussian_process(X, Y):
"""Deep Gaussian Process Regression."""
noise = ab.pos_variable(.1)
net = (
ab.InputLayer(name="X", n_samples=n_samples_) >>
ab.RandomFourier(n_features=20, kernel=ab.RBF(learn_lenscale=True)) >>
ab.DenseVariational(output_dim=5, full=False) >>
ab.RandomFourier(n_features=10, kernel=ab.RBF(1., seed=1)) >>
ab.DenseVariational(output_dim=1, full=False, learn_prior=True)
)
f, kl = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y)
loss = ab.elbo(lkhood, kl, N)
return f, loss
def nnet_ncp(X, Y):
"""Noise contrastive prior network."""
noise = ab.pos_variable(.5)
lstd = 1.
perturb_noise = 10.
net = (
ab.InputLayer(name="X", n_samples=n_samples_) >>
ab.NCPContinuousPerturb(input_noise=perturb_noise) >>
ab.Dense(output_dim=32) >>
ab.Activation(tf.nn.selu) >>
ab.Dense(output_dim=16) >>
ab.Activation(tf.nn.selu) >>
ab.Dense(output_dim=8) >>
ab.Activation(tf.nn.selu) >>
ab.DenseNCP(output_dim=1, prior_std=.1, latent_std=lstd)
)
f, kl = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y)
loss = ab.elbo(lkhood, kl, N)
return f, loss
def main():
"""Run the demo."""
n_iters = int(round(n_epochs * N / batch_size))
print("Iterations = {}".format(n_iters))
# Get training and testing data
Xr, Yr, Xs, Ys = gp_draws(N, Ns, kern=kernel, noise=true_noise)
# Prediction points
Xq = np.linspace(-20, 20, Ns).astype(np.float32)[:, np.newaxis]
Yq = np.linspace(-4, 4, Ns).astype(np.float32)[:, np.newaxis]
# Set up the probability image query points
Xi, Yi = np.meshgrid(Xq, Yq)
Xi = Xi.astype(np.float32).reshape(-1, 1)
Yi = Yi.astype(np.float32).reshape(-1, 1)
_, D = Xr.shape
# Name the "data" parts of the graph
with tf.name_scope("Input"):
# This function will make a TensorFlow queue for shuffling and batching
# the data, and will run through n_epochs of the data.
Xb, Yb = batch_training(Xr,
Yr,
n_epochs=n_epochs,
batch_size=batch_size)
X_ = tf.placeholder_with_default(Xb, shape=(None, D))
Y_ = tf.placeholder_with_default(Yb, shape=(None, 1))
# This is where we build the actual GP model
with tf.name_scope("Deepnet"):
phi, reg = net(X=X_)
noise = ab.pos_variable(NOISE)
ll = tf.distributions.Normal(loc=phi, scale=noise).log_prob(Y_)
loss = ab.max_posterior(ll, reg)
# Set up the trainig graph
with tf.name_scope("Train"):
optimizer = tf.train.AdamOptimizer()
global_step = tf.train.create_global_step()
train = optimizer.minimize(loss, global_step=global_step)
# This is used for building the predictive density image
with tf.name_scope("Predict"):
logprob = tf.reduce_mean(ll, axis=0)
# Logging learning progress
log = tf.train.LoggingTensorHook({
'step': global_step,
'loss': loss
},
every_n_iter=1000)
# This is the main training "loop"
with tf.train.MonitoredTrainingSession(config=config,
save_summaries_steps=None,
save_checkpoint_secs=None,
hooks=[log]) as sess:
try:
while not sess.should_stop():
sess.run(
train,
feed_dict={
n_samples_: n_samples,
# tf.keras.backend.learning_phase(): 1
})
except tf.errors.OutOfRangeError:
print('Input queues have been exhausted!')
pass
# Prediction, the [[None]] is to stop the default placeholder queue
# we keep learning phase flag on even in prediction phase to draw
# samples from predictive distribution
Ey = sess.run(phi,
feed_dict={
X_: Xq,
Y_: [[None]],
n_samples_: p_samples
})
logPY = sess.run(logprob,
feed_dict={
Y_: Yi,
X_: Xi,
n_samples_: p_samples
})
Eymean = Ey.mean(axis=0) # Average samples to get mean predicted funtion
Py = np.exp(logPY.reshape(Ns, Ns)) # Turn log-prob into prob
# Plot
im_min = np.amin(Py)
im_size = np.amax(Py) - im_min
img = (Py - im_min) / im_size
f = bk.figure(tools='pan,box_zoom,reset', sizing_mode='stretch_both')
f.image(image=[img], x=-20., y=-4., dw=40., dh=8, palette=bp.Plasma256)
f.circle(Xr.flatten(), Yr.flatten(), fill_color='blue', legend='Training')
f.line(Xs.flatten(), Ys.flatten(), line_color='blue', legend='Truth')
for y in Ey:
f.line(Xq.flatten(),
y.flatten(),
line_color='red',
legend='Samples',
alpha=0.2)
f.line(Xq.flatten(), Eymean.flatten(), line_color='green', legend='Mean')
bk.show(f)
def main():
"""Run the imputation demo."""
# Fetch data, one-hot targets and standardise data
data = fetch_covtype()
Xo = data.data[:, :10]
Xc = data.data[:, 10:]
Y = (data.target - 1)
Xo[:, :10] = StandardScaler().fit_transform(Xo[:, :10])
# Network construction
n_samples_ = tf.placeholder_with_default(LSAMPLES, [])
data_input = ab.InputLayer(name='Xo', n_samples=n_samples_) # Data input
# Run this with imputation
if METHOD is not None:
print("Imputation method {}.".format(METHOD))
# Fake some missing data
rnd = np.random.RandomState(RSEED)
mask = rnd.rand(*Xo.shape) < FRAC_MISSING
Xo[mask] = MISSING_VAL
# Use Aboleth to imputate
mask_input = ab.MaskInputLayer(name='M') # Missing data mask input
xm = np.ma.array(Xo, mask=mask)
if METHOD == "LearnedNormalImpute":
mean = tf.Variable(np.ma.mean(xm, axis=0).data.astype(np.float32))
std = ab.pos_variable(np.ma.std(xm, axis=0)
.data.astype(np.float32))
input_layer = ab.NormalImpute(data_input, mask_input, mean, std)
elif METHOD == "LearnedScalarImpute":
scalar = tf.Variable(tf.zeros(Xo.shape[-1]))
input_layer = ab.ScalarImpute(data_input, mask_input, scalar)
elif METHOD == "FixedNormalImpute":
mean = np.ma.mean(xm, axis=0).data.astype(np.float32)
std = np.ma.std(xm, axis=0).data.astype(np.float32)
input_layer = ab.NormalImpute(data_input, mask_input, mean, std)
elif METHOD == "FixedScalarImpute":
mean = np.ma.mean(xm, axis=0).data.astype(np.float32)
input_layer = ab.ScalarImpute(data_input, mask_input, mean)
elif METHOD == "MeanImpute":
input_layer = ab.MeanImpute(data_input, mask_input)
else:
raise ValueError("Invalid method!")
# Run this without imputation
else:
print("No missing data")
input_layer = data_input
mask = np.zeros_like(Xo)
cat_layers = (
ab.InputLayer(name='Xc', n_samples=n_samples_) >>
ab.DenseVariational(output_dim=8)
)
con_layers = (
input_layer >>
ab.DenseVariational(output_dim=8)
)
net = (
ab.Concat(cat_layers, con_layers) >>
ab.Activation(tf.nn.selu) >>
ab.DenseVariational(output_dim=NCLASSES)
)
# Split the training and testing data
Xo_tr, Xo_ts, Xc_tr, Xc_ts, Y_tr, Y_ts, M_tr, M_ts = train_test_split(
Xo.astype(np.float32),
Xc.astype(np.float32),
Y.astype(np.int32),
mask,
test_size=FRAC_TEST,
random_state=RSEED
)
N_tr, Do = Xo_tr.shape
_, Dc = Xc_tr.shape
# Data
with tf.name_scope("Input"):
Xob, Xcb, Yb, Mb = batch_training(Xo_tr, Xc_tr, Y_tr, M_tr,
n_epochs=NEPOCHS, batch_size=BSIZE)
Xo_ = tf.placeholder_with_default(Xob, shape=(None, Do))
Xc_ = tf.placeholder_with_default(Xcb, shape=(None, Dc))
# Y_ has to be this dimension for compatability with Categorical
Y_ = tf.placeholder_with_default(Yb, shape=(None,))
if METHOD is not None:
M_ = tf.placeholder_with_default(Mb, shape=(None, Do))
with tf.name_scope("Deepnet"):
if METHOD is not None:
nn, kl = net(Xo=Xo_, Xc=Xc_, M=M_)
else:
nn, kl = net(Xo=Xo_, Xc=Xc_)
lkhood = tf.distributions.Categorical(logits=nn)
loss = ab.elbo(lkhood.log_prob(Y_), kl, N_tr)
prob = ab.sample_mean(lkhood.probs)
with tf.name_scope("Train"):
optimizer = tf.train.AdamOptimizer()
global_step = tf.train.create_global_step()
train = optimizer.minimize(loss, global_step=global_step)
# Logging learning progress
log = tf.train.LoggingTensorHook(
{'step': global_step, 'loss': loss},
every_n_iter=1000
)
# This is the main training "loop"
with tf.train.MonitoredTrainingSession(
config=CONFIG,
save_summaries_steps=None,
save_checkpoint_secs=None,
hooks=[log]
) as sess:
try:
while not sess.should_stop():
sess.run(train)
except tf.errors.OutOfRangeError:
print('Input queues have been exhausted!')
pass
# Prediction
feed_dict = {Xo_: Xo_ts, Xc_: Xc_ts, Y_: [0], n_samples_: PSAMPLES}
if METHOD is not None:
feed_dict[M_] = M_ts
p = sess.run(prob, feed_dict=feed_dict)
# Get mean of samples for prediction, and max probability assignments
Ey = p.argmax(axis=1)
# Score results
acc = accuracy_score(Y_ts, Ey)
ll = log_loss(Y_ts, p)
conf = confusion_matrix(Y_ts, Ey)
print("Final scores: {}".format(METHOD))
print("\tAccuracy = {}\n\tLog loss = {}\n\tConfusion =\n{}".
format(acc, ll, conf))