def test_categorical_likelihood(make_data):
"""Test aboleth with a tf.distributions.Categorical likelihood.
Since it is a bit of an odd half-multivariate case.
"""
x, y, _, = make_data
N, K = x.shape
# Make two classes (K = 2)
Y = np.zeros(len(y), dtype=np.int32)
Y[y[:, 0] > 0] = 1
layers = ab.stack(
ab.InputLayer(name='X', n_samples=10),
lambda X: (X, 0.0) # Mock a sampling layer, with 2-class output
)
nn, reg = layers(X=x.astype(np.float32))
like = tf.distributions.Categorical(logits=nn)
ELBO = ab.elbo(like, Y, N, reg)
MAP = ab.max_posterior(like, Y, reg)
tc = tf.test.TestCase()
with tc.test_session():
tf.global_variables_initializer().run()
assert like.probs.eval().shape == (10, N, K)
assert like.prob(Y).eval().shape == (10, N)
L = ELBO.eval()
assert np.isscalar(L)
L = MAP.eval()
assert np.isscalar(L)
def nnet_dropout(X, Y):
"""Neural net with dropout."""
reg = 0.001 # Weight prior
noise = .5 # Likelihood st. dev.
net = (
ab.InputLayer(name="X", n_samples=n_samples) >>
ab.DenseMAP(output_dim=30, l2_reg=reg, l1_reg=0.) >>
ab.Activation(tf.tanh) >>
ab.DropOut(keep_prob=0.95) >>
ab.DenseMAP(output_dim=20, l2_reg=reg, l1_reg=0.) >>
ab.Activation(tf.tanh) >>
ab.DropOut(keep_prob=0.95) >>
ab.DenseMAP(output_dim=10, l2_reg=reg, l1_reg=0.) >>
ab.Activation(tf.tanh) >>
ab.DropOut(keep_prob=0.95) >>
ab.DenseMAP(output_dim=5, l2_reg=reg, l1_reg=0.) >>
ab.Activation(tf.tanh) >>
ab.DenseMAP(output_dim=1, l2_reg=reg, l1_reg=0.)
)
phi, reg = net(X=X)
lkhood = tf.distributions.Normal(loc=phi, scale=noise)
loss = ab.max_posterior(lkhood, Y, reg)
return phi, loss
def main():
# Dataset
mnist_data = tf.contrib.learn.datasets.mnist.read_data_sets('./mnist_demo',
reshape=False)
N = mnist_data.train.images.shape[0]
X, Y = tf.data.Dataset.from_tensor_slices(
(np.asarray(mnist_data.train.images, dtype=np.float32),
np.asarray(mnist_data.train.labels, dtype=np.int64))
).repeat(n_epochs).shuffle(N).batch(batch_size) \
.make_one_shot_iterator().get_next()
# Xs, Ys = tf.data.Dataset.from_tensor_slices(
# (np.asarray(mnist_data.test.images, dtype=np.float32),
# np.asarray(mnist_data.test.labels, dtype=np.int64))
# ).repeat(n_epochs).make_one_shot_iterator().get_next()
Xs = np.asarray(mnist_data.test.images, dtype=np.float32)
Ys = np.asarray(mnist_data.test.labels, dtype=np.int64)
# Model specification
with tf.name_scope("model"):
logits, reg = net(X=X)
llh = tf.distributions.Categorical(logits=logits)
loss = ab.max_posterior(llh.log_prob(Y), reg)
probs = ab.sample_mean(llh.probs)
accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(probs, axis=1), Y), dtype=tf.float32))
# Training graph building
with tf.name_scope("train"):
optimizer = tf.train.AdamOptimizer(learning_rate=1e-3)
global_step = tf.train.get_or_create_global_step()
train = optimizer.minimize(loss, global_step=global_step)
logger = tf.train.LoggingTensorHook(dict(step=global_step,
loss=loss,
accuracy=accuracy),
every_n_secs=5)
with tf.train.MonitoredTrainingSession(config=config,
hooks=[logger]) as sess:
while not sess.should_stop():
step, _ = sess.run([global_step, train])
# this part is a bit superfluous. Could get it working with
# Datasets too but just wanted to sanity test for now.
if not step % 100:
val_acc = accuracy.eval(feed_dict={X: Xs, Y: Ys}, session=sess)
tf.logging.info("step = %d, validation accuracy: %f", step,
val_acc)
def test_map_likelihood(make_graph):
"""Test for expected output dimensions from deepnet."""
x, y, _, _, _, _, layers = make_graph
nn, reg = layers(X=x.astype(np.float32))
like = tf.distributions.Normal(nn, scale=1.)
loss = ab.max_posterior(like, y.astype(np.float32), reg)
tc = tf.test.TestCase()
with tc.test_session():
tf.global_variables_initializer().run()
L = loss.eval()
assert np.isscalar(L)
def linear(X, Y):
"""Linear regression with l2 regularization."""
lambda_ = 1e-4 # Weight regularizer
noise = 1. # Likelihood st. dev.
net = (ab.InputLayer(name="X") >> ab.DenseMAP(
output_dim=1, l2_reg=lambda_, l1_reg=0.))
Xw, reg = net(X=X)
lkhood = tf.distributions.Normal(loc=Xw, scale=noise)
loss = ab.max_posterior(lkhood, Y, reg)
# loss = 0.5 * tf.reduce_mean((Y - Xw)**2) + reg
return Xw, loss
def test_map_likelihood(make_graph):
"""Test for expected output dimensions from deepnet."""
x, y, N, X_, Y_, N_, layers = make_graph
nn, reg = layers(X=X_)
log_like = tf.distributions.Normal(nn, scale=1.).log_prob(Y_)
loss = ab.max_posterior(log_like, reg)
tc = tf.test.TestCase()
with tc.test_session():
tf.global_variables_initializer().run()
L = loss.eval(feed_dict={X_: x, Y_: y})
assert np.isscalar(L)
def linear(X, Y):
"""Linear regression with l2 regularization."""
reg = .01 # Weight prior
noise = .5 # Likelihood st. dev.
net = (
ab.InputLayer(name="X", n_samples=1) >>
ab.DenseMAP(output_dim=1, l2_reg=reg, l1_reg=0.)
)
phi, reg = net(X=X)
lkhood = tf.distributions.Normal(loc=phi, scale=noise)
loss = ab.max_posterior(lkhood, Y, reg)
return phi, loss
def nnet(X, Y):
"""Neural net with regularization."""
lambda_ = 1e-4 # Weight regularizer
noise = .5 # Likelihood st. dev.
net = (
ab.InputLayer(name="X", n_samples=1) >> ab.DenseMAP(
output_dim=40, l2_reg=lambda_, l1_reg=0.) >> ab.Activation(tf.tanh)
>> ab.DenseMAP(output_dim=20, l2_reg=lambda_,
l1_reg=0.) >> ab.Activation(tf.tanh) >>
ab.DenseMAP(output_dim=10, l2_reg=lambda_, l1_reg=0.) >> ab.Activation(
tf.tanh) >> ab.DenseMAP(output_dim=1, l2_reg=lambda_, l1_reg=0.))
f, reg = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=noise)
loss = ab.max_posterior(lkhood, Y, reg)
return f, loss
def test_categorical_likelihood(make_data, likelihood):
"""Test aboleth with discrete likelihoods.
Since these are kind of corner cases...
"""
x, y, _, = make_data
like, K = likelihood
N, _ = x.shape
# Make two classes (K = 2)
Y = np.zeros(len(y), dtype=np.int32)
Y[y[:, 0] > 0] = 1
if K == 1:
Y = Y[:, np.newaxis]
X_ = tf.placeholder(tf.float32, x.shape)
Y_ = tf.placeholder(tf.int32, Y.shape)
n_samples_ = tf.placeholder(tf.int32)
layers = ab.stack(
ab.InputLayer(name='X', n_samples=n_samples_),
ab.Dense(output_dim=K)
)
nn, reg = layers(X=X_)
like = like(logits=nn)
log_like = like.log_prob(Y_)
prob = like.prob(Y_)
ELBO = ab.elbo(log_like, reg, N)
MAP = ab.max_posterior(log_like, reg)
fd = {X_: x, Y_: Y, n_samples_: 10}
tc = tf.test.TestCase()
with tc.test_session():
tf.global_variables_initializer().run()
assert like.probs.eval(feed_dict=fd).shape == (10, N, K)
assert prob.eval(feed_dict=fd).shape == (10,) + Y.shape
L = ELBO.eval(feed_dict=fd)
L = MAP.eval(feed_dict=fd)
assert np.isscalar(L)
def nnet_dropout(X, Y):
"""Neural net with dropout."""
lambda_ = 1e-3 # Weight prior
noise = .5 # Likelihood st. dev.
net = (
ab.InputLayer(name="X", n_samples=n_samples_) >>
ab.Dense(output_dim=32, l2_reg=lambda_) >>
ab.Activation(tf.nn.selu) >>
ab.DropOut(keep_prob=0.9, independent=True) >>
ab.Dense(output_dim=16, l2_reg=lambda_) >>
ab.Activation(tf.nn.selu) >>
ab.DropOut(keep_prob=0.95, independent=True) >>
ab.Dense(output_dim=8, l2_reg=lambda_) >>
ab.Activation(tf.nn.selu) >>
ab.Dense(output_dim=1, l2_reg=lambda_)
)
f, reg = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y)
loss = ab.max_posterior(lkhood, reg)
return f, loss
def main():
"""Run the demo."""
n_iters = int(round(n_epochs * N / batch_size))
print("Iterations = {}".format(n_iters))
# Get training and testing data
Xr, Yr, Xs, Ys = gp_draws(N, Ns, kern=kernel, noise=true_noise)
# Prediction points
Xq = np.linspace(-20, 20, Ns).astype(np.float32)[:, np.newaxis]
Yq = np.linspace(-4, 4, Ns).astype(np.float32)[:, np.newaxis]
# Set up the probability image query points
Xi, Yi = np.meshgrid(Xq, Yq)
Xi = Xi.astype(np.float32).reshape(-1, 1)
Yi = Yi.astype(np.float32).reshape(-1, 1)
_, D = Xr.shape
# Name the "data" parts of the graph
with tf.name_scope("Input"):
# This function will make a TensorFlow queue for shuffling and batching
# the data, and will run through n_epochs of the data.
Xb, Yb = batch_training(Xr,
Yr,
n_epochs=n_epochs,
batch_size=batch_size)
X_ = tf.placeholder_with_default(Xb, shape=(None, D))
Y_ = tf.placeholder_with_default(Yb, shape=(None, 1))
# This is where we build the actual GP model
with tf.name_scope("Deepnet"):
phi, reg = net(X=X_)
noise = ab.pos_variable(NOISE)
ll = tf.distributions.Normal(loc=phi, scale=noise).log_prob(Y_)
loss = ab.max_posterior(ll, reg)
# Set up the trainig graph
with tf.name_scope("Train"):
optimizer = tf.train.AdamOptimizer()
global_step = tf.train.create_global_step()
train = optimizer.minimize(loss, global_step=global_step)
# This is used for building the predictive density image
with tf.name_scope("Predict"):
logprob = tf.reduce_mean(ll, axis=0)
# Logging learning progress
log = tf.train.LoggingTensorHook({
'step': global_step,
'loss': loss
},
every_n_iter=1000)
# This is the main training "loop"
with tf.train.MonitoredTrainingSession(config=config,
save_summaries_steps=None,
save_checkpoint_secs=None,
hooks=[log]) as sess:
try:
while not sess.should_stop():
sess.run(
train,
feed_dict={
n_samples_: n_samples,
# tf.keras.backend.learning_phase(): 1
})
except tf.errors.OutOfRangeError:
print('Input queues have been exhausted!')
pass
# Prediction, the [[None]] is to stop the default placeholder queue
# we keep learning phase flag on even in prediction phase to draw
# samples from predictive distribution
Ey = sess.run(phi,
feed_dict={
X_: Xq,
Y_: [[None]],
n_samples_: p_samples
})
logPY = sess.run(logprob,
feed_dict={
Y_: Yi,
X_: Xi,
n_samples_: p_samples
})
Eymean = Ey.mean(axis=0) # Average samples to get mean predicted funtion
Py = np.exp(logPY.reshape(Ns, Ns)) # Turn log-prob into prob
# Plot
im_min = np.amin(Py)
im_size = np.amax(Py) - im_min
img = (Py - im_min) / im_size
f = bk.figure(tools='pan,box_zoom,reset', sizing_mode='stretch_both')
f.image(image=[img], x=-20., y=-4., dw=40., dh=8, palette=bp.Plasma256)
f.circle(Xr.flatten(), Yr.flatten(), fill_color='blue', legend='Training')
f.line(Xs.flatten(), Ys.flatten(), line_color='blue', legend='Truth')
for y in Ey:
f.line(Xq.flatten(),
y.flatten(),
line_color='red',
legend='Samples',
alpha=0.2)
f.line(Xq.flatten(), Eymean.flatten(), line_color='green', legend='Mean')
bk.show(f)
def main():
"""Run the demo."""
data = load_breast_cancer()
X = data.data.astype(np.float32)
y = data.target.astype(np.int32)[:, np.newaxis]
X = StandardScaler().fit_transform(X).astype(np.float32)
N, D = X.shape
# Benchmark classifier
bcl = RandomForestClassifier(random_state=RSEED)
# Data
with tf.name_scope("Input"):
X_ = tf.placeholder(dtype=tf.float32, shape=(None, D))
Y_ = tf.placeholder(dtype=tf.float32, shape=(None, 1))
with tf.name_scope("Deepnet"):
nn, reg = net(X=X_)
lkhood = tf.distributions.Bernoulli(logits=nn)
loss = ab.max_posterior(lkhood.log_prob(Y_), reg)
prob = ab.sample_mean(lkhood.probs)
with tf.name_scope("Train"):
optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
train = optimizer.minimize(loss)
kfold = KFold(n_splits=FOLDS, shuffle=True, random_state=RSEED)
# Launch the graph.
acc, acc_o, ll, ll_o = [], [], [], []
init = tf.global_variables_initializer()
with tf.Session(config=CONFIG):
for k, (r_ind, s_ind) in enumerate(kfold.split(X)):
init.run()
Xr, Yr = X[r_ind], y[r_ind]
Xs, Ys = X[s_ind], y[s_ind]
batches = ab.batch(
{X_: Xr, Y_: Yr},
batch_size=BSIZE,
n_iter=NITER)
for i, data in enumerate(batches):
train.run(feed_dict=data)
if i % 1000 == 0:
loss_val = loss.eval(feed_dict=data)
print("Iteration {}, loss = {}".format(i, loss_val))
# Predict, NOTE: we use the mean of the likelihood to get the
# probabilies
ps = prob.eval(feed_dict={X_: Xs, n_samples_: PSAMPLES})
print("Fold {}:".format(k))
Ep = np.hstack((1. - ps, ps))
print_k_result(Ys, Ep, ll, acc, "BNN")
bcl.fit(Xr, Yr.flatten())
Ep_o = bcl.predict_proba(Xs)
print_k_result(Ys, Ep_o, ll_o, acc_o, "RF")
print("-----")
print_final_result(acc, ll, "BNN")
print_final_result(acc_o, ll_o, "RF")