def test_categorical_likelihood(make_data):
"""Test aboleth with a tf.distributions.Categorical likelihood.
Since it is a bit of an odd half-multivariate case.
"""
x, y, _, = make_data
N, K = x.shape
# Make two classes (K = 2)
Y = np.zeros(len(y), dtype=np.int32)
Y[y[:, 0] > 0] = 1
layers = ab.stack(
ab.InputLayer(name='X', n_samples=10),
lambda X: (X, 0.0) # Mock a sampling layer, with 2-class output
)
nn, reg = layers(X=x.astype(np.float32))
like = tf.distributions.Categorical(logits=nn)
ELBO = ab.elbo(like, Y, N, reg)
MAP = ab.max_posterior(like, Y, reg)
tc = tf.test.TestCase()
with tc.test_session():
tf.global_variables_initializer().run()
assert like.probs.eval().shape == (10, N, K)
assert like.prob(Y).eval().shape == (10, N)
L = ELBO.eval()
assert np.isscalar(L)
L = MAP.eval()
assert np.isscalar(L)
def main():
# Dataset
mnist_data = tf.contrib.learn.datasets.mnist.read_data_sets(
'./mnist_demo', reshape=True)
N, D = mnist_data.train.images.shape
X, Y = tf.data.Dataset.from_tensor_slices(
(np.asarray(mnist_data.train.images, dtype=np.float32),
np.asarray(mnist_data.train.labels, dtype=np.int64))
).repeat(n_epochs).shuffle(N).batch(batch_size) \
.make_one_shot_iterator().get_next()
# Xs, Ys = tf.data.Dataset.from_tensor_slices(
# (np.asarray(mnist_data.test.images, dtype=np.float32),
# np.asarray(mnist_data.test.labels, dtype=np.int64))
# ).repeat(n_epochs).make_one_shot_iterator().get_next()
Xs = np.asarray(mnist_data.test.images, dtype=np.float32)
Ys = np.asarray(mnist_data.test.labels, dtype=np.int64)
# Model specification
with tf.name_scope("model"):
logits, reg = net(X=X)
llh = tf.distributions.Categorical(logits=logits)
loss = ab.elbo(llh, Y, N, reg)
probs = ab.sample_mean(llh.probs)
accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(probs, axis=1), Y), dtype=tf.float32))
# Training graph building
with tf.name_scope("train"):
optimizer = tf.train.AdamOptimizer(learning_rate=1e-3)
global_step = tf.train.get_or_create_global_step()
train = optimizer.minimize(loss, global_step=global_step)
logger = tf.train.LoggingTensorHook(
dict(step=global_step, loss=loss, accuracy=accuracy),
every_n_secs=5
)
with tf.train.MonitoredTrainingSession(
config=config,
hooks=[logger]
) as sess:
while not sess.should_stop():
step, _ = sess.run([global_step, train])
# this part is a bit superfluous. Could get it working with
# Datasets too but just wanted to sanity test for now.
if not step % 100:
val_acc = accuracy.eval(feed_dict={X: Xs, Y: Ys}, session=sess)
tf.logging.info("step = %d, validation accuracy: %f",
step, val_acc)
def my_model(features, labels, mode, params):
N = params["N"]
n_samples = NSAMPLES if mode == tf.estimator.ModeKeys.TRAIN \
else NPREDICTSAMPLES
X = tf.feature_column.input_layer(features, params['feature_columns'])
kernel = ab.RBF(LENSCALE, learn_lenscale=True)
net = (
ab.InputLayer(name="X", n_samples=n_samples) >>
ab.RandomFourier(n_features=NFEATURES, kernel=kernel) >>
ab.Dense(output_dim=64, init_fn="autonorm") >>
ab.Activation(tf.nn.selu) >>
ab.DenseVariational(output_dim=1, full=False, prior_std=1.0,
learn_prior=True)
)
phi, kl = net(X=X)
std = ab.pos_variable(NOISE, name="noise")
ll_f = tf.distributions.Normal(loc=phi, scale=std)
predict_mean = ab.sample_mean(phi)
# Compute predictions.
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'predictions': predict_mean,
'samples': phi
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
ll = ll_f.log_prob(labels)
loss = ab.elbo(ll, kl, N)
tf.summary.scalar('loss', loss)
# Compute evaluation metrics.
mse = tf.metrics.mean_squared_error(labels=labels,
predictions=predict_mean,
name='mse_op')
r2 = r2_metric(labels, predict_mean)
metrics = {'mse': mse,
'r2': r2}
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(
mode, loss=loss, eval_metric_ops=metrics)
# Create training op.
assert mode == tf.estimator.ModeKeys.TRAIN
optimizer = tf.train.AdamOptimizer()
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
def bayesian_linear(X, Y):
"""Bayesian Linear Regression."""
lambda_ = 100.
std = (1 / lambda_)**.5 # Weight st. dev. prior
noise = tf.Variable(1.) # Likelihood st. dev. initialisation, and learning
net = (ab.InputLayer(name="X", n_samples=n_samples_) >>
ab.DenseVariational(output_dim=1, std=std, full=True))
f, kl = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=ab.pos(noise))
loss = ab.elbo(lkhood, Y, N, kl)
return f, loss
def bayesian_linear(X, Y):
"""Bayesian Linear Regression."""
noise = ab.pos_variable(1.0)
net = (
ab.InputLayer(name="X", n_samples=n_samples_) >>
ab.DenseVariational(output_dim=1, full=True)
)
f, kl = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y)
loss = ab.elbo(lkhood, kl, N)
return f, loss
def test_elbo_likelihood(make_graph):
"""Test for expected output dimensions from loss ."""
x, y, N, X_, Y_, N_, layers = make_graph
nn, kl = layers(X=X_)
like = tf.distributions.Normal(nn, scale=1.)
loss = ab.elbo(like, Y_, N, kl)
tc = tf.test.TestCase()
with tc.test_session():
tf.global_variables_initializer().run()
L = loss.eval(feed_dict={X_: x, Y_: y, N_: float(N)})
assert np.isscalar(L)
def bayesian_linear(X, Y):
"""Bayesian Linear Regression."""
reg = .01 # Initial weight prior std. dev, this is optimised later
noise = tf.Variable(.5) # Likelihood st. dev. initialisation, and learning
net = (
ab.InputLayer(name="X", n_samples=n_samples) >>
ab.DenseVariational(output_dim=1, std=reg, full=True)
)
phi, kl = net(X=X)
lkhood = tf.distributions.Normal(loc=phi, scale=ab.pos(noise))
loss = ab.elbo(lkhood, Y, N, kl)
return phi, loss
def nnet_bayesian(X, Y):
"""Bayesian neural net."""
lambda_ = 1e-1 # Weight prior
noise = tf.Variable(0.01) # Likelihood st. dev. initialisation
net = (ab.InputLayer(name="X", n_samples=n_samples_) >>
ab.DenseVariational(output_dim=20, std=lambda_) >> ab.Activation(
tf.nn.relu) >> ab.DenseVariational(output_dim=7, std=lambda_) >>
ab.Activation(tf.nn.relu) >> ab.DenseVariational(
output_dim=5, std=lambda_) >> ab.Activation(
tf.tanh) >> ab.DenseVariational(output_dim=1, std=lambda_))
f, kl = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=ab.pos(noise))
loss = ab.elbo(lkhood, Y, N, kl)
return f, loss
def gaussian_process(X, Y):
"""Gaussian Process Regression."""
noise = ab.pos_variable(.5)
kern = ab.RBF(learn_lenscale=False) # learn lengthscale
net = (
ab.InputLayer(name="X", n_samples=n_samples_) >>
ab.RandomFourier(n_features=50, kernel=kern) >>
ab.DenseVariational(output_dim=1, full=True, learn_prior=True)
)
f, kl = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y)
loss = ab.elbo(lkhood, kl, N)
return f, loss
def deep_gaussian_process(X, Y):
"""Deep Gaussian Process Regression."""
lambda_ = 0.1 # Initial weight prior std. dev, this is optimised later
noise = tf.Variable(.01) # Likelihood st. dev. initialisation
lenscale = tf.Variable(1.) # learn the length scale
net = (ab.InputLayer(name="X", n_samples=n_samples_) >> ab.RandomFourier(
n_features=20, kernel=ab.RBF(ab.pos(lenscale))) >> ab.DenseVariational(
output_dim=5, std=lambda_, full=False) >> ab.RandomFourier(
n_features=10, kernel=ab.RBF(1.)) >> ab.DenseVariational(
output_dim=1, std=lambda_, full=False))
f, kl = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=ab.pos(noise))
loss = ab.elbo(lkhood, Y, N, kl)
return f, loss
def deep_gaussian_process(X, Y):
"""Deep Gaussian Process Regression."""
noise = ab.pos_variable(.1)
net = (
ab.InputLayer(name="X", n_samples=n_samples_) >>
ab.RandomFourier(n_features=20, kernel=ab.RBF(learn_lenscale=True)) >>
ab.DenseVariational(output_dim=5, full=False) >>
ab.RandomFourier(n_features=10, kernel=ab.RBF(1., seed=1)) >>
ab.DenseVariational(output_dim=1, full=False, learn_prior=True)
)
f, kl = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y)
loss = ab.elbo(lkhood, kl, N)
return f, loss
def test_categorical_likelihood(make_data, likelihood):
"""Test aboleth with discrete likelihoods.
Since these are kind of corner cases...
"""
x, y, _, = make_data
like, K = likelihood
N, _ = x.shape
# Make two classes (K = 2)
Y = np.zeros(len(y), dtype=np.int32)
Y[y[:, 0] > 0] = 1
if K == 1:
Y = Y[:, np.newaxis]
X_ = tf.placeholder(tf.float32, x.shape)
Y_ = tf.placeholder(tf.int32, Y.shape)
n_samples_ = tf.placeholder(tf.int32)
layers = ab.stack(
ab.InputLayer(name='X', n_samples=n_samples_),
ab.Dense(output_dim=K)
)
nn, reg = layers(X=X_)
like = like(logits=nn)
log_like = like.log_prob(Y_)
prob = like.prob(Y_)
ELBO = ab.elbo(log_like, reg, N)
MAP = ab.max_posterior(log_like, reg)
fd = {X_: x, Y_: Y, n_samples_: 10}
tc = tf.test.TestCase()
with tc.test_session():
tf.global_variables_initializer().run()
assert like.probs.eval(feed_dict=fd).shape == (10, N, K)
assert prob.eval(feed_dict=fd).shape == (10,) + Y.shape
L = ELBO.eval(feed_dict=fd)
L = MAP.eval(feed_dict=fd)
assert np.isscalar(L)
def gaussian_process(X, Y):
"""Gaussian Process Regression."""
lambda_ = 0.1 # Initial weight prior std. dev, this is optimised later
noise = tf.Variable(.5) # Likelihood st. dev. initialisation, and learning
lenscale = tf.Variable(1.) # learn the length scale
kern = ab.RBF(lenscale=ab.pos(lenscale)) # keep the length scale positive
# kern = ab.RBFVariational(lenscale=ab.pos(lenscale))
net = (ab.InputLayer(name="X", n_samples=n_samples_) >> ab.RandomFourier(
n_features=50, kernel=kern) >> ab.DenseVariational(
output_dim=1, std=lambda_, full=True))
f, kl = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=ab.pos(noise))
# lkhood = tf.distributions.StudentT(df=1., loc=f, scale=ab.pos(noise))
loss = ab.elbo(lkhood, Y, N, kl)
return f, loss
def nnet_bayesian(X, Y):
"""Bayesian neural net."""
noise = 0.01
net = (
ab.InputLayer(name="X", n_samples=n_samples_) >>
ab.DenseVariational(output_dim=5) >>
ab.Activation(tf.nn.selu) >>
ab.DenseVariational(output_dim=4) >>
ab.Activation(tf.nn.selu) >>
ab.DenseVariational(output_dim=3) >>
ab.Activation(tf.nn.selu) >>
ab.DenseVariational(output_dim=1)
)
f, kl = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y)
loss = ab.elbo(lkhood, kl, N)
return f, loss
def nnet_ncp(X, Y):
"""Noise contrastive prior network."""
noise = ab.pos_variable(.5)
lstd = 1.
perturb_noise = 10.
net = (
ab.InputLayer(name="X", n_samples=n_samples_) >>
ab.NCPContinuousPerturb(input_noise=perturb_noise) >>
ab.Dense(output_dim=32) >>
ab.Activation(tf.nn.selu) >>
ab.Dense(output_dim=16) >>
ab.Activation(tf.nn.selu) >>
ab.Dense(output_dim=8) >>
ab.Activation(tf.nn.selu) >>
ab.DenseNCP(output_dim=1, prior_std=.1, latent_std=lstd)
)
f, kl = net(X=X)
lkhood = tf.distributions.Normal(loc=f, scale=noise).log_prob(Y)
loss = ab.elbo(lkhood, kl, N)
return f, loss
def main():
"""Run the demo."""
# Get Continuous and categorical data
df_train, df_test = fetch_data()
df = pd.concat((df_train, df_test))
X_con, X_cat, n_cats, Y = input_fn(df)
n_samples_ = tf.placeholder_with_default(T_SAMPLES, [])
# Define the continuous layers
con_layer = (
ab.InputLayer(name='con', n_samples=n_samples_) >>
ab.RandomFourier(100, kernel=ab.RBF(learn_lenscale=True)) >>
ab.Dense(output_dim=16, init_fn="autonorm")
)
# Now define the cateogrical layers, which we embed
# Note every Embed call can be different, this is just "lazy"
cat_layer_list = [ab.Embed(EMBED_DIMS, i, init_fn="autonorm")
for i in n_cats]
cat_layer = (
ab.InputLayer(name='cat', n_samples=n_samples_) >>
ab.PerFeature(*cat_layer_list) >> # Assign columns to embedding layers
ab.Activation(tf.nn.selu) >>
ab.Dense(16, init_fn="autonorm")
)
# Now we can feed the initial continuous and cateogrical layers to further
# "joint" layers after we concatenate them
net = (
ab.Concat(con_layer, cat_layer) >>
ab.Activation(tf.nn.selu) >>
ab.DenseVariational(output_dim=1)
)
# Split data into training and testing
Xt_con, Xs_con = np.split(X_con, [len(df_train)], axis=0)
Xt_cat, Xs_cat = np.split(X_cat, [len(df_train)], axis=0)
Yt, Ys = np.split(Y, [len(df_train)], axis=0)
# Graph place holders
X_con_ = tf.placeholder(tf.float32, [None, Xt_con.shape[1]])
X_cat_ = tf.placeholder(tf.int32, [None, Xt_cat.shape[1]])
Y_ = tf.placeholder(tf.float32, [None, 1])
# Feed dicts
train_dict = {X_con_: Xt_con, X_cat_: Xt_cat, Y_: Yt}
test_dict = {X_con_: Xs_con, X_cat_: Xs_cat, n_samples_: P_SAMPLES}
# Make model
N = len(Xt_con)
nn, kl = net(con=X_con_, cat=X_cat_)
likelihood = tf.distributions.Bernoulli(logits=nn)
prob = ab.sample_mean(likelihood.probs)
loss = ab.elbo(likelihood.log_prob(Y_), kl, N)
optimizer = tf.train.AdamOptimizer()
train = optimizer.minimize(loss)
init = tf.global_variables_initializer()
with tf.Session(config=CONFIG):
init.run()
# We're going to just use a feed_dict to feed in batches, which we
# generate here
batches = ab.batch(
train_dict,
batch_size=BSIZE,
n_iter=NITER)
for i, data in enumerate(batches):
train.run(feed_dict=data)
if i % 1000 == 0:
loss_val = loss.eval(feed_dict=data)
print("Iteration {}, loss = {}".format(i, loss_val))
# Predict
Ep = prob.eval(feed_dict=test_dict)
Ey = Ep > 0.5 # Max probability assignment
acc = accuracy_score(Ys.flatten(), Ey.flatten())
logloss = log_loss(Ys.flatten(), np.hstack((1 - Ep, Ep)))
print("Accuracy = {}, log loss = {}".format(acc, logloss))
def main():
"""Run the demo."""
n_iters = int(round(n_epochs * N / batch_size))
print("Iterations = {}".format(n_iters))
# Get training and testing data
Xr, Yr, Xs, Ys = gp_draws(N, Ns, kern=kernel, noise=true_noise)
# Prediction points
Xq = np.linspace(-20, 20, Ns).astype(np.float32)[:, np.newaxis]
Yq = np.linspace(-4, 4, Ns).astype(np.float32)[:, np.newaxis]
# Set up the probability image query points
Xi, Yi = np.meshgrid(Xq, Yq)
Xi = Xi.astype(np.float32).reshape(-1, 1)
Yi = Yi.astype(np.float32).reshape(-1, 1)
_, D = Xr.shape
# Name the "data" parts of the graph
with tf.name_scope("Input"):
# This function will make a TensorFlow queue for shuffling and batching
# the data, and will run through n_epochs of the data.
Xb, Yb = batch_training(Xr,
Yr,
n_epochs=n_epochs,
batch_size=batch_size)
X_ = tf.placeholder_with_default(Xb, shape=(None, D))
Y_ = tf.placeholder_with_default(Yb, shape=(None, 1))
# This is where we build the actual GP model
with tf.name_scope("Deepnet"):
phi, kl = net(X=X_)
noise = ab.pos_variable(NOISE)
ll = tf.distributions.Normal(loc=phi, scale=noise).log_prob(Y_)
loss = ab.elbo(ll, kl, N)
# Set up the training graph
with tf.name_scope("Train"):
optimizer = tf.train.AdamOptimizer()
global_step = tf.train.create_global_step()
train = optimizer.minimize(loss, global_step=global_step)
# This is used for building the predictive density image
with tf.name_scope("Predict"):
logprob = ab.sample_mean(ll)
# Logging learning progress
log = tf.train.LoggingTensorHook({
'step': global_step,
'loss': loss
},
every_n_iter=1000)
# This is the main training "loop"
with tf.train.MonitoredTrainingSession(config=config,
save_summaries_steps=None,
save_checkpoint_secs=None,
hooks=[log]) as sess:
try:
while not sess.should_stop():
sess.run(train, feed_dict={n_samples_: n_samples})
except tf.errors.OutOfRangeError:
print('Input queues have been exhausted!')
pass
# Prediction, the [[None]] is to stop the default placeholder queue
Ey = sess.run(phi,
feed_dict={
X_: Xq,
Y_: [[None]],
n_samples_: p_samples
})
logPY = sess.run(logprob,
feed_dict={
Y_: Yi,
X_: Xi,
n_samples_: p_samples
})
Eymean = Ey.mean(axis=0) # Average samples to get mean predicted funtion
Py = np.exp(logPY.reshape(Ns, Ns)) # Turn log-prob into prob
# Plot
im_min = np.amin(Py)
im_size = np.amax(Py) - im_min
img = (Py - im_min) / im_size
f = bk.figure(tools='pan,box_zoom,reset', sizing_mode='stretch_both')
f.image(image=[img], x=-20., y=-4., dw=40., dh=8, palette=bp.Plasma256)
f.circle(Xr.flatten(), Yr.flatten(), fill_color='blue', legend='Training')
f.line(Xs.flatten(), Ys.flatten(), line_color='blue', legend='Truth')
for y in Ey:
f.line(Xq.flatten(),
y.flatten(),
line_color='red',
legend='Samples',
alpha=0.2)
f.line(Xq.flatten(), Eymean.flatten(), line_color='green', legend='Mean')
bk.show(f)
def main():
"""Run the imputation demo."""
# Fetch data, one-hot targets and standardise data
data = fetch_covtype()
X = data.data
Y = (data.target - 1)
X = StandardScaler().fit_transform(X)
# Now fake some missing data with a mask
rnd = np.random.RandomState(RSEED)
mask = rnd.rand(*X.shape) < FRAC_MISSING
X[mask] = MISSING_VAL
# Use Aboleth to learn imputation statistics
if USE_ABOLETH:
net = ab.LearnedNormalImpute(data_input, mask_input) >> layers
# Or just mean impute
else:
net = data_input >> layers
imp = Imputer(missing_values=MISSING_VAL, strategy='mean')
X = imp.fit_transform(X)
# Split the training and testing data
X_tr, X_ts, Y_tr, Y_ts, M_tr, M_ts = train_test_split(X.astype(np.float32),
Y.astype(np.int32),
mask,
test_size=FRAC_TEST,
random_state=RSEED)
N_tr, D = X_tr.shape
# Data
with tf.name_scope("Input"):
Xb, Yb, Mb = batch_training(X_tr,
Y_tr,
M_tr,
n_epochs=NEPOCHS,
batch_size=BSIZE)
X_ = tf.placeholder_with_default(Xb, shape=(None, D))
# Y_ has to be this dimension for compatability with Categorical
Y_ = tf.placeholder_with_default(Yb, shape=(None, ))
M_ = tf.placeholder_with_default(Mb, shape=(None, D))
with tf.name_scope("Deepnet"):
# Conditionally assign a placeholder for masks if USE_ABOLETH
nn, kl = net(X=X_, M=M_) if USE_ABOLETH else net(X=X_)
lkhood = tf.distributions.Categorical(logits=nn)
loss = ab.elbo(lkhood, Y_, N_tr, kl)
with tf.name_scope("Train"):
optimizer = tf.train.AdamOptimizer()
global_step = tf.train.create_global_step()
train = optimizer.minimize(loss, global_step=global_step)
# Logging learning progress
log = tf.train.LoggingTensorHook({
'step': global_step,
'loss': loss
},
every_n_iter=1000)
# This is the main training "loop"
with tf.train.MonitoredTrainingSession(config=CONFIG,
save_summaries_steps=None,
save_checkpoint_secs=None,
hooks=[log]) as sess:
try:
while not sess.should_stop():
sess.run(train)
except tf.errors.OutOfRangeError:
print('Input queues have been exhausted!')
pass
# Prediction
feed_dict = {X_: X_ts, Y_: [0]}
if USE_ABOLETH:
feed_dict[M_] = M_ts
Ep = ab.predict_samples(lkhood.probs,
feed_dict=feed_dict,
n_groups=PSAMPLES,
session=sess)
# Get mean of samples for prediction, and max probability assignments
p = Ep.mean(axis=0)
Ey = p.argmax(axis=1)
# Score results
acc = accuracy_score(Y_ts, Ey)
ll = log_loss(Y_ts, p)
conf = confusion_matrix(Y_ts, Ey)
print("Final scores:")
print("\tAccuracy = {}\n\tLog loss = {}\n\tConfusion =\n{}".format(
acc, ll, conf))
def main():
"""Run the demo."""
data = fetch_gpml_sarcos_data()
Xr = data.train.data.astype(np.float32)
Yr = data.train.targets.astype(np.float32)[:, np.newaxis]
Xs = data.test.data.astype(np.float32)
Ys = data.test.targets.astype(np.float32)[:, np.newaxis]
N, D = Xr.shape
print("Iterations: {}".format(int(round(N * NEPOCHS / BATCH_SIZE))))
# Scale and centre the data, as per the original experiment
ss = StandardScaler()
Xr = ss.fit_transform(Xr)
Xs = ss.transform(Xs)
ym = Yr.mean()
Yr -= ym
Ys -= ym
# Training batches
data_tr = tf.data.Dataset.from_tensor_slices({'X': Xr, 'Y': Yr}) \
.shuffle(buffer_size=1000) \
.batch(BATCH_SIZE)
# Testing iterators
data_ts = tf.data.Dataset.from_tensors({'X': Xs, 'Y': Ys}).repeat()
with tf.name_scope("DataIterators"):
iterator = tf.data.Iterator.from_structure(data_tr.output_types,
data_tr.output_shapes)
data = iterator.get_next()
training_init = iterator.make_initializer(data_tr)
testing_init = iterator.make_initializer(data_ts)
with tf.name_scope("Deepnet"):
phi, kl = net(X=data['X'])
std = tf.Variable(NOISE, name="noise")
lkhood = tf.distributions.Normal(phi, scale=ab.pos(std))
loss = ab.elbo(lkhood, data['Y'], N, kl)
tf.summary.scalar('loss', loss)
with tf.name_scope("Train"):
optimizer = tf.train.AdamOptimizer()
global_step = tf.train.create_global_step()
train = optimizer.minimize(loss, global_step=global_step)
with tf.name_scope("Test"):
Ey = ab.sample_mean(phi)
# Logging
log = tf.train.LoggingTensorHook({
'step': global_step,
'loss': loss
},
every_n_iter=1000)
with tf.train.MonitoredTrainingSession(
config=CONFIG,
scaffold=tf.train.Scaffold(local_init_op=training_init),
checkpoint_dir="./sarcos/",
save_summaries_steps=None,
save_checkpoint_secs=20,
save_summaries_secs=20,
hooks=[log]) as sess:
for i in range(NEPOCHS):
# Train for one epoch
sess.run(training_init)
try:
while not sess.should_stop():
_, g = sess.run([train, global_step])
except tf.errors.OutOfRangeError:
pass
# Init testing and assess and log R-square score on test set
sess.run(testing_init)
Eymean = sess.run(Ey, feed_dict={n_samples_: NPREDICTSAMPLES})
r2 = r2_score(Ys, Eymean)
print("Training epoch {}, r-square = {}".format(i, r2))
rsquare_summary(r2, sess, g)
print("------------")
print("r-square: {:.4f}, smse: {:.4f}".format(r2, 1 - r2))
def main():
"""Run the imputation demo."""
# Fetch data, one-hot targets and standardise data
data = fetch_covtype()
Xo = data.data[:, :10]
Xc = data.data[:, 10:]
Y = (data.target - 1)
Xo[:, :10] = StandardScaler().fit_transform(Xo[:, :10])
# Network construction
n_samples_ = tf.placeholder_with_default(LSAMPLES, [])
data_input = ab.InputLayer(name='Xo', n_samples=n_samples_) # Data input
# Run this with imputation
if METHOD is not None:
print("Imputation method {}.".format(METHOD))
# Fake some missing data
rnd = np.random.RandomState(RSEED)
mask = rnd.rand(*Xo.shape) < FRAC_MISSING
Xo[mask] = MISSING_VAL
# Use Aboleth to imputate
mask_input = ab.MaskInputLayer(name='M') # Missing data mask input
xm = np.ma.array(Xo, mask=mask)
if METHOD == "LearnedNormalImpute":
mean = tf.Variable(np.ma.mean(xm, axis=0).data.astype(np.float32))
std = ab.pos_variable(np.ma.std(xm, axis=0)
.data.astype(np.float32))
input_layer = ab.NormalImpute(data_input, mask_input, mean, std)
elif METHOD == "LearnedScalarImpute":
scalar = tf.Variable(tf.zeros(Xo.shape[-1]))
input_layer = ab.ScalarImpute(data_input, mask_input, scalar)
elif METHOD == "FixedNormalImpute":
mean = np.ma.mean(xm, axis=0).data.astype(np.float32)
std = np.ma.std(xm, axis=0).data.astype(np.float32)
input_layer = ab.NormalImpute(data_input, mask_input, mean, std)
elif METHOD == "FixedScalarImpute":
mean = np.ma.mean(xm, axis=0).data.astype(np.float32)
input_layer = ab.ScalarImpute(data_input, mask_input, mean)
elif METHOD == "MeanImpute":
input_layer = ab.MeanImpute(data_input, mask_input)
else:
raise ValueError("Invalid method!")
# Run this without imputation
else:
print("No missing data")
input_layer = data_input
mask = np.zeros_like(Xo)
cat_layers = (
ab.InputLayer(name='Xc', n_samples=n_samples_) >>
ab.DenseVariational(output_dim=8)
)
con_layers = (
input_layer >>
ab.DenseVariational(output_dim=8)
)
net = (
ab.Concat(cat_layers, con_layers) >>
ab.Activation(tf.nn.selu) >>
ab.DenseVariational(output_dim=NCLASSES)
)
# Split the training and testing data
Xo_tr, Xo_ts, Xc_tr, Xc_ts, Y_tr, Y_ts, M_tr, M_ts = train_test_split(
Xo.astype(np.float32),
Xc.astype(np.float32),
Y.astype(np.int32),
mask,
test_size=FRAC_TEST,
random_state=RSEED
)
N_tr, Do = Xo_tr.shape
_, Dc = Xc_tr.shape
# Data
with tf.name_scope("Input"):
Xob, Xcb, Yb, Mb = batch_training(Xo_tr, Xc_tr, Y_tr, M_tr,
n_epochs=NEPOCHS, batch_size=BSIZE)
Xo_ = tf.placeholder_with_default(Xob, shape=(None, Do))
Xc_ = tf.placeholder_with_default(Xcb, shape=(None, Dc))
# Y_ has to be this dimension for compatability with Categorical
Y_ = tf.placeholder_with_default(Yb, shape=(None,))
if METHOD is not None:
M_ = tf.placeholder_with_default(Mb, shape=(None, Do))
with tf.name_scope("Deepnet"):
if METHOD is not None:
nn, kl = net(Xo=Xo_, Xc=Xc_, M=M_)
else:
nn, kl = net(Xo=Xo_, Xc=Xc_)
lkhood = tf.distributions.Categorical(logits=nn)
loss = ab.elbo(lkhood.log_prob(Y_), kl, N_tr)
prob = ab.sample_mean(lkhood.probs)
with tf.name_scope("Train"):
optimizer = tf.train.AdamOptimizer()
global_step = tf.train.create_global_step()
train = optimizer.minimize(loss, global_step=global_step)
# Logging learning progress
log = tf.train.LoggingTensorHook(
{'step': global_step, 'loss': loss},
every_n_iter=1000
)
# This is the main training "loop"
with tf.train.MonitoredTrainingSession(
config=CONFIG,
save_summaries_steps=None,
save_checkpoint_secs=None,
hooks=[log]
) as sess:
try:
while not sess.should_stop():
sess.run(train)
except tf.errors.OutOfRangeError:
print('Input queues have been exhausted!')
pass
# Prediction
feed_dict = {Xo_: Xo_ts, Xc_: Xc_ts, Y_: [0], n_samples_: PSAMPLES}
if METHOD is not None:
feed_dict[M_] = M_ts
p = sess.run(prob, feed_dict=feed_dict)
# Get mean of samples for prediction, and max probability assignments
Ey = p.argmax(axis=1)
# Score results
acc = accuracy_score(Y_ts, Ey)
ll = log_loss(Y_ts, p)
conf = confusion_matrix(Y_ts, Ey)
print("Final scores: {}".format(METHOD))
print("\tAccuracy = {}\n\tLog loss = {}\n\tConfusion =\n{}".
format(acc, ll, conf))
def main():
"""Run the demo."""
data = fetch_gpml_sarcos_data()
Xr = data.train.data.astype(np.float32)
Yr = data.train.targets.astype(np.float32)[:, np.newaxis]
Xs = data.test.data.astype(np.float32)
Ys = data.test.targets.astype(np.float32)[:, np.newaxis]
N, D = Xr.shape
print("Iterations: {}".format(int(round(N * NEPOCHS / BATCH_SIZE))))
# Scale and centre the data, as per the original experiment
ss = StandardScaler()
Xr = ss.fit_transform(Xr)
Xs = ss.transform(Xs)
ym = Yr.mean()
Yr -= ym
Ys -= ym
# Training batches
data_tr = Dataset.from_tensor_slices({'X': Xr, 'Y': Yr}) \
.shuffle(buffer_size=1000) \
.batch(BATCH_SIZE)
# Testing iterators
data_ts = Dataset.from_tensors({'X': Xs, 'Y': Ys}).repeat()
with tf.name_scope("DataIterators"):
iterator = Iterator.from_structure(data_tr.output_types,
data_tr.output_shapes)
data = iterator.get_next()
training_init = iterator.make_initializer(data_tr)
testing_init = iterator.make_initializer(data_ts)
with tf.name_scope("Deepnet"):
phi, kl = net(X=data['X'])
std = tf.Variable(NOISE, name="noise")
lkhood = tf.distributions.Normal(phi, scale=ab.pos(std))
loss = ab.elbo(lkhood, data['Y'], N, kl)
tf.summary.scalar('loss', loss)
with tf.name_scope("Train"):
optimizer = tf.train.AdamOptimizer()
global_step = tf.train.create_global_step()
train = optimizer.minimize(loss, global_step=global_step)
with tf.name_scope("Test"):
r2 = rsquare(data['Y'], phi)
# Logging
log = tf.train.LoggingTensorHook(
{'step': global_step, 'loss': loss},
every_n_iter=1000
)
with tf.train.MonitoredTrainingSession(
config=CONFIG,
scaffold=tf.train.Scaffold(local_init_op=training_init),
checkpoint_dir="./sarcos/",
save_summaries_steps=None,
save_checkpoint_secs=20,
save_summaries_secs=20,
hooks=[log]
) as sess:
summary_writer = sess._hooks[1]._summary_writer
for i in range(NEPOCHS):
# Train for one epoch
try:
while not sess.should_stop():
sess.run(train)
except tf.errors.OutOfRangeError:
pass
# Init testing and assess and log R-square score on test set
sess.run(testing_init)
r2_score = sess.run(r2)
score_sum = tf.Summary(value=[
tf.Summary.Value(tag='r-square', simple_value=r2_score)
])
summary_writer.add_summary(score_sum, sess.run(global_step))
# Re-init training
sess.run(training_init)
# Prediction
sess.run(testing_init)
Ey = ab.predict_samples(phi, feed_dict=None, n_groups=NPREDICTSAMPLES,
session=sess)
sigma = sess.run(std)
r2_score = sess.run(r2)
# Score mean standardised log likelihood
Eymean = Ey.mean(axis=0)
Eyvar = Ey.var(axis=0) + sigma**2 # add sigma2 for obervation noise
snlp = msll(Ys.flatten(), Eymean, Eyvar, Yr.flatten())
print("------------")
print("r-square: {:.4f}, smse: {:.4f}, msll: {:.4f}."
.format(r2_score, 1 - r2_score, snlp))
