class Emp:
"""A very simple wrapper around the Empirical class of Edward. Once
the model is built with some samples, one can then draw aditional
samples from it through the evaluate method
"""
def __init__(self, model_id):
self.model_id = model_id
def build(self,Y):
self.var = Empirical(Y)
return self
def getVar(self):
return self.var
def evaluate(self, num_samples):
return self.var.sample([num_samples]).eval(session=ed.get_session())
info_dict = inference.update()
inference.print_progress(info_dict)
t = info_dict['t']
if t % inference.n_print == 0:
print("\nInferred cluster means:")
last_mu = sess.run(running_cluster_means, {t_ph: t - 1})
inference.finalize()
print("Inference Done.")
Gibbs_inference_elapsedTime = time.time() - Gibbs_inference_startTime
posterior_mu = qmu.params.eval().mean(axis=0)
# Calculate likelihood for each data point and cluster assignment,
# averaged over many posterior samples. ``x_post`` has shape (N, 100, K, D).
print("Sampling from Posterior...")
mu_sample = qmu.sample(M)
sigmasq_sample = qsigma.sample(M)
pi_sample = qpi.sample(M)
x_post = Normal(loc=tf.ones([N, 1, 1, 1]) * mu_sample,
scale=tf.ones([N, 1, 1, 1]) * tf.sqrt(sigmasq_sample))
x_broadcasted = tf.tile(tf.reshape(train_img, [N, 1, 1, D]), [1, M, K, 1])
x_broadcasted = tf.cast(x_broadcasted, dtype=tf.float32)
# Sum over latent dimension, then average over posterior samples.
# ``log_liks`` ends up with shape (N, K).
log_liks = tf.reduce_mean(tf.reduce_sum(x_post.log_prob(x_broadcasted), 3), 1)
print("Calculating Cluster Assignment...")
clusters = tf.argmax(log_liks, 1).eval()
result_img_dirs = '../tmp/img_result/{}'.format(current_time)
os.makedirs(result_img_dirs)
info_dict = inference.update(feed_dict={x: X_batch, y_ph: Y_batch})
inference.print_progress(info_dict)
#print("Run ok")
'''
X_test = mnist.test.images
X_test = X_test - X_train.mean(axis=0);
Y_test = np.argmax(mnist.test.labels,axis=1)
'''
n_samples = 100
prob_lst = []
samples = []
w_samples = []
b_samples = []
for _ in range(n_samples):
w_samp = qw.sample()
b_samp = qb.sample()
w_samples.append(w_samp)
b_samples.append(b_samp)
prob = tf.nn.softmax(
tf.matmul(tf.cast(X_test.values, tf.float32), w_samp) + b_samp)
prob_lst.append(prob.eval())
sample = tf.concat([tf.reshape(w_samp, [-1]), b_samp], 0)
samples.append(sample.eval())
accy_test = []
for prob in prob_lst:
y_trn_prd = np.argmax(prob, axis=1).astype(np.float32)
acc = (y_trn_prd == Y_test.values.flatten()).mean() * 100
accy_test.append(acc)
def main(_):
outdir = setup_outdir()
ed.set_seed(FLAGS.seed)
((Xtrain, ytrain), (Xtest, ytest)) = blr_utils.get_data()
N, D = Xtrain.shape
N_test, D_test = Xtest.shape
print("Xtrain")
print(Xtrain)
print(Xtrain.shape)
if 'synthetic' in FLAGS.exp:
w = Normal(loc=tf.zeros(D), scale=1.0 * tf.ones(D))
X = tf.placeholder(tf.float32, [N, D])
y = Bernoulli(logits=ed.dot(X, w))
#n_posterior_samples = 100000
n_posterior_samples = 10
qw_empirical = Empirical(
params=tf.get_variable("qw/params", [n_posterior_samples, D]))
inference = ed.HMC({w: qw_empirical}, data={X: Xtrain, y: ytrain})
inference.initialize(n_print=10, step_size=0.6)
tf.global_variables_initializer().run()
inference.run()
empirical_samples = qw_empirical.sample(50).eval()
#fig, ax = plt.subplots()
#ax.scatter(posterior_samples[:,0], posterior_samples[:,1])
#plt.show()
weights, q_components = [], []
ll_trains, ll_tests, bin_ac_trains, bin_ac_tests, elbos, rocs, gaps = [], [], [], [], [], [], []
total_time, times = 0., []
for iter in range(0, FLAGS.n_fw_iter):
print("iter %d" % iter)
g = tf.Graph()
with g.as_default():
sess = tf.InteractiveSession()
with sess.as_default():
tf.set_random_seed(FLAGS.seed)
# MODEL
w = Normal(loc=tf.zeros(D), scale=1.0 * tf.ones(D))
X = tf.placeholder(tf.float32, [N, D])
y = Bernoulli(logits=ed.dot(X, w))
X_test = tf.placeholder(tf.float32, [N_test, D_test])
y_test = Bernoulli(logits=ed.dot(X_test, w))
qw = construct_base_dist([D], iter, 'qw')
inference_time_start = time.time()
inference = relbo.KLqp({w: qw},
fw_iterates=get_fw_iterates(
weights, w, q_components),
data={
X: Xtrain,
y: ytrain
},
fw_iter=iter)
tf.global_variables_initializer().run()
inference.run(n_iter=FLAGS.LMO_iter)
inference_time_end = time.time()
total_time += float(inference_time_end - inference_time_start)
joint = Joint(Xtrain, ytrain, sess)
if iter > 0:
qtw_prev = build_mixture(weights, q_components)
gap = compute_duality_gap(joint, qtw_prev, qw)
gaps.append(gap)
np.savetxt(os.path.join(outdir, "gaps.csv"),
gaps,
delimiter=',')
print("duality gap", gap)
# update weights
gamma = 2. / (iter + 2.)
weights = [(1. - gamma) * w for w in weights]
weights.append(gamma)
# update components
q_components = update_iterate(q_components, qw)
if len(q_components) > 1 and FLAGS.fw_variant == 'fc':
print("running fully corrective")
# overwrite the weights
weights = fully_corrective(
build_mixture(weights, q_components), joint)
if True:
# remove inactivate iterates
weights = list(weights)
for i in reversed(range(len(weights))):
if weights[i] == 0:
del weights[i]
del q_components[i]
weights = np.array(
weights
) # TODO type acrobatics to make elements deletable
elif len(q_components
) > 1 and FLAGS.fw_variant == 'line_search':
print("running line search")
weights = line_search(
build_mixture(weights[:-1], q_components[:-1]), qw,
joint)
qtw_new = build_mixture(weights, q_components)
if False:
for i, comp in enumerate(qtw_new.components):
print("component", i, "\tmean",
comp.mean().eval(), "\tstddev",
comp.stddev().eval())
train_lls = [
sess.run(y.log_prob(ytrain),
feed_dict={
X: Xtrain,
w: qtw_new.sample().eval()
}) for _ in range(50)
]
train_lls = np.mean(train_lls, axis=0)
ll_trains.append((np.mean(train_lls), np.std(train_lls)))
test_lls = [
sess.run(y_test.log_prob(ytest),
feed_dict={
X_test: Xtest,
w: qtw_new.sample().eval()
}) for _ in range(50)
]
test_lls = np.mean(test_lls, axis=0)
ll_tests.append((np.mean(test_lls), np.std(test_lls)))
logits = np.mean([
np.dot(Xtest,
qtw_new.sample().eval()) for _ in range(50)
],
axis=0)
ypred = tf.sigmoid(logits).eval()
roc_score = roc_auc_score(ytest, ypred)
rocs.append(roc_score)
print('roc_score', roc_score)
print('ytrain', np.mean(train_lls), np.std(train_lls))
print('ytest', np.mean(test_lls), np.std(test_lls))
order = np.argsort(ytest)
plt.scatter(range(len(ypred)), ypred[order], c=ytest[order])
plt.savefig(os.path.join(outdir, 'ypred%d.pdf' % iter))
plt.close()
np.savetxt(os.path.join(outdir, "train_lls.csv"),
ll_trains,
delimiter=',')
np.savetxt(os.path.join(outdir, "test_lls.csv"),
ll_tests,
delimiter=',')
np.savetxt(os.path.join(outdir, "rocs.csv"),
rocs,
delimiter=',')
x_post = ed.copy(y, {w: qtw_new})
x_post_t = ed.copy(y_test, {w: qtw_new})
print(
'log lik train',
ed.evaluate('log_likelihood',
data={
x_post: ytrain,
X: Xtrain
}))
print(
'log lik test',
ed.evaluate('log_likelihood',
data={
x_post_t: ytest,
X_test: Xtest
}))
#ll_train = ed.evaluate('log_likelihood', data={x_post: ytrain, X:Xtrain})
#ll_test = ed.evaluate('log_likelihood', data={x_post_t: ytest, X_test:Xtest})
bin_ac_train = ed.evaluate('binary_accuracy',
data={
x_post: ytrain,
X: Xtrain
})
bin_ac_test = ed.evaluate('binary_accuracy',
data={
x_post_t: ytest,
X_test: Xtest
})
print('binary accuracy train', bin_ac_train)
print('binary accuracy test', bin_ac_test)
#latest_elbo = elbo(qtw_new, joint, w)
#foo = ed.KLqp({w: qtw_new}, data={X: Xtrain, y: ytrain})
#op = myloss(foo)
#print("myloss", sess.run(op[0], feed_dict={X: Xtrain, y: ytrain}), sess.run(op[1], feed_dict={X: Xtrain, y: ytrain}))
#append_and_save(ll_trains, ll_train, "loglik_train.csv", np.savetxt)
#append_and_save(ll_tests, ll_train, "loglik_test.csv", np.savetxt) #append_and_save(bin_ac_trains, bin_ac_train, "bin_acc_train.csv", np.savetxt) #append_and_save(bin_ac_tests, bin_ac_test, "bin_acc_test.csv", np.savetxt)
##append_and_save(elbos, latest_elbo, "elbo.csv", np.savetxt)
#print('log-likelihood train ', ll_train)
#print('log-likelihood test ', ll_test)
#print('binary_accuracy train ', bin_ac_train)
#print('binary_accuracy test ', bin_ac_test)
#print('elbo', latest_elbo)
times.append(total_time)
np.savetxt(os.path.join(setup_outdir(), 'times.csv'), times)
tf.reset_default_graph()
inference = ed.HMC({w: qw, b: qb}, data={X: X_train, y: y_train})
inference.initialize(n_print=10, step_size=0.6)
tf.global_variables_initializer().run()
# Set up figure.
fig = plt.figure(figsize=(8, 8), facecolor='white')
ax = fig.add_subplot(111, frameon=False)
plt.ion()
plt.show(block=False)
# Build samples from inferred posterior.
n_samples = 50
inputs = np.linspace(-5, 3, num=400, dtype=np.float32).reshape((400, 1))
probs = tf.stack([tf.sigmoid(ed.dot(inputs, qw.sample()) + qb.sample())
for _ in range(n_samples)])
for t in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
if t % inference.n_print == 0:
outputs = probs.eval()
# Plot data and functions
plt.cla()
ax.plot(X_train[:], y_train, 'bx')
for s in range(n_samples):
ax.plot(inputs[:], outputs[s], alpha=0.2)
def main(_):
ed.set_seed(42)
# DATA
X_train, y_train = build_toy_dataset(FLAGS.N)
X_test, y_test = build_toy_dataset(FLAGS.N)
# MODEL
X = tf.placeholder(tf.float32, [FLAGS.N, FLAGS.D])
w = Normal(loc=tf.zeros(FLAGS.D), scale=tf.ones(FLAGS.D))
b = Normal(loc=tf.zeros(1), scale=tf.ones(1))
y = Normal(loc=ed.dot(X, w) + b, scale=tf.ones(FLAGS.N))
# INFERENCE
qw = Empirical(params=tf.get_variable("qw/params", [FLAGS.T, FLAGS.D]))
qb = Empirical(params=tf.get_variable("qb/params", [FLAGS.T, 1]))
inference = ed.SGHMC({w: qw, b: qb}, data={X: X_train, y: y_train})
inference.run(step_size=1e-3)
# CRITICISM
# Plot posterior samples.
sns.jointplot(qb.params.eval()[FLAGS.nburn:FLAGS.T:FLAGS.stride],
qw.params.eval()[FLAGS.nburn:FLAGS.T:FLAGS.stride])
plt.show()
# Posterior predictive checks.
y_post = ed.copy(y, {w: qw, b: qb})
# This is equivalent to
# y_post = Normal(loc=ed.dot(X, qw) + qb, scale=tf.ones(FLAGS.N))
print("Mean squared error on test data:")
print(ed.evaluate('mean_squared_error', data={X: X_test, y_post: y_test}))
print("Displaying prior predictive samples.")
n_prior_samples = 10
w_prior = w.sample(n_prior_samples).eval()
b_prior = b.sample(n_prior_samples).eval()
plt.scatter(X_train, y_train)
inputs = np.linspace(-1, 10, num=400)
for ns in range(n_prior_samples):
output = inputs * w_prior[ns] + b_prior[ns]
plt.plot(inputs, output)
plt.show()
print("Displaying posterior predictive samples.")
n_posterior_samples = 10
w_post = qw.sample(n_posterior_samples).eval()
b_post = qb.sample(n_posterior_samples).eval()
plt.scatter(X_train, y_train)
inputs = np.linspace(-1, 10, num=400)
for ns in range(n_posterior_samples):
output = inputs * w_post[ns] + b_post[ns]
plt.plot(inputs, output)
plt.show()
print("Displaying prior predictive samples.")
n_prior_samples = 10
w_prior = w.sample(n_prior_samples).eval()
b_prior = b.sample(n_prior_samples).eval()
plt.scatter(X_train, y_train)
inputs = np.linspace(-1, 10, num=400, dtype=np.float32)
for ns in range(n_prior_samples):
output = inputs * w_prior[ns] + b_prior[ns]
plt.plot(inputs, output)
plt.show()
print("Displaying posterior predictive samples.")
n_posterior_samples = 10
w_post = qw.sample(n_posterior_samples).eval()
b_post = qb.sample(n_posterior_samples).eval()
plt.scatter(X_train, y_train)
inputs = np.linspace(-1, 10, num=400, dtype=np.float32)
for ns in range(n_posterior_samples):
output = inputs * w_post[ns] + b_post[ns]
plt.plot(inputs, output)
plt.show()
class Mixed_gauss_model():
def __init__(self, num_data, num_cluster, vector_dim, num_mcmc_sample):
self.K = num_cluster
self.D = vector_dim
self.N = num_data
self.pi = Dirichlet(tf.ones(self.K))
self.mu = Normal(tf.zeros(self.D),
tf.ones(self.D),
sample_shape=self.K)
self.sigmasq = InverseGamma(tf.ones(self.D),
tf.ones(self.D),
sample_shape=self.K)
self.x = ParamMixture(self.pi, {
'loc': self.mu,
'scale_diag': tf.sqrt(self.sigmasq)
},
MultivariateNormalDiag,
sample_shape=self.N)
self.z = self.x.cat
self.T = num_mcmc_sample # number of MCMC samples
self.qpi = Empirical(tf.Variable(tf.ones([self.T, self.K]) / self.K))
self.qmu = Empirical(tf.Variable(tf.zeros([self.T, self.K, self.D])))
self.qsigmasq = Empirical(
tf.Variable(tf.ones([self.T, self.K, self.D])))
self.qz = Empirical(
tf.Variable(tf.zeros([self.T, self.N], dtype=tf.int32)))
def fit(self, x_train):
self.inference = ed.Gibbs(
{
self.pi: self.qpi,
self.mu: self.qmu,
self.sigmasq: self.qsigmasq,
self.z: self.qz
},
data={self.x: x_train})
self.inference.initialize()
sess = ed.get_session()
tf.global_variables_initializer().run()
t_ph = tf.placeholder(tf.int32, [])
running_cluster_means = tf.reduce_mean(self.qmu.params[:t_ph], 0)
for _ in range(self.inference.n_iter):
info_dict = self.inference.update()
self.inference.print_progress(info_dict)
t = info_dict['t']
if t % self.inference.n_print == 0:
print("\nInferred cluster means:")
print(sess.run(running_cluster_means, {t_ph: t - 1}))
def clustering(self, x_data):
mu_sample = self.qmu.sample(100)
sigmasq_sample = self.qsigmasq.sample(100)
x_post = Normal(loc=tf.ones([self.N, 1, 1, 1]) * mu_sample,
scale=tf.ones([self.N, 1, 1, 1]) *
tf.sqrt(sigmasq_sample))
x_broadcasted = tf.tile(tf.reshape(x_data, [self.N, 1, 1, self.D]),
[1, 100, self.K, 1])
log_liks = x_post.log_prob(x_broadcasted)
log_liks = tf.reduce_sum(log_liks, 3)
log_liks = tf.reduce_mean(log_liks, 1)
self.clusters = tf.argmax(log_liks, 1).eval()
def plot_clusters(self, x_data, C, axis=[0, 1], simu=False):
if simu == True:
fig = plt.figure()
ax1 = fig.add_subplot(1, 3, 1)
ax2 = fig.add_subplot(1, 3, 2)
ax3 = fig.add_subplot(1, 3, 3)
ax1.scatter(x_data[:, axis[0]], x_data[:, axis[1]])
ax1.axis([-4, 4, -4, 4])
#ax1.title("Simulated dataset")
ax2.scatter(x_data[:, axis[0]],
x_data[:, axis[1]],
c=C,
cmap=cm.Set1)
ax2.axis([-4, 4, -4, 4])
ax3.scatter(x_data[:, axis[0]],
x_data[:, axis[1]],
c=self.clusters,
cmap=cm.Set1)
ax3.axis([-4, 4, -4, 4])
#ax2.title("Predicted cluster assignments")
plt.show()
else:
plt.scatter(x_data[:axis[0]],
x_data[:, axis[1]],
c=C,
cmap=cm.Set1)
plt.show()
def __init__(self, n, xdim, n_mixtures=5, mc_samples=500):
# Compute the shape dynamically from placeholders
self.x_ph = tf.placeholder(tf.float32, [None, xdim])
self.k = k = n_mixtures
self.batch_size = n
self.d = d = xdim
self.sample_size = tf.placeholder(tf.int32, ())
# Build the priors over membership probabilities and mixture parameters
with tf.variable_scope("priors"):
pi = Dirichlet(tf.ones(k))
mu = Normal(tf.zeros(d), tf.ones(d), sample_shape=k)
sigmasq = InverseGamma(tf.ones(d), tf.ones(d), sample_shape=k)
# Build the conditional mixture model
with tf.variable_scope("likelihood"):
x = ParamMixture(pi, {'loc': mu, 'scale_diag': tf.sqrt(sigmasq)},
MultivariateNormalDiag,
sample_shape=n)
z = x.cat
# Build approximate posteriors as Empirical samples
t = mc_samples
with tf.variable_scope("posteriors_samples"):
qpi = Empirical(tf.get_variable(
"qpi/params", [t, k],
initializer=tf.constant_initializer(1.0 / k)))
qmu = Empirical(tf.get_variable(
"qmu/params", [t, k, d],
initializer=tf.zeros_initializer()))
qsigmasq = Empirical(tf.get_variable(
"qsigmasq/params", [t, k, d],
initializer=tf.ones_initializer()))
qz = Empirical(tf.get_variable(
"qz/params", [t, n],
initializer=tf.zeros_initializer(),
dtype=tf.int32))
# Build inference graph using Gibbs and conditionals
with tf.variable_scope("inference"):
self.inference = ed.Gibbs({
pi: qpi,
mu: qmu,
sigmasq: qsigmasq,
z: qz
}, data={
x: self.x_ph
})
self.inference.initialize()
# Build predictive posterior graph by taking samples
n_samples = self.sample_size
with tf.variable_scope("posterior"):
mu_smpl = qmu.sample(n_samples) # shape: [1, 100, k, d]
sigmasq_smpl = qsigmasq.sample(n_samples)
x_post = Normal(
loc=tf.ones((n, 1, 1, 1)) * mu_smpl,
scale=tf.ones((n, 1, 1, 1)) * tf.sqrt(sigmasq_smpl)
)
# NOTE: x_ph has shape [n, d]
x_broadcasted = tf.tile(
tf.reshape(self.x_ph, (n, 1, 1, d)),
(1, n_samples, k, 1)
)
x_ll = x_post.log_prob(x_broadcasted)
x_ll = tf.reduce_sum(x_ll, axis=3)
x_ll = tf.reduce_mean(x_ll, axis=1)
self.sample_t_ph = tf.placeholder(tf.int32, ())
self.eval_ops = {
'generative_post': x_post,
'qmu': qmu,
'qsigma': qsigma,
'post_running_mu': tf.reduce_mean(
qmu.params[:self.sample_t_ph],
axis=0
)
'post_log_prob': xll
}
def main(_):
ed.set_seed(42)
# DATA
X_train, y_train = build_toy_dataset(FLAGS.N)
X_test, y_test = build_toy_dataset(FLAGS.N)
# MODEL
X = tf.placeholder(tf.float32, [FLAGS.N, FLAGS.D])
w = Normal(loc=tf.zeros(FLAGS.D), scale=tf.ones(FLAGS.D))
b = Normal(loc=tf.zeros(1), scale=tf.ones(1))
y = Normal(loc=ed.dot(X, w) + b, scale=tf.ones(FLAGS.N))
# INFERENCE
qw = Empirical(params=tf.get_variable("qw/params", [FLAGS.T, FLAGS.D]))
qb = Empirical(params=tf.get_variable("qb/params", [FLAGS.T, 1]))
inference = ed.SGHMC({w: qw, b: qb}, data={X: X_train, y: y_train})
inference.run(step_size=1e-3)
# CRITICISM
# Plot posterior samples.
sns.jointplot(qb.params.eval()[FLAGS.nburn:FLAGS.T:FLAGS.stride],
qw.params.eval()[FLAGS.nburn:FLAGS.T:FLAGS.stride])
plt.show()
# Posterior predictive checks.
y_post = ed.copy(y, {w: qw, b: qb})
# This is equivalent to
# y_post = Normal(loc=ed.dot(X, qw) + qb, scale=tf.ones(FLAGS.N))
print("Mean squared error on test data:")
print(ed.evaluate('mean_squared_error', data={X: X_test, y_post: y_test}))
print("Displaying prior predictive samples.")
n_prior_samples = 10
w_prior = w.sample(n_prior_samples).eval()
b_prior = b.sample(n_prior_samples).eval()
plt.scatter(X_train, y_train)
inputs = np.linspace(-1, 10, num=400)
for ns in range(n_prior_samples):
output = inputs * w_prior[ns] + b_prior[ns]
plt.plot(inputs, output)
plt.show()
print("Displaying posterior predictive samples.")
n_posterior_samples = 10
w_post = qw.sample(n_posterior_samples).eval()
b_post = qb.sample(n_posterior_samples).eval()
plt.scatter(X_train, y_train)
inputs = np.linspace(-1, 10, num=400)
for ns in range(n_posterior_samples):
output = inputs * w_post[ns] + b_post[ns]
plt.plot(inputs, output)
plt.show()
#Y_base[n] ~ a + bX[n]
#\epsilon ~ N(0,\sigma_Y)
X = tf.placeholder(tf.float32, [N, 1])
a = Normal(mu=tf.zeros([1]), sigma=tf.ones([1]) * 500)
b = Normal(mu=tf.zeros([1]), sigma=tf.ones([1]) * 500)
sigma = Uniform(a=tf.ones([1]) * 1, b=tf.ones([1]) * 10)
Y = Normal(mu=ed.dot(X, b) + a, sigma=sigma)
#データ
data = {X: X_data, Y: Y_data}
##推論(変分ベイズ)
#qa = Normal(mu=tf.Variable(tf.random_normal([1])),\
# sigma=tf.nn.softplus(tf.Variable(tf.random_normal([1]))))
#qb = Normal(mu=tf.Variable(tf.random_normal([1])),\
# sigma=tf.nn.softplus(tf.Variable(tf.random_normal([1]))))
#qsigma = Normal(mu=tf.Variable(tf.random_normal([1])),\
# sigma=tf.nn.softplus(tf.Variable(tf.random_normal([1]))))
#inference = ed.KLqp({a: qa, b: qb, sigma : qsigma}, data)
#inference.run(n_samples=1, n_iter=10000)
#HMC
qa = Empirical(params=tf.Variable(tf.random_normal([10000, 1])))
qb = Empirical(params=tf.Variable(tf.random_normal([10000, 1])))
qsigma = Empirical(params=tf.Variable(tf.random_normal([10000, 1])))
inference = ed.HMC({a: qa, b: qb, sigma: qsigma}, data=data)
inference.run()
qa.sample(10).eval()
qb.sample(10000).eval()
def _test(self, params, n): x = Empirical(params=params) val_est = x.sample(n).shape.as_list() val_true = n + tf.convert_to_tensor(params).shape.as_list()[1:] self.assertEqual(val_est, val_true)
inference = ed.HMC({w: qw, b: qb}, data={X: X_train, y: y_train})
inference.initialize(n_print=10, step_size=0.6)
tf.global_variables_initializer().run()
# Set up figure.
fig = plt.figure(figsize=(8, 8), facecolor='white')
ax = fig.add_subplot(111, frameon=False)
plt.ion()
plt.show(block=False)
# Build samples from inferred posterior.
n_samples = 50
inputs = np.linspace(-5, 3, num=400, dtype=np.float32).reshape((400, 1))
probs = tf.stack([tf.sigmoid(ed.dot(inputs, qw.sample()) + qb.sample())
for _ in range(n_samples)])
for t in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
if t % inference.n_print == 0:
outputs = probs.eval()
# Plot data and functions
plt.cla()
ax.plot(X_train[:], y_train, 'bx')
for s in range(n_samples):
ax.plot(inputs[:], outputs[s], alpha=0.2)
num_examples = len(X_train)
new_y_train = y_train.values.flatten()
print("Training...")
print(inference.n_iter)
for i in range(inference.n_iter):
info_dict=inference.update(feed_dict={x: X_train, y_ph: new_y_train})
inference.print_progress(info_dict)
#saver = tf.train.Saver()
#saver.save(sess, './bayesianlenet')
print("Testing")
print("Test Set: {} samples".format(len(X_test)))
with tf.name_scope("testing"):
nTestSamples=30
result=[]
for i in range(nTestSamples):
tweights=qweights.sample()
tbias=qbias.sample()
dic={'weights':tweights,'bias':tbias}
ypred=Categorical(Softmax(X_test,dic))
print(ypred.eval())
print(y_test.values.flatten())
match = (ypred.eval()==y_test.values.flatten()).sum()
print(match)
print("Test Accuracy: {} %".format(str(round(match*100/len(X_test),6))))
#import matplotlib.pyplot as plt
#plt.hist(np.array(result)/100)
#plt.show()
def main(_):
ed.set_seed(42)
# DATA
X_train, y_train = build_toy_dataset(FLAGS.N)
# MODEL
X = tf.placeholder(tf.float32, [FLAGS.N, FLAGS.D])
w = Normal(loc=tf.zeros(FLAGS.D), scale=3.0 * tf.ones(FLAGS.D))
b = Normal(loc=tf.zeros([]), scale=3.0 * tf.ones([]))
y = Bernoulli(logits=ed.dot(X, w) + b)
# INFERENCE
qw = Empirical(params=tf.get_variable("qw/params", [FLAGS.T, FLAGS.D]))
qb = Empirical(params=tf.get_variable("qb/params", [FLAGS.T]))
inference = ed.HMC({w: qw, b: qb}, data={X: X_train, y: y_train})
inference.initialize(n_print=10, step_size=0.6)
# Alternatively, use variational inference.
# qw_loc = tf.get_variable("qw_loc", [FLAGS.D])
# qw_scale = tf.nn.softplus(tf.get_variable("qw_scale", [FLAGS.D]))
# qb_loc = tf.get_variable("qb_loc", []) + 10.0
# qb_scale = tf.nn.softplus(tf.get_variable("qb_scale", []))
# qw = Normal(loc=qw_loc, scale=qw_scale)
# qb = Normal(loc=qb_loc, scale=qb_scale)
# inference = ed.KLqp({w: qw, b: qb}, data={X: X_train, y: y_train})
# inference.initialize(n_print=10, n_iter=600)
tf.global_variables_initializer().run()
# Set up figure.
fig = plt.figure(figsize=(8, 8), facecolor='white')
ax = fig.add_subplot(111, frameon=False)
plt.ion()
plt.show(block=False)
# Build samples from inferred posterior.
n_samples = 50
inputs = np.linspace(-5, 3, num=400, dtype=np.float32).reshape((400, 1))
probs = tf.stack([
tf.sigmoid(ed.dot(inputs, qw.sample()) + qb.sample())
for _ in range(n_samples)
])
for t in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
if t % inference.n_print == 0:
outputs = probs.eval()
# Plot data and functions
plt.cla()
ax.plot(X_train[:], y_train, 'bx')
for s in range(n_samples):
ax.plot(inputs[:], outputs[s], alpha=0.2)
ax.set_xlim([-5, 3])
ax.set_ylim([-0.5, 1.5])
plt.draw()
plt.pause(1.0 / 60.0)
#proposal_theta = Beta(concentration1=1.0, concentration0=1.0, sample_shape=(1,))
# proposal_theta = Normal(loc=theta,scale=0.5)
# inference = ed.MetropolisHastings({theta: qtheta}, {theta: proposal_theta}, {formulas: data})
sess = ed.get_session()
inference = ed.HMC({theta: qtheta}, {formulas: data})
inference.initialize()
tf.global_variables_initializer().run()
for _ in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
inference.finalize()
train_writer = tf.summary.FileWriter('/tmp/tensorflow/',sess.graph)
# qtheta = Beta(tf.Variable(1.0), tf.Variable(1.0)) #Why need tf.Variable here?
# inference = ed.KLqp({theta: qtheta}, {formulas: data})
##Results:
qtheta_samples = qtheta.sample(1000).eval()
print(qtheta_samples.mean())
plt.hist(qtheta_samples)
plt.show()
print("Displaying prior predictive samples.")
n_prior_samples = 10
w_prior = w.sample(n_prior_samples).eval()
b_prior = b.sample(n_prior_samples).eval()
plt.scatter(X_train, y_train)
inputs = np.linspace(-1, 10, num=400)
for ns in range(n_prior_samples):
output = inputs * w_prior[ns] + b_prior[ns]
plt.plot(inputs, output)
plt.show()
print("Displaying posterior predictive samples.")
n_posterior_samples = 10
w_post = qw.sample(n_posterior_samples).eval()
b_post = qb.sample(n_posterior_samples).eval()
plt.scatter(X_train, y_train)
inputs = np.linspace(-1, 10, num=400)
for ns in range(n_posterior_samples):
output = inputs * w_post[ns] + b_post[ns]
plt.plot(inputs, output)
plt.show()
sess = ed.get_session()
tf.global_variables_initializer().run()
t_ph = tf.placeholder(tf.int32, [])
running_cluster_means = tf.reduce_mean(qmu.params[:t_ph], 0)
for _ in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
t = info_dict['t']
print("\nInferred cluster means:")
print(sess.run(running_cluster_means, {t_ph: t - 1}))
# Calculate likelihood for each data point and cluster assignment,
# averaged over many posterior samples. ``x_post`` has shape (N, 100, K, D).
mu_sample = qmu.sample(100)
sigmasq_sample = qsigmasq.sample(100)
x_post = Normal(loc=tf.ones([N, 1, 1, 1]) * mu_sample,
scale=tf.ones([N, 1, 1, 1]) * tf.sqrt(sigmasq_sample))
x_broadcasted = tf.tile(tf.reshape(x_train, [N, 1, 1, D]), [1, 100, K, 1])
# Sum over latent dimension, then average over posterior samples.
# ``log_liks`` ends up with shape (N, K).
log_liks = x_post.log_prob(x_broadcasted)
log_liks = tf.reduce_sum(log_liks, 3)
log_liks = tf.reduce_mean(log_liks, 1)
# Choose the cluster with the highest likelihood for each data point.
clusters = tf.argmax(log_liks, 1).eval()
plt.scatter(x_train[:, 0], x_train[:, 1], c=clusters, cmap=cm.bwr)
def _test(params, n): x = Empirical(params=params) val_est = get_dims(x.sample(n)) val_true = n + get_dims(params)[1:] assert val_est == val_true
inference = ed.HMC({w: qw, b: qb}, data={X: X_train, y: y_train})
inference.initialize(n_print=10, step_size=0.6)
tf.global_variables_initializer().run()
# criticism & set up figure
fig = plt.figure(figsize=(8, 8), facecolor='white')
ax = fig.add_subplot(111, frameon=False)
plt.ion()
plt.show(block=False)
n_samples = 50
inputs = np.linspace(-5, 3, num=400, dtype=np.float32).reshape((400, 1))
probs = tf.stack([
tf.sigmoid(ed.dot(inputs, qw.sample()) + qb.sample())
for _ in range(n_samples)
])
for t in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
if t % inference.n_print == 0:
outputs = probs.eval()
# plot data and functions
plt.cla()
ax.plot(X_train[:], y_train, 'bx')
for s in range(n_samples):
ax.plot(inputs[:], outputs[s], alpha=0.2)
qw.params.eval()[nburn:T:stride])
plt.show()
# posterior predictive check
# this is equivalent to
# y_post = Normal(loc=ed.dot(X, qw)+qb, scale=tf.ones(N))
y_post = ed.copy(y, {w:qw, b:qb})
print("mean squared error on test data:")
print(ed.evaluate("mean_squared_error", data={X:X_test, y_post:y_test}))
print("Displaying prior predictive samples")
n_prior_samples=10
w_prior = w.sample(n_prior_samples).eval()
b_prior = b.sample(n_prior_samples).eval()
plt.scatter(X_train, y_train)
inputs = np.linspace(-1, 10, num=400)
for ns in range(n_prior_samples):
output = inputs*w_prior[ns] + b_prior[ns]
plt.plot(inputs, output)
plt.show()
print("Displaying posterior predictive samples")
n_posterior_samples=20
w_prior = qw.sample(n_posterior_samples).eval()
b_prior = qb.sample(n_posterior_samples).eval()
plt.scatter(X_train, y_train)
inputs = np.linspace(-1, 10, num=400)
for ns in range(n_posterior_samples):
output = inputs*w_prior[ns] + b_prior[ns]
plt.plot(inputs, output)
plt.show()
