def __init__(self):
mnist = MNIST()
(xtrain, ytrain), _, _ = mnist.images()
N, height, width, n_channels = xtrain.shape
input_shape = [None, height, width, n_channels]
model = MnfMnist(N, input_shape, 2, 2, True, learn_p=False,flow_dim_h=50, thres_var=0.5)
x = tf.placeholder(tf.float32, [1, None, height, width, n_channels], name='x')
xx = tf.reshape(x,[-1, height, width, n_channels])
model.build_mnf_mnist(xx)
self.z = model.z
self.noise_size = model.noise_size
self.x = x
self.xx = xx
self.model = model
self.w1, self.b1, self.w2, self.b2, self.w3, self.b3, self.w4, self.b4 = model.layers[0].sample_weights() + model.layers[1].sample_weights() + model.layers[2].sample_weights() + model.layers[3].sample_weights()
self.y_ = tf.placeholder(tf.float32, [None, general_params.number_of_categories], name='y_')
self.y = model.predict(self.xx)
self.probabilities = tf.expand_dims(tf.nn.softmax(self.y),0)
self.prediction = tf.argmax(self.y, 1)
self.correct_prediction = tf.equal(self.prediction, tf.argmax(self.y_, 1))
self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32))
self.saver = tf.train.Saver()
self.graph = tf.get_default_graph()
def train():
mnist = MNIST()
(xtrain, ytrain), (xvalid, yvalid), (xtest, ytest) = mnist.images()
xtrain, xvalid, xtest = np.transpose(xtrain, [0, 2, 3, 1]), np.transpose(
xvalid, [0, 2, 3, 1]), np.transpose(xtest, [0, 2, 3, 1])
ytrain, yvalid, ytest = to_categorical(ytrain, 10), to_categorical(
yvalid, 10), to_categorical(ytest, 10)
N, height, width, n_channels = xtrain.shape
iter_per_epoch = N / 100
sess = tf.InteractiveSession()
input_shape = [None, height, width, n_channels]
x = tf.placeholder(tf.float32, input_shape, name='x')
y_ = tf.placeholder(tf.float32, [None, 10], name='y_')
model = MNFLeNet(N,
input_shape=input_shape,
flows_q=FLAGS.fq,
flows_r=FLAGS.fr,
use_z=not FLAGS.no_z,
learn_p=FLAGS.learn_p,
thres_var=FLAGS.thres_var,
flow_dim_h=FLAGS.flow_h)
tf.set_random_seed(FLAGS.seed)
np.random.seed(FLAGS.seed)
y = model.predict(x)
yd = model.predict(x, sample=False)
pyx = tf.nn.softmax(y)
with tf.name_scope('KL_prior'):
regs = model.get_reg()
tf.summary.scalar('KL prior', regs)
with tf.name_scope('cross_entropy'):
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=y, labels=y_))
tf.summary.scalar('Loglike', cross_entropy)
global_step = tf.Variable(0, trainable=False)
if FLAGS.anneal:
number_zero, original_zero = FLAGS.epzero, FLAGS.epochs / 2
with tf.name_scope('annealing_beta'):
max_zero_step = number_zero * iter_per_epoch
original_anneal = original_zero * iter_per_epoch
beta_t_val = tf.cast(
(tf.cast(global_step, tf.float32) - max_zero_step) /
original_anneal, tf.float32)
beta_t = tf.maximum(beta_t_val, 0.)
annealing = tf.minimum(
1.,
tf.cond(global_step < max_zero_step, lambda: tf.zeros(
(1, ))[0], lambda: beta_t))
tf.summary.scalar('annealing beta', annealing)
else:
annealing = 1.
with tf.name_scope('lower_bound'):
lowerbound = cross_entropy + annealing * regs
tf.summary.scalar('Lower bound', lowerbound)
train_step = tf.train.AdamOptimizer(learning_rate=FLAGS.lr).minimize(
lowerbound, global_step=global_step)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(yd, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('Accuracy', accuracy)
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train',
sess.graph)
tf.add_to_collection('logits', y)
tf.add_to_collection('logits_map', yd)
tf.add_to_collection('accuracy', accuracy)
tf.add_to_collection('x', x)
tf.add_to_collection('y', y_)
saver = tf.train.Saver(tf.global_variables())
print("---------------------------------------------")
# print(tf.global_variables())
print("---------------------------------------------")
tf.global_variables_initializer().run()
idx = np.arange(N)
steps = 0
model_dir = '../../models/mnf/model_lenet_mnist_fq{}_fr{}_usez{}_thres{}/'.format(
FLAGS.fq, FLAGS.fr, not FLAGS.no_z, FLAGS.thres_var)
if not os.path.exists(model_dir):
os.makedirs(model_dir)
print 'Will save model as: {}'.format(model_dir)
# Train
for epoch in xrange(FLAGS.epochs):
widgets = [
"epoch {}/{}|".format(epoch + 1, FLAGS.epochs),
Percentage(),
Bar(),
ETA()
]
pbar = ProgressBar(iter_per_epoch, widgets=widgets)
pbar.start()
np.random.shuffle(idx)
t0 = time.time()
for j in xrange(iter_per_epoch):
steps += 1
pbar.update(j)
batch = np.random.choice(idx, 100)
if j == (iter_per_epoch - 1):
summary, _ = sess.run([merged, train_step],
feed_dict={
x: xtrain[batch],
y_: ytrain[batch]
})
train_writer.add_summary(summary, steps)
train_writer.flush()
else:
sess.run(train_step,
feed_dict={
x: xtrain[batch],
y_: ytrain[batch]
})
# the accuracy here is calculated by a crude MAP so as to have fast evaluation
# it is much better if we properly integrate over the parameters by averaging across multiple samples
tacc = sess.run(accuracy, feed_dict={x: xvalid, y_: yvalid})
string = 'Epoch {}/{}, valid_acc: {:0.3f}'.format(
epoch + 1, FLAGS.epochs, tacc)
if (epoch + 1) % 10 == 0:
string += ', model_save: True'
saver.save(sess, model_dir)
string += ', dt: {:0.3f}'.format(time.time() - t0)
print string
saver.save(sess, model_dir + 'mnf')
train_writer.close()
print '------------------------------------------------'
print '- MNIST -'
preds = np.zeros_like(ytest)
widgets = ["Sampling |", Percentage(), Bar(), ETA()]
pbar = ProgressBar(FLAGS.L, widgets=widgets)
pbar.start()
for i in xrange(FLAGS.L):
pbar.update(i)
for j in xrange(xtest.shape[0] / 100):
pyxi = sess.run(pyx, feed_dict={x: xtest[j * 100:(j + 1) * 100]})
preds[j * 100:(j + 1) * 100] += pyxi / FLAGS.L
print
sample_accuracy = np.mean(
np.equal(np.argmax(preds, 1), np.argmax(ytest, 1)))
print ' - Sample test accuracy: {}'.format(sample_accuracy)
print '------------------------------------------------'
print '- MNIST rotated -'
data_path = '../../data/mnist/mnist_rotated.pkl'
if os.path.exists(data_path):
with open(data_path, 'rb') as f:
data = pickle.load(f)
X = data['X']
y = data['y']
else:
# X, y = test_mnist_rot(plot=False)
# save_mnist_to_file(X, y)
pass
X = X[0:10]
y = y[0:10]
X = X.reshape((X.shape[0], 1, 28, 28))
print X.shape
X = np.transpose(X, [0, 2, 3, 1])
# X = X[:, np.newaxis, :, :]
y = to_categorical(y, 10)
print ' - Data loaded'
preds = np.zeros_like(y)
widgets = ["Sampling |", Percentage(), Bar(), ETA()]
pbar = ProgressBar(10, widgets=widgets)
pbar.start()
for i in xrange(10):
pbar.update(i)
for j in xrange(1):
pyxi = sess.run(pyx, feed_dict={x: X})
preds[0:10] += pyxi / 10
print
sample_accuracy = np.mean(np.equal(np.argmax(preds, 1), np.argmax(y, 1)))
print ' - Sample test accuracy: {}'.format(sample_accuracy)
print '------------------------------------------------'
def experiment_one():
model_dir = './models/mnf_lenet_mnist_fq2_fr2_usezTrue_thres0.5/model/'
# pyx = tf.get_variable("pyx")
# with tf.Session() as sess:
# sess = tf.InteractiveSession()
sess = tf.Session()
mnist = MNIST()
(xtrain, ytrain), (xvalid, yvalid), (xtest, ytest) = mnist.images()
xtrain, xvalid, xtest = np.transpose(xtrain, [0, 2, 3, 1]), np.transpose(
xvalid, [0, 2, 3, 1]), np.transpose(xtest, [0, 2, 3, 1])
ytrain, yvalid, ytest = to_categorical(ytrain, 10), to_categorical(
yvalid, 10), to_categorical(ytest, 10)
N, height, width, n_channels = xtrain.shape
iter_per_epoch = N / 100
input_shape = [None, height, width, n_channels]
x = tf.placeholder(tf.float32, input_shape, name='x')
y_ = tf.placeholder(tf.float32, [None, 10], name='y_')
model = MNFLeNet(N,
input_shape=input_shape,
flows_q=2,
flows_r=2,
use_z=False,
learn_p=True,
thres_var=0.5,
flow_dim_h=50)
tf.set_random_seed(1)
np.random.seed(1)
y = model.predict(x)
yd = model.predict(x, sample=False)
pyx = tf.nn.softmax(y)
saver = tf.train.import_meta_graph(model_dir + 'mnf.meta')
saver.restore(sess, tf.train.latest_checkpoint(model_dir))
# saver = tf.train.latest_checkpoint(model_dir + '**mnf**')
# saver.restore(sess, model_dir + 'mnf.json')
# saver.restore(sess, model_dir + 'mnf')
all_vars = tf.get_collection('vars')
for v in all_vars:
v_ = sess.run(v)
print("loaded")
print '------------------------------------------------'
print '- MNIST rotated -'
data_path = '../../data/mnist/mnist_rotated.pkl'
if os.path.exists(data_path):
with open(data_path, 'rb') as f:
data = pickle.load(f)
X = data['X']
y = data['y']
else:
# X, y = test_mnist_rot(plot=False)
# save_mnist_to_file(X, y)
pass
X = X.reshape((X.shape[0], 1, 28, 28))
print X.shape
X = np.transpose(X, [0, 2, 3, 1])
# X = X[:, np.newaxis, :, :]
y = to_categorical(y, 10)
print 'Data loaded'
preds = np.zeros_like(y)
widgets = ["Sampling |", Percentage(), Bar(), ETA()]
pbar = ProgressBar(10, widgets=widgets)
pbar.start()
for i in xrange(10):
pbar.update(i)
for j in xrange(1):
pyxi = sess.run(pyx, feed_dict={x: X[0:10]})
preds[0:10] += pyxi / 10
print
sample_accuracy = np.mean(np.equal(np.argmax(preds, 1), np.argmax(y, 1)))
print 'Sample test accuracy: {}'.format(sample_accuracy)
print '------------------------------------------------'
def train():
mnist = MNIST()
(xtrain, ytrain), (xvalid, yvalid), (xtest, ytest) = mnist.images()
N, height, width, n_channels = xtrain.shape
iter_per_epoch = int(N / 100)
sess = tf.InteractiveSession()
input_shape = [None, height, width, n_channels]
x = tf.placeholder(tf.float32, input_shape, name='x')
y_ = tf.placeholder(tf.float32, [None, 10], name='y_')
model = MnfMnist(N,
input_shape=input_shape,
flows_q=FLAGS.fq,
flows_r=FLAGS.fr,
use_z=not FLAGS.no_z,
learn_p=FLAGS.learn_p,
thres_var=FLAGS.thres_var,
flow_dim_h=FLAGS.flow_h)
tf.set_random_seed(FLAGS.seed)
np.random.seed(FLAGS.seed)
y = model.predict(x)
yd = model.predict(x, sample=False)
pyx = tf.nn.softmax(y)
with tf.name_scope('KL_prior'):
regs = model.get_reg()
tf.summary.scalar('KL prior', regs)
with tf.name_scope('cross_entropy'):
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=y, labels=y_))
tf.summary.scalar('Loglike', cross_entropy)
global_step = tf.Variable(0, trainable=False)
if FLAGS.anneal:
number_zero, original_zero = FLAGS.epzero, FLAGS.epochs / 2
with tf.name_scope('annealing_beta'):
max_zero_step = number_zero * iter_per_epoch
original_anneal = original_zero * iter_per_epoch
beta_t_val = tf.cast(
(tf.cast(global_step, tf.float32) - max_zero_step) /
original_anneal, tf.float32)
beta_t = tf.maximum(beta_t_val, 0.)
annealing = tf.minimum(
1.,
tf.cond(global_step < max_zero_step, lambda: tf.zeros(
(1, ))[0], lambda: beta_t))
tf.summary.scalar('annealing beta', annealing)
else:
annealing = 1.
with tf.name_scope('lower_bound'):
lowerbound = cross_entropy + annealing * regs
tf.summary.scalar('Lower bound', lowerbound)
train_step = tf.train.AdamOptimizer(learning_rate=FLAGS.lr).minimize(
lowerbound, global_step=global_step)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(yd, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('Accuracy', accuracy)
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train',
sess.graph)
tf.add_to_collection('logits', y)
tf.add_to_collection('logits_map', yd)
tf.add_to_collection('accuracy', accuracy)
tf.add_to_collection('x', x)
tf.add_to_collection('y', y_)
saver = tf.train.Saver(tf.global_variables())
tf.global_variables_initializer().run()
idx = np.arange(N)
steps = 0
model_dir = './models/mnf_lenet_mnist_fq{}_fr{}_usez{}_thres{}/model/'.format(
FLAGS.fq, FLAGS.fr, not FLAGS.no_z, FLAGS.thres_var)
if not os.path.exists(model_dir):
os.makedirs(model_dir)
print('Will save model as: {}'.format(model_dir + 'model'))
# Train
for epoch in range(FLAGS.epochs):
np.random.shuffle(idx)
t0 = time.time()
for j in range(iter_per_epoch):
steps += 1
batch = np.random.choice(idx, 100)
if j == (iter_per_epoch - 1):
summary, _ = sess.run([merged, train_step],
feed_dict={
x: xtrain[batch],
y_: ytrain[batch]
})
train_writer.add_summary(summary, steps)
train_writer.flush()
else:
sess.run(train_step,
feed_dict={
x: xtrain[batch],
y_: ytrain[batch]
})
# the accuracy here is calculated by a crude MAP so as to have fast evaluation
# it is much better if we properly integrate over the parameters by averaging across multiple samples
tacc = sess.run(accuracy, feed_dict={x: xvalid, y_: yvalid})
string = 'Epoch {}/{}, valid_acc: {:0.3f}'.format(
epoch + 1, FLAGS.epochs, tacc)
if (epoch + 1) % 10 == 0:
string += ', model_save: True'
saver.save(sess, model_dir + 'model')
string += ', dt: {:0.3f}'.format(time.time() - t0)
print(string)
saver.save(sess, model_dir + 'model')
train_writer.close()
preds = np.zeros_like(ytest)
for i in range(FLAGS.L):
for j in range(int(xtest.shape[0] / 100)):
pyxi = sess.run(pyx, feed_dict={x: xtest[j * 100:(j + 1) * 100]})
preds[j * 100:(j + 1) * 100] += pyxi / FLAGS.L
print('')
sample_accuracy = np.mean(
np.equal(np.argmax(preds, 1), np.argmax(ytest, 1)))
print('Sample test accuracy: {}'.format(sample_accuracy))
