def train_and_test(train_op, num_iters, sess, model, x_train, y_train, x_test, y_test, batch_size=1,
output_debug=False):
for i in range(num_iters):
# Train...
x, y = benchmark_model.batch(x_train, y_train, batch_size=batch_size)
_, = sess.run([train_op], feed_dict={
model.inputs: x,
model.labels: y,
})
train_loss, train_acc = sess.run([model.cross_entropy, model.accuracy], feed_dict={
model.inputs: x,
model.labels: y,
})
if output_debug:
outputs = sess.run(model.outputs, feed_dict={
model.inputs: x,
model.labels: y,
})
print(outputs)
stats = []
# Compute test accuracy.
for _ in range(5):
x, y = benchmark_model.batch(x_test, y_test, batch_size=batch_size)
stats.append(sess.run([model.cross_entropy, model.accuracy], feed_dict={
model.inputs: x,
model.labels: y,
}))
test_loss, test_acc = np.mean(stats, axis=0)
print('\r({}/{}) [Train]loss {:.3f}, accuracy {:.3f} [Test]loss {:.3f}, accuracy {:.3f}'.format(
i + 1, num_iters,
train_loss, train_acc, test_loss, test_acc), end='', flush=True)
print()
def test_mnist(self):
config = agents.tools.AttrDict(default_config())
model = benchmark_model.build_tf_one_hot_model(config.batch_size)
from keras.utils.np_utils import to_categorical
from keras.datasets.mnist import load_data
(x_train, y_train), (x_test, y_test) = load_data('/tmp/mnist')
x_train = x_train.reshape((-1, 784)).astype(np.float64) / 255.
y_train = to_categorical(y_train, 10).astype(np.float64)
assert x_train.shape == (60000, 784)
assert y_train.shape == (60000, 10)
optimizer = optimizers.NMFOptimizer()
train_op = optimizer.minimize(model.frob_norm)
losses = []
init = tf.global_variables_initializer()
with self.test_session() as sess:
sess.run(init)
pprint(optimizer._layers)
for i in range(2):
x, y = benchmark_model.batch(x_train, y_train, config.batch_size)
_, new_loss, acc = sess.run([train_op, model.cross_entropy, model.accuracy], feed_dict={
model.inputs: x,
model.labels: y,
})
losses.append(new_loss)
print('\nloss {}, accuracy {}'.format(new_loss, acc), end='', flush=True)
def main(_):
# Set configuration
config = AttrDict(default_config())
# Build one hot mnist model.
model = benchmark_model.build_tf_one_hot_model(batch_size=config.batch_size,
use_bias=config.use_bias,
activation=config.activation)
# Load one hot mnist data.
(x_train, y_train), (x_test, y_test) = benchmark_model.load_one_hot_data(dataset=config.dataset)
# Testing whether the dataset have correct shape.
assert x_train.shape == (60000, 784)
assert y_train.shape == (60000, 10)
# Minimize model's loss with NMF optimizer.
# optimizer = NMFOptimizer(config)
optimizer = NMFOptimizer(config=config)
train_op = optimizer.minimize(model.frob_norm)
# Minimize model's loss with Adam optimizer.
bp_optimizer = tf.train.AdamOptimizer(config.learning_rate)
bp_train_op = bp_optimizer.minimize(model.cross_entropy)
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
for i in range(1000):
x, y = benchmark_model.batch(x_train, y_train, batch_size=config.batch_size)
loss, _ = sess.run([optimizer.autoencoder_loss, optimizer.autoencoder_train_op],
feed_dict={model.inputs: x})
print('\rloss {}'.format(loss), end='', flush=True)
print()
_train_and_test = functools.partial(train_and_test,
sess=sess, model=model,
x_train=x_train, y_train=y_train,
x_test=x_test, y_test=y_test,
batch_size=config.batch_size)
print('NMF-optimizer')
# Train with NMF optimizer.
_train_and_test(train_op, num_iters=config.num_mf_iters)
print('Adam-optimizer')
# Train with Adam optimizer.
_train_and_test(bp_train_op, num_iters=config.num_bp_iters)
def train_and_test(train_op, num_iters, sess, model, x_train, y_train, x_test, y_test, batch_size=1,
output_debug=False, add_loss_before_train=False) -> Tuple[np.ndarray, np.ndarray]:
# 1 epoch = num_timesteps * batch_size
total_train_losses = []
total_test_losses = []
num_timesteps = x_train.shape[0] // batch_size
# Before training
if add_loss_before_train:
train_losses = []
for _ in range(5):
x, y = benchmark_model.batch(x_train, y_train, batch_size=batch_size)
train_losses.append(sess.run(model.mse_losses, feed_dict={
model.inputs: x,
model.labels: y,
}))
total_train_losses.append(np.mean(train_losses, axis=0))
test_losses = []
for _ in range(5):
x, y = benchmark_model.batch(x_test, y_test, batch_size=batch_size)
test_losses.append(sess.run(model.mse_losses, feed_dict={
model.inputs: x,
model.labels: y,
}))
total_test_losses.append(np.mean(test_losses))
durations = [.0]
for i in range(num_iters):
# Train...
start_time = time.time()
train_losses = []
test_losses = []
# Inference and compute loss during training.
for t in range(num_timesteps):
x, y = benchmark_model.batch(x_train, y_train, batch_size=batch_size)
_, train_loss = sess.run([train_op, model.mse_losses], feed_dict={
model.inputs: x,
model.labels: y,
})
assert train_loss.shape == (4,), "train_loss shape {}".format(train_loss.shape)
train_losses.append(train_loss)
duration = time.time() - start_time
durations.append(duration)
# Compute test accuracy.
for _ in range(5):
x, y = benchmark_model.batch(x_test, y_test, batch_size=batch_size)
test_losses.append(sess.run(model.mse, feed_dict={
model.inputs: x,
model.labels: y,
}))
train_loss = np.mean(train_losses, axis=0)
assert train_loss.shape[0] == 4
test_loss = np.mean(test_losses)
print('\r({}/{}) [Train]loss {:.3f}, time, {:.3f} [Test]loss {:.3f}'.format(
i + 1, num_iters,
np.mean(train_loss), duration, test_loss), end='', flush=True)
total_train_losses.append(train_loss)
total_test_losses.append(test_loss)
print()
print('Mean training time {}'.format(np.mean(durations)))
total_train_losses = np.asarray(total_train_losses).T
total_test_losses = np.asarray(total_test_losses).T
return (total_train_losses, total_test_losses)
