def test_linear_fit(self):
epochs = 2000
iris = Iris()
x_node = node.VarNode('x')
yt_node = node.VarNode('yt')
dense = node.DenseLayer(x_node, 3)
softmax = act.Softmax(dense)
cross_entropy = loss.CrossEntropy(softmax, yt_node)
optimizer_func = core.np.Optimization.AdamOptimizer(lr=0.001)
optimizer = core.np.Optimization.OptimizerIterator([x_node, yt_node],
cross_entropy,
optimizer_func)
log_at_info()
epoch = 0
ctx = node.ComputeContext()
for x, y in iris.train_iterator(epochs, 8):
ctx['x'], ctx['yt'] = x, y
loss_now = optimizer.step(ctx, 1.0)
if epoch % 100 == 0:
info("[{}]\tloss_now = {}".format(epoch, loss_now))
epoch += 1
f = node.make_evaluator([x_node, yt_node], softmax)
total, correct = 40, 0
for x, y_actual in iris.test_iterator(total, one_hot=False):
ctx['x'], ctx['yt'] = x, y_actual
y_predicted = f.at(ctx)
max_idx = np.argmax(y_predicted)
if max_idx == y_actual:
correct += 1
percent = correct * 100 / total
print("Correct= {}%".format(percent))
def test_linear_training_tf_fast(self):
r"""
For fastest results, use batch size of 64, adam optimizer
and 3 epochs. You should get more than 97% accuracy
:return:
"""
# Build the network
x_node = node.VarNode('x')
yt_node = node.VarNode('yt')
linear1 = node.DenseLayer(x_node, 100, name="Dense-First")
relu1 = act.RelUNode(linear1, name="RelU-First")
linear2 = node.DenseLayer(relu1, 200, name="Dense-Second")
relu2 = act.RelUNode(linear2, name="RelU-Second")
linear3 = node.DenseLayer(relu2, 10, name="Dense-Third")
cross_entropy = loss.LogitsCrossEntropy(linear3, yt_node, name="XEnt")
# Set up optimizers and params
batch_size = 64
epochs = 5 # use 25 for SGD
optimizer_func = autodiff_optim.AdamOptimizer()
# optimizer_func = autodiff_optim.SGDOptimizer(lr=.1)
optimizer = autodiff_optim.OptimizerIterator([x_node, yt_node],
cross_entropy,
optimizer_func)
log_at_info()
losses = []
x_train, y_train, x_val, y_val, x_test, y_test = mn.load_dataset(
flatten=True)
iter_count = 1
predictor = node.make_evaluator([x_node, yt_node], linear3)
total_time = time.time()
ctx = node.ComputeContext({})
for epoch in range(epochs):
epoch_time = time.time()
for x, y in iterate_over_minibatches(x_train,
y_train,
batch_size=batch_size):
ctx['x'], ctx['yt'] = x.T, to_one_hot(y, max_cat_num=9)
iter_loss = optimizer.step(ctx, 1.0) / batch_size
losses.append(iter_loss)
iter_count += 1
epoch_time = time.time() - epoch_time
loss_av = np.array(losses[:-batch_size + 1])
loss_av = np.mean(loss_av)
ctx['x'], ctx['yt'] = x_val.T, to_one_hot(y_val, max_cat_num=9)
y_predicted = predictor(ctx)
arg_max = np.argmax(y_predicted, axis=0)
correct = arg_max == y_val
percent = np.mean(correct) * 100
info("Epoch {:2d}:: Validation "
"accuracy:[{:5.2f}%] loss av={:01.8f}, time:{:2.3f}s".format(
epoch, percent, loss_av, epoch_time))
self.assertTrue(percent > 95)
total_time = time.time() - total_time
info("[Mnist784DsTest.test_linear_training()] total_time = {:5.3f} s".
format(total_time))
def test_multi_layer(self):
r"""
This actually performs better with SGD and normal initialization.
Gets almost 99% with SGD and normal initialization
:return:
"""
iris = Iris()
x_node = node.VarNode('x')
yt_node = node.VarNode('yt')
dense = node.DenseLayer(x_node, 16)
tanh = act.TanhNode(dense)
dense2 = node.DenseLayer(tanh, 10)
relu = act.RelUNode(dense2)
dense3 = node.DenseLayer(relu, 3)
softmax = act.Softmax(dense3)
cross_entropy = loss.CrossEntropy(softmax, yt_node)
#optimizer_func = core.np.Optimization.AdamOptimizer()
optimizer_func = core.np.Optimization.SGDOptimizer(lr=0.01)
optimizer = core.np.Optimization.OptimizerIterator([x_node, yt_node],
cross_entropy,
optimizer_func)
log_at_info()
epoch = 0
epochs = 10000
batch_size = 8
ctx = node.ComputeContext(weight_initializer=None)
for x, y in iris.train_iterator(epochs, batch_size):
ctx['x'], ctx['yt'] = x, y
loss_now = optimizer.step(ctx, 1.0) / batch_size
if epoch % 500 == 0:
info("[{}]\tloss_now = {}".format(epoch, loss_now))
epoch += 1
f = node.make_evaluator([x_node, yt_node], softmax)
total, correct = 100, 0
for x, y_actual in iris.test_iterator(total, one_hot=False):
var_map = {'x': x, 'yt': y_actual}
y_predicted = f(var_map)
max_idx = np.argmax(y_predicted)
mark = 'x'
if max_idx == y_actual:
correct += 1
mark = u'\u2713'
print("X:{}, y_pred:{}, Actual={}, Predicted:{} {}".format(
x.T, y_predicted.T, y_actual[0], max_idx, mark))
percent = correct * 100 / total
print("Correct= {}%".format(percent))
self.assertTrue(percent > 95)
def test_linear_training(self):
r"""
For fastest results, use batch size of 64, adam optimizer
and 3 epochs. You should get more than 97% accuracy
:return:
"""
# Build the network
x_node = node.VarNode('x')
yt_node = node.VarNode('yt')
linear1 = node.DenseLayer(x_node,
100,
name="Dense-First",
weight_scale=0.01)
relu1 = act.RelUNode(linear1, name="RelU-First")
linear2 = node.DenseLayer(relu1,
200,
name="Dense-Second",
weight_scale=0.01)
relu2 = act.RelUNode(linear2, name="RelU-Second")
linear3 = node.DenseLayer(relu2,
10,
name="Dense-Third",
weight_scale=0.01)
cross_entropy = loss.LogitsCrossEntropy(linear3, yt_node, name="XEnt")
# Set up optimizers and params
batch_size = 64
epochs = 3
optimizer_func = autodiff_optim.AdamOptimizer()
# optimizer_func = autodiff_optim.SGDOptimizer(lr=.1)
optimizer = autodiff_optim.OptimizerIterator([x_node, yt_node],
cross_entropy,
optimizer_func)
log_at_info()
losses = []
iter_count = 1
predictor = node.make_evaluator([x_node, yt_node], linear3)
ctx = node.ComputeContext({})
mnist = Mnist784()
total_time = time.time()
for epoch in range(epochs):
epoch_time = time.time()
iter = 0
for x, y in mnist.train_iterator_seq(batch_size=batch_size):
ctx['x'], ctx['yt'] = x, y
iter_loss = optimizer.step(ctx, 1.0) / batch_size
losses.append(iter_loss)
iter += 1
if iter % 100 == 0:
print("iter:{}".format(iter))
loss_av = np.array(losses[:-batch_size + 1])
loss_av = np.mean(loss_av)
e, xv, yv = mnist.test_iterator(1, batch_size=-1, one_hot=False)
ctx['x'], ctx['yt'] = xv, yv
percent = self.measure_validation_perf(predictor, ctx, yv)
epoch_time = time.time() - epoch_time
info("Iter {:2d}:: Val:{:2.4f}% , loss av={:01.8f}, time:{:2.3f}s".
format(epoch, percent, loss_av, epoch_time))
total_time = time.time() - total_time
info("Total time taken:{:4.4f}".format(total_time))