def test_train(self):
num_iter = 100000
x_node = n.VarNode('x')
y_target_node = n.VarNode('y_target')
rnn_node = rnn.SimpleRnnLayer(x_node, self.name_ds.n_categories, 15)
loss_node = loss.SoftmaxCrossEntropy(rnn_node, y_target_node)
all_losses = []
optimizer_func = autodiff_optim.AdamOptimizer()
optimizer_func = autodiff_optim.SGDOptimizer(lr=0.0001)
optimizer = autodiff_optim.OptimizerIterator([x_node, y_target_node],
loss_node, optimizer_func)
ctx = n.ComputeContext({'x': "", 'y_target': ""})
log_at_info()
every = 500
t = time.time()
for i in range(1, num_iter + 1):
rnn_node.set_initial_state_to_zero()
c, l, category_index, name_tensor = self.name_ds.random_training_example(
)
cat_tensor = self.name_ds.category_idx_to_tensor([category_index])
ctx['x'] = name_tensor
ctx['y_target'] = cat_tensor
ctx['i'] = i
loss_value = optimizer.step(ctx, 1.0)
all_losses.append(loss_value)
if i % every == 0:
t = time.time() - t
last_10 = all_losses[-every:]
av = np.average(last_10)
info("[{:06d}] Avg. loss = {:10.6f}"
" | {:04.2f}s per {} | Total Iters set to:{}".format(
i, av, t, every, num_iter))
all_losses = []
t = time.time()
def test_rnn_layer(self):
x = np.array([[1, 2, 1], [-1, 0, -.5]]).T
x = x.reshape((3, 1, 2))
input_node = n.VarNode('x')
var_map = {'x': x}
rnn_layer = rnn.SimpleRnnLayer(input_node, 4, 2)
input_node.forward(var_map)
y = rnn_layer.value()
dely = y * .1
rnn_layer.backward(dely, self, var_map)
x_grad = input_node.total_incoming_gradient()
debug("[SimpleRnnCellTests.test_rnn_layer()] x_grad = np.{}".format(
repr(x_grad)))
def test_rnn_layer_with_loss(self):
debug(
"[RnnLayerFullTests.test_rnn_layer_with_loss()] self.data_dir = {}"
.format(self.data_dir))
x = self.name_ds.line_to_numpy('ABCD')
debug("[RnnLayerFullTests.test_rnn_layer_with_loss()] ABCD: x = np.{}".
format(repr(x)))
debug("------------------------------------------------------")
x = self.name_ds.line_to_numpy('Albert')
debug(
"[RnnLayerFullTests.test_rnn_layer_with_loss()] x = np.{}".format(
repr(x)))
debug("------------------------------------------------------")
log_at_info()
for i in range(5):
c, l, category_index, name_tensor = self.name_ds.random_training_example(
)
debug("[{}]:{}".format(c, l))
cat_tensor = self.name_ds.category_idx_to_tensor([category_index])
debug(
"[RnnLayerFullTests.test_rnn_layer_with_loss()] cat_tensor = np.{}"
.format(repr(cat_tensor)))
x_node = n.VarNode('x')
y_target_node = n.VarNode('y_target')
ctx = n.ComputeContext({'x': name_tensor, 'y_target': cat_tensor})
rnn_node = rnn.SimpleRnnLayer(x_node, self.name_ds.n_categories, 128)
loss_node = loss.LogitsCrossEntropy(rnn_node, y_target_node)
x_node.forward(ctx)
y_target_node.forward(ctx)
y = rnn_node.value()
info(
"[RnnLayerFullTests.test_rnn_layer_with_loss()] y = np.{}".format(
repr(y)))
loss_value = loss_node.value()
info("[RnnLayerFullTests.test_rnn_layer_with_loss()] loss = np.{}".
format(repr(loss_value)))
loss_node.backward(1.0, self, ctx)
grads = rnn_node.total_incoming_gradient()
info(grads)