def make__two_layer_model():
r"""
Designed to be used only with TestFullNetwork class.. variable names etc. are
used in debugging in the testing code
"""
w_node = node.VarNode('w', True)
x_node = node.VarNode('x')
ya_node = node.VarNode('y_a')
b_node = node.VarNode('b', True)
w2_node = node.VarNode('w2', True)
b2_node = node.VarNode('b2', True)
start_nodes = [w_node, x_node, b_node, ya_node, w2_node, b2_node]
w = np.array([[1, 3], [0, 1]])
x = (np.array([[1, -1, 2]])).T
b = np.array([[-2, -3]]).T
y_act = np.array([[.5, .7]]).T
w2 = np.array([[.1, .2], [.3, .07]])
b2 = np.array([[.02, .3]]).T
var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b, 'w2': w2, 'b2': b2}
wx_node = node.MatrixMultiplication(w_node, x_node, "wx")
sum_node = node.MatrixAddition(wx_node, b_node, "wx+b")
sigmoid_node = SigmoidNode(sum_node, "sig")
wx2_node = node.MatrixMultiplication(w2_node, sigmoid_node, "wx2")
sum2_node = node.MatrixAddition(wx2_node, b2_node, "wx2+b2")
l2_node = L2DistanceSquaredNorm(sum2_node, ya_node, "l2")
return var_map, start_nodes, l2_node
def test_basic(self):
w_node = node.VarNode('w')
x_node = node.VarNode('x')
w = np.array([[1, 3, 0], [0, 1, -1]])
x = np.array([[1, -1], [0, 2], [9, 1]])
mult_node = node.MatrixMultiplication(w_node, x_node)
var_map = {'x': x, 'w': w}
x_node.forward(var_map)
self.assertIsNone(mult_node.value())
w_node.forward(var_map)
value = mult_node.value()
expected = w @ x
np.testing.assert_array_almost_equal(expected, value)
print(value)
self.assertIsNotNone(x_node.value())
mult_node.reset_network_fwd()
# Just checking
# Not none because fwd should start from start vars
self.assertIsNotNone(x_node.value())
b_node = node.VarNode('b')
b = np.array([-1, -1])
var_map['b'] = b
sum_node = node.MatrixAddition(mult_node, b_node)
var_nodes = [x_node, w_node, b_node]
for var_node in var_nodes:
var_node.forward(var_map)
expected = expected + b
np.testing.assert_array_almost_equal(expected, sum_node.value())
print(sum_node.value())
def test_full_sgmoid_node(self):
w_node = node.VarNode('w', True)
x_node = node.VarNode('x')
ya_node = node.VarNode('y_a')
b_node = node.VarNode('b', True)
start_nodes = [w_node, x_node, b_node, ya_node]
w = np.array([[1, 3, 0], [0, 1, -1]])
x = (np.array([[1, -1, 2]])).T
b = np.array([[-2, -3]]).T
y_act = np.array([[.5, .7]]).T
var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b}
wx_node = node.MatrixMultiplication(w_node, x_node)
sum_node = node.MatrixAddition(wx_node, b_node)
sigmoid_node = SigmoidNode(sum_node)
l2_node = L2DistanceSquaredNorm(sigmoid_node, ya_node)
optim_func = self.rate_adjustable_optimizer_func(0.01)
optimizer = core.np.Optimization.OptimizerIterator(
start_nodes, l2_node, optim_func)
log_at_info()
losses = []
for i in range(100):
loss = optimizer.step(var_map, 1.0)
losses.append(loss)
if i % 10 == 0:
print("[{}] Loss:{}".format(i, loss))
print("Final loss:{}".format(loss))
print("w:{}".format(var_map['w']))
print("b:{}".format(var_map['b']))
def test_single_step(self):
w_node = node.VarNode('w')
x_node = node.VarNode('x')
ya_node = node.VarNode('y_a')
b_node = node.VarNode('b')
w = np.array([[1, 3, 0], [0, 1, -1]])
x = (np.array([[1, -1, 2]])).T
b = np.array([[-2, -3]]).T
y_act = np.array([[1, 2]]).T
var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b}
wx_node = node.MatrixMultiplication(w_node, x_node)
sum_node = node.MatrixAddition(wx_node, b_node)
l2_node = L2DistanceSquaredNorm(sum_node, ya_node)
w_node.forward(var_map)
x_node.forward(var_map)
b_node.forward(var_map)
ya_node.forward(var_map)
l2norm = l2_node.value()
info("L2Norm: {}".format(l2norm))
y_p = w @ x + b
y_del = y_p - y_act
expected = np.sum(np.square(y_del)) / y_del.size
debug("l2norm:{}".format(l2_node))
self.assertEqual(expected, l2norm)
l2_node.backward(1.0, self, var_map)
w_grad = w_node.total_incoming_gradient()
b_grad = b_node.total_incoming_gradient()
debug("----- w grad ------")
debug(w_grad)
debug("-------------------")
debug("----- b grad ------")
debug(b_grad)
debug("-------------------")
l2_node.reset_network_back()
self.assertIsNone(w_node.total_incoming_gradient())
l2_node.backward(1.0, self, var_map)
w_grad = w_node.total_incoming_gradient()
b_grad = b_node.total_incoming_gradient()
debug("----- w grad ------")
debug(w_grad)
debug("-------------------")
debug("----- b grad ------")
debug(b_grad)
debug("-------------------")
def test_network_optimizer(self):
w_node = node.VarNode('w', True)
x_node = node.VarNode('x')
ya_node = node.VarNode('y_a')
b_node = node.VarNode('b', True)
start_nodes = [w_node, x_node, b_node, ya_node]
w = np.array([[1, 3, 0], [0, 1, -1]])
x = (np.array([[1, -1, 2]])).T
b = np.array([[-2, -3]]).T
y_act = np.array([[1, 2]]).T
var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b}
wx_node = node.MatrixMultiplication(w_node, x_node)
sum_node = node.MatrixAddition(wx_node, b_node)
l2_node = L2DistanceSquaredNorm(sum_node, ya_node)
optimizer = optim.OptimizerIterator(start_nodes, l2_node)
log_at_info()
for _ in range(500):
loss = optimizer.step(var_map, 1.0)
info("Final loss:{}".format(loss))
self.assertTrue(math.fabs(loss) < 1e-25)
def test_matrix_sum(self):
a_node = node.VarNode('a')
b_node = node.VarNode('b')
a = np.array([[1, 2, 3], [-1, -3, 0]])
b = np.array([[0, 1, -1], [2, 0, 1]])
sum_node = node.MatrixAddition(a_node, b_node)
var_map = {'a': a, 'b': b}
a_node.forward(var_map)
b_node.forward(var_map)
matrix_sum = sum_node.value()
expected_sum = a + b
print(matrix_sum)
np.testing.assert_array_almost_equal(expected_sum, matrix_sum)
start_grad = np.ones_like(a)
sum_node.backward(start_grad, self, var_map)
grad_at_a = a_node.total_incoming_gradient()
grad_at_b = b_node.total_incoming_gradient()
print(grad_at_a)
print("-------------")
print(grad_at_b)
np.testing.assert_array_almost_equal(grad_at_a, start_grad)
np.testing.assert_array_almost_equal(grad_at_b, start_grad)