def make__two_layer_model(): r""" Designed to be used only with TestFullNetwork class.. variable names etc. are used in debugging in the testing code """ w_node = node.VarNode('w', True) x_node = node.VarNode('x') ya_node = node.VarNode('y_a') b_node = node.VarNode('b', True) w2_node = node.VarNode('w2', True) b2_node = node.VarNode('b2', True) start_nodes = [w_node, x_node, b_node, ya_node, w2_node, b2_node] w = np.array([[1, 3], [0, 1]]) x = (np.array([[1, -1, 2]])).T b = np.array([[-2, -3]]).T y_act = np.array([[.5, .7]]).T w2 = np.array([[.1, .2], [.3, .07]]) b2 = np.array([[.02, .3]]).T var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b, 'w2': w2, 'b2': b2} wx_node = node.MatrixMultiplication(w_node, x_node, "wx") sum_node = node.MatrixAddition(wx_node, b_node, "wx+b") sigmoid_node = SigmoidNode(sum_node, "sig") wx2_node = node.MatrixMultiplication(w2_node, sigmoid_node, "wx2") sum2_node = node.MatrixAddition(wx2_node, b2_node, "wx2+b2") l2_node = L2DistanceSquaredNorm(sum2_node, ya_node, "l2") return var_map, start_nodes, l2_node
def test_basic(self): w_node = node.VarNode('w') x_node = node.VarNode('x') w = np.array([[1, 3, 0], [0, 1, -1]]) x = np.array([[1, -1], [0, 2], [9, 1]]) mult_node = node.MatrixMultiplication(w_node, x_node) var_map = {'x': x, 'w': w} x_node.forward(var_map) self.assertIsNone(mult_node.value()) w_node.forward(var_map) value = mult_node.value() expected = w @ x np.testing.assert_array_almost_equal(expected, value) print(value) self.assertIsNotNone(x_node.value()) mult_node.reset_network_fwd() # Just checking # Not none because fwd should start from start vars self.assertIsNotNone(x_node.value()) b_node = node.VarNode('b') b = np.array([-1, -1]) var_map['b'] = b sum_node = node.MatrixAddition(mult_node, b_node) var_nodes = [x_node, w_node, b_node] for var_node in var_nodes: var_node.forward(var_map) expected = expected + b np.testing.assert_array_almost_equal(expected, sum_node.value()) print(sum_node.value())
def test_full_sgmoid_node(self): w_node = node.VarNode('w', True) x_node = node.VarNode('x') ya_node = node.VarNode('y_a') b_node = node.VarNode('b', True) start_nodes = [w_node, x_node, b_node, ya_node] w = np.array([[1, 3, 0], [0, 1, -1]]) x = (np.array([[1, -1, 2]])).T b = np.array([[-2, -3]]).T y_act = np.array([[.5, .7]]).T var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b} wx_node = node.MatrixMultiplication(w_node, x_node) sum_node = node.MatrixAddition(wx_node, b_node) sigmoid_node = SigmoidNode(sum_node) l2_node = L2DistanceSquaredNorm(sigmoid_node, ya_node) optim_func = self.rate_adjustable_optimizer_func(0.01) optimizer = core.np.Optimization.OptimizerIterator( start_nodes, l2_node, optim_func) log_at_info() losses = [] for i in range(100): loss = optimizer.step(var_map, 1.0) losses.append(loss) if i % 10 == 0: print("[{}] Loss:{}".format(i, loss)) print("Final loss:{}".format(loss)) print("w:{}".format(var_map['w'])) print("b:{}".format(var_map['b']))
def test_single_step(self): w_node = node.VarNode('w') x_node = node.VarNode('x') ya_node = node.VarNode('y_a') b_node = node.VarNode('b') w = np.array([[1, 3, 0], [0, 1, -1]]) x = (np.array([[1, -1, 2]])).T b = np.array([[-2, -3]]).T y_act = np.array([[1, 2]]).T var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b} wx_node = node.MatrixMultiplication(w_node, x_node) sum_node = node.MatrixAddition(wx_node, b_node) l2_node = L2DistanceSquaredNorm(sum_node, ya_node) w_node.forward(var_map) x_node.forward(var_map) b_node.forward(var_map) ya_node.forward(var_map) l2norm = l2_node.value() info("L2Norm: {}".format(l2norm)) y_p = w @ x + b y_del = y_p - y_act expected = np.sum(np.square(y_del)) / y_del.size debug("l2norm:{}".format(l2_node)) self.assertEqual(expected, l2norm) l2_node.backward(1.0, self, var_map) w_grad = w_node.total_incoming_gradient() b_grad = b_node.total_incoming_gradient() debug("----- w grad ------") debug(w_grad) debug("-------------------") debug("----- b grad ------") debug(b_grad) debug("-------------------") l2_node.reset_network_back() self.assertIsNone(w_node.total_incoming_gradient()) l2_node.backward(1.0, self, var_map) w_grad = w_node.total_incoming_gradient() b_grad = b_node.total_incoming_gradient() debug("----- w grad ------") debug(w_grad) debug("-------------------") debug("----- b grad ------") debug(b_grad) debug("-------------------")
def test_network_optimizer(self): w_node = node.VarNode('w', True) x_node = node.VarNode('x') ya_node = node.VarNode('y_a') b_node = node.VarNode('b', True) start_nodes = [w_node, x_node, b_node, ya_node] w = np.array([[1, 3, 0], [0, 1, -1]]) x = (np.array([[1, -1, 2]])).T b = np.array([[-2, -3]]).T y_act = np.array([[1, 2]]).T var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b} wx_node = node.MatrixMultiplication(w_node, x_node) sum_node = node.MatrixAddition(wx_node, b_node) l2_node = L2DistanceSquaredNorm(sum_node, ya_node) optimizer = optim.OptimizerIterator(start_nodes, l2_node) log_at_info() for _ in range(500): loss = optimizer.step(var_map, 1.0) info("Final loss:{}".format(loss)) self.assertTrue(math.fabs(loss) < 1e-25)
def test_matrix_sum(self): a_node = node.VarNode('a') b_node = node.VarNode('b') a = np.array([[1, 2, 3], [-1, -3, 0]]) b = np.array([[0, 1, -1], [2, 0, 1]]) sum_node = node.MatrixAddition(a_node, b_node) var_map = {'a': a, 'b': b} a_node.forward(var_map) b_node.forward(var_map) matrix_sum = sum_node.value() expected_sum = a + b print(matrix_sum) np.testing.assert_array_almost_equal(expected_sum, matrix_sum) start_grad = np.ones_like(a) sum_node.backward(start_grad, self, var_map) grad_at_a = a_node.total_incoming_gradient() grad_at_b = b_node.total_incoming_gradient() print(grad_at_a) print("-------------") print(grad_at_b) np.testing.assert_array_almost_equal(grad_at_a, start_grad) np.testing.assert_array_almost_equal(grad_at_b, start_grad)