def make__two_layer_model(): r""" Designed to be used only with TestFullNetwork class.. variable names etc. are used in debugging in the testing code """ w_node = node.VarNode('w', True) x_node = node.VarNode('x') ya_node = node.VarNode('y_a') b_node = node.VarNode('b', True) w2_node = node.VarNode('w2', True) b2_node = node.VarNode('b2', True) start_nodes = [w_node, x_node, b_node, ya_node, w2_node, b2_node] w = np.array([[1, 3], [0, 1]]) x = (np.array([[1, -1, 2]])).T b = np.array([[-2, -3]]).T y_act = np.array([[.5, .7]]).T w2 = np.array([[.1, .2], [.3, .07]]) b2 = np.array([[.02, .3]]).T var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b, 'w2': w2, 'b2': b2} wx_node = node.MatrixMultiplication(w_node, x_node, "wx") sum_node = node.MatrixAddition(wx_node, b_node, "wx+b") sigmoid_node = SigmoidNode(sum_node, "sig") wx2_node = node.MatrixMultiplication(w2_node, sigmoid_node, "wx2") sum2_node = node.MatrixAddition(wx2_node, b2_node, "wx2+b2") l2_node = L2DistanceSquaredNorm(sum2_node, ya_node, "l2") return var_map, start_nodes, l2_node
def test_basic(self): w_node = node.VarNode('w') x_node = node.VarNode('x') w = np.array([[1, 3, 0], [0, 1, -1]]) x = np.array([[1, -1], [0, 2], [9, 1]]) mult_node = node.MatrixMultiplication(w_node, x_node) var_map = {'x': x, 'w': w} x_node.forward(var_map) self.assertIsNone(mult_node.value()) w_node.forward(var_map) value = mult_node.value() expected = w @ x np.testing.assert_array_almost_equal(expected, value) print(value) self.assertIsNotNone(x_node.value()) mult_node.reset_network_fwd() # Just checking # Not none because fwd should start from start vars self.assertIsNotNone(x_node.value()) b_node = node.VarNode('b') b = np.array([-1, -1]) var_map['b'] = b sum_node = node.MatrixAddition(mult_node, b_node) var_nodes = [x_node, w_node, b_node] for var_node in var_nodes: var_node.forward(var_map) expected = expected + b np.testing.assert_array_almost_equal(expected, sum_node.value()) print(sum_node.value())
def test_matrix_prd(self): w_node = node.VarNode('w') x_node = node.VarNode('x') w = np.array([[1, 3, 0], [0, 1, -1]]) x = (np.array([[1, -1, 2]])).T w_grad_expected = np.array([x[:, 0], x[:, 0]]) local_grad = np.array([[1, 1]]).T x_grad_expected = np.multiply(w, local_grad).sum(axis=0).T x_grad_expected = np.reshape(x_grad_expected, (len(x_grad_expected), 1)) mult_node = node.MatrixMultiplication(w_node, x_node, name="wx") var_map = {'x': x, 'w': w} x_node.forward(var_map) self.assertIsNone(mult_node.value()) w_node.forward(var_map) value = mult_node.value() expected = w @ x np.testing.assert_array_almost_equal(expected, value) mult_node.backward(local_grad, self, var_map) w_grad = w_node.total_incoming_gradient() print("---- printing w_grad ---") print(w_grad) np.testing.assert_array_almost_equal(w_grad, w_grad_expected) print("---- end printing ----") x_grad = x_node.total_incoming_gradient() print("---- printing x_grad ---") print(x_grad) np.testing.assert_array_almost_equal(x_grad_expected, x_grad) print("---- end printing ----")
def test_full_sgmoid_node(self): w_node = node.VarNode('w', True) x_node = node.VarNode('x') ya_node = node.VarNode('y_a') b_node = node.VarNode('b', True) start_nodes = [w_node, x_node, b_node, ya_node] w = np.array([[1, 3, 0], [0, 1, -1]]) x = (np.array([[1, -1, 2]])).T b = np.array([[-2, -3]]).T y_act = np.array([[.5, .7]]).T var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b} wx_node = node.MatrixMultiplication(w_node, x_node) sum_node = node.MatrixAddition(wx_node, b_node) sigmoid_node = SigmoidNode(sum_node) l2_node = L2DistanceSquaredNorm(sigmoid_node, ya_node) optim_func = self.rate_adjustable_optimizer_func(0.01) optimizer = core.np.Optimization.OptimizerIterator( start_nodes, l2_node, optim_func) log_at_info() losses = [] for i in range(100): loss = optimizer.step(var_map, 1.0) losses.append(loss) if i % 10 == 0: print("[{}] Loss:{}".format(i, loss)) print("Final loss:{}".format(loss)) print("w:{}".format(var_map['w'])) print("b:{}".format(var_map['b']))
def test_linear_transformation(self): np.random.seed(100) w_node = node.VarNode('w') x_node = node.VarNode('x') ya_node = node.VarNode('y_a') w = np.array([[1, 2, 1], [2, 0, -1]]) x = np.array( [[0.54340494, 0.27836939, 0.42451759, 0.84477613, 0.00471886], [0.12156912, 0.67074908, 0.82585276, 0.13670659, 0.57509333], [0.89132195, 0.20920212, 0.18532822, 0.10837689, 0.21969749]]) y_act = np.array( [[0.97862378, 0.81168315, 0.17194101, 0.81622475, 0.27407375], [0.43170418, 0.94002982, 0.81764938, 0.33611195, 0.17541045]]) info("Printing x...") info(x) info("printing y_pred") info(y_act) var_map = {'w': w, 'x': x, 'y_a': y_act} wx_node = node.MatrixMultiplication(w_node, x_node) l2_node = L2DistanceSquaredNorm(wx_node, ya_node) log_at_info() w_node.forward(var_map) x_node.forward(var_map) ya_node.forward(var_map) l2_node.backward(1.0, self, var_map) info(wx_node.value()) info("grad...") info(wx_node.total_incoming_gradient())
def test_single_step(self): w_node = node.VarNode('w') x_node = node.VarNode('x') ya_node = node.VarNode('y_a') b_node = node.VarNode('b') w = np.array([[1, 3, 0], [0, 1, -1]]) x = (np.array([[1, -1, 2]])).T b = np.array([[-2, -3]]).T y_act = np.array([[1, 2]]).T var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b} wx_node = node.MatrixMultiplication(w_node, x_node) sum_node = node.MatrixAddition(wx_node, b_node) l2_node = L2DistanceSquaredNorm(sum_node, ya_node) w_node.forward(var_map) x_node.forward(var_map) b_node.forward(var_map) ya_node.forward(var_map) l2norm = l2_node.value() info("L2Norm: {}".format(l2norm)) y_p = w @ x + b y_del = y_p - y_act expected = np.sum(np.square(y_del)) / y_del.size debug("l2norm:{}".format(l2_node)) self.assertEqual(expected, l2norm) l2_node.backward(1.0, self, var_map) w_grad = w_node.total_incoming_gradient() b_grad = b_node.total_incoming_gradient() debug("----- w grad ------") debug(w_grad) debug("-------------------") debug("----- b grad ------") debug(b_grad) debug("-------------------") l2_node.reset_network_back() self.assertIsNone(w_node.total_incoming_gradient()) l2_node.backward(1.0, self, var_map) w_grad = w_node.total_incoming_gradient() b_grad = b_node.total_incoming_gradient() debug("----- w grad ------") debug(w_grad) debug("-------------------") debug("----- b grad ------") debug(b_grad) debug("-------------------")
def test_network_optimizer(self): w_node = node.VarNode('w', True) x_node = node.VarNode('x') ya_node = node.VarNode('y_a') b_node = node.VarNode('b', True) start_nodes = [w_node, x_node, b_node, ya_node] w = np.array([[1, 3, 0], [0, 1, -1]]) x = (np.array([[1, -1, 2]])).T b = np.array([[-2, -3]]).T y_act = np.array([[1, 2]]).T var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b} wx_node = node.MatrixMultiplication(w_node, x_node) sum_node = node.MatrixAddition(wx_node, b_node) l2_node = L2DistanceSquaredNorm(sum_node, ya_node) optimizer = optim.OptimizerIterator(start_nodes, l2_node) log_at_info() for _ in range(500): loss = optimizer.step(var_map, 1.0) info("Final loss:{}".format(loss)) self.assertTrue(math.fabs(loss) < 1e-25)