def test_backprop_large_network_both_inputs(self): X = np.array([[0.32, 0.68], [0.83, 0.02]]) Y = np.array([[0.75, 0.98], [0.75, 0.28]]) expected_gradients_theta_1 = np.array([[0.00804, 0.02564, 0.04987], [0.00666, 0.01837, 0.06719], [0.00973, 0.03196, 0.05252], [0.00776, 0.05037, 0.08492]]) expected_gradients_theta_2 = np.array( [[0.01071, 0.09068, 0.02512, 0.12597, 0.11586], [0.02442, 0.06780, 0.04164, 0.05308, 0.12677], [0.03056, 0.08924, 0.12094, 0.10270, 0.03078]]) expected_gradients_theta_3 = np.array( [[0.08135, 0.17935, 0.12476, 0.13186], [0.20982, 0.19195, 0.30434, 0.25249]]) nnet = NetworkBuilder.build_network_from_input_files( NETWORK_DEFINITION_TEST_FILE_2, WEIGHTS_DEFINITION_TEST_FILE_2) nnet.forward(X) nnet.backprop(Y) self.assertTrue( np.isclose(nnet.layers[-1].grad_mem, expected_gradients_theta_3, atol=TOLERANCE).all()) self.assertTrue( np.isclose(nnet.layers[-2].grad_mem, expected_gradients_theta_2, atol=TOLERANCE).all()) self.assertTrue( np.isclose(nnet.layers[-3].grad_mem, expected_gradients_theta_1, atol=TOLERANCE).all())
def test_backprop_small_network_first_input(self): X = np.array([[0.13]]) Y = np.array([[0.9]]) expected_delta_3 = np.array([[-0.10597]]) expected_delta_2 = np.array([[-0.01270, -0.01548]]) expected_gradients_theta_2 = np.array([[-0.1057, -0.06378, -0.06155]]) expected_gradients_theta_1 = np.array([[-0.01270, -0.00165], [-0.01548, -0.00201]]) nnet = NetworkBuilder.build_network_from_input_files( NETWORK_DEFINITION_TEST_FILE_1, WEIGHTS_DEFINITION_TEST_FILE_1) nnet.forward(X) nnet.backprop(Y) print(nnet.layers[-1].grad_mem) self.assertTrue( np.isclose(nnet.layers[-1].d_mem, expected_delta_3, atol=TOLERANCE).all()) self.assertTrue( np.isclose(nnet.layers[-1].grad_mem, expected_gradients_theta_2, atol=TOLERANCE).all()) self.assertTrue( np.isclose(nnet.layers[-2].d_mem, expected_delta_2, atol=TOLERANCE).all()) self.assertTrue( np.isclose(nnet.layers[-2].grad_mem, expected_gradients_theta_1, atol=TOLERANCE).all())
def test_layer_weights_parser(self): expected_input_weights = [[[0.4, 0.1], [0.3, 0.2]], [[0.7, 0.5, 0.6]]] layer_sizes = [1, 2, 1] actual_input_weights = NetworkBuilder.parse_weights_file(layer_sizes, WEIGHTS_DEFINITION_TEST_FILE_1) self.assertEqual(actual_input_weights, expected_input_weights)
def test_backprop_small_network_second_input(self): X = np.array([[0.42]]) Y = np.array([[0.23]]) expected_delta_3 = np.array([[0.56597]]) expected_delta_2 = np.array([[0.06740, 0.08184]]) expected_gradients_theta_2 = np.array([[0.56597, 0.34452, 0.33666]]) expected_gradients_theta_1 = np.array([[0.06740, 0.02831], [0.08184, 0.03437]]) nnet = NetworkBuilder.build_network_from_input_files( NETWORK_DEFINITION_TEST_FILE_1, WEIGHTS_DEFINITION_TEST_FILE_1) nnet.forward(X) nnet.backprop(Y) self.assertTrue( np.isclose(nnet.layers[-1].d_mem, expected_delta_3, atol=TOLERANCE).all()) self.assertTrue( np.isclose(nnet.layers[-1].grad_mem, expected_gradients_theta_2, atol=TOLERANCE).all()) self.assertTrue( np.isclose(nnet.layers[-2].d_mem, expected_delta_2, atol=TOLERANCE).all()) self.assertTrue( np.isclose(nnet.layers[-2].grad_mem, expected_gradients_theta_1, atol=TOLERANCE).all())
def test_layer_size_parser(self): expected_lambda = 0 expected_sizes = [1, 2, 1] actual_lambda, actual_sizes = NetworkBuilder.parse_network_file(NETWORK_DEFINITION_TEST_FILE_1) self.assertEqual(actual_lambda, expected_lambda) self.assertEqual(actual_sizes, expected_sizes)
def test_cost_function_calculation_second_example_with_regularization( self): network_output = np.array([[0.83318, 0.84132], [0.82953, 0.83832]]) y = np.array([[0.75, 0.98], [0.75, 0.28]]) expected_loss = 1.90351 nnet = NetworkBuilder.build_network_from_input_files( NETWORK_DEFINITION_TEST_FILE_2, WEIGHTS_DEFINITION_TEST_FILE_2) actual_loss = nnet.loss(network_output, y) self.assertLess(np.abs(actual_loss - expected_loss), TOLERANCE)
def test_build_network_from_input_files(self): expected_size_first_layer = 1 expected_input_weights_first_layer = np.array([[0.4, 0.1], [0.3, 0.2]]) expected_size_second_layer = 2 expected_input_weights_second_layer = np.array([[0.7, 0.5, 0.6]]) nnet = NetworkBuilder.build_network_from_input_files(NETWORK_DEFINITION_TEST_FILE_1, WEIGHTS_DEFINITION_TEST_FILE_1) self.assertEqual(len(nnet.layers), 2) self.assertEqual(nnet.layers[0].size, expected_size_first_layer) self.assertTrue((nnet.layers[0].W == expected_input_weights_first_layer).all()) self.assertEqual(nnet.layers[1].size, expected_size_second_layer) self.assertTrue((nnet.layers[1].W == expected_input_weights_second_layer).all())
def test_backprop_compared_to_gradient_checking_large(self): epsilon = 0.000001 X = np.array([[0.32, 0.68], [0.83, 0.02]]) Y = np.array([[0.75, 0.98], [0.75, 0.28]]) nnet = NetworkBuilder.build_network_from_input_files( NETWORK_DEFINITION_TEST_FILE_2, WEIGHTS_DEFINITION_TEST_FILE_2) grad_check = nnet.verify_gradient(X, Y, epsilon) nnet.forward(X) nnet.backprop(Y) self.assertLess(np.mean(nnet.layers[0].grad_mem - grad_check[0]), 10e-8) self.assertLess(np.mean(nnet.layers[1].grad_mem - grad_check[1]), 10e-8)
def test_forward_activation_small_network_first_input(self): x1 = np.array([[0.13]]) expected_z2 = np.array([0.413, 0.326]) expected_a2 = np.array([0.60181, 0.58079]) expected_z3 = np.array([1.34937]) expected_out = np.array([0.79403]) nnet = NetworkBuilder.build_network_from_input_files( NETWORK_DEFINITION_TEST_FILE_1, WEIGHTS_DEFINITION_TEST_FILE_1) output = nnet.forward(x1) self.assertTrue(np.isclose(nnet.layers[0].z_mem, expected_z2).all()) self.assertTrue(np.isclose(nnet.layers[0].a_mem, expected_a2).all()) self.assertTrue(np.isclose(nnet.layers[1].z_mem, expected_z3).all()) self.assertTrue(np.isclose(output, expected_out).all())
def test_build_network_from_input_files_large_network(self): expected_size_first_layer = 2 expected_input_weights_first_layer = np.array([[0.42, 0.15, 0.4], [0.72, 0.1, 0.54], [0.01, 0.19, 0.42], [0.3, 0.35, 0.68]]) expected_size_second_layer = 4 expected_input_weights_second_layer = np.array([[0.21, 0.67, 0.14, 0.96, 0.87], [0.87, 0.42, 0.2, 0.32, 0.89], [0.03, 0.56, 0.8, 0.69, 0.09]]) expected_size_third_layer = 3 expected_input_weights_third_layer = np.array([[0.04, 0.87, 0.42, 0.53], [0.17, 0.1, 0.95, 0.69]]) nnet = NetworkBuilder.build_network_from_input_files(NETWORK_DEFINITION_TEST_FILE_2, WEIGHTS_DEFINITION_TEST_FILE_2) self.assertEqual(len(nnet.layers), 3) self.assertEqual(nnet.layers[0].size, expected_size_first_layer) self.assertTrue((nnet.layers[0].W == expected_input_weights_first_layer).all()) self.assertEqual(nnet.layers[1].size, expected_size_second_layer) self.assertTrue((nnet.layers[1].W == expected_input_weights_second_layer).all()) self.assertEqual(nnet.layers[2].size, expected_size_third_layer) self.assertTrue((nnet.layers[2].W == expected_input_weights_third_layer).all())
def test_numerical_gradient_validation_small_network(self): epsilon = 0.000001 X = np.array([[0.13], [0.42]]) Y = np.array([[0.9], [0.23]]) expected_gradient_theta_1 = np.array([[0.02735, 0.01333], [0.03318, 0.01618]]) expected_gradient_theta_2 = np.array([[0.23, 0.14037, 0.13756]]) nnet = NetworkBuilder.build_network_from_input_files( NETWORK_DEFINITION_TEST_FILE_1, WEIGHTS_DEFINITION_TEST_FILE_1) actual_grads = nnet.verify_gradient(X, Y, epsilon) self.assertTrue( np.isclose(actual_grads[0], expected_gradient_theta_1, atol=TOLERANCE).all()) self.assertTrue( np.isclose(actual_grads[1], expected_gradient_theta_2, atol=TOLERANCE).all())
def test_forward_activation_large_network_second_input(self): x1 = np.array([[0.83, 0.02]]) expected_z2 = np.array([0.5525, 0.81380, 0.17610, 0.60410]) expected_a2 = np.array([0.63472, 0.69292, 0.54391, 0.64659]) expected_z3 = np.array([1.81696, 2.02468, 1.37327]) expected_a3 = np.array([0.86020, 0.88336, 0.79791]) expected_z4 = np.array([1.58228, 1.64577]) expected_out = np.array([0.82953, 0.83832]) nnet = NetworkBuilder.build_network_from_input_files( NETWORK_DEFINITION_TEST_FILE_2, WEIGHTS_DEFINITION_TEST_FILE_2) output = nnet.forward(x1) self.assertTrue(np.isclose(nnet.layers[0].z_mem, expected_z2).all()) self.assertTrue(np.isclose(nnet.layers[0].a_mem, expected_a2).all()) self.assertTrue(np.isclose(nnet.layers[1].z_mem, expected_z3).all()) self.assertTrue(np.isclose(nnet.layers[1].a_mem, expected_a3).all()) self.assertTrue(np.isclose(nnet.layers[2].z_mem, expected_z4).all()) self.assertTrue(np.isclose(output, expected_out).all())
def test_forward_activation_large_network_first_input(self): x1 = np.array([[0.32, 0.68]]) expected_z2 = np.array([0.74, 1.1192, 0.35640, 0.87440]) expected_a2 = np.array([0.677, 0.75384, 0.58817, 0.70566]) expected_z3 = np.array([1.94769, 2.12136, 1.48154]) expected_a3 = np.array([0.87519, 0.89296, 0.81480]) expected_z4 = np.array([1.60831, 1.66805]) expected_out = np.array([0.83318, 0.84132]) nnet = NetworkBuilder.build_network_from_input_files( NETWORK_DEFINITION_TEST_FILE_2, WEIGHTS_DEFINITION_TEST_FILE_2) output = nnet.forward(x1) self.assertTrue(np.isclose(nnet.layers[0].z_mem, expected_z2).all()) self.assertTrue(np.isclose(nnet.layers[0].a_mem, expected_a2).all()) self.assertTrue(np.isclose(nnet.layers[1].z_mem, expected_z3).all()) self.assertTrue(np.isclose(nnet.layers[1].a_mem, expected_a3).all()) self.assertTrue(np.isclose(nnet.layers[2].z_mem, expected_z4).all()) self.assertTrue(np.isclose(output, expected_out).all())