def test_backprop_large_network_both_inputs(self):
X = np.array([[0.32, 0.68], [0.83, 0.02]])
Y = np.array([[0.75, 0.98], [0.75, 0.28]])
expected_gradients_theta_1 = np.array([[0.00804, 0.02564, 0.04987],
[0.00666, 0.01837, 0.06719],
[0.00973, 0.03196, 0.05252],
[0.00776, 0.05037, 0.08492]])
expected_gradients_theta_2 = np.array(
[[0.01071, 0.09068, 0.02512, 0.12597, 0.11586],
[0.02442, 0.06780, 0.04164, 0.05308, 0.12677],
[0.03056, 0.08924, 0.12094, 0.10270, 0.03078]])
expected_gradients_theta_3 = np.array(
[[0.08135, 0.17935, 0.12476, 0.13186],
[0.20982, 0.19195, 0.30434, 0.25249]])
nnet = NetworkBuilder.build_network_from_input_files(
NETWORK_DEFINITION_TEST_FILE_2, WEIGHTS_DEFINITION_TEST_FILE_2)
nnet.forward(X)
nnet.backprop(Y)
self.assertTrue(
np.isclose(nnet.layers[-1].grad_mem,
expected_gradients_theta_3,
atol=TOLERANCE).all())
self.assertTrue(
np.isclose(nnet.layers[-2].grad_mem,
expected_gradients_theta_2,
atol=TOLERANCE).all())
self.assertTrue(
np.isclose(nnet.layers[-3].grad_mem,
expected_gradients_theta_1,
atol=TOLERANCE).all())
def test_backprop_small_network_first_input(self):
X = np.array([[0.13]])
Y = np.array([[0.9]])
expected_delta_3 = np.array([[-0.10597]])
expected_delta_2 = np.array([[-0.01270, -0.01548]])
expected_gradients_theta_2 = np.array([[-0.1057, -0.06378, -0.06155]])
expected_gradients_theta_1 = np.array([[-0.01270, -0.00165],
[-0.01548, -0.00201]])
nnet = NetworkBuilder.build_network_from_input_files(
NETWORK_DEFINITION_TEST_FILE_1, WEIGHTS_DEFINITION_TEST_FILE_1)
nnet.forward(X)
nnet.backprop(Y)
print(nnet.layers[-1].grad_mem)
self.assertTrue(
np.isclose(nnet.layers[-1].d_mem, expected_delta_3,
atol=TOLERANCE).all())
self.assertTrue(
np.isclose(nnet.layers[-1].grad_mem,
expected_gradients_theta_2,
atol=TOLERANCE).all())
self.assertTrue(
np.isclose(nnet.layers[-2].d_mem, expected_delta_2,
atol=TOLERANCE).all())
self.assertTrue(
np.isclose(nnet.layers[-2].grad_mem,
expected_gradients_theta_1,
atol=TOLERANCE).all())
def test_backprop_small_network_second_input(self):
X = np.array([[0.42]])
Y = np.array([[0.23]])
expected_delta_3 = np.array([[0.56597]])
expected_delta_2 = np.array([[0.06740, 0.08184]])
expected_gradients_theta_2 = np.array([[0.56597, 0.34452, 0.33666]])
expected_gradients_theta_1 = np.array([[0.06740, 0.02831],
[0.08184, 0.03437]])
nnet = NetworkBuilder.build_network_from_input_files(
NETWORK_DEFINITION_TEST_FILE_1, WEIGHTS_DEFINITION_TEST_FILE_1)
nnet.forward(X)
nnet.backprop(Y)
self.assertTrue(
np.isclose(nnet.layers[-1].d_mem, expected_delta_3,
atol=TOLERANCE).all())
self.assertTrue(
np.isclose(nnet.layers[-1].grad_mem,
expected_gradients_theta_2,
atol=TOLERANCE).all())
self.assertTrue(
np.isclose(nnet.layers[-2].d_mem, expected_delta_2,
atol=TOLERANCE).all())
self.assertTrue(
np.isclose(nnet.layers[-2].grad_mem,
expected_gradients_theta_1,
atol=TOLERANCE).all())
def test_cost_function_calculation_second_example_with_regularization(
self):
network_output = np.array([[0.83318, 0.84132], [0.82953, 0.83832]])
y = np.array([[0.75, 0.98], [0.75, 0.28]])
expected_loss = 1.90351
nnet = NetworkBuilder.build_network_from_input_files(
NETWORK_DEFINITION_TEST_FILE_2, WEIGHTS_DEFINITION_TEST_FILE_2)
actual_loss = nnet.loss(network_output, y)
self.assertLess(np.abs(actual_loss - expected_loss), TOLERANCE)
def test_build_network_from_input_files(self): expected_size_first_layer = 1 expected_input_weights_first_layer = np.array([[0.4, 0.1], [0.3, 0.2]]) expected_size_second_layer = 2 expected_input_weights_second_layer = np.array([[0.7, 0.5, 0.6]]) nnet = NetworkBuilder.build_network_from_input_files(NETWORK_DEFINITION_TEST_FILE_1, WEIGHTS_DEFINITION_TEST_FILE_1) self.assertEqual(len(nnet.layers), 2) self.assertEqual(nnet.layers[0].size, expected_size_first_layer) self.assertTrue((nnet.layers[0].W == expected_input_weights_first_layer).all()) self.assertEqual(nnet.layers[1].size, expected_size_second_layer) self.assertTrue((nnet.layers[1].W == expected_input_weights_second_layer).all())
def test_backprop_compared_to_gradient_checking_large(self):
epsilon = 0.000001
X = np.array([[0.32, 0.68], [0.83, 0.02]])
Y = np.array([[0.75, 0.98], [0.75, 0.28]])
nnet = NetworkBuilder.build_network_from_input_files(
NETWORK_DEFINITION_TEST_FILE_2, WEIGHTS_DEFINITION_TEST_FILE_2)
grad_check = nnet.verify_gradient(X, Y, epsilon)
nnet.forward(X)
nnet.backprop(Y)
self.assertLess(np.mean(nnet.layers[0].grad_mem - grad_check[0]),
10e-8)
self.assertLess(np.mean(nnet.layers[1].grad_mem - grad_check[1]),
10e-8)
def test_forward_activation_small_network_first_input(self):
x1 = np.array([[0.13]])
expected_z2 = np.array([0.413, 0.326])
expected_a2 = np.array([0.60181, 0.58079])
expected_z3 = np.array([1.34937])
expected_out = np.array([0.79403])
nnet = NetworkBuilder.build_network_from_input_files(
NETWORK_DEFINITION_TEST_FILE_1, WEIGHTS_DEFINITION_TEST_FILE_1)
output = nnet.forward(x1)
self.assertTrue(np.isclose(nnet.layers[0].z_mem, expected_z2).all())
self.assertTrue(np.isclose(nnet.layers[0].a_mem, expected_a2).all())
self.assertTrue(np.isclose(nnet.layers[1].z_mem, expected_z3).all())
self.assertTrue(np.isclose(output, expected_out).all())
def test_build_network_from_input_files_large_network(self): expected_size_first_layer = 2 expected_input_weights_first_layer = np.array([[0.42, 0.15, 0.4], [0.72, 0.1, 0.54], [0.01, 0.19, 0.42], [0.3, 0.35, 0.68]]) expected_size_second_layer = 4 expected_input_weights_second_layer = np.array([[0.21, 0.67, 0.14, 0.96, 0.87], [0.87, 0.42, 0.2, 0.32, 0.89], [0.03, 0.56, 0.8, 0.69, 0.09]]) expected_size_third_layer = 3 expected_input_weights_third_layer = np.array([[0.04, 0.87, 0.42, 0.53], [0.17, 0.1, 0.95, 0.69]]) nnet = NetworkBuilder.build_network_from_input_files(NETWORK_DEFINITION_TEST_FILE_2, WEIGHTS_DEFINITION_TEST_FILE_2) self.assertEqual(len(nnet.layers), 3) self.assertEqual(nnet.layers[0].size, expected_size_first_layer) self.assertTrue((nnet.layers[0].W == expected_input_weights_first_layer).all()) self.assertEqual(nnet.layers[1].size, expected_size_second_layer) self.assertTrue((nnet.layers[1].W == expected_input_weights_second_layer).all()) self.assertEqual(nnet.layers[2].size, expected_size_third_layer) self.assertTrue((nnet.layers[2].W == expected_input_weights_third_layer).all())
def test_numerical_gradient_validation_small_network(self):
epsilon = 0.000001
X = np.array([[0.13], [0.42]])
Y = np.array([[0.9], [0.23]])
expected_gradient_theta_1 = np.array([[0.02735, 0.01333],
[0.03318, 0.01618]])
expected_gradient_theta_2 = np.array([[0.23, 0.14037, 0.13756]])
nnet = NetworkBuilder.build_network_from_input_files(
NETWORK_DEFINITION_TEST_FILE_1, WEIGHTS_DEFINITION_TEST_FILE_1)
actual_grads = nnet.verify_gradient(X, Y, epsilon)
self.assertTrue(
np.isclose(actual_grads[0],
expected_gradient_theta_1,
atol=TOLERANCE).all())
self.assertTrue(
np.isclose(actual_grads[1],
expected_gradient_theta_2,
atol=TOLERANCE).all())
def test_forward_activation_large_network_second_input(self):
x1 = np.array([[0.83, 0.02]])
expected_z2 = np.array([0.5525, 0.81380, 0.17610, 0.60410])
expected_a2 = np.array([0.63472, 0.69292, 0.54391, 0.64659])
expected_z3 = np.array([1.81696, 2.02468, 1.37327])
expected_a3 = np.array([0.86020, 0.88336, 0.79791])
expected_z4 = np.array([1.58228, 1.64577])
expected_out = np.array([0.82953, 0.83832])
nnet = NetworkBuilder.build_network_from_input_files(
NETWORK_DEFINITION_TEST_FILE_2, WEIGHTS_DEFINITION_TEST_FILE_2)
output = nnet.forward(x1)
self.assertTrue(np.isclose(nnet.layers[0].z_mem, expected_z2).all())
self.assertTrue(np.isclose(nnet.layers[0].a_mem, expected_a2).all())
self.assertTrue(np.isclose(nnet.layers[1].z_mem, expected_z3).all())
self.assertTrue(np.isclose(nnet.layers[1].a_mem, expected_a3).all())
self.assertTrue(np.isclose(nnet.layers[2].z_mem, expected_z4).all())
self.assertTrue(np.isclose(output, expected_out).all())
def test_forward_activation_large_network_first_input(self):
x1 = np.array([[0.32, 0.68]])
expected_z2 = np.array([0.74, 1.1192, 0.35640, 0.87440])
expected_a2 = np.array([0.677, 0.75384, 0.58817, 0.70566])
expected_z3 = np.array([1.94769, 2.12136, 1.48154])
expected_a3 = np.array([0.87519, 0.89296, 0.81480])
expected_z4 = np.array([1.60831, 1.66805])
expected_out = np.array([0.83318, 0.84132])
nnet = NetworkBuilder.build_network_from_input_files(
NETWORK_DEFINITION_TEST_FILE_2, WEIGHTS_DEFINITION_TEST_FILE_2)
output = nnet.forward(x1)
self.assertTrue(np.isclose(nnet.layers[0].z_mem, expected_z2).all())
self.assertTrue(np.isclose(nnet.layers[0].a_mem, expected_a2).all())
self.assertTrue(np.isclose(nnet.layers[1].z_mem, expected_z3).all())
self.assertTrue(np.isclose(nnet.layers[1].a_mem, expected_a3).all())
self.assertTrue(np.isclose(nnet.layers[2].z_mem, expected_z4).all())
self.assertTrue(np.isclose(output, expected_out).all())