def testThat_updateDeltaOfLastLayer_givesCorrectLastLayerDeltas_forMultipleBatchSize(
self):
batch_size = 3
layers = self.create_list_of_1D_layers_with_sigmoid_activations(
[3, 3, 2, 3, 3])
connection_indices = [(0, 1), (0, 2), (1, 3), (2, 4), (3, 4)]
connections = self.create_list_of_connections_having_fully_connection_type(
layers, connection_indices)
network1 = NetworkArchitecture(layers, connections)
input_arrays = [np.array([1, 2, 3])] * batch_size
actual_y_arrays = [
np.array([1, 0, 0]),
np.array([0, 1, 0]),
np.array([0, 0, 1])
]
expected_deltas = [
np.array([-0.5, 0.5, 0.5]),
np.array([0.5, -0.5, 0.5]),
np.array([0.5, 0.5, -0.5])
]
network1.update_delta_of_last_layer(actual_y_arrays, input_arrays)
actual_deltas = network1.all_layers[4].delta
for i in range(batch_size):
self.assertArrayEqual(expected_deltas[i], actual_deltas[i])
def testThat_backPropagate_flowsCorrectly_withMultipleBatchSizeAndSingleNeuronInLastLayer(
self):
batch_size = 4
layers = self.create_list_of_1D_layers_with_sigmoid_activations(
[3, 1, 2, 3, 1])
connection_indices = [(0, 1), (0, 2), (1, 3), (2, 4), (3, 4)]
connections = self.create_list_of_connections_having_fully_connection_type(
layers, connection_indices)
for index in range(len(connection_indices)):
connections[index].weights = np.ones(
connections[index].weights.shape)
network1 = NetworkArchitecture(layers, connections)
input_arrays = [np.zeros([3])] * batch_size
actual_y_arrays = [np.ones([1])] * batch_size
training_data = [(input_arrays[i], actual_y_arrays[i])
for i in range(batch_size)]
expected_gradient_for_biases = np.array([-0.0364, -0.0364, -0.0364])
expected_gradient_for_weights = np.array([[0, 0, 0], [0, 0, 0]])
gradient_for_biases, gradient_for_weights = network1.back_propagate_all_layers(
training_data, batch_size)
self.assertArrayEqual(expected_gradient_for_biases,
np.round(gradient_for_biases[0], 4))
self.assertArrayEqual(expected_gradient_for_weights,
np.round(gradient_for_weights[(0, 2)], 4))
def testThat_feedForwardAllLayers_givesOutputInCorrectDimensions_forSingleBatchSize(
self):
layers = self.create_list_of_1D_layers_with_sigmoid_activations(
[3, 3, 2, 3, 4])
connection_indices = [(0, 1), (0, 2), (1, 3), (2, 4), (3, 4)]
connections = self.create_list_of_connections_having_fully_connection_type(
layers, connection_indices)
network1 = NetworkArchitecture(layers, connections)
expected_last_layer_output = [np.array([0.5] * 4)]
network1.feed_forward_all_layers([np.array([0, 0, 0])])
self.assertArrayEqual(expected_last_layer_output[0],
network1.all_layers[4].output_array[0])
def testThat_getInputArrays_givesArraysInCorrectDimensions_ForMultipleInputsInSingleBatchSize(
self):
layers = self.create_list_of_1D_layers_with_sigmoid_activations(
[3, 3, 2, 3, 3])
connection_indices = [(0, 1), (0, 2), (1, 3), (2, 4), (3, 4)]
connections = self.create_list_of_connections_having_fully_connection_type(
layers, connection_indices)
network1 = NetworkArchitecture(layers, connections)
network1.all_layers[2].output_array = [np.zeros([2])]
network1.all_layers[3].output_array = [np.zeros([3])]
expected_arrays = [[np.zeros([3])], [np.zeros([3])]]
actual_arrays = network1.get_input_arrays_of_a_layer(4)
self.assertArrayEqual(expected_arrays[0][0], actual_arrays[0][0])
self.assertArrayEqual(expected_arrays[1][0], actual_arrays[1][0])
def testThat_backPropagationSequence_givesCorrectList(self):
layers = self.create_list_of_1D_layers_with_sigmoid_activations(
[3, 3, 3, 3, 3])
connection_indices = [(0, 1), (0, 2), (1, 3), (2, 4), (3, 4)]
connections = self.create_list_of_connections_having_fully_connection_type(
layers, connection_indices)
network1 = NetworkArchitecture(layers, connections)
expected_sequence = [4, 3, 2, 1, 0]
actual_sequence = network1.back_propagation_sequence
self.assertEqual(expected_sequence, actual_sequence)
def testThat_updateDeltaOfAllLayers_flowsCorrectly_withSingleInputSingleBatchSizeAndSingleNeuronInLastLayer(
self):
layers = self.create_list_of_1D_layers_with_sigmoid_activations(
[3, 3, 2, 3, 1])
connection_indices = [(0, 1), (0, 2), (1, 3), (2, 4), (3, 4)]
connections = self.create_list_of_connections_having_fully_connection_type(
layers, connection_indices)
for index in range(len(connection_indices)):
connections[index].weights = np.ones(
connections[index].weights.shape)
network1 = NetworkArchitecture(layers, connections)
input_arrays = [np.zeros([3])]
actual_y_arrays = [np.ones([1])]
expected_delta = [np.array([-0.0064, -0.0064, -0.0064])]
network1.update_deltas_of_all_layers(input_arrays, actual_y_arrays, 1)
self.assertArrayEqual(expected_delta[0],
np.round(network1.all_layers[0].delta[0], 4))
def testThat_getInputVector_givesVectorsInCorrectDimensions_ForMultipleInputsInMultipleBatchSize(
self):
batch_size = 4
layers = self.create_list_of_1D_layers_with_sigmoid_activations(
[3, 3, 2, 3])
connection_indices = [(0, 1), (0, 2), (1, 3), (2, 3)]
connections = self.create_list_of_connections_having_fully_connection_type(
layers, connection_indices)
network1 = NetworkArchitecture(layers, connections)
network1.all_layers[1].output_array = [np.zeros([3])] * batch_size
network1.all_layers[2].output_array = [np.ones([2])] * batch_size
network1.connections[(1, 3)].weights = np.ones([3, 3])
network1.connections[(2, 3)].weights = np.ones([3, 2])
expected_vectors = [[np.zeros([3])] * batch_size,
[np.ones([3]) * 2] * batch_size]
actual_vectors = network1.get_input_arrays_of_a_layer(3)
for i in range(2):
for j in range(batch_size):
self.assertArrayEqual(expected_vectors[i][j],
actual_vectors[i][j])
def testThat_updateDeltaOfAllLayers_flowsCorrectly_withMultipleSuccessorAndMultipleBatchSize(
self):
batch_size = 4
layers = self.create_list_of_1D_layers_with_sigmoid_activations(
[3, 3, 2, 3, 3])
connection_indices = [(0, 1), (0, 2), (1, 3), (2, 4), (3, 4)]
connections = self.create_list_of_connections_having_fully_connection_type(
layers, connection_indices)
for index in range(len(connection_indices)):
connections[index].weights = np.ones(
connections[index].weights.shape)
network1 = NetworkArchitecture(layers, connections)
input_arrays = [np.array([1, 2, 3])] * batch_size
actual_y_arrays = [np.array([0, 1, 0])] * batch_size
expected_deltas = [np.array([0.0023, 0.0012, 0.0005])] * batch_size
network1.update_deltas_of_all_layers(input_arrays, actual_y_arrays,
batch_size)
for i in range(batch_size):
self.assertArrayEqual(
expected_deltas[i],
np.around(network1.all_layers[0].delta[i], 4))
def testThat_createAllLayers_createsLayersWithCorrect_predecessorsAndSuccessors(
self):
layers = self.create_list_of_1D_layers_with_sigmoid_activations(
[3, 3, 3, 3])
connection_indices = [(0, 1), (0, 2), (1, 3), (2, 3)]
connections = self.create_list_of_connections_having_fully_connection_type(
layers, connection_indices)
network1 = NetworkArchitecture(layers, connections)
expected_predecessors_of_all_layers = [[], [0], [0], [1, 2]]
expected_successors_of_all_layers = [[1, 2], [3], [3], []]
actual_predecessors_of_all_layers = [
network1.all_layers[index].predecessors for index in range(4)
]
actual_successors_of_all_layers = [
network1.all_layers[index].successors for index in range(4)
]
self.assertEqual(expected_predecessors_of_all_layers,
actual_predecessors_of_all_layers)
self.assertEqual(expected_successors_of_all_layers,
actual_successors_of_all_layers)
def testThat_backPropagate_flowsCorrectly_for3DLayersWithMultipleBatchSizeAndSingleNeuronInLastLayer(
self):
batch_size = 4
# 3D representation of fully connected feed forward network
layer0 = Layer(id=0, shape=[2, 3, 2], activation=Sigmoid())
layer1 = Layer(id=1, shape=[3, 2, 3], activation=Sigmoid())
layer2 = Layer(id=2, shape=[2], activation=Sigmoid())
layers_3d = [layer0, layer1, layer2]
connection_indices = [(0, 1), (1, 2)]
connections_3d = self.create_list_of_connections_having_fully_connection_type(
layers_3d, connection_indices)
for index in range(len(connection_indices)):
connections_3d[index].weights = np.ones(
connections_3d[index].weights.shape)
input_arrays_3d = [np.zeros([2, 3, 2])] * batch_size
actual_y_arrays = [np.array([0, 1])] * batch_size
training_data_3d = [(input_arrays_3d[i], actual_y_arrays[i])
for i in range(batch_size)]
network1 = NetworkArchitecture(layers_3d, connections_3d)
bias_grad_3d, weight_grad_3d = network1.back_propagate_all_layers(
training_data_3d, batch_size)
# 1D representation of same feed forward network
layers_1d = self.create_list_of_1D_layers_with_sigmoid_activations(
[12, 18, 2])
connections_1d = self.create_list_of_connections_having_fully_connection_type(
layers_1d, connection_indices)
for index in range(len(connection_indices)):
connections_1d[index].weights = np.ones(
connections_1d[index].weights.shape)
input_arrays_1d = [np.zeros([12])] * batch_size
actual_y_arrays = [np.array([0, 1])] * batch_size
training_data_1d = [(input_arrays_1d[i], actual_y_arrays[i])
for i in range(batch_size)]
network2 = NetworkArchitecture(layers_1d, connections_1d)
bias_grad_1d, weight_grad_1d = network2.back_propagate_all_layers(
training_data_1d, batch_size)
for i in range(3):
self.assertArrayEqual(bias_grad_3d[i].reshape(layers_1d[i].shape),
bias_grad_1d[i])
for i in range(2):
self.assertArrayEqual(
weight_grad_3d[(i, i + 1)].reshape(
connections_1d[i].weights.shape),
weight_grad_1d[(i, i + 1)])