def test_derivatives_with_different_indices_in_jacobian_matrix(self): z = np.array([1, -1.5], float) j = Softmax.gradient(z) s = Softmax.activation(z) self.assertEqual(j[0, 1], s[0] * s[1]) s = Softmax.activation(z) self.assertEqual(j[1, 0], s[1] * s[0])
def test_for_2_element_vectors(self): z = np.array([1, 2], float) a = Softmax.activation(z) self.assertTrue(np.allclose( a, np.array([0.268941, 0.731058], float), )) z = np.array([0, 2], float) a = Softmax.activation(z) self.assertTrue( np.allclose( a, np.array([0.1192029, 0.880797], float), ))
def _forward_pass(self, x, third_layer_activation): ''' Performs forward pass of the network. a_i - results of applying weights for the data from precious layear z_i - result of activation We save the results for further backward pass. :param x: input data :param third_layer_activation: which activation to apply to third layer :return: multi-class prediction (n_class, 1) ''' self.x = x self.a_1 = self.w1.dot(x) + self.b1 self.z_1 = Tanh.activation(self.a_1) self.a_2 = self.w2.dot(self.z_1) + self.b2 self.z_2 = Relu.activation(self.a_2) self.z_2_with_skip_connection = self.z_2 + self.w_s.dot(x) self.a_3 = self.w_out.dot(self.z_2_with_skip_connection) + self.b_out if third_layer_activation == 'Softmax': self.y_pred = Softmax.activation(self.a_3) elif third_layer_activation == 'Tanh': self.y_pred = Tanh.activation(self.a_3) else: raise ValueError("Unknown activation type for 3rd layer")
def test_get_final_layer_error_for_arrays(self): cross_entropy = cost_functions.CrossEntropyCost(self.net) z_last = np.array([3, -1], float) z_last_prime = Softmax.gradient(z_last) y = np.array([0, 0.5], float) a_last = Softmax.activation(z_last) nabla = cross_entropy.get_final_layer_error(a_last, y, z_last_prime) self.assertAlmostEqual(nabla[0], a_last[0] - y[0], places=2) self.assertAlmostEqual(nabla[1], a_last[1] - y[1], places=2)
def test_results_add_to_1(self): z = np.array([-3, 0.1, 1, 20], float) a = Softmax.activation(z) self.assertAlmostEqual(a.sum(), 1)
def test_activations_in_correct_range(self): z = np.array([-1000, 0.1, 2, 200], float) a = Softmax.activation(z) self.assertTrue(np.all(0 <= a) and np.all(a <= 1))
def test_returns_array_of_valid_shape(self): z = np.array([1, 2], float) a = Softmax.activation(z) self.assertTupleEqual(a.shape, z.shape)
def test_on_vectors_with_huge_components(self): z = np.array([np.finfo(float).max, 2, np.finfo(float).max / 2], float) # won't raise OverflowError a = Softmax.activation(z)