def testMultipleLayersNetworkActivation(self): nn = neuralnet.NeuralNet(2, [2, 2], neuralnet.ReLu()) # 1 -- 0.5 --> [-0.1](A) -- 0.7 --> [-0.2](C) # ^ | ^ | # | 0.7 | 0.8 # 0.2 | 0.5 | # | v | v # 0 -- 0.6 --> [-0.1](B) -- 0.9 --> [-0.3](D) # (A) = 0.5 - 0.1 = 0.4 # (B) = 0.7 - 0.1 = 0.6 # (C) = 0.4x0.7 + 0.6x0.5 - 0.2 = 0.38 # (D) = 0.9x0.6 + 0.8x0.4 - 0.3 = 0.56 nn.layers[0][0].weights=[0.5, 0.2] nn.layers[0][0].bias=-0.1 nn.layers[0][1].weights=[0.7, 0.6] nn.layers[0][1].bias=-0.1 nn.layers[1][0].weights=[0.7, 0.5] nn.layers[1][0].bias=-0.2 nn.layers[1][1].weights=[0.8, 0.9] nn.layers[1][1].bias=-0.3 neuronA_output = nn.layers[0][0].output([1, 0], for_training=False) self.assertAlmostEqual(0.4, neuronA_output) neuronB_output = nn.layers[0][1].output([1, 0], for_training=False) self.assertAlmostEqual(0.6, neuronB_output) output = nn.evaluate([1, 0]) self.assertAlmostEqual(0.38, output[0]) self.assertAlmostEqual(0.56, output[1])
def testMultipleInputsNeuronActivation(self): n = neuralnet.Neuron(3, neuralnet.ReLu(), False) n.bias = -0.5 n.weights = [0.7, 0.5, 0.3] # 0.7*0.7 + 0.5*0.5 + 0.3*0.3 - 0.5 = 0.49 + 0.25 + 0.09 - 0.5 = 0.83 - 0.5 = 0.33 self.assertAlmostEqual(0.33, n.output([0.7, 0.5, 0.3], for_training=False), 4)
def testSaveAndLoad(self): nn = neuralnet.NeuralNet(2, [2, 2], neuralnet.Sigmoid()) nn.layers[0][0].weights=[0.01, 0.02] nn.layers[0][0].bias=0.03 nn.layers[0][1].weights=[0.04, 0.05] nn.layers[0][1].bias=0.06 nn.layers[1][0].weights=[0.07, 0.08] nn.layers[1][0].bias=0.09 nn.layers[1][1].weights=[0.10, 0.11] nn.layers[1][1].bias=0.12 tmpfile = tempfile.mkstemp()[1] nn.save(tmpfile) nn2 = neuralnet.NeuralNet(1, [1, 1], neuralnet.ReLu()) nn2.load(tmpfile) # This should give a more readable error when there are differences self.assertEqual(nn.__str__(), nn2.__str__()) # This ensure the previous test didn't miss any important thing self.assertEqual(nn2.activation.name(), nn.activation.name()) self.assertEqual(nn2.average_gradient, nn.average_gradient) for i in range(0, 2): for j in range(0, 2): for k in range(0, 2): self.assertAlmostEqual(nn.layers[i][j].weights[k], nn2.layers[i][j].weights[k]) self.assertAlmostEqual(nn.layers[i][j].bias, nn2.layers[i][j].bias)
def testSingleInputNeuronActivation(self): # A neuron with a weight of 0.7 and a bias of -0.5 will return 0.2 for an input of 1. # Using ReLu allow to have this value directly in output. n = neuralnet.Neuron(1, neuralnet.ReLu(), False) n.bias = -0.5 n.weights = [0.7] self.assertAlmostEqual(0.2, n.output([1.0], for_training=False), 4)
def testBackpropagateErrorOnSingleNeuron(self): n = neuralnet.Neuron(1, neuralnet.ReLu(), False) n.bias = -0.5 n.weights = [0.7] self.assertAlmostEqual(0.2, n.output([1.0], for_training=True), 4) # Assume expected output was 1 -> provide error of 0.8 da = n.per_eval_backprop(0.8) self.assertAlmostEqual(1.6, n.db) self.assertAlmostEqual(1.6, n.dw[0]) self.assertAlmostEqual(1.12, da[0])
def testBackpropagateErrorOnNetwork(self): nn = neuralnet.NeuralNet(2, [2, 2], neuralnet.ReLu()) # 1 -- 0.5 --> [-0.1](A) -- 0.7 --> [-0.2](C) # ^ | ^ | # | 0.7 | 0.8 # 0.2 | 0.5 | # | v | v # 0 -- 0.6 --> [-0.1](B) -- 0.9 --> [-0.3](D) # (A) = 0.5 - 0.1 = 0.4 # (B) = 0.7 - 0.1 = 0.6 # (C) = 0.4x0.7 + 0.6x0.5 - 0.2 = 0.38 # (D) = 0.9x0.6 + 0.8x0.4 - 0.3 = 0.56 nn.layers[0][0].weights=[0.5, 0.2] nn.layers[0][0].bias=-0.1 nn.layers[0][1].weights=[0.7, 0.6] nn.layers[0][1].bias=-0.1 nn.layers[1][0].weights=[0.7, 0.5] nn.layers[1][0].bias=-0.2 nn.layers[1][1].weights=[0.8, 0.9] nn.layers[1][1].bias=-0.3 # Start with same evaluation as testMultipleLayersNetworkActivation output = nn.evaluate([1, 0], for_training=True) self.assertAlmostEqual(0.38, output[0]) self.assertAlmostEqual(0.56, output[1]) expected = [0, 1] error = [exp - res for exp, res in zip(expected, output)] # Verify errors values are as expected self.assertAlmostEqual(-0.38, error[0]) self.assertAlmostEqual(0.44, error[1]) nn.per_eval_backprop(error) # Verify backpropagation of layer 1: # a(n-1) w expected result error dE db dw da(n-1) # 0.4 0.7 0 0.38 -0.38 0.76 -0.76 -0.304 -0.532 # 0.6 0.5 - - - - - -0.456 -0.38 # 0.4 0.8 - - - - - 0.352 0.704 # 0.6 0.9 1 0.56 0.44 -0.88 0.88 0.528 0.792 self.assertAlmostEqual(-0.76, nn.layers[1][0].db) self.assertAlmostEqual(0.88, nn.layers[1][1].db) self.assertAlmostEqual(-0.304, nn.layers[1][0].dw[0]) self.assertAlmostEqual(-0.456, nn.layers[1][0].dw[1]) self.assertAlmostEqual(0.352, nn.layers[1][1].dw[0]) self.assertAlmostEqual(0.528, nn.layers[1][1].dw[1]) self.assertAlmostEqual(-0.532, nn.layers[1][0].da[0]) self.assertAlmostEqual(-0.38, nn.layers[1][0].da[1]) self.assertAlmostEqual(0.704, nn.layers[1][1].da[0]) self.assertAlmostEqual(0.792, nn.layers[1][1].da[1]) # Verify backpropagation of layer 2: # a(n-1) w expected result error dE db dw da(n-1) # 1 0.0 - 0.4 0.086 -0.172 0.172 0.172 0.086 # 0 0.2 - - - - - 0 0.0344 # 1 0.7 - - - - - 0.412 0.2884 # 0 0.6 - 0.6 0.206 -0.412 0.412 0 0.2472 self.assertAlmostEqual(0.172, nn.layers[0][0].db) self.assertAlmostEqual(0.412, nn.layers[0][1].db) self.assertAlmostEqual(0.172, nn.layers[0][0].dw[0]) self.assertAlmostEqual(0, nn.layers[0][0].dw[1]) self.assertAlmostEqual(0.412, nn.layers[0][1].dw[0]) self.assertAlmostEqual(0, nn.layers[0][1].dw[1]) self.assertAlmostEqual(0.086, nn.layers[0][0].da[0]) self.assertAlmostEqual(0.0344, nn.layers[0][0].da[1]) self.assertAlmostEqual(0.2884, nn.layers[0][1].da[0]) self.assertAlmostEqual(0.2472, nn.layers[0][1].da[1])