def test_sigmoid_derivative(): sigmoid = Sigmoid() # Do forward the check that backward updates error correctly sigmoid.forward(np.asarray([0])) npt.assert_array_equal(sigmoid.backward(np.ones(1)), np.asarray([0.25])) sigmoid.forward(np.asarray([0])) npt.assert_almost_equal(sigmoid.backward(np.ones(1)), finite_difference_derivative(sigmoid.forward, 0)) sigmoid.forward(np.asarray([3])) npt.assert_almost_equal(sigmoid.backward(np.ones(1)), finite_difference_derivative(sigmoid.forward, 3)) sigmoid.forward(np.asarray([-3])) npt.assert_almost_equal(sigmoid.backward(np.ones(1)), finite_difference_derivative(sigmoid.forward, -3))
def test_tanh_derivative(): tanh = Tanh() # For each case do forward and then check backward changes the error correctly tanh.forward(np.zeros(2)) npt.assert_array_equal(tanh.backward(np.ones(2)), np.ones(2)) tanh.forward(np.asarray([0])) npt.assert_almost_equal(tanh.backward(np.asarray([1])), finite_difference_derivative(tanh.forward, 0)) tanh.forward(np.asarray([3])) npt.assert_almost_equal(tanh.backward(np.asarray([1])), finite_difference_derivative(tanh.forward, 3)) tanh.forward(np.asarray([-3])) npt.assert_almost_equal(tanh.backward(np.asarray([1])), finite_difference_derivative(tanh.forward, -3))
def test_MES_derivative(): mse = MSE() npt.assert_equal(mse.backward(np.ones(3), np.zeros(3)), np.full(3, 1)) npt.assert_equal(mse.backward(np.full(3, 2), np.zeros(3)), np.full(3, 2)) # Check that the derivative is with respect to the output by fixing the target using finite difference target = np.random.random((4,4)) output = np.random.random((4,4)) def fixed_target_MSE(output): return mse.forward(output, target) npt.assert_almost_equal(mse.backward(output, target), finite_difference_derivative(fixed_target_MSE, output))
def test_relu_backward(): relu = ReLU() relu.forward(np.full((3, ), -1)) npt.assert_array_equal(relu.backward(np.ones(3)), np.zeros(3)) relu.forward(np.full((3, 1), 2)) npt.assert_array_equal(relu.backward(np.ones((3, 1))), np.ones((3, 1))) relu.forward(np.zeros((3, 3))) npt.assert_array_equal(relu.backward(np.ones((3, 3))), np.ones((3, 3))) x = np.asarray([[0, 2, -3], [-4, 5, -6], [7, 8, 9]]) d_x = np.asarray([[1, 1, 0], [0, 1, 0], [1, 1, 1]]) relu.forward(x) npt.assert_array_equal(relu.backward(np.ones(x.shape)), d_x) relu.forward(np.asarray([3])) npt.assert_almost_equal(relu.backward(np.asarray([1])), finite_difference_derivative(relu.forward, 3)) relu.forward(np.asarray([-3])) npt.assert_almost_equal(relu.backward(np.asarray([1])), finite_difference_derivative(relu.forward, -3))
def test_finite_difference(): npt.assert_almost_equal(finite_difference_derivative(lambda x: x, 4), 1) npt.assert_almost_equal(finite_difference_derivative(lambda x: x**2, 4), 2 * 4)