def test_sigmoid_derivative():
sigmoid = Sigmoid()
# Do forward the check that backward updates error correctly
sigmoid.forward(np.asarray([0]))
npt.assert_array_equal(sigmoid.backward(np.ones(1)), np.asarray([0.25]))
sigmoid.forward(np.asarray([0]))
npt.assert_almost_equal(sigmoid.backward(np.ones(1)),
finite_difference_derivative(sigmoid.forward, 0))
sigmoid.forward(np.asarray([3]))
npt.assert_almost_equal(sigmoid.backward(np.ones(1)),
finite_difference_derivative(sigmoid.forward, 3))
sigmoid.forward(np.asarray([-3]))
npt.assert_almost_equal(sigmoid.backward(np.ones(1)),
finite_difference_derivative(sigmoid.forward, -3))
def test_tanh_derivative():
tanh = Tanh()
# For each case do forward and then check backward changes the error correctly
tanh.forward(np.zeros(2))
npt.assert_array_equal(tanh.backward(np.ones(2)), np.ones(2))
tanh.forward(np.asarray([0]))
npt.assert_almost_equal(tanh.backward(np.asarray([1])),
finite_difference_derivative(tanh.forward, 0))
tanh.forward(np.asarray([3]))
npt.assert_almost_equal(tanh.backward(np.asarray([1])),
finite_difference_derivative(tanh.forward, 3))
tanh.forward(np.asarray([-3]))
npt.assert_almost_equal(tanh.backward(np.asarray([1])),
finite_difference_derivative(tanh.forward, -3))
def test_MES_derivative():
mse = MSE()
npt.assert_equal(mse.backward(np.ones(3), np.zeros(3)), np.full(3, 1))
npt.assert_equal(mse.backward(np.full(3, 2), np.zeros(3)), np.full(3, 2))
# Check that the derivative is with respect to the output by fixing the target using finite difference
target = np.random.random((4,4))
output = np.random.random((4,4))
def fixed_target_MSE(output):
return mse.forward(output, target)
npt.assert_almost_equal(mse.backward(output, target), finite_difference_derivative(fixed_target_MSE, output))
def test_relu_backward():
relu = ReLU()
relu.forward(np.full((3, ), -1))
npt.assert_array_equal(relu.backward(np.ones(3)), np.zeros(3))
relu.forward(np.full((3, 1), 2))
npt.assert_array_equal(relu.backward(np.ones((3, 1))), np.ones((3, 1)))
relu.forward(np.zeros((3, 3)))
npt.assert_array_equal(relu.backward(np.ones((3, 3))), np.ones((3, 3)))
x = np.asarray([[0, 2, -3], [-4, 5, -6], [7, 8, 9]])
d_x = np.asarray([[1, 1, 0], [0, 1, 0], [1, 1, 1]])
relu.forward(x)
npt.assert_array_equal(relu.backward(np.ones(x.shape)), d_x)
relu.forward(np.asarray([3]))
npt.assert_almost_equal(relu.backward(np.asarray([1])),
finite_difference_derivative(relu.forward, 3))
relu.forward(np.asarray([-3]))
npt.assert_almost_equal(relu.backward(np.asarray([1])),
finite_difference_derivative(relu.forward, -3))
def test_finite_difference():
npt.assert_almost_equal(finite_difference_derivative(lambda x: x, 4), 1)
npt.assert_almost_equal(finite_difference_derivative(lambda x: x**2, 4),
2 * 4)