def test_broadcast_backward(self):
"""
Test broadcast functionality.
Arrays of different shapes are broadcast together.
The values of the gradients are compared with the gradients
from the slope formula.
"""
np.random.seed(100)
a = Tensor(np.random.randn(1, 10))
b = Tensor(np.random.randn(10, 1))
c = Tensor(np.random.randn(1, 1))
mse = MSE(a @ b, c)
a_idx = (0, 1)
b_idx = (0, 0)
forward_val = mse.forward()
assert forward_val.shape == ()
mse.backward()
a_grad = a.backward_val
b_grad = b.backward_val
assert a.value.shape == a.backward_val.shape
assert b.value.shape == b.backward_val.shape
a_numeric_grad = calculate_numerical_gradient(mse, a, a_idx)
b_numeric_grad = calculate_numerical_gradient(mse, b, b_idx)
assert np.abs(a_grad[a_idx] - a_numeric_grad) < 0.000001
assert np.abs(b_grad[b_idx] - b_numeric_grad) < 0.000001
def test_backward(self):
"""
Compare gradient value using newton's method with the gradients
computed by the graph.
This test passes if the two values are within some error tolerance
:return:
"""
# Setup graph with random values
np_inputs = np.random.randn(5)
np_targets = np.zeros(5)
np_targets[0] = 1.0
array_slice = (0, )
error_tolerance = 0.00000001
inputs = Tensor(np_inputs)
targets = Tensor(np_targets)
entropy = SoftmaxWithCrossEntropy(inputs, targets, axis=0)
entropy.backward()
# compute derivatives from graph
inputs_derivative = inputs.backward_val.copy()[array_slice]
entropy.clear()
inputs_numerical_deriv = calculate_numerical_gradient(
entropy, inputs, array_slice)
assert np.all(
np.abs(inputs_numerical_deriv -
inputs_derivative) < error_tolerance)
def test_backward(self):
"""
Test sigmoid gradient calculation.
For simplifying testing, a graph with multiple nodes is used.
if d(root)/d(x) is same from numerical as well as programmatic
gradients, the test is passed.
"""
inputs = Tensor(np.random.randn(10))
targets = Tensor(np.random.randn(10))
sigmoid = Sigmoid(inputs)
mse = MSE(sigmoid, targets)
a_idx = (0,)
forward_val = mse.forward()
assert forward_val.shape == ()
mse.backward()
a_grad = inputs.backward_val
assert inputs.value.shape == inputs.backward_val.shape
a_numeric_grad = calculate_numerical_gradient(mse, inputs, a_idx)
diff = np.abs(a_grad[a_idx] - a_numeric_grad)
assert diff < 0.0000001
def test_backward(self):
"""
Compare gradient value using newton's method with the gradients
computed by the graph.
This test passes if the two values are within some error tolerance
:return:
"""
# Setup graph with random values
np_inputs = np.random.randn(1)
np_targets = np.random.randn(1)
error_tolerance = 0.00001
inputs = Tensor(np_inputs)
targets = Tensor(np_targets)
mse = MSE(inputs, targets)
mse.backward()
# compute derivatives from graph
inputs_derivative = inputs.backward_val.copy()
targets_derivative = targets.backward_val.copy()
mse.clear()
inputs_numerical_deriv = calculate_numerical_gradient(
mse, inputs, (0, ))
targets_numerical_deriv = calculate_numerical_gradient(
mse, targets, (0, ))
assert (np.abs(inputs_numerical_deriv - inputs_derivative) <
error_tolerance)
assert (np.abs(targets_numerical_deriv - targets_derivative) <
error_tolerance)
def test_broadcast_backward(self):
"""
Test broadcast functionality.
Arrays of different shapes are broadcast together.
The values of the gradients are compared with the gradients
from the slope formula.
"""
a = Tensor(np.random.randn(1, 10, 3))
b = Tensor(np.random.randn(4, 10, 3))
a_idx = (0, 0, 1)
b_idx = (0, 1, 1)
add_op = a - b
forward_val = add_op.forward()
assert forward_val.shape == (4, 10, 3)
add_op.backward()
a_grad = a.backward_val
b_grad = b.backward_val
assert a.value.shape == a.backward_val.shape
assert b.value.shape == b.backward_val.shape
a_numeric_grad = calculate_numerical_gradient(add_op, a, a_idx)
b_numeric_grad = calculate_numerical_gradient(add_op, b, b_idx)
assert np.abs(a_grad[a_idx] - a_numeric_grad) < 0.000001
assert np.abs(b_grad[b_idx] - b_numeric_grad) < 0.000001
def test_vector_forward(self):
"""
Test to see if element-wise vector forward values are computed correctly
"""
a = np.random.randn(10)
t1 = Tensor(a)
b = np.random.randn(10)
t2 = Tensor(b)
add_op = t1 + t2
add_op.forward()
assert np.all(add_op.forward_val == (a + b))
def test_scalar_forward(self):
"""
Test to see if scalar forward values are computed correctly
"""
a = np.random.randn(1)
t1 = Tensor(a)
b = np.random.randn(1)
t2 = Tensor(b)
add_op = t1 + t2
add_op.forward()
assert add_op.forward_val == (a + b)
def test_graph_draw(self):
"""
Test graph drawing functionality.
The test passes if the graph image was successfully created.
"""
a = Tensor(np.random.randn(5))
b = Tensor(np.random.randn(5))
o = a + b
fname = str(hash(os.times())) + ".png"
full_name = os.path.join(tempfile.gettempdir(), fname)
o.draw_graph(full_name)
assert os.path.exists(full_name)
os.remove(full_name)
def test_scalar_backward(self):
"""
Test of add scalar backward
# Sympyle Graph used to calculate expect values
"""
a = Tensor(np.random.rand(1))
b = Tensor(np.random.rand(1))
add_op = a + b
add_op.backward()
t1_grad = a.backward_val
t2_grad = b.backward_val
a_numeric_grad = calculate_numerical_gradient(add_op, a, (0, ))
b_numeric_grad = calculate_numerical_gradient(add_op, b, (0, ))
assert np.abs(t1_grad - a_numeric_grad) < 0.00001
assert np.abs(t2_grad - b_numeric_grad) < 0.00001
def test_backward(self):
"""
Test Relu's gradient calculation.
For simplifying testing, a graph with multiple nodes is used.
if d(root)/d(x) is same from numerical as well as programmatic
gradients, the test is passed.
"""
a = Tensor(np.random.randn(10, 10))
b = Tensor(np.random.randn(10, 1))
c = Tensor(np.random.randn(10, 1))
matmul = a @ b
relu = Relu(matmul)
mse = MSE(relu, c)
a_idx = (0, 0)
b_idx = (0, 0)
forward_val = mse.forward()
assert forward_val.shape == ()
mse.backward()
a_grad = a.backward_val
b_grad = b.backward_val
assert a.value.shape == a.backward_val.shape
assert b.value.shape == b.backward_val.shape
a_numeric_grad = calculate_numerical_gradient(mse, a, a_idx)
b_numeric_grad = calculate_numerical_gradient(mse, b, b_idx)
assert np.abs(a_grad[a_idx] - a_numeric_grad) < 0.000001
assert np.abs(b_grad[b_idx] - b_numeric_grad) < 0.000001