def test_learn_xor_relu():
random.seed(1)
input_nodes = InputNode.make_input_nodes(2)
first_layer = [LinearNode(input_nodes) for i in range(10)]
first_layer_relu = [ReluNode(L) for L in first_layer]
second_layer = [LinearNode(first_layer_relu) for i in range(10)]
second_layer_relu = [ReluNode(L) for L in second_layer]
linear_output = LinearNode(second_layer_relu)
output = linear_output
error_node = L2ErrorNode(output)
network = NeuralNetwork(output,
input_nodes,
error_node=error_node,
step_size=0.05)
examples = [[0, 0], [0, 1], [1, 0], [1, 1]]
labels = [0, 1, 1, 0]
dataset = list(zip(examples, labels))
network.train(dataset, max_steps=1000)
for (example, label) in dataset:
assert abs(network.evaluate(example) - label) < 0.1
assert_that(network.error_on_dataset(dataset)).is_equal_to(0.0)
def test_linear_with_relu_evaluate():
input_nodes = InputNode.make_input_nodes(3)
inputs = [1, 2, 3]
initial_weights = [-20, 3, 2, 1]
linear_node = LinearNode(input_nodes, initial_weights=initial_weights)
relu_node = ReluNode(linear_node)
assert_that(relu_node.evaluate(inputs)).is_equal_to(0)
assert_that(linear_node.output).is_equal_to(-10)
def test_neural_network_error():
input_node = InputNode(0)
relu = ReluNode(input_node)
network = NeuralNetwork(relu, [input_node])
inputs = [-2]
label = 1
assert_that(network.evaluate(inputs)).is_equal_to(0)
assert_that(network.compute_error(inputs, label)).is_equal_to(1)
def build_network():
input_nodes = InputNode.make_input_nodes(28 * 28)
first_layer = [LinearNode(input_nodes) for i in range(10)]
first_layer_relu = [ReluNode(L) for L in first_layer]
second_layer = [LinearNode(first_layer_relu) for i in range(10)]
second_layer_relu = [ReluNode(L) for L in second_layer]
linear_output = LinearNode(second_layer_relu)
output = SigmoidNode(linear_output)
error_node = L2ErrorNode(output)
network = NeuralNetwork(output,
input_nodes,
error_node=error_node,
step_size=0.05)
return network
def test_pretty_print():
const = ConstantNode()
input_node = InputNode(0)
sigmoid = SigmoidNode(const)
sigmoid.evaluate([])
relu = ReluNode(input_node)
relu.evaluate([2])
assert_that(sigmoid.pretty_print()).is_equal_to(
"Sigmoid output=0.73\n Constant(1)\n")
assert_that(relu.pretty_print()).is_equal_to(
"Relu output=2.00\n InputNode(0) output = 2.00\n")
network = single_linear_relu_network(3, [-20, 3, 2, 1])
network.evaluate([1, 2, 3])
network.compute_error([1, 2, 3], 1)
assert_that(network.pretty_print()).is_equal_to("""Relu output=0.00
Linear weights=-20.00,3.00,2.00,1.00 gradient=0.00,0.00,0.00,0.00 output=-10.00
Constant(1)
InputNode(0) output = 1.00
InputNode(1) output = 2.00
InputNode(2) output = 3.00
""")
def test_neural_network_backpropagation_step():
input_nodes = InputNode.make_input_nodes(2)
initial_weights = [3, 2, 1]
linear_node = LinearNode(input_nodes, initial_weights=initial_weights)
relu_node = ReluNode(linear_node)
error_node = L2ErrorNode(relu_node)
network = NeuralNetwork(relu_node, input_nodes, error_node=error_node)
example = [2, -2]
label = 1
step_size = 0.5
network.backpropagation_step(example, label, step_size=step_size)
new_weights = [-1.0, -6.0, 9.0]
# ∂E/∂w_i = [8, 16, -16], delta is [-4, -8, 8]
assert_that(linear_node.weights).is_equal_to(new_weights)
def test_neural_network_gradients():
input_nodes = InputNode.make_input_nodes(2)
initial_weights = [3, 2, 1]
linear_node = LinearNode(input_nodes, initial_weights=initial_weights)
relu_node = ReluNode(linear_node)
error_node = L2ErrorNode(relu_node)
network = NeuralNetwork(relu_node, input_nodes, error_node=error_node)
example = [2, -2]
label = 1
'''
l(w, x): linear node
r(z): relu node
f(w, x) = r(l(w, x))
E(w, x, y): (r(l(w, x)) - y) ^ 2
'''
# f(w, x) = 5
# E(w, x, y) = 16
assert_that(network.evaluate(example)).is_equal_to(5)
assert relu_node.output > 0
assert_that(network.compute_error(example, label)).is_equal_to(16)
# ∂E/∂E = 1, ∂E/∂f = 8
assert_that(error_node.global_gradient).is_equal_to(1)
assert_that(error_node.local_gradient).is_equal_to([8])
# ∂E/∂z = 8, ∂r/∂z = 1
assert_that(relu_node.global_gradient).is_equal_to(8)
assert_that(relu_node.local_gradient).is_equal_to([1])
assert_that(relu_node.global_parameter_gradient).is_equal_to([])
assert_that(relu_node.local_parameter_gradient).is_equal_to([])
# ∂E/∂l = 8, ∂l/∂x_i = [3, 2, 1]
assert_that(linear_node.global_gradient).is_equal_to(8)
assert_that(linear_node.local_gradient).is_equal_to([3, 2, 1])
# ∂l/∂w_i = [1, 2, -2], ∂E/∂w_i = [8, 16, -16]
assert_that(linear_node.local_parameter_gradient).is_equal_to([1, 2, -2])
assert_that(linear_node.global_parameter_gradient).is_equal_to(
[8, 16, -16])
def test_relu_evaluate_negative():
input_node = InputNode(0)
relu = ReluNode(input_node)
assert_that(relu.evaluate([-2])).is_equal_to(0)
def single_linear_relu(input_nodes, initial_weights=None):
return ReluNode(LinearNode(input_nodes, initial_weights=initial_weights))
def test_relu_local_parameter_gradient_empty():
input_node = InputNode(0)
relu = ReluNode(input_node)
relu.evaluate([3])
assert_that(len(relu.local_parameter_gradient)).is_equal_to(0)
def test_relu_local_gradient_negative():
input_node = InputNode(0)
relu = ReluNode(input_node)
relu.evaluate([-3])
assert_that(relu.local_gradient_for_argument(input_node)).is_equal_to(0)