def test_update_biases_decreases_cost(self):
cost_func = QuadraticCost(self.nnet)
cost_before = cost_func.get_cost(self.examples)
calculator = BackPropagationBasedCalculator(data_src=self.examples,
cost_function=cost_func,
neural_net=self.nnet)
for i in range(5):
w_grad, b_grad = calculator.compute_gradients()
self.grad_descent.update_biases(bias_gradient=b_grad)
cost_after = cost_func.get_cost(self.examples)
self.assertLess(cost_after, cost_before)
def training_epoch(self, data_src):
"""
Perform a single iteration of gradient descent.
:param data_src: an instance of DataSource sub class
:return: None
"""
cost_function = self._cost_function
gradient_calculator = BackPropagationBasedCalculator(
data_src=data_src,
neural_net=self._nnet,
cost_function=cost_function)
wgrad, bgrad = gradient_calculator.compute_gradients()
self.update_weights(weight_gradient=wgrad)
self.update_biases(bias_gradient=bgrad)
def test_compute_gradients_with_cross_entropy_cost(self):
nnet = NetFactory.create_neural_net(sizes=[4, 2, 10])
nnet.randomize_parameters()
cost_func = cost_functions.CrossEntropyCost(neural_net=nnet)
examples = helpers.generate_random_examples(10, 4, 10)
calculator = BackPropagationBasedCalculator(
data_src=PreloadSource(examples),
neural_net=nnet,
cost_function=cost_func)
numerical_calculator = NumericalCalculator(
data_src=PreloadSource(examples),
neural_net=nnet,
cost_function=cost_func)
w_grad1, b_grad1 = calculator.compute_gradients()
w_grad2, b_grad2 = numerical_calculator.compute_gradients()
self.compare_grads(grad1=w_grad1, grad2=w_grad2)
self.compare_grads(grad1=b_grad1, grad2=b_grad2)
def test_that_returned_type_is_array(self):
nnet = NetFactory.create_neural_net(sizes=[2, 1, 2])
cost_func = cost_functions.QuadraticCost(neural_net=nnet)
x = np.array([5, 2], float)
y = np.array([0.25, 0], float)
examples = ([x], [y])
calculator = BackPropagationBasedCalculator(
data_src=PreloadSource(examples),
neural_net=nnet,
cost_function=cost_func)
w_grad, b_grad = calculator.compute_gradients()
self.assertIsInstance(w_grad, list)
self.assertIsInstance(w_grad[0], np.ndarray)
self.assertIsInstance(w_grad[1], np.ndarray)
self.assertIsInstance(b_grad, list)
self.assertIsInstance(b_grad[0], np.ndarray)
self.assertIsInstance(b_grad[1], np.ndarray)
def test_update_with_multiple_examples(self):
self.nnet.randomize_parameters()
self.examples = PreloadSource(
([np.array([5, 2], float),
np.array([5, 22], float)],
[np.array([0.25, 0, 1], float),
np.array([0.5, 1, 0], float)]))
cost_func = QuadraticCost(self.nnet)
cost_before = cost_func.get_cost(self.examples)
calculator = BackPropagationBasedCalculator(data_src=self.examples,
cost_function=cost_func,
neural_net=self.nnet)
for i in range(10):
w_grad, b_grad = calculator.compute_gradients()
self.grad_descent.update_weights(weight_gradient=w_grad)
self.grad_descent.update_biases(bias_gradient=b_grad)
cost_after = cost_func.get_cost(self.examples)
self.assertLess(cost_after, cost_before)
def test_with_rectifer_activation_and_quadratic_cost(self):
nnet = NetFactory.create_neural_net(
sizes=[4, 2, 10],
hidden_layer_activation=activation_functions.Rectifier,
output_layer_activation=activation_functions.Rectifier)
nnet.randomize_parameters()
cost_func = cost_functions.QuadraticCost(neural_net=nnet)
examples = helpers.generate_random_examples(10, 4, 10)
calculator = BackPropagationBasedCalculator(
data_src=PreloadSource(examples),
neural_net=nnet,
cost_function=cost_func)
numerical_calculator = NumericalCalculator(
data_src=PreloadSource(examples),
neural_net=nnet,
cost_function=cost_func)
w_grad1, b_grad1 = calculator.compute_gradients()
w_grad2, b_grad2 = numerical_calculator.compute_gradients()
self.compare_grads(grad1=w_grad1, grad2=w_grad2)
self.compare_grads(grad1=b_grad1, grad2=b_grad2)
def test_with_regularized_quadratic_loss(self):
nnet = NetFactory.create_neural_net(sizes=[4, 2, 10])
nnet.randomize_parameters()
reglambda = 2.5
cost_func = cost_functions.QuadraticCost(neural_net=nnet,
l2_reg_term=reglambda)
examples = helpers.generate_random_examples(10, 4, 10)
calculator = BackPropagationBasedCalculator(
data_src=PreloadSource(examples),
neural_net=nnet,
cost_function=cost_func)
numerical_calculator = NumericalCalculator(
data_src=PreloadSource(examples),
neural_net=nnet,
cost_function=cost_func)
w_grad1, b_grad1 = calculator.compute_gradients()
w_grad2, b_grad2 = numerical_calculator.compute_gradients()
self.compare_grads(grad1=w_grad1, grad2=w_grad2)
self.compare_grads(grad1=b_grad1, grad2=b_grad2)
def step(context):
cost_func = context.cost_function
calculator = BackPropagationBasedCalculator(data_src=context.training_data,
neural_net=context.nnet,
cost_function=cost_func)
context.back_prop_gradients = calculator.compute_gradients()