def step(context, seq_len): slen = int(seq_len) training_data = examples_drawn_from_distributions( number_of_examples=200, seq_len=slen, create_example=create_example) context.training_data = PreloadSource(training_data) test_data = examples_drawn_from_distributions( number_of_examples=50, seq_len=slen, create_example=create_example) context.test_data = PreloadSource(test_data)
def _load_mnist_examples(self): mnist.download_dataset() train_data = mnist.get_training_data() test_data = mnist.get_test_data() dataset_size = self._config['dataset_size'] if dataset_size: train_size = dataset_size test_size = dataset_size else: train_size = len(train_data) test_size = len(test_data) self._data_src = PreloadSource(train_data[:train_size]) self._test_data_src = PreloadSource(test_data[:test_size])
def train(self, pixels_to_categories, nepochs=1): examples = self.prepare_train_examples(pixels_to_categories) cost_func = cost_functions.CrossEntropyCost(self._nnet) gd = gradient_descent.StochasticGradientDescent( self._nnet, cost_function=cost_func, learning_rate=0.1) self._nnet.randomize_parameters() gd.train(data_src=PreloadSource(examples), nepochs=nepochs)
def setUp(self): x = np.array([5, 2], float) y = np.array([0.25, 0, 1], float) self.examples = PreloadSource(([x], [y])) nnet = NetFactory.create_neural_net(sizes=[2, 3, 3]) self.nnet = nnet self.Descent = GradientDescent self.cost_function = QuadraticCost(self.nnet)
def test_quadratic_cost(self): inputs = [np.array([0.7, 0.6, 0.1], float), np.array([1, 0, 0], float)] outputs = [np.array([0, 0.5], float), np.array([0, 0], float)] quadracost = cost_functions.QuadraticCost(neural_net=self.net) src = PreloadSource((inputs, outputs)) c = quadracost.get_cost(data_src=src) self.assertAlmostEqual(c, 0.75 / 4.0, places=3)
def squared_sin_data_set(): def f(x): return math.sin(x)**2 return PreloadSource( helpers.generate_data(f=f, start_value=0, end_value=3.14, step_value=0.1))
def test_get_cost_initial(self): nnet = NetFactory.create_neural_net(sizes=[1, 1, 1]) xes = [np.array([-10], float), np.array([100], float)] ys = [np.array([0.5], float), np.array([0.75], float)] examples = PreloadSource((xes, ys)) cost_func = cost_functions.QuadraticCost(nnet) cost = cost_func.get_cost(examples) self.assertAlmostEqual(cost, 1.0 / 64, places=4)
def setUp(self): x = np.array([5, 2], float) y = np.array([0.25, 0, 1], float) self.examples = PreloadSource(([x], [y])) nnet = NetFactory.create_neural_net(sizes=[2, 3, 3]) nnet.randomize_parameters() self.nnet = nnet cost_func = QuadraticCost(nnet) self.grad_descent = GradientDescent(neural_net=nnet, cost_function=cost_func)
def test_compute_gradients_with_cross_entropy_cost(self): nnet = NetFactory.create_neural_net(sizes=[4, 2, 10]) nnet.randomize_parameters() cost_func = cost_functions.CrossEntropyCost(neural_net=nnet) examples = helpers.generate_random_examples(10, 4, 10) calculator = BackPropagationBasedCalculator( data_src=PreloadSource(examples), neural_net=nnet, cost_function=cost_func) numerical_calculator = NumericalCalculator( data_src=PreloadSource(examples), neural_net=nnet, cost_function=cost_func) w_grad1, b_grad1 = calculator.compute_gradients() w_grad2, b_grad2 = numerical_calculator.compute_gradients() self.compare_grads(grad1=w_grad1, grad2=w_grad2) self.compare_grads(grad1=b_grad1, grad2=b_grad2)
def test_training_epoch_2_examples(self): cost_func = QuadraticCost(self.nnet) self.examples = PreloadSource( ([np.array([5, 2], float), np.array([5, 22], float)], [np.array([0.25, 0, 1], float), np.array([0.5, 1, 0], float)])) cost_before = cost_func.get_cost(self.examples) self.grad_descent.training_epoch(data_src=self.examples) cost_after = cost_func.get_cost(self.examples) self.assertLess(cost_after, cost_before)
def test_gives_correct_output_on_training_data(self): nnet = NetFactory.create_neural_net(sizes=[1, 1, 1]) cost_func = QuadraticCost(neural_net=nnet) gd = GradientDescent(neural_net=nnet, cost_function=cost_func) xes = [np.array([-10], float), np.array([100], float)] ys = [np.array([0.5], float), np.array([0.75], float)] gd.train(data_src=PreloadSource((xes, ys)), nepochs=100) for i in range(len(xes)): res = nnet.feed(xes[i]) self.assertAlmostEqual(res[0], ys[i][0], places=1)
def test_with_rectifer_activation_and_quadratic_cost(self): nnet = NetFactory.create_neural_net( sizes=[4, 2, 10], hidden_layer_activation=activation_functions.Rectifier, output_layer_activation=activation_functions.Rectifier) nnet.randomize_parameters() cost_func = cost_functions.QuadraticCost(neural_net=nnet) examples = helpers.generate_random_examples(10, 4, 10) calculator = BackPropagationBasedCalculator( data_src=PreloadSource(examples), neural_net=nnet, cost_function=cost_func) numerical_calculator = NumericalCalculator( data_src=PreloadSource(examples), neural_net=nnet, cost_function=cost_func) w_grad1, b_grad1 = calculator.compute_gradients() w_grad2, b_grad2 = numerical_calculator.compute_gradients() self.compare_grads(grad1=w_grad1, grad2=w_grad2) self.compare_grads(grad1=b_grad1, grad2=b_grad2)
def test_with_regularized_quadratic_loss(self): nnet = NetFactory.create_neural_net(sizes=[4, 2, 10]) nnet.randomize_parameters() reglambda = 2.5 cost_func = cost_functions.QuadraticCost(neural_net=nnet, l2_reg_term=reglambda) examples = helpers.generate_random_examples(10, 4, 10) calculator = BackPropagationBasedCalculator( data_src=PreloadSource(examples), neural_net=nnet, cost_function=cost_func) numerical_calculator = NumericalCalculator( data_src=PreloadSource(examples), neural_net=nnet, cost_function=cost_func) w_grad1, b_grad1 = calculator.compute_gradients() w_grad2, b_grad2 = numerical_calculator.compute_gradients() self.compare_grads(grad1=w_grad1, grad2=w_grad2) self.compare_grads(grad1=b_grad1, grad2=b_grad2)
def test_costs_match(self): nnet = NetFactory.create_neural_net(sizes=[2, 3, 1, 10, 2]) nnet.randomize_parameters() cost_func = CrossEntropyCost(nnet) X = [np.array([90, 23], float), np.array([0, 2], float)] Y = [np.array([0.4, 0.6], float), np.array([0.3, 0])] examples = PreloadSource((X, Y)) c1 = cost_func.get_cost(examples) fname = os.path.join('test_temp', 'nets_params.json') nnet.save(dest_fname=fname) nnet = NeuralNet.create_from_file(fname) c2 = cost_func.get_cost(examples) self.assertAlmostEqual(c1, c2, places=4)
def test_back_propagation_slow(self): nnet = NetFactory.create_neural_net(sizes=[1, 1, 1]) cost_func = QuadraticCost(neural_net=nnet) x = np.array([5], float) y = np.array([0.25], float) examples = ([x], [y]) numerical = NumericalCalculator(data_src=PreloadSource(examples), neural_net=nnet, cost_function=cost_func) w_grad, b_grad = numerical.compute_gradients() w_grad_expected = [np.array([[0]], float), np.array([[1 / 32]], float)] b_grad_expected = [np.array([[0]], float), np.array([[1 / 16]], float)] self.compare_grads(w_grad, w_grad_expected) self.compare_grads(b_grad, b_grad_expected)
def test_back_propagation_slow_type_array(self): nnet = NetFactory.create_neural_net(sizes=[2, 1, 2]) cost_func = QuadraticCost(neural_net=nnet) x = np.array([5, 2], float) y = np.array([0.25, 0], float) examples = ([x], [y]) numerical = NumericalCalculator(data_src=PreloadSource(examples), neural_net=nnet, cost_function=cost_func) w_grad, b_grad = numerical.compute_gradients() self.assertIsInstance(w_grad, list) self.assertIsInstance(w_grad[0], np.ndarray) self.assertIsInstance(w_grad[1], np.ndarray) self.assertIsInstance(b_grad, list) self.assertIsInstance(b_grad[0], np.ndarray) self.assertIsInstance(b_grad[1], np.ndarray)
def test_gives_correct_output_for_unseen_data(self): nnet = NetFactory.create_neural_net(sizes=[1, 10, 1]) cost_func = QuadraticCost(neural_net=nnet) gd = GradientDescent(neural_net=nnet, cost_function=cost_func) def parabola(x): return x**2 examples = helpers.generate_data(f=parabola, start_value=-0.6, end_value=-0.4, step_value=0.005) gd.train(data_src=PreloadSource(examples), nepochs=10) xval = -0.5000125 yval = parabola(xval) net_input = np.array([xval], float) output = nnet.feed(net_input) self.assertAlmostEqual(output[0], yval, places=1)
def test_returns_correct_gradient_shape(self): nnet = NetFactory.create_neural_net(sizes=[3, 2, 2, 5]) cost_func = cost_functions.QuadraticCost(neural_net=nnet) x = np.array([5, 2, -0.5], float) y = np.array([0.25, 0, 0, 0.7, 0.2], float) examples = ([x], [y]) numerical_calculator = NumericalCalculator( data_src=PreloadSource(examples), neural_net=nnet, cost_function=cost_func) w_grad, b_grad = numerical_calculator.compute_gradients() self.assertEqual(len(w_grad), 3) self.assertEqual(len(b_grad), 3) self.assertTupleEqual(w_grad[0].shape, (2, 3)) self.assertTupleEqual(w_grad[1].shape, (2, 2)) self.assertTupleEqual(w_grad[2].shape, (5, 2)) self.assertTupleEqual(b_grad[0].shape, (2, )) self.assertTupleEqual(b_grad[1].shape, (2, )) self.assertTupleEqual(b_grad[2].shape, (5, ))
def test_that_returned_type_is_array(self): nnet = NetFactory.create_neural_net(sizes=[2, 1, 2]) cost_func = cost_functions.QuadraticCost(neural_net=nnet) x = np.array([5, 2], float) y = np.array([0.25, 0], float) examples = ([x], [y]) calculator = BackPropagationBasedCalculator( data_src=PreloadSource(examples), neural_net=nnet, cost_function=cost_func) w_grad, b_grad = calculator.compute_gradients() self.assertIsInstance(w_grad, list) self.assertIsInstance(w_grad[0], np.ndarray) self.assertIsInstance(w_grad[1], np.ndarray) self.assertIsInstance(b_grad, list) self.assertIsInstance(b_grad[0], np.ndarray) self.assertIsInstance(b_grad[1], np.ndarray)
def test_update_with_multiple_examples(self): self.nnet.randomize_parameters() self.examples = PreloadSource( ([np.array([5, 2], float), np.array([5, 22], float)], [np.array([0.25, 0, 1], float), np.array([0.5, 1, 0], float)])) cost_func = QuadraticCost(self.nnet) cost_before = cost_func.get_cost(self.examples) calculator = BackPropagationBasedCalculator(data_src=self.examples, cost_function=cost_func, neural_net=self.nnet) for i in range(10): w_grad, b_grad = calculator.compute_gradients() self.grad_descent.update_weights(weight_gradient=w_grad) self.grad_descent.update_biases(bias_gradient=b_grad) cost_after = cost_func.get_cost(self.examples) self.assertLess(cost_after, cost_before)
def step(context): mnist.download_dataset() context.training_data = PreloadSource(mnist.get_training_data()) context.test_data = PreloadSource(mnist.get_test_data())