def test_nn_init_logging(self): with catch_stdout() as out: gdnet = algorithms.GradientDescent((2, 3, 1), verbose=False) terminal_output = out.getvalue() self.assertEqual("", terminal_output.strip()) with catch_stdout() as out: gdnet = algorithms.GradientDescent((2, 3, 1), verbose=True) terminal_output = out.getvalue() self.assertNotEqual("", terminal_output.strip()) self.assertIn("verbose = True", terminal_output)
def test_shared_parameters_between_layers(self): hidden_layer_1 = layers.Relu(10) network = layers.Input(10) > hidden_layer_1 hidden_layer_2 = layers.Relu( size=10, weight=hidden_layer_1.weight, bias=hidden_layer_1.bias) network = network > hidden_layer_2 self.assertIs(hidden_layer_1.weight, hidden_layer_2.weight) self.assertIs(hidden_layer_1.bias, hidden_layer_2.bias) # Check that it is able to train network without errors x_train = y_train = asfloat(np.random.random((15, 10))) gdnet = algorithms.GradientDescent(network, batch_size='all') gdnet.train(x_train, y_train, epochs=5) np.testing.assert_array_almost_equal( self.eval(hidden_layer_1.weight), self.eval(hidden_layer_2.weight), ) np.testing.assert_array_almost_equal( self.eval(hidden_layer_1.bias), self.eval(hidden_layer_2.bias), )
def test_prelu_param_updates(self): x_train, _, y_train, _ = simple_classification() prelu_layer1 = layers.PRelu(20, alpha=0.25) prelu_layer2 = layers.PRelu(1, alpha=0.25) gdnet = algorithms.GradientDescent([ layers.Input(10), prelu_layer1, prelu_layer2, ]) prelu1_alpha_before_training = prelu_layer1.alpha.get_value() prelu2_alpha_before_training = prelu_layer2.alpha.get_value() gdnet.train(x_train, y_train, epochs=10) prelu1_alpha_after_training = prelu_layer1.alpha.get_value() prelu2_alpha_after_training = prelu_layer2.alpha.get_value() self.assertTrue( all( np.not_equal( prelu1_alpha_before_training, prelu1_alpha_after_training, ))) self.assertTrue( all( np.not_equal( prelu2_alpha_before_training, prelu2_alpha_after_training, )))
def test_pandas_for_bp(self): dataset = datasets.load_diabetes() target = dataset.target.reshape(-1, 1) input_scaler = preprocessing.MinMaxScaler() target_scaler = preprocessing.MinMaxScaler() n_features = dataset.data.shape[1] input_columns = ['column_' + str(i) for i in range(n_features)] pandas_data = pd.DataFrame(dataset.data, columns=input_columns) pandas_data['target'] = target_scaler.fit_transform(target) pandas_data[input_columns] = input_scaler.fit_transform( pandas_data[input_columns]) x_train, x_test, y_train, y_test = train_test_split( pandas_data[input_columns], pandas_data['target'], test_size=0.15) bpnet = algorithms.GradientDescent(connection=[ layers.Input(10), layers.Sigmoid(30), layers.Sigmoid(1), ], show_epoch=100) bpnet.train(x_train, y_train, epochs=50) y_predict = bpnet.predict(x_test).reshape(-1, 1) y_test = y_test.values.reshape(-1, 1) error = estimators.rmsle( target_scaler.inverse_transform(y_test), target_scaler.inverse_transform(y_predict).round()) self.assertAlmostEqual(0.48, error, places=2)
def test_pipeline(self): dataset = datasets.load_diabetes() target_scaler = preprocessing.MinMaxScaler() target = dataset.target.reshape(-1, 1) x_train, x_test, y_train, y_test = train_test_split( asfloat(dataset.data), asfloat(target_scaler.fit_transform(target)), test_size=0.15 ) network = algorithms.GradientDescent( network=[ layers.Input(10), layers.Sigmoid(25), layers.Sigmoid(1), ], batch_size=None, show_epoch=100, verbose=False, ) pipeline = Pipeline([ ('min_max_scaler', preprocessing.MinMaxScaler()), ('gd', network), ]) pipeline.fit(x_train, y_train, gd__epochs=50) y_predict = pipeline.predict(x_test) error = objectives.rmsle( target_scaler.inverse_transform(y_test), target_scaler.inverse_transform(y_predict).round() ) error = self.eval(error) self.assertGreater(0.5, error)
def setUp(self): super(MixtureOfExpertsTestCase, self).setUp() self.networks = [ algorithms.GradientDescent((1, 20, 1), step=0.2, verbose=False), layers.join(layers.Input(1), layers.Sigmoid(20), layers.Sigmoid(1)), ]
def test_summary_table_delay_limit(self): with catch_stdout() as out: network = algorithms.GradientDescent((3, 2), verbose=True) network.train(simple_input_train, simple_target_train, epochs=20) terminal_output = out.getvalue() self.assertIn("Too many outputs", terminal_output)
def test_storage_pickle_save_and_load_during_the_training(self): tempdir = tempfile.mkdtemp() x_train, x_test, y_train, y_test = simple_classification() errors = {} def on_epoch_end(network): epoch = network.last_epoch errors[epoch] = network.prediction_error(x_test, y_test) if epoch == 4: storage.load_pickle( network.connection, os.path.join(tempdir, 'training-epoch-2')) raise StopTraining('Stop training process after 4th epoch') else: storage.save_pickle( network.connection, os.path.join(tempdir, 'training-epoch-{}'.format(epoch))) gdnet = algorithms.GradientDescent( connection=(10, 4, 1), epoch_end_signal=on_epoch_end, step=0.5 ) gdnet.train(x_train, y_train) validation_error = gdnet.prediction_error(x_test, y_test) self.assertGreater(errors[2], errors[4]) self.assertAlmostEqual(validation_error, errors[2]) self.assertNotAlmostEqual(validation_error, errors[4])
def gen_clf_neupy(datapath): x_train, y_train, x_test, y_test, y_train_label, y_test_label = get_data( datapath) network = layers.join( layers.Input((48)), layers.Relu(20), layers.Softmax(36), ) clf = algorithms.GradientDescent( network, step=0.1, shuffle_data=True, verbose=True, error='mse', # momentum=0.99 ) # x_train = np.array(x_train) # print x_train.shape clf.train(x_train, y_train, x_test, y_test, epochs=15) # plots.error_plot(clf) # print clf ypredicted = clf.predict(x_test) for i in range(0, len(y_test)): if np.argmax(y_test[i]) == np.argmax(ypredicted[i]): index = np.argmax(y_test[i]) print "the character is: " + str(dict[str(index)]) return clf
def test_batch_norm_storage(self): x_train, x_test, y_train, y_test = simple_classification() batch_norm = layers.BatchNorm() gdnet = algorithms.GradientDescent( [ layers.Input(10), layers.Relu(5), batch_norm, layers.Sigmoid(1), ], batch_size=10, verbose=True, # keep it as `True` ) gdnet.train(x_train, y_train, epochs=5) error_before_save = gdnet.prediction_error(x_test, y_test) mean_before_save = self.eval(batch_norm.running_mean) variance_before_save = self.eval(batch_norm.running_inv_std) with tempfile.NamedTemporaryFile() as temp: storage.save(gdnet, temp.name) storage.load(gdnet, temp.name) error_after_load = gdnet.prediction_error(x_test, y_test) mean_after_load = self.eval(batch_norm.running_mean) variance_after_load = self.eval(batch_norm.running_inv_std) self.assertAlmostEqual(error_before_save, error_after_load) np.testing.assert_array_almost_equal(mean_before_save, mean_after_load) np.testing.assert_array_almost_equal(variance_before_save, variance_after_load)
def reproducible_network_train(seed=0, epochs=500, **additional_params): """ Make a reproducible train for Gradient Descent based neural network with a XOR problem and return trained network. Parameters ---------- seed : int Random State seed number for reproducibility. Defaults to ``0``. epochs : int Number of epochs for training. Defaults to ``500``. **additional_params Aditional parameters for Neural Network. Returns ------- GradientDescent instance Returns trained network. """ np.random.seed(seed) network = algorithms.GradientDescent(connection=[ layers.Input(2), layers.Tanh(5), layers.Tanh(1), StepOutput(), ], **additional_params) network.train(xor_input_train, xor_target_train, epochs=epochs) return network
def test_training_with_multiple_inputs(self): network = algorithms.GradientDescent( [ [ layers.Input(2) > layers.Sigmoid(3), layers.Input(3) > layers.Sigmoid(5), ], layers.Concatenate(), layers.Sigmoid(1), ], step=0.1, verbose=False, shuffle_data=True, ) x_train, x_test, y_train, y_test = simple_classification(n_samples=100, n_features=5) x_train_2, x_train_3 = x_train[:, :2], x_train[:, 2:] x_test_2, x_test_3 = x_test[:, :2], x_test[:, 2:] network.train([x_train_2, x_train_3], y_train, [x_test_2, x_test_3], y_test, epochs=100) error = network.validation_errors[-1] self.assertAlmostEqual(error, 0.14, places=2)
def reproducible_network_train(seed=0, epochs=500, **additional_params): """ Make a reproducible train for Gradient Descent based neural network with a XOR problem and return trained network. Parameters ---------- seed : int Random State seed number for reproducibility. Defaults to ``0``. epochs : int Number of epochs for training. Defaults to ``500``. **additional_params Aditional parameters for Neural Network. Returns ------- GradientDescent instance Returns trained network. """ environment.reproducible(seed) xavier_normal = init.XavierNormal() tanh_weight1 = xavier_normal.sample((2, 5), return_array=True) tanh_weight2 = xavier_normal.sample((5, 1), return_array=True) network = algorithms.GradientDescent(connection=[ layers.Input(2), layers.Tanh(5, weight=tanh_weight1), layers.Tanh(1, weight=tanh_weight2), ], batch_size='all', **additional_params) network.train(xor_input_train, xor_target_train, epochs=epochs) return network
def test_pipeline(self): dataset = datasets.load_diabetes() target_scaler = preprocessing.MinMaxScaler() target = dataset.target.reshape(-1, 1) x_train, x_test, y_train, y_test = train_test_split( dataset.data, target_scaler.fit_transform(target), train_size=0.85) network = algorithms.GradientDescent( connection=[ layers.Sigmoid(10), layers.Sigmoid(25), layers.Output(1), ], show_epoch=100, verbose=False, ) pipeline = Pipeline([ ('min_max_scaler', preprocessing.MinMaxScaler()), ('gd', network), ]) pipeline.fit(x_train, y_train, gd__epochs=50) y_predict = pipeline.predict(x_test) error = rmsle(target_scaler.inverse_transform(y_test), target_scaler.inverse_transform(y_predict).round()) self.assertAlmostEqual(0.47, error, places=2)
def test_sew_together_when_cutted_piece_already_in_use(self): autoencoder = algorithms.Momentum([ layers.Input(25), layers.Sigmoid(15), layers.Sigmoid(25), ]) encoder = surgery.cut(autoencoder, start=0, end=2) self.assertEqual(len(encoder), 2) classifier = algorithms.Momentum(encoder > layers.Softmax(10)) network = algorithms.GradientDescent([ layers.Input(5), surgery.CutLine(), # <- first cut point layers.Sigmoid(10), layers.Sigmoid(20), layers.Sigmoid(30), surgery.CutLine(), # <- second cut point layers.Sigmoid(1), ]) _, hidden_layers, _ = surgery.cut_along_lines(network) self.assertEqual(len(hidden_layers), 3) connected_layers = surgery.sew_together([ encoder, layers.Relu(5), hidden_layers ]) self.assertEqual(len(connected_layers), 6)
def test_network_training_summary(self): with catch_stdout() as out: network = algorithms.GradientDescent( layers.join( layers.Input(2), layers.Sigmoid(3), layers.Sigmoid(1), ), verbose=False, batch_size=None, ) x = np.zeros((5, 2)) y = np.zeros((5, 1)) network.verbose = True n_epochs = 10 network.train(x, y, epochs=n_epochs) terminal_output = out.getvalue().strip() # `n_epochs - 1` because \n appears only between # inline summary lines. # Also network prints 5 additional lines at the beggining self.assertEqual(terminal_output.count('\n'), n_epochs - 1) self.assertEqual(terminal_output.count('train: '), n_epochs)
def test_show_epoch_valid_cases(self): Case = namedtuple("Case", "show_epoch should_be_n_times n_epochs") cases = ( # Show 10 epochs and the last one would be 11 Case(show_epoch='10 times', should_be_n_times=11, n_epochs=100), Case(show_epoch='1 time', should_be_n_times=2, n_epochs=10), Case(show_epoch='1 times', should_be_n_times=2, n_epochs=10), # Should be equal to the number of epochs Case(show_epoch='100 times', should_be_n_times=10, n_epochs=10), Case(show_epoch=5, should_be_n_times=3, n_epochs=10), Case(show_epoch=100, should_be_n_times=2, n_epochs=10), ) for case in cases: with catch_stdout() as out: bpnet = algorithms.GradientDescent( (2, 3, 1), step=0.1, verbose=True, show_epoch=case.show_epoch ) bpnet.train(xor_zero_input_train, xor_zero_target_train, epochs=case.n_epochs) terminal_output = out.getvalue() # One of the choices has to be true whether other # choices should give count equal to zero. time_counts = ( terminal_output.count(" μs ") + terminal_output.count(" ms ") + terminal_output.count(" ns ") ) self.assertEqual(case.should_be_n_times, time_counts)
def test_print_training_progress_signal(self): x_train = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]) y_train = np.array([[1, 0, 0, 1]]).T Case = namedtuple("Case", "show_epoch should_be_n_times n_epochs") cases = ( Case(show_epoch=5, should_be_n_times=3, n_epochs=10), Case(show_epoch=7, should_be_n_times=3, n_epochs=10), Case(show_epoch=100, should_be_n_times=2, n_epochs=10), ) for case in cases: with catch_stdout() as out: bpnet = algorithms.GradientDescent( layers.join( layers.Input(2), layers.Sigmoid(3), layers.Sigmoid(1), ), step=0.1, verbose=True, batch_size=None, show_epoch=case.show_epoch, ) bpnet.train(x_train, y_train, epochs=case.n_epochs) terminal_output = out.getvalue() # One of the choices has to be true whether other # choices should give count equal to zero. time_counts = (terminal_output.count(" μs]") + terminal_output.count(" ms]") + terminal_output.count(" ns]")) self.assertEqual(case.should_be_n_times, time_counts)
def test_error_plot_ax_none(self): ax = plt.gca() network = algorithms.GradientDescent((2, 3, 1)) ax_returned = plots.error_plot(network, ax=None, show=False) self.assertIs(ax_returned, ax)
def test_logging_info_about_the_data(self): network = algorithms.GradientDescent((2, 3, 1)) x = np.zeros((5, 2)) x_test = np.zeros((3, 2)) y = np.zeros((4, 1)) with self.assertRaisesRegexp(ValueError, "feature shape"): logging_info_about_the_data(network, x, y) with catch_stdout() as out: network = algorithms.GradientDescent((2, 3, 1), verbose=True) logging_info_about_the_data(network, [x, x], [x_test, x_test]) terminal_output = out.getvalue() self.assertIn("[(5, 2), (5, 2)]", terminal_output) self.assertIn("[(3, 2), (3, 2)]", terminal_output)
def test_custom_error_functions(self): # Test that everything works without fail def custom_mse(expected, predicted): return (0.5 * (predicted - expected)**2).mean() x_train, _, y_train, _ = simple_classification() gdnet = algorithms.GradientDescent((10, 10, 1), error=custom_mse) gdnet.train(x_train, y_train)
def test_network_training_with_unknown_summary_type(self): network = algorithms.GradientDescent((2, 3, 1)) x = np.zeros((5, 2)) y = np.zeros((5, 1)) with self.assertRaises(ValueError): network.train(x, y, summary='unknown')
def setUp(self): super(SurgeryCutTestCase, self).setUp() self.network = algorithms.GradientDescent([ layers.Input(30), layers.Sigmoid(10), layers.Sigmoid(20), layers.Sigmoid(1), ])
def test_gd_predict_wrong_arguments(self): optimizer = algorithms.GradientDescent( layers.Input(10) >> layers.Sigmoid(1), verbose=False, ) input = np.random.random((7, 10)) with self.assertRaisesRegexp(TypeError, "Unknown arguments"): optimizer.predict(input, batchsize=10)
def test_gd(self): x_train, _, y_train, _ = simple_classification() network = algorithms.GradientDescent( layers.Input(10) > layers.Tanh(20) > layers.Tanh(1), step=0.3, verbose=False) network.train(x_train, y_train, epochs=500) self.assertAlmostEqual(network.errors.last(), 0.014, places=3)
def test_init_logging(self): with catch_stdout() as out: algorithms.GradientDescent( layers.Input(2) > layers.Sigmoid(3) > layers.Sigmoid(1), verbose=False, ) terminal_output = out.getvalue() self.assertEqual("", terminal_output.strip()) with catch_stdout() as out: algorithms.GradientDescent( layers.Input(2) > layers.Sigmoid(3) > layers.Sigmoid(1), verbose=True, ) terminal_output = out.getvalue() self.assertNotEqual("", terminal_output.strip()) self.assertIn("verbose = True", terminal_output)
def test_error_plot_and_validation_error_warnings(self): with catch_stdout() as out: network = algorithms.GradientDescent((2, 3, 1), verbose=True) network.errors = ErrorHistoryList([1, 2]) network.validation_errors = ErrorHistoryList([None]) plots.error_plot(network, ax=None, show=False) terminal_output = out.getvalue() self.assertIn("error will be ignored", terminal_output)
def test_sew_together_cutted_pieces(self): network1 = algorithms.GradientDescent([ layers.Input(100), layers.Sigmoid(200), layers.Sigmoid(100), ]) network2 = algorithms.GradientDescent([ layers.Input(10), layers.Sigmoid(20), layers.Sigmoid(10), ]) first_part = surgery.cut(network1, start=0, end=2) self.assertEqual(first_part.output_shape, (200,)) self.assertEqual(first_part.input_shape, (100,)) second_part = surgery.cut(network2, start=0, end=2) self.assertEqual(second_part.output_shape, (20,)) self.assertEqual(second_part.input_shape, (10,))
def test_dan_repr(self): dan = algorithms.DynamicallyAveragedNetwork([ algorithms.Momentum((3, 2, 1)), algorithms.GradientDescent((3, 2, 1)), ]) dan_repr = str(dan) self.assertIn('DynamicallyAveragedNetwork', dan_repr) self.assertIn('Momentum', dan_repr) self.assertIn('GradientDescent', dan_repr)
def test_iter_until_converge_critical_cases(self): with catch_stdout() as out: network = algorithms.GradientDescent((2, 3, 1), verbose=True) iterator = iter_until_converge(network, epsilon=1e-5, max_epochs=5) for epoch in iterator: network.errors.append(np.nan) terminal_output = out.getvalue() self.assertIn('NaN or Inf', terminal_output)