def test_batch_norm_storage(self): x_train, x_test, y_train, y_test = simple_classification() batch_norm = layers.BatchNorm() gdnet = algorithms.GradientDescent( [ layers.Input(10), layers.Relu(5), batch_norm, layers.Sigmoid(1), ], batch_size=10, verbose=True, # keep it as `True` ) gdnet.train(x_train, y_train, epochs=5) error_before_save = gdnet.score(x_test, y_test) mean_before_save = self.eval(batch_norm.running_mean) variance_before_save = self.eval(batch_norm.running_inv_std) with tempfile.NamedTemporaryFile() as temp: storage.save(gdnet, temp.name) storage.load(gdnet, temp.name) error_after_load = gdnet.score(x_test, y_test) mean_after_load = self.eval(batch_norm.running_mean) variance_after_load = self.eval(batch_norm.running_inv_std) self.assertAlmostEqual(error_before_save, error_after_load) np.testing.assert_array_almost_equal(mean_before_save, mean_after_load) np.testing.assert_array_almost_equal(variance_before_save, variance_after_load)
def test_prelu_param_updates(self): x_train, _, y_train, _ = simple_classification() prelu_layer1 = layers.PRelu(20, alpha=0.25) prelu_layer2 = layers.PRelu(1, alpha=0.25) gdnet = algorithms.GradientDescent( [ layers.Input(10), prelu_layer1, prelu_layer2, ], batch_size=None, ) prelu1_alpha_before_training = self.eval(prelu_layer1.alpha) prelu2_alpha_before_training = self.eval(prelu_layer2.alpha) gdnet.train(x_train, y_train, epochs=10) prelu1_alpha_after_training = self.eval(prelu_layer1.alpha) prelu2_alpha_after_training = self.eval(prelu_layer2.alpha) self.assertTrue( all( np.not_equal( prelu1_alpha_before_training, prelu1_alpha_after_training, ))) self.assertTrue( all( np.not_equal( prelu2_alpha_before_training, prelu2_alpha_after_training, )))
def test_plot_errors_no_batch(self): x_train, x_test, y_train, y_test = simple_classification() optimizer = algorithms.Adadelta(self.network, batch_size=None) optimizer.train(x_train, y_train, x_test, y_test, epochs=10) optimizer.plot_errors(show=False) return plt.gcf()
def test_adagrad(self): x_train, x_test, y_train, y_test = simple_classification() optimizer = algorithms.Adagrad( self.network, step=0.1, batch_size=None, verbose=False, ) optimizer.train(x_train, y_train, x_test, y_test, epochs=150) self.assertGreater(0.15, optimizer.errors.valid[-1])
def test_adadelta(self): x_train, x_test, y_train, y_test = simple_classification() optimizer = algorithms.Adadelta( self.network, batch_size=None, verbose=False, rho=0.95, epsilon=1e-5, ) optimizer.train(x_train, y_train, x_test, y_test, epochs=100) self.assertGreater(0.05, optimizer.errors.train[-1]) self.assertGreater(0.15, optimizer.errors.valid[-1])
def test_rmsprop(self): x_train, x_test, y_train, y_test = simple_classification() optimizer = algorithms.RMSProp( self.network, step=0.02, batch_size=None, verbose=False, epsilon=1e-5, decay=0.9, ) optimizer.train(x_train, y_train, x_test, y_test, epochs=150) self.assertGreater(0.15, optimizer.errors.valid[-1])
def test_adam(self): x_train, x_test, y_train, y_test = simple_classification() optimizer = algorithms.Adam( self.network, step=0.1, verbose=False, epsilon=1e-4, beta1=0.9, beta2=0.99, ) optimizer.train(x_train, y_train, x_test, y_test, epochs=200) self.assertGreater(0.2, optimizer.errors.valid[-1])
def test_adamax(self): x_train, x_test, y_train, y_test = simple_classification() mnet = algorithms.Adamax( self.network, step=0.1, batch_size=None, verbose=False, epsilon=1e-7, beta1=0.9, beta2=0.999, ) mnet.train(x_train, y_train, x_test, y_test, epochs=50) self.assertGreater(0.15, mnet.errors.train[-1])
def test_momentum(self): x_train, x_test, y_train, y_test = simple_classification() optimizer = algorithms.Momentum( self.network, step=0.35, momentum=0.99, batch_size=None, verbose=False, nesterov=True, ) optimizer.train(x_train, y_train, x_test, y_test, epochs=30) self.assertGreater(0.15, optimizer.errors.valid[-1])
def test_training_with_l2_regularization(self): x_train, x_test, y_train, y_test = simple_classification() mnet = algorithms.Momentum( [layers.Input(10), layers.Sigmoid(20), layers.Sigmoid(1)], step=0.35, momentum=0.99, batch_size=None, verbose=False, nesterov=True, regularizer=algorithms.l2(0.001), ) mnet.train(x_train, y_train, x_test, y_test, epochs=40) self.assertGreater(0.15, mnet.errors.valid[-1])
def test_hessdiag(self): x_train, x_test, y_train, y_test = simple_classification() params = dict(weight=init.Uniform(-0.1, 0.1), bias=init.Uniform(-0.1, 0.1)) nw = algorithms.HessianDiagonal( network=[ layers.Input(10), layers.Sigmoid(20, **params), layers.Sigmoid(1, **params), ], step=0.1, shuffle_data=False, verbose=False, min_eigval=0.1, ) nw.train(x_train, y_train, epochs=50) self.assertGreater(0.2, nw.errors.train[-1])
def test_compare_bp_and_hessian(self): x_train, x_test, y_train, y_test = simple_classification() compare_networks( # Test classes partial(algorithms.GradientDescent, batch_size=None), partial(algorithms.Hessian, penalty_const=1), # Test data (x_train, y_train, x_test, y_test), # Network configurations network=[layers.Input(10), layers.Sigmoid(15), layers.Sigmoid(1)], shuffle_data=True, verbose=False, show_epoch=1, # Test configurations epochs=5, show_comparison_plot=False)
def test_plot_errors_show_triggered_automatically(self): x_train, x_test, y_train, y_test = simple_classification() optimizer = algorithms.Adadelta( self.network, shuffle_data=True, batch_size=10, ) optimizer.train(x_train, y_train, epochs=100) events = [] def mocked_show(*args, **kwargs): events.append('show') with mock.patch('matplotlib.pyplot.show', side_effect=mocked_show): optimizer.plot_errors(show=True) self.assertSequenceEqual(events, ['show']) return plt.gcf()
def test_conjgrad(self): cgnet = algorithms.ConjugateGradient( [ layers.Input(10), layers.Sigmoid(5), layers.Sigmoid(1), ], loss='binary_crossentropy', shuffle_data=True, verbose=False, update_function='fletcher_reeves', ) x_train, x_test, y_train, y_test = simple_classification() cgnet.train(x_train, y_train, x_test, y_test, epochs=50) actual_prediction = cgnet.predict(x_test).round().T error = metrics.accuracy_score(actual_prediction[0], y_test) self.assertAlmostEqual(error, 0.9, places=1)
def test_quasi_newton_dfp(self): x_train, x_test, y_train, y_test = simple_classification() qnnet = algorithms.QuasiNewton( network=[ layers.Input(10), layers.Sigmoid(30, weight=init.Orthogonal()), layers.Sigmoid(1, weight=init.Orthogonal()), ], shuffle_data=True, verbose=False, update_function='dfp', h0_scale=2, ) qnnet.train(x_train, y_train, x_test, y_test, epochs=10) result = qnnet.predict(x_test).round() roc_curve_score = metrics.roc_auc_score(result, y_test) self.assertAlmostEqual(0.92, roc_curve_score, places=2)
def test_momentum_with_minibatch(self): x_train, _, y_train, _ = simple_classification() compare_networks( # Test classes partial(algorithms.Momentum, batch_size=None), partial(algorithms.Momentum, batch_size=1), # Test data (x_train, y_train), # Network configurations network=[ layers.Input(10), layers.Sigmoid(20), layers.Sigmoid(1) ], step=0.25, momentum=0.1, shuffle_data=True, verbose=False, # Test configurations epochs=40, show_comparison_plot=False, )
def test_compare_bp_and_hessian(self): x_train, _, y_train, _ = simple_classification() params = dict(weight=init.Uniform(-0.1, 0.1), bias=init.Uniform(-0.1, 0.1)) compare_networks( # Test classes partial(algorithms.GradientDescent, batch_size=None), partial(algorithms.HessianDiagonal, min_eigval=0.1), # Test data (x_train, y_train), # Network configurations network=[ layers.Input(10), layers.Sigmoid(20, **params), layers.Sigmoid(1, **params), ], step=0.1, shuffle_data=True, verbose=False, # Test configurations epochs=50, show_comparison_plot=False)
def test_storage_pickle_save_and_load_during_the_training(self): tempdir = tempfile.mkdtemp() x_train, x_test, y_train, y_test = simple_classification() errors = {} def on_epoch_end(network): epoch = network.last_epoch errors[epoch] = network.score(x_test, y_test) if epoch == 4: storage.load_pickle( network.network, os.path.join(tempdir, 'training-epoch-2')) raise StopTraining('Stop training process after 4th epoch') else: storage.save_pickle( network.network, os.path.join(tempdir, 'training-epoch-{}'.format(epoch))) gdnet = algorithms.GradientDescent( network=[ layers.Input(10), layers.Sigmoid(4), layers.Sigmoid(1) ], signals=on_epoch_end, batch_size=None, step=0.5 ) gdnet.train(x_train, y_train) validation_error = gdnet.score(x_test, y_test) self.assertGreater(errors[2], errors[4]) self.assertAlmostEqual(validation_error, errors[2]) self.assertNotAlmostEqual(validation_error, errors[4])
def test_compare_bp_and_cg(self): x_train, x_test, y_train, y_test = simple_classification() compare_networks( # Test classes partial( partial(algorithms.GradientDescent, batch_size=None), step=1.0, ), partial(algorithms.ConjugateGradient, update_function='fletcher_reeves'), # Test data (asfloat(x_train), asfloat(y_train)), # Network configurations network=layers.join( layers.Input(10), layers.Sigmoid(5), layers.Sigmoid(1), ), loss='mse', shuffle_data=True, # Test configurations epochs=50, show_comparison_plot=False)
def test_gd_custom_target(self): def custom_loss(actual, predicted): actual_shape = tf.shape(actual) n_samples = actual_shape[0] actual = tf.reshape(actual, (n_samples, 1)) return objectives.rmse(actual, predicted) optimizer = algorithms.GradientDescent( layers.Input(10) >> layers.Sigmoid(1), step=0.2, shuffle_data=True, batch_size=None, loss=custom_loss, target=tf.placeholder(tf.float32, shape=(None, 1, 1)), ) x_train, _, y_train, _ = simple_classification() error_message = "Cannot feed value of shape \(60, 1\) for Tensor" with self.assertRaisesRegexp(ValueError, error_message): optimizer.train(x_train, y_train, epochs=1) optimizer.train(x_train, y_train.reshape(-1, 1, 1), epochs=1)
def assert_invalid_step_values(self, step, initial_value, final_value, epochs): x_train, x_test, y_train, y_test = simple_classification() optimizer = algorithms.Momentum( [ layers.Input(10), layers.Sigmoid(5), layers.Sigmoid(1), ], step=step, momentum=0.99, batch_size=None, verbose=False, nesterov=True, ) step = self.eval(optimizer.step) self.assertAlmostEqual(step, initial_value) optimizer.train(x_train, y_train, x_test, y_test, epochs=epochs) step = self.eval(optimizer.step) self.assertAlmostEqual(step, final_value)