def test_batch_norm_storage(self):
x_train, x_test, y_train, y_test = simple_classification()
batch_norm = layers.BatchNorm()
gdnet = algorithms.GradientDescent(
[
layers.Input(10),
layers.Relu(5),
batch_norm,
layers.Sigmoid(1),
],
batch_size=10,
verbose=True, # keep it as `True`
)
gdnet.train(x_train, y_train, epochs=5)
error_before_save = gdnet.score(x_test, y_test)
mean_before_save = self.eval(batch_norm.running_mean)
variance_before_save = self.eval(batch_norm.running_inv_std)
with tempfile.NamedTemporaryFile() as temp:
storage.save(gdnet, temp.name)
storage.load(gdnet, temp.name)
error_after_load = gdnet.score(x_test, y_test)
mean_after_load = self.eval(batch_norm.running_mean)
variance_after_load = self.eval(batch_norm.running_inv_std)
self.assertAlmostEqual(error_before_save, error_after_load)
np.testing.assert_array_almost_equal(mean_before_save,
mean_after_load)
np.testing.assert_array_almost_equal(variance_before_save,
variance_after_load)
def test_prelu_param_updates(self):
x_train, _, y_train, _ = simple_classification()
prelu_layer1 = layers.PRelu(20, alpha=0.25)
prelu_layer2 = layers.PRelu(1, alpha=0.25)
gdnet = algorithms.GradientDescent(
[
layers.Input(10),
prelu_layer1,
prelu_layer2,
],
batch_size=None,
)
prelu1_alpha_before_training = self.eval(prelu_layer1.alpha)
prelu2_alpha_before_training = self.eval(prelu_layer2.alpha)
gdnet.train(x_train, y_train, epochs=10)
prelu1_alpha_after_training = self.eval(prelu_layer1.alpha)
prelu2_alpha_after_training = self.eval(prelu_layer2.alpha)
self.assertTrue(
all(
np.not_equal(
prelu1_alpha_before_training,
prelu1_alpha_after_training,
)))
self.assertTrue(
all(
np.not_equal(
prelu2_alpha_before_training,
prelu2_alpha_after_training,
)))
def test_plot_errors_no_batch(self):
x_train, x_test, y_train, y_test = simple_classification()
optimizer = algorithms.Adadelta(self.network, batch_size=None)
optimizer.train(x_train, y_train, x_test, y_test, epochs=10)
optimizer.plot_errors(show=False)
return plt.gcf()
def test_adagrad(self):
x_train, x_test, y_train, y_test = simple_classification()
optimizer = algorithms.Adagrad(
self.network,
step=0.1,
batch_size=None,
verbose=False,
)
optimizer.train(x_train, y_train, x_test, y_test, epochs=150)
self.assertGreater(0.15, optimizer.errors.valid[-1])
def test_adadelta(self):
x_train, x_test, y_train, y_test = simple_classification()
optimizer = algorithms.Adadelta(
self.network,
batch_size=None,
verbose=False,
rho=0.95,
epsilon=1e-5,
)
optimizer.train(x_train, y_train, x_test, y_test, epochs=100)
self.assertGreater(0.05, optimizer.errors.train[-1])
self.assertGreater(0.15, optimizer.errors.valid[-1])
def test_rmsprop(self):
x_train, x_test, y_train, y_test = simple_classification()
optimizer = algorithms.RMSProp(
self.network,
step=0.02,
batch_size=None,
verbose=False,
epsilon=1e-5,
decay=0.9,
)
optimizer.train(x_train, y_train, x_test, y_test, epochs=150)
self.assertGreater(0.15, optimizer.errors.valid[-1])
def test_adam(self):
x_train, x_test, y_train, y_test = simple_classification()
optimizer = algorithms.Adam(
self.network,
step=0.1,
verbose=False,
epsilon=1e-4,
beta1=0.9,
beta2=0.99,
)
optimizer.train(x_train, y_train, x_test, y_test, epochs=200)
self.assertGreater(0.2, optimizer.errors.valid[-1])
def test_adamax(self):
x_train, x_test, y_train, y_test = simple_classification()
mnet = algorithms.Adamax(
self.network,
step=0.1,
batch_size=None,
verbose=False,
epsilon=1e-7,
beta1=0.9,
beta2=0.999,
)
mnet.train(x_train, y_train, x_test, y_test, epochs=50)
self.assertGreater(0.15, mnet.errors.train[-1])
def test_momentum(self):
x_train, x_test, y_train, y_test = simple_classification()
optimizer = algorithms.Momentum(
self.network,
step=0.35,
momentum=0.99,
batch_size=None,
verbose=False,
nesterov=True,
)
optimizer.train(x_train, y_train, x_test, y_test, epochs=30)
self.assertGreater(0.15, optimizer.errors.valid[-1])
def test_training_with_l2_regularization(self):
x_train, x_test, y_train, y_test = simple_classification()
mnet = algorithms.Momentum(
[layers.Input(10),
layers.Sigmoid(20),
layers.Sigmoid(1)],
step=0.35,
momentum=0.99,
batch_size=None,
verbose=False,
nesterov=True,
regularizer=algorithms.l2(0.001),
)
mnet.train(x_train, y_train, x_test, y_test, epochs=40)
self.assertGreater(0.15, mnet.errors.valid[-1])
def test_hessdiag(self):
x_train, x_test, y_train, y_test = simple_classification()
params = dict(weight=init.Uniform(-0.1, 0.1),
bias=init.Uniform(-0.1, 0.1))
nw = algorithms.HessianDiagonal(
network=[
layers.Input(10),
layers.Sigmoid(20, **params),
layers.Sigmoid(1, **params),
],
step=0.1,
shuffle_data=False,
verbose=False,
min_eigval=0.1,
)
nw.train(x_train, y_train, epochs=50)
self.assertGreater(0.2, nw.errors.train[-1])
def test_compare_bp_and_hessian(self):
x_train, x_test, y_train, y_test = simple_classification()
compare_networks(
# Test classes
partial(algorithms.GradientDescent, batch_size=None),
partial(algorithms.Hessian, penalty_const=1),
# Test data
(x_train, y_train, x_test, y_test),
# Network configurations
network=[layers.Input(10),
layers.Sigmoid(15),
layers.Sigmoid(1)],
shuffle_data=True,
verbose=False,
show_epoch=1,
# Test configurations
epochs=5,
show_comparison_plot=False)
def test_plot_errors_show_triggered_automatically(self):
x_train, x_test, y_train, y_test = simple_classification()
optimizer = algorithms.Adadelta(
self.network,
shuffle_data=True,
batch_size=10,
)
optimizer.train(x_train, y_train, epochs=100)
events = []
def mocked_show(*args, **kwargs):
events.append('show')
with mock.patch('matplotlib.pyplot.show', side_effect=mocked_show):
optimizer.plot_errors(show=True)
self.assertSequenceEqual(events, ['show'])
return plt.gcf()
def test_conjgrad(self):
cgnet = algorithms.ConjugateGradient(
[
layers.Input(10),
layers.Sigmoid(5),
layers.Sigmoid(1),
],
loss='binary_crossentropy',
shuffle_data=True,
verbose=False,
update_function='fletcher_reeves',
)
x_train, x_test, y_train, y_test = simple_classification()
cgnet.train(x_train, y_train, x_test, y_test, epochs=50)
actual_prediction = cgnet.predict(x_test).round().T
error = metrics.accuracy_score(actual_prediction[0], y_test)
self.assertAlmostEqual(error, 0.9, places=1)
def test_quasi_newton_dfp(self):
x_train, x_test, y_train, y_test = simple_classification()
qnnet = algorithms.QuasiNewton(
network=[
layers.Input(10),
layers.Sigmoid(30, weight=init.Orthogonal()),
layers.Sigmoid(1, weight=init.Orthogonal()),
],
shuffle_data=True,
verbose=False,
update_function='dfp',
h0_scale=2,
)
qnnet.train(x_train, y_train, x_test, y_test, epochs=10)
result = qnnet.predict(x_test).round()
roc_curve_score = metrics.roc_auc_score(result, y_test)
self.assertAlmostEqual(0.92, roc_curve_score, places=2)
def test_momentum_with_minibatch(self):
x_train, _, y_train, _ = simple_classification()
compare_networks(
# Test classes
partial(algorithms.Momentum, batch_size=None),
partial(algorithms.Momentum, batch_size=1),
# Test data
(x_train, y_train),
# Network configurations
network=[
layers.Input(10),
layers.Sigmoid(20),
layers.Sigmoid(1)
],
step=0.25,
momentum=0.1,
shuffle_data=True,
verbose=False,
# Test configurations
epochs=40,
show_comparison_plot=False,
)
def test_compare_bp_and_hessian(self):
x_train, _, y_train, _ = simple_classification()
params = dict(weight=init.Uniform(-0.1, 0.1),
bias=init.Uniform(-0.1, 0.1))
compare_networks(
# Test classes
partial(algorithms.GradientDescent, batch_size=None),
partial(algorithms.HessianDiagonal, min_eigval=0.1),
# Test data
(x_train, y_train),
# Network configurations
network=[
layers.Input(10),
layers.Sigmoid(20, **params),
layers.Sigmoid(1, **params),
],
step=0.1,
shuffle_data=True,
verbose=False,
# Test configurations
epochs=50,
show_comparison_plot=False)
def test_storage_pickle_save_and_load_during_the_training(self):
tempdir = tempfile.mkdtemp()
x_train, x_test, y_train, y_test = simple_classification()
errors = {}
def on_epoch_end(network):
epoch = network.last_epoch
errors[epoch] = network.score(x_test, y_test)
if epoch == 4:
storage.load_pickle(
network.network,
os.path.join(tempdir, 'training-epoch-2'))
raise StopTraining('Stop training process after 4th epoch')
else:
storage.save_pickle(
network.network,
os.path.join(tempdir, 'training-epoch-{}'.format(epoch)))
gdnet = algorithms.GradientDescent(
network=[
layers.Input(10),
layers.Sigmoid(4),
layers.Sigmoid(1)
],
signals=on_epoch_end,
batch_size=None,
step=0.5
)
gdnet.train(x_train, y_train)
validation_error = gdnet.score(x_test, y_test)
self.assertGreater(errors[2], errors[4])
self.assertAlmostEqual(validation_error, errors[2])
self.assertNotAlmostEqual(validation_error, errors[4])
def test_compare_bp_and_cg(self):
x_train, x_test, y_train, y_test = simple_classification()
compare_networks(
# Test classes
partial(
partial(algorithms.GradientDescent, batch_size=None),
step=1.0,
),
partial(algorithms.ConjugateGradient,
update_function='fletcher_reeves'),
# Test data
(asfloat(x_train), asfloat(y_train)),
# Network configurations
network=layers.join(
layers.Input(10),
layers.Sigmoid(5),
layers.Sigmoid(1),
),
loss='mse',
shuffle_data=True,
# Test configurations
epochs=50,
show_comparison_plot=False)
def test_gd_custom_target(self):
def custom_loss(actual, predicted):
actual_shape = tf.shape(actual)
n_samples = actual_shape[0]
actual = tf.reshape(actual, (n_samples, 1))
return objectives.rmse(actual, predicted)
optimizer = algorithms.GradientDescent(
layers.Input(10) >> layers.Sigmoid(1),
step=0.2,
shuffle_data=True,
batch_size=None,
loss=custom_loss,
target=tf.placeholder(tf.float32, shape=(None, 1, 1)),
)
x_train, _, y_train, _ = simple_classification()
error_message = "Cannot feed value of shape \(60, 1\) for Tensor"
with self.assertRaisesRegexp(ValueError, error_message):
optimizer.train(x_train, y_train, epochs=1)
optimizer.train(x_train, y_train.reshape(-1, 1, 1), epochs=1)
def assert_invalid_step_values(self, step, initial_value,
final_value, epochs):
x_train, x_test, y_train, y_test = simple_classification()
optimizer = algorithms.Momentum(
[
layers.Input(10),
layers.Sigmoid(5),
layers.Sigmoid(1),
],
step=step,
momentum=0.99,
batch_size=None,
verbose=False,
nesterov=True,
)
step = self.eval(optimizer.step)
self.assertAlmostEqual(step, initial_value)
optimizer.train(x_train, y_train, x_test, y_test, epochs=epochs)
step = self.eval(optimizer.step)
self.assertAlmostEqual(step, final_value)
