def test_nn_init_logging(self):
with catch_stdout() as out:
gdnet = algorithms.GradientDescent((2, 3, 1), verbose=False)
terminal_output = out.getvalue()
self.assertEqual("", terminal_output.strip())
with catch_stdout() as out:
gdnet = algorithms.GradientDescent((2, 3, 1), verbose=True)
terminal_output = out.getvalue()
self.assertNotEqual("", terminal_output.strip())
self.assertIn("verbose = True", terminal_output)
def test_shared_parameters_between_layers(self):
hidden_layer_1 = layers.Relu(10)
network = layers.Input(10) > hidden_layer_1
hidden_layer_2 = layers.Relu(
size=10,
weight=hidden_layer_1.weight,
bias=hidden_layer_1.bias)
network = network > hidden_layer_2
self.assertIs(hidden_layer_1.weight, hidden_layer_2.weight)
self.assertIs(hidden_layer_1.bias, hidden_layer_2.bias)
# Check that it is able to train network without errors
x_train = y_train = asfloat(np.random.random((15, 10)))
gdnet = algorithms.GradientDescent(network, batch_size='all')
gdnet.train(x_train, y_train, epochs=5)
np.testing.assert_array_almost_equal(
self.eval(hidden_layer_1.weight),
self.eval(hidden_layer_2.weight),
)
np.testing.assert_array_almost_equal(
self.eval(hidden_layer_1.bias),
self.eval(hidden_layer_2.bias),
)
def test_prelu_param_updates(self):
x_train, _, y_train, _ = simple_classification()
prelu_layer1 = layers.PRelu(20, alpha=0.25)
prelu_layer2 = layers.PRelu(1, alpha=0.25)
gdnet = algorithms.GradientDescent([
layers.Input(10),
prelu_layer1,
prelu_layer2,
])
prelu1_alpha_before_training = prelu_layer1.alpha.get_value()
prelu2_alpha_before_training = prelu_layer2.alpha.get_value()
gdnet.train(x_train, y_train, epochs=10)
prelu1_alpha_after_training = prelu_layer1.alpha.get_value()
prelu2_alpha_after_training = prelu_layer2.alpha.get_value()
self.assertTrue(
all(
np.not_equal(
prelu1_alpha_before_training,
prelu1_alpha_after_training,
)))
self.assertTrue(
all(
np.not_equal(
prelu2_alpha_before_training,
prelu2_alpha_after_training,
)))
def test_pandas_for_bp(self):
dataset = datasets.load_diabetes()
target = dataset.target.reshape(-1, 1)
input_scaler = preprocessing.MinMaxScaler()
target_scaler = preprocessing.MinMaxScaler()
n_features = dataset.data.shape[1]
input_columns = ['column_' + str(i) for i in range(n_features)]
pandas_data = pd.DataFrame(dataset.data, columns=input_columns)
pandas_data['target'] = target_scaler.fit_transform(target)
pandas_data[input_columns] = input_scaler.fit_transform(
pandas_data[input_columns])
x_train, x_test, y_train, y_test = train_test_split(
pandas_data[input_columns], pandas_data['target'], test_size=0.15)
bpnet = algorithms.GradientDescent(connection=[
layers.Input(10),
layers.Sigmoid(30),
layers.Sigmoid(1),
],
show_epoch=100)
bpnet.train(x_train, y_train, epochs=50)
y_predict = bpnet.predict(x_test).reshape(-1, 1)
y_test = y_test.values.reshape(-1, 1)
error = estimators.rmsle(
target_scaler.inverse_transform(y_test),
target_scaler.inverse_transform(y_predict).round())
self.assertAlmostEqual(0.48, error, places=2)
def test_pipeline(self):
dataset = datasets.load_diabetes()
target_scaler = preprocessing.MinMaxScaler()
target = dataset.target.reshape(-1, 1)
x_train, x_test, y_train, y_test = train_test_split(
asfloat(dataset.data),
asfloat(target_scaler.fit_transform(target)),
test_size=0.15
)
network = algorithms.GradientDescent(
network=[
layers.Input(10),
layers.Sigmoid(25),
layers.Sigmoid(1),
],
batch_size=None,
show_epoch=100,
verbose=False,
)
pipeline = Pipeline([
('min_max_scaler', preprocessing.MinMaxScaler()),
('gd', network),
])
pipeline.fit(x_train, y_train, gd__epochs=50)
y_predict = pipeline.predict(x_test)
error = objectives.rmsle(
target_scaler.inverse_transform(y_test),
target_scaler.inverse_transform(y_predict).round()
)
error = self.eval(error)
self.assertGreater(0.5, error)
def setUp(self):
super(MixtureOfExpertsTestCase, self).setUp()
self.networks = [
algorithms.GradientDescent((1, 20, 1), step=0.2, verbose=False),
layers.join(layers.Input(1), layers.Sigmoid(20),
layers.Sigmoid(1)),
]
def test_summary_table_delay_limit(self):
with catch_stdout() as out:
network = algorithms.GradientDescent((3, 2), verbose=True)
network.train(simple_input_train, simple_target_train, epochs=20)
terminal_output = out.getvalue()
self.assertIn("Too many outputs", terminal_output)
def test_storage_pickle_save_and_load_during_the_training(self):
tempdir = tempfile.mkdtemp()
x_train, x_test, y_train, y_test = simple_classification()
errors = {}
def on_epoch_end(network):
epoch = network.last_epoch
errors[epoch] = network.prediction_error(x_test, y_test)
if epoch == 4:
storage.load_pickle(
network.connection,
os.path.join(tempdir, 'training-epoch-2'))
raise StopTraining('Stop training process after 4th epoch')
else:
storage.save_pickle(
network.connection,
os.path.join(tempdir, 'training-epoch-{}'.format(epoch)))
gdnet = algorithms.GradientDescent(
connection=(10, 4, 1),
epoch_end_signal=on_epoch_end,
step=0.5
)
gdnet.train(x_train, y_train)
validation_error = gdnet.prediction_error(x_test, y_test)
self.assertGreater(errors[2], errors[4])
self.assertAlmostEqual(validation_error, errors[2])
self.assertNotAlmostEqual(validation_error, errors[4])
def gen_clf_neupy(datapath):
x_train, y_train, x_test, y_test, y_train_label, y_test_label = get_data(
datapath)
network = layers.join(
layers.Input((48)),
layers.Relu(20),
layers.Softmax(36),
)
clf = algorithms.GradientDescent(
network,
step=0.1,
shuffle_data=True,
verbose=True,
error='mse',
# momentum=0.99
)
# x_train = np.array(x_train)
# print x_train.shape
clf.train(x_train, y_train, x_test, y_test, epochs=15)
# plots.error_plot(clf)
# print clf
ypredicted = clf.predict(x_test)
for i in range(0, len(y_test)):
if np.argmax(y_test[i]) == np.argmax(ypredicted[i]):
index = np.argmax(y_test[i])
print "the character is: " + str(dict[str(index)])
return clf
def test_batch_norm_storage(self):
x_train, x_test, y_train, y_test = simple_classification()
batch_norm = layers.BatchNorm()
gdnet = algorithms.GradientDescent(
[
layers.Input(10),
layers.Relu(5),
batch_norm,
layers.Sigmoid(1),
],
batch_size=10,
verbose=True, # keep it as `True`
)
gdnet.train(x_train, y_train, epochs=5)
error_before_save = gdnet.prediction_error(x_test, y_test)
mean_before_save = self.eval(batch_norm.running_mean)
variance_before_save = self.eval(batch_norm.running_inv_std)
with tempfile.NamedTemporaryFile() as temp:
storage.save(gdnet, temp.name)
storage.load(gdnet, temp.name)
error_after_load = gdnet.prediction_error(x_test, y_test)
mean_after_load = self.eval(batch_norm.running_mean)
variance_after_load = self.eval(batch_norm.running_inv_std)
self.assertAlmostEqual(error_before_save, error_after_load)
np.testing.assert_array_almost_equal(mean_before_save,
mean_after_load)
np.testing.assert_array_almost_equal(variance_before_save,
variance_after_load)
def reproducible_network_train(seed=0, epochs=500, **additional_params):
"""
Make a reproducible train for Gradient Descent based neural
network with a XOR problem and return trained network.
Parameters
----------
seed : int
Random State seed number for reproducibility. Defaults to ``0``.
epochs : int
Number of epochs for training. Defaults to ``500``.
**additional_params
Aditional parameters for Neural Network.
Returns
-------
GradientDescent instance
Returns trained network.
"""
np.random.seed(seed)
network = algorithms.GradientDescent(connection=[
layers.Input(2),
layers.Tanh(5),
layers.Tanh(1),
StepOutput(),
],
**additional_params)
network.train(xor_input_train, xor_target_train, epochs=epochs)
return network
def test_training_with_multiple_inputs(self):
network = algorithms.GradientDescent(
[
[
layers.Input(2) > layers.Sigmoid(3),
layers.Input(3) > layers.Sigmoid(5),
],
layers.Concatenate(),
layers.Sigmoid(1),
],
step=0.1,
verbose=False,
shuffle_data=True,
)
x_train, x_test, y_train, y_test = simple_classification(n_samples=100,
n_features=5)
x_train_2, x_train_3 = x_train[:, :2], x_train[:, 2:]
x_test_2, x_test_3 = x_test[:, :2], x_test[:, 2:]
network.train([x_train_2, x_train_3],
y_train, [x_test_2, x_test_3],
y_test,
epochs=100)
error = network.validation_errors[-1]
self.assertAlmostEqual(error, 0.14, places=2)
def reproducible_network_train(seed=0, epochs=500, **additional_params):
"""
Make a reproducible train for Gradient Descent based neural
network with a XOR problem and return trained network.
Parameters
----------
seed : int
Random State seed number for reproducibility. Defaults to ``0``.
epochs : int
Number of epochs for training. Defaults to ``500``.
**additional_params
Aditional parameters for Neural Network.
Returns
-------
GradientDescent instance
Returns trained network.
"""
environment.reproducible(seed)
xavier_normal = init.XavierNormal()
tanh_weight1 = xavier_normal.sample((2, 5), return_array=True)
tanh_weight2 = xavier_normal.sample((5, 1), return_array=True)
network = algorithms.GradientDescent(connection=[
layers.Input(2),
layers.Tanh(5, weight=tanh_weight1),
layers.Tanh(1, weight=tanh_weight2),
],
batch_size='all',
**additional_params)
network.train(xor_input_train, xor_target_train, epochs=epochs)
return network
def test_pipeline(self):
dataset = datasets.load_diabetes()
target_scaler = preprocessing.MinMaxScaler()
target = dataset.target.reshape(-1, 1)
x_train, x_test, y_train, y_test = train_test_split(
dataset.data, target_scaler.fit_transform(target), train_size=0.85)
network = algorithms.GradientDescent(
connection=[
layers.Sigmoid(10),
layers.Sigmoid(25),
layers.Output(1),
],
show_epoch=100,
verbose=False,
)
pipeline = Pipeline([
('min_max_scaler', preprocessing.MinMaxScaler()),
('gd', network),
])
pipeline.fit(x_train, y_train, gd__epochs=50)
y_predict = pipeline.predict(x_test)
error = rmsle(target_scaler.inverse_transform(y_test),
target_scaler.inverse_transform(y_predict).round())
self.assertAlmostEqual(0.47, error, places=2)
def test_sew_together_when_cutted_piece_already_in_use(self):
autoencoder = algorithms.Momentum([
layers.Input(25),
layers.Sigmoid(15),
layers.Sigmoid(25),
])
encoder = surgery.cut(autoencoder, start=0, end=2)
self.assertEqual(len(encoder), 2)
classifier = algorithms.Momentum(encoder > layers.Softmax(10))
network = algorithms.GradientDescent([
layers.Input(5),
surgery.CutLine(), # <- first cut point
layers.Sigmoid(10),
layers.Sigmoid(20),
layers.Sigmoid(30),
surgery.CutLine(), # <- second cut point
layers.Sigmoid(1),
])
_, hidden_layers, _ = surgery.cut_along_lines(network)
self.assertEqual(len(hidden_layers), 3)
connected_layers = surgery.sew_together([
encoder,
layers.Relu(5),
hidden_layers
])
self.assertEqual(len(connected_layers), 6)
def test_network_training_summary(self):
with catch_stdout() as out:
network = algorithms.GradientDescent(
layers.join(
layers.Input(2),
layers.Sigmoid(3),
layers.Sigmoid(1),
),
verbose=False,
batch_size=None,
)
x = np.zeros((5, 2))
y = np.zeros((5, 1))
network.verbose = True
n_epochs = 10
network.train(x, y, epochs=n_epochs)
terminal_output = out.getvalue().strip()
# `n_epochs - 1` because \n appears only between
# inline summary lines.
# Also network prints 5 additional lines at the beggining
self.assertEqual(terminal_output.count('\n'), n_epochs - 1)
self.assertEqual(terminal_output.count('train: '), n_epochs)
def test_show_epoch_valid_cases(self):
Case = namedtuple("Case", "show_epoch should_be_n_times n_epochs")
cases = (
# Show 10 epochs and the last one would be 11
Case(show_epoch='10 times', should_be_n_times=11, n_epochs=100),
Case(show_epoch='1 time', should_be_n_times=2, n_epochs=10),
Case(show_epoch='1 times', should_be_n_times=2, n_epochs=10),
# Should be equal to the number of epochs
Case(show_epoch='100 times', should_be_n_times=10, n_epochs=10),
Case(show_epoch=5, should_be_n_times=3, n_epochs=10),
Case(show_epoch=100, should_be_n_times=2, n_epochs=10),
)
for case in cases:
with catch_stdout() as out:
bpnet = algorithms.GradientDescent(
(2, 3, 1),
step=0.1,
verbose=True,
show_epoch=case.show_epoch
)
bpnet.train(xor_zero_input_train, xor_zero_target_train,
epochs=case.n_epochs)
terminal_output = out.getvalue()
# One of the choices has to be true whether other
# choices should give count equal to zero.
time_counts = (
terminal_output.count(" μs ")
+ terminal_output.count(" ms ")
+ terminal_output.count(" ns ")
)
self.assertEqual(case.should_be_n_times, time_counts)
def test_print_training_progress_signal(self):
x_train = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
y_train = np.array([[1, 0, 0, 1]]).T
Case = namedtuple("Case", "show_epoch should_be_n_times n_epochs")
cases = (
Case(show_epoch=5, should_be_n_times=3, n_epochs=10),
Case(show_epoch=7, should_be_n_times=3, n_epochs=10),
Case(show_epoch=100, should_be_n_times=2, n_epochs=10),
)
for case in cases:
with catch_stdout() as out:
bpnet = algorithms.GradientDescent(
layers.join(
layers.Input(2),
layers.Sigmoid(3),
layers.Sigmoid(1),
),
step=0.1,
verbose=True,
batch_size=None,
show_epoch=case.show_epoch,
)
bpnet.train(x_train, y_train, epochs=case.n_epochs)
terminal_output = out.getvalue()
# One of the choices has to be true whether other
# choices should give count equal to zero.
time_counts = (terminal_output.count(" μs]") +
terminal_output.count(" ms]") +
terminal_output.count(" ns]"))
self.assertEqual(case.should_be_n_times, time_counts)
def test_error_plot_ax_none(self):
ax = plt.gca()
network = algorithms.GradientDescent((2, 3, 1))
ax_returned = plots.error_plot(network, ax=None, show=False)
self.assertIs(ax_returned, ax)
def test_logging_info_about_the_data(self):
network = algorithms.GradientDescent((2, 3, 1))
x = np.zeros((5, 2))
x_test = np.zeros((3, 2))
y = np.zeros((4, 1))
with self.assertRaisesRegexp(ValueError, "feature shape"):
logging_info_about_the_data(network, x, y)
with catch_stdout() as out:
network = algorithms.GradientDescent((2, 3, 1), verbose=True)
logging_info_about_the_data(network, [x, x], [x_test, x_test])
terminal_output = out.getvalue()
self.assertIn("[(5, 2), (5, 2)]", terminal_output)
self.assertIn("[(3, 2), (3, 2)]", terminal_output)
def test_custom_error_functions(self):
# Test that everything works without fail
def custom_mse(expected, predicted):
return (0.5 * (predicted - expected)**2).mean()
x_train, _, y_train, _ = simple_classification()
gdnet = algorithms.GradientDescent((10, 10, 1), error=custom_mse)
gdnet.train(x_train, y_train)
def test_network_training_with_unknown_summary_type(self):
network = algorithms.GradientDescent((2, 3, 1))
x = np.zeros((5, 2))
y = np.zeros((5, 1))
with self.assertRaises(ValueError):
network.train(x, y, summary='unknown')
def setUp(self):
super(SurgeryCutTestCase, self).setUp()
self.network = algorithms.GradientDescent([
layers.Input(30),
layers.Sigmoid(10),
layers.Sigmoid(20),
layers.Sigmoid(1),
])
def test_gd_predict_wrong_arguments(self):
optimizer = algorithms.GradientDescent(
layers.Input(10) >> layers.Sigmoid(1),
verbose=False,
)
input = np.random.random((7, 10))
with self.assertRaisesRegexp(TypeError, "Unknown arguments"):
optimizer.predict(input, batchsize=10)
def test_gd(self):
x_train, _, y_train, _ = simple_classification()
network = algorithms.GradientDescent(
layers.Input(10) > layers.Tanh(20) > layers.Tanh(1),
step=0.3,
verbose=False)
network.train(x_train, y_train, epochs=500)
self.assertAlmostEqual(network.errors.last(), 0.014, places=3)
def test_init_logging(self):
with catch_stdout() as out:
algorithms.GradientDescent(
layers.Input(2) > layers.Sigmoid(3) > layers.Sigmoid(1),
verbose=False,
)
terminal_output = out.getvalue()
self.assertEqual("", terminal_output.strip())
with catch_stdout() as out:
algorithms.GradientDescent(
layers.Input(2) > layers.Sigmoid(3) > layers.Sigmoid(1),
verbose=True,
)
terminal_output = out.getvalue()
self.assertNotEqual("", terminal_output.strip())
self.assertIn("verbose = True", terminal_output)
def test_error_plot_and_validation_error_warnings(self):
with catch_stdout() as out:
network = algorithms.GradientDescent((2, 3, 1), verbose=True)
network.errors = ErrorHistoryList([1, 2])
network.validation_errors = ErrorHistoryList([None])
plots.error_plot(network, ax=None, show=False)
terminal_output = out.getvalue()
self.assertIn("error will be ignored", terminal_output)
def test_sew_together_cutted_pieces(self):
network1 = algorithms.GradientDescent([
layers.Input(100),
layers.Sigmoid(200),
layers.Sigmoid(100),
])
network2 = algorithms.GradientDescent([
layers.Input(10),
layers.Sigmoid(20),
layers.Sigmoid(10),
])
first_part = surgery.cut(network1, start=0, end=2)
self.assertEqual(first_part.output_shape, (200,))
self.assertEqual(first_part.input_shape, (100,))
second_part = surgery.cut(network2, start=0, end=2)
self.assertEqual(second_part.output_shape, (20,))
self.assertEqual(second_part.input_shape, (10,))
def test_dan_repr(self):
dan = algorithms.DynamicallyAveragedNetwork([
algorithms.Momentum((3, 2, 1)),
algorithms.GradientDescent((3, 2, 1)),
])
dan_repr = str(dan)
self.assertIn('DynamicallyAveragedNetwork', dan_repr)
self.assertIn('Momentum', dan_repr)
self.assertIn('GradientDescent', dan_repr)
def test_iter_until_converge_critical_cases(self):
with catch_stdout() as out:
network = algorithms.GradientDescent((2, 3, 1), verbose=True)
iterator = iter_until_converge(network, epsilon=1e-5, max_epochs=5)
for epoch in iterator:
network.errors.append(np.nan)
terminal_output = out.getvalue()
self.assertIn('NaN or Inf', terminal_output)
