def test_network_convergence(self):
with catch_stdout() as out:
bpnet = algorithms.Backpropagation((2, 3, 1),
step=0.1,
verbose=True,
show_epoch=100)
bpnet.train(xor_zero_input_train,
xor_zero_target_train,
epochs=3,
epsilon=1e-5)
terminal_output = out.getvalue()
self.assertEqual(1, terminal_output.count("Network didn't converge"))
with catch_stdout() as out:
bpnet = algorithms.Backpropagation((2, 3, 1),
step=0.1,
verbose=True,
show_epoch=100)
bpnet.train(xor_zero_input_train,
xor_zero_target_train,
epochs=1e3,
epsilon=1e-3)
terminal_output = out.getvalue()
self.assertEqual(1, terminal_output.count("Network converged"))
def test_handle_errors(self):
data, target = datasets.make_classification(300, n_features=4,
n_classes=2)
x_train, x_test, y_train, y_test = cross_validation.train_test_split(
data, target, train_size=0.7
)
with self.assertRaises(ValueError):
# First network has two output layers and the second
# just one.
ensemble.DynamicallyAveragedNetwork([
algorithms.RPROP((4, 10, 2), step=0.1),
algorithms.Backpropagation((4, 10, 1), step=0.1)
])
with self.assertRaises(ValueError):
# Use ensemble with less than one network
ensemble.DynamicallyAveragedNetwork([
algorithms.Backpropagation((4, 10, 1), step=0.1)
])
with self.assertRaises(ValueError):
# Output between -1 and 1
dan = ensemble.DynamicallyAveragedNetwork([
algorithms.Backpropagation(
SigmoidLayer(4) > TanhLayer(10) > OutputLayer(1),
step=0.01
),
algorithms.RPROP((4, 10, 1), step=0.1)
])
dan.train(x_train, y_train, epochs=10)
dan.predict(x_test)
def test_pandas_for_bp(self):
dataset = datasets.load_diabetes()
input_scaler = preprocessing.MinMaxScaler()
target_scaler = preprocessing.MinMaxScaler()
n_features = dataset.data.shape[1]
input_columns = ['column_' + str(i) for i in range(n_features)]
pandas_data = pd.DataFrame(dataset.data, columns=input_columns)
pandas_data['target'] = target_scaler.fit_transform(dataset.target)
pandas_data[input_columns] = input_scaler.fit_transform(
pandas_data[input_columns])
x_train, x_test, y_train, y_test = train_test_split(
pandas_data[input_columns], pandas_data['target'], train_size=0.85)
bpnet = algorithms.Backpropagation(connection=[
layers.SigmoidLayer(10),
layers.SigmoidLayer(40),
layers.OutputLayer(1),
],
use_bias=True,
show_epoch=100)
bpnet.train(x_train, y_train, epochs=1000)
y_predict = bpnet.predict(x_test)
error = rmsle(target_scaler.inverse_transform(y_test),
target_scaler.inverse_transform(y_predict).round())
self.assertAlmostEqual(0.4477, error, places=4)
def test_show_epoch_valid_cases(self):
Case = namedtuple("Case", "show_epoch should_be_n_times n_epochs")
cases = (
# Show 10 epochs and the last one would be 11
Case(show_epoch='10 times', should_be_n_times=11, n_epochs=100),
Case(show_epoch='1 time', should_be_n_times=2, n_epochs=10),
Case(show_epoch='1 times', should_be_n_times=2, n_epochs=10),
# Should be equal to the number of epochs
Case(show_epoch='100 times', should_be_n_times=10, n_epochs=10),
Case(show_epoch=5, should_be_n_times=3, n_epochs=10),
Case(show_epoch=100, should_be_n_times=2, n_epochs=10),
)
for case in cases:
with catch_stdout() as out:
bpnet = algorithms.Backpropagation((2, 3, 1),
step=0.1,
verbose=True,
show_epoch=case.show_epoch)
bpnet.train(xor_zero_input_train,
xor_zero_target_train,
epochs=case.n_epochs)
terminal_output = out.getvalue()
self.assertEqual(case.should_be_n_times,
terminal_output.count("Train error"))
def test_pipeline(self):
dataset = datasets.load_diabetes()
target_scaler = preprocessing.MinMaxScaler()
x_train, x_test, y_train, y_test = train_test_split(
dataset.data,
target_scaler.fit_transform(dataset.target),
train_size=0.85)
network = algorithms.Backpropagation(
connection=[
layers.SigmoidLayer(10),
layers.SigmoidLayer(40),
layers.OutputLayer(1),
],
use_bias=True,
show_epoch=100,
verbose=False,
)
pipeline = Pipeline([
('min_max_scaler', preprocessing.MinMaxScaler()),
('backpropagation', network),
])
pipeline.fit(x_train, y_train, backpropagation__epochs=1000)
y_predict = pipeline.predict(x_test)
error = rmsle(target_scaler.inverse_transform(y_test),
target_scaler.inverse_transform(y_predict).round())
self.assertAlmostEqual(0.4481, error, places=4)
def test_mixture_of_experts(self):
dataset = datasets.load_diabetes()
data, target = dataset.data, dataset.target
insize, outsize = data.shape[1], 1
input_scaler = preprocessing.MinMaxScaler((-1, 1))
output_scaler = preprocessing.MinMaxScaler()
x_train, x_test, y_train, y_test = cross_validation.train_test_split(
input_scaler.fit_transform(data),
output_scaler.fit_transform(target.reshape(-1, 1)),
train_size=0.8)
n_epochs = 300
scaled_y_test = output_scaler.inverse_transform(y_test).reshape(
(y_test.size, 1))
# -------------- Train single Backpropagation -------------- #
bpnet = algorithms.Backpropagation((insize, 20, outsize),
step=0.1,
verbose=False)
bpnet.train(x_train, y_train, epochs=n_epochs)
network_output = bpnet.predict(x_test)
network_error = rmsle(output_scaler.inverse_transform(network_output),
scaled_y_test)
# -------------- Train ensemlbe -------------- #
moe = ensemble.MixtureOfExperts(
networks=[
algorithms.Backpropagation((insize, 20, outsize),
step=0.1,
verbose=False),
algorithms.Backpropagation((insize, 20, outsize),
step=0.1,
verbose=False),
],
gating_network=algorithms.Backpropagation(
layers.SoftmaxLayer(insize) > layers.OutputLayer(2),
step=0.1,
verbose=False))
moe.train(x_train, y_train, epochs=n_epochs)
ensemble_output = moe.predict(x_test)
ensemlbe_error = rmsle(
output_scaler.inverse_transform(ensemble_output), scaled_y_test)
self.assertGreater(network_error, ensemlbe_error)
def test_terminal_output_frequency(self):
with catch_stdout() as out:
data = np.random.random((1000, 2))
target = np.random.random((1000, 1))
bpnet = algorithms.Backpropagation((2, 1, 1),
verbose=True,
show_epoch=1)
bpnet.train(data, target, epochs=100)
terminal_output = out.getvalue()
self.assertEqual(1, terminal_output.count("Too many outputs"))
def test_dan(self):
data, target = datasets.make_classification(300, n_features=4,
n_classes=2)
x_train, x_test, y_train, y_test = cross_validation.train_test_split(
data, target, train_size=0.7
)
dan = ensemble.DynamicallyAveragedNetwork([
algorithms.RPROP((4, 100, 1), step=0.1, maximum_step=1),
algorithms.Backpropagation((4, 5, 1), step=0.1),
algorithms.ConjugateGradient((4, 5, 1), step=0.01),
])
dan.train(x_train, y_train, epochs=500)
result = dan.predict(x_test)
ensemble_result = metrics.accuracy_score(y_test, result)
self.assertAlmostEqual(0.9333, ensemble_result, places=4)
def test_dynamic_classes(self):
optimization_classes = [
algorithms.WeightDecay, algorithms.SearchThenConverge
]
bpnet = algorithms.Backpropagation(
(3, 5, 1),
optimizations=optimization_classes,
verbose=False,
)
data, target = datasets.make_regression(n_features=3, n_targets=1)
data = preprocessing.MinMaxScaler().fit_transform(data)
target = preprocessing.MinMaxScaler().fit_transform(target)
with tempfile.NamedTemporaryFile() as temp:
valid_class_name = bpnet.__class__.__name__
dill.dump(bpnet, temp)
temp.file.seek(0)
restored_bpnet = dill.load(temp)
restored_class_name = restored_bpnet.__class__.__name__
temp.file.seek(0)
self.assertEqual(valid_class_name, restored_class_name)
self.assertEqual(optimization_classes,
restored_bpnet.optimizations)
bpnet.train(data, target, epochs=10)
real_bpnet_error = bpnet.error(bpnet.predict(data), target)
updated_input_weight = bpnet.input_layer.weight.copy()
dill.dump(bpnet, temp)
temp.file.seek(0)
restored_bpnet2 = dill.load(temp)
temp.file.seek(0)
actual = restored_bpnet2.predict(data)
restored_bpnet_error = restored_bpnet2.error(actual, target)
np.testing.assert_array_equal(updated_input_weight,
restored_bpnet2.input_layer.weight)
# Error must be big, because we didn't normalize data
self.assertEqual(real_bpnet_error, restored_bpnet_error)
def test_show_epoch_invalid_cases(self):
wrong_input_values = (
'time 10',
'good time',
'100',
'super power',
'0 times',
'-1 times',
0,
-100,
)
for wrong_input_value in wrong_input_values:
with self.assertRaises(ValueError):
bpnet = algorithms.Backpropagation(
(2, 3, 1),
step=0.1,
verbose=False,
show_epoch=wrong_input_value)
def test_simple_storage(self):
bpnet = algorithms.Backpropagation((2, 3, 1), step=0.25, verbose=False)
data, target = datasets.make_regression(n_features=2, n_targets=1)
data = preprocessing.MinMaxScaler().fit_transform(data)
target = preprocessing.MinMaxScaler().fit_transform(target)
with tempfile.NamedTemporaryFile() as temp:
test_layer_weights = bpnet.input_layer.weight.copy()
dill.dump(bpnet, temp)
temp.file.seek(0)
restored_bpnet = dill.load(temp)
temp.file.seek(0)
layers_sizes = [
layer.input_size for layer in restored_bpnet.layers
]
self.assertEqual(0.25, restored_bpnet.step)
self.assertEqual([2, 3, 1], layers_sizes)
np.testing.assert_array_equal(test_layer_weights,
restored_bpnet.input_layer.weight)
bpnet.train(data, target, epochs=5)
real_bpnet_error = bpnet.error(bpnet.predict(data), target)
updated_input_weight = bpnet.input_layer.weight.copy()
dill.dump(bpnet, temp)
temp.file.seek(0)
restored_bpnet2 = dill.load(temp)
temp.file.seek(0)
actual = restored_bpnet2.predict(data)
restored_bpnet_error = restored_bpnet2.error(actual, target)
np.testing.assert_array_equal(updated_input_weight,
restored_bpnet2.input_layer.weight)
# Error must be big, because we didn't normalize data
self.assertEqual(real_bpnet_error, restored_bpnet_error)
def test_ensemble(self):
data, target = datasets.make_classification(300,
n_features=4,
n_classes=2)
x_train, x_test, y_train, y_test = train_test_split(data,
target,
train_size=0.7)
dan = ensemble.DynamicallyAveragedNetwork([
algorithms.RPROP((4, 100, 1), step=0.1, maximum_step=1),
algorithms.Backpropagation((4, 5, 1), step=0.1),
algorithms.ConjugateGradient((4, 5, 1), step=0.01),
])
pipeline = Pipeline([
('min_max_scaler', preprocessing.StandardScaler()),
('dan', dan),
])
pipeline.fit(x_train, y_train, dan__epochs=500)
result = pipeline.predict(x_test)
ensemble_result = metrics.accuracy_score(y_test, result)
self.assertAlmostEqual(0.9222, ensemble_result, places=4)
bpn.train(input_data, target_data, epsilon=0.125)
weights = weights[:, :bpn.epoch + 1]
weight_quiver(weights, color=color)
label = "{name} ({n} steps)".format(name=name, n=bpn.epoch)
return mpatches.Patch(color=color, label=label)
def target_function(network, x, y):
network.input_layer.weight = np.array([[x], [y]])
predicted = network.predict(input_data)
return network.error(predicted, target_data)
# Get data for countour plot
bp_network = algorithms.Backpropagation(get_connection(), **network_settings)
network_target_function = partial(target_function, bp_network)
plt.figure()
plt.title("Approximation function contour plot")
plt.xlabel("First weight")
plt.ylabel("Second weight")
draw_countour(np.linspace(-4.5, 4, 50), np.linspace(-4.5, 4, 50),
network_target_function)
cgnet_class = partial(algorithms.ConjugateGradient,
optimizations=[algorithms.LinearSearch])
algorithms = (
(algorithms.Backpropagation, 'Gradient Descent', 'k'),
def test_handle_errors(self):
networks = [
algorithms.Backpropagation((1, 20, 1), step=0.2),
algorithms.Backpropagation((1, 20, 1), step=0.2),
]
with self.assertRaises(ValueError):
# Ivalid network (not Backpropagation)
ensemble.MixtureOfExperts(
networks=networks + [algorithms.GRNN()],
gating_network=algorithms.Backpropagation(
layers.SigmoidLayer(1) > layers.OutputLayer(3), ))
with self.assertRaises(ValueError):
# Ivalid number of outputs in third network
ensemble.MixtureOfExperts(
networks=networks +
[algorithms.Backpropagation((1, 20, 2), step=0.2)],
gating_network=algorithms.Backpropagation(
layers.SigmoidLayer(1) > layers.OutputLayer(3), ))
with self.assertRaises(ValueError):
# Ivalid gating network output layer size
ensemble.MixtureOfExperts(
networks=networks,
gating_network=algorithms.Backpropagation(
layers.SoftmaxLayer(1) > layers.OutputLayer(1), ))
with self.assertRaises(ValueError):
# Ivalid gating network input layer
ensemble.MixtureOfExperts(
networks=networks,
gating_network=algorithms.Backpropagation(
layers.SigmoidLayer(1) > layers.OutputLayer(2), ))
with self.assertRaises(ValueError):
# Ivalid gating network output layer
ensemble.MixtureOfExperts(
networks=networks,
gating_network=algorithms.Backpropagation(
layers.SoftmaxLayer(1) > layers.RoundOutputLayer(2)))
with self.assertRaises(ValueError):
# Ivalid network error function
ensemble.MixtureOfExperts(
networks=networks + [
algorithms.Backpropagation(
(1, 20, 1), step=0.2, error=rmsle)
],
gating_network=algorithms.Backpropagation(
layers.SigmoidLayer(1) > layers.OutputLayer(3), ))
with self.assertRaises(ValueError):
moe = ensemble.MixtureOfExperts(
# Ivalid gating error function
networks=networks,
gating_network=algorithms.Backpropagation(
layers.SoftmaxLayer(1) > layers.OutputLayer(2),
error=rmsle))
moe = ensemble.MixtureOfExperts(
# Ivalid gating network output layer
networks=networks,
gating_network=algorithms.Backpropagation(
layers.SoftmaxLayer(1) > layers.OutputLayer(2)))
with self.assertRaises(ValueError):
# Wrong number of train input features
moe.train(np.array([[1, 2]]), np.array([[0]]))
with self.assertRaises(ValueError):
# Wrong number of train output features
moe.train(np.array([[1]]), np.array([[0, 0]]))