def test_handle_errors(self):
data, target = datasets.make_classification(300,
n_features=4,
n_classes=2)
x_train, x_test, y_train, y_test = model_selection.train_test_split(
data, target, test_size=0.3)
with self.assertRaises(ValueError):
# First network has two output layers and the second
# just one.
algorithms.DynamicallyAveragedNetwork([
algorithms.RPROP((4, 10, 2), step=0.1),
algorithms.GradientDescent((4, 10, 1), step=0.1)
])
with self.assertRaises(ValueError):
# Use ensemble with less than one network
algorithms.DynamicallyAveragedNetwork(
[algorithms.GradientDescent((4, 10, 1), step=0.1)])
with self.assertRaises(ValueError):
# Output greater than 1
dan = algorithms.DynamicallyAveragedNetwork([
algorithms.GradientDescent([
Input(4),
Sigmoid(10),
Relu(1, weight=init.Uniform(), bias=init.Uniform()),
],
step=0.01),
algorithms.RPROP((4, 10, 1), step=0.01),
])
dan.train(x_train, y_train, epochs=10)
dan.predict(x_test)
def test_irpropplus(self):
options = dict(minstep=0.001,
maxstep=1,
increase_factor=1.1,
decrease_factor=0.1,
step=1,
verbose=False)
uniform = init.Uniform()
params1 = dict(
weight=uniform.sample((3, 10), return_array=True),
bias=uniform.sample((10, ), return_array=True),
)
params2 = dict(
weight=uniform.sample((10, 2), return_array=True),
bias=uniform.sample((2, ), return_array=True),
)
network = layers.join(
Input(3),
Sigmoid(10, **params1),
Sigmoid(2, **params2),
)
nw = algorithms.IRPROPPlus(copy.deepcopy(network), **options)
nw.train(simple_x_train, simple_y_train, epochs=100)
irprop_plus_error = nw.errors.train[-1]
self.assertGreater(1e-4, nw.errors.train[-1])
nw = algorithms.RPROP(copy.deepcopy(network), **options)
nw.train(simple_x_train, simple_y_train, epochs=100)
rprop_error = nw.errors.train[-1]
self.assertGreater(rprop_error, irprop_plus_error)
def test_uniformal_initializer(self):
uniform = init.Uniform(minval=-10, maxval=10)
weight = uniform.sample((30, 30))
self.assertUniformlyDistributed(weight)
self.assertAlmostEqual(-10, np.min(weight), places=1)
self.assertAlmostEqual(10, np.max(weight), places=1)
def test_hessdiag(self):
x_train, x_test, y_train, y_test = simple_classification()
params = dict(weight=init.Uniform(-0.1, 0.1),
bias=init.Uniform(-0.1, 0.1))
nw = algorithms.HessianDiagonal(
network=[
layers.Input(10),
layers.Sigmoid(20, **params),
layers.Sigmoid(1, **params),
],
step=0.1,
shuffle_data=False,
verbose=False,
min_eigval=0.1,
)
nw.train(x_train, y_train, epochs=50)
self.assertGreater(0.2, nw.errors.train[-1])
def test_uniform_reprodusible_with_outside_seed(self):
uniform = init.Uniform(minval=-10, maxval=10)
np.random.seed(0)
weight1 = uniform.sample((10, 4), return_array=True)
np.random.seed(0)
weight2 = uniform.sample((10, 4), return_array=True)
np.testing.assert_array_almost_equal(weight1, weight2)
def test_hessdiag(self):
x_train, x_test, y_train, y_test = simple_classification()
nw = algorithms.HessianDiagonal(
connection=[
layers.Input(10),
layers.Sigmoid(20,
weight=init.Uniform(-1, 1),
bias=init.Uniform(-1, 1)),
layers.Sigmoid(1,
weight=init.Uniform(-1, 1),
bias=init.Uniform(-1, 1)),
],
step=0.1,
shuffle_data=False,
verbose=False,
min_eigval=0.01,
)
nw.train(x_train / 2, y_train, epochs=10)
self.assertAlmostEqual(0.10, nw.errors.last(), places=2)
def initialize(self):
self.network = algorithms.Momentum(
[
layers.Input(20),
layers.Linear(20, weight=init.Uniform(-0.5, 0.5)) ,
layers.LeakyRelu(15, weight=init.Uniform(-0.5, 0.5)),
layers.LeakyRelu(15, weight=init.Uniform(-0.5, 0.5)),
layers.LeakyRelu(12, weight=init.Uniform(-0.5, 0.5)),
layers.Linear(9, weight=init.Uniform(-0.5, 0.5)),
],
error='categorical_crossentropy',
step=0.01,
verbose=False,
shuffle_data=True,
momentum=0.99,
nesterov=True,
)
self.network.architecture()
def test_irpropplus(self):
options = dict(minstep=0.001,
maxstep=1,
increase_factor=1.1,
decrease_factor=0.1,
step=1,
verbose=False)
connection = [
Input(3),
Sigmoid(10, weight=init.Uniform(), bias=init.Uniform()),
Sigmoid(2, weight=init.Uniform(), bias=init.Uniform()),
]
nw = algorithms.IRPROPPlus(copy.deepcopy(connection), **options)
nw.train(simple_input_train, simple_target_train, epochs=100)
irprop_plus_error = nw.errors.last()
self.assertGreater(1e-4, nw.errors.last())
nw = algorithms.RPROP(copy.deepcopy(connection), **options)
nw.train(simple_input_train, simple_target_train, epochs=100)
rprop_error = nw.errors.last()
self.assertGreater(rprop_error, irprop_plus_error)
def initialize(self):
self.network = algorithms.Momentum(
[
layers.Input(20),
layers.Relu(30, weight=init.Uniform(-1, 1)),
layers.Tanh(40, weight=init.Uniform(-1, 1)),
# layers.Embedding(40, 1),
# layers.GRU(40),
layers.Relu(25, weight=init.Uniform(-1, 1)),
layers.Linear(9, weight=init.Uniform(-1, 1)),
],
error='categorical_crossentropy',
step=0.01,
verbose=False,
shuffle_data=True,
momentum=0.99,
nesterov=True,
)
self.network.architecture()
def test_compare_bp_and_hessian(self):
x_train, _, y_train, _ = simple_classification()
params = dict(weight=init.Uniform(-0.1, 0.1),
bias=init.Uniform(-0.1, 0.1))
compare_networks(
# Test classes
partial(algorithms.GradientDescent, batch_size=None),
partial(algorithms.HessianDiagonal, min_eigval=0.1),
# Test data
(x_train, y_train),
# Network configurations
network=[
layers.Input(10),
layers.Sigmoid(20, **params),
layers.Sigmoid(1, **params),
],
step=0.1,
shuffle_data=True,
verbose=False,
# Test configurations
epochs=50,
show_comparison_plot=False)
def test_uniform_initializer_repr(self):
uniform_initializer = init.Uniform(minval=0, maxval=1)
self.assertEqual("Uniform(0, 1)", str(uniform_initializer))
from utils import plot_2d_grid, make_circle, make_elipse, make_square
plt.style.use('ggplot')
utils.reproducible()
if __name__ == '__main__':
GRID_WIDTH = 4
GRID_HEIGHT = 4
datasets = [
make_square(),
make_circle(),
make_elipse(corr=0.7),
]
configurations = [{
'weight_init': init.Uniform(0, 1),
'title': 'Random uniform initialization',
}, {
'weight_init': 'sample_from_data',
'title': 'Sampled from the data',
}, {
'weight_init': 'init_pca',
'title': 'Initialize with PCA',
}]
plt.figure(figsize=(15, 15))
plt.title("Compare weight initialization methods for SOFM")
red, blue = ('#E24A33', '#348ABD')
n_columns = len(configurations)
n_rows = len(datasets)