def reproducible_network_train(seed=0, epochs=500, **additional_params):
"""
Make a reproducible train for Gradient Descent based neural
network with a XOR problem and return trained network.
Parameters
----------
seed : int
Random State seed number for reproducibility. Defaults to ``0``.
epochs : int
Number of epochs for training. Defaults to ``500``.
**additional_params
Aditional parameters for Neural Network.
Returns
-------
GradientDescent instance
Returns trained network.
"""
np.random.seed(seed)
network = algorithms.GradientDescent(connection=[
layers.Input(2),
layers.Tanh(5),
layers.Tanh(1),
StepOutput(),
],
**additional_params)
network.train(xor_input_train, xor_target_train, epochs=epochs)
return network
def reproducible_network_train(seed=0, epochs=500, **additional_params):
"""
Make a reproducible train for Gradient Descent based neural
network with a XOR problem and return trained network.
Parameters
----------
seed : int
Random State seed number for reproducibility. Defaults to ``0``.
epochs : int
Number of epochs for training. Defaults to ``500``.
**additional_params
Aditional parameters for Neural Network.
Returns
-------
GradientDescent instance
Returns trained network.
"""
environment.reproducible(seed)
xavier_normal = init.XavierNormal()
tanh_weight1 = xavier_normal.sample((2, 5), return_array=True)
tanh_weight2 = xavier_normal.sample((5, 1), return_array=True)
network = algorithms.GradientDescent(connection=[
layers.Input(2),
layers.Tanh(5, weight=tanh_weight1),
layers.Tanh(1, weight=tanh_weight2),
],
batch_size='all',
**additional_params)
network.train(xor_input_train, xor_target_train, epochs=epochs)
return network
def setUp(self):
super(LearningRateUpdatesTestCase, self).setUp()
self.first_step = 0.3
self.connection = [
layers.Tanh(2),
layers.Tanh(3),
layers.StepOutput(1, output_bounds=(-1, 1))
]
def test_gd(self):
x_train, _, y_train, _ = simple_classification()
network = algorithms.GradientDescent(
layers.Input(10) > layers.Tanh(20) > layers.Tanh(1),
step=0.3,
verbose=False)
network.train(x_train, y_train, epochs=500)
self.assertAlmostEqual(network.errors.last(), 0.014, places=3)
def test_gd(self):
environment.reproducible()
x_train, _, y_train, _ = simple_classification()
network = algorithms.BaseGradientDescent(
layers.Input(10) > layers.Tanh(20) > layers.Tanh(1),
step=0.1,
verbose=False)
network.train(x_train, y_train, epochs=100)
self.assertLess(network.errors.last(), 0.05)
def test_that_alg_works(self):
network = algorithms.GradientDescent([
layers.Input(2),
layers.Tanh(3),
layers.Tanh(1),
],
step=0.3,
decay_rate=0.0001,
addons=[algorithms.WeightDecay])
network.train(xor_input_train, xor_target_train, epochs=500)
self.assertAlmostEqual(network.errors.last(), 0, places=2)
def test_that_alg_works(self):
network = algorithms.GradientDescent(
[
layers.Tanh(2),
layers.Tanh(3),
layers.StepOutput(1, output_bounds=(-1, 1))
],
step=0.3,
zero_weight=20,
addons=[algorithms.WeightElimination])
network.train(xor_input_train, xor_target_train, epochs=350)
self.assertAlmostEqual(network.errors.last(), 0, places=2)
def test_that_alg_works(self):
network = algorithms.GradientDescent(
[
layers.Input(2),
layers.Tanh(3),
layers.Tanh(1),
],
step=0.3,
batch_size='all',
zero_weight=20,
addons=[algorithms.WeightElimination])
network.train(xor_input_train, xor_target_train, epochs=350)
self.assertAlmostEqual(network.errors.last(), 0, places=2)
def test_errdiff(self):
network = algorithms.GradientDescent(
[
layers.Tanh(2),
layers.Tanh(3),
layers.StepOutput(1, output_bounds=(-1, 1))
],
step=0.3,
update_for_smaller_error=1.05,
update_for_bigger_error=0.7,
error_difference=1.04,
addons=[algorithms.ErrDiffStepUpdate])
network.train(xor_input_train, xor_target_train, epochs=200)
self.assertAlmostEqual(network.errors.last(), 0, places=5)
def test_network_initializations(self):
possible_networks = (
# as a list
[layers.Input(2), layers.Sigmoid(3), layers.Tanh(1)],
# as forward sequence with inline operators
layers.Input(2) > layers.Relu(10) > layers.Tanh(1),
layers.Input(2) >> layers.Relu(10) >> layers.Tanh(1),
)
for i, network in enumerate(possible_networks, start=1):
optimizer = algorithms.GradientDescent(network)
message = "[Test #{}] Network: {}".format(i, network)
self.assertEqual(len(optimizer.network.layers), 3, msg=message)
def test_minibatch_gd(self):
x_train, _, y_train, _ = simple_classification()
compare_networks(
# Test classes
algorithms.GradientDescent,
partial(algorithms.MinibatchGradientDescent, batch_size=1),
# Test data
(x_train, y_train),
# Network configurations
connection=(layers.Input(10) > layers.Tanh(20) > layers.Tanh(1)),
step=0.1,
shuffle_data=True,
verbose=False,
# Test configurations
epochs=40,
show_comparison_plot=False)
def test_simple_learning_rate_minimization(self):
first_step = 0.3
network = algorithms.GradientDescent([
layers.Input(2),
layers.Tanh(3),
layers.Tanh(1),
],
step=first_step,
reduction_freq=50,
addons=[algorithms.StepDecay])
network.train(xor_input_train, xor_target_train, epochs=100)
self.assertAlmostEqual(
network.variables.step.get_value(),
asfloat(first_step / 3),
places=5,
)
def test_leak_step_adaptation(self):
compare_networks(
# Test classes
algorithms.GradientDescent,
partial(
algorithms.GradientDescent,
leak_size=0.05,
alpha=0.05,
beta=5,
addons=[algorithms.LeakStepAdaptation]
),
# Test data
(even_input_train, even_target_train),
# Network configurations
connection=[
layers.Sigmoid(2),
layers.Tanh(3),
layers.Output(1)
],
step=0.1,
verbose=False,
shuffle_data=True,
epochs=30,
# show_comparison_plot=True,
)
def test_search_then_converge(self):
network = algorithms.GradientDescent(
[
layers.Input(2),
layers.Tanh(3),
layers.Tanh(1),
],
step=0.3,
reduction_freq=50,
rate_coefitient=0.2,
addons=[algorithms.SearchThenConverge])
network.train(xor_input_train, xor_target_train, epochs=6)
self.assertAlmostEqual(
network.variables.step.get_value(),
0.18,
places=5,
)
def test_inline_definition(self):
network = layers.Input(2) >> layers.Relu(10) >> layers.Tanh(1)
self.assertShapesEqual(network.input_shape, (None, 2))
self.assertShapesEqual(network.output_shape, (None, 1))
input_value = asfloat(np.random.random((10, 2)))
output_value = self.eval(network.output(input_value))
self.assertEqual(output_value.shape, (10, 1))
def test_search_then_converge(self):
network = algorithms.GradientDescent(
[
layers.Tanh(2),
layers.Tanh(3),
layers.StepOutput(1, output_bounds=(-1, 1))
],
step=0.3,
epochs_step_minimizator=50,
rate_coefitient=0.2,
addons=[algorithms.SearchThenConverge])
network.train(xor_input_train, xor_target_train, epochs=6)
self.assertAlmostEqual(
network.variables.step.get_value(),
0.18,
places=5,
)
def Initialize_Connection(self):
neuronsLayers=self.NeuronsInEveryLayer
activationFsLayers=self.ActivationFunctionInEveryLayer
connectionInit=[layerNeupy.Input(neuronsLayers[0])]
for i in range(1,len(self.NeuronsInEveryLayer)):
if(activationFsLayers[i]=='logsig'):connectionInit.append(layerNeupy.Sigmoid(neuronsLayers[i]))
elif(activationFsLayers[i]=='tansig'):connectionInit.append(layerNeupy.Tanh(neuronsLayers[i]))
else:connectionInit.append(layerNeupy.Linear(neuronsLayers[i]))
return connectionInit
def test_errdiff(self):
initial_step = 0.3
network = algorithms.GradientDescent(
[
layers.Input(2),
layers.Tanh(3),
layers.Tanh(1),
],
batch_size='all',
step=initial_step,
update_for_smaller_error=1.05,
update_for_bigger_error=0.7,
error_difference=1.04,
addons=[algorithms.ErrDiffStepUpdate]
)
network.train(xor_input_train, xor_target_train, epochs=200)
self.assertNotEqual(self.eval(network.variables.step), initial_step)
self.assertAlmostEqual(network.errors.last(), 0, places=4)
def test_connection_initializations(self):
possible_connections = (
(2, 3, 1),
# as a list
[layers.Input(2),
layers.Sigmoid(3),
layers.Tanh(1)],
# as forward sequence with inline operators
layers.Input(2) > layers.Relu(10) > layers.Tanh(1),
layers.Input(2) >> layers.Relu(10) >> layers.Tanh(1),
# as backward sequence with inline operators
layers.Tanh(1) < layers.Relu(10) < layers.Input(2),
layers.Tanh(1) << layers.Relu(10) << layers.Input(2),
)
for connection in possible_connections:
network = algorithms.GradientDescent(connection)
self.assertEqual(len(network.layers), 3, msg=connection)
def test_connection_initializations(self):
possible_connections = (
(2, 3, 1),
# as a list
[layers.Input(2),
layers.Sigmoid(3),
layers.Tanh(1)],
# as forward sequence with inline operators
layers.Input(2) > layers.Relu(10) > layers.Tanh(1),
layers.Input(2) >> layers.Relu(10) >> layers.Tanh(1),
# as backward sequence with inline operators
layers.Tanh(1) < layers.Relu(10) < layers.Input(2),
layers.Tanh(1) << layers.Relu(10) << layers.Input(2),
)
for i, connection in enumerate(possible_connections, start=1):
network = algorithms.GradientDescent(connection)
message = "[Test #{}] Connection: {}".format(i, connection)
self.assertEqual(len(network.layers), 3, msg=message)
def init_network(self, member): network = layers.join(layers.Input(self.inputneurons)) for index in range(0, len(member[1][0])): if member[1][1][index] == 1: network = network > layers.Sigmoid(member[1][0][index]) elif member[1][1][index] == 2: network = network > layers.Relu(member[1][0][index]) elif member[1][1][index] == 3: network = network > layers.Softmax(member[1][0][index]) elif member[1][1][index] == 4: network = network > layers.Tanh(member[1][0][index]) elif member[1][1][index] == 5: network = network > layers.LeakyRelu(member[1][0][index]) network = network > layers.Sigmoid(self.outputneurons) return(network)
def __init__(self):
self.population = []
self.size_population = 20
self.inputneurons = 4
self.outputneurons = 1
self.data = datasets.load_iris()
for i in range(0,self.size_population):
#connections
network = layers.join(layers.Input(self.inputneurons))
num = random.randint(1,4)
temp1 = list(random.randint(1,50) for i in range(0, num))
#print(temp1, end="\n\n")
temp2 = []
for neu in temp1:
n = random.randint(1,5)
temp2.append(n)
if n == 1:
network = network > layers.Sigmoid(neu)
elif n == 2:
network = network > layers.Relu(neu)
elif n == 3:
network = network > layers.Softmax(neu)
elif n == 4:
network = network > layers.Tanh(neu)
elif n == 5:
network = network > layers.LeakyRelu(neu)
#print(network, end="\n~\n")
network = network > layers.Sigmoid(self.outputneurons)
attributes = [temp1, temp2]
self.population.append([network, attributes, 0]) # 0 --> fitness
#print(self.population)
self.run()
while self.best_members[0][2][0] > 1:
print("next iteration")
print(self.population)
self.run()
file = open("pickle_bestnet.txt", "w")
def model_network(self, algorithm='LevenbergMarquardt', model=None, opt=None):
model = self.decode_model(model)
if model is None:
model = [
[1, 'hidden', 15, 'Linear'],
[2, 'hidden', 10, 'Linear'],
[3, 'output', self.output_classes, 'Elu']
]
# [Input(4), Elu(1)]
# [Input(4), Elu(6), Elu(1)] EP: 100
layer_model = [layers.Input(self.input_features)]
for layer in model:
if layer[3] == 'Linear':
layer_model.append(layers.Linear(layer[2]))
if layer[3] == 'Relu':
layer_model.append(layers.Relu(layer[2]))
if layer[3] == 'Sigmoid':
layer_model.append(layers.Sigmoid(layer[2]))
if layer[3] == 'HardSigmoid':
layer_model.append(layers.HardSigmoid(layer[2]))
if layer[3] == 'Step':
layer_model.append(layers.Step(layer[2]))
if layer[3] == 'Tanh':
layer_model.append(layers.Tanh(layer[2]))
if layer[3] == 'Softplus':
layer_model.append(layers.Softplus(layer[2]))
if layer[3] == 'Softmax':
layer_model.append(layers.Softmax(layer[2]))
if layer[3] == 'Elu':
layer_model.append(layers.Elu(layer[2]))
if layer[3] == 'PRelu':
layer_model.append(layers.PRelu(layer[2]))
if layer[3] == 'LeakyRelu':
layer_model.append(layers.LeakyRelu(layer[2]))
print('Model warstw: ' + str(layer_model))
self.layers = layer_model
self.select_algorithm(algorithm, options=opt)
def initialize(self):
self.network = algorithms.Momentum(
[
layers.Input(20),
layers.Relu(30, weight=init.Uniform(-1, 1)),
layers.Tanh(40, weight=init.Uniform(-1, 1)),
# layers.Embedding(40, 1),
# layers.GRU(40),
layers.Relu(25, weight=init.Uniform(-1, 1)),
layers.Linear(9, weight=init.Uniform(-1, 1)),
],
error='categorical_crossentropy',
step=0.01,
verbose=False,
shuffle_data=True,
momentum=0.99,
nesterov=True,
)
self.network.architecture()
network = architectures.mixture_of_experts([
layers.join(
layers.Input(58),
layers.Softmax(22),
layers.Softmax(1),
),
layers.join(
layers.Input(58),
layers.Relu(60),
layers.Relu(40),
layers.Softmax(22),
layers.Softmax(1),
),
layers.join(
layers.Input(58),
layers.Tanh(12),
layers.Tanh(25),
layers.Tanh(1),
),
])
network
gdnet = algorithms.Adam(network, verbose=True)
print("Start moe training")
gdnet.train(x_train, y_train, epochs=500)
gresult = gdnet.predict(x_test)
gerror = estimators.rmse(gresult, y_test)
print(gerror)
def test_tanh_layer(self):
layer1 = layers.Tanh(1)
self.assertGreater(1, self.eval(layer1.activation_function(1.)))
training_set = numpy.loadtxt('training_set.txt')
print training_set.shape
testing_set = numpy.loadtxt('testing_set.txt')
print testing_set.shape
testing_attributes = numpy.delete(testing_set, 18, 1)
training_attributes = numpy.delete(training_set, 18, 1)
training_results = training_set[:, [18]]
testing_results = testing_set[:, [18]]
network = algorithms.Momentum(
[layers.Input(18),
layers.Sigmoid(40),
layers.Sigmoid(20),
layers.Tanh(1)],
error='categorical_crossentropy',
step=0.15,
verbose=True,
shuffle_data=True,
momentum=0.99,
nesterov=True)
network.architecture()
network.train(training_attributes,
training_results,
testing_attributes,
testing_results,
epochs=20)
'''for layer in network:
def test_connection_shapes(self):
connection = layers.Input(2) > layers.Relu(10) > layers.Tanh(1)
self.assertEqual(connection.input_shape, (2, ))
self.assertEqual(connection.output_shape, (1, ))
mean = (encoder > mu).output(x)
log_var = (encoder > sigma).output(x)
epsilon = 1e-7
predicted = tf.clip_by_value(predicted, epsilon, 1.0 - epsilon)
crossentropy_loss = binary_crossentropy(expected, predicted)
kl_loss = tf.reduce_sum(
1 + 2 * log_var - tf.square(mean) - tf.exp(2 * log_var),
axis=1
)
return tf.reduce_mean(crossentropy_loss - 0.5 * kl_loss)
# Construct Variational Autoencoder
encoder = layers.Input(784, name='input') > layers.Tanh(256)
# Two is the maximum number of dimensions that we can visualize
mu = layers.Linear(2, name='mu')
sigma = layers.Linear(2, name='sigma')
sampler = [mu, sigma] > GaussianSample()
decoder = layers.Tanh(256) > layers.Sigmoid(784)
# Train network
network = algorithms.RMSProp(
[
encoder,
sampler,
decoder,
],
predicted = T.clip(predicted, epsilon, 1.0 - epsilon)
crossentropy_loss = T.sum(
T.nnet.binary_crossentropy(predicted, expected),
axis=1
)
kl_loss = -0.5 * T.sum(
1 + 2 * log_var - T.square(mean) - T.exp(2 * log_var),
axis=1
)
return (crossentropy_loss + kl_loss).mean()
# Construct Variational Autoencoder
encoder = layers.Input(784) > layers.Tanh(500)
mu = layers.Linear(2, name='mu')
sigma = layers.Linear(2, name='sigma')
sampler = [mu, sigma] > GaussianSample()
decoder = layers.Tanh(500) > layers.Sigmoid(784)
# Train network
network = algorithms.RMSProp(
encoder > sampler > decoder,
error=vae_loss,
batch_size=128,
shuffle_data=True,
step=0.001,
