from neural_network.activations import * from neural_network.data_manipulation import load_easy_data from neural_network.plots import plot_data_2d, plot_measure_results_data from neural_network.learn_network import learn_network import matplotlib.pyplot as plt # load dataset train, test = load_easy_data() # x and y - observations and values for them x = train[:, 0:2] y = train[:, 2:3] # plot data classes plt.figure(figsize=(12.8, 9.6)) plt.subplot(221) plot_data_2d(x[:, 0], x[:, 1], y[:, 0], title='True classes of points on the plane', show=False) # learn model and plot result classes plt.subplot(222) mse_linear = learn_network(x, y, [20], [sigmoid, linear], iterations=100, regression=False, plot_title='Predicted classes for linear function', plot_show=False) plt.subplot(223) mse_softmax = learn_network(x, y, [20], [sigmoid, softmax], iterations=100, regression=False, plot_title='Predicted classes for softmax function', plot_show=False) plt.subplot(224) plot_measure_results_data([mse_linear, mse_softmax], labels=['linear', 'softmax'], title_base="Accuracy", ylabel="Accuracy", title_ending=" for last layer activation function", show=False) plt.show()
train, test = load_data(file_name='rings5-sparse',
folder='classification',
classification=True)
# x and y - observations and values for them
x = train[:, 0:2]
y = train[:, 2:3]
x_test = test[:, 0:2]
y_test = test[:, 2:3]
# plot data classes
plt.figure(figsize=(12.8, 4.8))
plt.subplot(121)
plot_data_2d(x[:, 0],
x[:, 1],
y[:, 0],
title='True classes of points from rings5 sparse dataset',
show=False)
neurons = [50, 50, 50]
labels = [
'no regularization', 'L1, lambda: 0.001', 'L1, lambda: 0.0001',
'L1, lambda: 0.00001', 'L2, lambda: 0.001', 'L2, lambda: 0.0001',
'L2, lambda: 0.00001'
]
# learn model and plot result classes
i = 2
activation = sigmoid
iterations = 10000
return -np.sum((results - y_ohe)**2) / res[2].shape[0] / 3
else:
results = results.transpose()
return np.sum(train[:,
2] == np.argmax(results, axis=0)) / res[2].shape[0]
# res = {2: np.array([1,2,3]), 3: np.array([3,1,2]), 4: np.array([0,4,0])}
data = {0: train[:, 0], 1: train[:, 1]}
# plot data classes
plt.figure(figsize=(12.8, 9.6))
plt.subplot(2, 2, 1)
plot_data_2d(x[:, 0],
x[:, 1],
y[:, 0],
title='True classes of points on the plane',
show=False)
n_input = 2
n_output = 3
n = 200
scores, accuracies = [], []
labels = [
"Without speciation, maximize accuracy",
"With speciation, maximize accuracy", "With speciation, maximize score"
]
for index, score_type, speciation in zip([2, 3, 4],
["accuracy", "accuracy", "score"],
[False, True, True]):
neat = NEAT(n_input,
train, test = load_data(file_name='xor3-balance',
folder='classification',
classification=True)
# x and y - observations and values for them
x = train[:, 0:2]
y = train[:, 2:3]
x_test = test[:, 0:2]
y_test = test[:, 2:3]
# plot data classes
plt.figure(figsize=(12.8, 4.8))
plt.subplot(121)
plot_data_2d(x[:, 0],
x[:, 1],
y[:, 0],
title='True classes of points from xor3 balance dataset',
show=False)
neurons = [50, 50, 50]
labels = [
'no regularization', 'L1, lambda: 0.001', 'L1, lambda: 0.0001',
'L1, lambda: 0.00001', 'L2, lambda: 0.001', 'L2, lambda: 0.0001',
'L2, lambda: 0.00001'
]
# learn model and plot result classes
i = 2
activation = sigmoid
iterations = 10000
from neural_network.plots import plot_data_2d
from kohonen.train import find_epochs_and_neighbour
# load data
data, classes = load_data("hexagon")
find_epochs_and_neighbour(data, classes, 5, 5, [5, 10, 20], [0.1, 0.2, 0.5, 1])
find_epochs_and_neighbour(data, classes, 5, 5, [5, 10, 20], [0.005, 0.01, 0.015, 0.03], distance_type="mexican")
# Gauss
# create network
network = Kohonen(5, 5, 2, neighbour_param=0.2, distance_type="gauss")
network.learn_epochs(data, epochs=10)
# plot network
plot_data_2d(data[:, :1], data[:, 1:], classes, show=False, title="Hexagon dataset, Gauss metric")
network.plot_weights()
plt.show()
# Mexican hat
# create network
network = Kohonen(5, 5, 2, neighbour_param=0.015, distance_type="mexican")
network.learn_epochs(data, epochs=10)
# plot network
plot_data_2d(data[:, :1], data[:, 1:], classes, show=False, title="Hexagon dataset, Mexican hat metric")
network.plot_weights()
plt.show()
# Results:
# it is really hard to train Kohonen using mexican hat metric,
def learn_network(x,
y,
hidden_layers,
activations,
x_test=None,
y_test=None,
use_test_and_stop_learning=False,
no_change_epochs_to_stop=50,
initialize_weights='norm',
momentum_type='RMSProp',
lambda_momentum=0.9,
beta=0.01,
eta=0.01,
epochs=1,
batch_size=32,
iterations=20,
regularization_lambda=0,
regularization_type="L2",
regression=True,
plot_title=None,
return_result=False,
plot=True,
plot_show=True):
# for classification problem we need one hot encoded y
if regression:
y_numeric = -1
else:
y_numeric = y
y = one_hot_encode(y)
# determine network parameters and initialize it
n_input = x.shape[1]
n_output = y.shape[1]
network = Network(n_input,
hidden_layers,
n_output,
activations,
initialize_weights=initialize_weights)
# initialize error list, predict and save default error
errors = []
# if use test dataset and regularization should test on test dataset
if use_test_and_stop_learning:
res = network.forward(x_test)
append_errors(errors, res, y_test, one_hot_encode(y_test), regression)
# else can test on train dataset
else:
res = network.forward(x)
append_errors(errors, res, y, y_numeric, regression)
# values needed for have in memory best network result and information how many epochs ago was positive change
best_result = set_up_best_result(regression)
any_change = 0
# in for loop train network
for i in range(iterations):
# train network (with backward propagation) depends on type of training (nothing, momentum, RMSProp)
if momentum_type == 'normal':
network.backward(x,
y,
eta=eta,
epochs=epochs,
batch_size=batch_size,
regularization_lambda=regularization_lambda,
regularization_type=regularization_type)
if momentum_type == 'momentum':
network.backward(x,
y,
eta=eta,
epochs=epochs,
batch_size=batch_size,
regularization_lambda=regularization_lambda,
regularization_type=regularization_type,
lambda_momentum=lambda_momentum,
moment_type='momentum')
if momentum_type == 'RMSProp':
network.backward(x,
y,
eta=eta,
epochs=epochs,
batch_size=batch_size,
regularization_lambda=regularization_lambda,
regularization_type=regularization_type,
beta=beta,
moment_type='RMSProp')
# predict and save error after i*epochs epochs
res = network.forward(x)
append_errors(errors, res, y, y_numeric, regression)
measure_name = "MSE" if regression else "accuracy"
print('Iteration {0}, {1}: {2:0.8f}'.format(i, measure_name,
errors[-1]))
# determine if should stop learning because of any better result in last 10 epochs
if use_test_and_stop_learning:
should_stop, best_result, any_change = stop_learning(
errors[-1],
best_result,
any_change,
regression,
no_change_epochs_to_stop=no_change_epochs_to_stop)
if should_stop:
break
# plot data after all training
if plot:
if regression:
if plot_title is not None:
plot_data_1d(x, y, res, title=plot_title)
else:
plot_data_1d(x, y, res)
else:
res = np.argmax(res, axis=1)
if plot_title is not None:
plot_data_2d(x[:, 0],
x[:, 1],
res,
title=plot_title,
show=plot_show)
else:
plot_data_2d(x[:, 0],
x[:, 1],
res,
title='Predicted classes of points on the plane',
show=plot_show)
if return_result:
return errors, res
return errors