from neural_network.activations import * from neural_network.data_manipulation import load_easy_data from neural_network.plots import plot_data_2d, plot_measure_results_data from neural_network.learn_network import learn_network import matplotlib.pyplot as plt # load dataset train, test = load_easy_data() # x and y - observations and values for them x = train[:, 0:2] y = train[:, 2:3] # plot data classes plt.figure(figsize=(12.8, 9.6)) plt.subplot(221) plot_data_2d(x[:, 0], x[:, 1], y[:, 0], title='True classes of points on the plane', show=False) # learn model and plot result classes plt.subplot(222) mse_linear = learn_network(x, y, [20], [sigmoid, linear], iterations=100, regression=False, plot_title='Predicted classes for linear function', plot_show=False) plt.subplot(223) mse_softmax = learn_network(x, y, [20], [sigmoid, softmax], iterations=100, regression=False, plot_title='Predicted classes for softmax function', plot_show=False) plt.subplot(224) plot_measure_results_data([mse_linear, mse_softmax], labels=['linear', 'softmax'], title_base="Accuracy", ylabel="Accuracy", title_ending=" for last layer activation function", show=False) plt.show()
regularization_type=reg_type, plot_title="Prediction with sigmoid activation function and " + str(i) + " hidden layers", plot=False, return_result=True, x_test=x_test, y_test=y_test, use_test_and_stop_learning=True, no_change_epochs_to_stop=no_change_epochs_to_stop) mses.append(mse_reg) results.append(res_reg) plt.figure(figsize=(12.8, 4.8)) plt.subplot(121) plot_data_1d_compare( x, y, results, labels=["true"] + labels, title= "Comparison of different regularization methods on multimodal sparse dataset", show=False) # plot from 5th iteration to better see differences measures = add_the_same_value_to_the_same_length(mses) plt.subplot(122) plot_measure_results_data(measures, labels=labels, title_ending=" for multimodal sparse dataset", from_error=50) plt.show()
mse_Xavier, Xavier = learn_network(x, y, [20], [sigmoid, linear], initialize_weights='Xavier', iterations=100, momentum_type='normal', plot=False, return_result=True) labels = ['Zero weights', 'Uniform distribution', 'Xavier'] plot_data_1d_compare(x, y, [zeros, normal, Xavier], labels=["true"] + labels, title="Comparison of initialization weights") plot_measure_results_data([mse_zeros, mse_normal, mse_Xavier], labels=labels, title_ending=" for initialization weights", from_error=5) # batch size experiment mse_all, res_all = learn_network(x, y, [20], [sigmoid, linear], initialize_weights='Xavier', eta=0.00001, batch_size=10000, iterations=100, momentum_type='normal', plot=False, return_result=True) mse_batch, batch = learn_network(x, y, [20], [sigmoid, linear], initialize_weights='Xavier',
x, y, neurons[:i], [activation] * i + [softmax], beta=0.01, eta=0.01, epochs=1, iterations=iterations, regression=False, regularization_lambda=reg_lambda, regularization_type=reg_type, plot_title="Prediction with sigmoid activation function and " + str(i) + " hidden layers", plot=True, return_result=False, x_test=x_test, y_test=y_test, use_test_and_stop_learning=True, no_change_epochs_to_stop=no_change_epochs_to_stop) mses.append(mse_reg) # plot from 5th iteration to better see differences measures = add_the_same_value_to_the_same_length(mses) plt.subplot(122) plot_measure_results_data(measures, labels=labels, title_base="accuracy", ylabel="accuracy", title_ending=" for different regularization methods", from_error=200) plt.show()
best_results = [] gen = Genetic(500, 5, eval_function, mutation_coef=1, mutation_percentage=mutation_percentage) for i in range(100): gen.learn_population(epochs=1) best_results.append(gen.get_best()[0]) mses1.append(best_results) print(f'Ended {mutation_percentage}') mses2 = [] mutation_coefs = [0.1, 0.2, 0.5, 1] for mutation_coef in mutation_coefs: best_results = [] gen = Genetic(500, 5, eval_function, mutation_coef=mutation_coef, mutation_percentage=0.3) for i in range(100): gen.learn_population(epochs=1) best_results.append(gen.get_best()[0]) mses2.append(best_results) print(f'Ended {mutation_coef}') # plot results title = "Function value " y_label = "Logarithm of function value " labels = ["Population size = " + str(val) for val in population_size] plot_measure_results_data(mses0, title_base=title, ylabel=y_label, labels=labels, from_error=0, y_log=True) labels = ["Mutation percentage = " + str(val) for val in mutation_percentages] plot_measure_results_data(mses1, title_base=title, ylabel=y_label, labels=labels, from_error=0, y_log=True) labels = ["Mutation coefficient = " + str(val) for val in mutation_coefs] plot_measure_results_data(mses2, title_base=title, ylabel=y_label, labels=labels, from_error=0, y_log=True)
str(i) + " hidden layers", plot=False) mse_tanh = learn_network( x, y, neurons[:i], [tanh] * i + [softmax], beta=0.01, eta=0.01, epochs=1, iterations=1000, regression=False, plot_title="Prediction with tanh activation function and " + str(i) + " hidden layers", plot=False) plt.subplot(220 + i + 1) plot_measure_results_data([mse_linear, mse_ReLU, mse_sigmoid, mse_tanh], labels=['linear', 'ReLU', 'sigmoid', 'tanh'], title_base="accuracy", ylabel="accuracy", title_ending=" for " + str(i) + " layers networks", show=False, from_error=5) plt.show() # Results # The best activation is tanh, then (ReLU and sigmoid)*, linear is the worst one # * for bigger networks sigmoid is better (and as good as tanh) than ReLU and for 3 layers ReLU had nan in gradient and then it fails # Linear activation is the worst for any number of layers # The more layers the better results achieve networks
15, score_type, use_speciation=speciation) for i in range(50): print(f'Start of epoch: {i+1}') neat.epoch(i) scores.append(neat.best_scores) accuracies.append(neat.best_accuracies) best = neat.get_best_population("accuracy")[0] res = neat.population[best].evaluate(copy.deepcopy(data)) results = np.vstack([res[2], res[3], res[4]]) y_predicted = np.argmax(results, axis=0) plt.subplot(2, 2, index) plot_data_2d(x[:, 0], x[:, 1], y_predicted, title=labels[index - 2], show=False) plt.show() # plot_measure_results_data(scores, title_base="Score ", ylabel="Score ", labels=labels, from_error=0, y_log=True) plot_measure_results_data(accuracies, title_base="Accuracy ", ylabel="Accuracy ", labels=labels, from_error=0)
indexes = np.argsort(cut.evaluate()) scores = [] for i in range(50): cut.learn_population(1) indexes = np.argsort(cut.evaluate()) res = cut.__evaluate_one__(indexes[0], return_area=False) scores.append(res / cut.best_rectangle_score_per_unit / circle) Ascores1.append(scores) AAscores1.append(Ascores1) plt.figure(figsize=(12.8, 14.4)) for i, Ascores1 in enumerate(AAscores1): plt.subplot(3, 2, i + 1) plot_measure_results_data(Ascores1, title_base=f"Score for R = {r_labels[i]}", labels=labels1, ylabel="Score normalized", show=False) plt.show() AAscores2, labels2, r_labels = [], [], [] for r in [800, 850, 1000, 1100, 1200]: r_labels.append(str(r)) Ascores1 = [] for mutation_percentage in [0.1, 0.2, 0.3, 0.5]: labels2.append(f'Mutation percentage = {mutation_percentage}') cut = CuttingStock(r, "r" + str(r) + ".csv", n=1500) circle = r * r * math.pi indexes = np.argsort(cut.evaluate()) scores = [] for i in range(50):
" hidden layers", plot=False, return_result=True) plt.subplot(420 + 2 * i - 1) plot_data_1d_compare(x, y, [res_linear, res_ReLU, res_sigmoid, res_tanh], labels=["true"] + labels, title="Comparison of activation functions for " + str(i) + " hidden layers networks", show=False) # plot from 5th iteration to better see differences plt.subplot(420 + 2 * i) plot_measure_results_data([mse_linear, mse_ReLU, mse_sigmoid, mse_tanh], labels=labels, title_ending=" for " + str(i) + " layers networks", from_error=5, show=False) if i == 3: plt.subplot(420 + 2 * i + 2) plot_measure_results_data([mse_linear, mse_sigmoid, mse_tanh], labels=labels[:1] + labels[2:4], colors=['red', 'cyan', 'yellow'], title_ending=" for " + str(i) + " layers networks", from_error=5, show=False) plt.show() # Results # mainly the same as in steps-large, only differences:
momentum_type='momentum', lambda_momentum=0.9, eta=0.01, epochs=1, iterations=100, plot=False, return_result=True) # learn model with RMSProp and show model mse_rms, rms = learn_network(x, y, [20], [sigmoid, linear], momentum_type='RMSProp', beta=0.01, eta=0.1, epochs=1, iterations=100, plot=False, return_result=True) labels = ['No momentum', 'Momentum', 'RMSProp'] # plot data and mse plot_data_1d_compare(x, y, [base, mom, rms], labels=["true"] + labels, title="Comparison of momentum learning") plot_measure_results_data([mse, mse_mom, mse_rms], labels=labels, title_ending=" for momentum learning", from_error=5)