def run_epoch_weight_init(X_train, X_test, y_train, y_test, layers, epochs, weight_inits, figure_folder="../fig", try_get_pickle=True, **kwargs): """Compares two weight inits.""" param_dict = {"weight_inits": weight_inits} param_dict.update(kwargs) print_parameters(**param_dict) # Double for-loop for all results pickle_fname = "mlp_epoch_weight_inits_results.pkl" if os.path.isfile(pickle_fname) and try_get_pickle: data = load_pickle(pickle_fname) else: data = {wi: {} for wi in weight_inits} for i, wi in enumerate(weight_inits): print("Weight init: {}".format(wi)) res_ = nn_core(X_train, X_test, y_train, y_test, layers, weight_init=wi, return_weights=True, epochs=epochs, **kwargs) data[wi] = convert_nn_core_to_dict(res_) data[wi]["label"] = wi.capitalize() data[wi]["x"] = np.arange(epochs) data[wi]["y"] = \ np.array(data[wi]["epoch_evaluations"]) / X_test.shape[0] save_pickle(pickle_fname, data) figname = "mlp_epoch_weight_inits.pdf" plot_epoch_accuracy(data, r"Epoch", r"Accuracy", figname, vmin=0.0, vmax=1.0)
def run_epoch_cost_functions(X_train, X_test, y_train, y_test, layers, epochs, cost_functions, try_get_pickle=True, figure_folder="../fig", **kwargs): """Compares cost functions over epochs inits: mse, log-loss""" param_dict = {"cost_functions": cost_functions} param_dict.update(kwargs) print_parameters(**param_dict) # Double for-loop for all results pickle_fname = "mlp_epoch_cost_functions_results.pkl" if os.path.isfile(pickle_fname) and try_get_pickle: data = load_pickle(pickle_fname) else: data = {cf: {} for cf in cost_functions} for i, cf in enumerate(cost_functions): print("Cost function: {}".format(cf)) res_ = nn_core(X_train, X_test, y_train, y_test, layers, cost_function=cf, return_weights=True, epochs=epochs, **kwargs) data[cf] = convert_nn_core_to_dict(res_) data[cf]["label"] = cf.capitalize() data[cf]["x"] = np.arange(epochs) data[cf]["y"] = \ np.array(data[cf]["epoch_evaluations"]) / X_test.shape[0] save_pickle(pickle_fname, data) figname = "mlp_epoch_cost_functions.pdf" plot_epoch_accuracy(data, r"Epoch", r"Accuracy", figname, vmin=0.0, vmax=1.0)
def run_epoch_activations(X_train, X_test, y_train, y_test, layers, epochs, activations, try_get_pickle=True, figure_folder="../fig", **kwargs): """Compares different layer activations.""" param_dict = {"activations": activations} param_dict.update(kwargs) print_parameters(**param_dict) # Double for-loop for all results pickle_fname = "mlp_epoch_activations_{}_results.pkl".format(kwargs["cost_function"]) if os.path.isfile(pickle_fname) and try_get_pickle: data = load_pickle(pickle_fname) else: data = {act: {} for act in activations} for i, act in enumerate(activations): print("Activation: {}".format(act)) res_ = nn_core(X_train, X_test, y_train, y_test, layers, activation=act, return_weights=True, epochs=epochs, **kwargs) data[act] = convert_nn_core_to_dict(res_) data[act]["label"] = act.capitalize() data[act]["x"] = np.arange(epochs) data[act]["y"] = \ np.array(data[act]["epoch_evaluations"]) / X_test.shape[0] save_pickle(pickle_fname, data) figname = "mlp_epoch_activations_{}_optimal_20mb_10neurons.pdf".format(kwargs["cost_function"]) plot_epoch_accuracy(data, r"Epoch", r"Accuracy", figname, vmin=0.0, vmax=1.0)
def run_lambda_learning_rate_comparison(X_train, X_test, y_train, y_test, lmbdas=None, learning_rates=None, try_get_pickle=True, figure_folder="../fig", **kwargs): param_dict = { "lmbdas": lmbdas, "sklearn": True, "figure_folder": figure_folder, "learning_rates": learning_rates } param_dict.update(kwargs) print_parameters(**param_dict) test_accuracy_values = [] train_accuracy_values = [] for lr in learning_rates: print("Learning rate: ", lr) pickle_fname = ("lambda_lr_accuracy_penalty{}_actsigmoid" "_solver{}_lr{}_mom0.0_tol1e-06." "pkl".format(kwargs["solver"], kwargs["penalty"], str(lr))) if os.path.isfile(pickle_fname) and try_get_pickle: res_ = load_pickle(pickle_fname) else: res_ = logreg_core(X_train, X_test, y_train, y_test, use_sk_learn=False, lmbdas=lmbdas, learning_rate=lr, store_pickle=True, pickle_fname=pickle_fname, **kwargs) train_accuracy, test_accuracy, critical_accuracy = res_ test_accuracy_values.append(test_accuracy) train_accuracy_values.append(train_accuracy) lr_labels = [r"Optimized"] lr_labels += [r"$\eta={0:.2f}$".format(lr) for lr in learning_rates[1:]] plot_accuracy_scores(lmbdas, train_accuracy_values, test_accuracy_values, lr_labels, "accuracy_learning_rate_scores", r"$\lambda$", r"Accuracy")
def run_lambda_solver(X_train, X_test, y_train, y_test, lmbdas=None, solvers=None, try_get_pickle=True, figure_folder="../fig", **kwargs): param_dict = { "lmbdas": lmbdas, "sklearn": True, "figure_folder": figure_folder, "solver": solvers } param_dict.update(kwargs) print_parameters(**param_dict) test_accuracy_values = [] train_accuracy_values = [] for solver in solvers: print("Solver:", solver) pickle_fname = ("lambda_solver_accuracy_penalty{}_actsigmoid" "_solver{}_lr{}_mom0.0_tol1e-06." "pkl".format(kwargs["penalty"], solver, kwargs["learning_rate"])) if os.path.isfile(pickle_fname) and try_get_pickle: res_ = load_pickle(pickle_fname) else: res_ = logreg_core(X_train, X_test, y_train, y_test, use_sk_learn=False, lmbdas=lmbdas, solver=solver, store_pickle=True, pickle_fname=pickle_fname, **kwargs) train_accuracy, test_accuracy, critical_accuracy = res_ test_accuracy_values.append(test_accuracy) train_accuracy_values.append(train_accuracy) plot_accuracy_scores( lmbdas, train_accuracy_values, test_accuracy_values, [r"Optimized Gradient Descent", r"Conjugate Gradient", r"Newtons-CG"], "accuracy_solver_scores", r"$\lambda$", r"Accuracy")
def run_lambda_penalty(X_train, X_test, y_train, y_test, lmbdas=None, penalties=None, try_get_pickle=True, figure_folder="../fig", **kwargs): param_dict = { "lmbdas": lmbdas, "sklearn": True, "figure_folder": figure_folder, "penalties": penalties } param_dict.update(kwargs) print_parameters(**param_dict) test_accuracy_values = [] train_accuracy_values = [] for penalty in penalties: print("Regularisation:", penalty) pickle_fname = ("lambda_penalty_accuracy_penalty{}_actsigmoid" "_solver{}_lrinverse_mom0.0_tol1e-06." "pkl".format(penalty, kwargs["solver"])) if os.path.isfile(pickle_fname) and try_get_pickle: res_ = load_pickle(pickle_fname) else: res_ = logreg_core(X_train, X_test, y_train, y_test, lmbdas=lmbdas, penalty=penalty, store_pickle=True, pickle_fname=pickle_fname, **kwargs) train_accuracy, test_accuracy, critical_accuracy = res_ test_accuracy_values.append(test_accuracy) train_accuracy_values.append(train_accuracy) plot_accuracy_scores(lmbdas, train_accuracy_values, test_accuracy_values, [r"$L^1$", r"$L^2$", r"Elastic net"], "accuracy_regularisation_scores", r"$\lambda$", r"Accuracy")
def run_lambda_momentum(X_train, X_test, y_train, y_test, lmbdas=None, momentums=None, try_get_pickle=True, figure_folder="../fig", **kwargs): param_dict = { "lmbdas": lmbdas, "sklearn": True, "figure_folder": figure_folder, "momentums": momentums } param_dict.update(kwargs) print_parameters(**param_dict) test_accuracy_values = [] train_accuracy_values = [] for momentum in momentums: print("Momentum: ", momentum) pickle_fname = ("lambda_mom_accuracy_penalty{}_actsigmoid" "_solverlr-gd_lrinverse_mom{}_tol1e-06." "pkl".format(kwargs["penalty"], str(momentum))) if os.path.isfile(pickle_fname) and try_get_pickle: res_ = load_pickle(pickle_fname) else: res_ = logreg_core(X_train, X_test, y_train, y_test, use_sk_learn=False, lmbdas=lmbdas, momentum=momentum, store_pickle=True, pickle_fname=pickle_fname, **kwargs) train_accuracy, test_accuracy, critical_accuracy = res_ test_accuracy_values.append(test_accuracy) train_accuracy_values.append(train_accuracy) plot_accuracy_scores(lmbdas, train_accuracy_values, test_accuracy_values, [r"$\gamma={0:.1e}$".format(m) for m in momentums], "accuracy_momentum_scores", r"$\lambda$", r"Accuracy")
def run_lambda_mini_batches(X_train, X_test, y_train, y_test, layers, lmbdas=None, mini_batch_sizes=None, try_get_pickle=True, figure_folder="../fig", **kwargs): """Compares mini batch sizes for lambda values.""" param_dict = {"lmbdas": lmbdas, "mini_batch_sizes": mini_batch_sizes} param_dict.update(kwargs) print_parameters(**param_dict) # Double for-loop for all results pickle_fname = "mlp_lambda_mini_batch_sizes_results.pkl" if os.path.isfile(pickle_fname) and try_get_pickle: data = load_pickle(pickle_fname) else: data = {lmbda: {mb: {} for mb in mini_batch_sizes} for lmbda in lmbdas} for i, lmbda in enumerate(lmbdas): for j, mb in enumerate(mini_batch_sizes): print("Lambda: {} MB: {}".format(lmbda, mb)) res_ = nn_core(X_train, X_test, y_train, y_test, layers, lmbda=lmbda, mini_batch_size=mb, return_weights=True, **kwargs) data[lmbda][mb] = res_ save_pickle(pickle_fname, data) # Maps values to matrix plot_data = np.empty((len(lmbdas), len(mini_batch_sizes))) # Populates plot data for i, lmbda in enumerate(lmbdas): for j, mb in enumerate(mini_batch_sizes): plot_data[i, j] = data[lmbda][mb][1] heatmap_plotter(lmbdas, mini_batch_sizes, plot_data.T, "mlp_lambda_mini_batch_size.pdf", tick_param_fs=8, label_fs=10, vmin=0.0, vmax=1.0, xlabel=r"$\lambda$", ylabel=r"$N_\mathrm{MB}$", cbartitle=r"Accuracy", x_tick_mode="exp", y_tick_mode="int")
def run_neurons_training_size(layers, neurons, training_sizes, data_size, data_path, try_get_pickle=True, figure_folder="../fig", **kwargs): """Compares different neurons for different training sizese.""" param_dict = {"neurons": neurons, "training_sizes": training_sizes} param_dict.update(kwargs) print_parameters(**param_dict) # Double for-loop for all results pickle_fname = "mlp_neurons_training_size_results.pkl" if os.path.isfile(pickle_fname) and try_get_pickle: data = load_pickle(pickle_fname) else: data = {n: {ts: {} for ts in training_sizes} for n in neurons} for i, neuron in enumerate(neurons): for j, ts in enumerate(training_sizes): inlay, outlay, X_train, X_test, y_train, y_test = \ retrieve_2d_data_formatted(data_path, data_size, ts) print("Neurons: {} Training size: {}".format(neuron, ts)) layers[1] = neuron print(X_train.shape, X_test.shape) res_ = nn_core(X_train, X_test, y_train, y_test, layers, return_weights=True, **kwargs) data[neuron][ts] = res_ save_pickle(pickle_fname, data) # Maps values to matrix plot_data = np.empty((len(neurons), len(training_sizes))) # Populates plot data for i, n in enumerate(neurons): for j, ts in enumerate(training_sizes): plot_data[i, j] = data[n][ts][1] heatmap_plotter(neurons, training_sizes, plot_data.T, "mlp_neurons_training_size.pdf", tick_param_fs=8, label_fs=10, xlabel=r"Neurons", ylabel=r"Training size", cbartitle=r"Accuracy", vmin=0.0, vmax=1.0, x_tick_mode="int", y_tick_mode="float")
def run_neurons_eta(X_train, X_test, y_train, y_test, layers, neurons=None, learning_rates=None, try_get_pickle=True, figure_folder="../fig", **kwargs): """Compares different neuron sizes for different etas.""" param_dict = {"neurons": neurons, "learning_rates": learning_rates} param_dict.update(kwargs) print_parameters(**param_dict) # Double for-loop for all results pickle_fname = "mlp_neurons_eta_results.pkl" if os.path.isfile(pickle_fname) and try_get_pickle: data = load_pickle(pickle_fname) else: data = {n: {eta: {} for eta in learning_rates} for n in neurons} for i, neuron in enumerate(neurons): for j, eta in enumerate(learning_rates): print("Neuron: {} Eta: {}".format(neuron, eta)) layers[1] = neuron res_ = nn_core(X_train, X_test, y_train, y_test, layers, return_weights=True, learning_rate=eta, **kwargs) data[neuron][eta] = res_ save_pickle(pickle_fname, data) # Maps values to matrix plot_data = np.empty((len(neurons), len(learning_rates))) # Populates plot data for i, n in enumerate(neurons): for j, eta in enumerate(learning_rates): plot_data[i, j] = data[n][eta][1] heatmap_plotter(neurons, learning_rates, plot_data.T, "mlp_neurons_eta.pdf", tick_param_fs=8, label_fs=10, xlabel=r"Neurons", ylabel=r"$\eta$", cbartitle=r"Accuracy", vmin=0.0, vmax=1.0, x_tick_mode="int", y_tick_mode="exp")
def run_lambda_neurons(X_train, X_test, y_train, y_test, layers, lmbdas=None, neurons=None, try_get_pickle=True, figure_folder="../fig", **kwargs): """Compares different lambdas for different neuron sizes.""" param_dict = {"lmbdas": lmbdas, "neurons": neurons} param_dict.update(kwargs) print_parameters(**param_dict) # Double for-loop for all results pickle_fname = "mlp_lambda_neurons_results.pkl" if os.path.isfile(pickle_fname) and try_get_pickle: data = load_pickle(pickle_fname) else: data = {lmbda: {neuron: {} for neuron in neurons} for lmbda in lmbdas} for i, lmbda in enumerate(lmbdas): for j, neuron in enumerate(neurons): print("Lambda: {} Neuron: {}".format(lmbda, neuron)) layers[1] = neuron res_ = nn_core(X_train, X_test, y_train, y_test, layers, lmbda=lmbda, return_weights=True, **kwargs) data[lmbda][neuron] = res_ save_pickle(pickle_fname, data) # Maps values to matrix plot_data = np.empty((len(lmbdas), len(neurons))) # Populates plot data for i, lmbda in enumerate(lmbdas): for j, n in enumerate(neurons): plot_data[i, j] = data[lmbda][n][1] heatmap_plotter(lmbdas, neurons, plot_data.T, "mlp_lambda_neurons.pdf", tick_param_fs=8, label_fs=10, xlabel=r"$\lambda$", ylabel=r"Neurons", vmin=0.0, vmax=1.0, cbartitle=r"Accuracy", x_tick_mode="exp", y_tick_mode="int")
def run_lambda_eta(X_train, X_test, y_train, y_test, layers, lmbdas=None, learning_rates=None, try_get_pickle=True, figure_folder="../fig", **kwargs): """Runs NN for different lambdas and etas.""" param_dict = {"lmbdas": lmbdas, "learning_rates": learning_rates} param_dict.update(kwargs) print_parameters(**param_dict) # Double for-loop for all results pickle_fname = "mlp_lambda_eta_results.pkl" if os.path.isfile(pickle_fname) and try_get_pickle: data = load_pickle(pickle_fname) else: data = {lmbda: {eta: {} for eta in learning_rates} for lmbda in lmbdas} for i, lmbda in enumerate(lmbdas): for j, eta in enumerate(learning_rates): print("Lambda: {} Eta: {}".format(lmbda, eta)) res_ = nn_core(X_train, X_test, y_train, y_test, layers, lmbda=lmbda, return_weights=True, learning_rate=eta, **kwargs) data[lmbda][eta] = res_ save_pickle(pickle_fname, data) # Maps values to matrix plot_data = np.empty((len(lmbdas), len(learning_rates))) # Populates plot data for i, lmbda in enumerate(lmbdas): for j, eta in enumerate(learning_rates): plot_data[i, j] = data[lmbda][eta][1] heatmap_plotter(lmbdas, learning_rates, plot_data.T, "mlp_lambda_eta.pdf", tick_param_fs=8, label_fs=10, xlabel=r"$\lambda$", ylabel=r"$\eta$", cbartitle=r"Accuracy", vmin=0.0, vmax=1.0, x_tick_mode="exp", y_tick_mode="exp")
def run_sk_comparison(X_train, X_test, y_train, y_test, lmbdas=None, penalty=None, activation=None, solver=None, learning_rate=None, momentum=None, mini_batch_size=None, max_iter=None, tolerance=None, verbose=False, try_get_pickle=True, figure_folder="../fig"): """Runs a comparison between sk learn and our method.""" param_dict = { "lmbdas": lmbdas, "sklearn": True, "penalty": penalty, "activation": activation, "solver": solver, "learning_rate": learning_rate, "momentum": momentum, "mini_batch_size": mini_batch_size, "max_iter": max_iter, "tolerance": tolerance } print_parameters(**param_dict) pickle_fname = ("sk_comparison_accuracy_sklearn_penalty{}_actsigmoid" "_solverlr-gd_lrinverse_mom0.0_tol1e-06" ".pkl".format(penalty)) if os.path.isfile(pickle_fname) and try_get_pickle: res_ = load_pickle(pickle_fname) else: res_ = logreg_core(X_train, X_test, y_train, y_test, use_sk_learn=True, lmbdas=lmbdas, penalty=penalty, activation=activation, solver=solver, learning_rate=learning_rate, momentum=momentum, mini_batch_size=mini_batch_size, max_iter=max_iter, tolerance=tolerance, store_pickle=True, pickle_fname=pickle_fname, verbose=verbose) # Retrieves results train_accuracy, test_accuracy, critical_accuracy, \ train_accuracy_SK, test_accuracy_SK, critical_accuracy_SK, \ train_accuracy_SGD, test_accuracy_SGD, critical_accuracy_SGD = res_ print('Mean accuracy: train, test') print("HomeMade: {0:0.4f} +/- {1:0.2f}, {2:0.4f} +/- {3:0.2f}".format( np.mean(train_accuracy), np.std(train_accuracy), np.mean(test_accuracy), np.std(test_accuracy))) print("SK: {0:0.4f} +/- {1:0.2f}, {2:0.4f} +/- {3:0.2f}".format( np.mean(train_accuracy_SK), np.std(train_accuracy_SK), np.mean(test_accuracy_SK), np.std(test_accuracy_SK))) print("SGD: {0:0.4f} +/- {1:0.2f}, {2:0.4f} +/- {3:0.2f}".format( np.mean(train_accuracy_SGD), np.std(train_accuracy_SGD), np.mean(test_accuracy_SGD), np.std(test_accuracy_SGD))) plot_accuracy_comparison(lmbdas, train_accuracy, test_accuracy, train_accuracy_SK, test_accuracy_SK, train_accuracy_SGD, test_accuracy_SGD, "logistic_accuracy_sklearn_comparison", figure_folder)