def run_epoch_weight_init(X_train, X_test, y_train, y_test, layers,
epochs, weight_inits,
figure_folder="../fig",
try_get_pickle=True,
**kwargs):
"""Compares two weight inits."""
param_dict = {"weight_inits": weight_inits}
param_dict.update(kwargs)
print_parameters(**param_dict)
# Double for-loop for all results
pickle_fname = "mlp_epoch_weight_inits_results.pkl"
if os.path.isfile(pickle_fname) and try_get_pickle:
data = load_pickle(pickle_fname)
else:
data = {wi: {} for wi in weight_inits}
for i, wi in enumerate(weight_inits):
print("Weight init: {}".format(wi))
res_ = nn_core(X_train, X_test, y_train, y_test, layers,
weight_init=wi, return_weights=True,
epochs=epochs, **kwargs)
data[wi] = convert_nn_core_to_dict(res_)
data[wi]["label"] = wi.capitalize()
data[wi]["x"] = np.arange(epochs)
data[wi]["y"] = \
np.array(data[wi]["epoch_evaluations"]) / X_test.shape[0]
save_pickle(pickle_fname, data)
figname = "mlp_epoch_weight_inits.pdf"
plot_epoch_accuracy(data, r"Epoch", r"Accuracy",
figname, vmin=0.0, vmax=1.0)
def run_epoch_cost_functions(X_train, X_test, y_train, y_test, layers,
epochs, cost_functions,
try_get_pickle=True,
figure_folder="../fig", **kwargs):
"""Compares cost functions over epochs inits: mse, log-loss"""
param_dict = {"cost_functions": cost_functions}
param_dict.update(kwargs)
print_parameters(**param_dict)
# Double for-loop for all results
pickle_fname = "mlp_epoch_cost_functions_results.pkl"
if os.path.isfile(pickle_fname) and try_get_pickle:
data = load_pickle(pickle_fname)
else:
data = {cf: {} for cf in cost_functions}
for i, cf in enumerate(cost_functions):
print("Cost function: {}".format(cf))
res_ = nn_core(X_train, X_test, y_train, y_test, layers,
cost_function=cf, return_weights=True,
epochs=epochs, **kwargs)
data[cf] = convert_nn_core_to_dict(res_)
data[cf]["label"] = cf.capitalize()
data[cf]["x"] = np.arange(epochs)
data[cf]["y"] = \
np.array(data[cf]["epoch_evaluations"]) / X_test.shape[0]
save_pickle(pickle_fname, data)
figname = "mlp_epoch_cost_functions.pdf"
plot_epoch_accuracy(data, r"Epoch", r"Accuracy",
figname, vmin=0.0, vmax=1.0)
def run_epoch_activations(X_train, X_test, y_train, y_test, layers,
epochs, activations,
try_get_pickle=True,
figure_folder="../fig", **kwargs):
"""Compares different layer activations."""
param_dict = {"activations": activations}
param_dict.update(kwargs)
print_parameters(**param_dict)
# Double for-loop for all results
pickle_fname = "mlp_epoch_activations_{}_results.pkl".format(kwargs["cost_function"])
if os.path.isfile(pickle_fname) and try_get_pickle:
data = load_pickle(pickle_fname)
else:
data = {act: {} for act in activations}
for i, act in enumerate(activations):
print("Activation: {}".format(act))
res_ = nn_core(X_train, X_test, y_train, y_test, layers,
activation=act, return_weights=True,
epochs=epochs, **kwargs)
data[act] = convert_nn_core_to_dict(res_)
data[act]["label"] = act.capitalize()
data[act]["x"] = np.arange(epochs)
data[act]["y"] = \
np.array(data[act]["epoch_evaluations"]) / X_test.shape[0]
save_pickle(pickle_fname, data)
figname = "mlp_epoch_activations_{}_optimal_20mb_10neurons.pdf".format(kwargs["cost_function"])
plot_epoch_accuracy(data, r"Epoch", r"Accuracy",
figname, vmin=0.0, vmax=1.0)
def run_lambda_mini_batches(X_train, X_test, y_train, y_test, layers,
lmbdas=None, mini_batch_sizes=None,
try_get_pickle=True,
figure_folder="../fig", **kwargs):
"""Compares mini batch sizes for lambda values."""
param_dict = {"lmbdas": lmbdas, "mini_batch_sizes": mini_batch_sizes}
param_dict.update(kwargs)
print_parameters(**param_dict)
# Double for-loop for all results
pickle_fname = "mlp_lambda_mini_batch_sizes_results.pkl"
if os.path.isfile(pickle_fname) and try_get_pickle:
data = load_pickle(pickle_fname)
else:
data = {lmbda: {mb: {} for mb in mini_batch_sizes} for lmbda in lmbdas}
for i, lmbda in enumerate(lmbdas):
for j, mb in enumerate(mini_batch_sizes):
print("Lambda: {} MB: {}".format(lmbda, mb))
res_ = nn_core(X_train, X_test, y_train, y_test, layers,
lmbda=lmbda,
mini_batch_size=mb,
return_weights=True,
**kwargs)
data[lmbda][mb] = res_
save_pickle(pickle_fname, data)
# Maps values to matrix
plot_data = np.empty((len(lmbdas), len(mini_batch_sizes)))
# Populates plot data
for i, lmbda in enumerate(lmbdas):
for j, mb in enumerate(mini_batch_sizes):
plot_data[i, j] = data[lmbda][mb][1]
heatmap_plotter(lmbdas, mini_batch_sizes, plot_data.T,
"mlp_lambda_mini_batch_size.pdf",
tick_param_fs=8, label_fs=10,
vmin=0.0, vmax=1.0, xlabel=r"$\lambda$",
ylabel=r"$N_\mathrm{MB}$",
cbartitle=r"Accuracy",
x_tick_mode="exp", y_tick_mode="int")
def run_neurons_training_size(layers, neurons, training_sizes,
data_size, data_path, try_get_pickle=True,
figure_folder="../fig", **kwargs):
"""Compares different neurons for different training sizese."""
param_dict = {"neurons": neurons, "training_sizes": training_sizes}
param_dict.update(kwargs)
print_parameters(**param_dict)
# Double for-loop for all results
pickle_fname = "mlp_neurons_training_size_results.pkl"
if os.path.isfile(pickle_fname) and try_get_pickle:
data = load_pickle(pickle_fname)
else:
data = {n: {ts: {} for ts in training_sizes} for n in neurons}
for i, neuron in enumerate(neurons):
for j, ts in enumerate(training_sizes):
inlay, outlay, X_train, X_test, y_train, y_test = \
retrieve_2d_data_formatted(data_path, data_size, ts)
print("Neurons: {} Training size: {}".format(neuron, ts))
layers[1] = neuron
print(X_train.shape, X_test.shape)
res_ = nn_core(X_train, X_test, y_train, y_test, layers,
return_weights=True, **kwargs)
data[neuron][ts] = res_
save_pickle(pickle_fname, data)
# Maps values to matrix
plot_data = np.empty((len(neurons), len(training_sizes)))
# Populates plot data
for i, n in enumerate(neurons):
for j, ts in enumerate(training_sizes):
plot_data[i, j] = data[n][ts][1]
heatmap_plotter(neurons, training_sizes, plot_data.T,
"mlp_neurons_training_size.pdf",
tick_param_fs=8, label_fs=10,
xlabel=r"Neurons", ylabel=r"Training size",
cbartitle=r"Accuracy", vmin=0.0, vmax=1.0,
x_tick_mode="int", y_tick_mode="float")
def run_neurons_eta(X_train, X_test, y_train, y_test, layers,
neurons=None, learning_rates=None,
try_get_pickle=True,
figure_folder="../fig", **kwargs):
"""Compares different neuron sizes for different etas."""
param_dict = {"neurons": neurons, "learning_rates": learning_rates}
param_dict.update(kwargs)
print_parameters(**param_dict)
# Double for-loop for all results
pickle_fname = "mlp_neurons_eta_results.pkl"
if os.path.isfile(pickle_fname) and try_get_pickle:
data = load_pickle(pickle_fname)
else:
data = {n: {eta: {} for eta in learning_rates} for n in neurons}
for i, neuron in enumerate(neurons):
for j, eta in enumerate(learning_rates):
print("Neuron: {} Eta: {}".format(neuron, eta))
layers[1] = neuron
res_ = nn_core(X_train, X_test, y_train, y_test, layers,
return_weights=True,
learning_rate=eta, **kwargs)
data[neuron][eta] = res_
save_pickle(pickle_fname, data)
# Maps values to matrix
plot_data = np.empty((len(neurons), len(learning_rates)))
# Populates plot data
for i, n in enumerate(neurons):
for j, eta in enumerate(learning_rates):
plot_data[i, j] = data[n][eta][1]
heatmap_plotter(neurons, learning_rates, plot_data.T,
"mlp_neurons_eta.pdf",
tick_param_fs=8, label_fs=10,
xlabel=r"Neurons", ylabel=r"$\eta$",
cbartitle=r"Accuracy", vmin=0.0, vmax=1.0,
x_tick_mode="int", y_tick_mode="exp")
def run_lambda_neurons(X_train, X_test, y_train, y_test, layers,
lmbdas=None, neurons=None,
try_get_pickle=True,
figure_folder="../fig", **kwargs):
"""Compares different lambdas for different neuron sizes."""
param_dict = {"lmbdas": lmbdas, "neurons": neurons}
param_dict.update(kwargs)
print_parameters(**param_dict)
# Double for-loop for all results
pickle_fname = "mlp_lambda_neurons_results.pkl"
if os.path.isfile(pickle_fname) and try_get_pickle:
data = load_pickle(pickle_fname)
else:
data = {lmbda: {neuron: {} for neuron in neurons} for lmbda in lmbdas}
for i, lmbda in enumerate(lmbdas):
for j, neuron in enumerate(neurons):
print("Lambda: {} Neuron: {}".format(lmbda, neuron))
layers[1] = neuron
res_ = nn_core(X_train, X_test, y_train, y_test, layers,
lmbda=lmbda, return_weights=True,
**kwargs)
data[lmbda][neuron] = res_
save_pickle(pickle_fname, data)
# Maps values to matrix
plot_data = np.empty((len(lmbdas), len(neurons)))
# Populates plot data
for i, lmbda in enumerate(lmbdas):
for j, n in enumerate(neurons):
plot_data[i, j] = data[lmbda][n][1]
heatmap_plotter(lmbdas, neurons, plot_data.T, "mlp_lambda_neurons.pdf",
tick_param_fs=8, label_fs=10,
xlabel=r"$\lambda$",
ylabel=r"Neurons", vmin=0.0, vmax=1.0,
cbartitle=r"Accuracy", x_tick_mode="exp",
y_tick_mode="int")
def nn_loop_wrapper(loop_arg, store_pickle, pickle_fname, *args, **kwargs):
train_accuracy_values = []
test_accuracy_values = []
train_accuracy_epochs_values = []
test_accuracy_epochs_values = []
for hyperparam in kwargs[loop_arg]:
res_ = nn_core(*args, loop_arg=hyperparam, **kwargs)
train_accuracy_values.append(res_[0])
test_accuracy_values.append(res_[1])
train_accuracy_epochs_values.append(res_[2])
test_accuracy_epochs_values.append(res_[3])
results = [train_accuracy_values, test_accuracy_values,
train_accuracy_epochs_values, test_accuracy_epochs_values]
if store_pickle:
fname = ("mlp_{}.pkl".format(pickle_fname))
save_pickle(fname, results)
return results
def run_lambda_eta(X_train, X_test, y_train, y_test, layers,
lmbdas=None, learning_rates=None, try_get_pickle=True,
figure_folder="../fig", **kwargs):
"""Runs NN for different lambdas and etas."""
param_dict = {"lmbdas": lmbdas, "learning_rates": learning_rates}
param_dict.update(kwargs)
print_parameters(**param_dict)
# Double for-loop for all results
pickle_fname = "mlp_lambda_eta_results.pkl"
if os.path.isfile(pickle_fname) and try_get_pickle:
data = load_pickle(pickle_fname)
else:
data = {lmbda: {eta: {} for eta in learning_rates} for lmbda in lmbdas}
for i, lmbda in enumerate(lmbdas):
for j, eta in enumerate(learning_rates):
print("Lambda: {} Eta: {}".format(lmbda, eta))
res_ = nn_core(X_train, X_test, y_train, y_test, layers,
lmbda=lmbda, return_weights=True,
learning_rate=eta, **kwargs)
data[lmbda][eta] = res_
save_pickle(pickle_fname, data)
# Maps values to matrix
plot_data = np.empty((len(lmbdas), len(learning_rates)))
# Populates plot data
for i, lmbda in enumerate(lmbdas):
for j, eta in enumerate(learning_rates):
plot_data[i, j] = data[lmbda][eta][1]
heatmap_plotter(lmbdas, learning_rates, plot_data.T, "mlp_lambda_eta.pdf",
tick_param_fs=8, label_fs=10,
xlabel=r"$\lambda$", ylabel=r"$\eta$",
cbartitle=r"Accuracy", vmin=0.0, vmax=1.0,
x_tick_mode="exp", y_tick_mode="exp")
def logreg_core(X_train,
X_test,
y_train,
y_test,
use_sk_learn=False,
lmbdas=[None],
penalty=None,
activation=None,
solver=None,
learning_rate=None,
momentum=None,
mini_batch_size=None,
max_iter=None,
tolerance=None,
store_pickle=False,
pickle_fname=None,
verbose=False):
"""Method for retrieveing data for given lists of hyperparameters
Args:
X_train (ndarray)
X_test (ndarray)
y_train (ndarray)
y_test (ndarray)
use_sk_learn (bool): if we are to use SK-learn. Default is False.
lmbdas (float): list of lmbdas
penalty (str): penalty type. Choices: l1, l2, elastic_net
activation (str): activation function.
solver(str): solver function.
learning_rate (str|float): learning rate. Options: float, inverse
momentum (float): momentum strength
mini_batch_size (float): minibatch size
tolerance (float): tolerance, at what point we cut off the parameter
search.
store_pickle (bool): saves output as pickle
pickle_fname (str): pickle filename
verbose (bool): more verbose output. Default is False
Returns:
Dictionary with logreg accuracy scores and times
SK-learn dictionary with accuracy scores and times
SGD-SK-learn dictionary with accuracy scores and times
"""
# Sets up data arrays
train_accuracy = np.zeros(lmbdas.shape, np.float64)
test_accuracy = np.zeros(lmbdas.shape, np.float64)
critical_accuracy = np.zeros(lmbdas.shape, np.float64)
if use_sk_learn:
train_accuracy_SK = np.zeros(lmbdas.shape, np.float64)
test_accuracy_SK = np.zeros(lmbdas.shape, np.float64)
critical_accuracy_SK = np.zeros(lmbdas.shape, np.float64)
train_accuracy_SGD = np.zeros(lmbdas.shape, np.float64)
test_accuracy_SGD = np.zeros(lmbdas.shape, np.float64)
critical_accuracy_SGD = np.zeros(lmbdas.shape, np.float64)
# Loops over regularisation strength
for i, lmbda in enumerate(lmbdas):
if verbose:
print("")
print("=" * 80)
print("Lambda = ", lmbda)
if use_sk_learn:
# Sets up lambda for SK learn logreg methods
if lmbda == 0.0:
sk_lmbda = 1.0 / 10000000
else:
sk_lmbda = 1.0 / lmbda
# Set sk penalty
if penalty == "elastic_net":
sk_penalty = "l2"
else:
sk_penalty = penalty
# Define SK-learn logistic regressor
logreg_SK = sk_model.LogisticRegression(penalty=sk_penalty,
fit_intercept=False,
C=sk_lmbda,
max_iter=max_iter,
tol=tolerance)
# SK learn fit
with warnings.catch_warnings():
warnings.simplefilter("ignore")
logreg_SK.fit(cp.deepcopy(X_train), cp.deepcopy(y_train))
if verbose:
print("SK-learn method done")
# Retrieve accuracy scores for SK-learn
train_accuracy_SK[i] = logreg_SK.score(X_train, y_train)
test_accuracy_SK[i] = logreg_SK.score(X_test, y_test)
# Sets up learning rate for SK-learn SGD
if learning_rate == "inverse":
sk_learning_rate = "optimal"
eta0 = 0.0
else:
sk_learning_rate = "constant"
eta0 = learning_rate
# Sets up regularisation for SK-learn SGD
if penalty == "elastic_net":
sk_penalty = "elasticnet"
# define SGD-based logistic regression
logreg_SGD = sk_model.SGDClassifier(loss="log",
penalty=sk_penalty,
alpha=lmbda,
max_iter=max_iter,
shuffle=True,
random_state=1,
eta0=eta0,
learning_rate=sk_learning_rate)
# fit training data
logreg_SGD.fit(X_train, y_train)
if verbose:
print("SGA SK-learn method done")
# check accuracy
train_accuracy_SGD[i] = logreg_SGD.score(X_train, y_train)
test_accuracy_SGD[i] = logreg_SGD.score(X_test, y_test)
# critical_accuracy_SGD[i]=logreg_SGD.score(X_critical,Y_critical)
# Our implementation of logistic regression
log_reg = logreg.LogisticRegression(penalty=penalty,
solver=solver,
activation=activation,
tol=tolerance,
alpha=lmbda,
momentum=momentum,
mini_batch_size=mini_batch_size,
max_iter=max_iter)
# Fit training data
log_reg.fit(cp.deepcopy(X_train),
cp.deepcopy(y_train),
eta=learning_rate)
if verbose:
print("HomeMade method done")
# Accuracy score for our implementation
train_accuracy[i] = log_reg.score(X_train, y_train)
test_accuracy[i] = log_reg.score(X_test, y_test)
# critical_accuracy[i]=log_reg.score(X_critical,Y_critical)
# Prints result from single lambda run
if verbose:
print('Accuracy scores: train, test, critical')
print('HomeMade: {0:0.4f}, {1:0.4f}, {2:0.4f}'.format(
train_accuracy[i], test_accuracy[i], critical_accuracy[i]))
if use_sk_learn and verbose:
print('SK: {0:0.4f}, {1:0.4f}, {2:0.4f}'.format(
train_accuracy_SK[i], test_accuracy_SK[i],
critical_accuracy_SK[i]))
print('SGD: %0.4f, %0.4f, %0.4f' %
(train_accuracy_SGD[i], test_accuracy_SGD[i],
critical_accuracy_SGD[i]))
# Prints iteration values
if verbose:
print("Finished computing {}/11 iterations".format(i + 1))
if verbose:
print("")
results = [train_accuracy, test_accuracy, critical_accuracy]
if use_sk_learn:
results += [
train_accuracy_SK, test_accuracy_SK, critical_accuracy_SK,
train_accuracy_SGD, test_accuracy_SGD, critical_accuracy_SGD
]
if store_pickle:
if use_sk_learn:
sk_str = "sklearn_"
else:
sk_str = ""
if isinstance(pickle_fname, type(None)):
fname = ("accuracy_{}penalty{}_act{}_solver{}_lr{}_mom{}_tol{}."
"pkl".format(sk_str, penalty, activation, solver,
str(learning_rate), str(momentum),
str(tolerance)))
else:
fname = pickle_fname
save_pickle(fname, results)
return results