def run_training():
x_train, y_train, x_val, y_val, x_test, y_test = load_data_wrapper()
learning_rate = 1e-1
batch_size = 50
max_epochs = 8
mlp_model = MLP(input_dim=784,
output_dim=10,
hidden_dims=[30],
activation_functions=[sigmoid],
init_parameters_sd=1,
optimizer=SGD(learning_rate=learning_rate))
print(mlp_model)
train_model(mlp_model,
x_train,
y_train,
lr=learning_rate,
batch_size=batch_size,
max_epochs=max_epochs,
x_val=x_val,
y_val=y_val,
plot=True)
def run_training_and_evaluation():
x_train, y_train, x_val, y_val, x_test, y_test = load_data_wrapper()
hidden_dims = [100]
activation_functions = [sigmoid, sigmoid]
init_parameters_sd = 1
learning_rate = 2e-1
batch_size = 50
max_epochs = 20
mlp_model = MLP(input_dim=784,
output_dim=10,
hidden_dims=hidden_dims,
activation_functions=activation_functions,
init_parameters_sd=init_parameters_sd,
optimizer=SGD(learning_rate=learning_rate))
print(mlp_model)
train_model(mlp_model,
x_train,
y_train,
batch_size=batch_size,
max_epochs=max_epochs,
x_val=x_val,
y_val=y_val,
plot=True,
early_stop=True,
patience=2)
file_name = f'mlp_model_{hidden_dims}_sd={init_parameters_sd}' + \
f'_lr={learning_rate}_b={batch_size}_{datetime.now().strftime("%m-%d-%Y_%H.%M")}.pkl'
mlp_model.save_model(file_name)
evaluate_model(mlp_model, x_test, y_test)
def analyze_activation_functions():
x_train, y_train, x_val, y_val, x_test, y_test = load_data_wrapper()
simulation_number = 5
max_epochs = 7
batch_size = 50
weight_sd = 1.0
learning_rate = 1e-1
act_functions = [sigmoid, relu]
act_functions_names = ['sigmoid', 'relu']
training_data_dictionary = {}
for act_fn, act_fn_name in zip(act_functions, act_functions_names):
epochs_num = []
training_losses = []
validation_losses = []
validation_accuracies = []
for i in range(simulation_number):
print(
f'\nActivation function : {act_fn_name}, simulation {i + 1}/{simulation_number}'
)
mlp_model = MLP(input_dim=784,
output_dim=10,
hidden_dims=[30],
activation_functions=[act_fn],
init_parameters_sd=weight_sd,
optimizer=SGD(learning_rate=learning_rate))
sim_overall_epoch_num, sim_training_losses, sim_validation_losses, sim_validation_accuracies = \
train_model(
mlp_model, x_train, y_train,
batch_size=batch_size, max_epochs=max_epochs,
x_val=x_val, y_val=y_val, plot=False
)
epochs_num.append(sim_overall_epoch_num)
training_losses.append(sim_training_losses)
validation_losses.append(sim_validation_losses)
validation_accuracies.append(sim_validation_accuracies)
training_data_dictionary[act_fn_name] = {
'epochs': epochs_num,
'train_losses': training_losses,
'val_losses': validation_losses,
'val_acc': validation_accuracies
}
file_name = f'act_functions_analysis_data_{act_functions_names}_{datetime.now().strftime("%m-%d-%Y_%H.%M")}.pkl'
with open(file_name, 'wb') as handle:
pkl.dump(training_data_dictionary,
handle,
protocol=pkl.HIGHEST_PROTOCOL)
plot_losses_results(training_data_dictionary)
plot_accuracies_results(training_data_dictionary)
def analyze_number_of_neurons():
x_train, y_train, x_val, y_val, x_test, y_test = load_data_wrapper()
simulation_number = 5
learning_rate = 1e-1
batch_size = 50
max_epochs = 7
hidden_neurons_numbers = [30, 100, 300, 500]
training_data_dictionary = {}
for neurons_number in hidden_neurons_numbers:
epochs_num = []
training_losses = []
validation_losses = []
validation_accuracies = []
for i in range(simulation_number):
print(f'\nHidden neurons: {neurons_number}, simulation {i + 1}/{simulation_number}')
mlp_model = MLP(
input_dim=784, output_dim=10, hidden_dims=[neurons_number],
activation_functions=[sigmoid],
init_parameters_sd=1,
optimizer=SGD(learning_rate=learning_rate)
)
sim_overall_epoch_num, sim_training_losses, sim_validation_losses, sim_validation_accuracies = \
train_model(
mlp_model, x_train, y_train,
batch_size=batch_size, max_epochs=max_epochs,
x_val=x_val, y_val=y_val, plot=False
)
epochs_num.append(sim_overall_epoch_num)
training_losses.append(sim_training_losses)
validation_losses.append(sim_validation_losses)
validation_accuracies.append(sim_validation_accuracies)
training_data_dictionary[
neurons_number] = {'epochs': epochs_num, 'train_losses': training_losses,
'val_losses': validation_losses, 'val_acc': validation_accuracies}
file_name = f'neuron_numbers_analysis_data_{hidden_neurons_numbers}_{datetime.now().strftime("%m-%d-%Y_%H.%M")}.pkl'
with open(file_name, 'wb') as handle:
pkl.dump(training_data_dictionary, handle, protocol=pkl.HIGHEST_PROTOCOL)
plot_losses_results(training_data_dictionary)
plot_accuracies_results(training_data_dictionary)
def evaluate_from_file():
file_name = 'mlp_model_[100]_sd=1_lr=0.2_b=50_11-01-2020_12.50.pkl'
x_train, y_train, x_val, y_val, x_test, y_test = load_data_wrapper()
hidden_dims = [100]
activation_functions = [sigmoid]
init_parameters_sd = 1
mlp_model = MLP(input_dim=784,
output_dim=10,
hidden_dims=hidden_dims,
activation_functions=activation_functions,
init_parameters_sd=init_parameters_sd)
print(mlp_model)
mlp_model.load_model(file_name)
evaluate_model(mlp_model, x_test, y_test)
def analyze_initializers():
x_train, y_train, x_val, y_val, x_test, y_test = load_data_wrapper()
initializers_names = ['Normal', 'Xavier', 'He']
data_dictionary = {}
processes = min(len(initializers_names), multiprocessing.cpu_count() - 1)
with multiprocessing.Pool(processes=processes) as pool:
results = pool.starmap(get_results_for_initializer,
[(initializer_name, x_train, x_val, y_train, y_val)
for initializer_name in initializers_names])
for name, res in zip(initializers_names, results):
data_dictionary[name] = res
file_name = f'initializers_analysis_data_{initializers_names}_sigmoid' \
f'_{datetime.now().strftime("%m-%d-%Y_%H.%M")}.pkl'
with open(file_name, 'wb') as handle:
pkl.dump(data_dictionary, handle, protocol=pkl.HIGHEST_PROTOCOL)
plot_losses_results(data_dictionary)
plot_accuracies_results(data_dictionary)
plot_accuracies_boxplot(data_dictionary)
def analyze_optimizers():
x_train, y_train, x_val, y_val, x_test, y_test = load_data_wrapper()
optimizers_names = ['SGD', 'Momentum', 'Nestorov', 'Adagrad', 'Adadelta', 'Adam']
data_dictionary = {}
processes = min(len(optimizers_names), multiprocessing.cpu_count() - 1)
with multiprocessing.Pool(processes=processes) as pool:
results = pool.starmap(get_results_for_optimizer,
[(optimizer_name, x_train, x_val, y_train, y_val)
for optimizer_name in optimizers_names])
for name, res in zip(optimizers_names, results):
data_dictionary[name] = res
file_name = f'optimizer_analysis_data_{optimizers_names}_relu' \
f'_{datetime.now().strftime("%m-%d-%Y_%H.%M")}.pkl'
with open(file_name, 'wb') as handle:
pkl.dump(data_dictionary, handle, protocol=pkl.HIGHEST_PROTOCOL)
plot_losses_results(data_dictionary)
plot_accuracies_results(data_dictionary)
plot_accuracies_boxplot(data_dictionary)
def analyze_models():
x_train, y_train, x_val, y_val, x_test, y_test = load_data_wrapper()
x_train = x_train[:train_data_num]
y_train = y_train[:train_data_num]
x_val = x_val[:val_data_num]
y_val = y_val[:val_data_num]
data_dictionary = {}
processes = min(simulation_number, multiprocessing.cpu_count() - 1)
with multiprocessing.Pool(processes=processes) as pool:
epochs_num = []
training_losses = []
validation_losses = []
validation_accuracies = []
results = pool.starmap(get_results_for_mlp,
[(x_train, x_val, y_train, y_val, i)
for i in range(simulation_number)])
for sim_overall_epoch_num, sim_training_losses, sim_validation_losses, sim_validation_accuracies in results:
epochs_num.append(sim_overall_epoch_num)
training_losses.append(sim_training_losses)
validation_losses.append(sim_validation_losses)
validation_accuracies.append(sim_validation_accuracies)
data_dictionary[f'MLP'] = \
{'epochs': epochs_num, 'train_losses': training_losses,
'val_losses': validation_losses, 'val_acc': validation_accuracies}
for k in kernel_sizes:
epochs_num = []
training_losses = []
validation_losses = []
validation_accuracies = []
results = pool.starmap(get_results_for_cnn,
[(x_train, x_val, y_train, y_val, k, i)
for i in range(simulation_number)])
for sim_overall_epoch_num, sim_training_losses, sim_validation_losses, sim_validation_accuracies in results:
epochs_num.append(sim_overall_epoch_num)
training_losses.append(sim_training_losses)
validation_losses.append(sim_validation_losses)
validation_accuracies.append(sim_validation_accuracies)
data_dictionary[f'CNN k={k}'] = \
{'epochs': epochs_num, 'train_losses': training_losses,
'val_losses': validation_losses, 'val_acc': validation_accuracies}
file_name = f'models_analysis_data' \
f'_{datetime.now().strftime("%m-%d-%Y_%H.%M")}.pkl'
with open(file_name, 'wb') as handle:
pkl.dump(data_dictionary, handle, protocol=pkl.HIGHEST_PROTOCOL)
plot_losses_results(data_dictionary)
plot_accuracies_results(data_dictionary)
plot_accuracies_boxplot(data_dictionary)
def run_training():
x_train, y_train, x_val, y_val, x_test, y_test = load_data_wrapper()
x_train = np.array([np.reshape(x, (28, 28)) for x in x_train])
x_val = np.array([np.reshape(x, (28, 28)) for x in x_val])
x_test = np.array([np.reshape(x, (28, 28)) for x in x_test])
# x_train = x_train[:5000]
# y_train = y_train[:5000]
#
x_val = x_val[:500]
y_val = y_val[:500]
learning_rate = 5e-3
batch_size = 50
max_epochs = 7
kernel_number = 4
kernel_size = 5
padding = 1
stride = 1
max_pooling = True
output_feature_map_dim = math.floor((28 - kernel_size + 2 * padding) /
stride + 1)
if max_pooling:
output_feature_map_dim = math.floor(output_feature_map_dim / 2)
conv_net = ConvolutionalNet(input_dim=(28, 28),
kernel_number=kernel_number,
kernel_size=kernel_size,
fc_input_dim=kernel_number *
output_feature_map_dim**2,
output_dim=10,
hidden_dims=[128],
activation_functions=[relu],
optimizer=Adam(learning_rate=learning_rate),
initializer=HeInitializer())
print(conv_net)
index = 1
x, y = x_test[index, :], y_test[index, :]
y_hat = conv_net(x)
print(f'y_real:\n{y}')
print('Before learning')
print(f'\ny_hat:\n{y_hat}')
train_model(conv_net,
x_train,
y_train,
batch_size=batch_size,
max_epochs=max_epochs,
x_val=x_val,
y_val=y_val,
plot=True)
y_hat = conv_net(x)
print(f'y_real:\n{y}')
print('After learning')
print(f'\ny_hat:\n{y_hat}')
evaluate_model(conv_net, x_test, y_test)
from dataset.mnist_dataset import load_data_wrapper
from functions.activation_functions import sigmoid
from models.neural_network_models.mlp import MLP
x_train, y_train, x_val, y_val, x_test, y_test = load_data_wrapper()
mlp_model = MLP(input_dim=784,
output_dim=10,
hidden_dims=[30],
activation_functions=[sigmoid],
init_parameters_sd=1)
print(mlp_model)
print()
limit = 3
i = 1
for x, y in zip(x_train, y_train):
y_hat = mlp_model(x)
print(f'y_real:\n{y}')
print(f'\ny_hat:\n{y_hat}')
i += 1
if i > limit:
break
