layer_size = 64
batch_size = 64
epochs = 60
def return_contexts():
contexts = {}
mse_model = MultiLayerPerceptron(input_dim=784,
output_dim=10,
hidden_sizes=[layer_size],
hidden_activation='sigmoid',
output_activation='softmax',
initializer='xavier')
mse_loss = MeanSquaredError()
mse_optim = Momentum(mse_model.parameters(), learning_rate=1e-3, momentum=0.9)
contexts['MSE'] = (mse_model, mse_loss, mse_optim)
ce_model = deepcopy(mse_model)
ce_model.change_output_activation('identity')
ce_loss = CrossEntropyWithLogitLoss()
ce_optim = Momentum(ce_model.parameters(), learning_rate=1e-3, momentum=0.9)
contexts['Cross Entropy'] = (ce_model, ce_loss, ce_optim)
return contexts, batch_size, epochs
results = mnist_experiment_on_models(return_contexts, N_REPEATS)
pickle_object(results, RESULTS_PATH, 'losses_result')
}),
'Adam': (AdamOptimizer, {
'learning_rate': 1e-3,
'beta_1': 0.9,
'beta_2': 0.999
})
}
def return_contexts():
contexts = {}
base_model = MultiLayerPerceptron(input_dim=784,
output_dim=10,
hidden_sizes=[layer_size],
hidden_activation='sigmoid',
output_activation='identity',
initializer='xavier')
for optim_name, optim in optimizers.items():
model = deepcopy(base_model)
loss = CrossEntropyWithLogitLoss()
optimizer = optim[0](model.parameters(), **optim[1])
contexts[optim_name] = (model, loss, optimizer)
return contexts, batch_size, epochs
results = mnist_experiment_on_models(return_contexts, N_REPEATS)
pickle_object(results, RESULTS_PATH, 'optimizers_result')
epochs = 100
activations = ['sigmoid', 'relu']
def return_contexts():
contexts = {}
base_activation = activations[0]
base_model = MultiLayerPerceptron(input_dim=784,
output_dim=10,
hidden_sizes=[layer_size],
hidden_activation=base_activation,
output_activation='identity',
initializer='plain')
base_loss = CrossEntropyWithLogitLoss()
base_optimizer = SGD(base_model.parameters(), learning_rate)
contexts[base_activation] = (base_model, base_loss, base_optimizer)
for act in activations[1:]:
model = deepcopy(base_model)
model.change_hidden_activation(act)
loss = CrossEntropyWithLogitLoss()
optimizer = SGD(model.parameters(), learning_rate)
contexts[act] = (model, loss, optimizer)
return contexts, batch_size, epochs
results = evaluate_mnist_experiment(return_contexts, N_REPEATS)
pickle_object(results, RESULTS_PATH, 'sigmoid_relu_experiment')
"""
Study on layer size influence
Each model is performed multiple times
"""
N_REPEATS = 3
layer_sizes = [10, 32, 128, 512, 1024]
learning_rate = 1e-2
batch_size = 64
epochs = 100
def return_contexts():
contexts = {}
for l_size in layer_sizes:
model = MultiLayerPerceptron(input_dim=784,
output_dim=10,
hidden_sizes=[l_size],
hidden_activation='sigmoid',
output_activation='identity',
initializer='plain')
loss = CrossEntropyWithLogitLoss()
optimizer = SGD(model.parameters(), learning_rate)
contexts[l_size] = (model, loss, optimizer)
return contexts, batch_size, epochs
results = evaluate_mnist_experiment(return_contexts, N_REPEATS)
pickle_object(results, RESULTS_PATH, 'layer_size_experiment')
def return_contexts():
multiple_contexts = []
base_batch_size = batch_sizes[0]
base_model = MultiLayerPerceptron(input_dim=784,
output_dim=10,
hidden_sizes=[layer_size],
hidden_activation='sigmoid',
output_activation='identity',
initializer='plain')
base_loss = CrossEntropyWithLogitLoss()
base_optimizer = SGD(base_model.parameters(), learning_rate)
contexts = {}
contexts[base_batch_size] = (base_model, base_loss, base_optimizer)
multiple_contexts.append((contexts, base_batch_size, epochs))
for b_size in batch_sizes[1:]:
contexts = {}
model = deepcopy(base_model)
loss = CrossEntropyWithLogitLoss()
optimizer = SGD(model.parameters(), learning_rate)
contexts[b_size] = (model, loss, optimizer)
multiple_contexts.append((contexts, b_size, epochs))
return multiple_contexts
results = evaluate_mnist_experiment(return_contexts, N_REPEATS)
pickle_object(results, RESULTS_PATH, 'batch_size_experiment')