def default_model():
"""
Function containing the code definition for training and evaluating the default required model
"""
model = Sequential(Linear(2, 25), ReLU(), Linear(25, 25), ReLU(),
Linear(25, 25), ReLU(), Linear(25, 2))
train_input, train_target = generate_disc_set(1000)
test_input, test_target = generate_disc_set(1000)
values = {"lr": [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1]}
cross_validate = args.cross_val
best_lr = 1e-4
optimizer = SGDCV(model,
nb_epochs=50,
mini_batch_size=1,
lr=best_lr,
criterion=LossMSE())
if cross_validate:
optimizer.cross_validate(k=5, values=values, verbose=True)
optimizer.set_params()
optimizer.train(train_input, train_target, verbose=True)
evaluator = Evaluator(model)
print("Train accuracy: {:.1f}%".format(
(evaluator.compute_accuracy(train_input, train_target) * 100).item()))
print("Test accuracy: {:.1f}%".format(
(evaluator.compute_accuracy(test_input, test_target) * 100).item()))
def __init__(self, model, nb_epochs=50, mini_batch_size=1, lr=1e-2, criterion=LossMSE()):
"""
SGDCV constructor
:param model: the model whose training to cross-validate, models.Sequential object (only one currently possible)
:param nb_epochs: maximum number of training epochs, positive int, optional, default is 50
:param mini_batch_size: number of samples per mini-batch, int in [1, num_train_samples], optional, default is 1
:param lr: learning rate, positive float, optional, default is 1e-2
:param criterion: loss function to optimize, models.Module object, optional, default is criteria.LossMSE
"""
super().__init__(model)
self.model = model
self.optimizer = SGD(model=model, nb_epochs=nb_epochs, mini_batch_size=mini_batch_size,
lr=lr, criterion=criterion)
def __init__(self, model, nb_epochs=50, mini_batch_size=1, lr=1e-3, criterion=LossMSE(),
b1=0.9, b2=0.999, epsilon=1e-8):
"""
AdamCV constructor
:param model: the model whose training to cross-validate, models.Sequential object (only one currently possible)
:param nb_epochs: maximum number of training epochs, positive int, optional, default is 50
:param mini_batch_size: number of samples per mini-batch, int in [1, num_train_samples], optional, default is 1
:param lr: learning rate, positive float, optional, default is 1e-3
:param criterion: loss function to optimize, models.Module object, optional, default is criteria.LossMSE
:param b1: Adam beta_1 parameter, float in [0, 1], optional, default is 0.9
:param b2: Adam beta_2 parameter, float in [0, 1], optional, default is 0.999
:param epsilon: Adam epsilon parameter, non-negative float, optional, default is 1e-8
"""
super().__init__(model)
self.model = model
self.optimizer = Adam(model=model, nb_epochs=nb_epochs, mini_batch_size=mini_batch_size,
lr=lr, criterion=criterion, b1=b1, b2=b2, epsilon=epsilon)
def __init__(self,
model,
nb_epochs=50,
mini_batch_size=1,
lr=1e-3,
criterion=LossMSE(),
b1=0.9,
b2=0.999,
epsilon=1e-8):
"""
Adam constructor
:param model: the model to train, models.Sequential object (only one currently possible)
:param nb_epochs: maximum number of training epochs, positive int, optional, default is 50
:param mini_batch_size: number of samples per mini-batch, int in [1, num_train_samples], optional, default is 1
:param lr: learning rate, positive float, optional, default is 1e-3
:param criterion: loss function to optimize, models.Module object, optional, default is criteria.LossMSE
:param b1: Adam beta_1 parameter, float in [0, 1], optional, default is 0.9
:param b2: Adam beta_2 parameter, float in [0, 1], optional, default is 0.999
:param epsilon: Adam epsilon parameter, non-negative float, optional, default is 1e-8
:raises ValueError if:
- `nb_epochs` is not a positive int
- `mini_batch_size` is not a positive int
- `lr` is not a positive float
- `b1` is not a float in [0, 1]
- `b2` is not a float in [0, 1]
- `epsilon` is not a positive float
"""
super().__init__(model, nb_epochs, mini_batch_size, lr, criterion)
if not isinstance(b1, float) or not 0 <= b1 <= 1:
raise ValueError("Beta 1 must be a float in [0, 1]")
if not isinstance(b2, float) or not 0 <= b2 <= 1:
raise ValueError("Beta 2 must be a float in [0, 1]")
if not isinstance(epsilon, float) or epsilon < 0:
raise ValueError("Epsilon must be a non-negative float")
self.b1 = b1
self.b2 = b2
self.epsilon = epsilon
self.t = 0
def __init__(self,
model,
nb_epochs=50,
mini_batch_size=1,
lr=1e-2,
criterion=LossMSE()):
"""
SGD constructor
:param model: the model to train, models.Sequential object (only one currently possible)
:param nb_epochs: maximum number of training epochs, positive int, optional, default is 50
:param mini_batch_size: number of samples per mini-batch, int in [1, num_train_samples], optional, default is 1
:param lr: learning rate, positive float, optional, default is 1e-2
:param criterion: loss function to optimize, models.Module object, optional, default is criteria.LossMSE
:raises ValueError if:
- `nb_epochs` is not a positive int
- `mini_batch_size` is not a positive int
- `lr` is not a positive float
"""
super().__init__(model, nb_epochs, mini_batch_size, lr, criterion)
def main():
"""
Function containing the main code definition, display all functionalities provided by the framework
"""
# Different activation functions and setting of automatic Xavier parameter initialization
relu_model = Sequential(Linear(2, 25),
ReLU(),
Linear(25, 25),
ReLU(),
Linear(25, 25),
ReLU(),
Linear(25, 2),
xavier_init=True)
leaky_relu_model = Sequential(Linear(2, 25),
LeakyReLU(),
Linear(25, 25),
LeakyReLU(),
Linear(25, 25),
LeakyReLU(),
Linear(25, 2),
xavier_init=True)
tanh_model = Sequential(Linear(2, 25),
Tanh(),
Linear(25, 25),
Tanh(),
Linear(25, 25),
Tanh(),
Linear(25, 2),
xavier_init=True)
sigmoid_model = Sequential(Linear(2, 25),
Sigmoid(),
Linear(25, 25),
Sigmoid(),
Linear(25, 25),
Sigmoid(),
Linear(25, 2),
xavier_init=False)
model_names = ["ReLU", "Leaky", "Tanh", "Sigmoid"]
train_input, train_target = generate_disc_set(1000)
test_input, test_target = generate_disc_set(1000)
# Model training without cross-validation of the optimizer parameters
optimizer = SGDCV(leaky_relu_model, nb_epochs=25)
optimizer.train(train_input, train_target)
evaluator = Evaluator(leaky_relu_model)
print("Train accuracy using LeakyReLU: {:.1f}%".format(
(evaluator.compute_accuracy(train_input, train_target) * 100).item()))
print("Test accuracy using LeakyReLU: {:.1f}%".format(
(evaluator.compute_accuracy(test_input, test_target) * 100).item()))
models = (relu_model, leaky_relu_model, tanh_model, sigmoid_model)
sgd_cross_val_param_grid = {"lr": [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1]}
adam_cross_val_param_grid = {
"lr": [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1],
"b1": [0.9, 0.8],
"b2": [0.999, 0.888],
"epsilon": [1e-8, 1e-7, 1e-6]
}
adam_params = {
"ReLU": {
"lr": [0.001],
"b1": [0.9],
"b2": [0.999],
"epsilon": [1e-08]
},
"Leaky": {
"lr": [0.001],
"b1": [0.9],
"b2": [0.999],
"epsilon": [1e-08]
},
"Tanh": {
"lr": [0.001],
"b1": [0.9],
"b2": [0.999],
"epsilon": [1e-08]
},
"Sigmoid": {
"lr": [0.001],
"b1": [0.9],
"b2": [0.999],
"epsilon": [1e-08]
}
}
sgd_params = {
"ReLU": {
"lr": [0.001]
},
"Leaky": {
"lr": [0.001]
},
"Tanh": {
"lr": [0.001]
},
"Sigmoid": {
"lr": [0.01]
}
}
mse_loss = not args.CE
optimizer_sgd = not args.Adam
cross_validate = args.cross_val
# Different loss functions
if mse_loss:
criterion = LossMSE()
else:
criterion = LossCrossEntropy()
for name, model in zip(model_names, models):
if optimizer_sgd:
# SGD optimizer parameter cross-validation
optimizer = SGDCV(model, mini_batch_size=10, criterion=criterion)
if cross_validate:
params = sgd_cross_val_param_grid
else:
params = sgd_params[name]
cross_val_results, best_params_score = optimizer.cross_validate(
values=params)
print("Best params for model using {} : (lr={:.3f})".format(
name, best_params_score["lr"]))
else:
# Adam optimizer parameter cross-validation
optimizer = AdamCV(model, mini_batch_size=10, criterion=criterion)
if cross_validate:
params = adam_cross_val_param_grid
else:
params = adam_params[name]
cross_val_results, best_params_score = optimizer.cross_validate(
values=params)
print(
"Best params for model using {} : (lr={:.3f}, b1={:.3f}, b2={:.3f}, epsilon={:.1e})"
.format(name, best_params_score["lr"], best_params_score["b1"],
best_params_score["b2"], best_params_score["epsilon"]))
print("Best score for model using {} : {:.3f} (+/- {:.3f})".format(
name, best_params_score["mean"], best_params_score["std"]))
def cross_val_results(verbose=True):
"""
Function for generating the accuracy results of four models presented in the report with their best parameters,
averaged over 10 runs and using different combinations of the available optimizers and loss
:param verbose: whether to print average results for each (Model, Optimizer, Loss) combination,
boolean, optional, default is True
:returns: list of tuples containing (mean, std) of each (Model, Optimizer, Loss) combination, each tuple in [0, 1]^2
"""
datasets = []
for i in range(10):
datasets.append((generate_disc_set(1000), generate_disc_set(1000)))
relu_model = Sequential(Linear(2, 25), ReLU(),
Linear(25, 25), ReLU(),
Linear(25, 25), ReLU(),
Linear(25, 2), xavier_init=True)
leaky_relu_model = Sequential(Linear(2, 25), LeakyReLU(),
Linear(25, 25), LeakyReLU(),
Linear(25, 25), LeakyReLU(),
Linear(25, 2), xavier_init=True)
tanh_model = Sequential(Linear(2, 25), Tanh(),
Linear(25, 25), Tanh(),
Linear(25, 25), Tanh(),
Linear(25, 2), xavier_init=True)
sigmoid_model = Sequential(Linear(2, 25), Sigmoid(),
Linear(25, 25), Sigmoid(),
Linear(25, 25), Sigmoid(),
Linear(25, 2))
models = [relu_model, leaky_relu_model, tanh_model, sigmoid_model]
final_scores = []
optimizers_names = ["SGD", "Adam"]
models_names = ["ReLU", "Leaky", "Tanh", "Sigmoid"]
losses_names = ["MSE", "CrossEntropy"]
losses = [LossMSE(), LossCrossEntropy()]
adam_params = {"ReLU": {"lr": 0.001, "b1": 0.9, "b2": 0.999, "epsilon": 1e-08},
"Leaky": {"lr": 0.001, "b1": 0.9, "b2": 0.999, "epsilon": 1e-08},
"Tanh": {"lr": 0.001, "b1": 0.9, "b2": 0.999, "epsilon": 1e-08},
"Sigmoid": {"lr": 0.001, "b1": 0.9, "b2": 0.999, "epsilon": 1e-08}}
sgd_params = {"ReLU": {"lr": 0.001},
"Leaky": {"lr": 0.001},
"Tanh": {"lr": 0.001},
"Sigmoid": {"lr": 0.01}}
for optim_name in optimizers_names:
for loss_name, loss in zip(losses_names, losses):
for model_name, model in zip(models_names, models):
if verbose:
print("Validating model {} with {} and {} loss...".format(model_name, optim_name, loss_name),
end='')
scores = []
if optim_name == "Adam":
params = adam_params[model_name]
optim = Adam(model, criterion=loss, nb_epochs=50, mini_batch_size=10, lr=params["lr"],
b1=params["b1"], b2=params["b2"], epsilon=params["epsilon"])
else:
params = sgd_params[model_name]
optim = SGD(relu_model, criterion=loss, nb_epochs=50, mini_batch_size=10, lr=params["lr"])
for ((train_input, train_target), (test_input, test_target)) in datasets:
optim.model = copy.deepcopy(model)
optim.train(train_input, train_target, verbose=False)
evaluator = Evaluator(optim.model)
accuracy = evaluator.compute_accuracy(test_input, test_target)
scores.append(accuracy)
scores = torch.FloatTensor(scores)
scores_mean = torch.mean(scores).item()
scores_var = torch.std(scores).item()
if verbose:
print("Score : {0:.3f} (+/- {1:.3f}) ".format(scores_mean, scores_var))
final_scores.append((scores_mean, scores_var))
return final_scores