def main():
# load the data
train_input, train_target, train_classes, test_input, test_target, test_classes = \
prologue.generate_pair_sets(nb=1000)
# normalize it
mean, std = train_input.mean(), train_input.std()
train_input.sub_(mean).div_(std)
test_input.sub_(mean).div_(std)
train_input, train_target, train_class = Variable(train_input), Variable(
train_target), Variable(train_classes)
test_input, test_target = Variable(test_input), Variable(test_target)
nb_epochs = NB_EPOCH
mini_batch_size = BATCH
learning_rates = LEARNING_RATE
# all our models and optimizers we implemented
# models= [Net1, Net2, NetSharing1, NetSharing2, NetSharing3 ,NetAux1, NetAux2, NetAux3, Net3]
# optimizers = [optim.SGD, optim.Adam, optim.RMSprop]
# test the following configuration
models = [NetAux3]
optimizers = [optim.SGD]
grid_search(models, optimizers, learning_rates, train_input, train_target,
train_class, test_input, test_target, nb_epochs,
mini_batch_size)
def pickle_output(param):
"""
"""
train_input, train_target, train_class, test_input, test_target, test_class = \
prologue.generate_pair_sets(N_PAIRS)
# Normalization of the data
train_input /= train_input.max()
test_input /= test_input.max()
model = create_Net(param)
model.load_state_dict(torch.load("nets/{}.pkl".format(param['net'])))
x_out_tr, y_out_tr, _ = model.eval()(train_input)
x_out_te, y_out_te, _ = model.eval()(test_input)
class_proba_tr, class_proba_te = torch.cat(
[x_out_tr, y_out_tr], 1), torch.cat([x_out_te, y_out_te], 1)
pickle.dump(
{
'tr_pred_proba': class_proba_tr,
'tr_target': train_target,
'tr_class': train_class,
'te_pred_proba': class_proba_te,
'te_target': test_target,
'te_class': test_class
}, open("data/{}_pred.pkl".format(param['net']), 'wb'))
def __init__(
self,
param={
"nb": 1000,
"nb_classes": 10,
"normalized": True,
"one_hot_labels": False,
"conv3D": False
}):
self.nb = param["nb"]
self.nb_classes = param["nb_classes"]
self.train_input, self.train_target, self.train_class, self.test_input, self.test_target, self.test_class = \
prologue.generate_pair_sets(self.nb)
# Normalization
if param["normalized"]:
self._normalize_data()
self.isNormalized = True
else:
self.isNormalized = False
# One hot labels
if param["one_hot_labels"]:
self._get_one_hot_labels()
self.hasHot = True
else:
self.hasHot = False
# Conv3D reshape dataset
if param["conv3D"]:
self._get_conv3D_data()
self.isConv3D = True
else:
self.isConv3D = False
def load_dataset(
N: int = 1000, batch_size: int = 50, standardize: bool = True
) -> Tuple[DataLoader, DataLoader]:
"""
Load MNIST dataset in the following format:
* input of size [N, 2, 14, 14]: two 14x14 images (a, b)
* target of size [1]: 1 if digit in image a <= digit in image b
* class of size [2]: digits in images (a, b)
Args:
N (int, optional): Number of samples to fetch for each set.
Defaults to 1000.
batch_size (int, optional): Batch size for DataLoader. Defaults to 50.
standardize (bool, optional): Standardize train and test sets. Defaults to True.
Returns:
Tuple[DataLoader, DataLoader]: train and test DataLoaders
"""
train_input, train_target, train_classes, \
test_input, test_target, test_classes = generate_pair_sets(N)
if standardize:
mu, sigma = train_input.mean(), train_input.std()
# Standardize train and test with train statistics
train_input = standardized(train_input, mu, sigma)
test_input = standardized(test_input, mu, sigma)
train = TensorDataset(train_input, train_target, train_classes)
test = TensorDataset(test_input, test_target, test_classes)
return DataLoader(train, batch_size=batch_size, shuffle=True), \
DataLoader(test, batch_size=batch_size)
def main():
# loading data
train_input, train_target, train_class, test_input, test_target, test_class = prologue.generate_pair_sets(
1000)
# if you wish to train an optimized CNN model
# just uncomment this and comment the other model
'''
temp_train_input,temp_train_target,temp_train_class=init.data_init(train_input,train_target, train_class, num=1000)
model_cnn=model.ConvNet_2(bn=True)
model_cnn.apply(init.weights_init)
model_cnn,train_loss_cnn,train_accuracy_cnn, test_loss_cnn,test_accuracy_cnn=\
train.train_model(model_cnn,\
temp_train_input,temp_train_target,\
test_input, test_target,\
if_print=True, epochs=45,
optim='Adam', learning_rate=0.01)
evaluate.evaluate_result(train_loss_cnn,train_accuracy_cnn, test_loss_cnn,test_accuracy_cnn, \
'Learning Curve of Optimized CNN')
'''
# if you wish to train a model with weight sharing and auxiliary loss
# just uncomment this and comment the other model
'''
temp_train_input,temp_train_target,temp_train_class=init.data_init(train_input,train_target, train_class, num=1000)
model_ws_al=model.ConvNet_WS()
model_ws_al.apply(init.weights_init)
model_ws_al,train_loss_ws_al,train_accuracy_ws_al, test_loss_ws_al,test_accuracy_ws_al=\
train.train_model_WS(model_ws_al, \
temp_train_input,temp_train_target,temp_train_class,\
test_input, test_target, test_class,\
optim='Adam', decay=True,
if_auxiliary_loss=True,epochs=32,if_print=True,
auxiliary_loss_ratio=5)
evaluate.evaluate_result(train_loss_ws_al,train_accuracy_ws_al, test_loss_ws_al,test_accuracy_ws_al,\
'Learning Curve of Second ConvNet with Weight Sharing and Auxiliart Loss')
'''
# training data random initialization
temp_train_input, temp_train_target, temp_train_class = init.data_init(
train_input, train_target, train_class, num=1000)
# import the model
model_digit = model.CNN_digit()
# model weight initialization
model_digit.apply(init.weights_init)
# get the training history of the model with input hyperparameters
_,_,_,train_accuracy_from_digit,_,_,test_accuracy_from_digit=train.train_by_digit(model_digit,\
temp_train_input,temp_train_target,temp_train_class, \
test_input, test_target, test_class, \
if_print=True, \
epochs=25,optim='Adam',learning_rate=0.01)
# plot the learning curves and print the accuracy on testing set
evaluate.evaluate_accuracy(train_accuracy_from_digit,test_accuracy_from_digit,\
'Accuracy of Boolean Classification from CNN Trained Directly on Digit Class')
def load_random_datasets():
""" Function to generate new random training and test sets for various runs """
train_input, train_target, train_classes, test_input, test_target, test_classes = \
prologue.generate_pair_sets(nb=1000)
""" Feature Normalization: We normalize the train and test datasets with the mean and variance of the training set (we don't want to introduce information of the test set into the training set by normalizing over the whole dataset)"""
train_input, mean_input, std_input = standardize(train_input)
test_input = (test_input - mean_input) / std_input
return train_input, train_target, train_classes, test_input, test_target, test_classes
def boosted_Net(net_dicts):
"""
Trains multiple nets and boosts the results to have better perfs
"""
print("Creating boosted prediction with {} nets".format(len(net_dicts)))
train_input, train_target, train_class, test_input, test_target, test_class = \
prologue.generate_pair_sets(N_PAIRS)
# Normalization of the data
train_input /= train_input.max()
test_input /= test_input.max()
pred_tr = []; pred_te = [];
for i, dic in enumerate(net_dicts):
print("Net {}/{}".format(i+1, len(net_dicts)))
for k in dic.keys():
print(" {}: {} |".format(k, dic[k]), end='')
print('')
# setting a random seed
if dic['seed'] != None:
torch.manual_seed(dic['seed'])
model = create_Net(dic)
# Training each of the models
model.train(True)
for i in [0, 1]:
train_model(model, Variable(train_input[:, i, :, :].view(-1, 1, 14, 14)),
Variable(train_class[:, i].long()), dic['batch_size'], dic['epochs'])
model.train(False)
print(test_model(model, test_input, test_target, test_class))
output_tr = []; output_te= [];
for i in [0, 1]:
out = model(Variable(train_input[:, i, :, :].view(-1, 1, 14, 14)))
output_tr.append(out.data.max(1)[1])
out = model(Variable(test_input[:, i, :, :].view(-1, 1, 14, 14)))
output_te.append(out.data.max(1)[1])
pred_tr.append(torch.cat(output_tr))
pred_te.append(torch.cat(output_te))
del model
target = torch.cat([train_class[:,0], train_class[:,1]])
boost_model = xgb.XGBClassifier().fit(torch.stack(pred_tr, 1), target)
boost_pred = boost_model.predict(torch.stack(pred_te, 1))
pred_classes = torch.Tensor(np.stack([boost_pred[:1000], boost_pred[1000:]], axis=1))
print("Classification error w/ boosting: {:.4}%".format((boost_pred != torch.cat([test_class[:,0], test_class[:,1]]).numpy()).sum() /boost_pred.shape[0]*100))
print("Comparison error w/ boosting: {:.4}%\n".format((compare_digits(pred_classes) - test_target.int()).abs().sum().item() / test_target.size(0)*100))
def load_normal_data(n_samples):
'''
Load and normalize the dataset.
Params:
n_samples : number of samples to be loaded
Returns:
train_X, train_Y, train_Class, test_X, test_Y, test_Class : the data
'''
train_X, train_Y, train_Class, test_X, test_Y, test_Class = prologue.generate_pair_sets(
n_samples)
mu, std = train_X.mean(), train_X.std()
train_X.sub_(mu).div_(std)
test_X.sub_(mu).div_(std)
return train_X, train_Y, train_Class, test_X, test_Y, test_Class
def data_generation():
# this function is to generate train and test data
global data_train, data_test, data_train_loader, data_test_loader
# using helper to generate 1000 pairs of train data and test data
data = prologue.generate_pair_sets(1000)
# create torch style data set
data_train = PairDataset(data, train=True, aux_labels=True)
data_test = PairDataset(data, train=False, aux_labels=True)
# create data loader
data_train_loader = DataLoader(data_train,
batch_size=100,
shuffle=True,
num_workers=12)
data_test_loader = DataLoader(data_test, batch_size=100, num_workers=12)
def load_data(n_pairs: int, batch_size: int) -> ty.Tuple[data.DataLoader]:
"""
Load MNIT image pair data.
:param n_pairs: number of image pairs
:param batch_size: DataLoader batch size
"""
# pair_sets = (train_input, train_target, train_classes, test_input, test_target, test_classes)
pair_sets = prologue.generate_pair_sets(n_pairs)
train_data = ImagePairDataset(pair_sets[0], pair_sets[1], pair_sets[2])
test_data = ImagePairDataset(pair_sets[3], pair_sets[4], pair_sets[5])
train_loader = data.DataLoader(train_data, batch_size=batch_size)
test_loader = data.DataLoader(test_data, batch_size=batch_size)
return train_loader, test_loader
def get_results(n_pairs=1000, rounds=10):
results = []
model_params = [[False, False, True], [True, False, True],
[False, True, True], [True, True, True],
[False, False, False], [True, False, False],
[False, True, False], [True, True, False]]
for p in model_params:
model = Model(weight_sharing=p[0], aux=p[1], binary=p[2])
accuracies = []
model_id = model_params.index(p) + 1
print("Model {}:".format(model_id))
for i in range(rounds):
# generate train/test data
train_input, train_target, train_classes, test_input, test_target, test_classes = \
prologue.generate_pair_sets(n_pairs)
# normalize inputs
train_input = train_input.sub_(torch.mean(train_input)).div_(
torch.std(train_input))
test_input = test_input.sub_(torch.mean(test_input)).div_(
torch.std(test_input))
# train model
train_model(model, train_input, train_target, train_classes)
# compute test accuracy
accuracy = compute_accuracy(test_input, test_target, model)
accuracies.append(accuracy)
accuracy_avg = round(sum(accuracies) / rounds, 2)
accuracy_stdev = round(
(sum([(x - accuracy_avg)**2
for x in accuracies]) / (rounds - 1))**0.5, 2)
results.append([model_id, accuracy_avg, accuracy_stdev])
print("Avg test acc %: {}".format(accuracy_avg))
print("Stdev test acc %: {}".format(accuracy_stdev))
print('-' * 30)
return results
def test_param(param):
print("Testing...", end='')
for k in param.keys():
print(" {}: {} |".format(k, param[k]), end='')
print('')
train_input, train_target, train_class, test_input, test_target, test_class = \
prologue.generate_pair_sets(N_PAIRS)
# Normalization of the data
train_input /= train_input.max()
test_input /= test_input.max()
# setting a random seed
if param['seed'] != None:
torch.manual_seed(param['seed'])
model = create_Net(param)
model.train(True)
for i in [0, 1]:
train_model(model, Variable(train_input[:, i, :, :].view(-1, 1, 14, 14)),
Variable(train_class[:, i].long()), param['batch_size'], param['epochs'])
model.train(False)
#TODO this is temp otherwise 2x computation
# nb_test_errors, _, _, _ = test_model(model, test_input, test_target, test_class)
nb_test_errors = 0
for i in [0, 1]:
nb_test_errors += compute_nb_errors(model, Variable(test_input[:, i, :, :].view(-1, 1, 14, 14)),
Variable(test_class[:, i].long()), 20)
print('Classification test error: {:0.2f}% {:d}/{:d}'.format( nb_test_errors / test_input.size(0) /2*100,
nb_test_errors, 2*test_input.size(0)))
pred_class = torch.stack([model(Variable(test_input[:, 0, :, :].view(-1, 1, 14, 14))).data.max(1)[1],
model(Variable(test_input[:, 1, :, :].view(-1, 1, 14, 14))).data.max(1)[1]], dim=1)
print("Comparison error: {:0.2f}% {:d}/{:d}".format( (compare_digits(pred_class) - test_target.int()).abs().sum().item() /test_target.size(0)*100,
(compare_digits(pred_class) - test_target.int()).abs().sum().item(),
test_input.size(0)))
del model
def test_param(param, save=False):
print("Testing...", end='')
for k in param.keys():
print(" {}: {} |".format(k, param[k]), end='')
print('')
train_input, train_target, train_class, test_input, test_target, test_class = \
prologue.generate_pair_sets(N_PAIRS)
# Normalization of the data
train_input /= train_input.max()
test_input /= test_input.max()
# setting a random seed
if param['seed'] != None:
torch.manual_seed(param['seed'])
model = create_Net(param)
model.train(True)
train_model(model, train_input, train_target, train_class, param['batch_size'], param['epochs'])
model.train(False)
class_err, class_per, comp_err, comp_per = test_model(model, test_input, test_target, test_class)
print('Classification test error: {:0.2f}% {:d}/{:d}'.format(class_per, class_err, 2*N_PAIRS))
print("Net comparison error: {:0.2f}% {:d}/{:d}".format(comp_per, comp_err, N_PAIRS))
with open("{}results.log".format(param['net']), mode='at') as f:
f.write("Class error: {:.4}%, Comp error: {:.4}%\n".format(class_per, comp_per))
if save:
if 'nets' not in os.listdir():
dir_ = 'nets'
try:
os.mkdir(dir_)
except OSError:
print ("Creation of the directory %s failed" % dir_)
else:
print ("Successfully created the directory %s " % dir_)
torch.save(model.state_dict(), "nets/{}.pkl".format(param['net']))
def load_data(N=1000, batch_size=50, seed=42):
'''
Load training and test data from MNIST
Data: pairs of MNIST images
Label: 1 if first digit is lesser or equal than the second, 0 otherwise
Parameters
-------
N
Number of examples to generate for each set
batch_size
Batch size (for loading datasets into DataLoader type)
seed
Random seed (for reproducibility)
Returns
-------
train_loader
DataLoader containing training examples, binary labels and true image classes
test_loader
DataLoader containing test examples, binary labels and true image classes
'''
# Generate pairs
trainX, trainY, trainC, testX, testY, testC = prologue.generate_pair_sets(N, seed)
# Retrieve mean and standard deviation of training set
mu, std = trainX.mean(), trainX.std()
# Standardize data
trainX, testX = [standardize(x, mu, std) for x in [trainX, testX]]
# Assemble all data
train_data = TensorDataset(trainX, trainY, trainC)
test_data = TensorDataset(testX, testY, testC)
# Load data in DataLoader and shuffle training set
torch.manual_seed(seed) # For reproducibility
train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
test_loader = DataLoader(test_data, batch_size=batch_size)
return train_loader, test_loader
def load_normal_data(n_samples):
'''
Load, normalize and flatten the dataset.
Params:
n_samples : number of samples to be loaded
Returns:
train_X, train_Y, test_X, test_Y : the data required for train and test
'''
train_X, train_Y, train_Class, test_X, test_Y, test_Class = prologue.generate_pair_sets(
n_samples)
# normalize the data
mu, std = train_X.mean(), train_X.std()
train_X.sub_(mu).div_(std)
test_X.sub_(mu).div_(std)
# flatten the data
train_X = train_X.view(train_X.size(0), -1)
test_X = test_X.view(test_X.size(0), -1)
return train_X, train_Y, test_X, test_Y
def generate_data(N=N, mode=mode):
"""
Generates pair datasets and returns them as DataLoader classes
:param N: number of samples in data
:param mode: mode for data preprocessing (none, oversampling, undersampling)
:return:
"""
train_input, train_class, train_digit, test_input, test_class, test_digit = generate_pair_sets(N)
# Normalising data
mean = torch.mean(train_input)
std = torch.std(train_input)
train_input = (train_input - mean) / std
test_input = (test_input - mean) / std
# Balancing data
if mode:
train_input, train_class, train_digit = balance_data_classes(train_input, train_class, train_digit, mode)
train_loader = DataLoader(list(zip(train_input, train_class, train_digit)), batch_size=64)
test_loader = DataLoader(list(zip(test_input, test_class, test_digit)), batch_size=64)
return train_loader, test_loader
def test():
N_PAIRS = 1000
train_input, train_target, train_classes, test_input, test_target, test_classes = \
prologue.generate_pair_sets(N_PAIRS)
train_input, train_target = Variable(train_input), Variable(train_target)
test_input, test_target = Variable(test_input), Variable(test_target)
""" Create and train model """
my_model = NetWithWeightSharing()
""" Create and train model which identifies each number and then compares them """
my_model.trainer(train_input, train_target)
print("Train error : %.1f%% \nTest error : %.1f%%" %
(my_model.nb_errors(train_input, train_target),
my_model.nb_errors(test_input, test_target)))
print("Train error : %.1f%% \nTest error : %.1f%%" %
(my_model.nb_errors(train_input, train_target),
my_model.nb_errors(test_input, test_target)))
return my_model.nb_errors(test_input, test_target)
def pipeline(net_class, name, rounds=2, epochs=25, mini_batch_size=50,
lr=1e-3*0.5, auxiliary=False, **model_kwargs):
"""
Full data generations, training, testing multiple times.
For a given situation (mini-batch size, network, number of samples) does:
* Data Generation
* Model training (logging of losses)
* Testing on training data (logging accuracy)
* Testing on testing data (logging accuracy)
* Printing results
Parameters:
net_class {nn.Module}: The class to train.
rounds {int; default=10}: how many time the pipeline must be ran.
epochs {int; default=25}: Number of epochs for the training.
mini_batch_size {int; default=50}: size of the mini-batch.
Must be a divisor of N.
Returns:
model: the model trained
train_losses: List of training losses for each epoch.
Shape: (rounds, num_epochs)
train_errors: Mean errors per round (on a scale [0-1]) on train data.
Shape: (rounds)
test_errors: Mean errors per round (on a scale [0-1]) on test data.
Shape: (rounds)
rounds_times: Total time (data gen, training, testint) for each round.
Shape: (rounds)
"""
N = 1000
train_losses = []
train_errors, test_errors = [], []
rounds_times = []
print("********************* Training model '{}'*********************"
.format(name))
for i in range(rounds):
print('Starting iteration {}'.format(i+1))
time_start = time()
# Generate Data
(train_data, train_target, train_classes,
test_data, test_target, test_classes) = prologue.generate_pair_sets(N)
if auxiliary:
# unpack the images to [2N, 1, 14, 14]
train_data = train_data.view(-1, 1, 14, 14)
train_classes = train_classes.view(-1)
test_data = test_data.view(-1, 1, 14, 14)
test_classes = test_classes.view(-1)
# Model training
time_train = time()
model = net_class(auxiliary=auxiliary, **model_kwargs)
train_epoch_losses = model.train_model(train_data,
train_target,
train_classes,
nb_epochs=epochs,
batch_size=mini_batch_size,
lr=lr)
train_losses.append(train_epoch_losses)
# Compute train error
time_train_errors = time()
nb_train_errors = compute_nb_errors(model, train_data,
train_target, mini_batch_size)
train_errors.append(nb_train_errors/train_data.shape[0])
# Compute train error
time_test_errors = time()
nb_test_errors = compute_nb_errors(model, test_data,
test_target, mini_batch_size)
test_errors.append(nb_test_errors/test_data.shape[0])
time_end = time()
rounds_times.append(time_end-time_start)
# Logging
print('Train error: {:0.2f}%'.format(
100 * (nb_train_errors / train_data.size(0))))
print('Test error: {:0.2f}%'
.format(100 * (nb_test_errors / test_data.size(0))))
print("Times:\n\
Data generation: {:.2f}s\tTraining: {:.2f}\n\
Errors compute train: {:.2f}s\tErrors compute test: {:.2f}s\n\
Full round: {:.2f}s"
.format(time_train-time_start,
time_train_errors-time_train,
time_test_errors-time_train_errors,
time_end-time_test_errors,
time_end-time_start))
print("\n############\n")
digest(train_errors, test_errors, rounds_times)
print("******************* Ending training of '{}'*******************\n\n"
.format(name))
return model, train_losses, train_errors, test_errors, rounds_times
############################################################################### # - CONSTANTS - # ############################################################################### #seed to retrieve the same results t.manual_seed(0) NB_PAIRS = 1000 NB_EPOCHS = args.epochs MINI_BATCH_SIZE = 100 NB_RUN = args.runs #fetch data train_input,train_target,train_classes,test_input,test_target,test_classes \ = generate_pair_sets(NB_PAIRS) #wrap both inputs so that the gradients can be computed train_input = Variable(train_input.float(), requires_grad=True) test_input = Variable(test_input.float(), requires_grad=True) #convert string learning rate to float to be used by optimizer lr_ = args.learning_rate ############################################################################### # - NETWORKS - # ############################################################################### # Baseline model that perform classification of a 14x14 MNIST image # Output is of size 10x1, containing the 10 classes power
def load_data(N=1000, batch_size=50, seed=42):
# Load data
train_input, train_target, train_classes, test_input, test_target, test_classes = prologue.generate_pair_sets(
N)
# Convert target to one-hot label
train_target = torch.nn.functional.one_hot(train_target)
test_target = torch.nn.functional.one_hot(test_target)
# Move data to the device
train_input = train_input.to(device)
train_target = train_target.to(device)
train_classes = train_classes.to(device)
test_input = test_input.to(device)
test_target = test_target.to(device)
test_classes = test_classes.to(device)
# Normalize data
mean, std = train_input.mean(), train_input.std()
train_input.sub_(mean).div_(std)
test_input.sub_(mean).div_(std)
# Generate dataset
train_data = TensorDataset(train_input, train_target, train_classes)
test_data = TensorDataset(test_input, test_target, test_classes)
# For reproducibility
torch.manual_seed(seed)
# Generate data loader
train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
test_loader = DataLoader(test_data, batch_size=batch_size)
return train_loader, test_loader
def get_data(N: int = 1000,
batch_size: int = 100,
shuffle: bool = True) -> tuple:
"""
Get train and test DataLoaders of size N
:param N: number of pairs to return for each data loader
:param batch_size: batch size
:param shuffle: activate random shuffling
:return: train and test loader
"""
# Get input tensors from provided prologue
train_input, train_target, train_classes, test_input, test_target, test_classes = generate_pair_sets(
N)
device = get_device()
# Normalization
mu, std = train_input.mean(), train_input.std()
train_input = train_input.sub(mu).div(std)
test_input = test_input.sub(mu).div(std)
# Move data to GPU if available
train_input = train_input.to(device)
train_target = train_target.to(device)
train_classes = train_classes.to(device)
test_input = test_input.to(device)
test_target = test_target.to(device)
test_classes = test_classes.to(device)
# Create train and test loader
train_loader = DataLoader(MNISTCompareDataset(train_input, train_target,
train_classes),
batch_size=batch_size,
shuffle=shuffle)
test_loader = DataLoader(MNISTCompareDataset(test_input, test_target,
test_classes),
batch_size=batch_size,
shuffle=shuffle)
return train_loader, test_loader
def perform_experiments(n_runs=10,
n_points=1000,
n_epochs=200,
run_best=False,
verbose=False):
"""
Perform experiments for 5 different neural network architectures and losses.
To run all experiments call this function with default params
:param n_runs: number of runs for which experiment should be repeated
:param n_points: number of training and testing data points used in the experiments
:param n_epochs: number of epochs every architecture should be trained on
:param run_best: If True only the best architecture (Siamese Network with auxiliary loss) is trained
:param verbose: If True, print training and validation loss every epoch
:returns: dictionary containing history of training (training, validation loss and accuracy)
"""
history_mlp_net = []
history_conv_net = []
history_conv_net_aux = []
history_siamese = []
history_siamese_aux = []
for n_run in range(n_runs):
data_set = generate_pair_sets(n_points)
MAX_VAL = 255.0
TRAIN_INPUT = Variable(data_set[0]) / MAX_VAL
TRAIN_TARGET = Variable(data_set[1])
TRAIN_CLASSES = Variable(data_set[2])
TEST_INPUT = Variable(data_set[3]) / MAX_VAL
TEST_TARGET = Variable(data_set[4])
TEST_CLASSES = Variable(data_set[5])
if not run_best:
##############################################################################
# Creates Multilayer Perceptron Network with ReLU activationss
mlp_net = MLPNet(in_features=392,
out_features=2,
n_layers=3,
n_hidden=16)
# Set train flag on (for dropouts)
mlp_net.train()
# Train the model and append the history
history_mlp_net.append(
train_model(mlp_net,
train_input=TRAIN_INPUT.view((n_points, -1)),
train_target=TRAIN_TARGET,
val_input=TEST_INPUT.view((n_points, -1)),
val_target=TEST_TARGET,
n_epochs=n_epochs,
verbose=verbose))
# Set train flag to False for getting accuracies on validation data
mlp_net.eval()
acc = get_accuracy(mlp_net, TEST_INPUT.view(
(n_points, -1)), TEST_TARGET) * 100.0
print("Run: {}, Mlp_net Test Accuracy: {:.3f} %".format(
n_run, acc))
##############################################################################
# Create ConvNet without auxiliary outputs
conv_net = ConvNet(n_classes=2, n_layers=3, n_features=16)
# Set train flag on (for dropouts)
conv_net.train()
# Train the model and append the history
history_conv_net.append(
train_model(conv_net,
train_input=TRAIN_INPUT,
train_target=TRAIN_TARGET,
val_input=TEST_INPUT,
val_target=TEST_TARGET,
n_epochs=n_epochs,
verbose=verbose))
# Set train flag to False for getting accuracies on validation data
conv_net.eval()
acc = get_accuracy(conv_net, TEST_INPUT, TEST_TARGET) * 100.0
print("Run: {}, ConvNet Test Accuracy: {:.3f} %".format(
n_run, acc))
##############################################################################
# Create ConvNet with auxiliary outputs
conv_net_aux = ConvNet(n_classes=22, n_layers=3, n_features=16)
# Set train flag on (for dropouts)
conv_net_aux.train()
# Train the model and append the history
history_conv_net_aux.append(
train_model(conv_net_aux,
train_input=TRAIN_INPUT,
train_target=TRAIN_TARGET,
aux_param=1.0,
train_classes=TRAIN_CLASSES,
val_input=TEST_INPUT,
val_target=TEST_TARGET,
val_classes=TEST_CLASSES,
n_epochs=n_epochs,
verbose=verbose))
# Set train flag to False for getting accuracies on validation data
conv_net_aux.eval()
acc = get_accuracy(conv_net_aux, TEST_INPUT, TEST_TARGET) * 100.0
print("Run: {}, ConvNet Auxilary Test Accuracy: {:.3f} %".format(
n_run, acc))
##############################################################################
# Create Siamese Network without auxiliary outputs
conv_net = BlockConvNet()
conv_net_siamese = DeepSiameseNet(conv_net)
# Set train flag on (for dropouts)
conv_net.train()
conv_net_siamese.train()
# Train the model and append the history
history_siamese.append(
train_model(conv_net_siamese,
train_input=TRAIN_INPUT,
train_target=TRAIN_TARGET,
val_input=TEST_INPUT,
val_target=TEST_TARGET,
n_epochs=n_epochs,
verbose=verbose))
# Set train flag to False for getting accuracies on validation data
conv_net.eval()
conv_net_siamese.eval()
acc = get_accuracy(conv_net_siamese, TEST_INPUT,
TEST_TARGET) * 100.0
print("Run: {}, Siamese Test Accuracy: {:.3f} %".format(
n_run, acc))
##############################################################################
# Create Siamese Network with auxiliary outputs
conv_net = BlockConvNet()
conv_net_siamese_aux = DeepSiameseNet(conv_net)
# Set train flag on (for dropouts)
conv_net.train()
conv_net_siamese_aux.train()
# Train the model and append the history
history_siamese_aux.append(
train_model(conv_net_siamese_aux,
train_input=TRAIN_INPUT,
train_target=TRAIN_TARGET,
train_classes=TRAIN_CLASSES,
val_input=TEST_INPUT,
val_target=TEST_TARGET,
val_classes=TEST_CLASSES,
aux_param=3.0,
n_epochs=n_epochs,
verbose=verbose))
# Set train flag to False for getting accuracies on validation data
conv_net.eval()
conv_net_siamese_aux.eval()
acc = get_accuracy(conv_net_siamese_aux, TEST_INPUT,
TEST_TARGET) * 100.0
print("Run: {}, Siamese Auxilary Test Accuracy: {:.3f} %".format(
n_run, acc))
##############################################################################
return {
'history_mlp_net': history_mlp_net,
'history_conv_net': history_conv_net,
'history_conv_net_aux': history_conv_net_aux,
'history_siamese': history_siamese,
'history_siamese_aux': history_siamese_aux
}
def benchmark_model(model,
train_function,
evaluate_function,
nb_trials=20,
N=1000,
mini_batch_size=250,
nb_epochs=25,
model_requires_target_and_classes=False,
_print=False):
# Benchmark of the basic network with Adam optimizer
performances = []
for trial in range(nb_trials):
# Generate Data
train_input, train_target, train_classes, test_input, test_target, test_classes = prologue.generate_pair_sets(
N)
test_target_one_hot = prologue.convert_to_one_hot_labels(
test_input, test_target)
# Define the model
model_total = model()
# Train the model
if model_requires_target_and_classes:
train_function(model_total,
train_input,
train_target,
train_classes,
mini_batch_size=mini_batch_size,
nb_epochs=nb_epochs,
use_optimizer="adam",
_print=_print)
else:
train_function(model_total,
train_input,
train_target,
mini_batch_size=mini_batch_size,
nb_epochs=nb_epochs,
use_optimizer="adam",
_print=_print)
# Evaluate performances
nb_test_errors = evaluate_function(model_total,
test_input,
test_target_one_hot,
mini_batch_size=mini_batch_size)
print('test error Net trial {:d} {:0.2f}% {:d}/{:d}'.format(
trial, (100 * nb_test_errors) / test_input.size(0), nb_test_errors,
test_input.size(0)))
performances.append(nb_test_errors)
mean_perf = 100 * sum(performances) / (N * nb_trials)
print(f"Average precision of this architecture {mean_perf}%")
std_dev = math.sqrt(sum(list(map(lambda x: x - mean_perf,
performances)))) / nb_trials
print(f"With standard deviation of {std_dev}")
return performances
def test(model_maker,
activation_fc=relu,
mean=True,
n_trials=5,
device=None,
output_file=None,
lr=10e-3,
nb_epochs=50,
batch_size=250,
infos='',
auxiliary=False):
"""Function that test multiple times a given network on MNIST and save the results.
Parameters
----------
model_maker : function
A function that returns a torch.nn.Module. This function should be able to accept arguments, even if not used.
activation_fc : torch.nn.functional
Activation function to be passed in model_maker (the default is relu).
mean : bool
If to compute the mean over multiple trials (the default is True).
n_trials : int
Number of times the model is to be tested, ignored if mean is False (the default is 5).
device : torch.device
The device to be used, by default is chosen according to computer resources (the default is None).
output_file : str
Name of the file where to save the results. If empty it prints on screen (the default is None).
lr : double
Learning rate (the default is 10e-3).
nb_epochs : int
Number of epochs (the default is 50).
batch_size : int
Batch size (the default is 100).
infos : str
Additional model infromations to be printed (the default is '').
auxiliary : bool
Wether to use auxiliary loss, it works only for siames net (the default is False).
"""
if device is None:
device = 'cuda' if torch.cuda.is_available() else 'cpu'
if not mean:
n_trials = 1
CELoss_tr = []
CELoss_te = []
Accuracy_tr = []
Accuracy_te = []
time_tr = []
model = model_maker(activation_fc)
model_name = type(model).__name__ + infos
if type(model).__name__ == 'SiameseNet' and auxiliary:
model_name += 'Auxiliary'
print('Training {}:'.format(model_name))
################################################################################
# The model testing follows
for trial in range(n_trials):
input_train, target_train, classes_train, input_test, target_test, classes_test = generate_pair_sets(
1000)
criterion = torch.nn.CrossEntropyLoss()
model = model_maker().to(device)
criterion = criterion.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr)
input_train, target_train = input_train.to(device), target_train.to(
device)
input_test, target_test = input_test.to(device), target_test.to(device)
if auxiliary:
classes_train, classes_test = classes_train.to(
device), classes_test.to(device)
########################################################################
#A model is trained for each trial
start_time = time.time()
for e in range(nb_epochs):
for input, targets in zip(input_train.split(batch_size),
target_train.split(batch_size)):
if type(model).__name__ != 'SiameseNet' or not auxiliary:
#the standart training
optimizer.zero_grad()
output = model(input)
loss = criterion(output, targets)
loss.backward()
optimizer.step()
else:
#first pass
#separating the data so that we train on the two images separatly, and learn to classify them properly
input_train1, classes_train1, input_train2, classes_train2 = split_channels(
input_train, classes_train)
#use the branch to perform handwritten digits classification
out1 = model.branch(input_train1)
out2 = model.branch(input_train2)
#auxiliary loss: learn to detect the handwritten digits directly
loss_aux = criterion(out1, classes_train1) + criterion(
out2, classes_train2)
#optimize based on this
model.zero_grad()
loss_aux.backward(retain_graph=True)
optimizer.step()
#second pass
#loss and optimization of the whole model
response = model.forward(input_train)
loss = criterion(response, target_train)
model.zero_grad()
loss.backward()
optimizer.step()
with torch.no_grad():
print('{:3}%| Trial {:>2}/{} - Iteration #{:>3}/{}:\t'.format(
int((trial * nb_epochs + e + 1) / n_trials / nb_epochs *
100), trial + 1, n_trials, e + 1, nb_epochs),
'Cross Entropy Loss on TRAIN :\t{:11.5}'.format(
criterion(model(input_train), target_train).item()),
end='\r')
########################################################################
# model evaluation
with torch.no_grad():
elapsed_time = time.time() - start_time
model.train(False)
this_CELoss_tr = criterion(model(input_train), target_train).item()
this_CELoss_te = criterion(model(input_test), target_test).item()
this_Accuracy_tr = nn_accuracy_score(model, input_train,
target_train)
this_Accuracy_te = nn_accuracy_score(model, input_test,
target_test)
if not mean:
print('Cross Entropy Loss on TRAIN :\t', this_CELoss_tr, '\n\
Cross Entropy Loss on TEST :\t', this_CELoss_te, '\n\
Accuracy score on TRAIN :\t', this_Accuracy_tr, '\n\
Accuracy score on TEST :\t', this_Accuracy_te)
CELoss_tr.append(this_CELoss_tr)
CELoss_te.append(this_CELoss_te)
Accuracy_tr.append(this_Accuracy_tr)
Accuracy_te.append(this_Accuracy_te)
time_tr.append(elapsed_time)
with torch.no_grad():
score_printing(CELoss_tr,
CELoss_te,
Accuracy_tr,
Accuracy_te,
time_tr,
model_name=model_name,
output=output_file)
return
# Import Pytorch Package
import torch
from torch import nn
from torch import optim
from torch.nn import functional as F
# Import Tool package
import dlc_practical_prologue as prologue
# Import the models and functions
from func import *
from models import *
# Load Data
N = 1000
train_input, train_target, train_classes, test_input, test_target, test_classes = prologue.generate_pair_sets(
N)
# Set training parameters
inpDim = train_input.view(N, -1).size()[1] # inpDim: Input dimension
nb_hidden1, nb_hidden2 = 160, 128 # nb_hidden1, nb_hidden2: Hidden dimensions
rounds = 15
mini_batch_size = 50
# Baseline Model
print("Train Baseline Model:")
print("--------------------------------------------------")
model1, loss_round_list1, train_errors_list1, test_errors_list1 = PipelineBase(
BaselineNet, rounds, mini_batch_size, N)
estiPerform(train_errors_list1, test_errors_list1)
print("The parameters number of Baseline Model:",
sum(p.numel() for p in model1.parameters()), '\n')
import torch
import torch.optim as optim
from torch.autograd import Variable
from torch import nn
from torch.nn import functional as F
from torchvision import datasets, transforms
import matplotlib.pyplot as plt
import dlc_practical_prologue as dlc
train_input, train_target, train_classes, \
test_input, test_target, test_classes = dlc.generate_pair_sets(10)
for imageind in range(0, train_input.size(0)):
plt.imshow(train_input[imageind][0].view(14, 14).numpy(), cmap="gray")
plt.show()
for targetee in range(0, train_classes.size(0)):
print(train_classes[targetee])
transform = transforms.Compose([
transforms.ToTensor(), # Convert each image into a torch.FloatTensor
transforms.Normalize(
(0.1307, ),
(0.3081, )) # Normalize the data to have zero mean and 1 stdv
])
train_set = datasets.MNIST('data',
train=True,
download=True,
transform=transform)
plt.imshow(train_set[2][0].view(28, 28).numpy(), cmap="gray")
def main(isplot=False):
"""
Main functoin
- Load data
- Create modeal
- train and evaluate
:return:
"""
# ----- PARAMETER --------------------
nb_pair = 1000
batch_size = [100]
nb_epochs = [100]
learning_rate = [5e-3]
nb_iteration = 10
for ep in nb_epochs:
for bs in batch_size:
for lr in learning_rate:
saved_train_accuracy = []
saved_test_accuracy = []
for i in range(nb_iteration):
print('\n------- ITERATION - %d -------' % (i + 1))
# ----- DATASET --------------------
train_input, train_target, _, test_input, test_target, _ = prologue.generate_pair_sets(
nb_pair)
# Normalize by dividing it to the max RGB value
train_input /= 255
test_input /= 255
# Split between training (80%) and validation (20%)
train_dataset = TensorDataset(train_input, train_target)
train_len = int(0.8 * train_dataset.__len__())
validation_len = train_dataset.__len__() - train_len
train_data, validation_data = random_split(
train_dataset, lengths=[train_len, validation_len])
train_loader = DataLoader(train_data,
batch_size=bs,
shuffle=False,
num_workers=2)
validation_loader = DataLoader(validation_data,
batch_size=bs,
shuffle=False,
num_workers=2)
# Test
test_dataset = TensorDataset(test_input, test_target)
test_loader = DataLoader(test_dataset,
batch_size=bs,
shuffle=False,
num_workers=2)
# ----- MODEL --------------------
# - Fully Connected model
model = FCModel()
# - Small cnn
# model = ConvModel()
# - Deeper cnn
# model = DeepConvModel()
# Optimizer
optimizer = optim.Adam(model.parameters(), lr=lr)
# Loss function
criterion = nn.CrossEntropyLoss()
# ----- TRAINING + VALIDATION --------------------
nb_batch_train = train_len // bs
nb_batch_validation = validation_len // bs
train_losses = []
train_accuracies = []
validation_losses = []
validation_accuracies = []
for epoch in range(ep):
# TRAIN
train_loss, train_accuracy = train(
train_loader, model, criterion, optimizer,
nb_batch_train)
train_losses.append(train_loss)
train_accuracies.append(train_accuracy)
# VALIDATION
validation_loss, validation_accuracy = validation(
validation_loader, model, criterion,
nb_batch_validation)
validation_losses.append(validation_loss)
validation_accuracies.append(validation_accuracy)
# Print progress
if (epoch + 1) % (ep / 10) == 0:
print(
'Epoch [%d/%d] --- TRAIN: Loss: %.4f - Accuracy: %d%% --- '
'VALIDATION: Loss: %.4f - Accuracy: %d%%' %
(epoch + 1, ep, train_loss, train_accuracy,
validation_loss, validation_accuracy))
# ----- PLOT --------------------
if isplot:
plt.figure()
plt.subplot(1, 2, 1)
plt.plot(train_losses, label='Train loss')
plt.plot(validation_losses, label='Validation loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(frameon=False)
plt.subplot(1, 2, 2)
plt.plot(train_accuracies, label='Train accuracy')
plt.plot(validation_accuracies,
label='Validation accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(frameon=False)
# ----- TEST --------------------
train_accuracy = test(train_loader, model)
saved_train_accuracy.append(train_accuracy)
test_accuracy = test(test_loader, model)
saved_test_accuracy.append(test_accuracy)
print('Accuracy on train set: %d %%' % train_accuracy)
print('Accuracy on test set: %d %%' % test_accuracy)
# ----- MEAN + STD OVER ITERATION --------------------
print('\n------- NB EPOCHS - %d -------' % ep)
print('------- LEARNING RATE - %f -------' % lr)
print('------- BATCH SIZE - %d -------' % bs)
print(
'Mean train accuracy {:.02f} --- Std train accuracy {:.02f} '
'\nMean test accuracy {:.02f} --- Std test accuracy {:.02f}'
.format(
torch.FloatTensor(saved_train_accuracy).mean(),
torch.FloatTensor(saved_train_accuracy).std(),
torch.FloatTensor(saved_test_accuracy).mean(),
torch.FloatTensor(saved_test_accuracy).std()))
def PipelineBase(Net, rounds=15, mini_batch_size=50, N=1000):
"""
Use "mini_batch_size" to optimize step by step in "rounds" rounds.
"""
loss_round_list = []
train_errors_list, test_errors_list = [], []
for k in range(rounds):
# Generate Data
train_input, train_target, train_classes, test_input, test_target, test_classes = prologue.generate_pair_sets(
N)
# Model training
model = Net()
loss_list = trainBaselineNet(model, train_input, train_target)
loss_round_list.append(loss_list)
# Predict and compute error
nb_train_errors = errorsCompute(model, train_input, train_target)
nb_test_errors = errorsCompute(model, test_input, test_target)
train_errors_list.append(nb_train_errors / train_input.size(0))
test_errors_list.append(nb_test_errors / test_input.size(0))
# Logging
print('Iteration ', k + 1)
print("---------------------- Error ---------------------")
print('Test error: {:0.2f}% {:d}/{:d}'.format(
(100 * nb_test_errors) / test_input.size(0), nb_test_errors,
test_input.size(0)))
print('Train error: {:0.2f}% {:d}/{:d}'.format(
(100 * nb_train_errors) / train_input.size(0), nb_train_errors,
train_input.size(0)))
print("--------------------------------------------------\n")
return model, loss_round_list, train_errors_list, test_errors_list
"""
Created on Wed Mar 6 10:36:43 2019
@author: Darcane
"""
from torch.autograd import Variable
import dlc_practical_prologue as prologue
from my_nets import *
""" Load Data """
N_PAIRS = 1000
train_input, train_target, train_classes, test_input, test_target, test_classes = \
prologue.generate_pair_sets(N_PAIRS)
train_input, train_target = Variable(train_input), Variable(train_target)
test_input, test_target = Variable(test_input), Variable(test_target)
print(test_input.shape)
print(train_classes.size())
""" Create and train model """
my_model = NetWithWeightSharingAndAuxiliaryLoss()
""" Create and train model which identifies each number and then compares them """
my_model.trainer(train_input, train_target,train_classes)
print("Train error : %.1f%% \nTest error : %.1f%%" %
(my_model.nb_errors(train_input, train_target),
my_model.nb_errors(test_input, test_target)))
def generate_data(normalization=True):
# Generate data using prologue
# param: normalization: To normalize the data or not
# return: data needed for training and testing
# Generate data using prologue
train_input, train_target, train_classes, test_input, test_target, test_classes = prologue.generate_pair_sets(
1000)
# Normalize data if requested
if normalization:
mean, std = train_input.mean(), train_input.std()
train_input.sub_(mean).div_(std)
test_input.sub_(mean).div_(std)
train_target = train_target.float()
test_target = test_target.float()
return train_input, train_target, train_classes, test_input, test_target, test_classes
