def main_program(path_data,nb_row_per_classe,
use_gpu,
nb_epoch,batch_size,learning_rate,
path_save_model):
# Label encoding and decoding dicts
enc_dict, dec_dict = create_encoding_deconding_dict(path_data)
#Data_set
size_image_train = 224
data_train=create_huge_data_set(path_data,nb_rows=nb_row_per_classe,size_image=size_image_train,encoding_dict=enc_dict)
data_valid=create_huge_data_set(path_data,nb_rows=100,size_image=size_image_train,skip_rows=range(1,nb_row_per_classe),encoding_dict=enc_dict)
# Model
model = create_model(use_gpu)
if use_gpu:
model.cuda()
#Loss
criterion = nn.CrossEntropyLoss()
#Optimiser
optimizer = optim.SGD(model.parameters(), lr=learning_rate)
# Scheduler LR
scheduler = LambdaLR(optimizer, lr_lambda=LRPolicy(start_lr=learning_rate))
#Data loader
train_loader=DataLoader(data_train,batch_size=batch_size,shuffle=True)
valid_loader=DataLoader(data_valid,batch_size=batch_size)
#Train
train_model(model,train_loader,valid_loader,nb_epoch,
scheduler,optimizer,criterion,use_gpu,
path_save=path_save_model)
pass
def show_mistakes(modele_name, path_data, use_gpu):
def calcul_metric_concours(model, val_loader, use_gpu, dec_dict):
model.train(False)
true = []
pred = []
val_loss = []
pred_top3 = []
criterion = nn.CrossEntropyLoss()
model.eval()
for j, batch in enumerate(val_loader):
inputs, targets = batch
print(targets)
if use_gpu:
inputs = inputs.cuda()
targets = targets.cuda()
inputs = Variable(inputs, volatile=True)
targets = Variable(targets, volatile=True)
output = model(inputs)
predictions = output.max(dim=1)[1]
t1 = inputs[0, 0, :, :].cpu()
label_true = dec_dict[targets.item()]
label_pred = dec_dict[predictions.item()]
imshow(t1, "True: {}, Pred: {}".format(label_true, label_pred))
predictions_top_3 = output.topk(3)[1]
model.train(True)
enc_dict, dec_dict = create_encoding_deconding_dict(path_data)
nb_ligne_dict = create_dict_nb_ligne(path_data)
# Model
model = create_model(use_gpu)
if use_gpu:
model.cuda()
data_test = create_huge_data_set(path_data,
nb_rows=100,
size_image=224,
skip_rows=range(1, 100),
encoding_dict=enc_dict)
test_loader = DataLoader(data_test, batch_size=1, shuffle=True)
model_final, history = load_model_weights(model,
modele_name,
type="best",
use_gpu=use_gpu,
get_history=True)
calcul_metric_concours(model_final, test_loader, use_gpu, dec_dict)
def main_program(path_data, path_save_model, path_load_existing_model,
path_model_weights_test, use_gpu, do_training, do_testing,
nb_row_per_classe, nb_generation_random_dataset_train,
nb_row_class_valid, nb_row_class_test, skip_test,
use_acc_proportionate_sampling, val_acc_class_save_name,
nb_epoch, batch_size, learning_rate, type_schedule, seed):
#Seed
torch.manual_seed(123)
np.random.seed(123)
random.seed(123)
torch.cuda.manual_seed(123)
torch.cuda.manual_seed_all(123)
# Label encoding, decoding dicts, nb_ligne dict
enc_dict, dec_dict = create_encoding_deconding_dict(path_data)
nb_ligne_dict = create_dict_nb_ligne(path_data)
# Model
model = create_model(use_gpu)
if use_gpu:
model.cuda()
#Loss
criterion = nn.CrossEntropyLoss()
#Optimiser
optimizer = optim.SGD(model.parameters(), lr=learning_rate)
# Scheduler LR
scheduler = create_scheduler(start_lr=learning_rate,
type=type_schedule,
optimizer=optimizer)
# Data_set
size_image_train = 224
# data_train = create_huge_data_set(path_data, nb_rows=nb_row_per_classe, size_image=size_image_train,encoding_dict=enc_dict)
data_valid = create_huge_data_set(path_data,
nb_rows=nb_row_class_valid,
size_image=size_image_train,
encoding_dict=enc_dict)
#Data loader
# train_loader=DataLoader(data_train,batch_size=batch_size,shuffle=True)
valid_loader = DataLoader(data_valid, batch_size=batch_size, shuffle=True)
#Train
if do_training:
for i in range(nb_generation_random_dataset_train):
data_train = generate_random_dataset(
path_data,
nb_row_class_valid,
nb_row_class_test,
nb_row_per_classe,
dict_nb_lignes=nb_ligne_dict,
size_image=size_image_train,
encoding_dict=enc_dict,
use_acc_proportionate_sampling=use_acc_proportionate_sampling,
val_acc_class_save_name=val_acc_class_save_name)
train_loader = DataLoader(data_train,
batch_size=batch_size,
shuffle=True)
if i > 0:
path_load_existing_model = path_save_model
train_model(
model,
train_loader,
valid_loader,
nb_epoch,
scheduler,
optimizer,
criterion,
use_gpu,
path_save=path_save_model,
path_start_from_existing_model=path_load_existing_model,
val_acc_class_save_name=val_acc_class_save_name)
#Test
if do_testing:
data_test = create_huge_data_set(path_data,
nb_rows=nb_row_class_test,
size_image=size_image_train,
skip_rows=skip_test,
encoding_dict=enc_dict)
test_loader = DataLoader(data_test, batch_size=batch_size)
model_final, history = load_model_weights(model,
path_model_weights_test,
type="best",
use_gpu=use_gpu,
get_history=True)
history.display()
acc, loss, score_top3, conf_mat, acc_per_class = calcul_metric_concours(
model_final, test_loader, use_gpu=use_gpu, show_acc_per_class=True)
print("Accuracy test: {}".format(acc))
print("Score top 3 concours: {}".format(score_top3))
print(acc_per_class)
#Log experiment
experiment_sacred.log_scalar("Test accuracy", acc)
experiment_sacred.log_scalar("Test loss", loss)
experiment_sacred.log_scalar("Test score top3", score_top3)
experiment_sacred.log_scalar("Test confusion matrix", conf_mat)
experiment_sacred.log_scalar("Test accuracy per class", acc_per_class)
def main_program(path_data, path_save_model, path_load_existing_model,
path_model_weights_test, use_gpu, do_training, do_testing,
nb_row_per_classe, nb_epoch, batch_size, learning_rate,
type_schedule):
# Label encoding and decoding dicts
enc_dict, dec_dict = create_encoding_deconding_dict(path_data)
#Data_set
size_image_train = 224
data_train = create_huge_data_set(path_data,
nb_rows=nb_row_per_classe,
size_image=size_image_train,
encoding_dict=enc_dict)
data_valid = create_huge_data_set(path_data,
nb_rows=100,
size_image=size_image_train,
skip_rows=range(1, nb_row_per_classe),
encoding_dict=enc_dict)
# Model
model = create_model(use_gpu)
if use_gpu:
model.cuda()
#Loss
criterion = nn.CrossEntropyLoss()
#Optimiser
optimizer = optim.SGD(model.parameters(), lr=learning_rate)
# Scheduler LR
scheduler = create_scheduler(start_lr=learning_rate,
type=type_schedule,
optimizer=optimizer)
#Data loader
train_loader = DataLoader(data_train, batch_size=batch_size, shuffle=True)
valid_loader = DataLoader(data_valid, batch_size=batch_size, shuffle=True)
#Train
if do_training:
train_model(model,
train_loader,
valid_loader,
nb_epoch,
scheduler,
optimizer,
criterion,
use_gpu,
path_save=path_save_model,
path_start_from_existing_model=path_load_existing_model)
#Test
if do_testing:
data_test = create_huge_data_set(path_data,
nb_rows=100,
size_image=size_image_train,
skip_rows=range(
1, nb_row_per_classe + 100),
encoding_dict=enc_dict)
test_loader = DataLoader(data_test, batch_size=batch_size)
model_final, history = load_model_weights(model,
path_model_weights_test,
type="best",
use_gpu=use_gpu,
get_history=True)
history.display()
acc, loss, score_top3 = calcul_metric_concours(model_final,
test_loader,
use_gpu=use_gpu,
show_acc_per_class=True)
print("Accuracy test: {}".format(acc))
print("Score top 3 concours: {}".format(score_top3))
def main_program(path_data, nb_row_per_classe, use_gpu, do_training,
do_testing, nb_epoch, batch_size, learning_rate,
path_save_model):
# Label encoding and decoding dicts
enc_dict, dec_dict = create_encoding_deconding_dict(path_data)
#Data_set
size_image_train = 224
data_train = create_huge_data_set(path_data,
nb_rows=nb_row_per_classe,
size_image=size_image_train,
encoding_dict=enc_dict)
data_valid = create_huge_data_set(path_data,
nb_rows=100,
size_image=size_image_train,
skip_rows=range(1, nb_row_per_classe),
encoding_dict=enc_dict)
# Model
model = create_model(use_gpu)
if use_gpu:
model.cuda()
#Loss
criterion = nn.CrossEntropyLoss()
#Optimiser
optimizer = optim.SGD(model.parameters(), lr=learning_rate)
# Scheduler LR
scheduler = LambdaLR(optimizer, lr_lambda=LRPolicy(start_lr=learning_rate))
#Data loader
train_loader = DataLoader(data_train, batch_size=batch_size, shuffle=True)
valid_loader = DataLoader(data_valid, batch_size=batch_size)
#Train
if do_training:
train_model(model,
train_loader,
valid_loader,
nb_epoch,
scheduler,
optimizer,
criterion,
use_gpu,
path_save=path_save_model)
score = validate(model, valid_loader, use_gpu=use_gpu)[0]
print(score)
#Test
if do_testing:
print("BIDON")
model_final, history = load_model_weights(model,
path_save_model,
type="best",
use_gpu=use_gpu,
get_history=True)
history.display()
score = validate(model, valid_loader, use_gpu=use_gpu)[0]
print(score)
pass