def main():
from optparse import OptionParser
parser = OptionParser()
parser.add_option("--slide",
dest="slide",
type="string",
help="slide name")
parser.add_option("--parameter",
dest="p",
type="string",
help="parameter file")
parser.add_option("--output",
dest="output",
type="string",
help="outputs name")
parser.add_option("--mean_file",
dest="mean_file",
type="string",
help="mean_file for the model")
parser.add_option(
"--log",
dest="log",
type="string",
help="log or weights for the model with feat being the after _")
(options, _) = parser.parse_args()
feat = int(options.log.split('_')[-1])
model = create_model(options.log, np.load(options.mean_file), feat)
predict_each_element(options.output, options.slide, options.p, model)
def main_program(path_data,nb_row_per_classe,
use_gpu,
nb_epoch,batch_size,learning_rate,
path_save_model):
# Label encoding and decoding dicts
enc_dict, dec_dict = create_encoding_deconding_dict(path_data)
#Data_set
size_image_train = 224
data_train=create_huge_data_set(path_data,nb_rows=nb_row_per_classe,size_image=size_image_train,encoding_dict=enc_dict)
data_valid=create_huge_data_set(path_data,nb_rows=100,size_image=size_image_train,skip_rows=range(1,nb_row_per_classe),encoding_dict=enc_dict)
# Model
model = create_model(use_gpu)
if use_gpu:
model.cuda()
#Loss
criterion = nn.CrossEntropyLoss()
#Optimiser
optimizer = optim.SGD(model.parameters(), lr=learning_rate)
# Scheduler LR
scheduler = LambdaLR(optimizer, lr_lambda=LRPolicy(start_lr=learning_rate))
#Data loader
train_loader=DataLoader(data_train,batch_size=batch_size,shuffle=True)
valid_loader=DataLoader(data_valid,batch_size=batch_size)
#Train
train_model(model,train_loader,valid_loader,nb_epoch,
scheduler,optimizer,criterion,use_gpu,
path_save=path_save_model)
pass
def show_mistakes(modele_name, path_data, use_gpu):
def calcul_metric_concours(model, val_loader, use_gpu, dec_dict):
model.train(False)
true = []
pred = []
val_loss = []
pred_top3 = []
criterion = nn.CrossEntropyLoss()
model.eval()
for j, batch in enumerate(val_loader):
inputs, targets = batch
print(targets)
if use_gpu:
inputs = inputs.cuda()
targets = targets.cuda()
inputs = Variable(inputs, volatile=True)
targets = Variable(targets, volatile=True)
output = model(inputs)
predictions = output.max(dim=1)[1]
t1 = inputs[0, 0, :, :].cpu()
label_true = dec_dict[targets.item()]
label_pred = dec_dict[predictions.item()]
imshow(t1, "True: {}, Pred: {}".format(label_true, label_pred))
predictions_top_3 = output.topk(3)[1]
model.train(True)
enc_dict, dec_dict = create_encoding_deconding_dict(path_data)
nb_ligne_dict = create_dict_nb_ligne(path_data)
# Model
model = create_model(use_gpu)
if use_gpu:
model.cuda()
data_test = create_huge_data_set(path_data,
nb_rows=100,
size_image=224,
skip_rows=range(1, 100),
encoding_dict=enc_dict)
test_loader = DataLoader(data_test, batch_size=1, shuffle=True)
model_final, history = load_model_weights(model,
modele_name,
type="best",
use_gpu=use_gpu,
get_history=True)
calcul_metric_concours(model_final, test_loader, use_gpu, dec_dict)
def prediction_data(loader, path_save_model, use_gpu, get_prob_pred=False):
# Model
model = create_model(use_gpu)
if use_gpu:
model.cuda()
model, history = load_model_weights(model,
path_save_model,
type="last",
use_gpu=use_gpu,
get_history=True)
model.train(False)
pred_top3 = []
pred_top3_prob = []
model.eval()
for j, inputs in enumerate(loader):
if type(inputs) is list:
inputs = inputs[0]
if use_gpu:
inputs = inputs.cuda()
inputs = Variable(inputs, volatile=True)
output = model(inputs)
predictions_top_3 = output.topk(3)[1]
pred_top3.extend(predictions_top_3.data.cpu().numpy().tolist())
pred_top3_prob.extend(F.softmax(output).topk(3)[0])
model.train(True)
if get_prob_pred:
return pred_top3, pred_top3_prob
return pred_top3
def main_program(path_data, path_save_model, path_load_existing_model,
path_model_weights_test, use_gpu, do_training, do_testing,
nb_row_per_classe, nb_generation_random_dataset_train,
nb_row_class_valid, nb_row_class_test, skip_test,
use_acc_proportionate_sampling, val_acc_class_save_name,
nb_epoch, batch_size, learning_rate, type_schedule, seed):
#Seed
torch.manual_seed(123)
np.random.seed(123)
random.seed(123)
torch.cuda.manual_seed(123)
torch.cuda.manual_seed_all(123)
# Label encoding, decoding dicts, nb_ligne dict
enc_dict, dec_dict = create_encoding_deconding_dict(path_data)
nb_ligne_dict = create_dict_nb_ligne(path_data)
# Model
model = create_model(use_gpu)
if use_gpu:
model.cuda()
#Loss
criterion = nn.CrossEntropyLoss()
#Optimiser
optimizer = optim.SGD(model.parameters(), lr=learning_rate)
# Scheduler LR
scheduler = create_scheduler(start_lr=learning_rate,
type=type_schedule,
optimizer=optimizer)
# Data_set
size_image_train = 224
# data_train = create_huge_data_set(path_data, nb_rows=nb_row_per_classe, size_image=size_image_train,encoding_dict=enc_dict)
data_valid = create_huge_data_set(path_data,
nb_rows=nb_row_class_valid,
size_image=size_image_train,
encoding_dict=enc_dict)
#Data loader
# train_loader=DataLoader(data_train,batch_size=batch_size,shuffle=True)
valid_loader = DataLoader(data_valid, batch_size=batch_size, shuffle=True)
#Train
if do_training:
for i in range(nb_generation_random_dataset_train):
data_train = generate_random_dataset(
path_data,
nb_row_class_valid,
nb_row_class_test,
nb_row_per_classe,
dict_nb_lignes=nb_ligne_dict,
size_image=size_image_train,
encoding_dict=enc_dict,
use_acc_proportionate_sampling=use_acc_proportionate_sampling,
val_acc_class_save_name=val_acc_class_save_name)
train_loader = DataLoader(data_train,
batch_size=batch_size,
shuffle=True)
if i > 0:
path_load_existing_model = path_save_model
train_model(
model,
train_loader,
valid_loader,
nb_epoch,
scheduler,
optimizer,
criterion,
use_gpu,
path_save=path_save_model,
path_start_from_existing_model=path_load_existing_model,
val_acc_class_save_name=val_acc_class_save_name)
#Test
if do_testing:
data_test = create_huge_data_set(path_data,
nb_rows=nb_row_class_test,
size_image=size_image_train,
skip_rows=skip_test,
encoding_dict=enc_dict)
test_loader = DataLoader(data_test, batch_size=batch_size)
model_final, history = load_model_weights(model,
path_model_weights_test,
type="best",
use_gpu=use_gpu,
get_history=True)
history.display()
acc, loss, score_top3, conf_mat, acc_per_class = calcul_metric_concours(
model_final, test_loader, use_gpu=use_gpu, show_acc_per_class=True)
print("Accuracy test: {}".format(acc))
print("Score top 3 concours: {}".format(score_top3))
print(acc_per_class)
#Log experiment
experiment_sacred.log_scalar("Test accuracy", acc)
experiment_sacred.log_scalar("Test loss", loss)
experiment_sacred.log_scalar("Test score top3", score_top3)
experiment_sacred.log_scalar("Test confusion matrix", conf_mat)
experiment_sacred.log_scalar("Test accuracy per class", acc_per_class)
def main_program(path_data, path_save_model, path_load_existing_model,
path_model_weights_test, use_gpu, do_training, do_testing,
nb_row_per_classe, nb_epoch, batch_size, learning_rate,
type_schedule):
# Label encoding and decoding dicts
enc_dict, dec_dict = create_encoding_deconding_dict(path_data)
#Data_set
size_image_train = 224
data_train = create_huge_data_set(path_data,
nb_rows=nb_row_per_classe,
size_image=size_image_train,
encoding_dict=enc_dict)
data_valid = create_huge_data_set(path_data,
nb_rows=100,
size_image=size_image_train,
skip_rows=range(1, nb_row_per_classe),
encoding_dict=enc_dict)
# Model
model = create_model(use_gpu)
if use_gpu:
model.cuda()
#Loss
criterion = nn.CrossEntropyLoss()
#Optimiser
optimizer = optim.SGD(model.parameters(), lr=learning_rate)
# Scheduler LR
scheduler = create_scheduler(start_lr=learning_rate,
type=type_schedule,
optimizer=optimizer)
#Data loader
train_loader = DataLoader(data_train, batch_size=batch_size, shuffle=True)
valid_loader = DataLoader(data_valid, batch_size=batch_size, shuffle=True)
#Train
if do_training:
train_model(model,
train_loader,
valid_loader,
nb_epoch,
scheduler,
optimizer,
criterion,
use_gpu,
path_save=path_save_model,
path_start_from_existing_model=path_load_existing_model)
#Test
if do_testing:
data_test = create_huge_data_set(path_data,
nb_rows=100,
size_image=size_image_train,
skip_rows=range(
1, nb_row_per_classe + 100),
encoding_dict=enc_dict)
test_loader = DataLoader(data_test, batch_size=batch_size)
model_final, history = load_model_weights(model,
path_model_weights_test,
type="best",
use_gpu=use_gpu,
get_history=True)
history.display()
acc, loss, score_top3 = calcul_metric_concours(model_final,
test_loader,
use_gpu=use_gpu,
show_acc_per_class=True)
print("Accuracy test: {}".format(acc))
print("Score top 3 concours: {}".format(score_top3))
def main_program(path_data, nb_row_per_classe, use_gpu, do_training,
do_testing, nb_epoch, batch_size, learning_rate,
path_save_model):
# Label encoding and decoding dicts
enc_dict, dec_dict = create_encoding_deconding_dict(path_data)
#Data_set
size_image_train = 224
data_train = create_huge_data_set(path_data,
nb_rows=nb_row_per_classe,
size_image=size_image_train,
encoding_dict=enc_dict)
data_valid = create_huge_data_set(path_data,
nb_rows=100,
size_image=size_image_train,
skip_rows=range(1, nb_row_per_classe),
encoding_dict=enc_dict)
# Model
model = create_model(use_gpu)
if use_gpu:
model.cuda()
#Loss
criterion = nn.CrossEntropyLoss()
#Optimiser
optimizer = optim.SGD(model.parameters(), lr=learning_rate)
# Scheduler LR
scheduler = LambdaLR(optimizer, lr_lambda=LRPolicy(start_lr=learning_rate))
#Data loader
train_loader = DataLoader(data_train, batch_size=batch_size, shuffle=True)
valid_loader = DataLoader(data_valid, batch_size=batch_size)
#Train
if do_training:
train_model(model,
train_loader,
valid_loader,
nb_epoch,
scheduler,
optimizer,
criterion,
use_gpu,
path_save=path_save_model)
score = validate(model, valid_loader, use_gpu=use_gpu)[0]
print(score)
#Test
if do_testing:
print("BIDON")
model_final, history = load_model_weights(model,
path_save_model,
type="best",
use_gpu=use_gpu,
get_history=True)
history.display()
score = validate(model, valid_loader, use_gpu=use_gpu)[0]
print(score)
pass
parser.add_argument('--hidden_units',
type=int,
default=2048,
help='units for hidden layer in classifier.')
parser.add_argument('--epochs',
type=int,
default=10,
help='number of epochs to train for')
parser.add_argument('--gpu', nargs='?', const=1, help='train on gpu.')
args = parser.parse_args()
data_dir = args.data_dir
train_dir = data_dir + '/train'
valid_dir = data_dir + '/valid'
test_dir = data_dir + '/test'
model, criterion, optimizer = create_model(args.hidden_units, args.arch,
args.learning_rate)
if args.gpu:
device = torch.device('cuda')
else: #args.gpu==0:
device = torch.device('cpu')
'''
else:
print('gpu preference not specified, using GPU if available.')
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
'''
print('Using device: ', device)
model.to(device)
#prepare data for training
#Data augmentation performed on training set
training_transform = transforms.Compose([
val_noisy_dir = '/media/bernardo/Storage/SIDD_patches_{0:04d}/both_datasets/{0:04d}'.format(SELECTED_ISO,SELECTED_ISO)
test_clean_dir = '/media/bernardo/Storage/SIDD_patches_{0:04d}/both_datasets/gts_val_classificator'.format(SELECTED_ISO)
test_noisy_dir = '/media/bernardo/Storage/SIDD_patches_{0:04d}/both_datasets/{0:04d}'.format(SELECTED_ISO,SELECTED_ISO)
# Run parameters
batch_size = 32
epochs = 3
# Data specific constants
image_size = 128,128
classes = ['artificial','natural']
num_classes = 1
model = create_model(image_size,num_classes)
model.summary()
datagen = ImageDataGenerator(horizontal_flip=True,vertical_flip=True)
datagen.config['random_crop_size'] = image_size
datagen.set_pipeline([random_transform,random_crop,random90rot,standardize,compute_fft2])
flow_train = datagen.flow_from_directory(train_clean_dir,batch_size=batch_size,color_mode='rgbfft',target_size=image_size)
flow_train.setCurrentISO(SELECTED_ISO,train_noisy_dir)
datagen_val = ImageDataGenerator()
datagen_val.config['center_crop_size'] = image_size