def main():
    from optparse import OptionParser
    parser = OptionParser()
    parser.add_option("--slide",
                      dest="slide",
                      type="string",
                      help="slide name")
    parser.add_option("--parameter",
                      dest="p",
                      type="string",
                      help="parameter file")
    parser.add_option("--output",
                      dest="output",
                      type="string",
                      help="outputs name")
    parser.add_option("--mean_file",
                      dest="mean_file",
                      type="string",
                      help="mean_file for the model")
    parser.add_option(
        "--log",
        dest="log",
        type="string",
        help="log or weights for the model with feat being the after _")
    (options, _) = parser.parse_args()

    feat = int(options.log.split('_')[-1])

    model = create_model(options.log, np.load(options.mean_file), feat)
    predict_each_element(options.output, options.slide, options.p, model)
def main_program(path_data,nb_row_per_classe,
                 use_gpu,
                 nb_epoch,batch_size,learning_rate,
                 path_save_model):




    # Label encoding and decoding dicts
    enc_dict, dec_dict = create_encoding_deconding_dict(path_data)




    #Data_set
    size_image_train = 224
    data_train=create_huge_data_set(path_data,nb_rows=nb_row_per_classe,size_image=size_image_train,encoding_dict=enc_dict)

    data_valid=create_huge_data_set(path_data,nb_rows=100,size_image=size_image_train,skip_rows=range(1,nb_row_per_classe),encoding_dict=enc_dict)





    # Model
    model = create_model(use_gpu)

    if use_gpu:
        model.cuda()

    #Loss
    criterion = nn.CrossEntropyLoss()

    #Optimiser
    optimizer = optim.SGD(model.parameters(), lr=learning_rate)

    # Scheduler LR
    scheduler = LambdaLR(optimizer, lr_lambda=LRPolicy(start_lr=learning_rate))


    #Data loader
    train_loader=DataLoader(data_train,batch_size=batch_size,shuffle=True)
    valid_loader=DataLoader(data_valid,batch_size=batch_size)

    #Train
    train_model(model,train_loader,valid_loader,nb_epoch,
                scheduler,optimizer,criterion,use_gpu,
                path_save=path_save_model)








    pass
def show_mistakes(modele_name, path_data, use_gpu):
    def calcul_metric_concours(model, val_loader, use_gpu, dec_dict):
        model.train(False)
        true = []
        pred = []
        val_loss = []
        pred_top3 = []

        criterion = nn.CrossEntropyLoss()
        model.eval()

        for j, batch in enumerate(val_loader):

            inputs, targets = batch
            print(targets)
            if use_gpu:
                inputs = inputs.cuda()
                targets = targets.cuda()

            inputs = Variable(inputs, volatile=True)
            targets = Variable(targets, volatile=True)
            output = model(inputs)
            predictions = output.max(dim=1)[1]

            t1 = inputs[0, 0, :, :].cpu()
            label_true = dec_dict[targets.item()]
            label_pred = dec_dict[predictions.item()]

            imshow(t1, "True: {}, Pred: {}".format(label_true, label_pred))

            predictions_top_3 = output.topk(3)[1]

        model.train(True)

    enc_dict, dec_dict = create_encoding_deconding_dict(path_data)
    nb_ligne_dict = create_dict_nb_ligne(path_data)

    # Model
    model = create_model(use_gpu)

    if use_gpu:
        model.cuda()
    data_test = create_huge_data_set(path_data,
                                     nb_rows=100,
                                     size_image=224,
                                     skip_rows=range(1, 100),
                                     encoding_dict=enc_dict)
    test_loader = DataLoader(data_test, batch_size=1, shuffle=True)

    model_final, history = load_model_weights(model,
                                              modele_name,
                                              type="best",
                                              use_gpu=use_gpu,
                                              get_history=True)

    calcul_metric_concours(model_final, test_loader, use_gpu, dec_dict)
예제 #4
0
def prediction_data(loader, path_save_model, use_gpu, get_prob_pred=False):

    # Model
    model = create_model(use_gpu)
    if use_gpu:
        model.cuda()

    model, history = load_model_weights(model,
                                        path_save_model,
                                        type="last",
                                        use_gpu=use_gpu,
                                        get_history=True)

    model.train(False)

    pred_top3 = []
    pred_top3_prob = []
    model.eval()

    for j, inputs in enumerate(loader):

        if type(inputs) is list:
            inputs = inputs[0]

        if use_gpu:
            inputs = inputs.cuda()

        inputs = Variable(inputs, volatile=True)

        output = model(inputs)

        predictions_top_3 = output.topk(3)[1]

        pred_top3.extend(predictions_top_3.data.cpu().numpy().tolist())
        pred_top3_prob.extend(F.softmax(output).topk(3)[0])

    model.train(True)

    if get_prob_pred:
        return pred_top3, pred_top3_prob

    return pred_top3
예제 #5
0
def main_program(path_data, path_save_model, path_load_existing_model,
                 path_model_weights_test, use_gpu, do_training, do_testing,
                 nb_row_per_classe, nb_generation_random_dataset_train,
                 nb_row_class_valid, nb_row_class_test, skip_test,
                 use_acc_proportionate_sampling, val_acc_class_save_name,
                 nb_epoch, batch_size, learning_rate, type_schedule, seed):

    #Seed
    torch.manual_seed(123)
    np.random.seed(123)
    random.seed(123)
    torch.cuda.manual_seed(123)
    torch.cuda.manual_seed_all(123)

    # Label encoding, decoding dicts, nb_ligne dict
    enc_dict, dec_dict = create_encoding_deconding_dict(path_data)
    nb_ligne_dict = create_dict_nb_ligne(path_data)

    # Model
    model = create_model(use_gpu)

    if use_gpu:
        model.cuda()

    #Loss
    criterion = nn.CrossEntropyLoss()

    #Optimiser
    optimizer = optim.SGD(model.parameters(), lr=learning_rate)

    # Scheduler LR
    scheduler = create_scheduler(start_lr=learning_rate,
                                 type=type_schedule,
                                 optimizer=optimizer)

    # Data_set
    size_image_train = 224
    # data_train = create_huge_data_set(path_data, nb_rows=nb_row_per_classe, size_image=size_image_train,encoding_dict=enc_dict)
    data_valid = create_huge_data_set(path_data,
                                      nb_rows=nb_row_class_valid,
                                      size_image=size_image_train,
                                      encoding_dict=enc_dict)

    #Data loader
    # train_loader=DataLoader(data_train,batch_size=batch_size,shuffle=True)
    valid_loader = DataLoader(data_valid, batch_size=batch_size, shuffle=True)

    #Train
    if do_training:

        for i in range(nb_generation_random_dataset_train):
            data_train = generate_random_dataset(
                path_data,
                nb_row_class_valid,
                nb_row_class_test,
                nb_row_per_classe,
                dict_nb_lignes=nb_ligne_dict,
                size_image=size_image_train,
                encoding_dict=enc_dict,
                use_acc_proportionate_sampling=use_acc_proportionate_sampling,
                val_acc_class_save_name=val_acc_class_save_name)

            train_loader = DataLoader(data_train,
                                      batch_size=batch_size,
                                      shuffle=True)

            if i > 0:
                path_load_existing_model = path_save_model

            train_model(
                model,
                train_loader,
                valid_loader,
                nb_epoch,
                scheduler,
                optimizer,
                criterion,
                use_gpu,
                path_save=path_save_model,
                path_start_from_existing_model=path_load_existing_model,
                val_acc_class_save_name=val_acc_class_save_name)

    #Test
    if do_testing:
        data_test = create_huge_data_set(path_data,
                                         nb_rows=nb_row_class_test,
                                         size_image=size_image_train,
                                         skip_rows=skip_test,
                                         encoding_dict=enc_dict)
        test_loader = DataLoader(data_test, batch_size=batch_size)

        model_final, history = load_model_weights(model,
                                                  path_model_weights_test,
                                                  type="best",
                                                  use_gpu=use_gpu,
                                                  get_history=True)
        history.display()

        acc, loss, score_top3, conf_mat, acc_per_class = calcul_metric_concours(
            model_final, test_loader, use_gpu=use_gpu, show_acc_per_class=True)

        print("Accuracy test: {}".format(acc))
        print("Score top 3 concours: {}".format(score_top3))
        print(acc_per_class)

        #Log experiment
        experiment_sacred.log_scalar("Test accuracy", acc)
        experiment_sacred.log_scalar("Test loss", loss)
        experiment_sacred.log_scalar("Test score top3", score_top3)
        experiment_sacred.log_scalar("Test confusion matrix", conf_mat)
        experiment_sacred.log_scalar("Test accuracy per class", acc_per_class)
def main_program(path_data, path_save_model, path_load_existing_model,
                 path_model_weights_test, use_gpu, do_training, do_testing,
                 nb_row_per_classe, nb_epoch, batch_size, learning_rate,
                 type_schedule):

    # Label encoding and decoding dicts
    enc_dict, dec_dict = create_encoding_deconding_dict(path_data)

    #Data_set
    size_image_train = 224
    data_train = create_huge_data_set(path_data,
                                      nb_rows=nb_row_per_classe,
                                      size_image=size_image_train,
                                      encoding_dict=enc_dict)
    data_valid = create_huge_data_set(path_data,
                                      nb_rows=100,
                                      size_image=size_image_train,
                                      skip_rows=range(1, nb_row_per_classe),
                                      encoding_dict=enc_dict)

    # Model
    model = create_model(use_gpu)

    if use_gpu:
        model.cuda()

    #Loss
    criterion = nn.CrossEntropyLoss()

    #Optimiser
    optimizer = optim.SGD(model.parameters(), lr=learning_rate)

    # Scheduler LR
    scheduler = create_scheduler(start_lr=learning_rate,
                                 type=type_schedule,
                                 optimizer=optimizer)

    #Data loader
    train_loader = DataLoader(data_train, batch_size=batch_size, shuffle=True)
    valid_loader = DataLoader(data_valid, batch_size=batch_size, shuffle=True)

    #Train
    if do_training:
        train_model(model,
                    train_loader,
                    valid_loader,
                    nb_epoch,
                    scheduler,
                    optimizer,
                    criterion,
                    use_gpu,
                    path_save=path_save_model,
                    path_start_from_existing_model=path_load_existing_model)

    #Test
    if do_testing:
        data_test = create_huge_data_set(path_data,
                                         nb_rows=100,
                                         size_image=size_image_train,
                                         skip_rows=range(
                                             1, nb_row_per_classe + 100),
                                         encoding_dict=enc_dict)
        test_loader = DataLoader(data_test, batch_size=batch_size)

        model_final, history = load_model_weights(model,
                                                  path_model_weights_test,
                                                  type="best",
                                                  use_gpu=use_gpu,
                                                  get_history=True)
        history.display()

        acc, loss, score_top3 = calcul_metric_concours(model_final,
                                                       test_loader,
                                                       use_gpu=use_gpu,
                                                       show_acc_per_class=True)

        print("Accuracy test: {}".format(acc))
        print("Score top 3 concours: {}".format(score_top3))
def main_program(path_data, nb_row_per_classe, use_gpu, do_training,
                 do_testing, nb_epoch, batch_size, learning_rate,
                 path_save_model):

    # Label encoding and decoding dicts
    enc_dict, dec_dict = create_encoding_deconding_dict(path_data)

    #Data_set
    size_image_train = 224
    data_train = create_huge_data_set(path_data,
                                      nb_rows=nb_row_per_classe,
                                      size_image=size_image_train,
                                      encoding_dict=enc_dict)

    data_valid = create_huge_data_set(path_data,
                                      nb_rows=100,
                                      size_image=size_image_train,
                                      skip_rows=range(1, nb_row_per_classe),
                                      encoding_dict=enc_dict)

    # Model
    model = create_model(use_gpu)

    if use_gpu:
        model.cuda()

    #Loss
    criterion = nn.CrossEntropyLoss()

    #Optimiser
    optimizer = optim.SGD(model.parameters(), lr=learning_rate)

    # Scheduler LR
    scheduler = LambdaLR(optimizer, lr_lambda=LRPolicy(start_lr=learning_rate))

    #Data loader
    train_loader = DataLoader(data_train, batch_size=batch_size, shuffle=True)
    valid_loader = DataLoader(data_valid, batch_size=batch_size)

    #Train
    if do_training:
        train_model(model,
                    train_loader,
                    valid_loader,
                    nb_epoch,
                    scheduler,
                    optimizer,
                    criterion,
                    use_gpu,
                    path_save=path_save_model)

        score = validate(model, valid_loader, use_gpu=use_gpu)[0]
        print(score)

    #Test
    if do_testing:
        print("BIDON")
        model_final, history = load_model_weights(model,
                                                  path_save_model,
                                                  type="best",
                                                  use_gpu=use_gpu,
                                                  get_history=True)
        history.display()
        score = validate(model, valid_loader, use_gpu=use_gpu)[0]
        print(score)

    pass
예제 #8
0
parser.add_argument('--hidden_units',
                    type=int,
                    default=2048,
                    help='units for hidden layer in classifier.')
parser.add_argument('--epochs',
                    type=int,
                    default=10,
                    help='number of epochs to train for')
parser.add_argument('--gpu', nargs='?', const=1, help='train on gpu.')

args = parser.parse_args()
data_dir = args.data_dir
train_dir = data_dir + '/train'
valid_dir = data_dir + '/valid'
test_dir = data_dir + '/test'
model, criterion, optimizer = create_model(args.hidden_units, args.arch,
                                           args.learning_rate)

if args.gpu:
    device = torch.device('cuda')
else:  #args.gpu==0:
    device = torch.device('cpu')
'''
else:
    print('gpu preference not specified, using GPU if available.')
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
'''
print('Using device: ', device)
model.to(device)
#prepare data for training
#Data augmentation performed on training set
training_transform = transforms.Compose([
예제 #9
0
val_noisy_dir = '/media/bernardo/Storage/SIDD_patches_{0:04d}/both_datasets/{0:04d}'.format(SELECTED_ISO,SELECTED_ISO)
test_clean_dir = '/media/bernardo/Storage/SIDD_patches_{0:04d}/both_datasets/gts_val_classificator'.format(SELECTED_ISO)
test_noisy_dir = '/media/bernardo/Storage/SIDD_patches_{0:04d}/both_datasets/{0:04d}'.format(SELECTED_ISO,SELECTED_ISO)


# Run parameters
batch_size = 32
epochs = 3

# Data specific constants
image_size = 128,128

classes = ['artificial','natural']
num_classes = 1

model = create_model(image_size,num_classes)
model.summary()


datagen = ImageDataGenerator(horizontal_flip=True,vertical_flip=True)

datagen.config['random_crop_size'] = image_size

datagen.set_pipeline([random_transform,random_crop,random90rot,standardize,compute_fft2])
flow_train = datagen.flow_from_directory(train_clean_dir,batch_size=batch_size,color_mode='rgbfft',target_size=image_size)
flow_train.setCurrentISO(SELECTED_ISO,train_noisy_dir)

datagen_val = ImageDataGenerator()

datagen_val.config['center_crop_size'] = image_size