Exemple #1
0
def test(args, model_load, imageSaver=None, EvaluatorIn=None):

    model = get_model(args)

    if model_load is not None:
        model.load_state_dict(torch.load(model_load), strict=False)

    logs = test_with_loaded_model(args, model, imageSaver, EvaluatorIn)

    return logs
Exemple #2
0
    def __init__(self, hparams: Namespace) -> None:
        super(MMClassifier, self).__init__()
        self.hparams = hparams

        self.model = get_model(self.hparams)

        if hparams.task_type == "multilabel":
            if hparams.weight_classes:
                freqs = [hparams.label_freqs[l] for l in hparams.labels]
                label_weights = (torch.FloatTensor(freqs) /
                                 hparams.train_data_len)**-1
                self.criterion = nn.BCEWithLogitsLoss(
                    pos_weight=label_weights.cuda())
            else:
                self.criterion = nn.BCEWithLogitsLoss()
        else:
            self.criterion = nn.CrossEntropyLoss()
Exemple #3
0
def train_normal(args, num_epoch, model_save=None, model_load=None, imgSaver=None, EvaluatorIn=None):
    # model_save is the name of the file name to save model to
    # model_load is the model to initialize training with

    if EvaluatorIn is None:
        evaluator = Evaluator(args)
    else:
        evaluator = EvaluatorIn(args)

    model = get_model(args)
    losses = get_losses(args)
    optimizer = get_optimizer(args, model)
    scheduler = get_scheduler(args, optimizer)
    valid_metric = None
    main_metric = evaluator.main_metric

    if model_load is not None:
        model.load_state_dict(torch.load(model_load), strict=False)

    train_dataset, valid_dataset = get_train_val_datasets(args)

    valid_logs, train_logs, best_loss_valid_metric = train(model, optimizer, scheduler, losses, train_dataset,
                                                           valid_dataset, num_epoch, args, evaluator, imgSaver)

    if args['valid_dataset'] and args['train_with_metric']:
        valid_metric = valid_logs['metrics'][main_metric]

    if args['save_best_loss_name'] is not None or args['save_on_best_total_training_loss']:
        valid_metric = best_loss_valid_metric

    if not args['valid_dataset']:
        valid_metric = train_logs['metrics'][main_metric]
        valid_logs = train_logs

    if model_save is not None:
        torch.save(model.state_dict(), model_save)

    return valid_logs, train_logs, valid_metric
Exemple #4
0
def main():
    parser = argparse.ArgumentParser(description='Chainer CIFAR example:')
    parser.add_argument('--model', default='c3f2')
    parser.add_argument('--batchsize', '-b', type=int, default=64)
    parser.add_argument('--learnrate', '-l', type=float, default=0.05)
    parser.add_argument('--epoch', '-e', type=int, default=300)
    parser.add_argument('--gpu', '-g', type=int, default=0)
    parser.add_argument('--N', type=int, default=9)
    parser.add_argument('--k', type=int, default=10)
    parser.add_argument('--out', '-o', default='result')
    parser.add_argument('--debug', action='store_true')
    parser.add_argument('--resume', '-r', default='')
    args = parser.parse_args()
    start = time.time()
    logger.initialize("outputs_" + args.model)
    logger.info(vars(args))
    save_dir = logger.get_savedir()
    logger.info("Written to {}".format(save_dir))
    logger.info('GPU: {}'.format(args.gpu))
    logger.info('# Minibatch-size: {}'.format(args.batchsize))
    logger.info('# epoch: {}'.format(args.epoch))
    logger.info('')
    train, test = get_cifar10()
    if args.debug:
        train, test = train[:200], test[:200]
    train_count, test_count = len(train), len(test)

    model = get_model(args.model, args.gpu, args.resume)

    optimizer = chainer.optimizers.MomentumSGD(args.learnrate)
    optimizer.setup(model)
    optimizer.add_hook(chainer.optimizer.WeightDecay(5e-4))

    train_iter = chainer.iterators.SerialIterator(train, args.batchsize)
    test_iter = chainer.iterators.SerialIterator(test,
                                                 args.batchsize,
                                                 repeat=False,
                                                 shuffle=False)

    sum_accuracy = 0
    sum_loss = 0
    st = time.time()
    iter_cnt = 0
    chainer.config.train = True
    chainer.config.enable_backprop = True
    while train_iter.epoch < args.epoch:
        batch = train_iter.next()
        if train_iter.epoch % 60 == 0 and train_iter.is_new_epoch:
            optimizer.lr *= 0.2
            logger.info('Reducing learning rate to: {}'.format(optimizer.lr))

        x_array, t_array = convert.concat_examples(batch, args.gpu)
        x = chainer.Variable(x_array)
        t = chainer.Variable(t_array)
        model.cleargrads()
        loss = model(x, t)
        loss.backward()
        loss.unchain_backward()
        optimizer.update()
        sum_loss += float(model.loss.data) * len(t.data)
        sum_accuracy += float(model.accuracy.data) * len(t.data)

        if train_iter.is_new_epoch:
            train_loss = sum_loss / train_count
            train_acc = sum_accuracy / train_count
            # evaluation
            sum_accuracy = 0
            sum_loss = 0
            chainer.config.train = False
            chainer.config.enable_backprop = False
            for batch in test_iter:
                x_array, t_array = convert.concat_examples(batch, args.gpu)
                x = chainer.Variable(x_array)
                t = chainer.Variable(t_array)
                loss = model(x, t)
                sum_loss += float(loss.data) * len(t.data)
                sum_accuracy += float(model.accuracy.data) * len(t.data)

            test_iter.reset()
            chainer.config.train = True
            chainer.config.enable_backprop = True
            test_loss = sum_loss / test_count
            test_acc = sum_accuracy / test_count
            message_str = "Epoch {}: train loss={:.4f}, acc={:.4f}, test loss={:.4f}, acc={:.4f}, elapsed={}"
            logger.info(
                message_str.format(train_iter.epoch, train_loss, train_acc,
                                   test_loss, test_acc,
                                   time.time() - st))
            st = time.time()
            sum_accuracy = 0
            sum_loss = 0
            if not args.debug:
                serializers.save_npz(
                    os.path.join(save_dir,
                                 "model_ep_{}.npz".format(train_iter.epoch)),
                    model)
        iter_cnt += 1
Exemple #5
0
def main():
    parser = argparse.ArgumentParser(description='Chainer CIFAR example:')
    parser.add_argument('--model', default='c3f2')
    parser.add_argument('--batchsize', '-b', type=int, default=64)
    parser.add_argument('--gpu', '-g', type=int, default=0)
    parser.add_argument('--N', type=int, default=9)
    parser.add_argument('--k', type=int, default=10)
    parser.add_argument('--nb_valid', type=int, default=10000)
    parser.add_argument('--seed', type=int, default=1701)
    parser.add_argument('--out', '-o', default='result')
    parser.add_argument('--debug', action='store_true')
    parser.add_argument('--test', action='store_true')
    parser.add_argument('--resume', '-r', default='')
    args = parser.parse_args()
    start = time.time()
    logger.initialize("evals_" + args.model)
    logger.info(vars(args))
    np.random.seed(args.seed)
    save_dir = logger.get_savedir()
    logger.info("Written to {}".format(save_dir))
    logger.info('GPU: {}'.format(args.gpu))
    logger.info('# Minibatch-size: {}'.format(args.batchsize))
    logger.info('')
    train_all, test = get_cifar10()

    if args.debug:
        valid = train_all[200:400]
        batchsize = 20
    else:
        valid_choice = np.random.choice(range(len(train_all)),
                                        args.nb_valid,
                                        replace=False)
        valid = [x for idx, x in enumerate(train_all) if idx in valid_choice]
        batchsize = args.batchsize

    valid_cnt = len(valid)
    print(valid_cnt)

    model = get_model(args.model, args.gpu, args.resume)
    valid_iter = chainer.iterators.SerialIterator(valid,
                                                  batchsize,
                                                  repeat=False,
                                                  shuffle=False)

    sum_accuracy = 0
    sum_loss = 0
    chainer.config.train = False
    chainer.config.enable_backprop = False
    predictions = []
    for idx, batch in enumerate(valid_iter):
        x_array, t_array = convert.concat_examples(batch, args.gpu)
        x = chainer.Variable(x_array)
        t = chainer.Variable(t_array)
        loss = model(x, t)
        sum_loss += float(loss.data) * len(t.data)
        sum_accuracy += float(model.accuracy.data) * len(t.data)
        predictions.extend(
            np.argmax(cuda.to_cpu(model.pred.data), axis=1).tolist())

    valid_iter.reset()
    valid_loss = sum_loss / valid_cnt
    valid_acc = sum_accuracy / valid_cnt
    message_str = "Valid loss={:.4f}, acc={:.4f}"
    logger.info(message_str.format(valid_loss, valid_acc))

    canvas = plot_error([x[0] for x in valid], [x[1] for x in valid],
                        predictions)
    cv2.imwrite("error.jpg", canvas)
    print(time.time() - start)
def main():
    parser = argparse.ArgumentParser(description='Chainer CIFAR example:')
    parser.add_argument('resume')
    parser.add_argument('--nb_trials', type=int, default=50)
    parser.add_argument('--model', default='c5')
    parser.add_argument('--gpu', '-g', type=int, default=0)
    parser.add_argument('--nb_valid', type=int, default=10000)
    parser.add_argument('--seed', type=int, default=1701)
    parser.add_argument('--debug', action='store_true')
    args = parser.parse_args()
    start = time.time()
    logger.initialize("grad_"+args.model)
    logger.info(vars(args))
    np.random.seed(args.seed)
    save_dir = logger.get_savedir()
    logger.info("Written to {}".format(save_dir))
    logger.info('GPU: {}'.format(args.gpu))
    train_all, test = get_cifar10()

    if args.debug:
        valid = train_all[200:400]
    else:
        valid_choice = np.random.choice(range(len(train_all)),
                                        args.nb_valid, replace=False)
        valid = [x for idx, x in enumerate(train_all) if idx in valid_choice]

    print(len(valid))

    model = get_model(args.model, args.gpu, args.resume)

    # Get one image per iteration
    valid_iter = chainer.iterators.SerialIterator(
        valid, 1, repeat=False, shuffle=False)

    if not os.path.exists("grads"):
        os.makedirs("grads")

    chainer.config.train = False
    chainer.config.enable_backprop = True
    for idx, tup in enumerate(valid_iter):
        print(idx)
        img = tup[0][0]
        # Tile image to calculate all the trials at once
        inp = np.tile(img.copy()[np.newaxis, ...], (args.nb_trials, 1, 1, 1))
        label = tup[0][1][np.newaxis, ...]
        sigma = (inp.max() - inp.min()) * 0.025  # noise level
        model.cleargrads()
        inp = inp + np.random.randn(*inp.shape).astype(np.float32) * sigma  # Add noise to every image
        x = Variable(cuda.to_gpu(inp, args.gpu))
        xp = cuda.get_array_module(x)
        pred = model.get_feature(x, False)
        # print(class_list[int(cuda.to_cpu(xp.argmax(pred.data)))], class_list[int(label)])
        pred.grad = xp.ones(pred.shape, dtype=np.float32)
        pred.backward()
        mean_grad = cuda.to_cpu(xp.mean(x.grad.copy(), axis=0))
        mean_grad = np.max(np.abs(mean_grad), axis=0)
        mean_grad = color.gray2rgb(mean_grad)
        mean_grad = clip_image(mean_grad)
        orig_img = np.transpose(img, (1, 2, 0))
        masked = orig_img * mean_grad
        out = np.concatenate((mean_grad, masked, orig_img), axis=1)
        plt.imsave("grads/{:05d}.png".format(idx), out)
        model.cleargrads()

    print(time.time()-start)
def one_stage_training_gauss(args, model_load):

    training_type = '_gauss_'
    output_dir = './run/experiments/models'
    add_to_file_path, model_save = create_file_name(args['dataset_name'],
                                                    args['model_name'],
                                                    args['im_size'],
                                                    training_type, output_dir)

    num_iter = 1
    num_epoch = 100

    args['num_epoch'] = num_epoch

    args['use_fixed_features'] = False
    args['mean_requires_grad'] = True

    args['learning_rate'] = 0.00001  #Max metric score: 0.8738366961479187

    # args['mean'] = torch.rand(args['num_classes'], args['num_features'])  # value is not important, for inititalization
    # args['var'] = torch.rand(args['num_classes'], args['num_features'])  # value is not important, for inititalization

    args['mean'] = torch.zeros(
        args['num_classes'],
        args['num_features'])  # value is not important, for inititalization
    args['var'] = torch.ones(
        args['num_classes'],
        args['num_features'])  # value is not important, for inititalization

    args['class_prob'] = torch.zeros(
        args['num_classes']) + 1 / args['num_classes']

    _, args['loss_names'] = gauss_mixture_combined(args)

    change_mean_from_gauss_first_iter = True
    change_mean_from_gauss_other_iter = False

    for iter in range(num_iter):
        print('\nIteration: ', iter,
              '****************************************')
        print("Testing current model")
        test_logs = T.test(args, model_load)

        if (change_mean_from_gauss_first_iter and iter == 0):
            mean, var, class_prob = compute_means_and_var(args, model_load)

            #args['var'] = var
            args['var'] = torch.zeros(
                args['num_classes'], args['num_features']
            )  # value is not important, for inititalization

            args['mean'] = mean
            args['class_prob'] = class_prob

            print("Test after changing mean before any iterations")
            test_logs = T.test(args, model_load)

        if iter > 0 and change_mean_from_gauss_other_iter:
            mean, var, class_prob = compute_means_and_var(args, model_load)

            args['var'] = var
            args['mean'] = mean
            args['class_prob'] = class_prob

            pretrained_dict = torch.load(model_load)
            model = get_model(args)
            model_dict = model.state_dict()

            # this ensures means and variances stay as specified in args[mean] and args[var]
            del pretrained_dict['mean']
            del pretrained_dict['sigma']

            model_dict.update(pretrained_dict)
            model.load_state_dict(model_dict)
            torch.save(model.state_dict(), model_load)

            print("Test after changing mean")
            test_logs = T.test(args, model_load)

        _, args['loss_names'] = gauss_mixture_combined(args)

        T.train_normal(args, num_epoch, model_save, model_load)
        model_load = model_save
Exemple #8
0
if not args.use_labels:
    args.clf_dir = os.path.join(args.model_dir, 'saved_classifier')
    if not os.path.exists(args.clf_dir):
        os.mkdir(args.clf_dir)

# Reset graph and set seed
tf.reset_default_graph()
print("Random seed: ", args.seed)
tf.random.set_random_seed(args.seed)
np.random.seed(args.seed)

### Load datasets
train_data, val_data, test_data = get_datasets(args)

### Get model
model = get_model(args, train_data)
elbo = model.loss

t_vars = tf.trainable_variables()
with tf.variable_scope(tf.get_variable_scope(), reuse=False):
    optimizer = tf.train.AdamOptimizer(learning_rate=args.lr)
    # Minimize elbo
    grads_and_vars = optimizer.compute_gradients(-elbo, var_list=t_vars)
    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
    with tf.control_dependencies(update_ops):
        train_op = optimizer.apply_gradients(grads_and_vars)

run_single_epoch = get_training_function(args.model_name)

# Initialize the variables (i.e. assign their default value)
init = tf.global_variables_initializer()
Exemple #9
0
def main():
    parser = argparse.ArgumentParser(description='Chainer CIFAR example:')
    parser.add_argument('--model', default='c3f2')
    parser.add_argument('--batchsize', '-b', type=int, default=64)
    parser.add_argument('--learnrate', '-l', type=float, default=0.05)
    parser.add_argument('--epoch', '-e', type=int, default=300)
    parser.add_argument('--gpu', '-g', type=int, default=0)
    parser.add_argument('--N', type=int, default=9)
    parser.add_argument('--k', type=int, default=10)
    parser.add_argument('--nb_valid', type=int, default=10000)
    parser.add_argument('--seed', type=int, default=1701)
    parser.add_argument('--out', '-o', default='result')
    parser.add_argument('--debug', action='store_true')
    parser.add_argument('--test', action='store_true')
    parser.add_argument('--resume', '-r', default='')
    args = parser.parse_args()
    start = time.time()
    logger.initialize("outputs_" + args.model)
    logger.info(vars(args))
    np.random.seed(args.seed)
    save_dir = logger.get_savedir()
    logger.info("Written to {}".format(save_dir))
    logger.info('GPU: {}'.format(args.gpu))
    logger.info('# Minibatch-size: {}'.format(args.batchsize))
    logger.info('# epoch: {}'.format(args.epoch))
    logger.info('')
    train_all, test = get_cifar10()

    if args.debug:
        train, valid, test = train_all[:200], train_all[200:400], test[:200]
        batchsize = 20
    else:
        valid_choice = np.random.choice(range(len(train_all)),
                                        args.nb_valid,
                                        replace=False)
        train = [
            x for idx, x in enumerate(train_all) if idx not in valid_choice
        ]
        valid = [x for idx, x in enumerate(train_all) if idx in valid_choice]
        batchsize = args.batchsize
        #import pdb;pdb.set_trace()

    train_cnt, valid_cnt, test_cnt = len(train), len(valid), len(test)
    print(train_cnt, valid_cnt, test_cnt)

    model = get_model(args.model, args.gpu, args.resume)
    optimizer = chainer.optimizers.MomentumSGD(args.learnrate)
    optimizer.setup(model)
    optimizer.add_hook(chainer.optimizer.WeightDecay(5e-4))

    train_iter = chainer.iterators.SerialIterator(train, batchsize)
    valid_iter = chainer.iterators.SerialIterator(valid,
                                                  batchsize,
                                                  repeat=False,
                                                  shuffle=False)

    sum_accuracy = 0
    sum_loss = 0
    st = time.time()
    iter_cnt = 0
    chainer.config.train = True
    chainer.config.enable_backprop = True
    logger.info("Training...")
    while train_iter.epoch < args.epoch:
        batch = train_iter.next()
        if train_iter.epoch % 60 == 0 and train_iter.is_new_epoch:
            optimizer.lr *= 0.2
            logger.info('Reducing learning rate to: {}'.format(optimizer.lr))

        x_array, t_array = convert.concat_examples(batch, args.gpu)
        x = chainer.Variable(x_array)
        t = chainer.Variable(t_array)
        model.cleargrads()
        loss = model(x, t)
        loss.backward()
        loss.unchain_backward()
        optimizer.update()
        sum_loss += float(model.loss.data) * len(t.data)
        sum_accuracy += float(model.accuracy.data) * len(t.data)

        if train_iter.is_new_epoch:
            train_loss = sum_loss / train_cnt
            train_acc = sum_accuracy / train_cnt

            # validation
            sum_accuracy = 0
            sum_loss = 0
            chainer.config.train = False
            chainer.config.enable_backprop = False
            for batch in valid_iter:
                x_array, t_array = convert.concat_examples(batch, args.gpu)
                x = chainer.Variable(x_array)
                t = chainer.Variable(t_array)
                loss = model(x, t)
                sum_loss += float(loss.data) * len(t.data)
                sum_accuracy += float(model.accuracy.data) * len(t.data)

            valid_iter.reset()
            valid_loss = sum_loss / valid_cnt
            valid_acc = sum_accuracy / valid_cnt
            message_str = "Epoch {}: train loss={:.4f}, acc={:.4f}, valid loss={:.4f}, acc={:.4f}, elapsed={}"
            logger.info(
                message_str.format(train_iter.epoch, train_loss, train_acc,
                                   valid_loss, valid_acc,
                                   time.time() - st))
            st = time.time()
            chainer.config.train = True
            chainer.config.enable_backprop = True
            sum_accuracy = 0
            sum_loss = 0
            if not args.debug:
                serializers.save_npz(
                    os.path.join(
                        save_dir,
                        "model_holdout_ep_{}.npz".format(train_iter.epoch)),
                    model)
        iter_cnt += 1

    if not test:
        print(time.time() - start)
        exit(1)

    logger.info("Test...")
    test_iter = chainer.iterators.SerialIterator(test,
                                                 batchsize,
                                                 repeat=False,
                                                 shuffle=False)
    sum_accuracy = 0
    sum_loss = 0
    chainer.config.train = False
    chainer.config.enable_backprop = False
    st = time.time()
    for batch in test_iter:
        x_array, t_array = convert.concat_examples(batch, args.gpu)
        x = chainer.Variable(x_array)
        t = chainer.Variable(t_array)
        loss = model(x, t)
        sum_loss += float(loss.data) * len(t.data)
        sum_accuracy += float(model.accuracy.data) * len(t.data)
    test_loss = sum_loss / test_cnt
    test_acc = sum_accuracy / test_cnt
    message_str = "test loss={:.4f}, acc={:.4f}, elapsed={}"
    logger.info(message_str.format(test_loss, test_acc, time.time() - st))
    print(time.time() - start)