예제 #1
0
def main(): 
    params = {'model': 'lenet',
          'display_step': 250, # we decreased this value for HW7
          'batch_size': 1024,
          'iterations': 30_000, # 5000 or 30000 
          'initial_lr': 0.05, # we decreased this value for HW7 
          'lr_decay': 0.5,
          'adjust_lr_step': 10_000,
          'initial_momentum': 0.5,
          'final_momentum': 0.95,
          'momentum_change_steps': 20_000,
          'adjust_momentum_step': 2_000,
          'apply_weight_norm': True,
          'weight_norm': 3.5,
          'adjust_norm_step': 5_000,
          'output_l2_decay': 0.001,
          'pooling': 'max', # TODO try three options: 'max', 'avg','no' 
          'activation':'relu',
          'random_seed': 42}
    # we increased n_train for HW7 
    data = prepare_MNIST_data(n_train=20_000, n_val=10_000, n_test=5_000,
                              use_norm_shift=False, use_norm_scale=True, use_data_augmentation=False)
    X_train, y_train, X_val, y_val, _, _ = data.values()
    insample_dataloader = get_dataloader(X_train, y_train, batch_size=params['batch_size'])
    outsample_dataloader = get_dataloader(X_val, y_val, batch_size=params['batch_size'])
    clf = LeNet(params)
    clf.fit(insample_dataloader, outsample_dataloader)
예제 #2
0
def train(config):
    seed = config["seed"]
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)

    trainTransform, testTransform = data.get_transforms(config)
    trainLoader = data.get_dataloader(config,
                                      train=True,
                                      transform=trainTransform)
    testLoader = data.get_dataloader(config,
                                     train=False,
                                     transform=testTransform)

    densenet = model.get_model(config).to(config["device"])
    opt, scheduler = optimizer.get_scheduled_optimizer(config, densenet)
    criterion = CrossEntropyLoss()
    recorder = Recorder(config)

    max_epoch = config["max_epoch"]
    for epoch in range(max_epoch):
        print("epoch:{:0>3}".format(epoch))
        train_out = _train(densenet, criterion, trainLoader, config["device"],
                           opt)
        recorder(epoch, train_out=train_out)
        test_out = _test(densenet,
                         criterion,
                         testLoader,
                         config["device"],
                         need_output_y=False)
        recorder(epoch, test_out=test_out)
        scheduler.step()
        torch.save(densenet, os.path.join(config["save_dir"], "latest.pth"))
    print("train finished")
    return None
예제 #3
0
def main():
    data = prepare_MNIST_data(n_train=10_000, n_val=10_000, n_test=5_000,
                              use_norm_shift=False, use_norm_scale=True, use_data_augmentation=False)

    X_train, y_train, X_val, y_val, _, _ = data.values()

    insample_dataloader = get_dataloader(X_train, y_train, batch_size=params['batch_size'])
    outsample_dataloader = get_dataloader(X_val, y_val, batch_size=params['batch_size'])

    clf = DropoutClassifier(params)
    clf.fit(insample_dataloader, outsample_dataloader)
예제 #4
0
def validate_dataset(model, split, tokenizer, dataset, topk=1):
    assert split in ('dev', 'test')
    fwd_dataloader = get_dataloader('xlnet', split, tokenizer, bwd=False, \
                        batch_size=16, num_workers=16, prefix=dataset)
    bwd_dataloader = get_dataloader('xlnet', split, tokenizer, bwd=True, \
                        batch_size=16, num_workers=16, prefix=dataset)
    em, f1, count = 0, 0, 0
    
    model.start_n_top = topk
    model.end_n_top = topk
    model.eval()
    for fwd_batch, bwd_batch in zip(fwd_dataloader, bwd_dataloader):
        # Forward
        input_ids, attention_mask, token_type_ids, fwd_input_tokens_no_unk, answers = fwd_batch
        input_ids = input_ids.cuda(device=device)
        attention_mask = attention_mask.cuda(device=device)
        token_type_ids = token_type_ids.cuda(device=device)
        with torch.no_grad():
            outputs = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
        fwd_start_index = outputs[1]
        fwd_end_index = outputs[3].view(-1, model.end_n_top, model.start_n_top).permute([0,2,1])[:,:,0]
        
        # Backward
        input_ids, attention_mask, token_type_ids, bwd_input_tokens_no_unk, answers = bwd_batch
        input_ids = input_ids.cuda(device=device)
        attention_mask = attention_mask.cuda(device=device)
        token_type_ids = token_type_ids.cuda(device=device)
        with torch.no_grad():
            outputs = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
        bwd_start_index = outputs[1]
        bwd_end_index = outputs[3].view(-1, model.end_n_top, model.start_n_top).permute([0,2,1])[:,:,0] + 1
        
        for i, answer in enumerate(answers):
            preds = []
            for k in range(model.start_n_top):
                fwd_pred_tokens = fwd_input_tokens_no_unk[i][fwd_start_index[i][k]:fwd_end_index[i][k]]
                bwd_pred_tokens = bwd_input_tokens_no_unk[i][bwd_start_index[i][k]:bwd_end_index[i][k]]
                preds.append(tokenizer.convert_tokens_to_string(fwd_pred_tokens))
                preds.append(tokenizer.convert_tokens_to_string(bwd_pred_tokens))

            norm_preds_tokens = [norm_tokenizer.basic_tokenizer.tokenize(pred) for pred in preds]
            norm_preds = [norm_tokenizer.convert_tokens_to_string(norm_pred_tokens) for norm_pred_tokens in norm_preds_tokens]
            norm_answer_tokens = [norm_tokenizer.basic_tokenizer.tokenize(ans) for ans in answer]
            norm_answer = [norm_tokenizer.convert_tokens_to_string(ans_tokens) for ans_tokens in norm_answer_tokens]

            em += max(metric_max_over_ground_truths(exact_match_score, norm_pred, norm_answer) for norm_pred in norm_preds)
            f1 += max(metric_max_over_ground_truths(f1_score, norm_pred, norm_answer) for norm_pred in norm_preds)
            count += 1
    del fwd_dataloader, bwd_dataloader
    return em, f1, count
예제 #5
0
def main():
    data = prepare_MNIST_data(n_train=10_000, n_val=10_000, n_test=5_000,
                              use_norm_shift=False, use_norm_scale=True, use_data_augmentation=False)

    X_train, y_train, X_val, y_val, _, _ = data.values()

    insample_dataloader = get_dataloader(X_train, y_train, batch_size=params['batch_size'])
    outsample_dataloader = get_dataloader(X_val, y_val, batch_size=params['batch_size'])

    clf = DropoutClassifier(params)
    clf.fit(insample_dataloader, outsample_dataloader)

    # To avoid unnecesary pain, we recommend you to save your classifiers
    clf.save_weights('./your_results_path/')
    clf.load_weights('./your_results_path/')
예제 #6
0
def test_simple_cnn_model():
    args = update_args(cfg_file='simple_cnn')
    device = 'cpu'

    train_data_dict = get_dataset(args).train_data_dict

    train_dataloader = get_dataloader(
        batch_size=args.TRAIN.BATCH_SIZE,
        dataset=train_data_dict['dataset'],
        num_workers=args.DATA.NUM_WORKERS,
        sampler=train_data_dict['train_sampler'],
    )

    validation_dataloader = get_dataloader(
        batch_size=args.TRAIN.BATCH_SIZE,
        dataset=train_data_dict['dataset'],
        num_workers=args.DATA.NUM_WORKERS,
        sampler=train_data_dict['validation_sampler'],
    )

    model = get_model(
        args=args,
        device=device,
        hparams={
            'learning rate': args.TRAIN.LR,
            'batch size': args.TRAIN.BATCH_SIZE,
        },
    )

    for data, labels in train_dataloader:
        data, labels = data.to(device), labels.to(device)
        outputs = model(data)
        assert data.shape[0] == labels.shape[0] == outputs.shape[
            0] == args.TRAIN.BATCH_SIZE
        assert data.shape[1] == 3
        assert outputs.shape[1] == len(args.DATA.CLASSES)

        break

    for data, labels in validation_dataloader:
        data, labels = data.to(device), labels.to(device)
        outputs = model(data)
        assert data.shape[0] == labels.shape[0] == outputs.shape[
            0] == args.TRAIN.BATCH_SIZE
        assert data.shape[1] == 3
        assert outputs.shape[1] == len(args.DATA.CLASSES)

        break
예제 #7
0
def task1():
    task_name = "base_dpn62_balance"
    makedir(os.path.join(cfg.log_dir, task_name))
    log = Logger(os.path.join(cfg.log_dir, task_name + '_log.txt'), mode="a")
    log("\n\n" + '-' * 51 +
        "[START %s]" % datetime.now().strftime('%Y-%m-%d %H:%M:%S') +
        "-" * 51 + "\n\n")
    print(cfg, file=log)
    train_loader, val_loader, test_loader = get_dataloader()
    model = DPN68()
    model.cuda()
    criterion1 = nn.BCEWithLogitsLoss().cuda()
    criterion2 = BalanceLoss().cuda()
    criterions = [criterion1, criterion2]
    optimizer = optim.SGD(model.parameters(),
                          lr=cfg.lr,
                          momentum=0.9,
                          weight_decay=1e-5)
    scheduler = lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)
    model = train(task_name,
                  model,
                  optimizer,
                  criterions,
                  scheduler,
                  train_loader,
                  val_loader,
                  log=log)
    submission_best_f1(task_name, model, test_loader, log=log)
예제 #8
0
def train_and_eval(cfg):
    train_loader, val_loader, test_loader, num_classes = get_dataloader(
        cfg.data.name, cfg.data.val_size, cfg.data.batch_size,
        cfg.data.download, cfg.augment, False)
    model = get_model(cfg.model.name, num_classes)
    optimizer = optim.SGD(cfg.optim.model.lr,
                          momentum=0.9,
                          weight_decay=cfg.optim.model.weight_decay)
    scheduler = lr_scheduler.MultiStepLR(cfg.optim.model.steps)
    tq = reporters.TQDMReporter(range(cfg.optim.epochs), verb=cfg.verb)
    callback = [
        callbacks.AccuracyCallback(),
        callbacks.LossCallback(),
        reporters.TensorboardReporter("."),
        reporters.IOReporter("."), tq
    ]

    with trainers.SupervisedTrainer(model,
                                    optimizer,
                                    F.cross_entropy,
                                    callbacks=callback,
                                    scheduler=scheduler) as trainer:
        for ep in tq:
            trainer.train(train_loader)
            trainer.test(val_loader, 'val')
            trainer.test(test_loader)
예제 #9
0
def train(args):
    #torch.manual_seed(args.seed)
    #torch.cuda.manual_seed(args.seed)

    dataset = data.get_dataset(args.dataset, training=True)
    model = DFVE(args.image_channels, args.image_size, args.n_latent, args.lambda_, args.gamma).to(args.device)
    optimizer = optim.Adam(model.parameters(),
                           lr=args.learning_rate,
                           betas=(args.adam_beta1, args.adam_beta2),
                           weight_decay=args.weight_decay)

    model.train()
    step = 0
    epoch = 0
    for _ in range(args.n_epochs):
        epoch += 1
        loader, _ = data.get_dataloader(dataset, args.batch_size)
        for samples, labels in loader:
            step += 1
            x = samples.to(args.device).float()
            z_mean, z_logvar = model(x)
            loss, mmd_loss_from_prior, mmd_loss_for_mi = model.loss(z_mean, z_logvar, args.repeats)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            if step % args.print_freq == 0:
                print('[Epoch {:d}, Step {:d}] loss: {:.4f}, mmd_loss_from_prior: {:.4f}, mmd_loss_for_mi: {:.4f}'.format(
                    epoch, step, loss.item(), mmd_loss_from_prior.item(), mmd_loss_for_mi.item()))

                monitor(z_mean, z_logvar, labels, epoch, step)
        model.monitor()
예제 #10
0
def train(args):
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)

    dataset = data.get_dataset(args.dataset, training=True)
    model = DFVE(args.image_channels, args.image_size, args.n_latent).to(args.device)
    optimizer = optim.Adam(model.parameters(),
                           lr=args.learning_rate,
                           betas=(args.adam_beta1, args.adam_beta2),
                           weight_decay=args.weight_decay)

    model.train()
    step = 0
    epoch = 0
    for _ in range(args.n_epochs):
        epoch += 1
        loader, _ = data.get_dataloader(dataset, args.batch_size)
        for samples, labels in loader:
            step += 1
            x = samples.to(args.device).float()
            z = model(x, args.repeats, args.noise_sigma)
            loss, mmd_loss_all, mmd_loss_avg = model.loss(z, args.gamma, args.kernel_gamma, args.kernel_power,
                                                          args.gnorm_mu, args.gnorm_sigma, args.gnorm_alpha)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            if step % args.print_freq == 0:
                print('[Epoch {:d}, Step {:d}] loss: {:.4f}, mmd_loss_all: {:.4f}, mmd_loss_avg: {:.4f}'.format(
                    epoch, step, loss.item(), mmd_loss_all.item(), mmd_loss_avg.item()))

            if step % args.show_freq == 0:
                monitor(z, labels, epoch, step)
                model.monitor()
예제 #11
0
def test():
    """
    train the model by the args
    :param args:
    :return:
    """
    # initialize model
    model = EfficientSeg(data_cfg.num_classes, compound_coef=cfg.compound_coef)
    load_state_dict(model)
    model = model.to(device)

    # test model
    model.eval()
    transforms = CommonTransforms(trans_cfg, "val")

    decode.device = device
    if data_cfg.test_dir is not None:
        # initialize the dataloader by dir
        test_dataloader = data.get_dataloader(data_cfg.batch_size,
                                              data_cfg.dataset,
                                              data_cfg.test_dir,
                                              with_label=False,
                                              phase="test",
                                              transforms=transforms)
        # foreach the images
        for iter_id, test_data in enumerate(test_dataloader):
            # to device
            inputs, infos = test_data
            handle_output(inputs, infos, model, transforms)
    else:
        img_path = data_cfg.test_image
        input_img = image.load_rgb_image(img_path)
        input, _, info = transforms(input_img, img_path=img_path)
        handle_output(input.unsqueeze(0), [info], model, transforms)
    logger.close()
예제 #12
0
def train(name, gen, disc, gen_train_ratio=5, epochs=-1):
    device = utils.get_device()
    dataloader = data.get_dataloader()

    loss_func = nn.BCELoss()
    g_optimizer = to.Adam(gen.parameters(), lr=.0003, betas=(.5, .9))
    d_optimizer = to.Adam(disc.parameters(), lr=.0003, betas=(.5, .9))

    iter_axis = []
    g_loss_axis = []
    d_loss_axis = []

    fixed_noise = torch.randn(64, gen.nz, device=device)
    epoch_iterator = range(epochs) if epochs >= 0 else count(0)
    iters = 0
    for epoch in epoch_iterator:
        for i, batch in enumerate(dataloader):
            loss_disc, acc_real, acc_fake = train_discriminator(
                gen, disc, batch, loss_func, d_optimizer)

            for _ in range(gen_train_ratio):
                loss_gen, acc_gen = train_generator(gen, disc, loss_func,
                                                    g_optimizer)

            # Training stats
            if i % 50 == 0:
                print(
                    f'[{epoch:2d}/{epochs}][{i:3d}/{len(dataloader)}]\t' +\
                     f'Loss_D: {loss_disc:3.4f}\tLoss_G: {loss_gen:3.4f}\t' +\
                     f'D(x): {acc_real:3.4f}\tD(G(z)): {acc_fake:3.4f} / {acc_gen:3.4f}'
                )

            if (iters % 10 == 0) or ((epoch == epochs - 1) and
                                     (i == len(dataloader) - 1)):
                iter_axis.append(iters)
                g_loss_axis.append(loss_gen)
                d_loss_axis.append(loss_disc)

            # Save output
            if (iters % 500 == 0) or ((epoch == epochs - 1) and
                                      (i == len(dataloader) - 1)):
                print('Saving images...')
                with torch.no_grad():
                    gen.mixing = False
                    fake = gen(fixed_noise).detach().cpu()
                    gen.mixing = True
                    utils.save_image(utils.make_torch_grid(fake),
                                     f'{name}_{iters}.png')

            # Save weights
            if (iters % 1000 == 0) or ((epoch == epochs - 1) and
                                       (i == len(dataloader) - 1)):
                print('Saving weights...')
                utils.save_weights(gen, f'{name}_gen.pt')
                utils.save_weights(disc, f'{name}_disc.pt')

            iters += 1

    utils.plot_losses(iter_axis, g_loss_axis, d_loss_axis)
예제 #13
0
def train_encoding(args):
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)

    tag = 'latents_{:d}_alpha_{:d}'.format(args.n_latent,
                                           int(args.gnorm_alpha))
    save_dir = os.path.join(args.save_base, tag)
    U.mkdir(save_dir)

    dataset = data.get_dataset(args.dataset, training=True)
    model = Model(args.image_channels, args.image_size, args.n_latent,
                  args.n_dims).to(args.device)
    optimizer = optim.Adam(model.encoder.parameters(),
                           lr=args.learning_rate,
                           betas=(args.adam_beta1, args.adam_beta2),
                           weight_decay=args.weight_decay)

    model.train()
    step = 0
    epoch = 0
    examples = 0
    while examples < args.max_examples:
        epoch += 1
        loader, _ = data.get_dataloader(dataset, args.batch_size)
        for samples, labels in loader:
            step += 1
            x = samples.to(args.device).float()  # B x C x H x W
            z = model(x, args.repeats, args.noise_sigma,
                      'encoding')  # B x repeats x n_latent
            loss, mmd_loss_all, mmd_loss_avg = model.encoding_loss(
                z, args.gamma, args.kernel_gamma, args.kernel_power,
                args.gnorm_mu, args.gnorm_sigma, args.gnorm_alpha)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            prev_examples = examples
            examples += x.size(0)

            if examples // BASE_N > prev_examples // BASE_N:
                print(
                    '[Epoch {:d}, Step {:d}, #Eg. {:d}] loss: {:.4f}, mmd_loss_all: {:.4f}, mmd_loss_avg: {:.4f}'
                    .format(epoch, step, examples, loss.item(),
                            mmd_loss_all.item(), mmd_loss_avg.item()))
                if examples // BASE_N in args.save_points:
                    path = os.path.join(
                        save_dir, 'training_examples_{:d}_10k.ckpt'.format(
                            examples // BASE_N))
                    print('save {}'.format(path))
                    torch.save(
                        {
                            'examples': examples // BASE_N * BASE_N,
                            'loss': loss.item(),
                            'mmd_loss_all': mmd_loss_all.item(),
                            'mmd_loss_avg': mmd_loss_avg.item(),
                            'model_state_dict': model.state_dict(),
                            'optimizer_state_dict': optimizer.state_dict()
                        }, path)
예제 #14
0
    def __init__(self, cfg):
        self.cfg = cfg
        self.model = build_model(cfg, 0)
        self.tng_dataloader, self.val_dataloader, self.num_classes, self.num_query = get_dataloader(
            cfg)
        self.model = self.model.cuda()
        self.model.load_params_wo_fc(torch.load(cfg.TEST.WEIGHT))

        self.get_distmat()
예제 #15
0
def get_dataloaders(args, config):
    if args.dataset == 'mnist':
        train_loader, test_loader = get_dataloader(args)
        retrain_loader, _ = get_dataloader(
            args, no_randomness=args.no_random_trainloaders)
    elif args.dataset.lower()[0:7] == 'cifar10':
        assert config is not None
        args.cifar_init_lr = config['optimizer_learning_rate']
        if args.second_model_name is not None:
            assert second_config is not None
            assert args.cifar_init_lr == second_config[
                'optimizer_learning_rate']
            # also the below things should be fine as it is just dataloader loading!
        print('loading {} dataloaders'.format(args.dataset.lower()))
        train_loader, test_loader = cifar_train.get_dataset(config)
        retrain_loader, _ = cifar_train.get_dataset(
            config, no_randomness=args.no_random_trainloaders)

    return train_loader, test_loader, retrain_loader
예제 #16
0
def train(args):
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)

    tag = 'vae_latents_{:d}'.format(args.n_latent)
    save_dir = os.path.join(args.save_base, tag)
    U.mkdir(save_dir)

    dataset = data.get_dataset(args.dataset, training=True)
    model = Model(args.image_channels, args.image_size, args.n_latent,
                  args.n_dims).to(args.device)
    optimizer = optim.Adam(model.parameters(),
                           lr=args.learning_rate,
                           betas=(args.adam_beta1, args.adam_beta2))

    model.train()
    step = 0
    epoch = 0
    examples = 0
    while examples < args.max_examples:
        epoch += 1
        loader, _ = data.get_dataloader(dataset, args.batch_size)
        for samples, labels in loader:
            step += 1
            x = samples.to(args.device).float()  # B x C x H x W
            z_mean, z_logvar, z, recon_x = model(x, args.repeats)
            loss, recon_loss, kl_loss = model.loss(x, z_mean, z_logvar, z,
                                                   recon_x)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            prev_examples = examples
            examples += x.size(0)

            if examples // BASE_N > prev_examples // BASE_N:
                print(
                    '[Epoch {:d}, Step {:d}, #Eg. {:d}] loss: {:.8f}, recon_loss: {:.8f}, kl_loss: {:.8f}'
                    .format(epoch, step, examples, loss.item(),
                            recon_loss.item(), kl_loss.item()))
                if examples // BASE_N in args.save_points:
                    path = os.path.join(
                        save_dir, 'training_examples_{:d}_10k.ckpt'.format(
                            examples // BASE_N))
                    print('save {}'.format(path))
                    torch.save(
                        {
                            'examples': examples // BASE_N * BASE_N,
                            'loss': loss.item(),
                            'recon_loss': recon_loss.item(),
                            'kl_loss': kl_loss.item(),
                            'model_state_dict': model.state_dict(),
                            'optimizer_state_dict': optimizer.state_dict()
                        }, path)
예제 #17
0
    def __init__(self, model: nn.Module, train_set: SentiDataset, valid_set: SentiDataset, test_set: SentiDataset,
                 batch_size: int, optim_cls: Callable[..., Optimizer], lr: float):
        """
        Learner for training binary sentiment analysis models.

        Args:
            model: Model.
            train_set: Training dataset.
            valid_set: Validation dataset.
            test_set: Testing dataset.
            batch_size: Batch size.
            optim_cls: Optimizer class.
            lr: Learning rate.
        """
        self.model = model.to(self.device)
        self.train_loader = get_dataloader(train_set, batch_size=batch_size)
        self.valid_loader = get_dataloader(valid_set, batch_size=batch_size)
        self.test_loader = get_dataloader(test_set, batch_size=batch_size)
        self.loss_fn = nn.BCEWithLogitsLoss()
        self.optimizer = optim_cls(self.model.parameters(), lr=lr)
예제 #18
0
    def __init__(self, args):
        super(Tester, self).__init__()
        self.args = args

        # dataloader
        self.dataloader = get_dataloader(self.args)

        # model
        self.device = torch.device(
            'cuda:0' if torch.cuda.is_available else 'cpu')
        self.model = LstmPuncModel(self.args)
예제 #19
0
    def execute(self, eval: bool) -> None:
        """Execution

        Execute train or eval.

        Args:
            eval: For evaluation mode.
                True: Execute eval.
                False: Execute train.

        """

        if not eval:
            self.train_dataloader, self.val_dataloader = get_dataloader(
                self.cfg, mode="trainval")
            self.train()

        else:
            self.test_dataloader = get_dataloader(self.cfg, mode="test")
            self.eval()
예제 #20
0
def train_encoding(args):
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)

    dataset = data.get_dataset(args.dataset, training=True)
    model = Model(args.image_channels, args.image_size, args.n_latent,
                  args.n_dims).to(args.device)
    optimizer = optim.Adam(model.encoder.parameters(),
                           lr=args.learning_rate,
                           betas=(args.adam_beta1, args.adam_beta2),
                           weight_decay=args.weight_decay)

    line_plotter = analysis.LinePlotter(
        title=
        'Learning Curve (In terms of averaged standard deviation of z given x)',
        legend=['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'All'],
        env='train_mnist')
    std_dict, count, examples = None, 0, 0

    model.train()
    step = 0
    epoch = 0
    for _ in range(args.n_epochs):
        epoch += 1
        loader, _ = data.get_dataloader(dataset, args.batch_size)
        for samples, labels in loader:
            step += 1
            x = samples.to(args.device).float()  # B x C x H x W
            z = model(x, args.repeats, args.noise_sigma,
                      'encoding')  # B x repeats x n_latent
            loss, mmd_loss_all, mmd_loss_avg = model.encoding_loss(
                z, args.gamma, args.kernel_gamma, args.kernel_power,
                args.gnorm_mu, args.gnorm_sigma, args.gnorm_alpha)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            std_dict, count = analysis.stats_avg_per_x_std(
                U.numpy(z),
                U.numpy(labels).astype(np.int), std_dict, count)
            examples += x.size(0)

            if step % args.print_freq == 0:
                print(
                    '[Epoch {:d}, Step {:d}] loss: {:.4f}, mmd_loss_all: {:.4f}, mmd_loss_avg: {:.4f}'
                    .format(epoch, step, loss.item(), mmd_loss_all.item(),
                            mmd_loss_avg.item()))
                line_plotter.append(
                    examples,
                    [std_dict[l] / count for l in line_plotter.legend])

        if epoch % args.show_freq == 0:
            monitor(z, labels, epoch, step)
예제 #21
0
def train(cfg, local_rank):
    # prepare dataset
    tng_loader, val_loader, num_classes, num_query = get_dataloader(cfg)

    do_train(
        cfg,
        local_rank,
        tng_loader,
        val_loader,
        num_classes,
        num_query,
    )
예제 #22
0
def train():
    """
    train the model by the args
    :return:
    """
    # initialize the dataloader by dir
    train_transforms = CommonTransforms(trans_cfg, "train")
    train_dataloader = data.get_dataloader(data_cfg.batch_size, data_cfg.dataset, data_cfg.train_dir,
                                           phase="train", transforms=train_transforms)

    eval_transforms = CommonTransforms(trans_cfg, "val")
    eval_dataloader = data.get_dataloader(data_cfg.batch_size, data_cfg.dataset, data_cfg.train_dir,
                                           phase="val", transforms=eval_transforms)

    # initialize model, optimizer, loss_fn
    model = EfficientSeg(data_cfg.num_classes, compound_coef=cfg.compound_coef,
                                 ratios=eval(cfg.anchors_ratios), scales=eval(cfg.anchors_scales))

    start_epoch, best_ap = load_state_dict(model, data_cfg.save_dir, cfg.pretrained_path)
    model = model.to(device)
    optimizer = get_optimizer(model, opt_cfg)
    loss_fn = ComposeLoss(device)

    # train model
    # foreach epoch
    for epoch in range(start_epoch, cfg.num_epochs):
        # each epoch includes two phase: train,val
        train_loss, train_loss_states = train_model_for_epoch(model, train_dataloader, loss_fn, optimizer, epoch)
        write_metric(train_loss_states, epoch, "train")
        executor.submit(save_checkpoint, model.state_dict(), epoch, best_ap, data_cfg.save_dir)

        if epoch >= cfg.start_eval_epoch:
            epoch, mAP, eval_results = evaluate_model(data_cfg, eval_dataloader, eval_transforms, model, epoch, data_cfg.dataset, decode_cfg, device, logger)
            # judge the model. if model is greater than current best loss
            if best_ap < mAP:
                best_ap = mAP
    logger.write("the best mAP:{}".format(best_ap))
    logger.close()
    executor.shutdown(wait=True)
예제 #23
0
파일: main.py 프로젝트: peterhan91/what
def main(config):
    config.device = torch.device(
        'cuda:{}'.format(config.gpu) if torch.cuda.is_available() else 'cpu')

    # load data_loader
    data_loader = get_dataloader(config)
    check_point = Checkpoint(config)
    operator = Operator(config, check_point)

    if config.is_train:
        operator.train(data_loader)
    else:
        operator.test(data_loader)
예제 #24
0
def main(model, epochs, learning_rate, upscale_factor, save_period,
         session_id):
    model = models.get_model(model,
                             upscale_factor=upscale_factor,
                             batch_norms=False)
    dataloader = data.get_dataloader(upscale_factor=upscale_factor)
    optimizer = optim.Adam(model.parameters(), lr=learning_rate)
    crit = nn.L1Loss()
    trainer = create_supervised_trainer(model, optimizer, crit)
    trainer.attach(ProgressBar(persist=False))
    trainer.attach(
        utils.tensorboard_logger.TensorboardLogger(session_id=session_id))
    trainer.attach(utils.saver.NetSaver(save_period, session_id))
    _ = trainer.run(dataloader, max_epochs=epochs)
예제 #25
0
def test(config, testLoader=None):
    # check if there's weight in work/* (* means the name of setting) before run test.
    if testLoader is None:
        trainTransform, testTransform = data.get_transforms(config)
        testLoader = data.get_dataloader(config,
                                         train=False,
                                         transform=testTransform)

    densenet = model.get_model(config)
    densenet.load_state_dict(os.path.join(config["save_dir"], "latest.pth"))
    criterion = CrossEntropyLoss()
    test_out = _test(densenet, criterion, testLoader, need_output_y=True)
    print("test finished")
    return test_out
예제 #26
0
def train(args):
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)
    np.random.seed(args.seed)

    loader, ds = data.get_dataloader(args.dataset,
                                     args.batch_size,
                                     max_steps=args.max_steps)
    ds_sampler = data.DspritesFullSampler(ds.dataset)

    model = DFVE(args.image_channels, args.n_latent, args.gamma,
                 args.kernel_gamma).to(args.device)
    optimizer = optim.Adam(model.parameters(),
                           lr=args.learning_rate,
                           betas=(args.adam_beta1, args.adam_beta2),
                           weight_decay=args.weight_decay)

    model.train()
    step = 0
    for batch in loader:
        step += 1
        _, samples = batch
        x = samples.to(args.device).float()
        z_mean, z_std = model(x)
        loss, mmd_loss, kl_loss = model.loss(z_mean, z_std, args.c)

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if step % args.print_freq == 0:
            print(
                '[Step {:d}] loss: {:.4f}, mmd_loss: {:.4f}, kl_loss: {:.4f}'.
                format(step, loss.item(), mmd_loss.item(), kl_loss.item()))
            print('  [z_mean (min)] {}'.format(
                z_mean.min(0)[0].detach().cpu().numpy()))
            print('  [z_mean (max)] {}'.format(
                z_mean.max(0)[0].detach().cpu().numpy()))
            print('  [z_mean (mean)] {}'.format(
                z_mean.mean(0).detach().cpu().numpy()))
            print('  [z_std (min)] {}'.format(
                z_std.min(0)[0].detach().cpu().numpy()))
            print('  [z_std (max)] {}'.format(
                z_std.max(0)[0].detach().cpu().numpy()))
            print('  [z_std (mean)] {}'.format(
                z_std.mean(0).detach().cpu().numpy()))

        if step % args.show_freq == 0:
            with torch.no_grad():
                show(ds_sampler, model, step)
예제 #27
0
def retrain_models(args, old_networks, train_loader, test_loader, config, tensorboard_obj=None, initial_acc=None, nicks=None):
    accuracies = []
    retrained_networks = []
    # nicks = []

    # assert len(old_networks) >= 4

    for i in range(len(old_networks)):
        nick = nicks[i]
        # if i == len(old_networks) - 1:
        #     nick = 'naive_averaging'
        # elif i == len(old_networks) - 2:
        #     nick = 'geometric'
        # else:
        #     nick = 'model_' + str(i)
        # nicks.append(nick)
        print("Retraining model : ", nick)

        if initial_acc is not None:
            start_acc = initial_acc[i]
        else:
            start_acc = -1
        if args.dataset.lower()[0:7] == 'cifar10':

            if args.reinit_trainloaders:
                print('reiniting trainloader')
                retrain_loader, _ = cifar_train.get_dataset(config, no_randomness=args.no_random_trainloaders)
            else:
                retrain_loader = train_loader

            output_root_dir = "{}/{}_models_ensembled/".format(args.baseroot, (args.dataset).lower())
            output_root_dir = os.path.join(output_root_dir, args.exp_name, nick)
            os.makedirs(output_root_dir, exist_ok=True)

            retrained_network, acc = cifar_train.get_retrained_model(args, retrain_loader, test_loader, old_networks[i], config, output_root_dir, tensorboard_obj=tensorboard_obj, nick=nick, start_acc=initial_acc[i])
            
        elif args.dataset.lower() == 'mnist':

            if args.reinit_trainloaders:
                print('reiniting trainloader')
                retrain_loader, _ = get_dataloader(args, no_randomness=args.no_random_trainloaders)
            else:
                retrain_loader = train_loader
                
            start_acc = initial_acc[i]
            retrained_network, acc = get_retrained_model(args, retrain_loader, test_loader, old_network=old_networks[i], tensorboard_obj=tensorboard_obj, nick=nick, start_acc=start_acc, retrain_seed=args.retrain_seed)
        retrained_networks.append(retrained_network)
        accuracies.append(acc)
    return retrained_networks, accuracies
예제 #28
0
    def get_data(self, mode, size=None):
        "Allow for custom mode and dataset size"
        size = self.config.dataset_size if size is None else size

        if self.config.preprocess_device == 'cpu':
            dataset = get_dataset(self.config,
                                  mode,
                                  size,
                                  preprocess=self.preprocess)
        else:
            dataset = get_dataset(self.config, mode, size)

        print(f"Dataset Size: {len(dataset)}")
        dataloader = get_dataloader(self.config, dataset)
        return dataloader
예제 #29
0
def test_get_residual():
    rain_path = 'E:\All_My_Files\All_My_DataSets\derain_datasets\Derain_ML_Proj\\final_testset\\rainy_image'
    gt_path = 'E:\All_My_Files\All_My_DataSets\derain_datasets\Derain_ML_Proj\\final_testset\ground_truth'
    batch = 3
    dataloader = iter(
        data.get_dataloader(gt_path=gt_path,
                            rain_path=rain_path,
                            batchSize=batch))
    gt_image, rain_image = next(dataloader)
    print(gt_image.size())
    print(rain_image.size())

    residual = get_residual_tensor(gt_image, rain_image)
    print(residual.size())
    save_image_from_tensor(residual, './test')
예제 #30
0
    def __init__(self, cfg, logger, writer):
        self.cfg, self.logger, self.writer = cfg, logger, writer
        # Define dataloader
        self.tng_dataloader, self.val_dataloader, self.num_classes, self.num_query = get_dataloader(
            cfg)
        # networks
        self.model = build_model(cfg, self.num_classes)
        # loss function
        self.ce_loss = nn.CrossEntropyLoss()
        self.triplet = TripletLoss(cfg.SOLVER.MARGIN)
        # optimizer and scheduler
        self.opt = make_optimizer(self.cfg, self.model)
        self.lr_sched = make_lr_scheduler(self.cfg, self.opt)

        self._construct()
예제 #31
0
def train(**kwargs):
    opt = Config()
    for k, v in kwargs.items():
        setattr(opt, k, v)
    device=t.device('cuda') if opt.use_gpu else t.device('cpu')

    opt.caption_data_path = 'caption.pth'  # 原始数据
    opt.test_img = ''  # 输入图片
    # opt.model_ckpt='caption_0914_1947' # 预训练的模型

    # 数据
    vis = Visualizer(env=opt.env)
    dataloader = get_dataloader(opt)
    _data = dataloader.dataset._data
    word2ix, ix2word = _data['word2ix'], _data['ix2word']

    # 模型
    model = CaptionModel(opt, word2ix, ix2word)
    if opt.model_ckpt:
        model.load(opt.model_ckpt)
    optimizer = model.get_optimizer(opt.lr)
    criterion = t.nn.CrossEntropyLoss()
   
    model.to(device)

    # 统计
    loss_meter = meter.AverageValueMeter()

    for epoch in range(opt.epoch):
        loss_meter.reset()
        for ii, (imgs, (captions, lengths), indexes) in tqdm.tqdm(enumerate(dataloader)):
            # 训练
            optimizer.zero_grad()
            imgs = imgs.to(device)
            captions = captions.to(device)
            input_captions = captions[:-1]
            target_captions = pack_padded_sequence(captions, lengths)[0]
            score, _ = model(imgs, input_captions, lengths)
            loss = criterion(score, target_captions)
            loss.backward()
            optimizer.step()
            loss_meter.add(loss.item())

            # 可视化
            if (ii + 1) % opt.plot_every == 0:
                if os.path.exists(opt.debug_file):
                    ipdb.set_trace()

                vis.plot('loss', loss_meter.value()[0])

                # 可视化原始图片 + 可视化人工的描述语句
                raw_img = _data['ix2id'][indexes[0]]
                img_path = opt.img_path + raw_img
                raw_img = Image.open(img_path).convert('RGB')
                raw_img = tv.transforms.ToTensor()(raw_img)

                raw_caption = captions.data[:, 0]
                raw_caption = ''.join([_data['ix2word'][ii] for ii in raw_caption])
                vis.text(raw_caption, u'raw_caption')
                vis.img('raw', raw_img, caption=raw_caption)

                # 可视化网络生成的描述语句
                results = model.generate(imgs.data[0])
                vis.text('</br>'.join(results), u'caption')
        model.save()