Exemplo n.º 1
0
class Trainer(OptunaMlFlow):
    def __init__(self, args, search_space):
        super().__init__(args, search_space)
        self.args=args
        self.best_pred = 0.0

        # 再現性を上げるためrandomを使用している場合はrandom.seed()でseedを設定する
        random.seed(self.args.seed)
        
        # numpyで再現性を上げるためのの設定
        np.random.seed(self.args.seed)
        
        # pytorchで再現性を上げるための設定
        torch.manual_seed(self.args.seed)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
        
        self.scaler = GradScaler()
        
        self.evaluator=Evaluator(self.args.nclass)

    def save_checkpoint(self, state, filename='checkpoint.pth.tar'):
        """Saves checkpoint to disk"""
        torch.save(state, filename)#, _use_new_zipfile_serialization=False)
        mlflow.log_artifact(filename)

    def training(self, epoch):
        train_loss = 0.0
        self.model.train()
        tbar = tqdm(self.train_loader)
        num_img_tr = len(self.train_loader)
        for i, sample in enumerate(tbar):
            image, target = sample
            if self.args.cuda:
                image, target = image.cuda(), target.cuda()
#            self.scheduler(self.optimizer, i, epoch, self.best_pred)
            self.optimizer.zero_grad()
            # for amp
            with autocast():
                output = self.model(image)
                loss = self.criterion(output, target)
            self.scaler.scale(loss).backward()
            # needed for horovod+amp
#            self.optimizer.synchronize()
#            with self.optimizer.skip_synchronize():
            self.scaler.step(self.optimizer)
            self.scaler.update()
            train_loss += loss.item()
            tbar.set_description('Train loss: %.3f' % (train_loss / (i + 1)))

            mlflow.log_metric('train/total_loss_iter', loss.item(), i + num_img_tr * epoch)

            if i > self.args.proc_batch_count:
                break

        mlflow.log_metric('train/total_loss_epoch', train_loss, epoch)
#        print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0]))
        print('Loss: %.3f' % train_loss)

        if self.args.no_val:
            # save checkpoint every epoch
            self.save_checkpoint(self.model.state_dict())

    def validating(self, epoch):
        self.model.eval()
        self.evaluator.reset()
        tbar = tqdm(self.val_loader, desc='\r')
        test_loss = 0.0
        for i, sample in enumerate(tbar):
            image, target = sample
            if self.args.cuda:
                image, target = image.cuda(), target.cuda()
            with autocast():
                with torch.no_grad():
                    output = self.model(image)
            output=output.float()
            loss = self.criterion(output, target)
            test_loss += loss.item()
            tbar.set_description('Test loss: %.3f' % (test_loss / (i + 1)))
#            pred = output.data.cpu().detach().numpy()
            pred = torch.argmax(output.data, dim=1).cpu().detach().numpy()
            target = target.cpu().detach().numpy()
#            target = nn.functional.one_hot(target, num_classes=10).cpu().detach().numpy().astype(np.float32)

            # Add batch sample into evaluator
            self.evaluator.add_batch(target, pred)
        
            if i > self.args.proc_batch_count:
                break

        # Fast test during the training
        Acc = self.evaluator.Pixel_Accuracy()
        """
        Acc_class = self.evaluator.Pixel_Accuracy_Class()
        """

        print('Validation:')
#        print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0]))
        
        new_pred = Acc
        if new_pred > self.best_pred:
            is_best = True
            self.best_pred = new_pred

            self.save_checkpoint(self.model.state_dict())

#            print("Acc:{}, Acc_class:{}, mIoU:{}, fwIoU: {}".format(Acc, Acc_class, mIoU, FWIoU))

        mlflow.log_metric('val/total_loss_epoch', test_loss, epoch)
        mlflow.log_metric("val/best_Acc", self.best_pred, epoch)

        mlflow.log_metric('val/Acc', Acc, epoch)
        """
        mlflow.log_metric('val/Acc_class', Acc_class, epoch)
        """

        return self.best_pred

    # ここで一つのパラメータの組み合わせについて評価する
    def trial_process(self,
                    trial,
                    optimizer,
                    learning_rate,
                    horizontal_flip,
                    horizontal_shift_ratio,
                    vertical_shift_ratio,
                    random_erasing):
        self.best_pred=0.0
        self.start_run(trial)

        # mlflowにtrialごとの情報をロギング
        self.log_trial(trial)

        self.model = EfficientNet.from_pretrained(self.args.backbone)

        # Unfreeze model weights
        for param in self.model.parameters():
            param.requires_grad = True

        num_ftrs = self.model._fc.in_features
        self.model._fc = nn.Linear(num_ftrs, self.args.nclass)

        if self.args.smry_viz:
            from torchinfo import summary
            from torchviz import make_dot
            dummy_image=torch.zeros((2, 3, 32,32))
            dummy_output=self.model(dummy_image)
            make_dot(dummy_output,params=dict(self.model.named_parameters())).render("torchviz", format="png")
            summary(self.model, (1,3, 32,32))
            import sys;sys.exit()

        if self.args.cuda:
            self.model = self.model.to('cuda')

        if optimizer=='SGD':
            self.optimizer = optim.SGD(self.model.parameters(), lr=learning_rate)
        elif optimizer=='Adam':
            self.optimizer = optim.Adam(self.model.parameters(), lr=learning_rate)


        pipeline = [
            T.ToTensor(),
            T.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
        ]

        if strtobool(horizontal_flip) == 1:
            pipeline.append(T.RandomHorizontalFlip(p=0.5))

        pipeline.append(T.RandomAffine(0,translate=(horizontal_shift_ratio,vertical_shift_ratio)))

        if strtobool(random_erasing) == 1:
            pipeline.append(T.RandomErasing())

        transform = T.Compose(pipeline)
                
        train_set = torchvision.datasets.CIFAR10(root='./data', train=True,
                                                download=True, transform=transform)
        self.train_loader = torch.utils.data.DataLoader(train_set, batch_size=self.args.batch_size,
                                                shuffle=True, num_workers=self.args.workers)

        val_set = torchvision.datasets.CIFAR10(root='./data', train=False,
                                                download=True, transform=transform)
        self.val_loader = torch.utils.data.DataLoader(val_set, batch_size=self.args.batch_size,
                                                shuffle=False, num_workers=self.args.workers)

        self.criterion = nn.CrossEntropyLoss()

        for epoch in range(self.args.start_epoch, self.args.epochs):
            self.training(epoch)
            if not self.args.no_val and epoch % self.args.eval_interval == (self.args.eval_interval - 1):
                best_score=self.validating(epoch)

        self.end_run()

        # scoring by best
        return 1.0 - best_score

    # ランダムおよびTPEサーチを行うための目的関数
    def objective_no_grid(self, trial):
        '''
        # Categorical parameter
        optimizer = trial.suggest_categorical('optimizer', self.args.optimizer)

        # Int parameter
        num_layers = trial.suggest_int('num_layers', self.args.num_layers[0], self.args.num_layers[1])

        # Uniform parameter
        dropout_rate = trial.suggest_uniform('dropout_rate', self.args.dropout_rate[0], self.args.dropout_rate[1])

        # Loguniform parameter
        learning_rate = trial.suggest_loguniform('learning_rate', self.args.learning_rate[0], self.args.learning_rate[1])

        # Discrete-uniform parameter
        drop_path_rate = trial.suggest_discrete_uniform('drop_path_rate', self.args.drop_path_rate[0], self.args.drop_path_rate[1], self.args.drop_path_rate[2])
        '''

        # Int parameter
#        num_layers = trial.suggest_int('num_layers', self.args.num_layers[0], self.args.num_layers[1])

        optimizer = trial.suggest_categorical('optimizer', self.args.optimizer)

        learning_rate = trial.suggest_loguniform('learning_rate', self.args.learning_rate[0], self.args.learning_rate[1])

        horizontal_flip = trial.suggest_categorical('horizontal_flip', self.args.horizontal_flip)

        horizontal_shift_ratio = trial.suggest_uniform('horizontal_shift_ratio', self.args.horizontal_shift_ratio[0], self.args.horizontal_shift_ratio[1])

        vertical_shift_ratio = trial.suggest_uniform('vertical_shift_ratio', self.args.vertical_shift_ratio[0], self.args.vertical_shift_ratio[1])

        random_erasing = trial.suggest_categorical('random_erasing', self.args.random_erasing)

        # ここで一つのパラメータの組み合わせについて評価する
        result=self.trial_process(trial,
                                optimizer,
                                learning_rate,
                                horizontal_flip,
                                horizontal_shift_ratio,
                                vertical_shift_ratio,
                                random_erasing)

        return result

    # 固定パラメータおよびグリッドサーチを行うための目的関数
    def objective_grid(self, trial):
        '''
        パラメータは原則trial,suggest_categorical()で指定する。 
        '''    

        optimizer = trial.suggest_categorical('optimizer', self.args.optimizer)

        learning_rate = trial.suggest_categorical('learning_rate', self.args.learning_rate)

        horizontal_flip = trial.suggest_categorical('horizontal_flip', self.args.horizontal_flip)

        horizontal_shift_ratio = trial.suggest_categorical('horizontal_shift_ratio', self.args.horizontal_shift_ratio)

        vertical_shift_ratio = trial.suggest_categorical('vertical_shift_ratio', self.args.vertical_shift_ratio)

        random_erasing = trial.suggest_categorical('random_erasing', self.args.random_erasing)

        # ここで一つのパラメータの組み合わせについて評価する
        result=self.trial_process(trial,
                                optimizer,
                                learning_rate,
                                horizontal_flip,
                                horizontal_shift_ratio,
                                vertical_shift_ratio,
                                random_erasing)

        return result
Exemplo n.º 2
0
class Epoch:
    def __init__(self,
                 model,
                 num_classes,
                 losses,
                 stage_name,
                 device='cpu',
                 verbose=True,
                 EvaluatorIn=None):
        self.model = model
        self.losses = losses
        self.stage_name = stage_name
        self.verbose = verbose
        self.device = device
        self._to_device()

        if EvaluatorIn is None:
            self.evaluator = Evaluator(num_classes, device)
        else:
            self.evaluator = EvaluatorIn

    def _to_device(self):
        for loss in self.losses:
            loss.to(self.device)
        self.model.to(self.device)

    @classmethod
    def _format_logs(self, logs):
        str_logs = ['{} - {:.4}'.format(k, v) for k, v in logs.items()]
        s = ', '.join(str_logs)
        return s

    def batch_update(self, x, y):
        raise NotImplementedError

    def on_epoch_start(self):
        pass

    def run(self, dataloader):

        self.on_epoch_start()

        logs = {}
        loss_meter = AverageValueMeter()
        losses_meters = {
            loss.__name__: AverageValueMeter()
            for loss in self.losses
        }

        with tqdm(dataloader,
                  desc=self.stage_name,
                  file=sys.stdout,
                  disable=not (self.verbose)) as iterator:
            for sample in iterator:
                x = sample['image']
                y = sample['label']
                if 'image_class' in sample:
                    image_class = sample['image_class']
                else:
                    image_class = None
                x, y = x.to(self.device), y.to(self.device)
                loss, losses, y_pred = self.batch_update(
                    x, y, image_class, sample)

                # update losses logs

                loss_value = loss.cpu().detach().numpy()
                loss_meter.add(loss_value)
                loss_logs = {'total_loss': loss_meter.mean}
                logs.update(loss_logs)

                for loss_fn in self.losses:
                    loss_value = loss_fn(y_pred, y, x, image_class)
                    if type(loss_value) == torch.Tensor:
                        loss_value = loss_value.cpu().detach().numpy()
                    losses_meters[loss_fn.__name__].add(loss_value)
                losses_logs = {k: v.mean for k, v in losses_meters.items()}
                logs.update(losses_logs)

                if self.verbose:
                    s = self._format_logs(logs)
                    iterator.set_postfix_str(s)

        logs['acc'] = self.evaluator.Pixel_Accuracy()
        logs['fscore'] = self.evaluator.f_score()
        logs['jaccard'] = self.evaluator.Jaccard()
        logs['miou'] = self.evaluator.Mean_Intersection_over_Union()
        logs['acc_class'] = self.evaluator.Pixel_Accuracy_Class()

        return logs
Exemplo n.º 3
0
class Trainer(object):
    def __init__(self, args):
        self.args = args

        # Define Saver
        self.saver = Saver(args)
        self.saver.save_experiment_config()
        # Define Tensorboard Summary
        self.summary = TensorboardSummary(args.logdir)
        self.writer = self.summary.create_summary()

        # Define Dataloader
        kwargs = {'num_workers': args.workers, 'pin_memory': True}
        dltrain = DLDataset('trainval', "./data/pascal_voc_seg/tfrecord/")
        dlval = DLDataset('val', "./data/pascal_voc_seg/tfrecord/")
        # dltrain = DLDataset('trainval', "./data/pascal_voc_seg/VOCdevkit/VOC2012/")
        # dlval = DLDataset('val', "./data/pascal_voc_seg/VOCdevkit/VOC2012/")
        self.train_loader = DataLoader(dltrain,
                                       batch_size=args.batch_size,
                                       shuffle=True,
                                       num_workers=args.workers,
                                       pin_memory=True)
        self.val_loader = DataLoader(dlval,
                                     batch_size=args.batch_size,
                                     shuffle=True,
                                     num_workers=args.workers,
                                     pin_memory=True)

        # Define network
        model = Deeplab()

        train_params = [{
            'params': model.get_1x_lr_params(),
            'lr': args.lr
        }, {
            'params': model.get_10x_lr_params(),
            'lr': args.lr * 10
        }]

        # Define Optimizer
        optimizer = torch.optim.SGD(train_params,
                                    momentum=args.momentum,
                                    weight_decay=args.weight_decay,
                                    nesterov=args.nesterov)

        # Define Criterion
        # whether to use class balanced weights
        self.criterion = nn.CrossEntropyLoss(ignore_index=255).cuda()
        self.model, self.optimizer = model, optimizer

        # Define Evaluator
        self.evaluator = Evaluator(21)
        # Define lr scheduler
        self.scheduler = optim.lr_scheduler.ReduceLROnPlateau(
            optimizer=optimizer)

        # Using cuda
        # if args.cuda:
        # self.model = torch.nn.DataParallel(self.model)
        self.model = self.model.cuda()

        # Resuming checkpoint
        self.best_pred = 0.0
        if args.resume is not None:
            if not os.path.isfile(args.resume):
                raise RuntimeError("=> no checkpoint found at '{}'".format(
                    args.resume))
            checkpoint = torch.load(args.resume)
            args.start_epoch = checkpoint['epoch']
            if args.cuda:
                self.model.module.load_state_dict(checkpoint['state_dict'])
            else:
                self.model.load_state_dict(checkpoint['state_dict'])
            if not args.ft:
                self.optimizer.load_state_dict(checkpoint['optimizer'])
            self.best_pred = checkpoint['best_pred']
            print("=> loaded checkpoint '{}' (epoch {})".format(
                args.resume, checkpoint['epoch']))

        # Clear start epoch if fine-tuning
        if args.ft:
            args.start_epoch = 0

    def training(self, epoch):
        train_loss = 0.0
        self.model.train()
        tbar = tqdm(self.train_loader)
        num_img_tr = len(self.train_loader)
        for i, (image, target) in enumerate(tbar):
            if self.args.cuda:
                image, target = image.cuda(), target.cuda()

            self.optimizer.zero_grad()

            output = self.model(image)
            loss = self.criterion(output, target.long())
            loss.backward()
            self.optimizer.step()

            train_loss += loss.item()
            tbar.set_description('Train loss: %.3f' % (train_loss / (i + 1)))
            self.writer.add_scalar('train/total_loss_iter', loss.item(),
                                   i + num_img_tr * epoch)

            # Show 10 * 3 inference results each epoch
            # if i % (num_img_tr // 10) == 0:
            if i % 10 == 0:
                global_step = i + num_img_tr * epoch
                self.summary.visualize_image(self.writer, self.args.dataset,
                                             image, target, output,
                                             global_step)
        self.scheduler.step(train_loss)
        self.writer.add_scalar('train/total_loss_epoch', train_loss, epoch)
        print('[Epoch: %d, numImages: %5d]' %
              (epoch, i * self.args.batch_size + image.data.shape[0]))
        print('Loss: %.3f' % train_loss)

        if self.args.no_val:
            # save checkpoint every epoch
            is_best = False
            self.saver.save_checkpoint(
                {
                    'epoch': epoch + 1,
                    'state_dict': self.model.module.state_dict(),
                    'optimizer': self.optimizer.state_dict(),
                    'best_pred': self.best_pred,
                }, is_best)

    def validation(self, epoch):
        self.model.eval()
        self.evaluator.reset()
        tbar = tqdm(self.val_loader, desc='\r')
        test_loss = 0.0
        for i, (image, target) in enumerate(tbar):

            if self.args.cuda:
                image, target = image.cuda(), target.cuda()

            with torch.no_grad():
                output = self.model(image)

            loss = self.criterion(output, target.long())
            test_loss += loss.item()
            tbar.set_description('Test loss: %.3f' % (test_loss / (i + 1)))
            pred = output.data.cpu().numpy()
            target = target.cpu().numpy()
            pred = np.argmax(pred, axis=1)
            # Add batch sample into evaluator
            self.evaluator.add_batch(target, pred)

        # Fast test during the training
        Acc = self.evaluator.Pixel_Accuracy()
        Acc_class = self.evaluator.Pixel_Accuracy_Class()
        mIoU = self.evaluator.Mean_Intersection_over_Union()
        FWIoU = self.evaluator.Frequency_Weighted_Intersection_over_Union()
        self.writer.add_scalar('val/total_loss_epoch', test_loss, epoch)
        self.writer.add_scalar('val/mIoU', mIoU, epoch)
        self.writer.add_scalar('val/Acc', Acc, epoch)
        self.writer.add_scalar('val/Acc_class', Acc_class, epoch)
        self.writer.add_scalar('val/fwIoU', FWIoU, epoch)
        print('Validation:')
        print('[Epoch: %d, numImages: %5d]' %
              (epoch, i * self.args.batch_size + image.data.shape[0]))
        print("Acc:{}, Acc_class:{}, mIoU:{}, fwIoU: {}".format(
            Acc, Acc_class, mIoU, FWIoU))
        print('Loss: %.3f' % test_loss)

        new_pred = mIoU
        if new_pred > self.best_pred:
            is_best = True
            self.best_pred = new_pred
            self.saver.save_checkpoint(
                {
                    'epoch': epoch + 1,
                    'state_dict': self.model.state_dict(),
                    'optimizer': self.optimizer.state_dict(),
                    'best_pred': self.best_pred,
                }, is_best)