def optimize(self):
        """Perform full optimization."""
        first_epoch = 0
        self.model = self.model.to(self.dev)
        N = len(self.pseudo_loader.dataset)
        # optimization times (spread exponentially), can also just be linear in practice (i.e. every n-th epoch)
        self.optimize_times = [(self.num_epochs+2)*N] + \
                              ((self.num_epochs+1.01)*N*(np.linspace(0, 1, args.nopts)**2)[::-1]).tolist()

        optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad,
                                           self.model.parameters()),
                                    weight_decay=self.weight_decay,
                                    momentum=self.momentum,
                                    lr=self.lr)

        if self.checkpoint_dir is not None and self.resume:
            self.L, first_epoch = files.load_checkpoint_all(
                self.checkpoint_dir, self.model, optimizer)
            print('found first epoch to be', first_epoch, flush=True)
            include = [(qq / N >= first_epoch) for qq in self.optimize_times]
            self.optimize_times = (np.array(
                self.optimize_times)[include]).tolist()
        print('We will optimize L at epochs:',
              [np.round(1.0 * t / N, 2) for t in self.optimize_times],
              flush=True)

        if first_epoch == 0:
            # initiate labels as shuffled.
            self.L = np.zeros((self.hc, N), dtype=np.int32)
            for nh in range(self.hc):
                for _i in range(N):
                    self.L[nh, _i] = _i % self.outs[nh]
                self.L[nh] = np.random.permutation(self.L[nh])
            self.L = torch.LongTensor(self.L).to(self.dev)

        # Perform optmization ###############################################################
        lowest_loss = 1e9
        epoch = first_epoch
        while epoch < (self.num_epochs + 1):
            m = self.optimize_epoch(optimizer,
                                    self.train_loader,
                                    epoch,
                                    validation=False)
            if m['loss'] < lowest_loss:
                lowest_loss = m['loss']
                files.save_checkpoint_all(self.checkpoint_dir,
                                          self.model,
                                          args.arch,
                                          optimizer,
                                          self.L,
                                          epoch,
                                          lowest=True)
            epoch += 1
        print(
            f"optimization completed. Saving model to {os.path.join(self.checkpoint_dir,'model_final.pth.tar')}"
        )
        torch.save(self.model,
                   os.path.join(self.checkpoint_dir, 'model_final.pth.tar'))
        return self.model
    def optimize(self, model, train_loader):
        """Perform full optimization."""
        first_epoch = 0
        model = model.to(self.dev)
        self.optimize_times = [0]
        optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad,
                                           model.parameters()),
                                    weight_decay=self.weight_decay,
                                    momentum=self.momentum,
                                    lr=self.lr)
        if self.checkpoint_dir is not None and self.resume:
            self.L, first_epoch = files.load_checkpoint_all(
                self.checkpoint_dir, model=None, opt=None)
            print('loaded from: ', self.checkpoint_dir, flush=True)
            print('first five entries of L: ', self.L[:5], flush=True)
            print('found first epoch to be', first_epoch, flush=True)
            first_epoch = 0
            self.optimize_times = [0]
            self.L = self.L.cuda()
            print("model.headcount ", model.headcount, flush=True)

        #####################################################################################
        # Perform optmization ###############################################################
        lowest_loss = 1e9
        epoch = first_epoch
        while epoch < (self.num_epochs + 1):
            if not args.val_only:
                m = self.optimize_epoch(model,
                                        optimizer,
                                        train_loader,
                                        epoch,
                                        validation=False)
                if m['loss'] < lowest_loss:
                    lowest_loss = m['loss']
                    files.save_checkpoint_all(self.checkpoint_dir,
                                              model,
                                              args.arch,
                                              optimizer,
                                              self.L,
                                              epoch,
                                              lowest=True)
            else:
                print('=' * 30 + ' doing only validation ' + "=" * 30)
                epoch = self.num_epochs
            m = self.optimize_epoch(model,
                                    optimizer,
                                    self.val_loader,
                                    epoch,
                                    validation=True)
            epoch += 1
        print(
            f"Model optimization completed. Saving final model to {os.path.join(self.checkpoint_dir, 'model_final.pth.tar')}"
        )
        torch.save(model,
                   os.path.join(self.checkpoint_dir, 'model_final.pth.tar'))
        return model
if __name__ == '__main__':
    # 模型
    model = AlexNet(num_classes=100)
    model.to('cuda:0')
    model.eval()

    # 优化器 SGD(loss func, grad func...)
    optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad,
                                       model.parameters()),
                                weight_decay=weight_decay,
                                momentum=momentum,
                                lr=lr)

    L, epoch = files.load_checkpoint_all(checkpoint_dir=checkpoint_dir,
                                         model=model,
                                         opt=optimizer)

    # 数据集
    test_loader = dataset.get_aug_dataloader(image_folder,
                                             batch_size=128,
                                             is_validation=True)

    total = 0
    correct = 0
    for data, label in tqdm(test_loader, desc='测试中...'):
        data = data.to(device)
        label = label.to(device)

        # 前向传播
        pre_label = model(data)