def __init__(self, args):
# Set the folder to save the records and checkpoints
log_base_dir = './logs/'
if not osp.exists(log_base_dir):
os.mkdir(log_base_dir)
meta_base_dir = osp.join(log_base_dir, 'meta')
if not osp.exists(meta_base_dir):
os.mkdir(meta_base_dir)
save_path1 = '_'.join([args.dataset, args.model_type, 'MTL'])
save_path2 = 'shot' + str(args.shot) + '_way' + str(args.way) + '_query' + str(args.train_query) + \
'_step' + str(args.step_size) + '_gamma' + str(args.gamma) + '_lr1' + str(args.meta_lr1) + '_lr2' + str(args.meta_lr2) + \
'_batch' + str(args.num_batch) + '_maxepoch' + str(args.max_epoch) + \
'_baselr' + str(args.base_lr) + '_updatestep' + str(args.update_step) + \
'_stepsize' + str(args.step_size) + '_' + args.meta_label
args.save_path = meta_base_dir + '/' + save_path1 + '_' + save_path2
ensure_path(args.save_path)
# Set args to be shareable in the class
self.args = args
# Load meta-train set
self.trainset = Dataset('train', self.args)
self.train_sampler = CategoriesSampler(self.trainset.label, self.args.num_batch, self.args.way, self.args.shot + self.args.train_query)
self.train_loader = DataLoader(dataset=self.trainset, batch_sampler=self.train_sampler, num_workers=8, pin_memory=True)
# Load meta-val set
self.valset = Dataset('val', self.args)
self.val_sampler = CategoriesSampler(self.valset.label, 600, self.args.way, self.args.shot + self.args.val_query)
self.val_loader = DataLoader(dataset=self.valset, batch_sampler=self.val_sampler, num_workers=8, pin_memory=True)
# Build meta-transfer learning model
self.model = MtlLearner(self.args)
# Set optimizer
self.optimizer = torch.optim.Adam([{'params': filter(lambda p: p.requires_grad, self.model.encoder.parameters())}, \
{'params': self.model.base_learner.parameters(), 'lr': self.args.meta_lr2}], lr=self.args.meta_lr1)
# Set learning rate scheduler
self.lr_scheduler = torch.optim.lr_scheduler.StepLR(self.optimizer, step_size=self.args.step_size, gamma=self.args.gamma)
# load pretrained model without FC classifier
self.model_dict = self.model.state_dict()
if self.args.init_weights is not None:
pretrained_dict = torch.load(self.args.init_weights)['params']
else:
pre_base_dir = osp.join(log_base_dir, 'pre')
pre_save_path1 = '_'.join([args.dataset, args.model_type])
pre_save_path2 = 'batchsize' + str(args.pre_batch_size) + '_lr' + str(args.pre_lr) + '_gamma' + str(args.pre_gamma) + '_step' + \
str(args.pre_step_size) + '_maxepoch' + str(args.pre_max_epoch)
pre_save_path = pre_base_dir + '/' + pre_save_path1 + '_' + pre_save_path2
pretrained_dict = torch.load(osp.join(pre_save_path, 'max_acc.pth'))['params']
pretrained_dict = {'encoder.'+k: v for k, v in pretrained_dict.items()}
pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in self.model_dict}
print(pretrained_dict.keys())
self.model_dict.update(pretrained_dict)
self.model.load_state_dict(self.model_dict)
# Set model to GPU
if torch.cuda.is_available():
torch.backends.cudnn.benchmark = True
self.model = self.model.cuda()
def __init__(self, args, phase):
# set args to self attr
for k, v in vars(args).items():
self.__setattr__(k, v)
self.dataset_dir = osp.join(self.dataset_basedir, self.dataset_name)
# set up gpu
self.use_gpu = len(self.gpus) != '-1' and torch.cuda.is_available()
# load training set
self.train_set = Dataset('train', self.dataset_dir, train_aug=True)
self.train_loader = DataLoader(dataset=self.train_set,
batch_size=self.batch_size_pre,
shuffle=True,
num_workers=8,
pin_memory=self.use_gpu)
# load validation set
self.val_set = Dataset('val', self.dataset_dir)
self.val_sampler = CategoriesSampler(label=self.val_set.labels,
n_batch=600,
n_cls=self.way_pre,
n_per=self.shot_pre + self.query_pre)
self.val_loader = DataLoader(dataset=self.val_set,
batch_sampler=self.val_sampler,
num_workers=8,
pin_memory=self.use_gpu)
def __init__(self, args):
# set args to self attr
for k, v in vars(args).items():
self.__setattr__(k, v)
self.dataset_dir = osp.join(self.dataset_basedir, self.dataset_name)
# set up gpu
self.use_gpu = self.gpu != '-1'
# load training set
self.train_set = Dataset('train', self.dataset_dir, train_aug=True)
self.train_loader = DataLoader(dataset=self.train_set,
batch_size=self.batch_size_pre,
shuffle=True,
num_workers=8,
pin_memory=self.use_gpu)
# load validation set
self.val_set = Dataset('val', self.dataset_dir)
self.val_sampler = CategoriesSampler(label=self.val_set.labels,
n_batch=600,
n_cls=self.way_pre,
n_per=self.shot_pre + self.query_pre)
self.val_loader = DataLoader(dataset=self.val_set,
batch_sampler=self.val_sampler,
num_workers=8,
pin_memory=self.use_gpu)
# set pretrain class number
num_class_pretrain = self.train_set.num_class
self.Learner = MetaTransformLearner(args=args, num_cls=num_class_pretrain)
# set the pretrain log
self.train_log = {
'train_loss': [],
'val_loss': [],
'train_acc': [],
'val_acc': [],
'max_acc': self.Learner.checkpoints_pre['max_metric'],
'max_acc_epoch': 0,
}
# set tensorboardX
self.log_dir = '/'.join([self.log_dir, self.Learner.time_pre])
self.writer = SummaryWriter(log_dir=self.log_dir)
# generate the label for eval
label = torch.arange(end=self.way_pre).repeat(self.shot_pre + self.query_pre)
if self.use_gpu:
self.Learner = self.Learner.cuda()
label = label.cuda()
self.label_shot = label[:self.way_pre * self.shot_pre] # for few train
self.label_query = label[self.way_pre * self.shot_pre:] # for eval
def __init__(self, args):
# Set the folder to save the records and checkpoints
log_base_dir = './logs/'
if not osp.exists(log_base_dir):
os.mkdir(log_base_dir)
pre_base_dir = osp.join(log_base_dir, 'pre')
if not osp.exists(pre_base_dir):
os.mkdir(pre_base_dir)
save_path1 = '_'.join([args.dataset, args.model_type])
save_path2 = 'batchsize' + str(args.pre_batch_size) + '_lr' + str(args.pre_lr) + '_gamma' + str(args.pre_gamma) + '_step' + \
str(args.pre_step_size) + '_maxepoch' + str(args.pre_max_epoch)
args.save_path = pre_base_dir + '/' + save_path1 + '_' + save_path2
ensure_path(args.save_path)
# Set args to be shareable in the class
self.args = args
# Load pretrain set
self.trainset = Dataset('train', self.args, train_aug=True)
self.train_loader = DataLoader(dataset=self.trainset,
batch_size=args.pre_batch_size,
shuffle=True,
num_workers=8,
pin_memory=True)
# Load meta-val set
self.valset = Dataset('val', self.args)
self.val_sampler = CategoriesSampler(
self.valset.label, 600, self.args.way,
self.args.shot + self.args.val_query)
self.val_loader = DataLoader(dataset=self.valset,
batch_sampler=self.val_sampler,
num_workers=8,
pin_memory=True)
# Set pretrain class number
num_class_pretrain = self.trainset.num_class
# Build pretrain model
self.model = MtlLearner(self.args,
mode='pre',
num_cls=num_class_pretrain)
# Set optimizer
self.optimizer = torch.optim.SGD([{'params': self.model.encoder.parameters(), 'lr': self.args.pre_lr}, \
{'params': self.model.pre_fc.parameters(), 'lr': self.args.pre_lr}], \
momentum=self.args.pre_custom_momentum, nesterov=True, weight_decay=self.args.pre_custom_weight_decay)
# Set learning rate scheduler
self.lr_scheduler = torch.optim.lr_scheduler.StepLR(self.optimizer, step_size=self.args.pre_step_size, \
gamma=self.args.pre_gamma)
# Set model to GPU
if torch.cuda.is_available():
torch.backends.cudnn.benchmark = True
self.model = self.model.cuda()
def get_base_means(self, normalize=False):
num_classes = self.num_classes
# save_dir = "pretrain/%s%s_%s_%s_mean.npy" % (pre, self.dataset, self.method, self.model_name)
# ensure_path("pretrain")
# self.means_save_dir = osp.join("logs/means", "%s_%s.npy" % (self.args.dataset, str(is_cosine_feature)))
# Load pretrain set
num_workers = 8
if self.args.debug:
num_workers = 0
self.trainset = Dataset('train', self.args, dataset=self.dataset, train_aug=False)
self.train_loader = DataLoader(dataset=self.trainset, batch_size=self.args.pre_batch_size, shuffle=True, num_workers=num_workers, pin_memory=True)
means = torch.zeros(num_classes, 640).cuda()
counts = torch.zeros(num_classes).cuda()
for epoch in range(1):
tqdm_gen = tqdm.tqdm(self.train_loader)
for i, batch in enumerate(tqdm_gen, 1):
data, _ = [_.cuda() for _ in batch]
label = batch[1]
with torch.no_grad():
data = self.encoder(data)
if normalize:
data = self.normalize(data)
for j in range(data.shape[0]):
means[label[j]] += data[j]
counts[label[j]] += 1
counts = counts.unsqueeze(1).expand_as(means)
means = means / counts
means_np = means.cpu().detach().numpy()
# np.save(save_dir, means_np)
return means_np
def __init__(self, args):
# Set the folder to save the records and checkpoints
log_base_dir = '../logs/'
if not osp.exists(log_base_dir):
os.mkdir(log_base_dir)
pre_base_dir = osp.join(log_base_dir, 'pre')
if not osp.exists(pre_base_dir):
os.mkdir(pre_base_dir)
save_path1 = '_'.join([args.dataset, args.model_type])
save_path2 = 'batchsize' + str(args.pre_batch_size) + '_lr' + str(args.pre_lr) + '_gamma' + str(args.pre_gamma) + '_step' + \
str(args.pre_step_size) + '_maxepoch' + str(args.pre_max_epoch)
args.save_path = pre_base_dir + '/' + save_path1 + '_' + save_path2
ensure_path(args.save_path)
# Set args to be shareable in the class
self.args = args
# Load pretrain set
self.trainset = Dataset('train', self.args)
self.train_loader = DataLoader(dataset=self.trainset, batch_size=args.pre_batch_size, shuffle=True, num_workers=8, pin_memory=True)
# Load pre-val set
self.valset = mDataset('val', self.args)
self.val_sampler = CategoriesSampler(self.valset.labeln,self.args.num_batch , self.args.way, self.args.shot + self.args.val_query,self.args.shot)
self.val_loader = DataLoader(dataset=self.valset, batch_sampler=self.val_sampler, num_workers=8, pin_memory=True)
# Build pretrain model
self.model = MtlLearner(self.args, mode='train')
print(self.model)
'''
if self.args.pre_init_weights is not None:
self.model_dict = self.model.state_dict()
pretrained_dict = torch.load(self.args.pre_init_weights)['params']
pretrained_dict = {'encoder.'+k: v for k, v in pretrained_dict.items()}
pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in self.model_dict}
print(pretrained_dict.keys())
self.model_dict.update(pretrained_dict)
self.model.load_state_dict(self.model_dict)
'''
self.FL=FocalLoss()
self.CD=CE_DiceLoss()
self.LS=LovaszSoftmax()
# Set optimizer
# Set optimizer
self.optimizer = torch.optim.SGD([{'params': self.model.encoder.parameters(), 'lr': self.args.pre_lr}], \
momentum=self.args.pre_custom_momentum, nesterov=True, weight_decay=self.args.pre_custom_weight_decay)
# Set learning rate scheduler
self.lr_scheduler = torch.optim.lr_scheduler.StepLR(self.optimizer, step_size=self.args.pre_step_size, \
gamma=self.args.pre_gamma)
# Set model to GPU
if torch.cuda.is_available():
torch.backends.cudnn.benchmark = True
self.model = self.model.cuda()
def eval(self):
"""The function for the meta-eval phase."""
# Load the logs
trlog = torch.load(osp.join(self.args.save_path, 'trlog'))
# Load meta-test set
test_set = Dataset('test', self.args)
sampler = CategoriesSampler(test_set.label, 600, self.args.way, self.args.shot + self.args.val_query)
loader = DataLoader(test_set, batch_sampler=sampler, num_workers=8, pin_memory=True)
# Set test accuracy recorder
test_acc_record = np.zeros((600,))
# Load model for meta-test phase
if self.args.eval_weights is not None:
self.model.load_state_dict(torch.load(self.args.eval_weights)['params'])
else:
self.model.load_state_dict(torch.load(osp.join(self.args.save_path, 'max_acc' + '.pth'))['params'])
# Set model to eval mode
self.model.eval()
# Set accuracy averager
ave_acc = Averager()
# Generate labels
label = torch.arange(self.args.way).repeat(self.args.val_query)
if torch.cuda.is_available():
label = label.type(torch.cuda.LongTensor)
else:
label = label.type(torch.LongTensor)
label_shot = torch.arange(self.args.way).repeat(self.args.shot)
if torch.cuda.is_available():
label_shot = label_shot.type(torch.cuda.LongTensor)
else:
label_shot = label_shot.type(torch.LongTensor)
# Start meta-test
for i, batch in enumerate(loader, 1):
if torch.cuda.is_available():
data, _ = [_.cuda() for _ in batch]
else:
data = batch[0]
k = self.args.way * self.args.shot
data_shot, data_query = data[:k], data[k:]
logits = self.model((data_shot, label_shot, data_query))
acc = count_acc(logits, label)
ave_acc.add(acc)
test_acc_record[i-1] = acc
if i % 100 == 0:
print('batch {}: {:.2f}({:.2f})'.format(i, ave_acc.item() * 100, acc * 100))
# Calculate the confidence interval, update the logs
m, pm = compute_confidence_interval(test_acc_record)
print('Val Best Epoch {}, Acc {:.4f}, Test Acc {:.4f}'.format(trlog['max_acc_epoch'], trlog['max_acc'], ave_acc.item()))
print('Test Acc {:.4f} + {:.4f}'.format(m, pm))
def eval(self):
"""The function for the meta-evaluate (test) phase."""
# Load the logs
trlog = torch.load(osp.join(self.args.save_path, 'trlog'))
# Load meta-test set
self.test_set = Dataset('test', self.args)
self.sampler = CategoriesSampler(self.test_set.labeln,
self.args.num_batch,
self.args.way + 1,
self.args.train_query,
self.args.test_query)
self.loader = DataLoader(dataset=self.test_set,
batch_sampler=self.sampler,
num_workers=8,
pin_memory=True)
# Load model for meta-test phase
if self.args.eval_weights is not None:
self.model.load_state_dict(
torch.load(self.args.eval_weights)['params'])
else:
self.model.load_state_dict(
torch.load(osp.join(self.args.save_path,
'max_iou' + '.pth'))['params'])
# Set model to eval mode
self.model.eval()
# Set accuracy(IoU) averager
ave_acc = Averager()
# Start meta-test
K = self.args.way + 1
N = self.args.train_query
Q = self.args.test_query
count = 1
for i, batch in enumerate(self.loader, 1):
if torch.cuda.is_available():
data, labels, _ = [_.cuda() for _ in batch]
else:
data = batch[0]
labels = batch[1]
p = K * N
im_train, im_test = data[:p], data[p:]
#Adjusting labels for each meta task
labels = downlabel(labels, K)
out_train, out_test = labels[:p], labels[p:]
if (torch.cuda.is_available()):
im_train = im_train.cuda()
im_test = im_test.cuda()
out_train = out_train.cuda()
out_test = out_test.cuda()
#Reshaping train set ouput
Ytr = out_train.reshape(-1)
Ytr = onehot(Ytr, K) #One hot encoding for loss
Yte = out_test.reshape(out_test.shape[0], -1)
if (torch.cuda.is_available()):
Ytr = Ytr.cuda()
Yte = Yte.cuda()
# Output logits for model
Gte = self.model(im_train, Ytr, im_test, Yte)
GteT = torch.transpose(Gte, 1, 2)
# Calculate meta-train accuracy
self._reset_metrics()
seg_metrics = eval_metrics(GteT, Yte, K)
self._update_seg_metrics(*seg_metrics)
pixAcc, mIoU, _ = self._get_seg_metrics(K).values()
ave_acc.add(mIoU)
#Saving Test Image, Ground Truth Image and Predicted Image
for j in range(K * Q):
x1 = im_test[j].detach().cpu()
y1 = out_test[j].detach().cpu()
z1 = GteT[j].detach().cpu()
z1 = torch.argmax(z1, axis=0)
m = int(math.sqrt(z1.shape[0]))
z2 = z1.reshape(m, m)
x = transforms.ToPILImage()(x1).convert("RGB")
y = Image.fromarray(decode_segmap(y1, K))
z = Image.fromarray(decode_segmap(z2, K))
px = self.args.save_image_dir + str(count) + 'a.jpg'
py = self.args.save_image_dir + str(count) + 'b.png'
pz = self.args.save_image_dir + str(count) + 'c.png'
x.save(px)
y.save(py)
z.save(pz)
count = count + 1
# Test mIoU
ave_acc = ave_acc.item()
print("=============================================================")
print('Average Test mIoU: {:.4f}'.format(ave_acc))
print("Images Saved!")
print("=============================================================")
def __init__(self, args):
# Set the folder to save the records and checkpoints
save_image_dir = '../results1/'
if not osp.exists(save_image_dir):
os.mkdir(save_image_dir)
log_base_dir = '../logs1/'
if not osp.exists(log_base_dir):
os.mkdir(log_base_dir)
meta_base_dir = osp.join(log_base_dir, 'meta')
if not osp.exists(meta_base_dir):
os.mkdir(meta_base_dir)
save_path1 = '_'.join([args.dataset, args.model_type, 'MTL'])
save_path2 = '_mtype' + str(args.mtype) + '_shot' + str(args.train_query) + '_way' + str(args.way) + '_query' + str(args.train_query) + \
'_step' + str(args.step_size) + '_gamma' + str(args.gamma) + '_lr' + str(args.meta_lr) + \
'_batch' + str(args.num_batch) + '_maxepoch' + str(args.max_epoch) + \
'_baselr' + str(args.base_lr) + '_updatestep' + str(args.update_step) + \
'_stepsize' + str(args.step_size) + '_' + args.meta_label
args.save_path = meta_base_dir + '/' + save_path1 + '_' + save_path2
args.save_image_dir = save_image_dir
ensure_path(args.save_path)
# Set args to be shareable in the class
self.args = args
# Load meta-train set
self.trainset = Dataset('train', self.args)
self.train_sampler = CategoriesSampler(self.trainset.labeln,
self.args.num_batch,
self.args.way + 1,
self.args.train_query,
self.args.test_query)
self.train_loader = DataLoader(dataset=self.trainset,
batch_sampler=self.train_sampler,
num_workers=8,
pin_memory=True)
# Load meta-val set
if (self.args.valdata == 'Yes'):
self.valset = Dataset('val', self.args)
self.val_sampler = CategoriesSampler(self.valset.labeln,
self.args.num_batch,
self.args.way + 1,
self.args.train_query,
self.args.test_query)
self.val_loader = DataLoader(dataset=self.valset,
batch_sampler=self.val_sampler,
num_workers=8,
pin_memory=True)
# Build meta-transfer learning model
self.model = MtlLearner(self.args)
self.CD = CE_DiceLoss()
self.FL = FocalLoss()
self.LS = LovaszSoftmax()
# Set model to GPU
if torch.cuda.is_available():
torch.backends.cudnn.benchmark = True
self.model = self.model.cuda()
# Set optimizer
self.optimizer = torch.optim.Adam([{
'params':
filter(lambda p: p.requires_grad, self.model.encoder.parameters())
}],
lr=self.args.meta_lr)
# Set learning rate scheduler
self.lr_scheduler = torch.optim.lr_scheduler.StepLR(
self.optimizer,
step_size=self.args.step_size,
gamma=self.args.gamma)
# load pretrained model
# Path should nbe changed accordingly
self.model.load_state_dict(
torch.load(osp.join(self.args.save_path,
'epoch24' + '.pth'))['params'])
self.optimizer.load_state_dict(
torch.load(osp.join(self.args.save_path,
'epoch24' + '_o.pth'))['params_o'])
self.lr_scheduler.load_state_dict(
torch.load(osp.join(self.args.save_path,
'epoch24' + '_s.pth'))['params_s'])
self.model_dict = self.model.state_dict()
self.optimizer_dict = self.optimizer.state_dict()
self.lr_scheduler_dict = self.lr_scheduler.state_dict()
#Total Model Parameters
pytorch_total_params = sum(p.numel() for p in self.model.parameters()
if p.requires_grad)
print("Total Trainable Parameters in the Model: " +
str(pytorch_total_params))
def eval(self):
"""The function for the meta-eval phase."""
# Load the logs
# trlog = torch.load(osp.join(self.args.save_path, 'trlog'))
num_workers = 8
if self.args.debug:
num_workers = 0
self.test_iter = 2000
# Load meta-test set
test_set = Dataset('test',
self.args,
dataset=self.args.param.dataset,
train_aug=False)
sampler = CategoriesSampler(test_set.label, self.test_iter,
self.args.way,
self.args.shot + self.args.val_query)
loader = DataLoader(test_set,
batch_sampler=sampler,
num_workers=num_workers,
pin_memory=True)
# Set test accuracy recorder
test_acc_record = np.zeros((self.test_iter, ))
# Load model for meta-test phase
if self.args.eval_weights is not None:
self.model.load_state_dict(
torch.load(self.args.eval_weights)['params'])
else:
# Load according to config file
args = self.args
base_path = "/data2/yuezhongqi/Model/ifsl/mtl"
if args.param.dataset == "tiered":
add_path = "tiered_"
else:
add_path = ""
if args.param.model == "ResNet10":
add_path += "resnet_"
elif args.param.model == "wideres":
add_path += "wrn_"
elif "baseline" in args.config:
add_path += "baseline_"
else:
add_path += "edsplit_"
add_path += str(args.param.shot)
self.add_path = add_path
self.model.load_state_dict(
torch.load(osp.join(base_path, add_path + '.pth'))['params'])
# Set model to eval mode
self.model.eval()
# Set accuracy averager
ave_acc = Averager()
# Generate labels
label = torch.arange(self.args.way).repeat(self.args.val_query)
if torch.cuda.is_available():
label = label.type(torch.cuda.LongTensor)
else:
label = label.type(torch.LongTensor)
label_shot = torch.arange(self.args.way).repeat(self.args.shot)
if torch.cuda.is_available():
label_shot = label_shot.type(torch.cuda.LongTensor)
else:
label_shot = label_shot.type(torch.LongTensor)
hacc = Hacc()
# Start meta-test
for i, batch in enumerate(loader, 1):
if torch.cuda.is_available():
data, _ = [_.cuda() for _ in batch]
else:
data = batch[0]
k = self.args.way * self.args.shot
data_shot, data_query = data[:k], data[k:]
logits = self.model((data_shot, label_shot, data_query, True))
acc = count_acc(logits, label)
hardness, correct = get_hardness_correct(logits, label_shot, label,
data_shot, data_query,
self.model.pretrain)
ave_acc.add(acc)
hacc.add_data(hardness, correct)
test_acc_record[i - 1] = acc
if i % 100 == 0:
#print('batch {}: {:.2f}({:.2f})'.format(i, ave_acc.item() * 100, acc * 100))
print("Average acc:{:.4f}, Average hAcc:{:.4f}".format(
ave_acc.item(), hacc.get_topk_hard_acc()))
# Modify add path to generate test case name:
test_case_name = self.add_path
if self.args.cross:
test_case_name += "_cross"
# Calculate the confidence interval, update the logs
m, pm = compute_confidence_interval(test_acc_record)
msg = test_case_name + ' Test Acc {:.4f} +- {:.4f}, hAcc {:.4f}'.format(
ave_acc.item() * 100, pm * 100, hacc.get_topk_hard_acc())
print(msg)
self.write_output_message(msg, test_case_name)
if self.args.save_hacc:
print("Saving hacc!")
pickle.dump(hacc, open("hacc/" + test_case_name, "wb"))
print('Test Acc {:.4f} + {:.4f}'.format(m, pm))
def __init__(self, args):
param = configs.__dict__[args.config]()
args.shot = param.shot
args.test = param.test
args.debug = param.debug
args.deconfound = param.deconfound
args.meta_label = param.meta_label
args.init_weights = param.init_weights
self.test_iter = param.test_iter
args.param = param
pprint(vars(args))
# Set the folder to save the records and checkpoints
log_base_dir = '/data2/yuezhongqi/Model/mtl/logs/'
if not osp.exists(log_base_dir):
os.mkdir(log_base_dir)
meta_base_dir = osp.join(log_base_dir, 'meta')
if not osp.exists(meta_base_dir):
os.mkdir(meta_base_dir)
save_path1 = '_'.join([args.dataset, args.model_type, 'MTL'])
save_path2 = 'shot' + str(args.shot) + '_way' + str(args.way) + '_query' + str(args.train_query) + \
'_step' + str(args.step_size) + '_gamma' + str(args.gamma) + '_lr1' + str(args.meta_lr1) + '_lr2' + str(args.meta_lr2) + \
'_batch' + str(args.num_batch) + '_maxepoch' + str(args.max_epoch) + \
'_baselr' + str(args.base_lr) + '_updatestep' + str(args.update_step) + \
'_stepsize' + str(args.step_size) + '_' + args.meta_label
args.save_path = meta_base_dir + '/' + save_path1 + '_' + save_path2
ensure_path(args.save_path)
# Set args to be shareable in the class
self.args = args
# Load meta-train set
self.trainset = Dataset('train',
self.args,
dataset=self.args.param.dataset,
train_aug=False)
num_workers = 8
if args.debug:
num_workers = 0
self.train_sampler = CategoriesSampler(
self.trainset.label, self.args.num_batch, self.args.way,
self.args.shot + self.args.train_query)
self.train_loader = DataLoader(dataset=self.trainset,
batch_sampler=self.train_sampler,
num_workers=num_workers,
pin_memory=True)
# Load meta-val set
self.valset = Dataset('val',
self.args,
dataset=self.args.param.dataset,
train_aug=False)
self.val_sampler = CategoriesSampler(
self.valset.label, self.test_iter, self.args.way,
self.args.shot + self.args.val_query)
self.val_loader = DataLoader(dataset=self.valset,
batch_sampler=self.val_sampler,
num_workers=num_workers,
pin_memory=True)
# Build meta-transfer learning model
self.model = MtlLearner(self.args)
# load pretrained model without FC classifier
self.model.load_pretrain_weight(self.args.init_weights)
'''
self.model_dict = self.model.state_dict()
if self.args.init_weights is not None:
pretrained_dict = torch.load(self.args.init_weights)['params']
else:
pre_base_dir = osp.join(log_base_dir, 'pre')
pre_save_path1 = '_'.join([args.dataset, args.model_type])
pre_save_path2 = 'batchsize' + str(args.pre_batch_size) + '_lr' + str(args.pre_lr) + '_gamma' + str(args.pre_gamma) + '_step' + \
str(args.pre_step_size) + '_maxepoch' + str(args.pre_max_epoch)
pre_save_path = pre_base_dir + '/' + pre_save_path1 + '_' + pre_save_path2
pretrained_dict = torch.load(osp.join(pre_save_path, 'max_acc.pth'))['params']
pretrained_dict = {'encoder.'+k: v for k, v in pretrained_dict.items()}
pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in self.model_dict}
print(pretrained_dict.keys())
self.model_dict.update(pretrained_dict)
self.model.load_state_dict(self.model_dict)
'''
# Set model to GPU
if torch.cuda.is_available():
torch.backends.cudnn.benchmark = True
self.model = self.model.cuda()
if self.args.param.model == "wideres":
print("Using Parallel")
self.model.encoder = torch.nn.DataParallel(
self.model.encoder).cuda()
# Set optimizer
self.optimizer = torch.optim.Adam(
[{
'params':
filter(lambda p: p.requires_grad,
self.model.encoder.parameters())
}, {
'params': self.model.base_learner.parameters(),
'lr': self.args.meta_lr2
}],
lr=self.args.meta_lr1)
# Set learning rate scheduler
self.lr_scheduler = torch.optim.lr_scheduler.StepLR(
self.optimizer,
step_size=self.args.step_size,
gamma=self.args.gamma)
if not self.args.deconfound:
self.criterion = torch.nn.CrossEntropyLoss().cuda()
else:
self.criterion = torch.nn.NLLLoss().cuda()
# Enable evaluation with Cross
if args.cross:
args.param.dataset = "cross"
def eval(self, gradcam=False, rise=False, test_on_val=False):
"""The function for the meta-eval phase."""
# Load the logs
if os.path.exists(osp.join(self.args.save_path, 'trlog')):
trlog = torch.load(osp.join(self.args.save_path, 'trlog'))
else:
trlog = None
torch.manual_seed(1)
np.random.seed(1)
# Load meta-test set
test_set = Dataset('val' if test_on_val else 'test', self.args)
sampler = CategoriesSampler(test_set.label, 600, self.args.way,
self.args.shot + self.args.val_query)
loader = DataLoader(test_set,
batch_sampler=sampler,
num_workers=8,
pin_memory=True)
# Set test accuracy recorder
test_acc_record = np.zeros((600, ))
# Load model for meta-test phase
if self.args.eval_weights is not None:
weights = self.addOrRemoveModule(
self.model,
torch.load(self.args.eval_weights)['params'])
self.model.load_state_dict(weights)
else:
self.model.load_state_dict(
torch.load(osp.join(self.args.save_path,
'max_acc' + '.pth'))['params'])
# Set model to eval mode
self.model.eval()
# Set accuracy averager
ave_acc = Averager()
# Generate labels
label = torch.arange(self.args.way).repeat(self.args.val_query)
if torch.cuda.is_available():
label = label.type(torch.cuda.LongTensor)
else:
label = label.type(torch.LongTensor)
label_shot = torch.arange(self.args.way).repeat(self.args.shot)
if torch.cuda.is_available():
label_shot = label_shot.type(torch.cuda.LongTensor)
else:
label_shot = label_shot.type(torch.LongTensor)
if gradcam:
self.model.layer3 = self.model.encoder.layer3
model_dict = dict(type="resnet",
arch=self.model,
layer_name='layer3')
grad_cam = GradCAM(model_dict, True)
grad_cam_pp = GradCAMpp(model_dict, True)
self.model.features = self.model.encoder
guided = GuidedBackprop(self.model)
if rise:
self.model.layer3 = self.model.encoder.layer3
score_mod = ScoreCam(self.model)
# Start meta-test
for i, batch in enumerate(loader, 1):
if torch.cuda.is_available():
data, _ = [_.cuda() for _ in batch]
else:
data = batch[0]
k = self.args.way * self.args.shot
data_shot, data_query = data[:k], data[k:]
if i % 5 == 0:
suff = "_val" if test_on_val else ""
if self.args.rep_vec or self.args.cross_att:
print('batch {}: {:.2f}({:.2f})'.format(
i,
ave_acc.item() * 100, acc * 100))
if self.args.cross_att:
label_one_hot = self.one_hot(label).to(label.device)
_, _, logits, simMapQuer, simMapShot, normQuer, normShot = self.model(
(data_shot, label_shot, data_query),
ytest=label_one_hot,
retSimMap=True)
else:
logits, simMapQuer, simMapShot, normQuer, normShot, fast_weights = self.model(
(data_shot, label_shot, data_query),
retSimMap=True)
torch.save(
simMapQuer,
"../results/{}/{}_simMapQuer{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
simMapShot,
"../results/{}/{}_simMapShot{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
data_query, "../results/{}/{}_dataQuer{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
data_shot, "../results/{}/{}_dataShot{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
normQuer, "../results/{}/{}_normQuer{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
normShot, "../results/{}/{}_normShot{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
else:
logits, normQuer, normShot, fast_weights = self.model(
(data_shot, label_shot, data_query),
retFastW=True,
retNorm=True)
torch.save(
normQuer, "../results/{}/{}_normQuer{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
normShot, "../results/{}/{}_normShot{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
if gradcam:
print("Saving gradmaps", i)
allMasks, allMasks_pp, allMaps = [], [], []
for l in range(len(data_query)):
allMasks.append(
grad_cam(data_query[l:l + 1], fast_weights, None))
allMasks_pp.append(
grad_cam_pp(data_query[l:l + 1], fast_weights,
None))
allMaps.append(
guided.generate_gradients(data_query[l:l + 1],
fast_weights))
allMasks = torch.cat(allMasks, dim=0)
allMasks_pp = torch.cat(allMasks_pp, dim=0)
allMaps = torch.cat(allMaps, dim=0)
torch.save(
allMasks, "../results/{}/{}_gradcamQuer{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
allMasks_pp,
"../results/{}/{}_gradcamppQuer{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
allMaps, "../results/{}/{}_guidedQuer{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
if rise:
print("Saving risemaps", i)
allScore = []
for l in range(len(data_query)):
allScore.append(
score_mod(data_query[l:l + 1], fast_weights))
else:
if self.args.cross_att:
label_one_hot = self.one_hot(label).to(label.device)
_, _, logits = self.model(
(data_shot, label_shot, data_query),
ytest=label_one_hot)
else:
logits = self.model((data_shot, label_shot, data_query))
acc = count_acc(logits, label)
ave_acc.add(acc)
test_acc_record[i - 1] = acc
# Calculate the confidence interval, update the logs
m, pm = compute_confidence_interval(test_acc_record)
if trlog is not None:
print('Val Best Epoch {}, Acc {:.4f}, Test Acc {:.4f}'.format(
trlog['max_acc_epoch'], trlog['max_acc'], ave_acc.item()))
print('Test Acc {:.4f} + {:.4f}'.format(m, pm))
return m
def __init__(self, args):
# Set the folder to save the records and checkpoints
log_base_dir = './logs/'
if not osp.exists(log_base_dir):
os.mkdir(log_base_dir)
meta_base_dir = osp.join(log_base_dir, 'meta')
if not osp.exists(meta_base_dir):
os.mkdir(meta_base_dir)
save_path1 = '_'.join([args.dataset, args.model_type, 'MTL'])
save_path2 = 'shot' + str(args.shot) + '_way' + str(args.way) + '_query' + str(args.train_query) + \
'_step' + str(args.step_size) + '_gamma' + str(args.gamma) + '_lr1' + str(args.meta_lr1) + '_lr2' + str(args.meta_lr2) + \
'_batch' + str(args.num_batch) + '_maxepoch' + str(args.max_epoch) + \
'_baselr' + str(args.base_lr) + '_updatestep' + str(args.update_step) + \
'_stepsize' + str(args.step_size) + '_' + args.meta_label
args.save_path = meta_base_dir + '/' + save_path1 + '_' + save_path2
ensure_path(args.save_path)
# Set args to be shareable in the class
self.args = args
# Load meta-train set
self.trainset = Dataset('train', self.args)
self.train_sampler = CategoriesSampler(
self.trainset.label, self.args.num_batch, self.args.way,
self.args.shot + self.args.train_query)
self.train_loader = DataLoader(dataset=self.trainset,
batch_sampler=self.train_sampler,
num_workers=args.num_workers,
pin_memory=True)
# Load meta-val set
self.valset = Dataset('val', self.args)
self.val_sampler = CategoriesSampler(
self.valset.label, 600, self.args.way,
self.args.shot + self.args.val_query)
self.val_loader = DataLoader(dataset=self.valset,
batch_sampler=self.val_sampler,
num_workers=args.num_workers,
pin_memory=True)
# Build meta-transfer learning model
self.model = MtlLearner(self.args,res="high" if (self.args.distill_id or self.args.high_res) else "low",multi_gpu=len(args.gpu.split(","))>1,\
crossAtt=self.args.cross_att)
if self.args.distill_id:
#self.teacher = MtlLearner(self.args,res="low")
#self.teacher.load_state_dict(torch.load(args.distill_id)["params"])
self.teacher = MtlLearner(self.args,
res="low",
repVecNb=self.args.nb_parts_teach,
multi_gpu=len(args.gpu.split(",")) > 1)
bestTeach = "../models/{}/meta_{}_trial{}_max_acc.pth".format(
self.args.exp_id, self.args.distill_id,
self.args.best_trial_teach - 1)
self.teacher.load_state_dict(torch.load(bestTeach)["params"])
# Set optimizer
self.optimizer = torch.optim.Adam([{'params': filter(lambda p: p.requires_grad, self.model.encoder.parameters())}, \
{'params': self.model.base_learner.parameters(), 'lr': self.args.meta_lr2}], lr=self.args.meta_lr1)
# Set learning rate scheduler
self.lr_scheduler = torch.optim.lr_scheduler.StepLR(
self.optimizer,
step_size=self.args.step_size,
gamma=self.args.gamma)
# load pretrained model without FC classifier
self.model_dict = self.model.state_dict()
if self.args.init_weights is not None:
pretrained_dict = torch.load(self.args.init_weights)['params']
pretrained_dict = {
'encoder.' + k: v
for k, v in pretrained_dict.items()
}
pretrained_dict = {
k: v
for k, v in pretrained_dict.items() if k in self.model_dict
}
self.model_dict.update(pretrained_dict)
self.model.load_state_dict(self.model_dict)
# Set model to GPU
if torch.cuda.is_available():
torch.backends.cudnn.benchmark = True
self.model = self.model.cuda()
if self.args.distill_id:
self.teacher = self.teacher.cuda()
if self.args.cross_att:
self.criterion = crossAttModule.CrossEntropyLoss()
