def val(self, episodes, val_folders):
self.net.eval()
self.classifier.eval()
accs = []
for i in range(episodes):
task = dataload.FewShotTask(val_folders, self.n_way, self.k_shot,
self.q_query, self.dataset)
spt_loader = dataload.get_Dataloader(task, self.k_shot, 'support')
qry_loader = dataload.get_Dataloader(task,
self.q_query,
'query',
shuffle=True)
spt, spt_y = iter(spt_loader).next()
qry, qry_y = iter(qry_loader).next()
if torch.cuda.is_available():
spt, spt_y, qry, qry_y = self.datas_on_cuda(
spt, spt_y, qry, qry_y)
spt_feature = self.net(spt)
qry_feature = self.net(qry)
dist = self.classifier(spt_feature, qry_feature)
predict_label = torch.argmin(dist, dim=1)
score = [
1 if predict_label[j] == qry_y[j].long() else 0
for j in range(len(qry_y))
]
acc = np.sum(score) / 1.0 / len(score)
accs.append(acc)
acc, pacc = compute_confidence_interval(accs)
print('acc is {:.4f} pacc is {:.4f}'.format(acc, pacc))
self.logger.info('acc is {:.4f} pacc is {:.4f}'.format(acc, pacc))
return acc, pacc
def evaluate(model, normalize, epoch, support_loader, n, k, q, device, logger):
accs_l2 = []
accs_cosine = []
model.eval()
with torch.no_grad():
for data in tqdm(support_loader):
imgs, labels = prepare_nshot_task(n, k, q, data, device)
_, outputs, _ = model(imgs, norm=normalize)
acc_l2 = evaluation(outputs, labels, n, k, q, 'l2')
acc_cosine = evaluation(outputs, labels, n, k, q, 'cosine')
accs_l2.append(acc_l2)
accs_cosine.append(acc_cosine)
m_l2, pm_l2 = compute_confidence_interval(accs_l2)
m_cosine, pm_cosine = compute_confidence_interval(accs_cosine)
# file_writer.write(f'{epoch:3d}.pth {n}-shot\tAccuracy_l2: {m_l2:.2f}+/-{pm_l2:.2f} Accuracy_cosine: {m_cosine:.2f}+/-{pm_cosine:.2f}\n')
logger.info(f'{epoch:3d}.pth: {n}-shot \t l2: {m_l2:.2f}+/-{pm_l2:.2f} \t '
f'cosine: {m_cosine:.2f}+/-{pm_cosine:.2f}')
def evaluate(model, normalize, epoch, support_loader, n, k, q, device, logger,
nodes, desc_embeddings, id_to_class_name, classFile_to_wikiID):
accs_l2 = []
accs_cosine = []
model.eval()
with torch.no_grad():
for data in tqdm(support_loader):
imgs, labels = prepare_nshot_task(n, k, q, data, device)
support_corr_nodeIndexs = find_nodeIndex_by_imgLabels(
nodes, data[1][:n * k], id_to_class_name, classFile_to_wikiID)
support_imgs, _, _, _, _ = model(imgs[:n * k],
desc_embeddings,
support_corr_nodeIndexs,
norm=normalize)
queries = []
for i in range(k):
query_corr_nodeIndexs = find_nodeIndex_by_imgLabels(
nodes, (q * k) * [data[1][0 + i * n]], id_to_class_name,
classFile_to_wikiID)
query_imgs, _, _, _, _ = model(imgs[n * k:],
desc_embeddings,
query_corr_nodeIndexs,
norm=normalize)
queries.append(query_imgs)
# import ipdb; ipdb.set_trace()
acc_l2 = argmax_evaluation(support_imgs, queries, labels, n, k, q,
'l2')
acc_cosine = argmax_evaluation(support_imgs, queries, labels, n, k,
q, 'cosine')
accs_l2.append(acc_l2)
accs_cosine.append(acc_cosine)
m_l2, pm_l2 = compute_confidence_interval(accs_l2)
m_cosine, pm_cosine = compute_confidence_interval(accs_cosine)
logger.info(f'{epoch:3d}.pth: {n}-shot \t l2: {m_l2:.2f}+/-{pm_l2:.2f} \t '
f'cosine: {m_cosine:.2f}+/-{pm_cosine:.2f}')
def test(model, label, args, few_shot_params):
if args.debug:
n_test = 10
print_freq = 2
else:
n_test = 1000
print_freq = 100
test_file = args.dataset_dir + 'test.json'
test_datamgr = SetDataManager(test_file, args.dataset_dir, args.image_size,
mode = 'val',n_episode = n_test ,**few_shot_params)
loader = test_datamgr.get_data_loader(aug=False)
test_acc_record = np.zeros((n_test,))
warmup_state = torch.load(osp.join(args.checkpoint_dir, 'max_acc' + '.pth'))['params']
model.load_state_dict(warmup_state, strict=False)
model.eval()
ave_acc = Averager()
with torch.no_grad():
for i, batch in enumerate(loader, 1):
data, index_label = batch[0].cuda(), batch[1].cuda()
logits = model(data, 'test')
acc = count_acc(logits, label)
ave_acc.add(acc)
test_acc_record[i - 1] = acc
if i % print_freq == 0:
print('batch {}: {:.2f}({:.2f})'.format(i, ave_acc.item() * 100, acc * 100))
m, pm = compute_confidence_interval(test_acc_record)
# print('Val Best Epoch {}, Acc {:.4f}, Test Acc {:.4f}'.format(trlog['max_acc_epoch'], trlog['max_acc'],
# ave_acc.item()))
print('Test Acc {:.4f} + {:.4f}'.format(m, pm))
acc_str = '%4.2f' % (m * 100)
with open(args.save_dir + '/result.txt', 'a') as f:
f.write('%s %s\n' % (acc_str, args.name))
ave_acc = Averager()
label = torch.arange(args.validation_way).repeat(args.query)
if torch.cuda.is_available():
label = label.type(torch.cuda.LongTensor)
else:
label = label.type(torch.LongTensor)
for i, batch in enumerate(loader, 1):
if torch.cuda.is_available():
data, _ = [_.cuda() for _ in batch]
else:
data = batch[0]
k = args.validation_way * args.shot
data_shot, data_query = data[:k], data[k:]
logits = model(data_shot, data_query)
acc = count_acc(logits, label)
ave_acc.add(acc)
test_acc_record[i - 1] = acc
print('batch {}: {:.2f}({:.2f})'.format(i,
ave_acc.item() * 100,
acc * 100))
m, pm = compute_confidence_interval(test_acc_record)
print('Val Best Acc {:.4f}, Test Acc {:.4f}'.format(
trlog['max_acc'], ave_acc.item()))
print('Test Acc {:.4f} + {:.4f}'.format(m, pm))
print(trlog)
test_data = SVD.transform(test_data)
x_train = torch.cuda.FloatTensor(train_data)
x_test = torch.nn.functional.normalize(torch.cuda.FloatTensor(test_data), dim=1, p=2)
adj = utils.generate_graphs(
x_train, k=examples_per_class, examples_per_class=examples_per_class, num_classes=num_classes)
x_train = torch.nn.functional.normalize(x_train, dim=1, p=2)
x_train_filtered, y_train_filtered = utils.generate_filtered_features(
x_train, adj, args.filter, alpha, num_classes=num_classes, examples_per_class=examples_per_class)
x_train_filtered = torch.nn.functional.normalize(x_train_filtered, dim=1, p=2)
results_1nn.append(utils.nearest_neighbor_classifier(
x_train, x_test, y_train, y_test))
results_filtered.append(utils.nearest_neighbor_classifier(
x_train_filtered, x_test, y_train_filtered, y_test))
results_ncm.append(utils.nearest_mean_classifier(torch.nn.functional.normalize(x_train, dim=1, p=2), torch.cuda.LongTensor(
y_train), torch.nn.functional.normalize(x_test, dim=1, p=2), torch.cuda.LongTensor(y_test)))
if run == 0:
print("Shape train {}, shape test {}".format(x_train_filtered.shape,x_test.shape))
mean_1nn, confiance_1nn = utils.compute_confidence_interval(results_1nn)
mean_filtered, confiance_filtered = utils.compute_confidence_interval(results_filtered)
mean_ncm, confiance_ncm = utils.compute_confidence_interval(results_ncm)
print("Results 1nn {}+-{}, Filtered {}+-{}, NCM {}+-{}".format(
mean_1nn, confiance_1nn, mean_filtered, confiance_filtered, mean_ncm, confiance_ncm))
