def valid_model(dataLoader, epoch_number, model, cfg, criterion, logger,
device, **kwargs):
model.eval()
num_classes = dataLoader.dataset.get_num_classes()
fusion_matrix = FusionMatrix(num_classes)
with torch.no_grad():
all_loss = AverageMeter()
acc = AverageMeter()
func = torch.nn.Softmax(dim=1)
for i, (image, label, meta) in enumerate(dataLoader):
image, label = image.to(device), label.to(device)
feature = model(image, feature_flag=True)
output = model(feature, classifier_flag=True)
loss = criterion(output, label)
score_result = func(output)
now_result = torch.argmax(score_result, 1)
all_loss.update(loss.data.item(), label.shape[0])
fusion_matrix.update(now_result.cpu().numpy(), label.cpu().numpy())
now_acc, cnt = accuracy(now_result.cpu().numpy(),
label.cpu().numpy())
acc.update(now_acc, cnt)
pbar_str = "------- Valid: Epoch:{:>3d} Valid_Loss:{:>5.3f} Valid_Acc:{:>5.2f}%-------".format(
epoch_number, all_loss.avg, acc.avg * 100)
logger.info(pbar_str)
return acc.avg, all_loss.avg
def valid_model(dataLoader, epoch_number, model, cfg, criterion, logger,
device, rank, distributed, **kwargs):
model.eval()
with torch.no_grad():
all_loss = AverageMeter()
acc_avg = AverageMeter()
func = torch.nn.Sigmoid() \
if cfg.LOSS.LOSS_TYPE in ['FocalLoss', 'ClassBalanceFocal'] else \
torch.nn.Softmax(dim=1)
for i, (image, label, meta) in enumerate(dataLoader):
image, label = image.to(device), label.to(device)
feature = model(image, feature_flag=True)
output = model(feature, classifier_flag=True, label=label)
loss = criterion(output, label, feature=feature)
score_result = func(output)
now_result = torch.argmax(score_result, 1)
acc, cnt = accuracy(now_result.cpu().numpy(), label.cpu().numpy())
if distributed:
world_size = float(os.environ.get("WORLD_SIZE", 1))
reduced_loss = reduce_tensor(loss.data, world_size)
reduced_acc = reduce_tensor(
torch.from_numpy(np.array([acc])).cuda(), world_size)
loss = reduced_loss.cpu().data
acc = reduced_acc.cpu().data
all_loss.update(loss.data.item(), label.shape[0])
if distributed:
acc_avg.update(acc.data.item(), cnt * world_size)
else:
acc_avg.update(acc, cnt)
pbar_str = "------- Valid: Epoch:{:>3d} Valid_Loss:{:>5.3f} Valid_Acc:{:>5.2f}%-------".format(
epoch_number, all_loss.avg, acc_avg.avg * 100)
if rank == 0:
logger.info(pbar_str)
return acc_avg.avg, all_loss.avg
def train_model(trainLoader,
model,
epoch,
epoch_number,
optimizer,
combiner,
criterion,
cfg,
logger,
rank=0,
**kwargs):
if cfg.EVAL_MODE:
model.eval()
else:
model.train()
trainLoader.dataset.update(epoch)
combiner.update(epoch)
criterion.update(epoch)
start_time = time.time()
number_batch = len(trainLoader)
all_loss = AverageMeter()
acc = AverageMeter()
for i, (image, label, meta) in enumerate(trainLoader):
cnt = label.shape[0]
loss, now_acc = combiner.forward(model, criterion, image, label, meta)
optimizer.zero_grad()
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
optimizer.step()
all_loss.update(loss.data.item(), cnt)
acc.update(now_acc, cnt)
if i % cfg.SHOW_STEP == 0 and rank == 0:
pbar_str = "Epoch:{:>3d} Batch:{:>3d}/{} Batch_Loss:{:>5.3f} Batch_Accuracy:{:>5.2f}% ".format(
epoch, i, number_batch, all_loss.val, acc.val * 100)
logger.info(pbar_str)
end_time = time.time()
pbar_str = "---Epoch:{:>3d}/{} Avg_Loss:{:>5.3f} Epoch_Accuracy:{:>5.2f}% Epoch_Time:{:>5.2f}min---".format(
epoch, epoch_number, all_loss.avg, acc.avg * 100,
(end_time - start_time) / 60)
if rank == 0:
logger.info(pbar_str)
return acc.avg, all_loss.avg
def train_model(
trainLoader,model,epoch,epoch_number,device,optimizer,criterion,cfg,logger,**kwargs
):
if cfg.EVAL_MODE:
model.eval()
else:
model.train()
start_time = time.time()
number_batch = len(trainLoader)
func = torch.nn.Softmax(dim=1)
all_loss = AverageMeter()
acc = AverageMeter()
for i, (image, label) in enumerate(trainLoader):
cnt = label.shape[0]
image, label = image.to(device), label.to(device)
output = model(image)
loss = criterion(output, label)
now_result = torch.argmax(func(output), 1)
now_acc = accuracy(now_result.cpu().numpy(), label.cpu().numpy())[0]
optimizer.zero_grad()
loss.backward()
optimizer.step()
all_loss.update(loss.data.item(), cnt)
acc.update(now_acc, cnt)
if i % cfg.SHOW_STEP == 0:
pbar_str = "Epoch:{:>3d} Batch:{:>3d}/{} Batch_Loss:{:>5.3f} Batch_Accuracy:{:>5.2f}% ".format(
epoch, i, number_batch, all_loss.val, acc.val * 100
)
logger.info(pbar_str)
end_time = time.time()
pbar_str = "---Epoch:{:>3d}/{} Avg_Loss:{:>5.3f} Epoch_Accuracy:{:>5.2f}% Epoch_Time:{:>5.2f}min---".format(
epoch, epoch_number, all_loss.avg, acc.avg * 100, (end_time - start_time) / 60
)
logger.info(pbar_str)
return acc.avg, all_loss.avg
def valid_model(
para_dict, dataLoader, epoch_number, model, cfg, criterion, logger):
model.eval()
num_classes = dataLoader.dataset.get_num_classes()
fusion_matrix = FusionMatrix(num_classes)
num_class_list = para_dict["num_class_list"]
device = para_dict["device"]
num_class_list = [i**cfg.LOSS.RATIO for i in num_class_list]
prior_prob = num_class_list / np.sum(num_class_list)
prior_prob = torch.FloatTensor(prior_prob).to(device)
with torch.no_grad():
all_loss = AverageMeter()
acc = AverageMeter()
func = torch.nn.Softmax(dim=1)
for i, (image, label) in enumerate(dataLoader):
image, label = image.to(device), label.to(device)
output = model(image)
loss = criterion(output, label)
score_result = func(output)
score_result = score_result / prior_prob
now_result = torch.argmax(score_result, 1)
all_loss.update(loss.data.item(), label.shape[0])
fusion_matrix.update(now_result.cpu().numpy(), label.cpu().numpy())
now_acc, cnt = accuracy(now_result.cpu().numpy(), label.cpu().numpy())
acc.update(now_acc, cnt)
pbar_str = "------- Valid: Epoch:{:>3d} Valid_Loss:{:>5.3f} Valid_Acc:{:>5.2f}%-------".format(
epoch_number, all_loss.avg, acc.avg * 100
)
logger.info(pbar_str)
# print(fusion_matrix.get_rec_per_class())
# print(fusion_matrix.get_pre_per_class())
return acc.avg, all_loss.avg
def valid_model(dataLoader, model, device, label_map, level_label_maps):
model.eval()
num_levels = label_map.shape[1]
num_classes = dataLoader.dataset.get_num_classes()
l1_cls_num = label_map[:, 0].max().item() + 1
l2_cls_num = label_map[:, 1].max().item() + 1
virtual_cls_num = l1_cls_num + l2_cls_num - num_classes
l1_raw_cls_num = l1_cls_num - virtual_cls_num
fusion_matrix1 = FusionMatrix(num_classes)
fusion_matrix2 = FusionMatrix(l1_cls_num)
func = torch.nn.Softmax(dim=1)
# 20 + 80
acc1 = AverageMeter()
p1_acc1 = AverageMeter()
p2_acc1 = AverageMeter()
level_accs = [AverageMeter() for _ in range(num_levels)]
all_labels1 = []
all_result1 = []
# 20 + 2
acc2 = AverageMeter()
p1_acc2 = AverageMeter()
p2_acc2 = AverageMeter()
all_labels2 = []
all_result2 = []
with torch.no_grad():
for i, (image, label, meta) in enumerate(dataLoader):
image, label = image.to(device), label.to(device)
batch_size = label.shape[0]
level_scores = []
level_probs = []
for level in range(num_levels):
level_score = model(image, level)
level_scores.append(level_score)
# for each level
level_label = label_map[label, level]
level_mask = level_label >= 0
level_label1 = level_label[level_mask]
level_score1 = level_score[level_mask]
level_top1 = torch.argmax(level_score1, 1)
level_acc1, level_cnt1 = accuracy(level_top1.cpu().numpy(),
level_label1.cpu().numpy())
level_accs[level].update(level_acc1, level_cnt1)
if level == 0:
level_prob = func(level_score)
level_probs.append(level_prob)
else:
high_lcid_to_curr_lcid = level_label_maps[level - 1]
level_prob = torch.zeros(level_score.shape).cuda()
for high_lcid in range(high_lcid_to_curr_lcid.shape[0]):
curr_lcid_mask = high_lcid_to_curr_lcid[high_lcid]
if curr_lcid_mask.sum().item() > 0:
level_prob[:, curr_lcid_mask] = func(
level_score[:, curr_lcid_mask])
level_probs.append(level_prob)
# =================== 20 + 80 ============================
all_probs = torch.ones((batch_size, num_classes)).cuda()
for level in range(num_levels):
level_prob = level_probs[level]
related_lcids = label_map[:, level]
related_lcids = related_lcids[related_lcids >= 0]
unrelated_class_num1 = (label_map[:, level] < 0).sum().item()
unrelated_class_num2 = label_map.shape[
0] - related_lcids.shape[0]
assert unrelated_class_num1 == unrelated_class_num2
all_probs[:,
unrelated_class_num1:] *= level_prob[:,
related_lcids]
l1_mask1 = label < l1_raw_cls_num
l1_scores1 = all_probs[l1_mask1]
l1_labels1 = label[l1_mask1]
l1_result1 = torch.argmax(l1_scores1, 1)
l1_now_acc1, l1_cnt1 = accuracy(l1_result1.cpu().numpy(),
l1_labels1.cpu().numpy())
p1_acc1.update(l1_now_acc1, l1_cnt1)
l2_mask1 = label >= l1_raw_cls_num
l2_scores1 = all_probs[l2_mask1]
l2_labels1 = label[l2_mask1]
l2_result1 = torch.argmax(l2_scores1, 1)
l2_now_acc1, l2_cnt1 = accuracy(l2_result1.cpu().numpy(),
l2_labels1.cpu().numpy())
p2_acc1.update(l2_now_acc1, l2_cnt1)
now_result = torch.argmax(all_probs, 1)
now_acc, cnt = accuracy(now_result.cpu().numpy(),
label.cpu().numpy())
acc1.update(now_acc, cnt)
fusion_matrix1.update(now_result.cpu().numpy(),
label.cpu().numpy())
all_labels1.extend(label.cpu().numpy().tolist())
all_result1.extend(now_result.cpu().numpy().tolist())
# ====================================================================
# ===================20 + 2 =================================
l1v_scores = level_probs[0]
l1v_labels = label_map[label, 0]
l1_mask2 = l1v_labels < l1_raw_cls_num
l1_scores2 = l1v_scores[l1_mask2]
l1_labels2 = l1v_labels[l1_mask2]
l1_result2 = torch.argmax(l1_scores2, 1)
l1_now_acc2, l1_cnt2 = accuracy(l1_result2.cpu().numpy(),
l1_labels2.cpu().numpy())
p1_acc2.update(l1_now_acc2, l1_cnt2)
l2_mask2 = l1v_labels >= l1_raw_cls_num
l2_scores2 = l1v_scores[l2_mask2]
l2_labels2 = l1v_labels[l2_mask2]
l2_result2 = torch.argmax(l2_scores2, 1)
l2_now_acc2, l2_cnt2 = accuracy(l2_result2.cpu().numpy(),
l2_labels2.cpu().numpy())
p2_acc2.update(l2_now_acc2, l2_cnt2)
l1v_result = torch.argmax(l1v_scores, 1)
l1v_now_acc, l1v_cnt = accuracy(l1v_result.cpu().numpy(),
l1v_labels.cpu().numpy())
acc2.update(l1v_now_acc, l1v_cnt)
fusion_matrix2.update(l1v_result.cpu().numpy(),
l1v_labels.cpu().numpy())
all_labels2.extend(l1v_labels.cpu().numpy().tolist())
all_result2.extend(l1v_result.cpu().numpy().tolist())
# ====================================================================
print('Acc (head+tail): %.4f %d' % (acc1.avg, acc1.count))
print('Acc P1 : %.4f %d' % (p1_acc1.avg, p1_acc1.count))
print('Acc P2 : %.4f %d' % (p2_acc1.avg, p2_acc1.count))
print('Acc L1 : %.4f %d' %
(level_accs[0].avg, level_accs[0].count))
print('Acc L2 : %.4f %d' %
(level_accs[1].avg, level_accs[1].count))
print('=' * 23)
print('Acc (head+v) : %.4f %d' % (acc2.avg, acc2.count))
print('Acc P1 : %.4f %d' % (p1_acc2.avg, p1_acc2.count))
print('Acc Pv : %.4f %d' % (p2_acc2.avg, p2_acc2.count))
print('=' * 23)
return fusion_matrix1, fusion_matrix2, all_labels1, all_result1, all_labels2, all_result2
def valid_model(dataLoader,
epoch_number,
model,
logger,
device,
label_map,
level_label_maps,
stage=0):
model.eval()
num_levels = label_map.shape[1]
num_classes = dataLoader.dataset.get_num_classes()
l1_cls_num = label_map[:, 0].max().item() + 1
l2_cls_num = label_map[:, 1].max().item() + 1
virtual_cls_num = l1_cls_num + l2_cls_num - num_classes
l1_raw_cls_num = l1_cls_num - virtual_cls_num
l2_raw_cls_num = l2_cls_num
fusion_matrix = FusionMatrix(num_classes)
func = torch.nn.Softmax(dim=1)
acc = AverageMeter()
l1_acc = AverageMeter()
l2_acc = AverageMeter()
with torch.no_grad():
for i, (image, label, meta) in enumerate(dataLoader):
image, label = image.to(device), label.to(device)
batch_size = label.shape[0]
if stage == 1 or stage == 0:
level_scores = []
level_probs = []
for level in range(num_levels):
level_score = model(image, level)
level_scores.append(level_score)
if level == 0:
level_prob = func(level_score)
level_probs.append(level_prob)
else:
high_lcid_to_curr_lcid = level_label_maps[level - 1]
level_prob = torch.zeros(level_score.shape).cuda()
for high_lcid in range(
high_lcid_to_curr_lcid.shape[0]):
curr_lcid_mask = high_lcid_to_curr_lcid[high_lcid]
if curr_lcid_mask.sum().item() > 0:
level_prob[:, curr_lcid_mask] = func(
level_score[:, curr_lcid_mask])
level_probs.append(level_prob)
all_probs = torch.ones((batch_size, num_classes)).cuda()
for level in range(num_levels):
level_prob = level_probs[level]
related_lcids = label_map[:, level]
related_lcids = related_lcids[related_lcids >= 0]
unrelated_class_num1 = (label_map[:, level] <
0).sum().item()
unrelated_class_num2 = label_map.shape[
0] - related_lcids.shape[0]
assert unrelated_class_num1 == unrelated_class_num2
all_probs[:,
unrelated_class_num1:] *= level_prob[:,
related_lcids]
elif stage == 2:
all_probs = model(image)
else:
print('ERROR STAGE: %d' % stage)
exit(-1)
# if stage == 1:
# all_probs = level_probs[0]
# label = label_map[label, 0]
l1_mask = label < l1_raw_cls_num
l1_scores = all_probs[l1_mask]
l1_labels = label[l1_mask]
l1_result = torch.argmax(l1_scores, 1)
l1_now_acc, l1_cnt = accuracy(l1_result.cpu().numpy(),
l1_labels.cpu().numpy())
l1_acc.update(l1_now_acc, l1_cnt)
l2_mask = label >= l1_raw_cls_num
l2_scores = all_probs[l2_mask]
l2_labels = label[l2_mask]
l2_result = torch.argmax(l2_scores, 1)
l2_now_acc, l2_cnt = accuracy(l2_result.cpu().numpy(),
l2_labels.cpu().numpy())
l2_acc.update(l2_now_acc, l2_cnt)
now_result = torch.argmax(all_probs, 1)
fusion_matrix.update(now_result.cpu().numpy(), label.cpu().numpy())
now_acc, cnt = accuracy(now_result.cpu().numpy(),
label.cpu().numpy())
acc.update(now_acc, cnt)
pbar_str = "------- Valid: Epoch:{:>3d} Valid_Acc:{:>5.4f} P1_Acc:{:>5.4f} P2_Acc:{:>5.4f}-------".format(
epoch_number, acc.avg, l1_acc.avg, l2_acc.avg)
logger.info(pbar_str)
return acc.avg