def evaluate(nets, loader, args):
loss_meter = AverageMeter()
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
# switch to eval mode
for net in nets:
net.eval()
for i, batch_data in enumerate(loader):
# forward pass
pred, err = forward_multiscale(nets, batch_data, args)
loss_meter.update(err.data[0])
# calculate accuracy
acc, pix = accuracy(batch_data, pred)
intersection, union = intersectionAndUnion(batch_data, pred,
args.segDepth)
acc_meter.update(acc, pix)
intersection_meter.update(intersection)
union_meter.update(union)
print('[{}] iter {}, loss: {}, accuracy: {}'.format(
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), i,
err.data[0], acc))
# visualization
if args.visualize:
visualize_result(batch_data, pred, args)
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
print('[Eval Summary]:')
print('Loss: {}, Mean IoU: {:.4}, Accurarcy: {:.2f}%'.format(
loss_meter.average(), iou.mean(),
acc_meter.average() * 100))
def cal_acc(data_list, pred_folder, classes, names):
intersection_meter = AverageMeter()
union_meter = AverageMeter()
target_meter = AverageMeter()
for i, (image_path, target_path) in enumerate(data_list):
image_name = image_path.split('/')[-1].split('.')[0]
pred = np.asarray(Image.open(os.path.join(pred_folder, image_name+'.png')))
target = np.asarray(Image.open(target_path))
intersection, union, target = intersectionAndUnion(pred, target, classes)
intersection_meter.update(intersection)
union_meter.update(union)
target_meter.update(target)
accuracy = sum(intersection_meter.val) / (sum(target_meter.val) + 1e-10)
logger.info('Evaluating {0}/{1} on image {2}, accuracy {3:.4f}.'.format(i + 1, len(data_list), image_name+'.png', accuracy))
iou_class = intersection_meter.sum / (union_meter.sum + 1e-10)
accuracy_class = intersection_meter.sum / (target_meter.sum + 1e-10)
mIoU = np.mean(iou_class)
mAcc = np.mean(accuracy_class)
allAcc = sum(intersection_meter.sum) / (sum(target_meter.sum) + 1e-10)
logger.info('Eval result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.'.format(mIoU, mAcc, allAcc))
for i in range(classes):
logger.info('Class_{} result: iou/accuracy {:.4f}/{:.4f}, name: {}.'.format(i, iou_class[i], accuracy_class[i], names[i]))
def evaluate(segmentation_module, loader, args):
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
segmentation_module.eval()
for i, batch_data in enumerate(loader):
# process data
batch_data = batch_data[0]
seg_label = as_numpy(batch_data['seg_label'][0])
img_resized_list = batch_data['img_data']
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
pred = torch.zeros(1, args.num_class, segSize[0], segSize[1])
pred = Variable(pred).cuda()
for img in img_resized_list:
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, args.gpu_id)
# forward pass
pred_tmp = segmentation_module(feed_dict, segSize=segSize)
pred = pred + pred_tmp / len(args.imgSize)
_, preds = torch.max(pred.data.cpu(), dim=1)
preds = as_numpy(preds.squeeze(0))
# calculate accuracy
acc, pix = accuracy(preds, seg_label)
intersection, union = intersectionAndUnion(preds, seg_label,
args.num_class)
acc_meter.update(acc, pix)
intersection_meter.update(intersection)
union_meter.update(union)
print('[{}] iter {}, accuracy: {}'.format(
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), i, acc))
# visualization
if args.visualize:
visualize_result(
(batch_data['img_ori'], seg_label, batch_data['info']), preds,
args)
if args.precompute:
precompute_result(batch_data['info'], preds, args)
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
print('class [{}], IoU: {}'.format(i, _iou))
print('[Eval Summary]:')
print('Mean IoU: {:.4}, Accuracy: {:.2f}%'.format(
iou.mean(),
acc_meter.average() * 100))
def evaluate(segmentation_module, loader, cfg, gpu, model_name,
paper_arxiv_id):
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
time_meter = AverageMeter()
evaluator = ADE20KEvaluator(model_name=model_name,
paper_arxiv_id=paper_arxiv_id)
segmentation_module.eval()
pbar = tqdm(total=len(loader))
for batch_data in loader:
# process data
batch_data = batch_data[0]
seg_label = as_numpy(batch_data['seg_label'][0])
img_resized_list = batch_data['img_data']
torch.cuda.synchronize()
tic = time.perf_counter()
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0],
segSize[1])
scores = async_copy_to(scores, gpu)
for img in img_resized_list:
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, gpu)
# forward pass
scores_tmp = segmentation_module(feed_dict, segSize=segSize)
scores = scores + scores_tmp / len(cfg.DATASET.imgSizes)
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
torch.cuda.synchronize()
time_meter.update(time.perf_counter() - tic)
# calculate accuracy
acc, pix = accuracy(pred, seg_label)
intersection, union = intersectionAndUnion(pred, seg_label,
cfg.DATASET.num_class)
acc_meter.update(acc, pix)
intersection_meter.update(intersection)
union_meter.update(union)
evaluator.add(outputs=pred.flatten(), targets=seg_label.flatten())
if evaluator.cache_exists:
break
pbar.update(1)
evaluator.save()
def validate(val_loader, model, criterion, classes, zoom_factor):
logger.info('>>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>')
batch_time = AverageMeter()
data_time = AverageMeter()
loss_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
target_meter = AverageMeter()
model.eval()
end = time.time()
for i, (input, target) in enumerate(val_loader):
data_time.update(time.time() - end)
input = input.cuda(async=True)
target = target.cuda(async=True)
input_var = torch.autograd.Variable(input)
target_var = torch.autograd.Variable(target)
output = model(input_var)
if zoom_factor != 8:
output = F.upsample(output, size=target_var.size()[1:], mode='bilinear', align_corners=True)
loss = criterion(output, target_var)
output = output.data.max(1)[1].cpu().numpy()
target = target.cpu().numpy()
intersection, union, target = intersectionAndUnion(output, target, args.classes, args.ignore_label)
intersection_meter.update(intersection)
union_meter.update(union)
target_meter.update(target)
accuracy = sum(intersection_meter.val) / (sum(target_meter.val) + 1e-10)
loss_meter.update(loss.data, input.size(0))
batch_time.update(time.time() - end)
end = time.time()
if (i + 1) % 10 == 0:
logger.info('Test: [{}/{}] '
'Data {data_time.val:.3f} ({data_time.avg:.3f}) '
'Batch {batch_time.val:.3f} ({batch_time.avg:.3f}) '
'Loss {loss_meter.val:.4f} ({loss_meter.avg:.4f}) '
'Accuracy {accuracy:.4f}.'.format(i + 1, len(val_loader),
data_time=data_time,
batch_time=batch_time,
loss_meter=loss_meter,
accuracy=accuracy))
iou_class = intersection_meter.sum / (union_meter.sum + 1e-10)
accuracy_class = intersection_meter.sum / (target_meter.sum + 1e-10)
mIoU = np.mean(iou_class)
mAcc = np.mean(accuracy_class)
allAcc = sum(intersection_meter.sum) / (sum(target_meter.sum) + 1e-10)
logger.info('Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.'.format(mIoU, mAcc, allAcc))
for i in range(classes):
logger.info('Class_{} Result: iou/accuracy {:.4f}/{:.4f}.'.format(i, iou_class[i], accuracy_class[i]))
logger.info('<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<')
return loss_meter.avg, mIoU, mAcc, allAcc
def evaluate(segmentation_module, loader, args):
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
cls_ious_meter = AverageMeter()
cls_mean_iou_meter = AverageMeter()
eval_model = segmentation_module
eval_model.eval()
f = file(args.val_list_file).readlines()
for i in range(len(loader)):
batch_data = next(loader)[0]
# process data
seg_label = as_numpy(batch_data['seg_label'])
with torch.no_grad():
segSize = (seg_label.shape[1], seg_label.shape[2])
pred = torch.zeros(1, args.num_class, segSize[0], segSize[1])
# forward pass
pred = eval_model(batch_data, segSize=segSize)
batch_data['data'] = batch_data['data'][:,:,:,::-1]
pred += eval_model(batch_data, segSize=segSize)[:,:,:,::-1]
_, preds = torch.max(pred.data.cpu(), dim=1)
preds = as_numpy(preds.squeeze(0))
img = misc.imread(args.root_dataset+'/'+f[i].strip().split(' ')[0])
misc.imsave('./tmp/'+f[i].strip().split(' ')[0].split('/')[-1].replace('.jpg','.png'), preds[:img.shape[0],:img.shape[1]].astype(np.uint8))
preds = preds[:img.shape[0],:img.shape[1]]
seg_label = seg_label[:,:img.shape[0],:img.shape[1]]
# calculate accuracy
acc, pix = accuracy(preds, seg_label, 255)
intersection, union = intersectionAndUnion(preds, seg_label, args.num_class, 255)
acc_meter.update(acc, pix)
intersection_meter.update(intersection)
union_meter.update(union)
mean_iou = (intersection/(union+1e-10))[union!=0].mean()
print('[{}] iter {}, accuracy: {:.5f}, mIoU: {:.5f}'
.format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), i, acc, mean_iou))
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
print('[{}] class [{}], IoU: {}'.format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), i, _iou))
print('[{}] [Eval Summary]:'.format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
print('Mean IoU: {:.4}, Accuracy: {:.2f}%'
.format(iou.mean(), acc_meter.average()*100))
return iou, iou.mean()
def evaluate(segmentation_module, loader_val, args):
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
time_meter = AverageMeter()
segmentation_module.eval()
pbar = tqdm(total=len(loader_val))
for batch_data in loader_val:
batch_data = batch_data[0]
seg_label = as_numpy(batch_data["mask"][0])
torch.cuda.synchronize()
batch_data["image"] = batch_data["image"].unsqueeze(0).cuda()
#batch_data["mask"][0] = batch_data["mask"][0].cuda()
#batch_data["mask"][1] = batch_data["mask"][1].cuda()
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
scores = torch.zeros(1, args.num_class, segSize[0], segSize[1])
scores = async_copy_to(scores, args.gpu)
feed_dict = batch_data.copy()
#print(torch.max(feed_dict['image']))
# forward pass
scores, edge, att, loss = segmentation_module(feed_dict, epoch=0, segSize=segSize)
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
torch.cuda.synchronize()
# calculate accuracy
acc, pix = accuracy(pred, seg_label)
intersection, union = intersectionAndUnion(pred, seg_label, args.num_class)
intersection_meter.update(intersection)
union_meter.update(union)
acc_meter.update(acc)
# visualization
if True:# args.visualize
visualize_result(
(batch_data['image'], seg_label, batch_data["name"]),
pred, edge, att, args)
#Free up memroy
#del sal
pbar.update(1)
# summary
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
print('class [{}], IoU: {:.4f}'.format(i, _iou))
print('[Eval Summary]:')
print('Mean IoU: {:.4f}, Accuracy: {:.2f}%, Inference Time: {:.4f}s'
.format(iou.mean(), acc_meter.average()*100, time_meter.average()))
def evaluate(segmentation_module, loader, args):
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
cls_ious_meter = AverageMeter()
cls_mean_iou_meter = AverageMeter()
if args.num_gpus > 1:
eval_model = segmentation_module.module
else:
eval_model = segmentation_module
eval_model.eval()
for i in range(len(loader)):
batch_data = next(loader)[0]
# process data
seg_label = as_numpy(batch_data['seg_label'])
with torch.no_grad():
segSize = (seg_label.shape[1], seg_label.shape[2])
pred = torch.zeros(1, args.num_class, segSize[0], segSize[1])
# forward pass
pred = eval_model(batch_data, segSize=segSize)
_, preds = torch.max(pred.data.cpu(), dim=1)
preds = as_numpy(preds.squeeze(0))
# calculate accuracy
acc, pix = accuracy(preds, seg_label, 255)
intersection, union = intersectionAndUnion(preds, seg_label,
args.num_class, 255)
acc_meter.update(acc, pix)
intersection_meter.update(intersection)
union_meter.update(union)
mean_iou = (intersection / (union + 1e-10))[union != 0].mean()
print('[{}] iter {}, accuracy: {:.5f}, mIoU: {:.5f}'.format(
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), i, acc,
mean_iou))
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
print('[{}] class [{}], IoU: {}'.format(
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), i, _iou))
print('[{}] [Eval Summary]:'.format(
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
print('Mean IoU: {:.4}, Accuracy: {:.2f}%'.format(
iou.mean(),
acc_meter.average() * 100))
return iou, iou.mean()
def evaluate(segmentation_module, loader, cfg, gpu_id, result_queue):
segmentation_module.eval()
for batch_data in loader:
# process data
batch_data = batch_data[0]
seg_label = as_numpy(batch_data['seg_label'][0])
img_resized_list = batch_data['img_data']
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0],
segSize[1])
scores = async_copy_to(scores, gpu_id)
for img in img_resized_list:
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, gpu_id)
# forward pass
#scores_tmp = segmentation_module(feed_dict, segSize=segSize)
scores_tmp = predict_sliding(segmentation_module,
feed_dict, (520, 520),
cfg.DATASET.num_class,
overlap=1.0 / 3.0)
scores_tmp = nn.functional.interpolate(scores_tmp,
size=segSize,
mode='bilinear',
align_corners=False)
scores = scores + scores_tmp / len(cfg.DATASET.imgSizes)
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
# calculate accuracy and SEND THEM TO MASTER
acc, pix = accuracy(pred, seg_label)
intersection, union = intersectionAndUnion(pred, seg_label,
cfg.DATASET.num_class)
result_queue.put_nowait((acc, pix, intersection, union))
# visualization
if cfg.VAL.visualize:
visualize_result(
(batch_data['img_ori'], seg_label, batch_data['info']), pred,
os.path.join(cfg.DIR, 'result'))
def eval(loader_val, segmentation_module, args, crit):
intersection_meter = AverageMeter()
union_meter = AverageMeter()
loss_meter = AverageMeter()
segmentation_module.eval()
for batch_data in loader_val:
batch_data = batch_data[0]
seg_label = as_numpy(batch_data["mask"][0])
torch.cuda.synchronize()
batch_data["image"] = batch_data["image"].unsqueeze(0).cuda()
print(batch_data["image"].shape)
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
scores = torch.zeros(1, args.num_class, segSize[0], segSize[1])
scores = async_copy_to(scores, args.gpu)
print("the score:", scores)
feed_dict = batch_data.copy()
# forward pass
scores_tmp, loss = segmentation_module(feed_dict, epoch=0, segSize=segSize)
scores = scores + scores_tmp
print("the new score:", scores)
loss_meter.update(loss)
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
print("pred shape:", pred.shape)
visualize_result(batch_data["image"].cpu().numpy(), seg_label, pred, args)
torch.cuda.synchronize()
# calculate accuracy
intersection, union = intersectionAndUnion(pred, seg_label, args.num_class)
intersection_meter.update(intersection)
union_meter.update(union)
# summary
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
if i >= 1:
print('class [{}], IoU: {:.4f}'.format(i, _iou))
print('loss: {:.4f}'.format(loss_meter.average()))
return iou[1:], loss_meter.average()
def evaluate(segmentation_module, loader, args, dev_id, result_queue):
segmentation_module.eval()
for i, batch_data in enumerate(loader):
# process data
batch_data = batch_data[0]
seg_label = as_numpy(batch_data['seg_label'][0])
img_resized_list = batch_data['img_data']
quadtree_resized_list = batch_data['quadtree']
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
pred = torch.zeros(1, args.num_class, segSize[0], segSize[1])
pred = Variable(pred).cuda()
for scale, img in enumerate(img_resized_list):
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
if args.eval_mode == 'gt':
feed_dict['qtree'] = quadtree_resized_list[scale]
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, dev_id)
# forward pass
pred_tmp = segmentation_module(feed_dict, segSize=segSize)
pred = pred + pred_tmp / len(args.imgSize)
_, preds = torch.max(pred.data.cpu(), dim=1)
preds = as_numpy(preds.squeeze(0))
# calculate accuracy and SEND THEM TO MASTER
acc, pix = accuracy(preds, seg_label)
intersection, union = intersectionAndUnion(preds, seg_label,
args.num_class)
result_queue.put_nowait((acc, pix, intersection, union))
# visualization
if args.visualize:
visualize_result(
(batch_data['img_ori'], seg_label, batch_data['info']), preds,
args)
def evaluate(segmentation_module, loader, cfg, gpu_id, result_queue):
segmentation_module.eval()
for i, batch_data in enumerate(loader):
# process data
batch_data = batch_data[0]
seg_label = as_numpy(batch_data['seg_label'][0])
img_resized_list = batch_data['img_data']
img_ref_resized_list = batch_data['img_refs']
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0],
segSize[1])
scores = async_copy_to(scores, gpu_id)
zip_list = zip(img_resized_list, img_ref_resized_list)
for img, img_refs in zip_list:
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
feed_dict['img_refs'] = img_refs
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, gpu_id)
scores_tmp = segmentation_module(feed_dict, segSize=segSize)
scores = scores + scores_tmp / len(cfg.DATASET.imgSizes)
#scores = scores_tmp
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
# calculate accuracy and SEND THEM TO MASTER
acc, pix = accuracy(pred, seg_label)
intersection, union = intersectionAndUnion(pred, seg_label,
cfg.DATASET.num_class)
result_queue.put_nowait((acc, pix, intersection, union))
# visualization
if cfg.VAL.visualize:
visualize_result(
(batch_data['img_ori'], seg_label, batch_data['info']), pred,
os.path.join(cfg.DIR, 'result'))
def get_metrics(pred, data):
metric = {}
# material
# valid... is originally set to 0 then turned to 1 if it is enabled
metric['valid_material'] = data['valid_material']
if metric['valid_material']:
metric['material'] = {}
material_gt, material_pred = data['seg_material'], pred['material']
metric["material"]["acc"] = ((material_gt == material_pred) *
(material_gt > 0)).sum() # ignore 0
metric["material"]["pixel"] = (material_gt > 0).sum() # ignore 0
metric["material"][
"inter"], metric["material"]["uni"] = intersectionAndUnion(
material_pred, material_gt,
broden_dataset.nr['material'] - 1) # ignore 0
return metric
def evaluate(segmentation_module, loader, args, dev_id, result_queue):
segmentation_module.eval()
for i, batch_data in enumerate(loader):
# process data
batch_data = batch_data[0]
seg_label = as_numpy(batch_data['seg_label'][0])
img_resized_list = batch_data['img_data']
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
pred = torch.zeros(1, args.num_class, segSize[0], segSize[1])
for img in img_resized_list:
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, dev_id)
# forward pass
pred_tmp = segmentation_module(feed_dict, segSize=segSize)
pred = pred + pred_tmp.cpu() / len(args.imgSize)
_, preds = torch.max(pred.data.cpu(), dim=1)
preds = as_numpy(preds.squeeze(0))
# calculate accuracy and SEND THEM TO MASTER
acc, pix = accuracy(preds, seg_label)
intersection, union = intersectionAndUnion(preds, seg_label, args.num_class)
result_queue.put_nowait((acc, pix, intersection, union))
# visualization
if args.visualize:
visualize_result(
(batch_data['img_ori'], seg_label, batch_data['info']),
preds, args)
def evaluate(segmentation_module, loader, args, gpu_id, result_queue):
segmentation_module.eval()
for batch_data in loader:
# process data
batch_data = batch_data[0]
seg_label = as_numpy(batch_data['seg_label'][0])
img_resized_list = batch_data['img_data']
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
scores = torch.zeros(1, args.num_class, segSize[0], segSize[1])
scores = async_copy_to(scores, gpu_id)
for img in img_resized_list:
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, gpu_id)
# forward pass
scores_tmp = segmentation_module(feed_dict, segSize=segSize)
scores = scores + scores_tmp / len(args.imgSize)
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
# calculate accuracy and SEND THEM TO MASTER
acc, pix = accuracy(pred, seg_label)
intersection, union = intersectionAndUnion(pred, seg_label,
args.num_class)
result_queue.put_nowait((acc, pix, intersection, union))
# visualization
if args.visualize:
visualize_result(
(batch_data['img_ori'], seg_label, batch_data['info']), pred,
args)
def eval(loader_val, segmentation_module, crit):
intersection_meter = AverageMeter()
union_meter = AverageMeter()
loss_meter = AverageMeter()
segmentation_module.eval()
for batch_data in loader_val:
batch_data = batch_data[0]
seg_label = as_numpy(batch_data["mask"])
torch.cuda.synchronize()
batch_data["image"] = batch_data["image"].unsqueeze(0).cuda()
batch_data["mask"] = batch_data["mask"].cuda()
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
scores = torch.zeros(1, 4, segSize[0], segSize[1])
feed_dict = batch_data.copy()
#print(torch.max(feed_dict['image']))
# forward pass
scores, loss = segmentation_module(feed_dict,
epoch=0,
segSize=segSize)
loss_meter.update(loss)
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
torch.cuda.synchronize()
# calculate accuracy
intersection, union = intersectionAndUnion(pred, seg_label, 4)
intersection_meter.update(intersection)
union_meter.update(union)
# summary
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
if i >= 1:
print('class [{}], IoU: {:.4f}'.format(i, _iou))
print('loss: {:.4f}'.format(loss_meter.average()))
return loss_meter.average()
def evaluate(segmentation_module, loader, cfg, task, gpu):
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
time_meter = AverageMeter()
segmentation_module.eval()
save_pred = cfg.ATTACK[task + '_save_pred']
save_image = cfg.ATTACK[task + '_save_pred_img']
preds = []
# confs = []
iou_map = dict()
pred_path = None
assert task in ['orig', 'pert'
], "task should be pred or orig for attack evaluation"
if save_pred or save_image:
pred_path = os.path.join(cfg.ATTACK.output_dir, 'pred')
if not os.path.isdir(pred_path):
create_dir(pred_path)
pbar = tqdm(total=len(loader))
for batch_data in loader:
# process data
batch_data = batch_data[0]
seg_label = as_numpy(batch_data['seg_label'][0])
img_resized_list = batch_data['img_data']
torch.cuda.synchronize()
tic = time.perf_counter()
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0],
segSize[1])
scores = async_copy_to(scores, gpu)
for img in img_resized_list:
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, gpu)
# forward pass
scores_tmp = segmentation_module(feed_dict, segSize=segSize)
scores = scores + scores_tmp / len(cfg.DATASET.imgSizes)
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
preds.append(pred)
# TODO: See if required
# confs.append(scores.detach().numpy())
torch.cuda.synchronize()
time_meter.update(time.perf_counter() - tic)
# calculate accuracy
acc, pix = accuracy(pred, seg_label)
intersection, union = intersectionAndUnion(pred, seg_label,
cfg.DATASET.num_class)
acc_meter.update(acc, pix)
iou_map[batch_data['info'].split('/')[-1].replace(
'.jpg', '')] = intersection.sum() / union.sum()
intersection_meter.update(intersection)
union_meter.update(union)
# save prediction
if save_pred:
img_name = batch_data['info'].split('/')[-1]
np.save(
os.path.join(
pred_path,
task + '_pred_' + img_name.replace('.jpg', '.npy')),
np.array(pred))
# save predicted image
if save_image:
img_name = batch_data['info'].split('/')[-1]
image_path = os.path.join(
pred_path, task + '_image_' + img_name.replace('.jpg', '.png'))
encode_save_image(pred, image_path)
pbar.update(1)
# summary
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
print('class [{}], IoU: {:.4f}'.format(i, _iou))
print('[Eval Summary]:')
print(
'Mean IoU: {:.4f}, Accuracy: {:.2f}%, Inference Time: {:.4f}s'.format(
iou.mean(),
acc_meter.average() * 100, time_meter.average()))
results = dict(iou=iou_map,
class_iou=iou,
accuracy=acc_meter.average() * 100)
return np.array(preds), None, results
def evaluate(segmentation_module, loader, cfg, gpu_id, result_queue):
segmentation_module.eval()
for i, batch_data in enumerate(loader):
# process data
batch_data = batch_data[0]
seg_label = as_numpy(batch_data['seg_label'][0])
img_resized_list = batch_data['img_data']
img_ref_rgb_resized_list = batch_data['img_refs_rgb']
img_ref_mask_resized_list = batch_data['img_refs_mask']
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0],
segSize[1])
scores = async_copy_to(scores, gpu_id)
if cfg.is_debug:
zip_list = zip(img_resized_list[-2:-1],
img_ref_rgb_resized_list[-2:-1],
img_ref_mask_resized_list[-2:-1])
else:
zip_list = zip(img_resized_list, img_ref_rgb_resized_list,
img_ref_mask_resized_list)
for img, img_refs_rgb, img_refs_mask in zip_list:
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
feed_dict['img_refs_rgb'] = img_refs_rgb
feed_dict['img_refs_mask'] = img_refs_mask
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, gpu_id)
# forward pass
if cfg.is_debug:
scores_tmp, qread, qval, qk_b, mk_b, mv_b, p, feature_enc, feature_memory = segmentation_module(
feed_dict, segSize=segSize)
np.save('debug/qread_%03d.npy' % (i),
qread.detach().cpu().float().numpy())
np.save('debug/qval_%03d.npy' % (i),
qval.detach().cpu().float().numpy())
np.save('debug/qk_b_%03d.npy' % (i),
qk_b.detach().cpu().float().numpy())
np.save('debug/mk_b_%03d.npy' % (i),
mk_b.detach().cpu().float().numpy())
np.save('debug/mv_b_%03d.npy' % (i),
mv_b.detach().cpu().float().numpy())
np.save('debug/p_%03d.npy' % (i),
p.detach().cpu().float().numpy())
np.save('debug/feature_enc_%03d.npy' % (i),
feature_enc[-1].detach().cpu().float().numpy())
np.save('debug/feature_memory_%03d.npy' % (i),
feature_memory[-1].detach().cpu().float().numpy())
print(batch_data['info'])
else:
if cfg.eval_att_voting:
scores_tmp, qread, qval, qk_b, mk_b, mv_b, p, feature_enc, feature_memory = segmentation_module(
feed_dict, segSize=segSize)
height, width = qread.shape[-2], qread.shape[-1]
assert p.shape[0] == height * width
img_refs_mask_resize = nn.functional.interpolate(
img_refs_mask[0].cuda(),
size=(height, width),
mode='nearest')
img_refs_mask_resize_flat = img_refs_mask_resize[:, 0, :, :].view(
img_refs_mask_resize.shape[0], -1)
mask_voting_flat = torch.mm(img_refs_mask_resize_flat,
p)
mask_voting = mask_voting_flat.view(
mask_voting_flat.shape[0], height, width)
mask_voting = torch.unsqueeze(mask_voting, 0)
scores_tmp = nn.functional.interpolate(
mask_voting[:, 1:],
size=segSize,
mode='bilinear',
align_corners=False)
'''np.save('debug/p_%03d.npy'%(i), p.detach().cpu().float().numpy())
np.save('debug/img_refs_mask_%03d.npy'%(i), img_refs_mask.cuda().detach().cpu().float().numpy())
np.save('debug/img_refs_mask_resize_%03d.npy'%(i), img_refs_mask_resize.detach().cpu().float().numpy())
np.save('debug/img_refs_mask_resize_flat_%03d.npy'%(i), img_refs_mask_resize_flat.detach().cpu().float().numpy())
np.save('debug/mask_voting_flat_%03d.npy'%(i), mask_voting_flat.detach().cpu().float().numpy())
np.save('debug/mask_voting_%03d.npy'%(i), mask_voting.detach().cpu().float().numpy())'''
else:
scores_tmp = segmentation_module(feed_dict,
segSize=segSize)
scores = scores + scores_tmp / len(cfg.DATASET.imgSizes)
#scores = scores_tmp
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
# calculate accuracy and SEND THEM TO MASTER
acc, pix = accuracy(pred, seg_label)
intersection, union = intersectionAndUnion(pred, seg_label,
cfg.DATASET.num_class)
result_queue.put_nowait((acc, pix, intersection, union))
# visualization
if cfg.VAL.visualize:
visualize_result(
(batch_data['img_ori'], seg_label, batch_data['info']), pred,
os.path.join(cfg.DIR, 'result'))
mask[label == 255] = 0
mask[label == 0] = 1
print(label.shape)
input = transforms.ToTensor()(input)
input = transforms.Normalize([0.5, 0.5, 0.5],
[0.5, 0.5, 0.5])(input)
input = input.view((-1, ) + input.shape)
# print(input.shape)
output = model(input.to(device))
# print(output[:,1].shape)
# print(output.shape)
result = np.zeros(size + (3, ))
new_mask = torch.argmax(output, dim=1)
print(new_mask.shape)
new_mask = new_mask.cpu().numpy()
area_intersection, area_union = intersectionAndUnion(
new_mask[0, :, :], mask, num_classes)
MIou = area_intersection[0] / area_union[0]
print("MIou:%.3f" % MIou)
print(area_intersection, area_union)
for i in range(256):
for j in range(256):
# print(output[0][:,i,j])
# print(new_mask[i,j,:])
if new_mask[0, i, j] == 0:
result[i, j] = COLOR_DICT[0]
else:
result[i, j] = COLOR_DICT[1]
# result = cv2.resize(result, (512,512))
cv2.namedWindow("test", cv2.WINDOW_NORMAL)
cv2.imshow("test", result)
cv2.imwrite("data/membrane/result/15.png", result)
def train(train_loader, model, criterion, optimizer, epoch, zoom_factor, batch_size, aux_weight, fcw):
batch_time = AverageMeter()
data_time = AverageMeter()
main_loss_meter = AverageMeter()
aux_loss_meter = AverageMeter()
loss_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
target_meter = AverageMeter()
model.train()
fcw.train()
world_size = dist.get_world_size()
rank = dist.get_rank()
end = time.time()
for i, (input, target) in enumerate(train_loader):
# to avoid bn problem in ppm module with bin size 1x1, sometimes n may get 1 on one gpu during the last batch, so just discard
# if input.shape[0] < batch_size:
# continue
data_time.update(time.time() - end)
current_iter = (epoch - 1) * len(train_loader) + i + 1
max_iter = args.epochs * len(train_loader)
if args.net_type == 0:
index_split = 4
elif args.net_type in [1, 2, 3]:
index_split = 5
poly_learning_rate(optimizer, args.base_lr, current_iter, max_iter, power=args.power, index_split=index_split)
input = input.cuda()
input_var = torch.autograd.Variable(input)
fcw_input = fcw(input_var)
output, aux = model(input_var, fcw_input)
if zoom_factor != 8:
h = int((target.size()[1]-1) / 8 * zoom_factor + 1)
w = int((target.size()[2] - 1) / 8 * zoom_factor + 1)
# 'nearest' mode doesn't support downsampling operation and while 'bilinear' mode is also fine
target = F.upsample(target.unsqueeze(1).float(), size=(h, w), mode='bilinear').squeeze(1).long()
target = target.data.cuda(async=True)
target_var = torch.autograd.Variable(target)
main_loss = criterion(output, target_var) / world_size
aux_loss = criterion(aux, target_var) / world_size
loss = main_loss + aux_weight * aux_loss
optimizer.zero_grad()
loss.backward()
average_gradients(model)
optimizer.step()
output = output.data.max(1)[1].cpu().numpy()
target = target.cpu().numpy()
intersection, union, target = intersectionAndUnion(output, target, args.classes, args.ignore_label)
intersection_meter.update(intersection)
union_meter.update(union)
target_meter.update(target)
accuracy = sum(intersection_meter.val) / (sum(target_meter.val) + 1e-10)
reduced_loss = loss.data.clone()
reduced_main_loss = main_loss.data.clone()
reduced_aux_loss = aux_loss.data.clone()
dist.all_reduce(reduced_loss)
dist.all_reduce(reduced_main_loss)
dist.all_reduce(reduced_aux_loss)
main_loss_meter.update(reduced_main_loss[0], input.size(0))
aux_loss_meter.update(reduced_aux_loss[0], input.size(0))
loss_meter.update(reduced_loss[0], input.size(0))
batch_time.update(time.time() - end)
end = time.time()
# calculate remain time
remain_iter = max_iter - current_iter
remain_time = remain_iter * batch_time.avg
t_m, t_s = divmod(remain_time, 60)
t_h, t_m = divmod(t_m, 60)
remain_time = '{:02d}:{:02d}:{:02d}'.format(int(t_h), int(t_m), int(t_s))
if rank == 0:
if (i + 1) % args.print_freq == 0:
logger.info('Epoch: [{}/{}][{}/{}] '
'Data {data_time.val:.3f} ({data_time.avg:.3f}) '
'Batch {batch_time.val:.3f} ({batch_time.avg:.3f}) '
'Remain {remain_time} '
'MainLoss {main_loss_meter.val:.4f} '
'AuxLoss {aux_loss_meter.val:.4f} '
'Loss {loss_meter.val:.4f} '
'Accuracy {accuracy:.4f}.'.format(epoch, args.epochs, i + 1, len(train_loader),
batch_time=batch_time,
data_time=data_time,
remain_time=remain_time,
main_loss_meter=main_loss_meter,
aux_loss_meter=aux_loss_meter,
loss_meter=loss_meter,
accuracy=accuracy))
writer.add_scalar('loss_train_batch', main_loss_meter.val, current_iter)
writer.add_scalar('mIoU_train_batch', np.mean(intersection / (union + 1e-10)), current_iter)
writer.add_scalar('mAcc_train_batch', np.mean(intersection / (target + 1e-10)), current_iter)
writer.add_scalar('allAcc_train_batch', accuracy, current_iter)
iou_class = intersection_meter.sum / (union_meter.sum + 1e-10)
accuracy_class = intersection_meter.sum / (target_meter.sum + 1e-10)
mIoU = np.mean(iou_class)
mAcc = np.mean(accuracy_class)
allAcc = sum(intersection_meter.sum) / (sum(target_meter.sum) + 1e-10)
if rank == 0:
logger.info('Train result at epoch [{}/{}]: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.'.format(epoch, args.epochs, mIoU, mAcc, allAcc))
return main_loss_meter.avg, mIoU, mAcc, allAcc
def evaluate(nets, loader, history, epoch, args):
print('Evaluating at {} epochs...'.format(epoch))
loss_meter = AverageMeter()
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
# switch to eval mode
for net in nets:
net.eval()
for i, batch_data in enumerate(loader):
# forward pass
torch.cuda.empty_cache()
pred, err, agreemap = forward_with_loss_agreemap(nets,
batch_data,
args,
is_train=False)
loss_meter.update(err.data[0])
print('[Eval] iter {}, loss: {}'.format(i, err.data[0]))
# calculate accuracy
acc, pix = accuracy(batch_data, pred)
acc_meter.update(acc, pix)
intersection, union = intersectionAndUnion(batch_data, pred,
args.num_class)
intersection_meter.update(intersection)
union_meter.update(union)
# visualization
visualize_2(batch_data, pred, agreemap, epoch, args)
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
print('class [{}], IoU: {}'.format(trainID2Class[i], _iou))
print('[Eval Summary]:')
print('Epoch: {}, Loss: {}, Mean IoU: {:.4}, Accurarcy: {:.2f}%'.format(
epoch, loss_meter.average(), iou.mean(),
acc_meter.average() * 100))
history['val']['epoch'].append(epoch)
history['val']['err'].append(loss_meter.average())
history['val']['acc'].append(acc_meter.average())
history['val']['mIoU'].append(iou.mean())
# Plot figure
if epoch > 0:
print('Plotting loss figure...')
fig = plt.figure()
plt.plot(np.asarray(history['train']['epoch']),
np.log(np.asarray(history['train']['err'])),
color='b',
label='training')
plt.plot(np.asarray(history['val']['epoch']),
np.log(np.asarray(history['val']['err'])),
color='c',
label='validation')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Log(loss)')
fig.savefig('{}/loss.png'.format(args.ckpt), dpi=200)
plt.close('all')
fig = plt.figure()
plt.plot(history['train']['epoch'],
history['train']['acc'],
color='b',
label='training')
plt.plot(history['val']['epoch'],
history['val']['acc'],
color='c',
label='validation')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
fig.savefig('{}/accuracy.png'.format(args.ckpt), dpi=200)
plt.close('all')
def evaluate(segmentation_module, loader, cfg, gpu):
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
groundthuth_meter = AverageMeter()
time_meter = AverageMeter()
segmentation_module.eval()
pbar = tqdm(total=len(loader))
for batch_data in loader:
# process data
batch_data = batch_data[0]
seg_label = as_numpy(batch_data['seg_label'][0])
img_resized_list = batch_data['img_data']
torch.cuda.synchronize()
tic = time.perf_counter()
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0],
segSize[1])
scores = async_copy_to(scores, gpu)
for img in img_resized_list:
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, gpu)
# forward pass
scores_tmp = segmentation_module(feed_dict, segSize=segSize)
scores = scores + scores_tmp / 1 #len(cfg.DATASET.imgSizes)
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
torch.cuda.synchronize()
time_meter.update(time.perf_counter() - tic)
# calculate accuracy
acc, pix = accuracy(pred, seg_label)
intersection, union, groundthuth = intersectionAndUnion(
pred, seg_label, cfg.DATASET.num_class)
acc_meter.update(acc, pix)
intersection_meter.update(intersection)
union_meter.update(union)
groundthuth_meter.update(groundthuth)
# visualization
if cfg.VAL.visualize:
visualize_result(
(batch_data['img_ori'], seg_label, batch_data['info']), pred,
os.path.join(cfg.DIR, 'val'))
pbar.update(1)
# summary
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
print('class [{}], IoU: {:.4f}'.format(i, _iou))
sensitive_iou = intersection_meter.sum / (groundthuth_meter.sum + 1e-10)
for i, _iou in enumerate(sensitive_iou):
print('class [{}], Sensitive_IoU: {:.4f}'.format(i, _iou))
print('[Eval Summary]:')
print(
'Mean IoU: {:.4f}, Accuracy: {:.2f}%, Inference Time: {:.4f}s'.format(
iou.mean(),
acc_meter.average() * 100, time_meter.average()))
def evaluate(nets, loader, loader_2, history, epoch, args):
print('Evaluating at {} epochs...'.format(epoch))
loss_meter = AverageMeter()
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
loss_meter_2 = AverageMeter()
acc_meter_2 = AverageMeter()
intersection_meter_2 = AverageMeter()
union_meter_2 = AverageMeter()
# switch to eval mode
for net in nets:
net.eval()
for i, batch_data in enumerate(loader):
# forward pass
pred, err = forward_with_loss(nets, batch_data, is_train=False)
loss_meter.update(err.data.item())
if i % args.disp_iter == 0:
print('[Eval] iter {}, loss: {}'.format(i, err.data.item()))
# calculate accuracy
acc, pix = accuracy(batch_data, pred)
acc_meter.update(acc, pix)
intersection, union = intersectionAndUnion(batch_data, pred,
args.num_class)
intersection_meter.update(intersection)
union_meter.update(union)
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
print('class [{}], IoU: {}'.format(trainID2Class[i], _iou))
print('[Cityscapes Eval Summary]:')
print('Epoch: {}, Loss: {}, Mean IoU: {:.4}, Accuracy: {:.2f}%'.format(
epoch, loss_meter.average(), iou.mean(),
acc_meter.average() * 100))
history['val']['epoch'].append(epoch)
history['val']['err'].append(loss_meter.average())
history['val']['acc'].append(acc_meter.average())
history['val']['mIoU'].append(iou.mean())
for i, batch_data in enumerate(loader_2):
# forward pass
pred, err = forward_with_loss(nets, batch_data, is_train=False)
loss_meter_2.update(err.data.item())
if i % args.disp_iter == 0:
print('[Eval] iter {}, loss: {}'.format(i, err.data.item()))
# calculate accuracy
acc, pix = accuracy(batch_data, pred)
acc_meter_2.update(acc, pix)
intersection, union = intersectionAndUnion(batch_data, pred,
args.num_class)
intersection_meter_2.update(intersection)
union_meter_2.update(union)
iou = intersection_meter_2.sum / (union_meter_2.sum + 1e-10)
for i, _iou in enumerate(iou):
print('class [{}], IoU: {}'.format(trainID2Class[i], _iou))
print('[BDD Eval Summary]:')
print('Epoch: {}, Loss: {}, Mean IoU: {:.4}, Accuracy: {:.2f}%'.format(
epoch, loss_meter_2.average(), iou.mean(),
acc_meter_2.average() * 100))
history['val_2']['epoch'].append(epoch)
history['val_2']['err'].append(loss_meter_2.average())
history['val_2']['acc'].append(acc_meter_2.average())
history['val_2']['mIoU'].append(iou.mean())
# Plot figure
if epoch > 0:
fig = plt.figure()
plt.plot(np.asarray(history['train']['epoch']),
np.log(np.asarray(history['train']['err'])),
color='b',
label='gta')
plt.plot(np.asarray(history['val']['epoch']),
np.log(np.asarray(history['val']['err'])),
color='c',
label='cityscapes')
plt.plot(np.asarray(history['val_2']['epoch']),
np.log(np.asarray(history['val_2']['err'])),
color='g',
label='bdd')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Log(loss)')
fig.savefig('{}/loss.png'.format(args.ckpt), dpi=200)
plt.close('all')
fig = plt.figure()
plt.plot(history['train']['epoch'],
history['train']['acc'],
color='b',
label='gta')
plt.plot(history['val']['epoch'],
history['val']['acc'],
color='c',
label='cityscapes')
plt.plot(history['val_2']['epoch'],
history['val_2']['acc'],
color='g',
label='bdd')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
fig.savefig('{}/accuracy.png'.format(args.ckpt), dpi=200)
plt.close('all')
def evaluate(segmentation_module, loader, loader_rec, cfg, gpu):
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
time_meter = AverageMeter()
segmentation_module.eval()
aurocs, auprs, fprs = [], [], []
pbar = tqdm(total=len(loader))
for batch_data, rec_data in zip(loader, loader_rec):
# process data
batch_data = batch_data[0]
seg_label = as_numpy(batch_data['seg_label'][0])
img_resized_list = batch_data['img_data']
#print(batch_data['name'])
full_name = batch_data['name']
img_folder, img_name = batch_data['name'].split('/')
del batch_data['name']
rec_data = rec_data[0]
seg_label_rec = as_numpy(rec_data['seg_label'][0])
img_resized_list_rec = rec_data['img_data']
#print(rec_data['name'])
del rec_data['name']
torch.cuda.synchronize()
tic = time.perf_counter()
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0],
segSize[1])
ft1 = torch.zeros(1, 4096, int(segSize[0] / 4),
int(segSize[1] / 4))
ft2 = torch.zeros(1, 4096, int(segSize[0] / 4),
int(segSize[1] / 4))
scores = async_copy_to(scores, gpu)
ft1 = async_copy_to(ft1, gpu)
ft2 = async_copy_to(ft2, gpu)
for img in img_resized_list:
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, gpu)
# forward pass
scores_tmp, ft_temp = segmentation_module(feed_dict,
segSize=segSize)
scores = scores + scores_tmp / len(cfg.DATASET.imgSizes)
ft_temp = nn.functional.interpolate(ft_temp,
size=ft1.shape[2:],
mode='bilinear',
align_corners=False)
ft1 = ft1 + ft_temp / len(cfg.DATASET.imgSizes)
for img in img_resized_list_rec:
feed_dict = rec_data.copy()
feed_dict['img_data'] = img
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, gpu)
# forward pass
_, ft_temp = segmentation_module(feed_dict, segSize=segSize)
ft_temp = nn.functional.interpolate(ft_temp,
size=ft2.shape[2:],
mode='bilinear',
align_corners=False)
ft2 = ft2 + ft_temp / len(cfg.DATASET.imgSizes)
tmp_scores = scores
if cfg.OOD.exclude_back:
tmp_scores = tmp_scores[:, 1:]
mask = None
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
#for evaluating MSP
if cfg.OOD.ood == "msp":
conf, _ = torch.max(tmp_scores, dim=1)
conf = as_numpy(conf.squeeze(0).cpu())
elif cfg.OOD.ood == "rec":
msp, _ = torch.max(tmp_scores, dim=1)
msp = msp.squeeze(0).cpu()
ft1 = nn.functional.normalize(ft1, dim=1)
ft2 = nn.functional.normalize(ft2, dim=1)
#ft_dist = torch.nn.SmoothL1Loss(reduction='none')(ft1, ft2)
ft_dist = nn.functional.cosine_similarity(ft1, ft2,
dim=1).unsqueeze(1)
ft_dist = nn.functional.interpolate(
ft_dist,
size=segSize,
mode='bilinear',
align_corners=False)[0, 0].cpu()
conf_rec = ft_dist
t = 0.999
conf = msp * (msp > t).float() + conf_rec * (msp <= t).float()
#conf = 1- (1 - conf_rec) * (msp <= 0.99).float()
conf = as_numpy(conf.squeeze(0).cpu())
res = eval_ood_measure(conf, seg_label, cfg, mask=mask)
if res is not None:
auroc, aupr, fpr = res
aurocs.append(auroc)
auprs.append(aupr), fprs.append(fpr)
else:
pass
res1 = eval_ood_measure(msp, seg_label, cfg, mask=mask)
if res is not None:
auroc1, aupr1, fpr1 = res1
#aurocs.append(auroc); auprs.append(aupr), fprs.append(fpr)
else:
pass
torch.cuda.synchronize()
time_meter.update(time.perf_counter() - tic)
# calculate accuracy
acc, pix = accuracy(pred, seg_label)
intersection, union = intersectionAndUnion(pred, seg_label,
cfg.DATASET.num_class)
acc_meter.update(acc, pix)
intersection_meter.update(intersection)
union_meter.update(union)
# visualization
if cfg.VAL.visualize:
visualize_result(
(batch_data['img_ori'], seg_label, batch_data['info']), pred,
os.path.join(cfg.DIR, 'result'))
pbar.update(1)
torch.cuda.empty_cache()
# summary
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
print('class [{}], IoU: {:.4f}'.format(i, _iou))
print('[Eval Summary]:')
print(
'Mean IoU: {:.4f}, Accuracy: {:.2f}%, Inference Time: {:.4f}s'.format(
iou.mean(),
acc_meter.average() * 100, time_meter.average()))
print("mean auroc = ", np.mean(aurocs), "mean aupr = ", np.mean(auprs),
" mean fpr = ", np.mean(fprs))
def evaluate(segmentation_module, loader, cfg, gpu):
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
time_meter = AverageMeter()
segmentation_module.eval()
aurocs, auprs, fprs = [], [], []
pbar = tqdm(total=len(loader))
for batch_data in loader:
# process data
batch_data = batch_data[0]
seg_label = as_numpy(batch_data['seg_label'][0])
img_resized_list = batch_data['img_data']
torch.cuda.synchronize()
tic = time.perf_counter()
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0],
segSize[1])
scores = async_copy_to(scores, gpu)
for img in img_resized_list:
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, gpu)
# forward pass
scores_tmp = segmentation_module(feed_dict, segSize=segSize)
scores = scores + scores_tmp / len(cfg.DATASET.imgSizes)
tmp_scores = scores
if cfg.OOD.exclude_back:
tmp_scores = tmp_scores[:, 1:]
mask = None
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
#for evaluating MSP
if cfg.OOD.ood == "msp":
conf, _ = torch.max(nn.functional.softmax(tmp_scores, dim=1),
dim=1)
conf = as_numpy(conf.squeeze(0).cpu())
elif cfg.OOD.ood == "maxlogit":
conf, _ = torch.max(tmp_scores, dim=1)
conf = as_numpy(conf.squeeze(0).cpu())
elif cfg.OOD.ood == "background":
conf = tmp_scores[:, 0]
conf = as_numpy(conf.squeeze(0).cpu())
elif cfg.OOD.ood == "crf":
import pydensecrf.densecrf as dcrf
from pydensecrf.utils import unary_from_softmax, create_pairwise_bilateral, create_pairwise_gaussian
ch, h, w = scores.squeeze(0).size()
d = dcrf.DenseCRF2D(h, w, ch) # width, height, nlabels
tmp_scores = as_numpy(
nn.functional.softmax(tmp_scores, dim=1).squeeze(0))
tmp_scores = as_numpy(tmp_scores)
U = unary_from_softmax(tmp_scores)
d.setUnaryEnergy(U)
pairwise_energy = create_pairwise_bilateral(sdims=(10, 10),
schan=13,
img=tmp_scores,
chdim=0)
d.addPairwiseEnergy(pairwise_energy, compat=10)
# Run inference for 100 iterations
Q_unary = d.inference(100)
# The Q is now the approximate posterior, we can get a MAP estimate using argmax.
map_soln_unary = np.argmax(Q_unary, axis=0)
# Unfortunately, the DenseCRF flattens everything, so get it back into picture form.
map_soln_unary = map_soln_unary.reshape((h, w))
conf = np.max(Q_unary, axis=0).reshape((h, w))
elif cfg.OOD.ood == "crf-gauss":
import pydensecrf.densecrf as dcrf
from pydensecrf.utils import unary_from_softmax, create_pairwise_bilateral, create_pairwise_gaussian
ch, h, w = scores.squeeze(0).size()
d = dcrf.DenseCRF2D(h, w, ch) # width, height, nlabels
tmp_scores = as_numpy(
nn.functional.softmax(tmp_scores, dim=1).squeeze(0))
tmp_scores = as_numpy(tmp_scores)
U = unary_from_softmax(tmp_scores)
d.setUnaryEnergy(U)
d.addPairwiseGaussian(
sxy=3,
compat=3) # `compat` is the "strength" of this potential.
# Run inference for 100 iterations
Q_unary = d.inference(100)
# The Q is now the approximate posterior, we can get a MAP estimate using argmax.
map_soln_unary = np.argmax(Q_unary, axis=0)
# Unfortunately, the DenseCRF flattens everything, so get it back into picture form.
map_soln_unary = map_soln_unary.reshape((h, w))
conf = np.max(Q_unary, axis=0).reshape((h, w))
res = eval_ood_measure(conf, seg_label, cfg, mask=mask)
if res is not None:
auroc, aupr, fpr = res
aurocs.append(auroc)
auprs.append(aupr), fprs.append(fpr)
else:
pass
torch.cuda.synchronize()
time_meter.update(time.perf_counter() - tic)
# calculate accuracy
acc, pix = accuracy(pred, seg_label)
intersection, union = intersectionAndUnion(pred, seg_label,
cfg.DATASET.num_class)
acc_meter.update(acc, pix)
intersection_meter.update(intersection)
union_meter.update(union)
# visualization
if cfg.VAL.visualize:
visualize_result(
(batch_data['img_ori'], seg_label, batch_data['info']), pred,
os.path.join(cfg.DIR, 'result'))
pbar.update(1)
# summary
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
print('class [{}], IoU: {:.4f}'.format(i, _iou))
print('[Eval Summary]:')
print(
'Mean IoU: {:.4f}, Accuracy: {:.2f}%, Inference Time: {:.4f}s'.format(
iou.mean(),
acc_meter.average() * 100, time_meter.average()))
print("mean auroc = ", np.mean(aurocs), "mean aupr = ", np.mean(auprs),
" mean fpr = ", np.mean(fprs))
def get_metrics(pred, data):
metric = {}
# scene
metric['scene'] = {}
metric['scene']['gt'] = data['scene_label']
if metric['scene']['gt'] != -1:
metric['scene']['top1'] = (pred['scene'] == metric['scene']['gt'])
# object, part
metric['valid_object'] = data['valid_object']
if metric['valid_object']:
# object
metric['object'] = {}
object_gt, object_pred = data['seg_object'], pred['object']
metric["object"]["acc"] = ((object_gt == object_pred) * (object_gt > 0)).sum() # ignore 0
metric["object"]["pixel"] = (object_gt > 0).sum() # ignore 0
metric["object"]["inter"], metric["object"]["uni"] = intersectionAndUnion(
object_pred, object_gt, broden_dataset.nr['object'] - 1) # ignore 0
# parts
metric['part'] = []
metric["valid_part"] = data["valid_part"]
for object_part_idx, object_label in enumerate(broden_dataset.object_with_part):
if not data['valid_part'][object_part_idx]:
metric["part"].append(None)
continue
# NOTE: nr part include background 0.
nr_part = len(broden_dataset.object_part[object_label])
object_pred_mask = (object_pred == object_label)
object_gt_mask = (object_gt == object_label)
parts_gt = data["seg_part"] * object_gt_mask
parts_pred = pred['part'][object_part_idx] * object_pred_mask
_result = {}
_result["acc"] = ((parts_gt == parts_pred) * (parts_gt > 0)).sum() # ignore 0
_result["pixel"] = (parts_gt > 0).sum() # ignore 0
_result["inter"], _result["uni"] = intersection_union_part(
parts_pred, parts_gt, nr_part) # mIoU-bg, include background 0
metric["part"].append(_result)
# material
metric['valid_material'] = data['valid_material']
if metric['valid_material']:
metric['material'] = {}
material_gt, material_pred = data['seg_material'], pred['material']
metric["material"]["acc"] = ((material_gt == material_pred) * (material_gt > 0)).sum() # ignore 0
metric["material"]["pixel"] = (material_gt > 0).sum() # ignore 0
metric["material"]["inter"], metric["material"]["uni"] = intersectionAndUnion(
material_pred, material_gt, broden_dataset.nr['material'] - 1) # ignore 0
return metric
def evaluate(segmentation_module, loader, args):
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
time_meter = AverageMeter()
segmentation_module.eval()
pbar = tqdm(total=len(loader))
for batch_data in loader:
# process data
batch_data = batch_data[0]
#print(batch_data[0])
seg_label = as_numpy(batch_data['seg_label'])
img_resized_list = batch_data['img_data'].unsqueeze(0)
#print(seg_label.shape)
#print(img_resized_list.shape)
'''
# fix the dimension permutation
for idx, item in enumerate(batch_data['img_data']):
print(item.shape)
batch_data['img_data'][idx] = item.permute(0, 3, 1, 2)
print()
for item in batch_data['img_data']:
print(item.shape)
'''
torch.cuda.synchronize()
tic = time.perf_counter()
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
scores = torch.zeros(1, args.num_class, segSize[0], segSize[1])
scores = async_copy_to(scores, args.gpu)
for img in img_resized_list:
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, args.gpu)
# forward pass
scores_tmp = segmentation_module(feed_dict, segSize=segSize)
scores = scores + scores_tmp / len(args.imgSize)
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
torch.cuda.synchronize()
time_meter.update(time.perf_counter() - tic)
# calculate accuracy
acc, pix = accuracy(pred, seg_label)
intersection, union = intersectionAndUnion(pred, seg_label,
args.num_class)
acc_meter.update(acc, pix)
intersection_meter.update(intersection)
union_meter.update(union)
# visualization
if True: # args.visualize
visualize_result(
(batch_data['img_ori'], seg_label, batch_data['info']), pred,
args)
pbar.update(1)
# summary
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
print('class [{}], IoU: {:.4f}'.format(i, _iou))
print('[Eval Summary]:')
print(
'Mean IoU: {:.4f}, Accuracy: {:.2f}%, Inference Time: {:.4f}s'.format(
iou.mean(),
acc_meter.average() * 100, time_meter.average()))
def get_metrics(pred, data):
metric = {}
# scene
metric['scene'] = {}
metric['scene']['gt'] = data['scene_label']
if metric['scene']['gt'] != -1:
metric['scene']['top1'] = (pred['scene'] == metric['scene']['gt'])
# object, part
metric['valid_object'] = data['valid_object']
if metric['valid_object']:
# object
metric['object'] = {}
object_gt, object_pred = data['seg_object'], pred['object']
metric["object"]["acc"] = ((object_gt == object_pred) * (object_gt > 0)).sum() # ignore 0
metric["object"]["pixel"] = (object_gt > 0).sum() # ignore 0
metric["object"]["inter"], metric["object"]["uni"] = intersectionAndUnion(
object_pred, object_gt, broden_dataset.nr['object'] - 1) # ignore 0
# parts
metric['part'] = []
metric["valid_part"] = data["valid_part"]
for object_part_idx, object_label in enumerate(broden_dataset.object_with_part):
if not data['valid_part'][object_part_idx]:
metric["part"].append(None)
continue
# NOTE: nr part include background 0.
nr_part = len(broden_dataset.object_part[object_label])
object_pred_mask = (object_pred == object_label)
object_gt_mask = (object_gt == object_label)
parts_gt = data["seg_part"] * object_gt_mask
parts_pred = pred['part'][object_part_idx] * object_pred_mask
_result = {}
_result["acc"] = ((parts_gt == parts_pred) * (parts_gt > 0)).sum() # ignore 0
_result["pixel"] = (parts_gt > 0).sum() # ignore 0
_result["inter"], _result["uni"] = intersection_union_part(
parts_pred, parts_gt, nr_part) # mIoU-bg, include background 0
metric["part"].append(_result)
# material
metric['valid_material'] = data['valid_material']
if metric['valid_material']:
metric['material'] = {}
material_gt, material_pred = data['seg_material'], pred['material']
metric["material"]["acc"] = ((material_gt == material_pred) * (material_gt > 0)).sum() # ignore 0
metric["material"]["pixel"] = (material_gt > 0).sum() # ignore 0
metric["material"]["inter"], metric["material"]["uni"] = intersectionAndUnion(
material_pred, material_gt, broden_dataset.nr['material'] - 1) # ignore 0
return metric
def evaluate(nets, loader, loader_2, history, epoch, args, isVis=True):
print('Evaluating at {} epochs...'.format(epoch))
loss_meter = AverageMeter()
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
loss_meter_2 = AverageMeter()
acc_meter_2 = AverageMeter()
intersection_meter_2 = AverageMeter()
union_meter_2 = AverageMeter()
# switch to eval mode
for net in nets:
net.eval()
for i, batch_data in enumerate(loader):
# forward pass
torch.cuda.empty_cache()
pred, err = forward_with_loss(nets, batch_data, is_train=False)
loss_meter.update(err.data.item())
print('[Eval] iter {}, loss: {}'.format(i, err.data.item()))
# calculate accuracy
acc, pix = accuracy(batch_data, pred)
acc_meter.update(acc, pix)
intersection, union = intersectionAndUnion(batch_data, pred,
args.num_class)
intersection_meter.update(intersection)
union_meter.update(union)
# visualization
if isVis:
visualize(batch_data, pred, args)
for i, batch_data in enumerate(loader_2):
# forward pass
torch.cuda.empty_cache()
pred, err = forward_with_loss(nets, batch_data, is_train=False)
loss_meter_2.update(err.data.item())
print('[Eval] iter {}, loss: {}'.format(i, err.data.item()))
# calculate accuracy
acc, pix = accuracy(batch_data, pred)
acc_meter_2.update(acc, pix)
intersection, union = intersectionAndUnion(batch_data, pred,
args.num_class)
intersection_meter_2.update(intersection)
union_meter_2.update(union)
# visualization
if isVis:
visualize(batch_data, pred, args)
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
print('class [{}], IoU: {}'.format(trainID2Class[i], _iou))
print('[Cityscapes Eval Summary]:')
print('Epoch: {}, Loss: {}, Mean IoU: {:.4}, Accuracy: {:.2f}%'
.format(epoch, loss_meter.average(), iou.mean(), acc_meter.average() * 100))
history['val']['epoch'].append(epoch)
history['val']['err'].append(loss_meter.average())
history['val']['acc'].append(acc_meter.average())
history['val']['mIoU'].append(iou.mean())
iou = intersection_meter_2.sum / (union_meter_2.sum + 1e-10)
for i, _iou in enumerate(iou):
print('class [{}], IoU: {}'.format(trainID2Class[i], _iou))
print('[BDD Eval Summary]:')
print('Epoch: {}, Loss: {}, Mean IoU: {:.4}, Accuracy: {:.2f}%'
.format(epoch, loss_meter_2.average(), iou.mean(), acc_meter_2.average() * 100))
history['val_2']['epoch'].append(epoch)
history['val_2']['err'].append(loss_meter_2.average())
history['val_2']['acc'].append(acc_meter_2.average())
history['val_2']['mIoU'].append(iou.mean())
def evaluate_segmentation(segmentation_module, loader, args):
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
time_meter = AverageMeter()
dataset = loader.dataset
segmentation_module.eval()
print('Start evaluation of %d batches' % (len(loader)))
pbar = tqdm(total=len(loader))
for batch_data in loader:
# process data
batch_data = batch_data[0]
seg_label = as_numpy(batch_data['seg_label'][0])
img_resized_list = batch_data['img_data']
torch.cuda.synchronize()
tic = time.perf_counter()
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
scores = torch.zeros(1, dataset.num_classes, segSize[0],
segSize[1])
scores = async_copy_to(scores, args.gpu)
for img in img_resized_list:
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, args.gpu)
# forward pass
scores_tmp = segmentation_module(feed_dict, segSize=segSize)
scores = scores + scores_tmp / len(args.imgSize)
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
torch.cuda.synchronize()
time_meter.update(time.perf_counter() - tic)
# calculate accuracy
acc, pix = accuracy(pred, seg_label)
intersection, union = intersectionAndUnion(pred, seg_label,
dataset.num_classes)
acc_meter.update(acc, pix)
intersection_meter.update(intersection)
union_meter.update(union)
# visualization
if args.visualize:
visualize_result(
(batch_data['img_ori'], seg_label, batch_data['info']), pred,
args)
pbar.update(1)
# summary
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
per_class_iou = []
for i, _iou in enumerate(iou):
print('class [{}], IoU: {:.4f}'.format(i, _iou))
per_class_iou.append(_iou)
print('[Eval Summary]:')
print(
'Mean IoU: {:.4f}%, Accuracy: {:.2f}%, Inference Time: {:.4f}s'.format(
iou.mean() * 100,
acc_meter.average() * 100, time_meter.average()))
metrics = {
'mean_iou': float(iou.mean()),
'acc': float(acc_meter.average()),
'per_class_iou': per_class_iou,
}
return metrics
def evaluate_surface_normals(segmentation_module, loader, args):
acc_meter = AverageMeter()
intersection_meter = AverageMeter()
union_meter = AverageMeter()
time_meter = AverageMeter()
dataset = loader.dataset
vocab_file = dataset.vocab_file
print('Loading normals')
img_id_to_normals_data = dataset.load_normals_file()
segmentation_module.eval()
normals_evaluator = NormalsEvaluator(vocab_file=vocab_file)
print('Start evaluation of %d batches' % (len(loader)))
pbar = tqdm(total=len(loader))
accum_probs = []
accum_gt = []
accum_masks = []
for batch_data in loader:
# process data
batch_data = batch_data[0]
seg_label = as_numpy(batch_data['seg_label'][0])
img_resized_list = batch_data['img_data']
img_id = batch_data['info']
raw_normals, valid_mask = img_id_to_normals_data(img_id)
torch.cuda.synchronize()
tic = time.perf_counter()
with torch.no_grad():
segSize = (seg_label.shape[0], seg_label.shape[1])
if len(img_resized_list) > 1:
scores = torch.zeros(1, dataset.num_classes, segSize[0],
segSize[1])
scores = async_copy_to(scores, args.gpu)
for img in img_resized_list:
feed_dict = batch_data.copy()
feed_dict['img_data'] = img
del feed_dict['img_ori']
del feed_dict['info']
feed_dict = async_copy_to(feed_dict, args.gpu)
# forward pass
scores_tmp = segmentation_module(feed_dict, segSize=segSize)
if len(img_resized_list) > 1:
scores = scores + scores_tmp / len(args.imgSize)
else:
scores = scores_tmp / len(args.imgSize)
probs = scores
_, pred = torch.max(scores, dim=1)
pred = as_numpy(pred.squeeze(0).cpu())
prob_np = as_numpy(probs)
torch.cuda.synchronize()
time_meter.update(time.perf_counter() - tic)
# calculate accuracy
acc, pix = accuracy(pred, seg_label)
intersection, union = intersectionAndUnion(pred, seg_label,
dataset.num_classes)
acc_meter.update(acc, pix)
intersection_meter.update(intersection)
union_meter.update(union)
# accumulate probs for normal
# since we compute median stats, we cannot average over images
accum_probs.append(prob_np.squeeze())
accum_gt.append(raw_normals)
accum_masks.append(valid_mask.astype(np.int32))
# visualization
if args.visualize:
visualize_result(
(batch_data['img_ori'], seg_label, batch_data['info']), pred,
args)
pbar.update(1)
# compute normal metrics
print("Computing pixelwise normal metrics")
metrics = normals_evaluator.compute_normals_metrics_from_network_probs(
accum_probs, accum_gt, accum_masks)
normals_evaluator.print_metrics(metrics)
# summary
iou = intersection_meter.sum / (union_meter.sum + 1e-10)
for i, _iou in enumerate(iou):
print('class [{}], IoU: {:.4f}'.format(i, _iou))
print('[Eval Summary]:')
print(
'Mean IoU: {:.4f}%, Accuracy: {:.2f}%, Inference Time: {:.4f}s'.format(
iou.mean() * 100,
acc_meter.average() * 100, time_meter.average()))
return metrics
def evaluate(model, loader, gpu_mode, num_class=7):
# output format
res = {
'loss': 0.1,
'acc': 0.2, # or acc for every category,
'iou': 0.3
}
# metric meters
loss_meter = AverageMeter()
acc_meter = AverageMeter()
inter_meter = AverageMeter()
union_meter = AverageMeter()
confusion_matrix = np.zeros((num_class, num_class))
for i_batch, (img, mask, _) in enumerate(loader):
if gpu_mode:
img = img.cuda()
mask = mask.cuda()
output = model(img)
# calculate loss
loss = cross_entropy2d(output, mask)
loss_value = loss.data.cpu().numpy()
loss_meter.update(loss_value)
output = output.max(1)[1]
# calculate accuracy
acc = accuracy(output, mask)
acc_meter.update(acc)
# if gpu_mode:
# output = output.int().cpu().detach().numpy()
# mask = mask.int().cpu().detach().numpy()
# seg_pred = np.array(output).reshape(1, -1)
# seg_gt = np.array(mask).reshape(1, -1)
# confusion_matrix += get_confusion_matrix(seg_gt, seg_pred, 7)
#
# pos = confusion_matrix.sum(1)
# res = confusion_matrix.sum(0)
# tp = np.diag(confusion_matrix)
#
# IU_array = (tp / np.maximum(1.0, pos + res - tp))
# mean_IU = IU_array.mean()
# calculate iou
intersection, union = intersectionAndUnion(output, mask, num_class)
inter_meter.update(intersection)
union_meter.update(union)
# summary
# iou = IU_array
# iou_mean = IU_array.mean()
iou = inter_meter.sum / (union_meter.sum + 1e-10)
iou_mean = iou.mean()
acc_mean = acc_meter.average()
loss_mean = loss_meter.average()
res['loss'] = loss_mean
res['acc'] = acc_mean
res['iou'] = iou
res['iou_mean'] = iou_mean
return res
