def validate(val_loader,
net,
criterion,
optim,
epoch,
calc_metrics=True,
dump_assets=False,
dump_all_images=False):
"""
Run validation for one epoch
:val_loader: data loader for validation
:net: the network
:criterion: loss fn
:optimizer: optimizer
:epoch: current epoch
:calc_metrics: calculate validation score
:dump_assets: dump attention prediction(s) images
:dump_all_images: dump all images, not just N
"""
val_time = time.perf_counter()
dumper = ImageDumper(val_len=len(val_loader),
dump_all_images=dump_all_images,
dump_assets=dump_assets,
dump_for_auto_labelling=args.dump_for_auto_labelling,
dump_for_submission=args.dump_for_submission,
rank=rank)
net.eval()
val_loss = AverageMeter()
iou_acc = 0
for val_idx, data in enumerate(val_loader):
input_images, labels, img_names, _ = data
if args.dump_for_auto_labelling or args.dump_for_submission:
submit_fn = '{}.png'.format(img_names[0])
if val_idx % 20 == 0:
logx.msg(
f'validating[Iter: {val_idx + 1} / {len(val_loader)}]')
if os.path.exists(os.path.join(dumper.save_dir, submit_fn)):
continue
# Run network
assets, _iou_acc = \
eval_minibatch(data, net, criterion, val_loss, calc_metrics,
args, val_idx)
iou_acc += _iou_acc
input_images, labels, img_names, _ = data
if optim.comm.rank == 0:
dumper.dump(
{
'gt_images': labels,
'input_images': input_images,
'img_names': img_names,
'assets': assets
}, val_idx)
if val_idx > 5 and args.test_mode:
break
if val_idx % 2 == 0 and optim.comm.rank == 0:
logx.msg(f'validating[Iter: {val_idx + 1} / {len(val_loader)}]')
# average the loss value
val_loss_tens = torch.tensor(val_loss.val)
optim.comm.Allreduce(MPI.IN_PLACE, val_loss_tens, MPI.SUM)
val_loss_tens = val_loss_tens.to(torch.float)
val_loss_tens /= float(optim.comm.size)
val_loss.val = val_loss_tens.item()
# sum up the iou_acc
optim.comm.Allreduce(MPI.IN_PLACE, iou_acc, MPI.SUM)
# was_best = False
if calc_metrics:
# was_best = eval_metrics(iou_acc, args, net, optim, val_loss, epoch)
_, mean_iu = eval_metrics(iou_acc, args, net, optim, val_loss, epoch)
optim.comm.bcast(mean_iu, root=0)
# was_best = optim.comm.bcast(was_best, root=0)
#
# # Write out a summary html page and tensorboard image table
# if not args.dump_for_auto_labelling and not args.dump_for_submission and optim.comm.rank == 0:
# dumper.write_summaries(was_best)
return val_loss.val, mean_iu, time.perf_counter() - val_time
def validate(val_loader, net, criterion, optim, epoch,
calc_metrics=True,
dump_assets=False,
dump_all_images=False):
"""
Run validation for one epoch
:val_loader: data loader for validation
:net: the network
:criterion: loss fn
:optimizer: optimizer
:epoch: current epoch
:calc_metrics: calculate validation score
:dump_assets: dump attention prediction(s) images
:dump_all_images: dump all images, not just N
"""
dumper = ImageDumper(val_len=len(val_loader),
dump_all_images=dump_all_images,
dump_assets=dump_assets,
dump_for_auto_labelling=args.dump_for_auto_labelling,
dump_for_submission=args.dump_for_submission)
net.eval()
val_loss = AverageMeter()
iou_acc = 0
for val_idx, data in enumerate(val_loader):
input_images, labels, img_names, _ = data
if args.dump_for_auto_labelling or args.dump_for_submission:
submit_fn = '{}.png'.format(img_names[0])
if val_idx % 20 == 0:
logx.msg(f'validating[Iter: {val_idx + 1} / {len(val_loader)}]')
if os.path.exists(os.path.join(dumper.save_dir, submit_fn)):
continue
# Run network
assets, _iou_acc = \
eval_minibatch(data, net, criterion, val_loss, calc_metrics,
args, val_idx)
iou_acc += _iou_acc
input_images, labels, img_names, _ = data
dumper.dump({'gt_images': labels,
'input_images': input_images,
'img_names': img_names,
'assets': assets}, val_idx)
if val_idx > 5 and args.test_mode:
break
if val_idx % 20 == 0:
logx.msg(f'validating[Iter: {val_idx + 1} / {len(val_loader)}]')
was_best = False
if calc_metrics:
was_best = eval_metrics(iou_acc, args, net, optim, val_loss, epoch)
# Write out a summary html page and tensorboard image table
if not args.dump_for_auto_labelling and not args.dump_for_submission:
dumper.write_summaries(was_best)
def validate(val_loader,
net,
criterion,
optim,
epoch,
calc_metrics=True,
dump_assets=False,
dump_all_images=False):
"""
Run validation for one epoch
:val_loader: data loader for validation
:net: the network
:criterion: loss fn
:optimizer: optimizer
:epoch: current epoch
:calc_metrics: calculate validation score
:dump_assets: dump attention prediction(s) images
:dump_all_images: dump all images, not just N
"""
dumper = ImageDumper(val_len=len(val_loader),
dump_all_images=dump_all_images,
dump_assets=dump_assets,
dump_for_auto_labelling=args.dump_for_auto_labelling,
dump_for_submission=args.dump_for_submission)
net.eval()
val_loss = AverageMeter()
iou_acc = 0
pred = dict()
for val_idx, data in enumerate(val_loader):
input_images, labels, img_names, _ = data
if args.dump_for_auto_labelling or args.dump_for_submission:
submit_fn = '{}.png'.format(img_names[0])
if val_idx % 20 == 0:
logx.msg(
f'validating[Iter: {val_idx + 1} / {len(val_loader)}]')
if os.path.exists(os.path.join(dumper.save_dir, submit_fn)):
continue
# Run network
assets, _iou_acc = \
eval_minibatch(data, net, criterion, val_loss, calc_metrics,
args, val_idx)
iou_acc += _iou_acc
input_images, labels, img_names, _ = data
if testing:
prediction = assets['predictions'][0]
values, counts = np.unique(prediction, return_counts=True)
pred.update({img_names[0]: dict(zip(values, counts))})
dumper.dump(
{
'gt_images': labels,
'input_images': input_images,
'img_names': img_names,
'assets': assets
},
val_idx,
testing=True,
grid=grid)
else:
dumper.dump(
{
'gt_images': labels,
'input_images': input_images,
'img_names': img_names,
'assets': assets
}, val_idx)
if val_idx > 5 and args.test_mode:
break
if val_idx % 20 == 0:
logx.msg(f'validating[Iter: {val_idx + 1} / {len(val_loader)}]')
was_best = False
if calc_metrics:
was_best = eval_metrics(iou_acc, args, net, optim, val_loss, epoch)
# Write out a summary html page and tensorboard image table
if not args.dump_for_auto_labelling and not args.dump_for_submission:
dumper.write_summaries(was_best)
if testing:
df = pd.DataFrame.from_dict(pred, orient='index')
df_p = df.div(df.sum(axis=1), axis=0)
df_p.to_csv(os.path.join(dumper.save_dir, 'freq.csv'),
mode='a+',
header=False)
def validate_topn(val_loader, net, criterion, optim, epoch, args):
"""
Find worse case failures ...
Only single GPU for now
First pass = calculate TP, FP, FN pixels per image per class
Take these stats and determine the top20 images to dump per class
Second pass = dump all those selected images
"""
assert args.bs_val == 1
######################################################################
# First pass
######################################################################
logx.msg('First pass')
image_metrics = {}
net.eval()
val_loss = AverageMeter()
iou_acc = 0
for val_idx, data in enumerate(val_loader):
# Run network
assets, _iou_acc = \
run_minibatch(data, net, criterion, val_loss, True, args, val_idx)
# per-class metrics
input_images, labels, img_names, _ = data
fp, fn = metrics_per_image(_iou_acc)
img_name = img_names[0]
image_metrics[img_name] = (fp, fn)
iou_acc += _iou_acc
if val_idx % 20 == 0:
logx.msg(f'validating[Iter: {val_idx + 1} / {len(val_loader)}]')
if val_idx > 5 and args.test_mode:
break
eval_metrics(iou_acc, args, net, optim, val_loss, epoch)
######################################################################
# Find top 20 worst failures from a pixel count perspective
######################################################################
from collections import defaultdict
worst_images = defaultdict(dict)
class_to_images = defaultdict(dict)
for classid in range(cfg.DATASET.NUM_CLASSES):
tbl = {}
for img_name in image_metrics.keys():
fp, fn = image_metrics[img_name]
fp = fp[classid]
fn = fn[classid]
tbl[img_name] = fp + fn
worst = sorted(tbl, key=tbl.get, reverse=True)
for img_name in worst[:args.dump_topn]:
fail_pixels = tbl[img_name]
worst_images[img_name][classid] = fail_pixels
class_to_images[classid][img_name] = fail_pixels
msg = str(worst_images)
logx.msg(msg)
# write out per-gpu jsons
# barrier
# make single table
######################################################################
# 2nd pass
######################################################################
logx.msg('Second pass')
attn_map = None
for val_idx, data in enumerate(val_loader):
in_image, gt_image, img_names, _ = data
# Only process images that were identified in first pass
if not args.dump_topn_all and img_names[0] not in worst_images:
continue
with torch.no_grad():
inputs = in_image.cuda()
inputs = {'images': inputs, 'gts': gt_image}
if cfg.MODEL.MSCALE:
output, attn_map = net(inputs)
else:
output = net(inputs)
output = torch.nn.functional.softmax(output, dim=1)
prob_mask, predictions = output.data.max(1)
predictions = predictions.cpu()
# this has shape [bs, h, w]
img_name = img_names[0]
for classid in worst_images[img_name].keys():
err_mask = calc_err_mask(predictions.numpy(), gt_image.numpy(),
cfg.DATASET.NUM_CLASSES, classid)
class_name = cfg.DATASET_INST.trainid_to_name[classid]
error_pixels = worst_images[img_name][classid]
logx.msg(f'{img_name} {class_name}: {error_pixels}')
img_names = [img_name + f'_{class_name}']
to_dump = {
'gt_images': gt_image,
'input_images': in_image,
'predictions': predictions.numpy(),
'err_mask': err_mask,
'prob_mask': prob_mask,
'img_names': img_names
}
if attn_map is not None:
to_dump['attn_maps'] = attn_map
# FIXME!
# do_dump_images([to_dump])
html_fn = os.path.join(args.result_dir, 'best_images',
'topn_failures.html')
from utils.results_page import ResultsPage
ip = ResultsPage('topn failures', html_fn)
for classid in class_to_images:
class_name = cfg.DATASET_INST.trainid_to_name[classid]
img_dict = class_to_images[classid]
for img_name in sorted(img_dict, key=img_dict.get, reverse=True):
fail_pixels = class_to_images[classid][img_name]
img_cls = f'{img_name}_{class_name}'
pred_fn = f'{img_cls}_prediction.png'
gt_fn = f'{img_cls}_gt.png'
inp_fn = f'{img_cls}_input.png'
err_fn = f'{img_cls}_err_mask.png'
prob_fn = f'{img_cls}_prob_mask.png'
img_label_pairs = [(pred_fn, 'pred'), (gt_fn, 'gt'),
(inp_fn, 'input'), (err_fn, 'errors'),
(prob_fn, 'prob')]
ip.add_table(img_label_pairs,
table_heading=f'{class_name}-{fail_pixels}')
ip.write_page()
return val_loss.avg