def validate(val_loader, net, criterion, optim, epoch, calc_metrics=True, dump_assets=False, dump_all_images=False): """ Run validation for one epoch :val_loader: data loader for validation :net: the network :criterion: loss fn :optimizer: optimizer :epoch: current epoch :calc_metrics: calculate validation score :dump_assets: dump attention prediction(s) images :dump_all_images: dump all images, not just N """ val_time = time.perf_counter() dumper = ImageDumper(val_len=len(val_loader), dump_all_images=dump_all_images, dump_assets=dump_assets, dump_for_auto_labelling=args.dump_for_auto_labelling, dump_for_submission=args.dump_for_submission, rank=rank) net.eval() val_loss = AverageMeter() iou_acc = 0 for val_idx, data in enumerate(val_loader): input_images, labels, img_names, _ = data if args.dump_for_auto_labelling or args.dump_for_submission: submit_fn = '{}.png'.format(img_names[0]) if val_idx % 20 == 0: logx.msg( f'validating[Iter: {val_idx + 1} / {len(val_loader)}]') if os.path.exists(os.path.join(dumper.save_dir, submit_fn)): continue # Run network assets, _iou_acc = \ eval_minibatch(data, net, criterion, val_loss, calc_metrics, args, val_idx) iou_acc += _iou_acc input_images, labels, img_names, _ = data if optim.comm.rank == 0: dumper.dump( { 'gt_images': labels, 'input_images': input_images, 'img_names': img_names, 'assets': assets }, val_idx) if val_idx > 5 and args.test_mode: break if val_idx % 2 == 0 and optim.comm.rank == 0: logx.msg(f'validating[Iter: {val_idx + 1} / {len(val_loader)}]') # average the loss value val_loss_tens = torch.tensor(val_loss.val) optim.comm.Allreduce(MPI.IN_PLACE, val_loss_tens, MPI.SUM) val_loss_tens = val_loss_tens.to(torch.float) val_loss_tens /= float(optim.comm.size) val_loss.val = val_loss_tens.item() # sum up the iou_acc optim.comm.Allreduce(MPI.IN_PLACE, iou_acc, MPI.SUM) # was_best = False if calc_metrics: # was_best = eval_metrics(iou_acc, args, net, optim, val_loss, epoch) _, mean_iu = eval_metrics(iou_acc, args, net, optim, val_loss, epoch) optim.comm.bcast(mean_iu, root=0) # was_best = optim.comm.bcast(was_best, root=0) # # # Write out a summary html page and tensorboard image table # if not args.dump_for_auto_labelling and not args.dump_for_submission and optim.comm.rank == 0: # dumper.write_summaries(was_best) return val_loss.val, mean_iu, time.perf_counter() - val_time
def validate(val_loader, net, criterion, optim, epoch, calc_metrics=True, dump_assets=False, dump_all_images=False): """ Run validation for one epoch :val_loader: data loader for validation :net: the network :criterion: loss fn :optimizer: optimizer :epoch: current epoch :calc_metrics: calculate validation score :dump_assets: dump attention prediction(s) images :dump_all_images: dump all images, not just N """ dumper = ImageDumper(val_len=len(val_loader), dump_all_images=dump_all_images, dump_assets=dump_assets, dump_for_auto_labelling=args.dump_for_auto_labelling, dump_for_submission=args.dump_for_submission) net.eval() val_loss = AverageMeter() iou_acc = 0 for val_idx, data in enumerate(val_loader): input_images, labels, img_names, _ = data if args.dump_for_auto_labelling or args.dump_for_submission: submit_fn = '{}.png'.format(img_names[0]) if val_idx % 20 == 0: logx.msg(f'validating[Iter: {val_idx + 1} / {len(val_loader)}]') if os.path.exists(os.path.join(dumper.save_dir, submit_fn)): continue # Run network assets, _iou_acc = \ eval_minibatch(data, net, criterion, val_loss, calc_metrics, args, val_idx) iou_acc += _iou_acc input_images, labels, img_names, _ = data dumper.dump({'gt_images': labels, 'input_images': input_images, 'img_names': img_names, 'assets': assets}, val_idx) if val_idx > 5 and args.test_mode: break if val_idx % 20 == 0: logx.msg(f'validating[Iter: {val_idx + 1} / {len(val_loader)}]') was_best = False if calc_metrics: was_best = eval_metrics(iou_acc, args, net, optim, val_loss, epoch) # Write out a summary html page and tensorboard image table if not args.dump_for_auto_labelling and not args.dump_for_submission: dumper.write_summaries(was_best)
def validate(val_loader, net, criterion, optim, epoch, calc_metrics=True, dump_assets=False, dump_all_images=False): """ Run validation for one epoch :val_loader: data loader for validation :net: the network :criterion: loss fn :optimizer: optimizer :epoch: current epoch :calc_metrics: calculate validation score :dump_assets: dump attention prediction(s) images :dump_all_images: dump all images, not just N """ dumper = ImageDumper(val_len=len(val_loader), dump_all_images=dump_all_images, dump_assets=dump_assets, dump_for_auto_labelling=args.dump_for_auto_labelling, dump_for_submission=args.dump_for_submission) net.eval() val_loss = AverageMeter() iou_acc = 0 pred = dict() for val_idx, data in enumerate(val_loader): input_images, labels, img_names, _ = data if args.dump_for_auto_labelling or args.dump_for_submission: submit_fn = '{}.png'.format(img_names[0]) if val_idx % 20 == 0: logx.msg( f'validating[Iter: {val_idx + 1} / {len(val_loader)}]') if os.path.exists(os.path.join(dumper.save_dir, submit_fn)): continue # Run network assets, _iou_acc = \ eval_minibatch(data, net, criterion, val_loss, calc_metrics, args, val_idx) iou_acc += _iou_acc input_images, labels, img_names, _ = data if testing: prediction = assets['predictions'][0] values, counts = np.unique(prediction, return_counts=True) pred.update({img_names[0]: dict(zip(values, counts))}) dumper.dump( { 'gt_images': labels, 'input_images': input_images, 'img_names': img_names, 'assets': assets }, val_idx, testing=True, grid=grid) else: dumper.dump( { 'gt_images': labels, 'input_images': input_images, 'img_names': img_names, 'assets': assets }, val_idx) if val_idx > 5 and args.test_mode: break if val_idx % 20 == 0: logx.msg(f'validating[Iter: {val_idx + 1} / {len(val_loader)}]') was_best = False if calc_metrics: was_best = eval_metrics(iou_acc, args, net, optim, val_loss, epoch) # Write out a summary html page and tensorboard image table if not args.dump_for_auto_labelling and not args.dump_for_submission: dumper.write_summaries(was_best) if testing: df = pd.DataFrame.from_dict(pred, orient='index') df_p = df.div(df.sum(axis=1), axis=0) df_p.to_csv(os.path.join(dumper.save_dir, 'freq.csv'), mode='a+', header=False)
def validate_topn(val_loader, net, criterion, optim, epoch, args, dump_assets=True, dump_all_images=True): """ Find worse case failures ... Only single GPU for now First pass = calculate TP, FP, FN pixels per image per class Take these stats and determine the top20 images to dump per class Second pass = dump all those selected images """ assert args.bs_val == 1 ###################################################################### # First pass ###################################################################### logx.msg('First pass') image_metrics = {} dumper = ImageDumper(val_len=len(val_loader), dump_all_images=dump_all_images, dump_assets=dump_assets, dump_for_auto_labelling=args.dump_for_auto_labelling, dump_for_submission=args.dump_for_submission) net.eval() val_loss = AverageMeter() iou_acc = 0 for val_idx, data in enumerate(val_loader): # Run network assets, _iou_acc = \ eval_minibatch(data, net, criterion, val_loss, True, args, val_idx) # per-class metrics input_images, labels, img_names, _ = data fp, fn = metrics_per_image(_iou_acc) img_name = img_names[0] image_metrics[img_name] = (fp, fn) iou_acc += _iou_acc if val_idx % 20 == 0: logx.msg(f'validating[Iter: {val_idx + 1} / {len(val_loader)}]') if val_idx > 5 and args.test_mode: break eval_metrics(iou_acc, args, net, optim, val_loss, epoch) ###################################################################### # Find top 20 worst failures from a pixel count perspective ###################################################################### from collections import defaultdict worst_images = defaultdict(dict) class_to_images = defaultdict(dict) for classid in range(cfg.DATASET.NUM_CLASSES): tbl = {} for img_name in image_metrics.keys(): fp, fn = image_metrics[img_name] fp = fp[classid] fn = fn[classid] tbl[img_name] = fp + fn worst = sorted(tbl, key=tbl.get, reverse=True) for img_name in worst[:args.dump_topn]: fail_pixels = tbl[img_name] worst_images[img_name][classid] = fail_pixels class_to_images[classid][img_name] = fail_pixels msg = str(worst_images) logx.msg(msg) # write out per-gpu jsons # barrier # make single table ###################################################################### # 2nd pass ###################################################################### logx.msg('Second pass') attn_map = None for val_idx, data in enumerate(val_loader): in_image, gt_image, img_names, _ = data # Only process images that were identified in first pass if not args.dump_topn_all and img_names[0] not in worst_images: continue with torch.no_grad(): inputs = in_image.cuda() inputs = {'images': inputs, 'gts': gt_image} if cfg.MODEL.MSCALE: #output, attn_map = net(inputs) output_dict = net(inputs) else: output_dict = net(inputs) output = output_dict['pred'] output_data = torch.nn.functional.softmax(output, dim=1).cpu().data op = output_data.cpu().detach().numpy() np.save(f'{cfg.RESULT_DIR}/output_{epoch}_{val_idx}.npy', op) prob_mask, predictions = output_data.max(1) #define assests based on the eval_minibatch function assets = {} for item in output_dict: smax = torch.nn.functional.softmax(output_dict[item],dim=1) _, pred = smax.data.max(1) assets[item] = pred.cpu().numpy() #print(assets) # this has shape [bs, h, w] img_name = img_names[0] for classid in worst_images[img_name].keys(): err_mask = calc_err_mask(predictions.numpy(), gt_image.numpy(), cfg.DATASET.NUM_CLASSES, classid) input_images, gt_image, img_names, _ = data class_name = cfg.DATASET_INST.trainid_to_name[classid] error_pixels = worst_images[img_name][classid] logx.msg(f'{img_name} {class_name}: {error_pixels}') img_names = [img_name + f'_{class_name}'] #add assests assets['predictions'] = predictions.numpy() assets['prob_mask'] = prob_mask assets['err_mask'] = err_mask to_dump = {'gt_images': gt_image, 'input_images': input_images, 'predictions': predictions.numpy(), 'err_mask': err_mask, 'prob_mask': prob_mask, 'img_names': img_names, 'assets': assets} if attn_map is not None: to_dump['attn_maps'] = attn_map # FIXME! # do_dump_images([to_dump]) #fixed dumper.dump(to_dump,val_idx) html_fn = os.path.join(args.result_dir, 'best_images', 'topn_failures.html') from utils.results_page import ResultsPage ip = ResultsPage('topn failures', html_fn) for classid in class_to_images: class_name = cfg.DATASET_INST.trainid_to_name[classid] img_dict = class_to_images[classid] for img_name in sorted(img_dict, key=img_dict.get, reverse=True): fail_pixels = class_to_images[classid][img_name] img_cls = f'{img_name}_{class_name}' pred_fn = f'{img_cls}_prediction.png' gt_fn = f'{img_cls}_gt.png' inp_fn = f'{img_cls}_input.png' err_fn = f'{img_cls}_err_mask.png' prob_fn = f'{img_cls}_prob_mask.png' img_label_pairs = [(pred_fn, 'pred'), (gt_fn, 'gt'), (inp_fn, 'input'), (err_fn, 'errors'), (prob_fn, 'prob')] ip.add_table(img_label_pairs, table_heading=f'{class_name}-{fail_pixels}') ip.write_page() return val_loss.avg