def evaluate(nets, loader, args): loss_meter = AverageMeter() acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() # switch to eval mode for net in nets: net.eval() for i, batch_data in enumerate(loader): # forward pass pred, err = forward_multiscale(nets, batch_data, args) loss_meter.update(err.data[0]) # calculate accuracy acc, pix = accuracy(batch_data, pred) intersection, union = intersectionAndUnion(batch_data, pred, args.segDepth) acc_meter.update(acc, pix) intersection_meter.update(intersection) union_meter.update(union) print('[{}] iter {}, loss: {}, accuracy: {}'.format( datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), i, err.data[0], acc)) # visualization if args.visualize: visualize_result(batch_data, pred, args) iou = intersection_meter.sum / (union_meter.sum + 1e-10) print('[Eval Summary]:') print('Loss: {}, Mean IoU: {:.4}, Accurarcy: {:.2f}%'.format( loss_meter.average(), iou.mean(), acc_meter.average() * 100))
def cal_acc(data_list, pred_folder, classes, names): intersection_meter = AverageMeter() union_meter = AverageMeter() target_meter = AverageMeter() for i, (image_path, target_path) in enumerate(data_list): image_name = image_path.split('/')[-1].split('.')[0] pred = np.asarray(Image.open(os.path.join(pred_folder, image_name+'.png'))) target = np.asarray(Image.open(target_path)) intersection, union, target = intersectionAndUnion(pred, target, classes) intersection_meter.update(intersection) union_meter.update(union) target_meter.update(target) accuracy = sum(intersection_meter.val) / (sum(target_meter.val) + 1e-10) logger.info('Evaluating {0}/{1} on image {2}, accuracy {3:.4f}.'.format(i + 1, len(data_list), image_name+'.png', accuracy)) iou_class = intersection_meter.sum / (union_meter.sum + 1e-10) accuracy_class = intersection_meter.sum / (target_meter.sum + 1e-10) mIoU = np.mean(iou_class) mAcc = np.mean(accuracy_class) allAcc = sum(intersection_meter.sum) / (sum(target_meter.sum) + 1e-10) logger.info('Eval result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.'.format(mIoU, mAcc, allAcc)) for i in range(classes): logger.info('Class_{} result: iou/accuracy {:.4f}/{:.4f}, name: {}.'.format(i, iou_class[i], accuracy_class[i], names[i]))
def evaluate(segmentation_module, loader, args): acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() segmentation_module.eval() for i, batch_data in enumerate(loader): # process data batch_data = batch_data[0] seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) pred = torch.zeros(1, args.num_class, segSize[0], segSize[1]) pred = Variable(pred).cuda() for img in img_resized_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, args.gpu_id) # forward pass pred_tmp = segmentation_module(feed_dict, segSize=segSize) pred = pred + pred_tmp / len(args.imgSize) _, preds = torch.max(pred.data.cpu(), dim=1) preds = as_numpy(preds.squeeze(0)) # calculate accuracy acc, pix = accuracy(preds, seg_label) intersection, union = intersectionAndUnion(preds, seg_label, args.num_class) acc_meter.update(acc, pix) intersection_meter.update(intersection) union_meter.update(union) print('[{}] iter {}, accuracy: {}'.format( datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), i, acc)) # visualization if args.visualize: visualize_result( (batch_data['img_ori'], seg_label, batch_data['info']), preds, args) if args.precompute: precompute_result(batch_data['info'], preds, args) iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): print('class [{}], IoU: {}'.format(i, _iou)) print('[Eval Summary]:') print('Mean IoU: {:.4}, Accuracy: {:.2f}%'.format( iou.mean(), acc_meter.average() * 100))
def evaluate(segmentation_module, loader, cfg, gpu, model_name, paper_arxiv_id): acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() time_meter = AverageMeter() evaluator = ADE20KEvaluator(model_name=model_name, paper_arxiv_id=paper_arxiv_id) segmentation_module.eval() pbar = tqdm(total=len(loader)) for batch_data in loader: # process data batch_data = batch_data[0] seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] torch.cuda.synchronize() tic = time.perf_counter() with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0], segSize[1]) scores = async_copy_to(scores, gpu) for img in img_resized_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, gpu) # forward pass scores_tmp = segmentation_module(feed_dict, segSize=segSize) scores = scores + scores_tmp / len(cfg.DATASET.imgSizes) _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) torch.cuda.synchronize() time_meter.update(time.perf_counter() - tic) # calculate accuracy acc, pix = accuracy(pred, seg_label) intersection, union = intersectionAndUnion(pred, seg_label, cfg.DATASET.num_class) acc_meter.update(acc, pix) intersection_meter.update(intersection) union_meter.update(union) evaluator.add(outputs=pred.flatten(), targets=seg_label.flatten()) if evaluator.cache_exists: break pbar.update(1) evaluator.save()
def validate(val_loader, model, criterion, classes, zoom_factor): logger.info('>>>>>>>>>>>>>>>> Start Evaluation >>>>>>>>>>>>>>>>') batch_time = AverageMeter() data_time = AverageMeter() loss_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() target_meter = AverageMeter() model.eval() end = time.time() for i, (input, target) in enumerate(val_loader): data_time.update(time.time() - end) input = input.cuda(async=True) target = target.cuda(async=True) input_var = torch.autograd.Variable(input) target_var = torch.autograd.Variable(target) output = model(input_var) if zoom_factor != 8: output = F.upsample(output, size=target_var.size()[1:], mode='bilinear', align_corners=True) loss = criterion(output, target_var) output = output.data.max(1)[1].cpu().numpy() target = target.cpu().numpy() intersection, union, target = intersectionAndUnion(output, target, args.classes, args.ignore_label) intersection_meter.update(intersection) union_meter.update(union) target_meter.update(target) accuracy = sum(intersection_meter.val) / (sum(target_meter.val) + 1e-10) loss_meter.update(loss.data, input.size(0)) batch_time.update(time.time() - end) end = time.time() if (i + 1) % 10 == 0: logger.info('Test: [{}/{}] ' 'Data {data_time.val:.3f} ({data_time.avg:.3f}) ' 'Batch {batch_time.val:.3f} ({batch_time.avg:.3f}) ' 'Loss {loss_meter.val:.4f} ({loss_meter.avg:.4f}) ' 'Accuracy {accuracy:.4f}.'.format(i + 1, len(val_loader), data_time=data_time, batch_time=batch_time, loss_meter=loss_meter, accuracy=accuracy)) iou_class = intersection_meter.sum / (union_meter.sum + 1e-10) accuracy_class = intersection_meter.sum / (target_meter.sum + 1e-10) mIoU = np.mean(iou_class) mAcc = np.mean(accuracy_class) allAcc = sum(intersection_meter.sum) / (sum(target_meter.sum) + 1e-10) logger.info('Val result: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.'.format(mIoU, mAcc, allAcc)) for i in range(classes): logger.info('Class_{} Result: iou/accuracy {:.4f}/{:.4f}.'.format(i, iou_class[i], accuracy_class[i])) logger.info('<<<<<<<<<<<<<<<<< End Evaluation <<<<<<<<<<<<<<<<<') return loss_meter.avg, mIoU, mAcc, allAcc
def evaluate(segmentation_module, loader, args): acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() cls_ious_meter = AverageMeter() cls_mean_iou_meter = AverageMeter() eval_model = segmentation_module eval_model.eval() f = file(args.val_list_file).readlines() for i in range(len(loader)): batch_data = next(loader)[0] # process data seg_label = as_numpy(batch_data['seg_label']) with torch.no_grad(): segSize = (seg_label.shape[1], seg_label.shape[2]) pred = torch.zeros(1, args.num_class, segSize[0], segSize[1]) # forward pass pred = eval_model(batch_data, segSize=segSize) batch_data['data'] = batch_data['data'][:,:,:,::-1] pred += eval_model(batch_data, segSize=segSize)[:,:,:,::-1] _, preds = torch.max(pred.data.cpu(), dim=1) preds = as_numpy(preds.squeeze(0)) img = misc.imread(args.root_dataset+'/'+f[i].strip().split(' ')[0]) misc.imsave('./tmp/'+f[i].strip().split(' ')[0].split('/')[-1].replace('.jpg','.png'), preds[:img.shape[0],:img.shape[1]].astype(np.uint8)) preds = preds[:img.shape[0],:img.shape[1]] seg_label = seg_label[:,:img.shape[0],:img.shape[1]] # calculate accuracy acc, pix = accuracy(preds, seg_label, 255) intersection, union = intersectionAndUnion(preds, seg_label, args.num_class, 255) acc_meter.update(acc, pix) intersection_meter.update(intersection) union_meter.update(union) mean_iou = (intersection/(union+1e-10))[union!=0].mean() print('[{}] iter {}, accuracy: {:.5f}, mIoU: {:.5f}' .format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), i, acc, mean_iou)) iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): print('[{}] class [{}], IoU: {}'.format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), i, _iou)) print('[{}] [Eval Summary]:'.format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) print('Mean IoU: {:.4}, Accuracy: {:.2f}%' .format(iou.mean(), acc_meter.average()*100)) return iou, iou.mean()
def evaluate(segmentation_module, loader_val, args): acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() time_meter = AverageMeter() segmentation_module.eval() pbar = tqdm(total=len(loader_val)) for batch_data in loader_val: batch_data = batch_data[0] seg_label = as_numpy(batch_data["mask"][0]) torch.cuda.synchronize() batch_data["image"] = batch_data["image"].unsqueeze(0).cuda() #batch_data["mask"][0] = batch_data["mask"][0].cuda() #batch_data["mask"][1] = batch_data["mask"][1].cuda() with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) scores = torch.zeros(1, args.num_class, segSize[0], segSize[1]) scores = async_copy_to(scores, args.gpu) feed_dict = batch_data.copy() #print(torch.max(feed_dict['image'])) # forward pass scores, edge, att, loss = segmentation_module(feed_dict, epoch=0, segSize=segSize) _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) torch.cuda.synchronize() # calculate accuracy acc, pix = accuracy(pred, seg_label) intersection, union = intersectionAndUnion(pred, seg_label, args.num_class) intersection_meter.update(intersection) union_meter.update(union) acc_meter.update(acc) # visualization if True:# args.visualize visualize_result( (batch_data['image'], seg_label, batch_data["name"]), pred, edge, att, args) #Free up memroy #del sal pbar.update(1) # summary iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): print('class [{}], IoU: {:.4f}'.format(i, _iou)) print('[Eval Summary]:') print('Mean IoU: {:.4f}, Accuracy: {:.2f}%, Inference Time: {:.4f}s' .format(iou.mean(), acc_meter.average()*100, time_meter.average()))
def evaluate(segmentation_module, loader, args): acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() cls_ious_meter = AverageMeter() cls_mean_iou_meter = AverageMeter() if args.num_gpus > 1: eval_model = segmentation_module.module else: eval_model = segmentation_module eval_model.eval() for i in range(len(loader)): batch_data = next(loader)[0] # process data seg_label = as_numpy(batch_data['seg_label']) with torch.no_grad(): segSize = (seg_label.shape[1], seg_label.shape[2]) pred = torch.zeros(1, args.num_class, segSize[0], segSize[1]) # forward pass pred = eval_model(batch_data, segSize=segSize) _, preds = torch.max(pred.data.cpu(), dim=1) preds = as_numpy(preds.squeeze(0)) # calculate accuracy acc, pix = accuracy(preds, seg_label, 255) intersection, union = intersectionAndUnion(preds, seg_label, args.num_class, 255) acc_meter.update(acc, pix) intersection_meter.update(intersection) union_meter.update(union) mean_iou = (intersection / (union + 1e-10))[union != 0].mean() print('[{}] iter {}, accuracy: {:.5f}, mIoU: {:.5f}'.format( datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), i, acc, mean_iou)) iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): print('[{}] class [{}], IoU: {}'.format( datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), i, _iou)) print('[{}] [Eval Summary]:'.format( datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) print('Mean IoU: {:.4}, Accuracy: {:.2f}%'.format( iou.mean(), acc_meter.average() * 100)) return iou, iou.mean()
def evaluate(segmentation_module, loader, cfg, gpu_id, result_queue): segmentation_module.eval() for batch_data in loader: # process data batch_data = batch_data[0] seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0], segSize[1]) scores = async_copy_to(scores, gpu_id) for img in img_resized_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, gpu_id) # forward pass #scores_tmp = segmentation_module(feed_dict, segSize=segSize) scores_tmp = predict_sliding(segmentation_module, feed_dict, (520, 520), cfg.DATASET.num_class, overlap=1.0 / 3.0) scores_tmp = nn.functional.interpolate(scores_tmp, size=segSize, mode='bilinear', align_corners=False) scores = scores + scores_tmp / len(cfg.DATASET.imgSizes) _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) # calculate accuracy and SEND THEM TO MASTER acc, pix = accuracy(pred, seg_label) intersection, union = intersectionAndUnion(pred, seg_label, cfg.DATASET.num_class) result_queue.put_nowait((acc, pix, intersection, union)) # visualization if cfg.VAL.visualize: visualize_result( (batch_data['img_ori'], seg_label, batch_data['info']), pred, os.path.join(cfg.DIR, 'result'))
def eval(loader_val, segmentation_module, args, crit): intersection_meter = AverageMeter() union_meter = AverageMeter() loss_meter = AverageMeter() segmentation_module.eval() for batch_data in loader_val: batch_data = batch_data[0] seg_label = as_numpy(batch_data["mask"][0]) torch.cuda.synchronize() batch_data["image"] = batch_data["image"].unsqueeze(0).cuda() print(batch_data["image"].shape) with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) scores = torch.zeros(1, args.num_class, segSize[0], segSize[1]) scores = async_copy_to(scores, args.gpu) print("the score:", scores) feed_dict = batch_data.copy() # forward pass scores_tmp, loss = segmentation_module(feed_dict, epoch=0, segSize=segSize) scores = scores + scores_tmp print("the new score:", scores) loss_meter.update(loss) _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) print("pred shape:", pred.shape) visualize_result(batch_data["image"].cpu().numpy(), seg_label, pred, args) torch.cuda.synchronize() # calculate accuracy intersection, union = intersectionAndUnion(pred, seg_label, args.num_class) intersection_meter.update(intersection) union_meter.update(union) # summary iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): if i >= 1: print('class [{}], IoU: {:.4f}'.format(i, _iou)) print('loss: {:.4f}'.format(loss_meter.average())) return iou[1:], loss_meter.average()
def evaluate(segmentation_module, loader, args, dev_id, result_queue): segmentation_module.eval() for i, batch_data in enumerate(loader): # process data batch_data = batch_data[0] seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] quadtree_resized_list = batch_data['quadtree'] with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) pred = torch.zeros(1, args.num_class, segSize[0], segSize[1]) pred = Variable(pred).cuda() for scale, img in enumerate(img_resized_list): feed_dict = batch_data.copy() feed_dict['img_data'] = img if args.eval_mode == 'gt': feed_dict['qtree'] = quadtree_resized_list[scale] del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, dev_id) # forward pass pred_tmp = segmentation_module(feed_dict, segSize=segSize) pred = pred + pred_tmp / len(args.imgSize) _, preds = torch.max(pred.data.cpu(), dim=1) preds = as_numpy(preds.squeeze(0)) # calculate accuracy and SEND THEM TO MASTER acc, pix = accuracy(preds, seg_label) intersection, union = intersectionAndUnion(preds, seg_label, args.num_class) result_queue.put_nowait((acc, pix, intersection, union)) # visualization if args.visualize: visualize_result( (batch_data['img_ori'], seg_label, batch_data['info']), preds, args)
def evaluate(segmentation_module, loader, cfg, gpu_id, result_queue): segmentation_module.eval() for i, batch_data in enumerate(loader): # process data batch_data = batch_data[0] seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] img_ref_resized_list = batch_data['img_refs'] with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0], segSize[1]) scores = async_copy_to(scores, gpu_id) zip_list = zip(img_resized_list, img_ref_resized_list) for img, img_refs in zip_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img feed_dict['img_refs'] = img_refs del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, gpu_id) scores_tmp = segmentation_module(feed_dict, segSize=segSize) scores = scores + scores_tmp / len(cfg.DATASET.imgSizes) #scores = scores_tmp _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) # calculate accuracy and SEND THEM TO MASTER acc, pix = accuracy(pred, seg_label) intersection, union = intersectionAndUnion(pred, seg_label, cfg.DATASET.num_class) result_queue.put_nowait((acc, pix, intersection, union)) # visualization if cfg.VAL.visualize: visualize_result( (batch_data['img_ori'], seg_label, batch_data['info']), pred, os.path.join(cfg.DIR, 'result'))
def get_metrics(pred, data): metric = {} # material # valid... is originally set to 0 then turned to 1 if it is enabled metric['valid_material'] = data['valid_material'] if metric['valid_material']: metric['material'] = {} material_gt, material_pred = data['seg_material'], pred['material'] metric["material"]["acc"] = ((material_gt == material_pred) * (material_gt > 0)).sum() # ignore 0 metric["material"]["pixel"] = (material_gt > 0).sum() # ignore 0 metric["material"][ "inter"], metric["material"]["uni"] = intersectionAndUnion( material_pred, material_gt, broden_dataset.nr['material'] - 1) # ignore 0 return metric
def evaluate(segmentation_module, loader, args, dev_id, result_queue): segmentation_module.eval() for i, batch_data in enumerate(loader): # process data batch_data = batch_data[0] seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) pred = torch.zeros(1, args.num_class, segSize[0], segSize[1]) for img in img_resized_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, dev_id) # forward pass pred_tmp = segmentation_module(feed_dict, segSize=segSize) pred = pred + pred_tmp.cpu() / len(args.imgSize) _, preds = torch.max(pred.data.cpu(), dim=1) preds = as_numpy(preds.squeeze(0)) # calculate accuracy and SEND THEM TO MASTER acc, pix = accuracy(preds, seg_label) intersection, union = intersectionAndUnion(preds, seg_label, args.num_class) result_queue.put_nowait((acc, pix, intersection, union)) # visualization if args.visualize: visualize_result( (batch_data['img_ori'], seg_label, batch_data['info']), preds, args)
def evaluate(segmentation_module, loader, args, gpu_id, result_queue): segmentation_module.eval() for batch_data in loader: # process data batch_data = batch_data[0] seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) scores = torch.zeros(1, args.num_class, segSize[0], segSize[1]) scores = async_copy_to(scores, gpu_id) for img in img_resized_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, gpu_id) # forward pass scores_tmp = segmentation_module(feed_dict, segSize=segSize) scores = scores + scores_tmp / len(args.imgSize) _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) # calculate accuracy and SEND THEM TO MASTER acc, pix = accuracy(pred, seg_label) intersection, union = intersectionAndUnion(pred, seg_label, args.num_class) result_queue.put_nowait((acc, pix, intersection, union)) # visualization if args.visualize: visualize_result( (batch_data['img_ori'], seg_label, batch_data['info']), pred, args)
def eval(loader_val, segmentation_module, crit): intersection_meter = AverageMeter() union_meter = AverageMeter() loss_meter = AverageMeter() segmentation_module.eval() for batch_data in loader_val: batch_data = batch_data[0] seg_label = as_numpy(batch_data["mask"]) torch.cuda.synchronize() batch_data["image"] = batch_data["image"].unsqueeze(0).cuda() batch_data["mask"] = batch_data["mask"].cuda() with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) scores = torch.zeros(1, 4, segSize[0], segSize[1]) feed_dict = batch_data.copy() #print(torch.max(feed_dict['image'])) # forward pass scores, loss = segmentation_module(feed_dict, epoch=0, segSize=segSize) loss_meter.update(loss) _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) torch.cuda.synchronize() # calculate accuracy intersection, union = intersectionAndUnion(pred, seg_label, 4) intersection_meter.update(intersection) union_meter.update(union) # summary iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): if i >= 1: print('class [{}], IoU: {:.4f}'.format(i, _iou)) print('loss: {:.4f}'.format(loss_meter.average())) return loss_meter.average()
def evaluate(segmentation_module, loader, cfg, task, gpu): acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() time_meter = AverageMeter() segmentation_module.eval() save_pred = cfg.ATTACK[task + '_save_pred'] save_image = cfg.ATTACK[task + '_save_pred_img'] preds = [] # confs = [] iou_map = dict() pred_path = None assert task in ['orig', 'pert' ], "task should be pred or orig for attack evaluation" if save_pred or save_image: pred_path = os.path.join(cfg.ATTACK.output_dir, 'pred') if not os.path.isdir(pred_path): create_dir(pred_path) pbar = tqdm(total=len(loader)) for batch_data in loader: # process data batch_data = batch_data[0] seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] torch.cuda.synchronize() tic = time.perf_counter() with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0], segSize[1]) scores = async_copy_to(scores, gpu) for img in img_resized_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, gpu) # forward pass scores_tmp = segmentation_module(feed_dict, segSize=segSize) scores = scores + scores_tmp / len(cfg.DATASET.imgSizes) _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) preds.append(pred) # TODO: See if required # confs.append(scores.detach().numpy()) torch.cuda.synchronize() time_meter.update(time.perf_counter() - tic) # calculate accuracy acc, pix = accuracy(pred, seg_label) intersection, union = intersectionAndUnion(pred, seg_label, cfg.DATASET.num_class) acc_meter.update(acc, pix) iou_map[batch_data['info'].split('/')[-1].replace( '.jpg', '')] = intersection.sum() / union.sum() intersection_meter.update(intersection) union_meter.update(union) # save prediction if save_pred: img_name = batch_data['info'].split('/')[-1] np.save( os.path.join( pred_path, task + '_pred_' + img_name.replace('.jpg', '.npy')), np.array(pred)) # save predicted image if save_image: img_name = batch_data['info'].split('/')[-1] image_path = os.path.join( pred_path, task + '_image_' + img_name.replace('.jpg', '.png')) encode_save_image(pred, image_path) pbar.update(1) # summary iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): print('class [{}], IoU: {:.4f}'.format(i, _iou)) print('[Eval Summary]:') print( 'Mean IoU: {:.4f}, Accuracy: {:.2f}%, Inference Time: {:.4f}s'.format( iou.mean(), acc_meter.average() * 100, time_meter.average())) results = dict(iou=iou_map, class_iou=iou, accuracy=acc_meter.average() * 100) return np.array(preds), None, results
def evaluate(segmentation_module, loader, cfg, gpu_id, result_queue): segmentation_module.eval() for i, batch_data in enumerate(loader): # process data batch_data = batch_data[0] seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] img_ref_rgb_resized_list = batch_data['img_refs_rgb'] img_ref_mask_resized_list = batch_data['img_refs_mask'] with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0], segSize[1]) scores = async_copy_to(scores, gpu_id) if cfg.is_debug: zip_list = zip(img_resized_list[-2:-1], img_ref_rgb_resized_list[-2:-1], img_ref_mask_resized_list[-2:-1]) else: zip_list = zip(img_resized_list, img_ref_rgb_resized_list, img_ref_mask_resized_list) for img, img_refs_rgb, img_refs_mask in zip_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img feed_dict['img_refs_rgb'] = img_refs_rgb feed_dict['img_refs_mask'] = img_refs_mask del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, gpu_id) # forward pass if cfg.is_debug: scores_tmp, qread, qval, qk_b, mk_b, mv_b, p, feature_enc, feature_memory = segmentation_module( feed_dict, segSize=segSize) np.save('debug/qread_%03d.npy' % (i), qread.detach().cpu().float().numpy()) np.save('debug/qval_%03d.npy' % (i), qval.detach().cpu().float().numpy()) np.save('debug/qk_b_%03d.npy' % (i), qk_b.detach().cpu().float().numpy()) np.save('debug/mk_b_%03d.npy' % (i), mk_b.detach().cpu().float().numpy()) np.save('debug/mv_b_%03d.npy' % (i), mv_b.detach().cpu().float().numpy()) np.save('debug/p_%03d.npy' % (i), p.detach().cpu().float().numpy()) np.save('debug/feature_enc_%03d.npy' % (i), feature_enc[-1].detach().cpu().float().numpy()) np.save('debug/feature_memory_%03d.npy' % (i), feature_memory[-1].detach().cpu().float().numpy()) print(batch_data['info']) else: if cfg.eval_att_voting: scores_tmp, qread, qval, qk_b, mk_b, mv_b, p, feature_enc, feature_memory = segmentation_module( feed_dict, segSize=segSize) height, width = qread.shape[-2], qread.shape[-1] assert p.shape[0] == height * width img_refs_mask_resize = nn.functional.interpolate( img_refs_mask[0].cuda(), size=(height, width), mode='nearest') img_refs_mask_resize_flat = img_refs_mask_resize[:, 0, :, :].view( img_refs_mask_resize.shape[0], -1) mask_voting_flat = torch.mm(img_refs_mask_resize_flat, p) mask_voting = mask_voting_flat.view( mask_voting_flat.shape[0], height, width) mask_voting = torch.unsqueeze(mask_voting, 0) scores_tmp = nn.functional.interpolate( mask_voting[:, 1:], size=segSize, mode='bilinear', align_corners=False) '''np.save('debug/p_%03d.npy'%(i), p.detach().cpu().float().numpy()) np.save('debug/img_refs_mask_%03d.npy'%(i), img_refs_mask.cuda().detach().cpu().float().numpy()) np.save('debug/img_refs_mask_resize_%03d.npy'%(i), img_refs_mask_resize.detach().cpu().float().numpy()) np.save('debug/img_refs_mask_resize_flat_%03d.npy'%(i), img_refs_mask_resize_flat.detach().cpu().float().numpy()) np.save('debug/mask_voting_flat_%03d.npy'%(i), mask_voting_flat.detach().cpu().float().numpy()) np.save('debug/mask_voting_%03d.npy'%(i), mask_voting.detach().cpu().float().numpy())''' else: scores_tmp = segmentation_module(feed_dict, segSize=segSize) scores = scores + scores_tmp / len(cfg.DATASET.imgSizes) #scores = scores_tmp _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) # calculate accuracy and SEND THEM TO MASTER acc, pix = accuracy(pred, seg_label) intersection, union = intersectionAndUnion(pred, seg_label, cfg.DATASET.num_class) result_queue.put_nowait((acc, pix, intersection, union)) # visualization if cfg.VAL.visualize: visualize_result( (batch_data['img_ori'], seg_label, batch_data['info']), pred, os.path.join(cfg.DIR, 'result'))
mask[label == 255] = 0 mask[label == 0] = 1 print(label.shape) input = transforms.ToTensor()(input) input = transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])(input) input = input.view((-1, ) + input.shape) # print(input.shape) output = model(input.to(device)) # print(output[:,1].shape) # print(output.shape) result = np.zeros(size + (3, )) new_mask = torch.argmax(output, dim=1) print(new_mask.shape) new_mask = new_mask.cpu().numpy() area_intersection, area_union = intersectionAndUnion( new_mask[0, :, :], mask, num_classes) MIou = area_intersection[0] / area_union[0] print("MIou:%.3f" % MIou) print(area_intersection, area_union) for i in range(256): for j in range(256): # print(output[0][:,i,j]) # print(new_mask[i,j,:]) if new_mask[0, i, j] == 0: result[i, j] = COLOR_DICT[0] else: result[i, j] = COLOR_DICT[1] # result = cv2.resize(result, (512,512)) cv2.namedWindow("test", cv2.WINDOW_NORMAL) cv2.imshow("test", result) cv2.imwrite("data/membrane/result/15.png", result)
def train(train_loader, model, criterion, optimizer, epoch, zoom_factor, batch_size, aux_weight, fcw): batch_time = AverageMeter() data_time = AverageMeter() main_loss_meter = AverageMeter() aux_loss_meter = AverageMeter() loss_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() target_meter = AverageMeter() model.train() fcw.train() world_size = dist.get_world_size() rank = dist.get_rank() end = time.time() for i, (input, target) in enumerate(train_loader): # to avoid bn problem in ppm module with bin size 1x1, sometimes n may get 1 on one gpu during the last batch, so just discard # if input.shape[0] < batch_size: # continue data_time.update(time.time() - end) current_iter = (epoch - 1) * len(train_loader) + i + 1 max_iter = args.epochs * len(train_loader) if args.net_type == 0: index_split = 4 elif args.net_type in [1, 2, 3]: index_split = 5 poly_learning_rate(optimizer, args.base_lr, current_iter, max_iter, power=args.power, index_split=index_split) input = input.cuda() input_var = torch.autograd.Variable(input) fcw_input = fcw(input_var) output, aux = model(input_var, fcw_input) if zoom_factor != 8: h = int((target.size()[1]-1) / 8 * zoom_factor + 1) w = int((target.size()[2] - 1) / 8 * zoom_factor + 1) # 'nearest' mode doesn't support downsampling operation and while 'bilinear' mode is also fine target = F.upsample(target.unsqueeze(1).float(), size=(h, w), mode='bilinear').squeeze(1).long() target = target.data.cuda(async=True) target_var = torch.autograd.Variable(target) main_loss = criterion(output, target_var) / world_size aux_loss = criterion(aux, target_var) / world_size loss = main_loss + aux_weight * aux_loss optimizer.zero_grad() loss.backward() average_gradients(model) optimizer.step() output = output.data.max(1)[1].cpu().numpy() target = target.cpu().numpy() intersection, union, target = intersectionAndUnion(output, target, args.classes, args.ignore_label) intersection_meter.update(intersection) union_meter.update(union) target_meter.update(target) accuracy = sum(intersection_meter.val) / (sum(target_meter.val) + 1e-10) reduced_loss = loss.data.clone() reduced_main_loss = main_loss.data.clone() reduced_aux_loss = aux_loss.data.clone() dist.all_reduce(reduced_loss) dist.all_reduce(reduced_main_loss) dist.all_reduce(reduced_aux_loss) main_loss_meter.update(reduced_main_loss[0], input.size(0)) aux_loss_meter.update(reduced_aux_loss[0], input.size(0)) loss_meter.update(reduced_loss[0], input.size(0)) batch_time.update(time.time() - end) end = time.time() # calculate remain time remain_iter = max_iter - current_iter remain_time = remain_iter * batch_time.avg t_m, t_s = divmod(remain_time, 60) t_h, t_m = divmod(t_m, 60) remain_time = '{:02d}:{:02d}:{:02d}'.format(int(t_h), int(t_m), int(t_s)) if rank == 0: if (i + 1) % args.print_freq == 0: logger.info('Epoch: [{}/{}][{}/{}] ' 'Data {data_time.val:.3f} ({data_time.avg:.3f}) ' 'Batch {batch_time.val:.3f} ({batch_time.avg:.3f}) ' 'Remain {remain_time} ' 'MainLoss {main_loss_meter.val:.4f} ' 'AuxLoss {aux_loss_meter.val:.4f} ' 'Loss {loss_meter.val:.4f} ' 'Accuracy {accuracy:.4f}.'.format(epoch, args.epochs, i + 1, len(train_loader), batch_time=batch_time, data_time=data_time, remain_time=remain_time, main_loss_meter=main_loss_meter, aux_loss_meter=aux_loss_meter, loss_meter=loss_meter, accuracy=accuracy)) writer.add_scalar('loss_train_batch', main_loss_meter.val, current_iter) writer.add_scalar('mIoU_train_batch', np.mean(intersection / (union + 1e-10)), current_iter) writer.add_scalar('mAcc_train_batch', np.mean(intersection / (target + 1e-10)), current_iter) writer.add_scalar('allAcc_train_batch', accuracy, current_iter) iou_class = intersection_meter.sum / (union_meter.sum + 1e-10) accuracy_class = intersection_meter.sum / (target_meter.sum + 1e-10) mIoU = np.mean(iou_class) mAcc = np.mean(accuracy_class) allAcc = sum(intersection_meter.sum) / (sum(target_meter.sum) + 1e-10) if rank == 0: logger.info('Train result at epoch [{}/{}]: mIoU/mAcc/allAcc {:.4f}/{:.4f}/{:.4f}.'.format(epoch, args.epochs, mIoU, mAcc, allAcc)) return main_loss_meter.avg, mIoU, mAcc, allAcc
def evaluate(nets, loader, history, epoch, args): print('Evaluating at {} epochs...'.format(epoch)) loss_meter = AverageMeter() acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() # switch to eval mode for net in nets: net.eval() for i, batch_data in enumerate(loader): # forward pass torch.cuda.empty_cache() pred, err, agreemap = forward_with_loss_agreemap(nets, batch_data, args, is_train=False) loss_meter.update(err.data[0]) print('[Eval] iter {}, loss: {}'.format(i, err.data[0])) # calculate accuracy acc, pix = accuracy(batch_data, pred) acc_meter.update(acc, pix) intersection, union = intersectionAndUnion(batch_data, pred, args.num_class) intersection_meter.update(intersection) union_meter.update(union) # visualization visualize_2(batch_data, pred, agreemap, epoch, args) iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): print('class [{}], IoU: {}'.format(trainID2Class[i], _iou)) print('[Eval Summary]:') print('Epoch: {}, Loss: {}, Mean IoU: {:.4}, Accurarcy: {:.2f}%'.format( epoch, loss_meter.average(), iou.mean(), acc_meter.average() * 100)) history['val']['epoch'].append(epoch) history['val']['err'].append(loss_meter.average()) history['val']['acc'].append(acc_meter.average()) history['val']['mIoU'].append(iou.mean()) # Plot figure if epoch > 0: print('Plotting loss figure...') fig = plt.figure() plt.plot(np.asarray(history['train']['epoch']), np.log(np.asarray(history['train']['err'])), color='b', label='training') plt.plot(np.asarray(history['val']['epoch']), np.log(np.asarray(history['val']['err'])), color='c', label='validation') plt.legend() plt.xlabel('Epoch') plt.ylabel('Log(loss)') fig.savefig('{}/loss.png'.format(args.ckpt), dpi=200) plt.close('all') fig = plt.figure() plt.plot(history['train']['epoch'], history['train']['acc'], color='b', label='training') plt.plot(history['val']['epoch'], history['val']['acc'], color='c', label='validation') plt.legend() plt.xlabel('Epoch') plt.ylabel('Accuracy') fig.savefig('{}/accuracy.png'.format(args.ckpt), dpi=200) plt.close('all')
def evaluate(segmentation_module, loader, cfg, gpu): acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() groundthuth_meter = AverageMeter() time_meter = AverageMeter() segmentation_module.eval() pbar = tqdm(total=len(loader)) for batch_data in loader: # process data batch_data = batch_data[0] seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] torch.cuda.synchronize() tic = time.perf_counter() with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0], segSize[1]) scores = async_copy_to(scores, gpu) for img in img_resized_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, gpu) # forward pass scores_tmp = segmentation_module(feed_dict, segSize=segSize) scores = scores + scores_tmp / 1 #len(cfg.DATASET.imgSizes) _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) torch.cuda.synchronize() time_meter.update(time.perf_counter() - tic) # calculate accuracy acc, pix = accuracy(pred, seg_label) intersection, union, groundthuth = intersectionAndUnion( pred, seg_label, cfg.DATASET.num_class) acc_meter.update(acc, pix) intersection_meter.update(intersection) union_meter.update(union) groundthuth_meter.update(groundthuth) # visualization if cfg.VAL.visualize: visualize_result( (batch_data['img_ori'], seg_label, batch_data['info']), pred, os.path.join(cfg.DIR, 'val')) pbar.update(1) # summary iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): print('class [{}], IoU: {:.4f}'.format(i, _iou)) sensitive_iou = intersection_meter.sum / (groundthuth_meter.sum + 1e-10) for i, _iou in enumerate(sensitive_iou): print('class [{}], Sensitive_IoU: {:.4f}'.format(i, _iou)) print('[Eval Summary]:') print( 'Mean IoU: {:.4f}, Accuracy: {:.2f}%, Inference Time: {:.4f}s'.format( iou.mean(), acc_meter.average() * 100, time_meter.average()))
def evaluate(nets, loader, loader_2, history, epoch, args): print('Evaluating at {} epochs...'.format(epoch)) loss_meter = AverageMeter() acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() loss_meter_2 = AverageMeter() acc_meter_2 = AverageMeter() intersection_meter_2 = AverageMeter() union_meter_2 = AverageMeter() # switch to eval mode for net in nets: net.eval() for i, batch_data in enumerate(loader): # forward pass pred, err = forward_with_loss(nets, batch_data, is_train=False) loss_meter.update(err.data.item()) if i % args.disp_iter == 0: print('[Eval] iter {}, loss: {}'.format(i, err.data.item())) # calculate accuracy acc, pix = accuracy(batch_data, pred) acc_meter.update(acc, pix) intersection, union = intersectionAndUnion(batch_data, pred, args.num_class) intersection_meter.update(intersection) union_meter.update(union) iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): print('class [{}], IoU: {}'.format(trainID2Class[i], _iou)) print('[Cityscapes Eval Summary]:') print('Epoch: {}, Loss: {}, Mean IoU: {:.4}, Accuracy: {:.2f}%'.format( epoch, loss_meter.average(), iou.mean(), acc_meter.average() * 100)) history['val']['epoch'].append(epoch) history['val']['err'].append(loss_meter.average()) history['val']['acc'].append(acc_meter.average()) history['val']['mIoU'].append(iou.mean()) for i, batch_data in enumerate(loader_2): # forward pass pred, err = forward_with_loss(nets, batch_data, is_train=False) loss_meter_2.update(err.data.item()) if i % args.disp_iter == 0: print('[Eval] iter {}, loss: {}'.format(i, err.data.item())) # calculate accuracy acc, pix = accuracy(batch_data, pred) acc_meter_2.update(acc, pix) intersection, union = intersectionAndUnion(batch_data, pred, args.num_class) intersection_meter_2.update(intersection) union_meter_2.update(union) iou = intersection_meter_2.sum / (union_meter_2.sum + 1e-10) for i, _iou in enumerate(iou): print('class [{}], IoU: {}'.format(trainID2Class[i], _iou)) print('[BDD Eval Summary]:') print('Epoch: {}, Loss: {}, Mean IoU: {:.4}, Accuracy: {:.2f}%'.format( epoch, loss_meter_2.average(), iou.mean(), acc_meter_2.average() * 100)) history['val_2']['epoch'].append(epoch) history['val_2']['err'].append(loss_meter_2.average()) history['val_2']['acc'].append(acc_meter_2.average()) history['val_2']['mIoU'].append(iou.mean()) # Plot figure if epoch > 0: fig = plt.figure() plt.plot(np.asarray(history['train']['epoch']), np.log(np.asarray(history['train']['err'])), color='b', label='gta') plt.plot(np.asarray(history['val']['epoch']), np.log(np.asarray(history['val']['err'])), color='c', label='cityscapes') plt.plot(np.asarray(history['val_2']['epoch']), np.log(np.asarray(history['val_2']['err'])), color='g', label='bdd') plt.legend() plt.xlabel('Epoch') plt.ylabel('Log(loss)') fig.savefig('{}/loss.png'.format(args.ckpt), dpi=200) plt.close('all') fig = plt.figure() plt.plot(history['train']['epoch'], history['train']['acc'], color='b', label='gta') plt.plot(history['val']['epoch'], history['val']['acc'], color='c', label='cityscapes') plt.plot(history['val_2']['epoch'], history['val_2']['acc'], color='g', label='bdd') plt.legend() plt.xlabel('Epoch') plt.ylabel('Accuracy') fig.savefig('{}/accuracy.png'.format(args.ckpt), dpi=200) plt.close('all')
def evaluate(segmentation_module, loader, loader_rec, cfg, gpu): acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() time_meter = AverageMeter() segmentation_module.eval() aurocs, auprs, fprs = [], [], [] pbar = tqdm(total=len(loader)) for batch_data, rec_data in zip(loader, loader_rec): # process data batch_data = batch_data[0] seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] #print(batch_data['name']) full_name = batch_data['name'] img_folder, img_name = batch_data['name'].split('/') del batch_data['name'] rec_data = rec_data[0] seg_label_rec = as_numpy(rec_data['seg_label'][0]) img_resized_list_rec = rec_data['img_data'] #print(rec_data['name']) del rec_data['name'] torch.cuda.synchronize() tic = time.perf_counter() with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0], segSize[1]) ft1 = torch.zeros(1, 4096, int(segSize[0] / 4), int(segSize[1] / 4)) ft2 = torch.zeros(1, 4096, int(segSize[0] / 4), int(segSize[1] / 4)) scores = async_copy_to(scores, gpu) ft1 = async_copy_to(ft1, gpu) ft2 = async_copy_to(ft2, gpu) for img in img_resized_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, gpu) # forward pass scores_tmp, ft_temp = segmentation_module(feed_dict, segSize=segSize) scores = scores + scores_tmp / len(cfg.DATASET.imgSizes) ft_temp = nn.functional.interpolate(ft_temp, size=ft1.shape[2:], mode='bilinear', align_corners=False) ft1 = ft1 + ft_temp / len(cfg.DATASET.imgSizes) for img in img_resized_list_rec: feed_dict = rec_data.copy() feed_dict['img_data'] = img del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, gpu) # forward pass _, ft_temp = segmentation_module(feed_dict, segSize=segSize) ft_temp = nn.functional.interpolate(ft_temp, size=ft2.shape[2:], mode='bilinear', align_corners=False) ft2 = ft2 + ft_temp / len(cfg.DATASET.imgSizes) tmp_scores = scores if cfg.OOD.exclude_back: tmp_scores = tmp_scores[:, 1:] mask = None _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) #for evaluating MSP if cfg.OOD.ood == "msp": conf, _ = torch.max(tmp_scores, dim=1) conf = as_numpy(conf.squeeze(0).cpu()) elif cfg.OOD.ood == "rec": msp, _ = torch.max(tmp_scores, dim=1) msp = msp.squeeze(0).cpu() ft1 = nn.functional.normalize(ft1, dim=1) ft2 = nn.functional.normalize(ft2, dim=1) #ft_dist = torch.nn.SmoothL1Loss(reduction='none')(ft1, ft2) ft_dist = nn.functional.cosine_similarity(ft1, ft2, dim=1).unsqueeze(1) ft_dist = nn.functional.interpolate( ft_dist, size=segSize, mode='bilinear', align_corners=False)[0, 0].cpu() conf_rec = ft_dist t = 0.999 conf = msp * (msp > t).float() + conf_rec * (msp <= t).float() #conf = 1- (1 - conf_rec) * (msp <= 0.99).float() conf = as_numpy(conf.squeeze(0).cpu()) res = eval_ood_measure(conf, seg_label, cfg, mask=mask) if res is not None: auroc, aupr, fpr = res aurocs.append(auroc) auprs.append(aupr), fprs.append(fpr) else: pass res1 = eval_ood_measure(msp, seg_label, cfg, mask=mask) if res is not None: auroc1, aupr1, fpr1 = res1 #aurocs.append(auroc); auprs.append(aupr), fprs.append(fpr) else: pass torch.cuda.synchronize() time_meter.update(time.perf_counter() - tic) # calculate accuracy acc, pix = accuracy(pred, seg_label) intersection, union = intersectionAndUnion(pred, seg_label, cfg.DATASET.num_class) acc_meter.update(acc, pix) intersection_meter.update(intersection) union_meter.update(union) # visualization if cfg.VAL.visualize: visualize_result( (batch_data['img_ori'], seg_label, batch_data['info']), pred, os.path.join(cfg.DIR, 'result')) pbar.update(1) torch.cuda.empty_cache() # summary iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): print('class [{}], IoU: {:.4f}'.format(i, _iou)) print('[Eval Summary]:') print( 'Mean IoU: {:.4f}, Accuracy: {:.2f}%, Inference Time: {:.4f}s'.format( iou.mean(), acc_meter.average() * 100, time_meter.average())) print("mean auroc = ", np.mean(aurocs), "mean aupr = ", np.mean(auprs), " mean fpr = ", np.mean(fprs))
def evaluate(segmentation_module, loader, cfg, gpu): acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() time_meter = AverageMeter() segmentation_module.eval() aurocs, auprs, fprs = [], [], [] pbar = tqdm(total=len(loader)) for batch_data in loader: # process data batch_data = batch_data[0] seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] torch.cuda.synchronize() tic = time.perf_counter() with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) scores = torch.zeros(1, cfg.DATASET.num_class, segSize[0], segSize[1]) scores = async_copy_to(scores, gpu) for img in img_resized_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, gpu) # forward pass scores_tmp = segmentation_module(feed_dict, segSize=segSize) scores = scores + scores_tmp / len(cfg.DATASET.imgSizes) tmp_scores = scores if cfg.OOD.exclude_back: tmp_scores = tmp_scores[:, 1:] mask = None _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) #for evaluating MSP if cfg.OOD.ood == "msp": conf, _ = torch.max(nn.functional.softmax(tmp_scores, dim=1), dim=1) conf = as_numpy(conf.squeeze(0).cpu()) elif cfg.OOD.ood == "maxlogit": conf, _ = torch.max(tmp_scores, dim=1) conf = as_numpy(conf.squeeze(0).cpu()) elif cfg.OOD.ood == "background": conf = tmp_scores[:, 0] conf = as_numpy(conf.squeeze(0).cpu()) elif cfg.OOD.ood == "crf": import pydensecrf.densecrf as dcrf from pydensecrf.utils import unary_from_softmax, create_pairwise_bilateral, create_pairwise_gaussian ch, h, w = scores.squeeze(0).size() d = dcrf.DenseCRF2D(h, w, ch) # width, height, nlabels tmp_scores = as_numpy( nn.functional.softmax(tmp_scores, dim=1).squeeze(0)) tmp_scores = as_numpy(tmp_scores) U = unary_from_softmax(tmp_scores) d.setUnaryEnergy(U) pairwise_energy = create_pairwise_bilateral(sdims=(10, 10), schan=13, img=tmp_scores, chdim=0) d.addPairwiseEnergy(pairwise_energy, compat=10) # Run inference for 100 iterations Q_unary = d.inference(100) # The Q is now the approximate posterior, we can get a MAP estimate using argmax. map_soln_unary = np.argmax(Q_unary, axis=0) # Unfortunately, the DenseCRF flattens everything, so get it back into picture form. map_soln_unary = map_soln_unary.reshape((h, w)) conf = np.max(Q_unary, axis=0).reshape((h, w)) elif cfg.OOD.ood == "crf-gauss": import pydensecrf.densecrf as dcrf from pydensecrf.utils import unary_from_softmax, create_pairwise_bilateral, create_pairwise_gaussian ch, h, w = scores.squeeze(0).size() d = dcrf.DenseCRF2D(h, w, ch) # width, height, nlabels tmp_scores = as_numpy( nn.functional.softmax(tmp_scores, dim=1).squeeze(0)) tmp_scores = as_numpy(tmp_scores) U = unary_from_softmax(tmp_scores) d.setUnaryEnergy(U) d.addPairwiseGaussian( sxy=3, compat=3) # `compat` is the "strength" of this potential. # Run inference for 100 iterations Q_unary = d.inference(100) # The Q is now the approximate posterior, we can get a MAP estimate using argmax. map_soln_unary = np.argmax(Q_unary, axis=0) # Unfortunately, the DenseCRF flattens everything, so get it back into picture form. map_soln_unary = map_soln_unary.reshape((h, w)) conf = np.max(Q_unary, axis=0).reshape((h, w)) res = eval_ood_measure(conf, seg_label, cfg, mask=mask) if res is not None: auroc, aupr, fpr = res aurocs.append(auroc) auprs.append(aupr), fprs.append(fpr) else: pass torch.cuda.synchronize() time_meter.update(time.perf_counter() - tic) # calculate accuracy acc, pix = accuracy(pred, seg_label) intersection, union = intersectionAndUnion(pred, seg_label, cfg.DATASET.num_class) acc_meter.update(acc, pix) intersection_meter.update(intersection) union_meter.update(union) # visualization if cfg.VAL.visualize: visualize_result( (batch_data['img_ori'], seg_label, batch_data['info']), pred, os.path.join(cfg.DIR, 'result')) pbar.update(1) # summary iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): print('class [{}], IoU: {:.4f}'.format(i, _iou)) print('[Eval Summary]:') print( 'Mean IoU: {:.4f}, Accuracy: {:.2f}%, Inference Time: {:.4f}s'.format( iou.mean(), acc_meter.average() * 100, time_meter.average())) print("mean auroc = ", np.mean(aurocs), "mean aupr = ", np.mean(auprs), " mean fpr = ", np.mean(fprs))
def get_metrics(pred, data): metric = {} # scene metric['scene'] = {} metric['scene']['gt'] = data['scene_label'] if metric['scene']['gt'] != -1: metric['scene']['top1'] = (pred['scene'] == metric['scene']['gt']) # object, part metric['valid_object'] = data['valid_object'] if metric['valid_object']: # object metric['object'] = {} object_gt, object_pred = data['seg_object'], pred['object'] metric["object"]["acc"] = ((object_gt == object_pred) * (object_gt > 0)).sum() # ignore 0 metric["object"]["pixel"] = (object_gt > 0).sum() # ignore 0 metric["object"]["inter"], metric["object"]["uni"] = intersectionAndUnion( object_pred, object_gt, broden_dataset.nr['object'] - 1) # ignore 0 # parts metric['part'] = [] metric["valid_part"] = data["valid_part"] for object_part_idx, object_label in enumerate(broden_dataset.object_with_part): if not data['valid_part'][object_part_idx]: metric["part"].append(None) continue # NOTE: nr part include background 0. nr_part = len(broden_dataset.object_part[object_label]) object_pred_mask = (object_pred == object_label) object_gt_mask = (object_gt == object_label) parts_gt = data["seg_part"] * object_gt_mask parts_pred = pred['part'][object_part_idx] * object_pred_mask _result = {} _result["acc"] = ((parts_gt == parts_pred) * (parts_gt > 0)).sum() # ignore 0 _result["pixel"] = (parts_gt > 0).sum() # ignore 0 _result["inter"], _result["uni"] = intersection_union_part( parts_pred, parts_gt, nr_part) # mIoU-bg, include background 0 metric["part"].append(_result) # material metric['valid_material'] = data['valid_material'] if metric['valid_material']: metric['material'] = {} material_gt, material_pred = data['seg_material'], pred['material'] metric["material"]["acc"] = ((material_gt == material_pred) * (material_gt > 0)).sum() # ignore 0 metric["material"]["pixel"] = (material_gt > 0).sum() # ignore 0 metric["material"]["inter"], metric["material"]["uni"] = intersectionAndUnion( material_pred, material_gt, broden_dataset.nr['material'] - 1) # ignore 0 return metric
def evaluate(segmentation_module, loader, args): acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() time_meter = AverageMeter() segmentation_module.eval() pbar = tqdm(total=len(loader)) for batch_data in loader: # process data batch_data = batch_data[0] #print(batch_data[0]) seg_label = as_numpy(batch_data['seg_label']) img_resized_list = batch_data['img_data'].unsqueeze(0) #print(seg_label.shape) #print(img_resized_list.shape) ''' # fix the dimension permutation for idx, item in enumerate(batch_data['img_data']): print(item.shape) batch_data['img_data'][idx] = item.permute(0, 3, 1, 2) print() for item in batch_data['img_data']: print(item.shape) ''' torch.cuda.synchronize() tic = time.perf_counter() with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) scores = torch.zeros(1, args.num_class, segSize[0], segSize[1]) scores = async_copy_to(scores, args.gpu) for img in img_resized_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, args.gpu) # forward pass scores_tmp = segmentation_module(feed_dict, segSize=segSize) scores = scores + scores_tmp / len(args.imgSize) _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) torch.cuda.synchronize() time_meter.update(time.perf_counter() - tic) # calculate accuracy acc, pix = accuracy(pred, seg_label) intersection, union = intersectionAndUnion(pred, seg_label, args.num_class) acc_meter.update(acc, pix) intersection_meter.update(intersection) union_meter.update(union) # visualization if True: # args.visualize visualize_result( (batch_data['img_ori'], seg_label, batch_data['info']), pred, args) pbar.update(1) # summary iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): print('class [{}], IoU: {:.4f}'.format(i, _iou)) print('[Eval Summary]:') print( 'Mean IoU: {:.4f}, Accuracy: {:.2f}%, Inference Time: {:.4f}s'.format( iou.mean(), acc_meter.average() * 100, time_meter.average()))
def evaluate(nets, loader, loader_2, history, epoch, args, isVis=True): print('Evaluating at {} epochs...'.format(epoch)) loss_meter = AverageMeter() acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() loss_meter_2 = AverageMeter() acc_meter_2 = AverageMeter() intersection_meter_2 = AverageMeter() union_meter_2 = AverageMeter() # switch to eval mode for net in nets: net.eval() for i, batch_data in enumerate(loader): # forward pass torch.cuda.empty_cache() pred, err = forward_with_loss(nets, batch_data, is_train=False) loss_meter.update(err.data.item()) print('[Eval] iter {}, loss: {}'.format(i, err.data.item())) # calculate accuracy acc, pix = accuracy(batch_data, pred) acc_meter.update(acc, pix) intersection, union = intersectionAndUnion(batch_data, pred, args.num_class) intersection_meter.update(intersection) union_meter.update(union) # visualization if isVis: visualize(batch_data, pred, args) for i, batch_data in enumerate(loader_2): # forward pass torch.cuda.empty_cache() pred, err = forward_with_loss(nets, batch_data, is_train=False) loss_meter_2.update(err.data.item()) print('[Eval] iter {}, loss: {}'.format(i, err.data.item())) # calculate accuracy acc, pix = accuracy(batch_data, pred) acc_meter_2.update(acc, pix) intersection, union = intersectionAndUnion(batch_data, pred, args.num_class) intersection_meter_2.update(intersection) union_meter_2.update(union) # visualization if isVis: visualize(batch_data, pred, args) iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): print('class [{}], IoU: {}'.format(trainID2Class[i], _iou)) print('[Cityscapes Eval Summary]:') print('Epoch: {}, Loss: {}, Mean IoU: {:.4}, Accuracy: {:.2f}%' .format(epoch, loss_meter.average(), iou.mean(), acc_meter.average() * 100)) history['val']['epoch'].append(epoch) history['val']['err'].append(loss_meter.average()) history['val']['acc'].append(acc_meter.average()) history['val']['mIoU'].append(iou.mean()) iou = intersection_meter_2.sum / (union_meter_2.sum + 1e-10) for i, _iou in enumerate(iou): print('class [{}], IoU: {}'.format(trainID2Class[i], _iou)) print('[BDD Eval Summary]:') print('Epoch: {}, Loss: {}, Mean IoU: {:.4}, Accuracy: {:.2f}%' .format(epoch, loss_meter_2.average(), iou.mean(), acc_meter_2.average() * 100)) history['val_2']['epoch'].append(epoch) history['val_2']['err'].append(loss_meter_2.average()) history['val_2']['acc'].append(acc_meter_2.average()) history['val_2']['mIoU'].append(iou.mean())
def evaluate_segmentation(segmentation_module, loader, args): acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() time_meter = AverageMeter() dataset = loader.dataset segmentation_module.eval() print('Start evaluation of %d batches' % (len(loader))) pbar = tqdm(total=len(loader)) for batch_data in loader: # process data batch_data = batch_data[0] seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] torch.cuda.synchronize() tic = time.perf_counter() with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) scores = torch.zeros(1, dataset.num_classes, segSize[0], segSize[1]) scores = async_copy_to(scores, args.gpu) for img in img_resized_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, args.gpu) # forward pass scores_tmp = segmentation_module(feed_dict, segSize=segSize) scores = scores + scores_tmp / len(args.imgSize) _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) torch.cuda.synchronize() time_meter.update(time.perf_counter() - tic) # calculate accuracy acc, pix = accuracy(pred, seg_label) intersection, union = intersectionAndUnion(pred, seg_label, dataset.num_classes) acc_meter.update(acc, pix) intersection_meter.update(intersection) union_meter.update(union) # visualization if args.visualize: visualize_result( (batch_data['img_ori'], seg_label, batch_data['info']), pred, args) pbar.update(1) # summary iou = intersection_meter.sum / (union_meter.sum + 1e-10) per_class_iou = [] for i, _iou in enumerate(iou): print('class [{}], IoU: {:.4f}'.format(i, _iou)) per_class_iou.append(_iou) print('[Eval Summary]:') print( 'Mean IoU: {:.4f}%, Accuracy: {:.2f}%, Inference Time: {:.4f}s'.format( iou.mean() * 100, acc_meter.average() * 100, time_meter.average())) metrics = { 'mean_iou': float(iou.mean()), 'acc': float(acc_meter.average()), 'per_class_iou': per_class_iou, } return metrics
def evaluate_surface_normals(segmentation_module, loader, args): acc_meter = AverageMeter() intersection_meter = AverageMeter() union_meter = AverageMeter() time_meter = AverageMeter() dataset = loader.dataset vocab_file = dataset.vocab_file print('Loading normals') img_id_to_normals_data = dataset.load_normals_file() segmentation_module.eval() normals_evaluator = NormalsEvaluator(vocab_file=vocab_file) print('Start evaluation of %d batches' % (len(loader))) pbar = tqdm(total=len(loader)) accum_probs = [] accum_gt = [] accum_masks = [] for batch_data in loader: # process data batch_data = batch_data[0] seg_label = as_numpy(batch_data['seg_label'][0]) img_resized_list = batch_data['img_data'] img_id = batch_data['info'] raw_normals, valid_mask = img_id_to_normals_data(img_id) torch.cuda.synchronize() tic = time.perf_counter() with torch.no_grad(): segSize = (seg_label.shape[0], seg_label.shape[1]) if len(img_resized_list) > 1: scores = torch.zeros(1, dataset.num_classes, segSize[0], segSize[1]) scores = async_copy_to(scores, args.gpu) for img in img_resized_list: feed_dict = batch_data.copy() feed_dict['img_data'] = img del feed_dict['img_ori'] del feed_dict['info'] feed_dict = async_copy_to(feed_dict, args.gpu) # forward pass scores_tmp = segmentation_module(feed_dict, segSize=segSize) if len(img_resized_list) > 1: scores = scores + scores_tmp / len(args.imgSize) else: scores = scores_tmp / len(args.imgSize) probs = scores _, pred = torch.max(scores, dim=1) pred = as_numpy(pred.squeeze(0).cpu()) prob_np = as_numpy(probs) torch.cuda.synchronize() time_meter.update(time.perf_counter() - tic) # calculate accuracy acc, pix = accuracy(pred, seg_label) intersection, union = intersectionAndUnion(pred, seg_label, dataset.num_classes) acc_meter.update(acc, pix) intersection_meter.update(intersection) union_meter.update(union) # accumulate probs for normal # since we compute median stats, we cannot average over images accum_probs.append(prob_np.squeeze()) accum_gt.append(raw_normals) accum_masks.append(valid_mask.astype(np.int32)) # visualization if args.visualize: visualize_result( (batch_data['img_ori'], seg_label, batch_data['info']), pred, args) pbar.update(1) # compute normal metrics print("Computing pixelwise normal metrics") metrics = normals_evaluator.compute_normals_metrics_from_network_probs( accum_probs, accum_gt, accum_masks) normals_evaluator.print_metrics(metrics) # summary iou = intersection_meter.sum / (union_meter.sum + 1e-10) for i, _iou in enumerate(iou): print('class [{}], IoU: {:.4f}'.format(i, _iou)) print('[Eval Summary]:') print( 'Mean IoU: {:.4f}%, Accuracy: {:.2f}%, Inference Time: {:.4f}s'.format( iou.mean() * 100, acc_meter.average() * 100, time_meter.average())) return metrics
def evaluate(model, loader, gpu_mode, num_class=7): # output format res = { 'loss': 0.1, 'acc': 0.2, # or acc for every category, 'iou': 0.3 } # metric meters loss_meter = AverageMeter() acc_meter = AverageMeter() inter_meter = AverageMeter() union_meter = AverageMeter() confusion_matrix = np.zeros((num_class, num_class)) for i_batch, (img, mask, _) in enumerate(loader): if gpu_mode: img = img.cuda() mask = mask.cuda() output = model(img) # calculate loss loss = cross_entropy2d(output, mask) loss_value = loss.data.cpu().numpy() loss_meter.update(loss_value) output = output.max(1)[1] # calculate accuracy acc = accuracy(output, mask) acc_meter.update(acc) # if gpu_mode: # output = output.int().cpu().detach().numpy() # mask = mask.int().cpu().detach().numpy() # seg_pred = np.array(output).reshape(1, -1) # seg_gt = np.array(mask).reshape(1, -1) # confusion_matrix += get_confusion_matrix(seg_gt, seg_pred, 7) # # pos = confusion_matrix.sum(1) # res = confusion_matrix.sum(0) # tp = np.diag(confusion_matrix) # # IU_array = (tp / np.maximum(1.0, pos + res - tp)) # mean_IU = IU_array.mean() # calculate iou intersection, union = intersectionAndUnion(output, mask, num_class) inter_meter.update(intersection) union_meter.update(union) # summary # iou = IU_array # iou_mean = IU_array.mean() iou = inter_meter.sum / (union_meter.sum + 1e-10) iou_mean = iou.mean() acc_mean = acc_meter.average() loss_mean = loss_meter.average() res['loss'] = loss_mean res['acc'] = acc_mean res['iou'] = iou res['iou_mean'] = iou_mean return res