def evaluate_from_results_file(gt_data_loader, results_from_file): coco = get_coco_api_from_dataset(gt_data_loader.dataset) iou_types = ["bbox"] # NOTE(rjbruin): hardcoded to only do bboxes coco_evaluator = CocoEvaluator(coco, iou_types) coco_evaluator.put_results(results_from_file) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(torch.get_num_threads()) return coco_evaluator
def perform_eval_inference(model, data_loader, device): # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) results_for_file = {iou_type: [] for iou_type in coco_evaluator.iou_types} for image, targets in metric_logger.log_every(data_loader, 100, header): image = list(img.to(device) for img in image) torch.cuda.synchronize() model_time = time.time() outputs = model(image) model_time = time.time() - model_time # DEBUG if isinstance(targets, dict): targets = [targets] targets = [{ k: v.to(cpu_device).detach().numpy() for k, v in t.items() } for t in targets] outputs = [{ k: v.to(cpu_device).detach().numpy() for k, v in o.items() } for o in outputs] res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() batch_results_for_file = coco_evaluator.update_inference(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) for iou_type in coco_evaluator.iou_types: results_for_file[iou_type].extend(batch_results_for_file[iou_type]) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) # coco_evaluator.synchronize_between_processes() return results_for_file
def evaluate(model, data_loader, device): n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) for image, targets in metric_logger.log_every(data_loader, 100, header): image = list(img.to(device) for img in image) targets = [{k: v.to(device) for k, v in t.items()} for t in targets] torch.cuda.synchronize() model_time = time.time() outputs = model(image) outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] model_time = time.time() - model_time res = { target["image_id"].item(): output for target, output in zip(targets, outputs) } evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) return coco_evaluator
def save_groundtruths_coco(gt_data_loader, filepath): coco = get_coco_api_from_dataset(gt_data_loader.dataset) with open(filepath + '.json', 'w') as f: json.dump(coco.dataset, f)