def check_completed_tags():
# Evaluator is only valid on master rank - all others will have nothing.
# So, assemble lists of completed runs on master
if is_main_process():
evaluator = get_evaluator()
# loop over all all epoch, result pairs that have finished
all_results = {}
for t, r in evaluator.finished_tasks().items():
# Note: one indirection due to possibility of multiple test datasets
# we only care about the first
map_results = r# [0]
bbox_map = map_results.results["bbox"]['AP']
segm_map = map_results.results["segm"]['AP']
all_results.update({ t : (bbox_map, segm_map) })
return all_results
return {}
def do_coco_evaluation(
dataset,
predictions,
box_only,
output_folder,
iou_types,
expected_results,
expected_results_sigma_tol,
):
logger = logging.getLogger("maskrcnn_benchmark.inference")
# Different path here, fast parallel method not available, fall back to effectively the old
# path.
if box_only:
predictions = _accumulate_predictions_from_multiple_gpus(predictions)
if not is_main_process():
return
logger.info("Evaluating bbox proposals")
areas = {"all": "", "small": "s", "medium": "m", "large": "l"}
res = COCOResults("box_proposal")
for limit in [100, 1000]:
for area, suffix in areas.items():
stats = evaluate_box_proposals(
predictions, dataset, area=area, limit=limit
)
key = "AR{}@{:d}".format(suffix, limit)
res.results["box_proposal"][key] = stats["ar"].item()
logger.info(res)
check_expected_results(res, expected_results, expected_results_sigma_tol)
if output_folder:
torch.save(res, os.path.join(output_folder, "box_proposals.pth"))
return
logger.info("Preparing results for COCO format")
coco_results = {}
if "bbox" in iou_types:
logger.info("Preparing bbox results")
coco_results["bbox"] = prepare_for_coco_detection(predictions, dataset)
if "segm" in iou_types:
logger.info("Preparing segm results")
coco_results["segm"] = prepare_for_coco_segmentation(predictions, dataset)
if 'keypoints' in iou_types:
logger.info('Preparing keypoints results')
coco_results['keypoints'] = prepare_for_coco_keypoint(predictions, dataset)
# Gather all prepared predictions of each type from all ranks
if "bbox" in iou_types:
temp_bbox_list = all_gather(coco_results["bbox"])
if "segm" in iou_types:
temp_segm_list = all_gather(coco_results["segm"])
if "keypoints" in iou_types:
temp_keypoints_list = all_gather(coco_results["keypoints"])
# Only main process will call COCO
if not is_main_process():
return
# Unpack the gathered results into a single List[Entry]
if "bbox" in iou_types:
coco_results["bbox"] = [i for j in temp_bbox_list for i in j]
if "segm" in iou_types:
coco_results["segm"] = [i for j in temp_segm_list for i in j]
if "keypoints" in iou_types:
coco_results["keypoints"] = [i for j in temp_keypoints_list for i in j]
# Submit to async evaluator
get_evaluator().submit_task(get_tag(),
evaluate_coco,
dataset,
coco_results,
iou_types,
output_folder)
# Note: None of these are possible now
# logger.info(results)
# check_expected_results(results, expected_results, expected_results_sigma_tol)
# if output_folder:
# torch.save(results, os.path.join(output_folder, "coco_results.pth"))
# Note: results is now empty, the relevant future is held in the hidden
# AsyncEvaluator object
return None, coco_results