def custom_coco_evaluation( results: List[Any], dataset: CustomDataset) -> Optional[Dict[str, float]]: """Evaluate results using dataset.evaluate for different sampled architectures. Parameters ---------- results: List of results for different architectures. dataset: mmdetection dataset with evaluate method. Returns ------- : Average COCO metrics for sample architectures. """ coco_total_metrics = [] results = sort_and_normalize_results(results) for arch_res in results: arch_metrics = dataset.evaluate(arch_res) coco_total_metrics.append(arch_metrics) result = None for coco_arch in coco_total_metrics: if result is None: result = dict() for res_name in coco_arch: if isinstance(coco_arch[res_name], float): result[res_name] = coco_arch[res_name] else: for res_name in result: if isinstance(result[res_name], float): result[res_name] += coco_arch[res_name] for res_name in result: result[res_name] /= len(results) return result
def test_dataset_evaluation(): tmp_dir = tempfile.TemporaryDirectory() # create dummy data fake_json_file = osp.join(tmp_dir.name, 'fake_data.json') _create_dummy_coco_json(fake_json_file) # test single coco dataset evaluation coco_dataset = CocoDataset( ann_file=fake_json_file, classes=('car', ), pipeline=[]) fake_results = _create_dummy_results() eval_results = coco_dataset.evaluate(fake_results, classwise=True) assert eval_results['bbox_mAP'] == 1 assert eval_results['bbox_mAP_50'] == 1 assert eval_results['bbox_mAP_75'] == 1 # test concat dataset evaluation fake_concat_results = _create_dummy_results() + _create_dummy_results() # build concat dataset through two config dict coco_cfg = dict( type='CocoDataset', ann_file=fake_json_file, classes=('car', ), pipeline=[]) concat_cfgs = [coco_cfg, coco_cfg] concat_dataset = build_dataset(concat_cfgs) eval_results = concat_dataset.evaluate(fake_concat_results) assert eval_results['0_bbox_mAP'] == 1 assert eval_results['0_bbox_mAP_50'] == 1 assert eval_results['0_bbox_mAP_75'] == 1 assert eval_results['1_bbox_mAP'] == 1 assert eval_results['1_bbox_mAP_50'] == 1 assert eval_results['1_bbox_mAP_75'] == 1 # build concat dataset through concatenated ann_file coco_cfg = dict( type='CocoDataset', ann_file=[fake_json_file, fake_json_file], classes=('car', ), pipeline=[]) concat_dataset = build_dataset(coco_cfg) eval_results = concat_dataset.evaluate(fake_concat_results) assert eval_results['0_bbox_mAP'] == 1 assert eval_results['0_bbox_mAP_50'] == 1 assert eval_results['0_bbox_mAP_75'] == 1 assert eval_results['1_bbox_mAP'] == 1 assert eval_results['1_bbox_mAP_50'] == 1 assert eval_results['1_bbox_mAP_75'] == 1 # create dummy data fake_pkl_file = osp.join(tmp_dir.name, 'fake_data.pkl') _create_dummy_custom_pkl(fake_pkl_file) # test single custom dataset evaluation custom_dataset = CustomDataset( ann_file=fake_pkl_file, classes=('car', ), pipeline=[]) fake_results = _create_dummy_results() eval_results = custom_dataset.evaluate(fake_results) assert eval_results['mAP'] == 1 # test concat dataset evaluation fake_concat_results = _create_dummy_results() + _create_dummy_results() # build concat dataset through two config dict custom_cfg = dict( type='CustomDataset', ann_file=fake_pkl_file, classes=('car', ), pipeline=[]) concat_cfgs = [custom_cfg, custom_cfg] concat_dataset = build_dataset(concat_cfgs) eval_results = concat_dataset.evaluate(fake_concat_results) assert eval_results['0_mAP'] == 1 assert eval_results['1_mAP'] == 1 # build concat dataset through concatenated ann_file concat_cfg = dict( type='CustomDataset', ann_file=[fake_pkl_file, fake_pkl_file], classes=('car', ), pipeline=[]) concat_dataset = build_dataset(concat_cfg) eval_results = concat_dataset.evaluate(fake_concat_results) assert eval_results['0_mAP'] == 1 assert eval_results['1_mAP'] == 1 # build concat dataset through explicit type concat_cfg = dict( type='ConcatDataset', datasets=[custom_cfg, custom_cfg], separate_eval=False) concat_dataset = build_dataset(concat_cfg) eval_results = concat_dataset.evaluate(fake_concat_results, metric='mAP') assert eval_results['mAP'] == 1 assert len(concat_dataset.datasets[0].data_infos) == \ len(concat_dataset.datasets[1].data_infos) assert len(concat_dataset.datasets[0].data_infos) == 1 tmp_dir.cleanup()