def test_compare_pc_with_golden_value(self): sample_data_dir = os.path.join(_TEST_DIR) gt_json_file = os.path.join(sample_data_dir, 'coco_gt.json') gt_folder = os.path.join(sample_data_dir, 'coco_gt') pred_json_file = os.path.join(sample_data_dir, 'coco_pred.json') pred_folder = os.path.join(sample_data_dir, 'coco_pred') deeplab_results = eval_coco_format.eval_coco_format( gt_json_file, pred_json_file, gt_folder, pred_folder, metric='pc', num_categories=7, ignored_label=0, max_instances_per_category=256, intersection_offset=(256 * 256), normalize_by_image_size=False) self.assertCountEqual(deeplab_results.keys(), ['All', 'Things', 'Stuff']) for cat_group in ['All', 'Things', 'Stuff']: self.assertCountEqual(deeplab_results[cat_group], ['pc', 'n']) self.assertAlmostEqual(deeplab_results['All']['pc'], 0.68210561) self.assertEqual(deeplab_results['All']['n'], 6) self.assertAlmostEqual(deeplab_results['Things']['pc'], 0.5890529) self.assertEqual(deeplab_results['Things']['n'], 4) self.assertAlmostEqual(deeplab_results['Stuff']['pc'], 0.86821097) self.assertEqual(deeplab_results['Stuff']['n'], 2)
def test_compare_pq_with_reference_eval(self): sample_data_dir = os.path.join(_TEST_DIR) gt_json_file = os.path.join(sample_data_dir, 'coco_gt.json') gt_folder = os.path.join(sample_data_dir, 'coco_gt') pred_json_file = os.path.join(sample_data_dir, 'coco_pred.json') pred_folder = os.path.join(sample_data_dir, 'coco_pred') panopticapi_results = panopticapi_eval.pq_compute( gt_json_file, pred_json_file, gt_folder, pred_folder) deeplab_results = eval_coco_format.eval_coco_format( gt_json_file, pred_json_file, gt_folder, pred_folder, metric='pq', num_categories=7, ignored_label=0, max_instances_per_category=256, intersection_offset=(256 * 256)) self.assertCountEqual(deeplab_results.keys(), ['All', 'Things', 'Stuff']) for cat_group in ['All', 'Things', 'Stuff']: self.assertCountEqual(deeplab_results[cat_group], ['pq', 'sq', 'rq', 'n']) for metric in ['pq', 'sq', 'rq', 'n']: self.assertAlmostEqual(deeplab_results[cat_group][metric], panopticapi_results[cat_group][metric])
def test_compare_pc_with_golden_value(self): sample_data_dir = os.path.join(_TEST_DIR) gt_json_file = os.path.join(sample_data_dir, 'coco_gt.json') gt_folder = os.path.join(sample_data_dir, 'coco_gt') pred_json_file = os.path.join(sample_data_dir, 'coco_pred.json') pred_folder = os.path.join(sample_data_dir, 'coco_pred') deeplab_results = eval_coco_format.eval_coco_format( gt_json_file, pred_json_file, gt_folder, pred_folder, metric='pc', num_categories=7, ignored_label=0, max_instances_per_category=256, intersection_offset=(256 * 256), normalize_by_image_size=False) self.assertCountEqual(deeplab_results.keys(), ['All', 'Things', 'Stuff']) for cat_group in ['All', 'Things', 'Stuff']: self.assertCountEqual(deeplab_results[cat_group], ['pc', 'n']) self.assertAlmostEqual(deeplab_results['All']['pc'], 0.68210561) self.assertEqual(deeplab_results['All']['n'], 6) self.assertAlmostEqual(deeplab_results['Things']['pc'], 0.5890529) self.assertEqual(deeplab_results['Things']['n'], 4) self.assertAlmostEqual(deeplab_results['Stuff']['pc'], 0.86821097) self.assertEqual(deeplab_results['Stuff']['n'], 2)
def test_compare_pq_with_reference_eval(self): sample_data_dir = os.path.join(_TEST_DIR) gt_json_file = os.path.join(sample_data_dir, 'coco_gt.json') gt_folder = os.path.join(sample_data_dir, 'coco_gt') pred_json_file = os.path.join(sample_data_dir, 'coco_pred.json') pred_folder = os.path.join(sample_data_dir, 'coco_pred') panopticapi_results = panopticapi_eval.pq_compute( gt_json_file, pred_json_file, gt_folder, pred_folder) deeplab_results = eval_coco_format.eval_coco_format( gt_json_file, pred_json_file, gt_folder, pred_folder, metric='pq', num_categories=7, ignored_label=0, max_instances_per_category=256, intersection_offset=(256 * 256)) self.assertCountEqual(deeplab_results.keys(), ['All', 'Things', 'Stuff']) for cat_group in ['All', 'Things', 'Stuff']: self.assertCountEqual(deeplab_results[cat_group], ['pq', 'sq', 'rq', 'n']) for metric in ['pq', 'sq', 'rq', 'n']: self.assertAlmostEqual(deeplab_results[cat_group][metric], panopticapi_results[cat_group][metric])
def test_compare_pc_with_golden_value_normalize_by_size(self): sample_data_dir = os.path.join(_TEST_DIR) gt_json_file = os.path.join(sample_data_dir, 'coco_gt.json') gt_folder = os.path.join(sample_data_dir, 'coco_gt') pred_json_file = os.path.join(sample_data_dir, 'coco_pred.json') pred_folder = os.path.join(sample_data_dir, 'coco_pred') deeplab_results = eval_coco_format.eval_coco_format( gt_json_file, pred_json_file, gt_folder, pred_folder, metric='pc', num_categories=7, ignored_label=0, max_instances_per_category=256, intersection_offset=(256 * 256), normalize_by_image_size=True) self.assertCountEqual( list(deeplab_results.keys()), ['All', 'Things', 'Stuff']) self.assertAlmostEqual(deeplab_results['All']['pc'], 0.68214908840)
def test_pc_with_multiple_workers(self): sample_data_dir = os.path.join(_TEST_DIR) gt_json_file = os.path.join(sample_data_dir, 'coco_gt.json') gt_folder = os.path.join(sample_data_dir, 'coco_gt') pred_json_file = os.path.join(sample_data_dir, 'coco_pred.json') pred_folder = os.path.join(sample_data_dir, 'coco_pred') deeplab_results = eval_coco_format.eval_coco_format( gt_json_file, pred_json_file, gt_folder, pred_folder, metric='pc', num_categories=7, ignored_label=0, max_instances_per_category=256, intersection_offset=(256 * 256), num_workers=3, normalize_by_image_size=False) self.assertCountEqual(deeplab_results.keys(), ['All', 'Things', 'Stuff']) self.assertAlmostEqual(deeplab_results['All']['pc'], 0.68210561668)