def main(unused_argv): flags.mark_flag_as_required('input_annotations_boxes') flags.mark_flag_as_required('input_annotations_labels') flags.mark_flag_as_required('input_predictions') flags.mark_flag_as_required('input_class_labelmap') flags.mark_flag_as_required('output_metrics') all_location_annotations = pd.read_csv(FLAGS.input_annotations_boxes) all_label_annotations = pd.read_csv(FLAGS.input_annotations_labels) all_label_annotations.rename( columns={'Confidence': 'ConfidenceImageLabel'}, inplace=True) is_instance_segmentation_eval = False if FLAGS.input_annotations_segm: is_instance_segmentation_eval = True all_segm_annotations = pd.read_csv(FLAGS.input_annotations_segm) # Note: this part is unstable as it requires the float point numbers in both # csvs are exactly the same; # Will be replaced by more stable solution: merge on LabelName and ImageID # and filter down by IoU. all_location_annotations = utils.merge_boxes_and_masks( all_location_annotations, all_segm_annotations) all_annotations = pd.concat( [all_location_annotations, all_label_annotations]) class_label_map, categories = _load_labelmap(FLAGS.input_class_labelmap) challenge_evaluator = ( object_detection_evaluation.OpenImagesChallengeEvaluator( categories, evaluate_masks=is_instance_segmentation_eval)) all_predictions = pd.read_csv(FLAGS.input_predictions) images_processed = 0 for _, groundtruth in enumerate(all_annotations.groupby('ImageID')): logging.info('Processing image %d', images_processed) image_id, image_groundtruth = groundtruth groundtruth_dictionary = utils.build_groundtruth_dictionary( image_groundtruth, class_label_map) challenge_evaluator.add_single_ground_truth_image_info( image_id, groundtruth_dictionary) prediction_dictionary = utils.build_predictions_dictionary_kaggle( all_predictions.loc[all_predictions['ImageID'] == image_id], class_label_map) challenge_evaluator.add_single_detected_image_info( image_id, prediction_dictionary) images_processed += 1 metrics = challenge_evaluator.evaluate() with open(FLAGS.output_metrics, 'w') as fid: io_utils.write_csv(fid, metrics)
def proc_one_image(gt_and_preds, categories, class_label_map, evaluate_masks): groundtruth, predictions = gt_and_preds image_id = groundtruth['ImageID'].iloc[0] tfod_evaluator = tfod_evaluation.OpenImagesChallengeEvaluator( categories, evaluate_masks=evaluate_masks) gt_dict = tfod_utils.build_groundtruth_dictionary( groundtruth, class_label_map) tfod_evaluator.add_single_ground_truth_image_info(image_id, gt_dict) pred_dict = tfod_utils.build_predictions_dictionary( predictions, class_label_map) tfod_evaluator.add_single_detected_image_info(image_id, pred_dict) return tfod_evaluator.get_internal_state()
def __init__(self, class_label_map_path, iou_threshold=0.5): """ Args: class_label_map_path: path to the label map .pbtxt file """ self._iou_threshold = iou_threshold self._class_label_map, self._categories = self.load_labelmap( class_label_map_path) self._reverse_label_map = { v: k for k, v in self._class_label_map.items() } self._evaluator = object_detection_evaluation.OpenImagesChallengeEvaluator( self._categories, evaluate_masks=False, matching_iou_threshold=self._iou_threshold, )
def test_returns_correct_instance_segm_metric_values(self): categories = [{'id': 1, 'name': 'cat'}, {'id': 2, 'name': 'dog'}] oivchallenge_evaluator = ( object_detection_evaluation.OpenImagesChallengeEvaluator( categories, evaluate_masks=True)) image_key = 'img1' groundtruth_boxes = np.array( [[0, 0, 1, 1], [0, 0, 2, 2], [0, 0, 3, 3]], dtype=float) groundtruth_class_labels = np.array([1, 2, 1], dtype=int) groundtruth_is_group_of_list = np.array([False, False, True], dtype=bool) groundtruth_verified_labels = np.array([1, 2, 3], dtype=int) groundtruth_mask_0 = np.array( [[1, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], dtype=np.uint8) zero_mask = np.array([[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], dtype=np.uint8) groundtruth_masks = np.stack( [groundtruth_mask_0, zero_mask, zero_mask], axis=0) oivchallenge_evaluator.add_single_ground_truth_image_info( image_key, { standard_fields.InputDataFields.groundtruth_boxes: groundtruth_boxes, standard_fields.InputDataFields.groundtruth_classes: groundtruth_class_labels, standard_fields.InputDataFields.groundtruth_group_of: groundtruth_is_group_of_list, standard_fields.InputDataFields.groundtruth_image_classes: groundtruth_verified_labels, standard_fields.InputDataFields.groundtruth_instance_masks: groundtruth_masks }) image_key = 'img3' groundtruth_boxes = np.array([[0, 0, 1, 1]], dtype=float) groundtruth_class_labels = np.array([2], dtype=int) groundtruth_mask_0 = np.array( [[1, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], dtype=np.uint8) groundtruth_masks = np.stack([groundtruth_mask_0], axis=0) oivchallenge_evaluator.add_single_ground_truth_image_info( image_key, { standard_fields.InputDataFields.groundtruth_boxes: groundtruth_boxes, standard_fields.InputDataFields.groundtruth_classes: groundtruth_class_labels, standard_fields.InputDataFields.groundtruth_instance_masks: groundtruth_masks }) image_key = 'img1' detected_boxes = np.array([[0, 0, 2, 2], [2, 2, 3, 3]], dtype=float) detection_mask_0 = np.array([[1, 1, 0, 0], [1, 1, 0, 0], [0, 0, 0, 0]], dtype=np.uint8) detected_masks = np.stack([detection_mask_0, zero_mask], axis=0) detected_class_labels = np.array([2, 1], dtype=int) detected_scores = np.array([0.7, 0.8], dtype=float) oivchallenge_evaluator.add_single_detected_image_info( image_key, { standard_fields.DetectionResultFields.detection_boxes: detected_boxes, standard_fields.DetectionResultFields.detection_scores: detected_scores, standard_fields.DetectionResultFields.detection_classes: detected_class_labels, standard_fields.DetectionResultFields.detection_masks: detected_masks }) image_key = 'img3' detected_boxes = np.array([[0, 0, 1, 1]], dtype=float) detected_class_labels = np.array([2], dtype=int) detected_scores = np.array([0.5], dtype=float) detected_mask_0 = np.array([[1, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], dtype=np.uint8) detected_masks = np.stack([detected_mask_0], axis=0) oivchallenge_evaluator.add_single_detected_image_info( image_key, { standard_fields.DetectionResultFields.detection_boxes: detected_boxes, standard_fields.DetectionResultFields.detection_scores: detected_scores, standard_fields.DetectionResultFields.detection_classes: detected_class_labels, standard_fields.DetectionResultFields.detection_masks: detected_masks }) metrics = oivchallenge_evaluator.evaluate() expected_metric_name = 'OpenImagesInstanceSegmentationChallenge' self.assertAlmostEqual( metrics[expected_metric_name + '_PerformanceByCategory/[email protected]/dog'], 1.0) self.assertAlmostEqual( metrics[expected_metric_name + '_PerformanceByCategory/[email protected]/cat'], 0) self.assertAlmostEqual( metrics[expected_metric_name + '_Precision/[email protected]'], 0.5) oivchallenge_evaluator.clear() self.assertFalse(oivchallenge_evaluator._image_ids)
def test_returns_correct_detection_metric_values(self): categories = [{ 'id': 1, 'name': 'cat' }, { 'id': 2, 'name': 'dog' }, { 'id': 3, 'name': 'elephant' }] oivchallenge_evaluator = ( object_detection_evaluation.OpenImagesChallengeEvaluator( categories, evaluate_masks=False, group_of_weight=0.5)) image_key = 'img1' groundtruth_boxes = np.array( [[0, 0, 1, 1], [0, 0, 2, 2], [0, 0, 3, 3]], dtype=float) groundtruth_class_labels = np.array([1, 3, 1], dtype=int) groundtruth_is_group_of_list = np.array([False, False, True], dtype=bool) groundtruth_verified_labels = np.array([1, 2, 3], dtype=int) oivchallenge_evaluator.add_single_ground_truth_image_info( image_key, { standard_fields.InputDataFields.groundtruth_boxes: groundtruth_boxes, standard_fields.InputDataFields.groundtruth_classes: groundtruth_class_labels, standard_fields.InputDataFields.groundtruth_group_of: groundtruth_is_group_of_list, standard_fields.InputDataFields.groundtruth_image_classes: groundtruth_verified_labels, }) image_key = 'img2' groundtruth_boxes = np.array( [[10, 10, 11, 11], [500, 500, 510, 510], [10, 10, 12, 12]], dtype=float) groundtruth_class_labels = np.array([1, 1, 3], dtype=int) groundtruth_is_group_of_list = np.array([False, False, True], dtype=bool) oivchallenge_evaluator.add_single_ground_truth_image_info( image_key, { standard_fields.InputDataFields.groundtruth_boxes: groundtruth_boxes, standard_fields.InputDataFields.groundtruth_classes: groundtruth_class_labels, standard_fields.InputDataFields.groundtruth_group_of: groundtruth_is_group_of_list }) image_key = 'img3' groundtruth_boxes = np.array([[0, 0, 1, 1]], dtype=float) groundtruth_class_labels = np.array([2], dtype=int) oivchallenge_evaluator.add_single_ground_truth_image_info( image_key, { standard_fields.InputDataFields.groundtruth_boxes: groundtruth_boxes, standard_fields.InputDataFields.groundtruth_classes: groundtruth_class_labels }) image_key = 'img1' detected_boxes = np.array([[10, 10, 11, 11], [100, 100, 120, 120]], dtype=float) detected_class_labels = np.array([2, 2], dtype=int) detected_scores = np.array([0.7, 0.8], dtype=float) oivchallenge_evaluator.add_single_detected_image_info( image_key, { standard_fields.DetectionResultFields.detection_boxes: detected_boxes, standard_fields.DetectionResultFields.detection_scores: detected_scores, standard_fields.DetectionResultFields.detection_classes: detected_class_labels }) image_key = 'img2' detected_boxes = np.array([[10, 10, 11, 11], [100, 100, 120, 120], [100, 100, 220, 220], [10, 10, 11, 11]], dtype=float) detected_class_labels = np.array([1, 1, 2, 3], dtype=int) detected_scores = np.array([0.7, 0.8, 0.5, 0.9], dtype=float) oivchallenge_evaluator.add_single_detected_image_info( image_key, { standard_fields.DetectionResultFields.detection_boxes: detected_boxes, standard_fields.DetectionResultFields.detection_scores: detected_scores, standard_fields.DetectionResultFields.detection_classes: detected_class_labels }) image_key = 'img3' detected_boxes = np.array([[0, 0, 1, 1]], dtype=float) detected_class_labels = np.array([2], dtype=int) detected_scores = np.array([0.5], dtype=float) oivchallenge_evaluator.add_single_detected_image_info( image_key, { standard_fields.DetectionResultFields.detection_boxes: detected_boxes, standard_fields.DetectionResultFields.detection_scores: detected_scores, standard_fields.DetectionResultFields.detection_classes: detected_class_labels }) metrics = oivchallenge_evaluator.evaluate() expected_metric_name = 'OpenImagesDetectionChallenge' self.assertAlmostEqual( metrics[expected_metric_name + '_PerformanceByCategory/[email protected]/dog'], 0.3333333333) self.assertAlmostEqual( metrics[expected_metric_name + '_PerformanceByCategory/[email protected]/elephant'], 0.333333333333) self.assertAlmostEqual( metrics[expected_metric_name + '_PerformanceByCategory/[email protected]/cat'], 0.142857142857) self.assertAlmostEqual( metrics[expected_metric_name + '_Precision/[email protected]'], 0.269841269) oivchallenge_evaluator.clear() self.assertFalse(oivchallenge_evaluator._image_ids)
def evaluate(self): tfod_eval_dicts = [] if self._distributed: from datetime import datetime tic = datetime.now() for sid in self.tfod_eval_dicts.keys(): comm.synchronize() shard = comm.gather(self.tfod_eval_dicts[sid], dst=0) tfod_eval_dicts.extend(list(itertools.chain(*shard))) if not comm.is_main_process(): return {} print('>' * 30 + f'gather cost: {(datetime.now() - tic).total_seconds()}s') else: for sid in self.tfod_eval_dicts.keys(): tfod_eval_dicts.extend(self.tfod_eval_dicts[sid]) pass if len(tfod_eval_dicts) == 0: print("_" * 60 + "There's no predictions.") return # Order doesn't matter. image_ids = list(set([x['ImageID'] for x in tfod_eval_dicts])) all_predictions = pd.DataFrame.from_records(tfod_eval_dicts) all_location_annotations = self.gt_bboxes[ self.gt_bboxes['ImageID'].isin(image_ids)] all_label_annotations = self.gt_image_labels[ self.gt_image_labels['ImageID'].isin(image_ids)] is_instance_segmentation_eval = True if 'instance-segmentation-task': all_segm_annotations = self.gt_masks[self.gt_masks['ImageID'].isin( image_ids)] # Note: this part is unstable as it requires the float point numbers in both # csvs are exactly the same; # Will be replaced by more stable solution: merge on LabelName and ImageID # and filter down by IoU. all_location_annotations = pd.merge(all_location_annotations, all_segm_annotations, how='outer', on=[ 'LabelName', 'ImageID', 'XMin', 'XMax', 'YMin', 'YMax', ]) all_annotations = pd.concat( [all_location_annotations, all_label_annotations]) class_label_map, categories = self.label_map, self.categories with Pool() as pool: func = partial(self.proc_one_image, categories=categories, class_label_map=class_label_map, evaluate_masks=is_instance_segmentation_eval) itr = zip([ all_annotations[all_annotations['ImageID'] == x] for x in image_ids ], [ all_predictions[all_predictions['ImageID'] == x] for x in image_ids ]) state_and_ids_list = list(tqdm(pool.imap(func, itr))) if 'merge-tfod-evaluator': merged = tfod_evaluation.OpenImagesChallengeEvaluator( categories, is_instance_segmentation_eval) for state, ids in state_and_ids_list: merged.merge_internal_state(ids, state) metrics = merged.evaluate() return OrderedDict(instance_segmentation=metrics)