def get_eval_metric_ops_for_evaluators(evaluation_metrics, categories, eval_dict, include_metrics_per_category=False): """Returns a dictionary of eval metric ops to use with `tf.EstimatorSpec`. Args: evaluation_metrics: List of evaluation metric names. Current options are 'coco_detection_metrics' and 'coco_mask_metrics'. categories: A list of dicts, each of which has the following keys - 'id': (required) an integer id uniquely identifying this category. 'name': (required) string representing category name e.g., 'cat', 'dog'. eval_dict: An evaluation dictionary, returned from result_dict_for_single_example(). include_metrics_per_category: If True, additionally include per-category metrics. Returns: A dictionary of metric names to tuple of value_op and update_op that can be used as eval metric ops in tf.EstimatorSpec. Raises: ValueError: If any of the metrics in `evaluation_metric` is not 'coco_detection_metrics' or 'coco_mask_metrics'. """ evaluation_metrics = list(set(evaluation_metrics)) input_data_fields = fields.InputDataFields detection_fields = fields.DetectionResultFields eval_metric_ops = {} for metric in evaluation_metrics: if metric == 'coco_detection_metrics': coco_evaluator = coco_evaluation.CocoDetectionEvaluator( categories, include_metrics_per_category=include_metrics_per_category) eval_metric_ops.update( coco_evaluator.get_estimator_eval_metric_ops( image_id=eval_dict[input_data_fields.key], groundtruth_boxes=eval_dict[input_data_fields.groundtruth_boxes], groundtruth_classes=eval_dict[ input_data_fields.groundtruth_classes], detection_boxes=eval_dict[detection_fields.detection_boxes], detection_scores=eval_dict[detection_fields.detection_scores], detection_classes=eval_dict[detection_fields.detection_classes], groundtruth_is_crowd=eval_dict.get( input_data_fields.groundtruth_is_crowd))) elif metric == 'coco_mask_metrics': coco_mask_evaluator = coco_evaluation.CocoMaskEvaluator( categories, include_metrics_per_category=include_metrics_per_category) eval_metric_ops.update( coco_mask_evaluator.get_estimator_eval_metric_ops( image_id=eval_dict[input_data_fields.key], groundtruth_boxes=eval_dict[input_data_fields.groundtruth_boxes], groundtruth_classes=eval_dict[ input_data_fields.groundtruth_classes], groundtruth_instance_masks=eval_dict[ input_data_fields.groundtruth_instance_masks], detection_scores=eval_dict[detection_fields.detection_scores], detection_classes=eval_dict[detection_fields.detection_classes], detection_masks=eval_dict[detection_fields.detection_masks], groundtruth_is_crowd=eval_dict.get( input_data_fields.groundtruth_is_crowd),)) else: raise ValueError('The only evaluation metrics supported are ' '"coco_detection_metrics" and "coco_mask_metrics". ' 'Found {} in the evaluation metrics'.format(metric)) return eval_metric_ops
def testGetOneMAPWithMatchingGroundtruthAndDetections(self): coco_evaluator = coco_evaluation.CocoMaskEvaluator(_get_categories_list()) coco_evaluator.add_single_ground_truth_image_info( image_id='image1', groundtruth_dict={ standard_fields.InputDataFields.groundtruth_boxes: np.array([[100., 100., 200., 200.]]), standard_fields.InputDataFields.groundtruth_classes: np.array([1]), standard_fields.InputDataFields.groundtruth_instance_masks: np.pad(np.ones([1, 100, 100], dtype=np.uint8), ((0, 0), (10, 10), (10, 10)), mode='constant') }) coco_evaluator.add_single_detected_image_info( image_id='image1', detections_dict={ standard_fields.DetectionResultFields.detection_boxes: np.array([[100., 100., 200., 200.]]), standard_fields.DetectionResultFields.detection_scores: np.array([.8]), standard_fields.DetectionResultFields.detection_classes: np.array([1]), standard_fields.DetectionResultFields.detection_masks: np.pad(np.ones([1, 100, 100], dtype=np.uint8), ((0, 0), (10, 10), (10, 10)), mode='constant') }) coco_evaluator.add_single_ground_truth_image_info( image_id='image2', groundtruth_dict={ standard_fields.InputDataFields.groundtruth_boxes: np.array([[50., 50., 100., 100.]]), standard_fields.InputDataFields.groundtruth_classes: np.array([1]), standard_fields.InputDataFields.groundtruth_instance_masks: np.pad(np.ones([1, 50, 50], dtype=np.uint8), ((0, 0), (10, 10), (10, 10)), mode='constant') }) coco_evaluator.add_single_detected_image_info( image_id='image2', detections_dict={ standard_fields.DetectionResultFields.detection_boxes: np.array([[50., 50., 100., 100.]]), standard_fields.DetectionResultFields.detection_scores: np.array([.8]), standard_fields.DetectionResultFields.detection_classes: np.array([1]), standard_fields.DetectionResultFields.detection_masks: np.pad(np.ones([1, 50, 50], dtype=np.uint8), ((0, 0), (10, 10), (10, 10)), mode='constant') }) coco_evaluator.add_single_ground_truth_image_info( image_id='image3', groundtruth_dict={ standard_fields.InputDataFields.groundtruth_boxes: np.array([[25., 25., 50., 50.]]), standard_fields.InputDataFields.groundtruth_classes: np.array([1]), standard_fields.InputDataFields.groundtruth_instance_masks: np.pad(np.ones([1, 25, 25], dtype=np.uint8), ((0, 0), (10, 10), (10, 10)), mode='constant') }) coco_evaluator.add_single_detected_image_info( image_id='image3', detections_dict={ standard_fields.DetectionResultFields.detection_boxes: np.array([[25., 25., 50., 50.]]), standard_fields.DetectionResultFields.detection_scores: np.array([.8]), standard_fields.DetectionResultFields.detection_classes: np.array([1]), standard_fields.DetectionResultFields.detection_masks: np.pad(np.ones([1, 25, 25], dtype=np.uint8), ((0, 0), (10, 10), (10, 10)), mode='constant') }) metrics = coco_evaluator.evaluate() self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP'], 1.0) coco_evaluator.clear() self.assertFalse(coco_evaluator._image_id_to_mask_shape_map) self.assertFalse(coco_evaluator._image_ids_with_detections) self.assertFalse(coco_evaluator._groundtruth_list) self.assertFalse(coco_evaluator._detection_masks_list)
def testGetOneMAPWithMatchingGroundtruthAndDetectionsBatched(self): coco_evaluator = coco_evaluation.CocoMaskEvaluator(_get_categories_list()) batch_size = 3 image_id = tf.placeholder(tf.string, shape=(batch_size)) groundtruth_boxes = tf.placeholder(tf.float32, shape=(batch_size, None, 4)) groundtruth_classes = tf.placeholder(tf.float32, shape=(batch_size, None)) groundtruth_masks = tf.placeholder( tf.uint8, shape=(batch_size, None, None, None)) detection_scores = tf.placeholder(tf.float32, shape=(batch_size, None)) detection_classes = tf.placeholder(tf.float32, shape=(batch_size, None)) detection_masks = tf.placeholder( tf.uint8, shape=(batch_size, None, None, None)) input_data_fields = standard_fields.InputDataFields detection_fields = standard_fields.DetectionResultFields eval_dict = { input_data_fields.key: image_id, input_data_fields.groundtruth_boxes: groundtruth_boxes, input_data_fields.groundtruth_classes: groundtruth_classes, input_data_fields.groundtruth_instance_masks: groundtruth_masks, detection_fields.detection_scores: detection_scores, detection_fields.detection_classes: detection_classes, detection_fields.detection_masks: detection_masks, } eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops(eval_dict) _, update_op = eval_metric_ops['DetectionMasks_Precision/mAP'] with self.test_session() as sess: sess.run( update_op, feed_dict={ image_id: ['image1', 'image2', 'image3'], groundtruth_boxes: np.array([[[100., 100., 200., 200.]], [[50., 50., 100., 100.]], [[25., 25., 50., 50.]]]), groundtruth_classes: np.array([[1], [1], [1]]), groundtruth_masks: np.stack([ np.pad( np.ones([1, 100, 100], dtype=np.uint8), ((0, 0), (0, 0), (0, 0)), mode='constant'), np.pad( np.ones([1, 50, 50], dtype=np.uint8), ((0, 0), (25, 25), (25, 25)), mode='constant'), np.pad( np.ones([1, 25, 25], dtype=np.uint8), ((0, 0), (37, 38), (37, 38)), mode='constant') ], axis=0), detection_scores: np.array([[.8], [.8], [.8]]), detection_classes: np.array([[1], [1], [1]]), detection_masks: np.stack([ np.pad( np.ones([1, 100, 100], dtype=np.uint8), ((0, 0), (0, 0), (0, 0)), mode='constant'), np.pad( np.ones([1, 50, 50], dtype=np.uint8), ((0, 0), (25, 25), (25, 25)), mode='constant'), np.pad( np.ones([1, 25, 25], dtype=np.uint8), ((0, 0), (37, 38), (37, 38)), mode='constant') ], axis=0) }) metrics = {} for key, (value_op, _) in eval_metric_ops.items(): metrics[key] = value_op metrics = sess.run(metrics) self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP (large)'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP (medium)'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP (small)'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@1'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@10'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100 (large)'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100 (medium)'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100 (small)'], 1.0) self.assertFalse(coco_evaluator._groundtruth_list) self.assertFalse(coco_evaluator._image_ids_with_detections) self.assertFalse(coco_evaluator._image_id_to_mask_shape_map) self.assertFalse(coco_evaluator._detection_masks_list)
def testGetOneMAPWithMatchingGroundtruthAndDetections(self): category_list = [{ 'id': 0, 'name': 'person' }, { 'id': 1, 'name': 'cat' }, { 'id': 2, 'name': 'dog' }] coco_evaluator = coco_evaluation.CocoMaskEvaluator(category_list) image_id = tf.placeholder(tf.string, shape=()) groundtruth_boxes = tf.placeholder(tf.float32, shape=(None, 4)) groundtruth_classes = tf.placeholder(tf.float32, shape=(None)) groundtruth_masks = tf.placeholder(tf.uint8, shape=(None, None, None)) detection_scores = tf.placeholder(tf.float32, shape=(None)) detection_classes = tf.placeholder(tf.float32, shape=(None)) detection_masks = tf.placeholder(tf.uint8, shape=(None, None, None)) eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops( image_id, groundtruth_boxes, groundtruth_classes, groundtruth_masks, detection_scores, detection_classes, detection_masks) _, update_op = eval_metric_ops['DetectionMasks_Precision/mAP'] with self.test_session() as sess: sess.run(update_op, feed_dict={ image_id: 'image1', groundtruth_boxes: np.array([[100., 100., 200., 200.]]), groundtruth_classes: np.array([1]), groundtruth_masks: np.pad(np.ones([1, 100, 100], dtype=np.uint8), ((0, 0), (10, 10), (10, 10)), mode='constant'), detection_scores: np.array([.8]), detection_classes: np.array([1]), detection_masks: np.pad(np.ones([1, 100, 100], dtype=np.uint8), ((0, 0), (10, 10), (10, 10)), mode='constant') }) sess.run(update_op, feed_dict={ image_id: 'image2', groundtruth_boxes: np.array([[50., 50., 100., 100.]]), groundtruth_classes: np.array([1]), groundtruth_masks: np.pad(np.ones([1, 50, 50], dtype=np.uint8), ((0, 0), (10, 10), (10, 10)), mode='constant'), detection_scores: np.array([.8]), detection_classes: np.array([1]), detection_masks: np.pad(np.ones([1, 50, 50], dtype=np.uint8), ((0, 0), (10, 10), (10, 10)), mode='constant') }) sess.run(update_op, feed_dict={ image_id: 'image3', groundtruth_boxes: np.array([[25., 25., 50., 50.]]), groundtruth_classes: np.array([1]), groundtruth_masks: np.pad(np.ones([1, 25, 25], dtype=np.uint8), ((0, 0), (10, 10), (10, 10)), mode='constant'), detection_scores: np.array([.8]), detection_classes: np.array([1]), detection_masks: np.pad(np.ones([1, 25, 25], dtype=np.uint8), ((0, 0), (10, 10), (10, 10)), mode='constant') }) metrics = {} for key, (value_op, _) in eval_metric_ops.iteritems(): metrics[key] = value_op metrics = sess.run(metrics) self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP (large)'], 1.0) self.assertAlmostEqual( metrics['DetectionMasks_Precision/mAP (medium)'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Precision/mAP (small)'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@1'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@10'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100 (large)'], 1.0) self.assertAlmostEqual( metrics['DetectionMasks_Recall/AR@100 (medium)'], 1.0) self.assertAlmostEqual(metrics['DetectionMasks_Recall/AR@100 (small)'], 1.0) self.assertFalse(coco_evaluator._groundtruth_list) self.assertFalse(coco_evaluator._image_ids_with_detections) self.assertFalse(coco_evaluator._image_id_to_mask_shape_map) self.assertFalse(coco_evaluator._detection_masks_list)