def testRejectionOnDuplicateGroundtruth(self): """Tests that groundtruth cannot be added more than once for an image.""" coco_evaluator = coco_evaluation.CocoDetectionEvaluator( _get_categories_list()) # Add groundtruth image_key1 = 'img1' groundtruth_boxes1 = np.array([[0, 0, 1, 1], [0, 0, 2, 2], [0, 0, 3, 3]], dtype=float) groundtruth_class_labels1 = np.array([1, 3, 1], dtype=int) coco_evaluator.add_single_ground_truth_image_info(image_key1, { standard_fields.InputDataFields.groundtruth_boxes: groundtruth_boxes1, standard_fields.InputDataFields.groundtruth_classes: groundtruth_class_labels1 }) groundtruth_lists_len = len(coco_evaluator._groundtruth_list) # Add groundtruth with the same image id. coco_evaluator.add_single_ground_truth_image_info(image_key1, { standard_fields.InputDataFields.groundtruth_boxes: groundtruth_boxes1, standard_fields.InputDataFields.groundtruth_classes: groundtruth_class_labels1 }) self.assertEqual(groundtruth_lists_len, len(coco_evaluator._groundtruth_list))
def testRejectionOnDuplicateDetections(self): """Tests that detections cannot be added more than once for an image.""" coco_evaluator = coco_evaluation.CocoDetectionEvaluator( _get_categories_list()) # Add groundtruth coco_evaluator.add_single_ground_truth_image_info( image_id='image1', groundtruth_dict={ standard_fields.InputDataFields.groundtruth_boxes: np.array([[99., 100., 200., 200.]]), standard_fields.InputDataFields.groundtruth_classes: np.array([1]) }) coco_evaluator.add_single_detected_image_info( image_id='image1', detections_dict={ standard_fields.DetectionResultFields.detection_boxes: np.array([[100., 100., 200., 200.]]), standard_fields.DetectionResultFields.detection_scores: np.array([.8]), standard_fields.DetectionResultFields.detection_classes: np.array([1]) }) detections_lists_len = len(coco_evaluator._detection_boxes_list) coco_evaluator.add_single_detected_image_info( image_id='image1', # Note that this image id was previously added. detections_dict={ standard_fields.DetectionResultFields.detection_boxes: np.array([[100., 100., 200., 200.]]), standard_fields.DetectionResultFields.detection_scores: np.array([.8]), standard_fields.DetectionResultFields.detection_classes: np.array([1]) }) self.assertEqual(detections_lists_len, len(coco_evaluator._detection_boxes_list))
def evaluate_single_image(image_path, annotation_path): """ Evaluate mAP on image args: image_path: path to image annotation_path: path to groundtruth in Pascal VOC format .xml """ categories = create_categories_from_labelmap(args.label_file) label_map_dict = get_label_map_dict(args.label_file) coco_evaluator = coco_evaluation.CocoDetectionEvaluator(categories) image_name = os.path.basename(image_path).split('.')[0] # Read groundtruth from XML file in Pascal VOC format gt_boxes, gt_classes = voc_parser(annotation_path, label_map_dict) dt_boxes, dt_classes, dt_scores, num_det = postprocess_output(image_path) coco_evaluator.add_single_ground_truth_image_info( image_id=image_name, groundtruth_dict={ standard_fields.InputDataFields.groundtruth_boxes: np.array(gt_boxes), standard_fields.InputDataFields.groundtruth_classes: np.array(gt_classes) }) coco_evaluator.add_single_detected_image_info( image_id=image_name, detections_dict={ standard_fields.DetectionResultFields.detection_boxes: dt_boxes, standard_fields.DetectionResultFields.detection_scores: dt_scores, standard_fields.DetectionResultFields.detection_classes: dt_classes }) coco_evaluator.evaluate()
def testGetOneMAPWithMatchingGroundtruthAndDetectionsSkipCrowd(self): """Tests computing mAP with is_crowd GT boxes skipped.""" coco_evaluator = coco_evaluation.CocoDetectionEvaluator( _get_categories_list()) coco_evaluator.add_single_ground_truth_image_info( image_id='image1', groundtruth_dict={ standard_fields.InputDataFields.groundtruth_boxes: np.array([[100., 100., 200., 200.], [99., 99., 200., 200.]]), standard_fields.InputDataFields.groundtruth_classes: np.array([1, 2]), standard_fields.InputDataFields.groundtruth_is_crowd: np.array([0, 1]) }) coco_evaluator.add_single_detected_image_info( image_id='image1', detections_dict={ standard_fields.DetectionResultFields.detection_boxes: np.array([[100., 100., 200., 200.]]), standard_fields.DetectionResultFields.detection_scores: np.array([.8]), standard_fields.DetectionResultFields.detection_classes: np.array([1]) }) metrics = coco_evaluator.evaluate() self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0)
def testGetOneMAPWithMatchingGroundtruthAndDetections(self): """Tests that mAP is calculated correctly on GT and Detections.""" coco_evaluator = coco_evaluation.CocoDetectionEvaluator( _get_categories_list()) coco_evaluator.add_single_ground_truth_image_info( image_id='image1', groundtruth_dict={ standard_fields.InputDataFields.groundtruth_boxes: np.array([[100., 100., 200., 200.]]), standard_fields.InputDataFields.groundtruth_classes: np.array([1]) }) coco_evaluator.add_single_detected_image_info( image_id='image1', detections_dict={ standard_fields.DetectionResultFields.detection_boxes: np.array([[100., 100., 200., 200.]]), standard_fields.DetectionResultFields.detection_scores: np.array([.8]), standard_fields.DetectionResultFields.detection_classes: np.array([1]) }) coco_evaluator.add_single_ground_truth_image_info( image_id='image2', groundtruth_dict={ standard_fields.InputDataFields.groundtruth_boxes: np.array([[50., 50., 100., 100.]]), standard_fields.InputDataFields.groundtruth_classes: np.array([1]) }) coco_evaluator.add_single_detected_image_info( image_id='image2', detections_dict={ standard_fields.DetectionResultFields.detection_boxes: np.array([[50., 50., 100., 100.]]), standard_fields.DetectionResultFields.detection_scores: np.array([.8]), standard_fields.DetectionResultFields.detection_classes: np.array([1]) }) coco_evaluator.add_single_ground_truth_image_info( image_id='image3', groundtruth_dict={ standard_fields.InputDataFields.groundtruth_boxes: np.array([[25., 25., 50., 50.]]), standard_fields.InputDataFields.groundtruth_classes: np.array([1]) }) coco_evaluator.add_single_detected_image_info( image_id='image3', detections_dict={ standard_fields.DetectionResultFields.detection_boxes: np.array([[25., 25., 50., 50.]]), standard_fields.DetectionResultFields.detection_scores: np.array([.8]), standard_fields.DetectionResultFields.detection_classes: np.array([1]) }) metrics = coco_evaluator.evaluate() self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0)
def testGetOneMAPWithMatchingGroundtruthAndDetectionsEmptyCrowd(self): """Tests computing mAP with empty is_crowd array passed in.""" category_list = [{ 'id': 0, 'name': 'person' }, { 'id': 1, 'name': 'cat' }, { 'id': 2, 'name': 'dog' }] coco_evaluator = coco_evaluation.CocoDetectionEvaluator(category_list) coco_evaluator.add_single_ground_truth_image_info( image_id='image1', groundtruth_dict={ standard_fields.InputDataFields.groundtruth_boxes: np.array([[100., 100., 200., 200.]]), standard_fields.InputDataFields.groundtruth_classes: np.array([1]), standard_fields.InputDataFields.groundtruth_is_crowd: np.array([]) }) coco_evaluator.add_single_detected_image_info( image_id='image1', detections_dict={ standard_fields.DetectionResultFields.detection_boxes: np.array([[100., 100., 200., 200.]]), standard_fields.DetectionResultFields.detection_scores: np.array([.8]), standard_fields.DetectionResultFields.detection_classes: np.array([1]) }) metrics = coco_evaluator.evaluate() self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0)
def get_evaluate_ops(probs, pbbox, glabel, categories): """ :param probs: tensor of shape [1, anchor_num, class_num] :param pbbox: tensor of shape [1, anchor_num, 4] :param glabel: dict of tensor, 'labels': shape [1, anchor_num] 'bboxes': shape [1, anchor_num, 4] :param categories: A list of dicts, each of which has the following keys - 'id': (required) an integer id uniquely identifying this category. 'name': (required) string representing category name e.g., 'cat', 'dog'. :return: a dict of tensor, representing metrics. """ anchors = tf.constant(anchorsUtil.get_all_layer_anchors(), dtype=tf.float32) labels = tf.unstack(glabel['labels'], axis=0) bboxes = tf.unstack(glabel['bboxes'], axis=0) # Decode predicted bbox. pbboxes = lossUtil.decodeBBox(pbbox, anchors) pclasses = tf.argmax(probs, axis=1) # Compute mAP metrics. evaluator = coco_evaluation.CocoDetectionEvaluator(categories) metric_ops = evaluator.get_estimator_eval_metric_ops( groundtruth_boxes=bboxes, groundtruth_classes=labels, detection_boxes=pbboxes, detection_scores=probs, detection_classes=pclasses) return metric_ops
def testGetOneMAPWithMatchingGroundtruthAndDetectionsBatched(self): category_list = [{'id': 0, 'name': 'person'}, {'id': 1, 'name': 'cat'}, {'id': 2, 'name': 'dog'}] coco_evaluator = coco_evaluation.CocoDetectionEvaluator(category_list) batch_size = 3 image_id = tf.placeholder(tf.string, shape=(batch_size)) groundtruth_boxes = tf.placeholder(tf.float32, shape=(batch_size, None, 4)) groundtruth_classes = tf.placeholder(tf.float32, shape=(batch_size, None)) detection_boxes = tf.placeholder(tf.float32, shape=(batch_size, None, 4)) detection_scores = tf.placeholder(tf.float32, shape=(batch_size, None)) detection_classes = tf.placeholder(tf.float32, shape=(batch_size, None)) eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops( image_id, groundtruth_boxes, groundtruth_classes, detection_boxes, detection_scores, detection_classes) _, update_op = eval_metric_ops['DetectionBoxes_Precision/mAP'] with self.test_session() as sess: sess.run(update_op, feed_dict={ image_id: ['image1', 'image2', 'image3'], groundtruth_boxes: np.array([[[100., 100., 200., 200.]], [[50., 50., 100., 100.]], [[25., 25., 50., 50.]]]), groundtruth_classes: np.array([[1], [3], [2]]), detection_boxes: np.array([[[100., 100., 200., 200.]], [[50., 50., 100., 100.]], [[25., 25., 50., 50.]]]), detection_scores: np.array([[.8], [.7], [.9]]), detection_classes: np.array([[1], [3], [2]]) }) metrics = {} for key, (value_op, _) in eval_metric_ops.items(): metrics[key] = value_op metrics = sess.run(metrics) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (large)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (medium)'], -1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (small)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@1'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@10'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (large)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (medium)'], -1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (small)'], 1.0) self.assertFalse(coco_evaluator._groundtruth_list) self.assertFalse(coco_evaluator._detection_boxes_list) self.assertFalse(coco_evaluator._image_ids)
def testExceptionRaisedWithMissingGroundtruth(self): """Tests that exception is raised for detection with missing groundtruth.""" coco_evaluator = coco_evaluation.CocoDetectionEvaluator( _get_categories_list()) with self.assertRaises(ValueError): coco_evaluator.add_single_detected_image_info( image_id='image1', detections_dict={ standard_fields.DetectionResultFields.detection_boxes: np.array([[100., 100., 200., 200.]]), standard_fields.DetectionResultFields.detection_scores: np.array([.8]), standard_fields.DetectionResultFields.detection_classes: np.array([1]) })
def testExceptionRaisedWithMissingGroundtruth(self): """Tests that exception is raised for detection with missing groundtruth.""" categories = [{'id': 1, 'name': 'cat'}, {'id': 2, 'name': 'dog'}, {'id': 3, 'name': 'elephant'}] coco_evaluator = coco_evaluation.CocoDetectionEvaluator(categories) with self.assertRaises(ValueError): coco_evaluator.add_single_detected_image_info( image_id='image1', detections_dict={ standard_fields.DetectionResultFields.detection_boxes: np.array([[100., 100., 200., 200.]]), standard_fields.DetectionResultFields.detection_scores: np.array([.8]), standard_fields.DetectionResultFields.detection_classes: np.array([1]) })
def evaluate_via_tf(categories_list, results_dir, txt_annotatins_dir, full_report): ''' Calculate COCO metrics via evaluator class included in TF models repository https://github.com/tensorflow/models/tree/master/research/object_detection/metrics This method uses annotation converted to txt files. This convertation is done by installation dataset-coco-2014 package. ''' class_name_to_id_map = {} for category in categories_list: # Converted txt annotation lacks spaces in class names # and we have to remove spaces from labelmap's class names too # to be able to find class id by class name from annotation class_name = category['name'].split() class_name_no_spaces = ''.join(class_name) class_name_to_id_map[class_name_no_spaces] = category['id'] evaluator = coco_evaluation.CocoDetectionEvaluator(categories_list) total_dets_count = 0 total_gts_count = 0 not_found_gts = [] files = ck_utils.get_files(results_dir) for file_index, file_name in enumerate(files): if full_report: print('Loading detections and annotations for {} ({} of {}) ...'. format(file_name, file_index + 1, len(files))) elif (file_index + 1) % 100 == 0: print('Loading detections and annotations: {} of {} ...'.format( file_index + 1, len(files))) gt_file = os.path.join(txt_annotatins_dir, file_name) det_file = os.path.join(results_dir, file_name) # Skip files for which there is no groundtruth # e.g. COCO_val2014_000000013466.jpg gts = load_groundtruth(gt_file, class_name_to_id_map) if not gts: not_found_gts.append(file_name) continue dets = load_detections(det_file) gts_count = gts[gt_field.groundtruth_boxes].shape[0] dets_count = dets[det_field.detection_boxes].shape[0] total_gts_count += gts_count total_dets_count += dets_count if full_report: print(' Detections: {}'.format(dets_count)) print(' Groundtruth: {}'.format(gts_count)) # Groundtruth should be added first, as adding image checks if there is groundtrush for it evaluator.add_single_ground_truth_image_info(image_id=file_name, groundtruth_dict=gts) evaluator.add_single_detected_image_info(image_id=file_name, detections_dict=dets) all_metrics = evaluator.evaluate() if not_found_gts: print('Groundtrush not found for {} results:'.format( len(not_found_gts))) for file_name in not_found_gts: print(' {}'.format(file_name)) print('Total detections: {}'.format(total_dets_count)) print('Total groundtruths: {}'.format(total_gts_count)) print('Detection rate: {}'.format( float(total_dets_count) / float(total_gts_count))) mAP = all_metrics['DetectionBoxes_Precision/mAP'] recall = all_metrics['DetectionBoxes_Recall/AR@100'] return mAP, recall, all_metrics
def testGetOneMAPWithMatchingGroundtruthAndDetectionsPaddedBatches(self): coco_evaluator = coco_evaluation.CocoDetectionEvaluator( _get_categories_list()) batch_size = 3 image_id = tf.placeholder(tf.string, shape=(batch_size)) groundtruth_boxes = tf.placeholder(tf.float32, shape=(batch_size, None, 4)) groundtruth_classes = tf.placeholder(tf.float32, shape=(batch_size, None)) num_gt_boxes_per_image = tf.placeholder(tf.int32, shape=(None)) detection_boxes = tf.placeholder(tf.float32, shape=(batch_size, None, 4)) detection_scores = tf.placeholder(tf.float32, shape=(batch_size, None)) detection_classes = tf.placeholder(tf.float32, shape=(batch_size, None)) num_det_boxes_per_image = tf.placeholder(tf.int32, shape=(None)) input_data_fields = standard_fields.InputDataFields detection_fields = standard_fields.DetectionResultFields eval_dict = { input_data_fields.key: image_id, input_data_fields.groundtruth_boxes: groundtruth_boxes, input_data_fields.groundtruth_classes: groundtruth_classes, detection_fields.detection_boxes: detection_boxes, detection_fields.detection_scores: detection_scores, detection_fields.detection_classes: detection_classes, 'num_groundtruth_boxes_per_image': num_gt_boxes_per_image, 'num_det_boxes_per_image': num_det_boxes_per_image } eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops(eval_dict) _, update_op = eval_metric_ops['DetectionBoxes_Precision/mAP'] with self.test_session() as sess: sess.run( update_op, feed_dict={ image_id: ['image1', 'image2', 'image3'], groundtruth_boxes: np.array([[[100., 100., 200., 200.], [-1, -1, -1, -1]], [[50., 50., 100., 100.], [-1, -1, -1, -1]], [[25., 25., 50., 50.], [10., 10., 15., 15.]]]), groundtruth_classes: np.array([[1, -1], [3, -1], [2, 2]]), num_gt_boxes_per_image: np.array([1, 1, 2]), detection_boxes: np.array([[[100., 100., 200., 200.], [0., 0., 0., 0.], [0., 0., 0., 0.]], [[50., 50., 100., 100.], [0., 0., 0., 0.], [0., 0., 0., 0.]], [[25., 25., 50., 50.], [10., 10., 15., 15.], [10., 10., 15., 15.]]]), detection_scores: np.array([[.8, 0., 0.], [.7, 0., 0.], [.95, .9, 0.9]]), detection_classes: np.array([[1, -1, -1], [3, -1, -1], [2, 2, 2]]), num_det_boxes_per_image: np.array([1, 1, 3]), }) # Check the number of bounding boxes added. self.assertEqual(len(coco_evaluator._groundtruth_list), 4) self.assertEqual(len(coco_evaluator._detection_boxes_list), 5) metrics = {} for key, (value_op, _) in eval_metric_ops.items(): metrics[key] = value_op metrics = sess.run(metrics) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (large)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (medium)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (small)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@1'], 0.83333331) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@10'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (large)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (medium)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (small)'], 1.0) self.assertFalse(coco_evaluator._groundtruth_list) self.assertFalse(coco_evaluator._detection_boxes_list) self.assertFalse(coco_evaluator._image_ids)
def testGetOneMAPWithMatchingGroundtruthAndDetectionsBatched(self): coco_evaluator = coco_evaluation.CocoDetectionEvaluator( _get_categories_list()) batch_size = 3 image_id = tf.placeholder(tf.string, shape=(batch_size)) groundtruth_boxes = tf.placeholder(tf.float32, shape=(batch_size, None, 4)) groundtruth_classes = tf.placeholder(tf.float32, shape=(batch_size, None)) detection_boxes = tf.placeholder(tf.float32, shape=(batch_size, None, 4)) detection_scores = tf.placeholder(tf.float32, shape=(batch_size, None)) detection_classes = tf.placeholder(tf.float32, shape=(batch_size, None)) input_data_fields = standard_fields.InputDataFields detection_fields = standard_fields.DetectionResultFields eval_dict = { input_data_fields.key: image_id, input_data_fields.groundtruth_boxes: groundtruth_boxes, input_data_fields.groundtruth_classes: groundtruth_classes, detection_fields.detection_boxes: detection_boxes, detection_fields.detection_scores: detection_scores, detection_fields.detection_classes: detection_classes } eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops(eval_dict) _, update_op = eval_metric_ops['DetectionBoxes_Precision/mAP'] with self.test_session() as sess: sess.run(update_op, feed_dict={ image_id: ['image1', 'image2', 'image3'], groundtruth_boxes: np.array([[[100., 100., 200., 200.]], [[50., 50., 100., 100.]], [[25., 25., 50., 50.]]]), groundtruth_classes: np.array([[1], [3], [2]]), detection_boxes: np.array([[[100., 100., 200., 200.]], [[50., 50., 100., 100.]], [[25., 25., 50., 50.]]]), detection_scores: np.array([[.8], [.7], [.9]]), detection_classes: np.array([[1], [3], [2]]) }) metrics = {} for key, (value_op, _) in eval_metric_ops.items(): metrics[key] = value_op metrics = sess.run(metrics) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (large)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (medium)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (small)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@1'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@10'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (large)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (medium)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (small)'], 1.0) self.assertFalse(coco_evaluator._groundtruth_list) self.assertFalse(coco_evaluator._detection_boxes_list) self.assertFalse(coco_evaluator._image_ids)
def testGetOneMAPWithMatchingGroundtruthAndDetectionsIsAnnotated(self): coco_evaluator = coco_evaluation.CocoDetectionEvaluator( _get_categories_list()) image_id = tf.placeholder(tf.string, shape=()) groundtruth_boxes = tf.placeholder(tf.float32, shape=(None, 4)) groundtruth_classes = tf.placeholder(tf.float32, shape=(None)) is_annotated = tf.placeholder(tf.bool, shape=()) detection_boxes = tf.placeholder(tf.float32, shape=(None, 4)) detection_scores = tf.placeholder(tf.float32, shape=(None)) detection_classes = tf.placeholder(tf.float32, shape=(None)) input_data_fields = standard_fields.InputDataFields detection_fields = standard_fields.DetectionResultFields eval_dict = { input_data_fields.key: image_id, input_data_fields.groundtruth_boxes: groundtruth_boxes, input_data_fields.groundtruth_classes: groundtruth_classes, 'is_annotated': is_annotated, detection_fields.detection_boxes: detection_boxes, detection_fields.detection_scores: detection_scores, detection_fields.detection_classes: detection_classes } eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops(eval_dict) _, update_op = eval_metric_ops['DetectionBoxes_Precision/mAP'] with self.test_session() as sess: sess.run(update_op, feed_dict={ image_id: 'image1', groundtruth_boxes: np.array([[100., 100., 200., 200.]]), groundtruth_classes: np.array([1]), is_annotated: True, detection_boxes: np.array([[100., 100., 200., 200.]]), detection_scores: np.array([.8]), detection_classes: np.array([1]) }) sess.run(update_op, feed_dict={ image_id: 'image2', groundtruth_boxes: np.array([[50., 50., 100., 100.]]), groundtruth_classes: np.array([3]), is_annotated: True, detection_boxes: np.array([[50., 50., 100., 100.]]), detection_scores: np.array([.7]), detection_classes: np.array([3]) }) sess.run(update_op, feed_dict={ image_id: 'image3', groundtruth_boxes: np.array([[25., 25., 50., 50.]]), groundtruth_classes: np.array([2]), is_annotated: True, detection_boxes: np.array([[25., 25., 50., 50.]]), detection_scores: np.array([.9]), detection_classes: np.array([2]) }) sess.run(update_op, feed_dict={ image_id: 'image4', groundtruth_boxes: np.zeros((0, 4)), groundtruth_classes: np.zeros((0)), is_annotated: False, # Note that this image isn't annotated. detection_boxes: np.array([[25., 25., 50., 50.], [25., 25., 70., 50.], [25., 25., 80., 50.], [25., 25., 90., 50.]]), detection_scores: np.array([0.6, 0.7, 0.8, 0.9]), detection_classes: np.array([1, 2, 2, 3]) }) metrics = {} for key, (value_op, _) in eval_metric_ops.items(): metrics[key] = value_op metrics = sess.run(metrics) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (large)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (medium)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (small)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@1'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@10'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (large)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (medium)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (small)'], 1.0) self.assertFalse(coco_evaluator._groundtruth_list) self.assertFalse(coco_evaluator._detection_boxes_list) self.assertFalse(coco_evaluator._image_ids)
def testGetOneMAPWithMatchingGroundtruthAndDetections(self): coco_evaluator = coco_evaluation.CocoDetectionEvaluator( _get_categories_list()) image_id = tf.placeholder(tf.string, shape=()) groundtruth_boxes = tf.placeholder(tf.float32, shape=(None, 4)) groundtruth_classes = tf.placeholder(tf.float32, shape=(None)) detection_boxes = tf.placeholder(tf.float32, shape=(None, 4)) detection_scores = tf.placeholder(tf.float32, shape=(None)) detection_classes = tf.placeholder(tf.float32, shape=(None)) input_data_fields = standard_fields.InputDataFields detection_fields = standard_fields.DetectionResultFields eval_dict = { input_data_fields.key: image_id, input_data_fields.groundtruth_boxes: groundtruth_boxes, input_data_fields.groundtruth_classes: groundtruth_classes, detection_fields.detection_boxes: detection_boxes, detection_fields.detection_scores: detection_scores, detection_fields.detection_classes: detection_classes } eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops(eval_dict) _, update_op = eval_metric_ops['DetectionBoxes_Precision/mAP'] with self.draw_roc() as sess: tf.metrics.auc( labels = detection_classes, predictions = detection_scores, weights=None, num_thresholds=200, metrics_collections=None, updates_collections=None, curve='ROC', name=None, summation_method='trapezoidal' ) print(auc) with self.test_session() as sess: sess.run(update_op, feed_dict={ image_id: 'image1', groundtruth_boxes: np.array([[100., 100., 200., 200.]]), groundtruth_classes: np.array([1]), detection_boxes: np.array([[100., 100., 200., 200.]]), detection_scores: np.array([.8]), detection_classes: np.array([1]) }) sess.run(update_op, feed_dict={ image_id: 'image2', groundtruth_boxes: np.array([[50., 50., 100., 100.]]), groundtruth_classes: np.array([3]), detection_boxes: np.array([[50., 50., 100., 100.]]), detection_scores: np.array([.7]), detection_classes: np.array([3]) }) sess.run(update_op, feed_dict={ image_id: 'image3', groundtruth_boxes: np.array([[25., 25., 50., 50.]]), groundtruth_classes: np.array([2]), detection_boxes: np.array([[25., 25., 50., 50.]]), detection_scores: np.array([.9]), detection_classes: np.array([2]) }) metrics = {} for key, (value_op, _) in eval_metric_ops.items(): metrics[key] = value_op metrics = sess.run(metrics) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (large)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (medium)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (small)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@1'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@10'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (large)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (medium)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (small)'], 1.0) self.assertFalse(coco_evaluator._groundtruth_list) self.assertFalse(coco_evaluator._detection_boxes_list) self.assertFalse(coco_evaluator._image_ids)
def model_fn(features, labels, mode, params=None): """Constructs the object detection model. Args: features: Dictionary of feature tensors, returned from `input_fn`. labels: Dictionary of groundtruth tensors if mode is TRAIN or EVAL, otherwise None. mode: Mode key from tf.estimator.ModeKeys. params: Parameter dictionary passed from the estimator. Returns: An `EstimatorSpec` that encapsulates the model and its serving configurations. """ params = params or {} total_loss, train_op, detections, export_outputs = None, None, None, None is_training = mode == tf.estimator.ModeKeys.TRAIN detection_model = detection_model_fn(is_training=is_training, add_summaries=(not use_tpu)) scaffold_fn = None if mode == tf.estimator.ModeKeys.TRAIN: labels = unstack_batch( labels, unpad_groundtruth_tensors=train_config.unpad_groundtruth_tensors) elif mode == tf.estimator.ModeKeys.EVAL: labels = unstack_batch(labels, unpad_groundtruth_tensors=False) if mode in (tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL): gt_boxes_list = labels[fields.InputDataFields.groundtruth_boxes] gt_classes_list = labels[fields.InputDataFields.groundtruth_classes] gt_masks_list = None if fields.InputDataFields.groundtruth_instance_masks in labels: gt_masks_list = labels[ fields.InputDataFields.groundtruth_instance_masks] gt_keypoints_list = None if fields.InputDataFields.groundtruth_keypoints in labels: gt_keypoints_list = labels[fields.InputDataFields.groundtruth_keypoints] detection_model.provide_groundtruth( groundtruth_boxes_list=gt_boxes_list, groundtruth_classes_list=gt_classes_list, groundtruth_masks_list=gt_masks_list, groundtruth_keypoints_list=gt_keypoints_list) preprocessed_images = features[fields.InputDataFields.image] prediction_dict = detection_model.predict( preprocessed_images, features[fields.InputDataFields.true_image_shape]) detections = detection_model.postprocess( prediction_dict, features[fields.InputDataFields.true_image_shape]) if mode == tf.estimator.ModeKeys.TRAIN: if train_config.fine_tune_checkpoint and hparams.load_pretrained: asg_map = detection_model.restore_map( from_detection_checkpoint=train_config.from_detection_checkpoint, load_all_detection_checkpoint_vars=( train_config.load_all_detection_checkpoint_vars)) available_var_map = ( variables_helper.get_variables_available_in_checkpoint( asg_map, train_config.fine_tune_checkpoint, include_global_step=False)) if use_tpu: def tpu_scaffold(): tf.train.init_from_checkpoint(train_config.fine_tune_checkpoint, available_var_map) return tf.train.Scaffold() scaffold_fn = tpu_scaffold else: tf.train.init_from_checkpoint(train_config.fine_tune_checkpoint, available_var_map) if mode in (tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL): losses_dict = detection_model.loss( prediction_dict, features[fields.InputDataFields.true_image_shape]) losses = [loss_tensor for loss_tensor in losses_dict.itervalues()] total_loss = tf.add_n(losses, name='total_loss') if mode == tf.estimator.ModeKeys.TRAIN: global_step = tf.train.get_or_create_global_step() training_optimizer, optimizer_summary_vars = optimizer_builder.build( train_config.optimizer) if use_tpu: training_optimizer = tpu_optimizer.CrossShardOptimizer( training_optimizer) # Optionally freeze some layers by setting their gradients to be zero. trainable_variables = None if train_config.freeze_variables: trainable_variables = tf.contrib.framework.filter_variables( tf.trainable_variables(), exclude_patterns=train_config.freeze_variables) clip_gradients_value = None if train_config.gradient_clipping_by_norm > 0: clip_gradients_value = train_config.gradient_clipping_by_norm if not use_tpu: for var in optimizer_summary_vars: tf.summary.scalar(var.op.name, var) summaries = [] if use_tpu else None train_op = tf.contrib.layers.optimize_loss( loss=total_loss, global_step=global_step, learning_rate=None, clip_gradients=clip_gradients_value, optimizer=training_optimizer, variables=trainable_variables, summaries=summaries, name='') # Preventing scope prefix on all variables. if mode == tf.estimator.ModeKeys.PREDICT: export_outputs = { tf.saved_model.signature_constants.PREDICT_METHOD_NAME: tf.estimator.export.PredictOutput(detections) } eval_metric_ops = None if mode == tf.estimator.ModeKeys.EVAL: # Detection summaries during eval. class_agnostic = (fields.DetectionResultFields.detection_classes not in detections) groundtruth = _get_groundtruth_data(detection_model, class_agnostic) eval_dict = eval_util.result_dict_for_single_example( tf.expand_dims(features[fields.InputDataFields.original_image][0], 0), features[inputs.HASH_KEY][0], detections, groundtruth, class_agnostic=class_agnostic, scale_to_absolute=False) if class_agnostic: category_index = label_map_util.create_class_agnostic_category_index() else: category_index = label_map_util.create_category_index_from_labelmap( eval_input_config.label_map_path) detection_and_groundtruth = vis_utils.draw_side_by_side_evaluation_image( eval_dict, category_index, max_boxes_to_draw=20, min_score_thresh=0.2) if not use_tpu: tf.summary.image('Detections_Left_Groundtruth_Right', detection_and_groundtruth) # Eval metrics on a single image. detection_fields = fields.DetectionResultFields() input_data_fields = fields.InputDataFields() coco_evaluator = coco_evaluation.CocoDetectionEvaluator( category_index.values()) eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops( image_id=eval_dict[input_data_fields.key], groundtruth_boxes=eval_dict[input_data_fields.groundtruth_boxes], groundtruth_classes=eval_dict[input_data_fields.groundtruth_classes], detection_boxes=eval_dict[detection_fields.detection_boxes], detection_scores=eval_dict[detection_fields.detection_scores], detection_classes=eval_dict[detection_fields.detection_classes]) if use_tpu: return tf.contrib.tpu.TPUEstimatorSpec( mode=mode, scaffold_fn=scaffold_fn, predictions=detections, loss=total_loss, train_op=train_op, eval_metrics=eval_metric_ops, export_outputs=export_outputs) else: return tf.estimator.EstimatorSpec( mode=mode, predictions=detections, loss=total_loss, train_op=train_op, eval_metric_ops=eval_metric_ops, export_outputs=export_outputs)
def main(_): pipeline_proto = _load_pipeline_proto(FLAGS.pipeline_proto) if FLAGS.model_dir: pipeline_proto.model_dir = FLAGS.model_dir tf.logging.info("Override model checkpoint dir: %s", FLAGS.model_dir) if FLAGS.shard_indicator: pipeline_proto.eval_reader.shard_indicator = FLAGS.shard_indicator tf.logging.info("Override shard_indicator: %s", FLAGS.shard_indicator) if FLAGS.input_pattern: while len(pipeline_proto.eval_reader.wsod_reader.input_pattern) > 0: pipeline_proto.eval_reader.wsod_reader.input_pattern.pop() pipeline_proto.eval_reader.wsod_reader.input_pattern.append( FLAGS.input_pattern) tf.logging.info("Override input_pattern: %s", FLAGS.input_pattern) tf.logging.info("Pipeline configure: %s", '=' * 128) tf.logging.info(pipeline_proto) # Load the vocabulary file. categories = [] category_to_id = {} with open(FLAGS.vocabulary_file, 'r') as fp: for line_id, line in enumerate(fp.readlines()): categories.append({'id': 1 + line_id, 'name': line.strip('\n')}) category_to_id[line.strip('\n')] = 1 + line_id tf.logging.info("\n%s", json.dumps(categories, indent=2)) # Create the evaluator. number_of_evaluators = max(1, FLAGS.number_of_evaluators) if FLAGS.evaluator.lower() == 'pascal': evaluators = [ object_detection_evaluation.PascalDetectionEvaluator(categories) for i in range(number_of_evaluators) ] elif FLAGS.evaluator.lower() == 'coco': evaluators = [ coco_evaluation.CocoDetectionEvaluator(categories) for i in range(number_of_evaluators) ] else: raise ValueError('Invalid evaluator {}.'.format(FLAGS.evaluator)) if not FLAGS.run_once: # Evaluation loop. latest_step = None while True: checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir) if checkpoint_path is not None: global_step = int(checkpoint_path.split('-')[-1]) if global_step != latest_step and global_step >= FLAGS.min_eval_steps: # Evaluate the checkpoint. latest_step = global_step tf.logging.info('Start to evaluate checkpoint %s.', checkpoint_path) summary, metric = _run_evaluation(pipeline_proto, checkpoint_path, evaluators, category_to_id, categories) step_best, metric_best = save_model_if_it_is_better( global_step, metric, checkpoint_path, FLAGS.saved_ckpts_dir) # Write summary. summary.value.add(tag='loss/best_metric', simple_value=metric_best) summary_writer = tf.summary.FileWriter(FLAGS.eval_log_dir) summary_writer.add_summary(summary, global_step=global_step) summary_writer.close() tf.logging.info("Summary is written.") continue tf.logging.info("Wait for 10 seconds.") time.sleep(10) else: # Run once. if FLAGS.eval_best_model: checkpoint_path = get_best_model_checkpoint(FLAGS.saved_ckpts_dir) else: checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir) tf.logging.info('Start to evaluate checkpoint %s.', checkpoint_path) summary, metric = _run_evaluation(pipeline_proto, checkpoint_path, evaluators, category_to_id, categories, save_report_to_file=True) tf.logging.info('Done')
def testGetOneMAPWithMatchingGroundtruthAndDetectionsPadded(self): category_list = [{ 'id': 0, 'name': 'person' }, { 'id': 1, 'name': 'cat' }, { 'id': 2, 'name': 'dog' }] coco_evaluator = coco_evaluation.CocoDetectionEvaluator(category_list) image_id = tf.placeholder(tf.string, shape=()) groundtruth_boxes = tf.placeholder(tf.float32, shape=(None, 4)) groundtruth_classes = tf.placeholder(tf.float32, shape=(None)) detection_boxes = tf.placeholder(tf.float32, shape=(None, 4)) detection_scores = tf.placeholder(tf.float32, shape=(None)) detection_classes = tf.placeholder(tf.float32, shape=(None)) input_data_fields = standard_fields.InputDataFields detection_fields = standard_fields.DetectionResultFields eval_dict = { input_data_fields.key: image_id, input_data_fields.groundtruth_boxes: groundtruth_boxes, input_data_fields.groundtruth_classes: groundtruth_classes, detection_fields.detection_boxes: detection_boxes, detection_fields.detection_scores: detection_scores, detection_fields.detection_classes: detection_classes } eval_metric_ops = coco_evaluator.get_estimator_eval_metric_ops( eval_dict) _, update_op = eval_metric_ops['DetectionBoxes_Precision/mAP'] with self.test_session() as sess: sess.run(update_op, feed_dict={ image_id: 'image1', groundtruth_boxes: np.array([[100., 100., 200., 200.], [-1, -1, -1, -1]]), groundtruth_classes: np.array([1, -1]), detection_boxes: np.array([[100., 100., 200., 200.], [0., 0., 0., 0.]]), detection_scores: np.array([.8, 0.]), detection_classes: np.array([1, -1]) }) sess.run(update_op, feed_dict={ image_id: 'image2', groundtruth_boxes: np.array([[50., 50., 100., 100.], [-1, -1, -1, -1]]), groundtruth_classes: np.array([3, -1]), detection_boxes: np.array([[50., 50., 100., 100.], [0., 0., 0., 0.]]), detection_scores: np.array([.7, 0.]), detection_classes: np.array([3, -1]) }) sess.run(update_op, feed_dict={ image_id: 'image3', groundtruth_boxes: np.array([[25., 25., 50., 50.], [10., 10., 15., 15.]]), groundtruth_classes: np.array([2, 2]), detection_boxes: np.array([[25., 25., 50., 50.], [10., 10., 15., 15.]]), detection_scores: np.array([.95, .9]), detection_classes: np.array([2, 2]) }) metrics = {} for key, (value_op, _) in eval_metric_ops.iteritems(): metrics[key] = value_op metrics = sess.run(metrics) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/[email protected]'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (large)'], 1.0) self.assertAlmostEqual( metrics['DetectionBoxes_Precision/mAP (medium)'], -1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP (small)'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@1'], 0.75) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@10'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100'], 1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (large)'], 1.0) self.assertAlmostEqual( metrics['DetectionBoxes_Recall/AR@100 (medium)'], -1.0) self.assertAlmostEqual(metrics['DetectionBoxes_Recall/AR@100 (small)'], 1.0) self.assertFalse(coco_evaluator._groundtruth_list) self.assertFalse(coco_evaluator._detection_boxes_list) self.assertFalse(coco_evaluator._image_ids)
def get_eval_metric_ops_for_evaluators(evaluation_metrics, categories, eval_dict, include_metrics_per_category=False): """Returns a dictionary of eval metric ops to use with `tf.EstimatorSpec`. Args: evaluation_metrics: List of evaluation metric names. Current options are 'coco_detection_metrics' and 'coco_mask_metrics'. categories: A list of dicts, each of which has the following keys - 'id': (required) an integer id uniquely identifying this category. 'name': (required) string representing category name e.g., 'cat', 'dog'. eval_dict: An evaluation dictionary, returned from result_dict_for_single_example(). include_metrics_per_category: If True, additionally include per-category metrics. Returns: A dictionary of metric names to tuple of value_op and update_op that can be used as eval metric ops in tf.EstimatorSpec. Raises: ValueError: If any of the metrics in `evaluation_metric` is not 'coco_detection_metrics' or 'coco_mask_metrics'. """ evaluation_metrics = list(set(evaluation_metrics)) input_data_fields = fields.InputDataFields detection_fields = fields.DetectionResultFields eval_metric_ops = {} for metric in evaluation_metrics: if metric == 'coco_detection_metrics': coco_evaluator = coco_evaluation.CocoDetectionEvaluator( categories, include_metrics_per_category=include_metrics_per_category) eval_metric_ops.update( coco_evaluator.get_estimator_eval_metric_ops( image_id=eval_dict[input_data_fields.key], groundtruth_boxes=eval_dict[input_data_fields.groundtruth_boxes], groundtruth_classes=eval_dict[ input_data_fields.groundtruth_classes], detection_boxes=eval_dict[detection_fields.detection_boxes], detection_scores=eval_dict[detection_fields.detection_scores], detection_classes=eval_dict[detection_fields.detection_classes], groundtruth_is_crowd=eval_dict.get( input_data_fields.groundtruth_is_crowd))) elif metric == 'coco_mask_metrics': coco_mask_evaluator = coco_evaluation.CocoMaskEvaluator( categories, include_metrics_per_category=include_metrics_per_category) eval_metric_ops.update( coco_mask_evaluator.get_estimator_eval_metric_ops( image_id=eval_dict[input_data_fields.key], groundtruth_boxes=eval_dict[input_data_fields.groundtruth_boxes], groundtruth_classes=eval_dict[ input_data_fields.groundtruth_classes], groundtruth_instance_masks=eval_dict[ input_data_fields.groundtruth_instance_masks], detection_scores=eval_dict[detection_fields.detection_scores], detection_classes=eval_dict[detection_fields.detection_classes], detection_masks=eval_dict[detection_fields.detection_masks], groundtruth_is_crowd=eval_dict.get( input_data_fields.groundtruth_is_crowd),)) else: raise ValueError('The only evaluation metrics supported are ' '"coco_detection_metrics" and "coco_mask_metrics". ' 'Found {} in the evaluation metrics'.format(metric)) return eval_metric_ops
def count_coco_metrics(true_images_data: List[ImageData], raw_pred_images_data: List[ImageData] = None, class_names: List[str] = None) -> Dict: from object_detection.metrics import coco_evaluation from object_detection.core.standard_fields import (InputDataFields, DetectionResultFields) if class_names is None: cocoevaluator = coco_evaluation.CocoDetectionEvaluator( categories=[{ 'id': 1, 'name': 'Label' }]) else: cocoevaluator = coco_evaluation.CocoDetectionEvaluator( categories=[{ 'id': i + 1, 'name': class_name } for i, class_name in enumerate(class_names)]) for i, (true_image_data, raw_pred_image_data) in enumerate( zip(true_images_data, raw_pred_images_data)): true_bboxes_data = true_image_data.bboxes_data raw_pred_bboxes_data = raw_pred_image_data.bboxes_data true_bboxes = [(true_bbox_data.ymin, true_bbox_data.xmin, true_bbox_data.ymax, true_bbox_data.xmax) for true_bbox_data in true_bboxes_data] raw_pred_bboxes = [(raw_pred_bbox_data.ymin, raw_pred_bbox_data.xmin, raw_pred_bbox_data.ymax, raw_pred_bbox_data.xmax) for raw_pred_bbox_data in raw_pred_bboxes_data] raw_pred_scores = [ raw_pred_bbox_data.detection_score for raw_pred_bbox_data in raw_pred_bboxes_data ] groundtruth_boxes = np.array(true_bboxes, dtype=np.float32).reshape(-1, 4) if class_names is None: groundtruth_classes = np.array([1] * len(true_bboxes)) else: groundtruth_classes = np.array([ class_names.index(bbox_data.label) + 1 for bbox_data in true_bboxes_data ]) groundtruth_dict = { InputDataFields.groundtruth_boxes: groundtruth_boxes, InputDataFields.groundtruth_classes: groundtruth_classes } cocoevaluator.add_single_ground_truth_image_info( image_id=i, groundtruth_dict=groundtruth_dict) detection_boxes = np.array(raw_pred_bboxes, dtype=np.float32).reshape(-1, 4) detection_scores = np.array(raw_pred_scores, dtype=np.float32) if class_names is None: detection_classes = np.array([1] * len(detection_boxes)) else: detection_classes = np.array([ class_names.index(bbox_data.label) + 1 for bbox_data in raw_pred_bboxes_data ]) detections_dict = { DetectionResultFields.detection_boxes: detection_boxes, DetectionResultFields.detection_scores: detection_scores, DetectionResultFields.detection_classes: detection_classes } cocoevaluator.add_single_detected_image_info( image_id=i, detections_dict=detections_dict) with open(os.devnull, "w") as f, contextlib.redirect_stdout(f): coco_metrics = cocoevaluator.evaluate() return coco_metrics