def setUp(self):
num_groundtruth_classes = 1
matching_iou_threshold_high_iou = 0.5
matching_iou_threshold_low_iou = 0.1
nms_iou_threshold = 1.0
nms_max_output_boxes = 10000
self.eval_high_iou = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes, matching_iou_threshold_high_iou,
nms_iou_threshold, nms_max_output_boxes)
self.eval_low_iou = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes, matching_iou_threshold_low_iou,
nms_iou_threshold, nms_max_output_boxes)
self.detected_boxes = np.array([[0, 0, 1, 1], [0, 0, 2, 2], [0, 0, 3, 3]],
dtype=float)
self.detected_scores = np.array([0.6, 0.8, 0.5], dtype=float)
detected_masks_0 = np.array([[0, 1, 1, 0],
[0, 0, 1, 0],
[0, 0, 0, 0]], dtype=np.uint8)
detected_masks_1 = np.array([[1, 0, 0, 0],
[1, 1, 0, 0],
[0, 0, 0, 0]], dtype=np.uint8)
detected_masks_2 = np.array([[0, 0, 0, 0],
[0, 1, 1, 0],
[0, 1, 0, 0]], dtype=np.uint8)
self.detected_masks = np.stack(
[detected_masks_0, detected_masks_1, detected_masks_2], axis=0)
def __init__(self,
num_groundtruth_classes,
matching_iou_threshold=0.5,
nms_iou_threshold=1.0,
nms_max_output_boxes=10000):
self.per_image_eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes, matching_iou_threshold, nms_iou_threshold,
nms_max_output_boxes)
self.num_class = num_groundtruth_classes
self.groundtruth_boxes = {}
self.groundtruth_class_labels = {}
self.groundtruth_is_difficult_list = {}
self.num_gt_instances_per_class = np.zeros(self.num_class, dtype=int)
self.num_gt_imgs_per_class = np.zeros(self.num_class, dtype=int)
self.detection_keys = set()
self.scores_per_class = [[] for _ in range(self.num_class)]
self.tp_fp_labels_per_class = [[] for _ in range(self.num_class)]
self.num_images_correctly_detected_per_class = np.zeros(self.num_class)
self.average_precision_per_class = np.empty(
self.num_class, dtype=float)
self.average_precision_per_class.fill(np.nan)
self.precisions_per_class = []
self.recalls_per_class = []
self.corloc_per_class = np.ones(self.num_class, dtype=float)
def __init__(self,
num_groundtruth_classes,
matching_iou_threshold=0.5,
nms_iou_threshold=1.0,
nms_max_output_boxes=10000,
use_weighted_mean_ap=False,
label_id_offset=0,
group_of_weight=0.0):
if num_groundtruth_classes < 1:
raise ValueError(
'Need at least 1 groundtruth class for evaluation.')
self.per_image_eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes=num_groundtruth_classes,
matching_iou_threshold=matching_iou_threshold,
nms_iou_threshold=nms_iou_threshold,
nms_max_output_boxes=nms_max_output_boxes,
group_of_weight=group_of_weight)
self.group_of_weight = group_of_weight
self.num_class = num_groundtruth_classes
self.use_weighted_mean_ap = use_weighted_mean_ap
self.label_id_offset = label_id_offset
self.groundtruth_boxes = {}
self.groundtruth_class_labels = {}
self.groundtruth_masks = {}
self.groundtruth_is_difficult_list = {}
self.groundtruth_is_group_of_list = {}
self.num_gt_instances_per_class = np.zeros(self.num_class, dtype=float)
self.num_gt_imgs_per_class = np.zeros(self.num_class, dtype=int)
self._initialize_detections()
def setUp(self):
num_groundtruth_classes = 1
matching_iou_threshold = 0.5
nms_iou_threshold = 1.0
nms_max_output_boxes = 10000
self.group_of_weight = 0.5
self.eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes, matching_iou_threshold, nms_iou_threshold,
nms_max_output_boxes, self.group_of_weight)
self.detected_boxes = np.array(
[[0, 0, 1, 1], [0, 0, 2, 1], [0, 0, 3, 1]], dtype=float)
self.detected_scores = np.array([0.8, 0.6, 0.5], dtype=float)
detected_masks_0 = np.array(
[[0, 1, 1, 0], [0, 0, 1, 0], [0, 0, 0, 0]], dtype=np.uint8)
detected_masks_1 = np.array(
[[1, 0, 0, 0], [1, 1, 0, 0], [0, 0, 0, 0]], dtype=np.uint8)
detected_masks_2 = np.array(
[[0, 0, 0, 0], [0, 1, 1, 0], [0, 1, 0, 0]], dtype=np.uint8)
self.detected_masks = np.stack(
[detected_masks_0, detected_masks_1, detected_masks_2], axis=0)
self.groundtruth_boxes = np.array(
[[0, 0, 1, 1], [0, 0, 5, 5], [10, 10, 20, 20]], dtype=float)
groundtruth_masks_0 = np.array(
[[1, 0, 0, 0], [1, 0, 0, 0], [1, 0, 0, 0]], dtype=np.uint8)
groundtruth_masks_1 = np.array(
[[0, 0, 1, 0], [0, 0, 1, 0], [0, 0, 1, 0]], dtype=np.uint8)
groundtruth_masks_2 = np.array(
[[0, 1, 0, 0], [0, 1, 0, 0], [0, 1, 0, 0]], dtype=np.uint8)
self.groundtruth_masks = np.stack(
[groundtruth_masks_0, groundtruth_masks_1, groundtruth_masks_2], axis=0)
def test_tp_fp(self):
num_groundtruth_classes = 3
matching_iou_threshold = 0.5
nms_iou_threshold = 1.0
nms_max_output_boxes = 10000
eval1 = per_image_evaluation.PerImageEvaluation(num_groundtruth_classes,
matching_iou_threshold,
nms_iou_threshold,
nms_max_output_boxes)
detected_boxes = np.array([[0, 0, 1, 1], [10, 10, 5, 5], [0, 0, 2, 2],
[5, 10, 10, 5], [10, 5, 5, 10], [0, 0, 3, 3]],
dtype=float)
detected_scores = np.array([0.8, 0.1, 0.8, 0.9, 0.7, 0.8], dtype=float)
detected_class_labels = np.array([0, 1, 1, 2, 0, 2], dtype=int)
groundtruth_boxes = np.array([[0, 0, 1, 1], [0, 0, 3.5, 3.5]], dtype=float)
groundtruth_class_labels = np.array([0, 2], dtype=int)
groundtruth_groundtruth_is_difficult_list = np.zeros(2, dtype=float)
groundtruth_groundtruth_is_group_of_list = np.array(
[False, False], dtype=bool)
scores, tp_fp_labels, _ = eval1.compute_object_detection_metrics(
detected_boxes, detected_scores, detected_class_labels,
groundtruth_boxes, groundtruth_class_labels,
groundtruth_groundtruth_is_difficult_list,
groundtruth_groundtruth_is_group_of_list)
expected_scores = [np.array([0.8], dtype=float)] * 3
expected_tp_fp_labels = [np.array([True]), np.array([False]), np.array([True
])]
for i in range(len(expected_scores)):
self.assertTrue(np.allclose(expected_scores[i], scores[i]))
self.assertTrue(np.array_equal(expected_tp_fp_labels[i], tp_fp_labels[i]))
def setUp(self):
num_groundtruth_classes = 1
matching_iou_threshold1 = 0.5
matching_iou_threshold2 = 0.1
nms_iou_threshold = 1.0
nms_max_output_boxes = 10000
self.eval1 = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes, matching_iou_threshold1,
nms_iou_threshold, nms_max_output_boxes)
self.eval2 = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes, matching_iou_threshold2,
nms_iou_threshold, nms_max_output_boxes)
self.detected_boxes = np.array(
[[0, 0, 1, 1], [0, 0, 2, 2], [0, 0, 3, 3]], dtype=float)
self.detected_scores = np.array([0.6, 0.8, 0.5], dtype=float)
def setUp(self):
num_groundtruth_classes = 1
matching_iou_threshold_iou = 0.5
nms_iou_threshold = 1.0
nms_max_output_boxes = 10000
self.eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes, matching_iou_threshold_iou, nms_iou_threshold,
nms_max_output_boxes)
def __init__(self,
num_groundtruth_classes,
matching_iou_threshold=0.5,
nms_iou_threshold=1.0,
nms_max_output_boxes=10000,
use_weighted_mean_ap=False,
label_id_offset=0):
self.per_image_eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes, matching_iou_threshold, nms_iou_threshold,
nms_max_output_boxes)
self.num_class = num_groundtruth_classes
self.label_id_offset = label_id_offset
self.groundtruth_boxes = {}
self.groundtruth_class_labels = {}
self.groundtruth_is_difficult_list = {}
self.groundtruth_is_group_of_list = {}
self.num_gt_instances_per_class = np.zeros(self.num_class, dtype=int)
self.num_gt_imgs_per_class = np.zeros(self.num_class, dtype=int)
self.detection_keys = set()
self.scores_per_class = [[] for _ in range(self.num_class)]
self.tp_fp_labels_per_class = [[] for _ in range(self.num_class)]
self.num_images_correctly_detected_per_class = np.zeros(self.num_class)
self.average_precision_per_class = np.empty(self.num_class,
dtype=float)
self.average_precision_per_class.fill(np.nan)
self.precisions_per_class = []
self.recalls_per_class = []
self.corloc_per_class = np.ones(self.num_class, dtype=float)
self.iou_list = [.5, .55, .6, .65, .7, .75, .8, .85, .9, .95]
self.scores_per_class_per_iou = \
[ [[] for _ in range(self.num_class)] for _ in range(len(self.iou_list)) ]
self.tp_fp_labels_per_class_per_iou = \
[ [[] for _ in range(self.num_class)] for _ in range(len(self.iou_list)) ]
self.num_images_correctly_detected_per_class_per_iou = \
[np.zeros(self.num_class) for _ in range(len(self.iou_list)) ]
self.average_precision_per_class_per_iou = \
[np.empty(self.num_class, dtype=float) for _ in range(len(self.iou_list))]
for average_precision_per_class in self.average_precision_per_class_per_iou:
average_precision_per_class.fill(np.nan)
self.precisions_per_class_per_iou = \
[[] for _ in range(len(self.iou_list))]
self.recalls_per_class_per_iou = \
[[] for _ in range(len(self.iou_list))]
self.corloc_per_class_per_iou = \
[np.ones(self.num_class, dtype=float) for _ in range(len(self.iou_list))]
self.use_weighted_mean_ap = use_weighted_mean_ap
def __init__(self,
num_groundtruth_classes,
matching_iou_threshold=0.5,
nms_type='standard',
nms_iou_threshold=1.0,
nms_max_output_boxes=10000,
soft_nms_sigma=0.5,
subset_names=('default', )):
self.per_image_eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes, matching_iou_threshold, nms_type,
nms_iou_threshold, nms_max_output_boxes, soft_nms_sigma)
self.num_class = num_groundtruth_classes
self.subset_names = subset_names
self.groundtruth_boxes = {}
self.groundtruth_class_labels = {}
self.groundtruth_subset = {s: {} for s in self.subset_names}
self.num_gt_instances_per_class = {
s: np.zeros(self.num_class, dtype=int)
for s in self.subset_names
}
self.num_gt_imgs_per_class = np.zeros(self.num_class, dtype=int)
self.detection_keys = set()
self.scores_per_class = {
s: [[] for _ in range(self.num_class)]
for s in self.subset_names
}
self.tp_fp_labels_per_class = {
s: [[] for _ in range(self.num_class)]
for s in self.subset_names
}
self.num_images_correctly_detected_per_class = np.zeros(self.num_class)
self.average_precision_per_class \
= {s: np.empty(self.num_class, dtype=float)
for s in self.subset_names}
for s in self.subset_names:
self.average_precision_per_class[s].fill(np.nan)
self.precisions_per_class = {s: [] for s in self.subset_names}
self.recalls_per_class = {s: [] for s in self.subset_names}
self.corloc_per_class = np.ones(self.num_class, dtype=float)
def __init__(self,
num_groundtruth_classes,
matching_iou_threshold=0.5,
nms_iou_threshold=1.0,
nms_max_output_boxes=10000,
use_weighted_mean_ap=False,
label_id_offset=0):
self.per_image_eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes, matching_iou_threshold, nms_iou_threshold,
nms_max_output_boxes)
self.num_class = num_groundtruth_classes
self.label_id_offset = label_id_offset
self.groundtruth_boxes = {}
self.groundtruth_class_labels = {}
self.groundtruth_is_difficult_list = {}
self.groundtruth_is_group_of_list = {}
self.num_gt_instances_per_class = np.zeros(self.num_class, dtype=int)
self.num_gt_imgs_per_class = np.zeros(self.num_class, dtype=int)
self.detection_keys = set()
self.scores_per_class = [[] for _ in range(self.num_class)]
self.tp_fp_labels_per_class = [[] for _ in range(self.num_class)]
self.num_images_correctly_detected_per_class = np.zeros(self.num_class)
self.average_precision_per_class = np.empty(self.num_class,
dtype=float)
self.average_precision_per_class.fill(np.nan)
self.precisions_per_class = []
self.recalls_per_class = []
self.corloc_per_class = np.ones(self.num_class, dtype=float)
self.use_weighted_mean_ap = use_weighted_mean_ap
# for image-level classification
self.groundtruth_image_class_labels = []
self.detected_image_scores = []
self.detected_image_class_labels = []
# eval result for class in bounding boxes
self.eval_result_in_box_classifcation = []
def test_compute_corloc_with_very_large_iou_threshold(self):
num_groundtruth_classes = 3
matching_iou_threshold = 0.9
nms_iou_threshold = 1.0
nms_max_output_boxes = 10000
eval1 = per_image_evaluation.PerImageEvaluation(num_groundtruth_classes,
matching_iou_threshold,
nms_iou_threshold,
nms_max_output_boxes)
detected_boxes = np.array([[0, 0, 1, 1], [0, 0, 2, 2], [0, 0, 3, 3],
[0, 0, 5, 5]], dtype=float)
detected_scores = np.array([0.9, 0.9, 0.1, 0.9], dtype=float)
detected_class_labels = np.array([0, 1, 0, 2], dtype=int)
groundtruth_boxes = np.array([[0, 0, 1, 1], [0, 0, 3, 3], [0, 0, 6, 6]],
dtype=float)
groundtruth_class_labels = np.array([0, 0, 2], dtype=int)
is_class_correctly_detected_in_image = eval1._compute_cor_loc(
detected_boxes, detected_scores, detected_class_labels,
groundtruth_boxes, groundtruth_class_labels)
expected_result = np.array([1, 0, 0], dtype=int)
self.assertTrue(np.array_equal(expected_result,
is_class_correctly_detected_in_image))
def __init__(self,
num_classes: int,
matching_iou_threshold: float = 0.5,
detections_nms_iou_threshold: float = 1.0,
detections_nms_max_output_boxes: int = 10000,
image_size=None,
groundtruth_boxes_in_normalized: bool = True,
detection_boxes_in_normalized: bool = True,
**kpi_plugin_kwargs):
super().__init__(**kpi_plugin_kwargs)
self._per_image_evaluation = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes=num_classes,
matching_iou_threshold=matching_iou_threshold,
nms_iou_threshold=detections_nms_iou_threshold,
nms_max_output_boxes=detections_nms_max_output_boxes,
)
self.num_classes = num_classes
self.matching_iou_threshold = matching_iou_threshold
self.detections_nms_iou_threshold = detections_nms_iou_threshold
self.detections_nms_max_output_boxes = detections_nms_max_output_boxes
self.image_size = image_size
self.groundtruth_boxes_in_normalized = groundtruth_boxes_in_normalized
self.detection_boxes_in_normalized = detection_boxes_in_normalized
def compute_precision_recall_per_cat(detection_file, db_file):
print('Loading detection file...')
with open(detection_file) as f:
detection_results = pickle.load(f)
with open(db_file, 'r') as f:
data = json.load(f)
im_to_seq = {}
for im in data['images']:
im_to_seq[im['id']] = im['seq_id']
im_to_cat = {}
for ann in data['annotations']:
im_to_cat[ann['image_id']] = ann['category_id']
#add empty category
empty_id = max([cat['id'] for cat in data['categories']]) + 1
data['categories'].append({'name': 'empty', 'id': empty_id})
#add all images that don't have annotations, with cat empty
for im in data['images']:
if im['id'] not in im_to_cat:
im_to_cat[im['id']] = empty_id
cat_id_to_cat = {}
for cat in data['categories']:
cat_id_to_cat[cat['id']] = cat['name']
cat_to_ims = {cat_id: [] for cat_id in cat_id_to_cat}
for im in data['images']:
cat_to_ims[im_to_cat[im['id']]].append(im['id'])
seqs = {}
for im in detection_results['images']:
if im in im_to_seq:
if im_to_seq[im] not in seqs:
seqs[im_to_seq[im]] = []
seqs[im_to_seq[im]].append(im)
print('Clustering detections by image...')
#print(detection_results.keys())
# group the detections and gts by image id:
per_image_detections, per_image_gts = cluster_detections_by_image(
detection_results)
per_image_eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes=1,
matching_iou_threshold=0.5,
nms_iou_threshold=1.0,
nms_max_output_boxes=10000)
detection_labels = {cat: [] for cat in cat_to_ims}
detection_scores = {cat: [] for cat in cat_to_ims}
num_total_gts = {cat: 0 for cat in cat_to_ims}
count = {cat: 0 for cat in cat_to_ims}
precision = {}
recall = {}
average_precision = {}
for cat, images in cat_to_ims.iteritems():
for image_id in images:
if image_id not in per_image_detections:
#print(image_id)
count[cat] += 1
continue
scores, tp_fp_labels = get_results_per_image(
per_image_detections[image_id], per_image_gts[image_id],
per_image_eval)
detection_labels[cat].append(tp_fp_labels)
detection_scores[cat].append(scores)
num_gts = len(per_image_gts[image_id]['bboxes'])
num_total_gts[cat] += num_gts
if len(detection_scores[cat]) > 0:
scores = np.concatenate(detection_scores[cat])
labels = np.concatenate(detection_labels[cat]).astype(np.bool)
#print(len(scores))
#print(len(labels))
precision[cat], recall[cat] = metrics.compute_precision_recall(
scores, labels, num_total_gts[cat])
average_precision[cat] = metrics.compute_average_precision(
precision[cat], recall[cat])
else:
print("no detections for " + cat_id_to_cat[cat])
print(cat_id_to_cat[cat], count[cat], len(images))
return precision, recall, average_precision, cat_id_to_cat
def compute_precision_recall(detection_file,
detection_results=None,
images_to_consider='all',
get_night_day=None):
if detection_results == None:
print('Loading detection file...')
with open(detection_file) as f:
detection_results = pickle.load(f)
print('Clustering detections by image...')
#print(detection_results.keys())
# group the detections by image id:
use_im = get_images_to_consider(detection_results, images_to_consider,
get_night_day)
per_image_detections, per_image_gts = cluster_detections_by_image(
detection_results, use_im)
per_image_eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes=1,
matching_iou_threshold=0.5,
nms_iou_threshold=1.0,
nms_max_output_boxes=10000)
print('Running per-object analysis...')
detection_labels = []
detection_scores = []
num_total_gts = 0
count = 0
for image_id, dets in per_image_detections.iteritems():
num_detections = len(dets['bboxes'])
# [ymin, xmin, ymax, xmax] in absolute image coordinates.
detected_boxes = np.zeros([num_detections, 4], dtype=np.float32)
# detection scores for the boxes
detected_scores = np.zeros([num_detections], dtype=np.float32)
# 0-indexed detection classes for the boxes
detected_class_labels = np.zeros([num_detections], dtype=np.int32)
detected_masks = None
for i in range(num_detections):
x1, y1, x2, y2 = dets['bboxes'][i]
detected_boxes[i] = np.array([y1, x1, y2, x2])
detected_scores[i] = dets['scores'][i]
detected_class_labels[i] = dets['labels'][i] - 1
gts = per_image_gts[image_id]
#print(gts)
num_gts = len(gts['bboxes'])
#print(num_gts)
if num_gts > 0:
# [ymin, xmin, ymax, xmax] in absolute image coordinates
groundtruth_boxes = np.zeros([num_gts, 4], dtype=np.float32)
# 0-indexed groundtruth classes for the boxes
groundtruth_class_labels = np.zeros(num_gts, dtype=np.int32)
groundtruth_masks = None
groundtruth_is_difficult_list = np.zeros(num_gts, dtype=bool)
groundtruth_is_group_of_list = np.zeros(num_gts, dtype=bool)
for i in range(num_gts):
x1, y1, x2, y2 = gts['bboxes'][i]
groundtruth_boxes[i] = np.array([y1, x1, y2, x2])
groundtruth_class_labels[i] = gts['labels'][i] - 1
#print(groundtruth_boxes, groundtruth_class_labels,detected_scores[0],detected_boxes[0], detected_class_labels[:2])
scores, tp_fp_labels, is_class_correctly_detected_in_image = (
per_image_eval.compute_object_detection_metrics(
detected_boxes=detected_boxes,
detected_scores=detected_scores,
detected_class_labels=detected_class_labels,
groundtruth_boxes=groundtruth_boxes,
groundtruth_class_labels=groundtruth_class_labels,
groundtruth_is_difficult_list=groundtruth_is_difficult_list,
groundtruth_is_group_of_list=groundtruth_is_group_of_list,
detected_masks=detected_masks,
groundtruth_masks=groundtruth_masks))
#print(scores, tp_fp_labels)
detection_labels.append(tp_fp_labels[0])
detection_scores.append(scores[0])
num_total_gts += num_gts
count += 1
if count % 1000 == 0:
print(str(count) + ' images complete')
#if (tp_fp_labels[0].shape[0] != num_detections):
# print('Incorrect label length')
#if scores[0].shape[0] != num_detections:
# print('Incorrect score length')
#if tp_fp_labels[0].sum() > num_gts:
# print('Too many correct detections')
else:
detection_labels.append(np.zeros(num_detections, dtype=np.int32))
detection_scores.append(detected_scores)
scores = np.concatenate(detection_scores)
labels = np.concatenate(detection_labels).astype(np.bool)
precision, recall = metrics.compute_precision_recall(
scores, labels, num_total_gts)
average_precision = metrics.compute_average_precision(precision, recall)
return precision, recall, average_precision
def compute_precision_recall_with_images(detection_file):
print('Loading detection file...')
with open(detection_file) as f:
detection_results = pickle.load(f)
print('Clustering detections by image...')
# group the detections by image id:
per_image_detections = {
detection_results['images'][idx]: {
'bboxes': detection_results['detections'][idx],
'scores': detection_results['detection_scores'][idx],
'labels': detection_results['detection_labels'][idx]
}
for idx in range(len(detection_results['images']))
}
# group the ground truth annotations by image id:
per_image_gts = {
detection_results['images'][idx]: {
'bboxes': detection_results['gts'][idx],
'labels': detection_results['gt_labels'][idx]
}
for idx in range(len(detection_results['images']))
}
per_image_eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes=1,
matching_iou_threshold=0.5,
nms_iou_threshold=1.0,
nms_max_output_boxes=10000)
print('Running per-image analysis...')
detection_labels = []
detection_scores = []
num_total_gts = 0
count = 0
for image_id, dets in per_image_detections.iteritems():
im_detection_labels = []
im_detection_scores = []
im_num_gts = []
max_im_scores = []
num_detections = len(dets['bboxes'])
# [ymin, xmin, ymax, xmax] in absolute image coordinates.
detected_boxes = np.zeros([num_detections, 4], dtype=np.float32)
# detection scores for the boxes
detected_scores = np.zeros([num_detections], dtype=np.float32)
# 0-indexed detection classes for the boxes
detected_class_labels = np.zeros([num_detections], dtype=np.int32)
detected_masks = None
for i in range(num_detections):
x1, y1, x2, y2 = dets['bboxes'][i]
detected_boxes[i] = np.array([y1, x1, y2, x2])
detected_scores[i] = dets['scores'][i]
detected_class_labels[i] = dets['labels'][i] - 1
max_im_scores.append(np.max(detected_scores))
box_id = np.argmax(detected_scores)
gts = per_image_gts[image_id]
num_gts = len(gts['bboxes'])
im_num_gts = num_gts
if num_gts > 0:
# [ymin, xmin, ymax, xmax] in absolute image coordinates
groundtruth_boxes = np.zeros([num_gts, 4], dtype=np.float32)
# 0-indexed groundtruth classes for the boxes
groundtruth_class_labels = np.zeros(num_gts, dtype=np.int32)
groundtruth_masks = None
groundtruth_is_difficult_list = np.zeros(num_gts, dtype=bool)
groundtruth_is_group_of_list = np.zeros(num_gts, dtype=bool)
for i in range(num_gts):
x1, y1, x2, y2 = gts['bboxes'][i]
groundtruth_boxes[i] = np.array([y1, x1, y2, x2])
groundtruth_class_labels[i] = gts['labels'][i] - 1
ious = np_box_ops.iou(detected_boxes, groundtruth_boxes)
if np.max(ious[box_id, :]) < 0.5:
max_im_scores[-1] = 0
#print('detected animal box')
#print(groundtruth_boxes, groundtruth_class_labels,detected_scores[0],detected_boxes[0], detected_class_labels[0])
scores, tp_fp_labels, is_class_correctly_detected_in_image = (
per_image_eval.compute_object_detection_metrics(
detected_boxes=detected_boxes,
detected_scores=detected_scores,
detected_class_labels=detected_class_labels,
groundtruth_boxes=groundtruth_boxes,
groundtruth_class_labels=groundtruth_class_labels,
groundtruth_is_difficult_list=groundtruth_is_difficult_list,
groundtruth_is_group_of_list=groundtruth_is_group_of_list,
detected_masks=detected_masks,
groundtruth_masks=groundtruth_masks))
#print(scores, tp_fp_labels)
im_detection_labels = tp_fp_labels[0]
im_detection_scores = scores[0]
#num_total_gts += num_gts
count += 1
if count % 1000 == 0:
print(str(count) + ' images complete')
#if (tp_fp_labels[0].shape[0] != num_detections):
# print('Incorrect label length')
#if scores[0].shape[0] != num_detections:
# print('Incorrect score length')
#if tp_fp_labels[0].sum() > num_gts:
# print('Too many correct detections')
else:
im_detection_labels = np.zeros(num_detections, dtype=np.int32)
im_detection_scores = detected_scores
max_im_scores[-1] = 0
best_score = np.max(max_im_scores)
if best_score > 0:
#print('valid box')
best_im = np.argmax(max_im_scores)
#print(best_im, best_score)
temp_labels = np.zeros(len(im_detection_labels), dtype=np.int32)
temp_scores = np.zeros(len(im_detection_scores), dtype=np.float32)
for j in range(min(im_num_gts, len(im_detection_labels))):
temp_labels[
j] = True #TODO: this currently only works for oneclass?
temp_scores[j] = best_score
im_detection_labels = temp_labels
im_detection_scores = temp_scores
num_total_gts += im_num_gts
detection_labels.append(im_detection_labels)
detection_scores.append(im_detection_scores)
print(len(detection_scores), len(detection_scores[0]),
len(detection_scores[1]))
scores = np.concatenate(detection_scores)
labels = np.concatenate(detection_labels).astype(np.bool)
precision, recall = metrics.compute_precision_recall(
scores, labels, num_total_gts)
average_precision = metrics.compute_average_precision(precision, recall)
return precision, recall, average_precision
feed_dict={image_tensor: image_np_expanded})
use_time = time.time() - start_time
# vis_util.visualize_boxes_and_labels_on_image_array(
# image_np, np.squeeze(boxes), np.squeeze(classes).astype(np.int32), np.squeeze(scores),
# category_index, use_normalized_coordinates=True, min_score_thresh=0.8, line_thickness=2)
eval_dicts = {
'boxes': boxes,
'scores': scores,
'classes': classes,
'num_detections': num_detections
}
scores, tp_fp_labels, is_class_correctly_detected_in_image = per_image_evaluation.PerImageEvaluation(
).compute_object_detection_metrics(
detected_boxes=np.squeeze(boxes),
detected_scores=np.squeeze(scores),
detected_class_labels=np.squeeze(classes).astype(np.int32),
groundtruth_boxes=gt_boxes,
groundtruth_class_labels=gt_class_labels,
groundtruth_is_difficult_list=gt_is_difficult_list,
groundtruth_is_group_of_list=gt_is_group_of_list)
#scores=np.array(scores),
tp_fp_labels = np.array(tp_fp_labels)
precision, recall = metrics.compute_precision_recall(
np.array(scores), tp_fp_labels[1].astype(float), 2)
print(scores)
print('---------')
print(len(tp_fp_labels))
#f_name = re.split('/',path_f)
#print(category_index.get(value))
plt.figure(figsize=IMAGE_SIZE)
plt.imshow(image_np)
def compute_precision_recall_bbox(per_image_detections,
per_image_gts,
num_gt_classes,
matching_iou_threshold=0.5):
"""
Compute the precision and recall at each confidence level for detection results of various classes.
Args:
per_image_detections: dict of image_id to a dict with fields `boxes`, `scores` and `labels`
per_image_gts: dict of image_id to a dict with fields `gt_boxes` and `gt_labels`
num_gt_classes: number of classes in the ground truth labels
matching_iou_threshold: IoU above which a detected and a ground truth box are considered overlapping
Returns:
A dict `per_cat_metrics`, where the keys are the possible gt classes and `one_class` which considers
all classes. Each key corresponds to a dict with the fields precision, recall, average_precision, etc.
"""
per_image_eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes=num_gt_classes,
matching_iou_threshold=matching_iou_threshold,
nms_iou_threshold=1.0,
nms_max_output_boxes=10000)
print('Running per-object analysis...')
detection_tp_fp = defaultdict(
list) # key is the category; in each list, 1 is tp, 0 is fp
detection_scores = defaultdict(list)
num_total_gt = defaultdict(int)
for image_id, dets in tqdm(per_image_detections.items()):
detected_boxes = np.array(dets['boxes'], dtype=np.float32)
detected_scores = np.array(dets['scores'], dtype=np.float32)
# labels input to compute_object_detection_metrics() needs to start at 0, not 1
detected_labels = np.array(dets['labels'],
dtype=np.int) - 1 # start at 0
# num_detections = len(dets['boxes'])
gts = per_image_gts[image_id]
gt_boxes = np.array(gts['gt_boxes'], dtype=np.float32)
gt_labels = np.array(gts['gt_labels'], dtype=np.int) - 1 # start at 0
num_gts = len(gts['gt_boxes'])
groundtruth_is_difficult_list = np.zeros(
num_gts, dtype=bool) # place holders - we don't have these
groundtruth_is_group_of_list = np.zeros(num_gts, dtype=bool)
# to prevent 'Invalid dimensions for box data.' error
if num_gts == 0:
# this box will not match any detections
gt_boxes = np.array([[0, 0, 0, 0]], dtype=np.float32)
scores, tp_fp_labels, is_class_correctly_detected_in_image = (
per_image_eval.compute_object_detection_metrics(
detected_boxes=detected_boxes,
detected_scores=detected_scores,
detected_class_labels=detected_labels,
groundtruth_boxes=gt_boxes,
groundtruth_class_labels=gt_labels,
groundtruth_is_difficult_list=groundtruth_is_difficult_list,
groundtruth_is_group_of_list=groundtruth_is_group_of_list))
for i, tp_fp_labels_cat in enumerate(tp_fp_labels):
assert sum(tp_fp_labels_cat) <= sum(
gt_labels == i) # true positives < gt of that category
cat = i + 1 # categories start at 1
detection_tp_fp[cat].append(tp_fp_labels_cat)
detection_scores[cat].append(scores[i])
num_total_gt[cat] += sum(gt_labels == i) # gt_labels start at 0
all_scores = []
all_tp_fp = []
print('Computing precision recall for each category...')
per_cat_metrics = {}
for i in range(num_gt_classes):
cat = i + 1
scores_cat = np.concatenate(detection_scores[cat])
tp_fp_cat = np.concatenate(detection_tp_fp[cat]).astype(np.bool)
all_scores.append(scores_cat)
all_tp_fp.append(tp_fp_cat)
precision, recall = metrics.compute_precision_recall(
scores_cat, tp_fp_cat, num_total_gt[cat])
average_precision = metrics.compute_average_precision(
precision, recall)
per_cat_metrics[cat] = {
'category': cat,
'precision': precision,
'recall': recall,
'average_precision': average_precision,
'scores': scores_cat,
'tp_fp': tp_fp_cat,
'num_gt': num_total_gt[cat]
}
print('Number of ground truth in category {} is {}'.format(
cat, num_total_gt[cat]))
# compute one-class precision/recall/average precision (if every box is just of an object class)
all_scores = np.concatenate(all_scores)
all_tp_fp = np.concatenate(all_tp_fp)
overall_gt_count = sum(num_total_gt.values())
one_class_prec, one_class_recall = metrics.compute_precision_recall(
all_scores, all_tp_fp, overall_gt_count)
one_class_average_precision = metrics.compute_average_precision(
one_class_prec, one_class_recall)
per_cat_metrics['one_class'] = {
'category': 'one_class',
'precision': one_class_prec,
'recall': one_class_recall,
'average_precision': one_class_average_precision,
'scores': all_scores,
'tp_fp': all_tp_fp,
'num_gt': overall_gt_count
}
return per_cat_metrics
def compute_precision_recall_with_sequences(detection_file, db_file,detection_results=None,images_to_consider='all', get_night_day = None):
if detection_results == None:
print('Loading detection file...')
with open(detection_file) as f:
detection_results = pickle.load(f)
im_to_seq = get_im_to_seq_map(db_file)
seqs = {}
for im in detection_results['images']:
if im in im_to_seq:
if im_to_seq[im] not in seqs:
seqs[im_to_seq[im]] = []
seqs[im_to_seq[im]].append(im)
print('Clustering detections by image...')
use_im = get_images_to_consider(detection_results, images_to_consider, get_night_day)
per_image_detections, per_image_gts = cluster_detections_by_image(detection_results, use_im)
per_image_eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes=1,
matching_iou_threshold=0.5,
nms_iou_threshold=1.0,
nms_max_output_boxes=10000
)
print('Running per-image analysis...')
detection_labels = []
detection_scores = []
num_total_gts = 0
count = 0
for seq in seqs:
seq_detection_labels = []
seq_detection_scores = []
seq_num_gts = []
is_gt_in_seq = False
max_seq_scores = []
valid_max_scores = []
#print(seq)
for image_id in seqs[seq]:
#for image_id, dets in per_image_detections.iteritems():
dets = per_image_detections[image_id]
num_detections = len(dets['bboxes'])
# [ymin, xmin, ymax, xmax] in absolute image coordinates.
detected_boxes = np.zeros([num_detections, 4], dtype=np.float32)
# detection scores for the boxes
detected_scores = np.zeros([num_detections], dtype=np.float32)
# 0-indexed detection classes for the boxes
detected_class_labels = np.zeros([num_detections], dtype=np.int32)
detected_masks = None
count +=1
if count % 1000 == 0:
print(str(count) + ' images complete')
for i in range(num_detections):
x1, y1, x2, y2 = dets['bboxes'][i]
detected_boxes[i] = np.array([y1, x1, y2, x2])
detected_scores[i] = dets['scores'][i]
detected_class_labels[i] = dets['labels'][i] - 1
max_seq_scores.append(np.max(detected_scores))
valid_max_scores.append(np.max(detected_scores))
box_id = np.argmax(detected_scores)
gts = per_image_gts[image_id]
num_gts = len(gts['bboxes'])
#seq_num_gts.append(num_gts)
#print(num_gts)
if num_gts > 0:
seq_num_gts.append(1)
is_gt_in_seq = True
# [ymin, xmin, ymax, xmax] in absolute image coordinates
groundtruth_boxes = np.zeros([num_gts, 4], dtype=np.float32)
# 0-indexed groundtruth classes for the boxes
groundtruth_class_labels = np.zeros(num_gts, dtype=np.int32)
groundtruth_masks = None
groundtruth_is_difficult_list = np.zeros(num_gts, dtype=bool)
groundtruth_is_group_of_list = np.zeros(num_gts, dtype=bool)
for i in range(num_gts):
x1, y1, x2, y2 = gts['bboxes'][i]
groundtruth_boxes[i] = np.array([y1, x1, y2, x2])
groundtruth_class_labels[i] = gts['labels'][i] - 1
ious = np_box_ops.iou(detected_boxes,groundtruth_boxes)
if np.max(ious[box_id, :]) < 0.5:
valid_max_scores[-1] = 0
scores, tp_fp_labels, is_class_correctly_detected_in_image = (
per_image_eval.compute_object_detection_metrics(
detected_boxes=detected_boxes,
detected_scores=detected_scores,
detected_class_labels=detected_class_labels,
groundtruth_boxes=groundtruth_boxes,
groundtruth_class_labels=groundtruth_class_labels,
groundtruth_is_difficult_list=groundtruth_is_difficult_list,
groundtruth_is_group_of_list=groundtruth_is_group_of_list,
detected_masks=detected_masks,
groundtruth_masks=groundtruth_masks
)
)
seq_detection_labels.append(tp_fp_labels[0])
seq_detection_scores.append(scores[0])
#num_total_gts += 1
else:
seq_num_gts.append(0)
seq_detection_labels.append(np.zeros(num_detections, dtype=np.int32))
seq_detection_scores.append(detected_scores)
valid_max_scores[-1] = 0
seq_detection_label = np.zeros(1, dtype=np.int32)
seq_detection_score = np.zeros(1, dtype=np.float32)
best_score = np.max(valid_max_scores)
if best_score > 0:
if not is_gt_in_seq:
print(is_gt_in_seq)
print('matched box with no gt')
print(valid_max_scores)
#print('valid box')
best_im = np.argmax(max_seq_scores)
#print(best_im, best_score)
for i in range(len(seqs[seq])):
temp_labels = np.zeros(len(seq_detection_labels[i]), dtype=np.int32)
temp_scores = np.zeros(len(seq_detection_scores[i]), dtype=np.float32)
for j in range(min(seq_num_gts[i], len(temp_labels))):
temp_labels[j] = True #TODO: this currently only works for oneclass?
temp_scores[j] = best_score
seq_detection_labels[i] = temp_labels
seq_detection_scores[i] = temp_scores
seq_detection_label[0] = True
seq_detection_score[0] = best_score
else:
#print('no valid box')
seq_detection_label[0] = False
seq_detection_score[0] = np.max(max_seq_scores)
#if sum(seq_num_gts)>0:
if is_gt_in_seq:
num_total_gts+=1
detection_labels.append(seq_detection_label)
detection_scores.append(seq_detection_score)
scores = np.concatenate(detection_scores)
labels = np.concatenate(detection_labels).astype(np.bool)
print(count)
print(len(seqs.keys()))
print(sum([1 for i in range(len(detection_labels)) if detection_labels[i] == True]), num_total_gts)
precision, recall = metrics.compute_precision_recall(
scores, labels, num_total_gts
)
average_precision = metrics.compute_average_precision(precision, recall)
return precision, recall, average_precision
def compute_precision_recall_per_loc(detection_file, db_file):
print('Loading detection file...')
with open(detection_file) as f:
detection_results = pickle.load(f)
with open(db_file, 'r') as f:
data = json.load(f)
print('Images: ', len(data['images']))
print('Detection result Images: ', len(detection_results['images']))
loc_to_ims = {}
for im in data['images']:
if im['location'] not in loc_to_ims:
loc_to_ims[im['location']] = []
loc_to_ims[im['location']].append(im['id'])
print('Clustering detections by image...')
#print(detection_results.keys())
# group the detections and gts by image id:
per_image_detections, per_image_gts = cluster_detections_by_image(
detection_results)
per_image_eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes=1,
matching_iou_threshold=0.5,
nms_iou_threshold=1.0,
nms_max_output_boxes=10000)
detection_labels = {loc: [] for loc in loc_to_ims}
detection_scores = {loc: [] for loc in loc_to_ims}
num_total_gts = {loc: 0 for loc in loc_to_ims}
count = {loc: 0 for loc in loc_to_ims}
precision = {}
recall = {}
average_precision = {}
for cat, images in loc_to_ims.iteritems():
for image_id in images:
if image_id not in per_image_detections:
#print(image_id)
count[cat] += 1
continue
scores, tp_fp_labels = get_results_per_image(
per_image_detections[image_id], per_image_gts[image_id],
per_image_eval)
detection_labels[cat].append(tp_fp_labels)
detection_scores[cat].append(scores)
num_gts = len(per_image_gts[image_id]['bboxes'])
num_total_gts[cat] += num_gts
if len(detection_scores[cat]) > 0:
scores = np.concatenate(detection_scores[cat])
labels = np.concatenate(detection_labels[cat]).astype(np.bool)
#print(len(scores))
#print(len(labels))
precision[cat], recall[cat] = metrics.compute_precision_recall(
scores, labels, num_total_gts[cat])
average_precision[cat] = metrics.compute_average_precision(
precision[cat], recall[cat])
else:
print("no detections for " + cat)
print(cat, count[cat], len(images))
return precision, recall, average_precision
def get_detect_info(self, image_path, threshold):
jpg_file_name, jpg_file_path = get_file_name_path(image_path,
format_key=".jpg")
with self.detection_graph.as_default():
with tf.Session(graph=self.detection_graph) as sess:
# Expand dimensions since the model expects images to have shape: [1, None, None, 3]
computer_label = 3
# print(jpg_file_path)
info_list = []
for fp, i in zip(jpg_file_path, range(len(jpg_file_path))):
image = cv2.imread(fp)
# size = np.shape(image)
height, width, depth = image.shape
print(height, width, depth)
groundtruth_boxes = []
xml_info = read_xml(fp[:-4] + ".xml")
for x in range(len(xml_info)):
print(xml_info[x][1])
if int(xml_info[x][1]) == computer_label:
groundtruth_boxes.append(xml_info[x][2:])
print(groundtruth_boxes)
# print(size)
image_np_expanded = np.expand_dims(image, axis=0)
image_tensor = self.detection_graph.get_tensor_by_name(
'image_tensor:0')
boxes = self.detection_graph.get_tensor_by_name(
'detection_boxes:0')
scores = self.detection_graph.get_tensor_by_name(
'detection_scores:0')
classes = self.detection_graph.get_tensor_by_name(
'detection_classes:0')
num_detections = self.detection_graph.get_tensor_by_name(
'num_detections:0')
# Actual detection.
(boxes, scores, classes, num_detections) = sess.run(
[boxes, scores, classes, num_detections],
feed_dict={image_tensor: image_np_expanded})
py_scores = np.array(scores[0])
py_classes = np.array(classes[0])
py_boxes = np.array(boxes[0])
scores = np.squeeze(scores)
classes = np.squeeze(classes)
groundtruth_groundtruth_is_difficult_list = np.array(
np.ones(len(groundtruth_boxes)), dtype=bool)
groundtruth_groundtruth_is_group_of_list = np.array(
np.ones(len(groundtruth_boxes)), dtype=bool)
#print("=====:", groundtruth_groundtruth_is_group_of_list)
detected_boxes = []
detected_scores = []
for num in range(len(py_scores)):
# print(py_scores[0])
#print(num)
if classes[num] == computer_label:
#print(py_boxes[num])
boxes = [
float(py_boxes[num][0] * height),
float(py_boxes[num][1] * width),
float(py_boxes[num][2] * height),
float(py_boxes[num][3] * width)
]
detected_boxes.append(boxes)
detected_scores.append(scores[num])
#print(detected_boxes,"\n", ":", detected_scores)
print("++++++++++++++++++++++++++++++")
num_groundtruth_classes = 3
matching_iou_threshold = 0.5
nms_iou_threshold = 1.0
nms_max_output_boxes = 10000
group_of_weight = 0.5
eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes, matching_iou_threshold,
nms_iou_threshold, nms_max_output_boxes,
group_of_weight)
scores, tp_fp_labels = eval._compute_tp_fp_for_single_class(
np.array(detected_boxes), np.array(detected_scores),
np.array(groundtruth_boxes),
groundtruth_groundtruth_is_difficult_list,
groundtruth_groundtruth_is_group_of_list)
print(scores, "\n", tp_fp_labels)
def compute_precision_recall_bbox(
per_image_detections: Mapping[str, Mapping[str, Any]],
per_image_gts: Mapping[str, Mapping[str, Any]],
num_gt_classes: int,
matching_iou_threshold: float = 0.5
) -> Dict[Union[str, int], Dict[str, Any]]:
"""
Compute the precision and recall at each confidence level for detection
results of various classes.
Args:
per_image_detections: dict, image_id (str) => dict with fields
'boxes': array-like, shape [N, 4], type float, each row is
[ymin, xmin, ymax, xmax] in normalized coordinates
'scores': array-like, shape [N], float
'labels': array-like, shape [N], integers in [1, num_gt_classes]
per_image_gts: dic, image_id (str) => dict with fields
'gt_boxes': array-like, shape [M, 4], type float, each row is
[ymin, xmin, ymax, xmax] in normalized coordinates
'gt_labels': array-like, shape [M], integers in [1, num_gt_classes]
num_gt_classes: int, number of classes in the ground truth labels
matching_iou_threshold: float, IoU above which a detected and a ground
truth box are considered overlapping
Returns: dict, per-class metrics, keys are integers in [1, num_gt_classes]
and 'one_class' which considers all classes. Each value is a dict with
fields ['precision', 'recall', 'average_precision', ...]
"""
per_image_eval = per_image_evaluation.PerImageEvaluation(
num_groundtruth_classes=num_gt_classes,
matching_iou_threshold=matching_iou_threshold,
nms_iou_threshold=1.0,
nms_max_output_boxes=10000)
print('Running per-object analysis...', flush=True)
# keys are categories (int)
detection_tp_fp = defaultdict(list) # in each list, 1 is tp, 0 is fp
detection_scores = defaultdict(list)
num_total_gt: Dict[int, int] = defaultdict(int)
for image_id, dets in tqdm(per_image_detections.items()):
# we force *_boxes to have shape [N, 4], even in case that N = 0
detected_boxes = np.asarray(dets['boxes'],
dtype=np.float32).reshape(-1, 4)
detected_scores = np.asarray(dets['scores'])
# labels input to compute_object_detection_metrics() needs to start at 0, not 1
detected_labels = np.asarray(dets['labels'],
dtype=np.int) - 1 # start at 0
# num_detections = len(dets['boxes'])
gts = per_image_gts[image_id]
gt_boxes = np.asarray(gts['gt_boxes'], dtype=np.float32).reshape(-1, 4)
gt_labels = np.asarray(gts['gt_labels'],
dtype=np.int) - 1 # start at 0
num_gts = len(gts['gt_boxes'])
# place holders - we don't have these
groundtruth_is_difficult_list = np.zeros(num_gts, dtype=bool)
groundtruth_is_group_of_list = np.zeros(num_gts, dtype=bool)
results = per_image_eval.compute_object_detection_metrics(
detected_boxes=detected_boxes,
detected_scores=detected_scores,
detected_class_labels=detected_labels,
groundtruth_boxes=gt_boxes,
groundtruth_class_labels=gt_labels,
groundtruth_is_difficult_list=groundtruth_is_difficult_list,
groundtruth_is_group_of_list=groundtruth_is_group_of_list)
scores, tp_fp_labels, is_class_correctly_detected_in_image = results
for i, tp_fp_labels_cat in enumerate(tp_fp_labels):
# true positives < gt of that category
assert sum(tp_fp_labels_cat) <= sum(gt_labels == i)
cat = i + 1 # categories start at 1
detection_tp_fp[cat].append(tp_fp_labels_cat)
detection_scores[cat].append(scores[i])
num_total_gt[cat] += sum(gt_labels == i) # gt_labels start at 0
all_scores = []
all_tp_fp = []
print('Computing precision recall for each category...')
per_cat_metrics: Dict[Union[int, str], Dict[str, Any]] = {}
for i in range(num_gt_classes):
cat = i + 1
scores_cat = np.concatenate(detection_scores[cat])
tp_fp_cat = np.concatenate(detection_tp_fp[cat]).astype(np.bool)
all_scores.append(scores_cat)
all_tp_fp.append(tp_fp_cat)
precision, recall = metrics.compute_precision_recall(
scores_cat, tp_fp_cat, num_total_gt[cat])
average_precision = metrics.compute_average_precision(
precision, recall)
per_cat_metrics[cat] = {
'category': cat,
'precision': precision,
'recall': recall,
'average_precision': average_precision,
'scores': scores_cat,
'tp_fp': tp_fp_cat,
'num_gt': num_total_gt[cat]
}
print(f'Number of ground truth in category {cat}: {num_total_gt[cat]}')
# compute one-class precision/recall/average precision (if every box is just
# of an object class)
all_scores = np.concatenate(all_scores)
all_tp_fp = np.concatenate(all_tp_fp)
overall_gt_count = sum(num_total_gt.values())
one_class_prec, one_class_recall = metrics.compute_precision_recall(
all_scores, all_tp_fp, overall_gt_count)
one_class_average_precision = metrics.compute_average_precision(
one_class_prec, one_class_recall)
per_cat_metrics['one_class'] = {
'category': 'one_class',
'precision': one_class_prec,
'recall': one_class_recall,
'average_precision': one_class_average_precision,
'scores': all_scores,
'tp_fp': all_tp_fp,
'num_gt': overall_gt_count
}
return per_cat_metrics