def _compute_is_class_correctly_detected_in_image(self,
detected_boxes,
detected_scores,
groundtruth_boxes,
detected_masks=None,
groundtruth_masks=None):
"""Compute CorLoc score for a single class.
Args:
detected_boxes: A numpy array of shape [N, 4] representing detected box
coordinates
detected_scores: A 1-d numpy array of length N representing classification
score
groundtruth_boxes: A numpy array of shape [M, 4] representing ground truth
box coordinates
detected_masks: (optional) A np.uint8 numpy array of shape
[N, height, width]. If not None, the scores will be computed based
on masks.
groundtruth_masks: (optional) A np.uint8 numpy array of shape
[M, height, width].
Returns:
is_class_correctly_detected_in_image: An integer 1 or 0 denoting whether a
class is correctly detected in the image or not
"""
if detected_boxes.size > 0:
if groundtruth_boxes.size > 0:
max_score_id = np.argmax(detected_scores)
mask_mode = False
if detected_masks is not None and groundtruth_masks is not None:
mask_mode = True
if mask_mode:
detected_boxlist = np_box_mask_list.BoxMaskList(
box_data=np.expand_dims(detected_boxes[max_score_id],
axis=0),
mask_data=np.expand_dims(detected_masks[max_score_id],
axis=0))
gt_boxlist = np_box_mask_list.BoxMaskList(
box_data=groundtruth_boxes,
mask_data=groundtruth_masks)
iou = np_box_mask_list_ops.iou(detected_boxlist,
gt_boxlist)
else:
detected_boxlist = np_box_list.BoxList(
np.expand_dims(detected_boxes[max_score_id, :],
axis=0))
gt_boxlist = np_box_list.BoxList(groundtruth_boxes)
iou = np_box_list_ops.iou(detected_boxlist, gt_boxlist)
if np.max(iou) >= self.matching_iou_threshold:
return 1
return 0
def _get_overlaps_and_scores_mask_mode(self, detected_boxes,
detected_scores, detected_masks,
groundtruth_boxes,
groundtruth_masks,
groundtruth_is_group_of_list):
"""Computes overlaps and scores between detected and groudntruth masks.
Args:
detected_boxes: A numpy array of shape [N, 4] representing detected box
coordinates
detected_scores: A 1-d numpy array of length N representing classification
score
detected_masks: A uint8 numpy array of shape [N, height, width]. If not
None, the scores will be computed based on masks.
groundtruth_boxes: A numpy array of shape [M, 4] representing ground truth
box coordinates
groundtruth_masks: A uint8 numpy array of shape [M, height, width].
groundtruth_is_group_of_list: A boolean numpy array of length M denoting
whether a ground truth box has group-of tag. If a groundtruth box
is group-of box, every detection matching this box is ignored.
Returns:
iou: A float numpy array of size [num_detected_boxes, num_gt_boxes]. If
gt_non_group_of_boxlist.num_boxes() == 0 it will be None.
ioa: A float numpy array of size [num_detected_boxes, num_gt_boxes]. If
gt_group_of_boxlist.num_boxes() == 0 it will be None.
scores: The score of the detected boxlist.
num_boxes: Number of non-maximum suppressed detected boxes.
"""
detected_boxlist = np_box_mask_list.BoxMaskList(
box_data=detected_boxes, mask_data=detected_masks)
detected_boxlist.add_field('scores', detected_scores)
detected_boxlist = np_box_mask_list_ops.non_max_suppression(
detected_boxlist, self.nms_max_output_boxes,
self.nms_iou_threshold)
gt_non_group_of_boxlist = np_box_mask_list.BoxMaskList(
box_data=groundtruth_boxes[~groundtruth_is_group_of_list],
mask_data=groundtruth_masks[~groundtruth_is_group_of_list])
gt_group_of_boxlist = np_box_mask_list.BoxMaskList(
box_data=groundtruth_boxes[groundtruth_is_group_of_list],
mask_data=groundtruth_masks[groundtruth_is_group_of_list])
iou = np_box_mask_list_ops.iou(detected_boxlist,
gt_non_group_of_boxlist)
ioa = np.transpose(
np_box_mask_list_ops.ioa(gt_group_of_boxlist, detected_boxlist))
scores = detected_boxlist.get_field('scores')
num_boxes = detected_boxlist.num_boxes()
return iou, ioa, scores, num_boxes
def test_has_field_with_nonexisted_field(self):
boxes = np.array([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
[0.0, 0.0, 20.0, 20.0]],
dtype=float)
box_mask_list = np_box_mask_list.BoxMaskList(
box_data=boxes, mask_data=np.zeros([3, 3, 3], dtype=np.uint8))
self.assertFalse(box_mask_list.has_field('scores'))
def test_num_boxes(self):
boxes = np.array([[0., 0., 100., 100.], [10., 30., 50., 70.]], dtype=float)
masks = np.zeros([2, 5, 5], dtype=np.uint8)
box_mask_list = np_box_mask_list.BoxMaskList(
box_data=boxes, mask_data=masks)
expected_num_boxes = 2
self.assertEquals(box_mask_list.num_boxes(), expected_num_boxes)
def setUp(self):
boxes = np.array([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
[0.0, 0.0, 20.0, 20.0]],
dtype=float)
masks = np.zeros([3, 3, 3], dtype=np.uint8)
self.box_mask_list = np_box_mask_list.BoxMaskList(
box_data=boxes, mask_data=masks)
def test_get_field_with_nonexited_field(self):
boxes = np.array([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
[0.0, 0.0, 20.0, 20.0]],
dtype=float)
masks = np.zeros([3, 3, 3], dtype=np.uint8)
box_mask_list = np_box_mask_list.BoxMaskList(
box_data=boxes, mask_data=masks)
with self.assertRaises(ValueError):
box_mask_list.get_field('scores')
def test_get_field_with_existed_field(self):
boxes = np.array([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
[0.0, 0.0, 20.0, 20.0]],
dtype=float)
masks = np.zeros([3, 3, 3], dtype=np.uint8)
box_mask_list = np_box_mask_list.BoxMaskList(
box_data=boxes, mask_data=masks)
self.assertTrue(np.allclose(box_mask_list.get_field('boxes'), boxes))
self.assertTrue(np.allclose(box_mask_list.get_field('masks'), masks))
def test_with_no_scores_field(self):
box_mask_list = np_box_mask_list.BoxMaskList(box_data=self.boxes1,
mask_data=self.masks1)
max_output_size = 3
iou_threshold = 0.5
with self.assertRaises(ValueError):
np_box_mask_list_ops.non_max_suppression(box_mask_list,
max_output_size,
iou_threshold)
def test_nms_disabled_max_output_size_equals_one(self):
box_mask_list = np_box_mask_list.BoxMaskList(box_data=self.boxes2,
mask_data=self.masks2)
box_mask_list.add_field('scores', np.array([.9, .75, .6], dtype=float))
max_output_size = 1
iou_threshold = 1. # No NMS
expected_boxes = np.array([[3.0, 4.0, 6.0, 8.0]], dtype=float)
expected_masks = np.array([[[0, 1, 0], [1, 1, 1], [0, 0, 0]]],
dtype=np.uint8)
nms_box_mask_list = np_box_mask_list_ops.non_max_suppression(
box_mask_list, max_output_size, iou_threshold)
self.assertAllClose(nms_box_mask_list.get(), expected_boxes)
self.assertAllClose(nms_box_mask_list.get_masks(), expected_masks)
def setUp(self):
boxes1 = np.array([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]],
dtype=float)
masks1_0 = np.array(
[[0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 0, 0, 0, 0],
[1, 1, 1, 1, 0, 0, 0, 0]],
dtype=np.uint8)
masks1_1 = np.array(
[[1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0]],
dtype=np.uint8)
masks1 = np.stack([masks1_0, masks1_1])
boxes2 = np.array([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
[0.0, 0.0, 20.0, 20.0]],
dtype=float)
masks2_0 = np.array(
[[0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 0, 0, 0, 0],
[1, 1, 1, 1, 0, 0, 0, 0]],
dtype=np.uint8)
masks2_1 = np.array(
[[1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 0, 0, 0],
[1, 1, 1, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0]],
dtype=np.uint8)
masks2_2 = np.array(
[[1, 1, 1, 1, 1, 0, 0, 0], [1, 1, 1, 1, 1, 0, 0, 0],
[1, 1, 1, 1, 1, 0, 0, 0], [1, 1, 1, 1, 1, 0, 0, 0],
[1, 1, 1, 1, 1, 0, 0, 0]],
dtype=np.uint8)
masks2 = np.stack([masks2_0, masks2_1, masks2_2])
self.box_mask_list1 = np_box_mask_list.BoxMaskList(box_data=boxes1,
mask_data=masks1)
self.box_mask_list2 = np_box_mask_list.BoxMaskList(box_data=boxes2,
mask_data=masks2)
def test_invalid_box_mask_data(self):
with self.assertRaises(ValueError):
np_box_mask_list.BoxMaskList(
box_data=[0, 0, 1, 1],
mask_data=np.zeros([1, 3, 3], dtype=np.uint8))
with self.assertRaises(ValueError):
np_box_mask_list.BoxMaskList(
box_data=np.array([[0, 0, 1, 1]], dtype=int),
mask_data=np.zeros([1, 3, 3], dtype=np.uint8))
with self.assertRaises(ValueError):
np_box_mask_list.BoxMaskList(
box_data=np.array([0, 1, 1, 3, 4], dtype=float),
mask_data=np.zeros([1, 3, 3], dtype=np.uint8))
with self.assertRaises(ValueError):
np_box_mask_list.BoxMaskList(
box_data=np.array([[0, 1, 1, 3], [3, 1, 1, 5]], dtype=float),
mask_data=np.zeros([2, 3, 3], dtype=np.uint8))
with self.assertRaises(ValueError):
np_box_mask_list.BoxMaskList(
box_data=np.array([[0, 1, 1, 3], [1, 1, 1, 5]], dtype=float),
mask_data=np.zeros([3, 5, 5], dtype=np.uint8))
with self.assertRaises(ValueError):
np_box_mask_list.BoxMaskList(
box_data=np.array([[0, 1, 1, 3], [1, 1, 1, 5]], dtype=float),
mask_data=np.zeros([2, 5], dtype=np.uint8))
with self.assertRaises(ValueError):
np_box_mask_list.BoxMaskList(
box_data=np.array([[0, 1, 1, 3], [1, 1, 1, 5]], dtype=float),
mask_data=np.zeros([2, 5, 5, 5], dtype=np.uint8))
with self.assertRaises(ValueError):
np_box_mask_list.BoxMaskList(
box_data=np.array([[0, 1, 1, 3], [1, 1, 1, 5]], dtype=float),
mask_data=np.zeros([2, 5, 5], dtype=np.int32))
def box_list_to_box_mask_list(boxlist):
"""Converts a BoxList containing 'masks' into a BoxMaskList.
Args:
boxlist: An np_box_list.BoxList object.
Returns:
An np_box_mask_list.BoxMaskList object.
Raises:
ValueError: If boxlist does not contain `masks` as a field.
"""
if not boxlist.has_field('masks'):
raise ValueError('boxlist does not contain mask field.')
box_mask_list = np_box_mask_list.BoxMaskList(
box_data=boxlist.get(), mask_data=boxlist.get_field('masks'))
extra_fields = boxlist.get_extra_fields()
for key in extra_fields:
if key != 'masks':
box_mask_list.data[key] = boxlist.get_field(key)
return box_mask_list
def test_multiclass_nms(self):
boxes = np.array(
[[0.2, 0.4, 0.8, 0.8], [0.4, 0.2, 0.8, 0.8], [0.6, 0.0, 1.0, 1.0]],
dtype=np.float32)
mask0 = np.array([[0, 0, 0, 0, 0], [0, 0, 1, 1, 0], [0, 0, 1, 1, 0],
[0, 0, 1, 1, 0], [0, 0, 0, 0, 0]],
dtype=np.uint8)
mask1 = np.array([[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 1, 1, 1, 0],
[0, 1, 1, 1, 0], [0, 0, 0, 0, 0]],
dtype=np.uint8)
mask2 = np.array([[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0],
[1, 1, 1, 1, 1], [1, 1, 1, 1, 1]],
dtype=np.uint8)
masks = np.stack([mask0, mask1, mask2])
box_mask_list = np_box_mask_list.BoxMaskList(box_data=boxes,
mask_data=masks)
scores = np.array(
[[-0.2, 0.1, 0.5, -0.4, 0.3], [0.7, -0.7, 0.6, 0.2, -0.9],
[0.4, 0.34, -0.9, 0.2, 0.31]],
dtype=np.float32)
box_mask_list.add_field('scores', scores)
box_mask_list_clean = np_box_mask_list_ops.multi_class_non_max_suppression(
box_mask_list,
score_thresh=0.25,
iou_thresh=0.1,
max_output_size=3)
scores_clean = box_mask_list_clean.get_field('scores')
classes_clean = box_mask_list_clean.get_field('classes')
boxes = box_mask_list_clean.get()
masks = box_mask_list_clean.get_masks()
expected_scores = np.array([0.7, 0.6, 0.34, 0.31])
expected_classes = np.array([0, 2, 1, 4])
expected_boxes = np.array([[0.4, 0.2, 0.8, 0.8], [0.4, 0.2, 0.8, 0.8],
[0.6, 0.0, 1.0, 1.0], [0.6, 0.0, 1.0, 1.0]],
dtype=np.float32)
self.assertAllClose(scores_clean, expected_scores)
self.assertAllClose(classes_clean, expected_classes)
self.assertAllClose(boxes, expected_boxes)
def multi_class_non_max_suppression(box_mask_list, score_thresh, iou_thresh,
max_output_size):
"""Multi-class version of non maximum suppression.
This op greedily selects a subset of detection bounding boxes, pruning
away boxes that have high IOU (intersection over union) overlap (> thresh)
with already selected boxes. It operates independently for each class for
which scores are provided (via the scores field of the input box_list),
pruning boxes with score less than a provided threshold prior to
applying NMS.
Args:
box_mask_list: np_box_mask_list.BoxMaskList holding N boxes. Must contain a
'scores' field representing detection scores. This scores field is a
tensor that can be 1 dimensional (in the case of a single class) or
2-dimensional, in which case we assume that it takes the
shape [num_boxes, num_classes]. We further assume that this rank is known
statically and that scores.shape[1] is also known (i.e., the number of
classes is fixed and known at graph construction time).
score_thresh: scalar threshold for score (low scoring boxes are removed).
iou_thresh: scalar threshold for IOU (boxes that that high IOU overlap
with previously selected boxes are removed).
max_output_size: maximum number of retained boxes per class.
Returns:
a box_mask_list holding M boxes with a rank-1 scores field representing
corresponding scores for each box with scores sorted in decreasing order
and a rank-1 classes field representing a class label for each box.
Raises:
ValueError: if iou_thresh is not in [0, 1] or if input box_mask_list does
not have a valid scores field.
"""
if not 0 <= iou_thresh <= 1.0:
raise ValueError('thresh must be between 0 and 1')
if not isinstance(box_mask_list, np_box_mask_list.BoxMaskList):
raise ValueError('box_mask_list must be a box_mask_list')
if not box_mask_list.has_field('scores'):
raise ValueError('input box_mask_list must have \'scores\' field')
scores = box_mask_list.get_field('scores')
if len(scores.shape) == 1:
scores = np.reshape(scores, [-1, 1])
elif len(scores.shape) == 2:
if scores.shape[1] is None:
raise ValueError(
'scores field must have statically defined second '
'dimension')
else:
raise ValueError('scores field must be of rank 1 or 2')
num_boxes = box_mask_list.num_boxes()
num_scores = scores.shape[0]
num_classes = scores.shape[1]
if num_boxes != num_scores:
raise ValueError('Incorrect scores field length: actual vs expected.')
selected_boxes_list = []
for class_idx in range(num_classes):
box_mask_list_and_class_scores = np_box_mask_list.BoxMaskList(
box_data=box_mask_list.get(), mask_data=box_mask_list.get_masks())
class_scores = np.reshape(scores[0:num_scores, class_idx], [-1])
box_mask_list_and_class_scores.add_field('scores', class_scores)
box_mask_list_filt = filter_scores_greater_than(
box_mask_list_and_class_scores, score_thresh)
nms_result = non_max_suppression(box_mask_list_filt,
max_output_size=max_output_size,
iou_threshold=iou_thresh,
score_threshold=score_thresh)
nms_result.add_field(
'classes',
np.zeros_like(nms_result.get_field('scores')) + class_idx)
selected_boxes_list.append(nms_result)
selected_boxes = np_box_list_ops.concatenate(selected_boxes_list)
sorted_boxes = np_box_list_ops.sort_by_field(selected_boxes, 'scores')
return box_list_to_box_mask_list(boxlist=sorted_boxes)