def graph_fn():
corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
[0.0, 0.0, 20.0, 20.0]])
boxes1 = box_list.BoxList(corners1)
boxes2 = box_list.BoxList(corners2)
boxes_empty = box_list.BoxList(tf.zeros((0, 4)))
iou_empty_1 = box_list_ops.iou(boxes1, boxes_empty)
iou_empty_2 = box_list_ops.iou(boxes_empty, boxes2)
iou_empty_3 = box_list_ops.iou(boxes_empty, boxes_empty)
return iou_empty_1, iou_empty_2, iou_empty_3
def test_iouworks_on_empty_inputs(self):
corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
[0.0, 0.0, 20.0, 20.0]])
boxes1 = box_list.BoxList(corners1)
boxes2 = box_list.BoxList(corners2)
boxes_empty = box_list.BoxList(tf.zeros((0, 4)))
iou_empty_1 = box_list_ops.iou(boxes1, boxes_empty)
iou_empty_2 = box_list_ops.iou(boxes_empty, boxes2)
iou_empty_3 = box_list_ops.iou(boxes_empty, boxes_empty)
with self.test_session() as sess:
iou_output_1, iou_output_2, iou_output_3 = sess.run(
[iou_empty_1, iou_empty_2, iou_empty_3])
self.assertAllEqual(iou_output_1.shape, (2, 0))
self.assertAllEqual(iou_output_2.shape, (0, 3))
self.assertAllEqual(iou_output_3.shape, (0, 0))
def match_and_select_feature(groundtruth_boxes, anchors, feature_maps):
""" Select features on the groundtruth box locations
Args:
groundtruth_boxes: a tensor of batch_size x 4
anchors: generated anchor Box list
feature_maps: a list of feature pyramid, each element is a
tensor of batch_size x height_i x width_i x channel
Returns:
selected_features: a tensor of batch_size x 1 x 1 x channel
"""
groundtruth_boxeslists = [
box_list.BoxList(tf.expand_dims(box, 0))
for box in tf.unstack(groundtruth_boxes, axis=0)
]
feature_maps = [
tf.reshape(m, [m.get_shape()[0].value, -1,
m.get_shape()[-1].value]) for m in feature_maps
]
feature_maps = tf.unstack(tf.concat(feature_maps, axis=1), axis=0)
num_anchors_per_location = anchors.get().get_shape(
)[0].value / feature_maps[0].get_shape()[0].value
selected_feature = list()
for groundtruth_boxes, feature_map in zip(groundtruth_boxeslists,
feature_maps):
iou = box_list_ops.iou(groundtruth_boxes, anchors)
max_ind = tf.argmax(iou, axis=1) / num_anchors_per_location
selected_feature.append(tf.gather(feature_map, max_ind))
selected_feature = tf.concat(selected_feature, axis=0)
selected_feature = tf.expand_dims(tf.expand_dims(selected_feature, axis=1),
axis=1)
return selected_feature
def test_iouworks_on_empty_inputs(self):
corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
[0.0, 0.0, 20.0, 20.0]])
boxes1 = box_list.BoxList(corners1)
boxes2 = box_list.BoxList(corners2)
boxes_empty = box_list.BoxList(tf.zeros((0, 4)))
iou_empty_1 = box_list_ops.iou(boxes1, boxes_empty)
iou_empty_2 = box_list_ops.iou(boxes_empty, boxes2)
iou_empty_3 = box_list_ops.iou(boxes_empty, boxes_empty)
with self.test_session() as sess:
iou_output_1, iou_output_2, iou_output_3 = sess.run(
[iou_empty_1, iou_empty_2, iou_empty_3])
self.assertAllEqual(iou_output_1.shape, (2, 0))
self.assertAllEqual(iou_output_2.shape, (0, 3))
self.assertAllEqual(iou_output_3.shape, (0, 0))
def _compare(self, boxlist1, boxlist2):
"""Compute pairwise IOU similarity between the two BoxLists.
Args:
boxlist1: BoxList holding N boxes.
boxlist2: BoxList holding M boxes.
Returns:
A tensor with shape [N, M] representing pairwise iou scores.
"""
return box_list_ops.iou(boxlist1, boxlist2)
def graph_fn():
corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
[0.0, 0.0, 20.0, 20.0]])
boxes1 = box_list.BoxList(corners1)
boxes2 = box_list.BoxList(corners2)
iou = box_list_ops.iou(boxes1, boxes2)
return iou
def test_iou(self):
corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
[0.0, 0.0, 20.0, 20.0]])
exp_output = [[2.0 / 16.0, 0, 6.0 / 400.0], [1.0 / 16.0, 0.0, 5.0 / 400.0]]
boxes1 = box_list.BoxList(corners1)
boxes2 = box_list.BoxList(corners2)
iou = box_list_ops.iou(boxes1, boxes2)
with self.test_session() as sess:
iou_output = sess.run(iou)
self.assertAllClose(iou_output, exp_output)
def _compare(self, boxlist1, boxlist2):
"""Compute pairwise IOU similarity between the two BoxLists.
Args:
boxlist1: BoxList holding N boxes.
boxlist2: BoxList holding M boxes.
Returns:
A tensor with shape [N, M] representing pairwise iou scores.
"""
return box_list_ops.iou(boxlist1, boxlist2)
def graph_fn():
boxes = box_list.BoxList(
tf.constant([[0.1, 0.1, 0.4, 0.4],
[0.1, 0.1, 0.5, 0.5],
[0.6, 0.6, 0.8, 0.8],
[0.2, 0.2, 0.3, 0.3]], tf.float32))
sampled_boxes = box_list_ops.sample_boxes_by_jittering(
boxlist=boxes, num_boxes_to_sample=10)
iou = box_list_ops.iou(boxes, sampled_boxes)
iou_max = tf.reduce_max(iou, axis=0)
return sampled_boxes.get(), iou_max
def test_iou(self):
corners1 = tf.constant([[4.0, 3.0, 7.0, 5.0], [5.0, 6.0, 10.0, 7.0]])
corners2 = tf.constant([[3.0, 4.0, 6.0, 8.0], [14.0, 14.0, 15.0, 15.0],
[0.0, 0.0, 20.0, 20.0]])
exp_output = [[2.0 / 16.0, 0, 6.0 / 400.0], [1.0 / 16.0, 0.0, 5.0 / 400.0]]
boxes1 = box_list.BoxList(corners1)
boxes2 = box_list.BoxList(corners2)
iou = box_list_ops.iou(boxes1, boxes2)
with self.test_session() as sess:
iou_output = sess.run(iou)
self.assertAllClose(iou_output, exp_output)
def test_sample_boxes_by_jittering(self):
boxes = box_list.BoxList(
tf.constant([[0.1, 0.1, 0.4, 0.4],
[0.1, 0.1, 0.5, 0.5],
[0.6, 0.6, 0.8, 0.8],
[0.2, 0.2, 0.3, 0.3]], tf.float32))
sampled_boxes = box_list_ops.sample_boxes_by_jittering(
boxlist=boxes, num_boxes_to_sample=10)
iou = box_list_ops.iou(boxes, sampled_boxes)
iou_max = tf.reduce_max(iou, axis=0)
with self.test_session() as sess:
(np_sampled_boxes, np_iou_max) = sess.run([sampled_boxes.get(), iou_max])
self.assertAllEqual(np_sampled_boxes.shape, [10, 4])
self.assertAllGreater(np_iou_max, 0.5)
def test_sample_boxes_by_jittering(self):
boxes = box_list.BoxList(
tf.constant([[0.1, 0.1, 0.4, 0.4],
[0.1, 0.1, 0.5, 0.5],
[0.6, 0.6, 0.8, 0.8],
[0.2, 0.2, 0.3, 0.3]], tf.float32))
sampled_boxes = box_list_ops.sample_boxes_by_jittering(
boxlist=boxes, num_boxes_to_sample=10)
iou = box_list_ops.iou(boxes, sampled_boxes)
iou_max = tf.reduce_max(iou, axis=0)
with self.test_session() as sess:
(np_sampled_boxes, np_iou_max) = sess.run([sampled_boxes.get(), iou_max])
self.assertAllEqual(np_sampled_boxes.shape, [10, 4])
self.assertAllGreater(np_iou_max, 0.5)
def _compare(self, boxlist1, boxlist2):
"""Compute pairwise IOU similarity between the two BoxLists and score.
Args:
boxlist1: BoxList holding N boxes. Must have a score field.
boxlist2: BoxList holding M boxes.
Returns:
A tensor with shape [N, M] representing scores threholded by pairwise
iou scores.
"""
ious = box_list_ops.iou(boxlist1, boxlist2)
scores = boxlist1.get_field(fields.BoxListFields.scores)
scores = tf.expand_dims(scores, axis=1)
row_replicated_scores = tf.tile(scores, [1, tf.shape(ious)[-1]])
thresholded_ious = tf.where(ious > self._iou_threshold,
row_replicated_scores, tf.zeros_like(ious))
return thresholded_ious
def _compare(self, boxlist1, boxlist2):
"""Compute pairwise IOU similarity between the two BoxLists and score.
Args:
boxlist1: BoxList holding N boxes. Must have a score field.
boxlist2: BoxList holding M boxes.
Returns:
A tensor with shape [N, M] representing scores threholded by pairwise
iou scores.
"""
ious = box_list_ops.iou(boxlist1, boxlist2)
scores = boxlist1.get_field(fields.BoxListFields.scores)
scores = tf.expand_dims(scores, axis=1)
row_replicated_scores = tf.tile(scores, [1, tf.shape(ious)[-1]])
thresholded_ious = tf.where(ious > self._iou_threshold,
row_replicated_scores, tf.zeros_like(ious))
return thresholded_ious
def export_boxes_csv(sess, detection_graph, data_filename,
output_filename):
columns = description_features + ['num_objects'] + [
'num_detections', 'xmin', 'ymin', 'xmax', 'ymax', 'iou',
'detected', 'label', 'difficulty'
]
def _restart_res():
return pd.DataFrame(columns=columns)
res = _restart_res()
res.to_csv(output_filename)
dataset = tf.data.TFRecordDataset(data_filename, compression_type='')
num_images_saved = 0
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
for i, data in enumerate(tqdm(dataset)):
tf_record = parse_camera_tfrecord_example(data)
features = get_features(tf_record)
image = tf.image.decode_jpeg(tf_record["image/encoded"]).numpy()
image_expanded = np.expand_dims(np.array(image), axis=0)
detection_boxes = detection_graph.get_tensor_by_name(
'detection_boxes:0')
num_detections = detection_graph.get_tensor_by_name(
'num_detections:0')
(boxes,
n_detections) = sess.run([detection_boxes, num_detections],
feed_dict={image_tensor: image_expanded})
boxes = np.squeeze(boxes)[:int(n_detections)]
gt_boxes = tf.concat([
tf.expand_dims(tf_record["image/object/bbox/ymin"], axis=1),
tf.expand_dims(tf_record["image/object/bbox/xmin"], axis=1),
tf.expand_dims(tf_record["image/object/bbox/ymax"], axis=1),
tf.expand_dims(tf_record["image/object/bbox/xmax"], axis=1)
],
axis=1)
gt_boxes_list = BoxList(gt_boxes)
boxes_list = BoxList(tf.constant(boxes))
iou_values = box_list_ops.iou(gt_boxes_list, boxes_list)
iou_values = tf.math.reduce_max(iou_values, axis=1).numpy()
features['num_detections'] = int(n_detections)
labels = tf_record["image/object/class/label"].numpy()
difficulty_ls = tf_record["image/object/difficult"].numpy()
missed_boxes = []
for box, iou, label, difficulty, xmin, ymin, xmax, ymax in zip(
gt_boxes.numpy(), iou_values, labels, difficulty_ls,
tf_record["image/object/bbox/xmin"],
tf_record["image/object/bbox/ymin"],
tf_record["image/object/bbox/xmax"],
tf_record["image/object/bbox/ymax"]):
_row = features.copy()
_row['xmin'] = xmin.numpy().astype(np.float16)
_row['ymin'] = ymin.numpy().astype(np.float16)
_row['xmax'] = xmax.numpy().astype(np.float16)
_row['ymax'] = ymax.numpy().astype(np.float16)
_row['iou'] = iou.astype(np.float16)
iou_threshold = {1: 0.7, 2: 0.5, 3: 0.5}
_row['detected'] = float(iou) >= iou_threshold[int(label)]
_row['label'] = label.astype(np.int8)
_row['difficulty'] = difficulty.astype(np.int8)
res = res.append(_row, ignore_index=True)
if num_images_saved < NUMBER_IMAGES_TO_SAVE and float(
iou) < iou_threshold[int(label)]:
missed_boxes.append([
float(xmin.numpy()),
float(ymin.numpy()),
float(xmax.numpy()),
float(ymax.numpy())
])
if i % 5 and num_images_saved < NUMBER_IMAGES_TO_SAVE and missed_boxes:
num_images_saved += 1
fig = plot_image_with_boxes(image, missed_boxes)
fig_name = tf_record["image/source_id"].numpy().decode(
) + '.png'
fig.tight_layout()
fig.savefig(os.path.join(SAVE_IMAGES_DIR, fig_name))
res.to_csv(output_filename, mode='a', header=False)
res = _restart_res()
return res
ii: decoded_boxes = self._box_coder.decode(rpn_box_encodings, box_list.BoxList(anchors))
--> faster_rcnn_box_coder.FasterRcnnBoxCoder._decode()
objectness_scores = tf.nn.softmax(rpn_objectness_predictions_with_background)
iii:proposal_boxlist = post_processing.multiclass_non_max_suppression()
iv: padded_proposals = box_list_ops.pad_or_clip_box_list()
II: self._compute_second_stage_input_feature_maps()
III: box_classifier_features = self._feature_extractor.extract_box_classifier_features()
IV: box_predictions = self._mask_rcnn_box_predictor.predict()
V: absolute_proposal_boxes = ops.normalized_to_image_coordinates()
B: losses_dict = detection_model.loss
a. _loss_rpn
1. target_assigner.batch_assign_targets()
I: target_assigner.assign()
i: match_quality_matrix = self._similarity_calc.compare(groundtruth_boxes,anchors)
-->sim_calc.IouSimilarity()-->box_list_ops.iou()
ii: match = self._matcher.match(match_quality_matrix, **params)
-->argmax_matcher.ArgMaxMatcher._match()
iii: reg_targets = self._create_regression_targets(anchors,groundtruth_boxes,match)
iv: cls_targets = self._create_classification_targets(groundtruth_labels,match)
v: reg_weights = self._create_regression_weights(match)
vi: cls_weights = self._create_classification_weights(
match, self._positive_class_weight, self._negative_class_weight)
2. localization_losses = self._first_stage_localization_loss
-->losses.WeightedSmoothL1LocalizationLoss._compute_loss()
3. objectness_losses = self._first_stage_objectness_loss
-->losses.WeightedSoftmaxClassificationLoss._compute_loss()
b. _loss_box_classifier
1. paddings_indicator = self._padded_batched_proposals_indicator
2. second_stage_loc_losses = self._second_stage_localization_loss
-->losses.WeightedSmoothL1LocalizationLoss._compute_loss()
tf.enable_eager_execution()
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
# ymin, xmin, ymax, xmax
gt_boxes_array = tf.convert_to_tensor([[0.0, 0.0, 1.0, 1.0],
[0.0, 0.0, 0.5, 0.5]])
anchors_array = tf.convert_to_tensor([[0.0, 0.0, 0.9,
0.9], [0.0, 0.0, 0.5, 0.5],
[0.0, 0.0, 0.7, 0.7],
[0.1, 0.1, 0.2, 0.2]])
#anchors = [[0.0, 0.0, 0.5, 0.5],[0.0, 0.0, 0.25, 0.25] ]
gt_boxes = BoxList(tf.convert_to_tensor(gt_boxes_array))
anchors = BoxList(tf.convert_to_tensor(anchors_array))
iou_values = box_list_ops.iou(gt_boxes, anchors)
max_iou_values = tf.math.reduce_max(iou_values, axis=1).numpy()
target_assigner = create_target_assigner('FasterRCNN',
'detection',
negative_class_weight=1.0,
use_matmul_gather=False)
# Each row is a ground truth box, and each column is an anchor (proposal)
match_quality_matrix = target_assigner._similarity_calc.compare(
gt_boxes, anchors)
match = target_assigner._matcher.match(match_quality_matrix)
cls_targets, cls_weights, reg_targets, reg_weights, match_results = \
target_assigner.assign(anchors, gt_boxes, groundtruth_labels=None,
