def get_feature_map_spatial_dims(feature_maps):
"""Return list of spatial dimensions for each feature map in a list.
Args:
feature_maps: a list of tensors where the ith tensor has shape
[batch, height_i, width_i, depth_i].
Returns:
a list of pairs (height, width) for each feature map in feature_maps
"""
feature_map_shapes = [
shape_utils.combined_static_and_dynamic_shape(feature_map)
for feature_map in feature_maps
]
return [(shape[1], shape[2]) for shape in feature_map_shapes]
def _compute_clip_window(preprocessed_images, true_image_shapes):
resized_inputs_shape = shape_utils.combined_static_and_dynamic_shape(
preprocessed_images)
true_heights, true_widths, _ = np.split(true_image_shapes, 3, axis=1)
padded_height = float(resized_inputs_shape[1])
padded_width = float(resized_inputs_shape[2])
cliped_image = np.stack([
np.zeros_like(true_heights),
np.zeros_like(true_widths), true_heights / padded_height,
true_widths / padded_width
],
axis=1)
cliped_imaged = cliped_image.reshape(1, -1)
return cliped_imaged
def _batch_decode(anchors, box_encodings):
"""Decodes a batch of box encodings with respect to the anchors.
Args:
box_encodings: A float32 tensor of shape
[batch_size, num_anchors, box_code_size] containing box encodings.
Returns:
decoded_boxes: A float32 tensor of shape
[batch_size, num_anchors, 4] containing the decoded boxes.
decoded_keypoints: A float32 tensor of shape
[batch_size, num_anchors, num_keypoints, 2] containing the decoded
keypoints if present in the input `box_encodings`, None otherwise.
"""
combined_shape = shape_utils.combined_static_and_dynamic_shape(
box_encodings)
batch_size = combined_shape[0]
tiled_anchor_boxes = np.tile(np.expand_dims(anchors.get(), 0),
[batch_size, 1, 1])
tiled_anchors_boxlist = box_list.BoxList(
np.reshape(tiled_anchor_boxes, [-1, 4]))
box_coder = box_coder_builder.build("faster_rcnn_box_coder")
decoded_boxes = box_coder.decode(
np.reshape(box_encodings, [-1, box_coder.code_size]),
tiled_anchors_boxlist)
decoded_keypoints = None
if decoded_boxes.has_field(fields.BoxListFields.keypoints):
decoded_keypoints = decoded_boxes.get_field(
fields.BoxListFields.keypoints)
num_keypoints = decoded_keypoints.get_shape()[1]
decoded_keypoints = np.reshape(
decoded_keypoints,
np.stack([combined_shape[0], combined_shape[1], num_keypoints, 2]))
decoded_boxes = np.reshape(
decoded_boxes.get(),
np.stack([combined_shape[0], combined_shape[1], 4]))
return decoded_boxes, decoded_keypoints
def _batch_decode_boxes(box_encodings, anchor_boxes):
print("================ _batch_decode_boxes ==================")
combined_shape = shape_utils.combined_static_and_dynamic_shape(
box_encodings)
num_classes = combined_shape[2]
# num_classes = 1
# (1, 28728, 4)
print("====== anchor_boxes:", anchor_boxes[0][0])
anchor_boxes_exp_dim = np.expand_dims(anchor_boxes, 2)
# anchor_boxes_exp_dim: (1, 28728, 1, 4)
tiled_anchor_boxes = np.tile(anchor_boxes_exp_dim, [1, 1, num_classes, 1])
# tiled_anchor_boxes (1, 28728, 1, 4)
reshaped_anchors = np.reshape(tiled_anchor_boxes, [-1, 4])
# reshaped: (28728, 4)
print("====== reshaped_anchors:", reshaped_anchors[0])
tiled_anchors_boxlist = box_list.BoxList(reshaped_anchors)
_proposal_target_assigner = target_assigner.create_target_assigner(
'FasterRCNN', 'proposal')
_box_coder = _proposal_target_assigner.box_coder
print("================ _box_coder.decode ==================")
reshaped = np.reshape(box_encodings, [-1, _box_coder.code_size])
print("reshaped:", reshaped.shape)
decoded_boxes = _box_coder.decode(reshaped, tiled_anchors_boxlist)
decoded_boxes_reahpe = np.reshape(
decoded_boxes.get(),
np.stack([combined_shape[0], combined_shape[1], num_classes, 4]))
print("decoded_boxes_reahpe:", decoded_boxes_reahpe.shape)
return decoded_boxes_reahpe
def last_predict_part(boxes_encodings,
classes_predictions_with_background,
feature_maps,
preprocessed_inputs=None):
print("------------------ last_predict_part ------------------")
"""Predicts unpostprocessed tensors from input tensor.
This function takes an input batch of images and runs it through the forward
pass of the network to yield unpostprocessesed predictions.
A side effect of calling the predict method is that self._anchors is
populated with a box_list.BoxList of anchors. These anchors must be
constructed before the postprocess or loss functions can be called.
Args:
boxes_encodings:
classes_predictions_with_background:
feature_maps:
preprocessed_inputs: a [batch, height, width, channels] image tensor.
true_image_shapes: int32 tensor of shape [batch, 3] where each row is
of the form [height, width, channels] indicating the shapes
of true images in the resized images, as resized images can be padded
with zeros.
"""
anchor_generator = anchor_generator_builder.build()
num_predictions_per_location_list = anchor_generator.num_anchors_per_location(
)
# print("num_predictions_per_location_list:", num_predictions_per_location_list)
prediction_dict = post_processor(boxes_encodings,
classes_predictions_with_background,
feature_maps,
num_predictions_per_location_list)
image_shape = shape_utils.combined_static_and_dynamic_shape(
preprocessed_inputs)
feature_map_spatial_dims = get_feature_map_spatial_dims(feature_maps)
anchors_list = anchor_generator.generate(feature_map_spatial_dims,
im_height=image_shape[1],
im_width=image_shape[2])
anchors = box_list_ops.concatenate(anchors_list)
box_encodings = np.concatenate(prediction_dict['box_encodings'], axis=1)
if box_encodings.ndim == 4 and box_encodings.shape[2] == 1:
box_encodings = np.squeeze(box_encodings, axis=2)
class_predictions_with_background = np.concatenate(
prediction_dict['class_predictions_with_background'], axis=1)
predictions_dict = {
'preprocessed_inputs': preprocessed_inputs,
'box_encodings': box_encodings,
'class_predictions_with_background': class_predictions_with_background,
'feature_maps': feature_maps,
'anchors': anchors.get()
}
return predictions_dict, anchors
def _postprocess_rpn(rpn_box_encodings_batch,
rpn_objectness_predictions_with_background_batch, anchors,
image_shapes, first_stage_max_proposals):
first_stage_nms_score_threshold = config.cfg.POSTPROCESSOR.FIRST_STAGE_NMS_SCORE_THRESHOLD
first_stage_nms_iou_threshold = config.cfg.POSTPROCESSOR.FIRST_STAGE_NMS_IOU_THRESHOLD
rpn_box_encodings_batch = np.expand_dims(rpn_box_encodings_batch, axis=2)
rpn_encodings_shape = shape_utils.combined_static_and_dynamic_shape(
rpn_box_encodings_batch)
# print("=== anchors:", anchors[0])
tiled_anchor_boxes = np.tile(np.expand_dims(anchors, 0),
[rpn_encodings_shape[0], 1, 1])
# print("=== tiled_anchor_boxes:", tiled_anchor_boxes[0][0])
proposal_boxes = _batch_decode_boxes(rpn_box_encodings_batch,
tiled_anchor_boxes)
proposal_boxes = np.squeeze(proposal_boxes, axis=2)
# (1, 28728, 4)
# 0 [11.60262919 3.12900102 41.31160688 18.96688846]
# rpn_objectness_predictions_with_background_batch (1, 28728, 2)
# rpn_objectness_predictions_with_background_batch[:, :, 1][0][0] -2.7135613
rpn_objectness_softmax_without_background = ops.softmax(
rpn_objectness_predictions_with_background_batch)[:, :, 1]
# rpn_objectness_softmax_without_background: (1, 28728)
# ====== softmax score : 0.0032150035
print("====== softmax score :",
rpn_objectness_softmax_without_background[0][0])
clip_window = _compute_clip_window(image_shapes)
# [[ 0 0 600 1002]]
print("clip_window:", clip_window)
(proposal_boxes, proposal_scores, _, _, _,
num_proposals) = post_processing.batch_multiclass_non_max_suppression(
np.expand_dims(proposal_boxes, axis=2),
np.expand_dims(rpn_objectness_softmax_without_background, axis=2),
first_stage_nms_score_threshold,
first_stage_nms_iou_threshold,
first_stage_max_proposals,
first_stage_max_proposals,
clip_window=clip_window)
print("proposal_boxes:", proposal_boxes.shape)
print("proposal_boxes [0][0]:", proposal_boxes[0][0])
# import h5py
# with h5py.File('tf_proposal.h5', 'w') as f:
# f["tf_proposal"] = proposal_boxes[0]
print("proposal_scores:", proposal_scores.shape)
print("proposal_scores [0][0]:", proposal_scores[0][0])
# proposal_boxes [0][0]: [ 6.95569825 402.90691757 398.87478089 947.73357773]
# proposal_scores: (1, 100)
# proposal_scores [0][0]: 0.9992391
# caffe 'proposals final:', array([237.24371 , 18.908209, 561.04926 , 175.2929 ], dtype=float32))
# normalize proposal boxes
def normalize_boxes(args):
proposal_boxes_per_image = args[0][0]
image_shape = args[1][0]
normalized_boxes_per_image = box_list_ops.to_normalized_coordinates(
box_list.BoxList(proposal_boxes_per_image), image_shape[0],
image_shape[1]).get()
return normalized_boxes_per_image
normalized_proposal_boxes = shape_utils.static_or_dynamic_map_fn(
normalize_boxes, elems=[proposal_boxes, image_shapes])
return normalized_proposal_boxes, proposal_scores, num_proposals
def _flatten_first_two_dimensions(inputs):
combined_shape = shape_utils.combined_static_and_dynamic_shape(inputs)
flattened_shape = np.stack([combined_shape[0] * combined_shape[1]] +
combined_shape[2:])
return np.reshape(inputs, flattened_shape)
def second_stage_box_predictor(preprocessed_inputs, box_encoding_reshape,
class_prediction_reshape, rpn_features_to_crop,
rpn_box_encodings,
rpn_objectness_predictions_with_background,
true_image_shapes, rpn_box_predictor_features):
image_shape = shape_utils.combined_static_and_dynamic_shape(
preprocessed_inputs)
first_stage_anchor_generator = anchor_generator_builder.build()
clip_window = np.stack([0, 0, image_shape[1], image_shape[2]])
feature_map_shape = rpn_features_to_crop.shape
anchors_boxlist = box_list_ops.concatenate(
first_stage_anchor_generator.generate([(feature_map_shape[1],
feature_map_shape[2])]))
anchors_boxlist = box_list_ops.clip_to_window(anchors_boxlist, clip_window)
_anchors = anchors_boxlist
image_shape_2d = _image_batch_shape_2d(image_shape)
num_anchors_per_location = (
first_stage_anchor_generator.num_anchors_per_location())
if len(num_anchors_per_location) != 1:
raise RuntimeError('anchor_generator is expected to generate anchors '
'corresponding to a single feature map.')
box_predictions = _first_stage_box_predictor_predict(
[rpn_box_predictor_features], [rpn_box_encodings],
[rpn_objectness_predictions_with_background], num_anchors_per_location)
predictions_box_encodings = np.concatenate(box_predictions[BOX_ENCODINGS],
axis=1)
rpn_box_encodings = np.squeeze(predictions_box_encodings, axis=2)
rpn_objectness_predictions_with_background = np.concatenate(
box_predictions[CLASS_PREDICTIONS_WITH_BACKGROUND], axis=1)
first_stage_max_proposals = config.cfg.POSTPROCESSOR.FIRST_STAGE_MAX_PROPOSALS
proposal_boxes_normalized, _, num_proposals = _postprocess_rpn(
rpn_box_encodings,
rpn_objectness_predictions_with_background,
_anchors.get(),
image_shape_2d,
first_stage_max_proposals=first_stage_max_proposals)
print("proposal_boxes_normalized:", proposal_boxes_normalized.shape)
prediction_dict = {
'rpn_box_predictor_features':
rpn_box_predictor_features,
'rpn_features_to_crop':
rpn_features_to_crop,
'image_shape':
image_shape,
'rpn_box_encodings':
rpn_box_encodings,
'rpn_objectness_predictions_with_background':
rpn_objectness_predictions_with_background,
}
print("=========== box_encoding_reshape", box_encoding_reshape.shape)
refined_box_encodings = np.squeeze(box_encoding_reshape, axis=1)
print("=========== class_prediction_reshape",
class_prediction_reshape.shape)
class_predictions_with_background = np.squeeze(class_prediction_reshape,
axis=1)
_parallel_iterations = 16
proposal_boxes_normalized = proposal_boxes_normalized[0]
absolute_proposal_boxes = ops.normalized_to_image_coordinates(
proposal_boxes_normalized, image_shape, _parallel_iterations)
prediction_dict1 = {
'refined_box_encodings': refined_box_encodings,
'class_predictions_with_background': class_predictions_with_background,
'num_proposals': num_proposals,
'proposal_boxes': absolute_proposal_boxes,
}
prediction_dict.update(prediction_dict1)
result_output = second_postprocess(prediction_dict, true_image_shapes)
return result_output