def faster_rcnn_output(self):
# === prepare input images ===
image_ids = self.train_data_generator.dataset_coco.image_ids
inputs, anchor_targets, bbox_reg_targets = self.train_data_generator.gen_train_data_rpn_one(
image_ids[0])
print(inputs.shape, anchor_targets.shape, bbox_reg_targets.shape)
image = np.reshape(inputs[0, :, :, :],
(1, self.IMG_SHAPE[0], self.IMG_SHAPE[1], 3))
# === get proposed region boxes ===
rpn_anchor_pred, rpn_bbox_regression_pred = self.RPN.process_image(
image)
proposed_boxes = self._proposal_boxes(rpn_anchor_pred,
rpn_bbox_regression_pred,
self.anchor_candidates)
# === processing boxes with RoI header ===
pred_class, pred_box_reg = self.RoI.process_image(
[image, proposed_boxes])
# === processing the results ===
pred_class_sparse = np.argmax(a=pred_class, axis=1)
pred_class_sparse_value = np.max(a=pred_class, axis=1)
print(pred_class, pred_box_reg)
print(pred_class_sparse, pred_class_sparse_value)
print(np.max(proposed_boxes), np.max(pred_box_reg))
final_box = BboxTools.bbox_reg2truebox(base_boxes=proposed_boxes,
regs=pred_box_reg)
final_box = BboxTools.clip_boxes(final_box, self.IMG_SHAPE)
print(final_box)
print(final_box[pred_class_sparse_value > 0.8])
final_box = final_box[pred_class_sparse_value > 0.8]
self.cocotool.draw_bboxes(original_image=image[0],
bboxes=final_box.tolist(),
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='6PredRoISBoxes')
# === Non maximum suppression ===
final_box_temp = np.array(final_box).astype(np.int)
nms_boxes_list = []
while final_box_temp.shape[0] > 0:
ious = self.nms_loop_np(final_box_temp)
nms_boxes_list.append(
final_box_temp[0, :]
) # since it's sorted by the value, here we can pick the first one each time.
final_box_temp = final_box_temp[ious < Param.RPN_NMS_THRESHOLD]
debug_print('number of box after nms', len(nms_boxes_list))
self.cocotool.draw_bboxes(original_image=image[0],
bboxes=nms_boxes_list,
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='7PredRoINMSBoxes')
def faster_rcnn_output(self):
# === prepare input images ===
image_ids = self.train_data_generator.dataset_coco.image_ids
inputs, anchor_targets, bbox_reg_targets = self.train_data_generator.gen_train_data_rpn_one(
image_ids[0])
print(inputs.shape, anchor_targets.shape, bbox_reg_targets.shape)
image = np.reshape(inputs[0, :, :, :],
(1, self.IMG_SHAPE[0], self.IMG_SHAPE[1], 3))
# === get proposed region boxes ===
rpn_anchor_pred, rpn_bbox_regression_pred = self.RPN.process_image(
image)
proposed_boxes = self.RPN._proposal_boxes(
rpn_anchor_pred, rpn_bbox_regression_pred, self.anchor_candidates,
self.anchor_candidate_generator.h,
self.anchor_candidate_generator.w,
self.anchor_candidate_generator.n_anchors, Param.ANCHOR_PROPOSAL_N,
Param.ANCHOR_THRESHOLD)
# === processing boxes with RoI header ===
pred_class, pred_box_reg = self.RoI.process_image(
[image, proposed_boxes])
# === processing the results ===
pred_class_sparse = np.argmax(a=pred_class[:, :], axis=1)
pred_class_sparse_value = np.max(a=pred_class[:, :], axis=1)
print(pred_class, pred_box_reg)
print(pred_class_sparse, pred_class_sparse_value)
print(np.max(proposed_boxes), np.max(pred_box_reg))
final_box = BboxTools.bbox_reg2truebox(base_boxes=proposed_boxes,
regs=pred_box_reg)
final_box = BboxTools.clip_boxes(final_box, self.IMG_SHAPE)
# === output to official coco bbox result json file ===
temp_output_to_file = []
for i in range(pred_class_sparse.shape[0]):
temp_category = self.train_data_generator.dataset_coco.get_category_from_sparse(
pred_class_sparse[i])
temp_output_to_file.append({
"image_id":
f"{image_ids[0]}",
"bbox": [
final_box[i][0].item(), final_box[i][1].item(),
final_box[i][2].item(), final_box[i][3].item()
],
"score":
pred_class_sparse_value[i].item(),
"category":
f"{temp_category}"
})
with open("results.pkl.bbox.json", "w") as f:
json.dump(temp_output_to_file, f, indent=4)
# print(final_box)
print(final_box[pred_class_sparse_value > 0.9])
final_box = final_box[pred_class_sparse_value > 0.9]
self.cocotool.draw_bboxes(original_image=image[0],
bboxes=final_box.tolist(),
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='6PredRoISBoxes')
# === Non maximum suppression ===
final_box_temp = np.array(final_box).astype(np.int)
nms_boxes_list = []
while final_box_temp.shape[0] > 0:
ious = self.nms_loop_np(final_box_temp)
nms_boxes_list.append(
final_box_temp[0, :]
) # since it's sorted by the value, here we can pick the first one each time.
final_box_temp = final_box_temp[ious < Param.RPN_NMS_THRESHOLD]
debug_print('number of box after nms', len(nms_boxes_list))
self.cocotool.draw_bboxes(original_image=image[0],
bboxes=nms_boxes_list,
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='7PredRoINMSBoxes')
def test_proposal_visualization(self):
# === Prediction part ===
image_ids = self.train_data_generator.dataset_coco.image_ids
inputs, anchor_targets, bbox_reg_targets = self.train_data_generator.gen_train_data_rpn_one(
image_ids[0])
print(inputs.shape, anchor_targets.shape, bbox_reg_targets.shape)
input1 = np.reshape(inputs[0, :, :, :],
(1, self.IMG_SHAPE[0], self.IMG_SHAPE[1], 3))
rpn_anchor_pred, rpn_bbox_regression_pred = self.RPN.process_image(
input1)
print(rpn_anchor_pred.shape, rpn_bbox_regression_pred.shape)
# === Selection part ===
# top_values, top_indices = tf.math.top_k()
rpn_anchor_pred = tf.slice(rpn_anchor_pred, [0, 0, 0, 0, 1], [
1, self.anchor_candidate_generator.h,
self.anchor_candidate_generator.w,
self.anchor_candidate_generator.n_anchors, 1
]) # second channel is foreground
print(rpn_anchor_pred.shape, rpn_bbox_regression_pred.shape)
# squeeze the pred of anchor and bbox_reg
rpn_anchor_pred = tf.squeeze(rpn_anchor_pred)
rpn_bbox_regression_pred = tf.squeeze(rpn_bbox_regression_pred)
shape1 = tf.shape(rpn_anchor_pred)
print(rpn_anchor_pred.shape, rpn_bbox_regression_pred.shape)
# flatten the pred of anchor to get top N values and indices
rpn_anchor_pred = tf.reshape(rpn_anchor_pred, (-1, ))
n_anchor_proposal = Param.ANCHOR_PROPOSAL_N
top_values, top_indices = tf.math.top_k(
rpn_anchor_pred,
n_anchor_proposal) # top_k has sort function. it's important here
top_indices = tf.gather_nd(
top_indices,
tf.where(tf.greater(top_values, Param.ANCHOR_THRESHOLD)))
top_values = tf.gather_nd(
top_values, tf.where(tf.greater(top_values,
Param.ANCHOR_THRESHOLD)))
debug_print('top values', top_values)
# test_indices = tf.where(tf.greater(tf.reshape(RPN_Anchor_Pred, (-1,)), 0.9))
# print(test_indices)
top_indices = tf.reshape(top_indices, (-1, 1))
debug_print('top indices', top_indices)
update_value = tf.math.add(top_values, 1)
rpn_anchor_pred = tf.tensor_scatter_nd_update(rpn_anchor_pred,
top_indices,
update_value)
rpn_anchor_pred = tf.reshape(rpn_anchor_pred, shape1)
# --- find the base boxes ---
anchor_pred_top_indices = tf.where(tf.greater(rpn_anchor_pred, 1))
debug_print('original_indices shape', anchor_pred_top_indices.shape)
debug_print('original_indices', anchor_pred_top_indices)
base_boxes = tf.gather_nd(self.anchor_candidates,
anchor_pred_top_indices)
debug_print('base_boxes shape', base_boxes.shape)
debug_print('base_boxes', base_boxes)
base_boxes = np.array(base_boxes)
# --- find the bbox_regs ---
# flatten the bbox_reg by last dim to use top_indices to get final_box_reg
rpn_bbox_regression_pred_shape = tf.shape(rpn_bbox_regression_pred)
rpn_bbox_regression_pred = tf.reshape(
rpn_bbox_regression_pred, (-1, rpn_bbox_regression_pred_shape[-1]))
debug_print('RPN_BBOX_Regression_Pred shape',
rpn_bbox_regression_pred.shape)
final_box_reg = tf.gather_nd(rpn_bbox_regression_pred, top_indices)
debug_print('final box reg values', final_box_reg)
# Convert to numpy to plot
final_box_reg = np.array(final_box_reg)
debug_print('final box reg shape', final_box_reg.shape)
debug_print('max value of final box reg', np.max(final_box_reg))
final_box = BboxTools.bbox_reg2truebox(base_boxes=base_boxes,
regs=final_box_reg)
# === Non maximum suppression ===
final_box_temp = np.array(final_box).astype(np.int)
nms_boxes_list = []
while final_box_temp.shape[0] > 0:
ious = self.nms_loop_np(final_box_temp)
nms_boxes_list.append(
final_box_temp[0, :]
) # since it's sorted by the value, here we can pick the first one each time.
final_box_temp = final_box_temp[ious < Param.RPN_NMS_THRESHOLD]
debug_print('number of box after nms', len(nms_boxes_list))
# Need to convert above instructions to tf operations
# === visualization part ===
# clip the boxes to make sure they are legal boxes
debug_print('max value of final box', np.max(final_box))
final_box = BboxTools.clip_boxes(final_box, self.IMG_SHAPE)
original_boxes = self.cocotool.get_original_bboxes_list(
image_id=self.cocotool.image_ids[0])
self.cocotool.draw_bboxes(original_image=input1[0],
bboxes=original_boxes,
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='1GroundTruthBoxes')
target_anchor_boxes, target_classes = self.train_data_generator.gen_target_anchor_bboxes_classes_for_debug(
image_id=self.cocotool.image_ids[0])
self.cocotool.draw_bboxes(original_image=input1[0],
bboxes=target_anchor_boxes,
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='2TrueAnchorBoxes')
self.cocotool.draw_bboxes(original_image=input1[0],
bboxes=base_boxes.tolist(),
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='3PredAnchorBoxes')
self.cocotool.draw_bboxes(original_image=input1[0],
bboxes=final_box.tolist(),
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='4PredRegBoxes')
self.cocotool.draw_bboxes(original_image=input1[0],
bboxes=nms_boxes_list,
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='5PredNMSBoxes')
cv.rectangle(img=img1, rec=(bbox[1], bbox[0], bbox[3] - bbox[1], bbox[2] - bbox[0]), color=color,
thickness=4)
plt.imshow(img1)
plt.show()
def get_feature_map_h_w_with_n_stages(img_shape: tuple, n_stage: int):
# here round up the number since the tensorflow conv2d round strategy
n_stage_revert_factor = 2 ** n_stage
h = int(img_shape[0] / n_stage_revert_factor) + int((img_shape[0] % n_stage_revert_factor) > 0)
w = int(img_shape[1] / n_stage_revert_factor) + int((img_shape[1] % n_stage_revert_factor) > 0)
return h, w
if __name__ == '__main__':
t1 = GenCandidateAnchors(base_size=12)
debug_print("base anchors", t1.base_anchors)
debug_print("9 Anchors candidate at [0,0]", t1.anchor_candidates[0, 0, :, :])
debug_print("9 Anchors candidate at [10,10]", t1.anchor_candidates[10, 10, :, :])
# bboxes = [t1.anchors_candidate[10, 10, i, :] for i in range(9)]
bboxes = t1.anchor_candidates[13, 22, :, :].tolist()
t1._validate_bbox(bboxes)
# print(t1.all_anchors[23,40,:,:])
debug_print("1 Anchor candidate at [0,0,2]", t1.anchor_candidates[0, 0, 2, :])
# temp = t1.anchors_candidate.reshape((-1, 4))
temp = np.reshape(t1.anchor_candidates, newshape=(-1, 4))
debug_print("Same with Anchor candidate at [0,0,2] after reshape", temp[2, :])
temp2 = temp.reshape((t1.h, t1.w, t1.n_anchors, 4))
debug_print("Same with temp[2,:] after reshape", temp2[0, 0, 2, :])
debug_print("Same anchor in anchor list", t1.anchor_candidates_list[2])