def faster_rcnn_output(self): # === prepare input images === image_ids = self.train_data_generator.dataset_coco.image_ids inputs, anchor_targets, bbox_reg_targets = self.train_data_generator.gen_train_data_rpn_one( image_ids[0]) print(inputs.shape, anchor_targets.shape, bbox_reg_targets.shape) image = np.reshape(inputs[0, :, :, :], (1, self.IMG_SHAPE[0], self.IMG_SHAPE[1], 3)) # === get proposed region boxes === rpn_anchor_pred, rpn_bbox_regression_pred = self.RPN.process_image( image) proposed_boxes = self._proposal_boxes(rpn_anchor_pred, rpn_bbox_regression_pred, self.anchor_candidates) # === processing boxes with RoI header === pred_class, pred_box_reg = self.RoI.process_image( [image, proposed_boxes]) # === processing the results === pred_class_sparse = np.argmax(a=pred_class, axis=1) pred_class_sparse_value = np.max(a=pred_class, axis=1) print(pred_class, pred_box_reg) print(pred_class_sparse, pred_class_sparse_value) print(np.max(proposed_boxes), np.max(pred_box_reg)) final_box = BboxTools.bbox_reg2truebox(base_boxes=proposed_boxes, regs=pred_box_reg) final_box = BboxTools.clip_boxes(final_box, self.IMG_SHAPE) print(final_box) print(final_box[pred_class_sparse_value > 0.8]) final_box = final_box[pred_class_sparse_value > 0.8] self.cocotool.draw_bboxes(original_image=image[0], bboxes=final_box.tolist(), show=True, save_file=True, path=Param.PATH_DEBUG_IMG, save_name='6PredRoISBoxes') # === Non maximum suppression === final_box_temp = np.array(final_box).astype(np.int) nms_boxes_list = [] while final_box_temp.shape[0] > 0: ious = self.nms_loop_np(final_box_temp) nms_boxes_list.append( final_box_temp[0, :] ) # since it's sorted by the value, here we can pick the first one each time. final_box_temp = final_box_temp[ious < Param.RPN_NMS_THRESHOLD] debug_print('number of box after nms', len(nms_boxes_list)) self.cocotool.draw_bboxes(original_image=image[0], bboxes=nms_boxes_list, show=True, save_file=True, path=Param.PATH_DEBUG_IMG, save_name='7PredRoINMSBoxes')
def faster_rcnn_output(self): # === prepare input images === image_ids = self.train_data_generator.dataset_coco.image_ids inputs, anchor_targets, bbox_reg_targets = self.train_data_generator.gen_train_data_rpn_one( image_ids[0]) print(inputs.shape, anchor_targets.shape, bbox_reg_targets.shape) image = np.reshape(inputs[0, :, :, :], (1, self.IMG_SHAPE[0], self.IMG_SHAPE[1], 3)) # === get proposed region boxes === rpn_anchor_pred, rpn_bbox_regression_pred = self.RPN.process_image( image) proposed_boxes = self.RPN._proposal_boxes( rpn_anchor_pred, rpn_bbox_regression_pred, self.anchor_candidates, self.anchor_candidate_generator.h, self.anchor_candidate_generator.w, self.anchor_candidate_generator.n_anchors, Param.ANCHOR_PROPOSAL_N, Param.ANCHOR_THRESHOLD) # === processing boxes with RoI header === pred_class, pred_box_reg = self.RoI.process_image( [image, proposed_boxes]) # === processing the results === pred_class_sparse = np.argmax(a=pred_class[:, :], axis=1) pred_class_sparse_value = np.max(a=pred_class[:, :], axis=1) print(pred_class, pred_box_reg) print(pred_class_sparse, pred_class_sparse_value) print(np.max(proposed_boxes), np.max(pred_box_reg)) final_box = BboxTools.bbox_reg2truebox(base_boxes=proposed_boxes, regs=pred_box_reg) final_box = BboxTools.clip_boxes(final_box, self.IMG_SHAPE) # === output to official coco bbox result json file === temp_output_to_file = [] for i in range(pred_class_sparse.shape[0]): temp_category = self.train_data_generator.dataset_coco.get_category_from_sparse( pred_class_sparse[i]) temp_output_to_file.append({ "image_id": f"{image_ids[0]}", "bbox": [ final_box[i][0].item(), final_box[i][1].item(), final_box[i][2].item(), final_box[i][3].item() ], "score": pred_class_sparse_value[i].item(), "category": f"{temp_category}" }) with open("results.pkl.bbox.json", "w") as f: json.dump(temp_output_to_file, f, indent=4) # print(final_box) print(final_box[pred_class_sparse_value > 0.9]) final_box = final_box[pred_class_sparse_value > 0.9] self.cocotool.draw_bboxes(original_image=image[0], bboxes=final_box.tolist(), show=True, save_file=True, path=Param.PATH_DEBUG_IMG, save_name='6PredRoISBoxes') # === Non maximum suppression === final_box_temp = np.array(final_box).astype(np.int) nms_boxes_list = [] while final_box_temp.shape[0] > 0: ious = self.nms_loop_np(final_box_temp) nms_boxes_list.append( final_box_temp[0, :] ) # since it's sorted by the value, here we can pick the first one each time. final_box_temp = final_box_temp[ious < Param.RPN_NMS_THRESHOLD] debug_print('number of box after nms', len(nms_boxes_list)) self.cocotool.draw_bboxes(original_image=image[0], bboxes=nms_boxes_list, show=True, save_file=True, path=Param.PATH_DEBUG_IMG, save_name='7PredRoINMSBoxes')
def test_proposal_visualization(self): # === Prediction part === image_ids = self.train_data_generator.dataset_coco.image_ids inputs, anchor_targets, bbox_reg_targets = self.train_data_generator.gen_train_data_rpn_one( image_ids[0]) print(inputs.shape, anchor_targets.shape, bbox_reg_targets.shape) input1 = np.reshape(inputs[0, :, :, :], (1, self.IMG_SHAPE[0], self.IMG_SHAPE[1], 3)) rpn_anchor_pred, rpn_bbox_regression_pred = self.RPN.process_image( input1) print(rpn_anchor_pred.shape, rpn_bbox_regression_pred.shape) # === Selection part === # top_values, top_indices = tf.math.top_k() rpn_anchor_pred = tf.slice(rpn_anchor_pred, [0, 0, 0, 0, 1], [ 1, self.anchor_candidate_generator.h, self.anchor_candidate_generator.w, self.anchor_candidate_generator.n_anchors, 1 ]) # second channel is foreground print(rpn_anchor_pred.shape, rpn_bbox_regression_pred.shape) # squeeze the pred of anchor and bbox_reg rpn_anchor_pred = tf.squeeze(rpn_anchor_pred) rpn_bbox_regression_pred = tf.squeeze(rpn_bbox_regression_pred) shape1 = tf.shape(rpn_anchor_pred) print(rpn_anchor_pred.shape, rpn_bbox_regression_pred.shape) # flatten the pred of anchor to get top N values and indices rpn_anchor_pred = tf.reshape(rpn_anchor_pred, (-1, )) n_anchor_proposal = Param.ANCHOR_PROPOSAL_N top_values, top_indices = tf.math.top_k( rpn_anchor_pred, n_anchor_proposal) # top_k has sort function. it's important here top_indices = tf.gather_nd( top_indices, tf.where(tf.greater(top_values, Param.ANCHOR_THRESHOLD))) top_values = tf.gather_nd( top_values, tf.where(tf.greater(top_values, Param.ANCHOR_THRESHOLD))) debug_print('top values', top_values) # test_indices = tf.where(tf.greater(tf.reshape(RPN_Anchor_Pred, (-1,)), 0.9)) # print(test_indices) top_indices = tf.reshape(top_indices, (-1, 1)) debug_print('top indices', top_indices) update_value = tf.math.add(top_values, 1) rpn_anchor_pred = tf.tensor_scatter_nd_update(rpn_anchor_pred, top_indices, update_value) rpn_anchor_pred = tf.reshape(rpn_anchor_pred, shape1) # --- find the base boxes --- anchor_pred_top_indices = tf.where(tf.greater(rpn_anchor_pred, 1)) debug_print('original_indices shape', anchor_pred_top_indices.shape) debug_print('original_indices', anchor_pred_top_indices) base_boxes = tf.gather_nd(self.anchor_candidates, anchor_pred_top_indices) debug_print('base_boxes shape', base_boxes.shape) debug_print('base_boxes', base_boxes) base_boxes = np.array(base_boxes) # --- find the bbox_regs --- # flatten the bbox_reg by last dim to use top_indices to get final_box_reg rpn_bbox_regression_pred_shape = tf.shape(rpn_bbox_regression_pred) rpn_bbox_regression_pred = tf.reshape( rpn_bbox_regression_pred, (-1, rpn_bbox_regression_pred_shape[-1])) debug_print('RPN_BBOX_Regression_Pred shape', rpn_bbox_regression_pred.shape) final_box_reg = tf.gather_nd(rpn_bbox_regression_pred, top_indices) debug_print('final box reg values', final_box_reg) # Convert to numpy to plot final_box_reg = np.array(final_box_reg) debug_print('final box reg shape', final_box_reg.shape) debug_print('max value of final box reg', np.max(final_box_reg)) final_box = BboxTools.bbox_reg2truebox(base_boxes=base_boxes, regs=final_box_reg) # === Non maximum suppression === final_box_temp = np.array(final_box).astype(np.int) nms_boxes_list = [] while final_box_temp.shape[0] > 0: ious = self.nms_loop_np(final_box_temp) nms_boxes_list.append( final_box_temp[0, :] ) # since it's sorted by the value, here we can pick the first one each time. final_box_temp = final_box_temp[ious < Param.RPN_NMS_THRESHOLD] debug_print('number of box after nms', len(nms_boxes_list)) # Need to convert above instructions to tf operations # === visualization part === # clip the boxes to make sure they are legal boxes debug_print('max value of final box', np.max(final_box)) final_box = BboxTools.clip_boxes(final_box, self.IMG_SHAPE) original_boxes = self.cocotool.get_original_bboxes_list( image_id=self.cocotool.image_ids[0]) self.cocotool.draw_bboxes(original_image=input1[0], bboxes=original_boxes, show=True, save_file=True, path=Param.PATH_DEBUG_IMG, save_name='1GroundTruthBoxes') target_anchor_boxes, target_classes = self.train_data_generator.gen_target_anchor_bboxes_classes_for_debug( image_id=self.cocotool.image_ids[0]) self.cocotool.draw_bboxes(original_image=input1[0], bboxes=target_anchor_boxes, show=True, save_file=True, path=Param.PATH_DEBUG_IMG, save_name='2TrueAnchorBoxes') self.cocotool.draw_bboxes(original_image=input1[0], bboxes=base_boxes.tolist(), show=True, save_file=True, path=Param.PATH_DEBUG_IMG, save_name='3PredAnchorBoxes') self.cocotool.draw_bboxes(original_image=input1[0], bboxes=final_box.tolist(), show=True, save_file=True, path=Param.PATH_DEBUG_IMG, save_name='4PredRegBoxes') self.cocotool.draw_bboxes(original_image=input1[0], bboxes=nms_boxes_list, show=True, save_file=True, path=Param.PATH_DEBUG_IMG, save_name='5PredNMSBoxes')
cv.rectangle(img=img1, rec=(bbox[1], bbox[0], bbox[3] - bbox[1], bbox[2] - bbox[0]), color=color, thickness=4) plt.imshow(img1) plt.show() def get_feature_map_h_w_with_n_stages(img_shape: tuple, n_stage: int): # here round up the number since the tensorflow conv2d round strategy n_stage_revert_factor = 2 ** n_stage h = int(img_shape[0] / n_stage_revert_factor) + int((img_shape[0] % n_stage_revert_factor) > 0) w = int(img_shape[1] / n_stage_revert_factor) + int((img_shape[1] % n_stage_revert_factor) > 0) return h, w if __name__ == '__main__': t1 = GenCandidateAnchors(base_size=12) debug_print("base anchors", t1.base_anchors) debug_print("9 Anchors candidate at [0,0]", t1.anchor_candidates[0, 0, :, :]) debug_print("9 Anchors candidate at [10,10]", t1.anchor_candidates[10, 10, :, :]) # bboxes = [t1.anchors_candidate[10, 10, i, :] for i in range(9)] bboxes = t1.anchor_candidates[13, 22, :, :].tolist() t1._validate_bbox(bboxes) # print(t1.all_anchors[23,40,:,:]) debug_print("1 Anchor candidate at [0,0,2]", t1.anchor_candidates[0, 0, 2, :]) # temp = t1.anchors_candidate.reshape((-1, 4)) temp = np.reshape(t1.anchor_candidates, newshape=(-1, 4)) debug_print("Same with Anchor candidate at [0,0,2] after reshape", temp[2, :]) temp2 = temp.reshape((t1.h, t1.w, t1.n_anchors, 4)) debug_print("Same with temp[2,:] after reshape", temp2[0, 0, 2, :]) debug_print("Same anchor in anchor list", t1.anchor_candidates_list[2])