def faster_rcnn_output(self):
# === prepare input images ===
image_ids = self.train_data_generator.dataset_coco.image_ids
inputs, anchor_targets, bbox_reg_targets = self.train_data_generator.gen_train_data_rpn_one(
image_ids[0])
print(inputs.shape, anchor_targets.shape, bbox_reg_targets.shape)
image = np.reshape(inputs[0, :, :, :],
(1, self.IMG_SHAPE[0], self.IMG_SHAPE[1], 3))
# === get proposed region boxes ===
rpn_anchor_pred, rpn_bbox_regression_pred = self.RPN.process_image(
image)
proposed_boxes = self._proposal_boxes(rpn_anchor_pred,
rpn_bbox_regression_pred,
self.anchor_candidates)
# === processing boxes with RoI header ===
pred_class, pred_box_reg = self.RoI.process_image(
[image, proposed_boxes])
# === processing the results ===
pred_class_sparse = np.argmax(a=pred_class, axis=1)
pred_class_sparse_value = np.max(a=pred_class, axis=1)
print(pred_class, pred_box_reg)
print(pred_class_sparse, pred_class_sparse_value)
print(np.max(proposed_boxes), np.max(pred_box_reg))
final_box = BboxTools.bbox_reg2truebox(base_boxes=proposed_boxes,
regs=pred_box_reg)
final_box = BboxTools.clip_boxes(final_box, self.IMG_SHAPE)
print(final_box)
print(final_box[pred_class_sparse_value > 0.8])
final_box = final_box[pred_class_sparse_value > 0.8]
self.cocotool.draw_bboxes(original_image=image[0],
bboxes=final_box.tolist(),
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='6PredRoISBoxes')
# === Non maximum suppression ===
final_box_temp = np.array(final_box).astype(np.int)
nms_boxes_list = []
while final_box_temp.shape[0] > 0:
ious = self.nms_loop_np(final_box_temp)
nms_boxes_list.append(
final_box_temp[0, :]
) # since it's sorted by the value, here we can pick the first one each time.
final_box_temp = final_box_temp[ious < Param.RPN_NMS_THRESHOLD]
debug_print('number of box after nms', len(nms_boxes_list))
self.cocotool.draw_bboxes(original_image=image[0],
bboxes=nms_boxes_list,
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='7PredRoINMSBoxes')
def test():
base_path = '/Users/shuzhiliu/Google Drive/KyoceraRobotAI/mmdetection_tools/data'
imagefolder_path = '/Users/shuzhiliu/Google Drive/KyoceraRobotAI/mmdetection_tools/LocalData_Images'
dataset_id = '1940091026744'
image_id = '20191119T063709-cca043ed-32fe-4da0-ba75-e4a12b88eef4'
data1 = CocoTools(
json_file=f"{base_path}/{dataset_id}/annotations/train.json",
image_folder_path=imagefolder_path)
img1 = data1.get_original_image(image_id=image_id)
print(data1.images)
bboxes = data1.get_original_bboxes_list(image_id=image_id)
print(bboxes)
# img1 = np.zeros(shape=(720,1280,3), dtype=np.uint8)
for bbox in bboxes:
color = (random.randint(0, 255), random.randint(0, 255),
random.randint(0, 255))
# cv.rectangle(img1,(bbox[0],bbox[1]), (bbox[2],bbox[3]), 255)
# print(bbox)
# cv.rectangle(img=img1,rec=(bbox[1],bbox[0],bbox[3]-bbox[1],bbox[2]-bbox[0]), color=color, thickness=4)
cv.rectangle(img1, (bbox[1], bbox[0]), (bbox[3], bbox[2]), color, 4)
plt.imshow(img1)
plt.show()
g1 = GenCandidateAnchors()
print(len(g1.anchor_candidates_list))
ious = BboxTools.ious(g1.anchor_candidates_list, bboxes[0])
ious[np.argmax(ious)] = 1
print(len(ious))
ious_np = np.reshape(ious, newshape=(23, 40, 9))
index = np.where(ious_np == 1)
print(index)
def gen_target_anchor_bboxes_classes_for_debug(self,
image_id,
debuginfo=False):
bboxes = self.dataset_coco.get_original_bboxes_list(image_id=image_id)
sparse_targets = self.dataset_coco.get_original_category_sparse_list(
image_id=image_id)
bboxes_ious = [] # for each gt_bbox calculate ious with candidates
for bbox in bboxes:
ious = BboxTools.ious(
self.gen_candidate_anchors.anchor_candidates_list, bbox)
ious_temp = np.ones(shape=(len(ious)), dtype=np.float) * 0.5
# other author's implementations are use -1 to indicate ignoring, here use 0.5 to use max
ious_temp = np.where(
np.asarray(ious) > self.threshold_iou_rpn, 1, ious_temp)
ious_temp = np.where(np.asarray(ious) < 0.3, 0, ious_temp)
ious_temp[np.argmax(ious)] = 1
bboxes_ious.append(ious_temp)
# for each gt_box, determine the box reg target
target_anchor_bboxes = []
target_classes = []
for index, bbox_ious in enumerate(bboxes_ious):
ious_temp = np.reshape(
bbox_ious,
newshape=(self.gen_candidate_anchors.h,
self.gen_candidate_anchors.w,
self.gen_candidate_anchors.n_anchors))
candidate_boxes = self.gen_candidate_anchors.anchor_candidates[
np.where(ious_temp == 1)]
n = candidate_boxes.shape[0]
for i in range(n):
target_anchor_bboxes.append(candidate_boxes[i])
target_classes.append(sparse_targets[index])
return target_anchor_bboxes, target_classes
def test():
base_path = ''
imagefolder_path = ''
dataset_id = ''
image_id = ''
data1 = CocoTools(json_file=f"{base_path}/{dataset_id}/annotations/train.json",
image_folder_path=imagefolder_path)
img1 = data1.get_original_image(image_id=image_id)
print(data1.images)
bboxes = data1.get_original_bboxes_list(image_id=image_id)
print(bboxes)
# img1 = np.zeros(shape=(720,1280,3), dtype=np.uint8)
for bbox in bboxes:
color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
# cv.rectangle(img1,(bbox[0],bbox[1]), (bbox[2],bbox[3]), 255)
# print(bbox)
# cv.rectangle(img=img1,rec=(bbox[1],bbox[0],bbox[3]-bbox[1],bbox[2]-bbox[0]), color=color, thickness=4)
cv.rectangle(img1, (bbox[1], bbox[0]), (bbox[3], bbox[2]), color, 4)
plt.imshow(img1)
plt.show()
g1 = GenCandidateAnchors()
print(len(g1.anchor_candidates_list))
ious = BboxTools.ious(g1.anchor_candidates_list, bboxes[0])
ious[np.argmax(ious)] = 1
print(len(ious))
ious_np = np.reshape(ious, newshape=(23, 40, 9))
index = np.where(ious_np == 1)
print(index)
def gen_train_target_anchor_boxreg_for_rpn(self, image_id, debuginfo=False):
bboxes = self.dataset_coco.get_original_bboxes_list(image_id=image_id)
# === resize ===
bboxes_ious = [] # for each gt_bbox calculate ious with candidates
for bbox in bboxes:
ious = BboxTools.ious(self.gen_candidate_anchors.anchor_candidates_list, bbox)
ious_temp = np.ones(shape=(len(ious)), dtype=np.float) * 0.5
# other author's implementations are use -1 to indicate ignoring, here use 0.5 to use max
ious_temp = np.where(np.asarray(ious) > self.threshold_iou_rpn, 1, ious_temp)
ious_temp = np.where(np.asarray(ious) < 0.3, 0, ious_temp)
ious_temp[np.argmax(ious)] = 1
bboxes_ious.append(ious_temp)
# for each candidate anchor, determine the anchor target
anchors_target = np.array(bboxes_ious)
anchors_target = np.max(anchors_target, axis=0)
anchors_target = np.reshape(anchors_target, newshape=(
self.gen_candidate_anchors.h, self.gen_candidate_anchors.w, self.gen_candidate_anchors.n_anchors))
if debuginfo:
print(f"[Debug INFO] Number of total gt bboxes :{len(bboxes)}")
print(
f"[Debug INFO] Number of total target anchors: {anchors_target[np.where(anchors_target == 1)].shape[0]}")
print(f"[Debug INFO] Shape of anchors_target: {anchors_target.shape}")
print(
f"[Debug INFO] Selected anchors: \n {self.gen_candidate_anchors.anchor_candidates[np.where(anchors_target == 1)]}")
# test
# self.anchor_generator.anchors_candidate[np.where(anchors_target==1)] = self.anchor_generator.anchors_candidate[np.where(anchors_target==1)] +100
# print(f"Selected anchors: \n {self.anchor_generator.anchors_candidate[np.where(anchors_target == 1)]}")
# for each gt_box, determine the box reg target
bbox_reg_target = np.zeros(
shape=(self.gen_candidate_anchors.h, self.gen_candidate_anchors.w, self.gen_candidate_anchors.n_anchors, 4),
dtype=np.float)
for index, bbox_ious in enumerate(bboxes_ious):
ious_temp = np.reshape(bbox_ious, newshape=(
self.gen_candidate_anchors.h, self.gen_candidate_anchors.w, self.gen_candidate_anchors.n_anchors))
gt_box = bboxes[index]
candidate_boxes = self.gen_candidate_anchors.anchor_candidates[np.where(ious_temp == 1)]
# print(candidate_boxes,gt_box)
box_reg = BboxTools.bbox_regression_target(candidate_boxes, gt_box)
# print(box_reg)
# print(bbox_tools.bbox_reg2truebox(candidate_boxes, box_reg))
bbox_reg_target[np.where(ious_temp == 1)] = box_reg
return anchors_target, bbox_reg_target
def gen_train_data_roi_one(self, image_id, bbox_list):
gt_bboxes = self.dataset_coco.get_original_bboxes_list(
image_id=image_id)
sparse_targets = self.dataset_coco.get_original_category_sparse_list(
image_id=image_id)
bboxes_ious = [] # for each gt_bbox calculate ious with candidates
for bbox in gt_bboxes:
ious = BboxTools.ious(bbox_list, bbox)
ious_temp = np.zeros(shape=(len(ious)), dtype=np.float)
# other author's implementations are use -1 to indicate ignoring, here use 0.5 to use max
ious_temp = np.where(
np.asarray(ious) > self.threshold_iou_roi, 1, ious_temp)
ious_temp[np.argmax(ious)] = 1
bboxes_ious.append(ious_temp)
# for each gt_box, determine the box reg target
original_img = self.gen_train_input_one(image_id)
input_images = []
input_box_filtered_by_iou = []
target_classes = []
target_bbox_reg = []
for index_gt, bbox_ious in enumerate(bboxes_ious):
candidate_boxes = np.asarray(bbox_list)[np.where(bbox_ious == 1)]
n = candidate_boxes.shape[0]
for i in range(n):
input_box_filtered_by_iou.append(candidate_boxes[i].astype(
np.float))
box_reg = BboxTools.bbox_regression_target(
pred_boxes=candidate_boxes[i].reshape((1, 4)),
gt_box=gt_bboxes[index_gt])
target_bbox_reg.append(box_reg.ravel())
target_classes.append(sparse_targets[index_gt])
input_images.append(original_img.astype(np.float))
for index_gt, bbox_gt in enumerate(gt_bboxes):
input_images.append(original_img.astype(np.float))
input_box_filtered_by_iou.append(bbox_gt.astype(np.float))
target_classes.append(sparse_targets[index_gt])
target_bbox_reg.append(np.array([0, 0, 0, 0], dtype=np.float))
return np.asarray(input_images).astype(
np.float), np.asarray(input_box_filtered_by_iou), np.asarray(
target_classes), np.asarray(target_bbox_reg)
def _proposal_boxes(self,
rpn_anchor_pred,
rpn_bbox_regression_pred,
anchor_candidates,
h: int,
w: int,
n_anchors: int,
n_proposal: int,
anchor_threshold: float):
# === Selection part ===
# top_values, top_indices = tf.math.top_k()
rpn_anchor_pred = tf.slice(rpn_anchor_pred, [0, 0, 0, 0, 1], [1,
h,
w,
n_anchors,
1]) # second channel is foreground
# squeeze the pred of anchor and bbox_reg
rpn_anchor_pred = tf.squeeze(rpn_anchor_pred)
rpn_bbox_regression_pred = tf.squeeze(rpn_bbox_regression_pred)
shape1 = tf.shape(rpn_anchor_pred)
# flatten the pred of anchor to get top N values and indices
rpn_anchor_pred = tf.reshape(rpn_anchor_pred, (-1,))
n_anchor_proposal = n_proposal
top_values, top_indices = tf.math.top_k(rpn_anchor_pred,
n_anchor_proposal) # top_k has sort function. it's important here
top_indices = tf.gather_nd(top_indices, tf.where(tf.greater(top_values, anchor_threshold)))
top_values = tf.gather_nd(top_values, tf.where(tf.greater(top_values, anchor_threshold)))
top_indices = tf.reshape(top_indices, (-1, 1))
update_value = tf.math.add(top_values, 1)
rpn_anchor_pred = tf.tensor_scatter_nd_update(rpn_anchor_pred, top_indices, update_value)
rpn_anchor_pred = tf.reshape(rpn_anchor_pred, shape1)
# --- find the base boxes ---
anchor_pred_top_indices = tf.where(tf.greater(rpn_anchor_pred, 1))
base_boxes = tf.gather_nd(anchor_candidates, anchor_pred_top_indices)
# --- find the bbox_regs ---
# flatten the bbox_reg by last dim to use top_indices to get final_box_reg
rpn_bbox_regression_pred_shape = tf.shape(rpn_bbox_regression_pred)
rpn_bbox_regression_pred = tf.reshape(rpn_bbox_regression_pred, (-1, rpn_bbox_regression_pred_shape[-1]))
final_box_reg = tf.gather_nd(rpn_bbox_regression_pred, top_indices)
# Convert to numpy to plot
final_box = BboxTools.bbox_reg2truebox(base_boxes=base_boxes, regs=final_box_reg)
return np.array(final_box).astype(np.float)
def faster_rcnn_output(self):
# === prepare input images ===
image_ids = self.train_data_generator.dataset_coco.image_ids
inputs, anchor_targets, bbox_reg_targets = self.train_data_generator.gen_train_data_rpn_one(
image_ids[0])
print(inputs.shape, anchor_targets.shape, bbox_reg_targets.shape)
image = np.reshape(inputs[0, :, :, :],
(1, self.IMG_SHAPE[0], self.IMG_SHAPE[1], 3))
# === get proposed region boxes ===
rpn_anchor_pred, rpn_bbox_regression_pred = self.RPN.process_image(
image)
proposed_boxes = self.RPN._proposal_boxes(
rpn_anchor_pred, rpn_bbox_regression_pred, self.anchor_candidates,
self.anchor_candidate_generator.h,
self.anchor_candidate_generator.w,
self.anchor_candidate_generator.n_anchors, Param.ANCHOR_PROPOSAL_N,
Param.ANCHOR_THRESHOLD)
# === processing boxes with RoI header ===
pred_class, pred_box_reg = self.RoI.process_image(
[image, proposed_boxes])
# === processing the results ===
pred_class_sparse = np.argmax(a=pred_class[:, :], axis=1)
pred_class_sparse_value = np.max(a=pred_class[:, :], axis=1)
print(pred_class, pred_box_reg)
print(pred_class_sparse, pred_class_sparse_value)
print(np.max(proposed_boxes), np.max(pred_box_reg))
final_box = BboxTools.bbox_reg2truebox(base_boxes=proposed_boxes,
regs=pred_box_reg)
final_box = BboxTools.clip_boxes(final_box, self.IMG_SHAPE)
# === output to official coco bbox result json file ===
temp_output_to_file = []
for i in range(pred_class_sparse.shape[0]):
temp_category = self.train_data_generator.dataset_coco.get_category_from_sparse(
pred_class_sparse[i])
temp_output_to_file.append({
"image_id":
f"{image_ids[0]}",
"bbox": [
final_box[i][0].item(), final_box[i][1].item(),
final_box[i][2].item(), final_box[i][3].item()
],
"score":
pred_class_sparse_value[i].item(),
"category":
f"{temp_category}"
})
with open("results.pkl.bbox.json", "w") as f:
json.dump(temp_output_to_file, f, indent=4)
# print(final_box)
print(final_box[pred_class_sparse_value > 0.9])
final_box = final_box[pred_class_sparse_value > 0.9]
self.cocotool.draw_bboxes(original_image=image[0],
bboxes=final_box.tolist(),
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='6PredRoISBoxes')
# === Non maximum suppression ===
final_box_temp = np.array(final_box).astype(np.int)
nms_boxes_list = []
while final_box_temp.shape[0] > 0:
ious = self.nms_loop_np(final_box_temp)
nms_boxes_list.append(
final_box_temp[0, :]
) # since it's sorted by the value, here we can pick the first one each time.
final_box_temp = final_box_temp[ious < Param.RPN_NMS_THRESHOLD]
debug_print('number of box after nms', len(nms_boxes_list))
self.cocotool.draw_bboxes(original_image=image[0],
bboxes=nms_boxes_list,
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='7PredRoINMSBoxes')
def test_proposal_visualization(self):
# === Prediction part ===
image_ids = self.train_data_generator.dataset_coco.image_ids
inputs, anchor_targets, bbox_reg_targets = self.train_data_generator.gen_train_data_rpn_one(
image_ids[0])
print(inputs.shape, anchor_targets.shape, bbox_reg_targets.shape)
input1 = np.reshape(inputs[0, :, :, :],
(1, self.IMG_SHAPE[0], self.IMG_SHAPE[1], 3))
rpn_anchor_pred, rpn_bbox_regression_pred = self.RPN.process_image(
input1)
print(rpn_anchor_pred.shape, rpn_bbox_regression_pred.shape)
# === Selection part ===
# top_values, top_indices = tf.math.top_k()
rpn_anchor_pred = tf.slice(rpn_anchor_pred, [0, 0, 0, 0, 1], [
1, self.anchor_candidate_generator.h,
self.anchor_candidate_generator.w,
self.anchor_candidate_generator.n_anchors, 1
]) # second channel is foreground
print(rpn_anchor_pred.shape, rpn_bbox_regression_pred.shape)
# squeeze the pred of anchor and bbox_reg
rpn_anchor_pred = tf.squeeze(rpn_anchor_pred)
rpn_bbox_regression_pred = tf.squeeze(rpn_bbox_regression_pred)
shape1 = tf.shape(rpn_anchor_pred)
print(rpn_anchor_pred.shape, rpn_bbox_regression_pred.shape)
# flatten the pred of anchor to get top N values and indices
rpn_anchor_pred = tf.reshape(rpn_anchor_pred, (-1, ))
n_anchor_proposal = Param.ANCHOR_PROPOSAL_N
top_values, top_indices = tf.math.top_k(
rpn_anchor_pred,
n_anchor_proposal) # top_k has sort function. it's important here
top_indices = tf.gather_nd(
top_indices,
tf.where(tf.greater(top_values, Param.ANCHOR_THRESHOLD)))
top_values = tf.gather_nd(
top_values, tf.where(tf.greater(top_values,
Param.ANCHOR_THRESHOLD)))
debug_print('top values', top_values)
# test_indices = tf.where(tf.greater(tf.reshape(RPN_Anchor_Pred, (-1,)), 0.9))
# print(test_indices)
top_indices = tf.reshape(top_indices, (-1, 1))
debug_print('top indices', top_indices)
update_value = tf.math.add(top_values, 1)
rpn_anchor_pred = tf.tensor_scatter_nd_update(rpn_anchor_pred,
top_indices,
update_value)
rpn_anchor_pred = tf.reshape(rpn_anchor_pred, shape1)
# --- find the base boxes ---
anchor_pred_top_indices = tf.where(tf.greater(rpn_anchor_pred, 1))
debug_print('original_indices shape', anchor_pred_top_indices.shape)
debug_print('original_indices', anchor_pred_top_indices)
base_boxes = tf.gather_nd(self.anchor_candidates,
anchor_pred_top_indices)
debug_print('base_boxes shape', base_boxes.shape)
debug_print('base_boxes', base_boxes)
base_boxes = np.array(base_boxes)
# --- find the bbox_regs ---
# flatten the bbox_reg by last dim to use top_indices to get final_box_reg
rpn_bbox_regression_pred_shape = tf.shape(rpn_bbox_regression_pred)
rpn_bbox_regression_pred = tf.reshape(
rpn_bbox_regression_pred, (-1, rpn_bbox_regression_pred_shape[-1]))
debug_print('RPN_BBOX_Regression_Pred shape',
rpn_bbox_regression_pred.shape)
final_box_reg = tf.gather_nd(rpn_bbox_regression_pred, top_indices)
debug_print('final box reg values', final_box_reg)
# Convert to numpy to plot
final_box_reg = np.array(final_box_reg)
debug_print('final box reg shape', final_box_reg.shape)
debug_print('max value of final box reg', np.max(final_box_reg))
final_box = BboxTools.bbox_reg2truebox(base_boxes=base_boxes,
regs=final_box_reg)
# === Non maximum suppression ===
final_box_temp = np.array(final_box).astype(np.int)
nms_boxes_list = []
while final_box_temp.shape[0] > 0:
ious = self.nms_loop_np(final_box_temp)
nms_boxes_list.append(
final_box_temp[0, :]
) # since it's sorted by the value, here we can pick the first one each time.
final_box_temp = final_box_temp[ious < Param.RPN_NMS_THRESHOLD]
debug_print('number of box after nms', len(nms_boxes_list))
# Need to convert above instructions to tf operations
# === visualization part ===
# clip the boxes to make sure they are legal boxes
debug_print('max value of final box', np.max(final_box))
final_box = BboxTools.clip_boxes(final_box, self.IMG_SHAPE)
original_boxes = self.cocotool.get_original_bboxes_list(
image_id=self.cocotool.image_ids[0])
self.cocotool.draw_bboxes(original_image=input1[0],
bboxes=original_boxes,
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='1GroundTruthBoxes')
target_anchor_boxes, target_classes = self.train_data_generator.gen_target_anchor_bboxes_classes_for_debug(
image_id=self.cocotool.image_ids[0])
self.cocotool.draw_bboxes(original_image=input1[0],
bboxes=target_anchor_boxes,
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='2TrueAnchorBoxes')
self.cocotool.draw_bboxes(original_image=input1[0],
bboxes=base_boxes.tolist(),
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='3PredAnchorBoxes')
self.cocotool.draw_bboxes(original_image=input1[0],
bboxes=final_box.tolist(),
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='4PredRegBoxes')
self.cocotool.draw_bboxes(original_image=input1[0],
bboxes=nms_boxes_list,
show=True,
save_file=True,
path=Param.PATH_DEBUG_IMG,
save_name='5PredNMSBoxes')