# Apply refinement deltas to positive anchors
refined_anchors = utils.apply_box_deltas(
positive_anchors, target_rpn_bbox[:positive_anchors.shape[0]] *
model.config.RPN_BBOX_STD_DEV)
log(
"refined_anchors",
refined_anchors,
)
# In[12]:
# Display positive anchors before refinement (dotted) and
# after refinement (solid).
visualize.draw_boxes(image,
boxes=positive_anchors,
refined_boxes=refined_anchors,
ax=get_ax())
# ### 1.b RPN Predictions
#
# Here we run the RPN graph and display its predictions.
# In[13]:
# Run RPN sub-graph
pillar = model.keras_model.get_layer(
"ROI").output # node to start searching from
# TF 1.4 introduces a new version of NMS. Search for both names to support TF 1.3 and 1.4
nms_node = model.ancestor(pillar, "ROI/rpn_non_max_suppression:0")
if nms_node is None:
positive_target_deltas,
(0., 0., 0., 0.), (0.1, 0.1, 0.2, 0.2))
# In[45]:
print('rpn_target_matchs:\t', rpn_target_matchs[0].shape.as_list())
print('rpn_target_deltas:\t', rpn_target_deltas[0].shape.as_list())
print('positive_anchors:\t', positive_anchors.shape.as_list())
print('negative_anchors:\t', negative_anchors.shape.as_list())
print('neutral_anchors:\t', neutral_anchors.shape.as_list())
print('refined_anchors:\t', refined_anchors.shape.as_list())
# In[44]:
visualize.draw_boxes(rgb_img,
boxes=positive_anchors.numpy(),
refined_boxes=refined_anchors.numpy())
plt.show()
# #### 1.b RPN Predictions
# In[15]:
training = False
C2, C3, C4, C5 = model.backbone(batch_imgs, training=training)
P2, P3, P4, P5, P6 = model.neck([C2, C3, C4, C5], training=training)
rpn_feature_maps = [P2, P3, P4, P5, P6]
rcnn_feature_maps = [P2, P3, P4, P5]
print("Positive anchors: {}".format(len(positive_anchor_ids)))
negative_anchor_ids = np.where(rpn_match[b] == -1)[0]
print("Negative anchors: {}".format(len(negative_anchor_ids)))
neutral_anchor_ids = np.where(rpn_match[b] == 0)[0]
print("Neutral anchors: {}".format(len(neutral_anchor_ids)))
# ROI breakdown by class
for c, n in zip(dataset.class_names,
np.bincount(mrcnn_class_ids[b].flatten())):
if n:
print("{:23}: {}".format(c[:20], n))
# Show positive anchors
fig, ax = plt.subplots(1, figsize=(16, 16))
visualize.draw_boxes(sample_image,
boxes=anchors[positive_anchor_ids],
refined_boxes=refined_anchors,
ax=ax)
# In[17]:
# Show negative anchors
visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids])
# In[18]:
# Show neutral anchors. They don't contribute to training.
visualize.draw_boxes(sample_image,
boxes=anchors[np.random.choice(neutral_anchor_ids, 100)])
# ## ROIs
import os
import tensorflow as tf
import numpy as np
import visualize
import cv2
img = cv2.imread('image_demo_random.png')
import matplotlib.image as mpimg
image = mpimg.imread("image_demo_random.png")
bbox = np.array([[10, 10, 20, 20]])
class_ids = [0]
class_names = ['cat']
visualize.display_instances(image, bbox, class_ids, class_names)
visualize.draw_boxes(image, bbox)
'boxes': [], 'labels': [], 'scores': [], 'masks': []
}
score_threshold: float = 0.9
for box_idx, score in enumerate(out['scores']):
if score >= score_threshold:
removed['boxes'].append(out['boxes'][box_idx].tolist())
removed['scores'].append(float(out['scores'][box_idx]))
removed['masks'].append(out['masks'][box_idx])
out = removed
# Draw boxes and its masks.
# [x0, y0, x1, y1]
boxes: List[List[float]] = out['boxes']
labels: List[str] = ['gray' for _ in boxes]
scores: List[float] = removed['scores']
img_boxes = vis.draw_boxes(orig_img, boxes, labels, scores)
img_arr = np.array(orig_img)
masks: List[np.ndarray] = []
for mask in out['masks']:
_, h, w = mask.size()
mask_arr: np.ndarray = mask.reshape(h, w).detach().numpy()
masks.append(mask_arr)
colors: List[Tuple[int, ...]] = [
vis.random_cmap() for _ in range(len(masks))
]
img_mask = vis.apply_masks(img_arr, masks, colors)
img_mask = Image.fromarray(np.uint8(img_mask))
fig = plt.figure()
nms_node = model.ancestor(pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV2:0")
rpn = model.run_graph([image], [
("rpn_class", model.keras_model.get_layer("rpn_class").output),
("pre_nms_anchors", model.ancestor(pillar, "ROI/pre_nms_anchors:0")),
("refined_anchors", model.ancestor(pillar, "ROI/refined_anchors:0")),
("refined_anchors_clipped", model.ancestor(pillar, "ROI/refined_anchors_clipped:0")),
("post_nms_anchor_ix", nms_node),
("proposals", model.keras_model.get_layer("ROI").output),
])
# Show top anchors by score (before refinement)
limit = 100
sorted_anchor_ids = np.argsort(rpn['rpn_class'][:,:,1].flatten())[::-1]
visualize.draw_boxes(image, boxes=model.anchors[sorted_anchor_ids[:limit]], ax=get_ax())
# Show top anchors with refinement. Then with clipping to image boundaries
limit = 50
ax = get_ax(1, 2)
visualize.draw_boxes(image, boxes=rpn["pre_nms_anchors"][0, :limit],
refined_boxes=rpn["refined_anchors"][0, :limit], ax=ax[0])
visualize.draw_boxes(image, refined_boxes=rpn["refined_anchors_clipped"][0, :limit], ax=ax[1])
# Show refined anchors after non-max suppression
limit = 50
ixs = rpn["post_nms_anchor_ix"][:limit]
visualize.draw_boxes(image, refined_boxes=rpn["refined_anchors_clipped"][0, ixs], ax=get_ax())
# Show final proposals
# These are the same as the previous step (refined anchors