def draw_rois(image, rois, refined_rois, mask, class_ids, class_names, limit=10):
"""
anchors: [n, (y1, x1, y2, x2)] list of anchors in image coordinates.
proposals: [n, 4] the same anchors but refined to fit objects better.
"""
masked_image = image.copy()
# Pick random anchors in case there are too many.
ids = np.arange(rois.shape[0], dtype=np.int32)
ids = np.random.choice(
ids, limit, replace=False) if ids.shape[0] > limit else ids
fig, ax = plt.subplots(1, figsize=(12, 12))
if rois.shape[0] > limit:
plt.title("Showing {} random ROIs out of {}".format(
len(ids), rois.shape[0]))
else:
plt.title("{} ROIs".format(len(ids)))
# Show area outside image boundaries.
ax.set_ylim(image.shape[0] + 20, -20)
ax.set_xlim(-50, image.shape[1] + 20)
ax.axis('off')
for i, id in enumerate(ids):
color = np.random.rand(3)
class_id = class_ids[id]
# ROI
y1, x1, y2, x2 = rois[id]
p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,
edgecolor=color if class_id else "gray",
facecolor='none', linestyle="dashed")
ax.add_patch(p)
# Refined ROI
if class_id:
ry1, rx1, ry2, rx2 = refined_rois[id]
p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2,
edgecolor=color, facecolor='none')
ax.add_patch(p)
# Connect the top-left corners of the anchor and proposal for easy visualization
ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color))
# Label
label = class_names[class_id]
ax.text(rx1, ry1 + 8, "{}".format(label),
color='w', size=11, backgroundcolor="none")
# Mask
m = utils.unmold_mask(mask[id], rois[id]
[:4].astype(np.int32), image.shape)
masked_image = apply_mask(masked_image, m, color)
ax.imshow(masked_image)
# Print stats
print("Positive ROIs: ", class_ids[class_ids > 0].shape[0])
print("Negative ROIs: ", class_ids[class_ids == 0].shape[0])
print("Positive Ratio: {:.2f}".format(
class_ids[class_ids > 0].shape[0] / class_ids.shape[0]))
def draw_rois(image,
rois,
refined_rois,
mask,
class_ids,
class_names,
limit=10):
"""
anchors: [n, (y1, x1, y2, x2)] list of anchors in image coordinates.
proposals: [n, 4] the same anchors but refined to fit objects better.
"""
masked_image = image.copy()
# Pick random anchors in case there are too many.
ids = np.arange(rois.shape[0], dtype=np.int32)
ids = np.random.choice(ids, limit,
replace=False) if ids.shape[0] > limit else ids
fig, ax = plt.subplots(1, figsize=(12, 12))
if rois.shape[0] > limit:
plt.title("Showing {} random ROIs out of {}".format(
len(ids), rois.shape[0]))
else:
plt.title("{} ROIs".format(len(ids)))
# Show area outside image boundaries.
#ax.set_ylim(image.shape[0] + 20, -20)
#ax.set_xlim(-50, image.shape[1] + 20)
#ax.axis('off')
for i, id in enumerate(ids):
color = np.random.rand(3)
class_id = class_ids[id]
# ROI
y1, x1, y2, x2 = rois[id]
p = patches.Rectangle((x1, y1),
x2 - x1,
y2 - y1,
linewidth=2,
edgecolor=color if class_id else "gray",
facecolor='none',
linestyle="dashed")
ax.add_patch(p)
# Refined ROI
if class_id:
ry1, rx1, ry2, rx2 = refined_rois[id]
p = patches.Rectangle((rx1, ry1),
rx2 - rx1,
ry2 - ry1,
linewidth=2,
edgecolor=color,
facecolor='none')
ax.add_patch(p)
# Connect the top-left corners of the anchor and proposal for easy visualization
ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color))
# Label
label = class_names[class_id]
ax.text(rx1,
ry1 + 8,
"{}".format(label),
color='w',
size=11,
backgroundcolor="none")
# Mask
m = utils.unmold_mask(mask[id], rois[id][:4].astype(np.int32),
image.shape)
masked_image = apply_mask(masked_image, m, color)
ax.imshow(masked_image, origin="lower", cmap="jet")
#ax.imshow(masked_image, cmap="jet")
# Print stats
print("Positive ROIs: ", class_ids[class_ids > 0].shape[0])
print("Negative ROIs: ", class_ids[class_ids == 0].shape[0])
print("Positive Ratio: {:.5f}".format(class_ids[class_ids > 0].shape[0] /
class_ids.shape[0]))
])
# Get detection class IDs. Trim zero padding.
det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)
det_count = np.where(det_class_ids == 0)[0][0]
det_class_ids = det_class_ids[:det_count]
print("{} detections: {}".format(det_count,
np.array(dataset.class_names)[det_class_ids]))
# Masks
det_boxes = utils.denorm_boxes(mrcnn["detections"][0, :, :4], image.shape[:2])
det_mask_specific = np.array(
[mrcnn["masks"][0, i, :, :, c] for i, c in enumerate(det_class_ids)])
det_masks = np.array([
utils.unmold_mask(m, det_boxes[i], image.shape)
for i, m in enumerate(det_mask_specific)
])
log("det_mask_specific", det_mask_specific)
log("det_masks", det_masks)
display_images(det_mask_specific[:4] * 255, cmap="Blues", interpolation="none")
display_images(det_masks[:4] * 255, cmap="Blues", interpolation="none")
#%%
# Visualize Activations
# In some cases it helps to look at the output
# from different layers and visualize them to catch issues
# and odd patterns.
])
# Get detection class IDs. Trim zero padding.
det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)
det_count = np.where(det_class_ids == 0)[0][0]
det_class_ids = det_class_ids[:det_count]
print("{} detections: {}".format(
det_count, np.array(dataset.class_names)[det_class_ids]))
# Masks
det_boxes = utils.denorm_boxes(mrcnn["detections"][0, :, :4], image.shape[:2])
det_mask_specific = np.array([mrcnn["masks"][0, i, :, :, c]
for i, c in enumerate(det_class_ids)])
det_masks = np.array([utils.unmold_mask(m, det_boxes[i], image.shape)
for i, m in enumerate(det_mask_specific)])
log("det_mask_specific", det_mask_specific)
log("det_masks", det_masks)
display_images(det_mask_specific[:4] * 255, cmap="Blues", interpolation="none")
# Get activations of a few sample layers
activations = model.run_graph([image], [
("input_image", tf.identity(model.keras_model.get_layer("input_image").output)),
("res2c_out", model.keras_model.get_layer("res2c_out").output),
("res3c_out", model.keras_model.get_layer("res3c_out").output),
("rpn_bbox", model.keras_model.get_layer("rpn_bbox").output),
("roi", model.keras_model.get_layer("ROI").output),
])
def unmold_detections(detections, mrcnn_mask, original_image_shape,
image_shape, window):
"""Reformats the detections of one image from the format of the neural
network output to a format suitable for use in the rest of the
application.
detections: [N, (y1, x1, y2, x2, class_id, score)] in normalized coordinates
mrcnn_mask: [N, height, width, num_classes]
original_image_shape: [H, W, C] Original image shape before resizing
image_shape: [H, W, C] Shape of the image after resizing and padding
window: [y1, x1, y2, x2] Pixel coordinates of box in the image where the real
image is excluding the padding.
Returns:
boxes: [N, (y1, x1, y2, x2)] Bounding boxes in pixels
class_ids: [N] Integer class IDs for each bounding box
scores: [N] Float probability scores of the class_id
masks: [height, width, num_instances] Instance masks
"""
# How many detections do we have?
# Detections array is padded with zeros. Find the first class_id == 0.
zero_ix = np.where(detections[:, 4] == 0)[0]
N = zero_ix[0] if zero_ix.shape[0] > 0 else detections.shape[0]
# Extract boxes, class_ids, scores, and class-specific masks
boxes = detections[:N, :4]
class_ids = detections[:N, 4].astype(np.int32)
scores = detections[:N, 5]
masks = mrcnn_mask[np.arange(N), :, :, class_ids]
# Translate normalized coordinates in the resized image to pixel
# coordinates in the original image before resizing
window = utils.norm_boxes(window, image_shape[:2])
wy1, wx1, wy2, wx2 = window
shift = np.array([wy1, wx1, wy1, wx1])
wh = wy2 - wy1 # window height
ww = wx2 - wx1 # window width
scale = np.array([wh, ww, wh, ww])
# Convert boxes to normalized coordinates on the window
boxes = np.divide(boxes - shift, scale)
# Convert boxes to pixel coordinates on the original image
boxes = utils.denorm_boxes(boxes, original_image_shape[:2])
# Filter out detections with zero area. Happens in early training when
# network weights are still random
exclude_ix = np.where((boxes[:, 2] - boxes[:, 0]) *
(boxes[:, 3] - boxes[:, 1]) <= 0)[0]
if exclude_ix.shape[0] > 0:
boxes = np.delete(boxes, exclude_ix, axis=0)
class_ids = np.delete(class_ids, exclude_ix, axis=0)
scores = np.delete(scores, exclude_ix, axis=0)
masks = np.delete(masks, exclude_ix, axis=0)
N = class_ids.shape[0]
# Resize masks to original image size and set boundary threshold.
full_masks = []
for i in range(N):
# Convert neural network mask to full size mask
full_mask = utils.unmold_mask(masks[i], boxes[i], original_image_shape)
full_masks.append(full_mask)
full_masks = np.stack(full_masks, axis=-1)\
if full_masks else np.empty(original_image_shape[:2] + (0,))
return boxes, class_ids, scores, full_masks
def unmold_detections(self, detections, mrcnn_mask, image_shape, window):
'''
Reformats the detections of one image from the format of the neural
network output to a format suitable for use in the rest of the application.
detections : [N, (y1, x1, y2, x2, class_id, score)]
mrcnn_mask : [N, height, width, num_classes]
image_shape : [height, width, depth] Original size of the image before resizing
window : [y1, x1, y2, x2] Box in the image where the real image is
excluding the padding.
Returns:
boxes : [N, (y1, x1, y2, x2)] Bounding boxes in pixels
class_ids : [N] Integer class IDs for each bounding box
scores : [N] Float probability scores of the class_id
masks : [height, width, num_instances] Instance masks
'''
# print('>>> unmold_detections ')
# print(' detections.shape : ', detections.shape)
# print(' mrcnn_mask.shape : ', mrcnn_mask.shape)
# print(' image_shape.shape: ', image_shape)
# print(' window.shape : ', window)
# print(detections)
# How many detections do we have?
# Detections array is padded with zeros. detections[:,4] identifies the class
# Find all rows in detection array with class_id == 0 , and place their row indices
# into zero_ix. zero_ix[0] will identify the first row with class_id == 0.
np.set_printoptions(linewidth=100)
zero_ix = np.where(detections[:, 4] == 0)[0]
N = zero_ix[0] if zero_ix.shape[0] > 0 else detections.shape[0]
# print(' np.where() \n', np.where(detections[:, 4] == 0))
# print(' zero_ix.shape : ', zero_ix.shape)
# print(' N is :', N)
# Extract boxes, class_ids, scores, and class-specific masks
boxes = detections[:N, :4]
class_ids = detections[:N, 4].astype(np.int32)
scores = detections[:N, 5]
masks = mrcnn_mask[np.arange(N), :, :, class_ids]
# Compute scale and shift to translate coordinates to image domain.
h_scale = image_shape[0] / (window[2] - window[0])
w_scale = image_shape[1] / (window[3] - window[1])
scale = min(h_scale, w_scale)
shift = window[:2] # y, x
scales = np.array([scale, scale, scale, scale])
shifts = np.array([shift[0], shift[1], shift[0], shift[1]])
# Translate bounding boxes to image domain
boxes = np.multiply(boxes - shifts, scales).astype(np.int32)
# Filter out detections with zero area. Often only happens in early
# stages of training when the network weights are still a bit random.
exclude_ix = np.where((boxes[:, 2] - boxes[:, 0]) *
(boxes[:, 3] - boxes[:, 1]) <= 0)[0]
if exclude_ix.shape[0] > 0:
boxes = np.delete(boxes, exclude_ix, axis=0)
class_ids = np.delete(class_ids, exclude_ix, axis=0)
scores = np.delete(scores, exclude_ix, axis=0)
masks = np.delete(masks, exclude_ix, axis=0)
N = class_ids.shape[0]
# Resize masks to original image size and set boundary threshold.
full_masks = []
for i in range(N):
# Convert neural network mask to full size mask
full_mask = utils.unmold_mask(masks[i], boxes[i], image_shape)
full_masks.append(full_mask)
full_masks = np.stack(full_masks, axis=-1)\
if full_masks else np.empty((0,) + masks.shape[1:3])
return boxes, class_ids, scores, full_masks
("masks", model.keras_model.get_layer("mrcnn_mask").output),
])
detectedClasses = maskRCnnOutput['detections'][0, :, 4].astype(np.int32)
numberOfDetections = np.where(det_class_ids == 0)[0][0]
detectedClasses = detectedClasses[:numberOfDetections]
print("{} detections: {}".format(
numberOfDetections,
np.array(dataset.class_names)[detectedClasses]))
detectedBoundingBoxes = utils.denorm_boxes(
maskrcnnOutput["detections"][0, :, :4], (512, 512))
detectedmasks = np.array([
maskRCnnOutput["masks"][0, i, :, :, c]
for i, c in enumerate(detectedClasses)
])
# detectedMasks reshaped to (512,512,3) to use as input for ensemble model
detectedMasks = np.array([
utils.unmold_mask(m, detectedBoundingBoxes[i], (512, 512, 3))
for i, m in enumerate(detectedmasks)
])
print(np.unique(detectedMasks))
a = np.sum(detectedMasks, axis=0) > 0
# create directory using terminal if it doesn't already exist
plt.imsave(fname="maskrcnn_final_test_ensemble_input/" +
str(dataset.image_reference(iid) + ".png"),
arr=a,
cmap="Blues")
for image_id in dataset.image_ids:
print("Image id: ", image_id)
filename = "maskrcnn_ensemble_input/" + str(dataset.image_reference(image_id)) + ".png"
#if os.path.isfile(filename):
# continue
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
mrcnn = model.run_graph([image], [
("detections", model.keras_model.get_layer("mrcnn_detection").output),
("masks", model.keras_model.get_layer("mrcnn_mask").output),
])
det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)
det_count = np.where(det_class_ids == 0)[0][0]
det_class_ids = det_class_ids[:det_count]
print("{} detections: {}".format(det_count, np.array(dataset.class_names)[det_class_ids]))
det_boxes = utils.denorm_boxes(mrcnn["detections"][0, :, :4], (512,512))
det_mask_specific = np.array([mrcnn["masks"][0, i, :, :, c]
for i, c in enumerate(det_class_ids)])
det_masks = np.array([utils.unmold_mask(m, det_boxes[i], (512,512,3))
for i, m in enumerate(det_mask_specific)])
log("det_mask_specific", det_mask_specific)
log("det_masks", det_masks)
print(np.unique(det_masks))
a = np.sum(det_masks, axis=0) > 0
# create directory using terminal if it doesn't already exist
plt.imsave(fname = "maskrcnn_final_ensemble_input/" + str(dataset.image_reference(image_id)+".png"), arr = a, cmap="Blues")
"""
def draw_rois(image,
rois,
refined_rois,
mask,
class_ids,
class_names,
limit=10):
"""
anchors: [n, (y1, x1, y2, x2)] 图像坐标中的锚点列表.
proposals: [n, 4] 相同锚点，经过精炼以选择更好地合适对象.
"""
masked_image = image.copy()
# 若有太多，则选择随机几个锚来绘图draw_box.
ids = np.arange(rois.shape[0], dtype=np.int32)
ids = np.random.choice(ids, limit,
replace=False) if ids.shape[0] > limit else ids
fig, ax = plt.subplots(1, figsize=(12, 12))
if rois.shape[0] > limit:
plt.title("Showing {} random ROIs out of {}".format(
len(ids), rois.shape[0]))
else:
plt.title("{} ROIs".format(len(ids)))
# 显示超出图像边界的区域.
ax.set_ylim(image.shape[0] + 20, -20)
ax.set_xlim(-50, image.shape[1] + 20)
ax.axis('off')
for i, id in enumerate(ids):
color = np.random.rand(3)
class_id = class_ids[id]
# ROI
y1, x1, y2, x2 = rois[id]
p = patches.Rectangle((x1, y1),
x2 - x1,
y2 - y1,
linewidth=2,
edgecolor=color if class_id else "gray",
facecolor='none',
linestyle="dashed")
ax.add_patch(p)
# 调整 ROI
if class_id:
ry1, rx1, ry2, rx2 = refined_rois[id]
p = patches.Rectangle((rx1, ry1),
rx2 - rx1,
ry2 - ry1,
linewidth=2,
edgecolor=color,
facecolor='none')
ax.add_patch(p)
# 连接锚点和建议框的左上角来便于可视化
ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color))
# Label
label = class_names[class_id]
ax.text(rx1,
ry1 + 8,
"{}".format(label),
color='w',
size=11,
backgroundcolor="none")
# Mask
m = utils.unmold_mask(mask[id], rois[id][:4].astype(np.int32),
image.shape)
masked_image = apply_mask(masked_image, m, color)
ax.imshow(masked_image)
# 输出统计
print("Positive ROIs: ", class_ids[class_ids > 0].shape[0])
print("Negative ROIs: ", class_ids[class_ids == 0].shape[0])
print("Positive Ratio: {:.2f}".format(class_ids[class_ids > 0].shape[0] /
class_ids.shape[0]))
def calculate_mask(model, graph, image, rois):
'''
inputs:
model: sd mask rcnn model
graph: model graph (provide keras to calculate)
image: filled hole 8-bit depth cropped image (specified with rois)
rois: Full image ROI patch(x1,y1,x2,y2)
returns:
det_masks: mask on 512*512 scale (values=0 or 1)
det_mask_specific: predict mask on CNN (28*28) (values=0 or 1)
resized_mask_image: max facet detected image (ROI Size)
'''
print('MAKING SD MASK RCNN PREDICTIONS')
class_names = ['bg', 'fg']
image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
print(image.shape)
# Resize image for net
image, _, _, _, _ = resize_image(image, max_dim=512)
#cv2.imshow("Test",image)
#cv2.waitKey(0)
# Get predictions of mask head
#model.keras_model._make_predict_function()
with graph.as_default():
mrcnn = model.run_graph([image], [
("detections",
model.keras_model.get_layer("mrcnn_detection").output),
("masks", model.keras_model.get_layer("mrcnn_mask").output),
])
# Get detection class IDs. Trim zero padding.
det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)
det_count = np.where(det_class_ids == 0)[0][0]
det_class_ids = det_class_ids[:det_count]
print("{} detections: {}".format(det_count,
np.array(class_names)[det_class_ids]))
# Masks
det_boxes = m_utils.denorm_boxes(mrcnn["detections"][0, :, :4],
image.shape[:2])
det_mask_specific = np.array(
[mrcnn["masks"][0, i, :, :, c] for i, c in enumerate(det_class_ids)])
det_masks = np.array([
m_utils.unmold_mask(m, det_boxes[i], image.shape)
for i, m in enumerate(det_mask_specific)
])
if det_count != 0:
log("det_mask_specific", det_mask_specific)
log("det_masks", det_masks)
else:
print("No facet detected!!")
return
resized_mask_image, no_padding_range = detect_show_instance(
image, model, graph, class_names, rois)
return det_masks, det_mask_specific, resized_mask_image, no_padding_range