def execute(image):
if (config is None or model is None):
init()
molded_image, image_meta, windows = model.mold_inputs([image])
window = utils.norm_boxes(windows, molded_image[0].shape[:2])[0]
anchors = model.get_anchors(molded_image[0].shape)
anchors = np.broadcast_to(anchors,
(model.config.BATCH_SIZE, ) + anchors.shape)
dects, probs, deltas, masks, proposals, _, _ = model.keras_model.predict(
[molded_image, image_meta, anchors], verbose=0)
# Class ID, and score per proposal
class_ids = np.argmax(probs[0], axis=1)
class_scores = probs[0, np.arange(class_ids.shape[0]), class_ids]
refined_rois = refine_proposals(proposals, class_ids, deltas, window)
keep = filter_rois(refined_rois, class_ids, class_scores)
class_scores = probs[0][keep]
# Detections are in different order than the indices in the "keep" array.
# Thus, we need to identify the matching detections for correct ordering of
# class_scores.
bounding_boxes = utils.denorm_boxes(dects[0, :len(keep), :4],
molded_image[0].shape[:2])
roi_boxes = utils.denorm_boxes(refined_rois[keep],
molded_image[0].shape[:2])
perm = []
for i in np.arange(len(keep)):
perm = np.append(
perm,
np.where(np.all(roi_boxes == bounding_boxes[i],
axis=1))[0][0]).astype(np.int32)
class_scores = class_scores[perm]
bounding_boxes, _, _, segmentation = model.unmold_detections(
dects[0], masks[0], image.shape, molded_image[0].shape, windows[0])
keep_fusion = filter_fusion(segmentation)
return generate_result(bounding_boxes[keep_fusion],
segmentation[:, :, keep_fusion],
class_scores[keep_fusion])
def unmold_predictions(self, detections, mrcnn_mask, original_image_shape,
image_shape, window):
'''
copied from mrcnn, changes in loop allows return of probability map
'''
# Detections array is padded with zeros. Find the first class_id == 0.
zero_ix = np.where(detections[:, 4] == 0)[0]
N = zero_ix[0] if zero_ix.shape[0] > 0 else detections.shape[0]
# Extract boxes, class_ids, scores, and class-specific masks
boxes = detections[:N, :4]
class_ids = detections[:N, 4].astype(np.int32)
scores = detections[:N, 5]
masks = mrcnn_mask[np.arange(N), :, :, class_ids]
# Translate normalized coordinates in the resized image to pixel
# coordinates in the original image before resizing
window = utils.norm_boxes(window, image_shape[:2])
wy1, wx1, wy2, wx2 = window
shift = np.array([wy1, wx1, wy1, wx1])
wh = wy2 - wy1 # window height
ww = wx2 - wx1 # window width
scale = np.array([wh, ww, wh, ww])
# Convert boxes to normalized coordinates on the window
boxes = np.divide(boxes - shift, scale)
# Convert boxes to pixel coordinates on the original image
boxes = utils.denorm_boxes(boxes, original_image_shape[:2])
# Filter out detections with zero area. Happens in early training when
# network weights are still random
exclude_ix = np.where((boxes[:, 2] - boxes[:, 0]) *
(boxes[:, 3] - boxes[:, 1]) <= 0)[0]
if exclude_ix.shape[0] > 0:
boxes = np.delete(boxes, exclude_ix, axis=0)
class_ids = np.delete(class_ids, exclude_ix, axis=0)
scores = np.delete(scores, exclude_ix, axis=0)
masks = np.delete(masks, exclude_ix, axis=0)
N = class_ids.shape[0]
# Resize masks to original image size and set boundary threshold.
full_masks = []
full_pmap = []
for i in range(N):
# Convert neural network mask to full size mask
threshold = 0.5
y1, x1, y2, x2 = boxes[i]
full_pmap.append(
skimage.transform.resize(masks[i], (y2 - y1, x2 - x1),
order=1,
mode="constant"))
mask = np.where(full_pmap[i] >= threshold, 1, 0).astype(np.bool)
# Put the mask in the right location.
full_mask = np.zeros(image_shape[:2], dtype=np.bool)
full_mask[y1:y2, x1:x2] = mask
# full_mask = utils.unmold_mask(masks[i], boxes[i], original_image_shape)
full_masks.append(full_mask)
full_masks = np.stack(full_masks, axis=-1)\
if full_masks else np.empty(original_image_shape[:2] + (0,))
return boxes, class_ids, scores, full_masks, full_pmap
def unmold_detections(detections, mrcnn_mask, original_image_shape,
image_shape, window):
zero_ix = np.where(detections[:, 4] == 0)[0]
N = zero_ix[0] if zero_ix.shape[0] > 0 else detections.shape[0]
# Extract boxes, class_ids, scores, and class-specific masks
boxes = detections[:N, :4]
class_ids = detections[:N, 4].astype(np.int32)
scores = detections[:N, 5]
masks = mrcnn_mask[np.arange(N), :, :, class_ids] # [N, 28*28, class]
# Translate normalized coordinates in the resized image to pixel
# coordinates in the original image before resizing
window = utils.norm_boxes(window, image_shape[:2])
wy1, wx1, wy2, wx2 = window
shift = np.array([wy1, wx1, wy1, wx1])
wh = wy2 - wy1 # window height
ww = wx2 - wx1 # window width
scale = np.array([wh, ww, wh, ww])
# Convert boxes to normalized coordinates on the window
boxes = np.divide(boxes - shift, scale)
# Convert boxes to pixel coordinates on the original image
boxes = utils.denorm_boxes(boxes, original_image_shape[:2])
# Filter out detections with zero area. Happens in early training when
# network weights are still random
exclude_ix = np.where((boxes[:, 2] - boxes[:, 0]) *
(boxes[:, 3] - boxes[:, 1]) <= 0)[0]
if exclude_ix.shape[0] > 0:
boxes = np.delete(boxes, exclude_ix, axis=0)
class_ids = np.delete(class_ids, exclude_ix, axis=0)
scores = np.delete(scores, exclude_ix, axis=0)
masks = np.delete(masks, exclude_ix, axis=0)
N = class_ids.shape[0]
# Resize masks to original image size and set boundary threshold.
full_masks = []
for i in range(N):
# Convert neural network mask to full size mask
full_mask = utils.unmold_mask(masks[i], boxes[i], original_image_shape)
full_masks.append(full_mask)
full_masks = np.stack(full_masks, axis=-1)\
if full_masks else np.empty(original_image_shape[:2] + (0,))
return boxes, class_ids, scores, full_masks
def display_mrcnn_mask_prediction():
#################################### Mask Targets ##############################################
# gt_masks 的 shape 为 (image_height, image_width, num_instances)
resized_image, image_meta, gt_class_ids, gt_bboxes, gt_masks = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
display_images(np.transpose(gt_masks, [2, 0, 1]), cmap="Blues")
# Get predictions of mask head
mrcnn = model.run_graph([resized_image], [
("detections", model.keras_model.get_layer("mrcnn_detection").output),
("masks", model.keras_model.get_layer("mrcnn_mask").output),
])
# Get detection class IDs. Trim zero padding.
det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)
padding_start_ix = np.where(det_class_ids == 0)[0][0]
det_class_ids = det_class_ids[:padding_start_ix]
print("{} detections: {}".format(
padding_start_ix,
np.array(dataset.class_names)[det_class_ids]))
# Masks
det_boxes = utils.denorm_boxes(mrcnn["detections"][0, :, :4],
resized_image.shape[:2])
# mrcnn['masks'] 的 shape 为 (batch_size, num_instances, mask_height, mask_width, num_classes)
det_mask_specific = np.array(
[mrcnn["masks"][0, i, :, :, c] for i, c in enumerate(det_class_ids)])
det_masks = np.array([
utils.unmold_mask(mask, det_boxes[i], resized_image.shape)
for i, mask in enumerate(det_mask_specific)
])
log("det_mask_specific", det_mask_specific)
display_images(det_mask_specific[:4] * 255,
cmap="Blues",
interpolation="none")
log("det_masks", det_masks)
display_images(det_masks[:4] * 255, cmap="Blues", interpolation="none")
if verbose:
info = dataset.image_info[image_id]
meta = modellib.parse_image_meta(image_meta[np.newaxis, ...])
print("{:3} {} AP: {:.2f}".format(
meta["image_id"][0], meta["original_image_shape"][0], ap))
return APs
# Run on validation set
limit = 5
APs = compute_batch_ap(dataset, dataset.image_ids[:limit])
print("Mean AP overa {} images: {:.4f}".format(len(APs), np.mean(APs)))
# Get anchors and convert to pixel coordinates
anchors = model.get_anchors(image.shape)
anchors = utils.denorm_boxes(anchors, image.shape[:2])
log("anchors", anchors)
# Generate RPN trainig targets
# target_rpn_match is 1 for positive anchors, -1 for negative anchors
# and 0 for neutral anchors.
target_rpn_match, target_rpn_bbox = modellib.build_rpn_targets(
image.shape, anchors, gt_class_id, gt_bbox, model.config)
log("target_rpn_match", target_rpn_match)
log("target_rpn_bbox", target_rpn_bbox)
positive_anchor_ix = np.where(target_rpn_match[:] == 1)[0]
negative_anchor_ix = np.where(target_rpn_match[:] == -1)[0]
neutral_anchor_ix = np.where(target_rpn_match[:] == 0)[0]
positive_anchors = anchors[positive_anchor_ix]
negative_anchors = anchors[negative_anchor_ix]
print("{:3} {} AP: {:.2f}".format(
meta["image_id"][0], meta["original_image_shape"][0], ap))
return APs
# Run on validation set
limit = 5
APs = compute_batch_ap(dataset, dataset.image_ids[:limit])
print("Mean AP overa {} images: {:.4f}".format(len(APs), np.mean(APs)))
###################
# A RPN Targets
###################
# Get anchors and convert to pixel coordinates
anchors = model.get_anchors(image.shape)
anchors = utils.denorm_boxes(anchors, image.shape[:2])
log("anchors", anchors)
# Generate RPN trainig targets
# target_rpn_match is 1 for positive anchors, -1 for negative anchors
# and 0 for neutral anchors.
target_rpn_match, target_rpn_bbox = modellib.build_rpn_targets(
image.shape, anchors, gt_class_id, gt_bbox, model.config)
log("target_rpn_match", target_rpn_match)
log("target_rpn_bbox", target_rpn_bbox)
positive_anchor_ix = np.where(target_rpn_match[:] == 1)[0]
negative_anchor_ix = np.where(target_rpn_match[:] == -1)[0]
neutral_anchor_ix = np.where(target_rpn_match[:] == 0)[0]
positive_anchors = anchors[positive_anchor_ix]
negative_anchors = anchors[negative_anchor_ix]
model.ancestor(pillar, "ROI/refined_anchors_clipped:0")),
("post_nms_anchor_ix", nms_node),
("proposals", model.keras_model.get_layer("ROI").output),
])
# Show top anchors by score (before refinement)
limit = 100
sorted_anchor_ids = np.argsort(rpn['rpn_class'][:, :, 1].flatten())[::-1]
visualize.draw_boxes(image,
boxes=model.anchors[sorted_anchor_ids[:limit]],
ax=get_ax())
# Show top anchors with refinement. Then with clipping to image boundaries
limit = 50
ax = get_ax(1, 2)
pre_nms_anchors = utils.denorm_boxes(rpn["pre_nms_anchors"][0],
image.shape[:2])
refined_anchors = utils.denorm_boxes(rpn["refined_anchors"][0],
image.shape[:2])
refined_anchors_clipped = utils.denorm_boxes(rpn["refined_anchors_clipped"][0],
image.shape[:2])
visualize.draw_boxes(image,
boxes=pre_nms_anchors[:limit],
refined_boxes=refined_anchors[:limit],
ax=ax[0])
visualize.draw_boxes(image,
refined_boxes=refined_anchors_clipped[:limit],
ax=ax[1])
# Show refined anchors after non-max suppression
limit = 50
ixs = rpn["post_nms_anchor_ix"][:limit]
def unmold_detections(detections, mrcnn_mask, original_image_shape,
image_shape, window):
"""Reformats the detections of one image from the format of the neural
network output to a format suitable for use in the rest of the
application.
detections: [N, (y1, x1, y2, x2, class_id, score)] in normalized coordinates
mrcnn_mask: [N, height, width, num_classes]
original_image_shape: [H, W, C] Original image shape before resizing
image_shape: [H, W, C] Shape of the image after resizing and padding
window: [y1, x1, y2, x2] Pixel coordinates of box in the image where the real
image is excluding the padding.
Returns:
boxes: [N, (y1, x1, y2, x2)] Bounding boxes in pixels
class_ids: [N] Integer class IDs for each bounding box
scores: [N] Float probability scores of the class_id
masks: [height, width, num_instances] Instance masks
"""
# How many detections do we have?
# Detections array is padded with zeros. Find the first class_id == 0.
zero_ix = np.where(detections[:, 4] == 0)[0]
N = zero_ix[0] if zero_ix.shape[0] > 0 else detections.shape[0]
# Extract boxes, class_ids, scores, and class-specific masks
boxes = detections[:N, :4]
class_ids = detections[:N, 4].astype(np.int32)
scores = detections[:N, 5]
masks = mrcnn_mask[np.arange(N), :, :, class_ids]
# Translate normalized coordinates in the resized image to pixel
# coordinates in the original image before resizing
window = utils.norm_boxes(window, image_shape[:2])
wy1, wx1, wy2, wx2 = window
shift = np.array([wy1, wx1, wy1, wx1])
wh = wy2 - wy1 # window height
ww = wx2 - wx1 # window width
scale = np.array([wh, ww, wh, ww])
# Convert boxes to normalized coordinates on the window
boxes = np.divide(boxes - shift, scale)
# Convert boxes to pixel coordinates on the original image
boxes = utils.denorm_boxes(boxes, original_image_shape[:2])
# Filter out detections with zero area. Happens in early training when
# network weights are still random
exclude_ix = np.where((boxes[:, 2] - boxes[:, 0]) *
(boxes[:, 3] - boxes[:, 1]) <= 0)[0]
if exclude_ix.shape[0] > 0:
boxes = np.delete(boxes, exclude_ix, axis=0)
class_ids = np.delete(class_ids, exclude_ix, axis=0)
scores = np.delete(scores, exclude_ix, axis=0)
masks = np.delete(masks, exclude_ix, axis=0)
N = class_ids.shape[0]
# Resize masks to original image size and set boundary threshold.
full_masks = []
for i in range(N):
# Convert neural network mask to full size mask
full_mask = utils.unmold_mask(masks[i], boxes[i], original_image_shape)
full_masks.append(full_mask)
full_masks = np.stack(full_masks, axis=-1)\
if full_masks else np.empty(original_image_shape[:2] + (0,))
return boxes, class_ids, scores, full_masks
def generate_multiple_bbox(model, image):
mrcnn = model.run_graph(
[image],
[
("proposals", model.keras_model.get_layer("ROI").output),
("probs", model.keras_model.get_layer("mrcnn_class").output),
("deltas", model.keras_model.get_layer("mrcnn_bbox").output),
# ("masks", model.keras_model.get_layer("mrcnn_mask").output),
# ("detections", model.keras_model.get_layer("mrcnn_detection").output),
])
Proposal_limit = 1000
final_boxes_limit = 15
proposals = utils.denorm_boxes(mrcnn['proposals'][0, :Proposal_limit],
image.shape[:2])
roi_class_ids = np.argmax(mrcnn["probs"][0], axis=1)
roi_scores = mrcnn["probs"][0,
np.arange(roi_class_ids.shape[0]),
roi_class_ids]
roi_class_names = np.array(class_names)[roi_class_ids]
roi_positive_ixs = np.where(roi_class_ids > 0)[0]
# How many ROIs vs empty rows?
# print("{} Valid proposals out of {}".format(np.sum(np.any(proposals, axis=1)), proposals.shape[0]))
# print("{} Positive ROIs".format(len(roi_positive_ixs)))
keep = np.where(roi_class_ids > 0)[0]
# print("Keep {} detections:\n{}".format(keep.shape[0], keep))
keep = np.intersect1d(
keep,
np.where(roi_scores >= config.DETECTION_MIN_CONFIDENCE)[0])
# print("Remove boxes below {} confidence. Keep {}:\n{}".format(
# config.DETECTION_MIN_CONFIDENCE, keep.shape[0], keep))
### Refinement
roi_bbox_specific = mrcnn["deltas"][0,
np.arange(proposals.shape[0]),
roi_class_ids]
# Apply bounding box transformations
# Shape: [N, (y1, x1, y2, x2)]
refined_proposals = utils.apply_box_deltas(
proposals, roi_bbox_specific * config.BBOX_STD_DEV).astype(np.int32)
ids = np.arange(0, len(keep) - 1) # Display all
# ids = np.random.randint(0, len(roi_positive_ixs), limit) # Display random sample
# ids = roi_positive_ixs[:-1]
# captions = ["{} {:.3f}".format(class_names[c], s) if c > 0 else ""
# for c, s in zip(roi_class_ids[keep][ids], roi_scores[keep][ids])]
# visualize.draw_boxes(image,
# refined_boxes=refined_proposals[keep][ids],
# captions=captions, title="ROIs After Refinement",
# ax=get_ax())
final_bboxes_before_NMS = []
# colors = random_colors(50)
for bbox, score in zip(refined_proposals[keep][ids][:],
roi_scores[keep][ids][:]):
# centerCoord = (bbox[1] + int((bbox[3] - bbox[1]) / 2), bbox[0] + int((bbox[2] - bbox[0]) / 2))
# visualize.draw_box(bbox)
# color = [i * 255 for i in colors[random.randint(0,30)]]
# x1_y1_x2_y2_score = [bbox[1],bbox[0],bbox[3],bbox[2]]
# image = cv2.rectangle(image, (x1_y1_x2_y2_score[0], x1_y1_x2_y2_score[1]),
# (x1_y1_x2_y2_score[2], x1_y1_x2_y2_score[3]), color)
final_bboxes_before_NMS.append(
[bbox[1], bbox[0], bbox[3], bbox[2], score])
# for bbox in refined_proposals[keep][ids]:
# # print(centerCoord)
# # save_img = cv2.circle(save_img, centerCoord, 4, (255, 0, 255), -1)
# image = cv2.rectangle(image, (bbox[1], bbox[0]), (bbox[3], bbox[2]), (0, 0, 255))
# # save_img = cv2.rectangle(save_img,(box[1],box[0]),(box[3],box[2]),(0,0,255))
return image, final_bboxes_before_NMS
for image_id in dataset.image_ids:
print("Image id: ", image_id)
filename = "maskrcnn_ensemble_input/" + str(dataset.image_reference(image_id)) + ".png"
#if os.path.isfile(filename):
# continue
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
mrcnn = model.run_graph([image], [
("detections", model.keras_model.get_layer("mrcnn_detection").output),
("masks", model.keras_model.get_layer("mrcnn_mask").output),
])
det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)
det_count = np.where(det_class_ids == 0)[0][0]
det_class_ids = det_class_ids[:det_count]
print("{} detections: {}".format(det_count, np.array(dataset.class_names)[det_class_ids]))
det_boxes = utils.denorm_boxes(mrcnn["detections"][0, :, :4], (512,512))
det_mask_specific = np.array([mrcnn["masks"][0, i, :, :, c]
for i, c in enumerate(det_class_ids)])
det_masks = np.array([utils.unmold_mask(m, det_boxes[i], (512,512,3))
for i, m in enumerate(det_mask_specific)])
log("det_mask_specific", det_mask_specific)
log("det_masks", det_masks)
print(np.unique(det_masks))
a = np.sum(det_masks, axis=0) > 0
# create directory using terminal if it doesn't already exist
plt.imsave(fname = "maskrcnn_final_ensemble_input/" + str(dataset.image_reference(image_id)+".png"), arr = a, cmap="Blues")
gt_class_id,
gt_mask,
r['rois'],
r['class_ids'],
r['scores'],
r['masks'],
dataset.class_names,
ax=get_ax(),
show_box=False,
show_mask=False,
iou_threshold=0.5,
score_threshold=0.5)
# Get anchors and convert to pixel coordinates
anchors = model.get_anchors(image.shape)
anchors = utils.denorm_boxes(anchors, image.shape[:2])
log("anchors", anchors)
# Generate RPN trainig targets
# target_rpn_match is 1 for positive anchors, -1 for negative anchors
# and 0 for neutral anchors.
target_rpn_match, target_rpn_bbox = modellib.build_rpn_targets(
image.shape, anchors, gt_class_id, gt_bbox, model.config)
log("target_rpn_match", target_rpn_match)
log("target_rpn_bbox", target_rpn_bbox)
positive_anchor_ix = np.where(target_rpn_match[:] == 1)[0]
negative_anchor_ix = np.where(target_rpn_match[:] == -1)[0]
neutral_anchor_ix = np.where(target_rpn_match[:] == 0)[0]
positive_anchors = anchors[positive_anchor_ix]
negative_anchors = anchors[negative_anchor_ix]
def display_mrcnn_prediction():
resized_image, image_meta, gt_class_ids, gt_bboxes, gt_masks = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
# Get input and output to classifier and mask heads.
mrcnn = model.run_graph([resized_image], [
("proposals", model.keras_model.get_layer("ROI").output),
("probs", model.keras_model.get_layer("mrcnn_class").output),
("deltas", model.keras_model.get_layer("mrcnn_bbox").output),
("masks", model.keras_model.get_layer("mrcnn_mask").output),
("detections", model.keras_model.get_layer("mrcnn_detection").output),
])
ax = get_ax(1, 4)
################################## display detections ###############################################
# Get detection class IDs. Trim zero padding.
det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)
padding_start_ix = np.where(det_class_ids == 0)[0][0]
det_class_ids = det_class_ids[:padding_start_ix]
detections = mrcnn['detections'][0, :padding_start_ix]
log('trimmed_detection', detections)
print("{} detections: {}".format(
padding_start_ix,
np.array(dataset.class_names)[det_class_ids]))
captions = [
"{} {:.3f}".format(dataset.class_names[int(class_id)], score)
if class_id > 0 else ""
for class_id, score in zip(detections[:, 4], detections[:, 5])
]
visualize.draw_boxes(resized_image.copy(),
refined_boxes=utils.denorm_boxes(
detections[:, :4], resized_image.shape[:2]),
visibilities=[2] * len(detections),
captions=captions,
title="Detections",
ax=ax[0])
################################### display proposals ##########################################
# Proposals are in normalized coordinates. Scale them to image coordinates.
h, w = resized_image.shape[:2]
proposals = np.around(mrcnn["proposals"][0] *
np.array([h, w, h, w])).astype(np.int32)
# Class ID, score, and mask per proposal
# mrcnn 的 shape 为 (batch_size, num_proposals=1000, num_classes)
proposal_class_ids = np.argmax(mrcnn["probs"][0], axis=1)
proposal_class_scores = mrcnn["probs"][
0, np.arange(proposal_class_ids.shape[0]), proposal_class_ids]
proposal_class_names = np.array(dataset.class_names)[proposal_class_ids]
proposal_positive_ixs = np.where(proposal_class_ids > 0)[0]
# How many ROIs vs empty rows?
print("{} valid proposals out of {}".format(
np.sum(np.any(proposals, axis=1)), proposals.shape[0]))
print("{} positive ROIs".format(len(proposal_positive_ixs)))
# Class counts
print(list(zip(*np.unique(proposal_class_names, return_counts=True))))
# Display a random sample of proposals.
# Proposals classified as background are dotted, and
# the rest show their class and confidence score.
limit = 200
ixs = np.random.randint(0, proposals.shape[0], limit)
captions = [
"{} {:.3f}".format(dataset.class_names[c], s) if c > 0 else ""
for c, s in zip(proposal_class_ids[ixs], proposal_class_scores[ixs])
]
visualize.draw_boxes(resized_image.copy(),
boxes=proposals[ixs],
visibilities=np.where(proposal_class_ids[ixs] > 0, 2,
1),
captions=captions,
title="Proposals Before Refinement",
ax=ax[1])
#################################### apply bounding box refinement #############################
# Class-specific bounding box shifts.
# mrcnn['deltas'] 的 shape 为 (batch_size, num_proposals=1000, num_classes, 4)
proposal_deltas = mrcnn["deltas"][0,
np.arange(proposals.shape[0]),
proposal_class_ids]
log("proposals_deltas", proposal_deltas)
# Apply bounding box transformations
# Shape: (num_proposals=1000, (y1, x1, y2, x2)]
# NOTE: delta 是不分 normalized coordinates 和 pixel coordinates 的
refined_proposals = utils.apply_box_deltas(
proposals, proposal_deltas * config.BBOX_STD_DEV).astype(np.int32)
log("refined_proposals", refined_proposals)
# Show positive proposals
# ids = np.arange(proposals.shape[0]) # Display all
limit = 5
ids = np.random.randint(0, len(proposal_positive_ixs),
limit) # Display random sample
captions = [
"{} {:.3f}".format(dataset.class_names[class_id], score)
if class_id > 0 else "" for class_id, score in zip(
proposal_class_ids[proposal_positive_ixs][ids],
proposal_class_scores[proposal_positive_ixs][ids])
]
visualize.draw_boxes(
resized_image.copy(),
boxes=proposals[proposal_positive_ixs][ids],
refined_boxes=refined_proposals[proposal_positive_ixs][ids],
visibilities=np.where(
proposal_class_ids[proposal_positive_ixs][ids] > 0, 1, 0),
captions=captions,
title="ROIs After Refinement",
ax=ax[2])
#################################### more steps ################################################
# Remove boxes classified as background
keep_proposal_ixs = np.where(proposal_class_ids > 0)[0]
print("Remove background proposals. Keep {}:\n{}".format(
keep_proposal_ixs.shape[0], keep_proposal_ixs))
# Remove low confidence detections
keep_proposal_ixs = np.intersect1d(
keep_proposal_ixs,
np.where(proposal_class_scores >= config.DETECTION_MIN_CONFIDENCE)[0])
print("Remove proposals below {} confidence. Keep {}:\n{}".format(
config.DETECTION_MIN_CONFIDENCE, keep_proposal_ixs.shape[0],
keep_proposal_ixs))
# Apply per-class non-max suppression
pre_nms_proposals = refined_proposals[keep_proposal_ixs]
pre_nms_proposal_scores = proposal_class_scores[keep_proposal_ixs]
pre_nms_proposal_class_ids = proposal_class_ids[keep_proposal_ixs]
nms_keep_proposal_ixs = []
for class_id in np.unique(pre_nms_proposal_class_ids):
# Pick detections of this class
ixs = np.where(pre_nms_proposal_class_ids == class_id)[0]
# Apply NMS
class_keep = utils.non_max_suppression(pre_nms_proposals[ixs],
pre_nms_proposal_scores[ixs],
config.DETECTION_NMS_THRESHOLD)
# Map indicies
class_keep_proposal_ixs = keep_proposal_ixs[ixs[class_keep]]
nms_keep_proposal_ixs = np.union1d(nms_keep_proposal_ixs,
class_keep_proposal_ixs)
print("{:12}: {} -> {}".format(dataset.class_names[class_id][:10],
keep_proposal_ixs[ixs],
class_keep_proposal_ixs))
keep_proposal_ixs = np.intersect1d(keep_proposal_ixs,
nms_keep_proposal_ixs).astype(np.int32)
print("\nKeep after per-class NMS: {}\n{}".format(
keep_proposal_ixs.shape[0], keep_proposal_ixs))
#################################### Show final detections #####################################
ixs = np.arange(len(keep_proposal_ixs)) # Display all
# ixs = np.random.randint(0, len(keep), 10) # Display random sample
captions = [
"{} {:.3f}".format(dataset.class_names[c], s) if c > 0 else ""
for c, s in zip(proposal_class_ids[keep_proposal_ixs][ixs],
proposal_class_scores[keep_proposal_ixs][ixs])
]
visualize.draw_boxes(
resized_image.copy(),
boxes=proposals[keep_proposal_ixs][ixs],
refined_boxes=refined_proposals[keep_proposal_ixs][ixs],
visibilities=np.where(proposal_class_ids[keep_proposal_ixs][ixs] > 0,
1, 0),
captions=captions,
title="Detections after NMS",
ax=ax[3])
plt.show()
def display_rpn_prediction():
# Run RPN sub-graph
resized_image, image_meta, gt_class_ids, gt_bboxes, gt_masks = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
pillar = model.keras_model.get_layer(
"ROI").output # node to start searching from
# TF 1.4 and 1.9 introduce new versions of NMS. Search for all names to support TF 1.3~1.10
nms_node = model.ancestor(pillar, "ROI/rpn_non_max_suppression:0")
if nms_node is None:
nms_node = model.ancestor(
pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV2:0")
if nms_node is None: # TF 1.9-1.10
nms_node = model.ancestor(
pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV3:0")
rpn = model.run_graph([resized_image], [
("rpn_class", model.keras_model.get_layer("rpn_class").output),
("pre_nms_anchors", model.ancestor(pillar, "ROI/pre_nms_anchors:0")),
("refined_anchors", model.ancestor(pillar, "ROI/refined_anchors:0")),
("refined_anchors_clipped",
model.ancestor(pillar, "ROI/refined_anchors_clipped:0")),
("post_nms_anchor_ix", nms_node),
("proposals", model.keras_model.get_layer("ROI").output),
])
ax = get_ax(2, 3)
# Show top anchors by score (before refinement)
limit = 100
# np.flatten() 会把多维数组变成一维数组, 那么此处就默认 batch_size=1, 否则不能这样计算
# 按从大到小排序
sorted_anchor_ids = np.argsort(rpn['rpn_class'][:, :, 1].flatten())[::-1]
visualize.draw_boxes(resized_image,
boxes=model.anchors[sorted_anchor_ids[:limit]],
ax=ax[0, 0])
# Show top anchors with refinement. Then with clipping to image boundaries
limit = 50
pre_nms_anchors = utils.denorm_boxes(rpn["pre_nms_anchors"][0],
resized_image.shape[:2])
refined_anchors = utils.denorm_boxes(rpn["refined_anchors"][0],
resized_image.shape[:2])
visualize.draw_boxes(resized_image,
boxes=pre_nms_anchors[:limit],
refined_boxes=refined_anchors[:limit],
ax=ax[0, 1])
refined_anchors_clipped = utils.denorm_boxes(
rpn["refined_anchors_clipped"][0], resized_image.shape[:2])
visualize.draw_boxes(resized_image,
refined_boxes=refined_anchors_clipped[:limit],
ax=ax[0, 2])
# Show refined anchors after non-max suppression
ixs = rpn["post_nms_anchor_ix"][:limit]
visualize.draw_boxes(resized_image,
refined_boxes=refined_anchors_clipped[ixs],
ax=ax[1, 0])
# Show final proposals
# These are the same as the previous step (refined anchors
# after NMS) but with coordinates normalized to [0, 1] range.
# Convert back to image coordinates for display
h, w = resized_image.shape[:2]
proposals = rpn['proposals'][0, :limit] * np.array([h, w, h, w])
visualize.draw_boxes(resized_image, refined_boxes=proposals, ax=ax[1, 1])
plt.show()
def read_results_to_df(training_results_file):
cols=['epoch','y1','x1','y2','x2','0','1','2','3','4','5','6','7','8',\
'p0','p1','p2','p3','p4','p5','p6','p7','p8']
# df = pd.read_csv(address,index_col=None,sep=",")
res_arr=np.loadtxt(training_results_file,\
skiprows=0,delimiter=",",dtype={'names': tuple(cols),\
'formats': ('i4', 'f4', 'f4', 'f4', 'f4', 'i4', 'i4', 'i4',\
'i4', 'i4', 'i4', 'i4', 'i4', 'i4', 'f4', 'f4',\
'f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4')})
res_df = pd.DataFrame(res_arr, columns=cols)
# print('here:',image.shape, molded_images[i].shape,windows[i])
# (256, 256, 4) (256, 256, 4)
# unq_epochs=res_df.epoch.unique()
# res_df=res_df[res_df['epoch']==unq_epochs[1]].reset_index(drop=True)
boxes = res_df[['y1', 'x1', 'y2', 'x2']].values
image_shape = (256, 256, 4)
original_image_shape = (256, 256, 4)
window = [0, 0, 256, 256]
window = utils.norm_boxes(window, image_shape[:2])
wy1, wx1, wy2, wx2 = window
shift = np.array([wy1, wx1, wy1, wx1])
wh = wy2 - wy1 # window height
ww = wx2 - wx1 # window width
scale = np.array([wh, ww, wh, ww])
# Convert boxes to normalized coordinates on the window
boxes = np.divide(boxes - shift, scale)
# Convert boxes to pixel coordinates on the original image
boxes = utils.denorm_boxes(boxes, original_image_shape[:2])
res_df[['y1_', 'x1_', 'y2_', 'x2_']] = boxes
# Filter out detections with zero area. Happens in early training when
# network weights are still random
exclude_ix = np.where((boxes[:, 2] - boxes[:, 0]) *
(boxes[:, 3] - boxes[:, 1]) <= 0)[0]
exclude_ix_big_area = np.where((boxes[:, 2] - boxes[:, 0]) *
(boxes[:, 3] - boxes[:, 1]) > 20)[0]
# res_df[res_df]
update_df = res_df.drop(exclude_ix)
# update_df2 = update_df.drop(exclude_ix_big_area)
# update_df
# if exclude_ix.shape[0] > 0:
# boxes = np.delete(boxes, exclude_ix, axis=0)
# class_probs = np.delete(class_probs, exclude_ix, axis=0)
# class_ids = np.delete(class_ids, exclude_ix, axis=0)
# scores = np.delete(scores, exclude_ix, axis=0)
# masks = np.delete(masks, exclude_ix, axis=0)
# N = class_ids.shape[0]
# height = box[:, 2] - box[:, 0]
height = res_df['y2_'] - res_df['y1_']
width = res_df['x2_'] - res_df['x1_']
# width = box[:, 3] - box[:, 1]
update_df['bb_center_y'] = res_df['y1_'] + 0.5 * height
update_df['bb_center_x'] = res_df['x1_'] + 0.5 * width
return update_df
("detections", model.keras_model.get_layer("mrcnn_detection").output),
("masks", model.keras_model.get_layer("mrcnn_mask").output),
])
# Get detection class IDs. Trim zero padding.
det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)
# det_class_ids는 detection한 object수만큼 list가 생성되며 각 리스트에는 분류될 class 수(3)(Llane, car, Slane)가 들어간다.
det_count = np.where(det_class_ids == 0)[0][0]
det_class_ids = det_class_ids[:det_count]
print((det_class_ids[0]))
print((det_class_ids[1]))
#display_images(d)
print("{} detections: {}".format(det_count,
np.array(dataset.class_names)[det_class_ids]))
det_boxes = utils.denorm_boxes(mrcnn["detections"][0, :, :4], image.shape[:2])
det_mask_specific = np.array(
[mrcnn["masks"][0, i, :, :, c] for i, c in enumerate(det_class_ids)])
det_masks = np.array([
utils.unmold_mask(m, det_boxes[i], image.shape)
for i, m in enumerate(det_mask_specific)
])
print("here \n", det_mask_specific[1])
'''
one, two, thr, four, fiv, six, sev = 0,0,0,0,0,0,0
# det_masks는 1024*1024의 이미지를 detection Object의 갯수만큼 갖고 있다. EX. [2][1024][1024] 갯수*가로*세로 and 한pixel당 float형으로 표현
for a in range(len(det_mask_specific)): # a 는 detect Box 갯수
for i in range(len(det_mask_specific[a])): # i 는 세로 픽셀 수
for j in range(len(det_mask_specific[a][i])): # j 는 가로 픽셀 수
if 0.000 in (det_mask_specific[0][i][j], float):
if a == 0:
image, imageMeta, groundTruthClassId, groundTruthBoundingBox, groundTruthMask = modellib.load_image_gt(
dataset, config, iid, use_mini_mask=False)
maskrcnnOutput = model.run_graph([image], [
("detections", model.keras_model.get_layer("mrcnn_detection").output),
("masks", model.keras_model.get_layer("mrcnn_mask").output),
])
detectedClasses = maskRCnnOutput['detections'][0, :, 4].astype(np.int32)
numberOfDetections = np.where(det_class_ids == 0)[0][0]
detectedClasses = detectedClasses[:numberOfDetections]
print("{} detections: {}".format(
numberOfDetections,
np.array(dataset.class_names)[detectedClasses]))
detectedBoundingBoxes = utils.denorm_boxes(
maskrcnnOutput["detections"][0, :, :4], (512, 512))
detectedmasks = np.array([
maskRCnnOutput["masks"][0, i, :, :, c]
for i, c in enumerate(detectedClasses)
])
# detectedMasks reshaped to (512,512,3) to use as input for ensemble model
detectedMasks = np.array([
utils.unmold_mask(m, detectedBoundingBoxes[i], (512, 512, 3))
for i, m in enumerate(detectedmasks)
])
print(np.unique(detectedMasks))
a = np.sum(detectedMasks, axis=0) > 0
# create directory using terminal if it doesn't already exist
plt.imsave(fname="maskrcnn_final_test_ensemble_input/" +
str(dataset.image_reference(iid) + ".png"),
("proposals", model.keras_model.get_layer("ROI").output),
])
#%%
# Show top anchors by score (before refinement)
limit = 100
sorted_anchor_ids = np.argsort(rpn['rpn_class'][:, :, 1].flatten())[::-1]
visualize.draw_boxes(image,
boxes=model.anchors[sorted_anchor_ids[:limit]],
ax=get_ax())
#%%
# Show top anchors with refinement. Then with clipping to image boundaries
limit = 50
ax = get_ax(1, 2)
pre_nms_anchors = utils.denorm_boxes(rpn["pre_nms_anchors"][0],
image.shape[:2])
refined_anchors = utils.denorm_boxes(rpn["refined_anchors"][0],
image.shape[:2])
refined_anchors_clipped = utils.denorm_boxes(rpn["refined_anchors_clipped"][0],
image.shape[:2])
visualize.draw_boxes(image,
boxes=pre_nms_anchors[:limit],
refined_boxes=refined_anchors[:limit],
ax=ax[0])
visualize.draw_boxes(image,
refined_boxes=refined_anchors_clipped[:limit],
ax=ax[1])
#%%
# Show refined anchors after non-max suppression
limit = 50
def calculate_mask(model, graph, image, rois):
'''
inputs:
model: sd mask rcnn model
graph: model graph (provide keras to calculate)
image: filled hole 8-bit depth cropped image (specified with rois)
rois: Full image ROI patch(x1,y1,x2,y2)
returns:
det_masks: mask on 512*512 scale (values=0 or 1)
det_mask_specific: predict mask on CNN (28*28) (values=0 or 1)
resized_mask_image: max facet detected image (ROI Size)
'''
print('MAKING SD MASK RCNN PREDICTIONS')
class_names = ['bg', 'fg']
image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
print(image.shape)
# Resize image for net
image, _, _, _, _ = resize_image(image, max_dim=512)
#cv2.imshow("Test",image)
#cv2.waitKey(0)
# Get predictions of mask head
#model.keras_model._make_predict_function()
with graph.as_default():
mrcnn = model.run_graph([image], [
("detections",
model.keras_model.get_layer("mrcnn_detection").output),
("masks", model.keras_model.get_layer("mrcnn_mask").output),
])
# Get detection class IDs. Trim zero padding.
det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)
det_count = np.where(det_class_ids == 0)[0][0]
det_class_ids = det_class_ids[:det_count]
print("{} detections: {}".format(det_count,
np.array(class_names)[det_class_ids]))
# Masks
det_boxes = m_utils.denorm_boxes(mrcnn["detections"][0, :, :4],
image.shape[:2])
det_mask_specific = np.array(
[mrcnn["masks"][0, i, :, :, c] for i, c in enumerate(det_class_ids)])
det_masks = np.array([
m_utils.unmold_mask(m, det_boxes[i], image.shape)
for i, m in enumerate(det_mask_specific)
])
if det_count != 0:
log("det_mask_specific", det_mask_specific)
log("det_masks", det_masks)
else:
print("No facet detected!!")
return
resized_mask_image, no_padding_range = detect_show_instance(
image, model, graph, class_names, rois)
return det_masks, det_mask_specific, resized_mask_image, no_padding_range
