def draw_rois(image, rois, refined_rois, mask, class_ids, class_names, limit=10): """ anchors: [n, (y1, x1, y2, x2)] list of anchors in image coordinates. proposals: [n, 4] the same anchors but refined to fit objects better. """ masked_image = image.copy() # Pick random anchors in case there are too many. ids = np.arange(rois.shape[0], dtype=np.int32) ids = np.random.choice( ids, limit, replace=False) if ids.shape[0] > limit else ids fig, ax = plt.subplots(1, figsize=(12, 12)) if rois.shape[0] > limit: plt.title("Showing {} random ROIs out of {}".format( len(ids), rois.shape[0])) else: plt.title("{} ROIs".format(len(ids))) # Show area outside image boundaries. ax.set_ylim(image.shape[0] + 20, -20) ax.set_xlim(-50, image.shape[1] + 20) ax.axis('off') for i, id in enumerate(ids): color = np.random.rand(3) class_id = class_ids[id] # ROI y1, x1, y2, x2 = rois[id] p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2, edgecolor=color if class_id else "gray", facecolor='none', linestyle="dashed") ax.add_patch(p) # Refined ROI if class_id: ry1, rx1, ry2, rx2 = refined_rois[id] p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2, edgecolor=color, facecolor='none') ax.add_patch(p) # Connect the top-left corners of the anchor and proposal for easy visualization ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color)) # Label label = class_names[class_id] ax.text(rx1, ry1 + 8, "{}".format(label), color='w', size=11, backgroundcolor="none") # Mask m = utils.unmold_mask(mask[id], rois[id] [:4].astype(np.int32), image.shape) masked_image = apply_mask(masked_image, m, color) ax.imshow(masked_image) # Print stats print("Positive ROIs: ", class_ids[class_ids > 0].shape[0]) print("Negative ROIs: ", class_ids[class_ids == 0].shape[0]) print("Positive Ratio: {:.2f}".format( class_ids[class_ids > 0].shape[0] / class_ids.shape[0]))
def draw_rois(image, rois, refined_rois, mask, class_ids, class_names, limit=10): """ anchors: [n, (y1, x1, y2, x2)] list of anchors in image coordinates. proposals: [n, 4] the same anchors but refined to fit objects better. """ masked_image = image.copy() # Pick random anchors in case there are too many. ids = np.arange(rois.shape[0], dtype=np.int32) ids = np.random.choice(ids, limit, replace=False) if ids.shape[0] > limit else ids fig, ax = plt.subplots(1, figsize=(12, 12)) if rois.shape[0] > limit: plt.title("Showing {} random ROIs out of {}".format( len(ids), rois.shape[0])) else: plt.title("{} ROIs".format(len(ids))) # Show area outside image boundaries. #ax.set_ylim(image.shape[0] + 20, -20) #ax.set_xlim(-50, image.shape[1] + 20) #ax.axis('off') for i, id in enumerate(ids): color = np.random.rand(3) class_id = class_ids[id] # ROI y1, x1, y2, x2 = rois[id] p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2, edgecolor=color if class_id else "gray", facecolor='none', linestyle="dashed") ax.add_patch(p) # Refined ROI if class_id: ry1, rx1, ry2, rx2 = refined_rois[id] p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2, edgecolor=color, facecolor='none') ax.add_patch(p) # Connect the top-left corners of the anchor and proposal for easy visualization ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color)) # Label label = class_names[class_id] ax.text(rx1, ry1 + 8, "{}".format(label), color='w', size=11, backgroundcolor="none") # Mask m = utils.unmold_mask(mask[id], rois[id][:4].astype(np.int32), image.shape) masked_image = apply_mask(masked_image, m, color) ax.imshow(masked_image, origin="lower", cmap="jet") #ax.imshow(masked_image, cmap="jet") # Print stats print("Positive ROIs: ", class_ids[class_ids > 0].shape[0]) print("Negative ROIs: ", class_ids[class_ids == 0].shape[0]) print("Positive Ratio: {:.5f}".format(class_ids[class_ids > 0].shape[0] / class_ids.shape[0]))
]) # Get detection class IDs. Trim zero padding. det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32) det_count = np.where(det_class_ids == 0)[0][0] det_class_ids = det_class_ids[:det_count] print("{} detections: {}".format(det_count, np.array(dataset.class_names)[det_class_ids])) # Masks det_boxes = utils.denorm_boxes(mrcnn["detections"][0, :, :4], image.shape[:2]) det_mask_specific = np.array( [mrcnn["masks"][0, i, :, :, c] for i, c in enumerate(det_class_ids)]) det_masks = np.array([ utils.unmold_mask(m, det_boxes[i], image.shape) for i, m in enumerate(det_mask_specific) ]) log("det_mask_specific", det_mask_specific) log("det_masks", det_masks) display_images(det_mask_specific[:4] * 255, cmap="Blues", interpolation="none") display_images(det_masks[:4] * 255, cmap="Blues", interpolation="none") #%% # Visualize Activations # In some cases it helps to look at the output # from different layers and visualize them to catch issues # and odd patterns.
]) # Get detection class IDs. Trim zero padding. det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32) det_count = np.where(det_class_ids == 0)[0][0] det_class_ids = det_class_ids[:det_count] print("{} detections: {}".format( det_count, np.array(dataset.class_names)[det_class_ids])) # Masks det_boxes = utils.denorm_boxes(mrcnn["detections"][0, :, :4], image.shape[:2]) det_mask_specific = np.array([mrcnn["masks"][0, i, :, :, c] for i, c in enumerate(det_class_ids)]) det_masks = np.array([utils.unmold_mask(m, det_boxes[i], image.shape) for i, m in enumerate(det_mask_specific)]) log("det_mask_specific", det_mask_specific) log("det_masks", det_masks) display_images(det_mask_specific[:4] * 255, cmap="Blues", interpolation="none") # Get activations of a few sample layers activations = model.run_graph([image], [ ("input_image", tf.identity(model.keras_model.get_layer("input_image").output)), ("res2c_out", model.keras_model.get_layer("res2c_out").output), ("res3c_out", model.keras_model.get_layer("res3c_out").output), ("rpn_bbox", model.keras_model.get_layer("rpn_bbox").output), ("roi", model.keras_model.get_layer("ROI").output), ])
def unmold_detections(detections, mrcnn_mask, original_image_shape, image_shape, window): """Reformats the detections of one image from the format of the neural network output to a format suitable for use in the rest of the application. detections: [N, (y1, x1, y2, x2, class_id, score)] in normalized coordinates mrcnn_mask: [N, height, width, num_classes] original_image_shape: [H, W, C] Original image shape before resizing image_shape: [H, W, C] Shape of the image after resizing and padding window: [y1, x1, y2, x2] Pixel coordinates of box in the image where the real image is excluding the padding. Returns: boxes: [N, (y1, x1, y2, x2)] Bounding boxes in pixels class_ids: [N] Integer class IDs for each bounding box scores: [N] Float probability scores of the class_id masks: [height, width, num_instances] Instance masks """ # How many detections do we have? # Detections array is padded with zeros. Find the first class_id == 0. zero_ix = np.where(detections[:, 4] == 0)[0] N = zero_ix[0] if zero_ix.shape[0] > 0 else detections.shape[0] # Extract boxes, class_ids, scores, and class-specific masks boxes = detections[:N, :4] class_ids = detections[:N, 4].astype(np.int32) scores = detections[:N, 5] masks = mrcnn_mask[np.arange(N), :, :, class_ids] # Translate normalized coordinates in the resized image to pixel # coordinates in the original image before resizing window = utils.norm_boxes(window, image_shape[:2]) wy1, wx1, wy2, wx2 = window shift = np.array([wy1, wx1, wy1, wx1]) wh = wy2 - wy1 # window height ww = wx2 - wx1 # window width scale = np.array([wh, ww, wh, ww]) # Convert boxes to normalized coordinates on the window boxes = np.divide(boxes - shift, scale) # Convert boxes to pixel coordinates on the original image boxes = utils.denorm_boxes(boxes, original_image_shape[:2]) # Filter out detections with zero area. Happens in early training when # network weights are still random exclude_ix = np.where((boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) <= 0)[0] if exclude_ix.shape[0] > 0: boxes = np.delete(boxes, exclude_ix, axis=0) class_ids = np.delete(class_ids, exclude_ix, axis=0) scores = np.delete(scores, exclude_ix, axis=0) masks = np.delete(masks, exclude_ix, axis=0) N = class_ids.shape[0] # Resize masks to original image size and set boundary threshold. full_masks = [] for i in range(N): # Convert neural network mask to full size mask full_mask = utils.unmold_mask(masks[i], boxes[i], original_image_shape) full_masks.append(full_mask) full_masks = np.stack(full_masks, axis=-1)\ if full_masks else np.empty(original_image_shape[:2] + (0,)) return boxes, class_ids, scores, full_masks
def unmold_detections(self, detections, mrcnn_mask, image_shape, window): ''' Reformats the detections of one image from the format of the neural network output to a format suitable for use in the rest of the application. detections : [N, (y1, x1, y2, x2, class_id, score)] mrcnn_mask : [N, height, width, num_classes] image_shape : [height, width, depth] Original size of the image before resizing window : [y1, x1, y2, x2] Box in the image where the real image is excluding the padding. Returns: boxes : [N, (y1, x1, y2, x2)] Bounding boxes in pixels class_ids : [N] Integer class IDs for each bounding box scores : [N] Float probability scores of the class_id masks : [height, width, num_instances] Instance masks ''' # print('>>> unmold_detections ') # print(' detections.shape : ', detections.shape) # print(' mrcnn_mask.shape : ', mrcnn_mask.shape) # print(' image_shape.shape: ', image_shape) # print(' window.shape : ', window) # print(detections) # How many detections do we have? # Detections array is padded with zeros. detections[:,4] identifies the class # Find all rows in detection array with class_id == 0 , and place their row indices # into zero_ix. zero_ix[0] will identify the first row with class_id == 0. np.set_printoptions(linewidth=100) zero_ix = np.where(detections[:, 4] == 0)[0] N = zero_ix[0] if zero_ix.shape[0] > 0 else detections.shape[0] # print(' np.where() \n', np.where(detections[:, 4] == 0)) # print(' zero_ix.shape : ', zero_ix.shape) # print(' N is :', N) # Extract boxes, class_ids, scores, and class-specific masks boxes = detections[:N, :4] class_ids = detections[:N, 4].astype(np.int32) scores = detections[:N, 5] masks = mrcnn_mask[np.arange(N), :, :, class_ids] # Compute scale and shift to translate coordinates to image domain. h_scale = image_shape[0] / (window[2] - window[0]) w_scale = image_shape[1] / (window[3] - window[1]) scale = min(h_scale, w_scale) shift = window[:2] # y, x scales = np.array([scale, scale, scale, scale]) shifts = np.array([shift[0], shift[1], shift[0], shift[1]]) # Translate bounding boxes to image domain boxes = np.multiply(boxes - shifts, scales).astype(np.int32) # Filter out detections with zero area. Often only happens in early # stages of training when the network weights are still a bit random. exclude_ix = np.where((boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) <= 0)[0] if exclude_ix.shape[0] > 0: boxes = np.delete(boxes, exclude_ix, axis=0) class_ids = np.delete(class_ids, exclude_ix, axis=0) scores = np.delete(scores, exclude_ix, axis=0) masks = np.delete(masks, exclude_ix, axis=0) N = class_ids.shape[0] # Resize masks to original image size and set boundary threshold. full_masks = [] for i in range(N): # Convert neural network mask to full size mask full_mask = utils.unmold_mask(masks[i], boxes[i], image_shape) full_masks.append(full_mask) full_masks = np.stack(full_masks, axis=-1)\ if full_masks else np.empty((0,) + masks.shape[1:3]) return boxes, class_ids, scores, full_masks
("masks", model.keras_model.get_layer("mrcnn_mask").output), ]) detectedClasses = maskRCnnOutput['detections'][0, :, 4].astype(np.int32) numberOfDetections = np.where(det_class_ids == 0)[0][0] detectedClasses = detectedClasses[:numberOfDetections] print("{} detections: {}".format( numberOfDetections, np.array(dataset.class_names)[detectedClasses])) detectedBoundingBoxes = utils.denorm_boxes( maskrcnnOutput["detections"][0, :, :4], (512, 512)) detectedmasks = np.array([ maskRCnnOutput["masks"][0, i, :, :, c] for i, c in enumerate(detectedClasses) ]) # detectedMasks reshaped to (512,512,3) to use as input for ensemble model detectedMasks = np.array([ utils.unmold_mask(m, detectedBoundingBoxes[i], (512, 512, 3)) for i, m in enumerate(detectedmasks) ]) print(np.unique(detectedMasks)) a = np.sum(detectedMasks, axis=0) > 0 # create directory using terminal if it doesn't already exist plt.imsave(fname="maskrcnn_final_test_ensemble_input/" + str(dataset.image_reference(iid) + ".png"), arr=a, cmap="Blues")
for image_id in dataset.image_ids: print("Image id: ", image_id) filename = "maskrcnn_ensemble_input/" + str(dataset.image_reference(image_id)) + ".png" #if os.path.isfile(filename): # continue image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) mrcnn = model.run_graph([image], [ ("detections", model.keras_model.get_layer("mrcnn_detection").output), ("masks", model.keras_model.get_layer("mrcnn_mask").output), ]) det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32) det_count = np.where(det_class_ids == 0)[0][0] det_class_ids = det_class_ids[:det_count] print("{} detections: {}".format(det_count, np.array(dataset.class_names)[det_class_ids])) det_boxes = utils.denorm_boxes(mrcnn["detections"][0, :, :4], (512,512)) det_mask_specific = np.array([mrcnn["masks"][0, i, :, :, c] for i, c in enumerate(det_class_ids)]) det_masks = np.array([utils.unmold_mask(m, det_boxes[i], (512,512,3)) for i, m in enumerate(det_mask_specific)]) log("det_mask_specific", det_mask_specific) log("det_masks", det_masks) print(np.unique(det_masks)) a = np.sum(det_masks, axis=0) > 0 # create directory using terminal if it doesn't already exist plt.imsave(fname = "maskrcnn_final_ensemble_input/" + str(dataset.image_reference(image_id)+".png"), arr = a, cmap="Blues")
"""
def draw_rois(image, rois, refined_rois, mask, class_ids, class_names, limit=10): """ anchors: [n, (y1, x1, y2, x2)] 图像坐标中的锚点列表. proposals: [n, 4] 相同锚点,经过精炼以选择更好地合适对象. """ masked_image = image.copy() # 若有太多,则选择随机几个锚来绘图draw_box. ids = np.arange(rois.shape[0], dtype=np.int32) ids = np.random.choice(ids, limit, replace=False) if ids.shape[0] > limit else ids fig, ax = plt.subplots(1, figsize=(12, 12)) if rois.shape[0] > limit: plt.title("Showing {} random ROIs out of {}".format( len(ids), rois.shape[0])) else: plt.title("{} ROIs".format(len(ids))) # 显示超出图像边界的区域. ax.set_ylim(image.shape[0] + 20, -20) ax.set_xlim(-50, image.shape[1] + 20) ax.axis('off') for i, id in enumerate(ids): color = np.random.rand(3) class_id = class_ids[id] # ROI y1, x1, y2, x2 = rois[id] p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2, edgecolor=color if class_id else "gray", facecolor='none', linestyle="dashed") ax.add_patch(p) # 调整 ROI if class_id: ry1, rx1, ry2, rx2 = refined_rois[id] p = patches.Rectangle((rx1, ry1), rx2 - rx1, ry2 - ry1, linewidth=2, edgecolor=color, facecolor='none') ax.add_patch(p) # 连接锚点和建议框的左上角来便于可视化 ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color)) # Label label = class_names[class_id] ax.text(rx1, ry1 + 8, "{}".format(label), color='w', size=11, backgroundcolor="none") # Mask m = utils.unmold_mask(mask[id], rois[id][:4].astype(np.int32), image.shape) masked_image = apply_mask(masked_image, m, color) ax.imshow(masked_image) # 输出统计 print("Positive ROIs: ", class_ids[class_ids > 0].shape[0]) print("Negative ROIs: ", class_ids[class_ids == 0].shape[0]) print("Positive Ratio: {:.2f}".format(class_ids[class_ids > 0].shape[0] / class_ids.shape[0]))
def calculate_mask(model, graph, image, rois): ''' inputs: model: sd mask rcnn model graph: model graph (provide keras to calculate) image: filled hole 8-bit depth cropped image (specified with rois) rois: Full image ROI patch(x1,y1,x2,y2) returns: det_masks: mask on 512*512 scale (values=0 or 1) det_mask_specific: predict mask on CNN (28*28) (values=0 or 1) resized_mask_image: max facet detected image (ROI Size) ''' print('MAKING SD MASK RCNN PREDICTIONS') class_names = ['bg', 'fg'] image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB) print(image.shape) # Resize image for net image, _, _, _, _ = resize_image(image, max_dim=512) #cv2.imshow("Test",image) #cv2.waitKey(0) # Get predictions of mask head #model.keras_model._make_predict_function() with graph.as_default(): mrcnn = model.run_graph([image], [ ("detections", model.keras_model.get_layer("mrcnn_detection").output), ("masks", model.keras_model.get_layer("mrcnn_mask").output), ]) # Get detection class IDs. Trim zero padding. det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32) det_count = np.where(det_class_ids == 0)[0][0] det_class_ids = det_class_ids[:det_count] print("{} detections: {}".format(det_count, np.array(class_names)[det_class_ids])) # Masks det_boxes = m_utils.denorm_boxes(mrcnn["detections"][0, :, :4], image.shape[:2]) det_mask_specific = np.array( [mrcnn["masks"][0, i, :, :, c] for i, c in enumerate(det_class_ids)]) det_masks = np.array([ m_utils.unmold_mask(m, det_boxes[i], image.shape) for i, m in enumerate(det_mask_specific) ]) if det_count != 0: log("det_mask_specific", det_mask_specific) log("det_masks", det_masks) else: print("No facet detected!!") return resized_mask_image, no_padding_range = detect_show_instance( image, model, graph, class_names, rois) return det_masks, det_mask_specific, resized_mask_image, no_padding_range