def execute(image): if (config is None or model is None): init() molded_image, image_meta, windows = model.mold_inputs([image]) window = utils.norm_boxes(windows, molded_image[0].shape[:2])[0] anchors = model.get_anchors(molded_image[0].shape) anchors = np.broadcast_to(anchors, (model.config.BATCH_SIZE, ) + anchors.shape) dects, probs, deltas, masks, proposals, _, _ = model.keras_model.predict( [molded_image, image_meta, anchors], verbose=0) # Class ID, and score per proposal class_ids = np.argmax(probs[0], axis=1) class_scores = probs[0, np.arange(class_ids.shape[0]), class_ids] refined_rois = refine_proposals(proposals, class_ids, deltas, window) keep = filter_rois(refined_rois, class_ids, class_scores) class_scores = probs[0][keep] # Detections are in different order than the indices in the "keep" array. # Thus, we need to identify the matching detections for correct ordering of # class_scores. bounding_boxes = utils.denorm_boxes(dects[0, :len(keep), :4], molded_image[0].shape[:2]) roi_boxes = utils.denorm_boxes(refined_rois[keep], molded_image[0].shape[:2]) perm = [] for i in np.arange(len(keep)): perm = np.append( perm, np.where(np.all(roi_boxes == bounding_boxes[i], axis=1))[0][0]).astype(np.int32) class_scores = class_scores[perm] bounding_boxes, _, _, segmentation = model.unmold_detections( dects[0], masks[0], image.shape, molded_image[0].shape, windows[0]) keep_fusion = filter_fusion(segmentation) return generate_result(bounding_boxes[keep_fusion], segmentation[:, :, keep_fusion], class_scores[keep_fusion])
def unmold_predictions(self, detections, mrcnn_mask, original_image_shape, image_shape, window): ''' copied from mrcnn, changes in loop allows return of probability map ''' # Detections array is padded with zeros. Find the first class_id == 0. zero_ix = np.where(detections[:, 4] == 0)[0] N = zero_ix[0] if zero_ix.shape[0] > 0 else detections.shape[0] # Extract boxes, class_ids, scores, and class-specific masks boxes = detections[:N, :4] class_ids = detections[:N, 4].astype(np.int32) scores = detections[:N, 5] masks = mrcnn_mask[np.arange(N), :, :, class_ids] # Translate normalized coordinates in the resized image to pixel # coordinates in the original image before resizing window = utils.norm_boxes(window, image_shape[:2]) wy1, wx1, wy2, wx2 = window shift = np.array([wy1, wx1, wy1, wx1]) wh = wy2 - wy1 # window height ww = wx2 - wx1 # window width scale = np.array([wh, ww, wh, ww]) # Convert boxes to normalized coordinates on the window boxes = np.divide(boxes - shift, scale) # Convert boxes to pixel coordinates on the original image boxes = utils.denorm_boxes(boxes, original_image_shape[:2]) # Filter out detections with zero area. Happens in early training when # network weights are still random exclude_ix = np.where((boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) <= 0)[0] if exclude_ix.shape[0] > 0: boxes = np.delete(boxes, exclude_ix, axis=0) class_ids = np.delete(class_ids, exclude_ix, axis=0) scores = np.delete(scores, exclude_ix, axis=0) masks = np.delete(masks, exclude_ix, axis=0) N = class_ids.shape[0] # Resize masks to original image size and set boundary threshold. full_masks = [] full_pmap = [] for i in range(N): # Convert neural network mask to full size mask threshold = 0.5 y1, x1, y2, x2 = boxes[i] full_pmap.append( skimage.transform.resize(masks[i], (y2 - y1, x2 - x1), order=1, mode="constant")) mask = np.where(full_pmap[i] >= threshold, 1, 0).astype(np.bool) # Put the mask in the right location. full_mask = np.zeros(image_shape[:2], dtype=np.bool) full_mask[y1:y2, x1:x2] = mask # full_mask = utils.unmold_mask(masks[i], boxes[i], original_image_shape) full_masks.append(full_mask) full_masks = np.stack(full_masks, axis=-1)\ if full_masks else np.empty(original_image_shape[:2] + (0,)) return boxes, class_ids, scores, full_masks, full_pmap
def unmold_detections(detections, mrcnn_mask, original_image_shape, image_shape, window): zero_ix = np.where(detections[:, 4] == 0)[0] N = zero_ix[0] if zero_ix.shape[0] > 0 else detections.shape[0] # Extract boxes, class_ids, scores, and class-specific masks boxes = detections[:N, :4] class_ids = detections[:N, 4].astype(np.int32) scores = detections[:N, 5] masks = mrcnn_mask[np.arange(N), :, :, class_ids] # [N, 28*28, class] # Translate normalized coordinates in the resized image to pixel # coordinates in the original image before resizing window = utils.norm_boxes(window, image_shape[:2]) wy1, wx1, wy2, wx2 = window shift = np.array([wy1, wx1, wy1, wx1]) wh = wy2 - wy1 # window height ww = wx2 - wx1 # window width scale = np.array([wh, ww, wh, ww]) # Convert boxes to normalized coordinates on the window boxes = np.divide(boxes - shift, scale) # Convert boxes to pixel coordinates on the original image boxes = utils.denorm_boxes(boxes, original_image_shape[:2]) # Filter out detections with zero area. Happens in early training when # network weights are still random exclude_ix = np.where((boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) <= 0)[0] if exclude_ix.shape[0] > 0: boxes = np.delete(boxes, exclude_ix, axis=0) class_ids = np.delete(class_ids, exclude_ix, axis=0) scores = np.delete(scores, exclude_ix, axis=0) masks = np.delete(masks, exclude_ix, axis=0) N = class_ids.shape[0] # Resize masks to original image size and set boundary threshold. full_masks = [] for i in range(N): # Convert neural network mask to full size mask full_mask = utils.unmold_mask(masks[i], boxes[i], original_image_shape) full_masks.append(full_mask) full_masks = np.stack(full_masks, axis=-1)\ if full_masks else np.empty(original_image_shape[:2] + (0,)) return boxes, class_ids, scores, full_masks
def display_mrcnn_mask_prediction(): #################################### Mask Targets ############################################## # gt_masks 的 shape 为 (image_height, image_width, num_instances) resized_image, image_meta, gt_class_ids, gt_bboxes, gt_masks = \ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) display_images(np.transpose(gt_masks, [2, 0, 1]), cmap="Blues") # Get predictions of mask head mrcnn = model.run_graph([resized_image], [ ("detections", model.keras_model.get_layer("mrcnn_detection").output), ("masks", model.keras_model.get_layer("mrcnn_mask").output), ]) # Get detection class IDs. Trim zero padding. det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32) padding_start_ix = np.where(det_class_ids == 0)[0][0] det_class_ids = det_class_ids[:padding_start_ix] print("{} detections: {}".format( padding_start_ix, np.array(dataset.class_names)[det_class_ids])) # Masks det_boxes = utils.denorm_boxes(mrcnn["detections"][0, :, :4], resized_image.shape[:2]) # mrcnn['masks'] 的 shape 为 (batch_size, num_instances, mask_height, mask_width, num_classes) det_mask_specific = np.array( [mrcnn["masks"][0, i, :, :, c] for i, c in enumerate(det_class_ids)]) det_masks = np.array([ utils.unmold_mask(mask, det_boxes[i], resized_image.shape) for i, mask in enumerate(det_mask_specific) ]) log("det_mask_specific", det_mask_specific) display_images(det_mask_specific[:4] * 255, cmap="Blues", interpolation="none") log("det_masks", det_masks) display_images(det_masks[:4] * 255, cmap="Blues", interpolation="none")
if verbose: info = dataset.image_info[image_id] meta = modellib.parse_image_meta(image_meta[np.newaxis, ...]) print("{:3} {} AP: {:.2f}".format( meta["image_id"][0], meta["original_image_shape"][0], ap)) return APs # Run on validation set limit = 5 APs = compute_batch_ap(dataset, dataset.image_ids[:limit]) print("Mean AP overa {} images: {:.4f}".format(len(APs), np.mean(APs))) # Get anchors and convert to pixel coordinates anchors = model.get_anchors(image.shape) anchors = utils.denorm_boxes(anchors, image.shape[:2]) log("anchors", anchors) # Generate RPN trainig targets # target_rpn_match is 1 for positive anchors, -1 for negative anchors # and 0 for neutral anchors. target_rpn_match, target_rpn_bbox = modellib.build_rpn_targets( image.shape, anchors, gt_class_id, gt_bbox, model.config) log("target_rpn_match", target_rpn_match) log("target_rpn_bbox", target_rpn_bbox) positive_anchor_ix = np.where(target_rpn_match[:] == 1)[0] negative_anchor_ix = np.where(target_rpn_match[:] == -1)[0] neutral_anchor_ix = np.where(target_rpn_match[:] == 0)[0] positive_anchors = anchors[positive_anchor_ix] negative_anchors = anchors[negative_anchor_ix]
print("{:3} {} AP: {:.2f}".format( meta["image_id"][0], meta["original_image_shape"][0], ap)) return APs # Run on validation set limit = 5 APs = compute_batch_ap(dataset, dataset.image_ids[:limit]) print("Mean AP overa {} images: {:.4f}".format(len(APs), np.mean(APs))) ################### # A RPN Targets ################### # Get anchors and convert to pixel coordinates anchors = model.get_anchors(image.shape) anchors = utils.denorm_boxes(anchors, image.shape[:2]) log("anchors", anchors) # Generate RPN trainig targets # target_rpn_match is 1 for positive anchors, -1 for negative anchors # and 0 for neutral anchors. target_rpn_match, target_rpn_bbox = modellib.build_rpn_targets( image.shape, anchors, gt_class_id, gt_bbox, model.config) log("target_rpn_match", target_rpn_match) log("target_rpn_bbox", target_rpn_bbox) positive_anchor_ix = np.where(target_rpn_match[:] == 1)[0] negative_anchor_ix = np.where(target_rpn_match[:] == -1)[0] neutral_anchor_ix = np.where(target_rpn_match[:] == 0)[0] positive_anchors = anchors[positive_anchor_ix] negative_anchors = anchors[negative_anchor_ix]
model.ancestor(pillar, "ROI/refined_anchors_clipped:0")), ("post_nms_anchor_ix", nms_node), ("proposals", model.keras_model.get_layer("ROI").output), ]) # Show top anchors by score (before refinement) limit = 100 sorted_anchor_ids = np.argsort(rpn['rpn_class'][:, :, 1].flatten())[::-1] visualize.draw_boxes(image, boxes=model.anchors[sorted_anchor_ids[:limit]], ax=get_ax()) # Show top anchors with refinement. Then with clipping to image boundaries limit = 50 ax = get_ax(1, 2) pre_nms_anchors = utils.denorm_boxes(rpn["pre_nms_anchors"][0], image.shape[:2]) refined_anchors = utils.denorm_boxes(rpn["refined_anchors"][0], image.shape[:2]) refined_anchors_clipped = utils.denorm_boxes(rpn["refined_anchors_clipped"][0], image.shape[:2]) visualize.draw_boxes(image, boxes=pre_nms_anchors[:limit], refined_boxes=refined_anchors[:limit], ax=ax[0]) visualize.draw_boxes(image, refined_boxes=refined_anchors_clipped[:limit], ax=ax[1]) # Show refined anchors after non-max suppression limit = 50 ixs = rpn["post_nms_anchor_ix"][:limit]
def unmold_detections(detections, mrcnn_mask, original_image_shape, image_shape, window): """Reformats the detections of one image from the format of the neural network output to a format suitable for use in the rest of the application. detections: [N, (y1, x1, y2, x2, class_id, score)] in normalized coordinates mrcnn_mask: [N, height, width, num_classes] original_image_shape: [H, W, C] Original image shape before resizing image_shape: [H, W, C] Shape of the image after resizing and padding window: [y1, x1, y2, x2] Pixel coordinates of box in the image where the real image is excluding the padding. Returns: boxes: [N, (y1, x1, y2, x2)] Bounding boxes in pixels class_ids: [N] Integer class IDs for each bounding box scores: [N] Float probability scores of the class_id masks: [height, width, num_instances] Instance masks """ # How many detections do we have? # Detections array is padded with zeros. Find the first class_id == 0. zero_ix = np.where(detections[:, 4] == 0)[0] N = zero_ix[0] if zero_ix.shape[0] > 0 else detections.shape[0] # Extract boxes, class_ids, scores, and class-specific masks boxes = detections[:N, :4] class_ids = detections[:N, 4].astype(np.int32) scores = detections[:N, 5] masks = mrcnn_mask[np.arange(N), :, :, class_ids] # Translate normalized coordinates in the resized image to pixel # coordinates in the original image before resizing window = utils.norm_boxes(window, image_shape[:2]) wy1, wx1, wy2, wx2 = window shift = np.array([wy1, wx1, wy1, wx1]) wh = wy2 - wy1 # window height ww = wx2 - wx1 # window width scale = np.array([wh, ww, wh, ww]) # Convert boxes to normalized coordinates on the window boxes = np.divide(boxes - shift, scale) # Convert boxes to pixel coordinates on the original image boxes = utils.denorm_boxes(boxes, original_image_shape[:2]) # Filter out detections with zero area. Happens in early training when # network weights are still random exclude_ix = np.where((boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) <= 0)[0] if exclude_ix.shape[0] > 0: boxes = np.delete(boxes, exclude_ix, axis=0) class_ids = np.delete(class_ids, exclude_ix, axis=0) scores = np.delete(scores, exclude_ix, axis=0) masks = np.delete(masks, exclude_ix, axis=0) N = class_ids.shape[0] # Resize masks to original image size and set boundary threshold. full_masks = [] for i in range(N): # Convert neural network mask to full size mask full_mask = utils.unmold_mask(masks[i], boxes[i], original_image_shape) full_masks.append(full_mask) full_masks = np.stack(full_masks, axis=-1)\ if full_masks else np.empty(original_image_shape[:2] + (0,)) return boxes, class_ids, scores, full_masks
def generate_multiple_bbox(model, image): mrcnn = model.run_graph( [image], [ ("proposals", model.keras_model.get_layer("ROI").output), ("probs", model.keras_model.get_layer("mrcnn_class").output), ("deltas", model.keras_model.get_layer("mrcnn_bbox").output), # ("masks", model.keras_model.get_layer("mrcnn_mask").output), # ("detections", model.keras_model.get_layer("mrcnn_detection").output), ]) Proposal_limit = 1000 final_boxes_limit = 15 proposals = utils.denorm_boxes(mrcnn['proposals'][0, :Proposal_limit], image.shape[:2]) roi_class_ids = np.argmax(mrcnn["probs"][0], axis=1) roi_scores = mrcnn["probs"][0, np.arange(roi_class_ids.shape[0]), roi_class_ids] roi_class_names = np.array(class_names)[roi_class_ids] roi_positive_ixs = np.where(roi_class_ids > 0)[0] # How many ROIs vs empty rows? # print("{} Valid proposals out of {}".format(np.sum(np.any(proposals, axis=1)), proposals.shape[0])) # print("{} Positive ROIs".format(len(roi_positive_ixs))) keep = np.where(roi_class_ids > 0)[0] # print("Keep {} detections:\n{}".format(keep.shape[0], keep)) keep = np.intersect1d( keep, np.where(roi_scores >= config.DETECTION_MIN_CONFIDENCE)[0]) # print("Remove boxes below {} confidence. Keep {}:\n{}".format( # config.DETECTION_MIN_CONFIDENCE, keep.shape[0], keep)) ### Refinement roi_bbox_specific = mrcnn["deltas"][0, np.arange(proposals.shape[0]), roi_class_ids] # Apply bounding box transformations # Shape: [N, (y1, x1, y2, x2)] refined_proposals = utils.apply_box_deltas( proposals, roi_bbox_specific * config.BBOX_STD_DEV).astype(np.int32) ids = np.arange(0, len(keep) - 1) # Display all # ids = np.random.randint(0, len(roi_positive_ixs), limit) # Display random sample # ids = roi_positive_ixs[:-1] # captions = ["{} {:.3f}".format(class_names[c], s) if c > 0 else "" # for c, s in zip(roi_class_ids[keep][ids], roi_scores[keep][ids])] # visualize.draw_boxes(image, # refined_boxes=refined_proposals[keep][ids], # captions=captions, title="ROIs After Refinement", # ax=get_ax()) final_bboxes_before_NMS = [] # colors = random_colors(50) for bbox, score in zip(refined_proposals[keep][ids][:], roi_scores[keep][ids][:]): # centerCoord = (bbox[1] + int((bbox[3] - bbox[1]) / 2), bbox[0] + int((bbox[2] - bbox[0]) / 2)) # visualize.draw_box(bbox) # color = [i * 255 for i in colors[random.randint(0,30)]] # x1_y1_x2_y2_score = [bbox[1],bbox[0],bbox[3],bbox[2]] # image = cv2.rectangle(image, (x1_y1_x2_y2_score[0], x1_y1_x2_y2_score[1]), # (x1_y1_x2_y2_score[2], x1_y1_x2_y2_score[3]), color) final_bboxes_before_NMS.append( [bbox[1], bbox[0], bbox[3], bbox[2], score]) # for bbox in refined_proposals[keep][ids]: # # print(centerCoord) # # save_img = cv2.circle(save_img, centerCoord, 4, (255, 0, 255), -1) # image = cv2.rectangle(image, (bbox[1], bbox[0]), (bbox[3], bbox[2]), (0, 0, 255)) # # save_img = cv2.rectangle(save_img,(box[1],box[0]),(box[3],box[2]),(0,0,255)) return image, final_bboxes_before_NMS
for image_id in dataset.image_ids: print("Image id: ", image_id) filename = "maskrcnn_ensemble_input/" + str(dataset.image_reference(image_id)) + ".png" #if os.path.isfile(filename): # continue image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) mrcnn = model.run_graph([image], [ ("detections", model.keras_model.get_layer("mrcnn_detection").output), ("masks", model.keras_model.get_layer("mrcnn_mask").output), ]) det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32) det_count = np.where(det_class_ids == 0)[0][0] det_class_ids = det_class_ids[:det_count] print("{} detections: {}".format(det_count, np.array(dataset.class_names)[det_class_ids])) det_boxes = utils.denorm_boxes(mrcnn["detections"][0, :, :4], (512,512)) det_mask_specific = np.array([mrcnn["masks"][0, i, :, :, c] for i, c in enumerate(det_class_ids)]) det_masks = np.array([utils.unmold_mask(m, det_boxes[i], (512,512,3)) for i, m in enumerate(det_mask_specific)]) log("det_mask_specific", det_mask_specific) log("det_masks", det_masks) print(np.unique(det_masks)) a = np.sum(det_masks, axis=0) > 0 # create directory using terminal if it doesn't already exist plt.imsave(fname = "maskrcnn_final_ensemble_input/" + str(dataset.image_reference(image_id)+".png"), arr = a, cmap="Blues")
gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], dataset.class_names, ax=get_ax(), show_box=False, show_mask=False, iou_threshold=0.5, score_threshold=0.5) # Get anchors and convert to pixel coordinates anchors = model.get_anchors(image.shape) anchors = utils.denorm_boxes(anchors, image.shape[:2]) log("anchors", anchors) # Generate RPN trainig targets # target_rpn_match is 1 for positive anchors, -1 for negative anchors # and 0 for neutral anchors. target_rpn_match, target_rpn_bbox = modellib.build_rpn_targets( image.shape, anchors, gt_class_id, gt_bbox, model.config) log("target_rpn_match", target_rpn_match) log("target_rpn_bbox", target_rpn_bbox) positive_anchor_ix = np.where(target_rpn_match[:] == 1)[0] negative_anchor_ix = np.where(target_rpn_match[:] == -1)[0] neutral_anchor_ix = np.where(target_rpn_match[:] == 0)[0] positive_anchors = anchors[positive_anchor_ix] negative_anchors = anchors[negative_anchor_ix]
def display_mrcnn_prediction(): resized_image, image_meta, gt_class_ids, gt_bboxes, gt_masks = \ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) # Get input and output to classifier and mask heads. mrcnn = model.run_graph([resized_image], [ ("proposals", model.keras_model.get_layer("ROI").output), ("probs", model.keras_model.get_layer("mrcnn_class").output), ("deltas", model.keras_model.get_layer("mrcnn_bbox").output), ("masks", model.keras_model.get_layer("mrcnn_mask").output), ("detections", model.keras_model.get_layer("mrcnn_detection").output), ]) ax = get_ax(1, 4) ################################## display detections ############################################### # Get detection class IDs. Trim zero padding. det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32) padding_start_ix = np.where(det_class_ids == 0)[0][0] det_class_ids = det_class_ids[:padding_start_ix] detections = mrcnn['detections'][0, :padding_start_ix] log('trimmed_detection', detections) print("{} detections: {}".format( padding_start_ix, np.array(dataset.class_names)[det_class_ids])) captions = [ "{} {:.3f}".format(dataset.class_names[int(class_id)], score) if class_id > 0 else "" for class_id, score in zip(detections[:, 4], detections[:, 5]) ] visualize.draw_boxes(resized_image.copy(), refined_boxes=utils.denorm_boxes( detections[:, :4], resized_image.shape[:2]), visibilities=[2] * len(detections), captions=captions, title="Detections", ax=ax[0]) ################################### display proposals ########################################## # Proposals are in normalized coordinates. Scale them to image coordinates. h, w = resized_image.shape[:2] proposals = np.around(mrcnn["proposals"][0] * np.array([h, w, h, w])).astype(np.int32) # Class ID, score, and mask per proposal # mrcnn 的 shape 为 (batch_size, num_proposals=1000, num_classes) proposal_class_ids = np.argmax(mrcnn["probs"][0], axis=1) proposal_class_scores = mrcnn["probs"][ 0, np.arange(proposal_class_ids.shape[0]), proposal_class_ids] proposal_class_names = np.array(dataset.class_names)[proposal_class_ids] proposal_positive_ixs = np.where(proposal_class_ids > 0)[0] # How many ROIs vs empty rows? print("{} valid proposals out of {}".format( np.sum(np.any(proposals, axis=1)), proposals.shape[0])) print("{} positive ROIs".format(len(proposal_positive_ixs))) # Class counts print(list(zip(*np.unique(proposal_class_names, return_counts=True)))) # Display a random sample of proposals. # Proposals classified as background are dotted, and # the rest show their class and confidence score. limit = 200 ixs = np.random.randint(0, proposals.shape[0], limit) captions = [ "{} {:.3f}".format(dataset.class_names[c], s) if c > 0 else "" for c, s in zip(proposal_class_ids[ixs], proposal_class_scores[ixs]) ] visualize.draw_boxes(resized_image.copy(), boxes=proposals[ixs], visibilities=np.where(proposal_class_ids[ixs] > 0, 2, 1), captions=captions, title="Proposals Before Refinement", ax=ax[1]) #################################### apply bounding box refinement ############################# # Class-specific bounding box shifts. # mrcnn['deltas'] 的 shape 为 (batch_size, num_proposals=1000, num_classes, 4) proposal_deltas = mrcnn["deltas"][0, np.arange(proposals.shape[0]), proposal_class_ids] log("proposals_deltas", proposal_deltas) # Apply bounding box transformations # Shape: (num_proposals=1000, (y1, x1, y2, x2)] # NOTE: delta 是不分 normalized coordinates 和 pixel coordinates 的 refined_proposals = utils.apply_box_deltas( proposals, proposal_deltas * config.BBOX_STD_DEV).astype(np.int32) log("refined_proposals", refined_proposals) # Show positive proposals # ids = np.arange(proposals.shape[0]) # Display all limit = 5 ids = np.random.randint(0, len(proposal_positive_ixs), limit) # Display random sample captions = [ "{} {:.3f}".format(dataset.class_names[class_id], score) if class_id > 0 else "" for class_id, score in zip( proposal_class_ids[proposal_positive_ixs][ids], proposal_class_scores[proposal_positive_ixs][ids]) ] visualize.draw_boxes( resized_image.copy(), boxes=proposals[proposal_positive_ixs][ids], refined_boxes=refined_proposals[proposal_positive_ixs][ids], visibilities=np.where( proposal_class_ids[proposal_positive_ixs][ids] > 0, 1, 0), captions=captions, title="ROIs After Refinement", ax=ax[2]) #################################### more steps ################################################ # Remove boxes classified as background keep_proposal_ixs = np.where(proposal_class_ids > 0)[0] print("Remove background proposals. Keep {}:\n{}".format( keep_proposal_ixs.shape[0], keep_proposal_ixs)) # Remove low confidence detections keep_proposal_ixs = np.intersect1d( keep_proposal_ixs, np.where(proposal_class_scores >= config.DETECTION_MIN_CONFIDENCE)[0]) print("Remove proposals below {} confidence. Keep {}:\n{}".format( config.DETECTION_MIN_CONFIDENCE, keep_proposal_ixs.shape[0], keep_proposal_ixs)) # Apply per-class non-max suppression pre_nms_proposals = refined_proposals[keep_proposal_ixs] pre_nms_proposal_scores = proposal_class_scores[keep_proposal_ixs] pre_nms_proposal_class_ids = proposal_class_ids[keep_proposal_ixs] nms_keep_proposal_ixs = [] for class_id in np.unique(pre_nms_proposal_class_ids): # Pick detections of this class ixs = np.where(pre_nms_proposal_class_ids == class_id)[0] # Apply NMS class_keep = utils.non_max_suppression(pre_nms_proposals[ixs], pre_nms_proposal_scores[ixs], config.DETECTION_NMS_THRESHOLD) # Map indicies class_keep_proposal_ixs = keep_proposal_ixs[ixs[class_keep]] nms_keep_proposal_ixs = np.union1d(nms_keep_proposal_ixs, class_keep_proposal_ixs) print("{:12}: {} -> {}".format(dataset.class_names[class_id][:10], keep_proposal_ixs[ixs], class_keep_proposal_ixs)) keep_proposal_ixs = np.intersect1d(keep_proposal_ixs, nms_keep_proposal_ixs).astype(np.int32) print("\nKeep after per-class NMS: {}\n{}".format( keep_proposal_ixs.shape[0], keep_proposal_ixs)) #################################### Show final detections ##################################### ixs = np.arange(len(keep_proposal_ixs)) # Display all # ixs = np.random.randint(0, len(keep), 10) # Display random sample captions = [ "{} {:.3f}".format(dataset.class_names[c], s) if c > 0 else "" for c, s in zip(proposal_class_ids[keep_proposal_ixs][ixs], proposal_class_scores[keep_proposal_ixs][ixs]) ] visualize.draw_boxes( resized_image.copy(), boxes=proposals[keep_proposal_ixs][ixs], refined_boxes=refined_proposals[keep_proposal_ixs][ixs], visibilities=np.where(proposal_class_ids[keep_proposal_ixs][ixs] > 0, 1, 0), captions=captions, title="Detections after NMS", ax=ax[3]) plt.show()
def display_rpn_prediction(): # Run RPN sub-graph resized_image, image_meta, gt_class_ids, gt_bboxes, gt_masks = \ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) pillar = model.keras_model.get_layer( "ROI").output # node to start searching from # TF 1.4 and 1.9 introduce new versions of NMS. Search for all names to support TF 1.3~1.10 nms_node = model.ancestor(pillar, "ROI/rpn_non_max_suppression:0") if nms_node is None: nms_node = model.ancestor( pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV2:0") if nms_node is None: # TF 1.9-1.10 nms_node = model.ancestor( pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV3:0") rpn = model.run_graph([resized_image], [ ("rpn_class", model.keras_model.get_layer("rpn_class").output), ("pre_nms_anchors", model.ancestor(pillar, "ROI/pre_nms_anchors:0")), ("refined_anchors", model.ancestor(pillar, "ROI/refined_anchors:0")), ("refined_anchors_clipped", model.ancestor(pillar, "ROI/refined_anchors_clipped:0")), ("post_nms_anchor_ix", nms_node), ("proposals", model.keras_model.get_layer("ROI").output), ]) ax = get_ax(2, 3) # Show top anchors by score (before refinement) limit = 100 # np.flatten() 会把多维数组变成一维数组, 那么此处就默认 batch_size=1, 否则不能这样计算 # 按从大到小排序 sorted_anchor_ids = np.argsort(rpn['rpn_class'][:, :, 1].flatten())[::-1] visualize.draw_boxes(resized_image, boxes=model.anchors[sorted_anchor_ids[:limit]], ax=ax[0, 0]) # Show top anchors with refinement. Then with clipping to image boundaries limit = 50 pre_nms_anchors = utils.denorm_boxes(rpn["pre_nms_anchors"][0], resized_image.shape[:2]) refined_anchors = utils.denorm_boxes(rpn["refined_anchors"][0], resized_image.shape[:2]) visualize.draw_boxes(resized_image, boxes=pre_nms_anchors[:limit], refined_boxes=refined_anchors[:limit], ax=ax[0, 1]) refined_anchors_clipped = utils.denorm_boxes( rpn["refined_anchors_clipped"][0], resized_image.shape[:2]) visualize.draw_boxes(resized_image, refined_boxes=refined_anchors_clipped[:limit], ax=ax[0, 2]) # Show refined anchors after non-max suppression ixs = rpn["post_nms_anchor_ix"][:limit] visualize.draw_boxes(resized_image, refined_boxes=refined_anchors_clipped[ixs], ax=ax[1, 0]) # Show final proposals # These are the same as the previous step (refined anchors # after NMS) but with coordinates normalized to [0, 1] range. # Convert back to image coordinates for display h, w = resized_image.shape[:2] proposals = rpn['proposals'][0, :limit] * np.array([h, w, h, w]) visualize.draw_boxes(resized_image, refined_boxes=proposals, ax=ax[1, 1]) plt.show()
def read_results_to_df(training_results_file): cols=['epoch','y1','x1','y2','x2','0','1','2','3','4','5','6','7','8',\ 'p0','p1','p2','p3','p4','p5','p6','p7','p8'] # df = pd.read_csv(address,index_col=None,sep=",") res_arr=np.loadtxt(training_results_file,\ skiprows=0,delimiter=",",dtype={'names': tuple(cols),\ 'formats': ('i4', 'f4', 'f4', 'f4', 'f4', 'i4', 'i4', 'i4',\ 'i4', 'i4', 'i4', 'i4', 'i4', 'i4', 'f4', 'f4',\ 'f4', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4')}) res_df = pd.DataFrame(res_arr, columns=cols) # print('here:',image.shape, molded_images[i].shape,windows[i]) # (256, 256, 4) (256, 256, 4) # unq_epochs=res_df.epoch.unique() # res_df=res_df[res_df['epoch']==unq_epochs[1]].reset_index(drop=True) boxes = res_df[['y1', 'x1', 'y2', 'x2']].values image_shape = (256, 256, 4) original_image_shape = (256, 256, 4) window = [0, 0, 256, 256] window = utils.norm_boxes(window, image_shape[:2]) wy1, wx1, wy2, wx2 = window shift = np.array([wy1, wx1, wy1, wx1]) wh = wy2 - wy1 # window height ww = wx2 - wx1 # window width scale = np.array([wh, ww, wh, ww]) # Convert boxes to normalized coordinates on the window boxes = np.divide(boxes - shift, scale) # Convert boxes to pixel coordinates on the original image boxes = utils.denorm_boxes(boxes, original_image_shape[:2]) res_df[['y1_', 'x1_', 'y2_', 'x2_']] = boxes # Filter out detections with zero area. Happens in early training when # network weights are still random exclude_ix = np.where((boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) <= 0)[0] exclude_ix_big_area = np.where((boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) > 20)[0] # res_df[res_df] update_df = res_df.drop(exclude_ix) # update_df2 = update_df.drop(exclude_ix_big_area) # update_df # if exclude_ix.shape[0] > 0: # boxes = np.delete(boxes, exclude_ix, axis=0) # class_probs = np.delete(class_probs, exclude_ix, axis=0) # class_ids = np.delete(class_ids, exclude_ix, axis=0) # scores = np.delete(scores, exclude_ix, axis=0) # masks = np.delete(masks, exclude_ix, axis=0) # N = class_ids.shape[0] # height = box[:, 2] - box[:, 0] height = res_df['y2_'] - res_df['y1_'] width = res_df['x2_'] - res_df['x1_'] # width = box[:, 3] - box[:, 1] update_df['bb_center_y'] = res_df['y1_'] + 0.5 * height update_df['bb_center_x'] = res_df['x1_'] + 0.5 * width return update_df
("detections", model.keras_model.get_layer("mrcnn_detection").output), ("masks", model.keras_model.get_layer("mrcnn_mask").output), ]) # Get detection class IDs. Trim zero padding. det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32) # det_class_ids는 detection한 object수만큼 list가 생성되며 각 리스트에는 분류될 class 수(3)(Llane, car, Slane)가 들어간다. det_count = np.where(det_class_ids == 0)[0][0] det_class_ids = det_class_ids[:det_count] print((det_class_ids[0])) print((det_class_ids[1])) #display_images(d) print("{} detections: {}".format(det_count, np.array(dataset.class_names)[det_class_ids])) det_boxes = utils.denorm_boxes(mrcnn["detections"][0, :, :4], image.shape[:2]) det_mask_specific = np.array( [mrcnn["masks"][0, i, :, :, c] for i, c in enumerate(det_class_ids)]) det_masks = np.array([ utils.unmold_mask(m, det_boxes[i], image.shape) for i, m in enumerate(det_mask_specific) ]) print("here \n", det_mask_specific[1]) ''' one, two, thr, four, fiv, six, sev = 0,0,0,0,0,0,0 # det_masks는 1024*1024의 이미지를 detection Object의 갯수만큼 갖고 있다. EX. [2][1024][1024] 갯수*가로*세로 and 한pixel당 float형으로 표현 for a in range(len(det_mask_specific)): # a 는 detect Box 갯수 for i in range(len(det_mask_specific[a])): # i 는 세로 픽셀 수 for j in range(len(det_mask_specific[a][i])): # j 는 가로 픽셀 수 if 0.000 in (det_mask_specific[0][i][j], float): if a == 0:
image, imageMeta, groundTruthClassId, groundTruthBoundingBox, groundTruthMask = modellib.load_image_gt( dataset, config, iid, use_mini_mask=False) maskrcnnOutput = model.run_graph([image], [ ("detections", model.keras_model.get_layer("mrcnn_detection").output), ("masks", model.keras_model.get_layer("mrcnn_mask").output), ]) detectedClasses = maskRCnnOutput['detections'][0, :, 4].astype(np.int32) numberOfDetections = np.where(det_class_ids == 0)[0][0] detectedClasses = detectedClasses[:numberOfDetections] print("{} detections: {}".format( numberOfDetections, np.array(dataset.class_names)[detectedClasses])) detectedBoundingBoxes = utils.denorm_boxes( maskrcnnOutput["detections"][0, :, :4], (512, 512)) detectedmasks = np.array([ maskRCnnOutput["masks"][0, i, :, :, c] for i, c in enumerate(detectedClasses) ]) # detectedMasks reshaped to (512,512,3) to use as input for ensemble model detectedMasks = np.array([ utils.unmold_mask(m, detectedBoundingBoxes[i], (512, 512, 3)) for i, m in enumerate(detectedmasks) ]) print(np.unique(detectedMasks)) a = np.sum(detectedMasks, axis=0) > 0 # create directory using terminal if it doesn't already exist plt.imsave(fname="maskrcnn_final_test_ensemble_input/" + str(dataset.image_reference(iid) + ".png"),
("proposals", model.keras_model.get_layer("ROI").output), ]) #%% # Show top anchors by score (before refinement) limit = 100 sorted_anchor_ids = np.argsort(rpn['rpn_class'][:, :, 1].flatten())[::-1] visualize.draw_boxes(image, boxes=model.anchors[sorted_anchor_ids[:limit]], ax=get_ax()) #%% # Show top anchors with refinement. Then with clipping to image boundaries limit = 50 ax = get_ax(1, 2) pre_nms_anchors = utils.denorm_boxes(rpn["pre_nms_anchors"][0], image.shape[:2]) refined_anchors = utils.denorm_boxes(rpn["refined_anchors"][0], image.shape[:2]) refined_anchors_clipped = utils.denorm_boxes(rpn["refined_anchors_clipped"][0], image.shape[:2]) visualize.draw_boxes(image, boxes=pre_nms_anchors[:limit], refined_boxes=refined_anchors[:limit], ax=ax[0]) visualize.draw_boxes(image, refined_boxes=refined_anchors_clipped[:limit], ax=ax[1]) #%% # Show refined anchors after non-max suppression limit = 50
def calculate_mask(model, graph, image, rois): ''' inputs: model: sd mask rcnn model graph: model graph (provide keras to calculate) image: filled hole 8-bit depth cropped image (specified with rois) rois: Full image ROI patch(x1,y1,x2,y2) returns: det_masks: mask on 512*512 scale (values=0 or 1) det_mask_specific: predict mask on CNN (28*28) (values=0 or 1) resized_mask_image: max facet detected image (ROI Size) ''' print('MAKING SD MASK RCNN PREDICTIONS') class_names = ['bg', 'fg'] image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB) print(image.shape) # Resize image for net image, _, _, _, _ = resize_image(image, max_dim=512) #cv2.imshow("Test",image) #cv2.waitKey(0) # Get predictions of mask head #model.keras_model._make_predict_function() with graph.as_default(): mrcnn = model.run_graph([image], [ ("detections", model.keras_model.get_layer("mrcnn_detection").output), ("masks", model.keras_model.get_layer("mrcnn_mask").output), ]) # Get detection class IDs. Trim zero padding. det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32) det_count = np.where(det_class_ids == 0)[0][0] det_class_ids = det_class_ids[:det_count] print("{} detections: {}".format(det_count, np.array(class_names)[det_class_ids])) # Masks det_boxes = m_utils.denorm_boxes(mrcnn["detections"][0, :, :4], image.shape[:2]) det_mask_specific = np.array( [mrcnn["masks"][0, i, :, :, c] for i, c in enumerate(det_class_ids)]) det_masks = np.array([ m_utils.unmold_mask(m, det_boxes[i], image.shape) for i, m in enumerate(det_mask_specific) ]) if det_count != 0: log("det_mask_specific", det_mask_specific) log("det_masks", det_masks) else: print("No facet detected!!") return resized_mask_image, no_padding_range = detect_show_instance( image, model, graph, class_names, rois) return det_masks, det_mask_specific, resized_mask_image, no_padding_range