def overlay_mask(self, image, predictions): """ Adds the instances contours for each predicted object. Each label has a different color. Arguments: image (np.ndarray): an image as returned by OpenCV predictions (BoxList): the result of the computation by the model. It should contain the field `mask` and `labels`. """ masks = predictions.get_field("mask").numpy() labels = predictions.get_field("labels") colors = self.compute_colors_for_labels(labels).tolist() for mask, color in zip(masks, colors): thresh = mask[0, :, :, None] contours, hierarchy = cv2_util.findContours( thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) image = cv2.drawContours(image, contours, -1, color, 3) composite = image return composite
def vis_one_image( im, im_name, output_dir, boxes, segms=None, thresh=0.9, dpi=200, box_alpha=0.3, class_index_to_name=None, show_class=False, ext='tif', out_when_no_box=False): """Visual debugging of detections.""" if not os.path.exists(output_dir): os.makedirs(output_dir) if isinstance(boxes, list): boxes, segms, classes = convert_from_cls_format( boxes, segms) if (boxes is None or boxes.shape[0] == 0) and not out_when_no_box: return if (boxes.shape[1] == 5 and max(boxes[:, 4]) < thresh) and not out_when_no_box: return if segms is not None and len(segms) > 0: masks = mask_util.decode(segms) color_list = colormap.colormap(rgb=True) / 255 fig = plt.figure(frameon=False) fig.set_size_inches(im.shape[1] / dpi, im.shape[0] / dpi) ax = plt.Axes(fig, [0., 0., 1., 1.]) ax.axis('off') fig.add_axes(ax) ax.imshow(im) if boxes is None: sorted_inds = [] # avoid crash when 'boxes' is None else: # Display in largest to smallest order to reduce occlusion areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) sorted_inds = np.argsort(-areas) mask_color_id = 0 for i in sorted_inds: bbox = boxes[i, :4] score = boxes[i, -1] if boxes.shape[1] > 4 else 1.0 if score < thresh: continue # show box (off by default) ax.add_patch( plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=False, edgecolor='g', linewidth=1, alpha=box_alpha)) if show_class: ax.text( bbox[0], bbox[1] - 2, get_class_string(classes[i], score, class_index_to_name), fontsize=3, family='serif', bbox=dict( facecolor='g', alpha=0.4, pad=0, edgecolor='none'), color='white') # show mask if segms is not None and len(segms) > i: img = np.ones(im.shape) color_mask = color_list[mask_color_id % len(color_list), 0:3] mask_color_id += 1 w_ratio = .4 for c in range(3): color_mask[c] = color_mask[c] * (1 - w_ratio) + w_ratio for c in range(3): img[:, :, c] = color_mask[c] e = masks[:, :, i] contour = cv2_util.findContours( e.copy(), cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE)[-2] for c in contour: polygon = Polygon( c.reshape((-1, 2)), fill=True, facecolor=color_mask, edgecolor='w', linewidth=0, alpha=0.5, antialiased=False) ax.add_patch(polygon) output_name = os.path.basename(im_name) + '.' + ext fig.savefig(os.path.join(output_dir, '{}'.format(output_name)), dpi=dpi) plt.close('all')
def select_the_connected_mask(self, image, predictions): flag = False dictPosition = {} # 剔除提出一些可能会被判定为背景和纹理的类目 del_set = { "sink", "bench", "couch", "stop sign", "chair", "traffic light", "dining table", "broccoli" } dictPosition['width_position'] = None dictPosition['height_position'] = None dictPosition['angle'] = None dictPosition['sub'] = [] imageHeight, imageWidth = image.shape[:2] masks = predictions.get_field("mask").numpy() labels = predictions.get_field("labels") scores = predictions.get_field("scores").tolist() colors = self.compute_colors_for_labels(labels).tolist() point_array = None for mask, color, score, label in zip(masks, colors, scores, labels): thresh = mask[0, :, :, None] contours, hierarchy = cv2_util.findContours( thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) # 剔除提出一些可能会被判定为背景和纹理的类目 if not contours or self.CATEGORIES[label] in del_set: continue # 剔除一些低置信度的人类和鸟类的轮廓 if (self.CATEGORIES[label] == 'person' or self.CATEGORIES[label] == 'bird') and score < 0.5: continue image = cv2.drawContours(image, contours, -1, color, 3) # 高置信度的单独画框 if score >= 0.7: sub_dict = {} contours = [ np.reshape(unit, (unit.shape[0], -1)) for unit in contours ] if len(contours) > 1: contours_array = np.concatenate( (tuple([unit for unit in contours])), axis=0) else: contours_array = contours[0] rect_single = cv2.minAreaRect(contours_array) box_single = cv2.boxPoints(rect_single) box_single = np.int0(box_single) cv2.drawContours(image, [box_single], 0, (0, 0, 255), 3) width, height = rect_single[0] reWidth, reHeight = rect_single[1] angle = rect_single[2] sub_dict['class'] = self.CATEGORIES[label] sub_dict['width_position'] = width / imageWidth sub_dict['height_position'] = height / imageHeight if reWidth <= reHeight: sub_dict['angle'] = round(-angle, 2) else: sub_dict['angle'] = round(-angle + 90, 2) dictPosition['sub'].append(sub_dict) # 将所有mask的所有点集合合并 for contour in contours: pointset = np.reshape(contour, (contour.shape[0], -1)) if point_array is None: point_array = pointset else: point_array = np.concatenate((point_array, pointset), axis=0) #作出所有点集合的最小外接矩形 if point_array is not None: rect = cv2.minAreaRect(point_array) box = cv2.boxPoints(rect) box = np.int0(box) cv2.drawContours(image, [box], 0, (255, 0, 0), 3) width, height = rect[0] reWidth, reHeight = rect[1] angle = rect[2] # 根据矩形的中点 判断居中程度 dictPosition['width_position'] = width / imageWidth dictPosition['height_position'] = height / imageHeight #根据长宽比判断矩形的旋转角度是否大于90度度度,90-180度的倾斜程度和0-90度相同,但是方向不同 if reWidth <= reHeight: dictPosition['angle'] = round(-angle, 2) else: dictPosition['angle'] = round(-angle + 90, 2) if dictPosition['width_position']: flag = True composite = image return composite, dictPosition, flag
def overlay_mask(self, image, predictions, color_rgb, inference=False): """ Adds the instances contours for each predicted object. Each label has a different color. Arguments: image (np.ndarray): an image as returned by OpenCV predictions (BoxList): the result of the computation by the model. It should contain the field `mask` and `labels`. """ masks = predictions.get_field("mask").numpy() labels = predictions.get_field("labels") # colors for predictions colors = { i: [s / 255 for s in color] for i, color in enumerate(color_rgb) } colors = [colors[i.item()] for i in labels] # do this if you want to make inference if inference == True: polys = [] for mask, color in zip(masks, colors): mask = np.squeeze(mask, axis=0) contours = measure.find_contours(mask, 0.5, positive_orientation="low") polygons_seg = [] for contour in contours: for i in range(len(contour)): row, col = contour[i] contour[i] = (col - 1, row - 1) try: poly = Polygon_shapely(contour) poly = poly.simplify(1.0, preserve_topology=False) poly = np.array(poly.exterior.coords).ravel().tolist() polygons_seg.append(poly) except AttributeError: print('Attribute Error raised') continue if len(polygons_seg) > 1: polygons_seg = np.array(polygons_seg[0]) #print(polygons_seg, '___THIS') polygons_seg = polygons_seg.reshape( (int(len(polygons_seg) / 2), 2)) #polys.append(Polygon(polygons_seg)) elif len(polygons_seg) > 0: polygons_seg = np.array(polygons_seg)[0] #print(polygons_seg, '___THIS') polygons_seg = polygons_seg.reshape( (int(len(polygons_seg) / 2), 2)) else: continue polys.append(Polygon(polygons_seg)) return polys, colors # do it differently, thickness can only be changed to 1 (too thin) or 2 (too thick) at # least for images with the size 512x512 else: thickness_contour = int((image.shape[0] / 1024) * 2) print(thickness_contour) for mask, color in zip(masks, colors): # fix for CV2 version check issue #339 print(np.unique(mask)) thresh = mask[0, :, :, None] contours, hierarchy = cv2_util.findContours( thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) # change width of the mask here # cv2.drawContours(image, contours, contourIdx, color[, thickness[, lineType[, hierarchy[, maxLevel[, offset]]]]]) image = cv2.drawContours(image, contours, -1, [i * 255 for i in color], thickness_contour) composite = image return composite
def overlay_mask(self, image, predictions): """ Adds the instances contours for each predicted object. Each label has a different color. Arguments: image (np.ndarray): an image as returned by OpenCV predictions (BoxList): the result of the computation by the model. It should contain the field `mask` and `labels`. """ #masker = Masker(threshold=0.5, padding=1) masks = predictions.get_field("mask").numpy() #print(masks.shape) #https://code.ihub.org.cn/projects/578/repository/revisions/master/entry/maskrcnn_benchmark/engine/inference.py labels = predictions.get_field("labels").tolist() labels = [self.CATEGORIES[i] for i in labels] for mask, label in zip(masks, labels): thresh = mask[0, :, :, None] contours, hierarchy = cv2_util.findContours( thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) if label == "BE": #image = cv2.drawContours(image, contours, -1, (255,255,255), 2) mask2 = np.zeros_like(image) mask2 = cv2.drawContours(mask2, contours, -1, (255, 255, 255), -1) BE = np.zeros_like( image ) # Extract out the object and place into output image BE[mask2 == 255] = image[mask2 == 255] BE_gray = cv2.cvtColor(BE, cv2.COLOR_BGR2GRAY) (thresh, BE) = cv2.threshold(BE_gray, 10, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU) cv2.imwrite(save_path + name + '_BE.tif', BE) elif label == "suspicious": mask2 = np.zeros_like(image) mask2 = cv2.drawContours(mask2, contours, -1, (255, 255, 255), -1) S = np.zeros_like( image ) # Extract out the object and place into output image S[mask2 == 255] = image[mask2 == 255] S_gray = cv2.cvtColor(S, cv2.COLOR_BGR2GRAY) #(thresh, S) = cv2.threshold(S_gray, 128, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU) (thresh, S) = cv2.threshold(S_gray, 10, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU) cv2.imwrite(save_path + name + '_suspicious.tif', S) elif label == "HGD": mask2 = np.zeros_like(image) mask2 = cv2.drawContours(mask2, contours, -1, (255, 255, 255), -1) H = np.zeros_like( image ) # Extract out the object and place into output image H[mask2 == 255] = image[mask2 == 255] H_gray = cv2.cvtColor(H, cv2.COLOR_BGR2GRAY) (thresh, H) = cv2.threshold(H_gray, 10, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU) cv2.imwrite(save_path + name + '_HGD.tif', H) elif label == "cancer": mask2 = np.zeros_like(image) mask2 = cv2.drawContours(mask2, contours, -1, (255, 255, 255), -1) C = np.zeros_like( image ) # Extract out the object and place into output image C[mask2 == 255] = image[mask2 == 255] C_gray = cv2.cvtColor(C, cv2.COLOR_BGR2GRAY) (thresh, C) = cv2.threshold(C_gray, 10, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU) #C=C.convertTo(img8bit, CV_32FC3) cv2.imwrite(save_path + name + '_cancer.tif', C) elif label == "polyp": mask2 = np.zeros_like(image) mask2 = cv2.drawContours(mask2, contours, -1, (255, 255, 255), -1) P = np.zeros_like( image ) # Extract out the object and place into output image P[mask2 == 255] = image[mask2 == 255] P_gray = cv2.cvtColor(P, cv2.COLOR_BGR2GRAY) (thresh, P) = cv2.threshold(P_gray, 10, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU) cv2.imwrite(save_path + name + '_polyp.tif', P) #cv2.imwrite('/home/zst19phu/benchmark/benchmark-code/datasets/edd2020/image.jpg', image) composite = image return composite
def overlay_mask_full(self, image, predictions, boxsize, boxcrop, category, is_crop=True, draw=False, multi_object=False): masks = predictions.get_field("mask").numpy() labels = predictions.get_field("labels") lab_list = labels.tolist() colors = self.compute_colors_for_labels(labels).tolist() #print(masks[0][0, :, :, None].astype(np.uint8)) #print(masks[0][0, :, :, None].astype(np.uint8).shape) #single_mask = masks[4][0, :, :, None].astype(np.uint8) #print(colors) #mask3 = cv2.cvtColor(single_mask, cv2.COLOR_GRAY2BGR) #print(np.array(masks3)[0].astype(np.uint8)) #print(mask3) threshs = [] for mask, color, lablist in zip(masks, colors, lab_list): thresh = mask[0, :, :, None].astype(np.uint8) contours, hierarchy = cv2_util.findContours( thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) #print(np.array(contours)) if draw: image = cv2.drawContours(image, contours, -1, color, 3) #if lablist==3: thresh = thresh * 255 threshs.append(thresh) #image = image * mask.astype(np.uint8) #image = cv2.bitwise_and(image, mask3) #print(image.shape) car_thresh = [] car_boxsize = [] car_boxcrop = [] is_find = True for i in range(len(lab_list)): if lab_list[i] == category: car_thresh.append(threshs[i]) car_boxsize.append(boxsize[i]) car_boxcrop.append(boxcrop[i]) try: largest_thresh = np.argmax(np.array(car_boxsize)) if not multi_object: image = cv2.bitwise_not(image) image = cv2.bitwise_and( image, cv2.cvtColor(threshs[largest_thresh], cv2.COLOR_GRAY2BGR)) image = cv2.bitwise_not(image) else: image = cv2.bitwise_not(image) mask_or = self.add_images(car_thresh) image = cv2.bitwise_and( image, cv2.cvtColor(mask_or, cv2.COLOR_GRAY2BGR)) image = cv2.bitwise_not(image) #image = cv2.bitwise_not(image) if is_crop: x, x_ = car_boxcrop[largest_thresh][0][0], car_boxcrop[ largest_thresh][1][0] y, y_ = car_boxcrop[largest_thresh][0][1], car_boxcrop[ largest_thresh][1][1] image = image[y:y_, x:x_] except: print("Find nothing! Return original image!") is_find = False composite = image return composite, is_find
if has_labels: label = data["labels"][ix] if has_rrects: rr = data["rrects"][ix] img_copy = draw_anchors(img_copy, [rr], [[0, 0, 255]]) else: img_copy = cv2.rectangle(img_copy, tuple(bbox[:2]), tuple(bbox[2:]), (0, 0, 255), 2) if has_masks: mask = data["masks"][ix] contours, hierarchy = cv2_util.findContours( mask.astype(np.uint8) * 255, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) color = get_random_color() img_copy = cv2.drawContours(img_copy, contours, -1, color, 3) # from maskrcnn_benchmark.modeling.rotate_ops import merge_rrects_by_iou # if has_masks and has_rrects: # img_copy2 = img.copy() # # match_inds = merge_rrects_by_iou(data["rrects"], iou_thresh=0.5) # # masks = data["masks"] # for idx, inds in match_inds.items(): # if len(inds) == 0: # continue # mask = masks[inds[0]]