def overlay_mask(self, image, predictions):
"""
Adds the instances contours for each predicted object.
Each label has a different color.
Arguments:
image (np.ndarray): an image as returned by OpenCV
predictions (BoxList): the result of the computation by the model.
It should contain the field `mask` and `labels`.
"""
masks = predictions.get_field("mask").numpy()
labels = predictions.get_field("labels")
colors = self.compute_colors_for_labels(labels).tolist()
for mask, color in zip(masks, colors):
thresh = mask[0, :, :, None]
contours, hierarchy = cv2_util.findContours(
thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
image = cv2.drawContours(image, contours, -1, color, 3)
composite = image
return composite
def vis_one_image(
im, im_name, output_dir, boxes, segms=None, thresh=0.9,
dpi=200, box_alpha=0.3, class_index_to_name=None, show_class=False,
ext='tif', out_when_no_box=False):
"""Visual debugging of detections."""
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if isinstance(boxes, list):
boxes, segms, classes = convert_from_cls_format(
boxes, segms)
if (boxes is None or boxes.shape[0] == 0) and not out_when_no_box:
return
if (boxes.shape[1] == 5 and max(boxes[:, 4]) < thresh) and not out_when_no_box:
return
if segms is not None and len(segms) > 0:
masks = mask_util.decode(segms)
color_list = colormap.colormap(rgb=True) / 255
fig = plt.figure(frameon=False)
fig.set_size_inches(im.shape[1] / dpi, im.shape[0] / dpi)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.axis('off')
fig.add_axes(ax)
ax.imshow(im)
if boxes is None:
sorted_inds = [] # avoid crash when 'boxes' is None
else:
# Display in largest to smallest order to reduce occlusion
areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
sorted_inds = np.argsort(-areas)
mask_color_id = 0
for i in sorted_inds:
bbox = boxes[i, :4]
score = boxes[i, -1] if boxes.shape[1] > 4 else 1.0
if score < thresh:
continue
# show box (off by default)
ax.add_patch(
plt.Rectangle((bbox[0], bbox[1]),
bbox[2] - bbox[0],
bbox[3] - bbox[1],
fill=False, edgecolor='g',
linewidth=1, alpha=box_alpha))
if show_class:
ax.text(
bbox[0], bbox[1] - 2,
get_class_string(classes[i], score, class_index_to_name),
fontsize=3,
family='serif',
bbox=dict(
facecolor='g', alpha=0.4, pad=0, edgecolor='none'),
color='white')
# show mask
if segms is not None and len(segms) > i:
img = np.ones(im.shape)
color_mask = color_list[mask_color_id % len(color_list), 0:3]
mask_color_id += 1
w_ratio = .4
for c in range(3):
color_mask[c] = color_mask[c] * (1 - w_ratio) + w_ratio
for c in range(3):
img[:, :, c] = color_mask[c]
e = masks[:, :, i]
contour = cv2_util.findContours(
e.copy(), cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE)[-2]
for c in contour:
polygon = Polygon(
c.reshape((-1, 2)),
fill=True, facecolor=color_mask,
edgecolor='w', linewidth=0,
alpha=0.5, antialiased=False)
ax.add_patch(polygon)
output_name = os.path.basename(im_name) + '.' + ext
fig.savefig(os.path.join(output_dir, '{}'.format(output_name)), dpi=dpi)
plt.close('all')
def select_the_connected_mask(self, image, predictions):
flag = False
dictPosition = {}
# 剔除提出一些可能会被判定为背景和纹理的类目
del_set = {
"sink", "bench", "couch", "stop sign", "chair", "traffic light",
"dining table", "broccoli"
}
dictPosition['width_position'] = None
dictPosition['height_position'] = None
dictPosition['angle'] = None
dictPosition['sub'] = []
imageHeight, imageWidth = image.shape[:2]
masks = predictions.get_field("mask").numpy()
labels = predictions.get_field("labels")
scores = predictions.get_field("scores").tolist()
colors = self.compute_colors_for_labels(labels).tolist()
point_array = None
for mask, color, score, label in zip(masks, colors, scores, labels):
thresh = mask[0, :, :, None]
contours, hierarchy = cv2_util.findContours(
thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
# 剔除提出一些可能会被判定为背景和纹理的类目
if not contours or self.CATEGORIES[label] in del_set:
continue
# 剔除一些低置信度的人类和鸟类的轮廓
if (self.CATEGORIES[label] == 'person'
or self.CATEGORIES[label] == 'bird') and score < 0.5:
continue
image = cv2.drawContours(image, contours, -1, color, 3)
# 高置信度的单独画框
if score >= 0.7:
sub_dict = {}
contours = [
np.reshape(unit, (unit.shape[0], -1)) for unit in contours
]
if len(contours) > 1:
contours_array = np.concatenate(
(tuple([unit for unit in contours])), axis=0)
else:
contours_array = contours[0]
rect_single = cv2.minAreaRect(contours_array)
box_single = cv2.boxPoints(rect_single)
box_single = np.int0(box_single)
cv2.drawContours(image, [box_single], 0, (0, 0, 255), 3)
width, height = rect_single[0]
reWidth, reHeight = rect_single[1]
angle = rect_single[2]
sub_dict['class'] = self.CATEGORIES[label]
sub_dict['width_position'] = width / imageWidth
sub_dict['height_position'] = height / imageHeight
if reWidth <= reHeight:
sub_dict['angle'] = round(-angle, 2)
else:
sub_dict['angle'] = round(-angle + 90, 2)
dictPosition['sub'].append(sub_dict)
# 将所有mask的所有点集合合并
for contour in contours:
pointset = np.reshape(contour, (contour.shape[0], -1))
if point_array is None:
point_array = pointset
else:
point_array = np.concatenate((point_array, pointset),
axis=0)
#作出所有点集合的最小外接矩形
if point_array is not None:
rect = cv2.minAreaRect(point_array)
box = cv2.boxPoints(rect)
box = np.int0(box)
cv2.drawContours(image, [box], 0, (255, 0, 0), 3)
width, height = rect[0]
reWidth, reHeight = rect[1]
angle = rect[2]
# 根据矩形的中点 判断居中程度
dictPosition['width_position'] = width / imageWidth
dictPosition['height_position'] = height / imageHeight
#根据长宽比判断矩形的旋转角度是否大于90度度度，90-180度的倾斜程度和0-90度相同，但是方向不同
if reWidth <= reHeight:
dictPosition['angle'] = round(-angle, 2)
else:
dictPosition['angle'] = round(-angle + 90, 2)
if dictPosition['width_position']:
flag = True
composite = image
return composite, dictPosition, flag
def overlay_mask(self, image, predictions, color_rgb, inference=False):
"""
Adds the instances contours for each predicted object.
Each label has a different color.
Arguments:
image (np.ndarray): an image as returned by OpenCV
predictions (BoxList): the result of the computation by the model.
It should contain the field `mask` and `labels`.
"""
masks = predictions.get_field("mask").numpy()
labels = predictions.get_field("labels")
# colors for predictions
colors = {
i: [s / 255 for s in color]
for i, color in enumerate(color_rgb)
}
colors = [colors[i.item()] for i in labels]
# do this if you want to make inference
if inference == True:
polys = []
for mask, color in zip(masks, colors):
mask = np.squeeze(mask, axis=0)
contours = measure.find_contours(mask,
0.5,
positive_orientation="low")
polygons_seg = []
for contour in contours:
for i in range(len(contour)):
row, col = contour[i]
contour[i] = (col - 1, row - 1)
try:
poly = Polygon_shapely(contour)
poly = poly.simplify(1.0, preserve_topology=False)
poly = np.array(poly.exterior.coords).ravel().tolist()
polygons_seg.append(poly)
except AttributeError:
print('Attribute Error raised')
continue
if len(polygons_seg) > 1:
polygons_seg = np.array(polygons_seg[0])
#print(polygons_seg, '___THIS')
polygons_seg = polygons_seg.reshape(
(int(len(polygons_seg) / 2), 2))
#polys.append(Polygon(polygons_seg))
elif len(polygons_seg) > 0:
polygons_seg = np.array(polygons_seg)[0]
#print(polygons_seg, '___THIS')
polygons_seg = polygons_seg.reshape(
(int(len(polygons_seg) / 2), 2))
else:
continue
polys.append(Polygon(polygons_seg))
return polys, colors
# do it differently, thickness can only be changed to 1 (too thin) or 2 (too thick) at
# least for images with the size 512x512
else:
thickness_contour = int((image.shape[0] / 1024) * 2)
print(thickness_contour)
for mask, color in zip(masks, colors):
# fix for CV2 version check issue #339
print(np.unique(mask))
thresh = mask[0, :, :, None]
contours, hierarchy = cv2_util.findContours(
thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
# change width of the mask here
# cv2.drawContours(image, contours, contourIdx, color[, thickness[, lineType[, hierarchy[, maxLevel[, offset]]]]])
image = cv2.drawContours(image, contours, -1,
[i * 255 for i in color],
thickness_contour)
composite = image
return composite
def overlay_mask(self, image, predictions):
"""
Adds the instances contours for each predicted object.
Each label has a different color.
Arguments:
image (np.ndarray): an image as returned by OpenCV
predictions (BoxList): the result of the computation by the model.
It should contain the field `mask` and `labels`.
"""
#masker = Masker(threshold=0.5, padding=1)
masks = predictions.get_field("mask").numpy()
#print(masks.shape)
#https://code.ihub.org.cn/projects/578/repository/revisions/master/entry/maskrcnn_benchmark/engine/inference.py
labels = predictions.get_field("labels").tolist()
labels = [self.CATEGORIES[i] for i in labels]
for mask, label in zip(masks, labels):
thresh = mask[0, :, :, None]
contours, hierarchy = cv2_util.findContours(
thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
if label == "BE":
#image = cv2.drawContours(image, contours, -1, (255,255,255), 2)
mask2 = np.zeros_like(image)
mask2 = cv2.drawContours(mask2, contours, -1, (255, 255, 255),
-1)
BE = np.zeros_like(
image
) # Extract out the object and place into output image
BE[mask2 == 255] = image[mask2 == 255]
BE_gray = cv2.cvtColor(BE, cv2.COLOR_BGR2GRAY)
(thresh,
BE) = cv2.threshold(BE_gray, 10, 255,
cv2.THRESH_BINARY | cv2.THRESH_OTSU)
cv2.imwrite(save_path + name + '_BE.tif', BE)
elif label == "suspicious":
mask2 = np.zeros_like(image)
mask2 = cv2.drawContours(mask2, contours, -1, (255, 255, 255),
-1)
S = np.zeros_like(
image
) # Extract out the object and place into output image
S[mask2 == 255] = image[mask2 == 255]
S_gray = cv2.cvtColor(S, cv2.COLOR_BGR2GRAY)
#(thresh, S) = cv2.threshold(S_gray, 128, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)
(thresh,
S) = cv2.threshold(S_gray, 10, 255,
cv2.THRESH_BINARY | cv2.THRESH_OTSU)
cv2.imwrite(save_path + name + '_suspicious.tif', S)
elif label == "HGD":
mask2 = np.zeros_like(image)
mask2 = cv2.drawContours(mask2, contours, -1, (255, 255, 255),
-1)
H = np.zeros_like(
image
) # Extract out the object and place into output image
H[mask2 == 255] = image[mask2 == 255]
H_gray = cv2.cvtColor(H, cv2.COLOR_BGR2GRAY)
(thresh,
H) = cv2.threshold(H_gray, 10, 255,
cv2.THRESH_BINARY | cv2.THRESH_OTSU)
cv2.imwrite(save_path + name + '_HGD.tif', H)
elif label == "cancer":
mask2 = np.zeros_like(image)
mask2 = cv2.drawContours(mask2, contours, -1, (255, 255, 255),
-1)
C = np.zeros_like(
image
) # Extract out the object and place into output image
C[mask2 == 255] = image[mask2 == 255]
C_gray = cv2.cvtColor(C, cv2.COLOR_BGR2GRAY)
(thresh,
C) = cv2.threshold(C_gray, 10, 255,
cv2.THRESH_BINARY | cv2.THRESH_OTSU)
#C=C.convertTo(img8bit, CV_32FC3)
cv2.imwrite(save_path + name + '_cancer.tif', C)
elif label == "polyp":
mask2 = np.zeros_like(image)
mask2 = cv2.drawContours(mask2, contours, -1, (255, 255, 255),
-1)
P = np.zeros_like(
image
) # Extract out the object and place into output image
P[mask2 == 255] = image[mask2 == 255]
P_gray = cv2.cvtColor(P, cv2.COLOR_BGR2GRAY)
(thresh,
P) = cv2.threshold(P_gray, 10, 255,
cv2.THRESH_BINARY | cv2.THRESH_OTSU)
cv2.imwrite(save_path + name + '_polyp.tif', P)
#cv2.imwrite('/home/zst19phu/benchmark/benchmark-code/datasets/edd2020/image.jpg', image)
composite = image
return composite
def overlay_mask_full(self,
image,
predictions,
boxsize,
boxcrop,
category,
is_crop=True,
draw=False,
multi_object=False):
masks = predictions.get_field("mask").numpy()
labels = predictions.get_field("labels")
lab_list = labels.tolist()
colors = self.compute_colors_for_labels(labels).tolist()
#print(masks[0][0, :, :, None].astype(np.uint8))
#print(masks[0][0, :, :, None].astype(np.uint8).shape)
#single_mask = masks[4][0, :, :, None].astype(np.uint8)
#print(colors)
#mask3 = cv2.cvtColor(single_mask, cv2.COLOR_GRAY2BGR)
#print(np.array(masks3)[0].astype(np.uint8))
#print(mask3)
threshs = []
for mask, color, lablist in zip(masks, colors, lab_list):
thresh = mask[0, :, :, None].astype(np.uint8)
contours, hierarchy = cv2_util.findContours(
thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
#print(np.array(contours))
if draw:
image = cv2.drawContours(image, contours, -1, color, 3)
#if lablist==3:
thresh = thresh * 255
threshs.append(thresh)
#image = image * mask.astype(np.uint8)
#image = cv2.bitwise_and(image, mask3)
#print(image.shape)
car_thresh = []
car_boxsize = []
car_boxcrop = []
is_find = True
for i in range(len(lab_list)):
if lab_list[i] == category:
car_thresh.append(threshs[i])
car_boxsize.append(boxsize[i])
car_boxcrop.append(boxcrop[i])
try:
largest_thresh = np.argmax(np.array(car_boxsize))
if not multi_object:
image = cv2.bitwise_not(image)
image = cv2.bitwise_and(
image,
cv2.cvtColor(threshs[largest_thresh], cv2.COLOR_GRAY2BGR))
image = cv2.bitwise_not(image)
else:
image = cv2.bitwise_not(image)
mask_or = self.add_images(car_thresh)
image = cv2.bitwise_and(
image, cv2.cvtColor(mask_or, cv2.COLOR_GRAY2BGR))
image = cv2.bitwise_not(image)
#image = cv2.bitwise_not(image)
if is_crop:
x, x_ = car_boxcrop[largest_thresh][0][0], car_boxcrop[
largest_thresh][1][0]
y, y_ = car_boxcrop[largest_thresh][0][1], car_boxcrop[
largest_thresh][1][1]
image = image[y:y_, x:x_]
except:
print("Find nothing! Return original image!")
is_find = False
composite = image
return composite, is_find
if has_labels:
label = data["labels"][ix]
if has_rrects:
rr = data["rrects"][ix]
img_copy = draw_anchors(img_copy, [rr], [[0, 0, 255]])
else:
img_copy = cv2.rectangle(img_copy, tuple(bbox[:2]),
tuple(bbox[2:]), (0, 0, 255), 2)
if has_masks:
mask = data["masks"][ix]
contours, hierarchy = cv2_util.findContours(
mask.astype(np.uint8) * 255, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
color = get_random_color()
img_copy = cv2.drawContours(img_copy, contours, -1, color, 3)
# from maskrcnn_benchmark.modeling.rotate_ops import merge_rrects_by_iou
# if has_masks and has_rrects:
# img_copy2 = img.copy()
#
# match_inds = merge_rrects_by_iou(data["rrects"], iou_thresh=0.5)
#
# masks = data["masks"]
# for idx, inds in match_inds.items():
# if len(inds) == 0:
# continue
# mask = masks[inds[0]]