def create_frames_with_overlays(self, video, mode='gt'):
"""Build list of individual frames in a video with masks overlayed."""
# Overlay masks on top of images
frames_with_overlays = []
num_frames = self.videos[video]
alpha = 0.6
if mode == 'gt' or mode == 'gt_warped':
masks = self.masks_train
overlay_color = (255, 0, 0) # red
bbox_color = (0, 255, 255) # cyan
else:
masks = self.warped_prev_masks_train
overlay_color = (0, 255, 0) # green
bbox_color = (255, 255, 0) # yellow
for frame_number in range(num_frames):
frame_idx = self.video_frame_idx[video + '_' + str(frame_number)]
frame_with_overlay = visualize.apply_mask(self.images_train[frame_idx], masks[frame_idx], overlay_color, alpha)
bbox = bboxes.extract_bbox(masks[frame_idx])
frame_with_overlay = visualize.draw_box(frame_with_overlay, bbox, bbox_color) # y1, x1, y2, x2 order
frames_with_overlays.append(frame_with_overlay)
if mode == 'gt_warped':
masks = self.warped_prev_masks_train
overlay_color = (0, 255, 0) # green
bbox_color = (255, 255, 0) # yellow
for frame_number in range(num_frames):
frame_idx = self.video_frame_idx[video + '_' + str(frame_number)]
frame_with_overlay = visualize.apply_mask(frames_with_overlays[frame_number], masks[frame_idx], overlay_color, alpha, in_place = True)
bbox = bboxes.extract_bbox(masks[frame_idx])
frame_with_overlay = visualize.draw_box(frame_with_overlay, bbox, bbox_color) # y1, x1, y2, x2 order
frames_with_overlays[frame_number] = frame_with_overlay
return frames_with_overlays
def paint_detections(image, boxes, masks, class_names, class_ids, scores):
font = cv2.FONT_HERSHEY_SIMPLEX
# Number of instances
N = boxes.shape[0]
# Generate random colors
colors = visualize.random_colors(N)
#copy the image
masked_image = image.astype(np.uint32).copy()
#paint masks on it
for i in range(N):
color = colors[i]
mask = masks[:, :, i]
masked_image = visualize.apply_mask(masked_image, mask, color).astype('uint8')
# paint BB rectangles
y1, x1, y2, x2 = boxes[i]
cv2.rectangle(masked_image, (x1,y1), (x2, y2), (0,255,0),2)
class_id = class_ids[i]
score = scores[i] if scores is not None else None
label = class_names[class_id]
x = random.randint(x1, (x1 + x2) // 2)
caption = "{} {:.3f}".format(label, score) if score else label
cv2.putText(masked_image, caption,(x1+3, y1+8), font, 0.3,(255,255,255))
return masked_image
def cv2_display_keypoint(image,
boxes,
keypoints,
masks,
class_ids,
scores,
class_names,
skeleton=inference_config.LIMBS):
# Number of persons
N = boxes.shape[0]
if not N:
print("\n*** No persons to display *** \n")
else:
assert N == keypoints.shape[0] and N == class_ids.shape[0] and N==scores.shape[0],\
"shape must match: boxes,keypoints,class_ids, scores"
colors = visualize.random_colors(N)
for i in range(N):
color = colors[i]
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
cv2.rectangle(image, (x1, y1), (x2, y2), color, thickness=2)
for Joint in keypoints[i]:
if (Joint[2] != 0):
cv2.circle(image, (Joint[0], Joint[1]), 2, color, -1)
# #draw skeleton connection
# limb_colors = [[0, 0, 255], [0, 170, 255], [0, 255, 170], [0, 255, 0], [170, 255, 0],
# [255, 170, 0], [255, 0, 0], [255, 0, 170], [170, 0, 255], [170, 170, 0], [170, 0, 170]]
# if (len(skeleton)):
# skeleton = np.reshape(skeleton, (-1, 2))
# neck = np.array((keypoints[i, 5, :] + keypoints[i, 6, :]) / 2).astype(int)
# if (keypoints[i, 5, 2] == 0 or keypoints[i, 6, 2] == 0):
# neck = [0, 0, 0]
# limb_index = -1
# for limb in skeleton:
# limb_index += 1
# start_index, end_index = limb # connection joint index from 0 to 16
# if (start_index == -1):
# Joint_start = neck
# else:
# Joint_start = keypoints[i][start_index]
# if (end_index == -1):
# Joint_end = neck
# else:
# Joint_end = keypoints[i][end_index]
# # both are Annotated
# # Joint:(x,y,v)
# if ((Joint_start[2] != 0) & (Joint_end[2] != 0)):
# # print(color)
# cv2.line(image, tuple(Joint_start[:2]), tuple(Joint_end[:2]), limb_colors[limb_index],3)
mask = masks[:, :, i]
image = visualize.apply_mask(image, mask, color)
caption = "{} {:.3f}".format(class_names[class_ids[i]], scores[i])
cv2.putText(image, caption, (x1 + 5, y1 + 16),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, color)
return image
def apply_masks(image, masks):
N = masks.shape[-1]
colors = visualize.random_colors(N)
masked_image = image.astype(np.uint32).copy()
for i in range(N):
masked_image = visualize.apply_mask(masked_image, masks[:, :, i],
colors[i])
return masked_image, (masked_image - image).astype(np.uint32)
def cv_display_instances(image, boxes, masks, class_ids, class_names,
scores=None, title="",
figsize=(16, 16)):
# Number of instances
N = boxes.shape[0]
print(N)
if not N:
print("\n*** No instances to display *** \n")
else:
assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
# Generate random colors
colors = visualize.random_colors(N)
height, width = image.shape[:2]
masked_image = image.copy()#.astype(np.uint32).copy()
for i in range(N):
color = colors[i]
# Bounding box
if not np.any(boxes[i]):
continue
y1, x1, y2, x2 = boxes[i]
cv_color = (color[0] * 255, color[1] * 255, color[2] * 255)
cv2.rectangle(masked_image, (x1, y1), (x2, y2), cv_color , 1)
# Label
class_id = class_ids[i]
score = scores[i] if scores is not None else None
label = class_names[class_id]
print(label)
x = random.randint(x1, (x1 + x2) // 2)
caption = "{} {:.3f}".format(label, score) if score else label
font = cv2.FONT_HERSHEY_PLAIN
cv2.putText(masked_image,caption,(x1, y1),font, 1, cv_color)
# Mask
mask = masks[:, :, i]
masked_image = visualize.apply_mask(masked_image, mask, color)
masked_image.astype(np.uint8)
return masked_image
random.shuffle(COLORS)
print("Loading model weights......")
model = mask_rcnn.MaskRCNN(mode="inference",
config=configs,
model_dir=os.getcwd())
model.load_weights(args["weights"], by_name=True)
print("Making inferences with Mask R-CNN......")
val = model.detect([image], verbose=1)[0]
for i in range(0, val["rois"].shape[0]):
classID = val["class_ids"][i]
mask = val["masks"][:, :, i]
color = COLORS[classID][::-1]
image = visualize.apply_mask(img, mask, color, alpha=0.5)
for i in range(0, len(val["scores"])):
(startY, startX, endY, endX) = val["rois"][i]
classID = val["class_ids"][i]
label = LABELS[classID]
score = val["scores"][i]
color = [int(c) for c in np.array(COLORS[classID]) * 255]
# draw = ImageDraw.Draw(img)
# draw.rectangle(((startX, startY), (endX, endY)), fill="black")
# draw.text((20, 70), f'{label}, {score}', font=ImageFont.truetype("font_path123"))
#img.save("/content/", "JPEG")
cv2.rectangle(image, (startX, startY), (endX, endY), color, 2)
text = "{}: {:.3f}".format(label, score)
def main(argv=None):
ROOT_DIR = os.getcwd()
with tf.gfile.FastGFile("./pb_out_new.pb", 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
_ = tf.import_graph_def(graph_def, name='')
print('Graph loaded.')
with tf.Session() as sess:
#IMAGE_DIR = os.path.join(ROOT_DIR, 'images') #image of the size defined in the config
#file_names = next(os.walk(IMAGE_DIR))[2]
#filename = random.choice(file_names)
#print('choose image file is', filename)
image = skimage.io.imread("./151.jpg")
# os.system('eog %s &'%(os.path.join(IMAGE_DIR,filename)))
print(image.shape)
images = [image]
print("Processing {} images".format(len(images)))
print('RGB image loaded and preprocessed.')
molded_images, image_metas, windows = mold_inputs(images)
print(molded_images.shape)
image_shape = molded_images[0].shape
# Anchors
anchors = get_anchors(image_shape, inference_config)
# Duplicate across the batch dimension because Keras requires it
# TODO: can this be optimized to avoid duplicating the anchors?
inference_config.BATCH_SIZE = 1
image_anchors = np.broadcast_to(
anchors, (inference_config.BATCH_SIZE, ) + anchors.shape)
print('anchors shape is', image_anchors.shape, image_anchors.dtype)
img_ph = sess.graph.get_tensor_by_name('input_image:0')
print(img_ph)
img_anchors_ph = sess.graph.get_tensor_by_name('input_anchors:0')
print(img_anchors_ph)
img_meta_ph = sess.graph.get_tensor_by_name('input_image_meta:0')
print(img_meta_ph)
detectionsT = sess.graph.get_tensor_by_name('output_detections:0')
print('Found ', detectionsT)
mrcnn_classT = sess.graph.get_tensor_by_name('output_mrcnn_class:0')
print('Found ', mrcnn_classT)
mrcnn_bboxT = sess.graph.get_tensor_by_name('output_mrcnn_bbox:0')
print('Found ', mrcnn_bboxT)
mrcnn_maskT = sess.graph.get_tensor_by_name('output_mrcnn_mask:0')
print('Found ', mrcnn_maskT)
roisT = sess.graph.get_tensor_by_name('output_rois:0')
print('Found ', roisT)
np.set_printoptions(suppress=False, precision=4)
print('Windows', windows.shape, ' ', windows)
detections = sess.run(detectionsT,
feed_dict={
img_ph: molded_images,
img_meta_ph: image_metas,
img_anchors_ph: image_anchors
})
#print('Detections: ',detections[0].shape, detections[0])
mrcnn_class = sess.run(mrcnn_classT,
feed_dict={
img_ph: molded_images,
img_meta_ph: image_metas,
img_anchors_ph: image_anchors
})
#print('Classes: ',mrcnn_class[0].shape, mrcnn_class[0])
mrcnn_bbox = sess.run(mrcnn_bboxT,
feed_dict={
img_ph: molded_images,
img_meta_ph: image_metas,
img_anchors_ph: image_anchors
})
#print('BBoxes: ',mrcnn_bbox[0].shape, mrcnn_bbox[0])
mrcnn_mask = sess.run(mrcnn_maskT,
feed_dict={
img_ph: molded_images,
img_meta_ph: image_metas,
img_anchors_ph: image_anchors
})
#print('Masks: ',mrcnn_mask[0].shape )#, outputs1[0])
rois = sess.run(roisT,
feed_dict={
img_ph: molded_images,
img_meta_ph: image_metas,
img_anchors_ph: image_anchors
})
#print('Rois: ',rois[0].shape, rois[0])
results = []
for i, image in enumerate(images):
final_rois, final_class_ids, final_scores, final_masks =\
unmold_detections(detections[i], mrcnn_mask[i],
image.shape, molded_images[i].shape,
windows[i])
results.append({
"rois": final_rois,
"class_ids": final_class_ids,
"scores": final_scores,
"masks": final_masks,
})
r = results[0]
print(results)
#print('result is', r)
class_names = [
'BG', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus',
'train', 'truck', 'boat', 'traffic light', 'fire hydrant',
'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog',
'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe',
'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat',
'baseball glove', 'skateboard', 'surfboard', 'tennis racket',
'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl',
'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot',
'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',
'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop',
'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven',
'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase',
'scissors', 'teddy bear', 'hair drier', 'toothbrush'
]
#visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'])#, ax=get_ax())
print('Done')
print("mask shape", r['masks'].shape)
#image = cv2.imread(testImage)
N = r['rois'].shape[0] #find out how many different instances are there
try: #if no mask simply return image frame
mask = r['masks'][:, :, 0]
color = visualize.random_colors(N) #generates random instance colors
color = color[0]
image = visualize.apply_mask(image, mask, color)
except Exception as e:
print(e)
cv2.imshow('test', image)
cv2.waitKey(0)
return 0
#class_names, r['scores'])
#skimage.io.imsave('/home/ltp/图片/smplayer_screenshots/crop/I2/'+v,im)
# Load a random image from the images folderim,ax =plt.subplots(1,figsize = (16,16))
IMAGE_DIR = train_dir #'./images/'
file_names = '3870_426771.jpg' #next(os.walk(IMAGE_DIR))[2]
random.seed(120)
image = skimage.io.imread(IMAGE_DIR + file_names)
# Run detection
results = model.detect([image], verbose=0)
#results = model.detect_filter([image],f['label'][...].transpose(), verbose=0)
# Visualize results
r = results[0]
color = visualize.random_colors(N=config.NUM_CLASSES)
im = visualize.apply_mask(image,
r['masks'],
color=color,
class_ids=[v for v in range(1, config.NUM_CLASSES)])
att_prob = model.run_graph([image],
model.get_layers_by_name(['mask_att_prob']))
att_prob = scipy.ndimage.zoom(att_prob['mask_att_prob'][0], zoom=[8, 8, 1])
molded_images, image_metas, windows = model.mold_inputs([image])
a_prob, _ = model.unmold_detections(att_prob, att_prob[..., 0], image.shape,
windows[0])
print(a_prob.sum(2))
for v in range(3):
plt.subplot(1, 3, v + 1)
skimage.io.imshow(50 * (att_prob[..., v] - 0.33), cmap='rainbow')
plt.show()
plt.figure()
plt.imshow(im)
plt.show()
def cv2_display_keypoint(image, boxes, masks, class_ids, scores, class_names):
# Number of lips
N = boxes.shape[0]
print("number of lips " + str(N))
if (N < 2):
return image, "minimum not found"
if not N:
print("\n*** No lips to display *** \n")
else:
assert N == class_ids.shape[0] and N==scores.shape[0],\
"shape must match: boxes,keypoints,class_ids, scores"
colors = visualize.random_colors(N)
class1 = True
class2 = True
keypoints = []
classes = []
for i in range(N):
if class_ids[i] == 1:
classes.append(i)
break
for i in range(N):
if class_ids[i] == 2:
classes.append(i)
break
for k in range(len(classes)):
i = classes[k]
color = colors[i]
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
# cv2.rectangle(image, (x1, y1), (x2, y2), color, thickness=2)
mask = masks[:, :, i]
if scores[i] >= 0.9:
column_first_non_zero = first_nonzero(mask, axis=0, invalid_val=-1)
column_last_non_zero = last_nonzero(mask, axis=0, invalid_val=-1)
row_first_non_zero = first_nonzero(mask, axis=1, invalid_val=-1)
row_last_non_zero = last_nonzero(mask, axis=1, invalid_val=-1)
if (lip_orientation(column_first_non_zero, row_first_non_zero)
in "horizontal"):
#for class 1
if class_ids[i] == 1 and class1:
class1 = False
indexes_column_last_positive = np.where(
column_last_non_zero > -1)[0]
indexes_column_first_positive = np.where(
column_first_non_zero > -1)[0]
xf = indexes_column_last_positive[0]
xl = indexes_column_last_positive[
len(indexes_column_last_positive) - 1]
yf = column_last_non_zero[xf]
yl = column_last_non_zero[xl]
if (yf < yl):
#first corner up
xf = indexes_column_first_positive[0]
yf = column_first_non_zero[xf]
elif (yf > yl):
#last corner up
xl = indexes_column_first_positive[
len(indexes_column_first_positive) - 1]
yl = column_first_non_zero[xl]
cv2.circle(image, (xf, yf), 2, color, -1)
cv2.circle(image, (xl, yl), 2, color, -1)
keypoints.append([xf, yf])
keypoints.append([xl, yl])
#mid points of 2 corners
x_mid_line = int(round((xf + xl) / 2))
y_mid_line = int(round((yf + yl) / 2))
#spliting columns from mid point
index_largest_last = np.argmax(column_last_non_zero)
yl = column_last_non_zero[index_largest_last]
max_replaced_neg = np.where(column_first_non_zero == -1,
yl, column_first_non_zero)
column_first_half = max_replaced_neg[:x_mid_line]
column_last_half = max_replaced_neg[x_mid_line:]
#upper part upper lip
#upper lip upper left high point
temp = column_first_half[::-1]
upper_lip_left_high_x = len(temp) - np.argmin(temp) - 1
upper_lip_left_high_y = column_first_half[
upper_lip_left_high_x]
cv2.circle(image,
(upper_lip_left_high_x, upper_lip_left_high_y),
2, color, -1)
keypoints.append(
[upper_lip_left_high_x, upper_lip_left_high_y])
#center of left corner and left high point
center_left_corner_high_x = int(
round((xf + upper_lip_left_high_x) / 2))
center_left_corner_high_y = column_first_non_zero[
center_left_corner_high_x]
cv2.circle(
image,
(center_left_corner_high_x, center_left_corner_high_y),
2, color, -1)
keypoints.append(
[center_left_corner_high_x, center_left_corner_high_y])
# upper lip upper right high point
upper_lip_right_high_x = np.argmin(column_last_half)
upper_lip_right_high_y = column_last_half[
upper_lip_right_high_x]
cv2.circle(
image,
(len(column_first_half) + upper_lip_right_high_x,
upper_lip_right_high_y), 2, color, -1)
keypoints.append([
len(column_first_half) + upper_lip_right_high_x,
upper_lip_right_high_y
])
# center of right corner and right high point
center_right_corner_high_x = int(
round((xl + len(column_first_half) +
upper_lip_right_high_x) / 2))
center_right_corner_high_y = column_first_non_zero[
center_right_corner_high_x]
cv2.circle(image, (center_right_corner_high_x,
center_right_corner_high_y), 2, color,
-1)
keypoints.append([
center_right_corner_high_x, center_right_corner_high_y
])
#actual mid point of upper upper lip
mid_point_x = int(
round((upper_lip_left_high_x + len(column_first_half) +
upper_lip_right_high_x) / 2))
mid_point_y = column_first_non_zero[mid_point_x]
cv2.circle(image, (mid_point_x, mid_point_y), 2, color, -1)
keypoints.append([mid_point_x, mid_point_y])
#lower part of upper lip
# angle of lip with x-axis
angle = math.atan((yl - yf) / (xl - xf))
#mid point lower upper lip
length = column_last_non_zero[mid_point_x] - mid_point_y
x_mid_point_low = int(
round(mid_point_x + math.sin(angle) * length))
y_mid_point_low = int(
round(mid_point_y + math.cos(angle) * length))
cv2.circle(image, (x_mid_point_low, y_mid_point_low), 2,
color, -1)
keypoints.append([x_mid_point_low, y_mid_point_low])
# upper lip lower left high point
length = column_last_non_zero[
upper_lip_left_high_x] - upper_lip_left_high_y
upper_lip_left_lower_x = int(
round(upper_lip_left_high_x +
math.sin(angle) * length))
upper_lip_left_lower_y = int(
round(upper_lip_left_high_y +
math.cos(angle) * length))
cv2.circle(
image,
(upper_lip_left_lower_x, upper_lip_left_lower_y), 2,
color, -1)
keypoints.append(
[upper_lip_left_lower_x, upper_lip_left_lower_y])
# upper lip lower right high point
length = column_last_non_zero[
len(column_first_half) +
upper_lip_right_high_x] - upper_lip_right_high_y
upper_lip_right_lower_x = int(
round(
len(column_first_half) + upper_lip_right_high_x +
math.sin(angle) * length))
upper_lip_right_lower_y = int(
round(upper_lip_right_high_y +
math.cos(angle) * length))
cv2.circle(
image,
(upper_lip_right_lower_x, upper_lip_right_lower_y), 2,
color, -1)
keypoints.append(
[upper_lip_right_lower_x, upper_lip_right_lower_y])
# upper lip lower center of right corner and right high point
length = column_last_non_zero[
center_right_corner_high_x] - center_right_corner_high_y
upper_lip_right_corner_lower_x = int(
round(center_right_corner_high_x +
math.sin(angle) * length))
upper_lip_right__corner_lower_y = int(
round(center_right_corner_high_y +
math.cos(angle) * length))
cv2.circle(image, (upper_lip_right_corner_lower_x,
upper_lip_right__corner_lower_y), 2,
color, -1)
keypoints.append([
upper_lip_right_corner_lower_x,
upper_lip_right__corner_lower_y
])
# lower center of left corner and left high point
length = column_last_non_zero[
center_left_corner_high_x] - center_left_corner_high_y
upper_lip_left_corner_lower_x = int(
round(center_left_corner_high_x +
math.sin(angle) * length))
upper_lip_left_corner_lower_y = int(
round(center_left_corner_high_y +
math.cos(angle) * length))
cv2.circle(image, (upper_lip_left_corner_lower_x,
upper_lip_left_corner_lower_y), 2,
color, -1)
keypoints.append([
upper_lip_left_corner_lower_x,
upper_lip_left_corner_lower_y
])
if class_ids[i] == 2 and class2 and not class1:
class2 = False
#lower lip upper part
# mid point upper lower lip
length = column_first_non_zero[mid_point_x] - mid_point_y
x_mid_point_up_lower = int(
round(mid_point_x + math.sin(angle) * length))
y_mid_point_up_lower = int(
round(mid_point_y + math.cos(angle) * length))
cv2.circle(image,
(x_mid_point_up_lower, y_mid_point_up_lower), 2,
color, -1)
keypoints.append(
[x_mid_point_up_lower, y_mid_point_up_lower])
# lower lip upper left high point
length = column_first_non_zero[
upper_lip_left_high_x] - upper_lip_left_high_y
lower_lip_left_upper_x = int(
round(upper_lip_left_high_x +
math.sin(angle) * length))
lower_lip_left_upper_y = int(
round(upper_lip_left_high_y +
math.cos(angle) * length))
cv2.circle(
image,
(lower_lip_left_upper_x, lower_lip_left_upper_y), 2,
color, -1)
keypoints.append(
[lower_lip_left_upper_x, lower_lip_left_upper_y])
# lower lip upper right high point
length = column_first_non_zero[
len(column_first_half) +
upper_lip_right_high_x] - upper_lip_right_high_y
lower_lip_right_upper_x = int(
round(
len(column_first_half) + upper_lip_right_high_x +
math.sin(angle) * length))
lower_lip_right_upper_y = int(
round(upper_lip_right_high_y +
math.cos(angle) * length))
cv2.circle(
image,
(lower_lip_right_upper_x, lower_lip_right_upper_y), 2,
color, -1)
keypoints.append(
[lower_lip_right_upper_x, lower_lip_right_upper_y])
# lower lip upper center of right corner and right high point
length = column_first_non_zero[
center_right_corner_high_x] - center_right_corner_high_y
lower_lip_right_corner_upper_x = int(
round(center_right_corner_high_x +
math.sin(angle) * length))
lower_lip_right_corner_upper_y = int(
round(center_right_corner_high_y +
math.cos(angle) * length))
cv2.circle(image, (lower_lip_right_corner_upper_x,
lower_lip_right_corner_upper_y), 2,
color, -1)
keypoints.append([
lower_lip_right_corner_upper_x,
lower_lip_right_corner_upper_y
])
# lower center of left corner and left high point
length = column_first_non_zero[
center_left_corner_high_x] - center_left_corner_high_y
lower_lip_left_corner_upper_x = int(
round(center_left_corner_high_x +
math.sin(angle) * length))
lower_lip_left_corner_upper_y = int(
round(center_left_corner_high_y +
math.cos(angle) * length))
cv2.circle(image, (lower_lip_left_corner_upper_x,
lower_lip_left_corner_upper_y), 2,
color, -1)
keypoints.append([
lower_lip_left_corner_upper_x,
lower_lip_left_corner_upper_y
])
#lower lip lower part
# mid point lower lower lip
length = column_last_non_zero[
x_mid_point_up_lower] - y_mid_point_up_lower
x_mid_point_low_lower = int(
round(x_mid_point_up_lower + math.sin(angle) * length))
y_mid_point_low_lower = int(
round(y_mid_point_up_lower + math.cos(angle) * length))
cv2.circle(image,
(x_mid_point_low_lower, y_mid_point_low_lower),
2, color, -1)
keypoints.append(
[x_mid_point_low_lower, y_mid_point_low_lower])
# lower lip lower left high point
length = column_last_non_zero[
lower_lip_left_upper_x] - lower_lip_left_upper_y
lower_lip_left_lower_x = int(
round(lower_lip_left_upper_x +
math.sin(angle) * length))
lower_lip_left_lower_y = int(
round(lower_lip_left_upper_y +
math.cos(angle) * length))
cv2.circle(
image,
(lower_lip_left_lower_x, lower_lip_left_lower_y), 2,
color, -1)
keypoints.append(
[lower_lip_left_lower_x, lower_lip_left_lower_y])
# lower lip lower right high point
length = column_last_non_zero[
lower_lip_right_upper_x] - lower_lip_right_upper_y
lower_lip_right_lower_x = int(
round(lower_lip_right_upper_x +
math.sin(angle) * length))
lower_lip_right_lower_y = int(
round(lower_lip_right_upper_y +
math.cos(angle) * length))
cv2.circle(
image,
(lower_lip_right_lower_x, lower_lip_right_lower_y), 2,
color, -1)
keypoints.append(
[lower_lip_right_lower_x, lower_lip_right_lower_y])
# lower lip lower center of right corner and right high point
length = column_last_non_zero[
lower_lip_right_corner_upper_x] - lower_lip_right_corner_upper_y
lower_lip_right_corner_lower_x = int(
round(lower_lip_right_corner_upper_x +
math.sin(angle) * length))
lower_lip_right_corner_lower_y = int(
round(lower_lip_right_corner_upper_y +
math.cos(angle) * length))
cv2.circle(image, (lower_lip_right_corner_lower_x,
lower_lip_right_corner_lower_y), 2,
color, -1)
keypoints.append([
lower_lip_right_corner_lower_x,
lower_lip_right_corner_lower_y
])
# lower center of left corner and left high point
length = column_last_non_zero[
lower_lip_left_corner_upper_x] - lower_lip_left_corner_upper_y
lower_lip_left_corner_lower_x = int(
round(lower_lip_left_corner_upper_x +
math.sin(angle) * length))
lower_lip_left_corner_lower_y = int(
round(lower_lip_left_corner_upper_y +
math.cos(angle) * length))
cv2.circle(image, (lower_lip_left_corner_lower_x,
lower_lip_left_corner_lower_y), 2,
color, -1)
keypoints.append([
lower_lip_left_corner_lower_x,
lower_lip_left_corner_lower_y
])
image = visualize.apply_mask(image, mask, color)
# caption = "{} {:.3f}".format(class_names[class_ids[i]], scores[i])
# cv2.putText(image, caption, (x1 + 5, y1 + 16), cv2.FONT_HERSHEY_SIMPLEX,
# 0.5, color)
s = ""
if (len(keypoints) > 1 and len(keypoints[0]) > 1):
distance = math.sqrt(((keypoints[0][0] - keypoints[1][0])**2) +
((keypoints[0][1] - keypoints[1][1])**2))
else:
distance = 0
p = 0
while (p < len(keypoints)):
q = p + 1
while (q < len(keypoints)):
hor = math.sqrt(((keypoints[p][0] - keypoints[q][0])**2) +
((keypoints[p][1] - keypoints[q][1])**2))
if hor != 0:
s = s + str(format(distance / hor, '.3f')) + ", "
q += 1
p += 1
return image, s
def get_all_mask_class_and_draw_minrect(self, r, image):
num = r['masks'].shape[2]
object = []
_, ax = plt.subplots(1, figsize=(16, 16))
shape = image.shape
ax.set_ylim(shape[0] + 10, -10)
ax.set_xlim(-10, shape[1] + 10)
ax.axis('off')
ax.set_title("hhhh")
for i in range(num): # create mask class for every instance
obj = Mask(i, r['class_ids'][i],
self.dataset_val.class_names[r['class_ids'][i]],
r['masks'][:, :, i], r['rois'][i], r['scores'][i])
obj.get_min_rect()
object.append(obj)
# cv2.drawContours(image, [object[i].min_box], 0, (0, 0, 255), 3)
print(object[i].class_name)
for i in range(num):
# if (object[i].class_name is "lemon") | (object[i].class_name is "grape")| (object[i].class_name is "fig"):
shoe = object[i]
center_x = shoe.min_rect[0][0]
center_y = shoe.min_rect[0][1]
width = shoe.min_rect[1][0]
height = shoe.min_rect[1][1]
theta = shoe.min_rect[2]
PI = math.pi
the = (theta / 180) * PI
# print("object", i,":width height theta:", width, height, theta)
if theta == 0:
y = center_y - 3 * height / 4
x = center_x
point = (x, y)
w_h = (width / 2, width / 4)
elif width > height:
x = center_x + (5 * width * math.cos(the)) / 8
y = center_y + (5 * width * math.sin(the)) / 8
# x = center_x - (2 * width * math.cos(the)) / 4
# y = center_y - (2 * width * math.sin(the)) / 4
point = (x, y)
w_h = (height / 4, height / 2)
else:
x = center_x + (5 * height * math.sin(the)) / 8
y = center_y - (5 * height * math.cos(the)) / 8
# x = center_x - (2 * height * math.sin(the)) / 4
# y = center_y + (2 * height * math.cos(the)) / 4
point = (x, y)
w_h = (width / 2, width / 4)
# print("object", i, "center center_cut:", shoe.min_rect[0], center_x, center_y, x, y)
rect = (point, w_h, theta)
box = cv2.boxPoints(rect)
box = np.int0(box)
cv2.drawContours(image, [box], 0, (255, 0, 0), 2)
colors = random_colors(num)
masked_image = image.astype(np.uint32).copy()
captions = [
"lemon 0.988", "lemon 1.000", "lemon 0.995", "lemon 0.984",
"lemon 0.992"
]
for i in range(num):
# if (object[i].class_name is "lemon") | (object[i].class_name is "grape") | (object[i].class_name is "fig"):
shoe = object[i]
box = shoe.min_box
width = shoe.min_rect[1][0]
height = shoe.min_rect[1][1]
theta = shoe.min_rect[2]
score = shoe.scores
label = shoe.class_name
mask = shoe.mask
p = patches.Rectangle(
box[1],
width,
height,
angle=theta,
linewidth=2, # add rectangle dash
alpha=0.7,
linestyle="dashed",
edgecolor="blue",
facecolor='none')
ax.add_patch(p)
caption = "{} {:.3f}".format(
label, score) if score else label # add label
# caption = captions[i]
print(type(caption), caption)
ax.text(box[0][0],
box[0][1],
caption,
color='w',
size=8,
backgroundcolor="blue")
masked_image = apply_mask(masked_image, mask, colors[i])
ax.imshow(masked_image.astype(np.uint8))
plt.show()