def create_frames_with_overlays(self, video, mode='gt'): """Build list of individual frames in a video with masks overlayed.""" # Overlay masks on top of images frames_with_overlays = [] num_frames = self.videos[video] alpha = 0.6 if mode == 'gt' or mode == 'gt_warped': masks = self.masks_train overlay_color = (255, 0, 0) # red bbox_color = (0, 255, 255) # cyan else: masks = self.warped_prev_masks_train overlay_color = (0, 255, 0) # green bbox_color = (255, 255, 0) # yellow for frame_number in range(num_frames): frame_idx = self.video_frame_idx[video + '_' + str(frame_number)] frame_with_overlay = visualize.apply_mask(self.images_train[frame_idx], masks[frame_idx], overlay_color, alpha) bbox = bboxes.extract_bbox(masks[frame_idx]) frame_with_overlay = visualize.draw_box(frame_with_overlay, bbox, bbox_color) # y1, x1, y2, x2 order frames_with_overlays.append(frame_with_overlay) if mode == 'gt_warped': masks = self.warped_prev_masks_train overlay_color = (0, 255, 0) # green bbox_color = (255, 255, 0) # yellow for frame_number in range(num_frames): frame_idx = self.video_frame_idx[video + '_' + str(frame_number)] frame_with_overlay = visualize.apply_mask(frames_with_overlays[frame_number], masks[frame_idx], overlay_color, alpha, in_place = True) bbox = bboxes.extract_bbox(masks[frame_idx]) frame_with_overlay = visualize.draw_box(frame_with_overlay, bbox, bbox_color) # y1, x1, y2, x2 order frames_with_overlays[frame_number] = frame_with_overlay return frames_with_overlays
def paint_detections(image, boxes, masks, class_names, class_ids, scores): font = cv2.FONT_HERSHEY_SIMPLEX # Number of instances N = boxes.shape[0] # Generate random colors colors = visualize.random_colors(N) #copy the image masked_image = image.astype(np.uint32).copy() #paint masks on it for i in range(N): color = colors[i] mask = masks[:, :, i] masked_image = visualize.apply_mask(masked_image, mask, color).astype('uint8') # paint BB rectangles y1, x1, y2, x2 = boxes[i] cv2.rectangle(masked_image, (x1,y1), (x2, y2), (0,255,0),2) class_id = class_ids[i] score = scores[i] if scores is not None else None label = class_names[class_id] x = random.randint(x1, (x1 + x2) // 2) caption = "{} {:.3f}".format(label, score) if score else label cv2.putText(masked_image, caption,(x1+3, y1+8), font, 0.3,(255,255,255)) return masked_image
def cv2_display_keypoint(image, boxes, keypoints, masks, class_ids, scores, class_names, skeleton=inference_config.LIMBS): # Number of persons N = boxes.shape[0] if not N: print("\n*** No persons to display *** \n") else: assert N == keypoints.shape[0] and N == class_ids.shape[0] and N==scores.shape[0],\ "shape must match: boxes,keypoints,class_ids, scores" colors = visualize.random_colors(N) for i in range(N): color = colors[i] # Bounding box if not np.any(boxes[i]): # Skip this instance. Has no bbox. Likely lost in image cropping. continue y1, x1, y2, x2 = boxes[i] cv2.rectangle(image, (x1, y1), (x2, y2), color, thickness=2) for Joint in keypoints[i]: if (Joint[2] != 0): cv2.circle(image, (Joint[0], Joint[1]), 2, color, -1) # #draw skeleton connection # limb_colors = [[0, 0, 255], [0, 170, 255], [0, 255, 170], [0, 255, 0], [170, 255, 0], # [255, 170, 0], [255, 0, 0], [255, 0, 170], [170, 0, 255], [170, 170, 0], [170, 0, 170]] # if (len(skeleton)): # skeleton = np.reshape(skeleton, (-1, 2)) # neck = np.array((keypoints[i, 5, :] + keypoints[i, 6, :]) / 2).astype(int) # if (keypoints[i, 5, 2] == 0 or keypoints[i, 6, 2] == 0): # neck = [0, 0, 0] # limb_index = -1 # for limb in skeleton: # limb_index += 1 # start_index, end_index = limb # connection joint index from 0 to 16 # if (start_index == -1): # Joint_start = neck # else: # Joint_start = keypoints[i][start_index] # if (end_index == -1): # Joint_end = neck # else: # Joint_end = keypoints[i][end_index] # # both are Annotated # # Joint:(x,y,v) # if ((Joint_start[2] != 0) & (Joint_end[2] != 0)): # # print(color) # cv2.line(image, tuple(Joint_start[:2]), tuple(Joint_end[:2]), limb_colors[limb_index],3) mask = masks[:, :, i] image = visualize.apply_mask(image, mask, color) caption = "{} {:.3f}".format(class_names[class_ids[i]], scores[i]) cv2.putText(image, caption, (x1 + 5, y1 + 16), cv2.FONT_HERSHEY_SIMPLEX, 0.5, color) return image
def apply_masks(image, masks): N = masks.shape[-1] colors = visualize.random_colors(N) masked_image = image.astype(np.uint32).copy() for i in range(N): masked_image = visualize.apply_mask(masked_image, masks[:, :, i], colors[i]) return masked_image, (masked_image - image).astype(np.uint32)
def cv_display_instances(image, boxes, masks, class_ids, class_names, scores=None, title="", figsize=(16, 16)): # Number of instances N = boxes.shape[0] print(N) if not N: print("\n*** No instances to display *** \n") else: assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0] # Generate random colors colors = visualize.random_colors(N) height, width = image.shape[:2] masked_image = image.copy()#.astype(np.uint32).copy() for i in range(N): color = colors[i] # Bounding box if not np.any(boxes[i]): continue y1, x1, y2, x2 = boxes[i] cv_color = (color[0] * 255, color[1] * 255, color[2] * 255) cv2.rectangle(masked_image, (x1, y1), (x2, y2), cv_color , 1) # Label class_id = class_ids[i] score = scores[i] if scores is not None else None label = class_names[class_id] print(label) x = random.randint(x1, (x1 + x2) // 2) caption = "{} {:.3f}".format(label, score) if score else label font = cv2.FONT_HERSHEY_PLAIN cv2.putText(masked_image,caption,(x1, y1),font, 1, cv_color) # Mask mask = masks[:, :, i] masked_image = visualize.apply_mask(masked_image, mask, color) masked_image.astype(np.uint8) return masked_image
random.shuffle(COLORS) print("Loading model weights......") model = mask_rcnn.MaskRCNN(mode="inference", config=configs, model_dir=os.getcwd()) model.load_weights(args["weights"], by_name=True) print("Making inferences with Mask R-CNN......") val = model.detect([image], verbose=1)[0] for i in range(0, val["rois"].shape[0]): classID = val["class_ids"][i] mask = val["masks"][:, :, i] color = COLORS[classID][::-1] image = visualize.apply_mask(img, mask, color, alpha=0.5) for i in range(0, len(val["scores"])): (startY, startX, endY, endX) = val["rois"][i] classID = val["class_ids"][i] label = LABELS[classID] score = val["scores"][i] color = [int(c) for c in np.array(COLORS[classID]) * 255] # draw = ImageDraw.Draw(img) # draw.rectangle(((startX, startY), (endX, endY)), fill="black") # draw.text((20, 70), f'{label}, {score}', font=ImageFont.truetype("font_path123")) #img.save("/content/", "JPEG") cv2.rectangle(image, (startX, startY), (endX, endY), color, 2) text = "{}: {:.3f}".format(label, score)
def main(argv=None): ROOT_DIR = os.getcwd() with tf.gfile.FastGFile("./pb_out_new.pb", 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) _ = tf.import_graph_def(graph_def, name='') print('Graph loaded.') with tf.Session() as sess: #IMAGE_DIR = os.path.join(ROOT_DIR, 'images') #image of the size defined in the config #file_names = next(os.walk(IMAGE_DIR))[2] #filename = random.choice(file_names) #print('choose image file is', filename) image = skimage.io.imread("./151.jpg") # os.system('eog %s &'%(os.path.join(IMAGE_DIR,filename))) print(image.shape) images = [image] print("Processing {} images".format(len(images))) print('RGB image loaded and preprocessed.') molded_images, image_metas, windows = mold_inputs(images) print(molded_images.shape) image_shape = molded_images[0].shape # Anchors anchors = get_anchors(image_shape, inference_config) # Duplicate across the batch dimension because Keras requires it # TODO: can this be optimized to avoid duplicating the anchors? inference_config.BATCH_SIZE = 1 image_anchors = np.broadcast_to( anchors, (inference_config.BATCH_SIZE, ) + anchors.shape) print('anchors shape is', image_anchors.shape, image_anchors.dtype) img_ph = sess.graph.get_tensor_by_name('input_image:0') print(img_ph) img_anchors_ph = sess.graph.get_tensor_by_name('input_anchors:0') print(img_anchors_ph) img_meta_ph = sess.graph.get_tensor_by_name('input_image_meta:0') print(img_meta_ph) detectionsT = sess.graph.get_tensor_by_name('output_detections:0') print('Found ', detectionsT) mrcnn_classT = sess.graph.get_tensor_by_name('output_mrcnn_class:0') print('Found ', mrcnn_classT) mrcnn_bboxT = sess.graph.get_tensor_by_name('output_mrcnn_bbox:0') print('Found ', mrcnn_bboxT) mrcnn_maskT = sess.graph.get_tensor_by_name('output_mrcnn_mask:0') print('Found ', mrcnn_maskT) roisT = sess.graph.get_tensor_by_name('output_rois:0') print('Found ', roisT) np.set_printoptions(suppress=False, precision=4) print('Windows', windows.shape, ' ', windows) detections = sess.run(detectionsT, feed_dict={ img_ph: molded_images, img_meta_ph: image_metas, img_anchors_ph: image_anchors }) #print('Detections: ',detections[0].shape, detections[0]) mrcnn_class = sess.run(mrcnn_classT, feed_dict={ img_ph: molded_images, img_meta_ph: image_metas, img_anchors_ph: image_anchors }) #print('Classes: ',mrcnn_class[0].shape, mrcnn_class[0]) mrcnn_bbox = sess.run(mrcnn_bboxT, feed_dict={ img_ph: molded_images, img_meta_ph: image_metas, img_anchors_ph: image_anchors }) #print('BBoxes: ',mrcnn_bbox[0].shape, mrcnn_bbox[0]) mrcnn_mask = sess.run(mrcnn_maskT, feed_dict={ img_ph: molded_images, img_meta_ph: image_metas, img_anchors_ph: image_anchors }) #print('Masks: ',mrcnn_mask[0].shape )#, outputs1[0]) rois = sess.run(roisT, feed_dict={ img_ph: molded_images, img_meta_ph: image_metas, img_anchors_ph: image_anchors }) #print('Rois: ',rois[0].shape, rois[0]) results = [] for i, image in enumerate(images): final_rois, final_class_ids, final_scores, final_masks =\ unmold_detections(detections[i], mrcnn_mask[i], image.shape, molded_images[i].shape, windows[i]) results.append({ "rois": final_rois, "class_ids": final_class_ids, "scores": final_scores, "masks": final_masks, }) r = results[0] print(results) #print('result is', r) class_names = [ 'BG', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush' ] #visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'])#, ax=get_ax()) print('Done') print("mask shape", r['masks'].shape) #image = cv2.imread(testImage) N = r['rois'].shape[0] #find out how many different instances are there try: #if no mask simply return image frame mask = r['masks'][:, :, 0] color = visualize.random_colors(N) #generates random instance colors color = color[0] image = visualize.apply_mask(image, mask, color) except Exception as e: print(e) cv2.imshow('test', image) cv2.waitKey(0) return 0
#class_names, r['scores']) #skimage.io.imsave('/home/ltp/图片/smplayer_screenshots/crop/I2/'+v,im) # Load a random image from the images folderim,ax =plt.subplots(1,figsize = (16,16)) IMAGE_DIR = train_dir #'./images/' file_names = '3870_426771.jpg' #next(os.walk(IMAGE_DIR))[2] random.seed(120) image = skimage.io.imread(IMAGE_DIR + file_names) # Run detection results = model.detect([image], verbose=0) #results = model.detect_filter([image],f['label'][...].transpose(), verbose=0) # Visualize results r = results[0] color = visualize.random_colors(N=config.NUM_CLASSES) im = visualize.apply_mask(image, r['masks'], color=color, class_ids=[v for v in range(1, config.NUM_CLASSES)]) att_prob = model.run_graph([image], model.get_layers_by_name(['mask_att_prob'])) att_prob = scipy.ndimage.zoom(att_prob['mask_att_prob'][0], zoom=[8, 8, 1]) molded_images, image_metas, windows = model.mold_inputs([image]) a_prob, _ = model.unmold_detections(att_prob, att_prob[..., 0], image.shape, windows[0]) print(a_prob.sum(2)) for v in range(3): plt.subplot(1, 3, v + 1) skimage.io.imshow(50 * (att_prob[..., v] - 0.33), cmap='rainbow') plt.show() plt.figure() plt.imshow(im) plt.show()
def cv2_display_keypoint(image, boxes, masks, class_ids, scores, class_names): # Number of lips N = boxes.shape[0] print("number of lips " + str(N)) if (N < 2): return image, "minimum not found" if not N: print("\n*** No lips to display *** \n") else: assert N == class_ids.shape[0] and N==scores.shape[0],\ "shape must match: boxes,keypoints,class_ids, scores" colors = visualize.random_colors(N) class1 = True class2 = True keypoints = [] classes = [] for i in range(N): if class_ids[i] == 1: classes.append(i) break for i in range(N): if class_ids[i] == 2: classes.append(i) break for k in range(len(classes)): i = classes[k] color = colors[i] # Bounding box if not np.any(boxes[i]): # Skip this instance. Has no bbox. Likely lost in image cropping. continue y1, x1, y2, x2 = boxes[i] # cv2.rectangle(image, (x1, y1), (x2, y2), color, thickness=2) mask = masks[:, :, i] if scores[i] >= 0.9: column_first_non_zero = first_nonzero(mask, axis=0, invalid_val=-1) column_last_non_zero = last_nonzero(mask, axis=0, invalid_val=-1) row_first_non_zero = first_nonzero(mask, axis=1, invalid_val=-1) row_last_non_zero = last_nonzero(mask, axis=1, invalid_val=-1) if (lip_orientation(column_first_non_zero, row_first_non_zero) in "horizontal"): #for class 1 if class_ids[i] == 1 and class1: class1 = False indexes_column_last_positive = np.where( column_last_non_zero > -1)[0] indexes_column_first_positive = np.where( column_first_non_zero > -1)[0] xf = indexes_column_last_positive[0] xl = indexes_column_last_positive[ len(indexes_column_last_positive) - 1] yf = column_last_non_zero[xf] yl = column_last_non_zero[xl] if (yf < yl): #first corner up xf = indexes_column_first_positive[0] yf = column_first_non_zero[xf] elif (yf > yl): #last corner up xl = indexes_column_first_positive[ len(indexes_column_first_positive) - 1] yl = column_first_non_zero[xl] cv2.circle(image, (xf, yf), 2, color, -1) cv2.circle(image, (xl, yl), 2, color, -1) keypoints.append([xf, yf]) keypoints.append([xl, yl]) #mid points of 2 corners x_mid_line = int(round((xf + xl) / 2)) y_mid_line = int(round((yf + yl) / 2)) #spliting columns from mid point index_largest_last = np.argmax(column_last_non_zero) yl = column_last_non_zero[index_largest_last] max_replaced_neg = np.where(column_first_non_zero == -1, yl, column_first_non_zero) column_first_half = max_replaced_neg[:x_mid_line] column_last_half = max_replaced_neg[x_mid_line:] #upper part upper lip #upper lip upper left high point temp = column_first_half[::-1] upper_lip_left_high_x = len(temp) - np.argmin(temp) - 1 upper_lip_left_high_y = column_first_half[ upper_lip_left_high_x] cv2.circle(image, (upper_lip_left_high_x, upper_lip_left_high_y), 2, color, -1) keypoints.append( [upper_lip_left_high_x, upper_lip_left_high_y]) #center of left corner and left high point center_left_corner_high_x = int( round((xf + upper_lip_left_high_x) / 2)) center_left_corner_high_y = column_first_non_zero[ center_left_corner_high_x] cv2.circle( image, (center_left_corner_high_x, center_left_corner_high_y), 2, color, -1) keypoints.append( [center_left_corner_high_x, center_left_corner_high_y]) # upper lip upper right high point upper_lip_right_high_x = np.argmin(column_last_half) upper_lip_right_high_y = column_last_half[ upper_lip_right_high_x] cv2.circle( image, (len(column_first_half) + upper_lip_right_high_x, upper_lip_right_high_y), 2, color, -1) keypoints.append([ len(column_first_half) + upper_lip_right_high_x, upper_lip_right_high_y ]) # center of right corner and right high point center_right_corner_high_x = int( round((xl + len(column_first_half) + upper_lip_right_high_x) / 2)) center_right_corner_high_y = column_first_non_zero[ center_right_corner_high_x] cv2.circle(image, (center_right_corner_high_x, center_right_corner_high_y), 2, color, -1) keypoints.append([ center_right_corner_high_x, center_right_corner_high_y ]) #actual mid point of upper upper lip mid_point_x = int( round((upper_lip_left_high_x + len(column_first_half) + upper_lip_right_high_x) / 2)) mid_point_y = column_first_non_zero[mid_point_x] cv2.circle(image, (mid_point_x, mid_point_y), 2, color, -1) keypoints.append([mid_point_x, mid_point_y]) #lower part of upper lip # angle of lip with x-axis angle = math.atan((yl - yf) / (xl - xf)) #mid point lower upper lip length = column_last_non_zero[mid_point_x] - mid_point_y x_mid_point_low = int( round(mid_point_x + math.sin(angle) * length)) y_mid_point_low = int( round(mid_point_y + math.cos(angle) * length)) cv2.circle(image, (x_mid_point_low, y_mid_point_low), 2, color, -1) keypoints.append([x_mid_point_low, y_mid_point_low]) # upper lip lower left high point length = column_last_non_zero[ upper_lip_left_high_x] - upper_lip_left_high_y upper_lip_left_lower_x = int( round(upper_lip_left_high_x + math.sin(angle) * length)) upper_lip_left_lower_y = int( round(upper_lip_left_high_y + math.cos(angle) * length)) cv2.circle( image, (upper_lip_left_lower_x, upper_lip_left_lower_y), 2, color, -1) keypoints.append( [upper_lip_left_lower_x, upper_lip_left_lower_y]) # upper lip lower right high point length = column_last_non_zero[ len(column_first_half) + upper_lip_right_high_x] - upper_lip_right_high_y upper_lip_right_lower_x = int( round( len(column_first_half) + upper_lip_right_high_x + math.sin(angle) * length)) upper_lip_right_lower_y = int( round(upper_lip_right_high_y + math.cos(angle) * length)) cv2.circle( image, (upper_lip_right_lower_x, upper_lip_right_lower_y), 2, color, -1) keypoints.append( [upper_lip_right_lower_x, upper_lip_right_lower_y]) # upper lip lower center of right corner and right high point length = column_last_non_zero[ center_right_corner_high_x] - center_right_corner_high_y upper_lip_right_corner_lower_x = int( round(center_right_corner_high_x + math.sin(angle) * length)) upper_lip_right__corner_lower_y = int( round(center_right_corner_high_y + math.cos(angle) * length)) cv2.circle(image, (upper_lip_right_corner_lower_x, upper_lip_right__corner_lower_y), 2, color, -1) keypoints.append([ upper_lip_right_corner_lower_x, upper_lip_right__corner_lower_y ]) # lower center of left corner and left high point length = column_last_non_zero[ center_left_corner_high_x] - center_left_corner_high_y upper_lip_left_corner_lower_x = int( round(center_left_corner_high_x + math.sin(angle) * length)) upper_lip_left_corner_lower_y = int( round(center_left_corner_high_y + math.cos(angle) * length)) cv2.circle(image, (upper_lip_left_corner_lower_x, upper_lip_left_corner_lower_y), 2, color, -1) keypoints.append([ upper_lip_left_corner_lower_x, upper_lip_left_corner_lower_y ]) if class_ids[i] == 2 and class2 and not class1: class2 = False #lower lip upper part # mid point upper lower lip length = column_first_non_zero[mid_point_x] - mid_point_y x_mid_point_up_lower = int( round(mid_point_x + math.sin(angle) * length)) y_mid_point_up_lower = int( round(mid_point_y + math.cos(angle) * length)) cv2.circle(image, (x_mid_point_up_lower, y_mid_point_up_lower), 2, color, -1) keypoints.append( [x_mid_point_up_lower, y_mid_point_up_lower]) # lower lip upper left high point length = column_first_non_zero[ upper_lip_left_high_x] - upper_lip_left_high_y lower_lip_left_upper_x = int( round(upper_lip_left_high_x + math.sin(angle) * length)) lower_lip_left_upper_y = int( round(upper_lip_left_high_y + math.cos(angle) * length)) cv2.circle( image, (lower_lip_left_upper_x, lower_lip_left_upper_y), 2, color, -1) keypoints.append( [lower_lip_left_upper_x, lower_lip_left_upper_y]) # lower lip upper right high point length = column_first_non_zero[ len(column_first_half) + upper_lip_right_high_x] - upper_lip_right_high_y lower_lip_right_upper_x = int( round( len(column_first_half) + upper_lip_right_high_x + math.sin(angle) * length)) lower_lip_right_upper_y = int( round(upper_lip_right_high_y + math.cos(angle) * length)) cv2.circle( image, (lower_lip_right_upper_x, lower_lip_right_upper_y), 2, color, -1) keypoints.append( [lower_lip_right_upper_x, lower_lip_right_upper_y]) # lower lip upper center of right corner and right high point length = column_first_non_zero[ center_right_corner_high_x] - center_right_corner_high_y lower_lip_right_corner_upper_x = int( round(center_right_corner_high_x + math.sin(angle) * length)) lower_lip_right_corner_upper_y = int( round(center_right_corner_high_y + math.cos(angle) * length)) cv2.circle(image, (lower_lip_right_corner_upper_x, lower_lip_right_corner_upper_y), 2, color, -1) keypoints.append([ lower_lip_right_corner_upper_x, lower_lip_right_corner_upper_y ]) # lower center of left corner and left high point length = column_first_non_zero[ center_left_corner_high_x] - center_left_corner_high_y lower_lip_left_corner_upper_x = int( round(center_left_corner_high_x + math.sin(angle) * length)) lower_lip_left_corner_upper_y = int( round(center_left_corner_high_y + math.cos(angle) * length)) cv2.circle(image, (lower_lip_left_corner_upper_x, lower_lip_left_corner_upper_y), 2, color, -1) keypoints.append([ lower_lip_left_corner_upper_x, lower_lip_left_corner_upper_y ]) #lower lip lower part # mid point lower lower lip length = column_last_non_zero[ x_mid_point_up_lower] - y_mid_point_up_lower x_mid_point_low_lower = int( round(x_mid_point_up_lower + math.sin(angle) * length)) y_mid_point_low_lower = int( round(y_mid_point_up_lower + math.cos(angle) * length)) cv2.circle(image, (x_mid_point_low_lower, y_mid_point_low_lower), 2, color, -1) keypoints.append( [x_mid_point_low_lower, y_mid_point_low_lower]) # lower lip lower left high point length = column_last_non_zero[ lower_lip_left_upper_x] - lower_lip_left_upper_y lower_lip_left_lower_x = int( round(lower_lip_left_upper_x + math.sin(angle) * length)) lower_lip_left_lower_y = int( round(lower_lip_left_upper_y + math.cos(angle) * length)) cv2.circle( image, (lower_lip_left_lower_x, lower_lip_left_lower_y), 2, color, -1) keypoints.append( [lower_lip_left_lower_x, lower_lip_left_lower_y]) # lower lip lower right high point length = column_last_non_zero[ lower_lip_right_upper_x] - lower_lip_right_upper_y lower_lip_right_lower_x = int( round(lower_lip_right_upper_x + math.sin(angle) * length)) lower_lip_right_lower_y = int( round(lower_lip_right_upper_y + math.cos(angle) * length)) cv2.circle( image, (lower_lip_right_lower_x, lower_lip_right_lower_y), 2, color, -1) keypoints.append( [lower_lip_right_lower_x, lower_lip_right_lower_y]) # lower lip lower center of right corner and right high point length = column_last_non_zero[ lower_lip_right_corner_upper_x] - lower_lip_right_corner_upper_y lower_lip_right_corner_lower_x = int( round(lower_lip_right_corner_upper_x + math.sin(angle) * length)) lower_lip_right_corner_lower_y = int( round(lower_lip_right_corner_upper_y + math.cos(angle) * length)) cv2.circle(image, (lower_lip_right_corner_lower_x, lower_lip_right_corner_lower_y), 2, color, -1) keypoints.append([ lower_lip_right_corner_lower_x, lower_lip_right_corner_lower_y ]) # lower center of left corner and left high point length = column_last_non_zero[ lower_lip_left_corner_upper_x] - lower_lip_left_corner_upper_y lower_lip_left_corner_lower_x = int( round(lower_lip_left_corner_upper_x + math.sin(angle) * length)) lower_lip_left_corner_lower_y = int( round(lower_lip_left_corner_upper_y + math.cos(angle) * length)) cv2.circle(image, (lower_lip_left_corner_lower_x, lower_lip_left_corner_lower_y), 2, color, -1) keypoints.append([ lower_lip_left_corner_lower_x, lower_lip_left_corner_lower_y ]) image = visualize.apply_mask(image, mask, color) # caption = "{} {:.3f}".format(class_names[class_ids[i]], scores[i]) # cv2.putText(image, caption, (x1 + 5, y1 + 16), cv2.FONT_HERSHEY_SIMPLEX, # 0.5, color) s = "" if (len(keypoints) > 1 and len(keypoints[0]) > 1): distance = math.sqrt(((keypoints[0][0] - keypoints[1][0])**2) + ((keypoints[0][1] - keypoints[1][1])**2)) else: distance = 0 p = 0 while (p < len(keypoints)): q = p + 1 while (q < len(keypoints)): hor = math.sqrt(((keypoints[p][0] - keypoints[q][0])**2) + ((keypoints[p][1] - keypoints[q][1])**2)) if hor != 0: s = s + str(format(distance / hor, '.3f')) + ", " q += 1 p += 1 return image, s
def get_all_mask_class_and_draw_minrect(self, r, image): num = r['masks'].shape[2] object = [] _, ax = plt.subplots(1, figsize=(16, 16)) shape = image.shape ax.set_ylim(shape[0] + 10, -10) ax.set_xlim(-10, shape[1] + 10) ax.axis('off') ax.set_title("hhhh") for i in range(num): # create mask class for every instance obj = Mask(i, r['class_ids'][i], self.dataset_val.class_names[r['class_ids'][i]], r['masks'][:, :, i], r['rois'][i], r['scores'][i]) obj.get_min_rect() object.append(obj) # cv2.drawContours(image, [object[i].min_box], 0, (0, 0, 255), 3) print(object[i].class_name) for i in range(num): # if (object[i].class_name is "lemon") | (object[i].class_name is "grape")| (object[i].class_name is "fig"): shoe = object[i] center_x = shoe.min_rect[0][0] center_y = shoe.min_rect[0][1] width = shoe.min_rect[1][0] height = shoe.min_rect[1][1] theta = shoe.min_rect[2] PI = math.pi the = (theta / 180) * PI # print("object", i,":width height theta:", width, height, theta) if theta == 0: y = center_y - 3 * height / 4 x = center_x point = (x, y) w_h = (width / 2, width / 4) elif width > height: x = center_x + (5 * width * math.cos(the)) / 8 y = center_y + (5 * width * math.sin(the)) / 8 # x = center_x - (2 * width * math.cos(the)) / 4 # y = center_y - (2 * width * math.sin(the)) / 4 point = (x, y) w_h = (height / 4, height / 2) else: x = center_x + (5 * height * math.sin(the)) / 8 y = center_y - (5 * height * math.cos(the)) / 8 # x = center_x - (2 * height * math.sin(the)) / 4 # y = center_y + (2 * height * math.cos(the)) / 4 point = (x, y) w_h = (width / 2, width / 4) # print("object", i, "center center_cut:", shoe.min_rect[0], center_x, center_y, x, y) rect = (point, w_h, theta) box = cv2.boxPoints(rect) box = np.int0(box) cv2.drawContours(image, [box], 0, (255, 0, 0), 2) colors = random_colors(num) masked_image = image.astype(np.uint32).copy() captions = [ "lemon 0.988", "lemon 1.000", "lemon 0.995", "lemon 0.984", "lemon 0.992" ] for i in range(num): # if (object[i].class_name is "lemon") | (object[i].class_name is "grape") | (object[i].class_name is "fig"): shoe = object[i] box = shoe.min_box width = shoe.min_rect[1][0] height = shoe.min_rect[1][1] theta = shoe.min_rect[2] score = shoe.scores label = shoe.class_name mask = shoe.mask p = patches.Rectangle( box[1], width, height, angle=theta, linewidth=2, # add rectangle dash alpha=0.7, linestyle="dashed", edgecolor="blue", facecolor='none') ax.add_patch(p) caption = "{} {:.3f}".format( label, score) if score else label # add label # caption = captions[i] print(type(caption), caption) ax.text(box[0][0], box[0][1], caption, color='w', size=8, backgroundcolor="blue") masked_image = apply_mask(masked_image, mask, colors[i]) ax.imshow(masked_image.astype(np.uint8)) plt.show()