def detect(model, dataset_dir, subset): """Run detection on images in the given directory.""" print("Running on {}".format(dataset_dir)) # Create directory if not os.path.exists(RESULTS_DIR): os.makedirs(RESULTS_DIR) submit_dir = "submit_{:%Y%m%dT%H%M%S}".format(datetime.datetime.now()) submit_dir = os.path.join(RESULTS_DIR, submit_dir) os.makedirs(submit_dir) # Read dataset dataset = NucleusDataset() dataset.load_nucleus(dataset_dir, subset) dataset.prepare() # Load over images submission = [] for image_id in dataset.image_ids: # Load image and run detection image = dataset.load_image(image_id) # Detect objects r = model.detect([image], verbose=0)[0] # Encode image to RLE. Returns a string of multiple lines source_id = dataset.image_info[image_id]["id"] rle = mask_to_rle(source_id, r["masks"], r["scores"]) submission.append(rle) # Save image with masks visualize.display_instances( image, r['rois'], r['masks'], r['class_ids'], dataset.class_names, r['scores'], show_bbox=False, show_mask=False, title="Predictions") plt.savefig("{}/{}.png".format(submit_dir, dataset.image_info[image_id]["id"])) # Save to csv file submission = "ImageId,EncodedPixels\n" + "\n".join(submission) file_path = os.path.join(submit_dir, "submit.csv") with open(file_path, "w") as f: f.write(submission) print("Saved to ", submit_dir)
# COCO Class names # Index of the class in the list is its ID. For example, to get ID of # the teddy bear class, use: class_names.index('teddy bear') class_names = [ 'BG', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush' ] # Load a random image from the images folder file_names = next(os.walk(IMAGE_DIR))[2] image = skimage.io.imread(os.path.join(IMAGE_DIR, random.choice(file_names))) # Run detection results = model.detect([image], verbose=1) # Visualize results r = results[0] visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'])
IMAGE_MIN_DIM = 320 config = InferenceConfig() config.display() model = modellib.MaskRCNN(mode="inference", model_dir=MODEL_DIR, config=config) model_path = PRETRAINED_MODEL_PATH model.load_weights(model_path, by_name=True) # 测试模型 class_names = ['BG', 'building'] # 这里只设置了一个背景值和一个建筑物,所以模型只能识别建筑物 file_name = 'D:/codes/python/BuildingDetectMaster/data/test_tiff/s32.TIF' # 输入的tiff图像 test_image = skimage.io.imread(file_name) predictions = model.detect([test_image] * config.BATCH_SIZE, verbose=1) p = predictions[0] # 存储为shapefile masks = p['masks'] save_name = 'D:/codes/python/RS_Test/a.shp' # 输出路径 data_tiff = gdal.Open(file_name) ref = data_tiff.GetGeoTransform() geom = geo.create_geom_from_rcnnmask(masks, reference=ref) geo.convert_geom_to_shp(geom, outputfile_name=save_name) # 图像显示 visualize.display_instances(test_image, p['rois'], p['masks'], p['class_ids'], class_names, p['scores'])
# In[12]: # Test on a random image image_id = random.choice(dataset_val.image_ids) original_image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt( dataset_val, inference_config, image_id, use_mini_mask=False) log("original_image", original_image) log("image_meta", image_meta) log("gt_class_id", gt_class_id) log("gt_bbox", gt_bbox) log("gt_mask", gt_mask) visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, dataset_train.class_names, figsize=(8, 8)) # In[13]: results = model.detect([original_image], verbose=1) r = results[0] visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], dataset_val.class_names, r['scores'], ax=get_ax())
def get_labels(self, labels): dims = labels.shape unlabeled_labels = np.zeros((dims[0], dims[1], 1)) building_labels = np.zeros((dims[0], dims[1], 1)) fence_labels = np.zeros((dims[0], dims[1], 1)) other_labels = np.zeros((dims[0], dims[1], 1)) pedestrian_labels = np.zeros((dims[0], dims[1], 1)) pole_labels = np.zeros((dims[0], dims[1], 1)) road_line_labels = np.zeros((dims[0], dims[1], 1)) road_labels = np.zeros((dims[0], dims[1], 1)) sidewalk_labels = np.zeros((dims[0], dims[1], 1)) vegetation_labels = np.zeros((dims[0], dims[1], 1)) car_labels = np.zeros((dims[0], dims[1], 1)) wall_labels = np.zeros((dims[0], dims[1], 1)) traffic_sign_labels = np.zeros((dims[0], dims[1], 1)) unlabeled_index = np.all(labels == (0, 0, 0), axis=-1) building_index = np.all(labels == (70, 70, 70), axis=-1) fence_index = np.all(labels == (190, 153, 153), axis=-1) other_index = np.all(labels == (250, 170, 160), axis=-1) pedestrian_index = np.all(labels == (220, 20, 60), axis=-1) pole_index = np.all(labels == (153, 153, 153), axis=-1) road_line_index = np.all(labels == (157, 234, 50), axis=-1) road_index = np.all(labels == (128, 64, 128), axis=-1) sidewalk_index = np.all(labels == (244, 35, 232), axis=-1) vegetation_index = np.all(labels == (107, 142, 35), axis=-1) car_index = np.all(labels == (0, 0, 142), axis=-1) wall_index = np.all(labels == (102, 102, 156), axis=-1) traffic_sign_index = np.all(labels == (220, 220, 70), axis=-1) unlabeled_labels[unlabeled_index] = 1 building_labels[building_index] = 10 fence_labels[fence_index] = 10 other_labels[other_index] = 10 pedestrian_labels[pedestrian_index] = 10 pole_labels[pole_index] = 10 road_line_labels[road_line_index] = 10 road_labels[road_index] = 10 sidewalk_labels[sidewalk_index] = 10 vegetation_labels[vegetation_index] = 1 car_labels[car_index] = 10 wall_labels[wall_index] = 10 traffic_sign_labels[traffic_sign_index] = 10 return np.dstack([unlabeled_labels, building_labels, fence_labels, return np.dstack([unlabeled_labels, building_labels, fence_labels, other_labels, pedestrian_labels, pole_labels, road_line_labels, road_labels, sidewalk_labels, vegetation_labels, car_labels, wall_labels, traffic_sign_labels]) def image_reference(self, image_id): """Return the carla data of the image.""" info = self.image_info[image_id] if info["source"] == "carla": return info["id"] else: super(self.__class__).image_reference(self, image_id) config = CarlaConfig() config.STEPS_PER_EPOCH = NUMBER_OF_TRAIN_DATA//config.BATCH_SIZE config.VALIDATION_STEPS = NUMBER_OF_VAL_DATA//config.BATCH_SIZE config.display() dataset = carlaDataset() dataset.load_images(dir=RGB_TRAIN_DIR, type='train') # mask, a = train.load_mask(50) # print(a) dataset.prepare() print("Image Count: {}".format(len(dataset.image_ids))) print("Class Count: {}".format(dataset.num_classes)) for i, info in enumerate(dataset.class_info): print("{:3}. {:50}".format(i, info['name'])) image_ids = np.random.choice(dataset.image_ids, 4) for image_id in image_ids: image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) visualize.display_top_masks(image, mask, class_ids, dataset.class_names) # Load random image and mask. image_id = random.choice(dataset.image_ids) image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) # Compute Bounding box bbox = utils.extract_bboxes(mask) # Display image and additional stats print("image_id ", image_id) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) # Display image and instances visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # Load random image and mask. image_id = np.random.choice(dataset.image_ids, 1)[0] image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) original_shape = image.shape # Resize image, window, scale, padding, _ = utils.resize_image( image, min_dim=config.IMAGE_MIN_DIM, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding) # Compute Bounding box bbox = utils.extract_bboxes(mask) # Display image and additional stats print("image_id: ", image_id) print("Original shape: ", original_shape) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) # Display image and instances visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) image_id = np.random.choice(dataset.image_ids, 1)[0] image, image_meta, class_ids, bbox, mask = modellib.load_image_gt( dataset, config, image_id, use_mini_mask=False) log("image", image) log("image_meta", image_meta) log("class_ids", class_ids) log("bbox", bbox) log("mask", mask) display_images([image]+[mask[:,:,i] for i in range(min(mask.shape[-1], 7))]) visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # Generate Anchors backbone_shapes = modellib.compute_backbone_shapes(config, config.IMAGE_SHAPE) anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES, config.RPN_ANCHOR_RATIOS, backbone_shapes, config.BACKBONE_STRIDES, config.RPN_ANCHOR_STRIDE) # Print summary of anchors num_levels = len(backbone_shapes) anchors_per_cell = len(config.RPN_ANCHOR_RATIOS) print("Count: ", anchors.shape[0]) print("Scales: ", config.RPN_ANCHOR_SCALES) print("ratios: ", config.RPN_ANCHOR_RATIOS) print("Anchors per Cell: ", anchors_per_cell) print("Levels: ", num_levels) anchors_per_level = [] for l in range(num_levels): num_cells = backbone_shapes[l][0] * backbone_shapes[l][1] anchors_per_level.append(anchors_per_cell * num_cells // config.RPN_ANCHOR_STRIDE**2) print("Anchors in Level {}: {}".format(l, anchors_per_level[l])) ## Visualize anchors of one cell at the center of the feature map of a specific level # Load and draw random image image_id = np.random.choice(dataset.image_ids, 1)[0] image, image_meta, _, _, _ = modellib.load_image_gt(dataset, config, image_id) fig, ax = plt.subplots(1, figsize=(10, 10)) ax.imshow(image) levels = len(backbone_shapes) for level in range(levels): colors = visualize.random_colors(levels) # Compute the index of the anchors at the center of the image level_start = sum(anchors_per_level[:level]) # sum of anchors of previous levels level_anchors = anchors[level_start:level_start+anchors_per_level[level]] print("Level {}. Anchors: {:6} Feature map Shape: {}".format(level, level_anchors.shape[0], backbone_shapes[level])) center_cell = backbone_shapes[level] // 2 center_cell_index = (center_cell[0] * backbone_shapes[level][1] + center_cell[1]) level_center = center_cell_index * anchors_per_cell center_anchor = anchors_per_cell * ( (center_cell[0] * backbone_shapes[level][1] / config.RPN_ANCHOR_STRIDE**2) \ + center_cell[1] / config.RPN_ANCHOR_STRIDE) level_center = int(center_anchor) # Draw anchors. Brightness show the order in the array, dark to bright. for i, rect in enumerate(level_anchors[level_center:level_center+anchors_per_cell]): y1, x1, y2, x2 = rect p = patches.Rectangle((x1, y1), x2-x1, y2-y1, linewidth=2, facecolor='none', edgecolor=(i+1)*np.array(colors[level]) / anchors_per_cell) ax.add_patch(p) # Create data generator random_rois = 4000 g = modellib.data_generator( dataset, config, shuffle=True, random_rois=random_rois, batch_size=4, detection_targets=True) # Get Next Image if random_rois: [normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, rpn_rois, rois], \ [mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g) log("rois", rois) log("mrcnn_class_ids", mrcnn_class_ids) log("mrcnn_bbox", mrcnn_bbox) log("mrcnn_mask", mrcnn_mask) else: [normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes, gt_masks], _ = next(g) log("gt_class_ids", gt_class_ids) log("gt_boxes", gt_boxes) log("gt_masks", gt_masks) log("rpn_match", rpn_match, ) log("rpn_bbox", rpn_bbox) image_id = modellib.parse_image_meta(image_meta)["image_id"][0] print("image_id: ", image_id, dataset.image_reference(image_id)) # Remove the last dim in mrcnn_class_ids. It's only added # to satisfy Keras restriction on target shape. mrcnn_class_ids = mrcnn_class_ids[:, :, 0] b = 0 # Restore original image (reverse normalization) sample_image = modellib.unmold_image(normalized_images[b], config) # Compute anchor shifts. indices = np.where(rpn_match[b] == 1)[0] refined_anchors = utils.apply_box_deltas(anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV) log("anchors", anchors) log("refined_anchors", refined_anchors) # Get list of positive anchors positive_anchor_ids = np.where(rpn_match[b] == 1)[0] print("Positive anchors: {}".format(len(positive_anchor_ids))) negative_anchor_ids = np.where(rpn_match[b] == -1)[0] print("Negative anchors: {}".format(len(negative_anchor_ids))) neutral_anchor_ids = np.where(rpn_match[b] == 0)[0] print("Neutral anchors: {}".format(len(neutral_anchor_ids))) # ROI breakdown by class for c, n in zip(dataset.class_names, np.bincount(mrcnn_class_ids[b].flatten())): if n: print("{:23}: {}".format(c[:20], n)) # Show positive anchors visualize.draw_boxes(sample_image, boxes=anchors[positive_anchor_ids], refined_boxes=refined_anchors) # Show negative anchors visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids]) # Show neutral anchors. They don't contribute to training. visualize.draw_boxes(sample_image, boxes=anchors[np.random.choice(neutral_anchor_ids, 100)]) if random_rois: # Class aware bboxes bbox_specific = mrcnn_bbox[b, np.arange(mrcnn_bbox.shape[1]), mrcnn_class_ids[b], :] # Refined ROIs refined_rois = utils.apply_box_deltas(rois[b].astype(np.float32), bbox_specific[:, :4] * config.BBOX_STD_DEV) # Class aware masks mask_specific = mrcnn_mask[b, np.arange(mrcnn_mask.shape[1]), :, :, mrcnn_class_ids[b]] visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific, mrcnn_class_ids[b], dataset.class_names) # Any repeated ROIs? rows = np.ascontiguousarray(rois[b]).view(np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1]))) _, idx = np.unique(rows, return_index=True) print("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1])) if random_rois: # Dispalay ROIs and corresponding masks and bounding boxes ids = random.sample(range(rois.shape[1]), 8) images = [] titles = [] for i in ids: image = visualize.draw_box(sample_image.copy(), rois[b,i,:4].astype(np.int32), [255, 0, 0]) image = visualize.draw_box(image, refined_rois[i].astype(np.int64), [0, 255, 0]) images.append(image) titles.append("ROI {}".format(i)) images.append(mask_specific[i] * 255) titles.append(dataset.class_names[mrcnn_class_ids[b,i]][:20]) display_images(images, titles, cols=4, cmap="Blues", interpolation="none") # Check ratio of positive ROIs in a set of images. if random_rois: limit = 10 temp_g = modellib.data_generator( dataset, config, shuffle=True, random_rois=10000, batch_size=1, detection_targets=True) total = 0 for i in range(limit): _, [ids, _, _] = next(temp_g) positive_rois = np.sum(ids[0] > 0) total += positive_rois print("{:5} {:5.2f}".format(positive_rois, positive_rois/ids.shape[1])) print("Average percent: {:.2f}".format(total/(limit*ids.shape[1]))) exit()
def detect(self): self.model = modellib.MaskRCNN(mode="inference", config=self.detectconfig, model_dir=self.MODEL_DIR) # SHARP_MODEL_PATH=os.path.join(SHARP_MODEL_DIR,"mask_rcnn_shapes_0000.h5") # self.SHARP_MODEL_PATH="/home/ljt/Mask_RCNN_shoes/logs/shape20190613T1601/mask_rcnn_shape_0010.h5" #self.SHARP_MODEL_PATH="../logs/shape20190613T1601/mask_rcnn_shape_0010.h5" self.SHARP_MODEL_PATH="../logs/shape20191113T1047/mask_rcnn_shape_0029.h5" self.model.load_weights(self.SHARP_MODEL_PATH, by_name=True) print(self.SHARP_MODEL_PATH) import skimage # Quilt_DIR="/home/ljt/Shoe-data-V2/test" Quilt_DIR="../shoes_data/test" IMAGE_DIR=os.path.join(Quilt_DIR,"/") #image = skimage.io.imread(os.path.join(IMAGE_DIR, "17.png")) #image = skimage.io.imread("/home/ljt/Shoe-data-V2/test/1.png") # image = skimage.io.imread("C:/Users/VCC/Desktop/3.jpg") # Run detection # print(image.shape) #image = cv2.imread(os.path.join(IMAGE_DIR, "65.png")) # image = cv2.imread("/home/ljt/Shoe-data-V2/test/20.png") image = cv2.imread("../shoes_data/test/shoes3.jpg") results = self.model.detect([image], verbose=1) r = results[0] # ---------------------------------------------------------------------txl mask = r['masks'] print(type(mask)) tolist = mask.tolist() # print(type(mask)) # print(type(tolist)) # print(tolist) # print(len(tolist[0][0])) height = len(tolist) width = len(tolist[0]) object_num = len(tolist[0][0]) print(height, width, object_num) points = [] for n in range(object_num): point_of_n = [] for x in range(height): for y in range(width): if tolist[x][y][n]: point_of_n.append([y, x]) points.append(point_of_n) print(points[0]) print(len(points)) with open('position.txt', 'w') as f: for i in range(len(points)): cnt = np.array(points[i]) rect = cv2.minAreaRect(cnt) # 得到最小外接矩形的(中心(x,y), (宽,高), 旋转角度) print(rect) box = cv2.boxPoints(rect) # 获取最小外接矩形的4个顶点坐标(ps: cv2.boxPoints(rect) for OpenCV 3.x) box = np.int0(box) cv2.drawContours(image, [box], 0, (0, 0, 255), 3) # cv2.imwrite('contours.png', image) text = "" for i in range(2): for item in rect[i]: text = text + str(item) + " " text = text + str(rect[2]) + "\n" f.writelines(text) with open('position.txt', 'r') as f: poi = [] lines = f.readlines() for line in lines: min_rect = line.split() for i in range(len(min_rect)): min_rect[i] = float(min_rect[i]) poi.append(int(min_rect[0])) poi.append(int(min_rect[1])) cv2.arrowedLine(image, (poi[2],poi[3]), (poi[0],poi[1]),(255, 0, 0), 5) cv2.imwrite('contours.png', image) # ---------------------------------------------------------------------------------txl a = visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], self.dataset_val.class_names, r['scores'], show_bbox=False)
IoUs = np.arange(0.5, 1, 0.05) lis = [] mAPs = [] F1 = [] sub = 0 captions = list(range(100000)) image_ids = dataset_eval.image_ids for image_id in image_ids: begin = len(lis) original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset_eval, YeastInferenceConfig(), image_id, use_mini_mask=False) name = list(dataset_eval.image_info[image_id].values())[0] visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, dataset_eval.class_names, SAVE_DIR=os.path.join(DATA_SAVE_DIR, name)) ########################################################################################################################################## ### Detect on random image of random Images ########################################################################################################################################## results = model.detect([original_image], verbose=1) r = results[0] operations.display_instances(image_id, original_image, r['rois'], r['masks'],
start = time.time() results = model.detect([original_image], verbose=1) end = time.time() print("视频每帧时间:") print(end - start) r = results[0] # print("识别:", r) # visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], # dataset_val.class_names, r['scores'], ax=get_ax()) visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'], figsize=(8, 8)) # ## Evaluation # Compute VOC-Style mAP @ IoU=0.5 # Running on 10 images. Increase for better accuracy. # image_ids = np.random.choice(dataset_val.image_ids, 10) # APs = [] # for image_id in image_ids: # # Load image and ground truth data # image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(dataset_val, inference_config, # image_id, use_mini_mask=False) # molded_images = np.expand_dims(
IMAGES_PER_GPU = 1 if __name__ == '__main__': class_names = ['BG', 'arm', 'ring'] #加载模型 config = InferenceConfig() config.display() model = modellib.MaskRCNN(mode="inference", config=config, model_dir='/home/simon/logs/surgery_200') model_path = PRETRAINED_MODEL_PATH # or if you want to use the latest trained model, you can use : # model_path = model.find_last()[1] model.load_weights(model_path, by_name=True) colors = visualize.random_colors(len(class_names)) cap = cv2.VideoCapture(0) while True: _, frame = cap.read() predictions = model.detect([frame], verbose=1) # We are replicating the same image to fill up the batch_size p = predictions[0] output = visualize.display_instances(frame, p['rois'], p['masks'], p['class_ids'], class_names, p['scores'], colors=colors, real_time=True) cv2.imshow("Mask RCNN", output) k = cv2.waitKey(10) if k & 0xFF == ord('q'): break cap.release() cv2.destroyAllWindows()
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) info = dataset.image_info[image_id] print("image ID: {}.{} ({}) {}".format(info["source"], info["id"], image_id, dataset.image_reference(image_id))) # Run object detection results = model.detect([image], verbose=1) # Display results ax = get_ax(1) r = results[0] visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], dataset.class_names, r['scores'], ax=ax, title="Predictions", img_name="_detected_with_bbox") # log("gt_class_id", gt_class_id) # log("gt_bbox", gt_bbox) # log("gt_mask", gt_mask) splash = ship.color_splash(image, r['masks']) display_images([splash], cols=1, img_name="detected_withMask") # Generate RPN trainig targets # target_rpn_match is 1 for positive anchors, -1 for negative anchors # and 0 for neutral anchors. target_rpn_match, target_rpn_bbox = modellib.build_rpn_targets(
def detect_items_in_image(image, model): # # choose your color hsv = [(50 / 81, 1, 1) for i in range(81)] COLOR = list(map(lambda c: colorsys.hsv_to_rgb(*c), hsv)) random.shuffle(COLOR) # Root directory of the project # ROOT_DIR = os.path.abspath("/home/jiang/Grasp") # Import Mask RCNN sys.path.append(ROOT_DIR) # To find local version of the library from mrcnn import visualize class_names = [ 'BG', "red_pepper", 'green_pepper', "carrot", "turnip", "eggplant", "baozi", "croissant", "cupcake", "ginger", "cake", "corn", "grape", "banana", "kiwi", "lemon", "pear", "apple", "carambola", "train", "detergent", "plate_w", "plate_g", "paper_box", "plastic_box", "cup_", "mouse", "hand", 'watermelon' ] image = image[..., ::-1] # Run detection s_time = time.time() results = model.detect([image], verbose=1) e_time = time.time() print('mask rcnn using : ', int(round(e_time * 1000)) - int(round(s_time * 1000))) # Visualize results r = results[0] global __k save_image = image[..., ::-1] cv2.imwrite('/home/jiang/Grasp/result/res' + str(__k) + '.jpg', save_image) __k += 1 RESULT_SAVE_PATH = "/home/jiang/Grasp/result" # !!! change the function to choose which object to show in the results save_path_name = os.path.join(RESULT_SAVE_PATH, 'img.jpg') visualize.display_instances(save_path_name, image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'], colors=COLOR) print(r['masks'].shape) target_item_dict = {} for score, index, cood, index_i in zip(r['scores'], r['class_ids'], r['rois'], range(len(r['class_ids']))): mask_points = [] mask = r['masks'][:, :, index_i] print(class_names[index]) for x in range(mask.shape[0]): for y in range(mask.shape[1]): if mask[x][y]: mask_points.append([x, y]) mask_points = np.array(mask_points) if mask_points.shape[0] < 1500: continue target_item_dict[class_names[index]] = [mask_points] return target_item_dict
# image=cv2.resize(image,(512,512)) # Extending the size of the image to be (h,w,1) # 将图片二维扩展成3维,加了一个维度 image = image[..., np.newaxis] # Run object detection. verbose=0不显示日志 results = model.detect([image], verbose=0) # Display results #ax = get_ax(1) r = results[0] if (notcommit == 'test'): visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], classdict, r['scores'], title="Predictions", show_mask=False) T12_S1_index = -1 T12_S1_flag = False for class_id in r['class_ids']: class_id_dsc = classdict[class_id] T12_S1_index = T12_S1_index + 1 if (class_id_dsc == 'T12_S1'): #找到了'T12_S1',可能有多个T12_S1_index,但按得分排序的,所以会找到第一个靠谱的 #print('找到了T12_S1') T12_S1_flag = True break if (T12_S1_flag): #print('找到了T12_S1')
def detect_and_color_splash(model, image_path=None, video_path=None, out_dir=''): assert image_path or video_path class_names = ['BG', 'arm', 'ring'] # Image or video? if image_path: # Run model detection and generate the color splash effect print("Running on {}".format(args.image)) # Read image image = skimage.io.imread(args.image) # Detect objects r = model.detect([image], verbose=1)[0] # Color splash # splash = color_splash(image, r['masks']) visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'], making_image=True) file_name = 'splash.png' # Save output # file_name = "splash_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now()) # save_file_name = os.path.join(out_dir, file_name) # skimage.io.imsave(save_file_name, splash) elif video_path: import cv2 # Video capture vcapture = cv2.VideoCapture(video_path) # width = int(vcapture.get(cv2.CAP_PROP_FRAME_WIDTH)) # height = int(vcapture.get(cv2.CAP_PROP_FRAME_HEIGHT)) width = 1600 height = 1600 fps = vcapture.get(cv2.CAP_PROP_FPS) # Define codec and create video writer file_name = "splash_{:%Y%m%dT%H%M%S}.wmv".format( datetime.datetime.now()) vwriter = cv2.VideoWriter(file_name, cv2.VideoWriter_fourcc(*'MJPG'), fps, (width, height)) count = 0 success = True #For video, we wish classes keep the same mask in frames, generate colors for masks colors = visualize.random_colors(len(class_names)) while success: print("frame: ", count) # Read next image plt.clf() plt.close() success, image = vcapture.read() if success: # OpenCV returns images as BGR, convert to RGB image = image[..., ::-1] # Detect objects r = model.detect([image], verbose=0)[0] # Color splash # splash = color_splash(image, r['masks']) splash = visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'], colors=colors, making_video=True) # Add image to video writer vwriter.write(splash) count += 1 vwriter.release() print("Saved to ", file_name)
def vis_segmentation(image, r): visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'])
def get_mask(image, r): # Visualize results mask = visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores']) return mask
def runtest(img,model,dataset,filename,name): import json # image=img image=cv2.cvtColor(img,cv2.COLOR_BGR2RGB) # print(image.shape,"image shape1") image,_,scale,scale1,padding,_=utils.resize_image(image,min_dim=256, max_dim=1024) # print(scale, " ", scale1) results = model.detect([image], verbose=1) # print("iii") ax = get_ax(1) # print("iiii") r = results[0] # print(image.shape,"image shape before contours") ccc,contours=visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], dataset.class_names, r['scores'], ax=ax, title="Predictions",show_bbox=False,show_mask=True) # print(len(contours)) arr = np.array(contours) # print(arr[0][0][0][0],"shape") # print(contours[0][0],"contours") # print(contours[0]) cls=r['class_ids'] classes = ['Background','Hole(Virtual)','Hole(Physical)','Character Line Segment', 'Physical Degradation','Page Boundary','Character Component','Picture', 'Decorator','Library Marker','Boundary Line'] strt=""" { "_via_settings": { "ui": { "annotation_editor_height": 30, "annotation_editor_fontsize": 0.6000000000000001, "leftsidebar_width": 18, "image_grid": { "img_height": 80, "rshape_fill": "none", "rshape_fill_opacity": 0.3, "rshape_stroke": "yellow", "rshape_stroke_width": 2, "show_region_shape": true, "show_image_policy": "all" }, "image": { "region_label": "region_id", "region_label_font": "10px Sans" } }, "core": { "buffer_size": 18, "filepath": {}, "default_filepath": "" }, "project": { "name": "corrected_3" } }, "_via_img_metadata": { "": { "filename": \""""+str(filename)+"""\", "size": -1, "regions": [ """ end=""" ], "file_attributes": {} } }, "_via_attributes": { "region": { "Spatial Annotation": { "type": "dropdown", "description": "", "options": { "Hole(Virtual)": "", "Hole(Physical)": "", "Character Line Segment": "", "Boundary Line": "", "Physical Degradation": "", "Page Boundary": "", "Character Component": "", "Picture": "", "Decorator": "", "Library Marker": "" }, "default_options": {} }, "Comments": { "type": "text", "description": "", "default_value": "" } }, "file": {} }, "file": \""""+str(name)+"""\" } """ rgns="" # print(scale,"scale") # print(scale1,"scale1") arr1 = np.zeros(11) count1 = np.zeros(11) avg1 = np.zeros(11) # print("intial") # print(arr1,"arr") # print(count1,"count") ROOT_DIR = os.path.abspath("./Instance-segmentation-master") perimeter_path = ROOT_DIR+"/main/doc/" loaded_avg = np.load(perimeter_path+'perimeter.npy') # arr = arr # count = count for i in range(len(cls)): if i!=(-1): try: k = np.array(contours[i][0]) except: #error_out = """{ #"error": ["1"] #}""" #error_out = {} #error_out['error'] = 1 return [] # d = k[0][0] # print(type(d),"d") # print(scale,"scale") # k=(k *(1/scale)) k=k.astype(int) arr1=np.array(arr1) # count=count # print(k[0][0]) # k[:,0] = np.rint(k[:,0] * (1/ scale)) # k[:,1] = np.rint(k[:,1] * (1/ scale1)) # d = k[0][0] # print(type(d),"d") k= np.array(k) # print(k.shape,"kshape") # print(k," ith k",i) ln=len(contours[i][0]) mid = int(ln/2) # k1=k[0:mid,:] # k2=k[mid:ln-1,:] k1=k[0:ln-1,:] # print(k1,"ith k1",i) # print(k1.shape,"k1shape") # print(k2,"ith k2",i) # print(k2.shape,"k2shape") # from rdp import rdp # from simplification.cutil import simplify_coords, simplify_coordsvw # from polysimplify import VWSimplifier import visvalingamwyatt as vw # simplifier = vw.Simplifier(points) # Simplify by percentage of points to keep # simplifier.simplify(ratio=0.5) simplifier1 = vw.Simplifier(k1) # simplifier.simplify(ratio=0.5) n1=int(0.020*k1.shape[0]) # if(n1<4): # n1=4 # perimeter perimeter = Polygon(k1).length # print(perimeter,"pppppppppppp") sub_loaded = np.absolute(loaded_avg - perimeter) index_min = np.argmin(sub_loaded) # print(sub_loaded ,"subloaded") # print(index_min,"index_min") # if it is BG if(index_min)==0: n1=int(0.020*k1.shape[0]) #if it is H-V if(index_min)==1: n1=int(0.040*k1.shape[0]) #if it is H if(index_min)==2: n1=int(0.040*k1.shape[0]) # cls if(index_min)==3: n1=int(0.020*k1.shape[0]) # pd if(index_min)==4: n1=int(0.030*k1.shape[0]) # pb if(index_min)==5: n1=int(0.010*k1.shape[0]) # cc if(index_min)==6: n1=int(0.040*k1.shape[0]) # p if(index_min)==7: n1=int(0.040*k1.shape[0]) # d if(index_min)==8: n1=int(0.040*k1.shape[0]) # lm if(index_min)==9: n1=int(0.040*k1.shape[0]) # bl if(index_min)==10: n1=int(0.020*k1.shape[0]) if n1<4: n1=4 # print(n1,"n1") # n2=int(0.025*k2.shape[0]) # print(n2,"n2") rdpk1=np.array(simplifier1.simplify(number=n1)) # simplifier2 = vw.Simplifier(k2) # simplifier.simplify(ratio=0.5) # rdpk2=np.array(simplifier2.simplify(number=n2)) # rdpk1= rdp(k1,epsilon=1) # rdpk2= rdp(k2,epsilon=1) # print(rdpk1,"ith rdpk1",i) # print(rdpk1.shape,"rdpk1shape") # print(rdpk2,"ith rdpk2",i) # print(rdpk2.shape,"rdpk2shape") # final= np.concatenate((rdpk1, rdpk2), axis=0) final=rdpk1 # print(final.shape[0],"finalshape") # print(final[0][0],"final") i = int(i) finallist = np.array(final).tolist() # perimeter # perimeter = Polygon(final).length # # sub_loaded = abs.absolute(loaded_avg - perimeter) # print(perimeter,"perimeter of i - ", i) # print(cls[i]) arr1[cls[i]] += perimeter count1[cls[i]] += 1 avg1[cls[i]] = (arr1[cls[i]]/count1[cls[i]]) # print(arr1,"arr after i - ",i) # print(count1,"count after i - ",i) # print(avg1,"avg after i",i) # storingarray(arr,count) length=final.shape[0] str1="" str2="" for j in range(length): str1+=str(int((final[j][0]-padding[0][0])*(1/scale))) str1+="," str1+='\n' for j in range(length): str2+=str(int((final[j][1]-padding[1][0])*(1/scale))) # g=0 str2+="," str2+='\n' str1=str1[:-2] str2=str2[:-2] rg="""{ "shape_attributes": { "name": "polygon", "all_points_x": [ """+ str2+"""], "all_points_y": ["""+str1+ """] }, "region_attributes": { "Spatial Annotation":\""""+str(classes[cls[i]])+"""\", "Comments": "" }, "timestamp": { "StartingTime": 6016533, "EndTime": 6035060 } }""" if(i!=len(cls)-1): rg+="," rgns+=rg # if(i!=len(cls)-1): # rg+="," # rgns+=rg print("done") # for updating the perimeter values # print(np.load('perimeter.npy'),"old") # loaded_avg = np.load('perimeter.npy') new_avg= np.zeros(11) for i in range(11): if((int(count1[i])!=0) and (int(loaded_avg[i])!=0)): new_avg[i]= (loaded_avg[i] + avg1[i]) new_avg[i] = new_avg[i] / 2 else: new_avg[i] = loaded_avg[i] + avg1[i] np.save(perimeter_path+'perimeter.npy', new_avg) # print(np.load('perimeter.npy'),"new") #end of updating the perimeter values # k=np.array(contours) # print(k,"k") # print(k.shape[0],"shape1") # for i in range(len(cls)): # str1="" # str2="" # ln=len(contours[i][0]) # print(ln,"lrn") # for j in range(ln): # if(j%20==0): # str1+=str(contours[i][0][j][0]-padding[0][0]) # str1+=',' # str1+='\n' # for j in range(ln): # if(j%20==0): # str2+=str(contours[i][0][j][1]-padding[1][0]) # str2+=',' # str2+='\n' # str1=str1[:-2] # str2=str2[:-2] # rg="""{ # "shape_attributes": { # "name": "polygon", # "all_points_x": [ """+ str2+"""], # "all_points_y": ["""+str1+ """] # }, # "region_attributes": { # "Spatial Annotation":\""""+str(classes[cls[i]])+"""\", # "Comments": "" # }, # "timestamp": { # "StartingTime": 6016533, # "EndTime": 6035060 # } # }""" # if(i!=len(cls)-1): # rg+="," # rgns+=rg with open (perimeter_path+'save.json','w') as f: f.write(strt) f.write(rgns) f.write(end) h, w = image.shape[:2] image=image[padding[0][0]:h-padding[0][1],padding[1][0]:w-padding[1][1]] plt.savefig(OUTPUTPATH,bbox_inches='tight') # print(arr) # print(count) #final_output = strt + rgns + end final_output = '{"output": [' + rgns + '], "file": "' + str(name) + '", "url": "' + str(filename) + '"}' #print(final_output) return final_output
def s_benchmark( run_dir, dataset_real, inference_config, pred_mask_dir, pred_info_dir, vis_missed=False, ): """Runs supplementary benchmarking code.""" print("Computing Supplementary's bounding box metrics") results_dir = os.path.join(run_dir, "results_supplement") mkdir_if_missing(results_dir) image_ids = dataset_real.image_ids mkdir_if_missing(os.path.join(results_dir, "vis_fn")) ms = [[] for _ in range(10)] thresh_all = [0.25, 0.5, 0.75] for ov in thresh_all: for m in ms: m.append([]) ms.append(thresh_all) ms = list(zip(*ms)) for image_id in tqdm(image_ids): # Load image and ground truth data image, _, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt( dataset_real, inference_config, image_id, use_mini_mask=False) gt_stat, stat_name = compute_gt_stats(gt_bbox, gt_mask) r = np.load( os.path.join(pred_info_dir, "image_{:06}.npy".format(image_id))).item() r_masks = np.load( os.path.join(pred_mask_dir, "image_{:06}.npy".format(image_id))) # Must transpose from (n, h, w) to (h, w, n) r["masks"] = np.transpose(r_masks, (1, 2, 0)) # Make sure scores are sorted. sc = r["scores"] is_sorted = np.all(np.diff(sc) <= 0) assert is_sorted overlaps = utilslib.compute_overlaps(r["rois"], gt_bbox) dt = {"sc": sc[:, np.newaxis] * 1.0} gt = {"diff": np.zeros((gt_bbox.shape[0], 1), dtype=np.bool)} for ( tps, fps, scs, num_insts, dup_dets, inst_ids, ovs, tp_inds, fn_inds, gt_stats, thresh, ) in ms: tp, fp, sc, num_inst, dup_det, inst_id, ov = inst_bench_image( dt, gt, {"minoverlap": thresh}, overlaps) tp_ind = np.sort(inst_id[tp]) fn_ind = np.setdiff1d(np.arange(num_inst), tp_ind) tps.append(tp) fps.append(fp) scs.append(sc) num_insts.append(num_inst) dup_dets.append(dup_det) inst_ids.append(inst_id) ovs.append(ov) tp_inds.append(tp_ind) fn_inds.append(fn_ind) gt_stats.append(gt_stat) # Visualize missing objects fn_ind = ms[1][8][-1] # missing objects at threshold 0.5 if fn_ind.size > 0: _, _, axes = subplot(plt, (fn_ind.size + 1, 1), sz_y_sz_x=(5, 5)) ax = axes.pop() ax.imshow(image) ax.set_axis_off() class_names = {1: ""} for _ in range(fn_ind.size): j = fn_ind[_] ax = axes.pop() visualize.display_instances( image, gt_bbox[j:j + 1, :], gt_mask[:, :, j:j + 1], gt_class_id[j:j + 1], class_names, ax=ax, title="", ) file_name = os.path.join( results_dir, "vis_fn", "vis_{:06d}.png".format(dataset_real.image_id[image_id]), ) plt.savefig(file_name, bbox_inches="tight", pad_inches=0) plt.close() print("Computing AP and plotting PR curves...") # Compute AP for ( tps, fps, scs, num_insts, dup_dets, inst_ids, ovs, tp_inds, fn_inds, gt_stats, thresh, ) in ms: ap, rec, prec, npos, _ = inst_bench(None, None, None, tp=tps, fp=fps, score=scs, numInst=num_insts) str_ = "mAP: {:.3f}, prec: {:.3f}, rec: {:.3f}, npos: {:d}".format( ap[0], np.min(prec), np.max(rec), npos) # logging.error('%s', str_) # print("mAP: ", ap[0], "prec: ", np.max(prec), "rec: ", np.max(rec), "prec-1: ", # prec[-1], "npos: ", npos) plt.style.use("fivethirtyeight") # bmh') _, _, axes = subplot(plt, (3, 4), (8, 8), space_y_x=(0.2, 0.2)) ax = axes.pop() ax.plot(rec, prec, "r") ax.set_xlim([0, 1]) ax.set_ylim([0, 1]) ax.set_xlabel("Recall") ax.set_ylabel("Precision") ax.set_title(str_) #'{:5.3f}'.format(ap[0]*100)) plot_stats(stat_name, gt_stats, tp_inds, fn_inds, axes) file_name = os.path.join(results_dir, "pr_stats_{:d}.png".format(int(thresh * 100))) # logging.error('plot file name: %s', file_name) plt.savefig(file_name, bbox_inches="tight", pad_inches=0) plt.close()
image_path = str(DATA_DIR / 'test' / image_id) img = cv2.imread(image_path) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) result = model.detect([resize_image(image_path)]) r = result[0] if r['masks'].size > 0: masks = np.zeros((img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8) for m in range(r['masks'].shape[-1]): masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST) y_scale = img.shape[0] / IMAGE_SIZE x_scale = img.shape[1] / IMAGE_SIZE rois = (r['rois'] * [y_scale, x_scale, y_scale, x_scale]).astype(int) masks, rois = refine_masks(masks, rois) else: masks, rois = r['masks'], r['rois'] visualize.display_instances(img, rois, masks, r['class_ids'], ['bg'] + label_names, r['scores'], title=image_id, figsize=(12, 12))
fig = plt.figure(figsize=(10, 30)) for i in range(6): image_id = random.choice(dataset.image_ids) original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset_val, inference_config, image_id, use_mini_mask=False) print(original_image.shape) plt.subplot(6, 2, 2 * i + 1) visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, dataset.class_names, colors=get_colors_for_class_ids(gt_class_id), ax=fig.axes[-1]) plt.subplot(6, 2, 2 * i + 2) results = model.detect([original_image]) #, verbose=1) r = results[0] visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], dataset.class_names, r['scores'], colors=get_colors_for_class_ids( r['class_ids']),
def detect_and_color_splash(model, image_path=None, video_path=None): assert image_path or video_path class_names = [ "BODY", "Brain stem", "Chiasma", "L-eye", "L-lens", "L-Optic nerve", "L-Parotid", "R-eye", "R-lens", "R-Optic nerve", "R-Parotid", "Spinal cord", "Thyroid" ] # class_names = ['BG', 'L-eye', 'R-eye', 'Brain stem'] # class_names = ['BODY', 'Spinal cord', 'Lung', 'Spinal cord+5mm', 'Airway', 'Heart', 'Brain stem', 'L-Parotid', 'R-Parotid', 'Bladder', # 'Chiasma', 'Rectum', 'R-lens', 'L-lens', 'L-Optic nerve', 'R-eye', # 'R-Optic nerve', 'L-eye', 'Liver', 'Thyroid', "R't Lung", "L't Lung", 'GTV-N', 'CTV-L', "R't_kidney", "L't_kidney", # 'GTV-T', 'Stomach', 'Bladder wall', 'Rectum wall', 'Sig Colon', "R't kidney", # "L't Kidney", "L't OPN", "R't OPN", 'Spinal Cord', 'Body'] # Image or video? if image_path: # Run model detection and generate the color splash effect print("Running on {}".format(image)) # Read image image = skimage.io.imread(image) # Detect objects r = model.detect([image], verbose=1)[0] visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'], making_image=True) # Color splash # splash = color_splash(image, r['masks']) # Save output # file_name = "splash_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now()) # skimage.io.imsave(file_name, splash) elif video_path: import cv2 # Video capture vcapture = cv2.VideoCapture(video_path) width = int(vcapture.get(cv2.CAP_PROP_FRAME_WIDTH)) height = int(vcapture.get(cv2.CAP_PROP_FRAME_HEIGHT)) fps = vcapture.get(cv2.CAP_PROP_FPS) # Define codec and create video writer file_name = "splash_{:%Y%m%dT%H%M%S}.avi".format( datetime.datetime.now()) vwriter = cv2.VideoWriter(file_name, cv2.VideoWriter_fourcc(*'MJPG'), fps, (width, height)) count = 0 success = True colors = visualize.random_colors(len(class_names)) while success: print("frame: ", count) # Read next image success, image = vcapture.read() if success: # OpenCV returns images as BGR, convert to RGB image = image[..., ::-1] # Detect objects r = model.detect([image], verbose=0)[0] splash = visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'], colors=colors, making_video=True) # # Color splash # splash = color_splash(image, r['masks']) # # RGB -> BGR to save image to video # splash = splash[..., ::-1] # Add image to video writer vwriter.write(splash) count += 1 vwriter.release() print("Saved to ", file_name)
'scores'] > thre #index of the elements with prob > threshold new_rois = r['rois'][selected] new_masks = r['masks'][:, :, selected] new_classids = r['class_ids'][selected] new_scores = r['scores'][selected] temp['rois'] = new_rois temp['masks'] = new_masks temp['class_ids'] = new_classids temp['scores'] = new_scores if (not args.maskoff): #if displayed mask option is on image_ir = visualize.display_instances( image, temp['rois'], temp['masks'], temp['class_ids'], dataset_detect.class_names, temp['scores'], ax=ax, title="Predictions") # image_ir = visualize.display_instances_new(image, r['rois'], r['masks'],r['class_ids'], # dataset_detect.class_names, r['scores'],ax=ax, # ) # plt.imshow(image_ir) # plt.show() #write the detection result to a folder if (args.write): plt.savefig(os.path.join(save_path, os.path.split(image_path)[1]), bbox_inches='tight')
def test(examples="one", backbone="resnet"): config = InferenceConfig() config.display() inference_model = modellib.MaskRCNN(mode="inference", model_dir=MODEL_DIR, config=config, backbone=backbone) model_path = PRETRAINED_MODEL_PATH # or if you want to use the latest trained model, you can use : # model_path = model.find_last()[1] inference_model.load_weights(model_path, by_name=True) class_names = ['BG', 'building'] # In our case, we have 1 class for the background, and 1 class for building file_names = next(os.walk(IMAGE_DIR))[2] # Run on single example if (examples == "one"): file_name = random.choice(file_names) print(file_name) random_image = skimage.io.imread(os.path.join(IMAGE_DIR, file_name)) predictions = inference_model.detect([random_image] * config.BATCH_SIZE, verbose=1) # We are replicating the same image to fill up the batch_size p = predictions[0] visualize.display_instances(random_image, p['rois'], p['masks'], p['class_ids'], class_names, p['scores']) # Run on entire test set else: # Gather all JPG files in the test set as small batches files = glob.glob(os.path.join(IMAGE_DIR, "*.jpg")) ALL_FILES = [] _buffer = [] for _idx, _file in enumerate(files): if len(_buffer) == config.IMAGES_PER_GPU * config.GPU_COUNT: ALL_FILES.append(_buffer) _buffer = [] else: _buffer.append(_file) if len(_buffer) > 0: ALL_FILES.append(_buffer) # Iterate over all the batches and predict _final_object = [] for files in tqdm.tqdm(ALL_FILES): images = [skimage.io.imread(x) for x in files] predictions = inference_model.detect(images, verbose=0) for _idx, r in enumerate(predictions): _file = files[_idx] image_id = int(_file.split("/")[-1].replace(".jpg", "")) for _idx, class_id in enumerate(r["class_ids"]): if class_id == 1: mask = r["masks"].astype(np.uint8)[:, :, _idx] bbox = np.around(r["rois"][_idx], 1) bbox = [float(x) for x in bbox] _result = {} _result["image_id"] = image_id _result["category_id"] = 100 _result["score"] = float(r["scores"][_idx]) _mask = maskUtils.encode(np.asfortranarray(mask)) _mask["counts"] = _mask["counts"].decode("UTF-8") _result["segmentation"] = _mask _result["bbox"] = [bbox[1], bbox[0], bbox[3] - bbox[1], bbox[2] - bbox[0]] _final_object.append(_result)
def detect_path(input_path, partitioning=[1, 1, 0], overwrite=False): # get data to do with image and where to save it...... rows, cols, overlap = partitioning image_head, image_ext = os.path.splitext(input_path) image_title = os.path.basename(image_head) reaction_path = os.path.dirname(os.path.dirname(input_path)) rxn_id = os.path.basename(reaction_path) exp_path = os.path.dirname(reaction_path) exp_id = os.path.basename(exp_path) output_dir = os.path.join(output_root_dir, exp_id, rxn_id, image_title) os.makedirs(output_dir, exist_ok=True) image = skimage.io.imread(input_path) duplicate_path = os.path.join(output_dir, os.path.basename(input_path)) skimage.io.imsave(duplicate_path, image) image_height, image_width = image.shape[:2] # partition the image as described by rows, cols and overlap partitions = get_partitions( image, partitioning) # a list of dicts with keys: 't', 'b', 'l', 'r' # array with the same dimensions as partitioned partition_shape = np.reshape(range(rows * cols), (rows, cols)) # data to be collected all_rois = np.empty((0, 4)) all_masks = np.empty((image_height, image_width, 0)) all_ids = np.empty(0) masks_per_partition = [] fig, ax = plt.subplots() for part_idx, p in enumerate(partitions): # data about partition and neighbours. Need to check each neighbour for overlapping detections partition_coordinates = np.where(partition_shape == part_idx) x, y = partition_coordinates[0][0], partition_coordinates[1][0] prev_neighbour_coordinates = [[x - 1, y], [x, y - 1]] neighbours_to_check = [ i for i in prev_neighbour_coordinates if -1 not in i ] indexes_to_check = [ partition_shape[x][y] for x, y in neighbours_to_check ] # actual image to be detected image_partition = image[p['t']:p['b'], p['l']:p['r']] save_root = os.path.join(output_dir, '[{},{},{}]'.format(rows, cols, overlap)) save_folder = os.path.join(save_root, 'images') os.makedirs(save_folder, exist_ok=True) new_image_name = '{};({}-{})({}-{}).png'.format( image_title, p['t'], p['b'], p['l'], p['r']) save_path = os.path.join(save_folder, new_image_name) if not overwrite: # if we don't want to overwrite if os.path.exists(save_path): # if the path already exists continue # skip and go to next partition # detections p_detections = model.detect([image_partition], verbose=1)[0] visualize.display_instances(image_partition, p_detections['rois'], p_detections['masks'], p_detections['class_ids'], ['BG', ''], ax=ax) # saving data plt.savefig(save_path) print('saving labelled image', save_path) # reset matplotlib plt.clf() plt.close('all') fig, ax = plt.subplots(figsize=(int(image_height / screen_dpi), int(image_width / screen_dpi))) # can only select mask by index when the mask array is transposed masks = p_detections['masks'].T masks_in_partition = [] # iterate through each detected mask for roi_idx, roi in enumerate(p_detections['rois']): _t, _l, _b, _r = roi # roi in partition # corresponding roi in original image t, b = [rownum + p['t'] for rownum in [_t, _b]] l, r = [colnum + p['l'] for colnum in [_l, _r]] # add roi to roi data all_rois = np.vstack([all_rois, np.array([t, l, b, r])]) # get array of just the mask. Transposition needed because of format. mask = masks[roi_idx].T[_t:_b, _l:_r] # put mask array into original image dimensions mask_in_origin = np.zeros(image.shape[:2], np.uint8) mask_in_origin[t:b, l:r] = mask mask = mask.astype(np.uint8) # append to lists masks_in_partition.append(mask_in_origin) all_masks = np.dstack([all_masks, mask_in_origin]) # save data of this mask, even if it overlaps with a previous mask.... save_folder = os.path.join(save_root, 'raw_masks') os.makedirs(save_folder, exist_ok=True) mask_name = '{};({},{},{},{}).png'.format(part_idx, t, b, l, r) save_path = os.path.join(save_folder, mask_name) mask = mask.astype(int) * 255 skimage.io.imsave(save_path, mask) # add class id to data all_ids = np.append(all_ids, p_detections['class_ids'][roi_idx]).astype(int) # save a copy of masks which DO NOT overlap save_folder = os.path.join(save_root, 'masks') os.makedirs(save_folder, exist_ok=True) save_path = os.path.join(save_folder, mask_name) overlapping = False if 0.4 * (np.sum(mask) / 255) > np.sum( np.ones(mask.shape[:2], np.uint8)): # very large crystal is probably a false positive. Ignore. continue if part_idx == 0: # save everything in the first partition print('saving mask', save_path) skimage.io.imsave(save_path, mask.astype(np.uint8)) else: # this compares all masks with those in neighbouring partitions. (those that have already been analysed) # mask is saved if overlap is less than 10% for idx in indexes_to_check: masks_to_check = masks_per_partition[idx] for mask_to_check in masks_to_check: # see which pixels are present in both masks, i.e. the overlapping region mask_overlap = np.bitwise_and(mask_in_origin, mask_to_check) if np.sum(mask_overlap) > np.sum(mask_in_origin) * 0.1: # if the overlapping region accounts for more than 10% of the original crystal # we decide that the crystals do not overlap. i.e arise from different crystals overlapping = True break # we stop checking if we already know that the crystals are overlapping if overlapping: break # if the crystals are not overlapping then we save if not overlapping: print('saving image: {}'.format(save_path)) skimage.io.imsave(save_path, mask.astype(np.uint8)) # add all masks from the partition to a list once the partition has been assessed. this is for # checking previous partitions for overlaps. masks_per_partition.append(masks_in_partition) # once all partitions have been assessed... if not overwrite: # if we don't want to overwrite.... if os.path.exists(save_path): # and the path already exists... return # exit function # create new axes to display images fig, ax = plt.subplots(figsize=(int(image_height / screen_dpi), int(image_width / screen_dpi))) # new image will have all detections from each partition stitched together visualize.display_instances(image, all_rois, all_masks, all_ids, ['BG', ''], ax=ax) # save it save_name = '{};[{},{},{}].png'.format(image_title, *partitioning) save_path = os.path.join(save_root, save_name) plt.savefig(save_path) print('saving new stitched image: ', save_path) # tidy up pyplot plt.clf() plt.close('all') ax.clear()
# Test on a random image image_id = random.choice(dataset_val.image_ids) original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset_val, inference_config, image_id, use_mini_mask=False) log("original_image", original_image) log("image_meta", image_meta) log("gt_class_id", gt_class_id) log("gt_bbox", gt_bbox) log("gt_mask", gt_mask) visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, dataset_train.class_names, figsize=(8, 8), save_path='gt_instances_' + str(image_id) + '.png') # %% results = model.detect([original_image], verbose=1) r = results[0] visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], dataset_val.class_names, r['scores'], ax=get_ax(),
def evaluate_tabnet(dataset_path, model, dataset, tab, eval_type="bbox", limit=0, image_ids=None): image_ids = image_ids or dataset.image_ids # Limit to a subset if limit: image_ids = image_ids[:limit] tab_image_ids = [dataset.image_info[id]["id"] for id in image_ids] t_prediction = 0 t_start = time.time() results = [] for i, image_id in enumerate(image_ids): # Load image image = dataset.load_image(image_id) #image1 = cv2.imread(dataset.image_info[image_id]['path']) #image1 = cv2.imread(image) # Run detection t = time.time() r = model.detect([image], verbose=0)[0] t_prediction += (time.time() - t) image_results = build_tabnet_results(dataset, tab_image_ids[i:i + 1], r["rois"], r["class_ids"], r["scores"], r["masks"].astype(np.uint8)) row_adj = r["row_adj"] col_adj = r["col_adj"] results.extend([image_results, row_adj, col_adj]) #ey_instancesxtra visualiza result print("i==>", i) #print("box information=>",r['rois']) #print("score information=>", r['scores']) image_name = "AR_" + str(tab_image_ids[i]) + ".jpg" result_path = os.path.join(dataset_path + '/result_jpg/', image_name) result_col_path = os.path.join( dataset_path + '/result_col_pkl/', "AR_" + str(tab_image_ids[i]) + "_col.pkl") result_row_path = os.path.join( dataset_path + '/result_row_pkl/', "AR_" + str(tab_image_ids[i]) + "_row.pkl") #visualize.display_results(image, r['rois'], r['masks'], r['class_ids'], dataset.class_names, r['scores'], save_dir='/ssd_scratch/cvit/ajoy/data/coco/result/', img_name = image_name) import pickle f = open(result_row_path, 'wb') pickle.dump(row_adj, f) f.close() f = open(result_col_path, 'wb') pickle.dump(col_adj, f) f.close() output = visualize.display_instances(image_name, image, r['rois'], r['masks'], r['class_ids'], dataset.class_names, r['scores'], dataset_path=dataset_path) output.figure.savefig(result_path) # save the figure to file #plt.close(output) #output=np.asarray(output) #output = Image.fromarray(output) #output.save('/ssd_scratch/cvit/ajoy/annual_report_table/coco/result/'+image_name,'.jpg') #output = np.float32(output) #output = cv2.cvtColor(cv2.UMat(output),cv2.COLOR_BGR2RGB) #result_path = os.path.join('/ssd_scratch/cvit/ajoy/annual_report_table/coco/result/', image_name) #cv2.imwrite(result_path, output) #cv2.imshow('output_image', output) #cv2.waitKey(10101010101010101010) #cv2.destroyAllWindows() #plt.savefig("{}/{}.png".format('/ssd_scratch/cvit/ajoy/data/coco/result/', dataset.image_info[image_id]["id"])) # Load results. This modifies results with additional attributes. tab_results = tab.loadRes(results) # Evaluate tabEval = COCOeval(tab, tab_results, eval_type) tabEval.params.imgIds = tab_image_ids tabEval.evaluate() tabEval.accumulate() tabEval.summarize() print("Prediction time: {}. Average {}/image".format( t_prediction, t_prediction / len(image_ids))) print("Total time: ", time.time() - t_start)
image_id = i test_image = dataset_finaltest.load_image(image_id) #smy 显示原图 # image_show = Image.fromarray(original_image) height, width = test_image.shape[:2] _, ax = plt.subplots(1, figsize=(8, 8)) ax.set_ylim(height + 10, -10) ax.set_xlim(-10, width + 10) ax.axis('off') ax.set_title("") ax.imshow(test_image.astype(np.uint8)) plt.show() #smy 显示原图 results = model.detect([test_image], verbose=1) r = results[0] visualize.display_instances(test_image, r['rois'], r['masks'], r['class_ids'], dataset_finaltest.class_names, r['scores'], ax=get_ax()) images = r['masks'] f = open('./test_parameter.txt','w') info = dataset_finaltest.image_info[image_id] print(info) for j in range(images.shape[2]): np.set_printoptions(threshold=np.NaN) image = images[:,:,j] if image.shape[0] != 1024 or image.shape[1] != 1024: break print('shape111', image.shape) print(info['original_info']) image = image[info['original_info'][0][0]:info['original_info'][0][2], info['original_info'][0][1]:info['original_info'][0][3]] w = image.shape[0] h = image.shape[1]
def evaluate_s2D_voc_style(inference_config, dataset_test, limit=0, image_ids=None): # Test on a random image image_ids = [] print("First Running Evaluation on 1 random image") # Test on a random image # Addsitional code added by Shiva Badruswamy to pick unique choices image_ids = dataset_test.image_ids random.shuffle(image_ids) image_id = random.choice(image_ids) state = True while state: if image_id in image_ids: image_id = random.choice(dataset_test.image_ids) break else: image_ids = image_ids.append(image_id) original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset_test, inference_config, image_id, use_mini_mask=False) log("original_image", original_image) log("image_meta", image_meta) log("gt_class_id", gt_class_id) log("gt_bbox", gt_bbox) log("gt_mask", gt_mask) visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, dataset_test.class_names, figsize=(8, 8)) state = False # Compute VOC-Style mAP @ IoU=0.5 # Running on 10 images. Increase for better accuracy. image_ids = dataset_test.image_ids[:limit] np.random.shuffle(image_ids) count = 0 APs = [] Recalls = [] for image_id in image_ids: count += 1 # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset_test, inference_config, image_id, use_mini_mask=False) molded_images = np.expand_dims( modellib.mold_image(image, inference_config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP AP, precisions, recalls, overlaps =\ utils.compute_ap(count,image_id,gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks'], iou_threshold=0.3) recalls = np.sum(recalls) / len(recalls) #images with low recall due to mask error exclude from calculations if recalls < 0.2: continue APs.append(AP) Recalls.append(recalls) if len(Recalls) > 0: mAP = np.mean(APs) mAR = np.mean(Recalls) f1_score = (2 * mAP * mAR) / (mAP + mAR) print("mAP:", mAP, "mAR:", mAR, "f1-score:", f1_score) else: print("Error: All images had < 0.5 recall")
mask, class_ids = dataset.load_mask(image_id) #print('real mask shape:',np.shape(mask)) # Compute Bounding box bbox = utils.extract_bboxes(mask) if np.shape(bbox)[0]>3: continue is_show = False # Display image and additional stats if is_show == True: print("image_id ", image_id, dataset.image_reference(image_id)) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) # Display image and instances visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) results = model.detect([image], verbose=1) r = results[0] mask_pred = r['masks'] class_ids_pred = r['class_ids'] bbox_pred = utils.extract_bboxes(mask_pred) if len(bbox_pred) == 0: continue if is_show == True: visualize.display_instances(image, bbox_pred, mask_pred, class_ids_pred, dataset.class_names) #print('pred mask shape:',np.shape(mask_pred)) index = cal_bbox_error(bbox, bbox_pred) mask_error = cal_mask_error(mask, mask_pred,index) mean_error = np.mean(mask_error) img_patch_name = str(mean_error) + '__' + str(image_id) + '.png'
# print('the dist is ', panorama.get_distanse('1.JPG', 'IMG_0995.JPG')) # print('size card is;', card.card_size()) """"""""""""""""""""""""""""""""""""""""""""""" Detect objects """"""""""""""""""""""""""""""""""""""""""""""" class InferenceConfig(object_detect.BalloonConfig): GPU_COUNT = 1 IMAGES_PER_GPU = 1 NUM_CLASSES = 3 config = InferenceConfig() model = modellib.MaskRCNN(mode="inference", model_dir='./mrcnn/logs', config=config) model.load_weights('./mrcnn/heights/mask_rcnn_object_0250.h5', by_name=True) img_list = ['DSC07663.JPG'] # img_list = ['odm_orthophoto.original.tif'] for img in img_list: image = mpimg.imread('./test_img/' + str(img)) # Run object detection results = model.detect([image], verbose=1) r = results[0] visualize.display_instances(img, image, r['rois'], r['masks'], r['class_ids'], ['BG', 'tree', 'building']) # tif = gdal.Open(', 1) panorama.save_detect_info('/home/error/PycharmProjects/panorama/result/odm_orthophoto.original.tif', r['rois'], r['masks']) # track = [[-40, 15], [-250, 220], [-10, 600]]
results = model.detect([image], verbose=1) r = results[0] AP, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks']) visualize.plot_precision_recall(AP, precisions, recalls) plt.show() # Display results print("---------------------------------------------------------------------------") ax = get_ax(1) r = results[0] visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], dataset.class_names, r['scores'], ax=ax, title="Predictions") log("gt_class_id", gt_class_id) log("gt_bbox", gt_bbox) log("gt_mask", gt_mask) print("---------------------------------------------------------------------------") # Draw precision-recall curve image_ids = np.random.choice(dataset.image_ids, 29) APs = [] for image_id in image_ids: # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask =\ image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
def FluorescenceAnalysis(self, folder, round_num): """ # ============================================================================= # Given the folder and round number, return a dictionary for the round # that contains each scanning position as key and structured array of detailed # information about each identified cell as content. # # Returned structured array fields: # - BoundingBox of cell ROI # - Mean intensity of whole cell area # - Mean intensity of cell membrane part # - Contour soma ratio # ============================================================================= Parameters ---------- folder : string. The directory to folder where the screening data is stored. round_num : int. The target round number of analysis. Returns ------- cell_Data : pd.DataFrame. Sum of return from func: retrieveDataFromML, for whole round. """ RoundNumberList, CoordinatesList, fileNameList = self.retrive_scanning_scheme(folder) os.mkdir(os.path.join(folder, 'MLimages_{}'.format(round_num))) # Create the folder for EachRound in RoundNumberList: if EachRound == round_num: # Start numbering cells at each round self.cell_counted_inRound = 0 for EachCoord in CoordinatesList: # ============================================================================= # For tag fluorescence: # ============================================================================= print(EachCoord) #-------------- readin image--------------- for Eachfilename in enumerate(fileNameList): if EachCoord in Eachfilename[1] and EachRound in Eachfilename[1]: ImgNameInfor = Eachfilename[1][0:len(Eachfilename[1])-14] # get rid of '_PMT_0Zmax.tif' in the name. tag_imagefilename = os.path.join(folder, Eachfilename[1]) #------------------------------------------ # ========================================================================= # USING MASKRCNN... # ========================================================================= Imagepath = self.Detector._fixPathName(tag_imagefilename) Rawimage = imread(Imagepath) # if ClearImgBef == True: # # Clear out junk parts to make it esaier for ML detection. # RawimageCleared = self.preProcessMLimg(Rawimage, smallest_size=300, lowest_region_intensity=0.16) # else: # RawimageCleared = Rawimage.copy() image = self.Convert2Unit8(Imagepath, Rawimage) # Run the detection on input image. results = self.Detector.RunDetectionOnImage(image) MLresults = results[0] ax, fig = visualize.display_instances(image, MLresults['rois'], MLresults['masks'], MLresults['class_ids'], ['BG'] + self.config.ValidLabels, MLresults['scores'],ReturnImageHandle=True) ax.imshow(fig) # Save the detection image segmentationImg = Image.fromarray(fig) #generate an image object segmentationImg.save(os.path.join(folder, 'MLimages_{}\{}.tif'.format(round_num, ImgNameInfor)))#save as tif if self.cell_counted_inRound == 0: cell_Data = self.retrieveDataFromML(Rawimage, MLresults, str(ImgNameInfor)) else: Cell_Data_new = self.retrieveDataFromML(Rawimage, MLresults, str(ImgNameInfor)) if len(Cell_Data_new) > 0: cell_Data = cell_Data.append(Cell_Data_new) return cell_Data