def resize_image(image_id): # Load random image and mask. image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) original_shape = image.shape # Resize image, window, scale, padding, _ = utils.resize_image( image, min_dim=config.IMAGE_MIN_DIM, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding) # Compute Bounding box bboxes = utils.extract_bboxes(mask) # Display image and additional stats print("image_id: ", image_id, dataset.image_reference(image_id)) print("Original shape: ", original_shape) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bboxes) # Display image and instances visualize.display_instances(image, bboxes, mask, class_ids, dataset.class_names)
def load_and_resize_images(dataset, datacfg, dnncfg): ''' ## Resize Images To support multiple images per batch, images are resized to one size (1024x1024). Aspect ratio is preserved, though. If an image is not square, then zero padding is added at the top/bottom or right/left. ''' log.info("load_and_resize_images::-------------------------------->") image_id = np.random.choice(dataset.image_ids, 1)[0] image = dataset.load_image(image_id) mask, class_ids, keys, values = dataset.load_mask(image_id, datacfg) original_shape = image.shape # Resize image, window, scale, padding, _ = utils.resize_image( image, min_dim=dnncfg.IMAGE_MIN_DIM, max_dim=dnncfg.IMAGE_MAX_DIM, mode=dnncfg.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding) # Compute Bounding box bbox = utils.extract_bboxes(mask) # Display image and additional stats log.debug("Original shape: {}".format(original_shape)) # customlog("image", image) # customlog("mask", mask) # customlog("class_ids", class_ids) # customlog("bbox", bbox) ## Display image and instances class_names = dataset.class_names visualize.display_instances(image, bbox, mask, class_ids, class_names)
def load_image_gt(dataset, config, image_id, augment=False, augmentation=None, use_mini_mask=False): image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) original_shape = image.shape image, window, scale, padding, crop = utils.resize_image( image, min_dim=config.IMAGE_MIN_DIM, min_scale=config.IMAGE_MIN_SCALE, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding, crop) if augment: logging.warning("'augment' is deprecated. Use 'augmentation' instead.") if random.randint(0, 1): image = np.fliplr(image) mask = np.fliplr(mask) if augmentation: import imgaug MASK_AUGMENTERS = ["Sequential", "SomeOf", "OneOf", "Sometimes", "Fliplr", "Flipud", "CropAndPad", "Affine", "PiecewiseAffine"] def hook(images, augmenter, parents, default): """Determines which augmenters to apply to masks.""" return augmenter.__class__.__name__ in MASK_AUGMENTERS image_shape = image.shape mask_shape = mask.shape det = augmentation.to_deterministic() image = det.augment_image(image) mask = det.augment_image(mask.astype(np.uint8), hooks=imgaug.HooksImages(activator=hook)) assert image.shape == image_shape, "Augmentation shouldn't change image size" assert mask.shape == mask_shape, "Augmentation shouldn't change mask size" mask = mask.astype(np.bool) _idx = np.sum(mask, axis=(0, 1)) > 0 mask = mask[:, :, _idx] class_ids = class_ids[_idx] bbox = utils.extract_bboxes(mask) active_class_ids = np.zeros([dataset.num_classes], dtype=np.int32) source_class_ids = dataset.source_class_ids[dataset.image_info[image_id]["source"]] active_class_ids[source_class_ids] = 1 if use_mini_mask: mask = utils.minimize_mask(bbox, mask, config.MINI_MASK_SHAPE) image_meta = compose_image_meta(image_id, original_shape, image.shape, window, scale, active_class_ids) return image, image_meta, class_ids, bbox, mask
def get_fru_net_results(results_dir: str, dataset: VesicleDataset): WIN_NAME = 'img' cv.namedWindow(WIN_NAME) gt_boxes = [] gt_class_ids = [] gt_masks = [] results = [] images = [] for image_id in dataset.image_ids: origin_img = dataset.load_image(image_id) image, image_meta, gt_class_id, gt_bbox, gt_mask = \ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) gt_boxes.append(gt_bbox) gt_class_ids.append(gt_class_id) gt_masks.append(gt_mask) # load masks img_name = dataset.image_info[image_id]['id'] name, ext = img_name.rsplit('.', 1) path = os.path.join(results_dir, f'{name}_labels.{ext}') mask_img = cv.imread(path, cv.IMREAD_GRAYSCALE + cv.IMREAD_ANYDEPTH) n = np.max(mask_img) class_ids = np.ones(n, dtype=np.int32) scores = np.ones(n, dtype=np.float32) mask = get_bin_mask(mask_img) image, window, scale, padding, crop = utils.resize_image( origin_img, min_dim=config.IMAGE_MIN_DIM, min_scale=config.IMAGE_MIN_SCALE, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding, crop) rois = utils.extract_bboxes(mask) images.append(image) vis_img = image.copy() draw_masks_contours(vis_img, gt_mask, (0, 0, 255)) draw_masks_contours(vis_img, mask, (0, 255, 0)) cv.setWindowTitle(WIN_NAME, img_name) cv.imshow(WIN_NAME, vis_img) cv.waitKey(0) result = { 'class_ids': class_ids, 'scores': scores, 'masks': mask, 'rois': rois } results.append(result) return images, gt_boxes, gt_class_ids, gt_masks, results
def get_bbox_from_mask(self, mask_ori, image, class_id): """ """ config = self.config mask = np.transpose(mask_ori[:][:][:],(2,1,0)) mask_raw = utils.resize_mask(mask,1,0) mask = np.equal(mask_raw, 3) # class_id = self.map_source_class_id( "clothes.{}".format(class_id)) mask = mask.astype(np.bool) class_ids = np.array([class_id]).astype(np.int32) image, window, scale, padding, crop = utils.resize_image( image, min_dim=config.IMAGE_MIN_DIM, min_scale=config.IMAGE_MIN_SCALE, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding, crop) _idx = np.sum(mask, axis=(0, 1)) > 0 mask = mask[:, :, _idx] class_ids = class_ids[_idx] # Bounding boxes. Note that some boxes might be all zeros # if the corresponding mask got cropped out. # bbox: [num_instances, (y1, x1, y2, x2)] bbox = utils.extract_bboxes(mask) # Active classes # Different datasets have different classes, so track the # classes supported in the dataset of this image. # if bbox.shape[0]>1: # print("------------bbox num:", bbox.shape[0]) return bbox, class_ids
def enlarge_mask(image_ori=[], pred=[], gt=[], config=None): if len(list(np.shape(image_ori))) == 2: image_ori = np.stack([image_ori, image_ori, image_ori], axis=-1) image_enlarge, window, scale, padding, crop = utils.resize_image( image_ori, min_dim=config.IMAGE_MIN_DIM, min_scale=config.IMAGE_MIN_SCALE, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) pred_enlarge = [] image_enlarge_shape = np.shape(image_enlarge) if pred != []: pred_shape = np.shape(pred) # if len(list(np.shape(pred))) == 2: # pred = np.stack([pred, pred, pred], axis=-1) # if np.shape(pred) != (config.IMAGE_MIN_DIM, config.IMAGE_MAX_DIM, 3): # pred_enlarge = utils.resize_mask(pred, [config.IMAGE_MIN_DIM, config.IMAGE_MAX_DIM], padding) # else: # pred_enlarge = pred # pred_enlarge = pred_enlarge[:, :, 0] if not np.max(pred) == 0: pred = pred / np.max(pred) if pred_shape[:2] != (config.IMAGE_MIN_DIM, config.IMAGE_MAX_DIM): if pred_shape[0] == pred_shape[1]: pred = pred * 255 pred_enlarge = utils.resize_mask( pred, [config.IMAGE_MIN_DIM, config.IMAGE_MAX_DIM], [(0, 0), (0, 0)]) # pred should be 0-255 else: pred_enlarge = utils.rescale_mask( pred, scale, padding) # pred should be 0-1 else: pred_enlarge = pred else: pred_enlarge = np.zeros(image_enlarge_shape[:2], np.uint8) else: # pred_enlarge = np.zeros(image_enlarge_shape[:2], np.uint8) pass gt_enlarge = [] if gt != []: if len(list(np.shape(gt))) == 2: gt = np.stack([gt, gt, gt], axis=-1) if np.shape(gt) != (config.IMAGE_MIN_DIM, config.IMAGE_MAX_DIM, 3): gt_enlarge = utils.rescale_mask(gt, scale, padding) else: gt_enlarge = gt gt_enlarge = gt_enlarge[:, :, 0] return image_enlarge, pred_enlarge, gt_enlarge
def validate_test_img(dataset_test, test_image_name): image_ids = (dataset_test.image_ids) for image_id in range(len(image_ids)): if dataset_test.image_info[image_id]['id'] == test_image_name: # get the mask id print("image_id ", image_id, dataset_train.image_reference(image_id)) # load the mask mask, class_ids = dataset_test.load_mask(image_id) # load the image image = dataset_test.load_image(image_id) # original shape original_shape = image.shape # resize the image with reference to the config file image, window, scale, padding, _ = utils.resize_image( image, min_dim=Config.IMAGE_MIN_DIM, max_dim=Config.IMAGE_MAX_DIM, mode=Config.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding) # compute the mask bbox = utils.extract_bboxes(mask) # Display image and additional stats print("image_id: ", image_id, dataset_train.image_reference(image_id)) print("Original shape: ", original_shape) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) # display the original data visualize.display_instances(image, bbox, mask, class_ids, dataset_test.class_names, figsize=(8, 8)) print("\nThere are {} actual dams for test image '{}'.".format(mask.shape[-1], test_image_name)) for i in range(mask.shape[-1]): mask_pixel = mask[:,:, i] # get number of pixels for each mask print(mask_pixel.sum())
def load_and_resize_images(self, dataset, dnncfg, ds_datacfg): ''' ## Resize Images To support multiple images per batch, images are resized to one size (1024x1024). Aspect ratio is preserved, though. If an image is not square, then zero padding is added at the top/bottom or right/left. ''' print("load_and_resize_images::-------------------------------->") image_id = np.random.choice(dataset.image_ids, 1)[0] image = dataset.load_image(image_id) datacfg = None if ds_datacfg: info = dataset.image_info[image_id] ds_source = info['source'] datacfg = utils.get_datacfg(ds_datacfg, ds_source) mask, class_ids, keys, values = dataset.load_mask(image_id, datacfg) original_shape = image.shape # Resize image, window, scale, padding, _ = utils.resize_image( image, min_dim=dnncfg.IMAGE_MIN_DIM, max_dim=dnncfg.IMAGE_MAX_DIM, mode=dnncfg.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding) # Compute Bounding box bbox = utils.extract_bboxes(mask) # Display image and additional stats print("Original shape: ", original_shape) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) ## Display image and instances class_names = dataset.class_names self.display_instances(image, bbox, mask, class_ids, class_names)
def load_mask(self, image_id): """Generate instance masks for shapes of the given image ID. """ global iter_num info = self.image_info[image_id] mask_obj = np.load(info['mask_path']) mask = mask_obj['mask'] shapes = mask_obj['class_id'] shapes = shapes[0] # print(shapes) #print("############shapes",shapes) count = len(shapes) # num_obj = self.get_obj_number(mask) # mask = np.zeros([info['height'], info['width'], num_obj], dtype=np.uint8) #mask = np.array(mask) #resize mask if count == 1: mask = mask.transpose(1, 2, 0) #print(mask.shape) scale = info['scale'] padding = info['padding'] crop = info['crop'] mask = utils.resize_mask(mask, scale, padding, crop) occlusion = np.logical_not(mask[:, :, -1]).astype(np.uint8) for i in range(count - 2, -1, -1): mask[:, :, i] = mask[:, :, i] * occlusion occlusion = np.logical_and(occlusion, np.logical_not(mask[:, :, i])) # if count == 1: # class_ids = np.array([1]) # else: # class_ids = np.array(shapes) class_ids = np.array(shapes) #print(mask.shape) return mask.astype(np.bool), class_ids.astype(np.int32)
log("bbox", bbox) # Display image and instances visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # Load random image and mask. image_id = np.random.choice(dataset.image_ids, 1)[0] image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) original_shape = image.shape # Resize image, window, scale, padding, _ = utils.resize_image( image, min_dim=config.IMAGE_MIN_DIM, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding) # Compute Bounding box bbox = utils.extract_bboxes(mask) # Display image and additional stats print("image_id: ", image_id, dataset.image_reference(image_id)) print("Original shape: ", original_shape) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) # Display image and instances visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) image_id = np.random.choice(dataset.image_ids, 1)[0]
def load_image_gt(dataset, config, image_id, augment=False, use_mini_mask=False): """ Load and return ground truth data for an image (image, mask, bounding boxes). Inputs: -------- augment: If true, apply random image augmentation. Currently, only horizontal flipping is offered. use_mini_mask: If False, returns full-size masks that are the same height and width as the original image. These can be big, for example 1024x1024x100 (for 100 instances). Mini masks are smaller, typically, 224x224 and are generated by extracting the bounding box of the object and resizing it to MINI_MASK_SHAPE. Returns: --------- image: [height, width, 3] shape: the original shape of the image before resizing and cropping. class_ids: [instance_count] Integer class IDs bbox: [instance_count, (y1, x1, y2, x2)] mask: [height, width, instance_count]. The height and width are those of the image unless use_mini_mask is True, in which case they are defined in MINI_MASK_SHAPE. """ # Load image and mask # print('=========================') # print(' Load Image GT: ', image_id) # print('=========================') image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) # print(mask.shape, class_ids.shape) # for i in range( class_ids.shape[-1]) : # print( 'mask ',i, ' class_id :', class_ids[i], mask[:,:,i].shape) # print() # print(np.array2string(np.where(mask[:,:,i],1,0),max_line_width=134, separator = '')) shape = image.shape image, window, scale, padding = utils.resize_image( image, min_dim=config.IMAGE_MIN_DIM, max_dim=config.IMAGE_MAX_DIM, padding=config.IMAGE_PADDING) mask = utils.resize_mask(mask, scale, padding) # print('after resize_mask shape is :',mask.shape) # Random horizontal flips. if augment: if random.randint(0, 1): image = np.fliplr(image) mask = np.fliplr(mask) # Bounding boxes. Note that some boxes might be all zeros # if the corresponding mask got cropped out. # bbox: [num_instances, (y1, x1, y2, x2)] bbox = utils.extract_bboxes(mask) # print('boxes are: \n', bbox) ## Active classes # Different datasets have different classes, so track the # classes supported in the dataset of this image. active_class_ids = np.zeros([dataset.num_classes], dtype=np.int32) source_class_ids = dataset.source_class_ids[dataset.image_info[image_id] ["source"]] active_class_ids[source_class_ids] = 1 # Resize masks to smaller size to reduce memory usage if use_mini_mask: mask = utils.minimize_mask(bbox, mask, config.MINI_MASK_SHAPE) # print('after use_mini_mask shape is :',mask.shape) # Image meta data image_meta = utils.compose_image_meta(image_id, shape, window, active_class_ids) return image, image_meta, class_ids, bbox, mask
def get_labels(self, labels): dims = labels.shape unlabeled_labels = np.zeros((dims[0], dims[1], 1)) building_labels = np.zeros((dims[0], dims[1], 1)) fence_labels = np.zeros((dims[0], dims[1], 1)) other_labels = np.zeros((dims[0], dims[1], 1)) pedestrian_labels = np.zeros((dims[0], dims[1], 1)) pole_labels = np.zeros((dims[0], dims[1], 1)) road_line_labels = np.zeros((dims[0], dims[1], 1)) road_labels = np.zeros((dims[0], dims[1], 1)) sidewalk_labels = np.zeros((dims[0], dims[1], 1)) vegetation_labels = np.zeros((dims[0], dims[1], 1)) car_labels = np.zeros((dims[0], dims[1], 1)) wall_labels = np.zeros((dims[0], dims[1], 1)) traffic_sign_labels = np.zeros((dims[0], dims[1], 1)) unlabeled_index = np.all(labels == (0, 0, 0), axis=-1) building_index = np.all(labels == (70, 70, 70), axis=-1) fence_index = np.all(labels == (190, 153, 153), axis=-1) other_index = np.all(labels == (250, 170, 160), axis=-1) pedestrian_index = np.all(labels == (220, 20, 60), axis=-1) pole_index = np.all(labels == (153, 153, 153), axis=-1) road_line_index = np.all(labels == (157, 234, 50), axis=-1) road_index = np.all(labels == (128, 64, 128), axis=-1) sidewalk_index = np.all(labels == (244, 35, 232), axis=-1) vegetation_index = np.all(labels == (107, 142, 35), axis=-1) car_index = np.all(labels == (0, 0, 142), axis=-1) wall_index = np.all(labels == (102, 102, 156), axis=-1) traffic_sign_index = np.all(labels == (220, 220, 70), axis=-1) unlabeled_labels[unlabeled_index] = 1 building_labels[building_index] = 10 fence_labels[fence_index] = 10 other_labels[other_index] = 10 pedestrian_labels[pedestrian_index] = 10 pole_labels[pole_index] = 10 road_line_labels[road_line_index] = 10 road_labels[road_index] = 10 sidewalk_labels[sidewalk_index] = 10 vegetation_labels[vegetation_index] = 1 car_labels[car_index] = 10 wall_labels[wall_index] = 10 traffic_sign_labels[traffic_sign_index] = 10 return np.dstack([unlabeled_labels, building_labels, fence_labels, return np.dstack([unlabeled_labels, building_labels, fence_labels, other_labels, pedestrian_labels, pole_labels, road_line_labels, road_labels, sidewalk_labels, vegetation_labels, car_labels, wall_labels, traffic_sign_labels]) def image_reference(self, image_id): """Return the carla data of the image.""" info = self.image_info[image_id] if info["source"] == "carla": return info["id"] else: super(self.__class__).image_reference(self, image_id) config = CarlaConfig() config.STEPS_PER_EPOCH = NUMBER_OF_TRAIN_DATA//config.BATCH_SIZE config.VALIDATION_STEPS = NUMBER_OF_VAL_DATA//config.BATCH_SIZE config.display() dataset = carlaDataset() dataset.load_images(dir=RGB_TRAIN_DIR, type='train') # mask, a = train.load_mask(50) # print(a) dataset.prepare() print("Image Count: {}".format(len(dataset.image_ids))) print("Class Count: {}".format(dataset.num_classes)) for i, info in enumerate(dataset.class_info): print("{:3}. {:50}".format(i, info['name'])) image_ids = np.random.choice(dataset.image_ids, 4) for image_id in image_ids: image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) visualize.display_top_masks(image, mask, class_ids, dataset.class_names) # Load random image and mask. image_id = random.choice(dataset.image_ids) image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) # Compute Bounding box bbox = utils.extract_bboxes(mask) # Display image and additional stats print("image_id ", image_id) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) # Display image and instances visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # Load random image and mask. image_id = np.random.choice(dataset.image_ids, 1)[0] image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) original_shape = image.shape # Resize image, window, scale, padding, _ = utils.resize_image( image, min_dim=config.IMAGE_MIN_DIM, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding) # Compute Bounding box bbox = utils.extract_bboxes(mask) # Display image and additional stats print("image_id: ", image_id) print("Original shape: ", original_shape) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) # Display image and instances visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) image_id = np.random.choice(dataset.image_ids, 1)[0] image, image_meta, class_ids, bbox, mask = modellib.load_image_gt( dataset, config, image_id, use_mini_mask=False) log("image", image) log("image_meta", image_meta) log("class_ids", class_ids) log("bbox", bbox) log("mask", mask) display_images([image]+[mask[:,:,i] for i in range(min(mask.shape[-1], 7))]) visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # Generate Anchors backbone_shapes = modellib.compute_backbone_shapes(config, config.IMAGE_SHAPE) anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES, config.RPN_ANCHOR_RATIOS, backbone_shapes, config.BACKBONE_STRIDES, config.RPN_ANCHOR_STRIDE) # Print summary of anchors num_levels = len(backbone_shapes) anchors_per_cell = len(config.RPN_ANCHOR_RATIOS) print("Count: ", anchors.shape[0]) print("Scales: ", config.RPN_ANCHOR_SCALES) print("ratios: ", config.RPN_ANCHOR_RATIOS) print("Anchors per Cell: ", anchors_per_cell) print("Levels: ", num_levels) anchors_per_level = [] for l in range(num_levels): num_cells = backbone_shapes[l][0] * backbone_shapes[l][1] anchors_per_level.append(anchors_per_cell * num_cells // config.RPN_ANCHOR_STRIDE**2) print("Anchors in Level {}: {}".format(l, anchors_per_level[l])) ## Visualize anchors of one cell at the center of the feature map of a specific level # Load and draw random image image_id = np.random.choice(dataset.image_ids, 1)[0] image, image_meta, _, _, _ = modellib.load_image_gt(dataset, config, image_id) fig, ax = plt.subplots(1, figsize=(10, 10)) ax.imshow(image) levels = len(backbone_shapes) for level in range(levels): colors = visualize.random_colors(levels) # Compute the index of the anchors at the center of the image level_start = sum(anchors_per_level[:level]) # sum of anchors of previous levels level_anchors = anchors[level_start:level_start+anchors_per_level[level]] print("Level {}. Anchors: {:6} Feature map Shape: {}".format(level, level_anchors.shape[0], backbone_shapes[level])) center_cell = backbone_shapes[level] // 2 center_cell_index = (center_cell[0] * backbone_shapes[level][1] + center_cell[1]) level_center = center_cell_index * anchors_per_cell center_anchor = anchors_per_cell * ( (center_cell[0] * backbone_shapes[level][1] / config.RPN_ANCHOR_STRIDE**2) \ + center_cell[1] / config.RPN_ANCHOR_STRIDE) level_center = int(center_anchor) # Draw anchors. Brightness show the order in the array, dark to bright. for i, rect in enumerate(level_anchors[level_center:level_center+anchors_per_cell]): y1, x1, y2, x2 = rect p = patches.Rectangle((x1, y1), x2-x1, y2-y1, linewidth=2, facecolor='none', edgecolor=(i+1)*np.array(colors[level]) / anchors_per_cell) ax.add_patch(p) # Create data generator random_rois = 4000 g = modellib.data_generator( dataset, config, shuffle=True, random_rois=random_rois, batch_size=4, detection_targets=True) # Get Next Image if random_rois: [normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, rpn_rois, rois], \ [mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g) log("rois", rois) log("mrcnn_class_ids", mrcnn_class_ids) log("mrcnn_bbox", mrcnn_bbox) log("mrcnn_mask", mrcnn_mask) else: [normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes, gt_masks], _ = next(g) log("gt_class_ids", gt_class_ids) log("gt_boxes", gt_boxes) log("gt_masks", gt_masks) log("rpn_match", rpn_match, ) log("rpn_bbox", rpn_bbox) image_id = modellib.parse_image_meta(image_meta)["image_id"][0] print("image_id: ", image_id, dataset.image_reference(image_id)) # Remove the last dim in mrcnn_class_ids. It's only added # to satisfy Keras restriction on target shape. mrcnn_class_ids = mrcnn_class_ids[:, :, 0] b = 0 # Restore original image (reverse normalization) sample_image = modellib.unmold_image(normalized_images[b], config) # Compute anchor shifts. indices = np.where(rpn_match[b] == 1)[0] refined_anchors = utils.apply_box_deltas(anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV) log("anchors", anchors) log("refined_anchors", refined_anchors) # Get list of positive anchors positive_anchor_ids = np.where(rpn_match[b] == 1)[0] print("Positive anchors: {}".format(len(positive_anchor_ids))) negative_anchor_ids = np.where(rpn_match[b] == -1)[0] print("Negative anchors: {}".format(len(negative_anchor_ids))) neutral_anchor_ids = np.where(rpn_match[b] == 0)[0] print("Neutral anchors: {}".format(len(neutral_anchor_ids))) # ROI breakdown by class for c, n in zip(dataset.class_names, np.bincount(mrcnn_class_ids[b].flatten())): if n: print("{:23}: {}".format(c[:20], n)) # Show positive anchors visualize.draw_boxes(sample_image, boxes=anchors[positive_anchor_ids], refined_boxes=refined_anchors) # Show negative anchors visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids]) # Show neutral anchors. They don't contribute to training. visualize.draw_boxes(sample_image, boxes=anchors[np.random.choice(neutral_anchor_ids, 100)]) if random_rois: # Class aware bboxes bbox_specific = mrcnn_bbox[b, np.arange(mrcnn_bbox.shape[1]), mrcnn_class_ids[b], :] # Refined ROIs refined_rois = utils.apply_box_deltas(rois[b].astype(np.float32), bbox_specific[:, :4] * config.BBOX_STD_DEV) # Class aware masks mask_specific = mrcnn_mask[b, np.arange(mrcnn_mask.shape[1]), :, :, mrcnn_class_ids[b]] visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific, mrcnn_class_ids[b], dataset.class_names) # Any repeated ROIs? rows = np.ascontiguousarray(rois[b]).view(np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1]))) _, idx = np.unique(rows, return_index=True) print("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1])) if random_rois: # Dispalay ROIs and corresponding masks and bounding boxes ids = random.sample(range(rois.shape[1]), 8) images = [] titles = [] for i in ids: image = visualize.draw_box(sample_image.copy(), rois[b,i,:4].astype(np.int32), [255, 0, 0]) image = visualize.draw_box(image, refined_rois[i].astype(np.int64), [0, 255, 0]) images.append(image) titles.append("ROI {}".format(i)) images.append(mask_specific[i] * 255) titles.append(dataset.class_names[mrcnn_class_ids[b,i]][:20]) display_images(images, titles, cols=4, cmap="Blues", interpolation="none") # Check ratio of positive ROIs in a set of images. if random_rois: limit = 10 temp_g = modellib.data_generator( dataset, config, shuffle=True, random_rois=10000, batch_size=1, detection_targets=True) total = 0 for i in range(limit): _, [ids, _, _] = next(temp_g) positive_rois = np.sum(ids[0] > 0) total += positive_rois print("{:5} {:5.2f}".format(positive_rois, positive_rois/ids.shape[1])) print("Average percent: {:.2f}".format(total/(limit*ids.shape[1]))) exit()
def load_image_gt(dataset, config, image_id, augment=False, augmentation=None, use_mini_mask=False): """Load and return ground truth data for an image (image, mask, bounding boxes). augment: (deprecated. Use augmentation instead). If true, apply random image augmentation. Currently, only horizontal flipping is offered. augmentation: Optional. An imgaug (https://github.com/aleju/imgaug) augmentation. For example, passing imgaug.augmenters.Fliplr(0.5) flips images right/left 50% of the time. use_mini_mask: If False, returns full-size masks that are the same height and width as the original image. These can be big, for example 1024x1024x100 (for 100 instances). Mini masks are smaller, typically, 224x224 and are generated by extracting the bounding box of the object and resizing it to MINI_MASK_SHAPE. Returns: image: [height, width, 3] shape: the original shape of the image before resizing and cropping. class_ids: [instance_count] Integer class IDs bbox: [instance_count, (y1, x1, y2, x2)] mask: [height, width, instance_count]. The height and width are those of the image unless use_mini_mask is True, in which case they are defined in MINI_MASK_SHAPE. """ # Load image and mask image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) original_shape = image.shape image, window, scale, padding, crop = utils.resize_image( image, min_dim=config.IMAGE_MIN_DIM, min_scale=config.IMAGE_MIN_SCALE, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding, crop) # Random horizontal flips. # TODO: will be removed in a future update in favor of augmentation if augment: logging.warning("'augment' is deprecated. Use 'augmentation' instead.") if random.randint(0, 1): image = np.fliplr(image) mask = np.fliplr(mask) # Augmentation # This requires the imgaug lib (https://github.com/aleju/imgaug) if augmentation: import imgaug # Augmenters that are safe to apply to masks # Some, such as Affine, have settings that make them unsafe, so always # test your augmentation on masks MASK_AUGMENTERS = [ "Sequential", "SomeOf", "OneOf", "Sometimes", "Fliplr", "Flipud", "CropAndPad", "Affine", "PiecewiseAffine" ] def hook(images, augmenter, parents, default): """Determines which augmenters to apply to masks.""" return augmenter.__class__.__name__ in MASK_AUGMENTERS # Store shapes before augmentation to compare image_shape = image.shape mask_shape = mask.shape # Make augmenters deterministic to apply similarly to images and masks det = augmentation.to_deterministic() image = det.augment_image(image) # Change mask to np.uint8 because imgaug doesn't support np.bool mask = det.augment_image(mask.astype(np.uint8), hooks=imgaug.HooksImages(activator=hook)) # Verify that shapes didn't change assert image.shape == image_shape, "Augmentation shouldn't change image size" assert mask.shape == mask_shape, "Augmentation shouldn't change mask size" # Change mask back to bool mask = mask.astype(np.bool) # Note that some boxes might be all zeros if the corresponding mask got cropped out. # and here is to filter them out _idx = np.sum(mask, axis=(0, 1)) > 0 mask = mask[:, :, _idx] class_ids = class_ids[_idx] # Bounding boxes. Note that some boxes might be all zeros # if the corresponding mask got cropped out. # bbox: [num_instances, (y1, x1, y2, x2)] bbox = utils.extract_bboxes(mask) # Active classes # Different datasets have different classes, so track the # classes supported in the dataset of this image. active_class_ids = np.zeros([dataset.num_classes], dtype=np.int32) source_class_ids = dataset.source_class_ids[dataset.image_info[image_id] ["source"]] active_class_ids[source_class_ids] = 1 # Resize masks to smaller size to reduce memory usage if use_mini_mask: mask = utils.minimize_mask(bbox, mask, config.MINI_MASK_SHAPE) # Image meta data image_meta = compose_image_meta(image_id, original_shape, image.shape, window, scale, active_class_ids) return image, image_meta, class_ids, bbox, mask
def inspect_data(dataset, config): print("Image Count: {}".format(len(dataset.image_ids))) print("Class Count: {}".format(dataset.num_classes)) for i, info in enumerate(dataset.class_info): print("{:3}. {:50}".format(i, info['name'])) # ## Display Samples # # Load and display images and masks. # In[4]: # Load and display random samples image_ids = np.random.choice(dataset.image_ids, 4) for image_id in image_ids: image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) visualize.display_top_masks(image, mask, class_ids, dataset.class_names) # ## Bounding Boxes # # Rather than using bounding box coordinates provided by the source datasets, we compute the bounding boxes from masks instead. This allows us to handle bounding boxes consistently regardless of the source dataset, and it also makes it easier to resize, rotate, or crop images because we simply generate the bounding boxes from the updates masks rather than computing bounding box transformation for each type of image transformation. # In[5]: # Load random image and mask. image_id = random.choice(dataset.image_ids) image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) # Compute Bounding box bbox = utils.extract_bboxes(mask) # Display image and additional stats print("image_id ", image_id, dataset.image_reference(image_id)) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) # Display image and instances visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # ## Resize Images # # To support multiple images per batch, images are resized to one size (1024x1024). Aspect ratio is preserved, though. If an image is not square, then zero padding is added at the top/bottom or right/left. # In[6]: # Load random image and mask. image_id = np.random.choice(dataset.image_ids, 1)[0] image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) original_shape = image.shape # Resize image, window, scale, padding, _ = utils.resize_image( image, min_dim=config.IMAGE_MIN_DIM, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding) # Compute Bounding box bbox = utils.extract_bboxes(mask) # Display image and additional stats print("image_id: ", image_id, dataset.image_reference(image_id)) print("Original shape: ", original_shape) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) # Display image and instances visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # ## Mini Masks # # Instance binary masks can get large when training with high resolution images. For example, if training with 1024x1024 image then the mask of a single instance requires 1MB of memory (Numpy uses bytes for boolean values). If an image has 100 instances then that's 100MB for the masks alone. # # To improve training speed, we optimize masks by: # * We store mask pixels that are inside the object bounding box, rather than a mask of the full image. Most objects are small compared to the image size, so we save space by not storing a lot of zeros around the object. # * We resize the mask to a smaller size (e.g. 56x56). For objects that are larger than the selected size we lose a bit of accuracy. But most object annotations are not very accuracy to begin with, so this loss is negligable for most practical purposes. Thie size of the mini_mask can be set in the config class. # # To visualize the effect of mask resizing, and to verify the code correctness, we visualize some examples. # In[7]: image_id = np.random.choice(dataset.image_ids, 1)[0] image, image_meta, class_ids, bbox, mask = modellib.load_image_gt( dataset, config, image_id, use_mini_mask=False) log("image", image) log("image_meta", image_meta) log("class_ids", class_ids) log("bbox", bbox) log("mask", mask) display_images([image] + [mask[:, :, i] for i in range(min(mask.shape[-1], 7))]) # In[8]: visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # In[9]: # Add augmentation and mask resizing. image, image_meta, class_ids, bbox, mask = modellib.load_image_gt( dataset, config, image_id, augment=True, use_mini_mask=True) log("mask", mask) display_images([image] + [mask[:, :, i] for i in range(min(mask.shape[-1], 7))]) # In[10]: mask = utils.expand_mask(bbox, mask, image.shape) visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names) # ## Anchors # # The order of anchors is important. Use the same order in training and prediction phases. And it must match the order of the convolution execution. # # For an FPN network, the anchors must be ordered in a way that makes it easy to match anchors to the output of the convolution layers that predict anchor scores and shifts. # * Sort by pyramid level first. All anchors of the first level, then all of the second and so on. This makes it easier to separate anchors by level. # * Within each level, sort anchors by feature map processing sequence. Typically, a convolution layer processes a feature map starting from top-left and moving right row by row. # * For each feature map cell, pick any sorting order for the anchors of different ratios. Here we match the order of ratios passed to the function. # # **Anchor Stride:** # In the FPN architecture, feature maps at the first few layers are high resolution. For example, if the input image is 1024x1024 then the feature meap of the first layer is 256x256, which generates about 200K anchors (256*256*3). These anchors are 32x32 pixels and their stride relative to image pixels is 4 pixels, so there is a lot of overlap. We can reduce the load significantly if we generate anchors for every other cell in the feature map. A stride of 2 will cut the number of anchors by 4, for example. # # In this implementation we use an anchor stride of 2, which is different from the paper. # In[11]: # Generate Anchors backbone_shapes = modellib.compute_backbone_shapes(config, config.IMAGE_SHAPE) anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES, config.RPN_ANCHOR_RATIOS, backbone_shapes, config.BACKBONE_STRIDES, config.RPN_ANCHOR_STRIDE) # Print summary of anchors num_levels = len(backbone_shapes) anchors_per_cell = len(config.RPN_ANCHOR_RATIOS) print("Count: ", anchors.shape[0]) print("Scales: ", config.RPN_ANCHOR_SCALES) print("ratios: ", config.RPN_ANCHOR_RATIOS) print("Anchors per Cell: ", anchors_per_cell) print("Levels: ", num_levels) anchors_per_level = [] for l in range(num_levels): num_cells = backbone_shapes[l][0] * backbone_shapes[l][1] anchors_per_level.append(anchors_per_cell * num_cells // config.RPN_ANCHOR_STRIDE**2) print("Anchors in Level {}: {}".format(l, anchors_per_level[l])) # Visualize anchors of one cell at the center of the feature map of a specific level. # In[12]: ## Visualize anchors of one cell at the center of the feature map of a specific level # Load and draw random image image_id = np.random.choice(dataset.image_ids, 1)[0] image, image_meta, _, _, _ = modellib.load_image_gt( dataset, config, image_id) fig, ax = plt.subplots(1, figsize=(10, 10)) ax.imshow(image) levels = len(backbone_shapes) for level in range(levels): colors = visualize.random_colors(levels) # Compute the index of the anchors at the center of the image level_start = sum( anchors_per_level[:level]) # sum of anchors of previous levels level_anchors = anchors[level_start:level_start + anchors_per_level[level]] print("Level {}. Anchors: {:6} Feature map Shape: {}".format( level, level_anchors.shape[0], backbone_shapes[level])) center_cell = backbone_shapes[level] // 2 center_cell_index = (center_cell[0] * backbone_shapes[level][1] + center_cell[1]) level_center = center_cell_index * anchors_per_cell center_anchor = anchors_per_cell * ( (center_cell[0] * backbone_shapes[level][1] / config.RPN_ANCHOR_STRIDE**2) \ + center_cell[1] / config.RPN_ANCHOR_STRIDE) level_center = int(center_anchor) # Draw anchors. Brightness show the order in the array, dark to bright. for i, rect in enumerate(level_anchors[level_center:level_center + anchors_per_cell]): y1, x1, y2, x2 = rect p = patches.Rectangle( (x1, y1), x2 - x1, y2 - y1, linewidth=2, facecolor='none', edgecolor=(i + 1) * np.array(colors[level]) / anchors_per_cell) ax.add_patch(p) # ## Data Generator # # In[13]: # Create data generator random_rois = 2000 g = modellib.data_generator(dataset, config, shuffle=True, random_rois=random_rois, batch_size=4, detection_targets=True) # In[14]: # Uncomment to run the generator through a lot of images # to catch rare errors # for i in range(1000): # print(i) # _, _ = next(g) # In[15]: # Get Next Image if random_rois: [ normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, rpn_rois, rois ], [mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g) log("rois", rois) log("mrcnn_class_ids", mrcnn_class_ids) log("mrcnn_bbox", mrcnn_bbox) log("mrcnn_mask", mrcnn_mask) else: [ normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes, gt_masks ], _ = next(g) log("gt_class_ids", gt_class_ids) log("gt_boxes", gt_boxes) log("gt_masks", gt_masks) log( "rpn_match", rpn_match, ) log("rpn_bbox", rpn_bbox) image_id = modellib.parse_image_meta(image_meta)["image_id"][0] print("image_id: ", image_id, dataset.image_reference(image_id)) # Remove the last dim in mrcnn_class_ids. It's only added # to satisfy Keras restriction on target shape. mrcnn_class_ids = mrcnn_class_ids[:, :, 0] # In[16]: b = 0 # Restore original image (reverse normalization) sample_image = modellib.unmold_image(normalized_images[b], config) # Compute anchor shifts. indices = np.where(rpn_match[b] == 1)[0] refined_anchors = utils.apply_box_deltas( anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV) log("anchors", anchors) log("refined_anchors", refined_anchors) # Get list of positive anchors positive_anchor_ids = np.where(rpn_match[b] == 1)[0] print("Positive anchors: {}".format(len(positive_anchor_ids))) negative_anchor_ids = np.where(rpn_match[b] == -1)[0] print("Negative anchors: {}".format(len(negative_anchor_ids))) neutral_anchor_ids = np.where(rpn_match[b] == 0)[0] print("Neutral anchors: {}".format(len(neutral_anchor_ids))) # ROI breakdown by class for c, n in zip(dataset.class_names, np.bincount(mrcnn_class_ids[b].flatten())): if n: print("{:23}: {}".format(c[:20], n)) # Show positive anchors fig, ax = plt.subplots(1, figsize=(16, 16)) visualize.draw_boxes(sample_image, boxes=anchors[positive_anchor_ids], refined_boxes=refined_anchors, ax=ax) # In[17]: # Show negative anchors visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids]) # In[18]: # Show neutral anchors. They don't contribute to training. visualize.draw_boxes(sample_image, boxes=anchors[np.random.choice( neutral_anchor_ids, 100)]) # ## ROIs # In[19]: if random_rois: # Class aware bboxes bbox_specific = mrcnn_bbox[b, np.arange(mrcnn_bbox.shape[1]), mrcnn_class_ids[b], :] # Refined ROIs refined_rois = utils.apply_box_deltas( rois[b].astype(np.float32), bbox_specific[:, :4] * config.BBOX_STD_DEV) # Class aware masks mask_specific = mrcnn_mask[b, np.arange(mrcnn_mask.shape[1]), :, :, mrcnn_class_ids[b]] visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific, mrcnn_class_ids[b], dataset.class_names) # Any repeated ROIs? rows = np.ascontiguousarray(rois[b]).view( np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1]))) _, idx = np.unique(rows, return_index=True) print("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1])) # In[20]: if random_rois: # Dispalay ROIs and corresponding masks and bounding boxes ids = random.sample(range(rois.shape[1]), 8) images = [] titles = [] for i in ids: image = visualize.draw_box(sample_image.copy(), rois[b, i, :4].astype(np.int32), [255, 0, 0]) image = visualize.draw_box(image, refined_rois[i].astype(np.int64), [0, 255, 0]) images.append(image) titles.append("ROI {}".format(i)) images.append(mask_specific[i] * 255) titles.append(dataset.class_names[mrcnn_class_ids[b, i]][:20]) display_images(images, titles, cols=4, cmap="Blues", interpolation="none") # In[21]: # Check ratio of positive ROIs in a set of images. if random_rois: limit = 10 temp_g = modellib.data_generator(dataset, config, shuffle=True, random_rois=10000, batch_size=1, detection_targets=True) total = 0 for i in range(limit): _, [ids, _, _] = next(temp_g) positive_rois = np.sum(ids[0] > 0) total += positive_rois print("{:5} {:5.2f}".format(positive_rois, positive_rois / ids.shape[1])) print("Average percent: {:.2f}".format(total / (limit * ids.shape[1])))
# print(mask.shape) # print(class_ids.shape) # print(dataset.image_info[0]) # visualize.display_top_masks(image, mask, class_ids, dataset.class_names) # Load random image and mask. image_id = np.random.choice(dataset.image_ids, 1)[0] image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) original_shape = image.shape # Resize image, window, scale, padding, crop = utils.resize_image( image, min_dim=dataset_config.IMAGE_MIN_DIM, max_dim=dataset_config.IMAGE_MAX_DIM, mode=dataset_config.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding, crop=crop) # Compute Bounding box bbox = utils.extract_bboxes(mask) # Display image and additional stats print("image_id: ", image_id, dataset.image_reference(image_id)) print("Original shape: ", original_shape) log("image", image) log("mask", mask) log("class_ids", class_ids) log("bbox", bbox) # Display image and instances visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
"{}_segmentation.png".format(filename)]) out_segmentation_filename = os.path.join(output_dir, os.path.basename(segmentation_file)) print("filename##################################",filename) print("outfilename##################################",out_filename) print("segmentationfilename##################################",segmentation_file) # load the input image, convert it from BGR to RGB channel # ordering, and resize the image image = cv2.imread(image_file) image, window, scale, padding, crop = utils.resize_image(image, min_dim=config.IMAGE_MIN_DIM, min_scale=config.IMAGE_MIN_SCALE, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) groundtruth_image = cv2.imread(segmentation_file) groundtruth_mask = utils.resize_mask(groundtruth_image, scale, padding, crop) #image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) #image = imutils.resize(image, width=1024) image_mask = np.zeros(image.shape) print("#################input image shape",image.shape) print("#################ground truth mask shape",groundtruth_mask.shape) print("#################predicted mask shape",image_mask.shape) # perform a forward pass of the network to obtain the results r = model.detect([image], verbose=1)[0] # loop over of the detected object's bounding boxes and # masks, drawing each as we go along