def load_inference_data_obj_feat_gt(dataset, image_id, config): image = dataset.load_image(image_id) gt_ranks, sel_not_sal_obj_idx_list, shuffled_indices, chosen_obj_idx_order_list = dataset.load_gt_rank_order( image_id) object_roi_masks = dataset.load_object_roi_masks(image_id, sel_not_sal_obj_idx_list) original_shape = image.shape image, window, scale, padding, crop = utils.resize_image( image, min_dim=config.IMAGE_MIN_DIM, min_scale=config.IMAGE_MIN_SCALE, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) obj_mask = utils.resize_mask(object_roi_masks, scale, padding, crop) # bbox: [num_instances, (y1, x1, y2, x2)] obj_bbox = utils.extract_bboxes(obj_mask) # *********************** FILL REST, SHUFFLE ORDER *********************** # order is in salient objects then non-salient objects batch_obj_roi = np.zeros(shape=(config.SAL_OBJ_NUM, 4), dtype=np.int32) for i in range(len(chosen_obj_idx_order_list)): _idx = chosen_obj_idx_order_list[i] batch_obj_roi[_idx] = obj_bbox[i] # Normalize image image = model_utils.mold_image(image.astype(np.float32), config) # Active classes active_class_ids = np.ones([config.NUM_CLASSES], dtype=np.int32) img_id = image_id img_id = int(img_id[-12:]) # Image meta data image_meta = model_utils.compose_image_meta(img_id, original_shape, image.shape, window, scale, active_class_ids) # Expand input dimensions to consider batch image = np.expand_dims(image, axis=0) image_meta = np.expand_dims(image_meta, axis=0) batch_obj_roi = np.expand_dims(batch_obj_roi, axis=0) return [ image, image_meta, batch_obj_roi ], gt_ranks, sel_not_sal_obj_idx_list, shuffled_indices, chosen_obj_idx_order_list
def load_inference_data_obj_feat(dataset, image_id, config): image = dataset.load_image(image_id) object_roi_masks = dataset.load_object_roi_masks(image_id) original_shape = image.shape image, window, scale, padding, crop = utils.resize_image( image, min_dim=config.IMAGE_MIN_DIM, min_scale=config.IMAGE_MIN_SCALE, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) obj_mask = utils.resize_mask(object_roi_masks, scale, padding, crop) # bbox: [num_instances, (y1, x1, y2, x2)] obj_bbox = utils.extract_bboxes(obj_mask) # Normalize image image = model_utils.mold_image(image.astype(np.float32), config) # Active classes active_class_ids = np.ones([config.NUM_CLASSES], dtype=np.int32) img_id = image_id img_id = int(img_id[-12:]) # Image meta data image_meta = model_utils.compose_image_meta(img_id, original_shape, image.shape, window, scale, active_class_ids) # Expand input dimensions to consider batch image = np.expand_dims(image, axis=0) image_meta = np.expand_dims(image_meta, axis=0) batch_obj_roi = np.zeros(shape=(config.SAL_OBJ_NUM, 4), dtype=np.int32) batch_obj_roi[:len(obj_bbox)] = obj_bbox batch_obj_roi = np.expand_dims(batch_obj_roi, axis=0) return [image, image_meta, batch_obj_roi]
def mold_inputs(self, images): """Takes a list of images and modifies them to the format expected as an input to the neural network. images: List of image matrices [height,width,depth]. Images can have different sizes. Returns 3 Numpy matrices: molded_images: [N, h, w, 3]. Images resized and normalized. image_metas: [N, length of meta data]. Details about each image. windows: [N, (y1, x1, y2, x2)]. The portion of the image that has the original image (padding excluded). """ molded_images = [] image_metas = [] windows = [] for image in images: # Resize image # TODO: move resizing to mold_image() molded_image, window, scale, padding, crop = utils.resize_image( image, min_dim=self.config.IMAGE_MIN_DIM, min_scale=self.config.IMAGE_MIN_SCALE, max_dim=self.config.IMAGE_MAX_DIM, mode=self.config.IMAGE_RESIZE_MODE) molded_image = model_utils.mold_image(molded_image, self.config) # Build image_meta image_meta = model_utils.compose_image_meta( 0, image.shape, molded_image.shape, window, scale, np.zeros([self.config.NUM_CLASSES], dtype=np.int32)) # Append molded_images.append(molded_image) windows.append(window) image_metas.append(image_meta) # Pack into arrays molded_images = np.stack(molded_images) image_metas = np.stack(image_metas) windows = np.stack(windows) return molded_images, image_metas, windows
def load_image_gt(dataset, config, image_id, augmentation=None): """Load and return ground truth data for an image (image, mask, bounding boxes). augment: (deprecated. Use augmentation instead). If true, apply random image augmentation. Currently, only horizontal flipping is offered. augmentation: Optional. An imgaug (https://github.com/aleju/imgaug) augmentation. For example, passing imgaug.augmenters.Fliplr(0.5) flips images right/left 50% of the time. Returns: image: [height, width, 3] shape: the original shape of the image before resizing and cropping. class_ids: [instance_count] Integer class IDs bbox: [instance_count, (y1, x1, y2, x2)] """ # Load image image = dataset.load_image(image_id) mask, class_ids = dataset.load_mask(image_id) original_shape = image.shape image, window, scale, padding, crop = utils.resize_image( image, min_dim=config.IMAGE_MIN_DIM, min_scale=config.IMAGE_MIN_SCALE, max_dim=config.IMAGE_MAX_DIM, mode=config.IMAGE_RESIZE_MODE) mask = utils.resize_mask(mask, scale, padding, crop) # print("Image size RESIZE: ", image.shape) # Augmentation # This requires the imgaug lib (https://github.com/aleju/imgaug) if augmentation: import imgaug # Augmenters that are safe to apply to masks # Some, such as Affine, have settings that make them unsafe, so always # test your augmentation on masks MASK_AUGMENTERS = [ "Sequential", "SomeOf", "OneOf", "Sometimes", "Fliplr", "Flipud", "CropAndPad", "Affine", "PiecewiseAffine" ] def hook(images, augmenter, parents, default): """Determines which augmenters to apply to masks.""" return augmenter.__class__.__name__ in MASK_AUGMENTERS # Store shapes before augmentation to compare image_shape = image.shape mask_shape = mask.shape # Make augmenters deterministic to apply similarly to images and masks det = augmentation.to_deterministic() image = det.augment_image(image) # Change mask to np.uint8 because imgaug doesn't support np.bool mask = det.augment_image(mask.astype(np.uint8), hooks=imgaug.HooksImages(activator=hook)) # Verify that shapes didn't change assert image.shape == image_shape, "Augmentation shouldn't change image size" assert mask.shape == mask_shape, "Augmentation shouldn't change mask size" # Change mask back to bool mask = mask.astype(np.bool) # Note that some boxes might be all zeros if the corresponding mask got cropped out. # and here is to filter them out _idx = np.sum(mask, axis=(0, 1)) > 0 mask = mask[:, :, _idx] class_ids = class_ids[_idx] # Bounding boxes. Note that some boxes might be all zeros # if the corresponding mask got cropped out. # bbox: [num_instances, (y1, x1, y2, x2)] bbox = utils.extract_bboxes(mask) # Active classes active_class_ids = np.ones([dataset.num_classes], dtype=np.int32) # Image meta data image_meta = model_utils.compose_image_meta(image_id, original_shape, image.shape, window, scale, active_class_ids) return image, image_meta, class_ids, bbox