def load_inference_data_obj_feat_gt(dataset, image_id, config):
image = dataset.load_image(image_id)
gt_ranks, sel_not_sal_obj_idx_list, shuffled_indices, chosen_obj_idx_order_list = dataset.load_gt_rank_order(
image_id)
object_roi_masks = dataset.load_object_roi_masks(image_id,
sel_not_sal_obj_idx_list)
original_shape = image.shape
image, window, scale, padding, crop = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
min_scale=config.IMAGE_MIN_SCALE,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
obj_mask = utils.resize_mask(object_roi_masks, scale, padding, crop)
# bbox: [num_instances, (y1, x1, y2, x2)]
obj_bbox = utils.extract_bboxes(obj_mask)
# *********************** FILL REST, SHUFFLE ORDER ***********************
# order is in salient objects then non-salient objects
batch_obj_roi = np.zeros(shape=(config.SAL_OBJ_NUM, 4), dtype=np.int32)
for i in range(len(chosen_obj_idx_order_list)):
_idx = chosen_obj_idx_order_list[i]
batch_obj_roi[_idx] = obj_bbox[i]
# Normalize image
image = model_utils.mold_image(image.astype(np.float32), config)
# Active classes
active_class_ids = np.ones([config.NUM_CLASSES], dtype=np.int32)
img_id = image_id
img_id = int(img_id[-12:])
# Image meta data
image_meta = model_utils.compose_image_meta(img_id, original_shape,
image.shape, window, scale,
active_class_ids)
# Expand input dimensions to consider batch
image = np.expand_dims(image, axis=0)
image_meta = np.expand_dims(image_meta, axis=0)
batch_obj_roi = np.expand_dims(batch_obj_roi, axis=0)
return [
image, image_meta, batch_obj_roi
], gt_ranks, sel_not_sal_obj_idx_list, shuffled_indices, chosen_obj_idx_order_list
def load_inference_data_obj_feat(dataset, image_id, config):
image = dataset.load_image(image_id)
object_roi_masks = dataset.load_object_roi_masks(image_id)
original_shape = image.shape
image, window, scale, padding, crop = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
min_scale=config.IMAGE_MIN_SCALE,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
obj_mask = utils.resize_mask(object_roi_masks, scale, padding, crop)
# bbox: [num_instances, (y1, x1, y2, x2)]
obj_bbox = utils.extract_bboxes(obj_mask)
# Normalize image
image = model_utils.mold_image(image.astype(np.float32), config)
# Active classes
active_class_ids = np.ones([config.NUM_CLASSES], dtype=np.int32)
img_id = image_id
img_id = int(img_id[-12:])
# Image meta data
image_meta = model_utils.compose_image_meta(img_id, original_shape,
image.shape, window, scale,
active_class_ids)
# Expand input dimensions to consider batch
image = np.expand_dims(image, axis=0)
image_meta = np.expand_dims(image_meta, axis=0)
batch_obj_roi = np.zeros(shape=(config.SAL_OBJ_NUM, 4), dtype=np.int32)
batch_obj_roi[:len(obj_bbox)] = obj_bbox
batch_obj_roi = np.expand_dims(batch_obj_roi, axis=0)
return [image, image_meta, batch_obj_roi]
def mold_inputs(self, images):
"""Takes a list of images and modifies them to the format expected
as an input to the neural network.
images: List of image matrices [height,width,depth]. Images can have
different sizes.
Returns 3 Numpy matrices:
molded_images: [N, h, w, 3]. Images resized and normalized.
image_metas: [N, length of meta data]. Details about each image.
windows: [N, (y1, x1, y2, x2)]. The portion of the image that has the
original image (padding excluded).
"""
molded_images = []
image_metas = []
windows = []
for image in images:
# Resize image
# TODO: move resizing to mold_image()
molded_image, window, scale, padding, crop = utils.resize_image(
image,
min_dim=self.config.IMAGE_MIN_DIM,
min_scale=self.config.IMAGE_MIN_SCALE,
max_dim=self.config.IMAGE_MAX_DIM,
mode=self.config.IMAGE_RESIZE_MODE)
molded_image = model_utils.mold_image(molded_image, self.config)
# Build image_meta
image_meta = model_utils.compose_image_meta(
0, image.shape, molded_image.shape, window, scale,
np.zeros([self.config.NUM_CLASSES], dtype=np.int32))
# Append
molded_images.append(molded_image)
windows.append(window)
image_metas.append(image_meta)
# Pack into arrays
molded_images = np.stack(molded_images)
image_metas = np.stack(image_metas)
windows = np.stack(windows)
return molded_images, image_metas, windows
def load_image_gt(dataset, config, image_id, augmentation=None):
"""Load and return ground truth data for an image (image, mask, bounding boxes).
augment: (deprecated. Use augmentation instead). If true, apply random
image augmentation. Currently, only horizontal flipping is offered.
augmentation: Optional. An imgaug (https://github.com/aleju/imgaug) augmentation.
For example, passing imgaug.augmenters.Fliplr(0.5) flips images
right/left 50% of the time.
Returns:
image: [height, width, 3]
shape: the original shape of the image before resizing and cropping.
class_ids: [instance_count] Integer class IDs
bbox: [instance_count, (y1, x1, y2, x2)]
"""
# Load image
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
original_shape = image.shape
image, window, scale, padding, crop = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
min_scale=config.IMAGE_MIN_SCALE,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
mask = utils.resize_mask(mask, scale, padding, crop)
# print("Image size RESIZE: ", image.shape)
# Augmentation
# This requires the imgaug lib (https://github.com/aleju/imgaug)
if augmentation:
import imgaug
# Augmenters that are safe to apply to masks
# Some, such as Affine, have settings that make them unsafe, so always
# test your augmentation on masks
MASK_AUGMENTERS = [
"Sequential", "SomeOf", "OneOf", "Sometimes", "Fliplr", "Flipud",
"CropAndPad", "Affine", "PiecewiseAffine"
]
def hook(images, augmenter, parents, default):
"""Determines which augmenters to apply to masks."""
return augmenter.__class__.__name__ in MASK_AUGMENTERS
# Store shapes before augmentation to compare
image_shape = image.shape
mask_shape = mask.shape
# Make augmenters deterministic to apply similarly to images and masks
det = augmentation.to_deterministic()
image = det.augment_image(image)
# Change mask to np.uint8 because imgaug doesn't support np.bool
mask = det.augment_image(mask.astype(np.uint8),
hooks=imgaug.HooksImages(activator=hook))
# Verify that shapes didn't change
assert image.shape == image_shape, "Augmentation shouldn't change image size"
assert mask.shape == mask_shape, "Augmentation shouldn't change mask size"
# Change mask back to bool
mask = mask.astype(np.bool)
# Note that some boxes might be all zeros if the corresponding mask got cropped out.
# and here is to filter them out
_idx = np.sum(mask, axis=(0, 1)) > 0
mask = mask[:, :, _idx]
class_ids = class_ids[_idx]
# Bounding boxes. Note that some boxes might be all zeros
# if the corresponding mask got cropped out.
# bbox: [num_instances, (y1, x1, y2, x2)]
bbox = utils.extract_bboxes(mask)
# Active classes
active_class_ids = np.ones([dataset.num_classes], dtype=np.int32)
# Image meta data
image_meta = model_utils.compose_image_meta(image_id, original_shape,
image.shape, window, scale,
active_class_ids)
return image, image_meta, class_ids, bbox