def load_inference_data_obj_feat_gt(dataset, image_id, config):
image = dataset.load_image(image_id)
gt_ranks, sel_not_sal_obj_idx_list, shuffled_indices, chosen_obj_idx_order_list = dataset.load_gt_rank_order(
image_id)
object_roi_masks = dataset.load_object_roi_masks(image_id,
sel_not_sal_obj_idx_list)
original_shape = image.shape
image, window, scale, padding, crop = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
min_scale=config.IMAGE_MIN_SCALE,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
obj_mask = utils.resize_mask(object_roi_masks, scale, padding, crop)
# bbox: [num_instances, (y1, x1, y2, x2)]
obj_bbox = utils.extract_bboxes(obj_mask)
# *********************** FILL REST, SHUFFLE ORDER ***********************
# order is in salient objects then non-salient objects
batch_obj_roi = np.zeros(shape=(config.SAL_OBJ_NUM, 4), dtype=np.int32)
for i in range(len(chosen_obj_idx_order_list)):
_idx = chosen_obj_idx_order_list[i]
batch_obj_roi[_idx] = obj_bbox[i]
# Normalize image
image = model_utils.mold_image(image.astype(np.float32), config)
# Active classes
active_class_ids = np.ones([config.NUM_CLASSES], dtype=np.int32)
img_id = image_id
img_id = int(img_id[-12:])
# Image meta data
image_meta = model_utils.compose_image_meta(img_id, original_shape,
image.shape, window, scale,
active_class_ids)
# Expand input dimensions to consider batch
image = np.expand_dims(image, axis=0)
image_meta = np.expand_dims(image_meta, axis=0)
batch_obj_roi = np.expand_dims(batch_obj_roi, axis=0)
return [
image, image_meta, batch_obj_roi
], gt_ranks, sel_not_sal_obj_idx_list, shuffled_indices, chosen_obj_idx_order_list
def load_inference_data_obj_feat(dataset, image_id, config):
image = dataset.load_image(image_id)
object_roi_masks = dataset.load_object_roi_masks(image_id)
original_shape = image.shape
image, window, scale, padding, crop = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
min_scale=config.IMAGE_MIN_SCALE,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
obj_mask = utils.resize_mask(object_roi_masks, scale, padding, crop)
# bbox: [num_instances, (y1, x1, y2, x2)]
obj_bbox = utils.extract_bboxes(obj_mask)
# Normalize image
image = model_utils.mold_image(image.astype(np.float32), config)
# Active classes
active_class_ids = np.ones([config.NUM_CLASSES], dtype=np.int32)
img_id = image_id
img_id = int(img_id[-12:])
# Image meta data
image_meta = model_utils.compose_image_meta(img_id, original_shape,
image.shape, window, scale,
active_class_ids)
# Expand input dimensions to consider batch
image = np.expand_dims(image, axis=0)
image_meta = np.expand_dims(image_meta, axis=0)
batch_obj_roi = np.zeros(shape=(config.SAL_OBJ_NUM, 4), dtype=np.int32)
batch_obj_roi[:len(obj_bbox)] = obj_bbox
batch_obj_roi = np.expand_dims(batch_obj_roi, axis=0)
return [image, image_meta, batch_obj_roi]
def mold_inputs(self, images):
"""Takes a list of images and modifies them to the format expected
as an input to the neural network.
images: List of image matrices [height,width,depth]. Images can have
different sizes.
Returns 3 Numpy matrices:
molded_images: [N, h, w, 3]. Images resized and normalized.
image_metas: [N, length of meta data]. Details about each image.
windows: [N, (y1, x1, y2, x2)]. The portion of the image that has the
original image (padding excluded).
"""
molded_images = []
image_metas = []
windows = []
for image in images:
# Resize image
# TODO: move resizing to mold_image()
molded_image, window, scale, padding, crop = utils.resize_image(
image,
min_dim=self.config.IMAGE_MIN_DIM,
min_scale=self.config.IMAGE_MIN_SCALE,
max_dim=self.config.IMAGE_MAX_DIM,
mode=self.config.IMAGE_RESIZE_MODE)
molded_image = model_utils.mold_image(molded_image, self.config)
# Build image_meta
image_meta = model_utils.compose_image_meta(
0, image.shape, molded_image.shape, window, scale,
np.zeros([self.config.NUM_CLASSES], dtype=np.int32))
# Append
molded_images.append(molded_image)
windows.append(window)
image_metas.append(image_meta)
# Pack into arrays
molded_images = np.stack(molded_images)
image_metas = np.stack(image_metas)
windows = np.stack(windows)
return molded_images, image_metas, windows
def data_generator(dataset,
config,
shuffle=True,
augmentation=None,
random_rois=0,
batch_size=1,
detection_targets=False):
b = 0
image_index = -1
image_ids = np.copy(dataset.img_ids)
error_count = 0
# Anchors
# [anchor_count, (y1, x1, y2, x2)]
backbone_shapes = model_utils.compute_backbone_shapes(
config, config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Keras requires a generator to run indefinitely.
while True:
try:
# Increment index to pick next image. Shuffle if at the start of an epoch.
image_index = (image_index + 1) % len(image_ids)
if shuffle and image_index == 0:
np.random.shuffle(image_ids)
# Get GT bounding boxes and masks for image.
image_id = image_ids[image_index]
image, image_meta, gt_class_ids, gt_boxes = \
load_image_gt(dataset, config, image_id, augmentation=augmentation)
if not np.any(gt_class_ids > 0):
continue
# RPN Targets
rpn_match, rpn_bbox = build_rpn_targets(image.shape, anchors,
gt_class_ids, gt_boxes,
config)
# Feature Pyramid Network Targets
if random_rois:
rpn_rois = generate_random_rois(image.shape, random_rois,
gt_class_ids, gt_boxes)
if detection_targets:
rois, fpn_class_ids, fpn_bbox = build_detection_targets(
rpn_rois, gt_class_ids, gt_boxes, config)
# Init batch arrays
if b == 0:
batch_image_meta = np.zeros((batch_size, ) + image_meta.shape,
dtype=image_meta.dtype)
batch_rpn_match = np.zeros([batch_size, anchors.shape[0], 1],
dtype=rpn_match.dtype)
batch_rpn_bbox = np.zeros(
[batch_size, config.RPN_TRAIN_ANCHORS_PER_IMAGE, 4],
dtype=rpn_bbox.dtype)
batch_images = np.zeros((batch_size, ) + image.shape,
dtype=np.float32)
batch_gt_class_ids = np.zeros(
(batch_size, config.MAX_GT_INSTANCES), dtype=np.int32)
batch_gt_boxes = np.zeros(
(batch_size, config.MAX_GT_INSTANCES, 4), dtype=np.int32)
if random_rois:
batch_rpn_rois = np.zeros(
(batch_size, rpn_rois.shape[0], 4),
dtype=rpn_rois.dtype)
if detection_targets:
batch_rois = np.zeros((batch_size, ) + rois.shape,
dtype=rois.dtype)
batch_fpn_class_ids = np.zeros(
(batch_size, ) + fpn_class_ids.shape,
dtype=fpn_class_ids.dtype)
batch_fpn_bbox = np.zeros(
(batch_size, ) + fpn_bbox.shape,
dtype=fpn_bbox.dtype)
# If more instances than fits in the array, sub-sample from them.
if gt_boxes.shape[0] > config.MAX_GT_INSTANCES:
ids = np.random.choice(np.arange(gt_boxes.shape[0]),
config.MAX_GT_INSTANCES,
replace=False)
gt_class_ids = gt_class_ids[ids]
gt_boxes = gt_boxes[ids]
# Add to batch
batch_image_meta[b] = image_meta
batch_rpn_match[b] = rpn_match[:, np.newaxis]
batch_rpn_bbox[b] = rpn_bbox
batch_images[b] = model_utils.mold_image(image.astype(np.float32),
config)
batch_gt_class_ids[b, :gt_class_ids.shape[0]] = gt_class_ids
batch_gt_boxes[b, :gt_boxes.shape[0]] = gt_boxes
if random_rois:
batch_rpn_rois[b] = rpn_rois
if detection_targets:
batch_rois[b] = rois
batch_fpn_class_ids[b] = fpn_class_ids
batch_fpn_bbox[b] = fpn_bbox
b += 1
# Batch full?
if b >= batch_size:
inputs = [
batch_images, batch_image_meta, batch_rpn_match,
batch_rpn_bbox, batch_gt_class_ids, batch_gt_boxes
]
outputs = []
if random_rois:
inputs.extend([batch_rpn_rois])
if detection_targets:
inputs.extend([batch_rois])
# Keras requires that output and targets have the same number of dimensions
batch_fpn_class_ids = np.expand_dims(
batch_fpn_class_ids, -1)
outputs.extend([batch_fpn_class_ids, batch_fpn_bbox])
yield inputs, outputs
# start a new batch
b = 0
except (GeneratorExit, KeyboardInterrupt):
raise
except:
# Log it and skip the image
logging.exception("Error processing image {}".format(
dataset.get_image_info(image_id)))
error_count += 1
if error_count > 5:
raise
def data_generator(dataset, config, shuffle=True, augmentation=None, batch_size=1):
b = 0
image_index = -1
image_ids = np.copy(dataset.img_ids)
error_count = 0
# Anchors
# [anchor_count, (y1, x1, y2, x2)]
backbone_shapes = model_utils.compute_backbone_shapes(config, config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Keras requires a generator to run indefinitely.
while True:
try:
# Increment index to pick next image. Shuffle if at the start of an epoch.
image_index = (image_index + 1) % len(image_ids)
if shuffle and image_index == 0:
np.random.shuffle(image_ids)
image_id = image_ids[image_index]
# Get data
image, image_meta, gt_class_ids, gt_boxes, gt_masks = \
load_image_gt(dataset, config, image_id, augmentation=augmentation,
use_mini_mask=config.USE_MINI_MASK)
if not np.any(gt_class_ids > 0):
continue
# RPN Targets
rpn_match, rpn_bbox = build_rpn_targets(image.shape, anchors,
gt_class_ids, gt_boxes, config)
# Init batch arrays
if b == 0:
batch_image_meta = np.zeros(
(batch_size,) + image_meta.shape, dtype=image_meta.dtype)
batch_rpn_match = np.zeros(
[batch_size, anchors.shape[0], 1], dtype=rpn_match.dtype)
batch_rpn_bbox = np.zeros(
[batch_size, config.RPN_TRAIN_ANCHORS_PER_IMAGE, 4], dtype=rpn_bbox.dtype)
batch_images = np.zeros(
(batch_size,) + image.shape, dtype=np.float32)
batch_gt_class_ids = np.zeros(
(batch_size, config.MAX_GT_INSTANCES), dtype=np.int32)
batch_gt_boxes = np.zeros(
(batch_size, config.MAX_GT_INSTANCES, 4), dtype=np.int32)
batch_gt_masks = np.zeros(
(batch_size, gt_masks.shape[0], gt_masks.shape[1],
config.MAX_GT_INSTANCES), dtype=gt_masks.dtype)
# Add to batch
batch_image_meta[b] = image_meta
batch_rpn_match[b] = rpn_match[:, np.newaxis]
batch_rpn_bbox[b] = rpn_bbox
batch_images[b] = model_utils.mold_image(image.astype(np.float32), config)
batch_gt_class_ids[b, :gt_class_ids.shape[0]] = gt_class_ids
batch_gt_boxes[b, :gt_boxes.shape[0]] = gt_boxes
batch_gt_masks[b, :, :, :gt_masks.shape[-1]] = gt_masks
b += 1
# Batch full?
if b >= batch_size:
inputs = [batch_images, batch_image_meta, batch_rpn_match, batch_rpn_bbox,
batch_gt_class_ids, batch_gt_boxes, batch_gt_masks]
outputs = []
yield inputs, outputs
# start a new batch
b = 0
except(GeneratorExit, KeyboardInterrupt):
raise
except:
# Log it and skip the image
logging.exception("Error processing image {}".format(
dataset.get_image_info(image_id)))
error_count += 1
if error_count > 5:
raise