def get_one_target(category,
dataset,
config,
augmentation=None,
target_size_limit=0,
max_attempts=10,
return_all=False,
return_original_size=False):
np.random.seed(2)
n_attempts = 0
while True:
# Get index with corresponding images for each category
category_image_index = dataset.category_image_index
# Draw a random image
random_image_id = np.random.choice(category_image_index[category])
# Load image
target_image, target_image_meta, target_class_ids, target_boxes, target_masks = \
modellib.load_image_gt(dataset, config, random_image_id, augmentation=augmentation,
use_mini_mask=config.USE_MINI_MASK)
# print(random_image_id, category, target_class_ids)
if not np.any(target_class_ids == category):
continue
# try:
# box_ind = np.random.choice(np.where(target_class_ids == category)[0])
# except ValueError:
# return None
box_ind = np.random.choice(np.where(target_class_ids == category)[0])
tb = target_boxes[box_ind, :]
target = target_image[tb[0]:tb[2], tb[1]:tb[3], :]
original_size = target.shape
target, window, scale, padding, crop = utils.resize_image(
target,
min_dim=config.TARGET_MIN_DIM,
min_scale=config.IMAGE_MIN_SCALE, #Same scaling as the image
max_dim=config.TARGET_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE) #Same output format as the image
n_attempts = n_attempts + 1
if (min(original_size[:2]) >= target_size_limit) or (n_attempts >=
max_attempts):
break
if return_all:
return target, window, scale, padding, crop
elif return_original_size:
return target, original_size
else:
return target
def siamese_data_generator(dataset,
config,
shuffle=True,
augmentation=imgaug.augmenters.Fliplr(0.5),
random_rois=0,
batch_size=1,
detection_targets=False,
diverse=0):
"""A generator that returns images and corresponding target class ids,
bounding box deltas, and masks.
dataset: The Dataset object to pick data from
config: The model config object
shuffle: If True, shuffles the samples before every epoch
augment: If True, applies image augmentation to images (currently only
horizontal flips are supported)
random_rois: If > 0 then generate proposals to be used to train the
network classifier and mask heads. Useful if training
the Mask RCNN part without the RPN.
batch_size: How many images to return in each call
detection_targets: If True, generate detection targets (class IDs, bbox
deltas, and masks). Typically for debugging or visualizations because
in trainig detection targets are generated by DetectionTargetLayer.
diverse: Float in [0,1] indicatiing probability to draw a target
from any random class instead of one from the image classes
Returns a Python generator. Upon calling next() on it, the
generator returns two lists, inputs and outputs. The containtes
of the lists differs depending on the received arguments:
inputs list:
- images: [batch, H, W, C]
- image_meta: [batch, size of image meta]
- rpn_match: [batch, N] Integer (1=positive anchor, -1=negative, 0=neutral)
- rpn_bbox: [batch, N, (dy, dx, log(dh), log(dw))] Anchor bbox deltas.
- gt_class_ids: [batch, MAX_GT_INSTANCES] Integer class IDs
- gt_boxes: [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)]
- gt_masks: [batch, height, width, MAX_GT_INSTANCES]. The height and width
are those of the image unless use_mini_mask is True, in which
case they are defined in MINI_MASK_SHAPE.
outputs list: Usually empty in regular training. But if detection_targets
is True then the outputs list contains target class_ids, bbox deltas,
and masks.
"""
b = 0 # batch item index
image_index = -1
image_ids = np.copy(dataset.image_ids)
error_count = 0
# Anchors
# [anchor_count, (y1, x1, y2, x2)]
backbone_shapes = modellib.compute_backbone_shapes(config,
config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Keras requires a generator to run indefinately.
while True:
try:
# Increment index to pick next image. Shuffle if at the start of an epoch.
image_index = (image_index + 1) % len(image_ids)
if shuffle and image_index == 0:
np.random.shuffle(image_ids)
# Get GT bounding boxes and masks for image.
image_id = image_ids[image_index]
image, image_meta, gt_class_ids, gt_boxes, gt_masks = \
modellib.load_image_gt(dataset, config, image_id, augmentation=augmentation,
use_mini_mask=config.USE_MINI_MASK)
# Replace class ids with foreground/background info if binary
# class option is chosen
# if binary_classes == True:
# gt_class_ids = np.minimum(gt_class_ids, 1)
# Skip images that have no instances. This can happen in cases
# where we train on a subset of classes and the image doesn't
# have any of the classes we care about.
if not np.any(gt_class_ids > 0):
continue
# print(gt_class_ids)
# Use only positive class_ids
categories = np.unique(gt_class_ids)
_idx = categories > 0
categories = categories[_idx]
# Use only active classes
active_categories = []
for c in categories:
if any(c == dataset.ACTIVE_CLASSES):
active_categories.append(c)
# Skiop image if it contains no instance of any active class
if not np.any(np.array(active_categories) > 0):
continue
# Randomly select category
category = np.random.choice(active_categories)
# Generate siamese target crop
if not config.NUM_TARGETS:
config.NUM_TARGETS = 1
targets = []
for i in range(config.NUM_TARGETS):
targets.append(
get_one_target(category,
dataset,
config,
augmentation=augmentation))
# target = np.stack(target, axis=0)
# print(target_class_id)
target_class_id = category
target_class_ids = np.array([target_class_id])
idx = gt_class_ids == target_class_id
siamese_class_ids = idx.astype('int8')
# print(idx)
# print(gt_boxes.shape, gt_masks.shape)
siamese_class_ids = siamese_class_ids[idx]
gt_class_ids = gt_class_ids[idx]
gt_boxes = gt_boxes[idx, :]
gt_masks = gt_masks[:, :, idx]
image_meta = image_meta[:14]
# print(gt_boxes.shape, gt_masks.shape)
# RPN Targets
rpn_match, rpn_bbox = modellib.build_rpn_targets(
image.shape, anchors, gt_class_ids, gt_boxes, config)
# Mask R-CNN Targets
if random_rois:
rpn_rois = modellib.generate_random_rois(
image.shape, random_rois, gt_class_ids, gt_boxes)
if detection_targets:
rois, mrcnn_class_ids, mrcnn_bbox, mrcnn_mask =\
modellib.build_detection_targets(
rpn_rois, gt_class_ids, gt_boxes, gt_masks, config)
# Init batch arrays
if b == 0:
batch_image_meta = np.zeros((batch_size, ) + image_meta.shape,
dtype=image_meta.dtype)
batch_rpn_match = np.zeros([batch_size, anchors.shape[0], 1],
dtype=rpn_match.dtype)
batch_rpn_bbox = np.zeros(
[batch_size, config.RPN_TRAIN_ANCHORS_PER_IMAGE, 4],
dtype=rpn_bbox.dtype)
batch_images = np.zeros((batch_size, ) + image.shape,
dtype=np.float32)
batch_gt_class_ids = np.zeros(
(batch_size, config.MAX_GT_INSTANCES), dtype=np.int32)
batch_gt_boxes = np.zeros(
(batch_size, config.MAX_GT_INSTANCES, 4), dtype=np.int32)
batch_targets = np.zeros(
(batch_size, config.NUM_TARGETS) + targets[0].shape,
dtype=np.float32)
# batch_target_class_ids = np.zeros(
# (batch_size, config.MAX_TARGET_INSTANCES), dtype=np.int32)
if config.USE_MINI_MASK:
batch_gt_masks = np.zeros(
(batch_size, config.MINI_MASK_SHAPE[0],
config.MINI_MASK_SHAPE[1], config.MAX_GT_INSTANCES))
else:
batch_gt_masks = np.zeros(
(batch_size, image.shape[0], image.shape[1],
config.MAX_GT_INSTANCES))
if random_rois:
batch_rpn_rois = np.zeros(
(batch_size, rpn_rois.shape[0], 4),
dtype=rpn_rois.dtype)
if detection_targets:
batch_rois = np.zeros((batch_size, ) + rois.shape,
dtype=rois.dtype)
batch_mrcnn_class_ids = np.zeros(
(batch_size, ) + mrcnn_class_ids.shape,
dtype=mrcnn_class_ids.dtype)
batch_mrcnn_bbox = np.zeros(
(batch_size, ) + mrcnn_bbox.shape,
dtype=mrcnn_bbox.dtype)
batch_mrcnn_mask = np.zeros(
(batch_size, ) + mrcnn_mask.shape,
dtype=mrcnn_mask.dtype)
# If more instances than fits in the array, sub-sample from them.
if gt_boxes.shape[0] > config.MAX_GT_INSTANCES:
ids = np.random.choice(np.arange(gt_boxes.shape[0]),
config.MAX_GT_INSTANCES,
replace=False)
gt_class_ids = gt_class_ids[ids]
siamese_class_ids = siamese_class_ids[ids]
gt_boxes = gt_boxes[ids]
gt_masks = gt_masks[:, :, ids]
# Add to batch
batch_image_meta[b] = image_meta
batch_rpn_match[b] = rpn_match[:, np.newaxis]
batch_rpn_bbox[b] = rpn_bbox
batch_images[b] = modellib.mold_image(image.astype(np.float32),
config)
batch_targets[b] = np.stack([
modellib.mold_image(target.astype(np.float32), config)
for target in targets
],
axis=0)
batch_gt_class_ids[
b, :siamese_class_ids.shape[0]] = siamese_class_ids
# batch_target_class_ids[b, :target_class_ids.shape[0]] = target_class_ids
batch_gt_boxes[b, :gt_boxes.shape[0]] = gt_boxes
batch_gt_masks[b, :, :, :gt_masks.shape[-1]] = gt_masks
if random_rois:
batch_rpn_rois[b] = rpn_rois
if detection_targets:
batch_rois[b] = rois
batch_mrcnn_class_ids[b] = mrcnn_class_ids
batch_mrcnn_bbox[b] = mrcnn_bbox
batch_mrcnn_mask[b] = mrcnn_mask
b += 1
# Batch full?
if b >= batch_size:
inputs = [
batch_images, batch_image_meta, batch_targets,
batch_rpn_match, batch_rpn_bbox, batch_gt_class_ids,
batch_gt_boxes, batch_gt_masks
]
outputs = []
if random_rois:
inputs.extend([batch_rpn_rois])
if detection_targets:
inputs.extend([batch_rois])
# Keras requires that output and targets have the same number of dimensions
batch_mrcnn_class_ids = np.expand_dims(
batch_mrcnn_class_ids, -1)
outputs.extend([
batch_mrcnn_class_ids, batch_mrcnn_bbox,
batch_mrcnn_mask
])
yield inputs, outputs
# start a new batch
b = 0
except (GeneratorExit, KeyboardInterrupt):
raise
except:
# Log it and skip the image
modellib.logging.exception("Error processing image {}".format(
dataset.image_info[image_id]))
error_count += 1
if error_count > 5:
raise
def evaluate_dataset(model,
dataset,
dataset_object,
eval_type="bbox",
dataset_type='coco',
limit=0,
image_ids=None,
class_index=None,
verbose=1,
random_detections=False,
return_results=False):
"""Runs official COCO evaluation.
dataset: A Dataset object with valiadtion data
eval_type: "bbox" or "segm" for bounding box or segmentation evaluation
limit: if not 0, it's the number of images to use for evaluation
"""
assert dataset_type in ['coco']
# Pick COCO images from the dataset
image_ids = image_ids or dataset.image_ids
# Limit to a subset
if limit:
image_ids = image_ids[:limit]
print(image_ids)
# Get corresponding COCO image IDs.
dataset_image_ids = [dataset.image_info[id]["id"] for id in image_ids]
t_prediction = 0
t_start = time.time()
results = []
for i, image_id in enumerate(image_ids):
if i % 100 == 0 and verbose > 1:
print("Processing image {}/{} ...".format(i, len(image_ids)))
# Load GT data
_, _, gt_class_ids, _, _ = modellib.load_image_gt(
dataset,
model.config,
image_id,
augmentation=False,
use_mini_mask=model.config.USE_MINI_MASK)
# BOILERPLATE: Code duplicated in siamese_data_loader
# Skip images that have no instances. This can happen in cases
# where we train on a subset of classes and the image doesn't
# have any of the classes we care about.
if not np.any(gt_class_ids > 0):
continue
# Use only positive class_ids
categories = np.unique(gt_class_ids)
_idx = categories > 0
categories = categories[_idx]
# Use only active classes
active_categories = []
for c in categories:
if any(c == dataset.ACTIVE_CLASSES):
active_categories.append(c)
# Skiop image if it contains no instance of any active class
if not np.any(np.array(active_categories) > 0):
continue
# END BOILERPLATE
# Evaluate for every category individually
for category in active_categories:
# Load image
image = dataset.load_image(image_id)
# Draw random target
target = []
for k in range(model.config.NUM_TARGETS):
try:
target.append(
get_one_target(category, dataset, model.config))
except:
print('error fetching target of category', category)
continue
target = np.stack(target, axis=0)
# Run detection
t = time.time()
try:
r = model.detect([target], [image],
verbose=0,
random_detections=random_detections)[0]
except:
print('error running detection for category', category)
continue
t_prediction += (time.time() - t)
# Format detections
r["class_ids"] = np.array(
[category for i in range(r["class_ids"].shape[0])])
# Convert results to COCO format
# Cast masks to uint8 because COCO tools errors out on bool
if dataset_type == 'coco':
image_results = coco.build_coco_results(
dataset, dataset_image_ids[i:i + 1], r["rois"],
r["class_ids"], r["scores"], r["masks"].astype(np.uint8))
results.extend(image_results)
# Load results. This modifies results with additional attributes.
dataset_results = dataset_object.loadRes(results)
# allow evaluating bbox & segm:
if not isinstance(eval_type, (list, )):
eval_type = [eval_type]
for current_eval_type in eval_type:
# Evaluate
cocoEval = customCOCOeval(dataset_object, dataset_results,
current_eval_type)
cocoEval.params.imgIds = dataset_image_ids
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize(class_index=class_index, verbose=verbose)
if verbose > 0:
print("Prediction time: {}. Average {}/image".format(
t_prediction, t_prediction / len(image_ids)))
print("Total time: ", time.time() - t_start)
print(cocoEval.stats)
if return_results:
return cocoEval