def generate_anchors():
# Generate Anchors
# np.array([[256,256],[128,128],[64,64],[32,32],[16,16]])
backbone_shapes = modellib.compute_backbone_shapes(config,
config.IMAGE_SHAPE)
# shape 是 (256*256*3+128*128*3+64*64*3+32*32*3+16*16*3,4)=(261888,4)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Print summary of anchors
num_levels = len(backbone_shapes)
anchors_per_cell = len(config.RPN_ANCHOR_RATIOS)
print("Count: ", anchors.shape[0])
print("Scales: ", config.RPN_ANCHOR_SCALES)
print("ratios: ", config.RPN_ANCHOR_RATIOS)
print("Anchors per Cell: ", anchors_per_cell)
print("Levels: ", num_levels)
anchors_per_level = []
for l in range(num_levels):
num_cells = backbone_shapes[l][0] * backbone_shapes[l][1]
anchors_per_level.append(anchors_per_cell * num_cells //
config.RPN_ANCHOR_STRIDE**2)
print("Anchors in Level {}: {}".format(l, anchors_per_level[l]))
return backbone_shapes, anchors, anchors_per_level
def generate_anchors(dnncfg):
'''
## Anchors
The order of anchors is important. Use the same order in training and prediction phases.
And it must match the order of the convolution execution.
For an FPN network, the anchors must be ordered in a way that makes it easy to match anchors
to the output of the convolution layers that predict anchor scores and shifts.
* Sort by pyramid level first. All anchors of the first level, then all of the second and so on.
This makes it easier to separate anchors by level.
* Within each level, sort anchors by feature map processing sequence. Typically, a convolution layer
processes a feature map starting from top-left and moving right row by row.
* For each feature map cell, pick any sorting order for the anchors of different ratios.
Here we match the order of ratios passed to the function.
**Anchor Stride:**
In the FPN architecture, feature maps at the first few layers are high resolution.
For example, if the input image is 1024x1024 then the feature meap of the first layer is 256x256,
which generates about 200K anchors (256*256*3). These anchors are 32x32 pixels and their stride
relative to image pixels is 4 pixels, so there is a lot of overlap. We can reduce the load
significantly if we generate anchors for every other cell in the feature map. A stride of 2 will
cut the number of anchors by 4, for example.
In this implementation we use an anchor stride of 2, which is different from the paper.
'''
log.info("generate_anchors::-------------------------------->")
# Generate Anchors
backbone_shapes = modellib.compute_backbone_shapes(dnncfg, dnncfg.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(dnncfg.RPN_ANCHOR_SCALES,
dnncfg.RPN_ANCHOR_RATIOS,
backbone_shapes,
dnncfg.BACKBONE_STRIDES,
dnncfg.RPN_ANCHOR_STRIDE)
# Print summary of anchors
num_levels = len(backbone_shapes)
anchors_per_cell = len(dnncfg.RPN_ANCHOR_RATIOS)
log.debug("Count: {}".format(anchors.shape[0]))
log.debug("Scales: {}".format(dnncfg.RPN_ANCHOR_SCALES))
log.debug("ratios: {}".format(dnncfg.RPN_ANCHOR_RATIOS))
log.debug("Anchors per Cell: {}".format(anchors_per_cell))
log.debug("Levels: {}".format(num_levels))
anchors_per_level = []
for l in range(num_levels):
num_cells = backbone_shapes[l][0] * backbone_shapes[l][1]
anchors_per_level.append(anchors_per_cell * num_cells // dnncfg.RPN_ANCHOR_STRIDE**2)
log.debug("Anchors in Level {}: {}".format(l, anchors_per_level[l]))
return backbone_shapes, anchors, anchors_per_level, anchors_per_cell
def get_anchors(image_shape, config):
"""Returns anchor pyramid for the given image size."""
backbone_shapes = compute_backbone_shapes(config, image_shape)
# Cache anchors and reuse if image shape is the same
_anchor_cache = {}
if not tuple(image_shape) in _anchor_cache:
# Generate Anchors
a = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Keep a copy of the latest anchors in pixel coordinates because
# it's used in inspect_model notebooks.
# TODO: Remove this after the notebook are refactored to not use it
anchors = a
# Normalize coordinates
_anchor_cache[tuple(image_shape)] = utils.norm_boxes(
a, image_shape[:2])
return _anchor_cache[tuple(image_shape)]
# Add augmentation and mask resizing.
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, augment=True, use_mini_mask=True)
log("mask", mask)
display_images([image]+[mask[:,:,i] for i in range(min(mask.shape[-1], 7))])
mask = utils.expand_mask(bbox, mask, image.shape)
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
# 锚框生成
# Generate Anchors
backbone_shapes = modellib.compute_backbone_shapes(config, config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Print summary of anchors
num_levels = len(backbone_shapes)
anchors_per_cell = len(config.RPN_ANCHOR_RATIOS)
print("Count: ", anchors.shape[0])
print("Scales: ", config.RPN_ANCHOR_SCALES)
print("ratios: ", config.RPN_ANCHOR_RATIOS)
print("Anchors per Cell: ", anchors_per_cell)
print("Levels: ", num_levels)
anchors_per_level = []
for l in range(num_levels):
num_cells = backbone_shapes[l][0] * backbone_shapes[l][1]
anchors_per_level.append(anchors_per_cell * num_cells // config.RPN_ANCHOR_STRIDE**2)
def data_generator(dataset,
config,
shuffle=True,
augment=True,
random_rois=0,
batch_size=1,
detection_targets=False):
'''
A generator that returns images and corresponding target class ids,
bounding box deltas, and masks.
Inputs:
-------
dataset: The Dataset object to pick data from
config: The model config object
shuffle: If True, shuffles the samples before every epoch
augment: If True, applies image augmentation to images (currently only
horizontal flips are supported)
random_rois: If > 0 then generate proposals to be used to train the
network classifier and mask heads. Useful if training
the Mask RCNN part without the RPN.
batch_size: How many images to return in each call
detection_targets: If True, generate detection targets (class IDs, bbox,deltas, and masks).
Typically for debugging or visualizations because in trainig detection
targets are generated by DetectionTargetLayer.
Returns: A Python generator. Upon calling next() on it, the
-------- generator returns two lists, [inputs] and [outputs]. The containtes
of the lists differs depending on the received arguments:
[Inputs] return list:
--------------------
0 batch_images: [batch_sz, H, W, C] [1, 128,128,3]
1 batch_image_meta: [batch_sz, size of image meta] [1, 12]
2 batch_rpn_match: [batch_sz, N] Integer (1=positive anchor, -1=negative, 0=neutral) [1,4092, 1]
3 batch_rpn_bbox: [batch_sz, N, (dy, dx, log(dh), log(dw))] Anchor bbox deltas. [1, 256, 4]
4 batch_gt_class_ids: [batch_sz, MAX_GT_INSTANCES] Integer class IDs [1, 100]
5 batch_gt_boxes: [batch_sz, MAX_GT_INSTANCES, (y1, x1, y2, x2)] [1, 100, 4]
6 batch_gt_masks: [batch_sz, height, width, MAX_GT_INSTANCES]. The height and width [1, 56, 56, 100]
are those of the image unless use_mini_mask is True, in which
case they are defined in MINI_MASK_SHAPE.
>> if random_rois <> 0 , the following is generated by GENERATE_RANDOM_ROIS
batch_rpn_roi [batch_size, #random_rois, (y1, x1, y2, x2)] ROI boxes in pixels.
>> if random_rois <> 0 AND detection_targets == True following generated by BUILD_DETECTION_TARGETS
batch_roi [batch_sz, TRAIN_ROIS_PER_IMAGE, (y1, x1, y2, x2)]
batch_mrcnn_class_ids [batch_sz, TRAIN_ROIS_PER_IMAGE]. Integer class IDs.
batch_mrcnn_bbox [batch_sz, TRAIN_ROIS_PER_IMAGE, NUM_CLASSES, (y, x, log(h), log(w))].
Class-specific bbox refinments.
batch_mrcnn_mask [batch_sz, TRAIN_ROIS_PER_IMAGE, height, width, NUM_CLASSES).
Class specific masks cropped to bbox boundaries and resized to neural
network output size.
[Outputs] : Usually empty in regular training. But if detection_targets
----------- is True then the outputs list contains target class_ids, bbox deltas,
and masks.
Operation outline:
------------------
- generate_pyramid_anchors
- load ground truth information for current image using load_image_gt
- build rpn_targets -- using anchors, class_ids, and bounding boxes
- generate random rois (as rpn_rois instead of using those generated by the RPN network (normally not used)
- at to batch being created
- If batch is complete, build [Inputs] list
'''
b = 0 # batch item index
image_index = -1
image_ids = np.copy(dataset.image_ids)
error_count = 0
# Anchors
# [anchor_count, (y1, x1, y2, x2)]
anchors = utils.generate_pyramid_anchors(
config.RPN_ANCHOR_SCALES, # (8, 16, 32, 64, 128)
config.RPN_ANCHOR_RATIOS, # [0.5, 1, 2]
config.BACKBONE_SHAPES, # [ 4X4, 8X8, 16X16, 32X32, 64X64]
config.BACKBONE_STRIDES, # [ 4, 8, 16, 32, 64]
config.RPN_ANCHOR_STRIDE) # 1
# Keras requires a generator to run indefinately.
while True:
try:
#-----------------------------------------------------------------------
# Increment index to pick next image. Shuffle if at the start of an epoch.
#-----------------------------------------------------------------------
image_index = (image_index + 1) % len(image_ids)
if shuffle and image_index == 0:
np.random.shuffle(image_ids)
#-----------------------------------------------------------------------
# Get GT bounding boxes and masks for image.
#-----------------------------------------------------------------------
image_id = image_ids[image_index]
image, image_meta, \
gt_class_ids, gt_boxes, gt_masks = \
load_image_gt(dataset, config, image_id, augment=augment, use_mini_mask=config.USE_MINI_MASK)
#-----------------------------------------------------------------------
# Skip images that have no instances. This can happen in cases
# where we train on a subset of classes and the image doesn't
# have any of the classes we care about.
#-----------------------------------------------------------------------
if not np.any(gt_class_ids > 0):
continue
#-----------------------------------------------------------------------
# RPN Targets to assist in training Region Proposal Network stage
#-----------------------------------------------------------------------
rpn_match, rpn_bbox = build_rpn_targets(image.shape, anchors,
gt_class_ids, gt_boxes,
config)
#-----------------------------------------------------------------------
# IF random_rois <> 0 then we generate random proposals
# (instead of using those generated by the RPN network)
# Mask R-CNN roi Targets
#-----------------------------------------------------------------------
if random_rois:
rpn_rois = generate_random_rois(image.shape, random_rois,
gt_class_ids, gt_boxes)
if detection_targets:
rois, mrcnn_class_ids, mrcnn_bbox, mrcnn_mask =\
build_detection_targets(rpn_rois, gt_class_ids, gt_boxes, gt_masks, config)
#-----------------------------------------------------------------------
# Init batch arrays
#-----------------------------------------------------------------------
if b == 0:
batch_images = np.zeros((batch_size, ) + image.shape,
dtype=np.float32)
batch_image_meta = np.zeros((batch_size, ) + image_meta.shape,
dtype=image_meta.dtype)
batch_rpn_match = np.zeros([batch_size, anchors.shape[0], 1],
dtype=rpn_match.dtype)
batch_rpn_bbox = np.zeros(
[batch_size, config.RPN_TRAIN_ANCHORS_PER_IMAGE, 4],
dtype=rpn_bbox.dtype)
batch_gt_class_ids = np.zeros(
(batch_size, config.MAX_GT_INSTANCES), dtype=np.int32)
batch_gt_boxes = np.zeros(
(batch_size, config.MAX_GT_INSTANCES, 4), dtype=np.int32)
if config.USE_MINI_MASK:
batch_gt_masks = np.zeros(
(batch_size, config.MINI_MASK_SHAPE[0],
config.MINI_MASK_SHAPE[1], config.MAX_GT_INSTANCES))
else:
batch_gt_masks = np.zeros(
(batch_size, image.shape[0], image.shape[1],
config.MAX_GT_INSTANCES))
if random_rois:
batch_rpn_rois = np.zeros(
(batch_size, rpn_rois.shape[0], 4),
dtype=rpn_rois.dtype)
if detection_targets:
batch_rois = np.zeros((batch_size, ) + rois.shape,
dtype=rois.dtype)
batch_mrcnn_class_ids = np.zeros(
(batch_size, ) + mrcnn_class_ids.shape,
dtype=mrcnn_class_ids.dtype)
batch_mrcnn_bbox = np.zeros(
(batch_size, ) + mrcnn_bbox.shape,
dtype=mrcnn_bbox.dtype)
batch_mrcnn_mask = np.zeros(
(batch_size, ) + mrcnn_mask.shape,
dtype=mrcnn_mask.dtype)
#-----------------------------------------------------------------------
# If more instances than fits in the array, sub-sample from them.
#-----------------------------------------------------------------------
if gt_boxes.shape[0] > config.MAX_GT_INSTANCES:
ids = np.random.choice(np.arange(gt_boxes.shape[0]),
config.MAX_GT_INSTANCES,
replace=False)
gt_class_ids = gt_class_ids[ids]
gt_boxes = gt_boxes[ids]
gt_masks = gt_masks[:, :, ids]
#-----------------------------------------------------------------------
# Add to batch
#-----------------------------------------------------------------------
batch_image_meta[b] = image_meta
batch_rpn_match[b] = rpn_match[:, np.newaxis]
batch_rpn_bbox[b] = rpn_bbox
batch_images[b] = utils.mold_image(image.astype(np.float32),
config)
batch_gt_class_ids[b, :gt_class_ids.shape[0]] = gt_class_ids
batch_gt_boxes[b, :gt_boxes.shape[0]] = gt_boxes
batch_gt_masks[b, :, :, :gt_masks.shape[-1]] = gt_masks
if random_rois:
batch_rpn_rois[b] = rpn_rois
if detection_targets:
batch_rois[b] = rois
batch_mrcnn_class_ids[b] = mrcnn_class_ids
batch_mrcnn_bbox[b] = mrcnn_bbox
batch_mrcnn_mask[b] = mrcnn_mask
b += 1
#-----------------------------------------------------------------------
# Batch full? send out inputs, outputs
#-----------------------------------------------------------------------
if b >= batch_size:
inputs = [
batch_images, batch_image_meta, batch_rpn_match,
batch_rpn_bbox, batch_gt_class_ids, batch_gt_boxes,
batch_gt_masks
]
outputs = []
if random_rois:
inputs.extend([batch_rpn_rois])
if detection_targets:
inputs.extend([batch_rois])
# Keras requires that output and targets have the same number of dimensions
batch_mrcnn_class_ids = np.expand_dims(
batch_mrcnn_class_ids, -1)
outputs.extend([
batch_mrcnn_class_ids, batch_mrcnn_bbox,
batch_mrcnn_mask
])
yield inputs, outputs
# start a new batch
b = 0
except (GeneratorExit, KeyboardInterrupt):
raise
except:
# Log it and skip the image
logging.exception("Error processing image {}".format(
dataset.image_info[image_id]))
error_count += 1
if error_count > 5:
raise
def get_labels(self, labels):
dims = labels.shape
unlabeled_labels = np.zeros((dims[0], dims[1], 1))
building_labels = np.zeros((dims[0], dims[1], 1))
fence_labels = np.zeros((dims[0], dims[1], 1))
other_labels = np.zeros((dims[0], dims[1], 1))
pedestrian_labels = np.zeros((dims[0], dims[1], 1))
pole_labels = np.zeros((dims[0], dims[1], 1))
road_line_labels = np.zeros((dims[0], dims[1], 1))
road_labels = np.zeros((dims[0], dims[1], 1))
sidewalk_labels = np.zeros((dims[0], dims[1], 1))
vegetation_labels = np.zeros((dims[0], dims[1], 1))
car_labels = np.zeros((dims[0], dims[1], 1))
wall_labels = np.zeros((dims[0], dims[1], 1))
traffic_sign_labels = np.zeros((dims[0], dims[1], 1))
unlabeled_index = np.all(labels == (0, 0, 0), axis=-1)
building_index = np.all(labels == (70, 70, 70), axis=-1)
fence_index = np.all(labels == (190, 153, 153), axis=-1)
other_index = np.all(labels == (250, 170, 160), axis=-1)
pedestrian_index = np.all(labels == (220, 20, 60), axis=-1)
pole_index = np.all(labels == (153, 153, 153), axis=-1)
road_line_index = np.all(labels == (157, 234, 50), axis=-1)
road_index = np.all(labels == (128, 64, 128), axis=-1)
sidewalk_index = np.all(labels == (244, 35, 232), axis=-1)
vegetation_index = np.all(labels == (107, 142, 35), axis=-1)
car_index = np.all(labels == (0, 0, 142), axis=-1)
wall_index = np.all(labels == (102, 102, 156), axis=-1)
traffic_sign_index = np.all(labels == (220, 220, 70), axis=-1)
unlabeled_labels[unlabeled_index] = 1
building_labels[building_index] = 10
fence_labels[fence_index] = 10
other_labels[other_index] = 10
pedestrian_labels[pedestrian_index] = 10
pole_labels[pole_index] = 10
road_line_labels[road_line_index] = 10
road_labels[road_index] = 10
sidewalk_labels[sidewalk_index] = 10
vegetation_labels[vegetation_index] = 1
car_labels[car_index] = 10
wall_labels[wall_index] = 10
traffic_sign_labels[traffic_sign_index] = 10
return np.dstack([unlabeled_labels, building_labels, fence_labels,
return np.dstack([unlabeled_labels, building_labels, fence_labels,
other_labels, pedestrian_labels, pole_labels,
road_line_labels, road_labels, sidewalk_labels, vegetation_labels,
car_labels, wall_labels, traffic_sign_labels])
def image_reference(self, image_id):
"""Return the carla data of the image."""
info = self.image_info[image_id]
if info["source"] == "carla":
return info["id"]
else:
super(self.__class__).image_reference(self, image_id)
config = CarlaConfig()
config.STEPS_PER_EPOCH = NUMBER_OF_TRAIN_DATA//config.BATCH_SIZE
config.VALIDATION_STEPS = NUMBER_OF_VAL_DATA//config.BATCH_SIZE
config.display()
dataset = carlaDataset()
dataset.load_images(dir=RGB_TRAIN_DIR, type='train')
# mask, a = train.load_mask(50)
# print(a)
dataset.prepare()
print("Image Count: {}".format(len(dataset.image_ids)))
print("Class Count: {}".format(dataset.num_classes))
for i, info in enumerate(dataset.class_info):
print("{:3}. {:50}".format(i, info['name']))
image_ids = np.random.choice(dataset.image_ids, 4)
for image_id in image_ids:
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids, dataset.class_names)
# Load random image and mask.
image_id = random.choice(dataset.image_ids)
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id ", image_id)
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
# Load random image and mask.
image_id = np.random.choice(dataset.image_ids, 1)[0]
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
original_shape = image.shape
# Resize
image, window, scale, padding, _ = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
mask = utils.resize_mask(mask, scale, padding)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id: ", image_id)
print("Original shape: ", original_shape)
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
log("image", image)
log("image_meta", image_meta)
log("class_ids", class_ids)
log("bbox", bbox)
log("mask", mask)
display_images([image]+[mask[:,:,i] for i in range(min(mask.shape[-1], 7))])
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
# Generate Anchors
backbone_shapes = modellib.compute_backbone_shapes(config, config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Print summary of anchors
num_levels = len(backbone_shapes)
anchors_per_cell = len(config.RPN_ANCHOR_RATIOS)
print("Count: ", anchors.shape[0])
print("Scales: ", config.RPN_ANCHOR_SCALES)
print("ratios: ", config.RPN_ANCHOR_RATIOS)
print("Anchors per Cell: ", anchors_per_cell)
print("Levels: ", num_levels)
anchors_per_level = []
for l in range(num_levels):
num_cells = backbone_shapes[l][0] * backbone_shapes[l][1]
anchors_per_level.append(anchors_per_cell * num_cells // config.RPN_ANCHOR_STRIDE**2)
print("Anchors in Level {}: {}".format(l, anchors_per_level[l]))
## Visualize anchors of one cell at the center of the feature map of a specific level
# Load and draw random image
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, _, _, _ = modellib.load_image_gt(dataset, config, image_id)
fig, ax = plt.subplots(1, figsize=(10, 10))
ax.imshow(image)
levels = len(backbone_shapes)
for level in range(levels):
colors = visualize.random_colors(levels)
# Compute the index of the anchors at the center of the image
level_start = sum(anchors_per_level[:level]) # sum of anchors of previous levels
level_anchors = anchors[level_start:level_start+anchors_per_level[level]]
print("Level {}. Anchors: {:6} Feature map Shape: {}".format(level, level_anchors.shape[0],
backbone_shapes[level]))
center_cell = backbone_shapes[level] // 2
center_cell_index = (center_cell[0] * backbone_shapes[level][1] + center_cell[1])
level_center = center_cell_index * anchors_per_cell
center_anchor = anchors_per_cell * (
(center_cell[0] * backbone_shapes[level][1] / config.RPN_ANCHOR_STRIDE**2) \
+ center_cell[1] / config.RPN_ANCHOR_STRIDE)
level_center = int(center_anchor)
# Draw anchors. Brightness show the order in the array, dark to bright.
for i, rect in enumerate(level_anchors[level_center:level_center+anchors_per_cell]):
y1, x1, y2, x2 = rect
p = patches.Rectangle((x1, y1), x2-x1, y2-y1, linewidth=2, facecolor='none',
edgecolor=(i+1)*np.array(colors[level]) / anchors_per_cell)
ax.add_patch(p)
# Create data generator
random_rois = 4000
g = modellib.data_generator(
dataset, config, shuffle=True, random_rois=random_rois,
batch_size=4,
detection_targets=True)
# Get Next Image
if random_rois:
[normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, rpn_rois, rois], \
[mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g)
log("rois", rois)
log("mrcnn_class_ids", mrcnn_class_ids)
log("mrcnn_bbox", mrcnn_bbox)
log("mrcnn_mask", mrcnn_mask)
else:
[normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes, gt_masks], _ = next(g)
log("gt_class_ids", gt_class_ids)
log("gt_boxes", gt_boxes)
log("gt_masks", gt_masks)
log("rpn_match", rpn_match, )
log("rpn_bbox", rpn_bbox)
image_id = modellib.parse_image_meta(image_meta)["image_id"][0]
print("image_id: ", image_id, dataset.image_reference(image_id))
# Remove the last dim in mrcnn_class_ids. It's only added
# to satisfy Keras restriction on target shape.
mrcnn_class_ids = mrcnn_class_ids[:, :, 0]
b = 0
# Restore original image (reverse normalization)
sample_image = modellib.unmold_image(normalized_images[b], config)
# Compute anchor shifts.
indices = np.where(rpn_match[b] == 1)[0]
refined_anchors = utils.apply_box_deltas(anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV)
log("anchors", anchors)
log("refined_anchors", refined_anchors)
# Get list of positive anchors
positive_anchor_ids = np.where(rpn_match[b] == 1)[0]
print("Positive anchors: {}".format(len(positive_anchor_ids)))
negative_anchor_ids = np.where(rpn_match[b] == -1)[0]
print("Negative anchors: {}".format(len(negative_anchor_ids)))
neutral_anchor_ids = np.where(rpn_match[b] == 0)[0]
print("Neutral anchors: {}".format(len(neutral_anchor_ids)))
# ROI breakdown by class
for c, n in zip(dataset.class_names, np.bincount(mrcnn_class_ids[b].flatten())):
if n:
print("{:23}: {}".format(c[:20], n))
# Show positive anchors
visualize.draw_boxes(sample_image, boxes=anchors[positive_anchor_ids],
refined_boxes=refined_anchors)
# Show negative anchors
visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids])
# Show neutral anchors. They don't contribute to training.
visualize.draw_boxes(sample_image, boxes=anchors[np.random.choice(neutral_anchor_ids, 100)])
if random_rois:
# Class aware bboxes
bbox_specific = mrcnn_bbox[b, np.arange(mrcnn_bbox.shape[1]), mrcnn_class_ids[b], :]
# Refined ROIs
refined_rois = utils.apply_box_deltas(rois[b].astype(np.float32), bbox_specific[:, :4] * config.BBOX_STD_DEV)
# Class aware masks
mask_specific = mrcnn_mask[b, np.arange(mrcnn_mask.shape[1]), :, :, mrcnn_class_ids[b]]
visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific, mrcnn_class_ids[b], dataset.class_names)
# Any repeated ROIs?
rows = np.ascontiguousarray(rois[b]).view(np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1])))
_, idx = np.unique(rows, return_index=True)
print("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1]))
if random_rois:
# Dispalay ROIs and corresponding masks and bounding boxes
ids = random.sample(range(rois.shape[1]), 8)
images = []
titles = []
for i in ids:
image = visualize.draw_box(sample_image.copy(), rois[b,i,:4].astype(np.int32), [255, 0, 0])
image = visualize.draw_box(image, refined_rois[i].astype(np.int64), [0, 255, 0])
images.append(image)
titles.append("ROI {}".format(i))
images.append(mask_specific[i] * 255)
titles.append(dataset.class_names[mrcnn_class_ids[b,i]][:20])
display_images(images, titles, cols=4, cmap="Blues", interpolation="none")
# Check ratio of positive ROIs in a set of images.
if random_rois:
limit = 10
temp_g = modellib.data_generator(
dataset, config, shuffle=True, random_rois=10000,
batch_size=1, detection_targets=True)
total = 0
for i in range(limit):
_, [ids, _, _] = next(temp_g)
positive_rois = np.sum(ids[0] > 0)
total += positive_rois
print("{:5} {:5.2f}".format(positive_rois, positive_rois/ids.shape[1]))
print("Average percent: {:.2f}".format(total/(limit*ids.shape[1])))
exit()
def siamese_data_generator(dataset,
config,
shuffle=True,
augmentation=imgaug.augmenters.Fliplr(0.5),
random_rois=0,
batch_size=1,
detection_targets=False,
diverse=0):
"""A generator that returns images and corresponding target class ids,
bounding box deltas, and masks.
dataset: The Dataset object to pick data from
config: The model config object
shuffle: If True, shuffles the samples before every epoch
augment: If True, applies image augmentation to images (currently only
horizontal flips are supported)
random_rois: If > 0 then generate proposals to be used to train the
network classifier and mask heads. Useful if training
the Mask RCNN part without the RPN.
batch_size: How many images to return in each call
detection_targets: If True, generate detection targets (class IDs, bbox
deltas, and masks). Typically for debugging or visualizations because
in trainig detection targets are generated by DetectionTargetLayer.
diverse: Float in [0,1] indicatiing probability to draw a target
from any random class instead of one from the image classes
Returns a Python generator. Upon calling next() on it, the
generator returns two lists, inputs and outputs. The containtes
of the lists differs depending on the received arguments:
inputs list:
- images: [batch, H, W, C]
- image_meta: [batch, size of image meta]
- rpn_match: [batch, N] Integer (1=positive anchor, -1=negative, 0=neutral)
- rpn_bbox: [batch, N, (dy, dx, log(dh), log(dw))] Anchor bbox deltas.
- gt_class_ids: [batch, MAX_GT_INSTANCES] Integer class IDs
- gt_boxes: [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)]
- gt_masks: [batch, height, width, MAX_GT_INSTANCES]. The height and width
are those of the image unless use_mini_mask is True, in which
case they are defined in MINI_MASK_SHAPE.
outputs list: Usually empty in regular training. But if detection_targets
is True then the outputs list contains target class_ids, bbox deltas,
and masks.
"""
b = 0 # batch item index
image_index = -1
image_ids = np.copy(dataset.image_ids)
error_count = 0
# Anchors
# [anchor_count, (y1, x1, y2, x2)]
backbone_shapes = modellib.compute_backbone_shapes(config,
config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Keras requires a generator to run indefinately.
while True:
try:
# Increment index to pick next image. Shuffle if at the start of an epoch.
image_index = (image_index + 1) % len(image_ids)
if shuffle and image_index == 0:
np.random.shuffle(image_ids)
# Get GT bounding boxes and masks for image.
image_id = image_ids[image_index]
image, image_meta, gt_class_ids, gt_boxes, gt_masks = \
modellib.load_image_gt(dataset, config, image_id, augmentation=augmentation,
use_mini_mask=config.USE_MINI_MASK)
# Replace class ids with foreground/background info if binary
# class option is chosen
# if binary_classes == True:
# gt_class_ids = np.minimum(gt_class_ids, 1)
# Skip images that have no instances. This can happen in cases
# where we train on a subset of classes and the image doesn't
# have any of the classes we care about.
if not np.any(gt_class_ids > 0):
continue
# print(gt_class_ids)
# Use only positive class_ids
categories = np.unique(gt_class_ids)
_idx = categories > 0
categories = categories[_idx]
# Use only active classes
active_categories = []
for c in categories:
if any(c == dataset.ACTIVE_CLASSES):
active_categories.append(c)
# Skiop image if it contains no instance of any active class
if not np.any(np.array(active_categories) > 0):
continue
# Randomly select category
category = np.random.choice(active_categories)
# Generate siamese target crop
target = get_one_target(category,
dataset,
config,
augmentation=augmentation)
if target is None: # fix until a better ADE20K metadata is built
print('skip target')
continue
# print(target_class_id)
target_class_id = category
target_class_ids = np.array([target_class_id])
idx = gt_class_ids == target_class_id
siamese_class_ids = idx.astype('int8')
# print(idx)
# print(gt_boxes.shape, gt_masks.shape)
siamese_class_ids = siamese_class_ids[idx]
gt_class_ids = gt_class_ids[idx]
gt_boxes = gt_boxes[idx, :]
gt_masks = gt_masks[:, :, idx]
image_meta = image_meta[:14]
# print(gt_boxes.shape, gt_masks.shape)
# RPN Targets
rpn_match, rpn_bbox = modellib.build_rpn_targets(
image.shape, anchors, gt_class_ids, gt_boxes, config)
# Mask R-CNN Targets
if random_rois:
rpn_rois = modellib.generate_random_rois(
image.shape, random_rois, gt_class_ids, gt_boxes)
if detection_targets:
rois, mrcnn_class_ids, mrcnn_bbox, mrcnn_mask =\
modellib.build_detection_targets(
rpn_rois, gt_class_ids, gt_boxes, gt_masks, config)
# Init batch arrays
if b == 0:
batch_image_meta = np.zeros((batch_size, ) + image_meta.shape,
dtype=image_meta.dtype)
batch_rpn_match = np.zeros([batch_size, anchors.shape[0], 1],
dtype=rpn_match.dtype)
batch_rpn_bbox = np.zeros(
[batch_size, config.RPN_TRAIN_ANCHORS_PER_IMAGE, 4],
dtype=rpn_bbox.dtype)
batch_images = np.zeros((batch_size, ) + image.shape,
dtype=np.float32)
batch_gt_class_ids = np.zeros(
(batch_size, config.MAX_GT_INSTANCES), dtype=np.int32)
batch_gt_boxes = np.zeros(
(batch_size, config.MAX_GT_INSTANCES, 4), dtype=np.int32)
batch_targets = np.zeros((batch_size, ) + target.shape,
dtype=np.float32)
# batch_target_class_ids = np.zeros(
# (batch_size, config.MAX_TARGET_INSTANCES), dtype=np.int32)
if config.USE_MINI_MASK:
batch_gt_masks = np.zeros(
(batch_size, config.MINI_MASK_SHAPE[0],
config.MINI_MASK_SHAPE[1], config.MAX_GT_INSTANCES))
else:
batch_gt_masks = np.zeros(
(batch_size, image.shape[0], image.shape[1],
config.MAX_GT_INSTANCES))
if random_rois:
batch_rpn_rois = np.zeros(
(batch_size, rpn_rois.shape[0], 4),
dtype=rpn_rois.dtype)
if detection_targets:
batch_rois = np.zeros((batch_size, ) + rois.shape,
dtype=rois.dtype)
batch_mrcnn_class_ids = np.zeros(
(batch_size, ) + mrcnn_class_ids.shape,
dtype=mrcnn_class_ids.dtype)
batch_mrcnn_bbox = np.zeros(
(batch_size, ) + mrcnn_bbox.shape,
dtype=mrcnn_bbox.dtype)
batch_mrcnn_mask = np.zeros(
(batch_size, ) + mrcnn_mask.shape,
dtype=mrcnn_mask.dtype)
# If more instances than fits in the array, sub-sample from them.
if gt_boxes.shape[0] > config.MAX_GT_INSTANCES:
ids = np.random.choice(np.arange(gt_boxes.shape[0]),
config.MAX_GT_INSTANCES,
replace=False)
gt_class_ids = gt_class_ids[ids]
siamese_class_ids = siamese_class_ids[ids]
gt_boxes = gt_boxes[ids]
gt_masks = gt_masks[:, :, ids]
# Add to batch
batch_image_meta[b] = image_meta
batch_rpn_match[b] = rpn_match[:, np.newaxis]
batch_rpn_bbox[b] = rpn_bbox
batch_images[b] = modellib.mold_image(image.astype(np.float32),
config)
batch_targets[b] = modellib.mold_image(target.astype(np.float32),
config)
batch_gt_class_ids[
b, :siamese_class_ids.shape[0]] = siamese_class_ids
# batch_target_class_ids[b, :target_class_ids.shape[0]] = target_class_ids
batch_gt_boxes[b, :gt_boxes.shape[0]] = gt_boxes
batch_gt_masks[b, :, :, :gt_masks.shape[-1]] = gt_masks
if random_rois:
batch_rpn_rois[b] = rpn_rois
if detection_targets:
batch_rois[b] = rois
batch_mrcnn_class_ids[b] = mrcnn_class_ids
batch_mrcnn_bbox[b] = mrcnn_bbox
batch_mrcnn_mask[b] = mrcnn_mask
b += 1
# Batch full?
if b >= batch_size:
inputs = [
batch_images, batch_image_meta, batch_targets,
batch_rpn_match, batch_rpn_bbox, batch_gt_class_ids,
batch_gt_boxes, batch_gt_masks
]
outputs = []
if random_rois:
inputs.extend([batch_rpn_rois])
if detection_targets:
inputs.extend([batch_rois])
# Keras requires that output and targets have the same number of dimensions
batch_mrcnn_class_ids = np.expand_dims(
batch_mrcnn_class_ids, -1)
outputs.extend([
batch_mrcnn_class_ids, batch_mrcnn_bbox,
batch_mrcnn_mask
])
yield inputs, outputs
# start a new batch
b = 0
except (GeneratorExit, KeyboardInterrupt):
raise
except:
# Log it and skip the image
modellib.logging.exception("Error processing image {}".format(
dataset.image_info[image_id]))
error_count += 1
if error_count > 5:
raise
def data_generator(dataset,
config,
shuffle=True,
augment=False,
augmentation=None,
random_rois=0,
batch_size=1,
detection_targets=False,
no_augmentation_sources=None):
"""A generator that returns images and corresponding target class ids,
bounding box deltas, and masks.
dataset: The Dataset object to pick data from
config: The model config object
shuffle: If True, shuffles the samples before every epoch
augment: (deprecated. Use augmentation instead). If true, apply random
image augmentation. Currently, only horizontal flipping is offered.
augmentation: Optional. An imgaug (https://github.com/aleju/imgaug) augmentation.
For example, passing imgaug.augmenters.Fliplr(0.5) flips images
right/left 50% of the time.
random_rois: If > 0 then generate proposals to be used to train the
network classifier and mask heads. Useful if training
the Mask RCNN part without the RPN.
batch_size: How many images to return in each call
detection_targets: If True, generate detection targets (class IDs, bbox
deltas, and masks). Typically for debugging or visualizations because
in trainig detection targets are generated by DetectionTargetLayer.
no_augmentation_sources: Optional. List of sources to exclude for
augmentation. A source is string that identifies a dataset and is
defined in the Dataset class.
Returns a Python generator. Upon calling next() on it, the
generator returns two lists, inputs and outputs. The contents
of the lists differs depending on the received arguments:
inputs list:
- images: [batch, H, W, C]
- image_meta: [batch, (meta data)] Image details. See compose_image_meta()
- rpn_match: [batch, N] Integer (1=positive anchor, -1=negative, 0=neutral)
- rpn_bbox: [batch, N, (dy, dx, log(dh), log(dw))] Anchor bbox deltas.
- gt_class_ids: [batch, MAX_GT_INSTANCES] Integer class IDs
- gt_boxes: [batch, MAX_GT_INSTANCES, (y1, x1, y2, x2)]
- gt_masks: [batch, height, width, MAX_GT_INSTANCES]. The height and width
are those of the image unless use_mini_mask is True, in which
case they are defined in MINI_MASK_SHAPE.
outputs list: Usually empty in regular training. But if detection_targets
is True then the outputs list contains target class_ids, bbox deltas,
and masks.
"""
b = 0 # batch item index
image_index = -1
image_ids = np.copy(dataset.image_ids)
error_count = 0
no_augmentation_sources = no_augmentation_sources or []
# Anchors
# [anchor_count, (y1, x1, y2, x2)]
backbone_shapes = compute_backbone_shapes(config, config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Keras requires a generator to run indefinitely.
while True:
try:
# Increment index to pick next image. Shuffle if at the start of an epoch.
image_index = (image_index + 1) % len(image_ids)
if shuffle and image_index == 0:
np.random.shuffle(image_ids)
# Get GT bounding boxes and masks for image.
image_id = image_ids[image_index]
# If the image source is not to be augmented pass None as augmentation
if dataset.image_info[image_id][
'source'] in no_augmentation_sources:
image, image_meta, gt_class_ids, gt_boxes, gt_masks = load_image_gt(
dataset,
config,
image_id,
augment=augment,
augmentation=None,
use_mini_mask=config.USE_MINI_MASK)
else:
image, image_meta, gt_class_ids, gt_boxes, gt_masks = load_image_gt(
dataset,
config,
image_id,
augment=augment,
augmentation=augmentation,
use_mini_mask=config.USE_MINI_MASK)
# Skip images that have no instances. This can happen in cases
# where we train on a subset of classes and the image doesn't
# have any of the classes we care about.
if not np.any(gt_class_ids > 0):
continue
# RPN Targets
rpn_match, rpn_bbox = build_rpn_targets(image.shape, anchors,
gt_class_ids, gt_boxes,
config)
# Mask R-CNN Targets
if random_rois:
rpn_rois = generate_random_rois(image.shape, random_rois,
gt_class_ids, gt_boxes)
if detection_targets:
rois, mrcnn_class_ids, mrcnn_bbox, mrcnn_mask = build_detection_targets(
rpn_rois, gt_class_ids, gt_boxes, gt_masks, config)
# Init batch arrays
if b == 0:
batch_image_meta = np.zeros((batch_size, ) + image_meta.shape,
dtype=image_meta.dtype)
batch_rpn_match = np.zeros([batch_size, anchors.shape[0], 1],
dtype=rpn_match.dtype)
batch_rpn_bbox = np.zeros(
[batch_size, config.RPN_TRAIN_ANCHORS_PER_IMAGE, 4],
dtype=rpn_bbox.dtype)
batch_images = np.zeros((batch_size, ) + image.shape,
dtype=np.float32)
batch_gt_class_ids = np.zeros(
(batch_size, config.MAX_GT_INSTANCES), dtype=np.int32)
batch_gt_boxes = np.zeros(
(batch_size, config.MAX_GT_INSTANCES, 4), dtype=np.int32)
batch_gt_masks = np.zeros(
(batch_size, gt_masks.shape[0], gt_masks.shape[1],
config.MAX_GT_INSTANCES),
dtype=gt_masks.dtype)
if random_rois:
batch_rpn_rois = np.zeros(
(batch_size, rpn_rois.shape[0], 4),
dtype=rpn_rois.dtype)
if detection_targets:
batch_rois = np.zeros((batch_size, ) + rois.shape,
dtype=rois.dtype)
batch_mrcnn_class_ids = np.zeros(
(batch_size, ) + mrcnn_class_ids.shape,
dtype=mrcnn_class_ids.dtype)
batch_mrcnn_bbox = np.zeros(
(batch_size, ) + mrcnn_bbox.shape,
dtype=mrcnn_bbox.dtype)
batch_mrcnn_mask = np.zeros(
(batch_size, ) + mrcnn_mask.shape,
dtype=mrcnn_mask.dtype)
# If more instances than fits in the array, sub-sample from them.
if gt_boxes.shape[0] > config.MAX_GT_INSTANCES:
ids = np.random.choice(np.arange(gt_boxes.shape[0]),
config.MAX_GT_INSTANCES,
replace=False)
gt_class_ids = gt_class_ids[ids]
gt_boxes = gt_boxes[ids]
gt_masks = gt_masks[:, :, ids]
# Add to batch
batch_image_meta[b] = image_meta
batch_rpn_match[b] = rpn_match[:, np.newaxis]
batch_rpn_bbox[b] = rpn_bbox
batch_images[b] = mold_image(image.astype(np.float32), config)
batch_gt_class_ids[b, :gt_class_ids.shape[0]] = gt_class_ids
batch_gt_boxes[b, :gt_boxes.shape[0]] = gt_boxes
batch_gt_masks[b, :, :, :gt_masks.shape[-1]] = gt_masks
if random_rois:
batch_rpn_rois[b] = rpn_rois
if detection_targets:
batch_rois[b] = rois
batch_mrcnn_class_ids[b] = mrcnn_class_ids
batch_mrcnn_bbox[b] = mrcnn_bbox
batch_mrcnn_mask[b] = mrcnn_mask
b += 1
# Batch full?
if b >= batch_size:
inputs = [
batch_images, batch_image_meta, batch_rpn_match,
batch_rpn_bbox, batch_gt_class_ids, batch_gt_boxes,
batch_gt_masks
]
outputs = []
if random_rois:
inputs.extend([batch_rpn_rois])
if detection_targets:
inputs.extend([batch_rois])
# Keras requires that output and targets have the same number of dimensions
batch_mrcnn_class_ids = np.expand_dims(
batch_mrcnn_class_ids, -1)
outputs.extend([
batch_mrcnn_class_ids, batch_mrcnn_bbox,
batch_mrcnn_mask
])
yield inputs, outputs
# start a new batch
b = 0
except (GeneratorExit, KeyboardInterrupt):
raise
except:
# Log it and skip the image
logging.exception("Error processing image {}".format(
dataset.image_info[image_id]))
error_count += 1
if error_count > 5:
raise
def recall(model, class_names):
class_dict = {}
label_dict = ['background']
if args.label:
label_file = open(args.label)
label_lines = label_file.readlines()
label_id = 1
for label_line in label_lines:
label_line = label_line.replace('\n', '')
class_dict[label_line] = label_id
label_dict.append(label_line)
label_id = label_id + 1
# Validation dataset
dataset_val = MyDataset()
dataset_val.load_my(args.dataset, "val", class_dict)
dataset_val.prepare()
pre_correct_dict = {}
pre_total_dict = {}
pre_iou_dict = {}
pre_scores_dict = {}
gt_total_dict = {}
for i in range(1, len(class_dict) + 1):
pre_correct_dict[i] = 0
pre_total_dict[i] = 0
pre_iou_dict[i] = 0.0
pre_scores_dict[i] = 0.0
gt_total_dict[i] = 0
backbone_shapes = modellib.compute_backbone_shapes(config, [768, 1280, 3])
anchor_boxes = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
#utils.generate_anchors(300, config.RPN_ANCHOR_RATIOS, [40,40], 32, config.RPN_ANCHOR_STRIDE)
#print(anchor_boxes)
rois = []
obj_groups = []
# {image_file, [gt_class_id], [gt_box, (y1,x1,y2,x2)], [gt_bbox_area], [gt_wh_ratio], [gt_mask_area], [gt_mask_ratio], [gt_size], }
for image_id in dataset_val.image_ids:
image, image_meta, gt_class_id, gt_box, gt_mask = modellib.load_image_gt(
dataset_val, config, image_id, use_mini_mask=False)
#print(image.shape)
gt_detects = {}
gt_detects['image'] = dataset_val.image_reference(image_id)
gt_detects['gt_class_id'] = gt_class_id
gt_detects['gt_bbox'] = gt_box
gt_detects['gt_bbox_area'] = []
gt_detects['gt_wh_ratio'] = []
gt_detects['gt_mask_area'] = []
gt_detects['gt_mask_ratio'] = []
gt_detects['gt_size'] = []
for i in range(0, len(gt_class_id)):
gt_total_dict[gt_class_id[i]] = gt_total_dict[gt_class_id[i]] + 1
wh_ratio, box_size, box_area, square_box = toSquareBox(gt_box[i])
mask_area = np.sum(gt_mask[:, :, i] == True)
mask_ratio = mask_area / box_area
gt_detects['gt_bbox_area'].append(box_area)
gt_detects['gt_wh_ratio'].append(wh_ratio)
gt_detects['gt_mask_area'].append(mask_area)
gt_detects['gt_mask_ratio'].append(mask_ratio)
gt_detects['gt_size'].append(box_size)
molded_image = modellib.mold_image(image, config)
#print(molded_image.shape)
# Anchors
"""
anchors = model.get_anchors(molded_image.shape)
# Duplicate across the batch dimension because Keras requires it
# TODO: can this be optimized to avoid duplicating the anchors?
anchors = np.broadcast_to(anchors, (config.BATCH_SIZE,) + anchors.shape)
print(anchors)
# Run object detection
detections, mrcnn_class, mrcnn_bbox, mrcnn_mask, rpn_rois, rpn_class, rpn_bbox =\
model.keras_model.predict([np.expand_dims(molded_image, 0), np.expand_dims(image_meta, 0), anchors], verbose=0)
print(detections[0])
print(mrcnn_class[0])
print(rpn_class[0])
"""
#skimage.io.imsave("test.jpg", image)
start_time = time.time()
results = model.detect_molded(np.expand_dims(molded_image, 0),
np.expand_dims(image_meta, 0),
verbose=0)
end_time = time.time()
#print("Time: %s" % str(end_time - start_time))
#print(results)
r = results[0]
pre_class_ids = r['class_ids']
for i in range(0, len(pre_class_ids)):
pre_total_dict[
pre_class_ids[i]] = pre_total_dict[pre_class_ids[i]] + 1
pre_scores = r['scores']
#print(r['rois'])
for roi in r['rois']:
whr, bsize, _, _ = toSquareBox(roi)
rois.append([bsize, whr])
#print(gt_detects['gt_size'])
#overlaps = utils.compute_iou(roi, gt_detects['gt_bbox'], roi_area, gt_detects['gt_bbox_area'])
#print(overlaps)
gt_match, pred_match, overlap = display_differences(
image,
gt_box,
gt_class_id,
gt_mask,
r['rois'],
pre_class_ids,
pre_scores,
r['masks'],
class_names,
title="",
ax=None,
show_mask=True,
show_box=True,
iou_threshold=0.1,
score_threshold=0.1)
gt_detects['rois'] = r['rois']
gt_detects['gt_match'] = gt_match
gt_detects['pred_match'] = pred_match
#print(gt_match)
"""
visualize.display_differences(image,
gt_box, gt_class_id, gt_mask,
r['rois'], pre_class_ids, pre_scores, r['masks'],
class_names, title="", ax=None,
show_mask=True, show_box=True,
iou_threshold=0.1, score_threshold=0.1)
"""
for i in range(0, len(pred_match)):
if pred_match[i] > -1.0:
#print(r['rois'][i])
pre_correct_dict[
pre_class_ids[i]] = pre_correct_dict[pre_class_ids[i]] + 1
pre_iou_dict[pre_class_ids[i]] = pre_iou_dict[
pre_class_ids[i]] + overlap[i, int(pred_match[i])]
pre_scores_dict[pre_class_ids[i]] = pre_scores_dict[
pre_class_ids[i]] + pre_scores[i]
obj_groups.append(gt_detects)
#print(rois)
print("图片,类别,标注框,标注宽高比,标注尺寸,检测框,检测宽高比,检测尺寸,最大IOU")
for det in obj_groups:
for i in range(0, len(det['gt_class_id'])):
overlaped = utils.compute_overlaps(
anchor_boxes, np.reshape(det['gt_bbox'][i], (1, 4)))
omax = max(overlaped)
#if det['gt_size'][i] > 150 and det['gt_size'][i] < 367:
if omax[0] > 0.0:
print(det['image'], end='')
print(",",
label_dict[det['gt_class_id'][i]],
",",
det['gt_bbox'][i],
",",
det['gt_wh_ratio'][i],
",",
det['gt_size'][i],
end="")
if det['gt_match'][i] > -1.0:
idx = int(det['gt_match'][i])
#print(idx, det['rois'])
whr, bsize, _, _ = toSquareBox(det['rois'][idx])
print(",", det['rois'][idx], ",", whr, ",", bsize, ",",
omax[0])
else:
print(",", 0, ",", 0, ",", 0, ",", omax[0])
tol_pre_correct_dict = 0
tol_pre_total_dict = 0
tol_pre_iou_dict = 0
tol_pre_scores_dict = 0
tol_gt_total_dict = 0
lines = []
tile_line = 'Type,Number,Correct,Proposals,Total,Rps/img,Avg IOU,Avg score,Recall,Precision\n'
lines.append(tile_line)
for key in class_dict:
tol_pre_correct_dict = tol_pre_correct_dict + pre_correct_dict[
class_dict[key]]
tol_pre_total_dict = pre_total_dict[
class_dict[key]] + tol_pre_total_dict
tol_pre_iou_dict = pre_iou_dict[class_dict[key]] + tol_pre_iou_dict
tol_pre_scores_dict = pre_scores_dict[
class_dict[key]] + tol_pre_scores_dict
tol_gt_total_dict = gt_total_dict[class_dict[key]] + tol_gt_total_dict
type_rps_img = pre_total_dict[class_dict[key]] / len(
dataset_val.image_ids)
if pre_correct_dict[class_dict[key]] > 0:
type_avg_iou = pre_iou_dict[class_dict[key]] / pre_correct_dict[
class_dict[key]]
type_avg_score = pre_scores_dict[
class_dict[key]] / pre_correct_dict[class_dict[key]]
else:
type_avg_iou = 0
type_avg_score = 0
if gt_total_dict[class_dict[key]] > 0:
type_recall = pre_total_dict[class_dict[key]] / gt_total_dict[
class_dict[key]]
else:
type_recall = 0
if pre_total_dict[class_dict[key]] > 0:
type_precision = pre_correct_dict[
class_dict[key]] / pre_total_dict[class_dict[key]]
else:
type_precision = 0
line = '{:s},{:d},{:d},{:d},{:d},{:.2f},{:.2f}%,{:.2f},{:.2f}%,{:.2f}%\n'.format(
key, len(dataset_val.image_ids), pre_correct_dict[class_dict[key]],
pre_total_dict[class_dict[key]], gt_total_dict[class_dict[key]],
type_rps_img, type_avg_iou * 100, type_avg_score,
type_recall * 100, type_precision * 100)
lines.append(line)
print(line)
tol_rps_img = tol_pre_total_dict / len(dataset_val.image_ids)
if tol_pre_correct_dict > 0:
tol_avg_iou = tol_pre_iou_dict / tol_pre_correct_dict
tol_avg_score = tol_pre_scores_dict / tol_pre_correct_dict
else:
tol_avg_iou = 0
tol_avg_score = 0
if tol_gt_total_dict > 0:
tol_recall = tol_pre_total_dict / tol_gt_total_dict
else:
tol_recall = 0
if tol_pre_total_dict > 0:
tol_precision = tol_pre_correct_dict / tol_pre_total_dict
else:
tol_precision = 0
totle_line = '{:s},{:d},{:d},{:d},{:d},{:.2f},{:.2f}%,{:.2f},{:.2f}%,{:.2f}%\n'.format(
'Total', len(dataset_val.image_ids), tol_pre_correct_dict,
tol_pre_total_dict, tol_gt_total_dict, type_rps_img, tol_avg_iou * 100,
tol_avg_score, tol_recall * 100, tol_precision * 100)
print(totle_line)
lines.append(totle_line)
result_file_name = "result_{:%Y%m%dT%H%M%S}.csv".format(datetime.now())
result_file = open(result_file_name, 'w+')
result_file.writelines(lines)
result_file.close()
from mrcnn.model import log
log("img", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# 显示mask,以及bbox
visualize.display_instances(image, bbox, mask, class_ids,
dataset_train.class_names)
# 3、计算anchor结果
config = BalloonConfig()
config.BACKBONE_SHAPES = [[256, 256], [128, 128], [64, 64], [32, 32],
[16, 16]]
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
config.BACKBONE_SHAPES,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# 打印anchor相关信息
num_levels = len(config.BACKBONE_SHAPES)
anchors_per_cell = len(config.RPN_ANCHOR_RATIOS)
print("Count: ", anchors.shape[0])
print("Scales: ", config.RPN_ANCHOR_SCALES)
print("ratios: ", config.RPN_ANCHOR_RATIOS)
print("Anchors per Cell: ", anchors_per_cell)
print("Levels: ", num_levels)
anchors_per_level = []
for l in range(num_levels):
num_cells = config.BACKBONE_SHAPES[l][0] * config.BACKBONE_SHAPES[l][1]
anchors_per_level.append(anchors_per_cell * num_cells //
def inspect_data(dataset, config):
print("Image Count: {}".format(len(dataset.image_ids)))
print("Class Count: {}".format(dataset.num_classes))
for i, info in enumerate(dataset.class_info):
print("{:3}. {:50}".format(i, info['name']))
# ## Display Samples
#
# Load and display images and masks.
# In[4]:
# Load and display random samples
image_ids = np.random.choice(dataset.image_ids, 4)
for image_id in image_ids:
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids,
dataset.class_names)
# ## Bounding Boxes
#
# Rather than using bounding box coordinates provided by the source datasets, we compute the bounding boxes from masks instead. This allows us to handle bounding boxes consistently regardless of the source dataset, and it also makes it easier to resize, rotate, or crop images because we simply generate the bounding boxes from the updates masks rather than computing bounding box transformation for each type of image transformation.
# In[5]:
# Load random image and mask.
image_id = random.choice(dataset.image_ids)
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id ", image_id, dataset.image_reference(image_id))
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Resize Images
#
# To support multiple images per batch, images are resized to one size (1024x1024). Aspect ratio is preserved, though. If an image is not square, then zero padding is added at the top/bottom or right/left.
# In[6]:
# Load random image and mask.
image_id = np.random.choice(dataset.image_ids, 1)[0]
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
original_shape = image.shape
# Resize
image, window, scale, padding, _ = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
mask = utils.resize_mask(mask, scale, padding)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id: ", image_id, dataset.image_reference(image_id))
print("Original shape: ", original_shape)
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Mini Masks
#
# Instance binary masks can get large when training with high resolution images. For example, if training with 1024x1024 image then the mask of a single instance requires 1MB of memory (Numpy uses bytes for boolean values). If an image has 100 instances then that's 100MB for the masks alone.
#
# To improve training speed, we optimize masks by:
# * We store mask pixels that are inside the object bounding box, rather than a mask of the full image. Most objects are small compared to the image size, so we save space by not storing a lot of zeros around the object.
# * We resize the mask to a smaller size (e.g. 56x56). For objects that are larger than the selected size we lose a bit of accuracy. But most object annotations are not very accuracy to begin with, so this loss is negligable for most practical purposes. Thie size of the mini_mask can be set in the config class.
#
# To visualize the effect of mask resizing, and to verify the code correctness, we visualize some examples.
# In[7]:
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
log("image", image)
log("image_meta", image_meta)
log("class_ids", class_ids)
log("bbox", bbox)
log("mask", mask)
display_images([image] +
[mask[:, :, i] for i in range(min(mask.shape[-1], 7))])
# In[8]:
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# In[9]:
# Add augmentation and mask resizing.
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, augment=True, use_mini_mask=True)
log("mask", mask)
display_images([image] +
[mask[:, :, i] for i in range(min(mask.shape[-1], 7))])
# In[10]:
mask = utils.expand_mask(bbox, mask, image.shape)
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Anchors
#
# The order of anchors is important. Use the same order in training and prediction phases. And it must match the order of the convolution execution.
#
# For an FPN network, the anchors must be ordered in a way that makes it easy to match anchors to the output of the convolution layers that predict anchor scores and shifts.
# * Sort by pyramid level first. All anchors of the first level, then all of the second and so on. This makes it easier to separate anchors by level.
# * Within each level, sort anchors by feature map processing sequence. Typically, a convolution layer processes a feature map starting from top-left and moving right row by row.
# * For each feature map cell, pick any sorting order for the anchors of different ratios. Here we match the order of ratios passed to the function.
#
# **Anchor Stride:**
# In the FPN architecture, feature maps at the first few layers are high resolution. For example, if the input image is 1024x1024 then the feature meap of the first layer is 256x256, which generates about 200K anchors (256*256*3). These anchors are 32x32 pixels and their stride relative to image pixels is 4 pixels, so there is a lot of overlap. We can reduce the load significantly if we generate anchors for every other cell in the feature map. A stride of 2 will cut the number of anchors by 4, for example.
#
# In this implementation we use an anchor stride of 2, which is different from the paper.
# In[11]:
# Generate Anchors
backbone_shapes = modellib.compute_backbone_shapes(config,
config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Print summary of anchors
num_levels = len(backbone_shapes)
anchors_per_cell = len(config.RPN_ANCHOR_RATIOS)
print("Count: ", anchors.shape[0])
print("Scales: ", config.RPN_ANCHOR_SCALES)
print("ratios: ", config.RPN_ANCHOR_RATIOS)
print("Anchors per Cell: ", anchors_per_cell)
print("Levels: ", num_levels)
anchors_per_level = []
for l in range(num_levels):
num_cells = backbone_shapes[l][0] * backbone_shapes[l][1]
anchors_per_level.append(anchors_per_cell * num_cells //
config.RPN_ANCHOR_STRIDE**2)
print("Anchors in Level {}: {}".format(l, anchors_per_level[l]))
# Visualize anchors of one cell at the center of the feature map of a specific level.
# In[12]:
## Visualize anchors of one cell at the center of the feature map of a specific level
# Load and draw random image
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, _, _, _ = modellib.load_image_gt(
dataset, config, image_id)
fig, ax = plt.subplots(1, figsize=(10, 10))
ax.imshow(image)
levels = len(backbone_shapes)
for level in range(levels):
colors = visualize.random_colors(levels)
# Compute the index of the anchors at the center of the image
level_start = sum(
anchors_per_level[:level]) # sum of anchors of previous levels
level_anchors = anchors[level_start:level_start +
anchors_per_level[level]]
print("Level {}. Anchors: {:6} Feature map Shape: {}".format(
level, level_anchors.shape[0], backbone_shapes[level]))
center_cell = backbone_shapes[level] // 2
center_cell_index = (center_cell[0] * backbone_shapes[level][1] +
center_cell[1])
level_center = center_cell_index * anchors_per_cell
center_anchor = anchors_per_cell * (
(center_cell[0] * backbone_shapes[level][1] / config.RPN_ANCHOR_STRIDE**2) \
+ center_cell[1] / config.RPN_ANCHOR_STRIDE)
level_center = int(center_anchor)
# Draw anchors. Brightness show the order in the array, dark to bright.
for i, rect in enumerate(level_anchors[level_center:level_center +
anchors_per_cell]):
y1, x1, y2, x2 = rect
p = patches.Rectangle(
(x1, y1),
x2 - x1,
y2 - y1,
linewidth=2,
facecolor='none',
edgecolor=(i + 1) * np.array(colors[level]) / anchors_per_cell)
ax.add_patch(p)
# ## Data Generator
#
# In[13]:
# Create data generator
random_rois = 2000
g = modellib.data_generator(dataset,
config,
shuffle=True,
random_rois=random_rois,
batch_size=4,
detection_targets=True)
# In[14]:
# Uncomment to run the generator through a lot of images
# to catch rare errors
# for i in range(1000):
# print(i)
# _, _ = next(g)
# In[15]:
# Get Next Image
if random_rois:
[
normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids,
gt_boxes, gt_masks, rpn_rois, rois
], [mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g)
log("rois", rois)
log("mrcnn_class_ids", mrcnn_class_ids)
log("mrcnn_bbox", mrcnn_bbox)
log("mrcnn_mask", mrcnn_mask)
else:
[
normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes,
gt_masks
], _ = next(g)
log("gt_class_ids", gt_class_ids)
log("gt_boxes", gt_boxes)
log("gt_masks", gt_masks)
log(
"rpn_match",
rpn_match,
)
log("rpn_bbox", rpn_bbox)
image_id = modellib.parse_image_meta(image_meta)["image_id"][0]
print("image_id: ", image_id, dataset.image_reference(image_id))
# Remove the last dim in mrcnn_class_ids. It's only added
# to satisfy Keras restriction on target shape.
mrcnn_class_ids = mrcnn_class_ids[:, :, 0]
# In[16]:
b = 0
# Restore original image (reverse normalization)
sample_image = modellib.unmold_image(normalized_images[b], config)
# Compute anchor shifts.
indices = np.where(rpn_match[b] == 1)[0]
refined_anchors = utils.apply_box_deltas(
anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV)
log("anchors", anchors)
log("refined_anchors", refined_anchors)
# Get list of positive anchors
positive_anchor_ids = np.where(rpn_match[b] == 1)[0]
print("Positive anchors: {}".format(len(positive_anchor_ids)))
negative_anchor_ids = np.where(rpn_match[b] == -1)[0]
print("Negative anchors: {}".format(len(negative_anchor_ids)))
neutral_anchor_ids = np.where(rpn_match[b] == 0)[0]
print("Neutral anchors: {}".format(len(neutral_anchor_ids)))
# ROI breakdown by class
for c, n in zip(dataset.class_names,
np.bincount(mrcnn_class_ids[b].flatten())):
if n:
print("{:23}: {}".format(c[:20], n))
# Show positive anchors
fig, ax = plt.subplots(1, figsize=(16, 16))
visualize.draw_boxes(sample_image,
boxes=anchors[positive_anchor_ids],
refined_boxes=refined_anchors,
ax=ax)
# In[17]:
# Show negative anchors
visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids])
# In[18]:
# Show neutral anchors. They don't contribute to training.
visualize.draw_boxes(sample_image,
boxes=anchors[np.random.choice(
neutral_anchor_ids, 100)])
# ## ROIs
# In[19]:
if random_rois:
# Class aware bboxes
bbox_specific = mrcnn_bbox[b,
np.arange(mrcnn_bbox.shape[1]),
mrcnn_class_ids[b], :]
# Refined ROIs
refined_rois = utils.apply_box_deltas(
rois[b].astype(np.float32),
bbox_specific[:, :4] * config.BBOX_STD_DEV)
# Class aware masks
mask_specific = mrcnn_mask[b,
np.arange(mrcnn_mask.shape[1]), :, :,
mrcnn_class_ids[b]]
visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific,
mrcnn_class_ids[b], dataset.class_names)
# Any repeated ROIs?
rows = np.ascontiguousarray(rois[b]).view(
np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1])))
_, idx = np.unique(rows, return_index=True)
print("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1]))
# In[20]:
if random_rois:
# Dispalay ROIs and corresponding masks and bounding boxes
ids = random.sample(range(rois.shape[1]), 8)
images = []
titles = []
for i in ids:
image = visualize.draw_box(sample_image.copy(),
rois[b, i, :4].astype(np.int32),
[255, 0, 0])
image = visualize.draw_box(image, refined_rois[i].astype(np.int64),
[0, 255, 0])
images.append(image)
titles.append("ROI {}".format(i))
images.append(mask_specific[i] * 255)
titles.append(dataset.class_names[mrcnn_class_ids[b, i]][:20])
display_images(images,
titles,
cols=4,
cmap="Blues",
interpolation="none")
# In[21]:
# Check ratio of positive ROIs in a set of images.
if random_rois:
limit = 10
temp_g = modellib.data_generator(dataset,
config,
shuffle=True,
random_rois=10000,
batch_size=1,
detection_targets=True)
total = 0
for i in range(limit):
_, [ids, _, _] = next(temp_g)
positive_rois = np.sum(ids[0] > 0)
total += positive_rois
print("{:5} {:5.2f}".format(positive_rois,
positive_rois / ids.shape[1]))
print("Average percent: {:.2f}".format(total / (limit * ids.shape[1])))
