def detect(model, dataset_dir, subset):
"""Run detection on images in the given directory."""
print("Running on {}".format(dataset_dir))
# Create directory
if not os.path.exists(RESULTS_DIR):
os.makedirs(RESULTS_DIR)
submit_dir = "submit_{:%Y%m%dT%H%M%S}".format(datetime.datetime.now())
submit_dir = os.path.join(RESULTS_DIR, submit_dir)
os.makedirs(submit_dir)
# Read dataset
dataset = BreastDataset()
dataset.load_breast(dataset_dir, subset)
dataset.prepare()
# Load over images
#submission = []
for image_id in dataset.image_ids:
# Load image and run detection
image = dataset.load_image(image_id)
# molded_images
#h,w = image.shape[:2]
#scale = 512 / max(h,w)
#image = utils.resize(image,(round(h*scale),round(w*scale)),preserve_range=True)
# Detect objects
r = model.detect([image], verbose=0)[0]
# image = skimage.color.rgb2gray(image)
# Encode image to RLE. Returns a string of multiple lines
#source_id = dataset.image_info[image_id]["id"]
#rle = mask_to_rle(source_id, r["masks"], r["scores"])
#submission.append(rle)
# Save image with masks
visualize.draw_boxes(image, r['rois'],title="MN")
#visualize.display_instances(
# image, r['rois'], r['masks'], r['class_ids'],
# dataset.class_names, r['scores'],
# show_bbox=True, show_mask=True,
# title="Predictions")
plt.savefig("{}/{}.png".format(submit_dir, dataset.image_info[image_id]["id"]))
# Save to csv file
#submission = "ImageId,EncodedPixels\n" + "\n".join(submission)
#file_path = os.path.join(submit_dir, "submit.csv")
#with open(file_path, "w") as f:
# f.write(submission)
print("Saved to ", submit_dir)
def display_rpn_targets():
# Generate RPN trainig targets
resized_image, image_meta, gt_class_ids, gt_bboxes, gt_masks = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
image_info = dataset.image_info[image_id]
# Note: image_info 的 id 是 image 的 filename
print("Image ID: {}.{} ({}) {}".format(image_info["source"],
image_info["id"], image_id,
dataset.image_reference(image_id)))
# get_anchors 会把 pixel coordinates 赋值到 self.a
normalized_anchors = model.get_anchors(resized_image.shape)
anchors = model.anchors
# target_rpn_match is 1 for positive anchors, -1 for negative anchors
# and 0 for neutral anchors.
target_rpn_match, target_rpn_deltas = modellib.build_rpn_targets(
anchors, gt_class_ids, gt_bboxes, model.config)
log("target_rpn_match", target_rpn_match)
log("target_rpn_deltas", target_rpn_deltas)
positive_anchor_ix = np.where(target_rpn_match[:] == 1)[0]
negative_anchor_ix = np.where(target_rpn_match[:] == -1)[0]
neutral_anchor_ix = np.where(target_rpn_match[:] == 0)[0]
positive_anchors = model.anchors[positive_anchor_ix]
negative_anchors = model.anchors[negative_anchor_ix]
neutral_anchors = model.anchors[neutral_anchor_ix]
log("positive_anchors", positive_anchors)
log("negative_anchors", negative_anchors)
log("neutral anchors", neutral_anchors)
# Apply refinement deltas to positive anchors
refined_anchors = utils.apply_box_deltas(
positive_anchors, target_rpn_deltas[:positive_anchors.shape[0]] *
model.config.RPN_BBOX_STD_DEV)
log(
"refined_anchors",
refined_anchors,
)
# Display positive anchors before refinement (dotted) and
# after refinement (solid).
visualize.draw_boxes(resized_image,
boxes=positive_anchors,
refined_boxes=refined_anchors,
ax=get_ax())
plt.show()
def show_anchors():
g, _, _ = create_data_generator(num_random_rois=0)
# Get Next Image
[
normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids,
gt_boxes, gt_masks
], _ = next(g)
b = 0
# Restore original image (reverse normalization)
sample_image = modellib.unmold_image(normalized_images[b], config)
# Get list of positive anchors
positive_anchor_ids = np.where(rpn_match[b] == 1)[0]
# Generate anchors
backbone_shapes, anchors, anchors_per_level = generate_anchors()
# Compute anchor shifts.
refined_anchors = utils.apply_box_deltas(
anchors[positive_anchor_ids],
rpn_bbox[b, :len(positive_anchor_ids)] * config.RPN_BBOX_STD_DEV)
log("anchors", anchors)
print("Positive anchors: {}".format(len(positive_anchor_ids)))
log("refined_anchors", refined_anchors)
negative_anchor_ids = np.where(rpn_match[b] == -1)[0]
print("Negative anchors: {}".format(len(negative_anchor_ids)))
neutral_anchor_ids = np.where(rpn_match[b] == 0)[0]
print("Neutral anchors: {}".format(len(neutral_anchor_ids)))
# Show positive anchors
fig, ax = plt.subplots(1, figsize=(16, 16))
visualize.draw_boxes(sample_image,
boxes=anchors[positive_anchor_ids],
refined_boxes=refined_anchors,
ax=ax)
# Show negative anchors
visualize.draw_boxes(sample_image,
boxes=anchors[negative_anchor_ids],
ax=ax)
# Show neutral anchors. They don't contribute to training.
visualize.draw_boxes(sample_image,
boxes=anchors[np.random.choice(
neutral_anchor_ids, 100)],
ax=ax)
plt.show()
if (args.write):
plt.savefig(os.path.join(save_path,
os.path.split(image_path)[1]),
bbox_inches='tight')
plt.show()
else:
captions = [
"{} {:.3f}".format(dataset_detect.class_names[int(c)], s)
if c > 0 else ""
for c, s in zip(temp['class_ids'], temp['scores'])
]
visualize.draw_boxes(image,
refined_boxes=temp['rois'],
captions=captions,
title="Detections",
ax=ax)
#write the detection result to a folder
if (args.write):
plt.savefig(os.path.join(save_path,
os.path.split(image_path)[1]),
bbox_inches='tight')
plt.show()
#run detection on the input video
if (args.videopath):
dataset_detect = CocoDataset()
val_type = "minival"
log("negative_anchors", negative_anchors)
log("neutral anchors", neutral_anchors)
# Apply refinement deltas to positive anchors
refined_anchors = utils.apply_box_deltas(
positive_anchors, target_rpn_bbox[:positive_anchors.shape[0]] *
model.config.RPN_BBOX_STD_DEV)
log(
"refined_anchors",
refined_anchors,
)
# Display positive anchors before refinement (dotted) and
# after refinement (solid).
visualize.draw_boxes(image,
ax=get_ax(),
boxes=positive_anchors,
refined_boxes=refined_anchors)
# Run RPN sub-graph
pillar = model.keras_model.get_layer(
"ROI").output # node to start searching from
# TF 1.4 introduces a new version of NMS. Search for both names to support TF 1.3 and 1.4
nms_node = model.ancestor(pillar, "ROI/rpn_non_max_suppression:0")
if nms_node is None:
nms_node = model.ancestor(
pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV2:0")
rpn = model.run_graph(image[np.newaxis], [
("rpn_class", model.keras_model.get_layer("rpn_class").output),
("pre_nms_anchors", model.ancestor(pillar, "ROI/pre_nms_anchors:0")),
log("neutral anchors", neutral_anchors)
# Apply refinement deltas to positive anchors
refined_anchors = utils.apply_box_deltas(
positive_anchors, target_rpn_bbox[:positive_anchors.shape[0]] *
model.config.RPN_BBOX_STD_DEV)
log(
"refined_anchors",
refined_anchors,
)
# Display positive anchors before refinement (dotted) and
# after refinement (solid).
visualize.draw_boxes(image,
boxes=positive_anchors,
refined_boxes=refined_anchors,
ax=get_ax(),
img_name="detected_draw_boxes")
# Run RPN sub-graph
pillar = model.keras_model.get_layer(
"ROI").output # node to start searching from
# TF 1.4 and 1.9 introduce new versions of NMS. Search for all names to support TF 1.3~1.10
nms_node = model.ancestor(pillar, "ROI/rpn_non_max_suppression:0")
if nms_node is None:
nms_node = model.ancestor(
pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV2:0")
if nms_node is None: #TF 1.9-1.10
nms_node = model.ancestor(
pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV3:0")
def all_steps(dataset, datacfg, dnncfg):
'''
## Single entry point for all the steps for inspecting dataset
'''
## Uncomment for debugging
# inspectdata.load_and_display_dataset(dataset, datacfg)
# In[7]:
log.info("[7]. ---------------")
log.info("Load and display random images and masks---------------")
log.info("Bounding Boxes---------------")
load_and_display_random_sample(dataset, datacfg)
# In[9]:
log.info("[9]. ---------------")
log.info("Resize Images---------------")
load_and_resize_images(dataset, datacfg, dnncfg)
# In[10]:
log.info("[10]. ---------------")
log.info("Mini Masks---------------")
image_id = load_mini_masks(dataset, datacfg, dnncfg)
log.info("image_id: {}".format(image_id))
# In[11]:
log.info("[11]. ---------------")
log.info("Add augmentation and mask resizing---------------")
add_augmentation(dataset, datacfg, dnncfg, image_id)
info = dataset.image_info[image_id]
log.debug("info: {}".format(info))
# In[12]:
log.info("[12]. ---------------")
log.info("Anchors---------------")
backbone_shapes, anchors, anchors_per_level, anchors_per_cell = generate_anchors(dnncfg)
# In[13]:
log.info("[13]. ---------------")
log.info("Visualize anchors of one cell at the center of the feature map of a specific level---------------")
visualize_anchors_at_center(dataset, datacfg, dnncfg, backbone_shapes, anchors, anchors_per_level, anchors_per_cell)
# In[14]:
log.info("[14]. ---------------")
log.info("info---------------")
image_ids = dataset.image_ids
log.info(image_ids)
image_index = -1
image_index = (image_index + 1) % len(image_ids)
log.info("image_index:{}".format(image_index))
# In[15]:
log.info("[15]. ---------------")
log.info("data_generator---------------")
## Data Generator
# Create data generator
random_rois = 2000
g = modellib.data_generator(
dataset, datacfg, dnncfg, shuffle=True, random_rois=random_rois,
batch_size=4,
detection_targets=True)
# Uncomment to run the generator through a lot of images
# to catch rare errors
# for i in range(1000):
# log.debug(i)
# _, _ = next(g)
# Get Next Image
if random_rois:
[normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, rpn_rois, rois], [mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g)
customlog("rois", rois)
customlog("mrcnn_class_ids", mrcnn_class_ids)
customlog("mrcnn_bbox", mrcnn_bbox)
customlog("mrcnn_mask", mrcnn_mask)
else:
[normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes, gt_masks], _ = next(g)
customlog("gt_class_ids", gt_class_ids)
customlog("gt_boxes", gt_boxes)
customlog("gt_masks", gt_masks)
customlog("rpn_match", rpn_match, )
customlog("rpn_bbox", rpn_bbox)
image_id = modellib.parse_image_meta(image_meta)["image_id"][0]
# Remove the last dim in mrcnn_class_ids. It's only added
# to satisfy Keras restriction on target shape.
mrcnn_class_ids = mrcnn_class_ids[:,:,0]
# In[16]:
log.info("[16]. ---------------")
b = 0
# Restore original image (reverse normalization)
sample_image = modellib.unmold_image(normalized_images[b], dnncfg)
# Compute anchor shifts.
indices = np.where(rpn_match[b] == 1)[0]
refined_anchors = utils.apply_box_deltas(anchors[indices], rpn_bbox[b, :len(indices)] * dnncfg.RPN_BBOX_STD_DEV)
customlog("anchors", anchors)
customlog("refined_anchors", refined_anchors)
# Get list of positive anchors
positive_anchor_ids = np.where(rpn_match[b] == 1)[0]
log.info("Positive anchors: {}".format(len(positive_anchor_ids)))
negative_anchor_ids = np.where(rpn_match[b] == -1)[0]
log.info("Negative anchors: {}".format(len(negative_anchor_ids)))
neutral_anchor_ids = np.where(rpn_match[b] == 0)[0]
log.info("Neutral anchors: {}".format(len(neutral_anchor_ids)))
log.info("ROI breakdown by class---------------")
# ROI breakdown by class
for c, n in zip(dataset.class_names, np.bincount(mrcnn_class_ids[b].flatten())):
if n:
log.info("{:23}: {}".format(c[:20], n))
log.info("Show positive anchors---------------")
# Show positive anchors
fig, ax = plt.subplots(1, figsize=(16, 16))
visualize.draw_boxes(sample_image, boxes=anchors[positive_anchor_ids],
refined_boxes=refined_anchors, ax=ax)
# In[17]:
log.info("[17]. ---------------")
log.info("Show negative anchors---------------")
# Show negative anchors
visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids])
# In[18]:
log.info("[18]. ---------------")
log.info("Show neutral anchors. They don't contribute to training---------------")
# Show neutral anchors. They don't contribute to training.
visualize.draw_boxes(sample_image, boxes=anchors[np.random.choice(neutral_anchor_ids, 100)])
# In[19]:
log.info("[19]. ---------------")
log.info("ROIs---------------")
## ROIs
if random_rois:
# Class aware bboxes
bbox_specific = mrcnn_bbox[b, np.arange(mrcnn_bbox.shape[1]), mrcnn_class_ids[b], :]
# Refined ROIs
refined_rois = utils.apply_box_deltas(rois[b].astype(np.float32), bbox_specific[:,:4] * dnncfg.BBOX_STD_DEV)
# Class aware masks
mask_specific = mrcnn_mask[b, np.arange(mrcnn_mask.shape[1]), :, :, mrcnn_class_ids[b]]
visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific, mrcnn_class_ids[b], dataset.class_names)
# Any repeated ROIs?
rows = np.ascontiguousarray(rois[b]).view(np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1])))
_, idx = np.unique(rows, return_index=True)
log.info("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1]))
# In[20]:
log.info("[20]. ---------------")
log.info("Dispalay ROIs and corresponding masks and bounding boxes---------------")
if random_rois:
# Dispalay ROIs and corresponding masks and bounding boxes
ids = random.sample(range(rois.shape[1]), 8)
images = []
titles = []
for i in ids:
image = visualize.draw_box(sample_image.copy(), rois[b,i,:4].astype(np.int32), [255, 0, 0])
image = visualize.draw_box(image, refined_rois[i].astype(np.int64), [0, 255, 0])
images.append(image)
titles.append("ROI {}".format(i))
images.append(mask_specific[i] * 255)
titles.append(dataset.class_names[mrcnn_class_ids[b,i]][:20])
visualize.display_images(images, titles, cols=4, cmap="Blues", interpolation="none")
# In[21]:
log.info("[21]. ---------------")
log.info("Check ratio of positive ROIs in a set of images.---------------")
# Check ratio of positive ROIs in a set of images.
if random_rois:
limit = 10
temp_g = modellib.data_generator(
dataset, datacfg, dnncfg, shuffle=True, random_rois=10000,
batch_size=1, detection_targets=True)
total = 0
for i in range(limit):
_, [ids, _, _] = next(temp_g)
positive_rois = np.sum(ids[0] > 0)
total += positive_rois
log.info("{:5} {:5.2f}".format(positive_rois, positive_rois/ids.shape[1]))
log.info("Average percent: {:.2f}".format(total/(limit*ids.shape[1])))
log(
"refined_anchors",
refined_anchors,
)
# Apply refinement deltas to positive anchors
refined_anchors = utils.apply_box_deltas(
positive_anchors, target_rpn_bbox[:positive_anchors.shape[0]] *
model.config.RPN_BBOX_STD_DEV)
log(
"refined_anchors",
refined_anchors,
)
visualize.draw_boxes(image,
boxes=positive_anchors,
refined_boxes=refined_anchors,
ax=get_ax())
# Run RPN sub-graph
pillar = model.keras_model.get_layer(
"ROI").output # node to start searching from
# TF 1.4 and 1.9 introduce new versions of NMS. Search for all names to support TF 1.3~1.10
nms_node = model.ancestor(pillar, "ROI/rpn_non_max_suppression:0")
if nms_node is None:
nms_node = model.ancestor(
pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV2:0")
if nms_node is None: #TF 1.9-1.10
nms_node = model.ancestor(
pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV3:0")
log("neutral anchors", neutral_anchors)
# Apply refinement deltas to positive anchors
refined_anchors = utils.apply_box_deltas(
positive_anchors, target_rpn_bbox[:positive_anchors.shape[0]] *
model.config.RPN_BBOX_STD_DEV)
log(
"refined_anchors",
refined_anchors,
)
#%%
# Display positive anchors before refinement (dotted) and
# after refinement (solid).
visualize.draw_boxes(image,
boxes=positive_anchors,
refined_boxes=refined_anchors,
ax=get_ax())
#%% [markdown]
# ### 1.b RPN Predictions
#
# Here we run the RPN graph and display its predictions.
#%%
# Run RPN sub-graph
pillar = model.keras_model.get_layer(
"ROI").output # node to start searching from
# TF 1.4 and 1.9 introduce new versions of NMS. Search for all names to support TF 1.3~1.10
nms_node = model.ancestor(pillar, "ROI/rpn_non_max_suppression:0")
if nms_node is None:
def inspect_data(dataset, config):
print("Image Count: {}".format(len(dataset.image_ids)))
print("Class Count: {}".format(dataset.num_classes))
for i, info in enumerate(dataset.class_info):
print("{:3}. {:50}".format(i, info['name']))
# ## Display Samples
#
# Load and display images and masks.
# In[4]:
# Load and display random samples
image_ids = np.random.choice(dataset.image_ids, 4)
for image_id in image_ids:
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids,
dataset.class_names)
# ## Bounding Boxes
#
# Rather than using bounding box coordinates provided by the source datasets, we compute the bounding boxes from masks instead. This allows us to handle bounding boxes consistently regardless of the source dataset, and it also makes it easier to resize, rotate, or crop images because we simply generate the bounding boxes from the updates masks rather than computing bounding box transformation for each type of image transformation.
# In[5]:
# Load random image and mask.
image_id = random.choice(dataset.image_ids)
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id ", image_id, dataset.image_reference(image_id))
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Resize Images
#
# To support multiple images per batch, images are resized to one size (1024x1024). Aspect ratio is preserved, though. If an image is not square, then zero padding is added at the top/bottom or right/left.
# In[6]:
# Load random image and mask.
image_id = np.random.choice(dataset.image_ids, 1)[0]
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
original_shape = image.shape
# Resize
image, window, scale, padding, _ = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
mask = utils.resize_mask(mask, scale, padding)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id: ", image_id, dataset.image_reference(image_id))
print("Original shape: ", original_shape)
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Mini Masks
#
# Instance binary masks can get large when training with high resolution images. For example, if training with 1024x1024 image then the mask of a single instance requires 1MB of memory (Numpy uses bytes for boolean values). If an image has 100 instances then that's 100MB for the masks alone.
#
# To improve training speed, we optimize masks by:
# * We store mask pixels that are inside the object bounding box, rather than a mask of the full image. Most objects are small compared to the image size, so we save space by not storing a lot of zeros around the object.
# * We resize the mask to a smaller size (e.g. 56x56). For objects that are larger than the selected size we lose a bit of accuracy. But most object annotations are not very accuracy to begin with, so this loss is negligable for most practical purposes. Thie size of the mini_mask can be set in the config class.
#
# To visualize the effect of mask resizing, and to verify the code correctness, we visualize some examples.
# In[7]:
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
log("image", image)
log("image_meta", image_meta)
log("class_ids", class_ids)
log("bbox", bbox)
log("mask", mask)
display_images([image] +
[mask[:, :, i] for i in range(min(mask.shape[-1], 7))])
# In[8]:
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# In[9]:
# Add augmentation and mask resizing.
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, augment=True, use_mini_mask=True)
log("mask", mask)
display_images([image] +
[mask[:, :, i] for i in range(min(mask.shape[-1], 7))])
# In[10]:
mask = utils.expand_mask(bbox, mask, image.shape)
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Anchors
#
# The order of anchors is important. Use the same order in training and prediction phases. And it must match the order of the convolution execution.
#
# For an FPN network, the anchors must be ordered in a way that makes it easy to match anchors to the output of the convolution layers that predict anchor scores and shifts.
# * Sort by pyramid level first. All anchors of the first level, then all of the second and so on. This makes it easier to separate anchors by level.
# * Within each level, sort anchors by feature map processing sequence. Typically, a convolution layer processes a feature map starting from top-left and moving right row by row.
# * For each feature map cell, pick any sorting order for the anchors of different ratios. Here we match the order of ratios passed to the function.
#
# **Anchor Stride:**
# In the FPN architecture, feature maps at the first few layers are high resolution. For example, if the input image is 1024x1024 then the feature meap of the first layer is 256x256, which generates about 200K anchors (256*256*3). These anchors are 32x32 pixels and their stride relative to image pixels is 4 pixels, so there is a lot of overlap. We can reduce the load significantly if we generate anchors for every other cell in the feature map. A stride of 2 will cut the number of anchors by 4, for example.
#
# In this implementation we use an anchor stride of 2, which is different from the paper.
# In[11]:
# Generate Anchors
backbone_shapes = modellib.compute_backbone_shapes(config,
config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Print summary of anchors
num_levels = len(backbone_shapes)
anchors_per_cell = len(config.RPN_ANCHOR_RATIOS)
print("Count: ", anchors.shape[0])
print("Scales: ", config.RPN_ANCHOR_SCALES)
print("ratios: ", config.RPN_ANCHOR_RATIOS)
print("Anchors per Cell: ", anchors_per_cell)
print("Levels: ", num_levels)
anchors_per_level = []
for l in range(num_levels):
num_cells = backbone_shapes[l][0] * backbone_shapes[l][1]
anchors_per_level.append(anchors_per_cell * num_cells //
config.RPN_ANCHOR_STRIDE**2)
print("Anchors in Level {}: {}".format(l, anchors_per_level[l]))
# Visualize anchors of one cell at the center of the feature map of a specific level.
# In[12]:
## Visualize anchors of one cell at the center of the feature map of a specific level
# Load and draw random image
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, _, _, _ = modellib.load_image_gt(
dataset, config, image_id)
fig, ax = plt.subplots(1, figsize=(10, 10))
ax.imshow(image)
levels = len(backbone_shapes)
for level in range(levels):
colors = visualize.random_colors(levels)
# Compute the index of the anchors at the center of the image
level_start = sum(
anchors_per_level[:level]) # sum of anchors of previous levels
level_anchors = anchors[level_start:level_start +
anchors_per_level[level]]
print("Level {}. Anchors: {:6} Feature map Shape: {}".format(
level, level_anchors.shape[0], backbone_shapes[level]))
center_cell = backbone_shapes[level] // 2
center_cell_index = (center_cell[0] * backbone_shapes[level][1] +
center_cell[1])
level_center = center_cell_index * anchors_per_cell
center_anchor = anchors_per_cell * (
(center_cell[0] * backbone_shapes[level][1] / config.RPN_ANCHOR_STRIDE**2) \
+ center_cell[1] / config.RPN_ANCHOR_STRIDE)
level_center = int(center_anchor)
# Draw anchors. Brightness show the order in the array, dark to bright.
for i, rect in enumerate(level_anchors[level_center:level_center +
anchors_per_cell]):
y1, x1, y2, x2 = rect
p = patches.Rectangle(
(x1, y1),
x2 - x1,
y2 - y1,
linewidth=2,
facecolor='none',
edgecolor=(i + 1) * np.array(colors[level]) / anchors_per_cell)
ax.add_patch(p)
# ## Data Generator
#
# In[13]:
# Create data generator
random_rois = 2000
g = modellib.data_generator(dataset,
config,
shuffle=True,
random_rois=random_rois,
batch_size=4,
detection_targets=True)
# In[14]:
# Uncomment to run the generator through a lot of images
# to catch rare errors
# for i in range(1000):
# print(i)
# _, _ = next(g)
# In[15]:
# Get Next Image
if random_rois:
[
normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids,
gt_boxes, gt_masks, rpn_rois, rois
], [mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g)
log("rois", rois)
log("mrcnn_class_ids", mrcnn_class_ids)
log("mrcnn_bbox", mrcnn_bbox)
log("mrcnn_mask", mrcnn_mask)
else:
[
normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes,
gt_masks
], _ = next(g)
log("gt_class_ids", gt_class_ids)
log("gt_boxes", gt_boxes)
log("gt_masks", gt_masks)
log(
"rpn_match",
rpn_match,
)
log("rpn_bbox", rpn_bbox)
image_id = modellib.parse_image_meta(image_meta)["image_id"][0]
print("image_id: ", image_id, dataset.image_reference(image_id))
# Remove the last dim in mrcnn_class_ids. It's only added
# to satisfy Keras restriction on target shape.
mrcnn_class_ids = mrcnn_class_ids[:, :, 0]
# In[16]:
b = 0
# Restore original image (reverse normalization)
sample_image = modellib.unmold_image(normalized_images[b], config)
# Compute anchor shifts.
indices = np.where(rpn_match[b] == 1)[0]
refined_anchors = utils.apply_box_deltas(
anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV)
log("anchors", anchors)
log("refined_anchors", refined_anchors)
# Get list of positive anchors
positive_anchor_ids = np.where(rpn_match[b] == 1)[0]
print("Positive anchors: {}".format(len(positive_anchor_ids)))
negative_anchor_ids = np.where(rpn_match[b] == -1)[0]
print("Negative anchors: {}".format(len(negative_anchor_ids)))
neutral_anchor_ids = np.where(rpn_match[b] == 0)[0]
print("Neutral anchors: {}".format(len(neutral_anchor_ids)))
# ROI breakdown by class
for c, n in zip(dataset.class_names,
np.bincount(mrcnn_class_ids[b].flatten())):
if n:
print("{:23}: {}".format(c[:20], n))
# Show positive anchors
fig, ax = plt.subplots(1, figsize=(16, 16))
visualize.draw_boxes(sample_image,
boxes=anchors[positive_anchor_ids],
refined_boxes=refined_anchors,
ax=ax)
# In[17]:
# Show negative anchors
visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids])
# In[18]:
# Show neutral anchors. They don't contribute to training.
visualize.draw_boxes(sample_image,
boxes=anchors[np.random.choice(
neutral_anchor_ids, 100)])
# ## ROIs
# In[19]:
if random_rois:
# Class aware bboxes
bbox_specific = mrcnn_bbox[b,
np.arange(mrcnn_bbox.shape[1]),
mrcnn_class_ids[b], :]
# Refined ROIs
refined_rois = utils.apply_box_deltas(
rois[b].astype(np.float32),
bbox_specific[:, :4] * config.BBOX_STD_DEV)
# Class aware masks
mask_specific = mrcnn_mask[b,
np.arange(mrcnn_mask.shape[1]), :, :,
mrcnn_class_ids[b]]
visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific,
mrcnn_class_ids[b], dataset.class_names)
# Any repeated ROIs?
rows = np.ascontiguousarray(rois[b]).view(
np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1])))
_, idx = np.unique(rows, return_index=True)
print("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1]))
# In[20]:
if random_rois:
# Dispalay ROIs and corresponding masks and bounding boxes
ids = random.sample(range(rois.shape[1]), 8)
images = []
titles = []
for i in ids:
image = visualize.draw_box(sample_image.copy(),
rois[b, i, :4].astype(np.int32),
[255, 0, 0])
image = visualize.draw_box(image, refined_rois[i].astype(np.int64),
[0, 255, 0])
images.append(image)
titles.append("ROI {}".format(i))
images.append(mask_specific[i] * 255)
titles.append(dataset.class_names[mrcnn_class_ids[b, i]][:20])
display_images(images,
titles,
cols=4,
cmap="Blues",
interpolation="none")
# In[21]:
# Check ratio of positive ROIs in a set of images.
if random_rois:
limit = 10
temp_g = modellib.data_generator(dataset,
config,
shuffle=True,
random_rois=10000,
batch_size=1,
detection_targets=True)
total = 0
for i in range(limit):
_, [ids, _, _] = next(temp_g)
positive_rois = np.sum(ids[0] > 0)
total += positive_rois
print("{:5} {:5.2f}".format(positive_rois,
positive_rois / ids.shape[1]))
print("Average percent: {:.2f}".format(total / (limit * ids.shape[1])))
target_rpn_bbox[:positive_anchors.shape[0]] * model.config.RPN_BBOX_STD_DEV)
log("refined_anchors", refined_anchors, )
# Run RPN sub-graph
pillar = model.keras_model.get_layer("ROI").output # node to start searching from
# TF 1.4 and 1.9 introduce new versions of NMS. Search for all names to support TF 1.3~1.10
nms_node = model.ancestor(pillar, "ROI/rpn_non_max_suppression:0")
if nms_node is None:
nms_node = model.ancestor(pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV2:0")
if nms_node is None: #TF 1.9-1.10
nms_node = model.ancestor(pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV3:0")
rpn = model.run_graph([image], [
("rpn_class", model.keras_model.get_layer("rpn_class").output),
("pre_nms_anchors", model.ancestor(pillar, "ROI/pre_nms_anchors:0")),
("refined_anchors", model.ancestor(pillar, "ROI/refined_anchors:0")),
("refined_anchors_clipped", model.ancestor(pillar, "ROI/refined_anchors_clipped:0")),
("post_nms_anchor_ix", nms_node),
("proposals", model.keras_model.get_layer("ROI").output),
])
limit = 50
ax = get_ax(1, 2)
pre_nms_anchors = utils.denorm_boxes(rpn["pre_nms_anchors"][0], image.shape[:2])
refined_anchors = utils.denorm_boxes(rpn["refined_anchors"][0], image.shape[:2])
refined_anchors_clipped = utils.denorm_boxes(rpn["refined_anchors_clipped"][0], image.shape[:2])
visualize.draw_boxes(image, boxes=pre_nms_anchors[:limit],
refined_boxes=refined_anchors[:limit], ax=ax[0])
visualize.draw_boxes(image, refined_boxes=refined_anchors_clipped[:limit], ax=ax[1])
splash = balloon.color_splash(image, r['masks'])
display_images([splash], cols=1)
# Generate RPN trainig targets
# target_rpn_match is 1 for positive anchors, -1 for negative anchors
# and 0 for neutral anchors.
target_rpn_match, target_rpn_bbox = modellib.build_rpn_targets(
image.shape, model.anchors, gt_class_id, gt_bbox, model.config)
log("target_rpn_match", target_rpn_match)
log("target_rpn_bbox", target_rpn_bbox)
positive_anchor_ix = np.where(target_rpn_match[:] == 1)[0]
negative_anchor_ix = np.where(target_rpn_match[:] == -1)[0]
neutral_anchor_ix = np.where(target_rpn_match[:] == 0)[0]
positive_anchors = model.anchors[positive_anchor_ix]
negative_anchors = model.anchors[negative_anchor_ix]
neutral_anchors = model.anchors[neutral_anchor_ix]
log("positive_anchors", positive_anchors)
log("negative_anchors", negative_anchors)
log("neutral anchors", neutral_anchors)
# Apply refinement deltas to positive anchors
refined_anchors = utils.apply_box_deltas(
positive_anchors,
target_rpn_bbox[:positive_anchors.shape[0]] * model.config.RPN_BBOX_STD_DEV)
log("refined_anchors", refined_anchors, )
# Display positive anchors before refinement (dotted) and
# after refinement (solid).
visualize.draw_boxes(image, boxes=positive_anchors, refined_boxes=refined_anchors, ax=get_ax())
print("Positive anchors: {}".format(len(positive_anchor_ids)))
negative_anchor_ids = np.where(rpn_match[b] == -1)[0]
print("Negative anchors: {}".format(len(negative_anchor_ids)))
neutral_anchor_ids = np.where(rpn_match[b] == 0)[0]
print("Neutral anchors: {}".format(len(neutral_anchor_ids)))
# ROI breakdown by class
for c, n in zip(dataset.class_names,
np.bincount(mrcnn_class_ids[b].flatten())):
if n:
print("{:23}: {}".format(c[:20], n))
# Show positive anchors
fig, ax = plt.subplots(1, figsize=(16, 16))
visualize.draw_boxes(sample_image,
boxes=anchors[positive_anchor_ids],
refined_boxes=refined_anchors,
ax=ax)
positive_anchor_image_name = "pos_ancher_img.png"
plt.savefig(os.path.join(ASSETS_DIR, positive_anchor_image_name))
visualize.draw_boxes(sample_image,
boxes=anchors[negative_anchor_ids],
img_name="negative_anchors")
# Show neutral anchors. They don't contribute to training.
visualize.draw_boxes(sample_image,
boxes=anchors[np.random.choice(neutral_anchor_ids, 100)],
img_name="neutral_anchors")
if random_rois:
log("negative_anchors", negative_anchors)
log("neutral anchors", neutral_anchors)
# Apply refinement deltas to positive anchors
refined_anchors = utils.apply_box_deltas(
positive_anchors, target_rpn_bbox[:positive_anchors.shape[0]] *
model.config.RPN_BBOX_STD_DEV)
log(
"refined_anchors",
refined_anchors,
)
# Display positive anchors before refinement (dotted) and
# after refinement (solid).
visualize.draw_boxes(image,
ax=get_ax(),
boxes=positive_anchors,
refined_boxes=refined_anchors)
# Run RPN sub-graph
pillar = model.keras_model.get_layer(
"ROI").output # node to start searching from
# TF 1.4 introduces a new version of NMS. Search for both names to support TF 1.3 and 1.4
nms_node = model.ancestor(pillar, "ROI/rpn_non_max_suppression:0")
if nms_node is None:
nms_node = model.ancestor(
pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV2:0")
rpn = model.run_graph(image[np.newaxis], [
("rpn_class", model.keras_model.get_layer("rpn_class").output),
("pre_nms_anchors", model.ancestor(pillar, "ROI/pre_nms_anchors:0")),
def display_mrcnn_prediction():
resized_image, image_meta, gt_class_ids, gt_bboxes, gt_masks = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
# Get input and output to classifier and mask heads.
mrcnn = model.run_graph([resized_image], [
("proposals", model.keras_model.get_layer("ROI").output),
("probs", model.keras_model.get_layer("mrcnn_class").output),
("deltas", model.keras_model.get_layer("mrcnn_bbox").output),
("masks", model.keras_model.get_layer("mrcnn_mask").output),
("detections", model.keras_model.get_layer("mrcnn_detection").output),
])
ax = get_ax(1, 4)
################################## display detections ###############################################
# Get detection class IDs. Trim zero padding.
det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)
padding_start_ix = np.where(det_class_ids == 0)[0][0]
det_class_ids = det_class_ids[:padding_start_ix]
detections = mrcnn['detections'][0, :padding_start_ix]
log('trimmed_detection', detections)
print("{} detections: {}".format(
padding_start_ix,
np.array(dataset.class_names)[det_class_ids]))
captions = [
"{} {:.3f}".format(dataset.class_names[int(class_id)], score)
if class_id > 0 else ""
for class_id, score in zip(detections[:, 4], detections[:, 5])
]
visualize.draw_boxes(resized_image.copy(),
refined_boxes=utils.denorm_boxes(
detections[:, :4], resized_image.shape[:2]),
visibilities=[2] * len(detections),
captions=captions,
title="Detections",
ax=ax[0])
################################### display proposals ##########################################
# Proposals are in normalized coordinates. Scale them to image coordinates.
h, w = resized_image.shape[:2]
proposals = np.around(mrcnn["proposals"][0] *
np.array([h, w, h, w])).astype(np.int32)
# Class ID, score, and mask per proposal
# mrcnn 的 shape 为 (batch_size, num_proposals=1000, num_classes)
proposal_class_ids = np.argmax(mrcnn["probs"][0], axis=1)
proposal_class_scores = mrcnn["probs"][
0, np.arange(proposal_class_ids.shape[0]), proposal_class_ids]
proposal_class_names = np.array(dataset.class_names)[proposal_class_ids]
proposal_positive_ixs = np.where(proposal_class_ids > 0)[0]
# How many ROIs vs empty rows?
print("{} valid proposals out of {}".format(
np.sum(np.any(proposals, axis=1)), proposals.shape[0]))
print("{} positive ROIs".format(len(proposal_positive_ixs)))
# Class counts
print(list(zip(*np.unique(proposal_class_names, return_counts=True))))
# Display a random sample of proposals.
# Proposals classified as background are dotted, and
# the rest show their class and confidence score.
limit = 200
ixs = np.random.randint(0, proposals.shape[0], limit)
captions = [
"{} {:.3f}".format(dataset.class_names[c], s) if c > 0 else ""
for c, s in zip(proposal_class_ids[ixs], proposal_class_scores[ixs])
]
visualize.draw_boxes(resized_image.copy(),
boxes=proposals[ixs],
visibilities=np.where(proposal_class_ids[ixs] > 0, 2,
1),
captions=captions,
title="Proposals Before Refinement",
ax=ax[1])
#################################### apply bounding box refinement #############################
# Class-specific bounding box shifts.
# mrcnn['deltas'] 的 shape 为 (batch_size, num_proposals=1000, num_classes, 4)
proposal_deltas = mrcnn["deltas"][0,
np.arange(proposals.shape[0]),
proposal_class_ids]
log("proposals_deltas", proposal_deltas)
# Apply bounding box transformations
# Shape: (num_proposals=1000, (y1, x1, y2, x2)]
# NOTE: delta 是不分 normalized coordinates 和 pixel coordinates 的
refined_proposals = utils.apply_box_deltas(
proposals, proposal_deltas * config.BBOX_STD_DEV).astype(np.int32)
log("refined_proposals", refined_proposals)
# Show positive proposals
# ids = np.arange(proposals.shape[0]) # Display all
limit = 5
ids = np.random.randint(0, len(proposal_positive_ixs),
limit) # Display random sample
captions = [
"{} {:.3f}".format(dataset.class_names[class_id], score)
if class_id > 0 else "" for class_id, score in zip(
proposal_class_ids[proposal_positive_ixs][ids],
proposal_class_scores[proposal_positive_ixs][ids])
]
visualize.draw_boxes(
resized_image.copy(),
boxes=proposals[proposal_positive_ixs][ids],
refined_boxes=refined_proposals[proposal_positive_ixs][ids],
visibilities=np.where(
proposal_class_ids[proposal_positive_ixs][ids] > 0, 1, 0),
captions=captions,
title="ROIs After Refinement",
ax=ax[2])
#################################### more steps ################################################
# Remove boxes classified as background
keep_proposal_ixs = np.where(proposal_class_ids > 0)[0]
print("Remove background proposals. Keep {}:\n{}".format(
keep_proposal_ixs.shape[0], keep_proposal_ixs))
# Remove low confidence detections
keep_proposal_ixs = np.intersect1d(
keep_proposal_ixs,
np.where(proposal_class_scores >= config.DETECTION_MIN_CONFIDENCE)[0])
print("Remove proposals below {} confidence. Keep {}:\n{}".format(
config.DETECTION_MIN_CONFIDENCE, keep_proposal_ixs.shape[0],
keep_proposal_ixs))
# Apply per-class non-max suppression
pre_nms_proposals = refined_proposals[keep_proposal_ixs]
pre_nms_proposal_scores = proposal_class_scores[keep_proposal_ixs]
pre_nms_proposal_class_ids = proposal_class_ids[keep_proposal_ixs]
nms_keep_proposal_ixs = []
for class_id in np.unique(pre_nms_proposal_class_ids):
# Pick detections of this class
ixs = np.where(pre_nms_proposal_class_ids == class_id)[0]
# Apply NMS
class_keep = utils.non_max_suppression(pre_nms_proposals[ixs],
pre_nms_proposal_scores[ixs],
config.DETECTION_NMS_THRESHOLD)
# Map indicies
class_keep_proposal_ixs = keep_proposal_ixs[ixs[class_keep]]
nms_keep_proposal_ixs = np.union1d(nms_keep_proposal_ixs,
class_keep_proposal_ixs)
print("{:12}: {} -> {}".format(dataset.class_names[class_id][:10],
keep_proposal_ixs[ixs],
class_keep_proposal_ixs))
keep_proposal_ixs = np.intersect1d(keep_proposal_ixs,
nms_keep_proposal_ixs).astype(np.int32)
print("\nKeep after per-class NMS: {}\n{}".format(
keep_proposal_ixs.shape[0], keep_proposal_ixs))
#################################### Show final detections #####################################
ixs = np.arange(len(keep_proposal_ixs)) # Display all
# ixs = np.random.randint(0, len(keep), 10) # Display random sample
captions = [
"{} {:.3f}".format(dataset.class_names[c], s) if c > 0 else ""
for c, s in zip(proposal_class_ids[keep_proposal_ixs][ixs],
proposal_class_scores[keep_proposal_ixs][ixs])
]
visualize.draw_boxes(
resized_image.copy(),
boxes=proposals[keep_proposal_ixs][ixs],
refined_boxes=refined_proposals[keep_proposal_ixs][ixs],
visibilities=np.where(proposal_class_ids[keep_proposal_ixs][ixs] > 0,
1, 0),
captions=captions,
title="Detections after NMS",
ax=ax[3])
plt.show()
def display_rpn_prediction():
# Run RPN sub-graph
resized_image, image_meta, gt_class_ids, gt_bboxes, gt_masks = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
pillar = model.keras_model.get_layer(
"ROI").output # node to start searching from
# TF 1.4 and 1.9 introduce new versions of NMS. Search for all names to support TF 1.3~1.10
nms_node = model.ancestor(pillar, "ROI/rpn_non_max_suppression:0")
if nms_node is None:
nms_node = model.ancestor(
pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV2:0")
if nms_node is None: # TF 1.9-1.10
nms_node = model.ancestor(
pillar, "ROI/rpn_non_max_suppression/NonMaxSuppressionV3:0")
rpn = model.run_graph([resized_image], [
("rpn_class", model.keras_model.get_layer("rpn_class").output),
("pre_nms_anchors", model.ancestor(pillar, "ROI/pre_nms_anchors:0")),
("refined_anchors", model.ancestor(pillar, "ROI/refined_anchors:0")),
("refined_anchors_clipped",
model.ancestor(pillar, "ROI/refined_anchors_clipped:0")),
("post_nms_anchor_ix", nms_node),
("proposals", model.keras_model.get_layer("ROI").output),
])
ax = get_ax(2, 3)
# Show top anchors by score (before refinement)
limit = 100
# np.flatten() 会把多维数组变成一维数组, 那么此处就默认 batch_size=1, 否则不能这样计算
# 按从大到小排序
sorted_anchor_ids = np.argsort(rpn['rpn_class'][:, :, 1].flatten())[::-1]
visualize.draw_boxes(resized_image,
boxes=model.anchors[sorted_anchor_ids[:limit]],
ax=ax[0, 0])
# Show top anchors with refinement. Then with clipping to image boundaries
limit = 50
pre_nms_anchors = utils.denorm_boxes(rpn["pre_nms_anchors"][0],
resized_image.shape[:2])
refined_anchors = utils.denorm_boxes(rpn["refined_anchors"][0],
resized_image.shape[:2])
visualize.draw_boxes(resized_image,
boxes=pre_nms_anchors[:limit],
refined_boxes=refined_anchors[:limit],
ax=ax[0, 1])
refined_anchors_clipped = utils.denorm_boxes(
rpn["refined_anchors_clipped"][0], resized_image.shape[:2])
visualize.draw_boxes(resized_image,
refined_boxes=refined_anchors_clipped[:limit],
ax=ax[0, 2])
# Show refined anchors after non-max suppression
ixs = rpn["post_nms_anchor_ix"][:limit]
visualize.draw_boxes(resized_image,
refined_boxes=refined_anchors_clipped[ixs],
ax=ax[1, 0])
# Show final proposals
# These are the same as the previous step (refined anchors
# after NMS) but with coordinates normalized to [0, 1] range.
# Convert back to image coordinates for display
h, w = resized_image.shape[:2]
proposals = rpn['proposals'][0, :limit] * np.array([h, w, h, w])
visualize.draw_boxes(resized_image, refined_boxes=proposals, ax=ax[1, 1])
plt.show()
def get_labels(self, labels):
dims = labels.shape
unlabeled_labels = np.zeros((dims[0], dims[1], 1))
building_labels = np.zeros((dims[0], dims[1], 1))
fence_labels = np.zeros((dims[0], dims[1], 1))
other_labels = np.zeros((dims[0], dims[1], 1))
pedestrian_labels = np.zeros((dims[0], dims[1], 1))
pole_labels = np.zeros((dims[0], dims[1], 1))
road_line_labels = np.zeros((dims[0], dims[1], 1))
road_labels = np.zeros((dims[0], dims[1], 1))
sidewalk_labels = np.zeros((dims[0], dims[1], 1))
vegetation_labels = np.zeros((dims[0], dims[1], 1))
car_labels = np.zeros((dims[0], dims[1], 1))
wall_labels = np.zeros((dims[0], dims[1], 1))
traffic_sign_labels = np.zeros((dims[0], dims[1], 1))
unlabeled_index = np.all(labels == (0, 0, 0), axis=-1)
building_index = np.all(labels == (70, 70, 70), axis=-1)
fence_index = np.all(labels == (190, 153, 153), axis=-1)
other_index = np.all(labels == (250, 170, 160), axis=-1)
pedestrian_index = np.all(labels == (220, 20, 60), axis=-1)
pole_index = np.all(labels == (153, 153, 153), axis=-1)
road_line_index = np.all(labels == (157, 234, 50), axis=-1)
road_index = np.all(labels == (128, 64, 128), axis=-1)
sidewalk_index = np.all(labels == (244, 35, 232), axis=-1)
vegetation_index = np.all(labels == (107, 142, 35), axis=-1)
car_index = np.all(labels == (0, 0, 142), axis=-1)
wall_index = np.all(labels == (102, 102, 156), axis=-1)
traffic_sign_index = np.all(labels == (220, 220, 70), axis=-1)
unlabeled_labels[unlabeled_index] = 1
building_labels[building_index] = 10
fence_labels[fence_index] = 10
other_labels[other_index] = 10
pedestrian_labels[pedestrian_index] = 10
pole_labels[pole_index] = 10
road_line_labels[road_line_index] = 10
road_labels[road_index] = 10
sidewalk_labels[sidewalk_index] = 10
vegetation_labels[vegetation_index] = 1
car_labels[car_index] = 10
wall_labels[wall_index] = 10
traffic_sign_labels[traffic_sign_index] = 10
return np.dstack([unlabeled_labels, building_labels, fence_labels,
return np.dstack([unlabeled_labels, building_labels, fence_labels,
other_labels, pedestrian_labels, pole_labels,
road_line_labels, road_labels, sidewalk_labels, vegetation_labels,
car_labels, wall_labels, traffic_sign_labels])
def image_reference(self, image_id):
"""Return the carla data of the image."""
info = self.image_info[image_id]
if info["source"] == "carla":
return info["id"]
else:
super(self.__class__).image_reference(self, image_id)
config = CarlaConfig()
config.STEPS_PER_EPOCH = NUMBER_OF_TRAIN_DATA//config.BATCH_SIZE
config.VALIDATION_STEPS = NUMBER_OF_VAL_DATA//config.BATCH_SIZE
config.display()
dataset = carlaDataset()
dataset.load_images(dir=RGB_TRAIN_DIR, type='train')
# mask, a = train.load_mask(50)
# print(a)
dataset.prepare()
print("Image Count: {}".format(len(dataset.image_ids)))
print("Class Count: {}".format(dataset.num_classes))
for i, info in enumerate(dataset.class_info):
print("{:3}. {:50}".format(i, info['name']))
image_ids = np.random.choice(dataset.image_ids, 4)
for image_id in image_ids:
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids, dataset.class_names)
# Load random image and mask.
image_id = random.choice(dataset.image_ids)
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id ", image_id)
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
# Load random image and mask.
image_id = np.random.choice(dataset.image_ids, 1)[0]
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
original_shape = image.shape
# Resize
image, window, scale, padding, _ = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
mask = utils.resize_mask(mask, scale, padding)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id: ", image_id)
print("Original shape: ", original_shape)
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
log("image", image)
log("image_meta", image_meta)
log("class_ids", class_ids)
log("bbox", bbox)
log("mask", mask)
display_images([image]+[mask[:,:,i] for i in range(min(mask.shape[-1], 7))])
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
# Generate Anchors
backbone_shapes = modellib.compute_backbone_shapes(config, config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Print summary of anchors
num_levels = len(backbone_shapes)
anchors_per_cell = len(config.RPN_ANCHOR_RATIOS)
print("Count: ", anchors.shape[0])
print("Scales: ", config.RPN_ANCHOR_SCALES)
print("ratios: ", config.RPN_ANCHOR_RATIOS)
print("Anchors per Cell: ", anchors_per_cell)
print("Levels: ", num_levels)
anchors_per_level = []
for l in range(num_levels):
num_cells = backbone_shapes[l][0] * backbone_shapes[l][1]
anchors_per_level.append(anchors_per_cell * num_cells // config.RPN_ANCHOR_STRIDE**2)
print("Anchors in Level {}: {}".format(l, anchors_per_level[l]))
## Visualize anchors of one cell at the center of the feature map of a specific level
# Load and draw random image
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, _, _, _ = modellib.load_image_gt(dataset, config, image_id)
fig, ax = plt.subplots(1, figsize=(10, 10))
ax.imshow(image)
levels = len(backbone_shapes)
for level in range(levels):
colors = visualize.random_colors(levels)
# Compute the index of the anchors at the center of the image
level_start = sum(anchors_per_level[:level]) # sum of anchors of previous levels
level_anchors = anchors[level_start:level_start+anchors_per_level[level]]
print("Level {}. Anchors: {:6} Feature map Shape: {}".format(level, level_anchors.shape[0],
backbone_shapes[level]))
center_cell = backbone_shapes[level] // 2
center_cell_index = (center_cell[0] * backbone_shapes[level][1] + center_cell[1])
level_center = center_cell_index * anchors_per_cell
center_anchor = anchors_per_cell * (
(center_cell[0] * backbone_shapes[level][1] / config.RPN_ANCHOR_STRIDE**2) \
+ center_cell[1] / config.RPN_ANCHOR_STRIDE)
level_center = int(center_anchor)
# Draw anchors. Brightness show the order in the array, dark to bright.
for i, rect in enumerate(level_anchors[level_center:level_center+anchors_per_cell]):
y1, x1, y2, x2 = rect
p = patches.Rectangle((x1, y1), x2-x1, y2-y1, linewidth=2, facecolor='none',
edgecolor=(i+1)*np.array(colors[level]) / anchors_per_cell)
ax.add_patch(p)
# Create data generator
random_rois = 4000
g = modellib.data_generator(
dataset, config, shuffle=True, random_rois=random_rois,
batch_size=4,
detection_targets=True)
# Get Next Image
if random_rois:
[normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, rpn_rois, rois], \
[mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g)
log("rois", rois)
log("mrcnn_class_ids", mrcnn_class_ids)
log("mrcnn_bbox", mrcnn_bbox)
log("mrcnn_mask", mrcnn_mask)
else:
[normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes, gt_masks], _ = next(g)
log("gt_class_ids", gt_class_ids)
log("gt_boxes", gt_boxes)
log("gt_masks", gt_masks)
log("rpn_match", rpn_match, )
log("rpn_bbox", rpn_bbox)
image_id = modellib.parse_image_meta(image_meta)["image_id"][0]
print("image_id: ", image_id, dataset.image_reference(image_id))
# Remove the last dim in mrcnn_class_ids. It's only added
# to satisfy Keras restriction on target shape.
mrcnn_class_ids = mrcnn_class_ids[:, :, 0]
b = 0
# Restore original image (reverse normalization)
sample_image = modellib.unmold_image(normalized_images[b], config)
# Compute anchor shifts.
indices = np.where(rpn_match[b] == 1)[0]
refined_anchors = utils.apply_box_deltas(anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV)
log("anchors", anchors)
log("refined_anchors", refined_anchors)
# Get list of positive anchors
positive_anchor_ids = np.where(rpn_match[b] == 1)[0]
print("Positive anchors: {}".format(len(positive_anchor_ids)))
negative_anchor_ids = np.where(rpn_match[b] == -1)[0]
print("Negative anchors: {}".format(len(negative_anchor_ids)))
neutral_anchor_ids = np.where(rpn_match[b] == 0)[0]
print("Neutral anchors: {}".format(len(neutral_anchor_ids)))
# ROI breakdown by class
for c, n in zip(dataset.class_names, np.bincount(mrcnn_class_ids[b].flatten())):
if n:
print("{:23}: {}".format(c[:20], n))
# Show positive anchors
visualize.draw_boxes(sample_image, boxes=anchors[positive_anchor_ids],
refined_boxes=refined_anchors)
# Show negative anchors
visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids])
# Show neutral anchors. They don't contribute to training.
visualize.draw_boxes(sample_image, boxes=anchors[np.random.choice(neutral_anchor_ids, 100)])
if random_rois:
# Class aware bboxes
bbox_specific = mrcnn_bbox[b, np.arange(mrcnn_bbox.shape[1]), mrcnn_class_ids[b], :]
# Refined ROIs
refined_rois = utils.apply_box_deltas(rois[b].astype(np.float32), bbox_specific[:, :4] * config.BBOX_STD_DEV)
# Class aware masks
mask_specific = mrcnn_mask[b, np.arange(mrcnn_mask.shape[1]), :, :, mrcnn_class_ids[b]]
visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific, mrcnn_class_ids[b], dataset.class_names)
# Any repeated ROIs?
rows = np.ascontiguousarray(rois[b]).view(np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1])))
_, idx = np.unique(rows, return_index=True)
print("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1]))
if random_rois:
# Dispalay ROIs and corresponding masks and bounding boxes
ids = random.sample(range(rois.shape[1]), 8)
images = []
titles = []
for i in ids:
image = visualize.draw_box(sample_image.copy(), rois[b,i,:4].astype(np.int32), [255, 0, 0])
image = visualize.draw_box(image, refined_rois[i].astype(np.int64), [0, 255, 0])
images.append(image)
titles.append("ROI {}".format(i))
images.append(mask_specific[i] * 255)
titles.append(dataset.class_names[mrcnn_class_ids[b,i]][:20])
display_images(images, titles, cols=4, cmap="Blues", interpolation="none")
# Check ratio of positive ROIs in a set of images.
if random_rois:
limit = 10
temp_g = modellib.data_generator(
dataset, config, shuffle=True, random_rois=10000,
batch_size=1, detection_targets=True)
total = 0
for i in range(limit):
_, [ids, _, _] = next(temp_g)
positive_rois = np.sum(ids[0] > 0)
total += positive_rois
print("{:5} {:5.2f}".format(positive_rois, positive_rois/ids.shape[1]))
print("Average percent: {:.2f}".format(total/(limit*ids.shape[1])))
exit()
refined_anchors = utils.apply_box_deltas(
anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV)
log("anchors", anchors)
log("refined_anchors", refined_anchors)
# 获取其中默认第一张图片的数据，打印正样本标记结果和负样本标记结果
sample_image = model.unmold_image(normalized_images[b], config)
# ROI的类别数量
for c, n in zip(dataset_train.class_names,
np.bincount(mrcnn_class_ids[b].flatten())):
if n:
print("{:23}: {}".format(c[:20], n))
# 展示正样本输出结果
import matplotlib.pyplot as plt
fig, ax = plt.subplots(1, figsize=(16, 16))
visualize.draw_boxes(sample_image,
boxes=anchors[positive_anchor_ids],
refined_boxes=refined_anchors,
ax=ax)
# 展示负样本输出
visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids])
# 6、正负rois区域比例
print("Positive ROIs: ",
mrcnn_class_ids[b][mrcnn_class_ids[b] > 0].shape[0])
print("Negative ROIs: ",
mrcnn_class_ids[b][mrcnn_class_ids[b] == 0].shape[0])
print("Positive Ratio: {:.2f}".format(
mrcnn_class_ids[b][mrcnn_class_ids[b] > 0].shape[0] /
mrcnn_class_ids[b].shape[0]))
# Get list of positive anchors
positive_anchor_ids = np.where(rpn_match[b] == 1)[0]
print("Positive anchors: {}".format(len(positive_anchor_ids)))
negative_anchor_ids = np.where(rpn_match[b] == -1)[0]
print("Negative anchors: {}".format(len(negative_anchor_ids)))
neutral_anchor_ids = np.where(rpn_match[b] == 0)[0]
print("Neutral anchors: {}".format(len(neutral_anchor_ids)))
# ROI breakdown by class
for c, n in zip(dataset.class_names, np.bincount(mrcnn_class_ids[b].flatten())):
if n:
print("{:23}: {}".format(c[:20], n))
# Show positive anchors
visualize.draw_boxes(sample_image, boxes=anchors[positive_anchor_ids],
refined_boxes=refined_anchors)
# Show negative anchors
visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids])
# Show neutral anchors. They don't contribute to training.
visualize.draw_boxes(sample_image, boxes=anchors[np.random.choice(neutral_anchor_ids, 100)])
if random_rois:
# Class aware bboxes
bbox_specific = mrcnn_bbox[b, np.arange(mrcnn_bbox.shape[1]), mrcnn_class_ids[b], :]
# Refined ROIs
refined_rois = utils.apply_box_deltas(rois[b].astype(np.float32), bbox_specific[:, :4] * config.BBOX_STD_DEV)
# points = [[100.0, 100.0, 100.0]]
ranchor_index, x_points, rpoint_fmap = utils.rpoint_image_mapping(
points, orig_shape[:2],
(config.IMAGE_MAX_DIM, config.IMAGE_MAX_DIM), config)
# Run detection returns detection on input image
t = time.time()
r = model.detect([image],
rpoint_fmap,
ranchor_index,
net=args.net,
verbose=0)[0]
t_prediction += (time.time() - t)
N = r['rois'].shape[0]
for i in range(N):
print("bbox, class name, score", r['rois'][i],
dataset.class_names[r['class_ids'][i]], r['scores'][i])
visualize.draw_boxes(image,
x_points,
r['class_ids'],
dataset.class_names,
boxes=r['rois'])
visualize.display_instances(image, r['rois'], r['class_ids'],
dataset.class_names, r['scores'])
else:
print("'{}' is not recognized. "
"Use 'train' or 'evaluate'".format(args.command))
