def loadSameResImage(imagePath, imageShape):
mask = cv2.imread(imagePath, cv2.IMREAD_UNCHANGED)
if mask.shape[0] != imageShape[0] or mask.shape[1] != imageShape[1]:
bbox = getBboxFromName(imagePath)
mask = expand_mask(bbox, mask, image_shape=imageShape)
mask = mask.astype(np.uint8) * 255
return mask
def create_multiclass_mask(image_shape, results: dict, config: Config = None):
"""
Creates an image containing all the masks where pixel color is the mask's class ID
:param image_shape: the shape of the initial image
:param results: the results dictionary containing all the masks
:param config: the config object used to expand mini_masks if enabled
:return: the multi-mask image
"""
res = np.zeros((image_shape[0], image_shape[1]), np.uint8)
masks = results['masks']
class_ids = results['class_ids']
rois = results['rois']
indices = np.arange(len(class_ids))
classes_hierarchy = config.get_classes_hierarchy()
if classes_hierarchy is None:
levels = [[i + 1 for i in range(len(config.get_classes_info()))]]
else:
levels = utils.remove_redundant_classes(
utils.classes_level(classes_hierarchy), keepFirst=False)
for lvl in levels:
current_indices = indices[np.isin(class_ids, lvl)]
for idx in current_indices:
mask = masks[:, :, idx].astype(bool).astype(np.uint8) * 255
roi = rois[idx]
classID = int(class_ids[idx])
if config is not None and config.is_using_mini_mask():
shifted_bbox = utils.shift_bbox(roi)
mask = utils.expand_mask(shifted_bbox, mask, shifted_bbox[2:])
res = apply_mask(res, mask, classID, 1, roi)
return res
def mini_mask(image_id):
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
log("image", image)
log("image_meta", image_meta)
log("class_ids", class_ids)
log("bbox", bbox)
log("mask", mask)
display_images([image] +
[mask[:, :, i] for i in range(min(mask.shape[-1], 7))])
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# Add augmentation and mask resizing.
# 取 mask 的 bbox, resize 到固定大小
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, augment=True, use_mini_mask=True)
log("mask", mask)
display_images([image] +
[mask[:, :, i] for i in range(min(mask.shape[-1], 7))])
# 把 mask 恢复到原来大小
mask = utils.expand_mask(bbox, mask, image.shape)
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
def loadOnlyMask(imagePath, imageShape):
mask = cv2.imread(imagePath, cv2.IMREAD_UNCHANGED)
if mask.shape[0] != imageShape[0] or mask.shape[1] != imageShape[1]:
# Finding bbox coordinates from image name
bbox = getBboxFromName(imagePath)
shifted = shift_bbox(bbox)
y1, x1, y2, x2 = shifted
# Expanding mask to its real size
mask = expand_mask(shifted, mask, image_shape=shifted[2:])
mask = mask.astype(np.uint8) * 255
else:
# Extracting bbox of
bbox = extract_bboxes(mask)
y1, x1, y2, x2 = bbox
return mask[y1:y2, x1:x2, ...], bbox
def add_augmentation(dataset, datacfg, dnncfg, image_id=None):
'''
# Add augmentation and mask resizing.
'''
log.info("add_augmentation::-------------------------------->")
image_id = image_id if image_id==None else np.random.choice(dataset.image_ids, 1)[0]
# Add augmentation and mask resizing.
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(dataset, datacfg, dnncfg, image_id, augment=True, use_mini_mask=False)
# customlog("mask", mask)
visualize.display_images([image]+[mask[:,:,i] for i in range(min(mask.shape[-1], 7))])
mask = utils.expand_mask(bbox, mask, image.shape)
## Display image and instances
class_names = dataset.class_names
visualize.display_instances(image, bbox, mask, class_ids, class_names)
def add_augmentation(self, dataset, config, ds_datacfg, image_id):
'''
# Add augmentation and mask resizing.
'''
print("add_augmentation::-------------------------------->")
# image_id = image_id if image_id==None else np.random.choice(dataset.image_ids, 1)[0]
datacfg = None
if ds_datacfg:
info = dataset.image_info[image_id]
ds_source = info['source']
datacfg = utils.get_datacfg(ds_datacfg, ds_source)
# Add augmentation and mask resizing.
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, datacfg, config, image_id, augment=True, use_mini_mask=True)
log("mask", mask)
display_images([image]+[mask[:,:,i] for i in range(min(mask.shape[-1], 7))])
mask = utils.expand_mask(bbox, mask, image.shape)
## Display image and instances
class_names = dataset.class_names
self.display_instances(image, bbox, mask, class_ids, class_names)
log("image_meta", image_meta)
log("class_ids", class_ids)
log("bbox", bbox)
log("mask", mask)
display_images([image]+[mask[:,:,i] for i in range(min(mask.shape[-1], 7))])
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
# Add augmentation and mask resizing.
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, augment=True, use_mini_mask=True)
log("mask", mask)
display_images([image]+[mask[:,:,i] for i in range(min(mask.shape[-1], 7))])
mask = utils.expand_mask(bbox, mask, image.shape)
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
# 锚框生成
# Generate Anchors
backbone_shapes = modellib.compute_backbone_shapes(config, config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Print summary of anchors
num_levels = len(backbone_shapes)
anchors_per_cell = len(config.RPN_ANCHOR_RATIOS)
print("Count: ", anchors.shape[0])
def export_annotations(image_info: dict,
results: dict,
adapterClass: AnnotationAdapter.__class__,
save_path="predicted",
config: Config = None,
verbose=0):
"""
Exports predicted results to an XML annotation file using given XMLExporter
:param image_info: Dict with at least {"NAME": str, "HEIGHT": int, "WIDTH": int} about the inferred image
:param results: inference results of the image
:param adapterClass: class inheriting XMLExporter
:param save_path: path to the dir you want to save the annotation file
:param config: the config to get mini_mask informations
:param verbose: verbose level of the method (0 = nothing, 1 = information)
:return: None
"""
if config is None:
print("Cannot export annotations as config is not given.")
return
rois = results['rois']
masks = results['masks']
class_ids = results['class_ids']
height = masks.shape[0]
width = masks.shape[1]
adapter_instance = adapterClass(
{
"name": image_info['NAME'],
"height": image_info['HEIGHT'],
'width': image_info['WIDTH'],
'format': image_info['IMAGE_FORMAT']
},
verbose=verbose)
if verbose > 0:
print(
f"Exporting to {adapter_instance.getName()} annotation file format."
)
# For each prediction
for i in range(masks.shape[2]):
if config is not None and config.is_using_mini_mask():
shifted_roi = shift_bbox(rois[i])
shifted_roi += [5, 5, 5, 5]
image_size = shifted_roi[2:] + [5, 5]
mask = expand_mask(shifted_roi, masks[:, :, i], image_size)
yStart, xStart = rois[i][:2] - [5, 5]
else:
# Getting the RoI coordinates and the corresponding area
# y1, x1, y2, x2
yStart, xStart, yEnd, xEnd = rois[i]
yStart = max(yStart - 10, 0)
xStart = max(xStart - 10, 0)
yEnd = min(yEnd + 10, height)
xEnd = min(xEnd + 10, width)
mask = masks[yStart:yEnd, xStart:xEnd, i]
# Getting list of points coordinates and adding the prediction to XML
points = getPoints(np.uint8(mask),
xOffset=xStart,
yOffset=yStart,
show=False,
waitSeconds=0,
info=False)
if points is None:
continue
adapter_instance.addAnnotation(
config.get_classes_info()[class_ids[i] - 1], points)
for classInfo in config.get_classes_info():
adapter_instance.addAnnotationClass(classInfo)
os.makedirs(save_path, exist_ok=True)
if verbose > 0:
print(' - ', end='')
adapter_instance.saveToFile(save_path, image_info['NAME'])
def get_count_and_area(results: dict,
image_info: dict,
selected_classes: [str],
save=None,
display=True,
config: Config = None,
verbose=0):
"""
Computing count and area of classes from results
:param results: the results
:param image_info: Dict containing informations about the image
:param selected_classes: list of classes' names that you want to get statistics on
:param save: if given, path to the json file that will contains statistics
:param display: if True, will print the statistics
:param config: the config to get mini_mask informations
:param verbose: 0 : nothing, 1+ : errors/problems, 2 : general information, ...
:return: Dict of "className": {"count": int, "area": int} elements for each classes
"""
if config is None or (save is None and not display):
return
print(" - Computing statistics on predictions")
rois = results['rois']
masks = results['masks']
class_ids = results['class_ids']
indices = np.arange(len(class_ids))
mini_mask_used = config.is_using_mini_mask()
resize = config.get_param().get('resize', None)
ratio = 1
if resize is not None:
ratio = image_info['HEIGHT'] / resize[0]
ratio *= (image_info['WIDTH'] / resize[1])
if type(selected_classes) is str:
selected_classes_ = [selected_classes]
else:
selected_classes_ = selected_classes
# Getting the inferenceIDs of the wanted classes
if "all" in selected_classes_:
selectedClassesID = {
aClass['id']: aClass['name']
for aClass in config.get_classes_info()
}
else:
selectedClassesID = {
config.get_class_id(name): name
for name in selected_classes_
}
indices = indices[np.isin(class_ids, list(selectedClassesID.keys()))]
res = {
c_name: {
"display_name": config.get_class_name(c_id, display=True),
"count": 0,
"area": 0
}
for c_id, c_name in selectedClassesID.items()
}
# For each predictions, if class ID matching with one we want
for index in indices:
# Getting current values of count and area
className = selectedClassesID[class_ids[index]]
res[className]["count"] += 1
# Getting the area of current mask
if mini_mask_used:
shifted_roi = utils.shift_bbox(rois[index])
mask = utils.expand_mask(shifted_roi, masks[:, :, index],
shifted_roi[2:])
else:
yStart, xStart, yEnd, xEnd = rois[index]
mask = masks[yStart:yEnd, xStart:xEnd, index]
mask = mask.astype(np.uint8)
if "mask_areas" in results and results['mask_areas'][index] != -1:
area = int(results['mask_areas'][index])
else:
area, _ = utils.get_mask_area(mask)
if resize is None:
res[className][
"area"] += area # Cast to int to avoid "json 'int64' not serializable"
else:
res[className]["area"] += int(round(area * ratio))
if 'BASE_CLASS' in image_info:
mode = config.get_class_mode(image_info['BASE_CLASS'],
only_in_previous="current")[0]
res[image_info['BASE_CLASS']] = {
"display_name":
config.get_class_name(config.get_class_id(image_info['BASE_CLASS'],
mode),
mode,
display=True),
"count":
image_info['BASE_COUNT'],
"area":
image_info["BASE_AREA"]
}
if save is not None:
with open(os.path.join(save, f"{image_info['NAME']}_stats.json"),
"w") as saveFile:
try:
json.dump(res, saveFile, indent='\t')
except TypeError:
if verbose > 0:
print(" Failed to save statistics", flush=True)
if display:
for className in res:
mode = config.get_class_mode(className,
only_in_previous="current")[0]
displayName = config.get_class_name(config.get_class_id(
className, mode),
mode,
display=True)
stat = res[className]
print(
f" - {displayName} : count = {stat['count']}, area = {stat['area']} px"
)
return res
def mask_histo_per_base_mask(base_results,
results,
image_info,
classes=None,
box_epsilon: int = 0,
test_masks=True,
mask_threshold=0.9,
count_zeros=True,
config: Config = None,
display_per_base_mask=False,
display_global=False,
save=None,
verbose=0):
"""
Return an histogram of the number of each mask of a class inside each base mask
:param base_results: results of the previous inference mode or ground-truth
:param results: results of the current inference mode or ground-truth
:param image_info: Dict containing informations about the image
:param classes: dict that link previous classes to current classes that we want to count
:param box_epsilon: margin of the RoI to allow boxes that are not exactly inside
:param test_masks: if True, will test that masks are at least 'mask_threshold' inside the base mask
:param mask_threshold: threshold that will define if a mask is included inside the base mask
:param count_zeros: if True, base masks without included masks will be counted
:param config: Config object of the Inference Tool
:param display_per_base_mask: if True, will display each base mask histogram
:param display_global: if True, will display global histogram
:param save: if given, will be used as directory path to save json file of
:param verbose: 0 : nothing, 1+ : errors/problems, 2 : general information
:return: global histogram of how many base masks contain a certain amount an included class mask
"""
# If classes is None or empty, skip method
if classes is None or classes == {} or config is None \
or (save is None and not (display_per_base_mask or display_global)):
return results
print(" - Computing base masks histograms")
if box_epsilon < 0:
raise ValueError(f"box_epsilon ({box_epsilon}) cannot be negative")
def get_class_data(classname):
fromPreviousRes = config.get_previous_mode() in config.get_class_mode(
classname, "current")
class_id = config.get_class_id(
b_class, "previous" if fromPreviousRes else "current")
tempRes = base_results if fromPreviousRes else results
if 'histos' not in tempRes: # If histo does not exists, initiate it
tempRes['histos'] = np.empty(len(tempRes['class_ids']),
dtype=object)
return (class_id, tempRes['class_ids'], tempRes['rois'],
tempRes['masks'], tempRes['histos'],
np.arange(len(tempRes['class_ids']),
dtype=int), fromPreviousRes)
# Getting all the results/current data
c_class_ids = results['class_ids']
c_rois = results['rois']
c_masks = results['masks']
c_indices = np.arange(len(results['class_ids']), dtype=int)
c_areas = results.get('mask_areas',
np.ones(len(c_class_ids), dtype=int) * -1)
# For each base class that we want to get an histogram of the included current classes
for b_class in classes:
b_class_id, b_class_ids, b_rois, b_masks, histograms, b_indices, fromPrevious = get_class_data(
b_class)
b_cur_idx = b_indices[np.isin(b_class_ids, [b_class_id])]
if classes[b_class] == "all" or (type(classes[b_class]) is list
and "all" in classes[b_class]):
c_cur_idx = c_indices
else:
if type(classes[b_class]) is str:
temp_ = [classes[b_class]]
else:
temp_ = classes[b_class]
c_class_id = [config.get_class_id(aClass) for aClass in temp_]
c_cur_idx = c_indices[np.isin(c_class_ids, c_class_id)]
for b_idx in b_cur_idx: # For each base class mask
b_roi = b_rois[b_idx]
custom_shift = b_roi[:2] - box_epsilon
padded_size = b_roi[2:] - b_roi[:2] + (box_epsilon * 2)
if test_masks:
b_mask = b_masks[..., b_idx]
if config.is_using_mini_mask(config.get_previous_mode()):
b_shifted_roi = utils.shift_bbox(b_roi, custom_shift)
b_mask = utils.expand_mask(b_shifted_roi, b_mask,
padded_size)
else:
b_mask = np.pad(
b_mask[b_roi[0]:b_roi[2], b_roi[1]:b_roi[3]],
box_epsilon)
if histograms[b_idx] is None:
histograms[b_idx] = {}
for c_idx in c_cur_idx: # For each mask of one of the current classes
c_roi = c_rois[c_idx]
c_class = c_class_ids[c_idx]
if fromPrevious and c_class == b_class_id: # If using same results, skip base class elements
continue
# If the bbox of the current mask is inside the base bbox
if utils.in_roi(c_roi, b_roi, epsilon=box_epsilon):
if test_masks: # If we have to check that masks are included
c_mask = c_masks[..., c_idx]
if config.is_using_mini_mask():
c_shifted_roi = utils.shift_bbox(
c_roi, custom_shift)
c_mask = utils.expand_mask(c_shifted_roi, c_mask,
padded_size)
else:
c_mask = np.pad(
c_mask[b_roi[0]:b_roi[2], b_roi[1]:b_roi[3]],
box_epsilon)
if c_areas[c_idx] == -1:
c_areas[c_idx] = dD.getBWCount(c_mask)[1]
c_mask = np.bitwise_and(b_mask, c_mask)
c_area_in = dD.getBWCount(c_mask)[1]
if c_area_in <= c_areas[
c_idx] * mask_threshold: # If the included part is not enough, skip it
continue
if c_class not in histograms[b_idx]:
histograms[b_idx][c_class] = 0
histograms[b_idx][c_class] += 1
# Display of each individual histogram
if display_per_base_mask:
for res in [base_results, results]:
if 'histos' not in res:
continue
for idx, histogram in enumerate(res['histos']):
if histogram is not None:
print(
f" - mask n°{idx}:", ", ".join([
f"{nb} {config.get_class_name(c, display=True)}"
for c, nb in histogram.items()
]))
# Computing global histograms
first = True
for res in [base_results, results]:
if 'histos' in res:
if first:
first = False
global_histo = mask_to_class_histogram(res,
classes=classes,
count_zeros=count_zeros,
config=config)
else: # Updating manually global histo if there are base classes from both previous and current res
temp_histo = mask_to_class_histogram(res,
classes=classes,
count_zeros=count_zeros,
config=config)
for c in temp_histo:
if c not in global_histo:
global_histo[c] = temp_histo[c]
else:
for nb in temp_histo[c]:
if nb not in global_histo[c]:
global_histo[c][nb] = 0
global_histo[c][nb] += temp_histo[c][nb]
for key in global_histo.keys():
global_histo[key] = sort_dict(global_histo[key], key_type=int)
# Displaying global histogram if needed
baseName = 'BASE' if len(classes) > 1 else list(classes.keys())[0]
if display_global:
for class_, histogram in global_histo.items():
print(
f" - {class_}:", ", ".join([
f"{nb_elt} [{nb_mask} {baseName.lower()} mask{'s' if nb_mask > 1 else ''}]"
for nb_elt, nb_mask in histogram.items()
]))
if save is not None:
temp = {
'_comment':
f"<class A>: {{N: <nb {baseName} masks with N class A masks>}}"
}
temp.update(global_histo)
with open(os.path.join(save, f'{image_info["NAME"]}_histo.json'),
'w') as saveFile:
json.dump(temp, saveFile, indent='\t')
return global_histo
def inspect_data(dataset, config):
print("Image Count: {}".format(len(dataset.image_ids)))
print("Class Count: {}".format(dataset.num_classes))
for i, info in enumerate(dataset.class_info):
print("{:3}. {:50}".format(i, info['name']))
# ## Display Samples
#
# Load and display images and masks.
# In[4]:
# Load and display random samples
image_ids = np.random.choice(dataset.image_ids, 4)
for image_id in image_ids:
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids,
dataset.class_names)
# ## Bounding Boxes
#
# Rather than using bounding box coordinates provided by the source datasets, we compute the bounding boxes from masks instead. This allows us to handle bounding boxes consistently regardless of the source dataset, and it also makes it easier to resize, rotate, or crop images because we simply generate the bounding boxes from the updates masks rather than computing bounding box transformation for each type of image transformation.
# In[5]:
# Load random image and mask.
image_id = random.choice(dataset.image_ids)
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id ", image_id, dataset.image_reference(image_id))
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Resize Images
#
# To support multiple images per batch, images are resized to one size (1024x1024). Aspect ratio is preserved, though. If an image is not square, then zero padding is added at the top/bottom or right/left.
# In[6]:
# Load random image and mask.
image_id = np.random.choice(dataset.image_ids, 1)[0]
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
original_shape = image.shape
# Resize
image, window, scale, padding, _ = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
mask = utils.resize_mask(mask, scale, padding)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id: ", image_id, dataset.image_reference(image_id))
print("Original shape: ", original_shape)
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Mini Masks
#
# Instance binary masks can get large when training with high resolution images. For example, if training with 1024x1024 image then the mask of a single instance requires 1MB of memory (Numpy uses bytes for boolean values). If an image has 100 instances then that's 100MB for the masks alone.
#
# To improve training speed, we optimize masks by:
# * We store mask pixels that are inside the object bounding box, rather than a mask of the full image. Most objects are small compared to the image size, so we save space by not storing a lot of zeros around the object.
# * We resize the mask to a smaller size (e.g. 56x56). For objects that are larger than the selected size we lose a bit of accuracy. But most object annotations are not very accuracy to begin with, so this loss is negligable for most practical purposes. Thie size of the mini_mask can be set in the config class.
#
# To visualize the effect of mask resizing, and to verify the code correctness, we visualize some examples.
# In[7]:
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
log("image", image)
log("image_meta", image_meta)
log("class_ids", class_ids)
log("bbox", bbox)
log("mask", mask)
display_images([image] +
[mask[:, :, i] for i in range(min(mask.shape[-1], 7))])
# In[8]:
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# In[9]:
# Add augmentation and mask resizing.
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, augment=True, use_mini_mask=True)
log("mask", mask)
display_images([image] +
[mask[:, :, i] for i in range(min(mask.shape[-1], 7))])
# In[10]:
mask = utils.expand_mask(bbox, mask, image.shape)
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Anchors
#
# The order of anchors is important. Use the same order in training and prediction phases. And it must match the order of the convolution execution.
#
# For an FPN network, the anchors must be ordered in a way that makes it easy to match anchors to the output of the convolution layers that predict anchor scores and shifts.
# * Sort by pyramid level first. All anchors of the first level, then all of the second and so on. This makes it easier to separate anchors by level.
# * Within each level, sort anchors by feature map processing sequence. Typically, a convolution layer processes a feature map starting from top-left and moving right row by row.
# * For each feature map cell, pick any sorting order for the anchors of different ratios. Here we match the order of ratios passed to the function.
#
# **Anchor Stride:**
# In the FPN architecture, feature maps at the first few layers are high resolution. For example, if the input image is 1024x1024 then the feature meap of the first layer is 256x256, which generates about 200K anchors (256*256*3). These anchors are 32x32 pixels and their stride relative to image pixels is 4 pixels, so there is a lot of overlap. We can reduce the load significantly if we generate anchors for every other cell in the feature map. A stride of 2 will cut the number of anchors by 4, for example.
#
# In this implementation we use an anchor stride of 2, which is different from the paper.
# In[11]:
# Generate Anchors
backbone_shapes = modellib.compute_backbone_shapes(config,
config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Print summary of anchors
num_levels = len(backbone_shapes)
anchors_per_cell = len(config.RPN_ANCHOR_RATIOS)
print("Count: ", anchors.shape[0])
print("Scales: ", config.RPN_ANCHOR_SCALES)
print("ratios: ", config.RPN_ANCHOR_RATIOS)
print("Anchors per Cell: ", anchors_per_cell)
print("Levels: ", num_levels)
anchors_per_level = []
for l in range(num_levels):
num_cells = backbone_shapes[l][0] * backbone_shapes[l][1]
anchors_per_level.append(anchors_per_cell * num_cells //
config.RPN_ANCHOR_STRIDE**2)
print("Anchors in Level {}: {}".format(l, anchors_per_level[l]))
# Visualize anchors of one cell at the center of the feature map of a specific level.
# In[12]:
## Visualize anchors of one cell at the center of the feature map of a specific level
# Load and draw random image
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, _, _, _ = modellib.load_image_gt(
dataset, config, image_id)
fig, ax = plt.subplots(1, figsize=(10, 10))
ax.imshow(image)
levels = len(backbone_shapes)
for level in range(levels):
colors = visualize.random_colors(levels)
# Compute the index of the anchors at the center of the image
level_start = sum(
anchors_per_level[:level]) # sum of anchors of previous levels
level_anchors = anchors[level_start:level_start +
anchors_per_level[level]]
print("Level {}. Anchors: {:6} Feature map Shape: {}".format(
level, level_anchors.shape[0], backbone_shapes[level]))
center_cell = backbone_shapes[level] // 2
center_cell_index = (center_cell[0] * backbone_shapes[level][1] +
center_cell[1])
level_center = center_cell_index * anchors_per_cell
center_anchor = anchors_per_cell * (
(center_cell[0] * backbone_shapes[level][1] / config.RPN_ANCHOR_STRIDE**2) \
+ center_cell[1] / config.RPN_ANCHOR_STRIDE)
level_center = int(center_anchor)
# Draw anchors. Brightness show the order in the array, dark to bright.
for i, rect in enumerate(level_anchors[level_center:level_center +
anchors_per_cell]):
y1, x1, y2, x2 = rect
p = patches.Rectangle(
(x1, y1),
x2 - x1,
y2 - y1,
linewidth=2,
facecolor='none',
edgecolor=(i + 1) * np.array(colors[level]) / anchors_per_cell)
ax.add_patch(p)
# ## Data Generator
#
# In[13]:
# Create data generator
random_rois = 2000
g = modellib.data_generator(dataset,
config,
shuffle=True,
random_rois=random_rois,
batch_size=4,
detection_targets=True)
# In[14]:
# Uncomment to run the generator through a lot of images
# to catch rare errors
# for i in range(1000):
# print(i)
# _, _ = next(g)
# In[15]:
# Get Next Image
if random_rois:
[
normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids,
gt_boxes, gt_masks, rpn_rois, rois
], [mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g)
log("rois", rois)
log("mrcnn_class_ids", mrcnn_class_ids)
log("mrcnn_bbox", mrcnn_bbox)
log("mrcnn_mask", mrcnn_mask)
else:
[
normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes,
gt_masks
], _ = next(g)
log("gt_class_ids", gt_class_ids)
log("gt_boxes", gt_boxes)
log("gt_masks", gt_masks)
log(
"rpn_match",
rpn_match,
)
log("rpn_bbox", rpn_bbox)
image_id = modellib.parse_image_meta(image_meta)["image_id"][0]
print("image_id: ", image_id, dataset.image_reference(image_id))
# Remove the last dim in mrcnn_class_ids. It's only added
# to satisfy Keras restriction on target shape.
mrcnn_class_ids = mrcnn_class_ids[:, :, 0]
# In[16]:
b = 0
# Restore original image (reverse normalization)
sample_image = modellib.unmold_image(normalized_images[b], config)
# Compute anchor shifts.
indices = np.where(rpn_match[b] == 1)[0]
refined_anchors = utils.apply_box_deltas(
anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV)
log("anchors", anchors)
log("refined_anchors", refined_anchors)
# Get list of positive anchors
positive_anchor_ids = np.where(rpn_match[b] == 1)[0]
print("Positive anchors: {}".format(len(positive_anchor_ids)))
negative_anchor_ids = np.where(rpn_match[b] == -1)[0]
print("Negative anchors: {}".format(len(negative_anchor_ids)))
neutral_anchor_ids = np.where(rpn_match[b] == 0)[0]
print("Neutral anchors: {}".format(len(neutral_anchor_ids)))
# ROI breakdown by class
for c, n in zip(dataset.class_names,
np.bincount(mrcnn_class_ids[b].flatten())):
if n:
print("{:23}: {}".format(c[:20], n))
# Show positive anchors
fig, ax = plt.subplots(1, figsize=(16, 16))
visualize.draw_boxes(sample_image,
boxes=anchors[positive_anchor_ids],
refined_boxes=refined_anchors,
ax=ax)
# In[17]:
# Show negative anchors
visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids])
# In[18]:
# Show neutral anchors. They don't contribute to training.
visualize.draw_boxes(sample_image,
boxes=anchors[np.random.choice(
neutral_anchor_ids, 100)])
# ## ROIs
# In[19]:
if random_rois:
# Class aware bboxes
bbox_specific = mrcnn_bbox[b,
np.arange(mrcnn_bbox.shape[1]),
mrcnn_class_ids[b], :]
# Refined ROIs
refined_rois = utils.apply_box_deltas(
rois[b].astype(np.float32),
bbox_specific[:, :4] * config.BBOX_STD_DEV)
# Class aware masks
mask_specific = mrcnn_mask[b,
np.arange(mrcnn_mask.shape[1]), :, :,
mrcnn_class_ids[b]]
visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific,
mrcnn_class_ids[b], dataset.class_names)
# Any repeated ROIs?
rows = np.ascontiguousarray(rois[b]).view(
np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1])))
_, idx = np.unique(rows, return_index=True)
print("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1]))
# In[20]:
if random_rois:
# Dispalay ROIs and corresponding masks and bounding boxes
ids = random.sample(range(rois.shape[1]), 8)
images = []
titles = []
for i in ids:
image = visualize.draw_box(sample_image.copy(),
rois[b, i, :4].astype(np.int32),
[255, 0, 0])
image = visualize.draw_box(image, refined_rois[i].astype(np.int64),
[0, 255, 0])
images.append(image)
titles.append("ROI {}".format(i))
images.append(mask_specific[i] * 255)
titles.append(dataset.class_names[mrcnn_class_ids[b, i]][:20])
display_images(images,
titles,
cols=4,
cmap="Blues",
interpolation="none")
# In[21]:
# Check ratio of positive ROIs in a set of images.
if random_rois:
limit = 10
temp_g = modellib.data_generator(dataset,
config,
shuffle=True,
random_rois=10000,
batch_size=1,
detection_targets=True)
total = 0
for i in range(limit):
_, [ids, _, _] = next(temp_g)
positive_rois = np.sum(ids[0] > 0)
total += positive_rois
print("{:5} {:5.2f}".format(positive_rois,
positive_rois / ids.shape[1]))
print("Average percent: {:.2f}".format(total / (limit * ids.shape[1])))
def display_instances(image,
boxes,
masks,
class_ids,
class_names,
scores=None,
title="",
figsize=(16, 16),
ax=None,
fig=None,
image_format="jpg",
show_mask=True,
show_bbox=True,
colors=None,
colorPerClass=False,
captions=None,
fileName=None,
save_cleaned_img=False,
silent=False,
config: Config = None):
"""
boxes: [num_instance, (y1, x1, y2, x2, class_id)] in image coordinates.
masks: [height, width, num_instances]
class_ids: [num_instances]
class_names: list of class names of the dataset
scores: (optional) confidence scores for each box
title: (optional) Figure title
show_mask, show_bbox: To show masks and bounding boxes or not
figsize: (optional) the size of the image
colors: (optional) An array or colors to use with each object
captions: (optional) A list of strings to use as captions for each object
"""
# Number of instances
N = boxes.shape[0]
if not N:
print("\n*** No instances to display *** \n")
else:
assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
# If no axis is passed, create one and automatically call show()
auto_show = False
ownFig = False
if ax is None or fig is None:
ownFig = True
fig, ax = plt.subplots(1, figsize=figsize)
auto_show = not silent
# Generate random colors
nb_color = (len(class_names) - 1) if colorPerClass else N
colors = colors if colors is not None else random_colors(
nb_color, shuffle=(not colorPerClass))
if type(colors[0][0]) is int:
_colors = []
for color in colors:
_colors.append([c / 255. for c in color])
else:
_colors = colors
# Show area outside image boundaries.
height, width = image.shape[:2]
ax.set_ylim(height + 10, -10)
ax.set_xlim(-10, width + 10)
ax.axis('off')
ax.set_title(title)
# To be usable on Google Colab we do not make a copy of the image leading to too much ram usage if it is a biopsy
# or nephrectomy image
masked_image = image
# masked_image = image.astype(np.uint32).copy()
for i in range(N):
if colorPerClass:
color = _colors[class_ids[i] - 1]
else:
color = _colors[i]
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
if show_bbox:
p = patches.Rectangle((x1, y1),
x2 - x1,
y2 - y1,
linewidth=2,
alpha=0.7,
linestyle="dashed",
edgecolor=color,
facecolor='none')
ax.add_patch(p)
# Label
if not captions:
class_id = class_ids[i]
score = scores[i] if scores is not None else None
label = class_names[class_id]
caption = "{} {:.3f}".format(label, score) if score else label
else:
caption = captions[i]
ax.text(x1 + 4,
y1 + 19,
caption,
color=get_text_color(color[0], color[1], color[2]),
size=12,
backgroundcolor=color)
# Mask
mask = masks[:, :, i]
bbox = boxes[i]
shift = np.array([0, 0])
if config is not None and config.is_using_mini_mask():
shifted_bbox = utils.shift_bbox(bbox)
shift = bbox[:2]
mask = utils.expand_mask(shifted_bbox, mask,
tuple(shifted_bbox[2:]))
mask = mask.astype(np.uint8) * 255
if show_mask:
masked_image = apply_mask(masked_image, mask, color, bbox=bbox)
# Mask Polygon
# Pad to ensure proper polygons for masks that touch image edges.
padded_mask = np.zeros((mask.shape[0] + 2, mask.shape[1] + 2),
dtype=np.uint8)
padded_mask[1:-1, 1:-1] = mask
contours = find_contours(padded_mask, 0.5)
for verts in contours:
verts = verts + shift
# Subtract the padding and flip (y, x) to (x, y)
verts = np.fliplr(verts) - 1
p = Polygon(verts, facecolor="none", edgecolor=color)
ax.add_patch(p)
# masked_image = masked_image.astype(np.uint8)
ax.imshow(masked_image)
fig.tight_layout()
if fileName is not None:
fig.savefig(f"{fileName}.{image_format}")
if save_cleaned_img:
BGR_img = cv2.cvtColor(masked_image, cv2.COLOR_RGB2BGR)
cv2.imwrite(f"{fileName}_clean.{image_format}", BGR_img,
CV2_IMWRITE_PARAM)
if auto_show:
plt.show()
fig.clf()
if ownFig:
del ax, fig
return masked_image
