def visualize_instances(image,
boxes,
masks,
class_ids,
class_names,
scores=None,
ax=None,
file_name=None,
colors=None,
making_video=False,
making_image=False,
open_cv=False):
# Number of instances
N = boxes.shape[0]
if not N:
print("\n*** No instances to display *** \n")
else:
assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
if not making_video:
# Generate random colors
colors = visualize.random_colors(N)
fig, ax = get_ax()
# Show area outside image boundaries.
height, width = image.shape[:2]
ax.set_ylim(height + 10, -10)
ax.set_xlim(-10, width + 10)
ax.axis('off')
masked_image = image.astype(np.uint32).copy()
for i in range(N):
if not making_video:
color = colors[i]
else:
# all objects of a class get same color
class_id = class_ids[i]
color = colors[class_id - 1]
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
# show bbox
p = visualize.patches.Rectangle((x1, y1),
x2 - x1,
y2 - y1,
linewidth=2,
alpha=0.7,
linestyle="dashed",
edgecolor=color,
facecolor='none')
ax.add_patch(p)
# Label
class_id = class_ids[i]
score = scores[i] if scores is not None else None
try:
label = class_names[class_id]
except IndexError:
label = 'label_error'
x = random.randint(x1, (x1 + x2) // 2)
caption = "{} {:.3f}".format(label, score) if score else label
ax.text(x1,
y1 + 8,
caption,
color='w',
size=11,
backgroundcolor="none")
# Mask
mask = masks[:, :, i]
masked_image = visualize.apply_mask(masked_image, mask, color)
# Mask Polygon
# Pad to ensure proper polygons for masks that touch image edges.
padded_mask = np.zeros((mask.shape[0] + 2, mask.shape[1] + 2),
dtype=np.uint8)
padded_mask[1:-1, 1:-1] = mask
contours = visualize.find_contours(padded_mask, 0.5)
for verts in contours:
# Subtract the padding and flip (y, x) to (x, y)
verts = np.fliplr(verts) - 1
p = visualize.Polygon(verts, facecolor="none", edgecolor=color)
ax.add_patch(p)
ax.imshow(masked_image.astype(np.uint8))
# To transform the drawn figure into ndarray X
fig.canvas.draw()
string = fig.canvas.tostring_rgb()
masked = masked_image.astype(np.uint8)
X = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
X = X.reshape(fig.canvas.get_width_height()[::-1] + (3, ))
# open cv's RGB style: BGR
# if open_cv:
X = X[..., ::-1]
if making_video:
plt.close()
# return masked_image.astype(np.uint8)
# cv2.imshow('frame', X)
# cv2.waitKey(1)
return X
elif making_image:
# plt.savefig(file_name)
cv2.imwrite(file_name, X)
def inspect_data(dataset, config):
print("Image Count: {}".format(len(dataset.image_ids)))
print("Class Count: {}".format(dataset.num_classes))
for i, info in enumerate(dataset.class_info):
print("{:3}. {:50}".format(i, info['name']))
# ## Display Samples
#
# Load and display images and masks.
# In[4]:
# Load and display random samples
image_ids = np.random.choice(dataset.image_ids, 4)
for image_id in image_ids:
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids,
dataset.class_names)
# ## Bounding Boxes
#
# Rather than using bounding box coordinates provided by the source datasets, we compute the bounding boxes from masks instead. This allows us to handle bounding boxes consistently regardless of the source dataset, and it also makes it easier to resize, rotate, or crop images because we simply generate the bounding boxes from the updates masks rather than computing bounding box transformation for each type of image transformation.
# In[5]:
# Load random image and mask.
image_id = random.choice(dataset.image_ids)
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id ", image_id, dataset.image_reference(image_id))
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Resize Images
#
# To support multiple images per batch, images are resized to one size (1024x1024). Aspect ratio is preserved, though. If an image is not square, then zero padding is added at the top/bottom or right/left.
# In[6]:
# Load random image and mask.
image_id = np.random.choice(dataset.image_ids, 1)[0]
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
original_shape = image.shape
# Resize
image, window, scale, padding, _ = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
mask = utils.resize_mask(mask, scale, padding)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id: ", image_id, dataset.image_reference(image_id))
print("Original shape: ", original_shape)
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Mini Masks
#
# Instance binary masks can get large when training with high resolution images. For example, if training with 1024x1024 image then the mask of a single instance requires 1MB of memory (Numpy uses bytes for boolean values). If an image has 100 instances then that's 100MB for the masks alone.
#
# To improve training speed, we optimize masks by:
# * We store mask pixels that are inside the object bounding box, rather than a mask of the full image. Most objects are small compared to the image size, so we save space by not storing a lot of zeros around the object.
# * We resize the mask to a smaller size (e.g. 56x56). For objects that are larger than the selected size we lose a bit of accuracy. But most object annotations are not very accuracy to begin with, so this loss is negligable for most practical purposes. Thie size of the mini_mask can be set in the config class.
#
# To visualize the effect of mask resizing, and to verify the code correctness, we visualize some examples.
# In[7]:
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
log("image", image)
log("image_meta", image_meta)
log("class_ids", class_ids)
log("bbox", bbox)
log("mask", mask)
display_images([image] +
[mask[:, :, i] for i in range(min(mask.shape[-1], 7))])
# In[8]:
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# In[9]:
# Add augmentation and mask resizing.
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, augment=True, use_mini_mask=True)
log("mask", mask)
display_images([image] +
[mask[:, :, i] for i in range(min(mask.shape[-1], 7))])
# In[10]:
mask = utils.expand_mask(bbox, mask, image.shape)
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Anchors
#
# The order of anchors is important. Use the same order in training and prediction phases. And it must match the order of the convolution execution.
#
# For an FPN network, the anchors must be ordered in a way that makes it easy to match anchors to the output of the convolution layers that predict anchor scores and shifts.
# * Sort by pyramid level first. All anchors of the first level, then all of the second and so on. This makes it easier to separate anchors by level.
# * Within each level, sort anchors by feature map processing sequence. Typically, a convolution layer processes a feature map starting from top-left and moving right row by row.
# * For each feature map cell, pick any sorting order for the anchors of different ratios. Here we match the order of ratios passed to the function.
#
# **Anchor Stride:**
# In the FPN architecture, feature maps at the first few layers are high resolution. For example, if the input image is 1024x1024 then the feature meap of the first layer is 256x256, which generates about 200K anchors (256*256*3). These anchors are 32x32 pixels and their stride relative to image pixels is 4 pixels, so there is a lot of overlap. We can reduce the load significantly if we generate anchors for every other cell in the feature map. A stride of 2 will cut the number of anchors by 4, for example.
#
# In this implementation we use an anchor stride of 2, which is different from the paper.
# In[11]:
# Generate Anchors
backbone_shapes = modellib.compute_backbone_shapes(config,
config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Print summary of anchors
num_levels = len(backbone_shapes)
anchors_per_cell = len(config.RPN_ANCHOR_RATIOS)
print("Count: ", anchors.shape[0])
print("Scales: ", config.RPN_ANCHOR_SCALES)
print("ratios: ", config.RPN_ANCHOR_RATIOS)
print("Anchors per Cell: ", anchors_per_cell)
print("Levels: ", num_levels)
anchors_per_level = []
for l in range(num_levels):
num_cells = backbone_shapes[l][0] * backbone_shapes[l][1]
anchors_per_level.append(anchors_per_cell * num_cells //
config.RPN_ANCHOR_STRIDE**2)
print("Anchors in Level {}: {}".format(l, anchors_per_level[l]))
# Visualize anchors of one cell at the center of the feature map of a specific level.
# In[12]:
## Visualize anchors of one cell at the center of the feature map of a specific level
# Load and draw random image
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, _, _, _ = modellib.load_image_gt(
dataset, config, image_id)
fig, ax = plt.subplots(1, figsize=(10, 10))
ax.imshow(image)
levels = len(backbone_shapes)
for level in range(levels):
colors = visualize.random_colors(levels)
# Compute the index of the anchors at the center of the image
level_start = sum(
anchors_per_level[:level]) # sum of anchors of previous levels
level_anchors = anchors[level_start:level_start +
anchors_per_level[level]]
print("Level {}. Anchors: {:6} Feature map Shape: {}".format(
level, level_anchors.shape[0], backbone_shapes[level]))
center_cell = backbone_shapes[level] // 2
center_cell_index = (center_cell[0] * backbone_shapes[level][1] +
center_cell[1])
level_center = center_cell_index * anchors_per_cell
center_anchor = anchors_per_cell * (
(center_cell[0] * backbone_shapes[level][1] / config.RPN_ANCHOR_STRIDE**2) \
+ center_cell[1] / config.RPN_ANCHOR_STRIDE)
level_center = int(center_anchor)
# Draw anchors. Brightness show the order in the array, dark to bright.
for i, rect in enumerate(level_anchors[level_center:level_center +
anchors_per_cell]):
y1, x1, y2, x2 = rect
p = patches.Rectangle(
(x1, y1),
x2 - x1,
y2 - y1,
linewidth=2,
facecolor='none',
edgecolor=(i + 1) * np.array(colors[level]) / anchors_per_cell)
ax.add_patch(p)
# ## Data Generator
#
# In[13]:
# Create data generator
random_rois = 2000
g = modellib.data_generator(dataset,
config,
shuffle=True,
random_rois=random_rois,
batch_size=4,
detection_targets=True)
# In[14]:
# Uncomment to run the generator through a lot of images
# to catch rare errors
# for i in range(1000):
# print(i)
# _, _ = next(g)
# In[15]:
# Get Next Image
if random_rois:
[
normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids,
gt_boxes, gt_masks, rpn_rois, rois
], [mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g)
log("rois", rois)
log("mrcnn_class_ids", mrcnn_class_ids)
log("mrcnn_bbox", mrcnn_bbox)
log("mrcnn_mask", mrcnn_mask)
else:
[
normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes,
gt_masks
], _ = next(g)
log("gt_class_ids", gt_class_ids)
log("gt_boxes", gt_boxes)
log("gt_masks", gt_masks)
log(
"rpn_match",
rpn_match,
)
log("rpn_bbox", rpn_bbox)
image_id = modellib.parse_image_meta(image_meta)["image_id"][0]
print("image_id: ", image_id, dataset.image_reference(image_id))
# Remove the last dim in mrcnn_class_ids. It's only added
# to satisfy Keras restriction on target shape.
mrcnn_class_ids = mrcnn_class_ids[:, :, 0]
# In[16]:
b = 0
# Restore original image (reverse normalization)
sample_image = modellib.unmold_image(normalized_images[b], config)
# Compute anchor shifts.
indices = np.where(rpn_match[b] == 1)[0]
refined_anchors = utils.apply_box_deltas(
anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV)
log("anchors", anchors)
log("refined_anchors", refined_anchors)
# Get list of positive anchors
positive_anchor_ids = np.where(rpn_match[b] == 1)[0]
print("Positive anchors: {}".format(len(positive_anchor_ids)))
negative_anchor_ids = np.where(rpn_match[b] == -1)[0]
print("Negative anchors: {}".format(len(negative_anchor_ids)))
neutral_anchor_ids = np.where(rpn_match[b] == 0)[0]
print("Neutral anchors: {}".format(len(neutral_anchor_ids)))
# ROI breakdown by class
for c, n in zip(dataset.class_names,
np.bincount(mrcnn_class_ids[b].flatten())):
if n:
print("{:23}: {}".format(c[:20], n))
# Show positive anchors
fig, ax = plt.subplots(1, figsize=(16, 16))
visualize.draw_boxes(sample_image,
boxes=anchors[positive_anchor_ids],
refined_boxes=refined_anchors,
ax=ax)
# In[17]:
# Show negative anchors
visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids])
# In[18]:
# Show neutral anchors. They don't contribute to training.
visualize.draw_boxes(sample_image,
boxes=anchors[np.random.choice(
neutral_anchor_ids, 100)])
# ## ROIs
# In[19]:
if random_rois:
# Class aware bboxes
bbox_specific = mrcnn_bbox[b,
np.arange(mrcnn_bbox.shape[1]),
mrcnn_class_ids[b], :]
# Refined ROIs
refined_rois = utils.apply_box_deltas(
rois[b].astype(np.float32),
bbox_specific[:, :4] * config.BBOX_STD_DEV)
# Class aware masks
mask_specific = mrcnn_mask[b,
np.arange(mrcnn_mask.shape[1]), :, :,
mrcnn_class_ids[b]]
visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific,
mrcnn_class_ids[b], dataset.class_names)
# Any repeated ROIs?
rows = np.ascontiguousarray(rois[b]).view(
np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1])))
_, idx = np.unique(rows, return_index=True)
print("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1]))
# In[20]:
if random_rois:
# Dispalay ROIs and corresponding masks and bounding boxes
ids = random.sample(range(rois.shape[1]), 8)
images = []
titles = []
for i in ids:
image = visualize.draw_box(sample_image.copy(),
rois[b, i, :4].astype(np.int32),
[255, 0, 0])
image = visualize.draw_box(image, refined_rois[i].astype(np.int64),
[0, 255, 0])
images.append(image)
titles.append("ROI {}".format(i))
images.append(mask_specific[i] * 255)
titles.append(dataset.class_names[mrcnn_class_ids[b, i]][:20])
display_images(images,
titles,
cols=4,
cmap="Blues",
interpolation="none")
# In[21]:
# Check ratio of positive ROIs in a set of images.
if random_rois:
limit = 10
temp_g = modellib.data_generator(dataset,
config,
shuffle=True,
random_rois=10000,
batch_size=1,
detection_targets=True)
total = 0
for i in range(limit):
_, [ids, _, _] = next(temp_g)
positive_rois = np.sum(ids[0] > 0)
total += positive_rois
print("{:5} {:5.2f}".format(positive_rois,
positive_rois / ids.shape[1]))
print("Average percent: {:.2f}".format(total / (limit * ids.shape[1])))
def detect_and_color_splash(model,
image_path=None,
video_path=None,
out_dir=''):
assert image_path or video_path
class_names = ['BG', 'adidas', 'apple']
# Image or video?
if image_path:
# Run model detection and generate the color splash effect
print("Running on {}".format(args.image))
# Read image
image = skimage.io.imread(args.image)
# Detect objects
r = model.detect([image], verbose=1)[0]
# Color splash
# splash = color_splash(image, r['masks'])
visualize.display_instances(image,
r['rois'],
r['masks'],
r['class_ids'],
class_names,
r['scores'],
making_image=True)
file_name = 'splash.png'
# Save output
# file_name = "splash_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())
# save_file_name = os.path.join(out_dir, file_name)
# skimage.io.imsave(save_file_name, splash)
elif video_path:
import cv2
# Video capture
vcapture = cv2.VideoCapture(video_path)
# width = int(vcapture.get(cv2.CAP_PROP_FRAME_WIDTH))
# height = int(vcapture.get(cv2.CAP_PROP_FRAME_HEIGHT))
width = 1600
height = 1600
fps = vcapture.get(cv2.CAP_PROP_FPS)
# Define codec and create video writer
file_name = "splash_{:%Y%m%dT%H%M%S}.wmv".format(
datetime.datetime.now())
vwriter = cv2.VideoWriter(file_name, cv2.VideoWriter_fourcc(*'MJPG'),
fps, (width, height))
count = 0
success = True
#For video, we wish classes keep the same mask in frames, generate colors for masks
colors = visualize.random_colors(len(class_names))
while success:
print("frame: ", count)
# Read next image
plt.clf()
plt.close()
success, image = vcapture.read()
if success:
# OpenCV returns images as BGR, convert to RGB
image = image[..., ::-1]
# Detect objects
r = model.detect([image], verbose=0)[0]
# Color splash
# splash = color_splash(image, r['masks'])
splash = visualize.display_instances(image,
r['rois'],
r['masks'],
r['class_ids'],
class_names,
r['scores'],
colors=colors,
making_video=True)
# Add image to video writer
vwriter.write(splash)
count += 1
vwriter.release()
print("Saved to ", file_name)
def detect_and_color_splash(model: modellib.MaskRCNN,
image_path: str = None,
video_path: str = None,
out_dir: str = None) -> None:
"""Detect objects in image/video and highlight them
Args:
model: trained model
image_path: path of image to be detected
video_path: path of video to be detected
out_dir: output directory
"""
assert image_path or video_path
# Get original file name
if image_path:
fname = os.path.basename(image_path)
fname = os.path.splitext(fname)[0]
elif video_path:
fname = os.path.basename(video_path)
fname = os.path.splitext(fname)[0]
else:
fname = ''
fname += '_splash'
# Generate output path
path = os.path.join(out_dir, fname)
# Generate colors for classes
colors = random_colors(len(CLS))
# Image or video?
if image_path:
# Run model detection and generate the color splash effect
print("Running on {}".format(image_path))
# Read image
image = skimage.io.imread(image_path)
# Detect objects
r = model.detect([image], verbose=1)[0]
# Color splash
splash = color_splash(image,
r['rois'],
r['masks'],
r['class_ids'],
CLS,
r['scores'],
colors=colors)
# Save output
path += ".png"
skimage.io.imsave(path, splash)
print("Saved to", path)
elif video_path:
print('Splash video:', video_path)
# Video capture
vcapture = cv.VideoCapture(video_path)
width = int(vcapture.get(cv.CAP_PROP_FRAME_WIDTH))
height = int(vcapture.get(cv.CAP_PROP_FRAME_HEIGHT))
fps = vcapture.get(cv.CAP_PROP_FPS)
print('Video size: ({}, {})'.format(width, height))
print('FPS:', fps)
# Define codec and create video writer
path += ".avi"
vwriter = cv.VideoWriter(path, cv.VideoWriter_fourcc(*'MJPG'), fps,
(width, height))
count = 0
success = True
while success:
print("frame:", count)
# Read next image
success, image = vcapture.read()
if success:
# OpenCV returns images as BGR, convert to RGB
image = image[..., ::-1]
# Detect objects
r = model.detect([image], verbose=0)[0]
# Color splash
splash = color_splash(image,
r['rois'],
r['masks'],
r['class_ids'],
CLS,
r['scores'],
colors=colors)
# RGB -> BGR to save image to video
splash = splash[..., ::-1]
# Add image to video writer
vwriter.write(splash)
count += 1
vwriter.release()
print("Saved to", path)
return None
def process_masked_image(image,
boxes,
masks,
class_ids,
class_names,
mask_threshold=0.0,
scores=None,
show_mask=True,
show_bbox=True,
colors=None,
captions=None):
"""
boxes: [num_instance, (y1, x1, y2, x2, class_id)] in image coordinates.
masks: [height, width, num_instances]
class_ids: [num_instances]
class_names: list of class names of the dataset
scores: (optional) confidence scores for each box
show_mask, show_bbox: To show masks and bounding boxes or not
colors: (optional) An array or colors to use with each object
captions: (optional) A list of strings to use as captions for each object
"""
# Number of instances
N = boxes.shape[0]
if N:
assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
# Generate random colors
dcolors = colors or visualize.random_colors(N)
colors = generate_class_colors()
# Show area outside image boundaries.
height, width = image.shape[:2]
font = cv2.FONT_HERSHEY_COMPLEX_SMALL
masked_image = image.astype(np.uint8).copy()
for i in range(N):
color = dcolors[i]
score = scores[i] if scores is not None else None
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
if score != None:
if score < mask_threshold:
continue
y1, x1, y2, x2 = boxes[i]
# Label
if not captions:
class_id = class_ids[i]
label = class_names[class_id]
x = random.randint(x1, (x1 + x2) // 2)
caption = "{} {:.3f}".format(label, score) if score else label
color = colors[class_id]
else:
caption = captions[i]
# Mask
mask = masks[:, :, i]
masked_image = visualize.apply_mask(masked_image, mask, color)
#modify only after the mask application
color = tuple([int(ch * 255) for ch in color])
cv2.rectangle(masked_image, (x1, y1), (x2, y2), color, 2)
cv2.putText(masked_image, caption, (x1, y1 + 8), font, 1,
(255, 255, 255), 1, cv2.LINE_AA)
return masked_image
while True:
start_time = time.time()
ret, images = cap.read()
rgb_image = images[0]
depth_image = images[1]
# print(rgb_image.shape, depth_image.shape)
padding_image = np.zeros((960, 1280, 3), np.uint8)
padding_image[240:720, 320:960] = rgb_image # 近距離でも遠くに見えるようにパディングする
# Run detection
results = model.detect([padding_image], verbose=1)
# Visualize results
result = results[0]
N = result['rois'].shape[0] # 検出数
result_image = padding_image.copy()
colors = visualize.random_colors(N)
for i in range(N):
'''クラス関係なく1物体ごと処理を行う'''
if class_names[result['class_ids'][i]] in filtered_classNames:
# Color
color = colors[i]
rgb = (round(color[0] * 255), round(color[1] * 255), round(color[2] * 255))
font = cv2.FONT_HERSHEY_SIMPLEX
# Bbox
result_image = visualize.draw_box(result_image, result['rois'][i], rgb)
# Class & Score
print(result['rois'][i])
text_top = class_names[result['class_ids'][i]] + ':' + str(result['scores'][i])
def detect(model,
weights_path,
image_path=None,
video_path=None,
colors_each_class=None,
show=True):
assert image_path or video_path
# Image or video?
if image_path:
# Run model detection and generate the color splash effect
print("Running on {}".format(image_path))
# Read image
image = skimage.io.imread(image_path)
# Detect objects
r = model.detect([image], verbose=1)[0]
if not colors_each_class:
colors = visualize.random_colors(len(r['class_ids']))
else:
colors = [colors_each_class[cid] for cid in r['class_ids']]
# Convert RGB to BGR in opencv:
image_cv = image[:, :, ::-1]
image_cv = image_cv.copy()
# Draw on image:
new_img = draw_on_image(image_cv, r, colors=colors)
out_dir = weights_path.split(os.path.basename(weights_path))[0]
file_name = os.path.basename(image_path)
cv2.imwrite(os.path.join(out_dir, 'detect_' + file_name), new_img,
[int(cv2.IMWRITE_JPEG_QUALITY), 100])
print("Saved to ", file_name)
if show:
cv2.imshow('', new_img)
cv2.waitKey(0)
cv2.destroyAllWindows()
return new_img
elif video_path:
# Video capture
vcapture = cv2.VideoCapture(video_path)
width = int(vcapture.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(vcapture.get(cv2.CAP_PROP_FRAME_HEIGHT))
fps = vcapture.get(cv2.CAP_PROP_FPS)
# Define codec and create video writer
file_name = "detection_{:%Y%m%dT%H%M%S}.avi".format(
datetime.datetime.now())
vwriter = cv2.VideoWriter(file_name, cv2.VideoWriter_fourcc(*'MJPG'),
fps, (width, height))
count = 0
success = True
while success:
print("frame: ", count)
# Read next image
success, image_cv = vcapture.read()
if success:
# OpenCV returns images as BGR, convert to RGB
image = image_cv[..., ::-1].copy()
# Detect objects
r = model.detect([image], verbose=0)[0]
colors = [colors_each_class[cid] for cid in r['class_ids']]
# Draw on image:
new_img = draw_on_image(image_cv, r, colors=colors)
# Add image to video writer
vwriter.write(new_img)
count += 1
vwriter.release()
print("Saved to ", file_name)
def detect_and_color_splash(model,
image_path=None,
video_path=None,
out_dir=''):
assert image_path or video_path
class_names = ['BG', 'cable']
if image_path:
# Run model detection and generate the color splash effect
print("Running on {}".format(args.image))
# Read image
image = skimage.io.imread(args.image)
# Detect objects
r = model.detect([image], verbose=1)[0]
# Color splash
splash = color_splash(image, r['masks'])
visualize.display_instances(image,
r['rois'],
r['masks'],
r['class_ids'],
class_names,
r['scores'],
making_image=True)
file_name = 'splash.png'
#Save output
#file_name = "splash_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())
#save_file_name = os.path.join(out_dir, file_name)
#skimage.io.imsave(save_file_name, splash)
elif video_path:
VIDEO_SAVE_DIR = '../../../Videos/save/'
import cv2
#import glob
#batch_size=1
count = 0
capture = cv2.VideoCapture(video_path)
fps = capture.get(cv2.CAP_PROP_FPS)
while True:
ret, frame = capture.read()
#cv2.imshow('Hallo',frame)
#frame = frame.astype(np.uint8)
#
# Bail out when the video file ends
if not ret:
break
# Save each frame of the video to a list
#frames = []
count += 1
frame = cv2.cvtColor(
np.array(frame), cv2.COLOR_BGR2RGB
) #cv2.cvtColor(numpy.array(pil_image), cv2.COLOR_RGB2BGR)
#cv2.imshow('Hallo',frame)
#print(count)
#frames.append(frame)
#if len(frames) == batch_size:
r = model.detect([frame], verbose=1)[0]
#for i, item in enumerate(zip(frames, results)):
#frame = item[0]
#r = item[1]
colors = visualize.random_colors(len(class_names))
frame = visualize.display_instances_video(count,
frame,
r['rois'],
r['masks'],
r['class_ids'],
class_names,
r['scores'],
colors=colors)
#(major_ver, minor_ver, subminor_ver) = (cv2.__version__).split('.')
# Clear the frames array to start the next batch
#frames = []
# Get all image file paths to a list.
#frames = []
#images = list(glob.iglob(os.path.join(VIDEO_SAVE_DIR, '*.*')))
# Sort the images by name index.
#images = sorted(images, key=lambda x: float(os.path.split(x)[1][:-3]))
#video = cv2.VideoCapture(os.path.join(VIDEO_SAVE_DIR, 'trailer1.mp4'));
# Find OpenCV version
"""
images = []
for img in glob.glob(VIDEO_SAVE_DIR+"images/*.jpg"):
n= cv2.imread(img)
images.append(n)
video = cv2.VideoWriter('video.avi',-1,1,(1200,900))
for q in range(1,count):
video.write(images[q])
cv2.destroyAllWindows()
video.release()
(major_ver, minor_ver, subminor_ver) = (cv2.__version__).split('.')
if int(major_ver) < 3 :
fps = video.get(cv2.cv.CV_CAP_PROP_FPS)
print("Frames per second using video.get(cv2.cv.CV_CAP_PROP_FPS): {0}".format(fps))
else :
fps = video.get(cv2.CAP_PROP_FPS)
#print("Frames per second using video.get(cv2.CAP_PROP_FPS) : {0}".format(fps))
video.release()
"""
images = [
img for img in sorted(os.listdir(VIDEO_SAVE_DIR))
if img.endswith(".jpg")
]
frame = cv2.imread(os.path.join(VIDEO_SAVE_DIR, images[0]))
height, width, layers = frame.shape
video_name = 'Detection.avi'
video = cv2.VideoWriter(video_name, 0, fps, (width, height))
for image in images:
video.write(cv2.imread(os.path.join(VIDEO_SAVE_DIR, image)))
cv2.destroyAllWindows()
video.release()
"""elif video_path:
def display_instances(image, boxes, masks, class_ids, class_names, scores=None):
N = boxes.shape[0]
if not N:
print("\n*** No instances to display *** \n")
else:
assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
_, ax = plt.subplots(1, figsize=(6.4, 6.4))
# Generate random colors
colors = random_colors(N)
# Show area outside image boundaries.
height, width = image.shape[:2]
ax.set_ylim(height, 0)
ax.set_xlim(0, width)
ax.axis('off')
masked_image = image.astype(np.uint32).copy()
for i in range(N):
color = colors[i]
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,
alpha=0.7, linestyle="dashed",
edgecolor=color, facecolor='none')
ax.add_patch(p)
# Label
class_id = class_ids[i]
score = scores[i] if scores is not None else None
label = class_names[class_id]
x = random.randint(x1, (x1 + x2) // 2)
caption = "{} {:.3f}".format(label, score) if score else label
ax.text(x1, y1 + 8, caption,
color='w', size=11, backgroundcolor="none")
mask = masks[:, :, i]
masked_image = apply_mask(masked_image, mask, color)
# Mask Polygon
# Pad to ensure proper polygons for masks that touch image edges.
padded_mask = np.zeros(
(mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8)
padded_mask[1:-1, 1:-1] = mask
contours = find_contours(padded_mask, 0.5)
for verts in contours:
# Subtract the padding and flip (y, x) to (x, y)
verts = np.fliplr(verts) - 1
p = Polygon(verts, facecolor="none", edgecolor=color)
ax.add_patch(p)
ax.imshow(masked_image.astype(np.uint8))
plt.gca().set_axis_off()
plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)
plt.margins(0, 0)
plt.gca().xaxis.set_major_locator(plt.NullLocator())
plt.gca().yaxis.set_major_locator(plt.NullLocator())
plt.savefig("static/scored.jpg", dpi=80, facecolor='w', edgecolor='k', bbox_inches='tight', pad_inches = 0)
def draw_boxes(image,
boxes=None,
refined_boxes=None,
masks=None,
captions=None,
visibilities=None,
title="",
ax=None):
# Number of boxes
assert boxes is not None or refined_boxes is not None
N = boxes.shape[0] if boxes is not None else refined_boxes.shape[0]
# Matplotlib Axis
if not ax:
_, ax = plt.subplots(1, figsize=(12, 12))
# Generate random colors
colors = visualize.random_colors(N)
# Show area outside image boundaries.
margin = image.shape[0] // 10
ax.set_ylim(image.shape[0] + margin, -margin)
ax.set_xlim(-margin, image.shape[1] + margin)
ax.axis('off')
ax.set_title(title)
masked_image = image.astype(np.uint32).copy()
for i in range(N):
# Box visibility
visibility = visibilities[i] if visibilities is not None else 1
if visibility == 0:
color = "gray"
style = "dotted"
alpha = 0.5
elif visibility == 1:
color = colors[i]
style = "dotted"
alpha = 1
elif visibility == 2:
color = colors[i]
style = "solid"
alpha = 1
# Boxes
if boxes is not None:
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in cropping.
continue
y1, x1, y2, x2 = boxes[i]
p = patches.Rectangle((x1, y1),
x2 - x1,
y2 - y1,
linewidth=2,
alpha=alpha,
linestyle=style,
edgecolor=color,
facecolor='none')
ax.add_patch(p)
# Refined boxes
if refined_boxes is not None and visibility > 0:
ry1, rx1, ry2, rx2 = refined_boxes[i].astype(np.int32)
p = patches.Rectangle((rx1, ry1),
rx2 - rx1,
ry2 - ry1,
linewidth=2,
edgecolor=color,
facecolor='none')
ax.add_patch(p)
# Connect the top-left corners of the anchor and proposal
if boxes is not None:
ax.add_line(lines.Line2D([x1, rx1], [y1, ry1], color=color))
# Captions
if captions is not None:
caption = captions[i]
# If there are refined boxes, display captions on them
if refined_boxes is not None:
y1, x1, y2, x2 = ry1, rx1, ry2, rx2
ax.text(x1,
y1,
caption,
size=11,
verticalalignment='top',
color='w',
backgroundcolor="none",
bbox={
'facecolor': color,
'alpha': 0.5,
'pad': 2,
'edgecolor': 'none'
})
ax.imshow(masked_image.astype(np.uint8))
plt.show()
def predict(img_name, model):
image = skimage.io.imread(img_name)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
results = model.detect([image], verbose=1)
# boxes: [num_instance, (y1, x1, y2, x2, class_id)] in image coordinates
boxes = results[0]['rois']
# masks: [height, width, num_instances]
masks = results[0]['masks']
# class_ids: [num_instances]
class_ids = results[0]['class_ids']
scores = results[0]['scores']
print('scores:', scores)
N = boxes.shape[0]
print('N:', N)
if not N:
print("No object")
return None, None, None
else:
assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
score_index = np.argmax(scores)
print('score_index:', score_index)
# if N != 1:
# print('该张图片检测出多个物体')
# return None , None , None
# else:
# lf = LabelFile()
#
# label_img = np.zeros(masks.shape, dtype=np.uint8)
# label_img[np.where(masks == 1)] = 255
# label = class_names[class_ids[0]]
#
# colors = visualize.random_colors(N , bright=True)
#
# mask_image = visualize.apply_mask(image , masks[:,:,0] , colors[0])
#
# # Mask Polygon
# padded_mask = np.zeros((masks[:,:,0].shape[0] + 2 , masks[:,:,0].shape[1] + 2) , dtype=np.uint8)
# padded_mask[1:-1 , 1:-1] = masks[:,:,0]
# contours = find_contours(padded_mask , 0.5)
#
# json_info = {}
#
# json_info["shapes"] = []
# shape_info = {}
# shape_info["line_color"] = None
# shape_info["fill_color"] = None
# shape_info["label"] = "141"
# shape_info["points"] = []
#
# for verts in contours:
# verts = np.fliplr(verts) - 1
#
# for index , vt in enumerate(verts.tolist()):
# if index % 15 == 0:
# shape_info["points"].append(vt)
# json_info["shapes"].append(shape_info)
#
# print('json file:' , img_name.replace("png" , "json"))
# # write_json(json_info , img_name.replace("png" , "json"))
#
# lf.save(img_name.replace("png" , "json") , shapes=json_info["shapes"] , imagePath=img_name , fillColor=[255, 0, 0, 128] , lineColor=[0 , 255 , 0 , 128] , flags={})
#
#
# return label_img , label , mask_image
lf = LabelFile()
label_img = np.zeros(masks[:, :, score_index].shape, dtype=np.uint8)
label_img[np.where(masks[:, :, score_index] == 1)] = 255
label = class_names[class_ids[score_index]]
colors = visualize.random_colors(N, bright=True)
mask_image = visualize.apply_mask(image, masks[:, :, 0], colors[0])
# Mask Polygon
padded_mask = np.zeros((masks[:, :, score_index].shape[0] + 2,
masks[:, :, score_index].shape[1] + 2),
dtype=np.uint8)
padded_mask[1:-1, 1:-1] = masks[:, :, score_index]
contours = find_contours(padded_mask, 0.5)
json_info = {}
json_info["shapes"] = []
shape_info = {}
shape_info["line_color"] = None
shape_info["fill_color"] = None
shape_info["label"] = label
shape_info["points"] = []
for verts in contours:
verts = np.fliplr(verts) - 1
for index, vt in enumerate(verts.tolist()):
if index % 15 == 0:
shape_info["points"].append(vt)
json_info["shapes"].append(shape_info)
print('json file:', img_name.replace("png", "json"))
# write_json(json_info , img_name.replace("png" , "json"))
lf.save(img_name.replace("png", "json"),
shapes=json_info["shapes"],
imagePath=os.path.basename(img_name),
fillColor=[255, 0, 0, 128],
lineColor=[0, 255, 0, 128],
flags={})
return label_img, label, mask_image
def display_instances(self,
image,
boxes,
masks,
class_ids,
class_names,
scores=None,
title="",
figsize=(16, 16),
ax=None,
show_mask=True,
show_bbox=True,
colors=None,
captions=None):
# Number of instances
N = boxes.shape[0]
if not N:
logging.info("\n*** No instances to display *** \n")
else:
assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
_, ax = plt.subplots(1, figsize=figsize)
# Generate random colors
colors = colors or visualize.random_colors(N)
# Show area outside image boundaries.
height, width = image.shape[:2]
ax.set_ylim(height)
ax.set_xlim(width)
ax.axis('off')
masked_image = image.astype(np.uint32).copy()
for i in range(N):
color = colors[i]
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
if show_bbox:
p = visualize.patches.Rectangle((x1, y1),
x2 - x1,
y2 - y1,
linewidth=2,
alpha=0.7,
linestyle="dashed",
edgecolor=color,
facecolor='none')
ax.add_patch(p)
# Label
if not captions:
class_id = class_ids[i]
score = scores[i] if scores is not None else None
label = class_names[class_id]
caption = "{} {:.3f}".format(label, score) if score else label
else:
caption = captions[i]
ax.text(x1,
y1 + 8,
caption,
color='w',
size=11,
backgroundcolor="none")
# Mask
mask = masks[:, :, i]
if show_mask:
masked_image = visualize.apply_mask(masked_image, mask, color)
# Mask Polygon
# Pad to ensure proper polygons for masks that touch image edges.
padded_mask = np.zeros((mask.shape[0] + 2, mask.shape[1] + 2),
dtype=np.uint8)
padded_mask[1:-1, 1:-1] = mask
contours = visualize.find_contours(padded_mask, 0.5)
for verts in contours:
# Subtract the padding and flip (y, x) to (x, y)
verts = np.fliplr(verts) - 1
p = Polygon(verts, facecolor="none", edgecolor=color)
ax.add_patch(p)
ax.imshow(masked_image.astype(np.uint8))
return ax
def draw_predict(image,
image_name,
boxes,
masks,
class_ids,
class_names,
figsize=(12, 12),
show_mask=True):
# Number of instances
N = boxes.shape[0]
if not N:
print("\n*** No instances to display *** \n")
else:
assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
fig, ax = plt.subplots(1, figsize=figsize)
ax.imshow(image)
# Generate random colors
colors = visualize.random_colors(N)
# Show area outside image boundaries.
height, width = image.shape[:2]
ax.set_ylim(height + 10, -10)
ax.set_xlim(-10, width + 10)
ax.axis('off')
masked_image = image.astype(np.uint32).copy()
for i in range(N):
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
p = patches.Rectangle((x1, y1),
x2 - x1,
y2 - y1,
linewidth=2,
alpha=0.7,
linestyle="dashed",
edgecolor=colors[i],
facecolor='none')
ax.add_patch(p)
# Label
class_id = class_ids[i]
label = class_names[class_id]
caption = label
ax.text(x1, y1 + 8, caption, color='w', size=11, backgroundcolor="k")
# Mask
mask = masks[:, :, i]
masked_image = visualize.apply_mask(masked_image, mask, colors[i])
# Mask Polygon
# Pad to ensure proper polygons for masks that touch image edges.
padded_mask = np.zeros((mask.shape[0] + 2, mask.shape[1] + 2),
dtype=np.uint8)
padded_mask[1:-1, 1:-1] = mask
contours = find_contours(padded_mask, 0.5)
for verts in contours:
# Subtract the padding and flip (y, x) to (x, y)
verts = np.fliplr(verts) - 1
p = Polygon(verts, facecolor="none", edgecolor=colors[i])
ax.add_patch(p)
ax.imshow(masked_image.astype(np.uint8))
fig.savefig(os.path.join(UPLOAD_FOLDER, 'predict_' + image_name),
bbox_inches='tight',
pad_inches=0)
def display_instances(image, boxes, masks, class_ids, class_names,
scores=None, title="",
figsize=(16, 16), ax=None,
show_mask=True, show_bbox=True,
colors=None, captions=None):
"""
boxes: [num_instance, (y1, x1, y2, x2, class_id)] in image coordinates.
masks: [height, width, num_instances]
class_ids: [num_instances]
class_names: list of class names of the dataset
scores: (optional) confidence scores for each box
title: (optional) Figure title
show_mask, show_bbox: To show masks and bounding boxes or not
figsize: (optional) the size of the image
colors: (optional) An array or colors to use with each object
captions: (optional) A list of strings to use as captions for each object
"""
# Number of instances
N = boxes.shape[0]
if not N:
#print("\n*** No instances to display *** \n")
else:
assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
img = img_to_array(image)
image = img[:,:,:3]
# If no axis is passed, create one and automatically call show()
auto_show = False
if not ax:
fig, ax = plt.subplots(1, figsize=figsize)
auto_show = True
# Generate random colors
colors = colors or random_colors(N)
# Show area outside image boundaries.
height, width = image.shape[:2]
ax.set_ylim(height + 10, -10)
ax.set_xlim(-10, width + 10)
ax.axis('off')
ax.set_title(title)
masked_image = image.astype(np.uint32).copy()
for i in range(N):
color = colors[i]
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
if show_bbox:
p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,
alpha=0.7, linestyle="dashed",
edgecolor=color, facecolor='none')
ax.add_patch(p)
# Label
if not captions:
class_id = class_ids[i]
score = scores[i] if scores is not None else None
label = class_names[class_id]
caption = "{} {:.3f}".format(label, score) if score else label
else:
caption = captions[i]
ax.text(x1, y1 + 8, caption,
color='w', size=11, backgroundcolor="none")
# Mask
mask = masks[:, :, i]
if show_mask:
masked_image = apply_mask(masked_image, mask, color)
# Mask Polygon
# Pad to ensure proper polygons for masks that touch image edges.
padded_mask = np.zeros(
(mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8)
padded_mask[1:-1, 1:-1] = mask
contours = find_contours(padded_mask, 0.5)
for verts in contours:
# Subtract the padding and flip (y, x) to (x, y)
verts = np.fliplr(verts) - 1
p = Polygon(verts, facecolor="none", edgecolor=color)
ax.add_patch(p)
#masked_image = ax.imshow(masked_image.astype(np.uint8))
#mpld3.plugins.connect(fig, extra.InteractiveLegendPlugin())
#return fig_to_html(fig)
#return mpld3.fig_to_html(fig)
return masked_image.astype(np.uint8)
def display_results(target,
image,
boxes,
masks,
class_ids,
scores=None,
title="",
figsize=(16, 16),
ax=None,
show_mask=True,
show_bbox=True,
colors=None,
captions=None):
"""
boxes: [num_instance, (y1, x1, y2, x2, class_id)] in image coordinates.
masks: [height, width, num_instances]
class_ids: [num_instances]
class_names: list of class names of the dataset
scores: (optional) confidence scores for each box
title: (optional) Figure title
show_mask, show_bbox: To show masks and bounding boxes or not
figsize: (optional) the size of the image
colors: (optional) An array or colors to use with each object
captions: (optional) A list of strings to use as captions for each object
"""
# Number of instances
N = boxes.shape[0]
if not N:
print("\n*** No instances to display *** \n")
else:
assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
# If no axis is passed, create one and automatically call show()
auto_show = False
if not ax:
from matplotlib.gridspec import GridSpec
# Use GridSpec to show target smaller than image
fig = plt.figure(figsize=figsize)
gs = GridSpec(3, 3)
ax = plt.subplot(gs[:, 1:])
target_ax = plt.subplot(gs[1, 0])
auto_show = True
# Generate random colors
colors = colors or visualize.random_colors(N)
# Show area outside image boundaries.
height, width = image.shape[:2]
ax.set_ylim(height + 10, -10)
ax.set_xlim(-10, width + 10)
ax.axis('off')
ax.set_title(title)
target_height, target_width = target.shape[:2]
target_ax.set_ylim(target_height + 10, -10)
target_ax.set_xlim(-10, target_width + 10)
target_ax.axis('off')
# target_ax.set_title('target')
masked_image = image.astype(np.uint32).copy()
for i in range(N):
color = colors[i]
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
if show_bbox:
p = visualize.patches.Rectangle((x1, y1),
x2 - x1,
y2 - y1,
linewidth=2,
alpha=0.7,
linestyle="dashed",
edgecolor=color,
facecolor='none')
ax.add_patch(p)
# Label
if not captions:
class_id = class_ids[i]
score = scores[i] if scores is not None else None
x = random.randint(x1, (x1 + x2) // 2)
caption = "{:.3f}".format(score) if score else 'no score'
else:
caption = captions[i]
ax.text(x1,
y1 + 8,
caption,
color='w',
size=11,
backgroundcolor="none")
# Mask
mask = masks[:, :, i]
if show_mask:
masked_image = visualize.apply_mask(masked_image, mask, color)
# Mask Polygon
# Pad to ensure proper polygons for masks that touch image edges.
padded_mask = np.zeros((mask.shape[0] + 2, mask.shape[1] + 2),
dtype=np.uint8)
padded_mask[1:-1, 1:-1] = mask
contours = visualize.find_contours(padded_mask, 0.5)
for verts in contours:
# Subtract the padding and flip (y, x) to (x, y)
verts = np.fliplr(verts) - 1
p = visualize.Polygon(verts, facecolor="none", edgecolor=color)
ax.add_patch(p)
ax.imshow(masked_image.astype(np.uint8))
target_ax.imshow(target.astype(np.uint8))
if auto_show:
plt.show()
return
def visualize(self, image, results, ax=None):
from mrcnn.visualize import random_colors, apply_mask
from matplotlib import patches, lines
from matplotlib.patches import Polygon
from skimage.measure import find_contours
figsize=(16, 16)
scores=None
title=""
show_mask=True
show_bbox=True
colors=None
captions=None
boxes, masks, class_ids = [], [], []
for res in results:
boxes.append(res.roi)
masks.append(res.mask)
class_ids.append(res.cid)
boxes = np.array(boxes)
masks = np.array(masks)
masks = np.swapaxes(np.swapaxes(masks, 0, 2), 0, 1)
# print(boxes.shape, masks.shape)
class_ids = np.array(class_ids)
# print(masks.shape, image.shape)
# Number of instances
N = boxes.shape[0]
if not N:
print("\n*** No instances to display *** \n")
else:
assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
# If no axis is passed, create one and automatically call show()
auto_show = False
if not ax:
_, ax = plt.subplots(1, figsize=figsize)
auto_show = True
# Generate random colors
colors = colors or random_colors(N)
# Show area outside image boundaries.
height, width = image.shape[:2]
ax.set_ylim(height + 10, -10)
ax.set_xlim(-10, width + 10)
ax.axis('off')
ax.set_title(title)
masked_image = image.astype(np.uint32).copy()
for i in range(N):
color = colors[i]
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
if show_bbox:
p = patches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=2,
alpha=0.7, linestyle="dashed",
edgecolor=color, facecolor='none')
ax.add_patch(p)
# Label
if not captions:
class_id = class_ids[i]
score = scores[i] if scores is not None else None
label = class_names[class_id]
x = random.randint(x1, (x1 + x2) // 2)
caption = "{} {:.3f}".format(label, score) if score else label
else:
caption = captions[i]
ax.text(x1, y1 + 8, caption,
color='w', size=11, backgroundcolor="none")
# Mask
mask = masks[:, :, i]
if show_mask:
masked_image = apply_mask(masked_image, mask, color)
# Mask Polygon
# Pad to ensure proper polygons for masks that touch image edges.
padded_mask = np.zeros(
(mask.shape[0] + 2, mask.shape[1] + 2), dtype=np.uint8)
padded_mask[1:-1, 1:-1] = mask
contours = find_contours(padded_mask, 0.5)
for verts in contours:
# Subtract the padding and flip (y, x) to (x, y)
verts = np.fliplr(verts) - 1
p = Polygon(verts, facecolor="none", edgecolor=color)
ax.add_patch(p)
ax.imshow(masked_image.astype(np.uint8))
if auto_show:
plt.show()
def display_grid(target_list,
image_list,
boxes_list,
masks_list,
class_ids_list,
scores_list=None,
category_names_list=None,
title="",
figsize=(16, 16),
ax=None,
show_mask=True,
show_bbox=True,
colors=None,
captions=None,
show_scores=True,
target_shift=10,
fontsize=14,
linewidth=2,
save=False):
"""
boxes: [num_instance, (y1, x1, y2, x2, class_id)] in image coordinates.
masks: [height, width, num_instances]
class_ids: [num_instances]
class_names: list of class names of the dataset
scores: (optional) confidence scores for each box
title: (optional) Figure title
show_mask, show_bbox: To show masks and bounding boxes or not
figsize: (optional) the size of the image
colors: (optional) An array or colors to use with each object
captions: (optional) A list of strings to use as captions for each object
"""
if type(target_list) == list:
M = int(np.sqrt(len(target_list)))
if len(target_list) - M**2 > 1e-3:
M = M + 1
else:
M = 1
target_list = [target_list]
image_list = [image_list]
boxes_list = [boxes_list]
masks_list = [masks_list]
class_ids_list = [class_ids_list]
if scores_list is not None:
scores_list = [scores_list]
# If no axis is passed, create one and automatically call show()
auto_show = False
if not ax:
from matplotlib.gridspec import GridSpec
# Use GridSpec to show target smaller than image
fig = plt.figure(figsize=figsize)
gs = GridSpec(M,
M,
hspace=0.1,
wspace=0.02,
left=0,
right=1,
bottom=0,
top=1)
# auto_show = True REMOVE
index = 0
for m1 in range(M):
for m2 in range(M):
ax = plt.subplot(gs[m1, m2])
if index >= len(target_list):
continue
target = target_list[index]
image = image_list[index]
boxes = boxes_list[index]
masks = masks_list[index]
class_ids = class_ids_list[index]
scores = scores_list[index]
# Number of instances
N = boxes.shape[0]
if not N:
print("\n*** No instances to display *** \n")
else:
assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
# Generate random colors
colors = visualize.random_colors(N)
# Show area outside image boundaries.
height, width = image.shape[:2]
ax.set_ylim(height, 0)
ax.set_xlim(0, width)
ax.axis('off')
ax.set_title(title)
masked_image = image.astype(np.uint32).copy()
for i in range(N):
color = colors[i]
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
if show_bbox:
p = visualize.patches.Rectangle((x1, y1),
x2 - x1,
y2 - y1,
linewidth=linewidth,
alpha=0.7,
linestyle="dashed",
edgecolor=color,
facecolor='none')
ax.add_patch(p)
# Label
if not captions:
class_id = class_ids[i]
score = scores[i] if scores is not None else None
x = random.randint(x1, (x1 + x2) // 2)
caption = "{:.3f}".format(score) if score else 'no score'
else:
caption = captions[i]
if show_scores:
ax.text(x1,
y1 + 8,
caption,
color='w',
size=int(10 / 14 * fontsize),
backgroundcolor="none")
# Mask
mask = masks[:, :, i]
if show_mask:
masked_image = visualize.apply_mask(
masked_image, mask, color)
# Mask Polygon
# Pad to ensure proper polygons for masks that touch image edges.
padded_mask = np.zeros((mask.shape[0] + 2, mask.shape[1] + 2),
dtype=np.uint8)
padded_mask[1:-1, 1:-1] = mask
contours = visualize.find_contours(padded_mask, 0.5)
for verts in contours:
# Subtract the padding and flip (y, x) to (x, y)
verts = np.fliplr(verts) - 1
p = visualize.Polygon(verts,
facecolor="none",
edgecolor=color)
ax.add_patch(p)
ax.imshow(masked_image.astype(np.uint8))
target_height, target_width = target.shape[:2]
target_height = target_height // 2
target_width = target_width // 2
target_area = target_height * target_width
target_scaling = np.sqrt((192 // 2 * 96 // 2) / target_area)
target_height = int(target_height * target_scaling)
target_width = int(target_width * target_scaling)
ax.imshow(target,
extent=[
target_shift, target_shift + target_width * 2,
height - target_shift,
height - target_shift - target_height * 2
],
zorder=9)
rect = visualize.patches.Rectangle(
(target_shift, height - target_shift),
target_width * 2,
-target_height * 2,
linewidth=5,
edgecolor='white',
facecolor='none',
zorder=10)
ax.add_patch(rect)
if category_names_list is not None:
plt.title(category_names_list[index], fontsize=fontsize)
index = index + 1
if auto_show:
plt.show()
if save:
fig.savefig('grid.pdf', bbox_inches='tight')
return
def main():
# video = Video(Config.input_video_file)
output_video = None
if Config.create_masked_video:
# output_video = cv2.VideoWriter(CONFIG["output_video_file"], cv2.VideoWriter_fourcc(*"mp4v"),
# 30, frame.shape[:-1])
pass
background = cv2.imread(Config.background_file)
if Config.subtract_background:
background = cv2.imread(Config.background_file)
pickler = BATRPickle(in_file=Config.input_mask_file)
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))
for frame_n in range(Config.offset, Config.end):
store = []
out_frame = np.copy(background)
print(frame_n)
frame = cv2.imread(f"input/videos/frames/frame{frame_n:06d}.jpg")
results = pickler.unpickle("frame{:06d}".format(frame_n))
r = results[0]
n = r['rois'].shape[0]
colors = visualize.random_colors(n)
if Config.subtract_background:
frame = (background - frame) + (frame - background)
for i in range(n):
obj = DetectedObject(_type=class_names[r['class_ids'][i]],
_probability=1,
image=frame[r['rois'][i][0]:r['rois'][i][2], r['rois'][i][1]:r['rois'][i][3]],
_xa=r['rois'][i][1],
_ya=r['rois'][i][0],
_xb=r['rois'][i][3],
_yb=r['rois'][i][2],
_w=abs(r['rois'][i][1] - r['rois'][i][3]),
_h=abs(r['rois'][i][0] - r['rois'][i][2]))
cx_axis = int((obj.xa + obj.xb) / 2)
cy_axis = int((obj.ya + obj.yb) / 2)
mask = np.float32(255 * r['masks'][:, :, i])
# color_for_obj, obj_index, near_value = color_for_object(obj, store, colors)
cv2.imwrite("mask.jpg", mask)
# mask = None
_alpha = cv2.imread("mask.jpg")
# cv2.imshow("cc", mask)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
_forg = np.float32(obj.image)
_back = np.float32(background[obj.ya:obj.yb, obj.xa:obj.xb])
_alpha = np.float32(_alpha) / 255
_forg = cv2.multiply(_alpha[obj.ya:obj.yb, obj.xa:obj.xb], _forg)
_back = cv2.multiply(1.0 - _alpha[obj.ya:obj.yb, obj.xa:obj.xb], _back)
output = cv2.add(_forg, _back)
_forg = cv2.dilate(_forg, kernel, iterations=3)
# cv2.imshow("cc", _forg)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
out_frame = visualize.apply_mask(frame, r['masks'][:, :, i], random.choice(colors))
# if obj.type == "car":
# out_frame = visualize.apply_mask(frame, r['masks'][:, :, i], random.choice(colors))
# # out_frame[obj.ya:obj.yb, obj.xa:obj.xb] = output
#
# # cv2.putText(frame, "Frame # {}".format(frame_n),
# # (250, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 2, cv2.LINE_4)
#
# if Config.display_object_info:
# cv2.putText(frame, "({},{},{})".format(cx_axis, cy_axis, obj.type),
# (cx_axis, cy_axis + 100), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2, cv2.LINE_AA)
# fgMask = backSub.apply(background_copy)
# background_copy = frame
track_for_object(out_frame, store, "car")
if Config.display_image or Config.display_video:
cv2.imshow("output", out_frame)
# cv2.imshow("output", fgMa sk)
if Config.display_image:
cv2.waitKey(0)
cv2.destroyAllWindows()
elif Config.display_video:
if cv2.waitKey(1) & 0xFF == ord('q'):
break
#
if Config.create_masked_video:
# frame = cv2.cvtColor(background_copy, cv2.COLOR_BGR2RGB)
cv2.imwrite(f"output/ferozpur10012019_maskrcnn/frame{frame_n:06d}.jpg", out_frame)
# output_video.write(np.uint8(out_frame))
if Config.display_video:
cv2.destroyAllWindows()
if Config.create_masked_video and output_video:
output_video.release()
def detect_and_color_splash(model, image_path=None, video_path=None):
assert image_path or video_path
class_names = ["CTV", "CTV-SIB"]
# class_names = ['BG', 'L-eye', 'R-eye', 'Brain stem']
# class_names = ['BODY', 'Spinal cord', 'Lung', 'Spinal cord+5mm', 'Airway', 'Heart', 'Brain stem', 'L-Parotid', 'R-Parotid', 'Bladder',
# 'Chiasma', 'Rectum', 'R-lens', 'L-lens', 'L-Optic nerve', 'R-eye',
# 'R-Optic nerve', 'L-eye', 'Liver', 'Thyroid', "R't Lung", "L't Lung", 'GTV-N', 'CTV-L', "R't_kidney", "L't_kidney",
# 'GTV-T', 'Stomach', 'Bladder wall', 'Rectum wall', 'Sig Colon', "R't kidney",
# "L't Kidney", "L't OPN", "R't OPN", 'Spinal Cord', 'Body']
# Image or video?
if image_path:
# Run model detection and generate the color splash effect
print("Running on {}".format(image))
# Read image
image = skimage.io.imread(image)
# Detect objects
r = model.detect([image], verbose=1)[0]
visualize.display_instances(image,
r['rois'],
r['masks'],
r['class_ids'],
class_names,
r['scores'],
making_image=True)
# Color splash
# splash = color_splash(image, r['masks'])
# Save output
# file_name = "splash_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())
# skimage.io.imsave(file_name, splash)
elif video_path:
import cv2
# Video capture
vcapture = cv2.VideoCapture(video_path)
width = int(vcapture.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(vcapture.get(cv2.CAP_PROP_FRAME_HEIGHT))
fps = vcapture.get(cv2.CAP_PROP_FPS)
# Define codec and create video writer
file_name = "splash_{:%Y%m%dT%H%M%S}.avi".format(
datetime.datetime.now())
vwriter = cv2.VideoWriter(file_name, cv2.VideoWriter_fourcc(*'MJPG'),
fps, (width, height))
count = 0
success = True
colors = visualize.random_colors(len(class_names))
while success:
print("frame: ", count)
# Read next image
success, image = vcapture.read()
if success:
# OpenCV returns images as BGR, convert to RGB
image = image[..., ::-1]
# Detect objects
r = model.detect([image], verbose=0)[0]
splash = visualize.display_instances(image,
r['rois'],
r['masks'],
r['class_ids'],
class_names,
r['scores'],
colors=colors,
making_video=True)
# # Color splash
# splash = color_splash(image, r['masks'])
# # RGB -> BGR to save image to video
# splash = splash[..., ::-1]
# Add image to video writer
vwriter.write(splash)
count += 1
vwriter.release()
print("Saved to ", file_name)
boxes = r['rois']
masks = r['masks']
class_ids = r['class_ids']
scores = r['scores']
def meets_criteria(candidate_class, target_classes, candidate_score,
target_score):
return candidate_class in target_classes and candidate_score >= max(
target_score, 0.70)
# Number of instances
N = boxes.shape[0]
colors = random_colors(N)
# Filter indices based on class name and score
idx = [
i for i in range(N)
if meets_criteria(class_names[class_ids[i]], vehicles, scores[i], 0.70)
]
for i in idx:
mask = masks[:, :, i]
mask_image = visualize.convert_mask_to_image(image, mask, colors[i])
if mode == 'roi':
# create an empty image with the same dimensions as the original one
mask_image_uncropped = np.zeros(original.shape, dtype=np.uint8)
# paste the mask onto the empty image
def color_splash(image: np.ndarray,
boxes: np.ndarray,
masks: np.ndarray,
class_ids: np.ndarray,
class_names: List[str],
scores: np.ndarray,
show_mask: bool = True,
show_bbox: bool = True,
colors: List[Tuple[float]] = None) -> np.ndarray:
"""Paint color for detected objects
Args:
image: image to paint color on
boxes: matrix of bounding boxes
masks: matrix of boolean to indicate if the pixel is a part of detected
objects
class_ids: class id
class_names: class names excluding background, in the same order of
``class_ids``
scores: matrix of detection scores
show_mask: whether to paint masks on image
show_bbox: whether to paint bounding boxes on image
colors: colors to be used for painting
Returns:
masked_image: image with painting
"""
# Number of instances
N = boxes.shape[0]
if not N:
return image
else:
assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
# Generate random colors
if colors is None:
colors = random_colors(len(class_names))
masked_image = image.astype(np.uint8).copy()
for i in range(N):
# class id
class_id = class_ids[i]
# get color of class
colors_rgb = colors[class_id - 1]
color_bgr = colors_rgb[::-1]
color_bgr_255 = tuple(round(255 * x) for x in color_bgr)
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
if show_bbox:
masked_image = cv.rectangle(masked_image, (x1, y1), (x2, y2),
color_bgr_255, 2)
# Mask
mask = masks[:, :, i]
if show_mask:
masked_image = apply_mask(masked_image, mask, color_bgr)
# Mask Polygon
# Pad to ensure proper polygons for masks that touch image edges.
padded_mask = np.zeros((mask.shape[0] + 2, mask.shape[1] + 2),
dtype=np.uint8)
padded_mask[1:-1, 1:-1] = mask
contours = find_contours(padded_mask, 0.5)
for verts in contours:
# Subtract the padding and flip (y, x) to (x, y)
verts = np.fliplr(verts) - 1
masked_image = cv.polylines(masked_image, np.int32([verts]), True,
color_bgr_255, 3)
# Label
score = scores[i] if scores is not None else None
label = class_names[class_id - 1] # ids include bg but not in names
caption = "{} {:.3f}".format(label, score) if score else label
cv.putText(masked_image, caption, (x1, y2), cv.FONT_HERSHEY_PLAIN, 1.5,
(255, 255, 255), 2, cv.LINE_4)
return masked_image
# one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
GPU_COUNT = 1
IMAGES_PER_GPU = 1
if __name__ == '__main__':
class_names = ['BG', 'arm', 'ring']
config = InferenceConfig()
config.display()
model = modellib.MaskRCNN(mode="inference", config=config, model_dir='/home/simon/logs/surgery_200')
model_path = PRETRAINED_MODEL_PATH
# or if you want to use the latest trained model, you can use :
# model_path = model.find_last()[1]
model.load_weights(model_path, by_name=True)
colors = visualize.random_colors(len(class_names))
cap = cv2.VideoCapture(0)
while True:
_, frame = cap.read()
predictions = model.detect([frame],
verbose=1) # We are replicating the same image to fill up the batch_size
p = predictions[0]
output = visualize.display_instances(frame, p['rois'], p['masks'], p['class_ids'],
class_names, p['scores'], colors=colors, real_time=True)
cv2.imshow("Mask RCNN", output)
k = cv2.waitKey(10)
if k & 0xFF == ord('q'):
break
def make_visuals(model,
classNames=None,
imagePath=None,
videoPath=None,
outPath=None):
assert imagePath or videoPath
assert outPath
classNames = [
'BG', 'bypass-r', 'bypass-v', 'intake', 'ladderframe', 'pipe1', 'pipe2'
]
# Image or video?
if imagePath:
# Run model detection and generate the color splash effect
print("Running on {}".format(imagePath))
# Read image
image = skimage.io.imread(args.image)
# Detect objects
r = model.detect([image], verbose=1)[0]
# Create visual and save image
visualize_instances(image,
r['rois'],
r['masks'],
r['class_ids'],
classNames,
r['scores'],
making_image=True,
file_name=outPath)
elif videoPath:
import cv2
# Video capture
vcapture = cv2.VideoCapture(videoPath)
# width = int(vcapture.get(cv2.CAP_PROP_FRAME_WIDTH))
# height = int(vcapture.get(cv2.CAP_PROP_FRAME_HEIGHT))
width = 1600
height = 1600
fps = vcapture.get(cv2.CAP_PROP_FPS)
vwriter = cv2.VideoWriter(outPath, cv2.VideoWriter_fourcc(*'MJPG'),
fps, (width, height))
count = 0
success = True
# For video, we wish classes keep the same mask in frames, generate colors for masks
colors = visualize.random_colors(len(classNames))
while success:
print("frame: ", count)
# Read next image
plt.clf()
plt.close()
success, image = vcapture.read()
if success:
# OpenCV returns images as BGR, convert to RGB
image = image[..., ::-1]
# Detect objects
r = model.detect([image], verbose=0)[0]
# Color splash
# frame = color_splash(image, r['masks'])
frame = visualize_instances(image,
r['rois'],
r['masks'],
r['class_ids'],
classNames,
r['scores'],
colors=colors,
making_video=True)
frame = cv2.resize(frame, (width, height))
# Add image to video writer
vwriter.write(frame)
count += 1
vwriter.release()
print("Saved to ", outPath)
#%% [markdown]
# Visualize anchors of one cell at the center of the feature map of a specific level.
#%%
## Visualize anchors of one cell at the center of the feature map of a specific level
# Load and draw random image
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, _, _, _ = modellib.load_image_gt(dataset, config, image_id)
fig, ax = plt.subplots(1, figsize=(10, 10))
ax.imshow(image)
levels = len(backbone_shapes)
for level in range(levels):
colors = visualize.random_colors(levels)
# Compute the index of the anchors at the center of the image
level_start = sum(
anchors_per_level[:level]) # sum of anchors of previous levels
level_anchors = anchors[level_start:level_start + anchors_per_level[level]]
print("Level {}. Anchors: {:6} Feature map Shape: {}".format(
level, level_anchors.shape[0], backbone_shapes[level]))
center_cell = backbone_shapes[level] // 2
center_cell_index = (center_cell[0] * backbone_shapes[level][1] +
center_cell[1])
level_center = center_cell_index * anchors_per_cell
center_anchor = anchors_per_cell * (
(center_cell[0] * backbone_shapes[level][1] / config.RPN_ANCHOR_STRIDE**2) \
+ center_cell[1] / config.RPN_ANCHOR_STRIDE)
level_center = int(center_anchor)
def drawMatchedROI(self, img, reference_img, mtched, unmtched_landmarks,
unmtched_detections):
# First : unmatched landmarks
# Second : unmatched detections
n_mtches = len(mtched) + 2
colors = visualize.random_colors(n_mtches)
if not n_mtches:
logging.info("No instances to display!")
return img
def _apply_mask(image, mask, color, alpha=0.5):
"""Apply the given mask to the image.
"""
for c in range(3):
image[:, :, c] = np.where(
mask == 1, image[:, :, c] * (1 - alpha) + alpha * color[c],
image[:, :, c])
return image
def _drawROI(image, box, mask, color, label, score, _id):
masked_image = image.copy()
# Bounding box
if not np.any(box):
# Skip this instance. Has no bbox. Likely lost in image cropping.
return masked_image
y1, x1, y2, x2 = box
caption = "<Landmark #{} : {}({:.3f})>".format(_id, label, score) if score \
else "<landmark #{} : {}>".format(_id, label)
masked_image = visualize.apply_mask(masked_image, mask, color)
masked_image_with_boxes = cv2.rectangle(masked_image, (x1, y1),
(x2, y2),
np.array(color) * 255, 2)
# Mask Polygon
padded_mask = np.zeros((mask.shape[0] + 2, mask.shape[1] + 2),
dtype=np.uint8)
padded_mask[1:-1, 1:-1] = mask
# contours = find_contours(padded_mask, 0.5)
if CV_MAJOR_VERSION > 3:
contours, _ = cv2.findContours(padded_mask, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
else:
_, contours, _ = cv2.findContours(padded_mask,
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
masked_image_with_contours_plus_boxes = cv2.drawContours(
masked_image_with_boxes, contours, -1, (0, 255, 0), 1)
out = cv2.putText(masked_image_with_contours_plus_boxes, caption,
(x1, y1 - 4), cv2.FONT_HERSHEY_PLAIN, 0.8,
np.array(color) * 255, 1)
masked_image = out
return out
MATCHED_COLORS = colors[1:-1]
# print("mtched colors", MATCHED_COLORS)
UNMATCHED__LANDMARK_COLORS = colors[0]
UNMATCHED_DETECTION_COLORS = colors[-1]
masked_img = img.copy()
masked_reference_img = reference_img.copy()
# drawing extraction results
offset = 0
for mtch in mtched:
landmark, detection = mtch
landmark.color = landmark.color or MATCHED_COLORS[offset]
masked_reference_img = _drawROI(masked_reference_img,
landmark.roi_features['box'],
landmark.roi_features['mask'],
landmark.color, landmark.label,
landmark.score, landmark.seq)
masked_img = _drawROI(masked_img, detection.roi_features['box'],
detection.roi_features['mask'],
landmark.color, detection.label,
detection.score, landmark.seq)
offset += 1
for unmtched_landmark in unmtched_landmarks:
masked_reference_img = _drawROI(
masked_reference_img,
unmtched_landmark.roi_features['box'],
unmtched_landmark.roi_features['mask'],
(0., 1., 1.), # Yellow
unmtched_landmark.label,
unmtched_landmark.score,
unmtched_landmark.seq)
for unmtched_detection in unmtched_detections:
masked_img = _drawROI(
masked_img,
unmtched_detection.roi_features['box'],
unmtched_detection.roi_features['mask'],
(0., 0., 1.), # Red
unmtched_detection.label,
unmtched_detection.score,
unmtched_detection.seq)
# store the rendered images
self._masked_img = masked_img
self._masked_reference_img = masked_reference_img
# drawing bbox matching results
r1, c1 = masked_img.shape[0], masked_img.shape[1]
r2, c2 = masked_reference_img.shape[0], masked_reference_img.shape[1]
out = np.zeros((max([r1, r2]), c1 + c2, 3), dtype='uint8')
out[:r1, :c1] = np.dstack([masked_img])
out[:r2, c1:] = np.dstack([masked_reference_img])
# draw line between matched bbox
offset = 0
for mtch in mtched:
color = mtch[0].color or np.array(MATCHED_COLORS[offset]) * 255
y1_1, x1_1, y2_1, x2_1 = mtch[1].roi_features['box']
cy_1 = (y2_1 + y1_1) / 2.0
cx_1 = (x2_1 + x1_1) / 2.0
y1_2, x1_2, y2_2, x2_2 = mtch[0].roi_features['box']
cy_2 = (y2_2 + y1_2) / 2.0
cx_2 = (x2_2 + x1_2) / 2.0
# draw lines
cv2.line(out, (x1_1, y1_1), (x1_2 + c1, y1_2), color, 1)
cv2.line(out, (int(x2_1), int(y1_1)), (int(x2_2) + c1, int(y1_2)),
color, 1)
cv2.line(out, (int(x2_1), int(y2_1)), (int(x2_2) + c1, int(y2_2)),
color, 1)
cv2.line(out, (int(x1_1), int(y2_1)), (int(x1_2) + c1, int(y2_2)),
color, 1)
# cv2.line(out, (int(cx_1),int(cy_1)), (int(cx_2)+c1,int(cy_2)), color, 1)
offset += 1
return out
