def annotate_video(model, class_names, video_path_in, video_path_out):
tmp_dir = f"tmp_annotate_{np.random.randint(100000, 1000000)}"
os.mkdir(tmp_dir)
print("Extracting frames:")
os.system(f"ffmpeg -i {video_path_in} -vf fps=30 {tmp_dir}/frame_%05d.jpeg -v quiet -stats")
color_map = visualize.random_colors(len(class_names))
for p in tqdm(glob.glob(f"{tmp_dir}/frame_*.jpeg"), desc="Annotating frames"):
image = cv2.imread(p)
r = model.detect([image])[0]
#print(f"Annotating {p} with {r['class_ids'].shape} labels")
if r['class_ids'].shape[0] > 0:
r_fused = utils.fuse_instances(r)
else:
r_fused = r
img = visualize.render_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'], color_map=color_map)
cv2.imwrite(f"{tmp_dir}/annotated_{Path(p).name}", img)
print("Rendering video:")
os.system(f"ffmpeg -r 30 -i {tmp_dir}/annotated_frame_%05d.jpeg -c:v libx264 -vf fps=30 -pix_fmt yuv420p {video_path_out} -v quiet -stats")
shutil.rmtree(tmp_dir, ignore_errors=True)
def paint_detections(image, boxes, masks, class_names, class_ids, scores):
font = cv2.FONT_HERSHEY_SIMPLEX
# Number of instances
N = boxes.shape[0]
# Generate random colors
colors = visualize.random_colors(N)
#copy the image
masked_image = image.astype(np.uint32).copy()
#paint masks on it
for i in range(N):
color = colors[i]
mask = masks[:, :, i]
masked_image = visualize.apply_mask(masked_image, mask, color).astype('uint8')
# paint BB rectangles
y1, x1, y2, x2 = boxes[i]
cv2.rectangle(masked_image, (x1,y1), (x2, y2), (0,255,0),2)
class_id = class_ids[i]
score = scores[i] if scores is not None else None
label = class_names[class_id]
x = random.randint(x1, (x1 + x2) // 2)
caption = "{} {:.3f}".format(label, score) if score else label
cv2.putText(masked_image, caption,(x1+3, y1+8), font, 0.3,(255,255,255))
return masked_image
def cv2_display_keypoint(image,
boxes,
keypoints,
masks,
class_ids,
scores,
class_names,
skeleton=inference_config.LIMBS):
# Number of persons
N = boxes.shape[0]
if not N:
print("\n*** No persons to display *** \n")
else:
assert N == keypoints.shape[0] and N == class_ids.shape[0] and N==scores.shape[0],\
"shape must match: boxes,keypoints,class_ids, scores"
colors = visualize.random_colors(N)
for i in range(N):
color = colors[i]
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
cv2.rectangle(image, (x1, y1), (x2, y2), color, thickness=2)
for Joint in keypoints[i]:
if (Joint[2] != 0):
cv2.circle(image, (Joint[0], Joint[1]), 2, color, -1)
# #draw skeleton connection
# limb_colors = [[0, 0, 255], [0, 170, 255], [0, 255, 170], [0, 255, 0], [170, 255, 0],
# [255, 170, 0], [255, 0, 0], [255, 0, 170], [170, 0, 255], [170, 170, 0], [170, 0, 170]]
# if (len(skeleton)):
# skeleton = np.reshape(skeleton, (-1, 2))
# neck = np.array((keypoints[i, 5, :] + keypoints[i, 6, :]) / 2).astype(int)
# if (keypoints[i, 5, 2] == 0 or keypoints[i, 6, 2] == 0):
# neck = [0, 0, 0]
# limb_index = -1
# for limb in skeleton:
# limb_index += 1
# start_index, end_index = limb # connection joint index from 0 to 16
# if (start_index == -1):
# Joint_start = neck
# else:
# Joint_start = keypoints[i][start_index]
# if (end_index == -1):
# Joint_end = neck
# else:
# Joint_end = keypoints[i][end_index]
# # both are Annotated
# # Joint:(x,y,v)
# if ((Joint_start[2] != 0) & (Joint_end[2] != 0)):
# # print(color)
# cv2.line(image, tuple(Joint_start[:2]), tuple(Joint_end[:2]), limb_colors[limb_index],3)
mask = masks[:, :, i]
image = visualize.apply_mask(image, mask, color)
caption = "{} {:.3f}".format(class_names[class_ids[i]], scores[i])
cv2.putText(image, caption, (x1 + 5, y1 + 16),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, color)
return image
def apply_masks(image, masks):
N = masks.shape[-1]
colors = visualize.random_colors(N)
masked_image = image.astype(np.uint32).copy()
for i in range(N):
masked_image = visualize.apply_mask(masked_image, masks[:, :, i],
colors[i])
return masked_image, (masked_image - image).astype(np.uint32)
def generate(self):
color_map = {}
N = len(self.classes)
for i, c in enumerate(random_colors(N)):
color_map[self.classes[i]] = c
f_name = '%s%s' % (self.class_color_file_name,
self.class_color_file_ext)
with open(f_name, 'w') as f:
if self.class_color_file_ext == '.yaml':
logger.info('Generate yaml file.')
yaml.dump(color_map, f, default_flow_style=False)
logger.info('YAML %s file is generated.', f_name)
elif self.class_color_file_ext == '.json':
logger.info('Generate json file')
json.dump(color_map, f, sort_keys=True, indent=4)
logger.info('JSON %s file is generated.', f_name)
def cv_display_instances(image, boxes, masks, class_ids, class_names,
scores=None, title="",
figsize=(16, 16)):
# Number of instances
N = boxes.shape[0]
print(N)
if not N:
print("\n*** No instances to display *** \n")
else:
assert boxes.shape[0] == masks.shape[-1] == class_ids.shape[0]
# Generate random colors
colors = visualize.random_colors(N)
height, width = image.shape[:2]
masked_image = image.copy()#.astype(np.uint32).copy()
for i in range(N):
color = colors[i]
# Bounding box
if not np.any(boxes[i]):
continue
y1, x1, y2, x2 = boxes[i]
cv_color = (color[0] * 255, color[1] * 255, color[2] * 255)
cv2.rectangle(masked_image, (x1, y1), (x2, y2), cv_color , 1)
# Label
class_id = class_ids[i]
score = scores[i] if scores is not None else None
label = class_names[class_id]
print(label)
x = random.randint(x1, (x1 + x2) // 2)
caption = "{} {:.3f}".format(label, score) if score else label
font = cv2.FONT_HERSHEY_PLAIN
cv2.putText(masked_image,caption,(x1, y1),font, 1, cv_color)
# Mask
mask = masks[:, :, i]
masked_image = visualize.apply_mask(masked_image, mask, color)
masked_image.astype(np.uint8)
return masked_image
def main(argv=None):
ROOT_DIR = os.getcwd()
with tf.gfile.FastGFile("./pb_out_new.pb", 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
_ = tf.import_graph_def(graph_def, name='')
print('Graph loaded.')
with tf.Session() as sess:
#IMAGE_DIR = os.path.join(ROOT_DIR, 'images') #image of the size defined in the config
#file_names = next(os.walk(IMAGE_DIR))[2]
#filename = random.choice(file_names)
#print('choose image file is', filename)
image = skimage.io.imread("./151.jpg")
# os.system('eog %s &'%(os.path.join(IMAGE_DIR,filename)))
print(image.shape)
images = [image]
print("Processing {} images".format(len(images)))
print('RGB image loaded and preprocessed.')
molded_images, image_metas, windows = mold_inputs(images)
print(molded_images.shape)
image_shape = molded_images[0].shape
# Anchors
anchors = get_anchors(image_shape, inference_config)
# Duplicate across the batch dimension because Keras requires it
# TODO: can this be optimized to avoid duplicating the anchors?
inference_config.BATCH_SIZE = 1
image_anchors = np.broadcast_to(
anchors, (inference_config.BATCH_SIZE, ) + anchors.shape)
print('anchors shape is', image_anchors.shape, image_anchors.dtype)
img_ph = sess.graph.get_tensor_by_name('input_image:0')
print(img_ph)
img_anchors_ph = sess.graph.get_tensor_by_name('input_anchors:0')
print(img_anchors_ph)
img_meta_ph = sess.graph.get_tensor_by_name('input_image_meta:0')
print(img_meta_ph)
detectionsT = sess.graph.get_tensor_by_name('output_detections:0')
print('Found ', detectionsT)
mrcnn_classT = sess.graph.get_tensor_by_name('output_mrcnn_class:0')
print('Found ', mrcnn_classT)
mrcnn_bboxT = sess.graph.get_tensor_by_name('output_mrcnn_bbox:0')
print('Found ', mrcnn_bboxT)
mrcnn_maskT = sess.graph.get_tensor_by_name('output_mrcnn_mask:0')
print('Found ', mrcnn_maskT)
roisT = sess.graph.get_tensor_by_name('output_rois:0')
print('Found ', roisT)
np.set_printoptions(suppress=False, precision=4)
print('Windows', windows.shape, ' ', windows)
detections = sess.run(detectionsT,
feed_dict={
img_ph: molded_images,
img_meta_ph: image_metas,
img_anchors_ph: image_anchors
})
#print('Detections: ',detections[0].shape, detections[0])
mrcnn_class = sess.run(mrcnn_classT,
feed_dict={
img_ph: molded_images,
img_meta_ph: image_metas,
img_anchors_ph: image_anchors
})
#print('Classes: ',mrcnn_class[0].shape, mrcnn_class[0])
mrcnn_bbox = sess.run(mrcnn_bboxT,
feed_dict={
img_ph: molded_images,
img_meta_ph: image_metas,
img_anchors_ph: image_anchors
})
#print('BBoxes: ',mrcnn_bbox[0].shape, mrcnn_bbox[0])
mrcnn_mask = sess.run(mrcnn_maskT,
feed_dict={
img_ph: molded_images,
img_meta_ph: image_metas,
img_anchors_ph: image_anchors
})
#print('Masks: ',mrcnn_mask[0].shape )#, outputs1[0])
rois = sess.run(roisT,
feed_dict={
img_ph: molded_images,
img_meta_ph: image_metas,
img_anchors_ph: image_anchors
})
#print('Rois: ',rois[0].shape, rois[0])
results = []
for i, image in enumerate(images):
final_rois, final_class_ids, final_scores, final_masks =\
unmold_detections(detections[i], mrcnn_mask[i],
image.shape, molded_images[i].shape,
windows[i])
results.append({
"rois": final_rois,
"class_ids": final_class_ids,
"scores": final_scores,
"masks": final_masks,
})
r = results[0]
print(results)
#print('result is', r)
class_names = [
'BG', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus',
'train', 'truck', 'boat', 'traffic light', 'fire hydrant',
'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog',
'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe',
'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat',
'baseball glove', 'skateboard', 'surfboard', 'tennis racket',
'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl',
'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot',
'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',
'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop',
'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven',
'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase',
'scissors', 'teddy bear', 'hair drier', 'toothbrush'
]
#visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'])#, ax=get_ax())
print('Done')
print("mask shape", r['masks'].shape)
#image = cv2.imread(testImage)
N = r['rois'].shape[0] #find out how many different instances are there
try: #if no mask simply return image frame
mask = r['masks'][:, :, 0]
color = visualize.random_colors(N) #generates random instance colors
color = color[0]
image = visualize.apply_mask(image, mask, color)
except Exception as e:
print(e)
cv2.imshow('test', image)
cv2.waitKey(0)
return 0
for l in range(num_levels):
num_cells = BACKBONE_SHAPES[l][0] * BACKBONE_SHAPES[l][1]
anchors_per_level.append(anchors_per_cell * num_cells //
config.RPN_ANCHOR_STRIDE**2)
print("Anchors in Level {}: {}".format(l, anchors_per_level[l]))
## Visualize anchors of one cell at the center of the feature map of a specific level
# Load and draw random image
image, image_meta, _, _, _ = modellib.load_image_gt(dataset, config, image_id)
fig, ax = plt.subplots(1, figsize=(10, 10))
ax.imshow(image)
levels = len(BACKBONE_SHAPES)
for level in range(levels):
colors = visualize.random_colors(levels)
# Compute the index of the anchors at the center of the image
level_start = sum(
anchors_per_level[:level]) # sum of anchors of previous levels
level_anchors = anchors[level_start:level_start + anchors_per_level[level]]
print("Level {}. Anchors: {:6} Feature map Shape: {}".format(
level, level_anchors.shape[0], BACKBONE_SHAPES[level]))
center_cell = BACKBONE_SHAPES[level] // 2
center_cell_index = (center_cell[0] * BACKBONE_SHAPES[level][1] +
center_cell[1])
level_center = center_cell_index * anchors_per_cell
center_anchor = anchors_per_cell * (
(center_cell[0] * BACKBONE_SHAPES[level][1] / config.RPN_ANCHOR_STRIDE**2) \
+ center_cell[1] / config.RPN_ANCHOR_STRIDE)
level_center = int(center_anchor)
#im=visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'],
#class_names, r['scores'])
#skimage.io.imsave('/home/ltp/图片/smplayer_screenshots/crop/I2/'+v,im)
# Load a random image from the images folderim,ax =plt.subplots(1,figsize = (16,16))
IMAGE_DIR = train_dir #'./images/'
file_names = '3870_426771.jpg' #next(os.walk(IMAGE_DIR))[2]
random.seed(120)
image = skimage.io.imread(IMAGE_DIR + file_names)
# Run detection
results = model.detect([image], verbose=0)
#results = model.detect_filter([image],f['label'][...].transpose(), verbose=0)
# Visualize results
r = results[0]
color = visualize.random_colors(N=config.NUM_CLASSES)
im = visualize.apply_mask(image,
r['masks'],
color=color,
class_ids=[v for v in range(1, config.NUM_CLASSES)])
att_prob = model.run_graph([image],
model.get_layers_by_name(['mask_att_prob']))
att_prob = scipy.ndimage.zoom(att_prob['mask_att_prob'][0], zoom=[8, 8, 1])
molded_images, image_metas, windows = model.mold_inputs([image])
a_prob, _ = model.unmold_detections(att_prob, att_prob[..., 0], image.shape,
windows[0])
print(a_prob.sum(2))
for v in range(3):
plt.subplot(1, 3, v + 1)
skimage.io.imshow(50 * (att_prob[..., v] - 0.33), cmap='rainbow')
plt.show()
def display_keypoints(image,
boxes,
keypoints,
class_ids,
class_names,
skeleton=[],
scores=None,
title="",
figsize=(16, 16),
ax=None):
"""
boxes: [num_persons, (y1, x1, y2, x2)] in image coordinates.
keypoints: [num_persons, num_keypoint, 3]
class_ids: [num_persons]
class_names: list of class names of the dataset
scores: (optional) confidence scores for each box
figsize: (optional) the size of the image.
"""
# Number of persons
N = boxes.shape[0]
keypoints = np.array(keypoints).astype(int)
print("keypoint_shape:", np.shape(keypoints))
if not N:
print("\n*** No persons to display *** \n")
else:
assert boxes.shape[0] == keypoints.shape[0] == class_ids.shape[0]
if not ax:
_, ax = plt.subplots(1, figsize=figsize)
# Generate random colors
colors = visualize.random_colors(N)
# Show area outside image boundaries.
height, width = image.shape[:2]
ax.set_ylim(height + 10, -10)
ax.set_xlim(-10, width + 10)
ax.axis('off')
ax.set_title(title)
skeleton_image = image.astype(np.float32).copy()
for i in range(N):
color = colors[i]
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
p = patches.Rectangle((x1, y1),
x2 - x1,
y2 - y1,
linewidth=2,
alpha=0.7,
linestyle="dashed",
edgecolor=color,
facecolor='none')
ax.add_patch(p)
# Label
class_id = class_ids[i]
score = scores[i] if scores is not None else None
label = class_names[class_id]
# x = random.randint(x1, (x1 + x2) // 2)
caption = "{} {:.3f}".format(label, score) if score else label
ax.text(x1,
y1 + 8,
caption,
color='w',
size=11,
backgroundcolor="none")
# Keypoints: num_person, num_keypoint, 3
for Joint in keypoints[i]:
if (Joint[2] != 0):
circle = patches.Circle((Joint[0], Joint[1]),
radius=1,
edgecolor=color,
facecolor='none')
ax.add_patch(circle)
# Skeleton: 11*2
limb_colors = [[0, 0, 255], [0, 170, 255], [0, 255, 170], [0, 255, 0],
[170, 255, 0], [255, 170, 0], [255, 0, 0],
[255, 0, 170], [170, 0, 255], [170, 170, 0],
[170, 0, 170]]
if (len(skeleton)):
skeleton = np.reshape(skeleton, (-1, 2))
neck = np.array(
(keypoints[i, 5, :] + keypoints[i, 6, :]) / 2).astype(int)
if (keypoints[i, 5, 2] == 0 or keypoints[i, 6, 2] == 0):
neck = [0, 0, 0]
limb_index = -1
for limb in skeleton:
limb_index += 1
start_index, end_index = limb # connection joint index from 0 to 16
if (start_index == -1):
Joint_start = neck
else:
Joint_start = keypoints[i][start_index]
if (end_index == -1):
Joint_end = neck
else:
Joint_end = keypoints[i][end_index]
# both are Annotated
# Joint:(x,y,v)
if ((Joint_start[2] != 0) & (Joint_end[2] != 0)):
# print(color)
cv2.line(skeleton_image, tuple(Joint_start[:2]),
tuple(Joint_end[:2]), limb_colors[limb_index], 5)
ax.imshow(skeleton_image.astype(np.uint8))
plt.show()
def cv2_display_keypoint(image, boxes, masks, class_ids, scores, class_names):
# Number of lips
N = boxes.shape[0]
print("number of lips " + str(N))
if (N < 2):
return image, "minimum not found"
if not N:
print("\n*** No lips to display *** \n")
else:
assert N == class_ids.shape[0] and N==scores.shape[0],\
"shape must match: boxes,keypoints,class_ids, scores"
colors = visualize.random_colors(N)
class1 = True
class2 = True
keypoints = []
classes = []
for i in range(N):
if class_ids[i] == 1:
classes.append(i)
break
for i in range(N):
if class_ids[i] == 2:
classes.append(i)
break
for k in range(len(classes)):
i = classes[k]
color = colors[i]
# Bounding box
if not np.any(boxes[i]):
# Skip this instance. Has no bbox. Likely lost in image cropping.
continue
y1, x1, y2, x2 = boxes[i]
# cv2.rectangle(image, (x1, y1), (x2, y2), color, thickness=2)
mask = masks[:, :, i]
if scores[i] >= 0.9:
column_first_non_zero = first_nonzero(mask, axis=0, invalid_val=-1)
column_last_non_zero = last_nonzero(mask, axis=0, invalid_val=-1)
row_first_non_zero = first_nonzero(mask, axis=1, invalid_val=-1)
row_last_non_zero = last_nonzero(mask, axis=1, invalid_val=-1)
if (lip_orientation(column_first_non_zero, row_first_non_zero)
in "horizontal"):
#for class 1
if class_ids[i] == 1 and class1:
class1 = False
indexes_column_last_positive = np.where(
column_last_non_zero > -1)[0]
indexes_column_first_positive = np.where(
column_first_non_zero > -1)[0]
xf = indexes_column_last_positive[0]
xl = indexes_column_last_positive[
len(indexes_column_last_positive) - 1]
yf = column_last_non_zero[xf]
yl = column_last_non_zero[xl]
if (yf < yl):
#first corner up
xf = indexes_column_first_positive[0]
yf = column_first_non_zero[xf]
elif (yf > yl):
#last corner up
xl = indexes_column_first_positive[
len(indexes_column_first_positive) - 1]
yl = column_first_non_zero[xl]
cv2.circle(image, (xf, yf), 2, color, -1)
cv2.circle(image, (xl, yl), 2, color, -1)
keypoints.append([xf, yf])
keypoints.append([xl, yl])
#mid points of 2 corners
x_mid_line = int(round((xf + xl) / 2))
y_mid_line = int(round((yf + yl) / 2))
#spliting columns from mid point
index_largest_last = np.argmax(column_last_non_zero)
yl = column_last_non_zero[index_largest_last]
max_replaced_neg = np.where(column_first_non_zero == -1,
yl, column_first_non_zero)
column_first_half = max_replaced_neg[:x_mid_line]
column_last_half = max_replaced_neg[x_mid_line:]
#upper part upper lip
#upper lip upper left high point
temp = column_first_half[::-1]
upper_lip_left_high_x = len(temp) - np.argmin(temp) - 1
upper_lip_left_high_y = column_first_half[
upper_lip_left_high_x]
cv2.circle(image,
(upper_lip_left_high_x, upper_lip_left_high_y),
2, color, -1)
keypoints.append(
[upper_lip_left_high_x, upper_lip_left_high_y])
#center of left corner and left high point
center_left_corner_high_x = int(
round((xf + upper_lip_left_high_x) / 2))
center_left_corner_high_y = column_first_non_zero[
center_left_corner_high_x]
cv2.circle(
image,
(center_left_corner_high_x, center_left_corner_high_y),
2, color, -1)
keypoints.append(
[center_left_corner_high_x, center_left_corner_high_y])
# upper lip upper right high point
upper_lip_right_high_x = np.argmin(column_last_half)
upper_lip_right_high_y = column_last_half[
upper_lip_right_high_x]
cv2.circle(
image,
(len(column_first_half) + upper_lip_right_high_x,
upper_lip_right_high_y), 2, color, -1)
keypoints.append([
len(column_first_half) + upper_lip_right_high_x,
upper_lip_right_high_y
])
# center of right corner and right high point
center_right_corner_high_x = int(
round((xl + len(column_first_half) +
upper_lip_right_high_x) / 2))
center_right_corner_high_y = column_first_non_zero[
center_right_corner_high_x]
cv2.circle(image, (center_right_corner_high_x,
center_right_corner_high_y), 2, color,
-1)
keypoints.append([
center_right_corner_high_x, center_right_corner_high_y
])
#actual mid point of upper upper lip
mid_point_x = int(
round((upper_lip_left_high_x + len(column_first_half) +
upper_lip_right_high_x) / 2))
mid_point_y = column_first_non_zero[mid_point_x]
cv2.circle(image, (mid_point_x, mid_point_y), 2, color, -1)
keypoints.append([mid_point_x, mid_point_y])
#lower part of upper lip
# angle of lip with x-axis
angle = math.atan((yl - yf) / (xl - xf))
#mid point lower upper lip
length = column_last_non_zero[mid_point_x] - mid_point_y
x_mid_point_low = int(
round(mid_point_x + math.sin(angle) * length))
y_mid_point_low = int(
round(mid_point_y + math.cos(angle) * length))
cv2.circle(image, (x_mid_point_low, y_mid_point_low), 2,
color, -1)
keypoints.append([x_mid_point_low, y_mid_point_low])
# upper lip lower left high point
length = column_last_non_zero[
upper_lip_left_high_x] - upper_lip_left_high_y
upper_lip_left_lower_x = int(
round(upper_lip_left_high_x +
math.sin(angle) * length))
upper_lip_left_lower_y = int(
round(upper_lip_left_high_y +
math.cos(angle) * length))
cv2.circle(
image,
(upper_lip_left_lower_x, upper_lip_left_lower_y), 2,
color, -1)
keypoints.append(
[upper_lip_left_lower_x, upper_lip_left_lower_y])
# upper lip lower right high point
length = column_last_non_zero[
len(column_first_half) +
upper_lip_right_high_x] - upper_lip_right_high_y
upper_lip_right_lower_x = int(
round(
len(column_first_half) + upper_lip_right_high_x +
math.sin(angle) * length))
upper_lip_right_lower_y = int(
round(upper_lip_right_high_y +
math.cos(angle) * length))
cv2.circle(
image,
(upper_lip_right_lower_x, upper_lip_right_lower_y), 2,
color, -1)
keypoints.append(
[upper_lip_right_lower_x, upper_lip_right_lower_y])
# upper lip lower center of right corner and right high point
length = column_last_non_zero[
center_right_corner_high_x] - center_right_corner_high_y
upper_lip_right_corner_lower_x = int(
round(center_right_corner_high_x +
math.sin(angle) * length))
upper_lip_right__corner_lower_y = int(
round(center_right_corner_high_y +
math.cos(angle) * length))
cv2.circle(image, (upper_lip_right_corner_lower_x,
upper_lip_right__corner_lower_y), 2,
color, -1)
keypoints.append([
upper_lip_right_corner_lower_x,
upper_lip_right__corner_lower_y
])
# lower center of left corner and left high point
length = column_last_non_zero[
center_left_corner_high_x] - center_left_corner_high_y
upper_lip_left_corner_lower_x = int(
round(center_left_corner_high_x +
math.sin(angle) * length))
upper_lip_left_corner_lower_y = int(
round(center_left_corner_high_y +
math.cos(angle) * length))
cv2.circle(image, (upper_lip_left_corner_lower_x,
upper_lip_left_corner_lower_y), 2,
color, -1)
keypoints.append([
upper_lip_left_corner_lower_x,
upper_lip_left_corner_lower_y
])
if class_ids[i] == 2 and class2 and not class1:
class2 = False
#lower lip upper part
# mid point upper lower lip
length = column_first_non_zero[mid_point_x] - mid_point_y
x_mid_point_up_lower = int(
round(mid_point_x + math.sin(angle) * length))
y_mid_point_up_lower = int(
round(mid_point_y + math.cos(angle) * length))
cv2.circle(image,
(x_mid_point_up_lower, y_mid_point_up_lower), 2,
color, -1)
keypoints.append(
[x_mid_point_up_lower, y_mid_point_up_lower])
# lower lip upper left high point
length = column_first_non_zero[
upper_lip_left_high_x] - upper_lip_left_high_y
lower_lip_left_upper_x = int(
round(upper_lip_left_high_x +
math.sin(angle) * length))
lower_lip_left_upper_y = int(
round(upper_lip_left_high_y +
math.cos(angle) * length))
cv2.circle(
image,
(lower_lip_left_upper_x, lower_lip_left_upper_y), 2,
color, -1)
keypoints.append(
[lower_lip_left_upper_x, lower_lip_left_upper_y])
# lower lip upper right high point
length = column_first_non_zero[
len(column_first_half) +
upper_lip_right_high_x] - upper_lip_right_high_y
lower_lip_right_upper_x = int(
round(
len(column_first_half) + upper_lip_right_high_x +
math.sin(angle) * length))
lower_lip_right_upper_y = int(
round(upper_lip_right_high_y +
math.cos(angle) * length))
cv2.circle(
image,
(lower_lip_right_upper_x, lower_lip_right_upper_y), 2,
color, -1)
keypoints.append(
[lower_lip_right_upper_x, lower_lip_right_upper_y])
# lower lip upper center of right corner and right high point
length = column_first_non_zero[
center_right_corner_high_x] - center_right_corner_high_y
lower_lip_right_corner_upper_x = int(
round(center_right_corner_high_x +
math.sin(angle) * length))
lower_lip_right_corner_upper_y = int(
round(center_right_corner_high_y +
math.cos(angle) * length))
cv2.circle(image, (lower_lip_right_corner_upper_x,
lower_lip_right_corner_upper_y), 2,
color, -1)
keypoints.append([
lower_lip_right_corner_upper_x,
lower_lip_right_corner_upper_y
])
# lower center of left corner and left high point
length = column_first_non_zero[
center_left_corner_high_x] - center_left_corner_high_y
lower_lip_left_corner_upper_x = int(
round(center_left_corner_high_x +
math.sin(angle) * length))
lower_lip_left_corner_upper_y = int(
round(center_left_corner_high_y +
math.cos(angle) * length))
cv2.circle(image, (lower_lip_left_corner_upper_x,
lower_lip_left_corner_upper_y), 2,
color, -1)
keypoints.append([
lower_lip_left_corner_upper_x,
lower_lip_left_corner_upper_y
])
#lower lip lower part
# mid point lower lower lip
length = column_last_non_zero[
x_mid_point_up_lower] - y_mid_point_up_lower
x_mid_point_low_lower = int(
round(x_mid_point_up_lower + math.sin(angle) * length))
y_mid_point_low_lower = int(
round(y_mid_point_up_lower + math.cos(angle) * length))
cv2.circle(image,
(x_mid_point_low_lower, y_mid_point_low_lower),
2, color, -1)
keypoints.append(
[x_mid_point_low_lower, y_mid_point_low_lower])
# lower lip lower left high point
length = column_last_non_zero[
lower_lip_left_upper_x] - lower_lip_left_upper_y
lower_lip_left_lower_x = int(
round(lower_lip_left_upper_x +
math.sin(angle) * length))
lower_lip_left_lower_y = int(
round(lower_lip_left_upper_y +
math.cos(angle) * length))
cv2.circle(
image,
(lower_lip_left_lower_x, lower_lip_left_lower_y), 2,
color, -1)
keypoints.append(
[lower_lip_left_lower_x, lower_lip_left_lower_y])
# lower lip lower right high point
length = column_last_non_zero[
lower_lip_right_upper_x] - lower_lip_right_upper_y
lower_lip_right_lower_x = int(
round(lower_lip_right_upper_x +
math.sin(angle) * length))
lower_lip_right_lower_y = int(
round(lower_lip_right_upper_y +
math.cos(angle) * length))
cv2.circle(
image,
(lower_lip_right_lower_x, lower_lip_right_lower_y), 2,
color, -1)
keypoints.append(
[lower_lip_right_lower_x, lower_lip_right_lower_y])
# lower lip lower center of right corner and right high point
length = column_last_non_zero[
lower_lip_right_corner_upper_x] - lower_lip_right_corner_upper_y
lower_lip_right_corner_lower_x = int(
round(lower_lip_right_corner_upper_x +
math.sin(angle) * length))
lower_lip_right_corner_lower_y = int(
round(lower_lip_right_corner_upper_y +
math.cos(angle) * length))
cv2.circle(image, (lower_lip_right_corner_lower_x,
lower_lip_right_corner_lower_y), 2,
color, -1)
keypoints.append([
lower_lip_right_corner_lower_x,
lower_lip_right_corner_lower_y
])
# lower center of left corner and left high point
length = column_last_non_zero[
lower_lip_left_corner_upper_x] - lower_lip_left_corner_upper_y
lower_lip_left_corner_lower_x = int(
round(lower_lip_left_corner_upper_x +
math.sin(angle) * length))
lower_lip_left_corner_lower_y = int(
round(lower_lip_left_corner_upper_y +
math.cos(angle) * length))
cv2.circle(image, (lower_lip_left_corner_lower_x,
lower_lip_left_corner_lower_y), 2,
color, -1)
keypoints.append([
lower_lip_left_corner_lower_x,
lower_lip_left_corner_lower_y
])
image = visualize.apply_mask(image, mask, color)
# caption = "{} {:.3f}".format(class_names[class_ids[i]], scores[i])
# cv2.putText(image, caption, (x1 + 5, y1 + 16), cv2.FONT_HERSHEY_SIMPLEX,
# 0.5, color)
s = ""
if (len(keypoints) > 1 and len(keypoints[0]) > 1):
distance = math.sqrt(((keypoints[0][0] - keypoints[1][0])**2) +
((keypoints[0][1] - keypoints[1][1])**2))
else:
distance = 0
p = 0
while (p < len(keypoints)):
q = p + 1
while (q < len(keypoints)):
hor = math.sqrt(((keypoints[p][0] - keypoints[q][0])**2) +
((keypoints[p][1] - keypoints[q][1])**2))
if hor != 0:
s = s + str(format(distance / hor, '.3f')) + ", "
q += 1
p += 1
return image, s
def detectVideoOffline(model, video_path):
""" Detects instances in video, produces vector plots and saves to file
Notes
-----
- Saves to local directory:
Video with colour effect applied
Images of vector plots
- Requires matlab.engine to interface with camera model script for
pixelToVector calculations.
Parameters
----------
model : Tensorflow model
Mask-RCNN model in inference mode.
Generated in runNetwork().
video_path : file/path/to/video
Path to video upon which to do detection.
User-defined.
Returns
-------
None
"""
angle1 = -125
angle2 = -70
class_names = ['BG', 'Duckie']
# Open 3d plot with origin at [0,0,0]
fig = plt.figure(figsize=(7, 7 / 3))
ax2 = fig.add_subplot(121, projection='3d')
ax2.cla()
ax2.set_xlim(-1, 1)
ax2.set_ylim(-1, 1)
ax2.set_zlim(0, 1)
ax2.set_title('Optimised pixelToVector function')
ax2.set_xlabel("x", fontsize=16)
ax2.set_ylabel("y", fontsize=16)
ax2.set_zlabel("z", fontsize=16)
ax2.view_init(elev=angle1, azim=angle2)
# ax2 = fig.add_subplot(122, projection='3d')
# ax2.cla()
# ax2.set_xlim(-1, 1)
# ax2.set_ylim(-1, 1)
# ax2.set_zlim(0, 1)
# ax2.set_title('Non-optimised pixelToVector function')
# ax2.set_xlabel("x", fontsize=16)
# ax2.set_ylabel("y", fontsize=16)
# ax2.set_zlabel("z", fontsize=16)
# ax2.view_init(elev=angle1, azim=angle2)
# plt.tight_layout()
plt.show()
# Generate filenames and create directories
file_name_orig = "offline_rec_{:%Y%m%dT%H%M%S}".format(
datetime.datetime.now())
rootDir = ROOT_DIR + r'/Recordings/' + file_name_orig + r'/'
rootDir_orig = rootDir + r'orig/'
rootDir_vec = rootDir + r'vec/'
rootDir_col = rootDir + r'col/'
os.mkdir(rootDir)
os.mkdir(rootDir_orig)
os.mkdir(rootDir_vec)
os.mkdir(rootDir_col)
lines = []
lines2 = []
frameCount = -1
origin = [0, 0, 0]
height = 200
angularWidth = 0
distance = 0
dpi = 72
imageSize = (32, 32)
imDuckie = image.imread(r'PlotImages\duckie.png')
imCamera = image.imread(r'PlotImages\wheatley.png')
imDuckie = OffsetImage(imDuckie, zoom=0.25)
imCamera = OffsetImage(imCamera, zoom=0.1)
capture = cv2.VideoCapture(video_path)
while True:
frameCount += 1
print("Frame #", frameCount)
ret, frame = capture.read()
# Detect objects
results = model.detect([frame], verbose=0)
r = results[0]
# Get points of interest
tL, bL, tR, bT, centre = getPoints(r['rois'])
if (r['rois'].shape[0] > 0):
# Clear previous lines and reset format
ax2.cla()
ax2.set_xlim(-1, 1)
ax2.set_ylim(-1, 1)
ax2.set_zlim(0, 1)
ax2.set_title('Non-optimised pixelToVector function')
ax2.set_xlabel("x", fontsize=16)
ax2.set_ylabel("y", fontsize=16)
ax2.set_zlabel("z", fontsize=16)
#ax2.view_init(elev=angle1, azim=angle2)
angularWidth = []
distance = []
# Plot vector from origin to centre of each object
for i in range(r['rois'].shape[0]):
# Do pixeltovector function using upper left and bottom left
u_uLeft = float(tL[i][0])
v_uLeft = float(tL[i][1])
u_bLeft = float(bL[i][0])
v_bLeft = float(bL[i][1])
l_Vec = eng.pixeltovector(float(u_uLeft), float(v_uLeft))
r_Vec = eng.pixeltovector(float(u_bLeft), float(v_bLeft))
l_Vec = [l_Vec[0][3], l_Vec[0][4], l_Vec[0][5]]
r_Vec = [r_Vec[0][3], r_Vec[0][4], r_Vec[0][5]]
# Using diameter, not radius
angularWidth.append(vectorAngle(l_Vec, r_Vec))
# Estimating distance
# (probably doing this wrong, too sleepy to think)
distance.append(height / np.tan(angularWidth[i]))
# Do pixeltovector function using centre points
# Run from MATLAB
u_centre = float(centre[i][0])
v_centre = float(centre[i][1])
res = eng.pixeltovector(float(u_centre), float(v_centre))
noptVect = [res[0][0], res[0][1], res[0][2]]
optVect = [res[0][3], res[0][4], res[0][5]]
# Plot optimised vector
# duckieVector = optVect
# xs = asarray([origin[0], duckieVector[0]])
# ys = asarray([origin[1], duckieVector[1]])
# zs = asarray([origin[2], duckieVector[2]])
# colors = random_colors(r['rois'].shape[0])
# lines.append(ax.plot(xs, ys, zs, color=colors[i],
# mfc="None", mec="None",
# markersize=imageSize[0]*(dpi/96)))
#
# xOrigin, yOrigin, _ = proj3d.proj_transform(0,0,0,
# ax.get_proj())
#
# xDuck, yDuck, _ = proj3d.proj_transform(duckieVector[0],
# duckieVector[1],
# duckieVector[2],
# ax.get_proj())
# ab1 = AnnotationBbox(imDuckie, [xDuck,yDuck])
# ab2 = AnnotationBbox(imCamera, [xOrigin,yOrigin])
# ax.add_artist(ab1)
# ax.add_artist(ab2)
# Plot non-optimised vector
duckieVector = noptVect
xs = asarray([origin[0], duckieVector[0]])
ys = asarray([origin[1], duckieVector[1]])
zs = asarray([origin[2], duckieVector[2]])
plt.tight_layout()
colors = random_colors(r['rois'].shape[0])
lines2.append(
ax2.plot(xs,
ys,
zs,
color=colors[i],
mfc="None",
mec="None",
markersize=imageSize[0] * (dpi / 96)))
xOrigin, yOrigin, _ = proj3d.proj_transform(
0, 0, 0, ax2.get_proj())
xDuck, yDuck, _ = proj3d.proj_transform(
duckieVector[0], duckieVector[1], duckieVector[2],
ax2.get_proj())
ab1 = AnnotationBbox(imDuckie, [xDuck, yDuck])
ab2 = AnnotationBbox(imCamera, [xOrigin, yOrigin])
ax2.add_artist(ab1)
ax2.add_artist(ab2)
plt.tight_layout()
plt.pause(0.1)
frame = displayDetections(frame, r['rois'], r['masks'], r['class_ids'],
class_names, r['scores'], angularWidth,
distance)
# Save 3D vector plots
imagePath = rootDir_vec + file_name_orig + 'vector_' + "{:03d}".format(
frameCount) + '.png'
fig.savefig(imagePath)
# Write the annotated image to a file
imagePath = rootDir_col + file_name_orig + 'img_' + "{:03d}".format(
frameCount) + '.png'
cv2.imwrite(imagePath, frame)
cv2.imshow('frame', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
capture.release()
cv2.destroyAllWindows()
def __init__(self):
"""Initialize a MaskRCNNNode
Parameters
----------
Returns
----------
- <obj>: MaskRCNNNode
A Mask RCNN instantiation.
"""
self._cv_bridge = CvBridge()
self._class_names = rospy.get_param('~class_names', CLASS_NAMES)
self._class_colors = visualize.random_colors(len(CLASS_NAMES))
self._visualization = rospy.get_param('~visualization', True)
#
# network configuration
#
config = InferenceConfig()
config.display()
#
# model in inference mode
#
self._model = modellib.MaskRCNN(mode="inference",
model_dir="",
config=config)
#
# mutual exclusive lock for handling image messages.
#
self._last_msg = None
self._msg_lock = threading.Lock()
#
# publisher
#
self._detection_pub = rospy.Publisher('~detection',
mask_rcnn_ros.msg.Detection,
queue_size=1)
self._vis_pub = rospy.Publisher('~visualization',
sensor_msgs.msg.Image,
queue_size=1)
self._publish_rate = rospy.get_param('~publish_rate', 100)
#
# tensorflow graph
#
self._graph = tf.get_default_graph()
#
# Load weights trained on MS-COCO, download COCO trained weights from
# releases if needed
#
model_path = rospy.get_param('~model_path', COCO_MODEL_PATH)
if model_path == COCO_MODEL_PATH and not os.path.exists(
COCO_MODEL_PATH):
utils.download_trained_weights(COCO_MODEL_PATH)
self._model.load_weights(model_path, by_name=True)
def get_all_mask_class_and_draw_minrect(self, r, image):
num = r['masks'].shape[2]
object = []
_, ax = plt.subplots(1, figsize=(16, 16))
shape = image.shape
ax.set_ylim(shape[0] + 10, -10)
ax.set_xlim(-10, shape[1] + 10)
ax.axis('off')
ax.set_title("hhhh")
for i in range(num): # create mask class for every instance
obj = Mask(i, r['class_ids'][i],
self.dataset_val.class_names[r['class_ids'][i]],
r['masks'][:, :, i], r['rois'][i], r['scores'][i])
obj.get_min_rect()
object.append(obj)
# cv2.drawContours(image, [object[i].min_box], 0, (0, 0, 255), 3)
print(object[i].class_name)
for i in range(num):
# if (object[i].class_name is "lemon") | (object[i].class_name is "grape")| (object[i].class_name is "fig"):
shoe = object[i]
center_x = shoe.min_rect[0][0]
center_y = shoe.min_rect[0][1]
width = shoe.min_rect[1][0]
height = shoe.min_rect[1][1]
theta = shoe.min_rect[2]
PI = math.pi
the = (theta / 180) * PI
# print("object", i,":width height theta:", width, height, theta)
if theta == 0:
y = center_y - 3 * height / 4
x = center_x
point = (x, y)
w_h = (width / 2, width / 4)
elif width > height:
x = center_x + (5 * width * math.cos(the)) / 8
y = center_y + (5 * width * math.sin(the)) / 8
# x = center_x - (2 * width * math.cos(the)) / 4
# y = center_y - (2 * width * math.sin(the)) / 4
point = (x, y)
w_h = (height / 4, height / 2)
else:
x = center_x + (5 * height * math.sin(the)) / 8
y = center_y - (5 * height * math.cos(the)) / 8
# x = center_x - (2 * height * math.sin(the)) / 4
# y = center_y + (2 * height * math.cos(the)) / 4
point = (x, y)
w_h = (width / 2, width / 4)
# print("object", i, "center center_cut:", shoe.min_rect[0], center_x, center_y, x, y)
rect = (point, w_h, theta)
box = cv2.boxPoints(rect)
box = np.int0(box)
cv2.drawContours(image, [box], 0, (255, 0, 0), 2)
colors = random_colors(num)
masked_image = image.astype(np.uint32).copy()
captions = [
"lemon 0.988", "lemon 1.000", "lemon 0.995", "lemon 0.984",
"lemon 0.992"
]
for i in range(num):
# if (object[i].class_name is "lemon") | (object[i].class_name is "grape") | (object[i].class_name is "fig"):
shoe = object[i]
box = shoe.min_box
width = shoe.min_rect[1][0]
height = shoe.min_rect[1][1]
theta = shoe.min_rect[2]
score = shoe.scores
label = shoe.class_name
mask = shoe.mask
p = patches.Rectangle(
box[1],
width,
height,
angle=theta,
linewidth=2, # add rectangle dash
alpha=0.7,
linestyle="dashed",
edgecolor="blue",
facecolor='none')
ax.add_patch(p)
caption = "{} {:.3f}".format(
label, score) if score else label # add label
# caption = captions[i]
print(type(caption), caption)
ax.text(box[0][0],
box[0][1],
caption,
color='w',
size=8,
backgroundcolor="blue")
masked_image = apply_mask(masked_image, mask, colors[i])
ax.imshow(masked_image.astype(np.uint8))
plt.show()
def detectVideo(model, video_path=None, live=True):
""" Detects instances in video, produces vector plots and saves to file
Notes
-----
- Saves to local directory:
Video with colour effect applied
Images of vector plots
- Requires matlab.engine to interface with camera model script for
pixelToVector calculations.
Parameters
----------
model : Tensorflow model
Mask-RCNN model in inference mode.
Generated in runNetwork().
video_path : file/path/to/video
Path to video upon which to do detection.
User-defined.
Returns
-------
None
"""
# Video capture - live
if live is True:
print("Using webcam!")
capture = cv2.VideoCapture(0)
fps = 1
# Video capture - video path
else:
capture = cv2.VideoCapture(video_path)
fps = capture.get(cv2.CAP_PROP_FPS)
# Get width and height
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
# Open 3d plot with origin at [0,0,0]
fig = plt.figure(figsize=(15, 15 / 3))
ax = fig.add_subplot(121, projection='3d')
ax.cla()
ax.set_xlim(-1, 1)
ax.set_ylim(-1, 1)
ax.set_zlim(0, 1)
ax.set_title('Optimised pixelToVector function')
ax.set_xlabel("x", fontsize=16)
ax.set_ylabel("y", fontsize=16)
ax.set_zlabel("z", fontsize=16)
ax.view_init(elev=45, azim=45)
ax2 = fig.add_subplot(122, projection='3d')
ax2.cla()
ax2.set_xlim(-1, 1)
ax2.set_ylim(-1, 1)
ax2.set_zlim(0, 1)
ax2.set_title('Non-optimised pixelToVector function')
ax2.set_xlabel("x", fontsize=16)
ax2.set_ylabel("y", fontsize=16)
ax2.set_zlabel("z", fontsize=16)
ax2.view_init(elev=45, azim=45)
plt.tight_layout()
plt.show()
# Generate filenames and create directories
file_name_orig = "colourised_{:%Y%m%dT%H%M%S}".format(
datetime.datetime.now())
rootDir = r'C:/Users/Chloe/Desktop/MCHA3900/Machine Vision/' + file_name_orig + r'/'
rootDir_orig = rootDir + r'orig/'
rootDir_vec = rootDir + r'vec/'
rootDir_col = rootDir + r'col/'
os.mkdir(rootDir)
os.mkdir(rootDir_orig)
os.mkdir(rootDir_vec)
os.mkdir(rootDir_col)
# Define codec and create video writer
file_name = file_name_orig + ".avi"
vwriter = cv2.VideoWriter(rootDir + file_name,
cv2.VideoWriter_fourcc(*'MJPG'), fps,
(width, height))
lines = []
lines2 = []
frameCount = -1
origin = [0, 0, 0]
validFlag = True
height = 200
angularWidth = 0
distance = 0
while validFlag:
# Increment frame counter
frameCount += 1
print("Frame #", frameCount)
# Read next image
validFlag, image = capture.read()
if validFlag:
# OpenCV returns images as BGR, convert to RGB
image = image[..., ::-1]
# Detect objects
r = model.detect([image], verbose=0)[0]
# Get points of interest
tL, bL, tR, bT, centre = getPoints(r['rois'])
# Do plots if any objects have been detected
if (r['rois'].shape[0] > 0):
# Clear previous lines and reset format
ax.cla()
ax.set_xlim(-1, 1)
ax.set_ylim(-1, 1)
ax.set_zlim(0, 1)
ax.set_title('Optimised pixelToVector function')
ax.set_xlabel("x", fontsize=16)
ax.set_ylabel("y", fontsize=16)
ax.set_zlabel("z", fontsize=16)
ax.view_init(elev=45, azim=45)
ax2.cla()
ax2.set_xlim(-1, 1)
ax2.set_ylim(-1, 1)
ax2.set_zlim(0, 1)
ax2.set_title('Non-optimised pixelToVector function')
ax2.set_xlabel("x", fontsize=16)
ax2.set_ylabel("y", fontsize=16)
ax2.set_zlabel("z", fontsize=16)
ax2.view_init(elev=45, azim=45)
angularWidth = []
distance = []
# Plot vector from origin to centre of each object
for i in range(r['rois'].shape[0]):
#duckieDistance = pixel2mm*(rad[i])/duckieRadius
#duckieLocation = [centre[i][1], centre[i][0], 1]
#duckieVector = duckieLocation
# Do pixeltovector function using upper left and bottom left
u_uLeft = float(tL[i][0])
v_uLeft = float(tL[i][1])
u_bLeft = float(bL[i][0])
v_bLeft = float(bL[i][1])
l_Vec = eng.pixeltovector(float(u_uLeft), float(v_uLeft))
r_Vec = eng.pixeltovector(float(u_bLeft), float(v_bLeft))
l_Vec = [l_Vec[0][3], l_Vec[0][4], l_Vec[0][5]]
r_Vec = [r_Vec[0][3], r_Vec[0][4], r_Vec[0][5]]
# Diameter, not radius
angularWidth.append(vectorAngle(l_Vec, r_Vec))
# Estimating distance
# (probably doing this wrong, too sleepy to think)
distance.append(height / np.tan(angularWidth[i]))
# Do pixeltovector function using cansentre points
# Run from MATLAB
u_centre = float(centre[i][0])
v_centre = float(centre[i][1])
res = eng.pixeltovector(float(u_centre), float(v_centre))
noptVect = [res[0][0], res[0][1], res[0][2]]
optVect = [res[0][3], res[0][4], res[0][5]]
# Plot optimised vector
duckieVector = optVect
xs = asarray([origin[0], abs(duckieVector[0])])
ys = asarray([origin[1], abs(duckieVector[1])])
zs = asarray([origin[2], duckieVector[2]])
colors = random_colors(r['rois'].shape[0])
lines.append(
ax.plot(xs,
ys,
zs,
color=colors[i],
marker=r'$\bigotimes$',
markersize=22))
# Plot non-optimised vector
duckieVector = noptVect
xs = asarray([origin[0], abs(duckieVector[0])])
ys = asarray([origin[1], abs(duckieVector[1])])
zs = asarray([origin[2], duckieVector[2]])
plt.tight_layout()
colors = random_colors(r['rois'].shape[0])
lines2.append(
ax2.plot(xs,
ys,
zs,
color=colors[i],
marker=r'$\bigotimes$',
markersize=22))
plt.tight_layout()
plt.pause(0.1)
# Colourise detected regions
# detectFlag, image = colourRegion(image, r['masks'])
image = display_cv_instances(image, r['rois'], r['masks'],
r['class_ids'], ['BG', 'duckie'],
r['scores'], angularWidth, distance)
# RGB -> BGR to save image to video
image = image[..., ::-1]
# Write the annotated image to a file
imagePath = rootDir_col + file_name_orig + 'img_' + "{:03d}".format(
frameCount) + '.png'
#cv2.imwrite(imagePath,image)
# Add annotated image to video writer
if live is False:
vwriter.write(image)
# Save 3D vector plots
imagePath = rootDir_vec + file_name_orig + 'vector_' + "{:03d}".format(
frameCount) + '.png'
fig.savefig(imagePath)
# Show webcam, press x to exit
cv2.imshow('image', image.astype(np.int32))
if cv2.waitKey(1) & 0xFF == ord('x'):
break
# Release video writer
capture.release()
vwriter.release()
cv2.destroyAllWindows()
