def render_car_cv2(pose, car_name, car_models, intrinsic, mesh_color, ax):
"""Render a car instance given pose and car_name
"""
car = car_models[car_name]
pose = np.array(pose)
# project 3D points to 2d image plane
rmat = euler_angles_to_rotation_matrix(pose[:3])
rvect, _ = cv2.Rodrigues(rmat)
imgpts, jac = cv2.projectPoints(np.float32(car['vertices']), rvect, pose[3:], intrinsic, distCoeffs=None)
for face in car['faces'] - 1:
pts = np.array([[imgpts[idx, 0, 0], imgpts[idx, 0, 1]] for idx in face], np.int32)
polygon = Polygon(
pts.reshape((-1, 2)),
fill=True, facecolor=mesh_color,
edgecolor='w', linewidth=1.2,
alpha=0.5)
ax.add_patch(polygon)
def _add_gt_annotations_Car3d(self, entry):
"""Add ground truth annotation metadata to an roidb entry."""
entry_id = entry['entry_id']
# Make file_name an abs path
car_pose_file = os.path.join(self.Car3D.data_dir, 'car_poses',
entry_id + '.json')
assert os.path.exists(car_pose_file), 'Label \'{}\' not found'.format(
car_pose_file)
with open(car_pose_file) as f:
car_poses = json.load(f)
entry['height'] = self.Car3D.image_shape[0]
entry['width'] = self.Car3D.image_shape[1]
intrinsic_mat = self.Car3D.get_intrinsic_mat()
# Sanitize bboxes -- some are invalid
valid_objs = []
for i, car_pose in enumerate(car_poses):
car_name = self.Car3D.car_id2name[car_pose['car_id']].name
car = self.car_models[car_name]
pose = np.array(car_pose['pose'])
# project 3D points to 2d image plane
rot_mat = euler_angles_to_rotation_matrix(pose[:3])
rvect, _ = cv2.Rodrigues(rot_mat)
imgpts, jac = cv2.projectPoints(np.float32(car['vertices']),
rvect,
pose[3:],
intrinsic_mat,
distCoeffs=None)
imgpts = np.int32(imgpts).reshape(-1, 2)
x1, y1, x2, y2 = imgpts[:, 0].min(), imgpts[:, 1].min(
), imgpts[:, 0].max(), imgpts[:, 1].max()
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, entry['height'], entry['width'])
# Require non-zero seg area and more than 1x1 box size\
obj = {
'area': car_pose['area'],
'clean_bbox': [x1, y1, x2, y2],
'category_id': 33,
'car_id': car_pose['car_id'],
'visible_rate': car_pose['visible_rate'],
'pose': car_pose['pose']
}
valid_objs.append(obj)
num_valid_objs = len(valid_objs)
boxes = np.zeros((num_valid_objs, 4), dtype=np.float32)
# this is a legacy network from WAD Mask-RCNN
car_class = 4
gt_overlaps = np.zeros((num_valid_objs, 8), dtype=np.float32)
seg_areas = np.zeros((num_valid_objs), dtype=np.float32)
is_crowd = np.zeros((num_valid_objs), dtype=np.bool)
box_to_gt_ind_map = np.zeros((num_valid_objs), dtype=np.int32)
# newly added for 3d car
visible_rate = np.zeros((num_valid_objs), dtype=np.float32)
poses = np.zeros((num_valid_objs, 6), dtype=np.float32)
quaternions = np.zeros((num_valid_objs, 4), dtype=np.float32)
car_cat_classes = np.zeros((num_valid_objs), dtype=np.int32)
for ix, obj in enumerate(valid_objs):
cls = np.where(self.Car3D.unique_car_models == obj['car_id'])[0][0]
boxes[ix, :] = obj['clean_bbox']
car_cat_classes[ix] = cls
seg_areas[ix] = obj['area']
is_crowd[ix] = False # TODO: What's this flag for?
box_to_gt_ind_map[ix] = ix
gt_overlaps[ix, car_class] = 1.0
visible_rate[ix] = obj['visible_rate']
poses[ix] = obj['pose']
quaternions[ix] = euler_angles_to_quaternions(
np.array([obj['pose'][:3]]))
# ensure the quaternion is upper hemispher
quaternions[ix] = quaternion_upper_hemispher(quaternions[ix])
entry['boxes'] = np.append(entry['boxes'], boxes, axis=0)
entry['seg_areas'] = np.append(entry['seg_areas'], seg_areas)
entry['gt_overlaps'] = np.append(entry['gt_overlaps'].toarray(),
gt_overlaps,
axis=0)
entry['gt_overlaps'] = scipy.sparse.csr_matrix(entry['gt_overlaps'])
entry['is_crowd'] = np.append(entry['is_crowd'], is_crowd)
entry['box_to_gt_ind_map'] = np.append(entry['box_to_gt_ind_map'],
box_to_gt_ind_map)
# newly added for 3d car
entry['visible_rate'] = np.append(entry['visible_rate'], visible_rate)
entry['poses'] = np.append(entry['poses'], poses, axis=0)
entry['car_cat_classes'] = np.append(entry['car_cat_classes'],
car_cat_classes)
entry['quaternions'] = np.append(entry['quaternions'],
quaternions,
axis=0)
def vis_one_image_eccv2018_car_3d(im,
im_name,
output_dir,
boxes,
car_cls_prob=None,
euler_angle=None,
trans_pred=None,
car_models=None,
intrinsic=None,
segms=None,
keypoints=None,
thresh=0.9,
dpi=200,
box_alpha=0.0,
dataset=None,
ext='jpg'):
"""Visual debugging of detections."""
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if isinstance(boxes, list):
boxes, segms, keypoints, classes = convert_from_cls_format(
boxes, segms, keypoints)
car_cls = np.argmax(car_cls_prob, axis=1)
if boxes is None or boxes.shape[0] == 0 or max(boxes[:, 4]) < thresh:
return
if segms is not None:
masks = mask_util.decode(segms)
color_list = colormap(rgb=True) / 255
fig = plt.figure(frameon=False)
fig.set_size_inches(im.shape[1] / dpi, im.shape[0] / dpi)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.axis('off')
fig.add_axes(ax)
ax.imshow(im)
# Display in largest to smallest order to reduce occlusion
areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
sorted_inds = np.argsort(-areas)
mask_color_id = 0
# [35, 38, 36, 39, 40, 34, 33]
for i in sorted_inds:
bbox = boxes[i, :4]
score = boxes[i, -1]
if score < thresh:
continue
json_id = dataset.contiguous_category_id_to_json_id[classes[i]]
class_string = dataset.id_map_to_cat[json_id]
# car_cls
car_cls_i = car_cls[i]
euler_angle_i = euler_angle[i]
trans_pred_i = trans_pred[i]
car_model_i = dataset.unique_car_models[car_cls_i]
car_model_name_i = dataset.car_id2name[car_model_i]
if class_string == 'car':
print("%s: %.4f, car model: %s" %
(class_string, score, car_model_name_i.name) +
'. quaternion: ' +
", ".join(['{:.3f}'.format(i)
for i in euler_angle_i]) + '. Trans: ' +
", ".join(['{:.3f}'.format(i) for i in trans_pred_i]))
else:
print(class_string, score)
# show box (off by default, box_alpha=0.0)
ax.add_patch(
plt.Rectangle((bbox[0], bbox[1]),
bbox[2] - bbox[0],
bbox[3] - bbox[1],
fill=False,
edgecolor='g',
linewidth=0.5,
alpha=box_alpha))
if class_string == 'car':
#vis_string = car_model_name_i.name + '. quaternion: ' + ", ".join(['{:.3f}'.format(i) for i in rot_pred_i]) + '. Trans: ' + ", ".join(['{:.3f}'.format(i) for i in trans_pred_i])
vis_string = car_model_name_i.name
ax.text(bbox[0],
bbox[1] - 2,
vis_string,
fontsize=10,
family='serif',
bbox=dict(facecolor='g',
alpha=0.4,
pad=0,
edgecolor='none'),
color='white')
# Show predicted pos mesh here:
if trans_pred_i is not None and euler_angle_i is not None:
#euler_angle = quaternion_to_euler_angle(rot_pred_i)
pose = np.concatenate((euler_angle_i, trans_pred_i))
car_name = car_model_name_i.name
car = car_models[car_name]
pose = np.array(pose)
# project 3D points to 2d image plane
rmat = euler_angles_to_rotation_matrix(pose[:3])
rvect, _ = cv2.Rodrigues(rmat)
imgpts, jac = cv2.projectPoints(np.float32(car['vertices']),
rvect,
pose[3:],
intrinsic,
distCoeffs=None)
triangles = np.array(car['faces'] - 1).astype('int64')
x = np.squeeze(imgpts[:, :, 1])
y = np.squeeze(imgpts[:, :, 0])
triangles = triangles
color_mask = color_list[mask_color_id % len(color_list), 0:3]
ax.triplot(y,
x,
triangles,
alpha=0.8,
linewidth=1.2,
color=color_mask)
# show mask
if segms is not None and len(segms) > i:
img = np.ones(im.shape)
color_mask = color_list[mask_color_id % len(color_list), 0:3]
mask_color_id += 1
w_ratio = .4
for c in range(3):
color_mask[c] = color_mask[c] * (1 - w_ratio) + w_ratio
for c in range(3):
img[:, :, c] = color_mask[c]
e = masks[:, :, i]
_, contour, hier = cv2.findContours(e.copy(), cv2.RETR_CCOMP,
cv2.CHAIN_APPROX_NONE)
for c in contour:
polygon = Polygon(c.reshape((-1, 2)),
fill=True,
facecolor=color_mask,
edgecolor='w',
linewidth=1.2,
alpha=0.2)
ax.add_patch(polygon)
output_name = os.path.basename(im_name) + '.' + ext
fig.savefig(os.path.join(output_dir, '{}'.format(output_name)), dpi=dpi)
plt.close('all')
def open_3d_vis(args, output_dir):
"""
http://www.open3d.org/docs/tutorial/Basic/visualization.html
-- Mouse view control --
Left button + drag : Rotate.
Ctrl + left button + drag : Translate.
Wheel : Zoom in/out.
-- Keyboard view control --
[/] : Increase/decrease field of view.
R : Reset view point.
Ctrl/Cmd + C : Copy current view status into the clipboard. (A nice view has been saved as utilites/view.json
Ctrl/Cmd + V : Paste view status from clipboard.
-- General control --
Q, Esc : Exit window.
H : Print help message.
P, PrtScn : Take a screen capture.
D : Take a depth capture.
O : Take a capture of current rendering settings.
"""
json_dir = os.path.join(
output_dir, 'json_' + args.list_flag + '_trans') + '_iou' + str(1.0)
args.test_dir = json_dir
args.gt_dir = args.dataset_dir + 'car_poses'
args.res_file = os.path.join(output_dir,
'json_' + args.list_flag + '_res.txt')
args.simType = None
car_models = load_car_models(os.path.join(args.dataset_dir, 'car_models'))
det_3d_metric = Detect3DEval(args)
evalImgs = det_3d_metric.evaluate()
image_id = evalImgs['image_id']
print(image_id)
# We only draw the most loose constraint here
gtMatches = evalImgs['gtMatches'][0]
dtScores = evalImgs['dtScores']
mesh_car_all = []
# We also save road surface
xmin, xmax, ymin, ymax, zmin, zmax = np.inf, -np.inf, np.inf, -np.inf, np.inf, -np.inf
for car in det_3d_metric._gts[image_id]:
car_model = car_models[car['car_id']]
R = euler_angles_to_rotation_matrix(car['pose'][:3])
vertices = np.matmul(R, car_model['vertices'].T) + np.asarray(
car['pose'][3:])[:, None]
xmin, xmax, ymin, ymax, zmin, zmax = update_road_surface(
xmin, xmax, ymin, ymax, zmin, zmax, vertices)
mesh_car = TriangleMesh()
mesh_car.vertices = Vector3dVector(vertices.T)
mesh_car.triangles = Vector3iVector(car_model['faces'] - 1)
# Computing normal
mesh_car.compute_vertex_normals()
# we render the GT car in BLUE
mesh_car.paint_uniform_color([0, 0, 1])
mesh_car_all.append(mesh_car)
for i, car in enumerate(det_3d_metric._dts[image_id]):
if dtScores[i] > args.dtScores:
car_model = car_models[car['car_id']]
R = euler_angles_to_rotation_matrix(car['pose'][:3])
vertices = np.matmul(R, car_model['vertices'].T) + np.asarray(
car['pose'][3:])[:, None]
mesh_car = TriangleMesh()
mesh_car.vertices = Vector3dVector(vertices.T)
mesh_car.triangles = Vector3iVector(car_model['faces'] - 1)
# Computing normal
mesh_car.compute_vertex_normals()
if car['id'] in gtMatches:
# if it is a true positive, we render it as green
mesh_car.paint_uniform_color([0, 1, 0])
else:
# else we render it as red
mesh_car.paint_uniform_color([1, 0, 0])
mesh_car_all.append(mesh_car)
# Draw the road surface here
# x = np.linspace(xmin, xmax, 100)
# every 0.1 meter
xyz = get_road_surface_xyz(xmin, xmax, ymin, ymax, zmin, zmax)
# Pass xyz to Open3D.PointCloud and visualize
pcd_road = PointCloud()
pcd_road.points = Vector3dVector(xyz)
pcd_road.paint_uniform_color([0, 0, 1])
mesh_car_all.append(pcd_road)
# draw mesh frame
mesh_frame = create_mesh_coordinate_frame(size=3, origin=[0, 0, 0])
mesh_car_all.append(mesh_frame)
draw_geometries(mesh_car_all)
det_3d_metric.accumulate()
det_3d_metric.summarize()