def projection_2d(self, pose_pred, pose_targets, K, threshold=5):
model_2d_pred = project(self.mesh_model["pts"], K, pose_pred)
model_2d_targets = project(self.mesh_model["pts"], K, pose_targets)
proj_mean_diff = np.mean(
np.linalg.norm(model_2d_pred - model_2d_targets, axis=-1))
self.proj_2d.append(proj_mean_diff < threshold)
def test_render():
car_model = 'test.pkl'
with open(car_model, 'r') as f:
model = pkl.load(f)
vertices = model['vertices']
faces = model['faces']
scale = np.float32([1, 1, 1])
intrinsic = np.float64([250, 250, 160, 120])
imgsize = [240, 320]
intrinsic = np.float64([350, 350, 320, 92])
imgsize = [180, 624]
T = np.float32([0.0, 1.9, 0.0, -1.0, -1.1, 6.0])
vertices_r = uts.project(T, scale, vertices)
vertices_r[:, 1] = vertices_r[:, 1] - 0.3
vertices_r[:, 2] = vertices_r[:, 2] + 3.0
faces = np.float64(faces)
depth, mask = render_egl.renderMesh_py(vertices_r, faces, intrinsic,
imgsize[0], imgsize[1], 0.0)
assert np.max(depth) > 0
print 'passed'
def render_car(self, pose, car_name):
"""Render a car instance given pose and car_name
"""
car = self.car_models[car_name]
scale = np.ones((3, ))
pose = np.array(pose)
vert = uts.project(pose, scale, car['vertices'])
K = self.intrinsic
intrinsic = np.float64([K[0, 0], K[1, 1], K[0, 2], K[1, 2]])
depth, mask = render.renderMesh_py(np.float64(vert),
np.float64(car['faces']), intrinsic,
self.image_size[0],
self.image_size[1],
np.float64(self.linewidth))
return depth, mask
def test_render():
# car_model = 'test.pkl'
car_model = '019-SUV.pkl'
# car_model = '/home/zhurui/Documents/baidu/personal-code/car-fitting/rui_modelfitting/dataset-api/apolloscape/019-SUV.pkl'
with open(car_model, 'r') as f:
model = pkl.load(f)
# # off_path = '/home/zhurui/Documents/baidu/personal-code/car-fitting/rui_modelfitting/dataset-api/apolloscape/5_gt/019-SUV.off'
# off_path = '/home/zhurui/Documents/baidu/personal-code/car-fitting/rui_modelfitting/dataset-api/apolloscape/019-SUV.off'
# off_mesh = Mesh.from_off(off_path)
# model = {'vertices': off_mesh.vertices, 'faces': off_mesh.faces}
vertices = model['vertices'] / 100.
# vertices[:, [0, 2]] = vertices[:, [2, 0]]
print vertices.shape
# print np.amax(vertices, axis=0), np.amin(vertices, axis=0)
faces = model['faces']
scale = np.float32([1, 1, 1])
intrinsic = np.float64([250, 250, 160, 120])
imgsize = [240, 320]
intrinsic = np.float64([350, 350, 320, 92])
imgsize = [180, 624]
T = np.float32([0.2, 0., 0.1, -1.0, -0.1, 6.0])
vertices_r = uts.project(T, scale, vertices)
# vertices_r[:, 1] = vertices_r[:, 1] - 0.3
# vertices_r[:, 2] = vertices_r[:, 2] + 3.0
faces = np.float64(faces)
depth, mask = render_egl.renderMesh_py(vertices_r, faces, intrinsic,
imgsize[0], imgsize[1], 0.0)
print np.max(depth), np.min(depth)
# assert np.max(depth) > 0
print 'passed.'
print 'Shape of depth:', depth.shape, 'Shape of mask:', mask.shape
import scipy.misc
scipy.misc.imsave('depth_test.jpg', depth)
scipy.misc.imsave('mask_test.jpg', mask * 255)
print 'Depth and mask image are saved to depth_test.jpg and mask_test.jpg.'
def showAnn(self,
image_name,
if_result=False,
if_visualize=False,
if_save=False,
plot_path='tmp',
is_training=False):
"""Show the annotation of a pose file in an image
Input:
image_name: the name of image
Output:
depth: a rendered depth map of each car
masks: an instance mask of the label
image_vis: an image show the overlap of car model and image
"""
image_file = '%s/%s.jpg' % (self._data_config['image_dir'], image_name)
image = cv2.imread(image_file, cv2.IMREAD_UNCHANGED)[:, :, ::-1]
# print 'Original and rescaled image size: ', image.shape, self.image_size
intrinsic = self.dataset.get_intrinsic(image_name, 'Camera_5')
image_rescaled, self.intrinsic = self.rescale(image, intrinsic)
if is_training:
car_pose_file = '%s/%s.json' % (
self._data_config['pose_dir'] if not (if_result) else
self._data_config['pose_dir_result'], image_name)
with open(car_pose_file) as f:
car_poses = json.load(f)
self.depth = self.MAX_DEPTH * np.ones(self.image_size)
self.mask = np.zeros(self.depth.shape)
self.shape_id_map = np.zeros(self.depth.shape)
self.pose_map = np.zeros(
(self.depth.shape[0], self.depth.shape[1], 6)) + np.inf
self.shape_map = np.zeros(
(self.depth.shape[0], self.depth.shape[1], 10)) + np.inf
self.pose_list = []
self.rot_uvd_list = []
self.bbox_list = []
self.shape_id_list = []
plt.figure(figsize=(20, 10))
plt.imshow(image_rescaled)
for i, car_pose in enumerate(car_poses):
car_name = car_models.car_id2name[car_pose['car_id']].name
# if if_result:
# car_pose['pose'][-1] = 1./car_pose['pose'][-1]
depth, mask, vert, K = self.render_car(car_pose['pose'],
car_name)
self.mask, self.shape_id_map, self.depth, self.pose_map = self.merge_inst(
depth, i + 1, car_pose['car_id'] + 1, self.mask,
self.shape_id_map, self.depth, self.pose_map,
car_pose['pose'])
self.pose_list.append(car_pose['pose'])
self.shape_id_list.append(car_pose['car_id'])
scale = np.ones((3, ))
car = self.car_models[car_name]
pose = np.array(car_pose['pose'])
print 'GT pose: ', pose[3:]
vert = car['vertices']
vert = np.zeros((1, 3))
vert_transformed = uts.project(pose, scale, vert) # [*, 3]
print 'Center transformed: ', vert_transformed
vert_hom = np.hstack(
(vert_transformed, np.ones((vert.shape[0], 1))))
K_hom = np.hstack((K, np.zeros((3, 1))))
proj_uv_hom = np.matmul(K_hom, vert_hom.T)
proj_uv = np.vstack((proj_uv_hom[0, :] / proj_uv_hom[2, :],
proj_uv_hom[1, :] / proj_uv_hom[2, :]))
u = proj_uv[0:1, :] # [1, 1]
v = proj_uv[1:2, :]
d = proj_uv_hom[2:3, :]
rot_uvd = [
car_pose['pose'][0], car_pose['pose'][1],
car_pose['pose'][2], u[0, 0], v[0, 0], car_pose['pose'][5]
]
self.rot_uvd_list.append(rot_uvd)
plt.scatter(u, v, linewidths=20)
F1 = K_hom[0, 0]
W = K_hom[0, 2]
F2 = K_hom[1, 1]
H = K_hom[1, 2]
K_T = np.array([[1. / F1, 0., -W / F1], [0, 1. / F2, -H / F2],
[0., 0., 1.]])
# print K_T
# print self.intrinsic
# print F1, W, F2, H
uvd = np.vstack((u * d, v * d, d))
xyz = np.matmul(K_T, uvd)
print 'xyz / pose recovered: ', xyz
# print 'uvd:', rot_uvd
# print car_pose['pose'].shape, vert_transformed.shape
## Get bbox from mask
arr = np.expand_dims(np.int32(mask), -1)
# number of highest label:
labmax = 1
# maximum and minimum positions along each axis (initialized to very low and high values)
b_first = np.iinfo('int32').max * np.ones(
(3, labmax + 1), dtype='int32')
b_last = np.iinfo('int32').max * np.ones(
(3, labmax + 1), dtype='int32')
# run through all dimensions making 2D slices and marking all existing labels to b
for dim in range(2):
# create a generic slice object to make the slices
sl = [slice(None), slice(None), slice(None)]
bf = b_first[dim]
bl = b_last[dim]
# go through all slices in this dimension
for k in range(arr.shape[dim]):
# create the slice object
sl[dim] = k
# update the last "seen" vector
bl[arr[sl].flatten()] = k
# if we have smaller values in "last" than in "first", update
bf[:] = np.clip(bf, None, bl)
bbox = [
b_first[1, 1], b_last[1, 1], b_first[0, 1], b_last[0, 1]
] # [x_min, x_max, y_min, y_max]
self.bbox_list.append(bbox)
plt.imshow(mask)
print mask.shape
currentAxis = plt.gca()
# print (bbox[0], bbox[2]), bbox[1]-bbox[0], bbox[3]-bbox[2]
currentAxis.add_patch(
Rectangle((bbox[0], bbox[2]),
bbox[1] - bbox[0],
bbox[3] - bbox[2],
alpha=1,
edgecolor='r',
facecolor='none'))
# plt.show()
# break
plt.show()
self.depth[self.depth == self.MAX_DEPTH] = -1.0
image = 0.5 * image_rescaled
for i in range(len(car_poses)):
frame = np.float32(self.mask == i + 1)
frame = np.tile(frame[:, :, None], (1, 1, 3))
image = image + frame * 0.5 * self.colors[i, :]
if if_visualize:
uts.plot_images(
{
'image_vis': np.uint8(image),
'shape_id': self.shape_id_map,
'mask': self.mask,
'depth': self.depth
},
np.asarray(self.rot_uvd_list),
self.bbox_list,
layout=[1, 4],
fig_size=10,
save_fig=if_save,
fig_name=plot_path)
return image, self.mask, self.shape_id_map, self.depth, self.pose_map, image_rescaled, self.pose_list, self.shape_id_list, self.rot_uvd_list, self.bbox_list
else:
return None, None, None, None, None, image_rescaled, None, None, None, None