def projection_2d(self, pose_pred, pose_targets, K, threshold=5): model_2d_pred = project(self.mesh_model["pts"], K, pose_pred) model_2d_targets = project(self.mesh_model["pts"], K, pose_targets) proj_mean_diff = np.mean( np.linalg.norm(model_2d_pred - model_2d_targets, axis=-1)) self.proj_2d.append(proj_mean_diff < threshold)
def test_render(): car_model = 'test.pkl' with open(car_model, 'r') as f: model = pkl.load(f) vertices = model['vertices'] faces = model['faces'] scale = np.float32([1, 1, 1]) intrinsic = np.float64([250, 250, 160, 120]) imgsize = [240, 320] intrinsic = np.float64([350, 350, 320, 92]) imgsize = [180, 624] T = np.float32([0.0, 1.9, 0.0, -1.0, -1.1, 6.0]) vertices_r = uts.project(T, scale, vertices) vertices_r[:, 1] = vertices_r[:, 1] - 0.3 vertices_r[:, 2] = vertices_r[:, 2] + 3.0 faces = np.float64(faces) depth, mask = render_egl.renderMesh_py(vertices_r, faces, intrinsic, imgsize[0], imgsize[1], 0.0) assert np.max(depth) > 0 print 'passed'
def render_car(self, pose, car_name): """Render a car instance given pose and car_name """ car = self.car_models[car_name] scale = np.ones((3, )) pose = np.array(pose) vert = uts.project(pose, scale, car['vertices']) K = self.intrinsic intrinsic = np.float64([K[0, 0], K[1, 1], K[0, 2], K[1, 2]]) depth, mask = render.renderMesh_py(np.float64(vert), np.float64(car['faces']), intrinsic, self.image_size[0], self.image_size[1], np.float64(self.linewidth)) return depth, mask
def test_render(): # car_model = 'test.pkl' car_model = '019-SUV.pkl' # car_model = '/home/zhurui/Documents/baidu/personal-code/car-fitting/rui_modelfitting/dataset-api/apolloscape/019-SUV.pkl' with open(car_model, 'r') as f: model = pkl.load(f) # # off_path = '/home/zhurui/Documents/baidu/personal-code/car-fitting/rui_modelfitting/dataset-api/apolloscape/5_gt/019-SUV.off' # off_path = '/home/zhurui/Documents/baidu/personal-code/car-fitting/rui_modelfitting/dataset-api/apolloscape/019-SUV.off' # off_mesh = Mesh.from_off(off_path) # model = {'vertices': off_mesh.vertices, 'faces': off_mesh.faces} vertices = model['vertices'] / 100. # vertices[:, [0, 2]] = vertices[:, [2, 0]] print vertices.shape # print np.amax(vertices, axis=0), np.amin(vertices, axis=0) faces = model['faces'] scale = np.float32([1, 1, 1]) intrinsic = np.float64([250, 250, 160, 120]) imgsize = [240, 320] intrinsic = np.float64([350, 350, 320, 92]) imgsize = [180, 624] T = np.float32([0.2, 0., 0.1, -1.0, -0.1, 6.0]) vertices_r = uts.project(T, scale, vertices) # vertices_r[:, 1] = vertices_r[:, 1] - 0.3 # vertices_r[:, 2] = vertices_r[:, 2] + 3.0 faces = np.float64(faces) depth, mask = render_egl.renderMesh_py(vertices_r, faces, intrinsic, imgsize[0], imgsize[1], 0.0) print np.max(depth), np.min(depth) # assert np.max(depth) > 0 print 'passed.' print 'Shape of depth:', depth.shape, 'Shape of mask:', mask.shape import scipy.misc scipy.misc.imsave('depth_test.jpg', depth) scipy.misc.imsave('mask_test.jpg', mask * 255) print 'Depth and mask image are saved to depth_test.jpg and mask_test.jpg.'
def showAnn(self, image_name, if_result=False, if_visualize=False, if_save=False, plot_path='tmp', is_training=False): """Show the annotation of a pose file in an image Input: image_name: the name of image Output: depth: a rendered depth map of each car masks: an instance mask of the label image_vis: an image show the overlap of car model and image """ image_file = '%s/%s.jpg' % (self._data_config['image_dir'], image_name) image = cv2.imread(image_file, cv2.IMREAD_UNCHANGED)[:, :, ::-1] # print 'Original and rescaled image size: ', image.shape, self.image_size intrinsic = self.dataset.get_intrinsic(image_name, 'Camera_5') image_rescaled, self.intrinsic = self.rescale(image, intrinsic) if is_training: car_pose_file = '%s/%s.json' % ( self._data_config['pose_dir'] if not (if_result) else self._data_config['pose_dir_result'], image_name) with open(car_pose_file) as f: car_poses = json.load(f) self.depth = self.MAX_DEPTH * np.ones(self.image_size) self.mask = np.zeros(self.depth.shape) self.shape_id_map = np.zeros(self.depth.shape) self.pose_map = np.zeros( (self.depth.shape[0], self.depth.shape[1], 6)) + np.inf self.shape_map = np.zeros( (self.depth.shape[0], self.depth.shape[1], 10)) + np.inf self.pose_list = [] self.rot_uvd_list = [] self.bbox_list = [] self.shape_id_list = [] plt.figure(figsize=(20, 10)) plt.imshow(image_rescaled) for i, car_pose in enumerate(car_poses): car_name = car_models.car_id2name[car_pose['car_id']].name # if if_result: # car_pose['pose'][-1] = 1./car_pose['pose'][-1] depth, mask, vert, K = self.render_car(car_pose['pose'], car_name) self.mask, self.shape_id_map, self.depth, self.pose_map = self.merge_inst( depth, i + 1, car_pose['car_id'] + 1, self.mask, self.shape_id_map, self.depth, self.pose_map, car_pose['pose']) self.pose_list.append(car_pose['pose']) self.shape_id_list.append(car_pose['car_id']) scale = np.ones((3, )) car = self.car_models[car_name] pose = np.array(car_pose['pose']) print 'GT pose: ', pose[3:] vert = car['vertices'] vert = np.zeros((1, 3)) vert_transformed = uts.project(pose, scale, vert) # [*, 3] print 'Center transformed: ', vert_transformed vert_hom = np.hstack( (vert_transformed, np.ones((vert.shape[0], 1)))) K_hom = np.hstack((K, np.zeros((3, 1)))) proj_uv_hom = np.matmul(K_hom, vert_hom.T) proj_uv = np.vstack((proj_uv_hom[0, :] / proj_uv_hom[2, :], proj_uv_hom[1, :] / proj_uv_hom[2, :])) u = proj_uv[0:1, :] # [1, 1] v = proj_uv[1:2, :] d = proj_uv_hom[2:3, :] rot_uvd = [ car_pose['pose'][0], car_pose['pose'][1], car_pose['pose'][2], u[0, 0], v[0, 0], car_pose['pose'][5] ] self.rot_uvd_list.append(rot_uvd) plt.scatter(u, v, linewidths=20) F1 = K_hom[0, 0] W = K_hom[0, 2] F2 = K_hom[1, 1] H = K_hom[1, 2] K_T = np.array([[1. / F1, 0., -W / F1], [0, 1. / F2, -H / F2], [0., 0., 1.]]) # print K_T # print self.intrinsic # print F1, W, F2, H uvd = np.vstack((u * d, v * d, d)) xyz = np.matmul(K_T, uvd) print 'xyz / pose recovered: ', xyz # print 'uvd:', rot_uvd # print car_pose['pose'].shape, vert_transformed.shape ## Get bbox from mask arr = np.expand_dims(np.int32(mask), -1) # number of highest label: labmax = 1 # maximum and minimum positions along each axis (initialized to very low and high values) b_first = np.iinfo('int32').max * np.ones( (3, labmax + 1), dtype='int32') b_last = np.iinfo('int32').max * np.ones( (3, labmax + 1), dtype='int32') # run through all dimensions making 2D slices and marking all existing labels to b for dim in range(2): # create a generic slice object to make the slices sl = [slice(None), slice(None), slice(None)] bf = b_first[dim] bl = b_last[dim] # go through all slices in this dimension for k in range(arr.shape[dim]): # create the slice object sl[dim] = k # update the last "seen" vector bl[arr[sl].flatten()] = k # if we have smaller values in "last" than in "first", update bf[:] = np.clip(bf, None, bl) bbox = [ b_first[1, 1], b_last[1, 1], b_first[0, 1], b_last[0, 1] ] # [x_min, x_max, y_min, y_max] self.bbox_list.append(bbox) plt.imshow(mask) print mask.shape currentAxis = plt.gca() # print (bbox[0], bbox[2]), bbox[1]-bbox[0], bbox[3]-bbox[2] currentAxis.add_patch( Rectangle((bbox[0], bbox[2]), bbox[1] - bbox[0], bbox[3] - bbox[2], alpha=1, edgecolor='r', facecolor='none')) # plt.show() # break plt.show() self.depth[self.depth == self.MAX_DEPTH] = -1.0 image = 0.5 * image_rescaled for i in range(len(car_poses)): frame = np.float32(self.mask == i + 1) frame = np.tile(frame[:, :, None], (1, 1, 3)) image = image + frame * 0.5 * self.colors[i, :] if if_visualize: uts.plot_images( { 'image_vis': np.uint8(image), 'shape_id': self.shape_id_map, 'mask': self.mask, 'depth': self.depth }, np.asarray(self.rot_uvd_list), self.bbox_list, layout=[1, 4], fig_size=10, save_fig=if_save, fig_name=plot_path) return image, self.mask, self.shape_id_map, self.depth, self.pose_map, image_rescaled, self.pose_list, self.shape_id_list, self.rot_uvd_list, self.bbox_list else: return None, None, None, None, None, image_rescaled, None, None, None, None