def render(obj_path, camera_mat, return_depth=False, im_size=128): fuze_trimesh = trimesh.load(obj_path) if type(fuze_trimesh) == trimesh.base.Trimesh: m = pyrender.Mesh.from_trimesh(fuze_trimesh) scene = pyrender.Scene() scene.add_node(pyrender.Node(mesh=m)) else: assert type( fuze_trimesh) == trimesh.scene.scene.Scene, "Unrognized file" scene = pyrender.Scene.from_trimesh_scene(fuze_trimesh) camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0) s = np.sqrt(2) / 2 scene.add(camera, pose=camera_mat) light = pyrender.SpotLight(color=np.ones(3), intensity=4.0, innerConeAngle=np.pi / 16.0) scene.add(light, pose=camera_mat) light = pyrender.SpotLight(color=np.ones(3), intensity=6.0, innerConeAngle=0.2 * np.pi) light_pose = np.array( [[0, 1, 0, 0], [0, 0, 1, 1], [1, 0, 0, 0], [0, 0, 0, 1]], dtype=np.float32) scene.add(light, pose=light_pose) r = pyrender.OffscreenRenderer(im_size, im_size) color, depth = r.render(scene) r.delete() if return_depth: return color, depth return color
def _create_scene_and_offscreen_render(): """We will add (and remove) the meshes later. Unfortunately this requires some tuning of the position and rotation. """ scene = pyrender.Scene() camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0) # The position / translation [px, py, pz] and then rotation matrix. p = [0.45, -0.45, 0.90] theta = -45 * DEG_TO_RAD RX = np.array([ [1.0, 0.0, 0.0], [0.0, np.cos(theta), np.sin(theta)], [0.0, -np.sin(theta), np.cos(theta)], ]) R = RX camera_pose = np.array([ [R[0,0], R[0,1], R[0,2], p[0]], [R[1,0], R[1,1], R[1,2], p[1]], [R[2,0], R[2,1], R[2,2], p[2]], [ 0.0, 0.0, 0.0, 1.0], ]) scene.add(camera, pose=camera_pose) light = pyrender.SpotLight(color=np.ones(2), intensity=1.0, innerConeAngle=np.pi/16.0, outerConeAngle=np.pi/6.0) scene.add(light, pose=camera_pose) # Only for debugging #v = pyrender.Viewer(scene, use_raymond_lighting=True) rend = pyrender.OffscreenRenderer(640, 480) return scene, rend
def render_depth(path, override=None): # Load the FUZE bottle trimesh and put it in a scene fuze_trimesh = trimesh.load(path) fuze_trimesh.vertices = ( fuze_trimesh.vertices - np.amin(fuze_trimesh.vertices, axis=0)) / ( np.amax(fuze_trimesh.vertices, axis=0) - np.amin(fuze_trimesh.vertices, axis=0) + 0.001) - 0.5 mesh = pyrender.Mesh.from_trimesh(fuze_trimesh) scene = pyrender.Scene() scene.add(mesh) # Set up the camera -- z-axis away from the scene, x-axis right, y-axis up camera = pyrender.PerspectiveCamera(yfov=np.pi / 5.0, znear=0.15) camera_pose = getCameraPose(override) scene.add(camera, pose=camera_pose) # Set up the light -- a single spot light in the same spot as the camera light = pyrender.SpotLight(color=np.ones(3), intensity=3.0, innerConeAngle=np.pi / 16.0) scene.add(light, pose=camera_pose) # Render the scene r = pyrender.OffscreenRenderer(200, 200) depth = r.render(scene, flags=pyrender.RenderFlags.DEPTH_ONLY) depth = depth / (np.max(depth) + 0.0001) return depth
def rendering(R, fuze_trimesh): mesh = pyrender.Mesh.from_trimesh(fuze_trimesh, poses=[R]) scene = pyrender.Scene() scene.add(mesh) camera = pyrender.PerspectiveCamera(yfov=np.pi / 3, aspectRatio=1) m1 = np.array([ [1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1], ], dtype='float') m2 = np.array([ [1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0.4], [0, 0, 0, 1], ], dtype='float') camera_pose = m1.dot(m2) scene.add(camera, pose=camera_pose) light = pyrender.SpotLight(color=np.ones(3), intensity=3.0, innerConeAngle=np.pi / 16.0) scene.add(light, pose=camera_pose) r = pyrender.OffscreenRenderer(400, 400) color, depth = r.render(scene) color = cv2.cvtColor(color, cv2.COLOR_BGR2RGB) return color
def render_hmr_smpl(hmr_coeff, f_len=500., rend_size=224., req_model=False): # hmr_coeff is a 85-vector, named as theta in hmr hmr_coeff = np.asarray(hmr_coeff).tolist() # make scene scene = pyrender.Scene() # initialize camera camera = pyrender.PerspectiveCamera( yfov=np.arctan(rend_size * 0.5 / f_len) * 2, aspectRatio=1) camera_pose = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]]) scene.add(camera, pose=camera_pose) # initialize light light_posi1 = np.array([[1.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, -1.0], [0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]]) light_posi2 = np.array([[1.0, 0.0, 0.0, -1.0], [0.0, 0.0, 1.0, -1.0], [0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]]) light_posi3 = np.array([[1.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, 1.0], [0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]]) light = pyrender.SpotLight(color=np.array( [0.65098039, 0.74117647, 0.85882353]), intensity=100, innerConeAngle=np.pi / 16.0, outerConeAngle=np.pi / 6.0) scene.add(light, pose=light_posi1) scene.add(light, pose=light_posi2) scene.add(light, pose=light_posi3) # get renderer r = pyrender.OffscreenRenderer(viewport_width=rend_size, viewport_height=rend_size) # get verts from smpl coefficients smpl_op = load_model("./tf_smpl/neutral_smpl_with_cocoplus_reg.pkl") smpl_op.pose[:] = np.asarray(hmr_coeff[3:75]) smpl_op.betas[:] = np.array(hmr_coeff[75:85]) verts = np.array(smpl_op) global_t = np.array( [hmr_coeff[1], hmr_coeff[2], f_len / (0.5 * rend_size * hmr_coeff[0])]) verts = verts + global_t faces = np.load("./tf_smpl/smpl_faces.npy").astype(np.int32) # smooth and expand om_mesh = make_trimesh(verts, faces) om_mesh = smooth_mesh(om_mesh, 4) om_mesh = expand_mesh(om_mesh, 0.026) this_trimesh = trimesh.Trimesh(vertices=om_mesh.points(), faces=om_mesh.face_vertex_indices()) this_mesh = pyrender.Mesh.from_trimesh(this_trimesh) scene.add(this_mesh) rend_img, depth = r.render(scene) if req_model is True: return rend_img, verts, faces, depth else: return rend_img
def _init_scene(self): self._scene = pyrender.Scene() camera = pyrender.PerspectiveCamera( yfov=self._fov, aspectRatio=1.0, znear=0.001) # do not change aspect ratio theta, phi = 0, np.pi / 4 + np.random.rand() * (np.pi / 8) radius = 3 + (np.random.rand() - 1) x = radius * np.sin(phi) * np.cos(theta) y = radius * np.sin(phi) * np.sin(theta) z = radius * np.cos(phi) camera_pos = np.array([x, y, z]) #camera_pos = np.array([0,0,10]) #light_pos = np.array([0,0,10]) camera_pose = tra.euler_matrix(0, phi, 0) camera_pose[:3, 3] = camera_pos camera_pose[:3, :3] = camera_pose[:3, :3] self._scene.add(camera, pose=camera_pose, name='camera') self.camera_pose = camera_pose light = pyrender.SpotLight(color=np.ones(4), intensity=50., innerConeAngle=np.pi / 16, outerConeAngle=np.pi / 6.0) self._scene.add(light, pose=camera_pose, name='light')
def render_pred_coeff(pred_coeff): pred_coeff = np.asarray(pred_coeff).tolist() # make scene scene = pyrender.Scene() # initialize camera f_len = 366.474487 rend_size = 256. camera = pyrender.PerspectiveCamera(yfov=np.arctan(rend_size*0.5/f_len)*2, aspectRatio=1) camera_pose = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]]) scene.add(camera, pose=camera_pose) # initialize light light_posi1 = np.array([[1.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, -1.0], [0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]]) light_posi2 = np.array([[1.0, 0.0, 0.0, -1.0], [0.0, 0.0, 1.0, -1.0], [0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]]) light_posi3 = np.array([[1.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, 1.0], [0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]]) light = pyrender.SpotLight(color=np.array([0.65098039, 0.74117647, 0.85882353]), intensity=100, innerConeAngle=np.pi/16.0, outerConeAngle=np.pi/6.0) scene.add(light, pose=light_posi1) scene.add(light, pose=light_posi2) scene.add(light, pose=light_posi3) # get renderer r = pyrender.OffscreenRenderer(viewport_width=rend_size, viewport_height=rend_size) # get verts from smpl coefficients smpl_op = load_model("./tf_smpl/neutral_smpl_with_cocoplus_reg.pkl") verts_bias = pred_coeff[3:6] smpl_op.pose[:] = np.asarray([0]*3 + pred_coeff[6:75]) smpl_op.betas[:] = np.array(pred_coeff[75:85]) verts = np.array(smpl_op) rot_mat = cv2.Rodrigues(np.asarray(pred_coeff[3:6]))[0] verts = np.tensordot(verts, rot_mat, axes=([1],[1])) verts = verts + pred_coeff[:3] # make trimesh faces = np.load("./tf_smpl/smpl_faces.npy").astype(np.int32) this_trimesh = trimesh.Trimesh(vertices = verts, faces = faces) this_mesh = pyrender.Mesh.from_trimesh(this_trimesh) scene.add(this_mesh) rend_img, _ = r.render(scene) return rend_img
def render(self, camera_pose, target_id="", render_pc=True): """Render RGB/depth image, point cloud, and segmentation mask of the scene. Args: camera_pose (np.ndarray): Homogenous 4x4 matrix describing the pose of the camera in scene coordinates. target_id (str, optional): Object ID which is used to create the segmentation mask. Defaults to ''. render_pc (bool, optional): If true, point cloud is also returned. Defaults to True. Returns: np.ndarray: Color image. np.ndarray: Depth image. np.ndarray: Point cloud. np.ndarray: Segmentation mask. """ # Keep local to free OpenGl resources after use renderer = pyrender.OffscreenRenderer(viewport_width=self._width, viewport_height=self._height) # add camera and light to scene scene = self._scene.as_pyrender_scene() scene.add(self._camera, pose=camera_pose, name="camera") light = pyrender.SpotLight( color=np.ones(4), intensity=3.0, innerConeAngle=np.pi / 16, outerConeAngle=np.pi / 6.0, ) scene.add(light, pose=camera_pose, name="light") # render the full scene color, depth = renderer.render(scene) segmentation = np.zeros(depth.shape, dtype=np.uint8) # hide all objects for node in scene.mesh_nodes: node.mesh.is_visible = False # Render only target object and add to segmentation mask for node in scene.mesh_nodes: if node.name == target_id: node.mesh.is_visible = True _, object_depth = renderer.render(scene) mask = np.logical_and((np.abs(object_depth - depth) < 1e-6), np.abs(depth) > 0) segmentation[mask] = 1 for node in scene.mesh_nodes: node.mesh.is_visible = True if render_pc: pc = self._to_pointcloud(depth) else: pc = None return color, depth, pc, segmentation
def __init__(self, imgH, imgW): self.imgW = imgW self.imgH = imgH self.scene = pyrender.Scene(bg_color=[0.0, 0.0, 0.0]) self.nodesDict = {} self.light = pyrender.SpotLight(color=np.ones(3), intensity=3.0, innerConeAngle=np.pi / 16.0, outerConeAngle=np.pi / 6.0) self.scene.add(self.light, pose=np.eye(4))
def _init_scene(self): self._scene = pyrender.Scene() camera = pyrender.PerspectiveCamera( yfov=self._fov, aspectRatio=1.0, znear=0.001) # do not change aspect ratio camera_pose = tra.euler_matrix(np.pi, 0, 0) self._scene.add(camera, pose=camera_pose, name='camera') light = pyrender.SpotLight(color=np.ones(4), intensity=3., innerConeAngle=np.pi / 16, outerConeAngle=np.pi / 6.0) self._scene.add(light, pose=camera_pose, name='light') self.renderer = pyrender.OffscreenRenderer(400, 400)
def get_empty_scene(self): """Creates an empty scene with a camera and spot light.""" scene = pyrender.Scene(ambient_light=[0.2, 0.2, 0.2], bg_color=[0.1, 0.1, 0.1]) camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1) camera_pose = translation_matrix([0, 3, 4]) @ quaternion_matrix( quaternion_about_axis(np.deg2rad(-45.0), [1, 0, 0])) scene.add(camera, pose=camera_pose) light = pyrender.SpotLight(color=np.ones(3), intensity=3.0, innerConeAngle=np.pi / 16.0) scene.add(light, pose=camera_pose) return scene
def img_renderer(pointcloud): pcd = o3d.io.read_point_cloud(pointcloud) pcd = pcd.voxel_down_sample(voxel_size=0.05) pcd.estimate_normals() # estimate radius for rolling ball distances = pcd.compute_nearest_neighbor_distance() avg_dist = np.mean(distances) radius = 5 * avg_dist mesh = o3d.geometry.TriangleMesh.create_from_point_cloud_ball_pivoting( pcd, o3d.utility.DoubleVector([radius, radius * 2])) # create the triangular mesh with the vertices and faces from open3d tri_mesh = trimesh.Trimesh(np.asarray(mesh.vertices), np.asarray(mesh.triangles), vertex_normals=np.asarray(mesh.vertex_normals)) #tri_mesh = trimesh.convex.is_convex(tri_mesh) mesh = pyrender.Mesh.from_trimesh(tri_mesh) scene = pyrender.Scene() scene.add(mesh) camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0) s = np.sqrt(2)/2 camera_pose = np.array([ [0.0, -s, s, 0.3], [1.0, 0.0, 0.0, 0.0], [0.0, s, s, 0.35], [0.0, 0.0, 0.0, 1.0], ]) scene.add(camera, pose=camera_pose) light = pyrender.SpotLight(color=np.ones(3), intensity=3.0, innerConeAngle=np.pi/16.0, outerConeAngle=np.pi/6.0) scene.add(light, pose=camera_pose) pyrender.Viewer(scene) r = pyrender.OffscreenRenderer(512, 512) color, _ = r.render(scene) plt.figure(figsize=(8,8)) plt.imshow(color)
def render4mesh(self, idx, ratio=1): #the ratio=10 can make the rendered image be black vertices4view = self.get4viewManovertices(idx) import trimesh import pyrender from pyrender import RenderFlags np_rot_x = np.array([[1, 0, 0], [0, -1, 0], [0, 0, -1]], dtype=np.float32) np_rot_x = np.reshape(np.tile(np_rot_x, [1, 1]), [3, 3]) recolorlist = [] for iv in range(4): xyz = vertices4view[iv, :778, :3, 0].copy() cv = xyz @ np_rot_x tmesh = trimesh.Trimesh(vertices=cv / 1000 * ratio, faces=self.mano_right.faces) # tmesh.visual.vertex_colors = [.9, .7, .7, 1] # tmesh.show() mesh = pyrender.Mesh.from_trimesh(tmesh) scene = pyrender.Scene() scene.add(mesh) pycamera = pyrender.IntrinsicsCamera(self.camera[iv].fx, self.camera[iv].fy, self.camera[iv].cx, self.camera[iv].cy, znear=0.0001, zfar=3000) ccamera_pose = self.camera_pose[self.rootcameraidx] scene.add(pycamera, pose=ccamera_pose) light = pyrender.SpotLight(color=np.ones(3), intensity=2.0, innerConeAngle=np.pi / 16.0) # light = pyrender.PointLight(color=[1.0, 1.0, 1.0], intensity=2.0) scene.add(light, pose=ccamera_pose) r = pyrender.OffscreenRenderer(640, 480) # flags = RenderFlags.SHADOWS_DIRECTIONAL recolor, depth = r.render(scene) # cv2.imshow("depth", depth) recolorlist.append(recolor[:, :, :3]) meshcolor = np.hstack(recolorlist) return meshcolor
def texture_ply(self): # drop the RGB properties, and add two new properties (u, v) # vert_list = [] # for vert in self.vertices.data: # vert = vert.tolist() # convert to tuple # vertices_utm = np.reshape(self.vertices.data vertices_utm = np.stack( (self.vertices['x'], self.vertices['y'], self.vertices['z']), axis=1) vertices_enu = self.reconstruction.utm_to_enu(vertices_utm) # vert_list.append(xyz) # vert_list.append(vert[0:3]+(u, v)) # vertices = np.array(vert_list, # dtype=[('x', '<f4'), ('y', '<f4'), ('z', '<f4')]) # ('u', '<f4'), ('v', '<f4')]) # vert_el = PlyElement.describe(vertices, 'vertex', # comments=['point coordinate, texture coordinate']) # self.ply_textured = PlyData([vert_el, self.faces], text=True) print 'ply_vertices:', vertices_enu[0:2, :] renderer = pyrender.OffscreenRenderer(1000, 1000) for image, camera in self.reconstruction.cameras.items(): print camera.project(vertices_enu[0, :]) test_camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0) test_camera_pose = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 1000.0], [0, 0, 0, 1]]) # self.scene.add(camera.pyrender_camera, camera.pose) self.scene.add(test_camera, pose=test_camera_pose) light = pyrender.SpotLight(color=np.ones(3), intensity=3.0, innerConeAngle=np.pi / 16.0) # self.scene.add(light, pose=camera.pose) # self.scene.add(light, pose=test_camera_pose) # renderer = pyrender.OffscreenRenderer(camera.width, camera.height) color, depth = renderer.render(self.scene) # png.from_array(color, 'RGB').save(image + '_render.png') png.from_array(color, 'RGB').save('test_render.png')
def create_scene(self): self.scene = pyrender.Scene() # load table table_trimesh = trimesh.load(self.table_mesh_path) table_mesh = pyrender.Mesh.from_trimesh(table_trimesh) self.scene.add(table_mesh) # setup camera camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0) self.camera_pose = np.array([ [1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, self.dist], [0, 0, 0, 1] ]) nc = pyrender.Node(camera=camera, matrix=self.camera_pose) self.scene.add_node(nc) # get camera projection matrix self.proj_matrix = camera.get_projection_matrix(self.img_w, self.img_h) # add light light = pyrender.SpotLight(color=np.ones(3), intensity=1.0, innerConeAngle=np.pi/16.0, outerConeAngle=np.pi/6.0) light_pose = np.array([ [1, 0, 0, 0.2], [0, 1, 0, 0.2], [0, 0, 1, 1], [0, 0, 0, 1] ]) self.scene.add(light, pose=light_pose) # load mesh mesh_file = os.path.join(self.mesh_path, self.mesh) fuze_trimesh = trimesh.load(mesh_file) mesh_file = pyrender.Mesh.from_trimesh(fuze_trimesh) self.scene.add(mesh_file, pose=self.obj_pose)
def __init__(self, mesh): import pyrender # rotate 180 around x because the Z dir of the reference grid is down T = np.eye(4) T[0:3, 0:3] = rot_x(np.pi) mesh.apply_transform(T) # Load the trimesh and put it in a scene mesh = pyrender.Mesh.from_trimesh(mesh) scene = pyrender.Scene(bg_color=np.array([0, 0, 0, 0])) scene.add(mesh) # add temp cam self.camera = pyrender.IntrinsicsCamera(K[0, 0], K[1, 1], K[0, 2], K[1, 2], zfar=10000, name="cam") light_pose = np.array([ [1.0, 0, 0, 0.0], [0, 1.0, 0.0, 10.0], [0.0, 0, 1, 100.0], [0.0, 0.0, 0.0, 1.0], ]) self.cam_node = scene.add(self.camera, pose=light_pose) # Set up the light -- a single spot light in z+ light = pyrender.SpotLight(color=255 * np.ones(3), intensity=3000.0, innerConeAngle=np.pi / 16.0) scene.add(light, pose=light_pose) self.scene = scene self.r = pyrender.OffscreenRenderer(width, height) # add the A flag for the masking self.flag = pyrender.constants.RenderFlags.RGBA
def process_mesh(self): mesh_path = os.path.join(self.folder_name, self.name + '.ply') print('Attempting to load mesh: %s ...' % self.name) if not os.path.exists(mesh_path): print("ERROR: mesh file does not exist: %s" % mesh_path) # Load the mesh and put it in a scene #input_trimesh = trimesh.load('./example_data/pointclouds/teapot_normal_dense.obj') input_trimesh = trimesh.load_mesh(mesh_path) trimesh_scene = input_trimesh.scene() input_trimesh.vertices -= trimesh_scene.centroid volume = input_trimesh.bounding_sphere.volume radius = np.power((3*volume/(4*np.pi)),1/3) #input_trimesh.vertices *= (1/(4*radius)) input_trimesh.apply_scale(1/(1.2*input_trimesh.scale)) print("Mesh has %d vertices" % input_trimesh.vertices.shape[0] ) #scene = fuze_trimesh.scene() self.mesh = pyrender.Mesh.from_trimesh(input_trimesh) self.scene = pyrender.Scene(ambient_light=(0.5, 0.5, 0.5)) #scene = pyrender.Scene.from_trimesh_scene(fuze_trimesh) self.scene.add(self.mesh) # Set up the camera -- z-axis away from the scene, x-axis right, y-axis up # Add lights light = pyrender.SpotLight(color=np.ones(3), intensity=24.0, innerConeAngle=np.pi / 16.0) light_node = self.scene.add(light, pose= lookAt(np.array([2,2,2]), [0,0,0])) light_node = self.scene.add(light, pose= lookAt(np.array([2,6,3]), [0,0,0])) light_node = self.scene.add(light, pose= lookAt(np.array([2,-1,-3]), [0,0,0])) light_node = self.scene.add(light, pose= lookAt(np.array([-4,4,-3]), [0,0,0])) light_node = self.scene.add(light, pose= lookAt(np.array([-2,-2,-3]), [0,0,0]))
# Set up the camera -- z-axis away from the scene, x-axis right, y-axis up camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0) s = np.sqrt(2) / 2 camera_pose = np.array([ [0.0, -s, s, 0.3], [1.0, 0.0, 0.0, 0.0], [0.0, s, s, 0.35], [0.0, 0.0, 0.0, 1.0], ]) scene.add(camera, pose=camera_pose) # Set up the light -- a single spot light in the same spot as the camera light = pyrender.SpotLight(color=np.ones(3), intensity=3.0, innerConeAngle=np.pi / 16.0) scene.add(light, pose=camera_pose) # Render the scene r = pyrender.OffscreenRenderer(640, 480) color, depth = r.render(scene) # Show the images import matplotlib.pyplot as plt plt.figure() plt.subplot(1, 2, 1) plt.axis('off') plt.imshow(color) plt.subplot(1, 2, 2)
bin_pose[:3, :3] = bin_R bin_pose[:3, 3] = bin_t bin_render_mesh = pyrender.Mesh.from_trimesh(bin_trimesh) scene.add(bin_render_mesh, pose=bin_pose, name=binId) for key in obj_keys: obj_t, obj_q = pybullet.getBasePositionAndOrientation(key, physicsClient) obj_R = Rotation.from_quat(obj_q).as_matrix() obj_pose = np.eye(4) obj_pose[:3, :3] = obj_R obj_pose[:3, 3] = obj_t obj_render_mesh = pyrender.Mesh.from_trimesh(obj_trimesh) scene.add(obj_render_mesh, pose=obj_pose, name=str(key)) # Add light light = pyrender.SpotLight(color=np.ones(3), intensity=4.0) light_pose = np.eye(4) light_pose[2, 3] = 1.8 scene.add(light, pose=light_pose) # Add camera width = 2048 height = 1536 fx = 3700 fy = 3700 cx = width / 2 cy = height / 2 camera = pyrender.IntrinsicsCamera(fx, fy, cx, cy) camera_pose = np.eye(4) camera_pose[2, 3] = 1.6 scene.add(camera, pose=camera_pose)
def render_mesh_pc_bed_pyrender_everything_synth(self, smpl_verts, smpl_faces, camera_point, bedangle, RESULTS_DICT, smpl_verts_gt = None, pmat = None, smpl_render_points = False, markers = None, dropout_variance=None, tf_corners = None, save_name = 'test_synth'): pmat *= 0.75 pmat[pmat>0] += 10 viz_popup = False #print np.min(smpl_verts[:, 0]) #print np.min(smpl_verts[:, 1]) shift_estimate_sideways = np.min([-0.15, np.min(smpl_verts[:, 1])]) #print shift_estimate_sideways shift_estimate_sideways = 0.8 - shift_estimate_sideways top_smpl_vert = np.max(smpl_verts[:, 0]) extend_top_bottom = np.max([np.max(smpl_verts[:, 0]), 64*.0286]) - 64*.0286 print extend_top_bottom, 'extend top bot' shift_both_amount = np.max([0.9, np.max(smpl_verts[:, 1])]) #if smpl is bigger than 0.9 shift less shift_both_amount = 1.5 - shift_both_amount + (0.15 + np.min([-0.15, np.min(smpl_verts[:, 1])])) smpl_verts_quad = np.concatenate((smpl_verts, np.ones((smpl_verts.shape[0], 1))), axis = 1) smpl_verts_quad = np.swapaxes(smpl_verts_quad, 0, 1) smpl_verts_quad_gt = np.concatenate((smpl_verts_gt, np.ones((smpl_verts_gt.shape[0], 1))), axis = 1) smpl_verts_quad_gt = np.swapaxes(smpl_verts_quad_gt, 0, 1) #print smpl_verts_quad.shape shift_ground_truth = 1.3 transform_A = np.identity(4) transform_A[1, 3] = shift_both_amount transform_B = np.identity(4) transform_B[1, 3] = shift_estimate_sideways + shift_both_amount#4.0 #move things over smpl_verts_B = np.swapaxes(np.matmul(transform_B, smpl_verts_quad), 0, 1)[:, 0:3] transform_C = np.identity(4) transform_C[1, 3] = shift_estimate_sideways + shift_both_amount+shift_ground_truth #move things over smpl_verts_C = np.swapaxes(np.matmul(transform_C, smpl_verts_quad_gt), 0, 1)[:, 0:3] from matplotlib import cm human_mesh_vtx_all, human_mesh_face_all = self.get_human_mesh_parts(smpl_verts_B, smpl_faces, segment_limbs=False) #GET MESH WITH PMAT tm_curr = trimesh.base.Trimesh(vertices=np.array(human_mesh_vtx_all[0]), faces = np.array(human_mesh_face_all[0])) tm_list = [tm_curr] original_mesh = [tm_curr] mesh_list = [] mesh_list.append(pyrender.Mesh.from_trimesh(tm_list[0], material = self.human_mat, smooth=True))#wireframe = False)) #this is for the main human human_mesh_vtx_all_gt, human_mesh_face_all_gt = self.get_human_mesh_parts(smpl_verts_C, smpl_faces, segment_limbs=False) #GET MESH GT WITH PMAT tm_curr_gt = trimesh.base.Trimesh(vertices=np.array(human_mesh_vtx_all_gt[0]), faces = np.array(human_mesh_face_all_gt[0])) tm_list_gt = [tm_curr_gt] original_mesh_gt = [tm_curr_gt] mesh_list_gt = [] mesh_list_gt.append(pyrender.Mesh.from_trimesh(tm_list_gt[0], material = self.human_mat_gt, smooth=True))#wireframe = False)) #this is for the main human fig = plt.figure() if self.render == True: artag_meshes = [] artag_tm = trimesh.base.Trimesh(vertices=self.artag_r + [0.0, shift_estimate_sideways + shift_both_amount, 0.0], faces=self.artag_f, face_colors = self.artag_facecolors) artag_meshes.append(pyrender.Mesh.from_trimesh(artag_tm, smooth = False)) artag_meshes_gt = [] artag_tm_gt = trimesh.base.Trimesh(vertices=self.artag_r + [0.0, shift_estimate_sideways + shift_both_amount+shift_ground_truth, 0.0], faces=self.artag_f, face_colors = self.artag_facecolors_gt) artag_meshes_gt.append(pyrender.Mesh.from_trimesh(artag_tm_gt, smooth = False)) if pmat is not None: pmat_verts, pmat_faces, pmat_facecolors = self.get_3D_pmat_markers(pmat, bedangle) pmat_verts = np.array(pmat_verts) pmat_verts = np.concatenate((np.swapaxes(pmat_verts, 0, 1), np.ones((1, pmat_verts.shape[0]))), axis = 0) pmat_verts = np.swapaxes(np.matmul(transform_A, pmat_verts), 0, 1)[:, 0:3] pmat_tm = trimesh.base.Trimesh(vertices=pmat_verts, faces=pmat_faces, face_colors = pmat_facecolors) pmat_mesh = pyrender.Mesh.from_trimesh(pmat_tm, smooth = False) pmat_verts2, _, pmat_facecolors2 = self.get_3D_pmat_markers(pmat, bedangle, solidcolor = True) pmat_verts2 = np.array(pmat_verts2) pmat_verts2 = np.concatenate((np.swapaxes(pmat_verts2, 0, 1), np.ones((1, pmat_verts2.shape[0]))), axis = 0) pmat_verts2 = np.swapaxes(np.matmul(transform_B, pmat_verts2), 0, 1)[:, 0:3] pmat_tm2 = trimesh.base.Trimesh(vertices=pmat_verts2, faces=pmat_faces, face_colors = pmat_facecolors2) pmat_mesh2 = pyrender.Mesh.from_trimesh(pmat_tm2, smooth = False) else: pmat_mesh = None pmat_mesh2 = None #print "Viewing" if self.first_pass == True: for mesh_part in mesh_list: self.scene.add(mesh_part) for mesh_part_gt in mesh_list_gt: self.scene.add(mesh_part_gt) if pmat_mesh is not None: self.scene.add(pmat_mesh) if pmat_mesh2 is not None: self.scene.add(pmat_mesh2) for artag_mesh in artag_meshes: if artag_mesh is not None: self.scene.add(artag_mesh) for artag_mesh_gt in artag_meshes_gt: if artag_mesh_gt is not None: self.scene.add(artag_mesh_gt) lighting_intensity = 20. #self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True, lighting_intensity=lighting_intensity, # point_size=2, run_in_thread=True, viewport_size=(1200, 1200)) self.first_pass = False self.node_list = [] for mesh_part in mesh_list: for node in self.scene.get_nodes(obj=mesh_part): self.node_list.append(node) self.node_list_gt = [] for mesh_part_gt in mesh_list_gt: for node in self.scene.get_nodes(obj=mesh_part_gt): self.node_list_gt.append(node) self.artag_nodes = [] for artag_mesh in artag_meshes: if artag_mesh is not None: for node in self.scene.get_nodes(obj=artag_mesh): self.artag_nodes.append(node) self.artag_nodes_gt = [] for artag_mesh_gt in artag_meshes_gt: if artag_mesh_gt is not None: for node in self.scene.get_nodes(obj=artag_mesh_gt): self.artag_nodes_gt.append(node) if pmat_mesh is not None: for node in self.scene.get_nodes(obj=pmat_mesh): self.pmat_node = node if pmat_mesh2 is not None: for node in self.scene.get_nodes(obj=pmat_mesh2): self.pmat_node2 = node camera_pose = np.eye(4) # camera_pose[0,0] = -1.0 # camera_pose[1,1] = -1.0 camera_pose[0, 0] = np.cos(np.pi/2) camera_pose[0, 1] = np.sin(np.pi/2) camera_pose[1, 0] = -np.sin(np.pi/2) camera_pose[1, 1] = np.cos(np.pi/2) rot_udpim = np.eye(4) rot_y = 180*np.pi/180. rot_udpim[1,1] = np.cos(rot_y) rot_udpim[2,2] = np.cos(rot_y) rot_udpim[1,2] = np.sin(rot_y) rot_udpim[2,1] = -np.sin(rot_y) camera_pose = np.matmul(rot_udpim, camera_pose) camera_pose[0, 3] = 64*0.0286/2 # -1.0 camera_pose[1, 3] = 1.2 + 0.8 camera_pose[2, 3] = -1.0 if viz_popup == True: self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True, lighting_intensity=10., point_size=5, run_in_thread=True, viewport_size=(1000, 1000)) #camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0) magnify =(64*.0286) camera = pyrender.OrthographicCamera(xmag=magnify, ymag = magnify) self.scene.add(camera, pose=camera_pose) light = pyrender.SpotLight(color=np.ones(3), intensity=250.0, innerConeAngle=np.pi / 10.0, outerConeAngle=np.pi / 2.0) light_pose = np.copy(camera_pose) # light_pose[1, 3] = 2.0 light_pose[0, 3] = 0.8 light_pose[1, 3] = -0.5 light_pose[2, 3] = -2.5 light_pose2 = np.copy(camera_pose) light_pose2[0, 3] = 2.5 light_pose2[1, 3] = 1.0 light_pose2[2, 3] = -5.0 light_pose3 = np.copy(camera_pose) light_pose3[0, 3] = 1.0 light_pose3[1, 3] = 5.0 light_pose3[2, 3] = -4.0 #light_pose2[0, 3] = 1.0 #light_pose2[1, 3] = 2.0 #across #light_pose2[2, 3] = -1.5 # light_pose[1, ] self.scene.add(light, pose=light_pose) self.scene.add(light, pose=light_pose2) self.scene.add(light, pose=light_pose3) else: if viz_popup == True: self.viewer.render_lock.acquire() #reset the human mesh for idx in range(len(mesh_list)): self.scene.remove_node(self.node_list[idx]) self.scene.add(mesh_list[idx]) for node in self.scene.get_nodes(obj=mesh_list[idx]): self.node_list[idx] = node #reset the human mesh for idx in range(len(mesh_list_gt)): self.scene.remove_node(self.node_list_gt[idx]) self.scene.add(mesh_list_gt[idx]) for node in self.scene.get_nodes(obj=mesh_list_gt[idx]): self.node_list_gt[idx] = node #reset the artag meshes for artag_node in self.artag_nodes: self.scene.remove_node(artag_node) for artag_mesh in artag_meshes: if artag_mesh is not None: self.scene.add(artag_mesh) self.artag_nodes = [] for artag_mesh in artag_meshes: if artag_mesh is not None: for node in self.scene.get_nodes(obj=artag_mesh): self.artag_nodes.append(node) #reset the artag meshes for artag_node_gt in self.artag_nodes_gt: self.scene.remove_node(artag_node_gt) for artag_mesh_gt in artag_meshes_gt: if artag_mesh_gt is not None: self.scene.add(artag_mesh_gt) self.artag_nodes_gt = [] for artag_mesh_gt in artag_meshes_gt: if artag_mesh_gt is not None: for node in self.scene.get_nodes(obj=artag_mesh_gt): self.artag_nodes_gt.append(node) #reset the pmat mesh if pmat_mesh is not None: self.scene.remove_node(self.pmat_node) self.scene.add(pmat_mesh) for node in self.scene.get_nodes(obj=pmat_mesh): self.pmat_node = node #reset the pmat mesh if pmat_mesh2 is not None: self.scene.remove_node(self.pmat_node2) self.scene.add(pmat_mesh2) for node in self.scene.get_nodes(obj=pmat_mesh2): self.pmat_node2 = node #print self.scene.get_nodes() if viz_popup == True: self.viewer.render_lock.release() #time.sleep(100) if viz_popup == False: r = pyrender.OffscreenRenderer(880, 880) # r.render(self.scene) color_render, depth = r.render(self.scene) # plt.subplot(1, 2, 1) plt.axis('off') #im_to_show = np.concatenate((color_render, color_im), axis = 1) im_to_show = np.copy(color_render) im_to_show = im_to_show[130-int(extend_top_bottom*300):750+int(extend_top_bottom*300), :, :] #plt.imshow(color) plt.imshow(im_to_show) # plt.subplot(1, 2, 2) # plt.axis('off') # plt.imshow(depth, cmap=plt.cm.gray_r) >> > plt.show() fig.set_size_inches(15., 10.) fig.tight_layout() #save_name = 'f_hbh_'+'{:04}'.format(self.pic_num) print "saving!" fig.savefig('/media/henry/multimodal_data_2/CVPR2020_study/'+save_name+'_v2.png', dpi=300) self.pic_num += 1 #plt.show() #if self.pic_num == 20: # print "DONE" # time.sleep(1000000) #print "got here" #print X.shape return RESULTS_DICT
def vis(model_id, mesh_or_scene): """ based on https://colab.research.google.com/drive/1Z71mHIc-Sqval92nK290vAsHZRUkCjUx setup: ```bash sudo apt update sudo wget https://github.com/mmatl/travis_debs/raw/master/xenial/mesa_18.3.3-0.deb sudo dpkg -i ./mesa_18.3.3-0.deb || true sudo apt install -f git clone https://github.com/mmatl/pyopengl.git pip install ./pyopengl ``` """ # scene = pyrender.Scene() if isinstance(mesh_or_scene, trimesh.Scene): # for mesh in mesh_or_scene.geometry.values(): # scene.add(pyrender.Mesh.from_trimesh(mesh)) # tm_scene = mesh_or_scene meshes = mesh_or_scene.geometry.values() else: # scene.add(pyrender.Mesh.from_trimesh(mesh_or_scene)) # tm_scene = mesh_or_scene.scene() meshes = [mesh_or_scene] camera_transform = np.array([ [ 0.58991882, -0.3372407, 0.73366511, 0.86073076, ], [ 0.04221561, 0.92024442, 0.38906047, 0.45644301, ], [ -0.80635825, -0.19854197, 0.55710632, 0.65359322, ], [ 0.0, 0.0, 0.0, 1.0, ], ]) trimesh.scene.cameras.Camera(fov=(45.0, 45.0)) light = trimesh.scene.lighting.SpotLight(color=0.5 * np.ones(3), intensity=0.0, innerConeAngle=np.pi / 16.0) tm_scene = trimesh.scene.Scene(meshes, lights=[light]) for name in [light.name, tm_scene.camera.name]: tm_scene.graph[name] = camera_transform # tm_scene = trimesh.scene.Scene(meshes) # views.set_scene_view(tm_scene, resolution, **view_fn(model_id)) tm_scene.camera.resolution = resolution tm_scene.camera_transform = camera_transform tm_scene.show() # dist = np.linalg.norm(tm_scene.camera_transform[:3, 3]) camera = pyrender.PerspectiveCamera(yfov=np.pi / 180 * tm_scene.camera.fov[1], # znear=dist-0.5, zfar=dist+0.5 ) scene = pyrender.Scene.from_trimesh_scene(tm_scene) scene.add(camera, pose=camera_transform) light = pyrender.SpotLight(color=np.ones(3), intensity=10.0, innerConeAngle=np.pi / 16.0) scene.add(light, pose=camera_transform) # viewer = pyrender.Viewer(scene, use_direct_lighting=True) r = pyrender.OffscreenRenderer(*resolution) color, depth = r.render(scene) # Show the images plt.figure() plt.subplot(1, 2, 1) plt.axis("off") plt.imshow(color) plt.subplot(1, 2, 2) plt.axis("off") plt.imshow(depth, cmap=plt.cm.gray_r) # pylint: disable=no-member plt.show()
def render_mesh_pc_bed_pyrender_everything(self, smpl_verts, smpl_faces, camera_point, bedangle, RESULTS_DICT, pc=None, pmat=None, smpl_render_points=False, markers=None, dropout_variance=None, color_im=None, tf_corners=None, current_pose_type_ct=None, participant=None): #print np.min(smpl_verts[:, 0]) #print np.min(smpl_verts[:, 1]) shift_estimate_sideways = np.min([-0.15, np.min(smpl_verts[:, 1])]) #print shift_estimate_sideways shift_estimate_sideways = 0.8 - shift_estimate_sideways top_smpl_vert = np.max(smpl_verts[:, 0]) extend_top_bottom = np.max([np.max(smpl_verts[:, 0]), 64 * .0286 ]) - 64 * .0286 print extend_top_bottom, 'extend top bot' shift_both_amount = np.max([0.9, np.max(smpl_verts[:, 1]) ]) #if smpl is bigger than 0.9 shift less shift_both_amount = 1.5 - shift_both_amount + ( 0.15 + np.min([-0.15, np.min(smpl_verts[:, 1])])) #print np.max(smpl_verts[:, 1]), 'max smpl' #shift_both_amount = 0.6 #smpl_verts[:, 2] += 0.5 #pc[:, 2] += 0.5 pc[:, 0] = pc[:, 0] # - 0.17 - 0.036608 pc[:, 1] = pc[:, 1] # + 0.09 #adjust the point cloud #segment_limbs = True #if pmat is not None: # if np.sum(pmat) < 5000: # smpl_verts = smpl_verts * 0.001 smpl_verts_quad = np.concatenate( (smpl_verts, np.ones((smpl_verts.shape[0], 1))), axis=1) smpl_verts_quad = np.swapaxes(smpl_verts_quad, 0, 1) #print smpl_verts_quad.shape transform_A = np.identity(4) transform_A[1, 3] = shift_both_amount transform_B = np.identity(4) transform_B[ 1, 3] = shift_estimate_sideways + shift_both_amount #4.0 #move things over smpl_verts_B = np.swapaxes(np.matmul(transform_B, smpl_verts_quad), 0, 1)[:, 0:3] transform_C = np.identity(4) transform_C[1, 3] = 2.0 #2.0 #move things over smpl_verts_C = np.swapaxes(np.matmul(transform_C, smpl_verts_quad), 0, 1)[:, 0:3] from matplotlib import cm human_mesh_vtx_all, human_mesh_face_all = self.get_human_mesh_parts( smpl_verts_B, smpl_faces, segment_limbs=False) #GET MESH WITH PMAT tm_curr = trimesh.base.Trimesh(vertices=np.array( human_mesh_vtx_all[0]), faces=np.array(human_mesh_face_all[0])) tm_list = [tm_curr] original_mesh = [tm_curr] mesh_list = [] mesh_list.append( pyrender.Mesh.from_trimesh(tm_list[0], material=self.human_mat, smooth=False) ) #wireframe = False)) #this is for the main human print np.shape(color_im) print tf_corners top_idx = float(tf_corners[0, 1]) bot_idx = float(tf_corners[2, 1]) perc_total = (bot_idx - top_idx) / 880. print perc_total fig = plt.figure() if self.render == True: #print m.r #print artag_r #create mini meshes for AR tags artag_meshes = [] if markers is not None: for marker in markers: if markers[2] is None: artag_meshes.append(None) elif marker is None: artag_meshes.append(None) else: #print marker - markers[2] if marker is markers[2]: print "is markers 2", marker #artag_tm = trimesh.base.Trimesh(vertices=self.artag_r, faces=self.artag_f, face_colors = self.artag_facecolors_root) #artag_meshes.append(pyrender.Mesh.from_trimesh(artag_tm, smooth = False)) else: artag_tm = trimesh.base.Trimesh( vertices=self.artag_r + [ 0.0, shift_estimate_sideways + shift_both_amount, 0.0 ], faces=self.artag_f, face_colors=self.artag_facecolors) artag_meshes.append( pyrender.Mesh.from_trimesh(artag_tm, smooth=False)) if pmat is not None: pmat_verts, pmat_faces, pmat_facecolors = self.get_3D_pmat_markers( pmat, bedangle) pmat_verts = np.array(pmat_verts) pmat_verts = np.concatenate((np.swapaxes( pmat_verts, 0, 1), np.ones((1, pmat_verts.shape[0]))), axis=0) pmat_verts = np.swapaxes(np.matmul(transform_A, pmat_verts), 0, 1)[:, 0:3] pmat_tm = trimesh.base.Trimesh(vertices=pmat_verts, faces=pmat_faces, face_colors=pmat_facecolors) pmat_mesh = pyrender.Mesh.from_trimesh(pmat_tm, smooth=False) pmat_verts2, _, pmat_facecolors2 = self.get_3D_pmat_markers( pmat, bedangle, solidcolor=True) pmat_verts2 = np.array(pmat_verts2) pmat_verts2 = np.concatenate((np.swapaxes( pmat_verts2, 0, 1), np.ones((1, pmat_verts2.shape[0]))), axis=0) pmat_verts2 = np.swapaxes(np.matmul(transform_B, pmat_verts2), 0, 1)[:, 0:3] pmat_tm2 = trimesh.base.Trimesh(vertices=pmat_verts2, faces=pmat_faces, face_colors=pmat_facecolors2) pmat_mesh2 = pyrender.Mesh.from_trimesh(pmat_tm2, smooth=False) else: pmat_mesh = None pmat_mesh2 = None #print "Viewing" if self.first_pass == True: for mesh_part in mesh_list: self.scene.add(mesh_part) if pmat_mesh is not None: self.scene.add(pmat_mesh) if pmat_mesh2 is not None: self.scene.add(pmat_mesh2) for artag_mesh in artag_meshes: if artag_mesh is not None: self.scene.add(artag_mesh) lighting_intensity = 20. #self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True, lighting_intensity=lighting_intensity, # point_size=2, run_in_thread=True, viewport_size=(1200, 1200)) self.first_pass = False self.node_list = [] for mesh_part in mesh_list: for node in self.scene.get_nodes(obj=mesh_part): self.node_list.append(node) self.artag_nodes = [] for artag_mesh in artag_meshes: if artag_mesh is not None: for node in self.scene.get_nodes(obj=artag_mesh): self.artag_nodes.append(node) if pmat_mesh is not None: for node in self.scene.get_nodes(obj=pmat_mesh): self.pmat_node = node if pmat_mesh2 is not None: for node in self.scene.get_nodes(obj=pmat_mesh2): self.pmat_node2 = node camera_pose = np.eye(4) # camera_pose[0,0] = -1.0 # camera_pose[1,1] = -1.0 camera_pose[0, 0] = np.cos(np.pi / 2) camera_pose[0, 1] = np.sin(np.pi / 2) camera_pose[1, 0] = -np.sin(np.pi / 2) camera_pose[1, 1] = np.cos(np.pi / 2) rot_udpim = np.eye(4) rot_y = 180 * np.pi / 180. rot_udpim[1, 1] = np.cos(rot_y) rot_udpim[2, 2] = np.cos(rot_y) rot_udpim[1, 2] = np.sin(rot_y) rot_udpim[2, 1] = -np.sin(rot_y) camera_pose = np.matmul(rot_udpim, camera_pose) camera_pose[0, 3] = 64 * 0.0286 / 2 # -1.0 camera_pose[1, 3] = 1.2 camera_pose[2, 3] = -1.0 # self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True, # lighting_intensity=10., # point_size=5, run_in_thread=True, viewport_size=(1000, 1000)) # camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0) magnify = (64 * .0286) * 0.5 / perc_total camera = pyrender.OrthographicCamera(xmag=magnify, ymag=magnify) self.scene.add(camera, pose=camera_pose) light = pyrender.SpotLight(color=np.ones(3), intensity=200.0, innerConeAngle=np.pi / 10.0, outerConeAngle=np.pi / 2.0) light_pose = np.copy(camera_pose) # light_pose[1, 3] = 2.0 light_pose[0, 3] = 0.8 light_pose[1, 3] = -0.5 light_pose[2, 3] = -1.5 light_pose2 = np.copy(camera_pose) light_pose2[0, 3] = 2.5 light_pose2[1, 3] = 1.0 light_pose2[2, 3] = -4.0 # light_pose[1, ] self.scene.add(light, pose=light_pose) self.scene.add(light, pose=light_pose2) else: #self.viewer.render_lock.acquire() #reset the human mesh for idx in range(len(mesh_list)): self.scene.remove_node(self.node_list[idx]) self.scene.add(mesh_list[idx]) for node in self.scene.get_nodes(obj=mesh_list[idx]): self.node_list[idx] = node #reset the artag meshes for artag_node in self.artag_nodes: self.scene.remove_node(artag_node) for artag_mesh in artag_meshes: if artag_mesh is not None: self.scene.add(artag_mesh) self.artag_nodes = [] for artag_mesh in artag_meshes: if artag_mesh is not None: for node in self.scene.get_nodes(obj=artag_mesh): self.artag_nodes.append(node) #reset the pmat mesh if pmat_mesh is not None: self.scene.remove_node(self.pmat_node) self.scene.add(pmat_mesh) for node in self.scene.get_nodes(obj=pmat_mesh): self.pmat_node = node #reset the pmat mesh if pmat_mesh2 is not None: self.scene.remove_node(self.pmat_node2) self.scene.add(pmat_mesh2) for node in self.scene.get_nodes(obj=pmat_mesh2): self.pmat_node2 = node #print self.scene.get_nodes() #self.viewer.render_lock.release() #time.sleep(100) r = pyrender.OffscreenRenderer(880, 880) # r.render(self.scene) color_render, depth = r.render(self.scene) # plt.subplot(1, 2, 1) plt.axis('off') if 880. - bot_idx > top_idx: print 'shift im down by', 880. - bot_idx - top_idx downshift = int((880. - bot_idx) / 2 - top_idx / 2 + 0.5) color_im[downshift:880] = color_im[0:880 - downshift] elif top_idx > (880. - bot_idx): print 'shift im up by', top_idx - (880. - bot_idx) upshift = int(top_idx / 2 - (880. - bot_idx) / 2 + 0.5) color_im[0:880 - upshift] = color_im[upshift:880] print tf_corners print np.shape(color_render), np.shape(color_im) color_im = np.concatenate( (color_im[:, :, 2:3], color_im[:, :, 1:2], color_im[:, :, 0:1]), axis=2) color_im = color_im[:, int(tf_corners[0, 0] - 10):int(tf_corners[1, 0] + 10), :] im_to_show = np.concatenate((color_render, color_im), axis=1) im_to_show = im_to_show[130 - int(extend_top_bottom * 300):750 + int(extend_top_bottom * 300), :, :] #plt.imshow(color) plt.imshow(im_to_show) # plt.subplot(1, 2, 2) # plt.axis('off') # plt.imshow(depth, cmap=plt.cm.gray_r) >> > plt.show() fig.set_size_inches(15., 10.) fig.tight_layout() #save_name = 'f_hbh_'+'{:04}'.format(self.pic_num) save_name = participant + '_' + current_pose_type_ct fig.savefig('/media/henry/multimodal_data_2/CVPR2020_study/' + participant + '/estimated_poses/' + save_name + '.png', dpi=300) #fig.savefig('/media/henry/multimodal_data_2/CVPR2020_study/TEST.png', dpi=300) #plt.savefig('test2png.png', dpi=100) self.pic_num += 1 #plt.show() #if self.pic_num == 20: # print "DONE" # time.sleep(1000000) #print "got here" #print X.shape return RESULTS_DICT
import trimesh import pyrender import numpy as np import math import collada red = [1.0, 0.0, 0.0] green = [0.0, 1.0, 0.0] blue = [0.0, 0.0, 1.0] black = [0.0, 0.0, 0.0] gray = [0.5, 0.5, 0.5] white = [1.0, 1.0, 1.0] point_light = pyrender.PointLight(color=[1.0, 1.0, 1.0], intensity=2.0) spot_light = pyrender.SpotLight(color=[1.0, 1.0, 1.0], intensity=2.0, innerConeAngle=0.05, outerConeAngle=0.5) directional_light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=2.0) perspective_camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.414) ortho_camera = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0) def display(mesh, light=None, camera=None): scene = pyrender.Scene(ambient_light=white) scene.add(mesh) if light: scene.add(light) if camera:
def _init_light(self): """ Set up light """ # Load light from config file light = self.conf.sensor.lights origin = np.array(light.origin) xyz = [] if light.polar: # Apply polar coordinates thetas = light.xrtheta.thetas rs = light.xrtheta.rs xs = light.xrtheta.xs for i in range(len(thetas)): theta = np.pi / 180 * thetas[i] xyz.append( [xs[i], rs[i] * np.cos(theta), rs[i] * np.sin(theta)]) else: # Apply cartesian coordinates xyz = np.array(light.xyz.coords) colors = np.array(light.colors) intensities = light.intensities # Save light nodes self.light_nodes = [] self.light_poses0 = [] for i in range(len(colors)): if not self.spot_light_enabled: # create pyrender.PointLight color = colors[i] light_pose_0 = euler2matrix(angles=[0, 0, 0], translation=xyz[i] + origin) light = pyrender.PointLight(color=color, intensity=intensities[i]) elif self.spot_light_enabled: # create pyrender.SpotLight color = colors[i] theta = np.pi / 180 * (thetas[i] - 90) tuning_angle = -np.pi / 16 light_pose_0 = euler2matrix( xyz="yzx", angles=[0, tuning_angle, theta], translation=xyz[i] + origin, ) light = pyrender.SpotLight( color=color, intensity=intensities[i], innerConeAngle=0, outerConeAngle=np.pi / 3, ) light_node = pyrender.Node(light=light, matrix=light_pose_0) self.scene.add_node(light_node) self.light_nodes.append(light_node) self.light_poses0.append(light_pose_0) self.current_light_nodes.append(light_node) # Add extra light node into scene_depth light_node_depth = pyrender.Node(light=light, matrix=light_pose_0) self.scene_depth.add_node(light_node_depth)
def renderMeshEGL(mesh, cam_viewpoint, number_sampled_points, z_towards_mesh=True): """ Renders a number of points on a mesh from a viewpoint, considering occlusion. Faces not in sight are detected by casting rays from a camera viewpoint. Uses EGL hardware acceleration. Parameters ---------- mesh: trimesh.Mesh Mesh to sample cam_viewpoint: (3,) float Point from which to observe the mesh number_sampled_points: int Number of points to sample on the mesh Returns ---------- points: (n, 3) float Points sampled on the visible portion of the mesh, in the original mesh reference frame points_camera_frame: (n, 3) float Points sampled on the visible portion of the mesh, in the camera reference frame """ import os os.environ["PYOPENGL_PLATFORM"] = "egl" mesh = pyrender.Mesh.from_trimesh(mesh) scene = pyrender.Scene() scene.add(mesh) # Set up the camera and depth of field fov = np.pi / 3.0 height = 480 width = 640 fx = fy = width / 2.0 / np.tan(fov / 2.0) cx = width / 2.0 cy = height / 2.0 camera = pyrender.IntrinsicsCamera(fx=fx, fy=fy, cx=cx, cy=cy) # z axis is the ray from the mesh center to the camera axis center # This inversion is necessary because the camera reference system uses the cinematographic # convention # https://www.scratchapixel.com/lessons/mathematics-physics-for-computer-graphics/lookat-function mesh_centroid = mesh.centroid if z_towards_mesh: camera_z_axis = (cam_viewpoint - mesh_centroid ) / np.linalg.norm(mesh.centroid - cam_viewpoint) else: camera_z_axis = (mesh_centroid - cam_viewpoint ) / np.linalg.norm(mesh.centroid - cam_viewpoint) # Obtain the camera transformation as a 4x4 and add it camera_transform_4x4 = trimesh.geometry.align_vectors( np.array([0, 0, 1], dtype=np.float32), camera_z_axis) camera_transform_4x4[0:3, 3] = cam_viewpoint scene.add(camera, pose=camera_transform_4x4) # Set up the light -- a single spot light in the same spot as the camera light = pyrender.SpotLight(color=np.ones(3), intensity=3.0, innerConeAngle=np.pi / 16.0) scene.add(light, pose=camera_transform_4x4) # Render the scene r = pyrender.OffscreenRenderer(width, height) color, depth = r.render(scene) # Clip off depth values where there's nothing mask = np.where(depth > 0) x = mask[1] y = mask[0] # Compute coordinates given depth and image pixel coordinates points_x = (x.astype(np.float32) - cx) / fx * depth[y, x] points_y = (y.astype(np.float32) - cy) / fy * depth[y, x] points_z = depth[y, x] points_camera_frame = np.vstack((points_x, points_y, points_z)).T # Downsample if necessary if points_camera_frame.shape[0] > number_sampled_points: choice = np.random.choice(points_camera_frame.shape[0], number_sampled_points, replace=False) points_camera_frame = points_camera_frame[choice, :] # Obtain point coordinates in the object-centric ref system points_cam_homogeneous = np.hstack((points_camera_frame, np.ones( (points_camera_frame.shape[0], 1), dtype=np.float32))) points_world_frame_homogeneous = np.transpose( np.matmul(np.linalg.inv(camera_transform_4x4), np.transpose(points_cam_homogeneous))) points_world_frame = points_world_frame_homogeneous[:, 0:3] return points_world_frame, points_camera_frame
def render_pose_data(model_path, num_frames, width=400, height=400, show=False): r = pyrender.OffscreenRenderer(viewport_width=width, viewport_height=height, point_size=1.0) fuze_trimesh = trimesh.load(model_path) camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0) camera_pose = np.array([ [1.0, 0, 0, 0], [0, 1.0, 0.0, 0], [0.0, 0, 1, 1.3], [0.0, 0.0, 0.0, 1.0], ]) if isinstance(fuze_trimesh, trimesh.scene.scene.Scene): scene = pyrender.Scene.from_trimesh_scene(fuze_trimesh) nodes = list(scene.nodes) for i in nodes: scene.remove_node(i) meshParent = pyrender.Node("MsehNode", children=nodes) scene.add_node(meshParent) mesh_vert = np.vstack([i.vertices for i in fuze_trimesh.dump()]) else: scene = pyrender.Scene() meshParent = pyrender.Mesh.from_trimesh(fuze_trimesh) scene.add(meshParent) meshParent = list(scene.nodes)[0] mesh_vert = np.array(fuze_trimesh.vertices) scene.add(camera, pose=camera_pose) light = pyrender.SpotLight(color=np.ones(3), intensity=20.0, innerConeAngle=np.pi / 6.0, outerConeAngle=np.pi / 6.0) scene.add(light, pose=camera_pose) transform = np.eye(4) import time t_s = time.time() poses = [] images = [] depths = [] for i in range(num_frames): x = special_ortho_group.rvs(3) transform[:3, :3] = x scene.set_pose(meshParent, transform) color, depth = r.render(scene) images.append(color) poses.append(np.copy(transform)) depths.append(depth) if show: plt.imshow(color) plt.show() dt = time.time() - t_s print("Rendering Time used {:.4f}s".format(dt)) return images, depths, poses, mesh_vert
def render_trimesh( trimesh_mesh, eye, center, world_up, res=(640, 640), light_intensity=3.0, ambient_intensity=None, **kwargs, ): """Render a shapenet mesh using default settings. Args: trimesh_mesh: trimesh mesh instance, or a list of trimesh meshes (or point clouds). eye: array with shape [3,] containing the XYZ world space position of the camera. center: array with shape [3,] containing a position along the center of the camera's gaze. world_up: np.float32 array with shape [3,] specifying the world's up direction; the output camera will have no tilt with respect to this direction. res: 2-tuple of int, [width, height], resolution (in pixels) of output images. light_intensity: float, light intensity. ambient_intensity: float, ambient light intensity. kwargs: additional flags to pass to pyrender renderer. Returns: color_img: [*res, 3] color image. depth_img: [*res, 1] depth image. world_to_cam: [4, 4] camera to world matrix. projection_matrix: [4, 4] projection matrix, aka cam_to_img matrix. """ if not isinstance(trimesh_mesh, list): trimesh_mesh = [trimesh_mesh] eye = list2npy(eye).astype(np.float32) center = list2npy(center).astype(np.float32) world_up = list2npy(world_up).astype(np.float32) # setup camera pose matrix scene = pyrender.Scene(ambient_light=ambient_intensity * np.ones([3], dtype=float)) for tmesh in trimesh_mesh: if not (isinstance(tmesh, trimesh.Trimesh) or isinstance(tmesh, trimesh.PointCloud)): raise NotImplementedError( "All instances in trimesh_mesh must be either trimesh.Trimesh " f"or trimesh.PointCloud. Instead it is {type(tmesh)}.") if isinstance(tmesh, trimesh.Trimesh): mesh = pyrender.Mesh.from_trimesh(tmesh) elif isinstance(tmesh, trimesh.PointCloud): if tmesh.colors is not None: colors = np.array(tmesh.colors) else: colors = np.ones_like(tmesh.vertices) mesh = pyrender.Mesh.from_points(np.array(tmesh.vertices), colors=colors) scene.add(mesh) # Set up the camera -- z-axis away from the scene, x-axis right, y-axis up camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0) world_to_cam = look_at(eye[None], center[None], world_up[None]) world_to_cam = world_to_cam[0] cam_pose = np.linalg.inv(world_to_cam) scene.add(camera, pose=cam_pose) # Set up the light -- a single spot light in the same spot as the camera light = pyrender.SpotLight( color=np.ones(3, dtype=np.float32), intensity=light_intensity, innerConeAngle=np.pi / 16.0, ) scene.add(light, pose=cam_pose) # Render the scene r = pyrender.OffscreenRenderer(*res, **kwargs) color_img, depth_img = r.render(scene) return ( color_img, depth_img, world_to_cam, camera.get_projection_matrix(*res), )
def render_only_human_gt(self, m): bed1_verts = np.array([ [-1.3, -1.35, -3.0], [-0.9, -0.75, -4.0], [1.3, -1.35, -3.0], [0.9, -0.75, -4.0], [-1.2, 0.05, -4.0], [-1.2, 0.0, -4.0], [1.2, 0.05, -4.0], [1.2, 0.0, -4.0], [-1.3, -1.35, -3.0], [-1.3, -1.45, -3.0], [1.3, -1.35, -3.0], [1.3, -1.45, -3.0], [-1.2, 0.05, -4.0], [-1.0, 1.0, -4.0], [1.2, 0.05, -4.0], [1.0, 1.0, -4.0], ]) bed1_faces = np.array([ [0, 1, 2], [0, 2, 1], [1, 2, 3], [1, 3, 2], [4, 5, 6], [4, 6, 5], [5, 6, 7], [5, 7, 6], [8, 9, 10], [8, 10, 9], [9, 10, 11], [9, 11, 10], [12, 13, 14], [12, 14, 13], [13, 14, 15], [13, 15, 14], ]) bed1_facecolors = [] for i in range(bed1_faces.shape[0]): if 4 <= i < 12: bed1_facecolors.append([0.8, 0.8, 0.2]) else: bed1_facecolors.append([1.0, 1.0, 0.8]) smpl_verts = (m.r - m.J_transformed[0, :]) #smpl_verts = np.concatenate((smpl_verts[:, 1:2] - 1.5, smpl_verts[:, 0:1], -smpl_verts[:, 2:3]), axis = 1) smpl_verts = np.concatenate( (smpl_verts[:, 0:1] - 1.5, smpl_verts[:, 1:2], smpl_verts[:, 2:3]), axis=1) #smpl_verts = np.concatenate((smpl_verts[:, 1:2], smpl_verts[:, 0:1], -smpl_verts[:, 2:3]), axis = 1) smpl_faces = np.array(m.f) #smpl_faces = np.concatenate((smpl_faces[:, 0:1],smpl_faces[:, 2:3],smpl_faces[:, 1:2]), axis = 1) smpl_verts2 = np.concatenate( (smpl_verts[:, 1:2], smpl_verts[:, 2:3] - 2.0, smpl_verts[:, 0:1]), axis=1) smpl_verts3 = np.concatenate( (smpl_verts[:, 1:2], smpl_verts[:, 2:3] - 2.4, smpl_verts[:, 0:1]), axis=1) laying_rot_M = np.array([[1., 0., 0.], [0., 0.866025, -0.5], [0., 0.5, 0.866025]]) laying_rot_M2 = np.array([[1., 0., 0.], [0., 0.34202, -0.93969], [0., 0.93969, 0.34202]]) for i in range(smpl_verts2.shape[0]): smpl_verts2[i, :] = np.matmul(laying_rot_M, smpl_verts2[i, :]) smpl_verts3[i, :] = np.matmul(laying_rot_M2, smpl_verts3[i, :]) #break #smpl_verts2 = np.concatenate((-smpl_verts[:, 2:3] + 1.5, smpl_verts[:, 1:2], smpl_verts[:, 0:1]), axis = 1) #smpl_verts3 = np.concatenate((smpl_verts[:, 2:3] - 1.5, smpl_verts[:, 1:2], -smpl_verts[:, 0:1]), axis = 1) #print smpl_verts2.shape #tm = trimesh.base.Trimesh(vertices=smpl_verts, faces=smpl_faces) #mesh = pyrender.Mesh.from_trimesh(tm, material=self.human_mat_for_study, wireframe=False) tm2 = trimesh.base.Trimesh(vertices=smpl_verts2, faces=smpl_faces) mesh2 = pyrender.Mesh.from_trimesh(tm2, material=self.human_mat_for_study, wireframe=False) tm3 = trimesh.base.Trimesh(vertices=smpl_verts3, faces=smpl_faces) mesh3 = pyrender.Mesh.from_trimesh(tm3, material=self.human_mat_for_study, wireframe=False) tm_bed1 = trimesh.base.Trimesh(vertices=bed1_verts, faces=bed1_faces, face_colors=np.array(bed1_facecolors)) mesh_bed1 = pyrender.Mesh.from_trimesh( tm_bed1, material=self.human_bed_for_study, wireframe=False, smooth=False) fig = plt.figure() #plt.plot(np.arange(0, 400), np.arange(0, 400)*.5 + 800) if self.first_pass == True: self.scene.add(mesh_bed1) #self.scene.add(mesh) self.scene.add(mesh2) self.scene.add(mesh3) self.first_pass = False self.node_list_bed1 = [] for node in self.scene.get_nodes(obj=mesh_bed1): self.node_list_bed1.append(node) #self.node_list = [] #for node in self.scene.get_nodes(obj=mesh): # self.node_list.append(node) self.node_list_2 = [] for node in self.scene.get_nodes(obj=mesh2): self.node_list_2.append(node) self.node_list_3 = [] for node in self.scene.get_nodes(obj=mesh3): self.node_list_3.append(node) camera_pose = np.eye(4) # camera_pose[0,0] = -1.0 # camera_pose[1,1] = -1.0 camera_pose[0, 3] = 0.0 # -1.0 camera_pose[1, 3] = 0.0 # -1.0 camera_pose[2, 3] = 4.0 # self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True, # lighting_intensity=10., # point_size=5, run_in_thread=True, viewport_size=(1000, 1000)) # camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0) camera = pyrender.OrthographicCamera(xmag=2.0, ymag=2.0) self.scene.add(camera, pose=camera_pose) light = pyrender.SpotLight(color=np.ones(3), intensity=150.0, innerConeAngle=np.pi / 10.0, outerConeAngle=np.pi / 2.0) light_pose = np.copy(camera_pose) #light_pose[1, 3] = 2.0 light_pose[0, 3] = -1.0 light_pose[2, 3] = 5.0 light_pose2 = np.copy(camera_pose) light_pose2[0, 3] = 0.5 light_pose2[1, 3] = 0.5 light_pose2[2, 3] = 5.0 # light_pose[1, ] #self.scene.add(light, pose=light_pose) #self.scene.add(light, pose=light_pose2) else: #self.viewer.render_lock.acquire() # reset the human mesh self.scene.remove_node(self.node_list_bed1[0]) self.scene.add(mesh_bed1) for node in self.scene.get_nodes(obj=mesh_bed1): self.node_list_bed1[0] = node # reset the human mesh #self.scene.remove_node(self.node_list[0]) #self.scene.add(mesh) #for node in self.scene.get_nodes(obj=mesh): # self.node_list[0] = node self.scene.remove_node(self.node_list_2[0]) self.scene.add(mesh2) for node in self.scene.get_nodes(obj=mesh2): self.node_list_2[0] = node self.scene.remove_node(self.node_list_3[0]) self.scene.add(mesh3) for node in self.scene.get_nodes(obj=mesh3): self.node_list_3[0] = node #self.viewer.render_lock.release() r = pyrender.OffscreenRenderer(2000, 2000) # r.render(self.scene) color, depth = r.render(self.scene) # plt.subplot(1, 2, 1) plt.axis('off') plt.imshow(color) # plt.subplot(1, 2, 2) # plt.axis('off') # plt.imshow(depth, cmap=plt.cm.gray_r) >> > plt.show() fig.set_size_inches(15., 15.) fig.tight_layout() save_name = 'f_hbh_' + '{:04}'.format(self.pic_num) fig.savefig('/home/henry/Pictures/CVPR2020_study/' + save_name + '.png', dpi=300) #plt.savefig('test2png.png', dpi=100) self.pic_num += 1 #plt.show() if self.pic_num == 20: print "DONE" time.sleep(1000000) print "got here"