def __init__(self, width, height, camera_distance=0.05, pose_y=0.0, focal_length=None): self.camera = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0, znear=0.05) if focal_length: focal_length = focal_length * height self.camera = pyrender.IntrinsicsCamera(focal_length, focal_length, width / 2, height / 2, 0.05, 5.05) self.width = width self.height = height self.global_tr = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, pose_y], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]) self.camera_pose = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, camera_distance], [0.0, 0.0, 0.0, 1.0]]) self.tri_mesh = self.py_mesh = self.vertices = self.faces = self.render = self.py_scene = None
def render_face_transparent(mesh, background=None): """ mesh location should be normalized :param mesh: :param background: :return: """ mesh.visual.face_colors = np.array([0.5, 0.5, 0.5, 1]) # np.array([0.05, 0.1, 0.2, 1]) mesh = pyrender.Mesh.from_trimesh(mesh, smooth=False) # mesh = pyrender.Mesh.from_trimesh(mesh) scene.add(mesh, pose=np.eye(4)) camera_pose = np.eye(4) # camera_pose[0, 3] = 1 # camera_pose[1, 3] = 1 # camera_pose[2, 3] = -10 # camera_pose[0, 0] = 1 # camera_pose[1, 1] = -1 # camera_pose[2, 2] = -1 # # camera = pyrender.OrthographicCamera(xmag=1, ymag=1, zfar=100) camera_pose[0, 3] = 1 camera_pose[1, 3] = 1 camera_pose[2, 3] = 10 camera_pose[0, 0] = 1 camera_pose[1, 1] = 1 camera_pose[2, 2] = 1 camera = pyrender.OrthographicCamera(xmag=1, ymag=1, zfar=100) scene.add(camera, pose=camera_pose) light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=5.0) scene.add(light, pose=camera_pose) color, depth = r.render(scene, flags=pyrender.RenderFlags.RGBA) scene.clear() # print(color.shape) color = np.array(color) color = color[::-1] if background is not None: bg_rgba = Image.fromarray(np.array(background)).convert('RGBA') color[color[:, :, 0] == 255, 3] = 0 color[color[:, :, 3] == 255, 3] = 166 color = Image.fromarray(color).convert('RGBA') final = Image.alpha_composite(bg_rgba, color) color = np.array(final) # # new_color = np.array(background) # new_color[color != 255] = color[color != 255] # color = new_color return color
def renderVertices(vertices): mesh = pyrender.Mesh.from_points([[v[0], v[1], -1.0] for v in vertices]) cam = pyrender.OrthographicCamera(xmag=ORTHO_CAM[0], ymag=ORTHO_CAM[1]) pos = np.eye(4) scene = pyrender.Scene() scene.add(mesh) scene.add(cam, pose=pos) pyrender.Viewer(scene, VIEWPORT_SIZE, all_wireframe=True, cull_faces=False, use_perspective_cam=False)
def renderMesh(mesh): tm = trimeshFromMesh(mesh) sceneMesh = pyrender.Mesh.from_trimesh(tm, smooth=False) cam = pyrender.OrthographicCamera(xmag=ORTHO_CAM[0], ymag=ORTHO_CAM[1]) pos = np.eye(4) scene = pyrender.Scene() scene.add(sceneMesh) scene.add(cam, pose=pos) pyrender.Viewer(scene, VIEWPORT_SIZE, all_wireframe=True, cull_faces=False, use_perspective_cam=False)
def _get_projection(self, tm_mesh, render_size): tm_scene = tm_mesh.scene() tm_scene = tm_scene.scaled(0.8 / np.max(tm_scene.extents)) scene = pyrender.Scene.from_trimesh_scene(tm_scene) camera = pyrender.OrthographicCamera(xmag=0.5, ymag=0.5) camera_position = tm_mesh.scene().camera_transform scene.add(camera, pose=camera_position) r = pyrender.OffscreenRenderer(render_size, render_size) depth_buffer = r.render(scene, flags=RF.DEPTH_ONLY) r.delete() return depth_buffer
def get_image_array(model_path, camera_pose): mesh = pyrender.Mesh.from_trimesh(trimesh.load(model_path)) camera = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0) scene = pyrender.Scene() scene.add(mesh) scene.add(camera, pose=camera_pose) renderer = pyrender.OffscreenRenderer(resolution, resolution) color, depth = renderer.render(scene) depth_uint8 = (fitted_normalize(depth) * 255).astype(np.uint8) return depth_uint8
def projectMeshCachedDebug(scene, f, R, t, sensorSize, ortho, mag, debug): # In OpenGL, camera points toward -z by default, hence we don't need rFix like in the MATLAB code sensorWidth = sensorSize[0] sensorHeight = sensorSize[1] fovHorizontal = 2 * np.arctan((sensorWidth / 2) / f) fovVertical = 2 * np.arctan((sensorHeight / 2) / f) if ortho: camera = pyrender.OrthographicCamera(xmag=mag, ymag=mag) else: camera = pyrender.PerspectiveCamera(fovVertical) camera_pose = np.eye(4) camera_pose[0:3, 0:3] = R camera_pose[0:3, 3] = t cameraNode = scene.add(camera, pose=camera_pose) scene._ambient_light = np.ones((3, )) r = pyrender.OffscreenRenderer(sensorWidth, sensorHeight) meshProjection, depth = r.render( scene) # TODO: this thing consumes ~14 GB RAM!!! r.delete( ) # this releases that; but it should not require so much RAM in the first place # XYZ cut scaling = 1.0 / f spaceCoordinateSystem = np.eye(3) sensorCoordinateSystem = np.matmul(R, spaceCoordinateSystem) sensorXAxis = sensorCoordinateSystem[:, 0] sensorYAxis = -sensorCoordinateSystem[:, 1] # make camera point toward -z by default, as in OpenGL cameraDirection = -sensorCoordinateSystem[:, 2] # unit vector xyzCut, pts = buildXYZcut(sensorWidth, sensorHeight, t, cameraDirection, scaling, sensorXAxis, sensorYAxis, depth) XYZpc = -1 if debug: XYZpc = o3d.geometry.PointCloud() XYZpc.points = o3d.utility.Vector3dVector(pts) scene.remove_node(cameraNode) return meshProjection, xyzCut, depth, XYZpc
def renderMeshWithCircumcenters(mesh, circumcenters): tm = trimeshFromMesh(mesh) sceneMesh = pyrender.Mesh.from_trimesh(tm, smooth=False) scenePoints = pyrender.Mesh.from_points([[v[0], v[1], -1.0] for v in circumcenters]) cam = pyrender.OrthographicCamera(xmag=ORTHO_CAM[0], ymag=ORTHO_CAM[1]) pos = np.eye(4) scene = pyrender.Scene() scene.add(sceneMesh) scene.add(scenePoints) scene.add(cam, pose=pos) pyrender.Viewer(scene, VIEWPORT_SIZE, all_wireframe=True, cull_faces=False, use_perspective_cam=False, point_size=5)
def render_face_orthographic(mesh, background=None): """ mesh location should be normalized :param mesh: :param background: :return: """ mesh.visual.face_colors = np.array([0.05, 0.1, 0.2, 1]) mesh = pyrender.Mesh.from_trimesh(mesh, smooth=False) # mesh = pyrender.Mesh.from_trimesh(mesh) scene.add(mesh, pose=np.eye(4)) camera_pose = np.eye(4) # camera_pose[0, 3] = 1 # camera_pose[1, 3] = 1 # camera_pose[2, 3] = -10 # camera_pose[0, 0] = 1 # camera_pose[1, 1] = -1 # camera_pose[2, 2] = -1 # # camera = pyrender.OrthographicCamera(xmag=1, ymag=1, zfar=100) camera_pose[0, 3] = 1 camera_pose[1, 3] = 1 camera_pose[2, 3] = 10 camera_pose[0, 0] = 1 camera_pose[1, 1] = 1 camera_pose[2, 2] = 1 camera = pyrender.OrthographicCamera(xmag=1, ymag=1, zfar=100) scene.add(camera, pose=camera_pose) light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=5.0) scene.add(light, pose=camera_pose) color, depth = r.render(scene) scene.clear() # print(color.shape) color = np.array(color) color = color[::-1] if background is not None: new_color = np.array(background) new_color[color != 255] = color[color != 255] color = new_color return color
def renderLight(posmap, init_image=None, is_render=True): tex = np.ones((256, 256, 3)) / 2 mesh = UVmap2Mesh(posmap, tex, is_extra_triangle=False) vertices = mesh['vertices'] triangles = mesh['triangles'] colors = mesh['colors'] / np.max(mesh['colors']) file = 'tmp/light/test.obj' write_obj_with_colors(file, vertices, triangles, colors) obj = trimesh.load(file) # obj.visual.vertex_colors = np.random.uniform(size=obj.vertices.shape) obj.visual.face_colors = np.array([0.05, 0.1, 0.2]) mesh = pyrender.Mesh.from_trimesh(obj, smooth=False) scene.add(mesh, pose=np.eye(4)) camera_pose = np.eye(4) camera_pose[0, 3] = 128 camera_pose[1, 3] = 128 camera_pose[2, 3] = 300 camera = pyrender.OrthographicCamera(xmag=128, ymag=128, zfar=1000) scene.add(camera, pose=camera_pose) light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=8.0) scene.add(light, pose=camera_pose) color, depth = r.render(scene) if is_render: plt.imshow(color) plt.show() if init_image is not None: sum_mask = np.mean(color, axis=-1) fuse_img = color.copy() fuse_img[sum_mask > 128] = init_image[sum_mask > 128] if is_render: plt.imshow(fuse_img) plt.show() scene.clear() return fuse_img scene.clear() return color
def render(obj, cam_pos): # create scene scene = pyrender.Scene() # MESHES obj_trimesh = trimesh.load(obj) obj_trimesh.vertices = scaleVertices(obj_trimesh.vertices, feature_range=(-1, 1)) mesh = pyrender.Mesh.from_trimesh(obj_trimesh) nm = pyrender.Node(mesh=mesh, matrix=np.eye(4)) scene.add_node(nm) # CAMERA cam = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0) nc = pyrender.Node(camera=cam, matrix=cam_pos) scene.add_node(nc) # RENDER flags = RenderFlags.DEPTH_ONLY r = pyrender.OffscreenRenderer(400, 400) depth = r.render(scene, flags=flags) return (depth)
def evaluate(resnet50, regression, flamelayer, dataloader): NoWDataLoader = dataloader for batch_idx, data_batched in enumerate(NoWDataLoader): # print(batch_idx, data_batched['images'].shape) cur_batch, cur_batch_shape = data_batched['images'], data_batched[ 'images'].shape cur_facepos = data_batched['faceposes'] cur_facepos.cuda() reshaped_batch = cur_batch.permute(1, 0, 2, 3, 4) # reshaped_facepos = cur_facepos.permute(1, 0, 2, 3, 4) # print(cur_facepos.size()) # print(reshaped_facepos.size()) # Output for each image in the ring # regress_outputs, flame_vertices, flame_lmks, flame_cams, flame_proj_lmks = [], [], [], [], [] regress_outputs, flame_vertices, flame_proj_lmks = [], [], [] shape_norms, exp_norms = 0.0, 0.0 # print(reshaped_batch.size()) for img_batch in reshaped_batch: # ResNet50 # res_output = resnet50(img_batch.float()) res_output = resnet50(img_batch.float().cuda()) # Empty estimates as the initial value for concatenation regress_estimates = torch.zeros( [res_output.shape[0], regress_out_size]).cuda() # Regression model for _ in range(regress_iteration_cnt): # Preprocess regression input - concatenation regress_input = torch.cat([res_output, regress_estimates], 1) regress_estimates = regression(regress_input) regress_output = regress_estimates regress_outputs.append(regress_output) # FLAME model cam_params, pose_params = regress_output[0:, 0:3], regress_output[0:, 3:9] shape_params, exp_params = regress_output[ 0:, 9:109], regress_output[0:, 109:159] # print(cam_params.shape, pose_params.shape, shape_params.shape, exp_params.shape) flame_vert, flame_lmk = flamelayer(shape_params, exp_params, pose_params) flame_vertices.append(flame_vert) # flame_lmks.append(flame_lmk) # flame_cams.append(cam_params) # print("raw flame landmarks") # print(flame_lmk) # print("cam ") # print(cam_params) flame_proj_lmk = project_points(flame_lmk, cam_params) faces = flamelayer.faces vertices = flame_vert.cpu().detach().numpy()[0] joints = flame_lmk.cpu().detach().numpy()[0] # vertex_colors = np.ones([vertices.shape[0], 4]) * [0.3, 0.3, 0.3, 0.8] # print("All info about faces: 1. length 2. itself") # print(len(faces)) # print(faces) tri_mesh = trimesh.Trimesh(vertices, faces) mesh = pyrender.Mesh.from_trimesh(tri_mesh) scene = pyrender.Scene() scene.add(mesh) sm = trimesh.creation.uv_sphere(radius=0.005) sm.visual.vertex_colors = [0.9, 0.1, 0.1, 1.0] tfs = np.tile(np.eye(4), (len(joints), 1, 1)) tfs[:, :3, 3] = joints joints_pcl = pyrender.Mesh.from_trimesh(sm, poses=tfs) scene.add(joints_pcl) scale = cam_params[0, 0] oc = pyrender.OrthographicCamera(scale, scale) pyrender.Viewer(scene, use_raymond_lighting=True) flame_proj_lmks.append(flame_proj_lmk) shape_norms += torch.norm(shape_params) exp_norms += torch.norm(exp_params)
def render_mesh_pc_bed_pyrender_everything_synth(self, smpl_verts, smpl_faces, camera_point, bedangle, RESULTS_DICT, smpl_verts_gt = None, pmat = None, smpl_render_points = False, markers = None, dropout_variance=None, tf_corners = None, save_name = 'test_synth'): pmat *= 0.75 pmat[pmat>0] += 10 viz_popup = False #print np.min(smpl_verts[:, 0]) #print np.min(smpl_verts[:, 1]) shift_estimate_sideways = np.min([-0.15, np.min(smpl_verts[:, 1])]) #print shift_estimate_sideways shift_estimate_sideways = 0.8 - shift_estimate_sideways top_smpl_vert = np.max(smpl_verts[:, 0]) extend_top_bottom = np.max([np.max(smpl_verts[:, 0]), 64*.0286]) - 64*.0286 print extend_top_bottom, 'extend top bot' shift_both_amount = np.max([0.9, np.max(smpl_verts[:, 1])]) #if smpl is bigger than 0.9 shift less shift_both_amount = 1.5 - shift_both_amount + (0.15 + np.min([-0.15, np.min(smpl_verts[:, 1])])) smpl_verts_quad = np.concatenate((smpl_verts, np.ones((smpl_verts.shape[0], 1))), axis = 1) smpl_verts_quad = np.swapaxes(smpl_verts_quad, 0, 1) smpl_verts_quad_gt = np.concatenate((smpl_verts_gt, np.ones((smpl_verts_gt.shape[0], 1))), axis = 1) smpl_verts_quad_gt = np.swapaxes(smpl_verts_quad_gt, 0, 1) #print smpl_verts_quad.shape shift_ground_truth = 1.3 transform_A = np.identity(4) transform_A[1, 3] = shift_both_amount transform_B = np.identity(4) transform_B[1, 3] = shift_estimate_sideways + shift_both_amount#4.0 #move things over smpl_verts_B = np.swapaxes(np.matmul(transform_B, smpl_verts_quad), 0, 1)[:, 0:3] transform_C = np.identity(4) transform_C[1, 3] = shift_estimate_sideways + shift_both_amount+shift_ground_truth #move things over smpl_verts_C = np.swapaxes(np.matmul(transform_C, smpl_verts_quad_gt), 0, 1)[:, 0:3] from matplotlib import cm human_mesh_vtx_all, human_mesh_face_all = self.get_human_mesh_parts(smpl_verts_B, smpl_faces, segment_limbs=False) #GET MESH WITH PMAT tm_curr = trimesh.base.Trimesh(vertices=np.array(human_mesh_vtx_all[0]), faces = np.array(human_mesh_face_all[0])) tm_list = [tm_curr] original_mesh = [tm_curr] mesh_list = [] mesh_list.append(pyrender.Mesh.from_trimesh(tm_list[0], material = self.human_mat, smooth=True))#wireframe = False)) #this is for the main human human_mesh_vtx_all_gt, human_mesh_face_all_gt = self.get_human_mesh_parts(smpl_verts_C, smpl_faces, segment_limbs=False) #GET MESH GT WITH PMAT tm_curr_gt = trimesh.base.Trimesh(vertices=np.array(human_mesh_vtx_all_gt[0]), faces = np.array(human_mesh_face_all_gt[0])) tm_list_gt = [tm_curr_gt] original_mesh_gt = [tm_curr_gt] mesh_list_gt = [] mesh_list_gt.append(pyrender.Mesh.from_trimesh(tm_list_gt[0], material = self.human_mat_gt, smooth=True))#wireframe = False)) #this is for the main human fig = plt.figure() if self.render == True: artag_meshes = [] artag_tm = trimesh.base.Trimesh(vertices=self.artag_r + [0.0, shift_estimate_sideways + shift_both_amount, 0.0], faces=self.artag_f, face_colors = self.artag_facecolors) artag_meshes.append(pyrender.Mesh.from_trimesh(artag_tm, smooth = False)) artag_meshes_gt = [] artag_tm_gt = trimesh.base.Trimesh(vertices=self.artag_r + [0.0, shift_estimate_sideways + shift_both_amount+shift_ground_truth, 0.0], faces=self.artag_f, face_colors = self.artag_facecolors_gt) artag_meshes_gt.append(pyrender.Mesh.from_trimesh(artag_tm_gt, smooth = False)) if pmat is not None: pmat_verts, pmat_faces, pmat_facecolors = self.get_3D_pmat_markers(pmat, bedangle) pmat_verts = np.array(pmat_verts) pmat_verts = np.concatenate((np.swapaxes(pmat_verts, 0, 1), np.ones((1, pmat_verts.shape[0]))), axis = 0) pmat_verts = np.swapaxes(np.matmul(transform_A, pmat_verts), 0, 1)[:, 0:3] pmat_tm = trimesh.base.Trimesh(vertices=pmat_verts, faces=pmat_faces, face_colors = pmat_facecolors) pmat_mesh = pyrender.Mesh.from_trimesh(pmat_tm, smooth = False) pmat_verts2, _, pmat_facecolors2 = self.get_3D_pmat_markers(pmat, bedangle, solidcolor = True) pmat_verts2 = np.array(pmat_verts2) pmat_verts2 = np.concatenate((np.swapaxes(pmat_verts2, 0, 1), np.ones((1, pmat_verts2.shape[0]))), axis = 0) pmat_verts2 = np.swapaxes(np.matmul(transform_B, pmat_verts2), 0, 1)[:, 0:3] pmat_tm2 = trimesh.base.Trimesh(vertices=pmat_verts2, faces=pmat_faces, face_colors = pmat_facecolors2) pmat_mesh2 = pyrender.Mesh.from_trimesh(pmat_tm2, smooth = False) else: pmat_mesh = None pmat_mesh2 = None #print "Viewing" if self.first_pass == True: for mesh_part in mesh_list: self.scene.add(mesh_part) for mesh_part_gt in mesh_list_gt: self.scene.add(mesh_part_gt) if pmat_mesh is not None: self.scene.add(pmat_mesh) if pmat_mesh2 is not None: self.scene.add(pmat_mesh2) for artag_mesh in artag_meshes: if artag_mesh is not None: self.scene.add(artag_mesh) for artag_mesh_gt in artag_meshes_gt: if artag_mesh_gt is not None: self.scene.add(artag_mesh_gt) lighting_intensity = 20. #self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True, lighting_intensity=lighting_intensity, # point_size=2, run_in_thread=True, viewport_size=(1200, 1200)) self.first_pass = False self.node_list = [] for mesh_part in mesh_list: for node in self.scene.get_nodes(obj=mesh_part): self.node_list.append(node) self.node_list_gt = [] for mesh_part_gt in mesh_list_gt: for node in self.scene.get_nodes(obj=mesh_part_gt): self.node_list_gt.append(node) self.artag_nodes = [] for artag_mesh in artag_meshes: if artag_mesh is not None: for node in self.scene.get_nodes(obj=artag_mesh): self.artag_nodes.append(node) self.artag_nodes_gt = [] for artag_mesh_gt in artag_meshes_gt: if artag_mesh_gt is not None: for node in self.scene.get_nodes(obj=artag_mesh_gt): self.artag_nodes_gt.append(node) if pmat_mesh is not None: for node in self.scene.get_nodes(obj=pmat_mesh): self.pmat_node = node if pmat_mesh2 is not None: for node in self.scene.get_nodes(obj=pmat_mesh2): self.pmat_node2 = node camera_pose = np.eye(4) # camera_pose[0,0] = -1.0 # camera_pose[1,1] = -1.0 camera_pose[0, 0] = np.cos(np.pi/2) camera_pose[0, 1] = np.sin(np.pi/2) camera_pose[1, 0] = -np.sin(np.pi/2) camera_pose[1, 1] = np.cos(np.pi/2) rot_udpim = np.eye(4) rot_y = 180*np.pi/180. rot_udpim[1,1] = np.cos(rot_y) rot_udpim[2,2] = np.cos(rot_y) rot_udpim[1,2] = np.sin(rot_y) rot_udpim[2,1] = -np.sin(rot_y) camera_pose = np.matmul(rot_udpim, camera_pose) camera_pose[0, 3] = 64*0.0286/2 # -1.0 camera_pose[1, 3] = 1.2 + 0.8 camera_pose[2, 3] = -1.0 if viz_popup == True: self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True, lighting_intensity=10., point_size=5, run_in_thread=True, viewport_size=(1000, 1000)) #camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0) magnify =(64*.0286) camera = pyrender.OrthographicCamera(xmag=magnify, ymag = magnify) self.scene.add(camera, pose=camera_pose) light = pyrender.SpotLight(color=np.ones(3), intensity=250.0, innerConeAngle=np.pi / 10.0, outerConeAngle=np.pi / 2.0) light_pose = np.copy(camera_pose) # light_pose[1, 3] = 2.0 light_pose[0, 3] = 0.8 light_pose[1, 3] = -0.5 light_pose[2, 3] = -2.5 light_pose2 = np.copy(camera_pose) light_pose2[0, 3] = 2.5 light_pose2[1, 3] = 1.0 light_pose2[2, 3] = -5.0 light_pose3 = np.copy(camera_pose) light_pose3[0, 3] = 1.0 light_pose3[1, 3] = 5.0 light_pose3[2, 3] = -4.0 #light_pose2[0, 3] = 1.0 #light_pose2[1, 3] = 2.0 #across #light_pose2[2, 3] = -1.5 # light_pose[1, ] self.scene.add(light, pose=light_pose) self.scene.add(light, pose=light_pose2) self.scene.add(light, pose=light_pose3) else: if viz_popup == True: self.viewer.render_lock.acquire() #reset the human mesh for idx in range(len(mesh_list)): self.scene.remove_node(self.node_list[idx]) self.scene.add(mesh_list[idx]) for node in self.scene.get_nodes(obj=mesh_list[idx]): self.node_list[idx] = node #reset the human mesh for idx in range(len(mesh_list_gt)): self.scene.remove_node(self.node_list_gt[idx]) self.scene.add(mesh_list_gt[idx]) for node in self.scene.get_nodes(obj=mesh_list_gt[idx]): self.node_list_gt[idx] = node #reset the artag meshes for artag_node in self.artag_nodes: self.scene.remove_node(artag_node) for artag_mesh in artag_meshes: if artag_mesh is not None: self.scene.add(artag_mesh) self.artag_nodes = [] for artag_mesh in artag_meshes: if artag_mesh is not None: for node in self.scene.get_nodes(obj=artag_mesh): self.artag_nodes.append(node) #reset the artag meshes for artag_node_gt in self.artag_nodes_gt: self.scene.remove_node(artag_node_gt) for artag_mesh_gt in artag_meshes_gt: if artag_mesh_gt is not None: self.scene.add(artag_mesh_gt) self.artag_nodes_gt = [] for artag_mesh_gt in artag_meshes_gt: if artag_mesh_gt is not None: for node in self.scene.get_nodes(obj=artag_mesh_gt): self.artag_nodes_gt.append(node) #reset the pmat mesh if pmat_mesh is not None: self.scene.remove_node(self.pmat_node) self.scene.add(pmat_mesh) for node in self.scene.get_nodes(obj=pmat_mesh): self.pmat_node = node #reset the pmat mesh if pmat_mesh2 is not None: self.scene.remove_node(self.pmat_node2) self.scene.add(pmat_mesh2) for node in self.scene.get_nodes(obj=pmat_mesh2): self.pmat_node2 = node #print self.scene.get_nodes() if viz_popup == True: self.viewer.render_lock.release() #time.sleep(100) if viz_popup == False: r = pyrender.OffscreenRenderer(880, 880) # r.render(self.scene) color_render, depth = r.render(self.scene) # plt.subplot(1, 2, 1) plt.axis('off') #im_to_show = np.concatenate((color_render, color_im), axis = 1) im_to_show = np.copy(color_render) im_to_show = im_to_show[130-int(extend_top_bottom*300):750+int(extend_top_bottom*300), :, :] #plt.imshow(color) plt.imshow(im_to_show) # plt.subplot(1, 2, 2) # plt.axis('off') # plt.imshow(depth, cmap=plt.cm.gray_r) >> > plt.show() fig.set_size_inches(15., 10.) fig.tight_layout() #save_name = 'f_hbh_'+'{:04}'.format(self.pic_num) print "saving!" fig.savefig('/media/henry/multimodal_data_2/CVPR2020_study/'+save_name+'_v2.png', dpi=300) self.pic_num += 1 #plt.show() #if self.pic_num == 20: # print "DONE" # time.sleep(1000000) #print "got here" #print X.shape return RESULTS_DICT
def render_orthcam( model_in, # model name or trimesh xy_mag, rend_size, flat_shading=False, zfar=10000, znear=0.05): # Mesh creation if isinstance(model_in, str) is True: mesh = trimesh.load(model_in, process=False) else: mesh = model_in.copy() pr_mesh = pyrender.Mesh.from_trimesh(mesh) # Scene creation scene = pyrender.Scene() # Adding objects to the scene face_node = scene.add(pr_mesh) # Camera Creation if type(xy_mag) == float: cam = pyrender.OrthographicCamera(xmag=xy_mag, ymag=xy_mag, znear=znear, zfar=zfar) elif type(xy_mag) == tuple: cam = pyrender.OrthographicCamera(xmag=xy_mag[0], ymag=xy_mag[1], znear=znear, zfar=zfar) else: print("Error: xy_mag should be float or tuple") return False scene.add(cam, pose=np.eye(4)) # Set up the light light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=10.0) scene.add(light, pose=np.eye(4)) # Rendering offscreen from that camera r = pyrender.OffscreenRenderer(viewport_width=rend_size[1], viewport_height=rend_size[0], point_size=1.0) if flat_shading is True: color, depth = r.render(scene, flags=pyrender.constants.RenderFlags.FLAT) else: color, depth = r.render(scene) # rgb to bgr for cv2 color = color[:, :, [2, 1, 0]] # fix pyrender BUG of depth rendering, pyrender version: 0.1.43 depth[depth != 0] = (zfar + znear - ( (2.0 * znear * zfar) / depth[depth != 0])) / (zfar - znear) depth[depth != 0] = ((depth[depth != 0] + (zfar + znear) / (zfar - znear)) * (zfar - znear)) / 2.0 return depth, color
def main(opt): model = body_models.create(model_path='../3d_data/models', model_type='smpl', gender='male', ext='pkl') smpl = pickle.load(open('../3d_data/densepose_uv.pkl', 'rb')) faces = np.array(smpl['f_extended'], dtype=np.int64).reshape((-1, 3)) uv_faceid = io.loadmat( '../3d_data/DensePoseData/UV_data/UV_Processed.mat')['All_FaceIndices'] uv = smpl['uv'] # with open('../3d_data/nongrey_male_0110.jpg', 'rb') as file: texture = cv2.imread('../3d_data/nongrey_male_0110.jpg') global_tr = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]) # set up the rendering objects focal_length = opt.focal_length * opt.image_height # mesh_camera = pyrender.IntrinsicsCamera(focal_length, focal_length, opt.image_width / 2, opt.image_height / 2, # opt.znear, opt.zfar) mesh_camera = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0, znear=0.05) camera = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0, znear=0.05) camera_pose = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.05], [0.0, 0.0, 0.0, 1.0]]) mesh_tr = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, opt.global_y + 0.11], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]) mesh_camera_pose = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, opt.camera_distance], [0.0, 0.0, 0.0, 1.0]]) render = pyrender.OffscreenRenderer(opt.image_width, opt.image_height) output = model(return_verts=True) vertices = output.vertices.detach().cpu().numpy().squeeze() mesh_verts = np.array([vertices[i] for i in smpl['v_extended']]) visual_check = trimesh.visual.TextureVisuals(uv=uv, image=texture) tri_mesh_scene = trimesh.Trimesh(vertices=mesh_verts, faces=faces, visual=visual_check) mesh_body = pyrender.Mesh.from_trimesh(tri_mesh_scene) mesh_scene = pyrender.Scene(ambient_light=[0.5, 0.5, 0.5], bg_color=[-1.0, -1.0, -1.0]) mesh_scene.add(mesh_body, pose=mesh_tr) mesh_scene.add(mesh_camera, pose=mesh_camera_pose) rendered_uv, depth = render.render(scene=mesh_scene, flags=pyrender.RenderFlags.UV_RENDERING) rendered_uv = rendered_uv.copy() mask = rendered_uv[:, :, 2] != -1. temp_2 = rendered_uv[:, :, 2] temp_2[mask] = np.take(uv_faceid, temp_2[mask].astype('int')) rendered_uv[:, :, 2] = temp_2 cv2.imshow('UV', rendered_uv) bounds = tri_mesh_scene.bounding_box_oriented.extents mesh_verts -= mesh_scene.centroid mesh_verts /= bounds # mesh_verts *= 2 mesh_verts = mesh_verts + 1 / 2 face_select = faces[uv_faceid[:, 0] == 1] # verts = np.concatenate((uv, np.ones(uv.shape[:2] + (1,))), axis=2) # uv[:, 2] = 1 verts = (uv * 2) - 1 visual = trimesh.visual.ColorVisuals(vertex_colors=uv) tri_mesh = trimesh.Trimesh(vertices=verts, faces=face_select, visual=visual) # tri_mesh mesh = pyrender.Mesh.from_trimesh(tri_mesh) # tri_mesh.show() scene = pyrender.Scene(ambient_light=[0.5, 0.5, 0.5], bg_color=[-1.0, -1.0, -1.0]) scene.add(mesh, pose=global_tr) scene.add(camera, pose=camera_pose) rendered_color_visual, depth = render.render( scene=scene, flags=pyrender.RenderFlags.SKIP_CULL_FACES) # pyrender.Viewer(scene, render_flags={'cull_faces': False}) cv2.imshow('Part UV', rendered_color_visual) # cv2.waitKey(0) rendered_interp, _ = render.render( scene=scene, flags=pyrender.RenderFlags.BARYCENTRIC_COORDINATES | pyrender.RenderFlags.SKIP_CULL_FACES) tri_id, _ = render.render(scene=scene, flags=pyrender.RenderFlags.TRIANGLE_ID_RENDERING | pyrender.RenderFlags.SKIP_CULL_FACES) vertex_stream = np.take(mesh_verts, face_select, axis=0) tri_id = tri_id[:, :, 0] rendered_interp = rendered_interp.reshape(rendered_interp.shape + (1, )).repeat([3], axis=-1) out_view = vertex_stream[tri_id.astype('int')] * rendered_interp out_view = out_view.sum(axis=-2) # rendered_uv[rendered_uv == -1] = 0 # rendered_uv[:, :, 2] /= 255 out_view[rendered_color_visual < 0] = 0 # cv2.imwrite('../saves/checks/mesh_normalized_uv.jpg', (rendered_uv * 255).astype('uint8')) cv2.imshow('Coords', out_view) cv2.imwrite('../saves/checks/mesh_uv_render.jpg', (out_view * 255).astype('uint8')) cv2.waitKey(0)
# +X # -X df = [] for dist in range(0, 200, 5): scene = pyrender.Scene() scene.add(mesh) # TODO: figure out how to render corectly # camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0) camera = pyrender.OrthographicCamera(xmag=10.0, ymag=10.0) camera_pose = np.array([ [1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, dist], [0.0, 0.0, 0.0, 1.0], ]) scene.add(camera, pose=camera_pose) light = pyrender.SpotLight(color=np.ones(3), intensity=3.0, innerConeAngle=np.pi / 16.0) # scene.add(light, pose=camera_pose) r = pyrender.OffscreenRenderer(100, 100)
def render_mesh_pc_bed_pyrender_everything(self, smpl_verts, smpl_faces, camera_point, bedangle, RESULTS_DICT, pc=None, pmat=None, smpl_render_points=False, markers=None, dropout_variance=None, color_im=None, tf_corners=None, current_pose_type_ct=None, participant=None): #print np.min(smpl_verts[:, 0]) #print np.min(smpl_verts[:, 1]) shift_estimate_sideways = np.min([-0.15, np.min(smpl_verts[:, 1])]) #print shift_estimate_sideways shift_estimate_sideways = 0.8 - shift_estimate_sideways top_smpl_vert = np.max(smpl_verts[:, 0]) extend_top_bottom = np.max([np.max(smpl_verts[:, 0]), 64 * .0286 ]) - 64 * .0286 print extend_top_bottom, 'extend top bot' shift_both_amount = np.max([0.9, np.max(smpl_verts[:, 1]) ]) #if smpl is bigger than 0.9 shift less shift_both_amount = 1.5 - shift_both_amount + ( 0.15 + np.min([-0.15, np.min(smpl_verts[:, 1])])) #print np.max(smpl_verts[:, 1]), 'max smpl' #shift_both_amount = 0.6 #smpl_verts[:, 2] += 0.5 #pc[:, 2] += 0.5 pc[:, 0] = pc[:, 0] # - 0.17 - 0.036608 pc[:, 1] = pc[:, 1] # + 0.09 #adjust the point cloud #segment_limbs = True #if pmat is not None: # if np.sum(pmat) < 5000: # smpl_verts = smpl_verts * 0.001 smpl_verts_quad = np.concatenate( (smpl_verts, np.ones((smpl_verts.shape[0], 1))), axis=1) smpl_verts_quad = np.swapaxes(smpl_verts_quad, 0, 1) #print smpl_verts_quad.shape transform_A = np.identity(4) transform_A[1, 3] = shift_both_amount transform_B = np.identity(4) transform_B[ 1, 3] = shift_estimate_sideways + shift_both_amount #4.0 #move things over smpl_verts_B = np.swapaxes(np.matmul(transform_B, smpl_verts_quad), 0, 1)[:, 0:3] transform_C = np.identity(4) transform_C[1, 3] = 2.0 #2.0 #move things over smpl_verts_C = np.swapaxes(np.matmul(transform_C, smpl_verts_quad), 0, 1)[:, 0:3] from matplotlib import cm human_mesh_vtx_all, human_mesh_face_all = self.get_human_mesh_parts( smpl_verts_B, smpl_faces, segment_limbs=False) #GET MESH WITH PMAT tm_curr = trimesh.base.Trimesh(vertices=np.array( human_mesh_vtx_all[0]), faces=np.array(human_mesh_face_all[0])) tm_list = [tm_curr] original_mesh = [tm_curr] mesh_list = [] mesh_list.append( pyrender.Mesh.from_trimesh(tm_list[0], material=self.human_mat, smooth=False) ) #wireframe = False)) #this is for the main human print np.shape(color_im) print tf_corners top_idx = float(tf_corners[0, 1]) bot_idx = float(tf_corners[2, 1]) perc_total = (bot_idx - top_idx) / 880. print perc_total fig = plt.figure() if self.render == True: #print m.r #print artag_r #create mini meshes for AR tags artag_meshes = [] if markers is not None: for marker in markers: if markers[2] is None: artag_meshes.append(None) elif marker is None: artag_meshes.append(None) else: #print marker - markers[2] if marker is markers[2]: print "is markers 2", marker #artag_tm = trimesh.base.Trimesh(vertices=self.artag_r, faces=self.artag_f, face_colors = self.artag_facecolors_root) #artag_meshes.append(pyrender.Mesh.from_trimesh(artag_tm, smooth = False)) else: artag_tm = trimesh.base.Trimesh( vertices=self.artag_r + [ 0.0, shift_estimate_sideways + shift_both_amount, 0.0 ], faces=self.artag_f, face_colors=self.artag_facecolors) artag_meshes.append( pyrender.Mesh.from_trimesh(artag_tm, smooth=False)) if pmat is not None: pmat_verts, pmat_faces, pmat_facecolors = self.get_3D_pmat_markers( pmat, bedangle) pmat_verts = np.array(pmat_verts) pmat_verts = np.concatenate((np.swapaxes( pmat_verts, 0, 1), np.ones((1, pmat_verts.shape[0]))), axis=0) pmat_verts = np.swapaxes(np.matmul(transform_A, pmat_verts), 0, 1)[:, 0:3] pmat_tm = trimesh.base.Trimesh(vertices=pmat_verts, faces=pmat_faces, face_colors=pmat_facecolors) pmat_mesh = pyrender.Mesh.from_trimesh(pmat_tm, smooth=False) pmat_verts2, _, pmat_facecolors2 = self.get_3D_pmat_markers( pmat, bedangle, solidcolor=True) pmat_verts2 = np.array(pmat_verts2) pmat_verts2 = np.concatenate((np.swapaxes( pmat_verts2, 0, 1), np.ones((1, pmat_verts2.shape[0]))), axis=0) pmat_verts2 = np.swapaxes(np.matmul(transform_B, pmat_verts2), 0, 1)[:, 0:3] pmat_tm2 = trimesh.base.Trimesh(vertices=pmat_verts2, faces=pmat_faces, face_colors=pmat_facecolors2) pmat_mesh2 = pyrender.Mesh.from_trimesh(pmat_tm2, smooth=False) else: pmat_mesh = None pmat_mesh2 = None #print "Viewing" if self.first_pass == True: for mesh_part in mesh_list: self.scene.add(mesh_part) if pmat_mesh is not None: self.scene.add(pmat_mesh) if pmat_mesh2 is not None: self.scene.add(pmat_mesh2) for artag_mesh in artag_meshes: if artag_mesh is not None: self.scene.add(artag_mesh) lighting_intensity = 20. #self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True, lighting_intensity=lighting_intensity, # point_size=2, run_in_thread=True, viewport_size=(1200, 1200)) self.first_pass = False self.node_list = [] for mesh_part in mesh_list: for node in self.scene.get_nodes(obj=mesh_part): self.node_list.append(node) self.artag_nodes = [] for artag_mesh in artag_meshes: if artag_mesh is not None: for node in self.scene.get_nodes(obj=artag_mesh): self.artag_nodes.append(node) if pmat_mesh is not None: for node in self.scene.get_nodes(obj=pmat_mesh): self.pmat_node = node if pmat_mesh2 is not None: for node in self.scene.get_nodes(obj=pmat_mesh2): self.pmat_node2 = node camera_pose = np.eye(4) # camera_pose[0,0] = -1.0 # camera_pose[1,1] = -1.0 camera_pose[0, 0] = np.cos(np.pi / 2) camera_pose[0, 1] = np.sin(np.pi / 2) camera_pose[1, 0] = -np.sin(np.pi / 2) camera_pose[1, 1] = np.cos(np.pi / 2) rot_udpim = np.eye(4) rot_y = 180 * np.pi / 180. rot_udpim[1, 1] = np.cos(rot_y) rot_udpim[2, 2] = np.cos(rot_y) rot_udpim[1, 2] = np.sin(rot_y) rot_udpim[2, 1] = -np.sin(rot_y) camera_pose = np.matmul(rot_udpim, camera_pose) camera_pose[0, 3] = 64 * 0.0286 / 2 # -1.0 camera_pose[1, 3] = 1.2 camera_pose[2, 3] = -1.0 # self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True, # lighting_intensity=10., # point_size=5, run_in_thread=True, viewport_size=(1000, 1000)) # camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0) magnify = (64 * .0286) * 0.5 / perc_total camera = pyrender.OrthographicCamera(xmag=magnify, ymag=magnify) self.scene.add(camera, pose=camera_pose) light = pyrender.SpotLight(color=np.ones(3), intensity=200.0, innerConeAngle=np.pi / 10.0, outerConeAngle=np.pi / 2.0) light_pose = np.copy(camera_pose) # light_pose[1, 3] = 2.0 light_pose[0, 3] = 0.8 light_pose[1, 3] = -0.5 light_pose[2, 3] = -1.5 light_pose2 = np.copy(camera_pose) light_pose2[0, 3] = 2.5 light_pose2[1, 3] = 1.0 light_pose2[2, 3] = -4.0 # light_pose[1, ] self.scene.add(light, pose=light_pose) self.scene.add(light, pose=light_pose2) else: #self.viewer.render_lock.acquire() #reset the human mesh for idx in range(len(mesh_list)): self.scene.remove_node(self.node_list[idx]) self.scene.add(mesh_list[idx]) for node in self.scene.get_nodes(obj=mesh_list[idx]): self.node_list[idx] = node #reset the artag meshes for artag_node in self.artag_nodes: self.scene.remove_node(artag_node) for artag_mesh in artag_meshes: if artag_mesh is not None: self.scene.add(artag_mesh) self.artag_nodes = [] for artag_mesh in artag_meshes: if artag_mesh is not None: for node in self.scene.get_nodes(obj=artag_mesh): self.artag_nodes.append(node) #reset the pmat mesh if pmat_mesh is not None: self.scene.remove_node(self.pmat_node) self.scene.add(pmat_mesh) for node in self.scene.get_nodes(obj=pmat_mesh): self.pmat_node = node #reset the pmat mesh if pmat_mesh2 is not None: self.scene.remove_node(self.pmat_node2) self.scene.add(pmat_mesh2) for node in self.scene.get_nodes(obj=pmat_mesh2): self.pmat_node2 = node #print self.scene.get_nodes() #self.viewer.render_lock.release() #time.sleep(100) r = pyrender.OffscreenRenderer(880, 880) # r.render(self.scene) color_render, depth = r.render(self.scene) # plt.subplot(1, 2, 1) plt.axis('off') if 880. - bot_idx > top_idx: print 'shift im down by', 880. - bot_idx - top_idx downshift = int((880. - bot_idx) / 2 - top_idx / 2 + 0.5) color_im[downshift:880] = color_im[0:880 - downshift] elif top_idx > (880. - bot_idx): print 'shift im up by', top_idx - (880. - bot_idx) upshift = int(top_idx / 2 - (880. - bot_idx) / 2 + 0.5) color_im[0:880 - upshift] = color_im[upshift:880] print tf_corners print np.shape(color_render), np.shape(color_im) color_im = np.concatenate( (color_im[:, :, 2:3], color_im[:, :, 1:2], color_im[:, :, 0:1]), axis=2) color_im = color_im[:, int(tf_corners[0, 0] - 10):int(tf_corners[1, 0] + 10), :] im_to_show = np.concatenate((color_render, color_im), axis=1) im_to_show = im_to_show[130 - int(extend_top_bottom * 300):750 + int(extend_top_bottom * 300), :, :] #plt.imshow(color) plt.imshow(im_to_show) # plt.subplot(1, 2, 2) # plt.axis('off') # plt.imshow(depth, cmap=plt.cm.gray_r) >> > plt.show() fig.set_size_inches(15., 10.) fig.tight_layout() #save_name = 'f_hbh_'+'{:04}'.format(self.pic_num) save_name = participant + '_' + current_pose_type_ct fig.savefig('/media/henry/multimodal_data_2/CVPR2020_study/' + participant + '/estimated_poses/' + save_name + '.png', dpi=300) #fig.savefig('/media/henry/multimodal_data_2/CVPR2020_study/TEST.png', dpi=300) #plt.savefig('test2png.png', dpi=100) self.pic_num += 1 #plt.show() #if self.pic_num == 20: # print "DONE" # time.sleep(1000000) #print "got here" #print X.shape return RESULTS_DICT
def render_only_human_gt(self, m): bed1_verts = np.array([ [-1.3, -1.35, -3.0], [-0.9, -0.75, -4.0], [1.3, -1.35, -3.0], [0.9, -0.75, -4.0], [-1.2, 0.05, -4.0], [-1.2, 0.0, -4.0], [1.2, 0.05, -4.0], [1.2, 0.0, -4.0], [-1.3, -1.35, -3.0], [-1.3, -1.45, -3.0], [1.3, -1.35, -3.0], [1.3, -1.45, -3.0], [-1.2, 0.05, -4.0], [-1.0, 1.0, -4.0], [1.2, 0.05, -4.0], [1.0, 1.0, -4.0], ]) bed1_faces = np.array([ [0, 1, 2], [0, 2, 1], [1, 2, 3], [1, 3, 2], [4, 5, 6], [4, 6, 5], [5, 6, 7], [5, 7, 6], [8, 9, 10], [8, 10, 9], [9, 10, 11], [9, 11, 10], [12, 13, 14], [12, 14, 13], [13, 14, 15], [13, 15, 14], ]) bed1_facecolors = [] for i in range(bed1_faces.shape[0]): if 4 <= i < 12: bed1_facecolors.append([0.8, 0.8, 0.2]) else: bed1_facecolors.append([1.0, 1.0, 0.8]) smpl_verts = (m.r - m.J_transformed[0, :]) #smpl_verts = np.concatenate((smpl_verts[:, 1:2] - 1.5, smpl_verts[:, 0:1], -smpl_verts[:, 2:3]), axis = 1) smpl_verts = np.concatenate( (smpl_verts[:, 0:1] - 1.5, smpl_verts[:, 1:2], smpl_verts[:, 2:3]), axis=1) #smpl_verts = np.concatenate((smpl_verts[:, 1:2], smpl_verts[:, 0:1], -smpl_verts[:, 2:3]), axis = 1) smpl_faces = np.array(m.f) #smpl_faces = np.concatenate((smpl_faces[:, 0:1],smpl_faces[:, 2:3],smpl_faces[:, 1:2]), axis = 1) smpl_verts2 = np.concatenate( (smpl_verts[:, 1:2], smpl_verts[:, 2:3] - 2.0, smpl_verts[:, 0:1]), axis=1) smpl_verts3 = np.concatenate( (smpl_verts[:, 1:2], smpl_verts[:, 2:3] - 2.4, smpl_verts[:, 0:1]), axis=1) laying_rot_M = np.array([[1., 0., 0.], [0., 0.866025, -0.5], [0., 0.5, 0.866025]]) laying_rot_M2 = np.array([[1., 0., 0.], [0., 0.34202, -0.93969], [0., 0.93969, 0.34202]]) for i in range(smpl_verts2.shape[0]): smpl_verts2[i, :] = np.matmul(laying_rot_M, smpl_verts2[i, :]) smpl_verts3[i, :] = np.matmul(laying_rot_M2, smpl_verts3[i, :]) #break #smpl_verts2 = np.concatenate((-smpl_verts[:, 2:3] + 1.5, smpl_verts[:, 1:2], smpl_verts[:, 0:1]), axis = 1) #smpl_verts3 = np.concatenate((smpl_verts[:, 2:3] - 1.5, smpl_verts[:, 1:2], -smpl_verts[:, 0:1]), axis = 1) #print smpl_verts2.shape #tm = trimesh.base.Trimesh(vertices=smpl_verts, faces=smpl_faces) #mesh = pyrender.Mesh.from_trimesh(tm, material=self.human_mat_for_study, wireframe=False) tm2 = trimesh.base.Trimesh(vertices=smpl_verts2, faces=smpl_faces) mesh2 = pyrender.Mesh.from_trimesh(tm2, material=self.human_mat_for_study, wireframe=False) tm3 = trimesh.base.Trimesh(vertices=smpl_verts3, faces=smpl_faces) mesh3 = pyrender.Mesh.from_trimesh(tm3, material=self.human_mat_for_study, wireframe=False) tm_bed1 = trimesh.base.Trimesh(vertices=bed1_verts, faces=bed1_faces, face_colors=np.array(bed1_facecolors)) mesh_bed1 = pyrender.Mesh.from_trimesh( tm_bed1, material=self.human_bed_for_study, wireframe=False, smooth=False) fig = plt.figure() #plt.plot(np.arange(0, 400), np.arange(0, 400)*.5 + 800) if self.first_pass == True: self.scene.add(mesh_bed1) #self.scene.add(mesh) self.scene.add(mesh2) self.scene.add(mesh3) self.first_pass = False self.node_list_bed1 = [] for node in self.scene.get_nodes(obj=mesh_bed1): self.node_list_bed1.append(node) #self.node_list = [] #for node in self.scene.get_nodes(obj=mesh): # self.node_list.append(node) self.node_list_2 = [] for node in self.scene.get_nodes(obj=mesh2): self.node_list_2.append(node) self.node_list_3 = [] for node in self.scene.get_nodes(obj=mesh3): self.node_list_3.append(node) camera_pose = np.eye(4) # camera_pose[0,0] = -1.0 # camera_pose[1,1] = -1.0 camera_pose[0, 3] = 0.0 # -1.0 camera_pose[1, 3] = 0.0 # -1.0 camera_pose[2, 3] = 4.0 # self.viewer = pyrender.Viewer(self.scene, use_raymond_lighting=True, # lighting_intensity=10., # point_size=5, run_in_thread=True, viewport_size=(1000, 1000)) # camera = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.0) camera = pyrender.OrthographicCamera(xmag=2.0, ymag=2.0) self.scene.add(camera, pose=camera_pose) light = pyrender.SpotLight(color=np.ones(3), intensity=150.0, innerConeAngle=np.pi / 10.0, outerConeAngle=np.pi / 2.0) light_pose = np.copy(camera_pose) #light_pose[1, 3] = 2.0 light_pose[0, 3] = -1.0 light_pose[2, 3] = 5.0 light_pose2 = np.copy(camera_pose) light_pose2[0, 3] = 0.5 light_pose2[1, 3] = 0.5 light_pose2[2, 3] = 5.0 # light_pose[1, ] #self.scene.add(light, pose=light_pose) #self.scene.add(light, pose=light_pose2) else: #self.viewer.render_lock.acquire() # reset the human mesh self.scene.remove_node(self.node_list_bed1[0]) self.scene.add(mesh_bed1) for node in self.scene.get_nodes(obj=mesh_bed1): self.node_list_bed1[0] = node # reset the human mesh #self.scene.remove_node(self.node_list[0]) #self.scene.add(mesh) #for node in self.scene.get_nodes(obj=mesh): # self.node_list[0] = node self.scene.remove_node(self.node_list_2[0]) self.scene.add(mesh2) for node in self.scene.get_nodes(obj=mesh2): self.node_list_2[0] = node self.scene.remove_node(self.node_list_3[0]) self.scene.add(mesh3) for node in self.scene.get_nodes(obj=mesh3): self.node_list_3[0] = node #self.viewer.render_lock.release() r = pyrender.OffscreenRenderer(2000, 2000) # r.render(self.scene) color, depth = r.render(self.scene) # plt.subplot(1, 2, 1) plt.axis('off') plt.imshow(color) # plt.subplot(1, 2, 2) # plt.axis('off') # plt.imshow(depth, cmap=plt.cm.gray_r) >> > plt.show() fig.set_size_inches(15., 15.) fig.tight_layout() save_name = 'f_hbh_' + '{:04}'.format(self.pic_num) fig.savefig('/home/henry/Pictures/CVPR2020_study/' + save_name + '.png', dpi=300) #plt.savefig('test2png.png', dpi=100) self.pic_num += 1 #plt.show() if self.pic_num == 20: print "DONE" time.sleep(1000000) print "got here"
def fixed_prediction_animation(real_traj, pred_traj, loop_time, urdf_path='robots/right_hand_relative.urdf', real_color=np.array([71, 107, 107, 255]), pred_color_cmap=matplotlib.cm.get_cmap('tab10'), background_color=np.array([1.0, 1.0, 1.0]), hand_offset=0.2, title="Hand Motion Prediction", reverse=True, show=False): scene, origin, node_map, real_hand, pred_hands, pred_trajectories, times, fps = setup_animation_scene( real_traj=real_traj, pred_traj=pred_traj, hand_offset=hand_offset, loop_time=loop_time, urdf_path=urdf_path, real_color=real_color, pred_color_cmap=pred_color_cmap, background_color=background_color, reverse=reverse) # clear_output() # Set up the camera -- z-axis away from the scene, pred_joint_states-axis right, real_joint_states-axis up camera = pyrender.OrthographicCamera( xmag=(hand_offset * len(pred_trajectories)) * 1.2, ymag=0.3) camera_pose = np.array([ [1.0, 0.0, 0.0, (hand_offset * len(pred_trajectories)) / 2], [0.0, 0.0, -1.0, -0.25], [0.0, 1.0, 0.0, 0.10], [0.0, 0.0, 0.0, 1.0], ]) scene.add(camera, pose=camera_pose) # Set up the light -- a single spot light in the same spot as the camera light = pyrender.DirectionalLight(color=np.ones(3), intensity=5) scene.add(light, pose=camera_pose) # Render the scene height = 5 width = height * 1.5 dpi = 100 r = pyrender.OffscreenRenderer(width * dpi, height * dpi) rgb_sequence, depth_sequence = [], [] fig = plt.figure(figsize=(width, height)) plt.title(title) color, _ = r.render(scene) rgb_sequence.append(color) for i in tqdm(range(len(times)), position=0, leave=True, desc="Rendering"): real_cfg = {k: real_traj[k][i] for k in real_traj} pred_cfgs = [{k: pred_traj[k][i] for k in pred_traj} for pred_traj in pred_trajectories] # i = (i + 1) % len(times) fk_real = real_hand.visual_trimesh_fk(cfg=real_cfg) fk_preds = [ pred_hand.visual_trimesh_fk(cfg=pred_cfg) for pred_hand, pred_cfg in zip(pred_hands, pred_cfgs) ] for real_mesh in fk_real: real_pose = fk_real[real_mesh] node_map[real_mesh].matrix = real_pose for fk_pred in fk_preds: for pred_mesh in fk_pred: pred_pose = fk_pred[pred_mesh] node_map[pred_mesh].matrix = pred_pose color, _ = r.render(scene) rgb_sequence.append(color) r.delete() artists = [] for img in tqdm(rgb_sequence, position=0, leave=True, desc="Preparing PLT Animation"): img_color = plt.imshow(img, animated=True) artists.append([img_color]) # Create PLT animation ani = animation.ArtistAnimation(fig, artists, interval=1 / fps * 1000, blit=True, repeat_delay=500) plt.axis('off') if show: plt.show() else: plt.close() return ani
def render(self, pose_deg=25.0, light_pose_deg=-25.0, gamma=1.0): vertices = self.dims["shape"]["vals"] vertices /= np.max(np.abs(vertices)) faces = self._tl - 1 # these are linear (I think) but the renderer needs them in sRGB colours = self.dims["tex"]["vals"] srgb_colours = colours / 255.0 srgb_colours = np.where( srgb_colours < 0.0031308, srgb_colours * 12.92, 1.055 * (srgb_colours**(1.0 / 2.4)) - 0.055, ) srgb_colours *= 255.0 mesh = trimesh.Trimesh(vertices=vertices, faces=faces, vertex_colors=colours) mesh.invert() self._mesh = mesh camera = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0) pose_vec = scipy.spatial.transform.Rotation.from_rotvec( [0, np.radians(pose_deg), 0]) pose_mat = np.eye(4) pose_mat[:3, :3] = pose_vec.as_dcm() pose_mat[2, -1] = -100 material = pyrender.material.SpecularGlossinessMaterial( diffuseFactor=1.0, glossinessFactor=0.0) obj = pyrender.Mesh.from_trimesh( mesh=mesh, poses=pose_mat[np.newaxis, ...] # material=material ) self._obj = obj light = pyrender.DirectionalLight(intensity=5.0) light_pose_vec = scipy.spatial.transform.Rotation.from_rotvec( [0, np.radians(light_pose_deg), 0]) light_pose_mat = np.eye(4) light_pose_mat[:3, :3] = light_pose_vec.as_dcm() scene = pyrender.Scene(ambient_light=[0.1] * 3, bg_color=[0] * 4) scene.add(camera) scene.add(obj) scene.add(light, pose=light_pose_mat) self._renderer = pyrender.OffscreenRenderer(viewport_width=512, viewport_height=512) self._scene = scene flags = (pyrender.constants.RenderFlags.NONE | pyrender.constants.RenderFlags.RGBA | pyrender.constants.RenderFlags.SHADOWS_ALL) (img, _) = self._renderer.render(scene, flags=flags) img = (img / 255.0)**(1.0 / gamma) img = np.round(img * 255.0).astype("uint8") return img