def __init__(self, image_size): super(Renderer, self).__init__() self.image_size = image_size self.dog_obj = load_objs_as_meshes(['data/dog_B/dog_B/dog_B_tpose.obj']) raster_settings = RasterizationSettings( image_size=self.image_size, blur_radius=0.0, faces_per_pixel=1, bin_size=None ) R, T = look_at_view_transform(2.7, 0, 0) cameras = OpenGLPerspectiveCameras(device=R.device, R=R, T=T) lights = PointLights(device=R.device, location=[[0.0, 1.0, 0.0]]) self.renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings ), shader=SoftPhongShader( device=R.device, cameras=cameras, lights=lights ) )
def test(): with torch.no_grad(): for i in trange(len(perspectives)): R, T = perspectives[i] cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T) if isinstance(bsdf, ComposeSpatialVarying): got, _ = pt.pathtrace( shape, size=SIZE, chunk_size=SIZE, bundle_size=1, bsdf=bsdf, integrator=BasisBRDF(bsdf), cameras=cameras, lights=lights, device=device, silent=True, background=0, ) f, axes = plt.subplots(r, c) f.set_figheight(10) f.set_figwidth(10) got = got.unsqueeze(-1).expand(got.shape + (3, )) for k, img in enumerate(got.split(1, dim=-2)): img = img.squeeze(-2).cpu().numpy() axes[unroll(k, c)].imshow(img) axes[unroll(k, c)].axis('off') plt.subplots_adjust(wspace=0, hspace=0) plt.savefig(f"outputs/weights_{i:04}.png", bbox_inches="tight") plt.clf() plt.close(f) #normals, _ = pt.pathtrace( # shape, # size=SIZE, chunk_size=SIZE, bundle_size=1, bsdf=bsdf, integrator=Debug(), # cameras=cameras, lights=lights, device=device, silent=True, # background=0, #) #save_image(f"outputs/normals_{i:04}.png", normals) #illum, _ = pt.pathtrace( # shape, # size=SIZE, chunk_size=SIZE, bundle_size=1, bsdf=bsdf, integrator=Illumination(), # cameras=cameras, lights=lights, device=device, silent=True, #) #save_image(f"outputs/illum_{i:04}.png", illum) if (integrator is not None) and False: got, _ = pt.pathtrace( shape, size=SIZE, chunk_size=SIZE, bundle_size=1, bsdf=bsdf, integrator=integrator, cameras=cameras, lights=lights, device=device, silent=True, background=0, ) save_image(f"outputs/got_{i:04}.png", got)
def set_renderer(self): cameras = OpenGLPerspectiveCameras(device=self.cuda_device, degrees=True, fov=VIEW['fov'], znear=VIEW['znear'], zfar=VIEW['zfar']) raster_settings = RasterizationSettings(image_size=VIEW['viewport'][0], blur_radius=0.0, faces_per_pixel=1, bin_size=0) lights = DirectionalLights( device=self.cuda_device, direction=((-40, 200, 100), ), ambient_color=((0.5, 0.5, 0.5), ), diffuse_color=((0.5, 0.5, 0.5), ), specular_color=((0.0, 0.0, 0.0), ), ) self.renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=TexturedSoftPhongShader(device=self.cuda_device, cameras=cameras, lights=lights))
def init_renderer(self): # nsh_face_mesh = meshio.Mesh('data/mesh/nsh_bfm_face.obj') # self.nsh_face_tri = torch.from_numpy(nsh_face_mesh.triangles).type( # torch.int64).to(self.device) R, T = look_at_view_transform(10, 0, 0) cameras = OpenGLPerspectiveCameras(znear=0.001, zfar=30.0, aspect_ratio=1.0, fov=12.5936, degrees=True, R=R, T=T, device=self.device) raster_settings = RasterizationSettings(image_size=self.im_size, blur_radius=0.0, faces_per_pixel=1, bin_size=0, cull_backfaces=True) self.rasterizer = MeshRasterizer(cameras=cameras, raster_settings=raster_settings) lights = DirectionalLights(device=self.device) shader = TexturedSoftPhongShader(device=self.device, cameras=cameras, lights=lights) self.renderer = MeshRenderer(rasterizer=self.rasterizer, shader=shader)
def render_obj(verts, faces, distance, elevation, azimuth): device = torch.device("cuda:0") verts_rgb = torch.ones_like(verts)[None] textures = Textures(verts_rgb=verts_rgb.to(device)) cur_mesh = Meshes(verts=[verts.to(device)], faces=[faces.to(device)], textures=textures) cameras = OpenGLPerspectiveCameras(device=device) blend_params = BlendParams(sigma=1e-4, gamma=1e-4) raster_settings = RasterizationSettings(image_size=256, blur_radius=0.0, faces_per_pixel=1, bin_size=0) lights = PointLights(device=device, location=((2.0, 2.0, -2.0), )) phong_renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=PhongShader(device=device, lights=lights)) R, T = look_at_view_transform(distance, elevation, azimuth, device=device) return phong_renderer(meshes_world=cur_mesh, R=R, T=T).cpu().numpy()
def project_mesh(mesh, angle): start = time.time() m = Metadata() R, T = look_at_view_transform(1.75, -45, angle, up=((0, 1, 0), ), at=((0, -0.25, 0), )) cameras = OpenGLPerspectiveCameras(device=m.device, R=R, T=T) raster_settings = m.raster_settings lights = m.lights renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardFlatShader(cameras=cameras, device=m.device, lights=lights)) verts = mesh.verts_list()[0] # faces = meshes.faces_list()[0] verts_rgb = torch.ones_like(verts)[None] # (1, V, 3) # verts_rgb = torch.ones((len(mesh.verts_list()[0]), 1))[None] # (1, V, 3) textures = Textures(verts_rgb=verts_rgb.to(m.device)) mesh.textures = textures mesh.textures._num_faces_per_mesh = mesh._num_faces_per_mesh.tolist() mesh.textures._num_verts_per_mesh = mesh._num_verts_per_mesh.tolist() image = renderer(mesh) return image
def sphere_examples(bsdf, device="cuda", size=256, chunk_size=128, scale=100): from pytorch3d.pathtracer.shapes import Sphere from pytorch3d.pathtracer.bsdf import Diffuse from pytorch3d.pathtracer import pathtrace from pytorch3d.renderer import ( look_at_view_transform, OpenGLPerspectiveCameras, PointLights, ) import pytorch3d.pathtracer.integrators as integrators sphere = Sphere([0, 0, 0], 1, device=device) R, T = look_at_view_transform(dist=2., elev=0, azim=0) cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T) lights = PointLights(device=device, location=[[0., 1., 4.]], scale=scale) out = [] for basis in bsdf.bsdfs: expected = pathtrace( sphere, cameras=cameras, lights=lights, chunk_size=chunk_size, size=size, bsdf=basis, integrator=integrators.Direct(), device=device, silent=True, )[0] out.append(expected) return out
def sphere_render_bsdf(bsdf, integrator=None, device="cuda", size=256, chunk_size=128, scale=100): from pytorch3d.pathtracer.shapes import Sphere from pytorch3d.pathtracer.bsdf import Diffuse from pytorch3d.pathtracer import pathtrace from pytorch3d.renderer import ( look_at_view_transform, OpenGLPerspectiveCameras, PointLights, ) import pytorch3d.pathtracer.integrators as integrators sphere = Sphere([0, 0, 0], 1, device=device) R, T = look_at_view_transform(dist=2., elev=0, azim=0) cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T) lights = PointLights(device=device, location=[[0., 1., 4.]], scale=scale) if integrator is None: integrator = integrators.Direct() return pathtrace( sphere, cameras=cameras, lights=lights, chunk_size=chunk_size, size=size, bsdf=bsdf, integrator=integrator, device=device, silent=True, )[0]
def createRenderer(image_size,faces_per_pixel,lights_location): # Function: createRenderer # Inputs: image_size,faces_per_pixel,lights_location # Process: creates an image renderer # Output: returns renderer cameras = OpenGLPerspectiveCameras() #Settings for Raster raster_settings = RasterizationSettings( image_size=image_size, blur_radius=0.0, faces_per_pixel=faces_per_pixel, ) # We can add a point light in front of the object. lights = PointLights(location=(lights_location,)) created_renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings ), shader=HardPhongShader(cameras=cameras, lights=lights) ) return created_renderer
def visualize_pred(img, category, pred, image_name, mesh_path, down_sample_rate=8, device='cuda:0'): render_image_size = max(IMAGE_SIZES[category]) crop_size = IMAGE_SIZES[category] cameras = OpenGLPerspectiveCameras(device=device, fov=12.0) raster_settings = RasterizationSettings(image_size=render_image_size, blur_radius=0.0, faces_per_pixel=1, bin_size=0) raster_settings1 = RasterizationSettings(image_size=render_image_size // down_sample_rate, blur_radius=0.0, faces_per_pixel=1, bin_size=0) rasterizer = MeshRasterizer(cameras=cameras, raster_settings=raster_settings1) lights = PointLights(device=device, location=((2.0, 2.0, -2.0), )) phong_renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(device=device, lights=lights, cameras=cameras)) theta_pred = pred['theta'] elevation_pred = pred['elevation'] azimuth_pred = pred['azimuth'] distance_pred = pred['distance'] cad_idx = pred['cad_idx'] dx = pred['dx'] * down_sample_rate dy = pred['dy'] * down_sample_rate x3d, xface = load_off(mesh_path + '/%02d.off' % cad_idx) verts = torch.from_numpy(x3d).to(device) verts = pre_process_mesh_pascal(verts) faces = torch.from_numpy(xface).to(device) verts_rgb = torch.ones_like(verts)[None] textures = Textures(verts_rgb.to(device)) meshes = Meshes(verts=[verts], faces=[faces], textures=textures) img_ = get_img(theta_pred, elevation_pred, azimuth_pred, distance_pred, meshes, phong_renderer, crop_size, render_image_size, device) C = camera_position_from_spherical_angles(distance_pred, elevation_pred, azimuth_pred, degrees=False, device=device) # get_image = np.concatenate((img, alpha_merge_imgs(img, img_)), axis=1) img_ = shift_img(img_, dx, dy) get_image = alpha_merge_imgs(img, img_) img = Image.fromarray(get_image).save(image_name)
def render_with_batch_size(self, batch_size, dist, light_location, output_path): self.meshes = self.meshes.extend(batch_size) self.batch_size = batch_size elev = torch.linspace(0, 180, batch_size) azim = torch.linspace(-180, 180, batch_size) self.R, self.T = look_at_view_transform(dist=dist, elev=elev, azim=azim) self.cameras = OpenGLPerspectiveCameras(device=self.device, R=self.R, T=self.T) #set light locatioin self.light_location = light_location lights = PointLights(device=self.device, location=[self.light_location]) # call pytorch3d mesh renderer with shong shader renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=self.cameras, raster_settings=self.raster_settings), shader=TexturedSoftPhongShader(device=self.device, cameras=self.cameras, lights=lights)) images = renderer(self.meshes, cameras=self.cameras, lights=lights) for i in range(self.batch_size): img = images[i, ..., :3].cpu().numpy() * 255 img = img.astype('uint8') img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) cv2.imwrite(output_path + 'render-image-' + str(i) + '.png', img)
def init_camera(self, focal, fov): if fov is None: fov = 2 * np.arctan(self.size / (focal * 2)) * 180 / np.pi camera = OpenGLPerspectiveCameras(zfar=350, fov=fov, device=self.device) return camera
def load_perspective_cameras(): device = torch.device(config.cuda.device) return OpenGLPerspectiveCameras(device=device, degrees=True, fov=config.generate.fov, znear=config.generate.znear, zfar=config.generate.zfar)
def load_perspective_cameras(): device = torch.device(config.cuda.device) return OpenGLPerspectiveCameras(device=device, degrees=True, fov=30, znear=0.00001, zfar=10000)
class Renderer(torch.nn.Module): def __init__(self, image_size, device): super(Renderer, self).__init__() self.image_size = image_size R, T = look_at_view_transform(2.7, 0, 0, device=device) self.cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T) self.mesh_color = torch.FloatTensor(config.MESH_COLOR).to(device)[None, None, :] / 255.0 blend_params = BlendParams(sigma=1e-4, gamma=1e-4) raster_settings = RasterizationSettings( image_size=self.image_size, blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma, faces_per_pixel=100, ) self.silhouette_renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=self.cameras, raster_settings=raster_settings ), shader=SoftSilhouetteShader(blend_params=blend_params) ) raster_settings_color = RasterizationSettings( image_size=self.image_size, blur_radius=0.0, faces_per_pixel=1, ) lights = PointLights(device=device, location=[[0.0, 0.0, 3.0]]) self.color_renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=self.cameras, raster_settings=raster_settings_color ), shader=HardPhongShader( device=device, cameras=self.cameras, lights=lights, ) ) def forward(self, vertices, points, faces, render_texture=False): tex = torch.ones_like(vertices) * self.mesh_color # (1, V, 3) textures = Textures(verts_rgb=tex) mesh = Meshes(verts=vertices, faces=faces, textures=textures) sil_images = self.silhouette_renderer(mesh)[..., -1].unsqueeze(1) screen_size = torch.ones(vertices.shape[0], 2).to(vertices.device) * self.image_size proj_points = self.cameras.transform_points_screen(points, screen_size)[:, :, [1, 0]] if render_texture: color_image = self.color_renderer(mesh).permute(0, 3, 1, 2)[:, :3, :, :] return sil_images, proj_points, color_image else: return sil_images, proj_points
def createRenderer(device, camera, light, imageSize): ''' It creates a pytorch3D renderer with the given camera pose, light source and output image size. Parameters ---------- device : Device on which the renderer is created. camera : Camera pose. light : Position of the light source. imageSize : The size of the rendered image. Returns ------- renderer : Pytorch3D renderer. ''' if camera is None: camera = (2.0, -20.0, 180.0) if light is None: light = (0.0, 2.0, 0.0) # Initialize an OpenGL perspective camera. # With world coordinates +Y up, +X left and +Z into the screen. R, T = look_at_view_transform(camera[0], camera[1], camera[2]) cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T) # Define the settings for rasterization and shading. Here we set the output image to be of size # 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1 # and blur_radius=0.0. We also set bin_size and max_faces_per_bin to None which ensure that # the faster coarse-to-fine rasterization method is used. Refer to rasterize_meshes.py for # explanations of these parameters. Refer to docs/notes/renderer.md for an explanation of # the difference between naive and coarse-to-fine rasterization. raster_settings = RasterizationSettings( image_size=imageSize, blur_radius=0.0, faces_per_pixel=1, ) # Place a point light at -y direction. lights = PointLights(device=device, location=[[light[0], light[1], light[2]]]) # Create a phong renderer by composing a rasterizer and a shader. renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(device=device, cameras=cameras, lights=lights)) return renderer
def __init__(self, image_size, device): super(Renderer, self).__init__() self.image_size = image_size R, T = look_at_view_transform(2.7, 0, 0, device=device) self.cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T) self.mesh_color = torch.FloatTensor(config.MESH_COLOR).to(device)[None, None, :] / 255.0 blend_params = BlendParams(sigma=1e-4, gamma=1e-4) raster_settings = RasterizationSettings( image_size=self.image_size, blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma, faces_per_pixel=100, ) self.silhouette_renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=self.cameras, raster_settings=raster_settings ), shader=SoftSilhouetteShader(blend_params=blend_params) ) raster_settings_color = RasterizationSettings( image_size=self.image_size, blur_radius=0.0, faces_per_pixel=1, ) lights = PointLights(device=device, location=[[0.0, 0.0, 3.0]]) self.color_renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=self.cameras, raster_settings=raster_settings_color ), shader=HardPhongShader( device=device, cameras=self.cameras, lights=lights, ) )
def initialize_renderer(self): # Initialize an OpenGL perspective camera self.cameras = OpenGLPerspectiveCameras(device=self.device) self.raster_settings = RasterizationSettings( image_size = 512, blur_radius = 0.0, faces_per_pixel=2, ) self.set_phong_renderer([0.0,3.0,5.0])
def render(self, model_ids: Optional[List[str]] = None, categories: Optional[List[str]] = None, sample_nums: Optional[List[int]] = None, idxs: Optional[List[int]] = None, shader_type=HardPhongShader, device="cpu", **kwargs) -> torch.Tensor: """ If a list of model_ids are supplied, render all the objects by the given model_ids. If no model_ids are supplied, but categories and sample_nums are specified, randomly select a number of objects (number specified in sample_nums) in the given categories and render these objects. If instead a list of idxs is specified, check if the idxs are all valid and render models by the given idxs. Otherwise, randomly select a number (first number in sample_nums, default is set to be 1) of models from the loaded dataset and render these models. Args: model_ids: List[str] of model_ids of models intended to be rendered. categories: List[str] of categories intended to be rendered. categories and sample_nums must be specified at the same time. categories can be given in the form of synset offsets or labels, or a combination of both. sample_nums: List[int] of number of models to be randomly sampled from each category. Could also contain one single integer, in which case it will be broadcasted for every category. idxs: List[int] of indices of models to be rendered in the dataset. shader_type: Select shading. Valid options include HardPhongShader (default), SoftPhongShader, HardGouraudShader, SoftGouraudShader, HardFlatShader, SoftSilhouetteShader. device: torch.device on which the tensors should be located. **kwargs: Accepts any of the kwargs that the renderer supports. Returns: Batch of rendered images of shape (N, H, W, 3). """ paths = self._handle_render_inputs(model_ids, categories, sample_nums, idxs) meshes = load_objs_as_meshes(paths, device=device, load_textures=False) meshes.textures = Textures( verts_rgb=torch.ones_like(meshes.verts_padded(), device=device)) cameras = kwargs.get("cameras", OpenGLPerspectiveCameras()).to(device) renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=cameras, raster_settings=kwargs.get("raster_settings", RasterizationSettings()), ), shader=shader_type( device=device, cameras=cameras, lights=kwargs.get("lights", PointLights()).to(device), ), ) return renderer(meshes)
def __init__(self, cfgs): super().__init__() self.device = cfgs.get('device', 'cpu') self.image_size = cfgs.get('image_size', 64) self.min_depth = cfgs.get('min_depth', 0.9) self.max_depth = cfgs.get('max_depth', 1.1) self.rot_center_depth = cfgs.get('rot_center_depth', (self.min_depth + self.max_depth) / 2) self.border_depth = cfgs.get( 'border_depth', 0.3 * self.min_depth + 0.7 * self.max_depth) self.fov = cfgs.get('fov', 10) #### camera intrinsics # (u) (x) # d * K^-1 (v) = (y) # (1) (z) ## renderer for visualization R = [[[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]]] R = torch.FloatTensor(R).to(self.device) t = torch.zeros(1, 3, dtype=torch.float32).to(self.device) fx = (self.image_size - 1) / 2 / (math.tan( self.fov / 2 * math.pi / 180)) fy = (self.image_size - 1) / 2 / (math.tan( self.fov / 2 * math.pi / 180)) cx = (self.image_size - 1) / 2 cy = (self.image_size - 1) / 2 K = [[fx, 0., cx], [0., fy, cy], [0., 0., 1.]] K = torch.FloatTensor(K).to(self.device) self.inv_K = torch.inverse(K).unsqueeze(0) self.K = K.unsqueeze(0) # Initialize an OpenGL perspective camera. R = look_at_rotation(((0, 0, 0), ), at=((0, 0, 1), ), up=((0, -1, 0), )) cameras = OpenGLPerspectiveCameras(device=self.device, fov=self.fov, R=R) lights = DirectionalLights( ambient_color=((1.0, 1.0, 1.0), ), diffuse_color=((0.0, 0.0, 0.0), ), specular_color=((0.0, 0.0, 0.0), ), direction=((0, 1, 0), ), device=self.device, ) raster_settings = RasterizationSettings( image_size=self.image_size, blur_radius=0.0, faces_per_pixel=1, ) self.rasterizer_torch = MeshRasterizer(cameras=cameras, raster_settings=raster_settings)
def init_test(self): self.segmenter = Segment(self.device) up_line = 100 bt_line = 80 self.transfers = {} self.uv_creators = {} self.nsh_face_tris = {} self.nsh_meshes = {} self.nsh_face_meshes = {} for face_model in ['230']: self.transfers[face_model] = Shape_Transfer(face_model=face_model, device=self.device) self.uv_creators[face_model] = UVCreator( face_model=face_model, bfm_version=self.config.bfm_version, device=self.device) self.nsh_face_meshes[face_model] = meshio.Mesh( 'data/mesh/{}/nsh_bfm_face.obj'.format(face_model)) self.nsh_face_tris[face_model] = self.to_tensor( self.nsh_face_meshes[face_model].triangles, torch.int64) self.nsh_meshes[face_model] = meshio.Mesh( 'data/mesh/{}/nsh_std.obj'.format(face_model), group=True) self.up_line = int(up_line * (self.config.uv_size / 1024)) self.bt_line = int(bt_line * (self.config.uv_size / 1024)) self.eye_lm_idx = np.loadtxt('data/mesh/eye_lm_idx.txt', dtype=np.int32) self.cropper = ImageCropper(self.config.im_size, use_dlib=False) self.reconstructor = Deep3DFace(self.sess, self.graph) R, T = look_at_view_transform(10, 0, 0) self.cameras = OpenGLPerspectiveCameras(znear=0.001, zfar=30.0, aspect_ratio=1.0, fov=12.5936, degrees=True, R=R, T=T, device=self.device) raster_settings = RasterizationSettings(image_size=512, blur_radius=0.0, faces_per_pixel=1, bin_size=0, cull_backfaces=True) self.rasterizer = MeshRasterizer(cameras=self.cameras, raster_settings=raster_settings)
def build_renderer(_image_size): # Initialize an OpenGL perspective camera. cameras = OpenGLPerspectiveCameras(device=DEVICE, degrees=True, fov=FOV, znear=1e-4, zfar=100) raster_settings = RasterizationSettings(image_size=_image_size, blur_radius=0.0, faces_per_pixel=1, bin_size=0) lights = DirectionalLights(device=DEVICE, direction=((-40, 200, 100),), ambient_color=((0.5, 0.5, 0.5),), diffuse_color=((0.5, 0.5, 0.5),), specular_color=((0.0, 0.0, 0.0),), ) renderer = MeshRenderer(rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=TexturedSoftPhongShader(device=DEVICE, cameras=cameras, lights=lights)) return renderer
def renderMultiPose(mesh, render, batch_size=20, row: int = 4, col: int = 5, device='cuda'): ''' It generates multiple camera poses to render images with settings in the given renderer. The number of poses is the batch_size. Parameters ---------- mesh : Mesh to be rendered. render : Pytorch3D renderer. batch_size : optional The number of camera poses to be generated. The default is 20. row : int, optional The number of rows when show the images. The default is 4. col : int, optional The number of columns when show the images. The default is 5. device : optional The device where the rendering takes place. The default is 'cuda'. Returns ------- None. ''' # Create a batch of meshes by repeating the cow mesh and associated textures. # Meshes has a useful `extend` method which allows us do this very easily. # This also extends the textures. meshes = mesh.extend(batch_size) # Get a batch of viewing angles. elev = torch.linspace(0, 180, batch_size) azim = torch.linspace(-180, 180, batch_size) # All the cameras helper methods support mixed type inputs and broadcasting. So we can # view the camera from the same distance and specify dist=2.7 as a float, # and then specify elevation and azimuth angles for each viewpoint as tensors. R, T = look_at_view_transform(dist=2.0, elev=elev, azim=azim) cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T) # We can pass arbirary keyword arguments to the rasterizer/shader via the renderer # so the renderer does not need to be reinitialized if any of the settings change. images = render(meshes, cameras=cameras) image_grid(images.cpu().numpy(), rows=row, cols=col, rgb=True)
def test_nomal_rendering(self): device = torch.device('cuda:0') torch.cuda.set_device(device) # Set paths data_dir = Path(__file__).resolve().parent / 'data' data_dir.mkdir(exist_ok=True) obj_dir = Path( __file__).resolve().parent.parent / "docs/tutorials/data" obj_filename = obj_dir / "cow_mesh/cow.obj" # Load obj file mesh = load_objs_as_meshes([obj_filename], device=device) # try: # texture_image = mesh.textures.maps_padded() # except: # pass R, T = look_at_view_transform(2.55, 10, 180) cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T) # Define the settings for rasterization and shading. Here we set the output image to be of size # 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1 # and blur_radius=0.0. We also set bin_size and max_faces_per_bin to None which ensure that # the faster coarse-to-fine rasterization method is used. Refer to rasterize_meshes.py for # explanations of these parameters. Refer to docs/notes/renderer.md for an explanation of # the difference between naive and coarse-to-fine rasterization. raster_settings = RasterizationSettings( image_size=512, blur_radius=0.0, faces_per_pixel=1, perspective_correct=True, ) renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=NormalShader( device=device, cameras=cameras, )) images = renderer(mesh) # cv2.imshow('render_normal_texture.png', # ((255 * images[0, ..., :3]).squeeze().cpu().numpy().astype(np.uint8))[..., ::-1]) Image.fromarray( ((255 * images[0, ..., :3]).squeeze().cpu().numpy().astype( np.uint8))).save(str(data_dir / 'render_normal_texture.png')) # cv2.imwrite(str(data_dir / 'render_normal_texture.png'), # ((255 * images[0, ..., :3]).squeeze().cpu().numpy().astype(np.uint8))[..., ::-1]) self.assertTrue((data_dir / 'render_normal_texture.png').exists())
def render_with_different_azim_elev_size(self, dist, elev_size, azim_size, light_location, output_path): azims = torch.linspace(-180, 180, azim_size) elevs = torch.linspace(0, 180, elev_size) self.light_location = light_location lights = PointLights(device=self.device, location=[self.light_location]) index = 0 for elev in elevs: for azim in azims: R, T = look_at_view_transform(dist=dist, elev=elev, azim=azim) self.cameras = OpenGLPerspectiveCameras(device=self.device, R=R, T=T) renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=self.cameras, raster_settings=self.raster_settings), shader=TexturedSoftPhongShader(device=self.device, cameras=self.cameras, lights=lights)) images = renderer(self.meshes, cameras=self.cameras, lights=lights) img = images[0, ..., :3].cpu().numpy() * 255 img = img.astype('uint8') img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) extent = "{:06}".format(index) cv2.imwrite(output_path + 'frame-' + extent + '.render.png', img) # Save extrinsic M = np.zeros((4, 4)) M[:3, :3] = R.numpy() M[:3, 3] = T.numpy() M[3, :] = [0, 0, 0, 1] with open(output_path + "/frame-" + extent + ".pose.txt", "w") as f: np.savetxt(f, M[0, :], fmt="%.5f", newline=' ') f.write('\n') np.savetxt(f, M[1, :], fmt="%.5f", newline=' ') f.write('\n') np.savetxt(f, M[2, :], fmt="%.5f", newline=' ') f.write('\n') np.savetxt(f, M[3, :], fmt="%.5f", newline=' ') index += 1
def load_cameras(moon_view: MoonView): device = torch.device('cuda') eye = moon_view.eye at = moon_view.at up = moon_view.up R, T = look_at_view_transform(eye=((eye[0], eye[1], eye[2]), ), at=((at[0], at[1], at[2]), ), up=((up[0], up[1], up[2]), )) return OpenGLPerspectiveCameras(device=device, R=R, T=T, degrees=True, fov=moon_view.fov, znear=moon_view.znear, zfar=moon_view.zfar)
def visualize_gif(img, distance_ckpt, elevation_ckpt, azimuth_ckpt, theta_ckpt): cameras = OpenGLPerspectiveCameras(device=device, fov=12.0) raster_settings = RasterizationSettings(image_size=render_image_size, blur_radius=0.0, faces_per_pixel=1, bin_size=0) raster_settings1 = RasterizationSettings(image_size=render_image_size // down_smaple_rate, blur_radius=0.0, faces_per_pixel=1, bin_size=0) rasterizer = MeshRasterizer(cameras=cameras, raster_settings=raster_settings1) lights = PointLights(device=device, location=((2.0, 2.0, -2.0), )) phong_renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(device=device, lights=lights, cameras=cameras)) distance_ckpt = torch.from_numpy(np.array(distance_ckpt, dtype=np.float32)).to(device) elevation_ckpt = torch.from_numpy( np.array(elevation_ckpt, dtype=np.float32)).to(device) azimuth_ckpt = torch.from_numpy(np.array(azimuth_ckpt, dtype=np.float32)).to(device) theta_ckpt = torch.from_numpy(np.array(theta_ckpt, dtype=np.float32)).to(device) img_ = get_img(theta_pred, elevation_pred, azimuth_pred, distance_pred, phong_renderer) C = camera_position_from_spherical_angles(distance_pred, elevation_pred, azimuth_pred, degrees=False, device=device) get_images = [] for i in range(10): get_image = np.concatenate((img, alpha_merge_imgs(img, img_[i])), axis=1) get_images.append(Image.fromarray(get_image)) get_images[0].save('optim.gif', save_all=True, append_images=get_images[1::])
def forward(self, z): deform_verts = self.layers(z) new_meshes = self.src_meshes.offset_verts(deform_verts.view(-1, 3)) # elev = torch.FloatTensor(z.shape[0]).uniform_(-5, 5) r = 60 azim = torch.FloatTensor(z.shape[0]).uniform_(180 - r, 180 + r) elev = torch.FloatTensor(z.shape[0]).uniform_(0, 0) # azim = torch.FloatTensor(z.shape[0]).uniform_(160, 200) # azim = torch.FloatTensor(z.shape[0]).uniform_(-180, 180) R, T = look_at_view_transform(dist=2.7, elev=elev, azim=azim) self.cameras = OpenGLPerspectiveCameras(R=R, T=T, device=device) images_predicted = self.renderer_silhouette(new_meshes, cameras=self.cameras, lights=self.lights) predicted_silhouette = images_predicted[..., 3] return new_meshes, predicted_silhouette
def render(self, dist, light_location, azim, elev, image_file): self.R, self.T = look_at_view_transform(dist=dist, elev=elev, azim=azim) self.cameras = OpenGLPerspectiveCameras(device=self.device, R=self.R, T=self.T) self.light_location = light_location lights = PointLights(device=self.device, location=[self.light_location]) # call pytorch3d mesh renderer with shong shader renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=self.cameras, raster_settings=self.raster_settings), shader=TexturedSoftPhongShader(device=self.device, cameras=self.cameras, lights=lights)) images = renderer(self.meshes, cameras=self.cameras, lights=lights) img = images[0, ..., :3].cpu().numpy() * 255 img = img.astype('uint8') img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) cv2.imwrite(image_file, img)
def render_mesh(mesh, R, T, device, img_size=512, silhouette=False): cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T) if silhouette: blend_params = BlendParams(sigma=1e-4, gamma=1e-4) raster_settings = RasterizationSettings( image_size=img_size, blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma, faces_per_pixel=100, ) renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings ), shader=SoftSilhouetteShader(blend_params=blend_params) ) else: raster_settings = RasterizationSettings( image_size=img_size, blur_radius=0.0, faces_per_pixel=1, ) lights = PointLights(device=device, location=[[0.0, 5.0, -10.0]]) renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings ), shader=SoftPhongShader( device=device, cameras=cameras, lights=lights ) ) rendered_images = renderer(mesh, cameras=cameras) return rendered_images