def set_renderer(image_size=512, use_sfm=False): # Setup device = torch.device("cuda:0") torch.cuda.set_device(device) # Initialize an OpenGL perspective camera. R, T = look_at_view_transform(2.0, 0, 180) if use_sfm: cameras = SfMPerspectiveCameras(focal_length=580.0, device=device, R=R, T=T) else: cameras = OpenGLOrthographicCameras(device=device, R=R, T=T) raster_settings = RasterizationSettings(image_size=image_size, blur_radius=0.0, faces_per_pixel=1, bin_size=None, max_faces_per_bin=None) lights = PointLights(device=device, location=((2.0, 2.0, 2.0), )) rasterizer = MeshRasterizer(cameras=cameras, raster_settings=raster_settings) shader = HardPhongShader(device=device, cameras=cameras, lights=lights) if use_sfm: renderer = MeshRendererWithDepth(rasterizer=rasterizer, shader=shader) else: renderer = MeshRenderer(rasterizer=rasterizer, shader=shader) return renderer
def set_renderer(): # Setup device = torch.device("cuda:0") torch.cuda.set_device(device) # Initialize an OpenGL perspective camera. R, T = look_at_view_transform(2.0, 0, 180) cameras = OpenGLOrthographicCameras(device=device, R=R, T=T) raster_settings = RasterizationSettings( image_size=512, blur_radius=0.0, faces_per_pixel=1, bin_size = None, max_faces_per_bin = None ) lights = PointLights(device=device, location=((2.0, 2.0, 2.0),)) renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings ), shader=HardPhongShader( device=device, cameras=cameras, lights=lights ) ) return renderer
def create_renderer(self): self.num_angles = self.config.num_angles azim = torch.linspace(-1 * self.config.angle_range, self.config.angle_range, self.num_angles) R, T = look_at_view_transform(dist=1.0, elev=0, azim=azim) T[:, 1] = -85 T[:, 2] = 200 cameras = FoVPerspectiveCameras(device=self.device, R=R, T=T) raster_settings = RasterizationSettings( image_size=self.config.img_size, blur_radius=0.0, faces_per_pixel=1, ) lights = PointLights(device=self.device, location=[[0.0, 85, 100.0]]) renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(device=self.device, cameras=cameras, lights=lights)) return renderer
def visualize_pred(img, category, pred, image_name, mesh_path, down_sample_rate=8, device='cuda:0'): render_image_size = max(IMAGE_SIZES[category]) crop_size = IMAGE_SIZES[category] cameras = OpenGLPerspectiveCameras(device=device, fov=12.0) raster_settings = RasterizationSettings(image_size=render_image_size, blur_radius=0.0, faces_per_pixel=1, bin_size=0) raster_settings1 = RasterizationSettings(image_size=render_image_size // down_sample_rate, blur_radius=0.0, faces_per_pixel=1, bin_size=0) rasterizer = MeshRasterizer(cameras=cameras, raster_settings=raster_settings1) lights = PointLights(device=device, location=((2.0, 2.0, -2.0), )) phong_renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(device=device, lights=lights, cameras=cameras)) theta_pred = pred['theta'] elevation_pred = pred['elevation'] azimuth_pred = pred['azimuth'] distance_pred = pred['distance'] cad_idx = pred['cad_idx'] dx = pred['dx'] * down_sample_rate dy = pred['dy'] * down_sample_rate x3d, xface = load_off(mesh_path + '/%02d.off' % cad_idx) verts = torch.from_numpy(x3d).to(device) verts = pre_process_mesh_pascal(verts) faces = torch.from_numpy(xface).to(device) verts_rgb = torch.ones_like(verts)[None] textures = Textures(verts_rgb.to(device)) meshes = Meshes(verts=[verts], faces=[faces], textures=textures) img_ = get_img(theta_pred, elevation_pred, azimuth_pred, distance_pred, meshes, phong_renderer, crop_size, render_image_size, device) C = camera_position_from_spherical_angles(distance_pred, elevation_pred, azimuth_pred, degrees=False, device=device) # get_image = np.concatenate((img, alpha_merge_imgs(img, img_)), axis=1) img_ = shift_img(img_, dx, dy) get_image = alpha_merge_imgs(img, img_) img = Image.fromarray(get_image).save(image_name)
def createRenderer(image_size,faces_per_pixel,lights_location): # Function: createRenderer # Inputs: image_size,faces_per_pixel,lights_location # Process: creates an image renderer # Output: returns renderer cameras = OpenGLPerspectiveCameras() #Settings for Raster raster_settings = RasterizationSettings( image_size=image_size, blur_radius=0.0, faces_per_pixel=faces_per_pixel, ) # We can add a point light in front of the object. lights = PointLights(location=(lights_location,)) created_renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings ), shader=HardPhongShader(cameras=cameras, lights=lights) ) return created_renderer
def createRenderer(device, camera, light, imageSize): ''' It creates a pytorch3D renderer with the given camera pose, light source and output image size. Parameters ---------- device : Device on which the renderer is created. camera : Camera pose. light : Position of the light source. imageSize : The size of the rendered image. Returns ------- renderer : Pytorch3D renderer. ''' if camera is None: camera = (2.0, -20.0, 180.0) if light is None: light = (0.0, 2.0, 0.0) # Initialize an OpenGL perspective camera. # With world coordinates +Y up, +X left and +Z into the screen. R, T = look_at_view_transform(camera[0], camera[1], camera[2]) cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T) # Define the settings for rasterization and shading. Here we set the output image to be of size # 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1 # and blur_radius=0.0. We also set bin_size and max_faces_per_bin to None which ensure that # the faster coarse-to-fine rasterization method is used. Refer to rasterize_meshes.py for # explanations of these parameters. Refer to docs/notes/renderer.md for an explanation of # the difference between naive and coarse-to-fine rasterization. raster_settings = RasterizationSettings( image_size=imageSize, blur_radius=0.0, faces_per_pixel=1, ) # Place a point light at -y direction. lights = PointLights(device=device, location=[[light[0], light[1], light[2]]]) # Create a phong renderer by composing a rasterizer and a shader. renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(device=device, cameras=cameras, lights=lights)) return renderer
def render_phong(self, meshes): lights = PointLights(device=self.device, location=((0.0, 0.0, 2.0), )) raster_settings = RasterizationSettings( image_size=self.resulution, blur_radius=0.0, faces_per_pixel=1, ) phong_renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=self.cameras, raster_settings=raster_settings), shader=HardPhongShader(device=self.device, lights=lights)) return phong_renderer(meshes_world=meshes)
def set_phong_renderer(self, light_location): # Place a point light in front of the object self.light_location = light_location lights = PointLights(device=self.device, location=[light_location]) # Create a phong renderer by composing a rasterizer and a shader self.phong_renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=self.cameras, raster_settings=self.raster_settings ), shader=HardPhongShader(device=self.device, lights=lights) )
def render(mesh, model_id, shapenet_dataset, device, camera=None): # Rendering settings. # camera_distance = 1 # camera_elevation = 0.5 + 100 * random.random() # camera_azimuth = 30 + 90 * random.random() # R, T = look_at_view_transform(camera_distance, camera_elevation, camera_azimuth) # camera = FoVPerspectiveCameras(R=R, T=T, device=device) # raster_settings = RasterizationSettings(image_size=512) # lights = PointLights(location=torch.tensor([0.0, 1.0, -2.0], device=device)[None],device=device) # #rendering_settings = cameras, raster_settings, lights # image = shapenet_dataset.render( # model_ids=[model_id], # device=device, # cameras=camera, # raster_settings=raster_settings, # lights=lights, # )[..., :3] if not camera: camera_elevation = 0 + 180 * torch.rand( (1)) #torch.linspace(0, 180, batch_size) camera_azimuth = -180 + 2 * 180 * torch.rand( (1)) #torch.linspace(-180, 180, batch_size) #R, T = look_at_view_transform(camera_distance, camera_elevation, camera_azimuth) R, T = look_at_view_transform(1.9, camera_elevation, camera_azimuth) camera = FoVPerspectiveCameras(R=R, T=T, device=device) camera.eval() #necessary ? raster_settings = RasterizationSettings(image_size=224) # TODO ????? lights = PointLights(location=torch.tensor([0.0, 1.0, -2.0], device=device)[None], device=device) renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=camera, raster_settings=raster_settings), shader=HardPhongShader(device=device, cameras=camera)) renderer.eval() #rendering_settings = cameras, raster_settings, lights #image = shapenet_dataset.render( # model_ids=[model_id], # device=device, # cameras=camera, # raster_settings=raster_settings, # lights=lights, #)[..., :3] image = renderer(mesh)[..., :3] #plt.imshow(image.squeeze().detach().cpu().numpy()) #plt.show() image = image.permute(0, 3, 1, 2) return image, camera #TODO batch of images
def visualize_gif(img, distance_ckpt, elevation_ckpt, azimuth_ckpt, theta_ckpt): cameras = OpenGLPerspectiveCameras(device=device, fov=12.0) raster_settings = RasterizationSettings(image_size=render_image_size, blur_radius=0.0, faces_per_pixel=1, bin_size=0) raster_settings1 = RasterizationSettings(image_size=render_image_size // down_smaple_rate, blur_radius=0.0, faces_per_pixel=1, bin_size=0) rasterizer = MeshRasterizer(cameras=cameras, raster_settings=raster_settings1) lights = PointLights(device=device, location=((2.0, 2.0, -2.0), )) phong_renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(device=device, lights=lights, cameras=cameras)) distance_ckpt = torch.from_numpy(np.array(distance_ckpt, dtype=np.float32)).to(device) elevation_ckpt = torch.from_numpy( np.array(elevation_ckpt, dtype=np.float32)).to(device) azimuth_ckpt = torch.from_numpy(np.array(azimuth_ckpt, dtype=np.float32)).to(device) theta_ckpt = torch.from_numpy(np.array(theta_ckpt, dtype=np.float32)).to(device) img_ = get_img(theta_pred, elevation_pred, azimuth_pred, distance_pred, phong_renderer) C = camera_position_from_spherical_angles(distance_pred, elevation_pred, azimuth_pred, degrees=False, device=device) get_images = [] for i in range(10): get_image = np.concatenate((img, alpha_merge_imgs(img, img_[i])), axis=1) get_images.append(Image.fromarray(get_image)) get_images[0].save('optim.gif', save_all=True, append_images=get_images[1::])
def __init__(self, image_size, device): super(Renderer, self).__init__() self.image_size = image_size R, T = look_at_view_transform(2.7, 0, 0, device=device) self.cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T) self.mesh_color = torch.FloatTensor(config.MESH_COLOR).to(device)[None, None, :] / 255.0 blend_params = BlendParams(sigma=1e-4, gamma=1e-4) raster_settings = RasterizationSettings( image_size=self.image_size, blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma, faces_per_pixel=100, ) self.silhouette_renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=self.cameras, raster_settings=raster_settings ), shader=SoftSilhouetteShader(blend_params=blend_params) ) raster_settings_color = RasterizationSettings( image_size=self.image_size, blur_radius=0.0, faces_per_pixel=1, ) lights = PointLights(device=device, location=[[0.0, 0.0, 3.0]]) self.color_renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=self.cameras, raster_settings=raster_settings_color ), shader=HardPhongShader( device=device, cameras=self.cameras, lights=lights, ) )
def create_renderer(self): self.num_angles_train = self.config.num_angles_train self.num_angles_test = self.config.num_angles_test azim_train = torch.linspace(-1 * self.config.angle_range_train, self.config.angle_range_train, self.num_angles_train) azim_test = torch.linspace(-1 * self.config.angle_range_test, self.config.angle_range_test, self.num_angles_test) # Cameras for SMPL meshes: camera_dist = 2.2 R, T = look_at_view_transform(camera_dist, 6, azim_train) train_cameras = FoVPerspectiveCameras(device=self.device, R=R, T=T) self.train_cameras = train_cameras R, T = look_at_view_transform(camera_dist, 6, azim_test) test_cameras = FoVPerspectiveCameras(device=self.device, R=R, T=T) self.test_cameras = test_cameras raster_settings = RasterizationSettings( image_size=self.config.img_size, blur_radius=0.0, faces_per_pixel=1, ) lights = PointLights(device=self.device, location=[[0.0, 85, 100.0]]) renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=train_cameras, raster_settings=raster_settings ), shader=HardPhongShader( device=self.device, cameras=train_cameras, lights=lights ) ) return renderer
# Create a silhouette mesh renderer by composing a rasterizer and a shader. silhouette_renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=SoftSilhouetteShader(blend_params=blend_params)) # We will also create a phong renderer. This is simpler and only needs to render one face per pixel. raster_settings = RasterizationSettings(image_size=256, blur_radius=0.0, faces_per_pixel=1, bin_size=0) # We can add a point light in front of the object. lights = PointLights(device=device, location=((2.0, 2.0, -2.0), )) phong_renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(device=device, lights=lights)) # Select the viewpoint using spherical angles viewpoint = [1.5, 240.0, 10.0] # distance, elevation, azimuth, stuck.. #viewpoint = [1.5, 140.0, 10.0] # distance, elevation, azimuth, ok... #viewpoint = [1.5, 160.0, 10.0] # distance, elevation, azimuth, ok... # Get the position of the camera based on the spherical angles R, T = look_at_view_transform(viewpoint[0], viewpoint[1], viewpoint[2], device=device) # Render the teapot providing the values of R and T. silhouete = silhouette_renderer(meshes_world=teapot_mesh, R=R, T=T) image = phong_renderer(meshes_world=teapot_mesh, R=R, T=T)
def batch_render( verts, faces, faces_per_pixel=10, K=None, rot=None, trans=None, colors=None, color=(0.53, 0.53, 0.8), # light_purple ambient_col=0.5, specular_col=0.2, diffuse_col=0.3, face_colors=None, # color = (0.74117647, 0.85882353, 0.65098039), # light_blue image_sizes=None, out_res=512, bin_size=0, shading="soft", mode="rgb", blend_gamma=1e-4, min_depth=None, ): device = torch.device("cuda:0") K = K.to(device) width, height = image_sizes[0] out_size = int(max(image_sizes[0])) raster_settings = RasterizationSettings( image_size=out_size, blur_radius=0.0, faces_per_pixel=faces_per_pixel, bin_size=bin_size, ) fx = K[:, 0, 0] fy = K[:, 1, 1] focals = torch.stack([fx, fy], 1) px = K[:, 0, 2] py = K[:, 1, 2] principal_point = torch.stack([width - px, height - py], 1) if rot is None: rot = torch.eye(3).unsqueeze(0).to(device) if trans is None: trans = torch.zeros(3).unsqueeze(0).to(device) cameras = PerspectiveCameras( device=device, focal_length=focals, principal_point=principal_point, image_size=[(out_size, out_size) for _ in range(len(verts))], R=rot, T=trans, ) if mode == "rgb": lights = PointLights(device=device, location=[[0.0, 0.0, -3.0]]) lights = DirectionalLights( device=device, direction=((0.6, -0.6, -0.6), ), ambient_color=((ambient_col, ambient_col, ambient_col), ), diffuse_color=((diffuse_col, diffuse_col, diffuse_col), ), specular_color=((specular_col, specular_col, specular_col), ), ) if shading == "soft": shader = SoftPhongShader(device=device, cameras=cameras, lights=lights) elif shading == "hard": shader = HardPhongShader(device=device, cameras=cameras, lights=lights) else: raise ValueError( f"Shading {shading} for mode rgb not in [sort|hard]") elif mode == "silh": blend_params = BlendParams(sigma=1e-4, gamma=1e-4) shader = SoftSilhouetteShader(blend_params=blend_params) elif shading == "faceidx": shader = FaceIdxShader() elif (mode == "facecolor") and (shading == "hard"): shader = FaceColorShader(face_colors=face_colors) elif (mode == "facecolor") and (shading == "soft"): shader = SoftFaceColorShader(face_colors=face_colors, blend_gamma=blend_gamma) else: raise ValueError( f"Unhandled mode {mode} and shading {shading} combination") renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=shader, ) if min_depth is not None: verts = torch.cat([verts[:, :, :2], verts[:, :, 2:].clamp(min_depth)], 2) if mode == "rgb": if colors is None: colors = get_colors(verts, color) tex = textures.TexturesVertex(verts_features=colors) meshes = Meshes(verts=verts, faces=faces, textures=tex) elif mode in ["silh", "facecolor"]: meshes = Meshes(verts=verts, faces=faces) else: raise ValueError(f"Render mode {mode} not in [rgb|silh]") square_images = renderer(meshes, cameras=cameras) square_images = torch.flip(square_images, (1, 2)) height_off = abs(int(width - height)) if width > height: images = square_images[:, height_off:, :] else: images = square_images[:, :, height_off:] return images
raster_settings = RasterizationSettings( image_size=image_size, blur_radius=0.0, faces_per_pixel=1, bin_size=None, ) lights = PointLights(device=device, ambient_color=((0.5, 0.5, 0.5), ), location=((0.0, 0.0, 1e5), )) renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(device=device, cameras=cameras, lights=lights)) #-----render----- class AverageMeter(object): """Computes and stores the average and current value""" def __init__(self, name, fmt=':f'): self.name = name self.fmt = fmt self.reset() def reset(self): self.val = 0 self.avg = 0 self.sum = 0
def initRender(self, method, image_size): cameras = OpenGLPerspectiveCameras(device=self.device, fov=15) if (method == "soft-silhouette"): blend_params = BlendParams(sigma=1e-7, gamma=1e-7) raster_settings = RasterizationSettings( image_size=image_size, blur_radius=np.log(1. / 1e-7 - 1.) * blend_params.sigma, faces_per_pixel=self.faces_per_pixel) renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=SoftSilhouetteShader(blend_params=blend_params)) elif (method == "hard-silhouette"): blend_params = BlendParams(sigma=1e-7, gamma=1e-7) raster_settings = RasterizationSettings( image_size=image_size, blur_radius=np.log(1. / 1e-7 - 1.) * blend_params.sigma, faces_per_pixel=1) renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=SoftSilhouetteShader(blend_params=blend_params)) elif (method == "soft-depth"): # Soft Rasterizer - from https://github.com/facebookresearch/pytorch3d/issues/95 #blend_params = BlendParams(sigma=1e-7, gamma=1e-7) blend_params = BlendParams(sigma=1e-3, gamma=1e-4) raster_settings = RasterizationSettings( image_size=image_size, #blur_radius= np.log(1. / 1e-7 - 1.) * blend_params.sigma, blur_radius=np.log(1. / 1e-3 - 1.) * blend_params.sigma, faces_per_pixel=self.faces_per_pixel) renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=SoftDepthShader(blend_params=blend_params)) elif (method == "hard-depth"): raster_settings = RasterizationSettings(image_size=image_size, blur_radius=0, faces_per_pixel=20) renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardDepthShader()) elif (method == "blurry-depth"): # Soft Rasterizer - from https://github.com/facebookresearch/pytorch3d/issues/95 blend_params = BlendParams(sigma=1e-4, gamma=1e-4) raster_settings = RasterizationSettings( image_size=image_size, blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma, faces_per_pixel=self.faces_per_pixel) renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=SoftDepthShader(blend_params=blend_params)) elif (method == "soft-phong"): blend_params = BlendParams(sigma=1e-3, gamma=1e-3) raster_settings = RasterizationSettings( image_size=image_size, blur_radius=np.log(1. / 1e-3 - 1.) * blend_params.sigma, faces_per_pixel=self.faces_per_pixel) # lights = DirectionalLights(device=self.device, # ambient_color=[[0.25, 0.25, 0.25]], # diffuse_color=[[0.6, 0.6, 0.6]], # specular_color=[[0.15, 0.15, 0.15]], # direction=[[0.0, 1.0, 0.0]]) lights = DirectionalLights(device=self.device, direction=[[0.0, 1.0, 0.0]]) renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=SoftPhongShader(device=self.device, blend_params=blend_params, lights=lights)) elif (method == "hard-phong"): blend_params = BlendParams(sigma=1e-8, gamma=1e-8) raster_settings = RasterizationSettings(image_size=image_size, blur_radius=0.0, faces_per_pixel=1) lights = DirectionalLights(device=self.device, ambient_color=[[0.25, 0.25, 0.25]], diffuse_color=[[0.6, 0.6, 0.6]], specular_color=[[0.15, 0.15, 0.15]], direction=[[-1.0, -1.0, 1.0]]) renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(device=self.device, lights=lights)) else: print("Unknown render method!") return None return renderer
def main(): # Set the cuda device device = torch.device("cuda:0") torch.cuda.set_device(device) mesh = load_objs_as_meshes(obj_paths, device=device) texture_image = mesh.textures.maps_padded() cameras = OpenGLRealPerspectiveCameras( focal_length=((K[0, 0], K[1, 1]), ), # Nx2 principal_point=((K[0, 2], K[1, 2]), ), # Nx2 x0=0, y0=0, w=H, h=H, # HEIGHT, znear=ZNEAR, zfar=ZFAR, device=device, ) # To blend the 100 faces we set a few parameters which control the opacity and the sharpness of # edges. Refer to blending.py for more details. blend_params = BlendParams(sigma=1e-4, gamma=1e-4, background_color=(0.0, 0.0, 0.0)) # Define the settings for rasterization and shading. Here we set the output image to be of size # 640x640. To form the blended image we use 100 faces for each pixel. Refer to rasterize_meshes.py # for an explanation of this parameter. silhouette_raster_settings = RasterizationSettings( image_size=IMG_SIZE, # longer side or scaled longer side blur_radius=np.log(1.0 / 1e-4 - 1.0) * blend_params.sigma, faces_per_pixel= 100, # the nearest faces_per_pixel points along the z-axis. bin_size=0, ) # Create a silhouette mesh renderer by composing a rasterizer and a shader. silhouette_renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=silhouette_raster_settings), shader=SoftSilhouetteShader(blend_params=blend_params), ) # We will also create a phong renderer. This is simpler and only needs to render one face per pixel. phong_raster_settings = RasterizationSettings(image_size=IMG_SIZE, blur_radius=0.0, faces_per_pixel=1, bin_size=0) # We can add a point light in front of the object. lights = PointLights(device=device, location=((2.0, 2.0, -2.0), )) phong_renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=phong_raster_settings), shader=HardPhongShader(device=device, cameras=cameras, lights=lights, blend_params=blend_params), ) batch_R = torch.tensor(np.stack([R]), device=device, dtype=torch.float32).permute(0, 2, 1) # Bx3x3 batch_T = torch.tensor(np.stack([t]), device=device, dtype=torch.float32) # Bx3 silhouete = silhouette_renderer(meshes_world=mesh, R=batch_R, T=batch_T) image_ref = phong_renderer(meshes_world=mesh, R=batch_R, T=batch_T) # crop results silhouete = silhouete[:, :H, :W, :].cpu().numpy() image_ref = image_ref[:, :H, :W, :3].cpu().numpy() pred_images = image_ref opengl = mmcv.imread(osp.join("Render/OpenGL.png"), "color") / 255.0 for i in range(pred_images.shape[0]): pred_mask = silhouete[i, :, :, 3].astype("float32") print("num rendered images", pred_images.shape[0]) image = pred_images[i] diff_opengl = np.abs(opengl[:, :, ::-1].astype("float32") - image.astype("float32")) print("image", image.shape, image.min(), image.max()) print("dr mask area: ", pred_mask.sum()) print_stat(pred_mask, "pred_mask") show_ims = [image, diff_opengl, opengl[:, :, ::-1]] show_titles = ["image", "diff_opengl", "opengl"] grid_show(show_ims, show_titles, row=1, col=3)
def run_on_image(self, image, id_str, gt_verts, gt_faces): deprocess = imagenet_deprocess(rescale_image=False) with torch.no_grad(): voxel_scores, meshes_pred = self.predictor(image) sid, mid, iid = id_str.split('-') iid = int(iid) #Transform vertex space metadata_path = os.path.join('./datasets/shapenet/ShapeNetV1processed', sid, mid, "metadata.pt") metadata = torch.load(metadata_path) K = metadata["intrinsic"] RTs = metadata["extrinsics"] rot_y_90 = torch.tensor([[0, 0, 1, 0], [0, 1, 0, 0], [-1, 0, 0, 0], [0, 0, 0, 1]]).to(RTs) mesh = meshes_pred[-1][0] #For some strange reason all classes (expect vehicle class) require a 90 degree rotation about the y-axis #for the GT mesh invRT = torch.inverse(RTs[iid].mm(rot_y_90)) invRT_no_rot = torch.inverse(RTs[iid]) mesh._verts_list[0] = project_verts(mesh._verts_list[0], invRT.cpu()) #Get look at view extrinsics render_metadata_path = os.path.join( 'datasets/shapenet/ShapeNetRenderingExtrinsics', sid, mid, 'rendering_metadata.pt') render_metadata = torch.load(render_metadata_path) render_RTs = render_metadata['extrinsics'] plt.figure(figsize=(10, 10)) R = render_RTs[iid][:3, :3].unsqueeze(0) T = render_RTs[iid][:3, 3].unsqueeze(0) cameras = OpenGLPerspectiveCameras(R=R, T=T) #Phong Renderer lights = PointLights(location=[[0.0, 0.0, -3.0]]) raster_settings = RasterizationSettings(image_size=256, blur_radius=0.0, faces_per_pixel=1, bin_size=0) phong_renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(lights=lights)) #Silhouette Renderer blend_params = BlendParams(sigma=1e-4, gamma=1e-4) raster_settings = RasterizationSettings( image_size=256, blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma, faces_per_pixel=50, ) silhouette_renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=SoftSilhouetteShader(blend_params=blend_params)) verts, faces = mesh.get_mesh_verts_faces(0) verts_rgb = torch.ones_like(verts)[None] textures = Textures(verts_rgb=verts_rgb) mesh.textures = textures verts_rgb = torch.ones_like(gt_verts)[None] textures = Textures(verts_rgb=verts_rgb) #Invert without the rotation for the vehicle class if sid == '02958343': gt_verts = project_verts(gt_verts, invRT_no_rot.cpu()) else: gt_verts = project_verts(gt_verts, invRT.cpu()) gt_mesh = Meshes(verts=[gt_verts], faces=[gt_faces], textures=textures) img = image_to_numpy(deprocess(image[0])) mesh_image = phong_renderer(meshes_world=mesh, R=R, T=T) gt_silh_image = (silhouette_renderer(meshes_world=gt_mesh, R=R, T=T) > 0).float() silhouette_image = (silhouette_renderer(meshes_world=mesh, R=R, T=T) > 0).float() plt.subplot(2, 2, 1) plt.imshow(img) plt.title('input image') plt.subplot(2, 2, 2) plt.imshow(mesh_image[0, ..., :3].cpu().numpy()) plt.title('rendered mesh') plt.subplot(2, 2, 3) plt.imshow(gt_silh_image[0, ..., 3].cpu().numpy()) plt.title('silhouette of gt mesh') plt.subplot(2, 2, 4) plt.imshow(silhouette_image[0, ..., 3].cpu().numpy()) plt.title('silhouette of rendered mesh') plt.show() #plt.savefig('./output_demo/figures/'+id_str+'.png') vis_utils.visualize_prediction(id_str, img, mesh, self.output_dir)
# Renderer setting cameras = FoVOrthographicCameras(device=device) lights = PointLights( ambient_color=((0.99, 0.99, 0.99), ), diffuse_color=((0, 0, 0), ), specular_color=((0, 0, 0), ), device=device, ) raster_settings = RasterizationSettings(image_size=image_size) blend_params = BlendParams(background_color=(0.0, 0.0, 0.0), sigma=2e-4) renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(device=device, cameras=cameras, lights=lights, blend_params=blend_params), ) soft_raster_settings = RasterizationSettings( image_size=image_size, faces_per_pixel=3, blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma) renderer_silhouette = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=soft_raster_settings), shader=SoftSilhouetteShader(blend_params=blend_params)) # Optimization parameters N = len(meshes[0].verts_packed()) offset_values = torch.zeros((len(meshes), 2), device=device,