def __init__(self, image_size): super(Renderer, self).__init__() self.image_size = image_size self.dog_obj = load_objs_as_meshes(['data/dog_B/dog_B/dog_B_tpose.obj']) raster_settings = RasterizationSettings( image_size=self.image_size, blur_radius=0.0, faces_per_pixel=1, bin_size=None ) R, T = look_at_view_transform(2.7, 0, 0) cameras = OpenGLPerspectiveCameras(device=R.device, R=R, T=T) lights = PointLights(device=R.device, location=[[0.0, 1.0, 0.0]]) self.renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings ), shader=SoftPhongShader( device=R.device, cameras=cameras, lights=lights ) )
def set_renderer(): # Setup device = torch.device("cuda:0") torch.cuda.set_device(device) # Initialize an OpenGL perspective camera. R, T = look_at_view_transform(2.0, 0, 180) cameras = OpenGLOrthographicCameras(device=device, R=R, T=T) raster_settings = RasterizationSettings( image_size=512, blur_radius=0.0, faces_per_pixel=1, bin_size = None, max_faces_per_bin = None ) lights = PointLights(device=device, location=((2.0, 2.0, 2.0),)) renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings ), shader=HardPhongShader( device=device, cameras=cameras, lights=lights ) ) return renderer
def define_render(num): shapenet_cam_params_file = '../data/metadata/rendering_metadata.json' with open(shapenet_cam_params_file) as f: shapenet_cam_params = json.load(f) param_num = num R, T = look_at_view_transform( dist=shapenet_cam_params["distance"][param_num] * 5, elev=shapenet_cam_params["elevation"][param_num], azim=shapenet_cam_params["azimuth"][param_num]) cameras = FoVPerspectiveCameras( device=device, R=R, T=T, fov=shapenet_cam_params["field_of_view"][param_num]) raster_settings = RasterizationSettings( image_size=512, blur_radius=0.0, faces_per_pixel=1, ) lights = PointLights(device=device, location=[[0.0, 0.0, -3.0]]) renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=SoftPhongShader(device=device, cameras=cameras, lights=lights)) return renderer
def sphere_render_bsdf(bsdf, integrator=None, device="cuda", size=256, chunk_size=128, scale=100): from pytorch3d.pathtracer.shapes import Sphere from pytorch3d.pathtracer.bsdf import Diffuse from pytorch3d.pathtracer import pathtrace from pytorch3d.renderer import ( look_at_view_transform, OpenGLPerspectiveCameras, PointLights, ) import pytorch3d.pathtracer.integrators as integrators sphere = Sphere([0, 0, 0], 1, device=device) R, T = look_at_view_transform(dist=2., elev=0, azim=0) cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T) lights = PointLights(device=device, location=[[0., 1., 4.]], scale=scale) if integrator is None: integrator = integrators.Direct() return pathtrace( sphere, cameras=cameras, lights=lights, chunk_size=chunk_size, size=size, bsdf=bsdf, integrator=integrator, device=device, silent=True, )[0]
def rotate_mesh_around_axis( mesh, rot: list, renderer, dist: float = 3.5, save: str = "", device: str = "" ): if not device: device = torch.cuda.current_device() rot_x = RotateAxisAngle(rot[0], "X", device=device) rot_y = RotateAxisAngle(rot[1], "Y", device=device) rot_z = RotateAxisAngle(rot[2], "Z", device=device) rot = Transform3d(device=device).stack(rot_x, rot_y, rot_z) verts, faces = mesh.get_mesh_verts_faces(0) verts = rot_x.transform_points(verts) verts = rot_y.transform_points(verts) verts = rot_z.transform_points(verts) mesh = mesh.update_padded(verts.unsqueeze(0)) dist = dist elev = 0 azim = 0 R, T = look_at_view_transform(dist=dist, elev=elev, azim=azim, device=device) image_ref = renderer(meshes_world=mesh, R=R, T=T, device=device) image_ref = image_ref.cpu().numpy()[..., :3] plt.imshow(image_ref.squeeze()) plt.show() if save: verts, faces = mesh.get_mesh_verts_faces(0) save_obj(save, verts, faces) return mesh
def get_segmentation(obj_path, image_path, renderer): input_image = cv2.imread(image_path) input_image = input_image[:, :input_image.shape[1] // 3] input_image = cv2.resize(input_image, (1024, 1024)) # Setup device = torch.device("cuda:0") torch.cuda.set_device(device) # Load obj file verts_rgb_colors = get_verts_rgb_colors(obj_path) verts_rgb_colors = torch.from_numpy(verts_rgb_colors).to(device) textures = TexturesVertex(verts_features=verts_rgb_colors) # wo_textures = TexturesVertex(verts_features=torch.ones_like(verts_rgb_colors)*0.75) # Load obj mesh = load_objs_as_meshes([obj_path], device=device) # Set mesh vers = mesh._verts_list faces = mesh._faces_list mesh_w_tex = Meshes(vers, faces, textures) mesh_wo_tex = Meshes(vers, faces, wo_textures) R, T = look_at_view_transform(1.8, 0, 0, device=device) images_w_tex = renderer(mesh_w_tex, R=R, T=T) images_w_tex = np.clip(images_w_tex[0, ..., :3].cpu().numpy(), 0.0, 1.0)[:, :, ::-1] * 255 images_wo_tex = renderer(mesh_wo_tex, R=R, T=T) images_wo_tex = np.clip(images_wo_tex[0, ..., :3].cpu().numpy(), 0.0, 1.0)[:, :, ::-1] * 255 return input_image, images_w_tex, images_wo_tex
def project_mesh(mesh, angle): start = time.time() m = Metadata() R, T = look_at_view_transform(1.75, -45, angle, up=((0, 1, 0), ), at=((0, -0.25, 0), )) cameras = OpenGLPerspectiveCameras(device=m.device, R=R, T=T) raster_settings = m.raster_settings lights = m.lights renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardFlatShader(cameras=cameras, device=m.device, lights=lights)) verts = mesh.verts_list()[0] # faces = meshes.faces_list()[0] verts_rgb = torch.ones_like(verts)[None] # (1, V, 3) # verts_rgb = torch.ones((len(mesh.verts_list()[0]), 1))[None] # (1, V, 3) textures = Textures(verts_rgb=verts_rgb.to(m.device)) mesh.textures = textures mesh.textures._num_faces_per_mesh = mesh._num_faces_per_mesh.tolist() mesh.textures._num_verts_per_mesh = mesh._num_verts_per_mesh.tolist() image = renderer(mesh) return image
def render_obj(verts, faces, distance, elevation, azimuth): device = torch.device("cuda:0") verts_rgb = torch.ones_like(verts)[None] textures = Textures(verts_rgb=verts_rgb.to(device)) cur_mesh = Meshes(verts=[verts.to(device)], faces=[faces.to(device)], textures=textures) cameras = OpenGLPerspectiveCameras(device=device) blend_params = BlendParams(sigma=1e-4, gamma=1e-4) raster_settings = RasterizationSettings(image_size=256, blur_radius=0.0, faces_per_pixel=1, bin_size=0) lights = PointLights(device=device, location=((2.0, 2.0, -2.0), )) phong_renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=PhongShader(device=device, lights=lights)) R, T = look_at_view_transform(distance, elevation, azimuth, device=device) return phong_renderer(meshes_world=cur_mesh, R=R, T=T).cpu().numpy()
def set_renderer(image_size=512, use_sfm=False): # Setup device = torch.device("cuda:0") torch.cuda.set_device(device) # Initialize an OpenGL perspective camera. R, T = look_at_view_transform(2.0, 0, 180) if use_sfm: cameras = SfMPerspectiveCameras(focal_length=580.0, device=device, R=R, T=T) else: cameras = OpenGLOrthographicCameras(device=device, R=R, T=T) raster_settings = RasterizationSettings(image_size=image_size, blur_radius=0.0, faces_per_pixel=1, bin_size=None, max_faces_per_bin=None) lights = PointLights(device=device, location=((2.0, 2.0, 2.0), )) rasterizer = MeshRasterizer(cameras=cameras, raster_settings=raster_settings) shader = HardPhongShader(device=device, cameras=cameras, lights=lights) if use_sfm: renderer = MeshRendererWithDepth(rasterizer=rasterizer, shader=shader) else: renderer = MeshRenderer(rasterizer=rasterizer, shader=shader) return renderer
def render_mesh(verts, faces): device = verts[0].get_device() N = len(verts) num_verts_per_mesh = [] for i in range(N): num_verts_per_mesh.append(verts[i].shape[0]) verts_rgb = torch.ones((N, np.max(num_verts_per_mesh), 3), requires_grad=False, device=device) for i in range(N): verts_rgb[i, num_verts_per_mesh[i]:, :] = -1 textures = Textures(verts_rgb=verts_rgb) meshes = Meshes(verts=verts, faces=faces, textures=textures) elev = torch.rand(N) * 30 - 15 azim = torch.rand(N) * 360 - 180 R, T = look_at_view_transform(dist=2, elev=elev, azim=azim) cameras = FoVPerspectiveCameras(device=device, R=R, T=T) sigma = 1e-4 raster_settings = RasterizationSettings( image_size=128, blur_radius=np.log(1. / 1e-4 - 1.) * sigma, faces_per_pixel=40, perspective_correct=False) renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=SoftSilhouetteShader()) return renderer(meshes)
def test_load_apartment(self): """ This is the example habitat example scene from inside http://dl.fbaipublicfiles.com/habitat/habitat-test-scenes.zip The scene is "already lit", i.e. the textures reflect the lighting already, so we want to render them with full ambient light. """ self.skipTest("Data not available") glb = DATA_DIR / "apartment_1.glb" self.assertTrue(glb.is_file()) device = torch.device("cuda:0") mesh = _load(glb, device=device) if DEBUG: texturesuv_image_PIL(mesh.textures).save(DATA_DIR / "out_apartment.png") for i in range(19): # random locations in the apartment eye = ((np.random.uniform(-6, 0.5), np.random.uniform(-8, 2), 0), ) at = ((np.random.uniform(-6, 0.5), np.random.uniform(-8, 2), 0), ) up = ((0, 0, -1), ) RT = look_at_view_transform(eye=eye, at=at, up=up) _render(mesh, f"apartment_eau{i}", RT=RT, use_ambient=True) for i in range(12): # panning around the inner room from one location pan = axis_angle_to_matrix( torch.FloatTensor([0, radians(30 * i), 0])) _render(mesh, f"apartment{i}", 1.0, -90, pan, use_ambient=True)
def generate_video_from_obj(obj_path, video_path, renderer): # Setup device = torch.device("cuda:0") torch.cuda.set_device(device) # Load obj file verts_rgb_colors = get_verts_rgb_colors(obj_path) verts_rgb_colors = torch.from_numpy(verts_rgb_colors).to(device) textures = Textures(verts_rgb=verts_rgb_colors) wo_textures = Textures(verts_rgb=torch.ones_like(verts_rgb_colors) * 0.75) # Load obj mesh = load_objs_as_meshes([obj_path], device=device) # Set mesh vers = mesh._verts_list faces = mesh._faces_list mesh_w_tex = Meshes(vers, faces, textures) mesh_wo_tex = Meshes(vers, faces, wo_textures) # create VideoWriter fourcc = cv2.VideoWriter_fourcc(*'MP4V') out = cv2.VideoWriter(video_path, fourcc, 20.0, (1024, 512)) for i in tqdm(range(90)): R, T = look_at_view_transform(1.8, 0, i * 4, device=device) images_w_tex = renderer(mesh_w_tex, R=R, T=T) images_w_tex = np.clip(images_w_tex[0, ..., :3].cpu().numpy(), 0.0, 1.0)[:, :, ::-1] * 255 images_wo_tex = renderer(mesh_wo_tex, R=R, T=T) images_wo_tex = np.clip(images_wo_tex[0, ..., :3].cpu().numpy(), 0.0, 1.0)[:, :, ::-1] * 255 image = np.concatenate([images_w_tex, images_wo_tex], axis=1) out.write(image.astype('uint8')) out.release()
def _get_renderer(self, device): R, T = look_at_view_transform(10, 0, 0) # camera's position cameras = FoVPerspectiveCameras( device=device, R=R, T=T, znear=0.01, zfar=50, fov=2 * np.arctan(self.img_size // 2 / self.focal) * 180. / np.pi) lights = PointLights(device=device, location=[[0.0, 0.0, 1e5]], ambient_color=[[1, 1, 1]], specular_color=[[0., 0., 0.]], diffuse_color=[[0., 0., 0.]]) raster_settings = RasterizationSettings( image_size=self.img_size, blur_radius=0.0, faces_per_pixel=1, ) blend_params = blending.BlendParams(background_color=[0, 0, 0]) renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings), shader=SoftPhongShader(device=device, cameras=cameras, lights=lights, blend_params=blend_params)) return renderer
def setup(self, device): R, T = look_at_view_transform(self.viewpoint_distance, self.viewpoint_elevation, self.viewpoint_azimuth, device=device) cameras = FoVPerspectiveCameras(device=device, R=R, T=T) raster_settings = RasterizationSettings( image_size=self.opt.fast_image_size, blur_radius=self.opt.raster_blur_radius, faces_per_pixel=self.opt.raster_faces_per_pixel, ) rasterizer = MeshRasterizer(cameras=cameras, raster_settings=raster_settings) lights = PointLights(device=device, location=[self.opt.lights_location]) lights = DirectionalLights(device=device, direction=[self.opt.lights_direction]) shader = SoftPhongShader( device=device, cameras=cameras, lights=lights, blend_params=BlendParams( self.opt.blend_params_sigma, self.opt.blend_params_gamma, self.opt.blend_params_background_color, ), ) self.renderer = MeshRenderer( rasterizer=rasterizer, shader=shader, )
def __init__(self, dir: str, rasterization_settings: dict, znear: float = 1.0, zfar: float = 1000.0, scale_min: float = 0.5, scale_max: float = 2.0, device: str = 'cuda'): super(ToyNeuralGraphicsDataset, self).__init__() device = torch.device(device) self.device = device self.scale_min = scale_min self.scale_max = scale_max self.scale_range = scale_max - scale_min objs = [ os.path.join(dir, f) for f in os.listdir(dir) if f.endswith('.obj') ] self.meshes = load_objs_as_meshes(objs, device=device) R, T = look_at_view_transform(0, 0, 0) self.cameras = FoVPerspectiveCameras(R=R, T=T, znear=znear, zfar=zfar, device=device) self.renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=self.cameras, raster_settings=RasterizationSettings(**rasterization_settings), ), shader=HardFlatShader( device=device, cameras=self.cameras, ))
def render_with_batch_size(self, batch_size, dist, light_location, output_path): self.meshes = self.meshes.extend(batch_size) self.batch_size = batch_size elev = torch.linspace(0, 180, batch_size) azim = torch.linspace(-180, 180, batch_size) self.R, self.T = look_at_view_transform(dist=dist, elev=elev, azim=azim) self.cameras = OpenGLPerspectiveCameras(device=self.device, R=self.R, T=self.T) #set light locatioin self.light_location = light_location lights = PointLights(device=self.device, location=[self.light_location]) # call pytorch3d mesh renderer with shong shader renderer = MeshRenderer( rasterizer=MeshRasterizer(cameras=self.cameras, raster_settings=self.raster_settings), shader=TexturedSoftPhongShader(device=self.device, cameras=self.cameras, lights=lights)) images = renderer(self.meshes, cameras=self.cameras, lights=lights) for i in range(self.batch_size): img = images[i, ..., :3].cpu().numpy() * 255 img = img.astype('uint8') img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) cv2.imwrite(output_path + 'render-image-' + str(i) + '.png', img)
def init_renderer(self): # nsh_face_mesh = meshio.Mesh('data/mesh/nsh_bfm_face.obj') # self.nsh_face_tri = torch.from_numpy(nsh_face_mesh.triangles).type( # torch.int64).to(self.device) R, T = look_at_view_transform(10, 0, 0) cameras = OpenGLPerspectiveCameras(znear=0.001, zfar=30.0, aspect_ratio=1.0, fov=12.5936, degrees=True, R=R, T=T, device=self.device) raster_settings = RasterizationSettings(image_size=self.im_size, blur_radius=0.0, faces_per_pixel=1, bin_size=0, cull_backfaces=True) self.rasterizer = MeshRasterizer(cameras=cameras, raster_settings=raster_settings) lights = DirectionalLights(device=self.device) shader = TexturedSoftPhongShader(device=self.device, cameras=cameras, lights=lights) self.renderer = MeshRenderer(rasterizer=self.rasterizer, shader=shader)
def create_renderer(self): self.num_angles = self.config.num_angles azim = torch.linspace(-1 * self.config.angle_range, self.config.angle_range, self.num_angles) R, T = look_at_view_transform(dist=1.0, elev=0, azim=azim) T[:, 1] = -85 T[:, 2] = 200 cameras = FoVPerspectiveCameras(device=self.device, R=R, T=T) raster_settings = RasterizationSettings( image_size=self.config.img_size, blur_radius=0.0, faces_per_pixel=1, ) lights = PointLights(device=self.device, location=[[0.0, 85, 100.0]]) renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(device=self.device, cameras=cameras, lights=lights)) return renderer
def render_rotating_volume(volume_model, device, n_frames=50, video_size=400, n_pts_per_ray=192): renderer = get_renderer(video_size, n_pts_per_ray) # Render frames. with torch.inference_mode(): print("Generating rotating volume ...") elev = 30 azimuths = torch.linspace(0., 360., n_frames, device=device) frames = [] for azim in tqdm(azimuths): R, T = look_at_view_transform(dist=args.camera_radius, elev=elev, azim=azim) batch_cameras = FoVPerspectiveCameras(device=device, R=R, T=T) rgbo = volume_model(batch_cameras, renderer) rgb = rgbo[Ellipsis, :3] opacity = rgbo[Ellipsis, 3:4] frame = opacity * rgb + 1 - opacity frame = frame.clamp(0.0, 1.0) frames.append(frame) frames = torch.cat(frames).clamp(0., 1.) frames = frames.movedim(-1, 1) # THWC to TCHW. return frames.cpu().numpy()
def sphere_examples(bsdf, device="cuda", size=256, chunk_size=128, scale=100): from pytorch3d.pathtracer.shapes import Sphere from pytorch3d.pathtracer.bsdf import Diffuse from pytorch3d.pathtracer import pathtrace from pytorch3d.renderer import ( look_at_view_transform, OpenGLPerspectiveCameras, PointLights, ) import pytorch3d.pathtracer.integrators as integrators sphere = Sphere([0, 0, 0], 1, device=device) R, T = look_at_view_transform(dist=2., elev=0, azim=0) cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T) lights = PointLights(device=device, location=[[0., 1., 4.]], scale=scale) out = [] for basis in bsdf.bsdfs: expected = pathtrace( sphere, cameras=cameras, lights=lights, chunk_size=chunk_size, size=size, bsdf=basis, integrator=integrators.Direct(), device=device, silent=True, )[0] out.append(expected) return out
def compute_sem_dis_loss(self, mesh, num_render, semantic_discriminator_net, device): # need to make sure this matches render settings for discriminator training set # TODO: alternatively, randomize the angles each time? # 0., 45., 90., 135., 180., 225., 270., 315. azims = torch.linspace(0, 360, num_render + 1)[:-1] elevs = torch.Tensor([25 for i in range(num_render)]) dists = torch.ones(num_render) * 1.7 R, T = look_at_view_transform(dists, elevs, azims) meshes = mesh.extend(num_render) renders = utils.render_mesh(meshes, R, T, device, img_size=224, silhouette=True) # converting from [num_render, 224, 224, 4] silhouette render (only channel 4 has info) # to [num_render, 224, 224, 3] rgb image (black/white) renders_binary_rgb = torch.unsqueeze(renders[..., 3], 3).repeat(1, 1, 1, 3) loss = torch.sigmoid( semantic_discriminator_net(renders_binary_rgb.permute(0, 3, 1, 2))) loss = torch.mean(loss) return loss, renders_binary_rgb
def translate_mesh_on_axis( mesh, t: list, renderer, dist: float = 3.5, save: str = "", device: str = "" ): translation = Transform3d(device=device).translate(t[0], t[1], t[2]) verts, faces = mesh.get_mesh_verts_faces(0) verts = translation.transform_points(verts) mesh = mesh.update_padded(verts.unsqueeze(0)) dist = dist elev = 0 azim = 0 R, T = look_at_view_transform(dist=dist, elev=elev, azim=azim, device=device) image_ref = renderer(meshes_world=mesh, R=R, T=T, device=device) image_ref = image_ref.cpu().numpy() plt.imshow(image_ref.squeeze()) plt.show() if save: verts, faces = mesh.get_mesh_verts_faces(0) save_obj(save, verts, faces) return mesh
def test_sampler(): heatmap = torch.zeros(1, 1, 5, 5) heatmap[0, 0, 2, 2] = 1.0 from filter import sample_particles_from_heatmap_2d hws, alphas = sample_particles_from_heatmap_2d(heatmap, {'cup': 1}, deterministic=True, h_min=-4.0, h_max=4.0, w_min=-4.0, w_max=4.0) vert = torch.tensor( [[-1, -1.1, 0], [1, -1.1, 0], [1, 1.1, 0], [-1, 1.1, 0]], dtype=torch.float) face = torch.LongTensor([[0, 1, 2], [0, 2, 3]]) white = torch.ones_like(vert) from tracker import World world = World() world.add_mesh('cup', vert, face, white) scene = world.create_scene(hws, alphas, 1) print(hws['cup']) print(scene.verts_padded().shape) plt.scatter(scene.verts_padded()[0, :, 0], scene.verts_padded()[0, :, 1]) plt.show() cameras = FoVOrthographicCameras(device=device, max_x=2.5, max_y=2.5, min_x=-2.5, min_y=-2.5, scale_xyz=((1, 1, 1), )) raster_settings = RasterizationSettings( image_size=5, blur_radius=0, faces_per_pixel=6, ) renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings, ), shader=IdentityShader()) distance, elevation, azimuth = 30, 0.0, 0 R, T = look_at_view_transform(distance, elevation, azimuth, device=device) image = renderer(meshes_world=scene.to(device), R=R, T=T) fig = plt.figure(figsize=(10, 10)) for i in range(image.size(-2)): panel = fig.add_subplot(3, 3, i + 1) panel.imshow(image[..., i, 0:3].squeeze().cpu()) plt.grid(False) plt.show()
def change_cameras(self, mode, camera_dist=2.2): azim_train = torch.linspace(-1 * self.config.angle_range_train, self.config.angle_range_train, self.num_angles_train) azim_test = torch.linspace(-1 * self.config.angle_range_test, self.config.angle_range_test, self.num_angles_test) R, T = look_at_view_transform(camera_dist, 6, azim_train) train_cameras = FoVPerspectiveCameras(device=self.device, R=R, T=T) self.train_cameras = train_cameras R, T = look_at_view_transform(camera_dist, 6, azim_test) test_cameras = FoVPerspectiveCameras(device=self.device, R=R, T=T) self.test_cameras = test_cameras if mode == 'train': self.renderer.rasterizer.cameras=self.train_cameras self.renderer.shader.cameras=self.train_cameras elif mode == 'test': self.renderer.rasterizer.cameras=self.test_cameras self.renderer.shader.cameras=self.test_cameras
def test_bounding_box(): verts = torch.tensor([[-2, -1, 0], [2, -1, 0], [2, 1, 0], [-2, 1, 0]], dtype=torch.float) * 20.0 faces = torch.LongTensor([[0, 1, 2], [0, 2, 3]]) white = torch.ones_like(verts) red = white * torch.tensor([1.0, 0.0, 0.0]) green = white * torch.tensor([0.0, 1.0, 0.0]) blue = white * torch.tensor([0.0, 0.0, 1.0]) meshes = Meshes(verts=[verts], faces=[faces], textures=TexturesVertex([blue])) distance = 30 elevation = 0.0 azimuth = 0 R, T = look_at_view_transform(distance, elevation, azimuth) cameras = FoVOrthographicCameras(max_x=64.0, max_y=64.0, min_x=-64.0, min_y=-64.0, scale_xyz=((1, 1, 1), ), R=R, T=T) bb = BoundingBoxes(meshes, cameras, screen_size=(128, 128)) raster_settings = RasterizationSettings( image_size=128, blur_radius=0, faces_per_pixel=6, ) renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings, ), shader=IdentityShader()) fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(10, 10), constrained_layout=False) ax.imshow(renderer(meshes)[0, :, :, 0, :]) boxes_rect = patches.Rectangle(bb.bottom_left(0), width=bb.width(0), height=bb.height(0), linewidth=4, edgecolor='r', facecolor='none') ax.add_patch(boxes_rect) plt.show()
def createRenderer(device, camera, light, imageSize): ''' It creates a pytorch3D renderer with the given camera pose, light source and output image size. Parameters ---------- device : Device on which the renderer is created. camera : Camera pose. light : Position of the light source. imageSize : The size of the rendered image. Returns ------- renderer : Pytorch3D renderer. ''' if camera is None: camera = (2.0, -20.0, 180.0) if light is None: light = (0.0, 2.0, 0.0) # Initialize an OpenGL perspective camera. # With world coordinates +Y up, +X left and +Z into the screen. R, T = look_at_view_transform(camera[0], camera[1], camera[2]) cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T) # Define the settings for rasterization and shading. Here we set the output image to be of size # 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1 # and blur_radius=0.0. We also set bin_size and max_faces_per_bin to None which ensure that # the faster coarse-to-fine rasterization method is used. Refer to rasterize_meshes.py for # explanations of these parameters. Refer to docs/notes/renderer.md for an explanation of # the difference between naive and coarse-to-fine rasterization. raster_settings = RasterizationSettings( image_size=imageSize, blur_radius=0.0, faces_per_pixel=1, ) # Place a point light at -y direction. lights = PointLights(device=device, location=[[light[0], light[1], light[2]]]) # Create a phong renderer by composing a rasterizer and a shader. renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings), shader=HardPhongShader(device=device, cameras=cameras, lights=lights)) return renderer
def renderImage(self): # Function: renderImage # Inputs: self # Process: returns image of the object R, T = look_at_view_transform(self.camera_position[0], self.camera_position[1], self.camera_position[2]) image = self.renderer(meshes_world=self.mesh.clone(),R=R,T=T) return image
def _render(self): """Render the unit cube at various camera poses. We deliberately sample the camera poses such that all vertices of the cube are always in view. """ opts = self.opts # TODO(ycho): is `no_grad()` here necessary? with th.no_grad(): # Sample a Unit-ray viewing direction. ray = th.randn(size=(opts.batch_size, 3), device=self.device) ray /= th.norm(ray, dim=1, keepdim=True) # Compute distance along the ray according to constraints. distance = (self.min_distance + (opts.max_distance - self.min_distance) * th.rand(size=(opts.batch_size, 1), device=self.device)) # NOTE(ycho): `sqrt(3)/2` here comes from max radius of unit cube. # The generic thing to do would be to compute the radius of our # cloud. We're not explicitly taking the cube into account. max_tangential_offset = th.clamp(distance * self.tan_half_fov - (0.5 * np.sqrt(3)), min=0.0) # NOTE(ycho): We're sampling an offset orthogonal to the view ray. # The constraint here is to be visible within the view plane. offset = th.randn(size=(opts.batch_size, 3), device=self.device) offset = offset - th.einsum('...a,...a->...', offset, ray)[:, None] * ray offset *= (max_tangential_offset / th.norm(offset, dim=1, keepdim=True)) # Finally, we have the full definition of the ray. at = offset pos = offset + ray * distance # Compose the `lookat` camera transform according to this position R, T = look_at_view_transform(eye=pos, at=at, device=self.device) if self.opts.use_mesh: # NOTE(ycho): Ignoring alpha dimension img = self.renderer(self.meshes, R=R, T=T)[..., :3] else: img = self.renderer(point_clouds=self.clouds, R=R, T=T) # pytorch3d uses `NHWC` convension, convert to NCHW. img = img.permute(0, 3, 1, 2) # pytorch3d uses a `float` tensor for representing images, # whereas we'd like for the image to come out as `uint8` ONLY for # compatibility with the output from the `Objectron` dataset. img = img.mul_(255.0).to(dtype=th.uint8) return (img, R, T)
def test_scene(): world = engine.World() world.add_mesh('red_box', verts, faces, red) world.add_mesh('green_box', verts, faces, green) world.add_mesh('blue_box', verts, faces, blue) scene_spec = [ {'red_box_0': 'red_box', 'green_box_0': 'green_box'}, {'blue_box_0': 'blue_box', 'blue_box_1': 'blue_box'} ] world.create_scenes(scene_spec) poses = [ [Translate(0, -30, 0), Translate(-10, -10, 0)], [Translate(40, 0, 0), Translate(-10, -10, 0)] ] world.update_scenes(poses) batch = world.batch() labels = world.labels() distance = 30 elevation = 0.0 azimuth = 0 R, T = look_at_view_transform(distance, elevation, azimuth) cameras = FoVOrthographicCameras(max_x=64.0, max_y=64.0, min_x=-64.0, min_y=-64.0, scale_xyz=((1, 1, 1),), R=R, T=T) raster_settings = RasterizationSettings( image_size=128, blur_radius=0, faces_per_pixel=6, ) renderer = MeshRenderer( rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings, ), shader=IdentityShader() ) boxes = world.bounding_boxes(cameras, (128, 128)) image = renderer(batch) fig, ax = plt.subplots(nrows=1, ncols=1) ax.imshow(image[0, :, :, 0, :]) for box in boxes[0]: ax.add_patch(box.get_patch()) plt.show()
def forward(self): R, T = look_at_view_transform(dist=self.dist, elev=self.elev, azim=self.azim, at=self.p, up=self.u, device=self.device) predict_image = self.renderer(meshes_world=self.mesh.clone(), R=R, T=T) ssim_loss = pytorch_ssim.SSIM(window_size=11) predict_image = predict_image[..., :3] loss = 1 - ssim_loss(self.gray_image_ref, predict_image) return loss