def render_depth(self, vertices, faces, K=None, R=None, t=None, dist_coeffs=None, orig_size=None): # fill back if self.fill_back: faces = torch.cat((faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() # viewpoint transformation if self.camera_mode == 'look_at': vertices = nr.look_at(vertices, self.eye) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == 'look': vertices = nr.look(vertices, self.eye, self.camera_direction) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == 'projection': if K is None: K = self.K if R is None: R = self.R if t is None: t = self.t if dist_coeffs is None: dist_coeffs = self.dist_coeffs if orig_size is None: orig_size = self.orig_size vertices = nr.projection(vertices, K, R, t, dist_coeffs, orig_size) # rasterization faces = nr.vertices_to_faces(vertices, faces) images = nr.rasterize_depth(faces, self.image_size, self.anti_aliasing) return images
def render_depth(self, vertices, faces): # fill back if self.fill_back: faces = torch.cat( (faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() # viewpoint transformation if self.camera_mode == 'look_at': vertices = nr.look_at(vertices, self.eye) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == 'look': vertices = nr.look(vertices, self.eye, self.camera_direction) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == 'projection': vertices = nr.projection(vertices, self.P, self.dist_coeffs, self.orig_size) elif self.camera_mode == 'projection_by_params': vertices = nr.projection_by_params(vertices, self.camera_f, self.camera_c, self.camera_rt, self.camera_t, self.dist_coeffs, self.orig_size) # rasterization faces = nr.vertices_to_faces(vertices, faces) images = nr.rasterize_depth(faces, self.image_size, self.anti_aliasing) return images
def forward( self, vertices, faces, Ks=None, Rs=None, ts=None, dist_coeffs=None, bbxs=None, image_size=64, orig_size=64, anti_aliasing=False, far=100.0, ): # batch_size = vertices.shape(0) if self.fill_back: faces = torch.cat( (faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1, ).to(vertices.device).detach() if Ks is None: print("K must not None if render depthmap") raise Exception() if Rs is None: Rs = torch.Tensor([ [1, 0, 0], [0, 1, 0], [0, 0, 1], ]).view((1, 3, 3)).to(vertices.device) if ts is None: ts = torch.Tensor([0, 0, 0]).view((1, 3)).to(vertices.device) if dist_coeffs is None: dist_coeffs = torch.Tensor([[0., 0., 0., 0., 0.]]).to(vertices.device) ''' xyz -> uvd ''' vertices = self.projection(vertices, Ks, Rs, ts, dist_coeffs, orig_size, bbxs=bbxs) faces = nr.vertices_to_faces(vertices, faces) # rasteriation rast = nr.rasterize_depth(faces, image_size, anti_aliasing, far=far) # normalize to 0~1 rend = self.normalize_depth(rast, far=far) return rend
def render_depth(self, cam, vertices): bs = cam.shape[0] faces = self.faces.repeat(bs, 1, 1) # set offset_z for persp proj proj_verts = self.proj_func(vertices, cam) # flipping the y-axis here to make it align with the image coordinate system! proj_verts[:, :, 1] *= -1 # rasterization faces = self.vertices_to_faces(proj_verts, faces) images = nr.rasterize_depth(faces, self.image_size, self.anti_aliasing) return images
def render_depth(self, vertices, faces, backgrounds=None): vertices = self.transform_vertices(vertices) images = neural_renderer.rasterize_depth( vertices, faces, background_color=self.background_color, backgrounds=backgrounds, image_size=self.image_size, near=self.near, far=self.far, anti_aliasing=self.anti_aliasing, draw_backside=self.draw_backside, ) return images
def forward(self, vertices, faces, textures=None, mode=None): ''' Implementation of forward rendering method The old API is preserved for back-compatibility with the Chainer implementation ''' _textures = textures if mode not in [None, 'silhouettes', 'depth']: raise ValueError("mode should be one of None, 'silhouettes' or 'depth'") # fill back if self.fill_back: faces = torch.cat((faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() if _textures is not None: _textures = torch.cat((_textures, textures.permute((0, 1, 4, 3, 2, 5))), dim=1) if textures is not None: # lighting faces_lighting = nr.vertices_to_faces(vertices, faces) _textures = nr.lighting( faces_lighting, _textures, self.light_intensity_ambient, self.light_intensity_directional, self.light_color_ambient, self.light_color_directional, self.light_direction) # projection vertices = nr.projection(vertices, self.camera) if self.camera.perspective: vertices = nr.perspective(vertices, angle=self.camera.viewing_angle) # rasterization faces = nr.vertices_to_faces(vertices, faces) if mode is None: images = nr.rasterize( faces, _textures, self.camera.image_size, self.anti_aliasing, self.camera.near, self.camera.far, self.rasterizer_eps, self.background_color) elif mode == 'silhouettes': images = nr.rasterize_silhouettes(faces, self.camera.image_size, self.anti_aliasing) elif mode == 'depth': images = nr.rasterize_depth(faces, self.camera.image_size, self.anti_aliasing) return images
def render_depth(self, vertices, faces): # fill back if self.fill_back: faces = cf.concat((faces, faces[:, :, ::-1]), axis=1).data # viewpoint transformation if self.camera_mode == 'look_at': vertices = nr.look_at(vertices, self.eye) elif self.camera_mode == 'look': vertices = nr.look(vertices, self.eye, self.camera_direction, self.up) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) # rasterization faces = nr.vertices_to_faces(vertices, faces) images = nr.rasterize_depth(faces, self.image_size, self.anti_aliasing) return images
def render_depth(self, vertices, faces): # fill back if self.fill_back: faces = cf.concat((faces, faces[:, :, ::-1]), axis=1).data # viewpoint transformation if self.camera_mode == 'look_at': vertices = neural_renderer.look_at(vertices, self.eye) elif self.camera_mode == 'look': vertices = neural_renderer.look(vertices, self.eye, self.camera_direction) # perspective transformation if self.perspective: vertices = neural_renderer.perspective(vertices, angle=self.viewing_angle) # rasterization faces = neural_renderer.vertices_to_faces(vertices, faces) images = neural_renderer.rasterize_depth(faces, self.image_size, self.anti_aliasing) return images
def forward(self, mesh, cam_param, img_affine_trans_mat, mesh): batch_size = mesh.shape[0] face_num = len(mesh['vi']) focal, princpt, campos, camrot = cam_param['focal'], cam_param[ 'princpt'], cam_param['campos'], cam_param['camrot'] # project mesh world -> camera space mesh = mesh - campos.view(-1, 1, 3) mesh = torch.cat([ torch.mm(camrot[i], mesh[i].permute(1, 0)).permute( 1, 0)[None, :, :] for i in range(batch_size) ], 0) mesh_3d_x = mesh[:, :, 0] mesh_3d_y = mesh[:, :, 1] mesh_z = mesh[:, :, 2] mesh_z = mesh_z + (mesh_z == 0).type('torch.cuda.FloatTensor') * 1e-4 # project camera -> image space mesh_2d_x = (mesh_3d_x / mesh_z * focal[:, 0].view(-1, 1) + princpt[:, 0].view(-1, 1))[:, :, None] mesh_2d_y = (mesh_3d_y / mesh_z * focal[:, 1].view(-1, 1) + princpt[:, 1].view(-1, 1))[:, :, None] mesh_2d = torch.cat([mesh_2d_x, mesh_2d_y, torch.ones_like(mesh_2d_x)], 2) # apply affine transform (crop and resize) mesh_2d = torch.bmm(img_affine_trans_mat, mesh_2d.permute(0, 2, 1)).permute(0, 2, 1) ################################## # neural renderer (for depth map rendering) mesh_2d_norm = torch.cat([mesh_2d[:,:,0:1]/cfg.rendered_img_shape[1]*2-1,\ (cfg.rendered_img_shape[0] - 1 - mesh_2d[:,:,1:2])/cfg.rendered_img_shape[0]*2-1, \ mesh_z[:,:,None]],2) # mesh_2d_v0, v1, v2: batch_size x face_num x 3. coordinates of vertices for each face mesh_2d_v0 = torch.cat([ index_selection_nd(mesh_2d_norm[i], mesh['vi'][:, 0], 0)[None, ...] for i in range(batch_size) ], dim=0) mesh_2d_v1 = torch.cat([ index_selection_nd(mesh_2d_norm[i], mesh['vi'][:, 1], 0)[None, ...] for i in range(batch_size) ], dim=0) mesh_2d_v2 = torch.cat([ index_selection_nd(mesh_2d_norm[i], mesh['vi'][:, 2], 0)[None, ...] for i in range(batch_size) ], dim=0) face_vertices = torch.cat([ mesh_2d_v0[:, :, None, :], mesh_2d_v1[:, :, None, :], mesh_2d_v2[:, :, None, :] ], 2) rendered_depthmap = nr.rasterize_depth(face_vertices, cfg.rendered_img_shape[0], False, near=cfg.depth_min, far=cfg.depth_max)[:, None, :, :] rendered_depthmap[rendered_depthmap == cfg.depth_max] = 0 return rendered_depthmap