def render(self, cam, vertices, textures, faces=None, get_fim=False): if faces is None: bs = cam.shape[0] faces = self.faces.repeat(bs, 1, 1) # lighting is inplace operation textures = textures.clone() # lighting faces_lighting = nr.vertices_to_faces(vertices, faces) textures = nr.lighting( faces_lighting, textures, self.light_intensity_ambient, self.light_intensity_directional, self.light_color_ambient, self.light_color_directional, self.light_direction) # set offset_z for persp proj proj_verts = self.proj_func(vertices, cam) # flipping the y-axis here to make it align with the image coordinate system! proj_verts[:, :, 1] *= -1 # calculate the look_at vertices. vertices = nr.look_at(proj_verts, self.eye) # rasterization faces = nr.vertices_to_faces(vertices, faces) images = nr.rasterize(faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, self.background_color) fim = None if get_fim: fim = nr.rasterize_face_index_map(faces, image_size=self.image_size, anti_aliasing=False, near=self.near, far=self.far, eps=self.rasterizer_eps) return images, fim
def render(self, vertices, texture=None, faces=None): if faces is None: bs = vertices.shape[0] faces = self.faces.repeat(bs, 1, 1) if texture is None: texture = self.debug_textures().to(vertices.device) texture = texture.unsqueeze(0).repeat(bs, 1, 1, 1, 1, 1) # lighting is inplace operation texture = texture.clone() # lighting faces_lighting = nr.vertices_to_faces(vertices, faces) texture = nr.lighting( faces_lighting, texture, self.light_intensity_ambient, self.light_intensity_directional, self.light_color_ambient, self.light_color_directional, self.light_direction) # set offset_z for persp proj #proj_verts = self.proj_func(vertices, cam) # flipping the y-axis here to make it align with the image coordinate system! #proj_verts[:, :, 1] *= -1 # calculate the look_at vertices. vertices = nr.look_at(vertices, self.eye) # rasterization faces = nr.vertices_to_faces(vertices, faces) image = nr.rasterize(faces, texture, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, self.background_color) return image
def render(self, vertices, faces, textures): # fill back if self.fill_back: faces = cf.concat((faces, faces[:, :, ::-1]), axis=1).data textures = cf.concat( (textures, textures.transpose((0, 1, 4, 3, 2, 5))), axis=1) # lighting faces_lighting = neural_renderer.vertices_to_faces(vertices, faces) textures = neural_renderer.lighting(faces_lighting, textures, self.light_intensity_ambient, self.light_intensity_directional, self.light_color_ambient, self.light_color_directional, self.light_direction) # viewpoint transformation if self.camera_mode == 'look_at': vertices = neural_renderer.look_at(vertices, self.eye) elif self.camera_mode == 'look': vertices = neural_renderer.look(vertices, self.eye, self.camera_direction) # perspective transformation if self.perspective: vertices = neural_renderer.perspective(vertices, angle=self.viewing_angle) # rasterization faces = neural_renderer.vertices_to_faces(vertices, faces) images = neural_renderer.rasterize(faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, self.background_color) return images
def render(self, vertices, faces, textures, K=None, R=None, t=None, at=None, up=None, dist_coeffs=None, orig_size=None): # fill back if self.fill_back: faces = torch.cat( (faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() textures = torch.cat((textures, textures.permute( (0, 1, 4, 3, 2, 5))), dim=1) # lighting faces_lighting = nr.vertices_to_faces(vertices, faces) textures = nr.lighting(faces_lighting, textures, self.light_intensity_ambient, self.light_intensity_directional, self.light_color_ambient, self.light_color_directional, self.light_direction) # viewpoint transformation if self.camera_mode == 'look_at': vertices = nr.look_at(vertices, self.eye, up=up) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == 'look': vertices = nr.look(vertices, self.eye, self.camera_direction) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == 'projection': if K is None: K = self.K if R is None: R = self.R if t is None: t = self.t if dist_coeffs is None: dist_coeffs = self.dist_coeffs if orig_size is None: orig_size = self.orig_size vertices = nr.projection(vertices, K, R, t, dist_coeffs, orig_size) # rasterization faces = nr.vertices_to_faces(vertices, faces) out = nr.rasterize_rgbad(faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, self.background_color) return out['rgb'], out['depth'], out['alpha']
def render(self, vertices, faces, textures): # fill back if self.fill_back: faces = cf.concat((faces, faces[:, :, ::-1]), axis=1).data textures = cf.concat((textures, textures.transpose((0, 1, 4, 3, 2, 5))), axis=1) # lighting faces_lighting = neural_renderer.vertices_to_faces(vertices, faces) textures = neural_renderer.lighting( faces_lighting, textures, self.light_intensity_ambient, self.light_intensity_directional, self.light_color_ambient, self.light_color_directional, self.light_direction) # viewpoint transformation if self.camera_mode == 'look_at': vertices = neural_renderer.look_at(vertices, self.eye) elif self.camera_mode == 'look': vertices = neural_renderer.look(vertices, self.eye, self.camera_direction) # perspective transformation if self.perspective: vertices = neural_renderer.perspective(vertices, angle=self.viewing_angle) # rasterization faces = neural_renderer.vertices_to_faces(vertices, faces) images = neural_renderer.rasterize( faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, self.background_color) return images
def render_fov(self, mesh, R, t, background_color=[0, 0, 0]): vertices, faces, textures = mesh if isinstance(t, np.ndarray): t = torch.cuda.FloatTensor(t) if isinstance(R, np.ndarray): R = torch.cuda.FloatTensor(R) t = t.view(-1, 1, 3) R = R.view(-1, 3, 3) assert len(vertices.shape) == 3 batch_size = max(t.shape[0], vertices.shape[0]) # expand dimension vertices = vertices.expand(batch_size, *vertices.shape[1:]) faces = faces.expand(batch_size, *faces.shape[1:]) textures = textures.expand(batch_size, *textures.shape[1:]) # fill back if self.fill_back: faces = torch.cat( (faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() textures = torch.cat((textures, textures.permute( (0, 1, 4, 3, 2, 5))), dim=1) # lighting faces_lighting = nr.vertices_to_faces(vertices, faces) textures = nr.lighting(faces_lighting, textures, self.light_intensity_ambient, self.light_intensity_directional, self.light_color_ambient, self.light_color_directional, self.light_direction) # viewpoint transformation vertices = nr.projection_fov(vertices, self.fov, R, t) # rasterization faces = nr.vertices_to_faces(vertices, faces) out = nr.rasterize_rgbad(faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, background_color=background_color) return out['rgb'], out['depth'], out['alpha']
def render(self, vertices, faces, textures): # fill back if self.fill_back: faces = cf.concat((faces, faces[:, :, ::-1]), axis=1).data textures = cf.concat( (textures, textures.transpose((0, 1, 4, 3, 2, 5))), axis=1) # lighting faces_lighting = neural_renderer.vertices_to_faces(vertices, faces) textures = neural_renderer.lighting(faces_lighting, textures, self.light_intensity_ambient, self.light_intensity_directional, self.light_color_ambient, self.light_color_directional, self.light_direction) # viewpoint transformation if self.camera_mode == 'look_at': vertices = neural_renderer.look_at(vertices, self.eye) elif self.camera_mode == 'look': vertices = neural_renderer.look(vertices, self.eye, self.camera_direction) # perspective transformation if self.perspective: vertices = neural_renderer.perspective(vertices, angle=self.viewing_angle) # rasterization faces = neural_renderer.vertices_to_faces(vertices, faces) # ==== TM changes ==== results_dict = neural_renderer.rasterize(faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, self.background_color) images = results_dict['rgb'] face_index_map = results_dict['face_index_map'] weight_map = results_dict['weight_map'] sampling_weight_map = results_dict['sampling_weight_map'] # ==== Making another dictionary (just for clarity) ==== return_dict = dict() return_dict['images'] = images return_dict['face_index_map'] = face_index_map return_dict['weight_map'] = weight_map return_dict['sampling_weight_map'] = sampling_weight_map return return_dict
def render(self, vertices, faces, textures): # fill back if self.fill_back: faces = torch.cat( (faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() textures = torch.cat((textures, textures.permute( (0, 1, 4, 3, 2, 5))), dim=1) # lighting faces_lighting = nr.vertices_to_faces(vertices, faces) textures = nr.lighting(faces_lighting, textures, self.light_intensity_ambient, self.light_intensity_directional, self.light_color_ambient, self.light_color_directional, self.light_direction) # viewpoint transformation if self.camera_mode == 'look_at': vertices = nr.look_at(vertices, self.eye) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == 'look': vertices = nr.look(vertices, self.eye, self.camera_direction) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == 'projection': vertices = nr.projection(vertices, self.P, self.dist_coeffs, self.orig_size) elif self.camera_mode == 'projection_by_params': vertices = nr.projection_by_params(vertices, self.camera_f, self.camera_c, self.camera_rt, self.camera_t, self.dist_coeffs, self.orig_size) # rasterization faces = nr.vertices_to_faces(vertices, faces) images = nr.rasterize(faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, self.background_color) return images
def forward(self, vertices, faces, textures=None, mode=None): ''' Implementation of forward rendering method The old API is preserved for back-compatibility with the Chainer implementation ''' _textures = textures if mode not in [None, 'silhouettes', 'depth']: raise ValueError("mode should be one of None, 'silhouettes' or 'depth'") # fill back if self.fill_back: faces = torch.cat((faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() if _textures is not None: _textures = torch.cat((_textures, textures.permute((0, 1, 4, 3, 2, 5))), dim=1) if textures is not None: # lighting faces_lighting = nr.vertices_to_faces(vertices, faces) _textures = nr.lighting( faces_lighting, _textures, self.light_intensity_ambient, self.light_intensity_directional, self.light_color_ambient, self.light_color_directional, self.light_direction) # projection vertices = nr.projection(vertices, self.camera) if self.camera.perspective: vertices = nr.perspective(vertices, angle=self.camera.viewing_angle) # rasterization faces = nr.vertices_to_faces(vertices, faces) if mode is None: images = nr.rasterize( faces, _textures, self.camera.image_size, self.anti_aliasing, self.camera.near, self.camera.far, self.rasterizer_eps, self.background_color) elif mode == 'silhouettes': images = nr.rasterize_silhouettes(faces, self.camera.image_size, self.anti_aliasing) elif mode == 'depth': images = nr.rasterize_depth(faces, self.camera.image_size, self.anti_aliasing) return images
def render(self, vertices, faces, textures): if self.camera_mode == 'look_at': vertices = neural_renderer.look_at(vertices, self.eye) elif self.camera_mode == 'look': vertices = neural_renderer.look(vertices, self.eye, self.camera_direction) if self.perspective: vertices = neural_renderer.perspective(vertices) faces = neural_renderer.vertices_to_faces(vertices, faces) if self.fill_back: faces = cf.concat((faces, faces[:, :, ::-1]), axis=1) textures = cf.concat((textures, textures.transpose((0, 1, 4, 3, 2, 5))), axis=1) textures = neural_renderer.lighting( faces, textures, self.light_intensity_ambient, self.light_intensity_directional) images = neural_renderer.rasterize( faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, self.background_color) return images
def render_shading(self, vertices, faces, textures): # lighting faces_lighting = nr.vertices_to_faces(vertices, faces) textures = nr.lighting(faces_lighting, textures, self.light_intensity_ambient, self.light_intensity_directional, self.light_color_ambient, self.light_color_directional, self.light_direction) # viewpoint transformation if self.camera_mode == 'look_at': vertices = nr.look_at(vertices, self.eye) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == 'look': vertices = nr.look(vertices, self.eye, self.camera_direction) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == 'projection': vertices = nr.projection(vertices, self.P, self.dist_coeffs, self.orig_size) # rasterization faces = nr.vertices_to_faces(vertices, faces) images_real = nr.rasterize_rgba(faces, textures, self.image_size, self.anti_aliasing) # import pdb;pdb.set_trace() images_real = torch.cat( [images_real['rgb'], images_real['alpha'].unsqueeze(1)], dim=1).detach() images_fake = self.forward_eff_render_shading( faces, textures, self.image_size, self.sigma_val, self.sigma_num, self.sigma_mul, self.gamma_val, self.gamma_num, self.gamma_mul, self.near, self.far) # import pdb;pdb.set_trace() return images_fake, images_real
def test_case1(self): """Test whether it is executable.""" faces = np.random.normal(size=(64, 16, 3, 3)).astype('float32') textures = np.random.normal(size=(64, 16, 8, 8, 8, 3)).astype('float32') neural_renderer.lighting(faces, textures)
def render(self, vertices, faces, textures, K=None, R=None, t=None, dist_coeffs=None, orig_size=None): # fill back # viewpoint transformation if self.camera_mode == 'look_at': vertices = nr.look_at(vertices, self.eye) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == 'look': vertices = nr.look(vertices, self.eye, self.camera_direction) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == 'projection': if K is None: K = self.K if R is None: R = self.R if t is None: t = self.t if dist_coeffs is None: dist_coeffs = self.dist_coeffs if orig_size is None: orig_size = self.orig_size vertices = nr.projection(vertices, K, R, t, dist_coeffs, orig_size) elif self.camera_mode == 'weak_projection': if K is None: K = self.K if R is None: R = self.R if t is None: t = self.t vertices = weak_projection(vertices, K, R, t, orig_size) # 正则化一下,让脸颊到鼻尖的向量和z轴负方向夹角小于90度 (a,b,c)*(0,0,-1)>0 -> c<0 # 另外将z值全部归到正轴上去 from MorphableModelFitting.models.face_model import FaceModelBFM front_idx, back_idx_1, back_idx_2 = FaceModelBFM().keypoints[ 30], FaceModelBFM().keypoints[0], FaceModelBFM().keypoints[16] for i in range(len(vertices)): back_z = (vertices[i, back_idx_1, 2] + vertices[i, back_idx_2, 2]) / 2 if (vertices[i, front_idx, 2] - back_z) > 0: vertices[i, :, 2] = -vertices[i, :, 2] vertices[:, :, 2] = vertices[:, :, 2] - torch.min( vertices[:, :, 2], dim=1)[0].unsqueeze(1) + 1 # lighting if self.light_mode == "parallel": textures = texture_from_point2faces(faces, textures) / 255 if self.fill_back: faces = torch.cat( (faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() textures = torch.cat( (textures, textures.permute((0, 1, 4, 3, 2, 5))), dim=1) faces_lighting = nr.vertices_to_faces(vertices, faces) textures = nr.lighting(faces_lighting, textures, self.light_intensity_ambient, self.light_intensity_directional, self.light_color_ambient, self.light_color_directional, self.light_direction) elif self.light_mode == "SH": point_buf = torch.from_numpy(self.facemodel.point_buf).long() - 1 point_norm = compute_point_norm( vertices, faces[0][:, list(reversed(range(faces.shape[-1])))], point_buf) # texture = texture_from_point2faces(triangles, texture).reshape((batch, -1, 3)) textures, _ = Illumination_SH(textures, point_norm, self.SH_coeff) textures = texture_from_point2faces(faces, textures) / 255 if self.fill_back: faces = torch.cat( (faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() textures = torch.cat( (textures, textures.permute((0, 1, 4, 3, 2, 5))), dim=1) else: return None # rasterization faces = nr.vertices_to_faces(vertices, faces) out = nr.rasterize_rgbad(faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, self.background_color) return out['rgb'], out['depth'], out['alpha']
def render( self, vertices, faces, textures, K=None, R=None, t=None, dist_coeffs=None, orig_size=None, detach_renders=False, ): # fill back if self.fill_back: faces = torch.cat( (faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() textures = torch.cat((textures, textures.permute( (0, 1, 4, 3, 2, 5))), dim=1) # lighting if not self.no_light: faces_lighting = nr.vertices_to_faces(vertices, faces) textures = nr.lighting( faces_lighting, textures, self.light_intensity_ambient, self.light_intensity_directional, self.light_color_ambient, self.light_color_directional, self.light_direction, ) # viewpoint transformation if self.camera_mode == "look_at": vertices = nr.look_at(vertices, self.eye) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == "look": vertices = nr.look(vertices, self.eye, self.camera_direction) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == "projection": vertices = self.project(vertices, K=K, R=R, t=t, dist_coeffs=dist_coeffs, orig_size=orig_size) # rasterization faces = nr.vertices_to_faces(vertices, faces) if detach_renders: faces = faces.detach() out = rasterize.rasterize_rgbad( faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, self.background_color, ) return out
def render_rgb(self, vertices, faces, textures, K=None, R=None, t=None, dist_coeffs=None, orig_size=None, lighting=True): # fill back if self.fill_back: faces = torch.cat( (faces, faces[:, :, list(reversed(range(faces.shape[-1])))]), dim=1).detach() textures = torch.cat((textures, textures.permute( (0, 1, 4, 3, 2, 5))), dim=1) # lighting if lighting and textures.size()[-1] == 3: # 3 channels (RGB) faces_lighting = nr.vertices_to_faces(vertices, faces) textures = nr.lighting(faces_lighting, textures, self.light_intensity_ambient, self.light_intensity_directional, self.light_color_ambient, self.light_color_directional, self.light_direction) # viewpoint transformation if self.camera_mode == 'look_at': vertices = nr.look_at(vertices, self.eye) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == 'look': vertices = nr.look(vertices, self.eye, self.camera_direction) # perspective transformation if self.perspective: vertices = nr.perspective(vertices, angle=self.viewing_angle) elif self.camera_mode == 'projection': if K is None: K = self.K if R is None: R = self.R if t is None: t = self.t if dist_coeffs is None: dist_coeffs = self.dist_coeffs if orig_size is None: orig_size = self.orig_size vertices = nr.projection(vertices, K, R, t, dist_coeffs, orig_size) # rasterization faces = nr.vertices_to_faces(vertices, faces) if isinstance(self.background_color, (tuple, list)): background = self.background_color else: assert (isinstance(self.background_color, (float, int))) background = [ self.background_color for _ in range(textures.size()[-1]) ] images = nr.rasterize(faces, textures, self.image_size, self.anti_aliasing, self.near, self.far, self.rasterizer_eps, background) return images
def test_case1(self): faces = np.random.normal(size=(64, 16, 3, 3)).astype('float32') textures = np.random.normal(size=(64, 16, 8, 8, 8, 3)).astype('float32') neural_renderer.lighting(faces, textures)