def test_empty_meshes(self): device = torch.device("cuda:0") target_length = 0 N = 10 V = 32 verts_list = [] faces_list = [] for _ in range(N): vn = torch.randint(3, high=V, size=(1, ))[0].item() verts = torch.rand((vn, 3), dtype=torch.float32, device=device) faces = torch.tensor([], dtype=torch.int64, device=device) verts_list.append(verts) faces_list.append(faces) mesh = Meshes(verts=verts_list, faces=faces_list) loss = mesh_edge_loss(mesh, target_length=target_length) self.assertTrue( torch.allclose( loss, torch.tensor([0.0], dtype=torch.float32, device=device))) self.assertTrue(loss.requires_grad)
def test_getitem(self): N = 5 V = 20 F = 10 source = {"atlas": torch.randn(size=(N, F, 4, 4, 3))} tex = TexturesAtlas(atlas=source["atlas"]) verts = torch.rand(size=(N, V, 3)) faces = torch.randint(size=(N, F, 3), high=V) meshes = Meshes(verts=verts, faces=faces, textures=tex) tryindex(self, 2, tex, meshes, source) tryindex(self, slice(0, 2, 1), tex, meshes, source) index = torch.tensor([1, 0, 1, 0, 0], dtype=torch.bool) tryindex(self, index, tex, meshes, source) index = torch.tensor([0, 0, 0, 0, 0], dtype=torch.bool) tryindex(self, index, tex, meshes, source) index = torch.tensor([1, 2], dtype=torch.int64) tryindex(self, index, tex, meshes, source) tryindex(self, [2, 4], tex, meshes, source)
def test_taubin(self): N = 3 device = get_random_cuda_device() mesh = ico_sphere(4, device).extend(N) ico_verts = mesh.verts_padded() ico_faces = mesh.faces_padded() rand_noise = torch.rand_like(ico_verts) * 0.2 - 0.1 z_mask = (ico_verts[:, :, -1] > 0).view(N, -1, 1) rand_noise = rand_noise * z_mask verts = ico_verts + rand_noise mesh = Meshes(verts=verts, faces=ico_faces) smooth_mesh = taubin_smoothing(mesh, num_iter=50) smooth_verts = smooth_mesh.verts_padded() smooth_dist = (smooth_verts - ico_verts).norm(dim=-1).mean() dist = (verts - ico_verts).norm(dim=-1).mean() self.assertTrue(smooth_dist < dist)
def __getitem__(self, index): path = self.paths[index][0] label = self.paths[index][1] # Load the obj and ignore the textures and materials. verts, faces_idx, _ = load_obj(path) faces = faces_idx.verts_idx # center = verts.mean(0) # verts = verts - center # scale = max(verts.abs().max(0)[0]) # verts = verts / scale # Initialize each vertex to be white in color. verts_rgb = torch.ones_like(verts)[None] # (1, V, 3) textures = Textures(verts_rgb=verts_rgb) # Create a Meshes object for the teapot. Here we have only one mesh in the batch. mesh = Meshes(verts=[verts], faces=[faces], textures=textures) return mesh, label
def _get_cube_mesh(self): # NOTE(ycho): duplicated from _get_cube_cloud() vertices = list( itertools.product(*zip([-0.5, -0.5, -0.5], [0.5, 0.5, 0.5]))) vertices = np.insert(vertices, 0, [0, 0, 0], axis=0) vertices = th.as_tensor(vertices, dtype=th.float32, device=self.device) # FIXME(ycho): Hardcoded face indices. # (We can't use Box.FACE since we need triangulated faces) faces = [[7, 3, 5], [5, 3, 1], [5, 1, 6], [6, 1, 2], [6, 2, 8], [8, 2, 4], [8, 4, 7], [7, 4, 3], [3, 4, 1], [1, 4, 2], [8, 7, 6], [6, 7, 5]] face_indices = th.as_tensor(faces, dtype=th.int32, device=self.device).reshape(-1, 3, 3) textures = TexturesVertex(verts_features=(0.5 + 0.5 * vertices)[None]) mesh = Meshes(verts=vertices[None], faces=face_indices[None], textures=textures) return mesh
def forward(self, vertices, faces, attributes=None): """ Args: meshes_world: a Meshes object representing a batch of meshes with coordinates in world space. Returns: Fragments: Rasterization outputs as a named tuple. """ fixed_vetices = vertices.clone() fixed_vetices[..., :2] = -fixed_vetices[..., :2] meshes_screen = Meshes(verts=fixed_vetices.float(), faces=faces.long()) raster_settings = self.raster_settings pix_to_face, zbuf, bary_coords, dists = rasterize_meshes( meshes_screen, image_size=raster_settings.image_size, blur_radius=raster_settings.blur_radius, faces_per_pixel=raster_settings.faces_per_pixel, bin_size=raster_settings.bin_size, max_faces_per_bin=raster_settings.max_faces_per_bin, perspective_correct=raster_settings.perspective_correct, ) vismask = (pix_to_face > -1).float() D = attributes.shape[-1] attributes = attributes.clone() attributes = attributes.view(attributes.shape[0] * attributes.shape[1], 3, attributes.shape[-1]) N, H, W, K, _ = bary_coords.shape #plt.imshow(zbuf[0,:,:,0]) #plt.show() mask = pix_to_face == -1 # [] pix_to_face = pix_to_face.clone() pix_to_face[mask] = 0 idx = pix_to_face.view(N * H * W * K, 1, 1).expand(N * H * W * K, 3, D) pixel_face_vals = attributes.gather(0, idx).view(N, H, W, K, 3, D) pixel_vals = (bary_coords[..., None] * pixel_face_vals).sum(dim=-2) pixel_vals[mask] = 0 # Replace masked values in output. pixel_vals = pixel_vals[:, :, :, 0].permute(0, 3, 1, 2) pixel_vals = torch.cat([pixel_vals, vismask[:, :, :, 0][:, None, :, :]], dim=1) # import ipdb; ipdb.set_trace() return pixel_vals
def test_interpolate_texture_map(self): barycentric_coords = torch.tensor([[0.5, 0.3, 0.2], [0.3, 0.6, 0.1]], dtype=torch.float32).view( 1, 1, 1, 2, -1) dummy_verts = torch.zeros(4, 3) vert_uvs = torch.tensor([[1, 0], [0, 1], [1, 1], [0, 0]], dtype=torch.float32) face_uvs = torch.tensor([[0, 1, 2], [1, 2, 3]], dtype=torch.int64) interpolated_uvs = torch.tensor( [[0.5 + 0.2, 0.3 + 0.2], [0.6, 0.3 + 0.6]], dtype=torch.float32) # Create a dummy texture map H = 2 W = 2 x = torch.linspace(0, 1, W).view(1, W).expand(H, W) y = torch.linspace(0, 1, H).view(H, 1).expand(H, W) tex_map = torch.stack([x, y], dim=2).view(1, H, W, 2) pix_to_face = torch.tensor([0, 1], dtype=torch.int64).view(1, 1, 1, 2) fragments = Fragments( pix_to_face=pix_to_face, bary_coords=barycentric_coords, zbuf=pix_to_face, dists=pix_to_face, ) tex = Textures( maps=tex_map, faces_uvs=face_uvs[None, ...], verts_uvs=vert_uvs[None, ...], ) meshes = Meshes(verts=[dummy_verts], faces=[face_uvs], textures=tex) texels = interpolate_texture_map(fragments, meshes) # Expected output pixel_uvs = interpolated_uvs * 2.0 - 1.0 pixel_uvs = pixel_uvs.view(2, 1, 1, 2) tex_map = torch.flip(tex_map, [1]) tex_map = tex_map.permute(0, 3, 1, 2) tex_map = torch.cat([tex_map, tex_map], dim=0) expected_out = F.grid_sample(tex_map, pixel_uvs, align_corners=False) self.assertTrue( torch.allclose(texels.squeeze(), expected_out.squeeze()))
def loss(self, data, epoch): pred = self.forward(data) # embed() CE_Loss = nn.CrossEntropyLoss() ce_loss = CE_Loss(pred[0][-1][3], data['y_voxels']) chamfer_loss = torch.tensor(0).float().cuda() edge_loss = torch.tensor(0).float().cuda() laplacian_loss = torch.tensor(0).float().cuda() normal_consistency_loss = torch.tensor(0).float().cuda() for c in range(self.config.num_classes-1): target = data['surface_points'][c].cuda() for k, (vertices, faces, _, _, _) in enumerate(pred[c][1:]): pred_mesh = Meshes(verts=list(vertices), faces=list(faces)) pred_points = sample_points_from_meshes(pred_mesh, 3000) chamfer_loss += chamfer_distance(pred_points, target)[0] laplacian_loss += mesh_laplacian_smoothing(pred_mesh, method="uniform") normal_consistency_loss += mesh_normal_consistency(pred_mesh) edge_loss += mesh_edge_loss(pred_mesh) loss = 1 * chamfer_loss + 1 * ce_loss + 0.1 * laplacian_loss + 1 * edge_loss + 0.1 * normal_consistency_loss log = {"loss": loss.detach(), "chamfer_loss": chamfer_loss.detach(), "ce_loss": ce_loss.detach(), "normal_consistency_loss": normal_consistency_loss.detach(), "edge_loss": edge_loss.detach(), "laplacian_loss": laplacian_loss.detach()} return loss, log
def forward_step(th_scan_meshes, smplx, init_smplx_meshes, search_tree, pen_distance, tri_filtering_module): """ Performs a forward step, given smplx and scan meshes. Then computes the losses. """ # forward # verts, _, _, _ = smplx() verts = smplx() th_SMPLX_meshes = [tm.from_tensors(vertices=v, faces=smplx.faces) for v in verts] p3d_meshes = Meshes(verts=verts, faces=smplx.faces.expand(1,-1,-1)) # losses loss = dict() loss['s2m'] = batch_point_to_surface([sm.vertices for sm in th_scan_meshes], th_SMPLX_meshes) loss['m2s'] = batch_point_to_surface([sm.vertices for sm in th_SMPLX_meshes], th_scan_meshes) loss['lap'] = torch.stack([laplacian_loss(sc, sm) for sc, sm in zip(init_smplx_meshes, th_SMPLX_meshes)]) # loss['offsets'] = torch.mean(torch.mean(smplx.offsets**2, axis=1), axis=1) # loss['normal'] = mesh_normal_consistency(p3d_meshes).unsqueeze(0) # loss['interpenetration'] = interpenetration_loss(verts, smplx.faces, search_tree, pen_distance, tri_filtering_module, 1.0) return loss
def test_verts_nan(self): num_verts = 30 num_faces = 50 for device in ["cpu", "cuda:0"]: for invalid in ["nan", "inf"]: verts = torch.rand((num_verts, 3), dtype=torch.float32, device=device) # randomly assign an invalid type verts[torch.randperm(num_verts)[:10]] = float(invalid) faces = torch.randint(num_verts, size=(num_faces, 3), dtype=torch.int64, device=device) meshes = Meshes(verts=[verts], faces=[faces]) with self.assertRaisesRegex(ValueError, "Meshes contain nan or inf."): sample_points_from_meshes(meshes, num_samples=100, return_normals=True)
def forward(self, images): ''' 输入: N张图片 N list C*H*W 输出: 对应N个视角的图片 : N list C*H*W ''' # 为每张图像提取特征 # embeddings = [] # for i in range(len(images)): # embeddings.append(self.img_embedding_model(images[i])) # features_cat = torch.zeros([embeddings[0].shape[0],self.f_dim*self.N]).cuda() # for i in range(len(embeddings)): # x = self.GAP(embeddings[i]) # x = x.view(x.size(0), -1) # features_cat[:,i*self.f_dim:(i+1)*self.f_dim] = x # 图像通道混合后直接unet features_cat = images[0] for i in range(len(images) - 1): features_cat = torch.cat((features_cat, images[i + 1]), dim=1) # 特征提取 # features_cat = self.img_embedding_model(features_cat) # features_cat = self.GAP(features_cat) # features_cat = features_cat.view(features_cat.size(0), -1) # 将特征输入解码器,得到displacement map和uv map pred_tex, _ = self.displace_net(features_cat) _, mesh_map = self.uvmap_net(features_cat) raw_vtx = self.meshtemp.get_vertex_positions(mesh_map) vertex_positions, mesh_faces, input_uvs, input_texture, mesh_face_textures = self.meshtemp.forward_renderer( raw_vtx, pred_tex) repro_imgs, img_probs = self.renderer(vertex_positions, mesh_faces, input_uvs, input_texture, mesh_face_textures) new_mesh = Meshes(verts=raw_vtx, faces=self.meshtemp.mesh.faces.unsqueeze(0).repeat( raw_vtx.shape[0], 1, 1)) return repro_imgs, raw_vtx, img_probs, self.meshtemp.mesh.faces, new_mesh, input_texture
def deform_mesh_by_closest_vertices(mesh, src_mesh, tar_mesh, device=torch.device("cpu")): """ mesh の頂点と最も近い src_mesh の頂点番号を元に、mesh を tar_mesh の形状に変更する。 [ToDo] 複数バッチ処理に対応 """ if (len(mesh.verts_packed().shape) == 2): batch_size = 1 else: batch_size = mesh.verts_packed().shape[0] # pytorch3d -> psbody.mesh への変換 mesh_face_pytorch3d = mesh.faces_packed() mesh = Mesh(mesh.verts_packed().detach().cpu().numpy(), mesh.faces_packed().detach().cpu().numpy()) src_mesh = Mesh(src_mesh.verts_packed().detach().cpu().numpy(), src_mesh.faces_packed().detach().cpu().numpy()) tar_mesh = Mesh(tar_mesh.verts_packed().detach().cpu().numpy(), tar_mesh.faces_packed().detach().cpu().numpy()) # verts_idx : 最も近い頂点番号 verts_idx, _ = src_mesh.closest_vertices(mesh.v) verts_idx = np.array(verts_idx) new_mesh_verts = mesh.v - src_mesh.v[verts_idx] + tar_mesh.v[verts_idx] #print( "verts_idx : ", verts_idx ) #print( "new_mesh_verts.shape : ", new_mesh_verts.shape ) # (7702, 3) #print( "mesh.f.shape : ", mesh.f.shape ) # (15180, 3) # psbody.mesh -> pytorch3d への変換 if (batch_size == 1): new_mesh = Meshes( torch.from_numpy( new_mesh_verts).requires_grad_(False).float().unsqueeze(0), mesh_face_pytorch3d.unsqueeze(0)).to(device) else: NotImplementedError() return new_mesh
def transform_meshes(meshes, camera_info): """ input: @meshes: mesh in local frame @camera_info: plane params from camera info, type = dict, must contain 'position' and 'rotation' as keys output: mesh in global frame. """ tran = camera_info["position"] rot = camera_info["rotation"] verts_packed = meshes.verts_packed() verts_packed = verts_packed * torch.tensor( [1.0, -1.0, -1.0], dtype=torch.float32) # suncg2habitat faces_list = meshes.faces_list() tex = meshes.textures rot_matrix = torch.tensor(quaternion.as_rotation_matrix(rot), dtype=torch.float32) verts_packed = torch.mm(rot_matrix, verts_packed.T).T + torch.tensor( tran, dtype=torch.float32) verts_list = list( verts_packed.split(meshes.num_verts_per_mesh().tolist(), dim=0)) return Meshes(verts=verts_list, faces=faces_list, textures=tex)
def test_texture_sampling(self): device = torch.device("cuda:0") batch_size = 6 # verts verts = torch.rand((batch_size, 6, 3), device=device, dtype=torch.float32) verts[:, :3, 2] = 1.0 verts[:, 3:, 2] = -1.0 # textures texts = torch.rand((batch_size, 6, 3), device=device, dtype=torch.float32) # faces faces = torch.tensor([[0, 1, 2], [3, 4, 5]], device=device, dtype=torch.int64) faces = faces.view(1, 2, 3).expand(batch_size, -1, -1) meshes = Meshes(verts=verts, faces=faces, textures=TexturesVertex(texts)) num_samples = 24 samples, normals, textures = sample_points_from_meshes( meshes, num_samples=num_samples, return_normals=True, return_textures=True ) textures_naive = torch.zeros( (batch_size, num_samples, 3), dtype=torch.float32, device=device ) for n in range(batch_size): for i in range(num_samples): p = samples[n, i] if p[2] > 0.0: # sampled from 1st face v0, v1, v2 = verts[n, 0, :2], verts[n, 1, :2], verts[n, 2, :2] w0, w1, w2 = barycentric_coordinates(p[:2], v0, v1, v2) t0, t1, t2 = texts[n, 0], texts[n, 1], texts[n, 2] else: # sampled from 2nd face v0, v1, v2 = verts[n, 3, :2], verts[n, 4, :2], verts[n, 5, :2] w0, w1, w2 = barycentric_coordinates(p[:2], v0, v1, v2) t0, t1, t2 = texts[n, 3], texts[n, 4], texts[n, 5] tt = w0 * t0 + w1 * t1 + w2 * t2 textures_naive[n, i] = tt self.assertClose(textures, textures_naive)
def load_objs_as_meshes(files: list, device=None, load_textures: bool = True): """ Load meshes from a list of .obj files using the load_obj function, and return them as a Meshes object. This only works for meshes which have a single texture image for the whole mesh. See the load_obj function for more details. material_colors and normals are not stored. Args: f: A list of file-like objects (with methods read, readline, tell, and seek), pathlib paths or strings containing file names. device: Desired device of returned Meshes. Default: uses the current device for the default tensor type. load_textures: Boolean indicating whether material files are loaded Returns: New Meshes object. """ mesh_list = [] for f_obj in files: # TODO: update this function to support the two texturing options. verts, faces, aux = load_obj(f_obj, load_textures=load_textures) verts = verts.to(device) tex = None tex_maps = aux.texture_images if tex_maps is not None and len(tex_maps) > 0: verts_uvs = aux.verts_uvs[None, ...].to(device) # (1, V, 2) faces_uvs = faces.textures_idx[None, ...].to(device) # (1, F, 3) image = list(tex_maps.values())[0].to(device)[None] tex = Textures(verts_uvs=verts_uvs, faces_uvs=faces_uvs, maps=image) mesh = Meshes(verts=[verts], faces=[faces.verts_idx.to(device)], textures=tex) mesh_list.append(mesh) if len(mesh_list) == 1: return mesh_list[0] return join_meshes_as_batch(mesh_list)
def generate_video_from_obj(obj_path, image_path, video_path, renderer): input_image = cv2.imread(image_path) input_image = input_image[:, :input_image.shape[1] // 3] input_image = cv2.resize(input_image, (512, 512)) # Setup device = torch.device("cuda:0") torch.cuda.set_device(device) # Load obj file verts_rgb_colors = get_verts_rgb_colors(obj_path) verts_rgb_colors = torch.from_numpy(verts_rgb_colors).to(device) textures = TexturesVertex(verts_features=verts_rgb_colors) # wo_textures = TexturesVertex(verts_features=torch.ones_like(verts_rgb_colors)*0.75) # Load obj mesh = load_objs_as_meshes([obj_path], device=device) # Set mesh vers = mesh._verts_list faces = mesh._faces_list mesh_w_tex = Meshes(vers, faces, textures) # mesh_wo_tex = Meshes(vers, faces, wo_textures) # create VideoWriter fourcc = cv2.VideoWriter_fourcc(*'MP4V') out = cv2.VideoWriter(video_path, fourcc, 20.0, (1024, 512)) for i in tqdm(range(90)): R, T = look_at_view_transform(1.8, 0, i * 4, device=device) images_w_tex = renderer(mesh_w_tex, R=R, T=T) images_w_tex = np.clip(images_w_tex[0, ..., :3].cpu().numpy(), 0.0, 1.0)[:, :, ::-1] * 255 # images_wo_tex = renderer(mesh_wo_tex, R=R, T=T) # images_wo_tex = np.clip(images_wo_tex[0, ..., :3].cpu().numpy(), 0.0, 1.0)[:, :, ::-1] * 255 image = np.concatenate([input_image, images_w_tex], axis=1) out.write(image.astype('uint8')) out.release()
def forward(self, coeffs, render=True): batch_num = coeffs.shape[0] id_coeff, exp_coeff, tex_coeff, angles, gamma, translation = self.split_coeffs( coeffs) vs = self.get_vs(id_coeff, exp_coeff) rotation = self.compute_rotation_matrix(angles) vs_t = self.rigid_transform( vs, rotation, translation) lms_t = self.get_lms(vs_t) lms_proj = self.project_vs(lms_t) lms_proj = torch.stack( [lms_proj[:, :, 0], self.img_size-lms_proj[:, :, 1]], dim=2) if render: face_texture = self.get_color(tex_coeff) face_norm = self.compute_norm(vs, self.tri, self.point_buf) face_norm_r = face_norm.bmm(rotation) face_color = self.add_illumination( face_texture, face_norm_r, gamma) face_color_tv = TexturesVertex(face_color) mesh = Meshes(vs_t, self.tri.repeat( batch_num, 1, 1), face_color_tv) rendered_img = self.renderer(mesh) rendered_img = torch.clamp(rendered_img, 0, 255) return {'rendered_img': rendered_img, 'lms_proj': lms_proj, 'face_texture': face_texture, 'vs': vs_t, 'tri': self.tri, 'color': face_color} else: return {'lms_proj': lms_proj}
def validate_training_AE(validation_generator, model): ''' This function is used to calculate validation loss during training NMF AE ''' print("Validating model......") with torch.no_grad(): total_loss = 0 items = 0 for input, _, _ in validation_generator: input = input.cuda() _, _, pred2, face = model( input ) # Point prediction after each deform block and face information (refer figure 4 in paper) mesh_p2 = Meshes(verts=pred2, faces=face) # Construct Differentiable mesh M_p2 pts2 = sample_points_from_meshes( mesh_p2, num_samples=2562 ) # Differentiably sample random points from mesh surfaces loss, _ = chamfer_distance(pts2, input) total_loss += loss.item() items += 1 return total_loss / items # Return average validation loss
def __call__(self, points, faces, colors=None, mean=None, std=None, grayscale=True): assert len(points.shape) == 4 and points.shape[1] == 3 colors = colors if colors is not None else torch.ones_like(points) points, colors = grid_to_list(points), grid_to_list(colors) if self.renderer is None: self.setup(points.device) textures = TexturesVertex(verts_features=colors) mesh = Meshes(verts=points, faces=faces, textures=textures) r_images = self.renderer(mesh) r_images = r_images.permute(0, 3, 1, 2).contiguous() r_images = r_images[:, :3, :, :] if grayscale: r_images = r_images.mean(dim=1, keepdim=True) if mean and std: r_images = (r_images - mean) / std return r_images
def collate_fn(batch): imgs, verts, faces, points, normals, voxels, Ps, id_strs = zip(*batch) imgs = torch.stack(imgs, dim=0) if verts[0] is not None and faces[0] is not None: # TODO(gkioxari) Meshes should accept tuples meshes = Meshes(verts=list(verts), faces=list(faces)) else: meshes = None if points[0] is not None and normals[0] is not None: points = torch.stack(points, dim=0) normals = torch.stack(normals, dim=0) else: points, normals = None, None if voxels[0] is None: voxels = None Ps = None elif voxels[0].dim() == 2: # They are voxel coords Ps = torch.stack(Ps, dim=0) elif voxels[0].dim() == 3: # They are actual voxels voxels = torch.stack(voxels, dim=0) return imgs, meshes, points, normals, voxels, Ps, id_strs
def generate_video_from_obj(obj_path, video_path, renderer): # Setup device = torch.device("cuda:0") torch.cuda.set_device(device) # Load obj file verts_rgb_colors = get_verts_rgb_colors(obj_path) verts_rgb_colors = torch.from_numpy(verts_rgb_colors).to(device) textures = TexturesVertex(verts_features=verts_rgb_colors) wo_textures = TexturesVertex( verts_features=torch.ones_like(verts_rgb_colors) * 0.75) # Load obj #mesh = load_objs_as_meshes([obj_path], device=device) mesh = trimesh.load(obj_path) mesh.vertices -= mesh.center_mass # Set mesh mesh_wo_tex = Meshes(torch.FloatTensor(mesh.vertices), torch.FloatTensor(mesh.faces), wo_textures) # create VideoWriter fourcc = cv2.VideoWriter_fourcc(*'MP4V') out = cv2.VideoWriter(video_path, fourcc, 20.0, (512, 512)) for i in tqdm(range(90)): R, T = look_at_view_transform(vers[0][1].mean(), 0, i * 4, device=device) images_wo_tex = renderer(mesh_wo_tex, R=R, T=T) images_wo_tex = np.clip(images_wo_tex[0, ..., :3].cpu().numpy(), 0.0, 1.0)[:, :, ::-1] * 255 #image = np.concatenate([images_w_tex, images_wo_tex], axis=1) image = images_wo_tex out.write(image.astype('uint8')) out.release()
def test_my_renderer(): vert, normal, st, color, face = load_blender_ply_mesh( '../data/meshes/background.ply') # V, _ = vert.shape # meshes = Meshes( # verts=[vert.to(device)], # faces=[face.to(device)], # textures=TexturesVertex([torch.cat((color.to(device), torch.ones(V, 1, device=device)), dim=1)]) # ) # vert = torch.tensor([ # [-1, -1, 0], # [1, -1, 0], # [1, 1, 0], # [-1, 1, 0] # ], dtype=torch.float) * 30 x = 80 y = 93 vert = torch.tensor([[-x, -y, 0], [x, -y, 0], [x, y, 0], [-x, y, 0]], dtype=torch.float) face = torch.LongTensor([[0, 1, 2], [0, 2, 3]]) meshes = Meshes(verts=[vert.to(device)], faces=[face.to(device)], textures=TexturesVertex([color.to(device)])) scene = join_meshes_as_scene(meshes) zbuffer_renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=cameras, raster_settings=raster_settings, ), shader=IdentityShader()) plot_channels(render(zbuffer_renderer, scene))
def compute_mesh_visibility(vert, tri, bfm_torch, H_img, W_img): N, nver, _ = vert.shape ntri = tri.shape[0] tri = torch.from_numpy(tri).to(vert.device).unsqueeze(0).expand(N, ntri, 3) # Transform to NDC vert_t = vert + torch.tensor((0.5, 0.5, 0), dtype=torch.float, device=vert.device).view(1, 1, 3) vert_t = vert_t * torch.tensor((-1, 1, -1), dtype=torch.float, device=vert.device).view(1, 1, 3) vert_t = vert_t * torch.tensor((2. / float(W_img), 2. / float(H_img), 1), dtype=torch.float, device=vert.device).view(1, 1, 3) # vert_t = vert * torch.tensor((-1, 1, -1), dtype=torch.float, device=vert.device).view(1, 1, 3) # vert_t = vert_t * torch.tensor((2. / (float(W_img) - 1.), 2. / (float(H_img) - 1.), 1), # dtype=torch.float, device=vert.device).view(1, 1, 3) z_offset = -vert_t[:, :, 2].min() + 10. vert_t = vert_t + torch.tensor((0, 0, z_offset.item()), dtype=torch.float, device=vert.device).view(1, 1, 3) # Render mesh_torch = Meshes(verts=vert_t, faces=tri) pix_to_face, zbuf, _, _ = rasterize_meshes(mesh_torch, image_size=H_img, faces_per_pixel=1, cull_backfaces=True) assert H_img == W_img # Compute visibility zbuf = (zbuf.view(N, 1, H_img, W_img) - z_offset) * -1 - 1. zbuf = torch.where(torch.eq(pix_to_face.view(N, 1, H_img, W_img), -1), torch.zeros_like(zbuf) - 2e2, zbuf) vert_zbuf = sample_per_vert_feat(zbuf, vert, bfm_torch, H_img, W_img).squeeze(1) mask = torch.gt(vert[:, :, 2], vert_zbuf) return mask, zbuf
def transform_meshes_to_camera_coord_system(meshes, boxes, zranges, Ks, imsize): device = meshes.device new_verts, new_faces = [], [] h, w = imsize im_size = torch.tensor([w, h], device=device).view(1, 2) assert len(meshes) == len(zranges) for i in range(len(meshes)): verts, faces = meshes.get_mesh_verts_faces(i) if verts.numel() == 0: verts, faces = ico_sphere(level=3, device=device).get_mesh_verts_faces(0) assert not torch.isnan(verts).any() assert not torch.isnan(faces).any() roi = boxes[i].view(1, 4) zrange = zranges[i].view(1, 2) K = Ks[i].view(1, 3) cub3D = shape_utils.box2D_to_cuboid3D(zrange, K, roi, im_size) txz, tyz = shape_utils.cuboid3D_to_unitbox3D(cub3D) # image to camera coords verts[:, 0] = -verts[:, 0] verts[:, 1] = -verts[:, 1] # transform to destination size xz = verts[:, [0, 2]] yz = verts[:, [1, 2]] pxz = txz.inverse(xz.view(1, -1, 2)).squeeze(0) pyz = tyz.inverse(yz.view(1, -1, 2)).squeeze(0) verts = torch.stack([pxz[:, 0], pyz[:, 0], pxz[:, 1]], dim=1).to(device, dtype=torch.float32) new_verts.append(verts) new_faces.append(faces) return Meshes(verts=new_verts, faces=new_faces)
def load_mesh(obj_path): device = torch.device('cuda') vertices, faces, aux = load_obj(obj_path) vertices_uvs = aux.verts_uvs[None, ...].to(device) faces_uvs = faces.textures_idx[None, ...].to(device) texture_maps = aux.texture_images texture_maps = list(texture_maps.values())[0] texture_maps = texture_maps[None, ...].to(device) textures = Textures( verts_uvs=vertices_uvs, faces_uvs=faces_uvs, maps=texture_maps, ) vertices = vertices.to(device) faces = faces.verts_idx.to(device) mesh = Meshes(verts=[vertices], faces=[faces], textures=textures) return mesh
def loss_rnet(self, dorig, drec, ds_name='train'): out_put = self.rhm_train(**drec) verts_rhand = out_put.vertices rh_mesh = Meshes(verts=verts_rhand, faces=self.rh_f).to( self.device).verts_normals_packed().view(-1, 778, 3) h2o_gt = dorig['h2o_gt'] o2h_signed, h2o, _ = point2point_signed(verts_rhand, dorig['obj_sampled_verts_gt'], rh_mesh) ######### dist loss loss_dist_h = 35 * (1. - self.cfg.kl_coef) * torch.mean( torch.einsum('ij,j->ij', torch.abs(h2o.abs() - h2o_gt.abs()), self.v_weights2)) ########## verts loss loss_mesh_rec_w = 20 * (1. - self.cfg.kl_coef) * torch.mean( torch.einsum( 'ijk,j->ijk', torch.abs( (dorig['hand_verts_gt'] - verts_rhand)), self.v_weights2)) ########## edge loss loss_edge = 10 * (1. - self.cfg.kl_coef) * self.LossL1( self.edges_for(verts_rhand, self.vpe), self.edges_for(dorig['hand_verts_gt'], self.vpe)) ########## loss_dict = { 'loss_edge_r': loss_edge, 'loss_mesh_rec_r': loss_mesh_rec_w, 'loss_dist_h_r': loss_dist_h, } loss_total = torch.stack(list(loss_dict.values())).sum() loss_dict['loss_total'] = loss_total return loss_total, loss_dict
def make_mesh(verts: torch.tensor, faces: np.ndarray, detach: bool, textures=None) -> Meshes: device = torch.device("cuda:0") if detach: verts = verts.detach() # Initialize each vertex to be white in color. if textures is None: verts_rgb = torch.ones_like(verts)[None] # (1, V, 3) textures = Textures(verts_rgb=verts_rgb.to(device)) faces = torch.tensor(np.int32(faces), dtype=torch.long).cuda() # return Meshes( # verts=[verts.to(device)], # faces=[faces.to(device)], # textures=textures # ) return Meshes(verts=verts.to(device), faces=faces.to(device).repeat(verts.shape[0], 1, 1), textures=textures.to(device))
def collate_batched_meshes(batch: List[Dict]): # pragma: no cover """ Take a list of objects in the form of dictionaries and merge them into a single dictionary. This function can be used with a Dataset object to create a torch.utils.data.Dataloader which directly returns Meshes objects. TODO: Add support for textures. Args: batch: List of dictionaries containing information about objects in the dataset. Returns: collated_dict: Dictionary of collated lists. If batch contains both verts and faces, a collated mesh batch is also returned. """ if batch is None or len(batch) == 0: return None collated_dict = {} for k in batch[0].keys(): collated_dict[k] = [d[k] for d in batch] collated_dict["mesh"] = None if {"verts", "faces"}.issubset(collated_dict.keys()): textures = None if "textures" in collated_dict: textures = TexturesAtlas(atlas=collated_dict["textures"]) collated_dict["mesh"] = Meshes( verts=collated_dict["verts"], faces=collated_dict["faces"], textures=textures, ) return collated_dict
def render_images(vertices, faces, texture, faces_uvs, verts_uvs, crop_size, device): textures = Textures(maps=texture, faces_uvs=faces_uvs, verts_uvs=verts_uvs) meshes = Meshes(vertices, faces, textures) R, T = look_at_view_transform(1.0, 0.5, 0, device=device) camera = OpenGLPerspectiveCameras(R=R, T=T, fov=20, device=device) raster_settings = RasterizationSettings(image_size=crop_size * 2, blur_radius=0.0, faces_per_pixel=1, bin_size=None, max_faces_per_bin=None) lights = PointLights(location=[[0.0, 0.0, -3.0]], device=device) renderer = MeshRenderer(rasterizer=MeshRasterizer( cameras=camera, raster_settings=raster_settings), shader=TexturedSoftPhongShader(cameras=camera, device=device, lights=lights)) images = renderer(meshes) return images
def test_sample_textures_error(self): N = 5 V = 20 verts = torch.rand(size=(N, V, 3)) faces = torch.randint(size=(N, 10, 3), high=V) tex = TexturesVertex(verts_features=torch.randn(size=(N, 10, 128))) # Verts features have the wrong number of verts with self.assertRaisesRegex(ValueError, "do not match the dimensions"): Meshes(verts=verts, faces=faces, textures=tex) # Verts features have the wrong batch dim tex = TexturesVertex(verts_features=torch.randn(size=(1, V, 128))) with self.assertRaisesRegex(ValueError, "do not match the dimensions"): Meshes(verts=verts, faces=faces, textures=tex) meshes = Meshes(verts=verts, faces=faces) meshes.textures = tex # Cannot use the texture attribute set on meshes for sampling # textures if the dimensions don't match with self.assertRaisesRegex(ValueError, "do not match the dimensions"): meshes.sample_textures(None)