def export_points(mesh, modelname, loc, scale, args): if not mesh.is_watertight: print('Warning: mesh %s is not watertight!' 'Cannot sample points.' % modelname) return filename = os.path.join(args.points_folder, modelname + '.npz') if not args.overwrite and os.path.exists(filename): print('Points already exist: %s' % filename) return n_points_uniform = int(args.points_size * args.points_uniform_ratio) n_points_noise_surface = args.points_size - n_points_uniform # uniform boxsize = 1 + args.points_padding points_uniform = np.random.rand(n_points_uniform, 3) points_uniform = boxsize * (points_uniform - 0.5) # surface + noise points_noise_surface, points_noise_index = mesh.sample( n_points_noise_surface, return_index=True) points_noise_normal = mesh.face_normals[points_noise_index] points_noise_ratio = np.random.rand(n_points_noise_surface) * 2. - 1. points_noise_surface += args.points_sigma * points_noise_ratio.reshape( n_points_noise_surface, 1) * points_noise_normal points = np.concatenate([points_uniform, points_noise_surface], axis=0) points_uniform_occupancies = check_mesh_contains( mesh, points_uniform).astype(np.float32) points_noise_surface_occupancies = check_mesh_contains( mesh, points_noise_surface).astype(np.float32) # adjustment points_noise_ratio = np.abs(points_noise_ratio) points_noise_ratio[points_noise_ratio > 1] = 1 points_noise_surface_occupancies = (points_noise_surface_occupancies - 0.5) * points_noise_ratio + 0.5 occupancies = np.concatenate( [points_uniform_occupancies, points_noise_surface_occupancies], axis=0) # Compress if args.float16: dtype = np.float16 else: dtype = np.float32 points = points.astype(dtype) occupancies = occupancies.astype(dtype) print('Writing points: %s' % filename) np.savez(filename, points=points, occupancies=occupancies, loc=loc, scale=scale)
def eval_mesh2(self, mesh, pointcloud_tgt, normals_tgt, points_iou, occ_tgt): ''' Evaluates a mesh. Args: mesh (trimesh): mesh which should be evaluated pointcloud_tgt (numpy array): target point cloud normals_tgt (numpy array): target normals points_iou (numpy_array): points tensor for IoU evaluation occ_tgt (numpy_array): GT occupancy values for IoU points ''' if len(mesh.vertices) != 0 and len(mesh.faces) != 0: pointcloud, idx = mesh.sample(self.n_points, return_index=True) pointcloud = pointcloud.astype(np.float32) normals = mesh.face_normals[idx] else: pointcloud = np.empty((0, 3)) normals = np.empty((0, 3)) out_dict = self.eval_pointcloud( pointcloud, pointcloud_tgt, normals, normals_tgt) if len(mesh.vertices) != 0 and len(mesh.faces) != 0: occ = check_mesh_contains(mesh, points_iou) out_dict['iou'] = compute_iou(occ, occ_tgt) else: out_dict['iou'] = 0. return out_dict
def export_points(mesh, modelname, loc, scale, args): if not mesh.is_watertight: print('Warning: mesh %s is not watertight!' 'Cannot sample points.' % modelname) return filename = os.path.join(args.points_folder, modelname + '.npz') if not args.overwrite and os.path.exists(filename): print('Points already exist: %s' % filename) return # uniform grid total_l = 1. + args.points_padding res = args.points_resolution points_uniform = make_3d_grid((-total_l/2.0 + total_l/(res*2),)*3, (total_l/2.0 - total_l/(res*2),)*3, (res,)*3).numpy() points_uniform_occupancies = check_mesh_contains(mesh, points_uniform).astype(np.float32) points = points_uniform occupancies = points_uniform_occupancies # Compress if args.float16: dtype = np.float16 else: dtype = np.float32 points = points.astype(dtype) occupancies = occupancies.astype(dtype) print('Writing points: %s' % filename) np.savez(filename, points=points, occupancies=occupancies, loc=loc, scale=scale)
def export_points(modelname, model_files, args): out_folder = os.path.join(args.points_folder, modelname) if os.path.exists(out_folder): if not args.overwrite: print('Points already exist: %s' % out_folder) return else: shutil.rmtree(out_folder) # Create out_folder os.makedirs(out_folder) n_points_uniform = int(args.points_size * args.points_uniform_ratio) n_points_surface = args.points_size - n_points_uniform for it, model_file in enumerate(model_files): out_file = os.path.join(out_folder, '%08d.npz' % it) mesh = trimesh.load(model_file, process=False) if not mesh.is_watertight: print('Warning: mesh %s is not watertight!') loc, scale = get_loc_scale(mesh, args) mesh.apply_translation(-loc) mesh.apply_scale(1 / scale) boxsize = 1 + args.points_padding points_uniform = np.random.rand(n_points_uniform, 3) points_uniform = boxsize * (points_uniform - 0.5) points_surface = mesh.sample(n_points_surface) points_surface += args.points_sigma * \ np.random.randn(n_points_surface, 3) points = np.concatenate([points_uniform, points_surface], axis=0) occupancies = check_mesh_contains(mesh, points) print('Writing points: %s' % out_file) # Compress if args.float16: dtype = np.float16 else: dtype = np.float32 points = points.astype(dtype) loc = loc.astype(dtype) scale = scale.astype(dtype) if args.packbits: occupancies = np.packbits(occupancies) np.savez(out_file, points=points, occupancies=occupancies, loc=loc, scale=scale)
def voxelize_interior(mesh, resolution): shape = (resolution, ) * 3 bb_min = (0.5, ) * 3 bb_max = (resolution - 0.5, ) * 3 # Create points. Add noise to break symmetry points = make_3d_grid(bb_min, bb_max, shape=shape).numpy() points = points + 0.1 * (np.random.rand(*points.shape) - 0.5) points = (points / resolution - 0.5) occ = check_mesh_contains(mesh, points) occ = occ.reshape(shape) return occ
def export_points(mesh, modelname, loc, scale, args): if not mesh.is_watertight: print('Warning: mesh %s is not watertight!' 'Cannot sample points.' % modelname) return filename = os.path.join(args.points_folder, modelname + '.npz') if not args.overwrite and os.path.exists(filename): print('Points already exist: %s' % filename) return n_points_uniform = int(args.points_size * args.points_uniform_ratio) n_points_surface = args.points_size - n_points_uniform boxsize = 1 + args.points_padding points_uniform = np.random.rand(n_points_uniform, 3) points_uniform = boxsize * (points_uniform - 0.5) points_surface = mesh.sample(n_points_surface) points_surface += args.points_sigma * np.random.randn(n_points_surface, 3) points = np.concatenate([points_uniform, points_surface], axis=0) occupancies = check_mesh_contains(mesh, points) # Compress if args.float16: dtype = np.float16 else: dtype = np.float32 points = points.astype(dtype) if args.packbits: occupancies = np.packbits(occupancies) print('Writing points: %s' % filename) np.savez(filename, points=points, occupancies=occupancies, loc=loc, scale=scale)
def __getitem__(self, idx): ''' Returns an item of the dataset. Args: idx (int): ID of data point ''' data_path = self.data[idx]['data_path'] subject = self.data[idx]['subject'] gender = self.data[idx]['gender'] data = {} aug_rot = self.augm_params().astype(np.float32) points_dict = np.load(data_path) # 3D models and points loc = points_dict['loc'].astype(np.float32) trans = points_dict['trans'].astype(np.float32) root_loc = points_dict['Jtr'][0].astype(np.float32) scale = points_dict['scale'].astype(np.float32) # Also get GT SMPL poses pose_body = points_dict['pose_body'] pose_hand = points_dict['pose_hand'] pose = np.concatenate([pose_body, pose_hand], axis=-1) pose = R.from_rotvec(pose.reshape([-1, 3])) body_mesh_a_pose = points_dict['a_pose_mesh_points'] # Break symmetry if given in float16: if body_mesh_a_pose.dtype == np.float16: body_mesh_a_pose = body_mesh_a_pose.astype(np.float32) body_mesh_a_pose += 1e-4 * np.random.randn(*body_mesh_a_pose.shape) else: body_mesh_a_pose = body_mesh_a_pose.astype(np.float32) n_smpl_points = body_mesh_a_pose.shape[0] bone_transforms = points_dict['bone_transforms'].astype(np.float32) # Apply rotation augmentation to bone transformations bone_transforms_aug = np.matmul(np.expand_dims(aug_rot, axis=0), bone_transforms) bone_transforms_aug[:, :3, -1] += root_loc - trans - np.dot(aug_rot[:3, :3], root_loc - trans) bone_transforms = bone_transforms_aug # Get augmented posed-mesh skinning_weights = self.skinning_weights[gender] if self.use_abs_bone_transforms: J_regressor = self.J_regressors[gender] T = np.dot(skinning_weights, bone_transforms.reshape([-1, 16])).reshape([-1, 4, 4]) homogen_coord = np.ones([n_smpl_points, 1], dtype=np.float32) a_pose_homo = np.concatenate([body_mesh_a_pose - trans, homogen_coord], axis=-1).reshape([n_smpl_points, 4, 1]) body_mesh = np.matmul(T, a_pose_homo)[:, :3, 0].astype(np.float32) + trans # Get extents of model. bb_min = np.min(body_mesh, axis=0) bb_max = np.max(body_mesh, axis=0) # total_size = np.sqrt(np.square(bb_max - bb_min).sum()) total_size = (bb_max - bb_min).max() # Scales all dimensions equally. scale = max(1.6, total_size) # 1.6 is the magic number from IPNet loc = np.array( [(bb_min[0] + bb_max[0]) / 2, (bb_min[1] + bb_max[1]) / 2, (bb_min[2] + bb_max[2]) / 2], dtype=np.float32 ) posed_trimesh = trimesh.Trimesh(vertices=body_mesh, faces=self.faces) # a_pose_trimesh = trimesh.Trimesh(vertices=(body_mesh_a_pose - trans) * 1.0 / scale * 1.5, faces=self.faces) n_points_uniform = int(self.points_size * self.points_uniform_ratio) n_points_surface = self.points_size - n_points_uniform boxsize = 1 + self.points_padding points_uniform = np.random.rand(n_points_uniform, 3) points_uniform = boxsize * (points_uniform - 0.5) # Scale points in (padded) unit box back to the original space points_uniform *= scale points_uniform += loc # Sample points around posed-mesh surface n_points_surface_cloth = n_points_surface // 2 if self.double_layer else n_points_surface points_surface = posed_trimesh.sample(n_points_surface_cloth + self.input_pointcloud_n) if self.input_type == 'pointcloud': input_pointcloud = points_surface[n_points_surface_cloth:] noise = self.input_pointcloud_noise * np.random.randn(*input_pointcloud.shape) input_pointcloud = (input_pointcloud + noise).astype(np.float32) points_surface = points_surface[:n_points_surface_cloth] points_surface += np.random.normal(scale=self.points_sigma, size=points_surface.shape) if self.double_layer: n_points_surface_minimal = n_points_surface // 2 posedir = self.posedirs[gender] minimal_shape_path = os.path.join(self.cape_path, 'cape_release', 'minimal_body_shape', subject, subject + '_minimal.npy') minimal_shape = np.load(minimal_shape_path) pose_mat = pose.as_matrix() ident = np.eye(3) pose_feature = (pose_mat - ident).reshape([207, 1]) pose_offsets = np.dot(posedir.reshape([-1, 207]), pose_feature).reshape([6890, 3]) minimal_shape += pose_offsets if self.use_abs_bone_transforms: Jtr_cano = np.dot(J_regressor, minimal_shape) Jtr_cano = Jtr_cano[IPNET2SMPL_IDX, :] a_pose_homo = np.concatenate([minimal_shape, homogen_coord], axis=-1).reshape([n_smpl_points, 4, 1]) minimal_body_mesh = np.matmul(T, a_pose_homo)[:, :3, 0].astype(np.float32) + trans minimal_posed_trimesh = trimesh.Trimesh(vertices=minimal_body_mesh, faces=self.faces) # Sample points around minimally clothed posed-mesh surface points_surface_minimal = minimal_posed_trimesh.sample(n_points_surface_minimal) points_surface_minimal += np.random.normal(scale=self.points_sigma, size=points_surface_minimal.shape) points_surface = np.vstack([points_surface, points_surface_minimal]) # Check occupancy values for sampled ponits query_points = np.vstack([points_uniform, points_surface]).astype(np.float32) if self.double_layer: # Double-layer occupancies, as was done in IPNet # 0: outside, 1: between body and cloth, 2: inside body mesh occupancies_cloth = check_mesh_contains(posed_trimesh, query_points) occupancies_minimal = check_mesh_contains(minimal_posed_trimesh, query_points) occupancies = occupancies_cloth.astype(np.int64) occupancies[occupancies_minimal] = 2 else: occupancies = check_mesh_contains(posed_trimesh, query_points).astype(np.float32) # Skinning inds by querying nearest SMPL vertex on the clohted mesh kdtree = KDTree(body_mesh if self.query_on_clothed else minimal_body_mesh) _, p_idx = kdtree.query(query_points) pts_W = skinning_weights[p_idx, :] skinning_inds_ipnet = self.part_labels[p_idx] # skinning inds (14 parts) skinning_inds_smpl = pts_W.argmax(1) # full skinning inds (24 parts) if self.num_joints == 14: skinning_inds = skinning_inds_ipnet else: skinning_inds = skinning_inds_smpl # Invert LBS to get query points in A-pose space T = np.dot(pts_W, bone_transforms.reshape([-1, 16])).reshape([-1, 4, 4]) T = np.linalg.inv(T) homogen_coord = np.ones([self.points_size, 1], dtype=np.float32) posed_homo = np.concatenate([query_points - trans, homogen_coord], axis=-1).reshape([self.points_size, 4, 1]) query_points_a_pose = np.matmul(T, posed_homo)[:, :3, 0].astype(np.float32) + trans if self.use_abs_bone_transforms: assert (not self.use_v_template and self.num_joints == 24) query_points_a_pose -= Jtr_cano[SMPL2IPNET_IDX[skinning_inds], :] if self.use_v_template: v_template = self.v_templates[gender] pose_shape_offsets = v_template - minimal_shape query_points_template = query_points_a_pose + pose_shape_offsets[p_idx, :] sc_factor = 1.0 / scale * 1.5 if self.normalized_scale else 1.0 # 1.5 is the magic number from IPNet offset = loc bone_transforms_inv = bone_transforms.copy() bone_transforms_inv[:, :3, -1] += trans - loc bone_transforms_inv = np.linalg.inv(bone_transforms_inv) bone_transforms_inv[:, :3, -1] *= sc_factor data = { None: (query_points - offset) * sc_factor, 'occ': occupancies, 'trans': trans, 'root_loc': root_loc, 'pts_a_pose': (query_points_a_pose - (trans if self.use_global_trans else offset)) * sc_factor, 'skinning_inds': skinning_inds, 'skinning_inds_ipnet': skinning_inds_ipnet, 'skinning_inds_smpl': skinning_inds_smpl, 'loc': loc, 'scale': scale, 'bone_transforms': bone_transforms, 'bone_transforms_inv': bone_transforms_inv, } if self.use_v_template: data.update({'pts_template': (query_points_template - (trans if self.use_global_trans else offset)) * sc_factor}) if self.mode in ['test']: data.update({'smpl_vertices': body_mesh, 'smpl_a_pose_vertices': body_mesh_a_pose}) if self.double_layer: data.update({'minimal_smpl_vertices': minimal_body_mesh}) data_out = {} field_name = 'points' if self.mode in ['train', 'test'] else 'points_iou' for k, v in data.items(): if k is None: data_out[field_name] = v else: data_out['%s.%s' % (field_name, k)] = v if self.input_type == 'pointcloud': data_out.update( {'inputs': (input_pointcloud - offset) * sc_factor, 'idx': idx, } ) elif self.input_type == 'voxel': voxels = np.unpackbits(points_dict['voxels_occ']).astype(np.float32) voxels = np.reshape(voxels, [self.voxel_res] * 3) data_out.update( {'inputs': voxels, 'idx': idx, } ) else: raise ValueError('Unsupported input type: {}'.format(self.input_type)) return data_out
def export_points(mesh, modelname, loc, scale, args): if not mesh.is_watertight: print('Warning: mesh %s is not watertight!' 'Cannot sample points.' % modelname) return filename = os.path.join(args.points_folder, modelname + '.npz') if not args.overwrite and os.path.exists(filename): print('Points already exist: %s' % filename) return ratio_world = mesh.volume / ((1 + args.points_padding)**3) ratio_bbox = mesh.volume / mesh.bounding_box.volume print('Volume ratio, world:%.4f; bbox:%.4f' % (ratio_world, ratio_bbox)) N = args.points_subsample bandwidth = args.bandwidth N_bandwidth = int(N * 0.7) # for surface bandwidth points: n_points_surface = int(N_bandwidth * 5) points_surface, points_index = mesh.sample(n_points_surface, return_index=True) points_normal = mesh.face_normals[points_index] points_offset = bandwidth * (np.random.rand(n_points_surface, 1) * 2. - 1.) points_surface += points_offset * points_normal # for other points #ratio_min = min(ratio_world, 1. - ratio_world) #N_other = int(N * 0.15) n_points_uniform = args.points_size boxsize = 1 + args.points_padding points_uniform = np.random.rand(n_points_uniform, 3) points_uniform = boxsize * (points_uniform - 0.5) points = np.concatenate([points_uniform, points_surface], axis=0) occupancies = check_mesh_contains(mesh, points) # calculate distance mesh_sampled_points = mesh.sample(count=100000) kdtree = KDTree(mesh_sampled_points) dist, _ = kdtree.query(points) sdf = (occupancies * (-2.) + 1.).astype(np.float32) * dist # Compress dtype = np.float32 points = points.astype(dtype) sdf = sdf.astype(dtype) #print("SDF: min", sdf.min(), "max:", sdf.max(), "avg:", sdf.mean()) #for i in range(10): # percent = i * 10 # print("SDF: below %d" % percent, np.percentile(sdf, percent)) if args.points_subsample != 0: # subsample points, sdf, occupancies = sample_points_sdf(args.points_subsample, points, sdf, occupancies, bandwidth=args.bandwidth) points = np.squeeze(points) sdf = np.squeeze(sdf) occupancies = np.squeeze(occupancies) # save print('Writing points: %s' % filename) np.savez(filename, points=points, sdf=sdf, occupancies=occupancies, loc=loc, scale=scale)