def export_points(mesh, modelname, loc, scale, args):
if not mesh.is_watertight:
print('Warning: mesh %s is not watertight!'
'Cannot sample points.' % modelname)
return
filename = os.path.join(args.points_folder, modelname + '.npz')
if not args.overwrite and os.path.exists(filename):
print('Points already exist: %s' % filename)
return
n_points_uniform = int(args.points_size * args.points_uniform_ratio)
n_points_noise_surface = args.points_size - n_points_uniform
# uniform
boxsize = 1 + args.points_padding
points_uniform = np.random.rand(n_points_uniform, 3)
points_uniform = boxsize * (points_uniform - 0.5)
# surface + noise
points_noise_surface, points_noise_index = mesh.sample(
n_points_noise_surface, return_index=True)
points_noise_normal = mesh.face_normals[points_noise_index]
points_noise_ratio = np.random.rand(n_points_noise_surface) * 2. - 1.
points_noise_surface += args.points_sigma * points_noise_ratio.reshape(
n_points_noise_surface, 1) * points_noise_normal
points = np.concatenate([points_uniform, points_noise_surface], axis=0)
points_uniform_occupancies = check_mesh_contains(
mesh, points_uniform).astype(np.float32)
points_noise_surface_occupancies = check_mesh_contains(
mesh, points_noise_surface).astype(np.float32)
# adjustment
points_noise_ratio = np.abs(points_noise_ratio)
points_noise_ratio[points_noise_ratio > 1] = 1
points_noise_surface_occupancies = (points_noise_surface_occupancies -
0.5) * points_noise_ratio + 0.5
occupancies = np.concatenate(
[points_uniform_occupancies, points_noise_surface_occupancies], axis=0)
# Compress
if args.float16:
dtype = np.float16
else:
dtype = np.float32
points = points.astype(dtype)
occupancies = occupancies.astype(dtype)
print('Writing points: %s' % filename)
np.savez(filename,
points=points,
occupancies=occupancies,
loc=loc,
scale=scale)
def eval_mesh2(self, mesh, pointcloud_tgt, normals_tgt,
points_iou, occ_tgt):
''' Evaluates a mesh.
Args:
mesh (trimesh): mesh which should be evaluated
pointcloud_tgt (numpy array): target point cloud
normals_tgt (numpy array): target normals
points_iou (numpy_array): points tensor for IoU evaluation
occ_tgt (numpy_array): GT occupancy values for IoU points
'''
if len(mesh.vertices) != 0 and len(mesh.faces) != 0:
pointcloud, idx = mesh.sample(self.n_points, return_index=True)
pointcloud = pointcloud.astype(np.float32)
normals = mesh.face_normals[idx]
else:
pointcloud = np.empty((0, 3))
normals = np.empty((0, 3))
out_dict = self.eval_pointcloud(
pointcloud, pointcloud_tgt, normals, normals_tgt)
if len(mesh.vertices) != 0 and len(mesh.faces) != 0:
occ = check_mesh_contains(mesh, points_iou)
out_dict['iou'] = compute_iou(occ, occ_tgt)
else:
out_dict['iou'] = 0.
return out_dict
def export_points(mesh, modelname, loc, scale, args):
if not mesh.is_watertight:
print('Warning: mesh %s is not watertight!'
'Cannot sample points.' % modelname)
return
filename = os.path.join(args.points_folder, modelname + '.npz')
if not args.overwrite and os.path.exists(filename):
print('Points already exist: %s' % filename)
return
# uniform grid
total_l = 1. + args.points_padding
res = args.points_resolution
points_uniform = make_3d_grid((-total_l/2.0 + total_l/(res*2),)*3, (total_l/2.0 - total_l/(res*2),)*3, (res,)*3).numpy()
points_uniform_occupancies = check_mesh_contains(mesh, points_uniform).astype(np.float32)
points = points_uniform
occupancies = points_uniform_occupancies
# Compress
if args.float16:
dtype = np.float16
else:
dtype = np.float32
points = points.astype(dtype)
occupancies = occupancies.astype(dtype)
print('Writing points: %s' % filename)
np.savez(filename, points=points, occupancies=occupancies,
loc=loc, scale=scale)
def export_points(modelname, model_files, args):
out_folder = os.path.join(args.points_folder, modelname)
if os.path.exists(out_folder):
if not args.overwrite:
print('Points already exist: %s' % out_folder)
return
else:
shutil.rmtree(out_folder)
# Create out_folder
os.makedirs(out_folder)
n_points_uniform = int(args.points_size * args.points_uniform_ratio)
n_points_surface = args.points_size - n_points_uniform
for it, model_file in enumerate(model_files):
out_file = os.path.join(out_folder, '%08d.npz' % it)
mesh = trimesh.load(model_file, process=False)
if not mesh.is_watertight:
print('Warning: mesh %s is not watertight!')
loc, scale = get_loc_scale(mesh, args)
mesh.apply_translation(-loc)
mesh.apply_scale(1 / scale)
boxsize = 1 + args.points_padding
points_uniform = np.random.rand(n_points_uniform, 3)
points_uniform = boxsize * (points_uniform - 0.5)
points_surface = mesh.sample(n_points_surface)
points_surface += args.points_sigma * \
np.random.randn(n_points_surface, 3)
points = np.concatenate([points_uniform, points_surface], axis=0)
occupancies = check_mesh_contains(mesh, points)
print('Writing points: %s' % out_file)
# Compress
if args.float16:
dtype = np.float16
else:
dtype = np.float32
points = points.astype(dtype)
loc = loc.astype(dtype)
scale = scale.astype(dtype)
if args.packbits:
occupancies = np.packbits(occupancies)
np.savez(out_file,
points=points,
occupancies=occupancies,
loc=loc,
scale=scale)
def voxelize_interior(mesh, resolution):
shape = (resolution, ) * 3
bb_min = (0.5, ) * 3
bb_max = (resolution - 0.5, ) * 3
# Create points. Add noise to break symmetry
points = make_3d_grid(bb_min, bb_max, shape=shape).numpy()
points = points + 0.1 * (np.random.rand(*points.shape) - 0.5)
points = (points / resolution - 0.5)
occ = check_mesh_contains(mesh, points)
occ = occ.reshape(shape)
return occ
def export_points(mesh, modelname, loc, scale, args):
if not mesh.is_watertight:
print('Warning: mesh %s is not watertight!'
'Cannot sample points.' % modelname)
return
filename = os.path.join(args.points_folder, modelname + '.npz')
if not args.overwrite and os.path.exists(filename):
print('Points already exist: %s' % filename)
return
n_points_uniform = int(args.points_size * args.points_uniform_ratio)
n_points_surface = args.points_size - n_points_uniform
boxsize = 1 + args.points_padding
points_uniform = np.random.rand(n_points_uniform, 3)
points_uniform = boxsize * (points_uniform - 0.5)
points_surface = mesh.sample(n_points_surface)
points_surface += args.points_sigma * np.random.randn(n_points_surface, 3)
points = np.concatenate([points_uniform, points_surface], axis=0)
occupancies = check_mesh_contains(mesh, points)
# Compress
if args.float16:
dtype = np.float16
else:
dtype = np.float32
points = points.astype(dtype)
if args.packbits:
occupancies = np.packbits(occupancies)
print('Writing points: %s' % filename)
np.savez(filename,
points=points,
occupancies=occupancies,
loc=loc,
scale=scale)
def __getitem__(self, idx):
''' Returns an item of the dataset.
Args:
idx (int): ID of data point
'''
data_path = self.data[idx]['data_path']
subject = self.data[idx]['subject']
gender = self.data[idx]['gender']
data = {}
aug_rot = self.augm_params().astype(np.float32)
points_dict = np.load(data_path)
# 3D models and points
loc = points_dict['loc'].astype(np.float32)
trans = points_dict['trans'].astype(np.float32)
root_loc = points_dict['Jtr'][0].astype(np.float32)
scale = points_dict['scale'].astype(np.float32)
# Also get GT SMPL poses
pose_body = points_dict['pose_body']
pose_hand = points_dict['pose_hand']
pose = np.concatenate([pose_body, pose_hand], axis=-1)
pose = R.from_rotvec(pose.reshape([-1, 3]))
body_mesh_a_pose = points_dict['a_pose_mesh_points']
# Break symmetry if given in float16:
if body_mesh_a_pose.dtype == np.float16:
body_mesh_a_pose = body_mesh_a_pose.astype(np.float32)
body_mesh_a_pose += 1e-4 * np.random.randn(*body_mesh_a_pose.shape)
else:
body_mesh_a_pose = body_mesh_a_pose.astype(np.float32)
n_smpl_points = body_mesh_a_pose.shape[0]
bone_transforms = points_dict['bone_transforms'].astype(np.float32)
# Apply rotation augmentation to bone transformations
bone_transforms_aug = np.matmul(np.expand_dims(aug_rot, axis=0), bone_transforms)
bone_transforms_aug[:, :3, -1] += root_loc - trans - np.dot(aug_rot[:3, :3], root_loc - trans)
bone_transforms = bone_transforms_aug
# Get augmented posed-mesh
skinning_weights = self.skinning_weights[gender]
if self.use_abs_bone_transforms:
J_regressor = self.J_regressors[gender]
T = np.dot(skinning_weights, bone_transforms.reshape([-1, 16])).reshape([-1, 4, 4])
homogen_coord = np.ones([n_smpl_points, 1], dtype=np.float32)
a_pose_homo = np.concatenate([body_mesh_a_pose - trans, homogen_coord], axis=-1).reshape([n_smpl_points, 4, 1])
body_mesh = np.matmul(T, a_pose_homo)[:, :3, 0].astype(np.float32) + trans
# Get extents of model.
bb_min = np.min(body_mesh, axis=0)
bb_max = np.max(body_mesh, axis=0)
# total_size = np.sqrt(np.square(bb_max - bb_min).sum())
total_size = (bb_max - bb_min).max()
# Scales all dimensions equally.
scale = max(1.6, total_size) # 1.6 is the magic number from IPNet
loc = np.array(
[(bb_min[0] + bb_max[0]) / 2,
(bb_min[1] + bb_max[1]) / 2,
(bb_min[2] + bb_max[2]) / 2],
dtype=np.float32
)
posed_trimesh = trimesh.Trimesh(vertices=body_mesh, faces=self.faces)
# a_pose_trimesh = trimesh.Trimesh(vertices=(body_mesh_a_pose - trans) * 1.0 / scale * 1.5, faces=self.faces)
n_points_uniform = int(self.points_size * self.points_uniform_ratio)
n_points_surface = self.points_size - n_points_uniform
boxsize = 1 + self.points_padding
points_uniform = np.random.rand(n_points_uniform, 3)
points_uniform = boxsize * (points_uniform - 0.5)
# Scale points in (padded) unit box back to the original space
points_uniform *= scale
points_uniform += loc
# Sample points around posed-mesh surface
n_points_surface_cloth = n_points_surface // 2 if self.double_layer else n_points_surface
points_surface = posed_trimesh.sample(n_points_surface_cloth + self.input_pointcloud_n)
if self.input_type == 'pointcloud':
input_pointcloud = points_surface[n_points_surface_cloth:]
noise = self.input_pointcloud_noise * np.random.randn(*input_pointcloud.shape)
input_pointcloud = (input_pointcloud + noise).astype(np.float32)
points_surface = points_surface[:n_points_surface_cloth]
points_surface += np.random.normal(scale=self.points_sigma, size=points_surface.shape)
if self.double_layer:
n_points_surface_minimal = n_points_surface // 2
posedir = self.posedirs[gender]
minimal_shape_path = os.path.join(self.cape_path, 'cape_release', 'minimal_body_shape', subject, subject + '_minimal.npy')
minimal_shape = np.load(minimal_shape_path)
pose_mat = pose.as_matrix()
ident = np.eye(3)
pose_feature = (pose_mat - ident).reshape([207, 1])
pose_offsets = np.dot(posedir.reshape([-1, 207]), pose_feature).reshape([6890, 3])
minimal_shape += pose_offsets
if self.use_abs_bone_transforms:
Jtr_cano = np.dot(J_regressor, minimal_shape)
Jtr_cano = Jtr_cano[IPNET2SMPL_IDX, :]
a_pose_homo = np.concatenate([minimal_shape, homogen_coord], axis=-1).reshape([n_smpl_points, 4, 1])
minimal_body_mesh = np.matmul(T, a_pose_homo)[:, :3, 0].astype(np.float32) + trans
minimal_posed_trimesh = trimesh.Trimesh(vertices=minimal_body_mesh, faces=self.faces)
# Sample points around minimally clothed posed-mesh surface
points_surface_minimal = minimal_posed_trimesh.sample(n_points_surface_minimal)
points_surface_minimal += np.random.normal(scale=self.points_sigma, size=points_surface_minimal.shape)
points_surface = np.vstack([points_surface, points_surface_minimal])
# Check occupancy values for sampled ponits
query_points = np.vstack([points_uniform, points_surface]).astype(np.float32)
if self.double_layer:
# Double-layer occupancies, as was done in IPNet
# 0: outside, 1: between body and cloth, 2: inside body mesh
occupancies_cloth = check_mesh_contains(posed_trimesh, query_points)
occupancies_minimal = check_mesh_contains(minimal_posed_trimesh, query_points)
occupancies = occupancies_cloth.astype(np.int64)
occupancies[occupancies_minimal] = 2
else:
occupancies = check_mesh_contains(posed_trimesh, query_points).astype(np.float32)
# Skinning inds by querying nearest SMPL vertex on the clohted mesh
kdtree = KDTree(body_mesh if self.query_on_clothed else minimal_body_mesh)
_, p_idx = kdtree.query(query_points)
pts_W = skinning_weights[p_idx, :]
skinning_inds_ipnet = self.part_labels[p_idx] # skinning inds (14 parts)
skinning_inds_smpl = pts_W.argmax(1) # full skinning inds (24 parts)
if self.num_joints == 14:
skinning_inds = skinning_inds_ipnet
else:
skinning_inds = skinning_inds_smpl
# Invert LBS to get query points in A-pose space
T = np.dot(pts_W, bone_transforms.reshape([-1, 16])).reshape([-1, 4, 4])
T = np.linalg.inv(T)
homogen_coord = np.ones([self.points_size, 1], dtype=np.float32)
posed_homo = np.concatenate([query_points - trans, homogen_coord], axis=-1).reshape([self.points_size, 4, 1])
query_points_a_pose = np.matmul(T, posed_homo)[:, :3, 0].astype(np.float32) + trans
if self.use_abs_bone_transforms:
assert (not self.use_v_template and self.num_joints == 24)
query_points_a_pose -= Jtr_cano[SMPL2IPNET_IDX[skinning_inds], :]
if self.use_v_template:
v_template = self.v_templates[gender]
pose_shape_offsets = v_template - minimal_shape
query_points_template = query_points_a_pose + pose_shape_offsets[p_idx, :]
sc_factor = 1.0 / scale * 1.5 if self.normalized_scale else 1.0 # 1.5 is the magic number from IPNet
offset = loc
bone_transforms_inv = bone_transforms.copy()
bone_transforms_inv[:, :3, -1] += trans - loc
bone_transforms_inv = np.linalg.inv(bone_transforms_inv)
bone_transforms_inv[:, :3, -1] *= sc_factor
data = {
None: (query_points - offset) * sc_factor,
'occ': occupancies,
'trans': trans,
'root_loc': root_loc,
'pts_a_pose': (query_points_a_pose - (trans if self.use_global_trans else offset)) * sc_factor,
'skinning_inds': skinning_inds,
'skinning_inds_ipnet': skinning_inds_ipnet,
'skinning_inds_smpl': skinning_inds_smpl,
'loc': loc,
'scale': scale,
'bone_transforms': bone_transforms,
'bone_transforms_inv': bone_transforms_inv,
}
if self.use_v_template:
data.update({'pts_template': (query_points_template - (trans if self.use_global_trans else offset)) * sc_factor})
if self.mode in ['test']:
data.update({'smpl_vertices': body_mesh, 'smpl_a_pose_vertices': body_mesh_a_pose})
if self.double_layer:
data.update({'minimal_smpl_vertices': minimal_body_mesh})
data_out = {}
field_name = 'points' if self.mode in ['train', 'test'] else 'points_iou'
for k, v in data.items():
if k is None:
data_out[field_name] = v
else:
data_out['%s.%s' % (field_name, k)] = v
if self.input_type == 'pointcloud':
data_out.update(
{'inputs': (input_pointcloud - offset) * sc_factor,
'idx': idx,
}
)
elif self.input_type == 'voxel':
voxels = np.unpackbits(points_dict['voxels_occ']).astype(np.float32)
voxels = np.reshape(voxels, [self.voxel_res] * 3)
data_out.update(
{'inputs': voxels,
'idx': idx,
}
)
else:
raise ValueError('Unsupported input type: {}'.format(self.input_type))
return data_out
def export_points(mesh, modelname, loc, scale, args):
if not mesh.is_watertight:
print('Warning: mesh %s is not watertight!'
'Cannot sample points.' % modelname)
return
filename = os.path.join(args.points_folder, modelname + '.npz')
if not args.overwrite and os.path.exists(filename):
print('Points already exist: %s' % filename)
return
ratio_world = mesh.volume / ((1 + args.points_padding)**3)
ratio_bbox = mesh.volume / mesh.bounding_box.volume
print('Volume ratio, world:%.4f; bbox:%.4f' % (ratio_world, ratio_bbox))
N = args.points_subsample
bandwidth = args.bandwidth
N_bandwidth = int(N * 0.7)
# for surface bandwidth points:
n_points_surface = int(N_bandwidth * 5)
points_surface, points_index = mesh.sample(n_points_surface,
return_index=True)
points_normal = mesh.face_normals[points_index]
points_offset = bandwidth * (np.random.rand(n_points_surface, 1) * 2. - 1.)
points_surface += points_offset * points_normal
# for other points
#ratio_min = min(ratio_world, 1. - ratio_world)
#N_other = int(N * 0.15)
n_points_uniform = args.points_size
boxsize = 1 + args.points_padding
points_uniform = np.random.rand(n_points_uniform, 3)
points_uniform = boxsize * (points_uniform - 0.5)
points = np.concatenate([points_uniform, points_surface], axis=0)
occupancies = check_mesh_contains(mesh, points)
# calculate distance
mesh_sampled_points = mesh.sample(count=100000)
kdtree = KDTree(mesh_sampled_points)
dist, _ = kdtree.query(points)
sdf = (occupancies * (-2.) + 1.).astype(np.float32) * dist
# Compress
dtype = np.float32
points = points.astype(dtype)
sdf = sdf.astype(dtype)
#print("SDF: min", sdf.min(), "max:", sdf.max(), "avg:", sdf.mean())
#for i in range(10):
# percent = i * 10
# print("SDF: below %d" % percent, np.percentile(sdf, percent))
if args.points_subsample != 0:
# subsample
points, sdf, occupancies = sample_points_sdf(args.points_subsample,
points,
sdf,
occupancies,
bandwidth=args.bandwidth)
points = np.squeeze(points)
sdf = np.squeeze(sdf)
occupancies = np.squeeze(occupancies)
# save
print('Writing points: %s' % filename)
np.savez(filename,
points=points,
sdf=sdf,
occupancies=occupancies,
loc=loc,
scale=scale)