def run_corresponding_points_alignment():
points_alignment.corresponding_points_alignment(
X,
X_t,
weights,
allow_reflection=allow_reflection,
estimate_scale=estimate_scale,
)
torch.cuda.synchronize()
def _compute_norm_sign_scaling_factor(c_cam, alphas, x_world, y, weight, eps=1e-9):
""" Given a solution, adjusts the scale and flip
Args:
c_cam: control points in camera coordinates
alphas: barycentric coordinates of the points
x_world: Batch of 3-dimensional points of shape `(minibatch, num_points, 3)`.
y: Batch of 2-dimensional points of shape `(minibatch, num_points, 2)`.
weights: Batch of non-negative weights of
shape `(minibatch, num_point)`. `None` means equal weights.
eps: epsilon to threshold negative `z` values
"""
# position of reference points in camera coordinates
x_cam = torch.matmul(alphas, c_cam)
x_cam = x_cam * (1.0 - 2.0 * (oputil.wmean(x_cam[..., 2:], weight) < 0).float())
if torch.any(x_cam[..., 2:] < -eps):
neg_rate = oputil.wmean((x_cam[..., 2:] < 0).float(), weight, dim=(0, 1)).item()
warnings.warn("\nEPnP: %2.2f%% points have z<0." % (neg_rate * 100.0))
R, T, s = points_alignment.corresponding_points_alignment(
x_world, x_cam, weight, estimate_scale=True
)
s = s.clamp(eps)
x_cam = x_cam / s[:, None, None]
T = T / s[:, None]
x_w_rotated = torch.matmul(x_world, R) + T[:, None, :]
err_2d = _reproj_error(x_w_rotated, y, weight)
err_3d = _algebraic_error(x_w_rotated, x_cam, weight)
return EpnpSolution(x_cam, R, T, err_2d, err_3d)
def align_and_get_mse(weights_):
R_n, T_n, s_n = points_alignment.corresponding_points_alignment(
X_noisy,
X_t,
weights_,
allow_reflection=allow_reflection,
estimate_scale=estimate_scale,
)
X_t_est = _apply_pcl_transformation(X_noisy, R_n, T_n, s=s_n)
return (((X_t_est - X_t) * weights[..., None])**
2).sum(dim=(1, 2)) / weights.sum(dim=-1)
def compute_eye_param(self, vertices, eye_lm_idx, face_model):
nsh_vert_lm = vertices[None, eye_lm_idx]
nsh_std_lm = self.to_tensor(
self.transfers[face_model].tgt_std_vert)[None, eye_lm_idx]
R, T, s = corresponding_points_alignment(nsh_vert_lm,
nsh_std_lm,
estimate_scale=True)
R = R.cpu().numpy()[0]
T = T.cpu().numpy()[0]
s = s.cpu().numpy()
angle = Rotation.from_matrix(R).as_euler('xyz')
eye_param = np.concatenate([angle, T, s])
return eye_param
def _test_single_corresponding_points_alignment(
self,
batch_size=10,
n_points=100,
dim=3,
use_pointclouds=False,
estimate_scale=False,
reflect=False,
allow_reflection=False,
random_weights=False,
):
"""
Executes a single test for `corresponding_points_alignment` for a
specific setting of the inputs / outputs.
"""
device = torch.device("cuda:0")
# initialize the a ground truth point cloud
X = TestCorrespondingPointsAlignment.init_point_cloud(
batch_size=batch_size,
n_points=n_points,
dim=dim,
device=device,
use_pointclouds=use_pointclouds,
random_pcl_size=True,
)
# generate the true transformation
R, T, s = TestCorrespondingPointsAlignment.generate_pcl_transformation(
batch_size=batch_size,
scale=estimate_scale,
reflect=reflect,
dim=dim,
device=device,
)
if reflect:
# generate random reflection M and apply to the rotations
M = TestCorrespondingPointsAlignment.generate_random_reflection(
batch_size=batch_size, dim=dim, device=device
)
R = torch.bmm(M, R)
weights = None
if random_weights:
template = X.points_padded() if use_pointclouds else X
weights = torch.rand_like(template[:, :, 0])
weights = weights / weights.sum(dim=1, keepdim=True)
# zero out some weights as zero weights are a common use case
# this guarantees there are no zero weight
weights *= (weights * template.size()[1] > 0.3).to(weights)
if use_pointclouds: # convert to List[Tensor]
weights = [
w[:npts] for w, npts in zip(weights, X.num_points_per_cloud())
]
# apply the generated transformation to the generated
# point cloud X
X_t = _apply_pcl_transformation(X, R, T, s=s)
# run the CorrespondingPointsAlignment algorithm
R_est, T_est, s_est = points_alignment.corresponding_points_alignment(
X,
X_t,
weights,
allow_reflection=allow_reflection,
estimate_scale=estimate_scale,
)
assert_error_message = (
f"Corresponding_points_alignment assertion failure for "
f"n_points={n_points}, "
f"dim={dim}, "
f"use_pointclouds={use_pointclouds}, "
f"estimate_scale={estimate_scale}, "
f"reflect={reflect}, "
f"allow_reflection={allow_reflection},"
f"random_weights={random_weights}."
)
# if we test the weighted case, check that weights help with noise
if random_weights and not use_pointclouds and n_points >= (dim + 10):
# add noise to 20% points with smallest weight
X_noisy = X_t.clone()
_, mink_idx = torch.topk(-weights, int(n_points * 0.2), dim=1)
mink_idx = mink_idx[:, :, None].expand(-1, -1, X_t.shape[-1])
X_noisy.scatter_add_(
1, mink_idx, 0.3 * torch.randn_like(mink_idx, dtype=X_t.dtype)
)
def align_and_get_mse(weights_):
R_n, T_n, s_n = points_alignment.corresponding_points_alignment(
X_noisy,
X_t,
weights_,
allow_reflection=allow_reflection,
estimate_scale=estimate_scale,
)
X_t_est = _apply_pcl_transformation(X_noisy, R_n, T_n, s=s_n)
return (((X_t_est - X_t) * weights[..., None]) ** 2).sum(
dim=(1, 2)
) / weights.sum(dim=-1)
# check that using weights leads to lower weighted_MSE(X_noisy, X_t)
self.assertTrue(
torch.all(align_and_get_mse(weights) <= align_and_get_mse(None))
)
if reflect and not allow_reflection:
# check that all rotations have det=1
self._assert_all_close(
torch.det(R_est), R_est.new_ones(batch_size), assert_error_message
)
else:
# mask out inputs with too few non-degenerate points for assertions
w = (
torch.ones_like(R_est[:, 0, 0])
if weights is None or n_points >= dim + 10
else (weights > 0.0).all(dim=1).to(R_est)
)
# check that the estimated tranformation is the same
# as the ground truth
if n_points >= (dim + 1):
# the checks on transforms apply only when
# the problem setup is unambiguous
msg = assert_error_message
self._assert_all_close(R_est, R, msg, w[:, None, None], atol=1e-5)
self._assert_all_close(T_est, T, msg, w[:, None])
self._assert_all_close(s_est, s, msg, w)
# check that the orthonormal part of the
# transformation has a correct determinant (+1/-1)
desired_det = R_est.new_ones(batch_size)
if reflect:
desired_det *= -1.0
self._assert_all_close(torch.det(R_est), desired_det, msg, w)
# check that the transformed point cloud
# X matches X_t
X_t_est = _apply_pcl_transformation(X, R_est, T_est, s=s_est)
self._assert_all_close(
X_t, X_t_est, assert_error_message, w[:, None, None], atol=1e-5
)
def predict(self,
image,
out_dir,
idx=None,
deploy=False,
face_model='230'):
'''deploy for nsh'''
if not deploy and idx is None:
idx = '{:>05d}'.format(idx)
images, uvmaps, params, nsh_face_vert, nsh_neu_vert = self.preprocess(
image, face_model)
fnames = []
gen_uvmaps = self.inpaint_model.forward(images[:, :3],
uvmaps,
nsh_face_vert[None],
params,
fix_uv=True,
deploy=deploy,
face_model=face_model)
nsh_uv = F.interpolate(gen_uvmaps.detach(),
size=1024,
mode='bilinear',
align_corners=False)[0]
fnames.append(os.path.join(out_dir, '{}_uv.png'.format(idx)))
self.imsave(fnames[-1], nsh_uv, False, True)
lm_idx = self.to_tensor(self.transfers[face_model].lm_icp_idx,
torch.int64)
nsh_vert_lm = nsh_neu_vert[None, lm_idx]
nsh_std_lm = self.to_tensor(
self.transfers[face_model].tgt_std_vert)[None, lm_idx]
R, T, s = corresponding_points_alignment(nsh_vert_lm,
nsh_std_lm,
estimate_scale=True)
s = s * 0.97
nsh_neu_vert_trans = (
s[:, None, None] * torch.bmm(nsh_neu_vert[None], R) +
T[:, None, :])[0]
nsh_neu_vert = nsh_neu_vert_trans.cpu().numpy()
nsh_neu_vert = self.transfers[face_model].normalize(nsh_neu_vert)
fnames.append(os.path.join(out_dir, '{}_neu.obj'.format(idx)))
meshio.write_obj(
fnames[-1],
nsh_neu_vert[self.uv_creators[face_model].nsh_face_start_idx:],
self.nsh_face_meshes[face_model].triangles,
texcoords=self.nsh_face_meshes[face_model].texcoords,
mtllib=True,
uv_name='{}_uv'.format(idx))
fnames.append(os.path.join(out_dir, '{}_neu.mtl'.format(idx)))
try:
self.imsave(os.path.join(out_dir, '{}_input.jpg'.format(idx)),
images[0, :3], True)
except:
pass
