def get_vertices_and_3D_landmarks(self):
pose_params = torch.cat([self.global_rot, self.jaw_pose], dim=1)
shape_params = (self.fixed_shape
if self.fixed_shape is not None else self.shape_params)
betas = torch.cat([shape_params, self.expression_params], dim=1)
# pose_params_numpy[:, :3] : global rotation
# pose_params_numpy[:, 3:] : jaw rotation
full_pose = torch.cat([
pose_params[:, :3], self.neck_pose, pose_params[:, 3:],
self.eye_pose
],
dim=1)
template_vertices = self.v_template.unsqueeze(0).repeat(
self.batch_size, 1, 1)
vertices, _ = lbs(betas,
full_pose,
template_vertices,
self.shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
dtype=self.dtype)
lmk_faces_idx = self.lmk_faces_idx.unsqueeze(dim=0)
lmk_bary_coords = self.lmk_bary_coords.unsqueeze(dim=0)
if self.use_face_contour:
dyn_lmk_faces_idx, dyn_lmk_bary_coords = self._find_dynamic_lmk_idx_and_bcoords(
vertices,
full_pose,
self.dynamic_lmk_faces_idx,
self.dynamic_lmk_bary_coords,
self.neck_kin_chain,
dtype=self.dtype)
lmk_faces_idx = torch.cat([dyn_lmk_faces_idx, lmk_faces_idx], 1)
lmk_bary_coords = torch.cat([dyn_lmk_bary_coords, lmk_bary_coords],
1)
landmarks_3d = vertices2landmarks(vertices, self.faces_tensor,
lmk_faces_idx, lmk_bary_coords)
landmarks_3d += self.transl.unsqueeze(dim=1)
vertices += self.transl.unsqueeze(dim=1)
landmarks_3d.squeeze_()
vertices.squeeze_()
return vertices, landmarks_3d
def forward(self, shape_params=None, expression_params=None, pose_params=None, neck_pose=None, eye_pose=None, transl=None):
"""
Input:
shape_params: N X number of shape parameters
expression_params: N X number of expression parameters
pose_params: N X number of pose parameters
return:
vertices: N X V X 3
landmarks: N X number of landmarks X 3
"""
betas = torch.cat([shape_params,self.shape_betas, expression_params, self.expression_betas], dim=1)
neck_pose = (neck_pose if neck_pose is not None else self.neck_pose)
eye_pose = (eye_pose if eye_pose is not None else self.eye_pose)
transl = (transl if transl is not None else self.transl)
full_pose = torch.cat([pose_params[:,:3], neck_pose, pose_params[:,3:], eye_pose], dim=1)
template_vertices = self.v_template.unsqueeze(0).repeat(self.batch_size, 1, 1)
vertices, _ = lbs(betas, full_pose, template_vertices,
self.shapedirs, self.posedirs,
self.J_regressor, self.parents,
# self.lbs_weights, dtype=self.dtype)
self.lbs_weights)
lmk_faces_idx = self.lmk_faces_idx.unsqueeze(dim=0).repeat(
self.batch_size, 1)
lmk_bary_coords = self.lmk_bary_coords.unsqueeze(dim=0).repeat(
self.batch_size, 1, 1)
if self.use_face_contour:
dyn_lmk_faces_idx, dyn_lmk_bary_coords = self._find_dynamic_lmk_idx_and_bcoords(
vertices, full_pose, self.dynamic_lmk_faces_idx,
self.dynamic_lmk_bary_coords,
self.neck_kin_chain, dtype=self.dtype)
lmk_faces_idx = torch.cat([dyn_lmk_faces_idx, lmk_faces_idx], 1)
lmk_bary_coords = torch.cat(
[dyn_lmk_bary_coords, lmk_bary_coords], 1)
landmarks = vertices2landmarks(vertices, self.faces_tensor,
lmk_faces_idx,
lmk_bary_coords)
if self.use_3D_translation:
landmarks += transl.unsqueeze(dim=1)
vertices += transl.unsqueeze(dim=1)
return vertices, landmarks
def forward(self,
shape_params=None,
expression_params=None,
pose_params=None,
neck_pose=None,
transl=None,
eye_pose=None):
"""
Input:
shape_params: N X number of shape parameters
expression_params: N X number of expression parameters
pose_params: N X number of pose parameters
return:
vertices: N X V X 3
"""
betas = torch.cat([shape_params, expression_params], dim=1)
# If we don't specify eye_pose use the default
eye_pose = (eye_pose if eye_pose is not None else self.eye_pose)
full_pose = torch.cat(
[pose_params[:, :3], neck_pose, pose_params[:, 3:], eye_pose],
dim=1)
template_vertices = self.v_template.unsqueeze(0).repeat(
pose_params.shape[0], 1, 1)
vertices, _ = lbs(betas,
full_pose,
template_vertices,
self.shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
dtype=self.dtype)
vertices += transl.unsqueeze(dim=1)
return vertices
def forward(self,
betas=None,
global_orient=None,
body_pose=None,
left_hand_pose=None,
right_hand_pose=None,
transl=None,
expression=None,
jaw_pose=None,
leye_pose=None,
reye_pose=None,
return_verts=True,
return_full_pose=False,
pose2rot=True,
**kwargs):
'''
Forward pass for the SMPLX model
Parameters
----------
global_orient: torch.tensor, optional, shape Bx3
If given, ignore the member variable and use it as the global
rotation of the body. Useful if someone wishes to predicts this
with an external model. (default=None)
betas: torch.tensor, optional, shape Bx10
If given, ignore the member variable `betas` and use it
instead. For example, it can used if shape parameters
`betas` are predicted from some external model.
(default=None)
expression: torch.tensor, optional, shape Bx10
If given, ignore the member variable `expression` and use it
instead. For example, it can used if expression parameters
`expression` are predicted from some external model.
body_pose: torch.tensor, optional, shape Bx(J*3)
If given, ignore the member variable `body_pose` and use it
instead. For example, it can used if someone predicts the
pose of the body joints are predicted from some external model.
It should be a tensor that contains joint rotations in
axis-angle format. (default=None)
left_hand_pose: torch.tensor, optional, shape BxP
If given, ignore the member variable `left_hand_pose` and
use this instead. It should either contain PCA coefficients or
joint rotations in axis-angle format.
right_hand_pose: torch.tensor, optional, shape BxP
If given, ignore the member variable `right_hand_pose` and
use this instead. It should either contain PCA coefficients or
joint rotations in axis-angle format.
jaw_pose: torch.tensor, optional, shape Bx3
If given, ignore the member variable `jaw_pose` and
use this instead. It should either joint rotations in
axis-angle format.
transl: torch.tensor, optional, shape Bx3
If given, ignore the member variable `transl` and use it
instead. For example, it can used if the translation
`transl` is predicted from some external model.
(default=None)
return_verts: bool, optional
Return the vertices. (default=True)
return_full_pose: bool, optional
Returns the full axis-angle pose vector (default=False)
Returns
-------
output: ModelOutput
A named tuple of type `ModelOutput`
'''
# If no shape and pose parameters are passed along, then use the
# ones from the module
global_orient = (global_orient
if global_orient is not None else self.global_orient)
body_pose = body_pose if body_pose is not None else self.body_pose
betas = betas if betas is not None else self.betas
left_hand_pose = (left_hand_pose if left_hand_pose is not None else
self.left_hand_pose)
right_hand_pose = (right_hand_pose if right_hand_pose is not None else
self.right_hand_pose)
jaw_pose = jaw_pose if jaw_pose is not None else self.jaw_pose
leye_pose = leye_pose if leye_pose is not None else self.leye_pose
reye_pose = reye_pose if reye_pose is not None else self.reye_pose
expression = expression if expression is not None else self.expression
apply_trans = transl is not None or hasattr(self, 'transl')
if transl is None:
if hasattr(self, 'transl'):
transl = self.transl
if self.use_pca:
left_hand_pose = torch.einsum(
'bi,ij->bj', [left_hand_pose, self.left_hand_components])
right_hand_pose = torch.einsum(
'bi,ij->bj', [right_hand_pose, self.right_hand_components])
batch_size = max(betas.shape[0], global_orient.shape[0],
body_pose.shape[0])
#Fixed rot matrix input issue. Assuming hands and others are still angle axis
if pose2rot == True:
full_pose = torch.cat([
global_orient, body_pose, jaw_pose, leye_pose, reye_pose,
left_hand_pose, right_hand_pose
],
dim=1)
full_pose += self.pose_mean
else:
full_pose = torch.cat([global_orient, body_pose],
dim=1) #(N, 22, 3, 3)
full_others = torch.cat([
jaw_pose, leye_pose, reye_pose, left_hand_pose, right_hand_pose
],
dim=1)
# full_pose += self.pose_mean
full_others += self.pose_mean[full_pose.shape[1] * 3:]
# batchsize_size = full_pose.shape[0]
full_pose_others = batch_rodrigues(full_others.view(-1, 3),
dtype=self.dtype).view(
[batch_size, -1, 3,
3]) #(N, 33, 3, 3)
full_pose = torch.cat([full_pose, full_pose_others],
dim=1) #(4,55, 3, 3)
# Add the mean pose of the model. Does not affect the body, only the
# hands when flat_hand_mean == False
# Concatenate the shape and expression coefficients
scale = int(batch_size / betas.shape[0])
if scale > 1:
betas = betas.expand(scale, -1)
shape_components = torch.cat([betas, expression], dim=-1)
vertices, joints = lbs(shape_components,
full_pose,
self.v_template,
self.shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
pose2rot=pose2rot,
dtype=self.dtype)
lmk_faces_idx = self.lmk_faces_idx.unsqueeze(dim=0).expand(
batch_size, -1).contiguous()
lmk_bary_coords = self.lmk_bary_coords.unsqueeze(dim=0).repeat(
self.batch_size, 1, 1)
if self.use_face_contour:
dyn_lmk_faces_idx, dyn_lmk_bary_coords = find_dynamic_lmk_idx_and_bcoords(
vertices,
full_pose,
self.dynamic_lmk_faces_idx,
self.dynamic_lmk_bary_coords,
self.neck_kin_chain,
dtype=self.dtype)
lmk_faces_idx = torch.cat([lmk_faces_idx, dyn_lmk_faces_idx], 1)
lmk_bary_coords = torch.cat([
lmk_bary_coords.expand(batch_size, -1, -1), dyn_lmk_bary_coords
], 1)
landmarks = vertices2landmarks(vertices, self.faces_tensor,
lmk_faces_idx, lmk_bary_coords)
# Add any extra joints that might be needed
joints = self.vertex_joint_selector(vertices, joints)
# Add the landmarks to the joints
joints = torch.cat([joints, landmarks], dim=1)
# Map the joints to the current dataset
if self.joint_mapper is not None:
joints = self.joint_mapper(joints=joints, vertices=vertices)
if apply_trans:
joints += transl.unsqueeze(dim=1)
vertices += transl.unsqueeze(dim=1)
output = ModelOutput(vertices=vertices if return_verts else None,
joints=joints,
betas=betas,
expression=expression,
global_orient=self.global_orient,
body_pose=body_pose,
left_hand_pose=self.left_hand_pose,
right_hand_pose=self.right_hand_pose,
jaw_pose=jaw_pose,
full_pose=full_pose if return_full_pose else None)
return output
def forward(self,
betas=None,
global_orient=None,
body_pose=None,
left_hand_pose=None,
right_hand_pose=None,
transl=None,
return_verts=True,
return_full_pose=False,
pose2rot=True,
**kwargs):
'''
'''
# If no shape and pose parameters are passed along, then use the
# ones from the module
global_orient = (global_orient
if global_orient is not None else self.global_orient)
body_pose = body_pose if body_pose is not None else self.body_pose
betas = betas if betas is not None else self.betas
left_hand_pose = (left_hand_pose if left_hand_pose is not None else
self.left_hand_pose)
right_hand_pose = (right_hand_pose if right_hand_pose is not None else
self.right_hand_pose)
apply_trans = transl is not None or hasattr(self, 'transl')
if transl is None:
if hasattr(self, 'transl'):
transl = self.transl
if self.use_pca:
left_hand_pose = torch.einsum(
'bi,ij->bj', [left_hand_pose, self.left_hand_components])
right_hand_pose = torch.einsum(
'bi,ij->bj', [right_hand_pose, self.right_hand_components])
full_pose = torch.cat(
[global_orient, body_pose, left_hand_pose, right_hand_pose], dim=1)
full_pose += self.pose_mean
vertices, joints = lbs(self.betas,
full_pose,
self.v_template,
self.shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
pose2rot=pose2rot,
dtype=self.dtype)
# Add any extra joints that might be needed
joints = self.vertex_joint_selector(vertices, joints)
if self.joint_mapper is not None:
joints = self.joint_mapper(joints)
if apply_trans:
joints += transl.unsqueeze(dim=1)
vertices += transl.unsqueeze(dim=1)
output = ModelOutput(vertices=vertices if return_verts else None,
joints=joints,
betas=betas,
global_orient=global_orient,
body_pose=body_pose,
left_hand_pose=left_hand_pose,
right_hand_pose=right_hand_pose,
full_pose=full_pose if return_full_pose else None)
return output
def forward(self,
betas=None,
body_pose=None,
global_orient=None,
transl=None,
return_verts=True,
return_full_pose=False,
pose2rot=True,
**kwargs):
''' Forward pass for the SMPL model
Parameters
----------
global_orient: torch.tensor, optional, shape Bx3
If given, ignore the member variable and use it as the global
rotation of the body. Useful if someone wishes to predicts this
with an external model. (default=None)
betas: torch.tensor, optional, shape Bx10
If given, ignore the member variable `betas` and use it
instead. For example, it can used if shape parameters
`betas` are predicted from some external model.
(default=None)
body_pose: torch.tensor, optional, shape Bx(J*3)
If given, ignore the member variable `body_pose` and use it
instead. For example, it can used if someone predicts the
pose of the body joints are predicted from some external model.
It should be a tensor that contains joint rotations in
axis-angle format. (default=None)
transl: torch.tensor, optional, shape Bx3
If given, ignore the member variable `transl` and use it
instead. For example, it can used if the translation
`transl` is predicted from some external model.
(default=None)
return_verts: bool, optional
Return the vertices. (default=True)
return_full_pose: bool, optional
Returns the full axis-angle pose vector (default=False)
Returns
-------
'''
# If no shape and pose parameters are passed along, then use the
# ones from the module
global_orient = (global_orient
if global_orient is not None else self.global_orient)
body_pose = body_pose if body_pose is not None else self.body_pose
betas = betas if betas is not None else self.betas
apply_trans = transl is not None or hasattr(self, 'transl')
if transl is None and hasattr(self, 'transl'):
transl = self.transl
full_pose = torch.cat([global_orient, body_pose], dim=1)
batch_size = max(betas.shape[0], global_orient.shape[0],
body_pose.shape[0])
if betas.shape[0] != batch_size:
num_repeats = int(batch_size / betas.shape[0])
betas = betas.expand(num_repeats, -1)
vertices, joints = lbs(betas,
full_pose,
self.v_template,
self.shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
pose2rot=pose2rot,
dtype=self.dtype)
joints = self.vertex_joint_selector(vertices, joints)
# Map the joints to the current dataset
if self.joint_mapper is not None:
joints = self.joint_mapper(joints)
if apply_trans:
joints += transl.unsqueeze(dim=1)
vertices += transl.unsqueeze(dim=1)
output = ModelOutput(vertices=vertices if return_verts else None,
global_orient=global_orient,
body_pose=body_pose,
joints=joints,
betas=betas,
full_pose=full_pose if return_full_pose else None)
return output
def forward(self,
root_orient=None,
pose_body=None,
pose_hand=None,
pose_jaw=None,
pose_eye=None,
betas=None,
trans=None,
**kwargs):
'''
:param root_orient: Nx3
:param pose_body:
:param pose_hand:
:param pose_jaw:
:param pose_eye:
:param kwargs:
:return:
'''
assert self.model_type in [
'smpl', 'smplh', 'smplhf', 'mano_left', 'mano_right'
], ValueError(
'model_type should be in smpl/smplh/smplhf/mano_left/mano_right.')
if root_orient is None: root_orient = self.root_orient
if self.model_type in ['smplh', 'smpl']:
if pose_body is None: pose_body = self.pose_body
if pose_hand is None: pose_hand = self.pose_hand
elif self.model_type == 'smplhf':
if pose_body is None: pose_body = self.pose_body
if pose_hand is None: pose_hand = self.pose_hand
if pose_jaw is None: pose_jaw = self.pose_jaw
if pose_eye is None: pose_eye = self.pose_eye
elif self.model_type in ['mano_left', 'mano_right']:
if pose_hand is None: pose_hand = self.pose_hand
if trans is None: trans = self.trans
if betas is None: betas = self.betas
if self.model_type in ['smplh', 'smpl']:
full_pose = torch.cat([root_orient, pose_body, pose_hand], dim=1)
elif self.model_type == 'smplhf':
full_pose = torch.cat(
[root_orient, pose_body, pose_jaw, pose_eye, pose_hand], dim=1
) # orient:3, body:63, jaw:3, eyel:3, eyer:3, handl, handr
elif self.model_type in ['mano_left', 'mano_right']:
full_pose = torch.cat([root_orient, pose_hand], dim=1)
if self.model_type == 'smplhf':
shape_components = torch.cat([betas, self.expression], dim=-1)
shapedirs = torch.cat([self.shapedirs, self.exprdirs], dim=-1)
else:
shape_components = betas
shapedirs = self.shapedirs
verts, joints = lbs(betas=shape_components,
pose=full_pose,
v_template=self.v_template,
shapedirs=shapedirs,
posedirs=self.posedirs,
J_regressor=self.J_regressor,
parents=self.kintree_table[0].long(),
lbs_weights=self.weights,
num_joints=int(full_pose.shape[1] / 3),
dtype=self.dtype)
Jtr = joints + trans.unsqueeze(dim=1)
verts = verts + trans.unsqueeze(dim=1)
class result_meta(object):
pass
res = result_meta()
res.v = verts
res.f = self.f
res.betas = self.betas
res.Jtr = Jtr #Todo: ik can be made with vposer
if self.model_type == 'smpl':
res.pose_body = pose_body
elif self.model_type == 'smplh':
res.pose_body = pose_body
res.pose_hand = pose_hand
elif self.model_type == 'smplhf':
res.pose_body = pose_body
res.pose_hand = pose_hand
res.pose_jaw = pose_jaw
res.pose_eye = pose_eye
elif self.model_type in ['mano_left', 'mano_right']:
res.pose_hand = pose_hand
res.full_pose = full_pose
return res
def forward(self,
root_orient=None,
pose_body=None,
pose_hand=None,
pose_jaw=None,
pose_eye=None,
betas=None,
trans=None,
dmpls=None,
expression=None,
return_dict=False,
v_template=None,
**kwargs):
'''
:param root_orient: Nx3
:param pose_body:
:param pose_hand:
:param pose_jaw:
:param pose_eye:
:param kwargs:
:return:
'''
assert not (v_template is not None and betas is not None), ValueError(
'vtemplate and betas could not be used jointly.')
assert self.model_type in [
'smpl', 'smplh', 'smplx', 'mano'
], ValueError('model_type should be in smpl/smplh/smplx/mano')
if root_orient is None: root_orient = self.root_orient
if self.model_type in ['smpl', 'smplh']:
if pose_body is None: pose_body = self.pose_body
if pose_hand is None: pose_hand = self.pose_hand
elif self.model_type == 'smplx':
if pose_body is None: pose_body = self.pose_body
if pose_hand is None: pose_hand = self.pose_hand
if pose_jaw is None: pose_jaw = self.pose_jaw
if pose_eye is None: pose_eye = self.pose_eye
elif self.model_type in ['mano']:
if pose_hand is None: pose_hand = self.pose_hand
if trans is None: trans = self.trans
if v_template is None: v_template = self.v_template
if betas is None: betas = self.betas
if self.model_type in ['smpl', 'smplh']:
full_pose = torch.cat([root_orient, pose_body, pose_hand], dim=1)
elif self.model_type == 'smplx':
full_pose = torch.cat(
[root_orient, pose_body, pose_jaw, pose_eye, pose_hand], dim=1
) # orient:3, body:63, jaw:3, eyel:3, eyer:3, handl, handr
elif self.model_type in ['mano']:
full_pose = torch.cat([root_orient, pose_hand], dim=1)
if self.use_dmpl:
if dmpls is None: dmpls = self.dmpls
shape_components = torch.cat([betas, dmpls], dim=-1)
shapedirs = torch.cat([self.shapedirs, self.dmpldirs], dim=-1)
elif self.model_type == 'smplx':
if expression is None: expression = self.expression
shape_components = torch.cat([betas, expression], dim=-1)
shapedirs = torch.cat([self.shapedirs, self.exprdirs], dim=-1)
else:
shape_components = betas
shapedirs = self.shapedirs
verts, joints = lbs(betas=shape_components,
pose=full_pose,
v_template=v_template,
shapedirs=shapedirs,
posedirs=self.posedirs,
J_regressor=self.J_regressor,
parents=self.kintree_table[0].long(),
lbs_weights=self.weights,
dtype=self.dtype)
Jtr = joints + trans.unsqueeze(dim=1)
verts = verts + trans.unsqueeze(dim=1)
res = {}
res['v'] = verts
res['f'] = self.f
res['betas'] = self.betas
res['Jtr'] = Jtr # Todo: ik can be made with vposer
if self.model_type == 'smpl':
res['pose_body'] = pose_body
elif self.model_type == 'smplh':
res['pose_body'] = pose_body
res['pose_hand'] = pose_hand
elif self.model_type == 'smplx':
res['pose_body'] = pose_body
res['pose_hand'] = pose_hand
res['pose_jaw'] = pose_jaw
res['pose_eye'] = pose_eye
elif self.model_type in ['mano']:
res['pose_hand'] = pose_hand
res['full_pose'] = full_pose
if not return_dict:
class result_meta(object):
pass
res_class = result_meta()
for k, v in res.items():
res_class.__setattr__(k, v)
res = res_class
return res
def forward(self,
shape_params=None,
expression_params=None,
pose_params=None,
neck_pose=None,
eye_pose=None,
transl=None):
"""
Input:
shape_params: N X number of shape parameters
expression_params: N X number of expression parameters
pose_params: N X number of pose parameters
return:
vertices: N X V X 3
landmarks: N X number of landmarks X 3
"""
batch_size = pose_params.shape[0]
betas = torch.cat([
shape_params, self.shape_betas, expression_params,
self.expression_betas
],
dim=1)
neck_pose = (neck_pose if neck_pose is not None else self.neck_pose)
eye_pose = (eye_pose if eye_pose is not None else self.eye_pose)
transl = (transl if transl is not None else self.transl)
full_pose = torch.cat(
[pose_params[:, :3], neck_pose, neck_pose, eye_pose], dim=1)
template_vertices = self.v_template.unsqueeze(0).repeat(
self.batch_size, 1, 1)
vertices, _ = lbs(betas,
full_pose,
template_vertices,
self.shapedirs,
self.posedirs,
self.J_regressor,
self.parents,
self.lbs_weights,
dtype=self.dtype)
print('vertices', vertices.shape)
scaled = (-torch.abs(pose_params[:, 4].unsqueeze(1)) *
vertices).reshape(batch_size, 3, -1) #
scaled[:, :2, :] = scaled[:, :2, :] + pose_params[:, 5:7].view(
batch_size, 2, 1)
# lmk_faces_idx = self.lmk_faces_idx.unsqueeze(dim=0).repeat(
# self.batch_size, 1)
# lmk_bary_coords = self.lmk_bary_coords.unsqueeze(dim=0).repeat(self.batch_size, 1, 1)
# if self.use_face_contour:
# dyn_lmk_faces_idx, dyn_lmk_bary_coords = self._find_dynamic_lmk_idx_and_bcoords(
# face_pixels, full_pose, self.dynamic_lmk_faces_idx,
# self.dynamic_lmk_bary_coords,
# self.neck_kin_chain, dtype=self.dtype)
# lmk_faces_idx = torch.cat([dyn_lmk_faces_idx, lmk_faces_idx], 1)
# lmk_bary_coords = torch.cat(
# [dyn_lmk_bary_coords, lmk_bary_coords], 1)
# landmarks = vertices2landmarks(face_pixels, self.faces_tensor,lmk_faces_idx,lmk_bary_coords)
return vertices
