def verts_core(pose, v, J, weights, kintree_table, want_Jtr=False):
A, A_global = global_rigid_transformation(pose, J, kintree_table)
T = A.dot(weights.T)
rest_shape_h = ch.vstack((v.T, np.ones((1, v.shape[0]))))
v = (T[:, 0, :] * rest_shape_h[0, :].reshape(
(1, -1)) + T[:, 1, :] * rest_shape_h[1, :].reshape(
(1, -1)) + T[:, 2, :] * rest_shape_h[2, :].reshape(
(1, -1)) + T[:, 3, :] * rest_shape_h[3, :].reshape((1, -1))).T
v = v[:, :3]
class result_meta(object):
pass
if not want_Jtr:
Jtr = None
else:
Jtr = ch.vstack([g[:3, 3] for g in A_global])
meta = result_meta()
meta.Jtr = Jtr
meta.A = A
meta.A_global = A_global
meta.A_weighted = T
return v, meta
def chumpy_compute_error(p1,
p2,
H,
Mcor,
error=0,
return_sigma=False,
sigma_precomputed=None):
"""Compute deviation of p1 and p2 from common epipole, given H.
"""
# Warp p2 estimates with new homography
p2_new_denom = H[2, 0] * p2[:, 0] + H[2, 1] * p2[:, 1] + H[2, 2]
p2_new_x = (H[0, 0] * p2[:, 0] + H[0, 1] * p2[:, 1] +
H[0, 2]) / p2_new_denom
p2_new_y = (H[1, 0] * p2[:, 0] + H[1, 1] * p2[:, 1] +
H[1, 2]) / p2_new_denom
p2_new = ch.vstack((p2_new_x, p2_new_y)).T
# Compute current best FoE
u = p2_new[:, 0] - p1[:, 0]
v = p2_new[:, 1] - p1[:, 1]
T = ch.vstack((-v, u, v * p1[:, 0] - u * p1[:, 1])).T
U, S, V = ch.linalg.svd(Mcor.dot(T.T.dot(T)).dot(Mcor))
qv = Mcor.dot(V[-1, :])
q = qv[:2] / qv[2]
d = T.dot(qv)
# Robust error norm
if error == 0:
d = d**2
elif error == 1:
sigma = np.median(np.abs(d() - np.median(d())))
d = ch.sqrt(d**2 + sigma**2)
elif error == 2:
# Geman-McClure
sigma = np.median(np.abs(d() - np.median(d()))) * 1.5
d = d**2 / (d**2 + sigma)
elif error == 3:
# Geman-McClure, corrected
if sigma_precomputed is not None:
sigma = sigma_precomputed
else:
sigma = 1.426 * np.median(
np.abs(d() - np.median(d()))) * np.sqrt(3.0)
d = d**2 / (d**2 + sigma**2)
# Correction
d = d * sigma
elif error == 4:
# Inverse exponential norm
sigma = np.median(np.abs(d() - np.median(d())))
d = -ch.exp(-d**2 / (2 * sigma**2)) * sigma
err = d.sum()
if return_sigma:
return sigma
else:
return err
def chumpy_get_H(p1, p2):
""" Compute differentiable homography from p1 to p2.
Parameters
----------
p1,p2 : array_like, shape (4,2)
Containing points to match.
"""
xmin = 0
ymin = 0
xmax = 1024
ymax = 576
p1 = 2 * p1 / ch.array([[xmax, ymax]])
p1 = p1 - 1.0
p2 = 2 * p2 / ch.array([[xmax, ymax]])
p2 = p2 - 1.0
N = p1.shape[0]
A1 = ch.vstack((ch.zeros((3, N)), -p1.T, -ch.ones(
(1, N)), p2[:, 1] * p1[:, 0], p2[:, 1] * p1[:, 1], p2[:, 1])).T
A2 = ch.vstack((p1.T, ch.ones((1, N)), ch.zeros(
(3, N)), -p2[:, 0] * p1[:, 0], -p2[:, 0] * p1[:, 1], -p2[:, 0])).T
A = ch.vstack((A1, A2))
U, S, V = ch.linalg.svd(A.T.dot(A))
H_new = V[-1, :].reshape((3, 3))
# Re-normalize
ML = ch.array([[xmax / 2.0, 0.0, xmax / 2.0], [0, ymax / 2.0, ymax / 2.0],
[0, 0, 1.0]])
MR = ch.array([[2.0 / xmax, 0.0, -1.0], [0, 2.0 / ymax, -1.0], [0, 0,
1.0]])
H_new = ML.dot(H_new).dot(MR)
return H_new / H_new[2, 2]
def __init__(self, t, rod, rad, length):
self.t = t # translation of the axis
self.rod = rod # rotation of the axis in Rodrigues form
self.rad = rad # radious of the capsule
self.length = length # length of the axis
axis0 = ch.vstack([0, ch.abs(self.length), 0])
self.axis = ch.vstack((t.T, (t + Rodrigues(rod).dot(axis0)).T))
v0 = ch.hstack([v[:26].T*rad, (v[26:].T*rad)+ axis0])
self.v = ((t + Rodrigues(rod).dot(v0)).T)
self.set_sphere_centers()
def chumpy_get_H(p1, p2):
""" Compute differentiable homography from p1 to p2.
"""
N = p1.shape[0]
A1 = ch.vstack((ch.zeros((3, N)), -p1.T, -ch.ones(
(1, N)), p2[:, 1] * p1[:, 0], p2[:, 1] * p1[:, 1], p2[:, 1])).T
A2 = ch.vstack((p1.T, ch.ones((1, N)), ch.zeros(
(3, N)), -p2[:, 0] * p1[:, 0], -p2[:, 0] * p1[:, 1], -p2[:, 0])).T
A = ch.vstack((A1, A2))
U, S, V = ch.linalg.svd(A.T.dot(A))
H_new = V[-1, :].reshape((3, 3))
return H_new
def guess_init(model, focal_length, j2d, init_pose):
"""Initialize the camera translation via triangle similarity, by using the torso joints .
:param model: SMPL model
:param focal_length: camera focal length (kept fixed)
:param j2d: 14x2 array of CNN joints
:param init_pose: 72D vector of pose parameters used for initialization (kept fixed)
:returns: 3D vector corresponding to the estimated camera translation
"""
cids = np.arange(0, 12)
# map from LSP to SMPL joints
j2d_here = j2d[cids]
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
opt_pose = ch.array(init_pose)
(_, A_global) = global_rigid_transformation(
opt_pose, model.J, model.kintree_table, xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
Jtr = Jtr[smpl_ids].r
# 9 is L shoulder, 3 is L hip
# 8 is R shoulder, 2 is R hip
diff3d = np.array([Jtr[9] - Jtr[3], Jtr[8] - Jtr[2]])
mean_height3d = np.mean(np.sqrt(np.sum(diff3d**2, axis=1)))
diff2d = np.array([j2d_here[9] - j2d_here[3], j2d_here[8] - j2d_here[2]])
mean_height2d = np.mean(np.sqrt(np.sum(diff2d**2, axis=1)))
est_d = focal_length * (mean_height3d / mean_height2d)
# just set the z value
init_t = np.array([0., 0., est_d])
return init_t
def _global_rigid_transformation(self):
results = {}
pose = self.pose.reshape((-1, 3))
parent = {
i: self.kintree_table[0, i]
for i in range(1, self.kintree_table.shape[1])
}
with_zeros = lambda x: ch.vstack((x, ch.array([[0.0, 0.0, 0.0, 1.0]])))
pack = lambda x: ch.hstack([ch.zeros((4, 3)), x.reshape((4, 1))])
results[0] = with_zeros(
ch.hstack((Rodrigues(pose[0, :]), self.J[0, :].reshape((3, 1)))))
for i in range(1, self.kintree_table.shape[1]):
results[i] = results[parent[i]].dot(
with_zeros(
ch.hstack((
Rodrigues(pose[i, :]), # rotation around bone endpoint
(self.J[i, :] - self.J[parent[i], :]).reshape(
(3, 1)) # bone
))))
results = [results[i] for i in sorted(results.keys())]
results_global = results
# subtract rotated J position
results2 = [
results[i] -
(pack(results[i].dot(ch.concatenate((self.J[i, :], [0])))))
for i in range(len(results))
]
result = ch.dstack(results2)
return result, results_global
def mesh_points_by_barycentric_coordinates( mesh_verts, mesh_faces, lmk_face_idx, lmk_b_coords ):
""" function: evaluation 3d points given mesh and landmark embedding
"""
dif1 = ch.vstack([(mesh_verts[mesh_faces[lmk_face_idx], 0] * lmk_b_coords).sum(axis=1),
(mesh_verts[mesh_faces[lmk_face_idx], 1] * lmk_b_coords).sum(axis=1),
(mesh_verts[mesh_faces[lmk_face_idx], 2] * lmk_b_coords).sum(axis=1)]).T
return dif1
def _set_up(self):
self.v_shaped = self.shapedirs.dot(self.betas) + self.v_template
self.v_shaped_personal = self.v_shaped + self.v_personal
if sp.issparse(self.J_regressor):
self.J = sp_dot(self.J_regressor, self.v_shaped)
else:
self.J = ch.sum(self.J_regressor.T.reshape(-1, 1, 24) *
self.v_shaped.reshape(-1, 3, 1),
axis=0).T
self.v_posevariation = self.posedirs.dot(
posemap(self.bs_type)(self.pose))
self.v_poseshaped = self.v_shaped_personal + self.v_posevariation
self.A, A_global = self._global_rigid_transformation()
self.Jtr = ch.vstack([g[:3, 3] for g in A_global])
self.J_transformed = self.Jtr + self.trans.reshape((1, 3))
self.V = self.A.dot(self.weights.T)
rest_shape_h = ch.hstack(
(self.v_poseshaped, ch.ones((self.v_poseshaped.shape[0], 1))))
self.v_posed = ch.sum(self.V.T * rest_shape_h.reshape(-1, 4, 1),
axis=1)[:, :3]
self.v = self.v_posed + self.trans
def createRendererGT(glMode, chAz, chEl, chDist, center, v, vc, f_list, vn,
light_color, chComponent, chVColors, targetPosition,
chDisplacement, width, height, uv, haveTextures_list,
textures_list, frustum, win):
renderer = TexturedRenderer()
renderer.set(glMode=glMode)
vflat = [item for sublist in v for item in sublist]
rangeMeshes = range(len(vflat))
# vch = [ch.array(vflat[mesh]) for mesh in rangeMeshes]
vch = vflat
if len(vch) == 1:
vstack = vch[0]
else:
vstack = ch.vstack(vch)
camera, modelRotation, _ = setupCamera(
vstack, chAz, chEl, chDist, center + targetPosition + chDisplacement,
width, height)
vnflat = [item for sublist in vn for item in sublist]
# vnch = [ch.array(vnflat[mesh]) for mesh in rangeMeshes]
# vnchnorm = [vnch[mesh]/ch.sqrt(vnch[mesh][:,0]**2 + vnch[mesh][:,1]**2 + vnch[mesh][:,2]**2).reshape([-1,1]) for mesh in rangeMeshes]
vcflat = [item for sublist in vc for item in sublist]
vcch = [vcflat[mesh] for mesh in rangeMeshes]
# vc_list = computeGlobalAndPointLighting(vch, vnch, vcch, lightPosGT, chGlobalConstantGT, light_colorGT)
setupTexturedRenderer(renderer, vstack, vch, f_list, vcch, vnflat, uv,
haveTextures_list, textures_list, camera, frustum,
win)
return renderer
def get_keypoints_from_mesh_ch(mesh_vertices, keypoints_regressed):
""" Assembles the full 21 keypoint set from the 16 Mano Keypoints and 5 mesh vertices for the fingers. """
keypoints = [0.0 for _ in range(21)] # init empty list
# fill keypoints which are regressed
mapping = {
0: 0, #Wrist
1: 5,
2: 6,
3: 7, #Index
4: 9,
5: 10,
6: 11, #Middle
7: 17,
8: 18,
9: 19, # Pinky
10: 13,
11: 14,
12: 15, # Ring
13: 1,
14: 2,
15: 3
} # Thumb
for manoId, myId in mapping.items():
keypoints[myId] = keypoints_regressed[manoId, :]
# get other keypoints from mesh
for myId, meshId in kpId2vertices.items():
keypoints[myId] = ch.mean(mesh_vertices[meshId, :], 0)
keypoints = ch.vstack(keypoints)
return keypoints
def createNewRendererTarget(glMode, chAz, chEl, chDist, center, v, vc, f_list,
vn, light_color, chComponent, chVColors,
targetPosition, chDisplacement, width, height, uv,
haveTextures_list, textures_list, frustum, win):
renderer = ResidualRendererOpenDR()
renderer.set(glMode=glMode)
vflat = [item for sublist in v for item in sublist]
rangeMeshes = range(len(vflat))
vnflat = [item for sublist in vn for item in sublist]
vcflat = [item for sublist in vc for item in sublist]
# vcch = [np.ones_like(vcflat[mesh])*chVColors.reshape([1,3]) for mesh in rangeMeshes]
vc_list = computeSphericalHarmonics(vnflat, vcflat, light_color,
chComponent)
if len(vflat) == 1:
vstack = vflat[0]
else:
vstack = ch.vstack(vflat)
camera, modelRotation, _ = setupCamera(
vstack, chAz, chEl, chDist, center + targetPosition + chDisplacement,
width, height)
setupTexturedRenderer(renderer, vstack, vflat, f_list, vc_list, vnflat, uv,
haveTextures_list, textures_list, camera, frustum,
win)
return renderer
def landmark_error_3d( scale ,trans_2_3d, mesh_verts,mesh_faces, target_lmk_3d_face, target_lmk_3d_body,lmk_face_idx, lmk_b_coords, lmk_facevtx_idx, lmk_bodyvtx_idx, face_weight=1.0,body_weight =1.0,use_lunkuo =False ):
""" function: 3d landmark error objective
"""
# select corresponding vertices
v_selected = mesh_points_by_barycentric_coordinates( mesh_verts, mesh_faces, lmk_face_idx, lmk_b_coords )
source_face_lmkvtx = mesh_verts[lmk_facevtx_idx[0:17]]
# lmk_num = lmk_face_idx.shape[0]
source_body_lmkvtx = mesh_verts[lmk_bodyvtx_idx]
# an index to select which landmark to use
# lmk_selection = np.arange(0,lmk_num).ravel() # use all
if use_lunkuo:
frame_landmark_idx = range(0,17)+range(17, 60) + range(61, 64) + range(65, 68)
else:
frame_landmark_idx = range(17, 60) + range(61, 64) + range(65, 68)
target_lmk_3d_face = target_lmk_3d_face[frame_landmark_idx,:]
if use_lunkuo:
v_selected_merge = ch.vstack([source_face_lmkvtx,v_selected])
else:
v_selected_merge = v_selected
# residual vectors
if( target_lmk_3d_face.shape[1] == 2):
cast_source_face_lmkvtx = scale*v_selected_merge[:,0:2]+trans_2_3d[0:2]
cast_source_body_lmkvtx = scale*source_body_lmkvtx[:,0:2]+trans_2_3d[0:2]
lmk3d_obj_face = face_weight * (target_lmk_3d_face[:,0:2] - cast_source_face_lmkvtx)
lmk3d_obj_body = body_weight * (target_lmk_3d_body[:,0:2] - cast_source_body_lmkvtx)
elif target_lmk_3d_face.shape[1] == 3:
cast_source_face_lmkvtx = scale * v_selected_merge[:,:] + trans_2_3d[:]
cast_source_body_lmkvtx = scale * source_body_lmkvtx[:, 0:2] + trans_2_3d[0:2]
lmk3d_obj_face = face_weight * (target_lmk_3d_face - cast_source_face_lmkvtx)
lmk3d_obj_body = body_weight * (target_lmk_3d_body[:,0:2] - cast_source_body_lmkvtx)
else:
pass
return lmk3d_obj_face,lmk3d_obj_body
def update_capsules_and_centers(self):
centers = [set_sphere_centers(capsule) for capsule in self.capsules]
count = 0
for capsule in self.capsules:
capsule.center_id = count
count += len(capsule.centers)
self.sph_vs = ch.vstack(centers)
self.sph_weights = get_sphere_bweights(self.sph_vs, self.capsules)
self.ids0 = []
self.ids1 = []
self.radiuss = []
self.caps_pairs = []
for collision in collisions:
if hasattr(self, 'no_hands'):
(id0, id1, rd) = capsule_dist(self.capsules[collision[0]],
self.capsules[collision[1]],
increase_hand=False)
else:
(id0, id1, rd) = capsule_dist(self.capsules[collision[0]],
self.capsules[collision[1]])
self.ids0.append(id0.r)
self.ids1.append(id1.r)
self.radiuss.append(rd)
self.caps_pairs.append(
['%02d_%02d' % (collision[0], collision[1])] * len(id0))
self.ids0 = np.concatenate(self.ids0).astype(int)
self.ids1 = np.concatenate(self.ids1).astype(int)
self.radiuss = np.concatenate(self.radiuss)
self.caps_pairs = np.concatenate(self.caps_pairs)
assert (self.caps_pairs.size == self.ids0.size)
assert (self.radiuss.size == self.ids0.size)
def ready_arguments(fname_or_dict, highRes):
import numpy as np
import pickle
import chumpy as ch
from chumpy.ch import MatVecMult
from dataset.smpl_layer.posemapper import posemap
import scipy.sparse as sp
if not isinstance(fname_or_dict, dict):
dd = pickle.load(open(fname_or_dict, 'rb'), encoding='latin1')
# dd = pickle.load(open(fname_or_dict, 'rb'))
else:
dd = fname_or_dict
want_shapemodel = 'shapedirs' in dd
nposeparms = dd['kintree_table'].shape[1] * 3
if 'trans' not in dd:
dd['trans'] = np.zeros(3)
if 'pose' not in dd:
dd['pose'] = np.zeros(nposeparms)
if 'shapedirs' in dd and 'betas' not in dd:
dd['betas'] = np.zeros(dd['shapedirs'].shape[-1])
for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J']:
if (s in dd) and isinstance(dd[s], ch.ch.Ch):
dd[s] = dd[s].r
if want_shapemodel:
dd['v_shaped'] = dd['shapedirs'].dot(dd['betas']) + dd['v_template']
v_shaped = dd['v_shaped']
J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:, 0])
J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:, 1])
J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:, 2])
dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
dd['v_posed'] = v_shaped + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
else:
dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
if highRes is not None:
with open(highRes, 'rb') as f:
mapping, hf = pickle.load(f, encoding='latin1')
num_betas = dd['shapedirs'].shape[-1]
hv = mapping.dot(dd['v_template'].ravel()).reshape(-1, 3)
J_reg = dd['J_regressor'].asformat('csr')
dd['f'] = hf
dd['v_template'] = hv
dd['weights'] = np.hstack([
np.expand_dims(
np.mean(
mapping.dot(np.repeat(np.expand_dims(dd['weights'][:, i], -1), 3)).reshape(-1, 3)
, axis=1),
axis=-1)
for i in range(24)
])
dd['posedirs'] = mapping.dot(dd['posedirs'].reshape((-1, 207))).reshape(-1, 3, 207)
dd['shapedirs'] = mapping.dot(dd['shapedirs'].reshape((-1, num_betas))).reshape(-1, 3, num_betas)
dd['J_regressor'] = sp.csr_matrix((J_reg.data, J_reg.indices, J_reg.indptr), shape=(24, hv.shape[0]))
return dd
def __init__(self, t, rod, rad, length):
assert (hasattr(t, 'dterms'))
# the translation should be a chumpy object (differentiable wrt shape)
self.t = t # translation of the axis
self.rod = rod # rotation of the axis in Rodrigues form
# the radius should be a chumpy object (differentiable wrt shape)
assert (hasattr(rad, 'dterms'))
self.rad = rad # radius of the capsule
# the length should be a chumpy object (differentiable wrt shape)
assert (hasattr(length, 'dterms'))
self.length = length # length of the axis
axis0 = ch.vstack([0, ch.abs(self.length), 0])
self.axis = ch.vstack((t.T, (t + Rodrigues(rod).dot(axis0)).T))
v0 = ch.hstack([v[:26].T * rad, (v[26:].T * rad) + axis0])
self.v = ((t + Rodrigues(rod).dot(v0)).T)
self.set_sphere_centers()
def joints_coco(smpl):
J = smpl.J_transformed
nose = smpl[VERT_NOSE]
ear_l = smpl[VERT_EAR_L]
ear_r = smpl[VERT_EAR_R]
eye_l = smpl[VERT_EYE_L]
eye_r = smpl[VERT_EYE_R]
shoulders_m = ch.sum(J[[14, 13]], axis=0) / 2.
neck = J[12] - 0.55 * (J[12] - shoulders_m)
return ch.vstack((
nose,
neck,
2.1 * (J[14] - shoulders_m) + neck,
J[[19, 21]],
2.1 * (J[13] - shoulders_m) + neck,
J[[18, 20]],
J[2] + 0.38 * (J[2] - J[1]),
J[[5, 8]],
J[1] + 0.38 * (J[1] - J[2]),
J[[4, 7]],
eye_r,
eye_l,
ear_r,
ear_l,
))
def ready_arguments(fname_or_dict):
if not isinstance(fname_or_dict, dict):
dd = pickle.load(open(fname_or_dict))
else:
dd = fname_or_dict
backwards_compatibility_replacements(dd) #insert or change the name of some index
want_shapemodel = 'shapedirs' in dd
nposeparms = dd['kintree_table'].shape[1]*3 #shape[1] = 24, shape[0] = 2, why shape[0] = 2?
# *3 is to sca.e the kintree_table to QUATERNION
if 'trans' not in dd:
dd['trans'] = np.zeros(3) # why trans = (0,0,0) what trans?
if 'pose' not in dd:
dd['pose'] = np.zeros(nposeparms) # dim of pose is 24*3 = 72
if 'shapedirs' in dd and 'betas' not in dd: # dim of shapedirs 6890 * 3 * 10
dd['betas'] = np.zeros(dd['shapedirs'].shape[-1]) # dim of beta 10
for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J']:
if (s in dd) and not hasattr(dd[s], 'dterms'):
dd[s] = ch.array(dd[s])
if want_shapemodel:
dd['v_shaped'] = dd['shapedirs'].dot(dd['betas'])+dd['v_template']
v_shaped = dd['v_shaped']
J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:,0])
J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:,1])
J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:,2])
dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
dd['v_posed'] = v_shaped + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
else:
dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
return dd
def ready_arguments(fname_or_dict):
if not isinstance(fname_or_dict, dict):
dd = pickle.load(open(fname_or_dict, 'rb'), encoding="latin1")
else:
dd = fname_or_dict
backwards_compatibility_replacements(dd)
want_shapemodel = 'shapedirs' in dd
nposeparms = dd['kintree_table'].shape[1]*3
if 'trans' not in dd:
dd['trans'] = np.zeros(3)
if 'pose' not in dd:
dd['pose'] = np.zeros(nposeparms)
if 'shapedirs' in dd and 'betas' not in dd:
dd['betas'] = np.zeros(dd['shapedirs'].shape[-1])
for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J']:
if (s in dd) and not hasattr(dd[s], 'dterms'):
dd[s] = ch.array(dd[s])
if want_shapemodel:
dd['v_shaped'] = dd['shapedirs'].dot(dd['betas'])+dd['v_template']
v_shaped = dd['v_shaped']
J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:,0])
J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:,1])
J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:,2])
dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
dd['v_posed'] = v_shaped + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
else:
dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
return dd
def __init__(self, t, rod, rad, length):
assert (hasattr(t, 'dterms'))
# the translation should be a chumpy object (differentiable wrt shape)
self.t = t # translation of the axis
self.rod = rod # rotation of the axis in Rodrigues form
# the radius should be a chumpy object (differentiable wrt shape)
assert (hasattr(rad, 'dterms'))
self.rad = rad # radius of the capsule
# the length should be a chumpy object (differentiable wrt shape)
assert (hasattr(length, 'dterms'))
self.length = length # length of the axis
axis0 = ch.vstack([0, ch.abs(self.length), 0])
self.axis = ch.vstack((t.T, (t + Rodrigues(rod).dot(axis0)).T))
v0 = ch.hstack([v[:26].T * rad, (v[26:].T * rad) + axis0])
self.v = ((t + Rodrigues(rod).dot(v0)).T)
self.set_sphere_centers()
def SecondFundamentalForm(v, f):
from chumpy import hstack, vstack
from chumpy.linalg import Pinv
nbrs = MatVecMult(FirstEdgesMtx(v, f, want_big=True), v.ravel()).reshape(
(-1, 3))
b0 = VertNormals(f=f, v=v)
b1 = NormalizedNx3(CrossProduct(b0, nbrs - v)).reshape((-1, 3))
b2 = NormalizedNx3(CrossProduct(b0, b1)).reshape((-1, 3))
cnct = get_vert_connectivity(np.asarray(v), f)
ffs = []
for i in range(v.size / 3):
nbrs = v[np.nonzero(np.asarray(cnct[i].todense()).ravel())[0]] - row(
v[i])
us = nbrs.dot(b2[i])
vs = nbrs.dot(b1[i])
hs = nbrs.dot(b0[i])
coeffs = Pinv(
hstack((col((us * .5)**2), col(us * vs), col(
(vs * .5)**2)))).dot(hs)
ffs.append(row(coeffs))
# if i == 3586:
# import pdb; pdb.set_trace()
ffs = vstack(ffs)
return ffs
def guess_init(model, focal_length, j2d, init_pose):
"""Initialize the camera translation via triangle similarity, by using the torso joints .
:param model: SMPL model
:param focal_length: camera focal length (kept fixed)
:param j2d: 14x2 array of CNN joints
:param init_pose: 72D vector of pose parameters used for initialization (kept fixed)
:returns: 3D vector corresponding to the estimated camera translation
"""
cids = np.arange(0, 12)
# map from LSP to SMPL joints
j2d_here = j2d[cids]
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
opt_pose = ch.array(init_pose)
(_, A_global) = global_rigid_transformation(opt_pose,
model.J,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
Jtr = Jtr[smpl_ids].r
# 9 is L shoulder, 3 is L hip
# 8 is R shoulder, 2 is R hip
diff3d = np.array([Jtr[9] - Jtr[3], Jtr[8] - Jtr[2]])
mean_height3d = np.mean(np.sqrt(np.sum(diff3d**2, axis=1)))
diff2d = np.array([j2d_here[9] - j2d_here[3], j2d_here[8] - j2d_here[2]])
mean_height2d = np.mean(np.sqrt(np.sum(diff2d**2, axis=1)))
est_d = focal_length * (mean_height3d / mean_height2d)
# just set the z value
init_t = np.array([0., 0., est_d])
return init_t
def _set_up(self):
self.v_shaped = self.shapedirs.dot(self.betas) + self.v_template
body_height = (self.v_shaped[2802, 1] + self.v_shaped[6262, 1]) - (
self.v_shaped[2237, 1] + self.v_shaped[6728, 1])
# print('111111111111111111111111111111111111111')
# print(np.array(body_height)[0])
# print(type(body_height))
# print('22222222222222222222222222222222222222222')
self.scale = 1.66 / np.array(body_height)[0]
self.v_shaped_personal = self.scale * self.v_shaped + self.v_personal
if sp.issparse(self.J_regressor):
self.J = self.scale * sp_dot(self.J_regressor, self.v_shaped)
else:
self.J = self.scale * ch.sum(self.J_regressor.T.reshape(-1, 1, 24)
* self.v_shaped.reshape(-1, 3, 1),
axis=0).T
self.v_posevariation = self.posedirs.dot(
posemap(self.bs_type)(self.pose))
self.v_poseshaped = self.v_shaped_personal + self.v_posevariation
self.A, A_global = self._global_rigid_transformation()
self.Jtr = ch.vstack([g[:3, 3] for g in A_global])
self.J_transformed = self.Jtr + self.trans.reshape((1, 3))
self.V = self.A.dot(self.weights.T)
rest_shape_h = ch.hstack(
(self.v_poseshaped, ch.ones((self.v_poseshaped.shape[0], 1))))
self.v_posed = ch.sum(self.V.T * rest_shape_h.reshape(-1, 4, 1),
axis=1)[:, :3]
self.v = self.v_posed + self.trans
def verts_decorated(trans, pose,
v_template, J, weights, kintree_table, bs_style, f,
bs_type=None, posedirs=None, betas=None, shapedirs=None, want_Jtr=False):
for which in [trans, pose, v_template, weights, posedirs, betas, shapedirs]:
if which is not None:
assert ischumpy(which)
v = v_template
if shapedirs is not None:
if betas is None:
betas = chumpy.zeros(shapedirs.shape[-1])
v_shaped = v + shapedirs.dot(betas)
else:
v_shaped = v
if posedirs is not None:
v_posed = v_shaped + posedirs.dot(posemap(bs_type)(pose))
else:
v_posed = v_shaped
v = v_posed
if sp.issparse(J):
regressor = J
J_tmpx = MatVecMult(regressor, v_shaped[:,0])
J_tmpy = MatVecMult(regressor, v_shaped[:,1])
J_tmpz = MatVecMult(regressor, v_shaped[:,2])
J = chumpy.vstack((J_tmpx, J_tmpy, J_tmpz)).T
else:
assert(ischumpy(J))
assert(bs_style=='lbs')
result, Jtr = lbs.verts_core(pose, v, J, weights, kintree_table, want_Jtr=True, xp=chumpy)
tr = trans.reshape((1,3))
result = result + tr
Jtr = Jtr + tr
result.trans = trans
result.f = f
result.pose = pose
result.v_template = v_template
result.J = J
result.weights = weights
result.kintree_table = kintree_table
result.bs_style = bs_style
result.bs_type =bs_type
if posedirs is not None:
result.posedirs = posedirs
result.v_posed = v_posed
if shapedirs is not None:
result.shapedirs = shapedirs
result.betas = betas
result.v_shaped = v_shaped
if want_Jtr:
result.J_transformed = Jtr
return result
def verts_decorated(trans, pose,
v_template, J, weights, kintree_table, bs_style, f,
bs_type=None, posedirs=None, betas=None, shapedirs=None, want_Jtr=False):
for which in [trans, pose, v_template, weights, posedirs, betas, shapedirs]:
if which is not None:
assert ischumpy(which)
v = v_template
if shapedirs is not None:
if betas is None:
betas = chumpy.zeros(shapedirs.shape[-1])
v_shaped = v + shapedirs.dot(betas)
else:
v_shaped = v
if posedirs is not None:
v_posed = v_shaped + posedirs.dot(posemap(bs_type)(pose))
else:
v_posed = v_shaped
v = v_posed
if sp.issparse(J):
regressor = J
J_tmpx = MatVecMult(regressor, v_shaped[:,0])
J_tmpy = MatVecMult(regressor, v_shaped[:,1])
J_tmpz = MatVecMult(regressor, v_shaped[:,2])
J = chumpy.vstack((J_tmpx, J_tmpy, J_tmpz)).T
else:
assert(ischumpy(J))
assert(bs_style=='lbs')
result, Jtr = verts_core(pose, v, J, weights, kintree_table, want_Jtr=True, xp=chumpy)
tr = trans.reshape((1,3))
result = result + tr
Jtr = Jtr + tr
result.trans = trans
result.f = f
result.pose = pose
result.v_template = v_template
result.J = J
result.weights = weights
result.kintree_table = kintree_table
result.bs_style = bs_style
result.bs_type =bs_type
if posedirs is not None:
result.posedirs = posedirs
result.v_posed = v_posed
if shapedirs is not None:
result.shapedirs = shapedirs
result.betas = betas
result.v_shaped = v_shaped
if want_Jtr:
result.J_transformed = Jtr
return result
def _calc_coords(self):
# calculate joint location and rotation
A, A_global = global_rigid_transformation(self.model.fullpose, self.model.J, self.model.kintree_table, ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
R = [g[:3, :3] for g in A_global]
coords_kp_xyz = get_keypoints_from_mesh_ch(self.model, Jtr)
coords_kp_xyz = coords_kp_xyz + self.global_trans
return coords_kp_xyz
def verts_core(pose, v, J, weights, kintree_table, want_Jtr=False):
A, A_global = global_rigid_transformation(pose, J, kintree_table)
T = A.dot(weights.T)
rest_shape_h = ch.vstack((v.T, np.ones((1, v.shape[0]))))
v = (T[:, 0, :] * rest_shape_h[0, :].reshape((1, -1)) +
T[:, 1, :] * rest_shape_h[1, :].reshape((1, -1)) +
T[:, 2, :] * rest_shape_h[2, :].reshape((1, -1)) +
T[:, 3, :] * rest_shape_h[3, :].reshape((1, -1))).T
v = v[:, :3]
if not want_Jtr:
return v
Jtr = ch.vstack([g[:3, 3] for g in A_global])
return (v, Jtr)
def mstack(vs, fs):
import chumpy as ch
import numpy as np
lengths = [v.shape[0] for v in vs]
f = np.vstack([fs[i]+np.sum(lengths[:i]).astype(np.uint32) for i in range(len(fs))])
v = ch.vstack(vs)
return v, f
def mstack(vs, fs):
import chumpy as ch
import numpy as np
lengths = [v.shape[0] for v in vs]
f = np.vstack([fs[i]+np.sum(lengths[:i]).astype(np.uint32) for i in range(len(fs))])
v = ch.vstack(vs)
return v, f
def guess_init(
model, # pylint: disable=too-many-locals, too-many-arguments
focal_length,
j2d,
weights,
init_pose,
use_gt_guess):
"""Initialize the camera depth using the torso points."""
cids = _np.arange(0, 12)
j2d_here = j2d[cids]
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# The initial pose is a t-pose with all body parts planar to the camera
# (i.e., they have their maximum length).
opt_pose = _ch.array(init_pose) # pylint: disable=no-member
(_, A_global) = _global_rigid_transformation(opt_pose,
model.J,
model.kintree_table,
xp=_ch)
Jtr = _ch.vstack([g[:3, 3] for g in A_global]) # pylint: disable=no-member
Jtr = Jtr[smpl_ids].r
if use_gt_guess:
# More robust estimate:
# Since there is no fore'lengthening', only foreshortening, take the longest
# visible body part as a robust estimate in the case of noise-free data.
longest_part = None
longest_part_length = -1.
for conn in connections_lsp:
if (conn[0] < 12 and conn[1] < 12 and weights[conn[0]] > 0.
and weights[conn[1]] > 0.):
part_length_proj_gt = _np.sqrt(
_np.sum(_np.array(j2d_here[conn[0]] -
j2d_here[conn[1]])**2,
axis=0))
if part_length_proj_gt > longest_part_length:
longest_part_length = part_length_proj_gt
longest_part = conn
#length2d = _np.sqrt(_np.sum(
# _np.array(j2d_here[longest_part[0]] - j2d_here[longest_part[1]])**2, axis=0))
part_length_3D = _np.sqrt(
_np.sum(_np.array(Jtr[longest_part[0]] - Jtr[longest_part[1]])**2,
axis=0))
est_d = focal_length * (part_length_3D / longest_part_length)
else:
diff3d = _np.array([Jtr[9] - Jtr[3], Jtr[8] - Jtr[2]])
mean_height3d = _np.mean(_np.sqrt(_np.sum(diff3d**2, axis=1)))
diff2d = _np.array(
[j2d_here[9] - j2d_here[3], j2d_here[8] - j2d_here[2]])
mean_height2d = _np.mean(_np.sqrt(_np.sum(diff2d**2, axis=1)))
f = focal_length # pylint: disable=redefined-outer-name
if mean_height2d == 0.:
_LOGGER.warn(
"Depth can not be correctly estimated. Guessing wildly.")
est_d = 60.
else:
est_d = f * (mean_height3d / mean_height2d)
init_t = _np.array([0., 0., est_d])
return init_t
def verts_decorated_quat(trans,
pose,
v_template,
J_regressor,
weights,
kintree_table,
f,
posedirs=None,
betas=None,
add_shape=True,
shapedirs=None,
want_Jtr=False):
for which in [
trans, pose, v_template, weights, posedirs, betas, shapedirs
]:
if which is not None:
assert ischumpy(which)
v = v_template
v_shaped = v + shapedirs.dot(betas) #Add Shape of the model.
quaternion_angles = axis2quat(pose.reshape((-1, 3))).reshape(-1)[4:]
shape_feat = betas[1]
feat = ch.concatenate([quaternion_angles, shape_feat], axis=0)
poseblends = posedirs.dot(feat)
v_posed = v_shaped + poseblends
J_tmpx = MatVecMult(J_regressor, v_shaped[:, 0])
J_tmpy = MatVecMult(J_regressor, v_shaped[:, 1])
J_tmpz = MatVecMult(J_regressor, v_shaped[:, 2])
J = chumpy.vstack((J_tmpx, J_tmpy, J_tmpz)).T
result, meta = verts_core(pose,
v,
J,
weights,
kintree_table,
want_Jtr=True)
Jtr = meta.Jtr if meta is not None else None
tr = trans.reshape((1, 3))
result = result + tr
Jtr = Jtr + tr
result.trans = trans
result.f = f
result.pose = pose
result.v_template = v_template
result.J = J
result.weights = weights
result.posedirs = posedirs
result.v_posed = v_posed
result.shapedirs = shapedirs
result.betas = betas
result.v_shaped = v_shaped
result.J_transformed = Jtr
return result
def ready_arguments(fname_or_dict,
posekey4vposed='pose',
shared_args=None,
chTrans=None,
chBetas=None):
import numpy as np
import pickle
import chumpy as ch
from chumpy.ch import MatVecMult
from mano.webuser.posemapper import posemap
if not isinstance(fname_or_dict, dict):
dd = pickle.load(open(fname_or_dict))
else:
dd = fname_or_dict
want_shapemodel = 'shapedirs' in dd
nposeparms = dd['kintree_table'].shape[1] * 3
if 'trans' not in dd:
dd['trans'] = np.zeros(3)
if 'pose' not in dd:
dd['pose'] = np.zeros(nposeparms)
if 'shapedirs' in dd and 'betas' not in dd:
dd['betas'] = np.zeros(dd['shapedirs'].shape[-1])
if chTrans is not None:
dd['trans'] = chTrans
if chTrans is not None:
dd['betas'] = chBetas
for s in [
'v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs',
'betas', 'J', 'fullpose', 'pose_dof'
]:
if (s in dd) and not hasattr(dd[s], 'dterms'):
if shared_args is not None and s in shared_args:
dd[s] = shared_args[s]
else:
dd[s] = ch.array(dd[s])
assert (posekey4vposed in dd)
if want_shapemodel:
dd['v_shaped'] = dd['shapedirs'].dot(dd['betas']) + dd['v_template']
v_shaped = dd['v_shaped']
J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:, 0])
J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:, 1])
J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:, 2])
dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
dd['v_posed'] = v_shaped + dd['posedirs'].dot(
posemap(dd['bs_type'])(dd[posekey4vposed]))
else:
dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(
posemap(dd['bs_type'])(dd[posekey4vposed]))
return dd
def smooth_rings(gar_rings, verts):
"""
:param gar_rings:
:param verts:
:returns
"""
error = ch.zeros([0, 3])
for ring in gar_rings:
N = len(ring)
aring = np.array(ring)
ring_0 = aring[np.arange(0, N)]
ring_m1 = aring[np.array([N - 1] + range(0, N - 1))]
ring_p1 = aring[np.array(range(1, N) + [0])]
err_ring = verts[ring_m1] - 2 * verts[ring_0] + verts[ring_p1]
error = ch.vstack([error, err_ring])
error = ch.vstack([error, err_ring])
return error
def _fit_rot_trans(
model, # pylint: disable=too-many-arguments, too-many-locals
j2d,
center,
init_t,
init_pose,
conf,
flength):
"""Find a rotation and translation to minimize the projection error of the pose."""
opt_pose = _ch.array(init_pose) # pylint: disable=no-member
opt_trans = _ch.array(init_t) # pylint: disable=no-member
(_, A_global) = _global_rigid_transformation(opt_pose,
model.J,
model.kintree_table,
xp=_ch)
Jtr = _ch.vstack([g[:3, 3] for g in A_global]) + opt_trans # pylint: disable=no-member
cam = _ProjectPoints(f=_np.array([flength, flength]),
rt=_np.zeros(3),
t=_np.zeros(3),
k=_np.zeros(5),
c=center)
cam_3d = _ProjectPoints3D(f=_np.array([flength, flength]),
rt=_np.zeros(3),
t=_np.zeros(3),
k=_np.zeros(5),
c=center)
cam.v = Jtr
cam_3d.v = Jtr
free_variables = [opt_pose[:3],
opt_trans] # Optimize global rotation and translation.
j2d_ids = range(12)
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
#_CID_TORSO = [2, 3, 8, 9]
#torso_smpl_ids = [2, 1, 17, 16]
_ch.minimize(
{
'j2d': (j2d.T[j2d_ids] - cam[smpl_ids]).T * conf[j2d_ids],
# 'dev^2': 1e2 * (opt_trans[2] - init_t[2])
},
x0=free_variables,
method='dogleg',
callback=None, #on_step_vis,
options={
'maxiter': 100,
'e_3': .0001,
'disp': 0
})
_LOGGER.debug("Global rotation: %s, global translation: %s.",
str(opt_pose[:3].r), str(opt_trans.r))
_LOGGER.debug("Points 3D: %s.", str(cam_3d.r[:10, :]))
_LOGGER.debug("Points 2D: %s.", str(cam.r[:10, :]))
return opt_pose[:3].r, opt_trans.r, cam_3d.r[:,
2].min(), cam_3d.r[:,
2].max()
def SecondFundamentalForm(v, f):
from chumpy import hstack, vstack
from chumpy.linalg import Pinv
nbrs = MatVecMult(FirstEdgesMtx(v, f, want_big=True), v.ravel()).reshape((-1,3))
b0 = NormalizedNx3(VertNormalsScaled(f=f, v=v)).reshape((-1,3))
b1 = NormalizedNx3(CrossProduct(b0, nbrs-v)).reshape((-1,3))
b2 = NormalizedNx3(CrossProduct(b0, b1)).reshape((-1,3))
cnct = get_vert_connectivity(v.r, f)
ffs = []
for i in range(v.size/3):
nbrs = v[np.nonzero(np.asarray(cnct[i].todense()).ravel())[0]] - row(v[i])
us = nbrs.dot(b2[i])
vs = nbrs.dot(b1[i])
hs = nbrs.dot(b0[i])
coeffs = Pinv(hstack((col((us*.5)**2), col(us*vs), col((vs*.5)**2)))).dot(hs)
ffs.append(row(coeffs))
# if i == 3586:
# import pdb; pdb.set_trace()
ffs = vstack(ffs)
return ffs
def get_capsules(model, wrt_betas=None, length_regs=None, rad_regs=None):
from opendr.geometry import Rodrigues
if length_regs is not None:
n_shape_dofs = length_regs.shape[0] - 1
else:
n_shape_dofs = model.betas.r.size
segm = np.argmax(model.weights_prior, axis=1)
J_off = ch.zeros((len(joint2name), 3))
rots = rots0.copy()
mujoco_t_mid = [0, 3, 6, 9]
if wrt_betas is not None:
# if we want to differentiate wrt betas (shape), we must have the
# regressors...
assert (length_regs is not None and rad_regs is not None)
# ... and betas must be a chumpy object
assert (hasattr(wrt_betas, 'dterms'))
pad = ch.concatenate(
(wrt_betas, ch.zeros(n_shape_dofs - len(wrt_betas)), ch.ones(1)))
lengths = pad.dot(length_regs)
rads = pad.dot(rad_regs)
else:
lengths = ch.ones(len(joint2name))
rads = ch.ones(len(joint2name))
betas = wrt_betas if wrt_betas is not None else model.betas
n_betas = len(betas)
# the joint regressors are the original, pre-optimized ones
# (middle of the part frontier)
myJ_regressor = model.J_regressor_prior
myJ0 = ch.vstack(
(ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 0] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 0]),
ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 1] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 1]),
ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 2] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 2]))).T
# with small adjustments for hips, spine and feet
myJ = ch.vstack(
[ch.concatenate([myJ0[0, 0], (
.6 * myJ0[0, 1] + .2 * myJ0[1, 1] + .2 * myJ0[2, 1]), myJ0[9, 2]]),
ch.vstack([myJ0[i] for i in range(1, 7)]), ch.concatenate(
[myJ0[7, 0], (1.1 * myJ0[7, 1] - .1 * myJ0[4, 1]), myJ0[7, 2]]),
ch.concatenate(
[myJ0[8, 0], (1.1 * myJ0[8, 1] - .1 * myJ0[5, 1]), myJ0[8, 2]]),
ch.concatenate(
[myJ0[9, 0], myJ0[9, 1], (.2 * myJ0[9, 2] + .8 * myJ0[12, 2])]),
ch.vstack([myJ0[i] for i in range(10, 24)])])
capsules = []
# create one capsule per mujoco joint
for ijoint, segms in enumerate(mujoco2segm):
if wrt_betas is None:
vidxs = np.asarray([segm == k for k in segms]).any(axis=0)
verts = model.v_template[vidxs].r
dims = (verts.max(axis=0) - verts.min(axis=0))
rads[ijoint] = .5 * ((dims[(np.argmax(dims) + 1) % 3] + dims[(
np.argmax(dims) + 2) % 3]) / 4.)
lengths[ijoint] = max(dims) - 2. * rads[ijoint].r
# the core joints are different, since the capsule is not in the joint
# but in the middle
if ijoint in mujoco_t_mid:
len_offset = ch.vstack([ch.zeros(1), ch.abs(lengths[ijoint]) / 2.,
ch.zeros(1)]).reshape(3, 1)
caps = Capsule(
(J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(
3, 1) - Rodrigues(rots[ijoint]).dot(len_offset),
rots[ijoint], rads[ijoint], lengths[ijoint])
else:
caps = Capsule(
(J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(3, 1),
rots[ijoint], rads[ijoint], lengths[ijoint])
caps.id = ijoint
capsules.append(caps)
return capsules
def optimize_on_joints(j2d,
model,
cam,
img,
prior,
try_both_orient,
body_orient,
n_betas=10,
regs=None,
conf=None,
viz=False):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit
:param body_orient: 3D vector, initialization for the body orientation
:param n_betas: number of shape coefficients considered during optimization
:param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term
:param conf: 14D vector storing the confidence values from the CNN
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the optimized model, its joints projected on image space, the camera translation
"""
t0 = time()
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
cids = range(12) + [13]
# joint ids for SMPL
# SMPL does not have a joint for head, instead we use a vertex for the head
# and append it later.
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# the vertex id for the joint corresponding to the head
head_id = 411
# weights assigned to each joint during optimization;
# the definition of hips in SMPL and LSP is significantly different so set
# their weights to zero
base_weights = np.array(
[1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=np.float64)
if try_both_orient:
flipped_orient = cv2.Rodrigues(body_orient)[0].dot(
cv2.Rodrigues(np.array([0., np.pi, 0]))[0])
flipped_orient = cv2.Rodrigues(flipped_orient)[0].ravel()
orientations = [body_orient, flipped_orient]
else:
orientations = [body_orient]
if try_both_orient:
# store here the final error for both orientations,
# and pick the orientation resulting in the lowest error
errors = []
svs = []
cams = []
for o_id, orient in enumerate(orientations):
# initialize the shape to the mean shape in the SMPL training set
betas = ch.zeros(n_betas)
# initialize the pose by using the optimized body orientation and the
# pose prior
init_pose = np.hstack((orient, prior.weights.dot(prior.means)))
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(
trans=ch.zeros(3),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# make the SMPL joints depend on betas
Jdirs = np.dstack([model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))])
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = global_rigid_transformation(
sv.pose, J_onbetas, model.kintree_table, xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
# add the head joint, corresponding to a vertex...
Jtr = ch.vstack((Jtr, sv[head_id]))
# ... and add the joint id to the list
if o_id == 0:
smpl_ids.append(len(Jtr) - 1)
# update the weights using confidence values
weights = base_weights * conf[
cids] if conf is not None else base_weights
# project SMPL joints on the image plane using the estimated camera
cam.v = Jtr
# data term: distance between observed and estimated joints in 2D
obj_j2d = lambda w, sigma: (
w * weights.reshape((-1, 1)) * GMOf((j2d[cids] - cam[smpl_ids]), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([my_exp(sv.pose[55]), my_exp(-sv.pose[
58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15])])
if viz:
import matplotlib.pyplot as plt
plt.ion()
def on_step(_):
"""Create visualization."""
plt.figure(1, figsize=(10, 10))
plt.subplot(1, 2, 1)
# show optimized joints in 2D
tmp_img = img.copy()
for coord, target_coord in zip(
np.around(cam.r[smpl_ids]).astype(int),
np.around(j2d[cids]).astype(int)):
if (coord[0] < tmp_img.shape[1] and coord[0] >= 0 and
coord[1] < tmp_img.shape[0] and coord[1] >= 0):
cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255])
if (target_coord[0] < tmp_img.shape[1] and
target_coord[0] >= 0 and
target_coord[1] < tmp_img.shape[0] and
target_coord[1] >= 0):
cv2.circle(tmp_img, tuple(target_coord), 3,
[0, 255, 0])
plt.imshow(tmp_img[:, :, ::-1])
plt.draw()
plt.show()
plt.pause(1e-2)
on_step(_)
else:
on_step = None
if regs is not None:
# interpenetration term
sp = SphereCollisions(
pose=sv.pose, betas=sv.betas, model=model, regs=regs)
sp.no_hands = True
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
if regs is not None:
objs['sph_coll'] = 1e3 * sp
ch.minimize(
objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=on_step,
options={'maxiter': 100,
'e_3': .0001,
'disp': 0})
t1 = time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if try_both_orient:
errors.append((objs['j2d'].r**2).sum())
svs.append(sv)
cams.append(cam)
if try_both_orient and errors[0] > errors[1]:
choose_id = 1
else:
choose_id = 0
if viz:
plt.ioff()
return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r)
def test_occlusion(self):
if visualize:
import matplotlib.pyplot as plt
plt.ion()
# Create renderer
import chumpy as ch
import numpy as np
from opendr.renderer import TexturedRenderer, ColoredRenderer
#rn = TexturedRenderer()
rn = ColoredRenderer()
# Assign attributes to renderer
from util_tests import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
rn.texture_image = m.texture_image
rn.ft = m.ft
rn.vt = m.vt
m.v[:,2] = np.mean(m.v[:,2])
# red is front and zero
# green is back and 1
t0 = ch.array([1,0,.1])
t1 = ch.array([-1,0,.1])
v0 = ch.array(m.v) + t0
if False:
v1 = ch.array(m.v*.4 + np.array([0,0,3.8])) + t1
else:
v1 = ch.array(m.v) + t1
vc0 = v0*0 + np.array([[.4,0,0]])
vc1 = v1*0 + np.array([[0,.4,0]])
vc = ch.vstack((vc0, vc1))
v = ch.vstack((v0, v1))
f = np.vstack((m.f, m.f+len(v0)))
w, h = (320, 240)
rn.camera = ProjectPoints(v=v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.camera.t = ch.array([0,0,-2.5])
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
m.vc = v.r*0 + np.array([[1,0,0]])
rn.set(v=v, f=f, vc=vc)
t0[:] = np.array([1.4, 0, .1-.02])
t1[:] = np.array([-0.6, 0, .1+.02])
target = rn.r
if visualize:
plt.figure()
plt.imshow(target)
plt.title('target')
plt.figure()
plt.show()
im_orig = rn.r.copy()
from cvwrap import cv2
tr = t0
eps_emp = .02
eps_pred = .02
#blur = lambda x : cv2.blur(x, ksize=(5,5))
blur = lambda x : x
for tr in [t0, t1]:
if tr is t0:
sum_limits = np.array([2.1e+2, 6.9e+1, 1.6e+2])
else:
sum_limits = [1., 5., 4.]
if visualize:
plt.figure()
for i in range(3):
dr_pred = np.array(rn.dr_wrt(tr[i]).todense()).reshape(rn.shape) * eps_pred
dr_pred = blur(dr_pred)
# central differences
tr[i] = tr[i].r + eps_emp/2.
rn_greater = rn.r.copy()
tr[i] = tr[i].r - eps_emp/1.
rn_lesser = rn.r.copy()
tr[i] = tr[i].r + eps_emp/2.
dr_emp = blur((rn_greater - rn_lesser) * eps_pred / eps_emp)
dr_pred_shown = np.clip(dr_pred, -.5, .5) + .5
dr_emp_shown = np.clip(dr_emp, -.5, .5) + .5
if visualize:
plt.subplot(3,3,i+1)
plt.imshow(dr_pred_shown)
plt.title('pred')
plt.axis('off')
plt.subplot(3,3,3+i+1)
plt.imshow(dr_emp_shown)
plt.title('empirical')
plt.axis('off')
plt.subplot(3,3,6+i+1)
diff = np.abs(dr_emp - dr_pred)
if visualize:
plt.imshow(diff)
diff = diff.ravel()
if visualize:
plt.title('diff (sum: %.2e)' % (np.sum(diff)))
plt.axis('off')
# print 'dr pred sum: %.2e' % (np.sum(np.abs(dr_pred.ravel())),)
# print 'dr emp sum: %.2e' % (np.sum(np.abs(dr_emp.ravel())),)
#import pdb; pdb.set_trace()
self.assertTrue(np.sum(diff) < sum_limits[i])
def initialize_camera(model,
j2d,
img,
init_pose,
flength=5000.,
pix_thsh=25.,
viz=False):
"""Initialize camera translation and body orientation
:param model: SMPL model
:param j2d: 14x2 array of CNN joints
:param img: h x w x 3 image
:param init_pose: 72D vector of pose parameters used for initialization
:param flength: camera focal length (kept fixed)
:param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D
is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both
the estimated one and its flip)
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the estimated camera,
a boolean deciding if both the optimized body orientation and its flip should be considered,
3D vector for the body orientation
"""
# optimize camera translation and body orientation based on torso joints
# LSP torso ids:
# 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder
torso_cids = [2, 3, 8, 9]
# corresponding SMPL torso ids
torso_smpl_ids = [2, 1, 17, 16]
center = np.array([img.shape[1] / 2, img.shape[0] / 2])
# initialize camera rotation
rt = ch.zeros(3)
# initialize camera translation
_LOGGER.info('initializing translation via similar triangles')
init_t = guess_init(model, flength, j2d, init_pose)
t = ch.array(init_t)
# check how close the shoulder joints are
try_both_orient = np.linalg.norm(j2d[8] - j2d[9]) < pix_thsh
opt_pose = ch.array(init_pose)
(_, A_global) = global_rigid_transformation(
opt_pose, model.J, model.kintree_table, xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
# initialize the camera
cam = ProjectPoints(
f=np.array([flength, flength]), rt=rt, t=t, k=np.zeros(5), c=center)
# we are going to project the SMPL joints
cam.v = Jtr
if viz:
viz_img = img.copy()
# draw the target (CNN) joints
for coord in np.around(j2d).astype(int):
if (coord[0] < img.shape[1] and coord[0] >= 0 and
coord[1] < img.shape[0] and coord[1] >= 0):
cv2.circle(viz_img, tuple(coord), 3, [0, 255, 0])
import matplotlib.pyplot as plt
plt.ion()
# draw optimized joints at each iteration
def on_step(_):
"""Draw a visualization."""
plt.figure(1, figsize=(5, 5))
plt.subplot(1, 1, 1)
viz_img = img.copy()
for coord in np.around(cam.r[torso_smpl_ids]).astype(int):
if (coord[0] < viz_img.shape[1] and coord[0] >= 0 and
coord[1] < viz_img.shape[0] and coord[1] >= 0):
cv2.circle(viz_img, tuple(coord), 3, [0, 0, 255])
plt.imshow(viz_img[:, :, ::-1])
plt.draw()
plt.show()
plt.pause(1e-3)
else:
on_step = None
# optimize for camera translation and body orientation
free_variables = [cam.t, opt_pose[:3]]
ch.minimize(
# data term defined over torso joints...
{'cam': j2d[torso_cids] - cam[torso_smpl_ids],
# ...plus a regularizer for the camera translation
'cam_t': 1e2 * (cam.t[2] - init_t[2])},
x0=free_variables,
method='dogleg',
callback=on_step,
options={'maxiter': 100,
'e_3': .0001,
# disp set to 1 enables verbose output from the optimizer
'disp': 0})
if viz:
plt.ioff()
return (cam, try_both_orient, opt_pose[:3].r)
