def test_nested_concatenate(self):
aa = ch.arange(3)
bb = ch.arange(4)
cc = ch.arange(5)
result = ch.concatenate((ch.concatenate((aa, bb)), cc))
self.assertTrue(result.m0 is aa)
self.assertTrue(result.m1 is bb)
self.assertTrue(result.m2 is cc)
self.assertTrue(result.dr_wrt(aa).nnz > 0)
self.assertTrue(result.dr_wrt(bb).nnz > 0)
self.assertTrue(result.dr_wrt(cc).nnz > 0)
def laplacian_pyramid(input_objective, imshape, normalization, n_levels, as_list):
if normalization is None:
norm2 = lambda x : x
elif normalization is 'SSE':
norm2 = lambda x : x / np.sqrt(np.sum(x.r**2.))
elif normalization is 'size':
norm2 = lambda x : x / x.r.size
else:
norm2 = normalization
output_objs = []
for level in range(n_levels):
blur_mtx = filter_for(imshape[0], imshape[1], imshape[2] if len(imshape)>2 else 1, kernel = GaussianKernel2D(3, 1))
blurred = MatVecMult(blur_mtx, input_objective).reshape(imshape)
output_objs.append(norm2(input_objective - blurred))
halfsampler_mtx, imshape = halfsampler_for(imshape)
input_objective = MatVecMult(halfsampler_mtx, blurred.ravel()).reshape(imshape)
output_objs.append(norm2(input_objective).reshape(imshape))
return output_objs if as_list else reduce(lambda x, y : ch.concatenate((x.ravel(), y.ravel())), output_objs)
def chZonalHarmonics(a):
zl0 = -ch.sqrt(ch.pi) * (-1.0 + ch.cos(a))
zl1 = 0.5 * ch.sqrt(3.0 * ch.pi) * ch.sin(a)**2
zl2 = -0.5 * ch.sqrt(
5.0 * ch.pi) * ch.cos(a) * (-1.0 + ch.cos(a)) * (ch.cos(a) + 1.0)
z = [zl0, zl1, zl2]
return ch.concatenate(z)
def obj_angle(w):
return w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
def gaussian_pyramid(input_objective, imshape=None, normalization='SSE', n_levels=3, as_list=False, label=None):
if imshape is None:
imshape = input_objective.shape
if normalization is None:
norm2 = lambda x : x
elif normalization is 'SSE':
norm2 = lambda x : x / np.sqrt(np.sum(x.r**2.))
elif normalization is 'size':
norm2 = lambda x : x / x.r.size
else:
norm2 = normalization
cur_imshape = deepcopy(imshape)
cur_obj = input_objective
input_objective = norm2(input_objective)
output_objectives = [input_objective]
for ik in range(n_levels):
cur_obj = GaussPyrDownOne(px=cur_obj, im_shape = cur_imshape)
cur_imshape = cur_obj.output_shape
output_objectives.append(norm2(cur_obj) if label is None else norm2(cur_obj) >> '%s%d' % (label,ik))
if not as_list:
andit = lambda a : reduce(lambda x, y : ch.concatenate((x.ravel(), y.ravel())), a)
output_objectives = andit(output_objectives)
return output_objectives
def global_rigid_transformation(pose, J, kintree_table):
def _rodrigues(x):
return Rodrigues(x)
results = {}
pose = pose.reshape((-1, 3))
id_to_col = {kintree_table[1, i]: i for i in range(kintree_table.shape[1])}
parent = {i: id_to_col[kintree_table[0, i]]
for i in range(1, kintree_table.shape[1])}
results[0] = with_zeros(
ch.hstack((_rodrigues(pose[0, :]), J[0, :].reshape((3, 1)))))
for i in range(1, kintree_table.shape[1]):
results[i] = results[parent[i]].dot(
with_zeros(ch.hstack((_rodrigues(pose[i, :]),
((J[i, :] - J[parent[i], :]).reshape((3, 1)))
))))
results = [results[i] for i in sorted(results.keys())]
results_global = results
results2 = [results[i] - (pack(
results[i].dot(ch.concatenate(((J[i, :]), 0))))
) for i in range(len(results))]
results = results2
result = ch.dstack(results)
return result, results_global
def _global_rigid_transformation(self):
results = {}
pose = self.pose.reshape((-1, 3))
parent = {
i: self.kintree_table[0, i]
for i in range(1, self.kintree_table.shape[1])
}
with_zeros = lambda x: ch.vstack((x, ch.array([[0.0, 0.0, 0.0, 1.0]])))
pack = lambda x: ch.hstack([ch.zeros((4, 3)), x.reshape((4, 1))])
results[0] = with_zeros(
ch.hstack((Rodrigues(pose[0, :]), self.J[0, :].reshape((3, 1)))))
for i in range(1, self.kintree_table.shape[1]):
results[i] = results[parent[i]].dot(
with_zeros(
ch.hstack((
Rodrigues(pose[i, :]), # rotation around bone endpoint
(self.J[i, :] - self.J[parent[i], :]).reshape(
(3, 1)) # bone
))))
results = [results[i] for i in sorted(results.keys())]
results_global = results
# subtract rotated J position
results2 = [
results[i] -
(pack(results[i].dot(ch.concatenate((self.J[i, :], [0])))))
for i in range(len(results))
]
result = ch.dstack(results2)
return result, results_global
def lrotmin(p):
if isinstance(p, np.ndarray):
p = p.ravel()[3:]
return np.concatenate([(cv2.Rodrigues(np.array(pp))[0]-np.eye(3)).ravel() for pp in p.reshape((-1,3))]).ravel()
if p.ndim != 2 or p.shape[1] != 3:
p = p.reshape((-1,3))
p = p[1:]
return ch.concatenate([(Rodrigues(pp)-ch.eye(3)).ravel() for pp in p]).ravel()
def lrotmin(p):
if isinstance(p, np.ndarray):
p = p.ravel()[3:]
return np.concatenate([(cv2.Rodrigues(np.array(pp))[0] - np.eye(3)).ravel() for pp in p.reshape((-1, 3))]).ravel()
if p.ndim != 2 or p.shape[1] != 3:
p = p.reshape((-1, 3))
p = p[1:]
return ch.concatenate([(Rodrigues(pp) - ch.eye(3)).ravel() for pp in p]).ravel()
def verts_decorated_quat(trans,
pose,
v_template,
J_regressor,
weights,
kintree_table,
f,
posedirs=None,
betas=None,
add_shape=True,
shapedirs=None,
want_Jtr=False):
for which in [
trans, pose, v_template, weights, posedirs, betas, shapedirs
]:
if which is not None:
assert ischumpy(which)
v = v_template
v_shaped = v + shapedirs.dot(betas) #Add Shape of the model.
quaternion_angles = axis2quat(pose.reshape((-1, 3))).reshape(-1)[4:]
shape_feat = betas[1]
feat = ch.concatenate([quaternion_angles, shape_feat], axis=0)
poseblends = posedirs.dot(feat)
v_posed = v_shaped + poseblends
J_tmpx = MatVecMult(J_regressor, v_shaped[:, 0])
J_tmpy = MatVecMult(J_regressor, v_shaped[:, 1])
J_tmpz = MatVecMult(J_regressor, v_shaped[:, 2])
J = chumpy.vstack((J_tmpx, J_tmpy, J_tmpz)).T
result, meta = verts_core(pose,
v,
J,
weights,
kintree_table,
want_Jtr=True)
Jtr = meta.Jtr if meta is not None else None
tr = trans.reshape((1, 3))
result = result + tr
Jtr = Jtr + tr
result.trans = trans
result.f = f
result.pose = pose
result.v_template = v_template
result.J = J
result.weights = weights
result.posedirs = posedirs
result.v_posed = v_posed
result.shapedirs = shapedirs
result.betas = betas
result.v_shaped = v_shaped
result.J_transformed = Jtr
return result
def capsule_dist(capsule0, capsule1, alpha=.3, increase_hand=True):
range0 = range(capsule0.center_id,
capsule0.center_id + len(capsule0.centers))
range1 = range(capsule1.center_id,
capsule1.center_id + len(capsule1.centers))
cnt0 = ch.concatenate([[cid] * len(range1) for cid in range0])
cnt1 = ch.concatenate([range1] * len(range0))
if increase_hand:
if (capsule0.id == 18) or (capsule0.id == 19) or (
capsule1.id == 18) or (capsule1.id == 19):
dst = (alpha * 1.2 * capsule0.rad.r)**2 + (alpha * 1.2 *
capsule1.rad.r)**2
else:
dst = (alpha * capsule0.rad.r)**2 + (alpha * capsule1.rad.r)**2
else:
dst = (alpha * capsule0.rad.r)**2 + (alpha * capsule1.rad.r)**2
radiuss = np.hstack([dst] * len(cnt0)).squeeze()
return (cnt0, cnt1, radiuss)
def capsule_dist(capsule0, capsule1, alpha=.3, increase_hand=True):
range0 = range(capsule0.center_id,
capsule0.center_id + len(capsule0.centers))
range1 = range(capsule1.center_id,
capsule1.center_id + len(capsule1.centers))
cnt0 = ch.concatenate([[cid] * len(range1) for cid in range0])
cnt1 = ch.concatenate([range1] * len(range0))
if increase_hand:
if (capsule0.id == 18) or (capsule0.id == 19) or (
capsule1.id == 18) or (capsule1.id == 19):
dst = (alpha * 1.2 * capsule0.rad.r)**2 + (alpha * 1.2 *
capsule1.rad.r)**2
else:
dst = (alpha * capsule0.rad.r)**2 + (alpha * capsule1.rad.r)**2
else:
dst = (alpha * capsule0.rad.r)**2 + (alpha * capsule1.rad.r)**2
radiuss = np.hstack([dst] * len(cnt0)).squeeze()
return (cnt0, cnt1, radiuss)
def chZonalToSphericalHarmonics(z, theta, phi):
sphCoeffs = []
for l in np.arange(len(z)):
for m in np.arange(np.int(-(l * 2 + 1) / 2),
np.int((l * 2 + 1) / 2) + 1):
ylm_d = chSpherical_harmonics[(l, m)](theta, phi)
sh = np.sqrt(4 * np.pi / (2 * l + 1)) * z[l] * ylm_d
sphCoeffs = sphCoeffs + [sh]
#Correct order in band l=1.
sphCoeffs[1], sphCoeffs[3] = sphCoeffs[3], sphCoeffs[1]
chSphCoeffs = ch.concatenate(sphCoeffs)
return chSphCoeffs
def axis2quat(p):
angle = ch.sqrt(ch.clip(ch.sum(ch.square(p), 1), 1e-16, 1e16))
norm_p = p / angle[:, np.newaxis]
cos_angle = ch.cos(angle / 2)
sin_angle = ch.sin(angle / 2)
qx = norm_p[:, 0] * sin_angle
qy = norm_p[:, 1] * sin_angle
qz = norm_p[:, 2] * sin_angle
qw = cos_angle - 1
return ch.concatenate([
qx[:, np.newaxis], qy[:, np.newaxis], qz[:, np.newaxis], qw[:,
np.newaxis]
],
axis=1)
def getHandModel():
globalJoints = ch.zeros((45, ))
globalBeta = ch.zeros((10, ))
chRot = ch.zeros((3, ))
chTrans = ch.array([0., 0., 0.5])
fullpose = ch.concatenate([chRot, globalJoints], axis=0)
m = load_model_withInputs(os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'../mano/models/MANO_RIGHT.pkl'),
fullpose,
chTrans,
globalBeta,
ncomps=15,
flat_hand_mean=True)
return m, chRot, globalJoints, chTrans, globalBeta
def global_rigid_transformation(pose, J, kintree_table):
results = {}
pose = pose.reshape((-1, 3))
id_to_col = {kintree_table[1, i]: i for i in range(kintree_table.shape[1])}
parent = {
i: id_to_col[kintree_table[0, i]]
for i in range(1, kintree_table.shape[1])
}
def with_zeros(x):
return ch.vstack((x, ch.array([[0.0, 0.0, 0.0, 1.0]])))
results[0] = with_zeros(
ch.hstack((Rodrigues(pose[0, :]), J[0, :].reshape((3, 1)))))
for i in range(1, kintree_table.shape[1]):
results[i] = results[parent[i]].dot(
with_zeros(
ch.hstack((Rodrigues(pose[i, :]),
((J[i, :] - J[parent[i], :]).reshape((3, 1)))))))
def pack(x):
return ch.hstack([np.zeros((4, 3)), x.reshape((4, 1))])
results = [results[i] for i in sorted(results.keys())]
results_global = results
if True:
results2 = [
results[i] - pack(results[i].dot(ch.concatenate(((J[i, :]), 0))))
for i in range(len(results))
]
results = results2
result = ch.dstack(results)
return result, results_global
def compute_r(self):
min_w = self.weights[self.min_component_idx]
# Add the sqrt(-log(weights))
return ch.concatenate((self.loglikelihoods[self.min_component_idx].r,
np.sqrt(-np.log(min_w))))
def optimize_on_joints(j2d,
model,
cam,
img,
prior,
try_both_orient,
body_orient,
exp_logistic,
n_betas=10,
inner_penetration=False,
silh=None,
conf=None,
viz=False):
"""Run the optimization."""
if silh is not None:
raise NotImplementedError("Silhouette fitting is not supported in "
"this code release due to dependencies on "
"proprietary code for the "
"distance computation.")
t0 = _time()
# define the mapping LSP joints -> SMPL joints
if j2d.shape[0] == 14:
cids = range(12) + [13]
elif j2d.shape[0] == 91:
cids = range(j2d.shape[0])
else:
raise Exception("Unknown number of joints: %d! Mapping not defined!" %
j2d.shape[0])
# joint ids for SMPL
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# weight given to each joint during optimization;
if j2d.shape[0] == 14:
weights = [1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]
else:
weights = [1] * (len(smpl_ids) + len(landmark_mesh_91))
# The non-skeleton vertex ids are added later.
if try_both_orient:
flipped_orient = _cv2.Rodrigues(body_orient)[0].dot(
_cv2.Rodrigues(_np.array([0., _np.pi, 0]))[0])
flipped_orient = _cv2.Rodrigues(flipped_orient)[0].ravel()
orientations = [body_orient, flipped_orient]
else:
orientations = [body_orient]
if try_both_orient:
errors = []
svs = []
cams = []
# rends = []
for o_id, orient in enumerate(orientations):
# initialize betas
betas = _ch.zeros(n_betas) # pylint: disable=no-member
init_pose = _np.hstack((orient, prior.weights.dot(prior.means)))
# 2D joint error term
# make the SMPL joint depend on betas
Jdirs = _np.dstack([
model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))
])
# pylint: disable=no-member
J_onbetas = _ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# instantiate the model
sv = verts_decorated(trans=_ch.zeros(3),
pose=_ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = _global_rigid_transformation(sv.pose,
J_onbetas,
model.kintree_table,
xp=_ch)
Jtr = _ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
if j2d.shape[0] == 14:
# add the "fake" joint for the head
head_id = _HEAD_REGR[0]
Jtr = _ch.vstack((Jtr, sv[head_id]))
if o_id == 0:
smpl_ids.append(len(Jtr) - 1)
else:
# add the plain vertex IDs on the mesh surface.
for vertex_id in landmark_mesh_91.values():
Jtr = _ch.vstack((Jtr, sv[vertex_id]))
# add the joint id
# for SMPL it's the last one added
if o_id == 0:
smpl_ids.append(len(Jtr) - 1)
weights = _np.array(weights, dtype=_np.float64)
if conf is not None:
weights *= conf[cids]
# we'll project the joints on the image plane
cam.v = Jtr
# data term: difference between observed and estimated joints
obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * _GMOf(
(j2d[cids] - cam[smpl_ids]), sigma))
# pose prior
pprior = lambda w: w * prior(sv.pose) # pylint: disable=cell-var-from-loop
# joint angles prior
# 55: left elbow, should bend -np.pi/2
# 58: right elbow, should bend np.pi/2
# 12: left knee, should bend np.pi/2
# 15: right knee, should bend np.pi/2
if exp_logistic:
_LOGGER.info('USING LOGISTIC')
# Skinny Logistic function. as 50-> inf we get a step function at
# 0.1. (0.1) is a margin bc 0 is still ok.
my_exp = lambda x: 1 / (1 + _ch.exp(100 * (0.1 + -x)))
else:
x_0 = 0 #10
alpha = 10
my_exp = lambda x: alpha * _ch.exp((x - x_0)) # pylint: disable=cell-var-from-loop
obj_angle = lambda w: w * _ch.concatenate([
my_exp(sv.pose[55]), # pylint: disable=cell-var-from-loop
my_exp(-sv.pose[58]), # pylint: disable=cell-var-from-loop
my_exp(-sv.pose[12]), # pylint: disable=cell-var-from-loop
my_exp(-sv.pose[15])
]) # pylint: disable=cell-var-from-loop
if viz:
from body.mesh.sphere import Sphere
from body.mesh.meshviewer import MeshViewer
import matplotlib.pyplot as plt
# set up visualization
# openGL window
mv = MeshViewer(window_width=120, window_height=120)
# and ids
show_ids = _np.array(smpl_ids)[weights > 0]
vc = _np.ones((len(Jtr), 3))
vc[show_ids] = [0, 1, 0]
plt.ion()
def on_step(_):
"""Create visualization."""
# show optimized joints in 3D
# pylint: disable=cell-var-from-loop
mv.set_dynamic_meshes([_Mesh(v=sv.r, f=[]),
Sphere(center=cam.t.r,
radius=.1).to_mesh()] \
+ [Sphere(center=jc, radius=.01).to_mesh(vc[ijc])
for ijc, jc in enumerate(Jtr.r)])
plt.figure(1, figsize=(10, 10))
plt.subplot(1, 2, 1)
# show optimized joints in 2D
tmp_img = img.copy()
for coord, target_coord in zip(
_np.around(cam.r[smpl_ids]).astype(int),
_np.around(j2d[cids]).astype(int)):
if (coord[0] < tmp_img.shape[1] and coord[0] >= 0
and coord[1] < tmp_img.shape[0] and coord[1] >= 0):
_cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255])
if (target_coord[0] < tmp_img.shape[1]
and target_coord[0] >= 0
and target_coord[1] < tmp_img.shape[0]
and target_coord[1] >= 0):
_cv2.circle(tmp_img, tuple(target_coord), 3,
[0, 255, 0])
plt.imshow(tmp_img)
plt.draw()
plt.show()
on_step(_)
else:
on_step = None
sp = _SphereCollisions(pose=sv.pose,
betas=sv.betas,
model=model,
regs=_REGRESSORS)
sp.no_hands = True
# configuration used with conf joints
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
#if stage < 2:
objs['j2d'] = obj_j2d(1., 100) # TODO: evaluate.
objs['pose'] = pprior(w)
# WEIGHT FOR ANGLE
if exp_logistic:
# Set to high weight always.
objs['pose_exp'] = obj_angle(5 * 1e3)
else:
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
if inner_penetration:
objs['sph_coll'] = 1e3 * sp
try:
_ch.minimize(objs.values(),
x0=[sv.betas, sv.pose],
method='dogleg',
callback=on_step,
options={
'maxiter': 100,
'e_3': .0001,
'disp': 0
})
except AssertionError:
# Divergence detected.
_LOGGER.warn("Diverging optimization! Breaking!")
break
t1 = _time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if try_both_orient:
errors.append((objs['j2d'].r**2).sum())
svs.append(sv)
cams.append(cam)
# rends.append(rend)
if try_both_orient and errors[0] > errors[1]:
choose_id = 1
else:
choose_id = 0
if viz:
plt.ioff()
return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r,
cams[choose_id].rt.r)
def setupTexturedRenderer(renderer,
vstack,
vch,
f_list,
vc_list,
vnch,
uv,
haveTextures_list,
textures_list,
camera,
frustum,
sharedWin=None):
f = []
f_listflat = [item for sublist in f_list for item in sublist]
lenMeshes = 0
for mesh_i, mesh in enumerate(f_listflat):
polygonLen = 0
for polygons in mesh:
f = f + [polygons + lenMeshes]
polygonLen += len(polygons)
lenMeshes += len(vch[mesh_i])
fstack = np.vstack(f)
if len(vnch) == 1:
vnstack = vnch[0]
else:
vnstack = ch.vstack(vnch)
if len(vc_list) == 1:
vcstack = vc_list[0]
else:
vcstack = ch.vstack(vc_list)
uvflat = [item for sublist in uv for item in sublist]
ftstack = np.vstack(uvflat)
texturesch = []
textures_listflat = [item for sublist in textures_list for item in sublist]
# import ipdb; ipdb.set_trace()
for texture_list in textures_listflat:
if texture_list != None:
for texture in texture_list:
if isinstance(texture, np.ndarray):
texturesch = texturesch + [ch.array(texture)]
elif texture != None:
texturesch = texturesch + [ch.array(texture)]
if len(texturesch) == 0:
texture_stack = ch.Ch([])
elif len(texturesch) == 1:
texture_stack = texturesch[0].ravel()
else:
texture_stack = ch.concatenate([tex.ravel() for tex in texturesch])
haveTextures_listflat = [
item for sublist in haveTextures_list for item in sublist
]
renderer.set(camera=camera,
frustum=frustum,
v=vstack,
f=fstack,
vn=vnstack,
vc=vcstack,
ft=ftstack,
texture_stack=texture_stack,
v_list=vch,
f_list=f_listflat,
vc_list=vc_list,
ft_list=uvflat,
textures_list=textures_listflat,
haveUVs_list=haveTextures_listflat,
bgcolor=ch.ones(3),
overdraw=True)
renderer.msaa = True
renderer.sharedWin = sharedWin
# likelihoodsZ = [chGenComponentsProbs[comp]*ch.exp( - (chInput - chZ)**2 / (2 * covars)) * (1/(ch.sqrt(covars) * np.sqrt(2 * np.pi))) for comp in range(nComps)]
# chLikelihoodsZ = ch.concatenate(likelihoods)
# chGenMarginalZ = ch.exp(ch.sum(ch.log(chLikelihoodsZ)))
gmmRec = mixture.GMM(n_components=nRecComps, covariance_type='spherical')
gmmRec.covars_ = gmm.covars_.copy()
#Update the mean of the gaussians and update the mixing weights.
methods = ['dogleg', 'minimize', 'BFGS', 'L-BFGS-B', 'Nelder-Mead']
free_vars = [recMeans.ravel(), recSoftmaxW]
print("Beginning optimization.")
while True:
gmmRec.weights_ = np.array(chRecSoftmax.r)
gmmRec.means_ = np.array(ch.concatenate(recMeans))
epsilon = np.random.randn(numVars)
u = choice(nRecComps, size=1, p=chRecSoftmax.r)
chZ[:] = chZRecComps[:, u].r.ravel() + recCovars * epsilon.ravel()
pu = chRecSoftmax
L = ch.log(pu[u]) + ch.sum(chLogJoint.ravel()) - ch.sum(
chRecLogLikelihoods[:, :, u].ravel())
drL = L.dr_wrt(recMeans) / numPixels
alpha = 0.1
recSoftmaxW[:] = recSoftmaxW.r[:] + alpha * L.dr_wrt(recSoftmaxW).reshape(
recSoftmaxW.shape) / numPixels
ipdb.set_trace()
chZ[:] = chZ.r[:] + alpha * L.dr_wrt(chZ).reshape(chZ.r.shape) / numPixels
chZRecComps[:, u] = chZ.r[:]
# ch.minimize({'raw': -L}, bounds=None, method=methods[1], x0=free_vars, callback=None, options={'disp':False, 'maxiter':1})
def get_capsules(model, wrt_betas=None, length_regs=None, rad_regs=None):
from opendr.geometry import Rodrigues
if length_regs is not None:
n_shape_dofs = length_regs.shape[0] - 1
else:
n_shape_dofs = model.betas.r.size
segm = np.argmax(model.weights_prior, axis=1)
J_off = ch.zeros((len(joint2name), 3))
rots = rots0.copy()
mujoco_t_mid = [0, 3, 6, 9]
if wrt_betas is not None:
# if we want to differentiate wrt betas (shape), we must have the
# regressors...
assert (length_regs is not None and rad_regs is not None)
# ... and betas must be a chumpy object
assert (hasattr(wrt_betas, 'dterms'))
pad = ch.concatenate(
(wrt_betas, ch.zeros(n_shape_dofs - len(wrt_betas)), ch.ones(1)))
lengths = pad.dot(length_regs)
rads = pad.dot(rad_regs)
else:
lengths = ch.ones(len(joint2name))
rads = ch.ones(len(joint2name))
betas = wrt_betas if wrt_betas is not None else model.betas
n_betas = len(betas)
# the joint regressors are the original, pre-optimized ones
# (middle of the part frontier)
myJ_regressor = model.J_regressor_prior
myJ0 = ch.vstack((ch.ch.MatVecMult(
myJ_regressor, model.v_template[:, 0] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 0]),
ch.ch.MatVecMult(
myJ_regressor, model.v_template[:, 1] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 1]),
ch.ch.MatVecMult(
myJ_regressor, model.v_template[:, 2] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 2]))).T
# with small adjustments for hips, spine and feet
myJ = ch.vstack([
ch.concatenate([
myJ0[0, 0], (.6 * myJ0[0, 1] + .2 * myJ0[1, 1] + .2 * myJ0[2, 1]),
myJ0[9, 2]
]),
ch.vstack([myJ0[i] for i in range(1, 7)]),
ch.concatenate(
[myJ0[7, 0], (1.1 * myJ0[7, 1] - .1 * myJ0[4, 1]), myJ0[7, 2]]),
ch.concatenate(
[myJ0[8, 0], (1.1 * myJ0[8, 1] - .1 * myJ0[5, 1]), myJ0[8, 2]]),
ch.concatenate(
[myJ0[9, 0], myJ0[9, 1], (.2 * myJ0[9, 2] + .8 * myJ0[12, 2])]),
ch.vstack([myJ0[i] for i in range(10, 24)])
])
capsules = []
# create one capsule per mujoco joint
for ijoint, segms in enumerate(mujoco2segm):
if wrt_betas is None:
vidxs = np.asarray([segm == k for k in segms]).any(axis=0)
verts = model.v_template[vidxs].r
dims = (verts.max(axis=0) - verts.min(axis=0))
rads[ijoint] = .5 * ((dims[(np.argmax(dims) + 1) % 3] + dims[
(np.argmax(dims) + 2) % 3]) / 4.)
lengths[ijoint] = max(dims) - 2. * rads[ijoint].r
# the core joints are different, since the capsule is not in the joint
# but in the middle
if ijoint in mujoco_t_mid:
len_offset = ch.vstack(
[ch.zeros(1),
ch.abs(lengths[ijoint]) / 2.,
ch.zeros(1)]).reshape(3, 1)
caps = Capsule(
(J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(3, 1) -
Rodrigues(rots[ijoint]).dot(len_offset), rots[ijoint],
rads[ijoint], lengths[ijoint])
else:
caps = Capsule(
(J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(3, 1),
rots[ijoint], rads[ijoint], lengths[ijoint])
caps.id = ijoint
capsules.append(caps)
return capsules
def diffHog(image, drconv=None, numOrient=9, cwidth=8, cheight=8):
imagegray = 0.3 * image[:, :, 0] + 0.59 * image[:, :,
1] + 0.11 * image[:, :, 2]
sy, sx = imagegray.shape
# gx = ch.empty(imagegray.shape, dtype=np.double)
gx = imagegray[:, 2:] - imagegray[:, :-2]
gx = ch.hstack([np.zeros([sy, 1]), gx, np.zeros([sy, 1])])
gy = imagegray[2:, :] - imagegray[:-2, :]
# gy = imagegray[:, 2:] - imagegray[:, :-2]
gy = ch.vstack([np.zeros([1, sx]), gy, np.zeros([1, sx])])
gx += 1e-5
# gy = imagegray[:-2,1:-1] - imagegray[2:,1:-1] + 0.00001
# gx = imagegray[1:-1,:-2] - imagegray[1:-1, 2:] + 0.00001
distFilter = np.ones([2 * cheight, 2 * cwidth], dtype=np.uint8)
distFilter[np.int(2 * cheight / 2), np.int(2 * cwidth / 2)] = 0
distFilter = (cv2.distanceTransform(distFilter, cv2.DIST_L2, 3) - np.max(
cv2.distanceTransform(distFilter, cv2.DIST_L2, 3))) / (
-np.max(cv2.distanceTransform(distFilter, cv2.DIST_L2, 3)))
magn = ch.sqrt(gy**2 + gx**2) * 180 / np.sqrt(2)
angles = ch.arctan(gy / gx) * 180 / np.pi + 90
# meanOrient = np.linspace(0, 180, numOrient)
orientations_arr = np.arange(numOrient)
meanOrient = orientations_arr / numOrient * 180
fb_resttmp = 1 - ch.abs(
ch.expand_dims(angles[:, :], 2) -
meanOrient[1:].reshape([1, 1, numOrient - 1])) * numOrient / 180
zeros_rest = np.zeros([sy, sx, numOrient - 1, 1])
fb_rest = ch.max(ch.concatenate([fb_resttmp[:, :, :, None], zeros_rest],
axis=3),
axis=3)
chMinOrient0 = ch.min(ch.concatenate([
ch.abs(
ch.expand_dims(angles[:, :], 2) -
meanOrient[0].reshape([1, 1, 1]))[:, :, :, None],
ch.abs(180 - ch.expand_dims(angles[:, :], 2) -
meanOrient[0].reshape([1, 1, 1]))[:, :, :, None]
],
axis=3),
axis=3)
zeros_fb0 = np.zeros([sy, sx, 1])
fb0_tmp = ch.concatenate(
[1 - chMinOrient0[:, :] * numOrient / 180, zeros_fb0], axis=2)
fb_0 = ch.max(fb0_tmp, axis=2)
fb = ch.concatenate([fb_0[:, :, None], fb_rest], axis=2)
# fb[:,:,0] = ch.max(1 - ch.abs(ch.expand_dims(angles,2) - meanOrient.reshape([1,1,numOrient]))*numOrient/180,0)
# fb = 1./(1. + ch.exp(1 - ch.abs(ch.expand_dims(angles,2) - meanOrient.reshape([1,1,numOrient]))*numOrient/180))
Fb = ch.expand_dims(magn, 2) * fb
if drconv is None:
drconv = dr_wrt_convolution(Fb[:, :, 0], distFilter)
Fs_list = [
convolve2D(x=Fb[:, :, Fbi], filter=distFilter,
convolve2DDr=drconv).reshape([Fb.shape[0], Fb.shape[1], 1])
for Fbi in range(numOrient)
]
# Fs_list = [scipy.signal.convolve2d(Fb[:,:,Fbi], distFilter).reshape([Fb.shape[0], Fb.shape[1],1]) for Fbi in range(numOrient)]
Fs = ch.concatenate(Fs_list, axis=2)
# cellCols = np.arange(start=cwidth/2, stop=Fs.shape[1]-cwidth/2 , step=cwidth)
# cellRows = np.arange(start=cheight/2, stop=Fs.shape[0]-cheight/2 , step=cheight)
Fcells = Fs[0:Fs.shape[0]:cheight, 0:Fs.shape[1]:cwidth, :]
epsilon = 1e-5
v = Fcells / ch.sqrt(ch.sum(Fcells**2) + epsilon)
# v = Fcells
# hog, hogim = skimage.feature.hog(imagegray, orientations=numOrient, pixels_per_cell=(cheight, cwidth), visualise=True)
hog_image = HogImage(image=image,
hog=Fcells,
numOrient=numOrient,
cwidth=cwidth,
cheight=cheight)
# plt.imshow(hog_image)
# plt.figure()
# plt.imshow(hogim)
# ipdb.set_trace()
return v, hog_image, drconv
def optimize_on_joints_and_silhouette(j2d,
sil,
model,
cam,
img,
prior,
init_pose,
init_shape,
n_betas=10,
conf=None):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param sil: h x w silhouette with soft boundaries (np.float32, range(-1, 1))
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param init_pose: 72D vector, pose prediction results provided by HMR
:param init_shape: 10D vector, shape prediction results provided by HMR
:param n_betas: number of shape coefficients considered during optimization
:param conf: 14D vector storing the confidence values from the CNN
:returns: a tuple containing the optimized model, its joints projected on image space, the
camera translation
"""
# define the mapping LSP joints -> SMPL joints
cids = range(12) + [13]
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
head_id = 411
# weights assigned to each joint during optimization;
# the definition of hips in SMPL and LSP is significantly different so set
# their weights to zero
base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1],
dtype=np.float64)
betas = ch.array(init_shape)
# instantiate the model:
sv = verts_decorated(trans=ch.zeros(3),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# make the SMPL joints depend on betas
Jdirs = np.dstack([
model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))
])
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = global_rigid_transformation(sv.pose,
J_onbetas,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
# add the head joint
Jtr = ch.vstack((Jtr, sv[head_id]))
smpl_ids.append(len(Jtr) - 1)
# update the weights using confidence values
weights = base_weights * conf[cids] if conf is not None else base_weights
# project SMPL joints and vertex on the image plane using the estimated camera
cam.v = ch.vstack([Jtr, sv])
# obtain a gradient map of the soft silhouette
grad_x = cv2.Sobel(sil, cv2.CV_32FC1, 1, 0) * 0.125
grad_y = cv2.Sobel(sil, cv2.CV_32FC1, 0, 1) * 0.125
# data term #1: distance between observed and estimated joints in 2D
obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf(
(j2d[cids] - cam[smpl_ids]), sigma))
# data term #2: distance between the observed and projected boundaries
obj_s2d = lambda w, sigma, flag, target_pose: (w * flag * GMOf(
(target_pose - cam[len(Jtr):(len(Jtr) + 6890)]), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
print('****** Optimization on joints')
curr_pose = sv.pose.r
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
objs['thetas'] = wbetas * (sv.pose - curr_pose
) # constrain theta changes
ch.minimize(objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=None,
options={
'maxiter': 100,
'e_3': .001,
'disp': 0
})
curr_pose = sv.pose.r
# cam.v = ch.vstack([Jtr, sv.r])
# run the optimization in 2 stages, progressively decreasing the
# weights for the priors
print('****** Optimization on silhouette and joints')
opt_weights = zip([57.4, 4.78], [2e2, 1e2])
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
# find the boundary vertices and estimate their expected location
smpl_vs = cam.r[len(Jtr):, :]
boundary_flag = np.zeros((smpl_vs.shape[0], 1))
expected_pos = np.zeros((smpl_vs.shape[0], 2))
for vi, v in enumerate(smpl_vs):
r, c = int(v[1]), int(v[0])
if r < 0 or r >= sil.shape[0] or c < 0 or c >= sil.shape[1]:
continue
sil_v = sil[r, c]
grad = np.array([grad_x[r, c], grad_y[r, c]])
grad_n = np.linalg.norm(grad)
if grad_n > 1e-1 and sil_v < 0.4: # vertex on or out of the boundaries
boundary_flag[vi] = 1.0
step = (grad / grad_n) * (sil_v / grad_n)
expected_pos[vi] = np.array([c - step[0], r - step[1]])
# run optimization
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['s2d'] = obj_s2d(5., 100, boundary_flag, expected_pos)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas # constrain beta changes
objs['thetas'] = wbetas * (sv.pose - curr_pose
) # constrain theta changes
ch.minimize(objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=None,
options={
'maxiter': 100,
'e_3': .001,
'disp': 0
})
return sv, cam.r, cam.t.r
def dump3DModel2DKpsHand(img,
m,
filename,
camMat,
est2DJoints=None,
gt2DJoints=None,
outDir=None,
camPose=np.eye(4, dtype=np.float32)):
'''
Saves the following in this order depending on availibility
1. GT 2D joint locations
2. 2D joint locations as estimated by the CPM
3. 2D joint locations after fitting the MANO model to the estimation 2D joints
4. 3D model of the hand in the estimated pose
:param img:
:param m:
:param filename:
:param camMat:
:param est2DJoints:
:param gt2DJoints:
:param outDir:
:return:
'''
if outDir is not None:
plt.ioff()
fig = plt.figure(figsize=(2, 2))
figManager = plt.get_current_fig_manager()
# figManager.window.showMaximized()
ax = fig.add_subplot(2, 2, 4, projection="3d")
plot3dVisualize(ax, m, flip_x=False, camPose=camPose)
ax.title.set_text("3D mesh")
J3DHomo = ch.concatenate([m.J_transformed, np.ones((21, 1))], axis=1)
J3DTrans = J3DHomo.dot(camPose.T)[:, :3]
projPts = utilsEval.chProjectPoints(J3DTrans, camMat, False)[jointsMap]
axEst = fig.add_subplot(2, 2, 3)
imgOutEst = utilsEval.showHandJoints(img.copy(),
np.copy(projPts.r).astype(np.float32),
estIn=None,
filename=None,
upscale=1,
lineThickness=3)
axEst.imshow(imgOutEst[:, :, [2, 1, 0]])
axEst.title.set_text("After fitting")
if est2DJoints is not None:
axGT = fig.add_subplot(2, 2, 2)
imgOutGt = utilsEval.showHandJoints(img.copy(),
est2DJoints.astype(np.float32),
estIn=None,
filename=None,
upscale=1,
lineThickness=3)
axGT.imshow(imgOutGt[:, :, [2, 1, 0]])
axGT.title.set_text("Before fitting")
if gt2DJoints is not None:
ax1 = fig.add_subplot(2, 2, 1)
ax1.imshow(gt2DJoints[:, :, [2, 1, 0]])
ax1.title.set_text("Ground Truth")
if outDir is not None:
if len(filename.split('/')) > 1:
if len(filename.split('/')) == 2:
if not os.path.exists(
os.path.join(outDir,
filename.split('/')[0])):
os.mkdir(os.path.join(outDir, filename.split('/')[0]))
elif len(filename.split('/')) == 3:
if not os.path.exists(
os.path.join(outDir,
filename.split('/')[0])):
os.mkdir(os.path.join(outDir, filename.split('/')[0]))
if not os.path.exists(
os.path.join(outDir,
filename.split('/')[0],
filename.split('/')[1])):
os.mkdir(
os.path.join(outDir,
filename.split('/')[0],
filename.split('/')[1]))
else:
raise NotImplementedError
fig.set_size_inches((11, 8.5), forward=False)
# plt.show()
plt.savefig(os.path.join(outDir, filename + '.jpg'), dpi=300)
plt.close(fig)
else:
plt.show()
def get_capsules(model, wrt_betas=None, length_regs=None, rad_regs=None):
from opendr.geometry import Rodrigues
if length_regs is not None:
n_shape_dofs = length_regs.shape[0] - 1
else:
n_shape_dofs = model.betas.r.size
segm = np.argmax(model.weights_prior, axis=1)
J_off = ch.zeros((len(joint2name), 3))
rots = rots0.copy()
mujoco_t_mid = [0, 3, 6, 9]
if wrt_betas is not None:
# if we want to differentiate wrt betas (shape), we must have the
# regressors...
assert (length_regs is not None and rad_regs is not None)
# ... and betas must be a chumpy object
assert (hasattr(wrt_betas, 'dterms'))
pad = ch.concatenate(
(wrt_betas, ch.zeros(n_shape_dofs - len(wrt_betas)), ch.ones(1)))
lengths = pad.dot(length_regs)
rads = pad.dot(rad_regs)
else:
lengths = ch.ones(len(joint2name))
rads = ch.ones(len(joint2name))
betas = wrt_betas if wrt_betas is not None else model.betas
n_betas = len(betas)
# the joint regressors are the original, pre-optimized ones
# (middle of the part frontier)
myJ_regressor = model.J_regressor_prior
myJ0 = ch.vstack(
(ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 0] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 0]),
ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 1] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 1]),
ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 2] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 2]))).T
# with small adjustments for hips, spine and feet
myJ = ch.vstack(
[ch.concatenate([myJ0[0, 0], (
.6 * myJ0[0, 1] + .2 * myJ0[1, 1] + .2 * myJ0[2, 1]), myJ0[9, 2]]),
ch.vstack([myJ0[i] for i in range(1, 7)]), ch.concatenate(
[myJ0[7, 0], (1.1 * myJ0[7, 1] - .1 * myJ0[4, 1]), myJ0[7, 2]]),
ch.concatenate(
[myJ0[8, 0], (1.1 * myJ0[8, 1] - .1 * myJ0[5, 1]), myJ0[8, 2]]),
ch.concatenate(
[myJ0[9, 0], myJ0[9, 1], (.2 * myJ0[9, 2] + .8 * myJ0[12, 2])]),
ch.vstack([myJ0[i] for i in range(10, 24)])])
capsules = []
# create one capsule per mujoco joint
for ijoint, segms in enumerate(mujoco2segm):
if wrt_betas is None:
vidxs = np.asarray([segm == k for k in segms]).any(axis=0)
verts = model.v_template[vidxs].r
dims = (verts.max(axis=0) - verts.min(axis=0))
rads[ijoint] = .5 * ((dims[(np.argmax(dims) + 1) % 3] + dims[(
np.argmax(dims) + 2) % 3]) / 4.)
lengths[ijoint] = max(dims) - 2. * rads[ijoint].r
# the core joints are different, since the capsule is not in the joint
# but in the middle
if ijoint in mujoco_t_mid:
len_offset = ch.vstack([ch.zeros(1), ch.abs(lengths[ijoint]) / 2.,
ch.zeros(1)]).reshape(3, 1)
caps = Capsule(
(J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(
3, 1) - Rodrigues(rots[ijoint]).dot(len_offset),
rots[ijoint], rads[ijoint], lengths[ijoint])
else:
caps = Capsule(
(J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(3, 1),
rots[ijoint], rads[ijoint], lengths[ijoint])
caps.id = ijoint
capsules.append(caps)
return capsules
def load_model_1(fname_or_dict,
ncomps=6,
flat_hand_mean=False,
v_template=None,
shared_args=None,
optwrt='pose_coeff',
relRot=np.eye(3),
relTrans=np.array([0.0, 0.0, 0.0])):
''' This model loads the fully articulable HAND SMPL model,
and replaces the pose DOFS by ncomps from PCA'''
import numpy as np
import chumpy as ch
import pickle
import scipy.sparse as sp
np.random.seed(1)
if not isinstance(fname_or_dict, dict):
with open(fname_or_dict, 'rb') as f:
smpl_data = pickle.load(f, encoding='latin1')
else:
smpl_data = fname_or_dict
rot = 3 # for global orientation!!!
dof = 20
# smpl_data['hands_components'] = np.eye(45)
from sklearn.preprocessing import normalize
smpl_data['hands_components'] = normalize(smpl_data['hands_components'],
axis=1)
hands_components = smpl_data['hands_components']
std = np.linalg.norm(hands_components, axis=1)
hands_mean = np.zeros(hands_components.shape[1]
) if flat_hand_mean else smpl_data['hands_mean']
hands_coeffs = smpl_data['hands_coeffs'][:, :ncomps]
selected_components = np.vstack((hands_components[:ncomps]))
hands_mean = hands_mean.copy()
if shared_args is not None and 'pose_coeffs' in shared_args:
pose_coeffs = ch.zeros(rot + selected_components.shape[0])
pose_coeffs[:len(shared_args['pose_coeffs']
)] = shared_args['pose_coeffs']
else:
pose_coeffs = ch.zeros(rot + selected_components.shape[0])
full_hand_pose = pose_coeffs[rot:(rot + ncomps)].dot(selected_components)
smpl_data['fullpose'] = ch.concatenate(
(pose_coeffs[:rot], hands_mean + full_hand_pose))
pose_dof = ch.zeros(rot + dof)
smpl_data['pose'] = pose_coeffs
smpl_data['pose_dof'] = pose_dof
Jreg = smpl_data['J_regressor']
if not sp.issparse(Jreg):
smpl_data['J_regressor'] = (sp.csc_matrix(
(Jreg.data, (Jreg.row, Jreg.col)), shape=Jreg.shape))
# slightly modify ready_arguments to make sure that it uses the fullpose
# (which will NOT be pose) for the computation of posedirs
dd = ready_arguments(smpl_data, posekey4vposed='fullpose')
# create the smpl formula with the fullpose,
# but expose the PCA coefficients as smpl.pose for compatibility
args = {
'pose': dd['fullpose'],
'v': dd['v_posed'],
'J': dd['J'],
'weights': dd['weights'],
'kintree_table': dd['kintree_table'],
'xp': ch,
'want_Jtr': True,
'bs_style': dd['bs_style'],
}
# print(dd['J'].r)
result_previous, meta = verts_core(**args)
result_noRel = result_previous + dd['trans'].reshape((1, 3))
result = result_noRel.dot(relRot) + relTrans
result.no_translation = result_previous
if meta is not None:
for field in ['Jtr', 'A', 'A_global', 'A_weighted']:
if (hasattr(meta, field)):
setattr(result, field, getattr(meta, field))
if hasattr(result, 'Jtr'):
result.J_transformed = (result.Jtr + dd['trans'].reshape(
(1, 3))).dot(relRot) + relTrans
for k, v in dd.items():
setattr(result, k, v)
if v_template is not None:
result.v_template[:] = v_template
return result
dd[s] = ch.array(dd[s])
else:
print type(dd[s])
dd['v_shaped'] = dd['shapedirs'].dot(dd['betas'])+dd['v_template']
v_shaped = dd['v_shaped']
J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:,0])
J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:,1])
J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:,2])
dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
if dd['pose'].ndim != 2 or p.shape[1] != 3:
p = dd['pose'].reshape((-1,3))
p = p[1:]
c= ch.concatenate([(Rodrigues(pp)-ch.eye(3)).ravel() for pp in p]).ravel()
dd['v_posed'] = v_shaped + dd['posedirs'].dot(c)
args = {
'pose': dd['pose'],
'v': dd['v_posed'],
'J': dd['J'],
'weights': dd['weights'],
'kintree_table': dd['kintree_table'],
'xp': ch,
'want_Jtr': True,
'bs_style': dd['bs_style']
}
pose=args['pose']
def optimize_on_joints(j2d,
model,
cam,
img,
prior,
try_both_orient,
body_orient,
n_betas=10,
regs=None,
conf=None,
viz=False):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit
:param body_orient: 3D vector, initialization for the body orientation
:param n_betas: number of shape coefficients considered during optimization
:param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term
:param conf: 14D vector storing the confidence values from the CNN
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the optimized model, its joints projected on image space, the camera translation
"""
t0 = time()
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
cids = range(12) + [13]
# joint ids for SMPL
# SMPL does not have a joint for head, instead we use a vertex for the head
# and append it later.
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# the vertex id for the joint corresponding to the head
head_id = 411
# weights assigned to each joint during optimization;
# the definition of hips in SMPL and LSP is significantly different so set
# their weights to zero
base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1],
dtype=np.float64)
if try_both_orient:
flipped_orient = cv2.Rodrigues(body_orient)[0].dot(
cv2.Rodrigues(np.array([0., np.pi, 0]))[0])
flipped_orient = cv2.Rodrigues(flipped_orient)[0].ravel()
orientations = [body_orient, flipped_orient]
else:
orientations = [body_orient]
if try_both_orient:
# store here the final error for both orientations,
# and pick the orientation resulting in the lowest error
errors = []
svs = []
cams = []
for o_id, orient in enumerate(orientations):
# initialize the shape to the mean shape in the SMPL training set
betas = ch.zeros(n_betas)
# initialize the pose by using the optimized body orientation and the
# pose prior
init_pose = np.hstack((orient, prior.weights.dot(prior.means)))
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(trans=ch.zeros(3),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# make the SMPL joints depend on betas
Jdirs = np.dstack([
model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))
])
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = global_rigid_transformation(sv.pose,
J_onbetas,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
# add the head joint, corresponding to a vertex...
Jtr = ch.vstack((Jtr, sv[head_id]))
# ... and add the joint id to the list
if o_id == 0:
smpl_ids.append(len(Jtr) - 1)
# update the weights using confidence values
weights = base_weights * conf[
cids] if conf is not None else base_weights
# project SMPL joints on the image plane using the estimated camera
cam.v = Jtr
# data term: distance between observed and estimated joints in 2D
obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf(
(j2d[cids] - cam[smpl_ids]), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
if viz:
import matplotlib.pyplot as plt
plt.ion()
def on_step(_):
"""Create visualization."""
plt.figure(1, figsize=(10, 10))
plt.subplot(1, 2, 1)
# show optimized joints in 2D
tmp_img = img.copy()
for coord, target_coord in zip(
np.around(cam.r[smpl_ids]).astype(int),
np.around(j2d[cids]).astype(int)):
if (coord[0] < tmp_img.shape[1] and coord[0] >= 0
and coord[1] < tmp_img.shape[0] and coord[1] >= 0):
cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255])
if (target_coord[0] < tmp_img.shape[1]
and target_coord[0] >= 0
and target_coord[1] < tmp_img.shape[0]
and target_coord[1] >= 0):
cv2.circle(tmp_img, tuple(target_coord), 3,
[0, 255, 0])
plt.imshow(tmp_img[:, :, ::-1])
plt.draw()
plt.show()
plt.pause(1e-2)
on_step(_)
else:
on_step = None
if regs is not None:
# interpenetration term
sp = SphereCollisions(pose=sv.pose,
betas=sv.betas,
model=model,
regs=regs)
sp.no_hands = True
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
if regs is not None:
objs['sph_coll'] = 1e3 * sp
ch.minimize(objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=on_step,
options={
'maxiter': 100,
'e_3': .0001,
'disp': 0
})
t1 = time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if try_both_orient:
errors.append((objs['j2d'].r**2).sum())
svs.append(sv)
cams.append(cam)
if try_both_orient and errors[0] > errors[1]:
choose_id = 1
else:
choose_id = 0
if viz:
plt.ioff()
return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r, Jtr)
def load_model(fname_or_dict,
ncomps=6,
flat_hand_mean=False,
v_template=None,
use_pca=True):
''' This model loads the fully articulable HAND SMPL model,
and replaces the pose DOFS by ncomps from PCA'''
from verts import verts_core
import numpy as np
import chumpy as ch
import pickle
import scipy.sparse as sp
np.random.seed(1)
if not isinstance(fname_or_dict, dict):
smpl_data = pickle.load(open(fname_or_dict))
else:
smpl_data = fname_or_dict
rot = 3 # for global orientation!!!
if use_pca:
hands_components = smpl_data[hands_components] # PCA components
else:
hands_components = np.eye(
45) # directly modify 15x3 articulation angles
hands_mean = np.zeros(hands_components.shape[1]
) if flat_hand_mean else smpl_data['hands_mean']
hands_coeffs = smpl_data['hands_coeffs'][:, :ncomps]
selected_components = np.vstack((hands_components[:ncomps]))
hands_mean = hands_mean.copy()
pose_coeffs = ch.zeros(rot + selected_components.shape[0])
full_hand_pose = pose_coeffs[rot:(rot + ncomps)].dot(selected_components)
smpl_data['fullpose'] = ch.concatenate(
(pose_coeffs[:rot], hands_mean + full_hand_pose))
smpl_data['pose'] = pose_coeffs
Jreg = smpl_data['J_regressor']
if not sp.issparse(Jreg):
smpl_data['J_regressor'] = (sp.csc_matrix(
(Jreg.data, (Jreg.row, Jreg.col)), shape=Jreg.shape))
# slightly modify ready_arguments to make sure that it uses the fullpose
# (which will NOT be pose) for the computation of posedirs
dd = ready_arguments(smpl_data, posekey4vposed='fullpose')
# create the smpl formula with the fullpose,
# but expose the PCA coefficients as smpl.pose for compatibility
args = {
'pose': dd['fullpose'],
'v': dd['v_posed'],
'J': dd['J'],
'weights': dd['weights'],
'kintree_table': dd['kintree_table'],
'xp': ch,
'want_Jtr': True,
'bs_style': dd['bs_style'],
}
result_previous, meta = verts_core(**args)
result = result_previous + dd['trans'].reshape((1, 3))
result.no_translation = result_previous
if meta is not None:
for field in ['Jtr', 'A', 'A_global', 'A_weighted']:
if (hasattr(meta, field)):
setattr(result, field, getattr(meta, field))
if hasattr(result, 'Jtr'):
result.J_transformed = result.Jtr + dd['trans'].reshape((1, 3))
for k, v in dd.items():
setattr(result, k, v)
if v_template is not None:
result.v_template[:] = v_template
result.dd = dd
return result
def optimize_on_joints(j2d,
model,
cam,
img,
prior,
try_both_orient,
body_orient,
n_betas=10,
regs=None,
conf=None,
viz=False):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit
:param body_orient: 3D vector, initialization for the body orientation
:param n_betas: number of shape coefficients considered during optimization
:param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term
:param conf: 14D vector storing the confidence values from the CNN
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the optimized model, its joints projected on image space, the camera translation
"""
t0 = time()
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
cids = range(12) + [13]
# joint ids for SMPL
# SMPL does not have a joint for head, instead we use a vertex for the head
# and append it later.
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# the vertex id for the joint corresponding to the head
head_id = 411
# weights assigned to each joint during optimization;
# the definition of hips in SMPL and LSP is significantly different so set
# their weights to zero
base_weights = np.array(
[1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=np.float64)
if try_both_orient:
flipped_orient = cv2.Rodrigues(body_orient)[0].dot(
cv2.Rodrigues(np.array([0., np.pi, 0]))[0])
flipped_orient = cv2.Rodrigues(flipped_orient)[0].ravel()
orientations = [body_orient, flipped_orient]
else:
orientations = [body_orient]
if try_both_orient:
# store here the final error for both orientations,
# and pick the orientation resulting in the lowest error
errors = []
svs = []
cams = []
for o_id, orient in enumerate(orientations):
# initialize the shape to the mean shape in the SMPL training set
betas = ch.zeros(n_betas)
# initialize the pose by using the optimized body orientation and the
# pose prior
init_pose = np.hstack((orient, prior.weights.dot(prior.means)))
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(
trans=ch.zeros(3),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# make the SMPL joints depend on betas
Jdirs = np.dstack([model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))])
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = global_rigid_transformation(
sv.pose, J_onbetas, model.kintree_table, xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
# add the head joint, corresponding to a vertex...
Jtr = ch.vstack((Jtr, sv[head_id]))
# ... and add the joint id to the list
if o_id == 0:
smpl_ids.append(len(Jtr) - 1)
# update the weights using confidence values
weights = base_weights * conf[
cids] if conf is not None else base_weights
# project SMPL joints on the image plane using the estimated camera
cam.v = Jtr
# data term: distance between observed and estimated joints in 2D
obj_j2d = lambda w, sigma: (
w * weights.reshape((-1, 1)) * GMOf((j2d[cids] - cam[smpl_ids]), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([my_exp(sv.pose[55]), my_exp(-sv.pose[
58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15])])
if viz:
import matplotlib.pyplot as plt
plt.ion()
def on_step(_):
"""Create visualization."""
plt.figure(1, figsize=(10, 10))
plt.subplot(1, 2, 1)
# show optimized joints in 2D
tmp_img = img.copy()
for coord, target_coord in zip(
np.around(cam.r[smpl_ids]).astype(int),
np.around(j2d[cids]).astype(int)):
if (coord[0] < tmp_img.shape[1] and coord[0] >= 0 and
coord[1] < tmp_img.shape[0] and coord[1] >= 0):
cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255])
if (target_coord[0] < tmp_img.shape[1] and
target_coord[0] >= 0 and
target_coord[1] < tmp_img.shape[0] and
target_coord[1] >= 0):
cv2.circle(tmp_img, tuple(target_coord), 3,
[0, 255, 0])
plt.imshow(tmp_img[:, :, ::-1])
plt.draw()
plt.show()
plt.pause(1e-2)
on_step(_)
else:
on_step = None
if regs is not None:
# interpenetration term
sp = SphereCollisions(
pose=sv.pose, betas=sv.betas, model=model, regs=regs)
sp.no_hands = True
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
if regs is not None:
objs['sph_coll'] = 1e3 * sp
ch.minimize(
objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=on_step,
options={'maxiter': 100,
'e_3': .0001,
'disp': 0})
t1 = time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if try_both_orient:
errors.append((objs['j2d'].r**2).sum())
svs.append(sv)
cams.append(cam)
if try_both_orient and errors[0] > errors[1]:
choose_id = 1
else:
choose_id = 0
if viz:
plt.ioff()
return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r)
def compute_r(self):
# pylint: disable=unsubscriptable-object
min_w = self.weights[self.min_component_idx]
# Add the sqrt(-log(weights)).
return ch.concatenate((self.loglikelihoods[self.min_component_idx].r,
np.sqrt(-np.log(min_w))))
def load_model_withInputs_poseCoeffs(
fname_or_dict,
chRot,
chPoseCoeff,
chTrans,
chBetas,
ncomps=6,
flat_hand_mean=False,
v_template=None,
shared_args=None,
):
import numpy as np
import chumpy as ch
import pickle
import scipy.sparse as sp
np.random.seed(1)
if not isinstance(fname_or_dict, dict):
# smpl_data = pickle.load(open(fname_or_dict))
smpl_data = pickle.load(open(fname_or_dict, 'rb'), encoding='latin1')
else:
smpl_data = fname_or_dict
rot = 3 # for global orientation!!!
dof = 20
# smpl_data['hands_components'] = np.eye(45)
from sklearn.preprocessing import normalize
smpl_data['hands_components'] = normalize(smpl_data['hands_components'],
axis=1)
hands_components = smpl_data['hands_components']
hands_mean = np.zeros(hands_components.shape[1]
) if flat_hand_mean else smpl_data['hands_mean']
hands_coeffs = smpl_data['hands_coeffs'][:, :ncomps]
selected_components = np.vstack((hands_components[:ncomps]))
hands_mean = hands_mean.copy()
pose_coeffs = ch.concatenate([chRot, chPoseCoeff], axis=0)
full_hand_pose = pose_coeffs[rot:(rot + ncomps)].dot(selected_components)
smpl_data['fullpose'] = ch.concatenate(
(pose_coeffs[:rot], hands_mean + full_hand_pose))
smpl_data['pose'] = pose_coeffs
Jreg = smpl_data['J_regressor']
if not sp.issparse(Jreg):
smpl_data['J_regressor'] = (sp.csc_matrix(
(Jreg.data, (Jreg.row, Jreg.col)), shape=Jreg.shape))
# slightly modify ready_arguments to make sure that it uses the fullpose
# (which will NOT be pose) for the computation of posedirs
dd = ready_arguments(smpl_data,
posekey4vposed='fullpose',
shared_args=shared_args,
chTrans=chTrans,
chBetas=chBetas)
# create the smpl formula with the fullpose,
# but expose the PCA coefficients as smpl.pose for compatibility
args = {
'pose': dd['fullpose'],
'v': dd['v_posed'],
'J': dd['J'],
'weights': dd['weights'],
'kintree_table': dd['kintree_table'],
'xp': ch,
'want_Jtr': True,
'bs_style': dd['bs_style'],
}
# print(dd['J'].r)
result_previous, meta = verts_core(**args)
result_noRel = result_previous + dd['trans'].reshape((1, 3))
result = result_noRel
result.no_translation = result_previous
if meta is not None:
for field in ['Jtr', 'A', 'A_global', 'A_weighted']:
if (hasattr(meta, field)):
setattr(result, field, getattr(meta, field))
if hasattr(result, 'Jtr'):
result.J_transformed = (result.Jtr + dd['trans'].reshape((1, 3)))
for k, v in dd.items():
setattr(result, k, v)
if v_template is not None:
result.v_template[:] = v_template
return result
def verts_decorated_quat(trans,
pose,
v_template,
J,
weights,
kintree_table,
f,
posedirs=None,
betas=None,
add_shape=True,
shapedirs=None,
want_Jtr=False):
for which in [
trans, pose, v_template, weights, posedirs, betas, shapedirs
]:
if which is not None:
assert ischumpy(which)
v = v_template
if shapedirs is not None:
if betas is None:
betas = chumpy.zeros(shapedirs.shape[-1])
if add_shape:
v_shaped = v + shapedirs.dot(betas) #Add Shape of the model.
else:
v_shaped = v
else:
v_shaped = v
quaternion_angles = axis2quat(pose.reshape((-1, 3))).reshape(-1)
shape_feat = betas[1]
feat = ch.concatenate([quaternion_angles, shape_feat], axis=0)
feat = quaternion_angles
poseblends = posedirs.dot(feat)
v_posed = v_shaped + poseblends
v = v_posed
regressor = J
J_tmpx = MatVecMult(regressor, v_shaped[:, 0])
J_tmpy = MatVecMult(regressor, v_shaped[:, 1])
J_tmpz = MatVecMult(regressor, v_shaped[:, 2])
J = chumpy.vstack((J_tmpx, J_tmpy, J_tmpz)).T
result, meta = verts_core(pose,
v,
J,
weights,
kintree_table,
want_Jtr=True)
Jtr = meta.Jtr if meta is not None else None
tr = trans.reshape((1, 3))
result = result + tr
Jtr = Jtr + tr
result.trans = trans
result.f = f
result.pose = pose
result.v_template = v_template
result.J = J
result.weights = weights
result.kintree_table = kintree_table
result.poseblends = poseblends
result.quats = quaternion_angles
if meta is not None:
for field in ['Jtr', 'A', 'A_global', 'A_weighted']:
if (hasattr(meta, field)):
setattr(result, field, getattr(meta, field))
if posedirs is not None:
result.posedirs = posedirs
result.v_posed = v_posed
if shapedirs is not None:
result.shapedirs = shapedirs
result.betas = betas
result.v_shaped = v_shaped
if want_Jtr:
result.J_transformed = Jtr
result.poseblends = poseblends
return result
def optimize_on_joints(j2d,
model,
cam,
img,
prior,
init_pose,
init_shape,
n_betas=10,
conf=None):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param init_pose: 72D vector, pose prediction results provided by HMR
:param init_shape: 10D vector, shape prediction results provided by HMR
:param n_betas: number of shape coefficients considered during optimization
:param conf: 14D vector storing the confidence values from the CNN
:returns: a tuple containing the optimized model, its joints projected on image space, the
camera translation
"""
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
cids = range(12) + [13]
# joint ids for SMPL
# SMPL does not have a joint for head, instead we use a vertex for the head
# and append it later.
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# the vertex id for the joint corresponding to the head
head_id = 411
# weights assigned to each joint during optimization;
# the definition of hips in SMPL and LSP is significantly different so set
# their weights to zero
base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1],
dtype=np.float64)
# initialize the shape to the mean shape in the SMPL training set
betas = ch.array(init_shape)
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(trans=ch.zeros(3),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# make the SMPL joints depend on betas
Jdirs = np.dstack([
model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))
])
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = global_rigid_transformation(sv.pose,
J_onbetas,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
# add the head joint, corresponding to a vertex...
Jtr = ch.vstack((Jtr, sv[head_id]))
# ... and add the joint id to the list
smpl_ids.append(len(Jtr) - 1)
# update the weights using confidence values
weights = base_weights * conf[cids] if conf is not None else base_weights
# project SMPL joints on the image plane using the estimated camera
cam.v = Jtr
# data term: distance between observed and estimated joints in 2D
obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf(
(j2d[cids] - cam[smpl_ids]), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
ch.minimize(objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=None,
options={
'maxiter': 100,
'e_3': .0001,
'disp': 0
})
return sv, cam.r, cam.t.r
def compute_r(self):
min_w = self.weights[self.min_component_idx]
# Add the sqrt(-log(weights))
return ch.concatenate((self.loglikelihoods[self.min_component_idx].r,
np.sqrt(-np.log(min_w))))
