def _global_rigid_transformation(self):
results = {}
pose = self.pose.reshape((-1, 3))
parent = {
i: self.kintree_table[0, i]
for i in range(1, self.kintree_table.shape[1])
}
with_zeros = lambda x: ch.vstack((x, ch.array([[0.0, 0.0, 0.0, 1.0]])))
pack = lambda x: ch.hstack([ch.zeros((4, 3)), x.reshape((4, 1))])
results[0] = with_zeros(
ch.hstack((Rodrigues(pose[0, :]), self.J[0, :].reshape((3, 1)))))
for i in range(1, self.kintree_table.shape[1]):
results[i] = results[parent[i]].dot(
with_zeros(
ch.hstack((
Rodrigues(pose[i, :]), # rotation around bone endpoint
(self.J[i, :] - self.J[parent[i], :]).reshape(
(3, 1)) # bone
))))
results = [results[i] for i in sorted(results.keys())]
results_global = results
# subtract rotated J position
results2 = [
results[i] -
(pack(results[i].dot(ch.concatenate((self.J[i, :], [0])))))
for i in range(len(results))
]
result = ch.dstack(results2)
return result, results_global
def global_rigid_transformation(pose, J, kintree_table):
def _rodrigues(x):
return Rodrigues(x)
results = {}
pose = pose.reshape((-1, 3))
id_to_col = {kintree_table[1, i]: i for i in range(kintree_table.shape[1])}
parent = {i: id_to_col[kintree_table[0, i]]
for i in range(1, kintree_table.shape[1])}
results[0] = with_zeros(
ch.hstack((_rodrigues(pose[0, :]), J[0, :].reshape((3, 1)))))
for i in range(1, kintree_table.shape[1]):
results[i] = results[parent[i]].dot(
with_zeros(ch.hstack((_rodrigues(pose[i, :]),
((J[i, :] - J[parent[i], :]).reshape((3, 1)))
))))
results = [results[i] for i in sorted(results.keys())]
results_global = results
results2 = [results[i] - (pack(
results[i].dot(ch.concatenate(((J[i, :]), 0))))
) for i in range(len(results))]
results = results2
result = ch.dstack(results)
return result, results_global
def volume(v, f):
# Construct a 3D matrix which is of size (nfaces x 3 x 3)
# Each row corresponds to a face; the third dimension indicates
# which triangle in that face is being referred to
vs = ch.dstack((v[f[:, 0], :], v[f[:, 1], :], v[f[:, 2], :]))
v321 = vs[:, 0, 2] * vs[:, 1, 1] * vs[:, 2, 0]
v231 = vs[:, 0, 1] * vs[:, 1, 2] * vs[:, 2, 0]
v312 = vs[:, 0, 2] * vs[:, 1, 0] * vs[:, 2, 1]
v132 = vs[:, 0, 0] * vs[:, 1, 2] * vs[:, 2, 1]
v213 = vs[:, 0, 1] * vs[:, 1, 0] * vs[:, 2, 2]
v123 = vs[:, 0, 0] * vs[:, 1, 1] * vs[:, 2, 2]
volumes = (-v321 + v231 + v312 - v132 - v213 + v123) * (1.0 / 6.0)
return ch.abs(ch.sum(volumes))
def volume(v, f):
# Construct a 3D matrix which is of size (nfaces x 3 x 3)
# Each row corresponds to a face; the third dimension indicates
# which triangle in that face is being referred to
vs = ch.dstack((v[f[:, 0], :], v[f[:, 1], :], v[f[:, 2], :]))
v321 = vs[:, 0, 2] * vs[:, 1, 1] * vs[:, 2, 0]
v231 = vs[:, 0, 1] * vs[:, 1, 2] * vs[:, 2, 0]
v312 = vs[:, 0, 2] * vs[:, 1, 0] * vs[:, 2, 1]
v132 = vs[:, 0, 0] * vs[:, 1, 2] * vs[:, 2, 1]
v213 = vs[:, 0, 1] * vs[:, 1, 0] * vs[:, 2, 2]
v123 = vs[:, 0, 0] * vs[:, 1, 1] * vs[:, 2, 2]
volumes = (-v321 + v231 + v312 - v132 - v213 + v123) * (1. / 6.)
return ch.abs(ch.sum(volumes))
def global_rigid_transformation(pose, J, kintree_table):
results = {}
pose = pose.reshape((-1, 3))
id_to_col = {kintree_table[1, i]: i for i in range(kintree_table.shape[1])}
parent = {
i: id_to_col[kintree_table[0, i]]
for i in range(1, kintree_table.shape[1])
}
def with_zeros(x):
return ch.vstack((x, ch.array([[0.0, 0.0, 0.0, 1.0]])))
results[0] = with_zeros(
ch.hstack((Rodrigues(pose[0, :]), J[0, :].reshape((3, 1)))))
for i in range(1, kintree_table.shape[1]):
results[i] = results[parent[i]].dot(
with_zeros(
ch.hstack((Rodrigues(pose[i, :]),
((J[i, :] - J[parent[i], :]).reshape((3, 1)))))))
def pack(x):
return ch.hstack([np.zeros((4, 3)), x.reshape((4, 1))])
results = [results[i] for i in sorted(results.keys())]
results_global = results
if True:
results2 = [
results[i] - pack(results[i].dot(ch.concatenate(((J[i, :]), 0))))
for i in range(len(results))
]
results = results2
result = ch.dstack(results)
return result, results_global
def optimize_on_DensePose(raw_img,
tar_img,
dp_iuv,
op_j2d,
w_j2d,
model,
cam,
cam_old,
prior,
init_trans,
init_pose,
init_betas,
n_betas=10,
viz=False,
imgname='tmp'):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit
:param body_orient: 3D vector, initialization for the body orientation
:param n_betas: number of shape coefficients considered during optimization
:param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term
:param conf: 14D vector storing the confidence values from the CNN
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the optimized model, its joints projected on image space, the camera translation
"""
outPath = os.path.join(ROOT_PATH, 'smplify/{}_stage'.format(imgname))
if not os.path.isdir(outPath):
os.mkdir(outPath)
print outPath
dp_iuv_tar = dp_iuv.copy()
width = dp_iuv_tar.shape[1]
height = dp_iuv_tar.shape[0]
dp_iuv_weight = np.zeros(dp_iuv_tar.shape) + 1
dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 0] = 1
dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 1] = 1
dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 2] = 255.0 / 24
dp_iuv_weight = ch.array(dp_iuv_weight)
t0 = time()
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
tarIUV = ch.array(dp_iuv)
ori_betas = init_betas.copy()
ori_betas_ch = ch.array(ori_betas)
# initialize the shape to the mean shape in the SMPL training set
betas = ch.array(init_betas)
# initialize the pose by using the optimized body orientation and the
# pose prior
pose = ch.array(init_pose)
#init_pose = np.hstack((body_orient, body_init))
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(trans=ch.array(init_trans),
pose=pose,
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=None)
J_tmpx = MatVecMult(J_reg, sv[:, 0])
J_tmpy = MatVecMult(J_reg, sv[:, 1])
J_tmpz = MatVecMult(J_reg, sv[:, 2])
Jtr = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
#Jtr = J_reg.dot(sv)
cam.v = Jtr
reIUV = render_model(sv,
model.f,
width,
height,
cam,
near=0.5,
far=25,
vc=dp_colors,
img=None)
reModel = render_model(sv,
model.f,
width,
height,
cam,
near=0.5,
far=25,
vc=None,
img=None)
fullModel = render_model(sv,
model.f,
raw_img.shape[1],
raw_img.shape[0],
cam_old,
near=0.5,
far=25,
vc=None,
img=None)
#gaussian_pyramid(input_objective, imshape=None, normalization='SSE', n_levels=3, as_list=False, label=None):
def obj_j2d(w, sigma):
return (w * w_j2d.reshape((-1, 1)) * GMOf((op_j2d - cam), sigma))
#input_objective, imshape, normalization, n_levels, as_list
err = (tarIUV - reIUV) * dp_iuv_weight
err_uv = 1 - ch.exp(-err[:, :, 0:2]**2 / 1)
err_inds = 1 - ch.exp(-err[:, :, 2]**2 / 0.001)
err_total = ch.dstack((err_uv, err_inds))
#obj_dense = gaussian_pyramid(err, n_levels=4, normalization='SSE')
obj_dense = gaussian_pyramid(err_total, n_levels=4, normalization=None)
# TODO
# the render error can be further extended to geodistic error.
# data term: distance between observed and estimated joints in 2D
# obj_dense = lambda w, sigma: (
# w * GMOf((tarIUV - reIUV).reshape(-1,3), sigma))
# mixture of gaussians pose prior
def pprior(w):
return w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
def my_exp(x):
return alpha * ch.exp(x)
def obj_angle(w):
return w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
def viz_func(stage_num):
#plt.figure(1, figsize=(10, 10))
# show optimized joints in 2D
tmp_img = reIUV.r.copy()
tmp_tar = tarIUV.copy()
tmp_model = reModel.r.copy()
full_model = fullModel.r.copy()
# w = tmp_tar.shape[1]
# h = tmp_tar.shape[0]
for aa in ax:
for cax in aa:
cax.clear()
ax[0][0].imshow(tar_img)
ax[0][0].imshow(tmp_img, alpha=0.5)
for j1, j2, w_ts in zip(cam.r, op_j2d, w_j2d):
if (w_ts > 0):
ax[0][0].plot([j1[0], j2[0]], [j1[1], j2[1]], 'r')
ax[0][0].set_xlim([0, width])
ax[0][0].set_ylim([height, 0])
ax[0][1].imshow(tar_img)
ax[0][1].imshow(tmp_tar, alpha=0.5)
ax[0][1].set_xlim([0, width])
ax[0][1].set_ylim([height, 0])
ax[0][2].imshow(tar_img)
tmp_model_alpha = np.ones((tmp_model.shape[0], tmp_model.shape[1]))
tmp_model_alpha[tmp_model[:, :, 0] < 1e-2] = 0
tmp_model = np.dstack((tmp_model, tmp_model_alpha))
ax[0][2].imshow(tmp_model)
ax[0][2].set_xlim([0, width])
ax[0][2].set_ylim([height, 0])
tmp_err = err_total.r.copy()
#show error
ax[1][0].imshow(tmp_err[:, :, 0])
ax[1][0].set_xlim([0, width])
ax[1][0].set_ylim([height, 0])
ax[1][1].imshow(tmp_err[:, :, 1])
ax[1][1].set_xlim([0, width])
ax[1][1].set_ylim([height, 0])
ax[1][2].imshow(tmp_err[:, :, 2])
ax[1][2].set_xlim([0, width])
ax[1][2].set_ylim([height, 0])
#show target
ax[2][0].imshow(tmp_tar[:, :, 0])
ax[2][0].set_xlim([0, width])
ax[2][0].set_ylim([height, 0])
ax[2][1].imshow(tmp_tar[:, :, 1])
ax[2][1].set_xlim([0, width])
ax[2][1].set_ylim([height, 0])
ax[2][2].imshow(tmp_tar[:, :, 2])
ax[2][2].set_xlim([0, width])
ax[2][2].set_ylim([height, 0])
#show reproj
ax[3][0].imshow(tmp_img[:, :, 0])
ax[3][0].set_xlim([0, width])
ax[3][0].set_ylim([height, 0])
ax[3][1].imshow(tmp_img[:, :, 1])
ax[3][1].set_xlim([0, width])
ax[3][1].set_ylim([height, 0])
ax[3][2].imshow(tmp_img[:, :, 2])
ax[3][2].set_xlim([0, width])
ax[3][2].set_ylim([height, 0])
for aa in ax:
aa[0].set_xticks(())
aa[0].set_yticks(())
aa[1].set_xticks(())
aa[1].set_yticks(())
aa[2].set_xticks(())
aa[2].set_yticks(())
#plt.tight_layout()
plt.draw()
plt.savefig(os.path.join(outPath, 'stage-{}.png'.format(stage_num)),
bbox_inches='tight')
full_model_int = full_model.copy()
full_model_int *= 255
full_model_int = full_model_int.astype(np.uint8)
cv2.imwrite(os.path.join(outPath, 'model-{}.png'.format(stage_num)),
full_model_int)
out_params = {
'pose': sv.pose.r,
'shape': sv.betas.r,
'trans': sv.trans.r
}
with open(os.path.join(outPath, 'param-{}.pkl'.format(stage_num)),
'w') as fio:
pickle.dump(out_params, fio, pickle.HIGHEST_PROTOCOL)
#if viz:
if viz and False:
def on_step(cstep):
"""Create visualization."""
# TODO this function is in vis_func
#plt.savefig(os.path.join(outPath,'{}.png'.format(strftime("%d_%H_%M_%S", gmtime()))),bbox_inches='tight')
on_step(stepI)
else:
on_step = None
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
# opt_weights = zip([4.78, 3.78, 2.78, 1.78],
# [5, 5, 5, 5],
# [1, 0.25, 0.10, 0])
opt_weights = zip([4.78], [50], [10])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas, w_joints) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
#objs['dense'] = obj_dense
objs['j2d'] = obj_j2d(w_joints, 50)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
#objs['betas'] = wbetas * (sv.betas - ori_betas_ch)
ch.minimize(
objs,
#x0=[sv.betas, sv.pose, sv.trans],
x0=[sv.pose, sv.trans],
method='dogleg',
callback=on_step,
options={
'maxiter': 10000,
'e_3': .001,
'disp': 1
})
viz_func(254)
t1 = time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if viz and False:
plt.ioff()
