def chumpy_compute_error(p1,
p2,
H,
Mcor,
error=0,
return_sigma=False,
sigma_precomputed=None):
"""Compute deviation of p1 and p2 from common epipole, given H.
"""
# Warp p2 estimates with new homography
p2_new_denom = H[2, 0] * p2[:, 0] + H[2, 1] * p2[:, 1] + H[2, 2]
p2_new_x = (H[0, 0] * p2[:, 0] + H[0, 1] * p2[:, 1] +
H[0, 2]) / p2_new_denom
p2_new_y = (H[1, 0] * p2[:, 0] + H[1, 1] * p2[:, 1] +
H[1, 2]) / p2_new_denom
p2_new = ch.vstack((p2_new_x, p2_new_y)).T
# Compute current best FoE
u = p2_new[:, 0] - p1[:, 0]
v = p2_new[:, 1] - p1[:, 1]
T = ch.vstack((-v, u, v * p1[:, 0] - u * p1[:, 1])).T
U, S, V = ch.linalg.svd(Mcor.dot(T.T.dot(T)).dot(Mcor))
qv = Mcor.dot(V[-1, :])
q = qv[:2] / qv[2]
d = T.dot(qv)
# Robust error norm
if error == 0:
d = d**2
elif error == 1:
sigma = np.median(np.abs(d() - np.median(d())))
d = ch.sqrt(d**2 + sigma**2)
elif error == 2:
# Geman-McClure
sigma = np.median(np.abs(d() - np.median(d()))) * 1.5
d = d**2 / (d**2 + sigma)
elif error == 3:
# Geman-McClure, corrected
if sigma_precomputed is not None:
sigma = sigma_precomputed
else:
sigma = 1.426 * np.median(
np.abs(d() - np.median(d()))) * np.sqrt(3.0)
d = d**2 / (d**2 + sigma**2)
# Correction
d = d * sigma
elif error == 4:
# Inverse exponential norm
sigma = np.median(np.abs(d() - np.median(d())))
d = -ch.exp(-d**2 / (2 * sigma**2)) * sigma
err = d.sum()
if return_sigma:
return sigma
else:
return err
def pixelLikelihoodRobustRegionCh(image, template, testMask, backgroundModel,
layerPrior, variances):
sigma = ch.sqrt(variances)
mask = testMask
if backgroundModel == 'FULL':
mask = np.ones(image.shape[0:2])
# mask = np.repeat(mask[..., np.newaxis], 3, 2)
repPriors = ch.tile(layerPrior, image.shape[0:2])
# sum = np.sum(np.log(layerPrior * scipy.stats.norm.pdf(image, location = template, scale=np.sqrt(variances) ) + (1 - repPriors)))
# uniformProbs = np.ones(image.shape)
imshape = image.shape
from opendr.filters import filter_for
from opendr.filters import GaussianKernel2D
blur_mtx = filter_for(imshape[0],
imshape[1],
imshape[2] if len(imshape) > 2 else 1,
kernel=GaussianKernel2D(3, 1))
blurred_image = MatVecMult(blur_mtx, image).reshape(imshape)
blurred_template = MatVecMult(blur_mtx, template).reshape(imshape)
probs = ch.exp(-(blurred_image - template)**2 /
(2 * variances)) * (1. / (sigma * np.sqrt(2 * np.pi)))
foregroundProbs = (probs[:, :, 0] * probs[:, :, 1] *
probs[:, :, 2]) * layerPrior + (1 - repPriors)
return foregroundProbs * mask + (1 - mask)
def pixelLikelihoodRobustCh(image, template, testMask, backgroundModel, layerPrior, variances):
sigma = ch.sqrt(variances)
mask = testMask
if backgroundModel == 'FULL':
mask = np.ones(image.shape[0:2])
# mask = np.repeat(mask[..., np.newaxis], 3, 2)
repPriors = ch.tile(layerPrior, image.shape[0:2])
# sum = np.sum(np.log(layerPrior * scipy.stats.norm.pdf(image, location = template, scale=np.sqrt(variances) ) + (1 - repPriors)))
# uniformProbs = np.ones(image.shape)
probs = ch.exp( - (image - template)**2 / (2 * variances)) * (1./(sigma * np.sqrt(2 * np.pi)))
foregroundProbs = (probs[:,:,0] * probs[:,:,1] * probs[:,:,2]) * layerPrior + (1 - repPriors)
return foregroundProbs * mask + (1-mask)
def layerPosteriorsRobustCh(image, template, testMask, backgroundModel, layerPrior, variances):
sigma = ch.sqrt(variances)
mask = testMask
if backgroundModel == 'FULL':
mask = np.ones(image.shape[0:2])
# mask = np.repeat(mask[..., np.newaxis], 3, 2)
repPriors = ch.tile(layerPrior, image.shape[0:2])
probs = ch.exp( - (image - template)**2 / (2 * variances)) * (1/(sigma * np.sqrt(2 * np.pi)))
foregroundProbs = probs[:,:,0] * probs[:,:,1] * probs[:,:,2] * layerPrior
backgroundProbs = np.ones(image.shape)
outlierProbs = ch.Ch(1-repPriors)
lik = pixelLikelihoodRobustCh(image, template, testMask, backgroundModel, layerPrior, variances)
# prodlik = np.prod(lik, axis=2)
# return np.prod(foregroundProbs*mask, axis=2)/prodlik, np.prod(outlierProbs*mask, axis=2)/prodlik
return foregroundProbs*mask/lik, outlierProbs*mask/lik
def logProb(self):
fgProb = ch.exp(-(self.renderer - self.groundtruth.r)**2 /
(2 * self.variances.r)) * (
1. /
(np.sqrt(self.variances.r) * np.sqrt(2 * np.pi)))
h = self.renderer.r.shape[0]
w = self.renderer.r.shape[1]
occProb = np.ones([h, w])
bgProb = np.ones([h, w])
fgMask = np.array(self.renderer.image_mesh_bool([0])).astype(np.bool)
errorFun = ch.log(fgMask[:, :, None] * (
(self.Q[0][:, :, None] * fgProb) +
(self.Q[1] * occProb + self.Q[2] * bgProb)[:, :, None]) +
(1 - fgMask[:, :, None]))
return errorFun
def optimize_on_joints(j2d,
model,
cam,
img,
prior,
try_both_orient,
body_orient,
exp_logistic,
n_betas=10,
inner_penetration=False,
silh=None,
conf=None,
viz=False):
"""Run the optimization."""
if silh is not None:
raise NotImplementedError("Silhouette fitting is not supported in "
"this code release due to dependencies on "
"proprietary code for the "
"distance computation.")
t0 = _time()
# define the mapping LSP joints -> SMPL joints
if j2d.shape[0] == 14:
cids = range(12) + [13]
elif j2d.shape[0] == 91:
cids = range(j2d.shape[0])
else:
raise Exception("Unknown number of joints: %d! Mapping not defined!" %
j2d.shape[0])
# joint ids for SMPL
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# weight given to each joint during optimization;
if j2d.shape[0] == 14:
weights = [1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]
else:
weights = [1] * (len(smpl_ids) + len(landmark_mesh_91))
# The non-skeleton vertex ids are added later.
if try_both_orient:
flipped_orient = _cv2.Rodrigues(body_orient)[0].dot(
_cv2.Rodrigues(_np.array([0., _np.pi, 0]))[0])
flipped_orient = _cv2.Rodrigues(flipped_orient)[0].ravel()
orientations = [body_orient, flipped_orient]
else:
orientations = [body_orient]
if try_both_orient:
errors = []
svs = []
cams = []
# rends = []
for o_id, orient in enumerate(orientations):
# initialize betas
betas = _ch.zeros(n_betas) # pylint: disable=no-member
init_pose = _np.hstack((orient, prior.weights.dot(prior.means)))
# 2D joint error term
# make the SMPL joint depend on betas
Jdirs = _np.dstack([
model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))
])
# pylint: disable=no-member
J_onbetas = _ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# instantiate the model
sv = verts_decorated(trans=_ch.zeros(3),
pose=_ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = _global_rigid_transformation(sv.pose,
J_onbetas,
model.kintree_table,
xp=_ch)
Jtr = _ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
if j2d.shape[0] == 14:
# add the "fake" joint for the head
head_id = _HEAD_REGR[0]
Jtr = _ch.vstack((Jtr, sv[head_id]))
if o_id == 0:
smpl_ids.append(len(Jtr) - 1)
else:
# add the plain vertex IDs on the mesh surface.
for vertex_id in landmark_mesh_91.values():
Jtr = _ch.vstack((Jtr, sv[vertex_id]))
# add the joint id
# for SMPL it's the last one added
if o_id == 0:
smpl_ids.append(len(Jtr) - 1)
weights = _np.array(weights, dtype=_np.float64)
if conf is not None:
weights *= conf[cids]
# we'll project the joints on the image plane
cam.v = Jtr
# data term: difference between observed and estimated joints
obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * _GMOf(
(j2d[cids] - cam[smpl_ids]), sigma))
# pose prior
pprior = lambda w: w * prior(sv.pose) # pylint: disable=cell-var-from-loop
# joint angles prior
# 55: left elbow, should bend -np.pi/2
# 58: right elbow, should bend np.pi/2
# 12: left knee, should bend np.pi/2
# 15: right knee, should bend np.pi/2
if exp_logistic:
_LOGGER.info('USING LOGISTIC')
# Skinny Logistic function. as 50-> inf we get a step function at
# 0.1. (0.1) is a margin bc 0 is still ok.
my_exp = lambda x: 1 / (1 + _ch.exp(100 * (0.1 + -x)))
else:
x_0 = 0 #10
alpha = 10
my_exp = lambda x: alpha * _ch.exp((x - x_0)) # pylint: disable=cell-var-from-loop
obj_angle = lambda w: w * _ch.concatenate([
my_exp(sv.pose[55]), # pylint: disable=cell-var-from-loop
my_exp(-sv.pose[58]), # pylint: disable=cell-var-from-loop
my_exp(-sv.pose[12]), # pylint: disable=cell-var-from-loop
my_exp(-sv.pose[15])
]) # pylint: disable=cell-var-from-loop
if viz:
from body.mesh.sphere import Sphere
from body.mesh.meshviewer import MeshViewer
import matplotlib.pyplot as plt
# set up visualization
# openGL window
mv = MeshViewer(window_width=120, window_height=120)
# and ids
show_ids = _np.array(smpl_ids)[weights > 0]
vc = _np.ones((len(Jtr), 3))
vc[show_ids] = [0, 1, 0]
plt.ion()
def on_step(_):
"""Create visualization."""
# show optimized joints in 3D
# pylint: disable=cell-var-from-loop
mv.set_dynamic_meshes([_Mesh(v=sv.r, f=[]),
Sphere(center=cam.t.r,
radius=.1).to_mesh()] \
+ [Sphere(center=jc, radius=.01).to_mesh(vc[ijc])
for ijc, jc in enumerate(Jtr.r)])
plt.figure(1, figsize=(10, 10))
plt.subplot(1, 2, 1)
# show optimized joints in 2D
tmp_img = img.copy()
for coord, target_coord in zip(
_np.around(cam.r[smpl_ids]).astype(int),
_np.around(j2d[cids]).astype(int)):
if (coord[0] < tmp_img.shape[1] and coord[0] >= 0
and coord[1] < tmp_img.shape[0] and coord[1] >= 0):
_cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255])
if (target_coord[0] < tmp_img.shape[1]
and target_coord[0] >= 0
and target_coord[1] < tmp_img.shape[0]
and target_coord[1] >= 0):
_cv2.circle(tmp_img, tuple(target_coord), 3,
[0, 255, 0])
plt.imshow(tmp_img)
plt.draw()
plt.show()
on_step(_)
else:
on_step = None
sp = _SphereCollisions(pose=sv.pose,
betas=sv.betas,
model=model,
regs=_REGRESSORS)
sp.no_hands = True
# configuration used with conf joints
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
#if stage < 2:
objs['j2d'] = obj_j2d(1., 100) # TODO: evaluate.
objs['pose'] = pprior(w)
# WEIGHT FOR ANGLE
if exp_logistic:
# Set to high weight always.
objs['pose_exp'] = obj_angle(5 * 1e3)
else:
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
if inner_penetration:
objs['sph_coll'] = 1e3 * sp
try:
_ch.minimize(objs.values(),
x0=[sv.betas, sv.pose],
method='dogleg',
callback=on_step,
options={
'maxiter': 100,
'e_3': .0001,
'disp': 0
})
except AssertionError:
# Divergence detected.
_LOGGER.warn("Diverging optimization! Breaking!")
break
t1 = _time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if try_both_orient:
errors.append((objs['j2d'].r**2).sum())
svs.append(sv)
cams.append(cam)
# rends.append(rend)
if try_both_orient and errors[0] > errors[1]:
choose_id = 1
else:
choose_id = 0
if viz:
plt.ioff()
return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r,
cams[choose_id].rt.r)
v_teapots, f_list_teapots, vc_teapots, vn_teapots, uv_teapots, haveTextures_list_teapots, textures_list_teapots, vflat, varray, center_teapots = scene_io_utils.loadTeapotsOpenDRData(renderTeapotsList, useBlender, unpackModelsFromBlender, targetModels) azimuth = np.pi chCosAz = ch.Ch([np.cos(azimuth)]) chSinAz = ch.Ch([np.sin(azimuth)]) chAz = 2*ch.arctan(chSinAz/(ch.sqrt(chCosAz**2 + chSinAz**2) + chCosAz)) chAz = ch.Ch([np.pi/4]) chObjAz = ch.Ch([np.pi/4]) chAzRel = chAz - chObjAz elevation = 0 chLogCosEl = ch.Ch(np.log(np.cos(elevation))) chLogSinEl = ch.Ch(np.log(np.sin(elevation))) chEl = 2*ch.arctan(ch.exp(chLogSinEl)/(ch.sqrt(ch.exp(chLogCosEl)**2 + ch.exp(chLogSinEl)**2) + ch.exp(chLogCosEl))) chEl = ch.Ch([0.95993109]) chDist = ch.Ch([camDistance]) chObjAzGT = ch.Ch([np.pi*3/2]) chAzGT = ch.Ch([np.pi*3/2]) chAzRelGT = chAzGT - chObjAzGT chElGT = ch.Ch(chEl.r[0]) chDistGT = ch.Ch([camDistance]) chComponentGT = ch.Ch(np.array([2, 0.25, 0.25, 0.12,-0.17,0.36,0.1,0.,0.])) chComponent = ch.Ch(np.array([2, 0.25, 0.25, 0.12,-0.17,0.36,0.1,0.,0.])) chPointLightIntensity = ch.Ch([1]) chPointLightIntensityGT = ch.Ch([1]) chLightAz = ch.Ch([0.0]) chLightEl = ch.Ch([np.pi/2])
def get_objective(self):
return ch.sum((ch.exp(-((ch.sum(
(self.sph_v[self.ids0] - self.sph_v[self.ids1])**2, axis=1)) /
(self.radiuss)) / 2.))**2)**.5
def optimize_on_joints_and_silhouette(j2d,
sil,
model,
cam,
img,
prior,
init_pose,
init_shape,
n_betas=10,
conf=None):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param sil: h x w silhouette with soft boundaries (np.float32, range(-1, 1))
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param init_pose: 72D vector, pose prediction results provided by HMR
:param init_shape: 10D vector, shape prediction results provided by HMR
:param n_betas: number of shape coefficients considered during optimization
:param conf: 14D vector storing the confidence values from the CNN
:returns: a tuple containing the optimized model, its joints projected on image space, the
camera translation
"""
# define the mapping LSP joints -> SMPL joints
cids = range(12) + [13]
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
head_id = 411
# weights assigned to each joint during optimization;
# the definition of hips in SMPL and LSP is significantly different so set
# their weights to zero
base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1],
dtype=np.float64)
betas = ch.array(init_shape)
# instantiate the model:
sv = verts_decorated(trans=ch.zeros(3),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# make the SMPL joints depend on betas
Jdirs = np.dstack([
model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))
])
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = global_rigid_transformation(sv.pose,
J_onbetas,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
# add the head joint
Jtr = ch.vstack((Jtr, sv[head_id]))
smpl_ids.append(len(Jtr) - 1)
# update the weights using confidence values
weights = base_weights * conf[cids] if conf is not None else base_weights
# project SMPL joints and vertex on the image plane using the estimated camera
cam.v = ch.vstack([Jtr, sv])
# obtain a gradient map of the soft silhouette
grad_x = cv2.Sobel(sil, cv2.CV_32FC1, 1, 0) * 0.125
grad_y = cv2.Sobel(sil, cv2.CV_32FC1, 0, 1) * 0.125
# data term #1: distance between observed and estimated joints in 2D
obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf(
(j2d[cids] - cam[smpl_ids]), sigma))
# data term #2: distance between the observed and projected boundaries
obj_s2d = lambda w, sigma, flag, target_pose: (w * flag * GMOf(
(target_pose - cam[len(Jtr):(len(Jtr) + 6890)]), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
print('****** Optimization on joints')
curr_pose = sv.pose.r
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
objs['thetas'] = wbetas * (sv.pose - curr_pose
) # constrain theta changes
ch.minimize(objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=None,
options={
'maxiter': 100,
'e_3': .001,
'disp': 0
})
curr_pose = sv.pose.r
# cam.v = ch.vstack([Jtr, sv.r])
# run the optimization in 2 stages, progressively decreasing the
# weights for the priors
print('****** Optimization on silhouette and joints')
opt_weights = zip([57.4, 4.78], [2e2, 1e2])
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
# find the boundary vertices and estimate their expected location
smpl_vs = cam.r[len(Jtr):, :]
boundary_flag = np.zeros((smpl_vs.shape[0], 1))
expected_pos = np.zeros((smpl_vs.shape[0], 2))
for vi, v in enumerate(smpl_vs):
r, c = int(v[1]), int(v[0])
if r < 0 or r >= sil.shape[0] or c < 0 or c >= sil.shape[1]:
continue
sil_v = sil[r, c]
grad = np.array([grad_x[r, c], grad_y[r, c]])
grad_n = np.linalg.norm(grad)
if grad_n > 1e-1 and sil_v < 0.4: # vertex on or out of the boundaries
boundary_flag[vi] = 1.0
step = (grad / grad_n) * (sil_v / grad_n)
expected_pos[vi] = np.array([c - step[0], r - step[1]])
# run optimization
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['s2d'] = obj_s2d(5., 100, boundary_flag, expected_pos)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas # constrain beta changes
objs['thetas'] = wbetas * (sv.pose - curr_pose
) # constrain theta changes
ch.minimize(objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=None,
options={
'maxiter': 100,
'e_3': .001,
'disp': 0
})
return sv, cam.r, cam.t.r
def optimize_on_joints(j2d,
model,
cam,
img,
prior,
init_pose,
init_shape,
n_betas=10,
conf=None):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param init_pose: 72D vector, pose prediction results provided by HMR
:param init_shape: 10D vector, shape prediction results provided by HMR
:param n_betas: number of shape coefficients considered during optimization
:param conf: 14D vector storing the confidence values from the CNN
:returns: a tuple containing the optimized model, its joints projected on image space, the
camera translation
"""
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
cids = range(12) + [13]
# joint ids for SMPL
# SMPL does not have a joint for head, instead we use a vertex for the head
# and append it later.
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# the vertex id for the joint corresponding to the head
head_id = 411
# weights assigned to each joint during optimization;
# the definition of hips in SMPL and LSP is significantly different so set
# their weights to zero
base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1],
dtype=np.float64)
# initialize the shape to the mean shape in the SMPL training set
betas = ch.array(init_shape)
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(trans=ch.zeros(3),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# make the SMPL joints depend on betas
Jdirs = np.dstack([
model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))
])
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = global_rigid_transformation(sv.pose,
J_onbetas,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
# add the head joint, corresponding to a vertex...
Jtr = ch.vstack((Jtr, sv[head_id]))
# ... and add the joint id to the list
smpl_ids.append(len(Jtr) - 1)
# update the weights using confidence values
weights = base_weights * conf[cids] if conf is not None else base_weights
# project SMPL joints on the image plane using the estimated camera
cam.v = Jtr
# data term: distance between observed and estimated joints in 2D
obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf(
(j2d[cids] - cam[smpl_ids]), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
ch.minimize(objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=None,
options={
'maxiter': 100,
'e_3': .0001,
'disp': 0
})
return sv, cam.r, cam.t.r
gmm = mixture.GMM(n_components=nComps, covariance_type='spherical')
win = 40
colors = image[image.shape[0] / 2 - win:image.shape[0] / 2 + win,
image.shape[1] / 2 - win:image.shape[1] / 2 + win, :].reshape(
[4 * win * win, 3])
gmm.fit(colors)
imshape = [win * 2, win * 2, 3]
numPixels = win * 2 * win * 2
chInput = ch.Ch(colors)
numVars = chInput.size
recSoftmaxW = ch.Ch(np.random.uniform(0, 1, [nRecComps, numVars]) / numVars)
chRecLogistic = ch.exp(ch.dot(recSoftmaxW, chInput.reshape([numVars, 1])))
chRecSoftmax = chRecLogistic.ravel() / ch.sum(chRecLogistic)
chZRecComps = ch.zeros([numVars, nRecComps])
chZ = ch.zeros([numVars])
recMeans = ch.Ch(np.random.uniform(0, 1, [3, nRecComps]))
recCovars = 0.2
chRecLogLikelihoods = -0.5 * (chZ.reshape([numPixels, 3, 1]) - ch.tile(
recMeans, [numPixels, 1, 1]))**2 - ch.log(
(2 * recCovars) * (1 / (ch.sqrt(recCovars) * np.sqrt(2 * np.pi))))
genZCovars = 0.2
chGenComponentsProbs = ch.Ch(gmm.weights_)
chCompMeans = ch.zeros([nComps, 3])
def optimize_on_joints(j2d,
model,
cam,
img,
prior,
try_both_orient,
body_orient,
n_betas=10,
regs=None,
conf=None,
viz=False):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit
:param body_orient: 3D vector, initialization for the body orientation
:param n_betas: number of shape coefficients considered during optimization
:param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term
:param conf: 14D vector storing the confidence values from the CNN
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the optimized model, its joints projected on image space, the camera translation
"""
t0 = time()
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
cids = range(12) + [13]
# joint ids for SMPL
# SMPL does not have a joint for head, instead we use a vertex for the head
# and append it later.
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# the vertex id for the joint corresponding to the head
head_id = 411
# weights assigned to each joint during optimization;
# the definition of hips in SMPL and LSP is significantly different so set
# their weights to zero
base_weights = np.array(
[1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=np.float64)
if try_both_orient:
flipped_orient = cv2.Rodrigues(body_orient)[0].dot(
cv2.Rodrigues(np.array([0., np.pi, 0]))[0])
flipped_orient = cv2.Rodrigues(flipped_orient)[0].ravel()
orientations = [body_orient, flipped_orient]
else:
orientations = [body_orient]
if try_both_orient:
# store here the final error for both orientations,
# and pick the orientation resulting in the lowest error
errors = []
svs = []
cams = []
for o_id, orient in enumerate(orientations):
# initialize the shape to the mean shape in the SMPL training set
betas = ch.zeros(n_betas)
# initialize the pose by using the optimized body orientation and the
# pose prior
init_pose = np.hstack((orient, prior.weights.dot(prior.means)))
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(
trans=ch.zeros(3),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# make the SMPL joints depend on betas
Jdirs = np.dstack([model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))])
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = global_rigid_transformation(
sv.pose, J_onbetas, model.kintree_table, xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
# add the head joint, corresponding to a vertex...
Jtr = ch.vstack((Jtr, sv[head_id]))
# ... and add the joint id to the list
if o_id == 0:
smpl_ids.append(len(Jtr) - 1)
# update the weights using confidence values
weights = base_weights * conf[
cids] if conf is not None else base_weights
# project SMPL joints on the image plane using the estimated camera
cam.v = Jtr
# data term: distance between observed and estimated joints in 2D
obj_j2d = lambda w, sigma: (
w * weights.reshape((-1, 1)) * GMOf((j2d[cids] - cam[smpl_ids]), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([my_exp(sv.pose[55]), my_exp(-sv.pose[
58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15])])
if viz:
import matplotlib.pyplot as plt
plt.ion()
def on_step(_):
"""Create visualization."""
plt.figure(1, figsize=(10, 10))
plt.subplot(1, 2, 1)
# show optimized joints in 2D
tmp_img = img.copy()
for coord, target_coord in zip(
np.around(cam.r[smpl_ids]).astype(int),
np.around(j2d[cids]).astype(int)):
if (coord[0] < tmp_img.shape[1] and coord[0] >= 0 and
coord[1] < tmp_img.shape[0] and coord[1] >= 0):
cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255])
if (target_coord[0] < tmp_img.shape[1] and
target_coord[0] >= 0 and
target_coord[1] < tmp_img.shape[0] and
target_coord[1] >= 0):
cv2.circle(tmp_img, tuple(target_coord), 3,
[0, 255, 0])
plt.imshow(tmp_img[:, :, ::-1])
plt.draw()
plt.show()
plt.pause(1e-2)
on_step(_)
else:
on_step = None
if regs is not None:
# interpenetration term
sp = SphereCollisions(
pose=sv.pose, betas=sv.betas, model=model, regs=regs)
sp.no_hands = True
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
if regs is not None:
objs['sph_coll'] = 1e3 * sp
ch.minimize(
objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=on_step,
options={'maxiter': 100,
'e_3': .0001,
'disp': 0})
t1 = time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if try_both_orient:
errors.append((objs['j2d'].r**2).sum())
svs.append(sv)
cams.append(cam)
if try_both_orient and errors[0] > errors[1]:
choose_id = 1
else:
choose_id = 0
if viz:
plt.ioff()
return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r)
def my_exp(x):
return alpha * ch.exp(x)
def optimize_on_DensePose(dp_iuv,
op_j2d,
w_j2d,
model,
cam,
prior,
body_orient,
body_trans,
body_init,
n_betas=10,
viz=False,
imageid=1):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit
:param body_orient: 3D vector, initialization for the body orientation
:param n_betas: number of shape coefficients considered during optimization
:param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term
:param conf: 14D vector storing the confidence values from the CNN
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the optimized model, its joints projected on image space, the camera translation
"""
outPath = '/home/xiul/databag/dbfusion/record0/smplify/vid{}'.format(
imageid)
if not os.path.isdir(outPath):
os.mkdir(outPath)
t0 = time()
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
tarIUV = ch.array(dp_iuv)
# initialize the shape to the mean shape in the SMPL training set
betas = ch.zeros(n_betas)
# initialize the pose by using the optimized body orientation and the
# pose prior
init_pose = np.hstack((body_orient, prior.weights.dot(prior.means)))
#init_pose = np.hstack((body_orient, body_init))
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(trans=ch.array(body_trans),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
J_tmpx = MatVecMult(J_reg, sv[:, 0])
J_tmpy = MatVecMult(J_reg, sv[:, 1])
J_tmpz = MatVecMult(J_reg, sv[:, 2])
Jtr = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
#Jtr = J_reg.dot(sv)
cam.v = Jtr
op_j2d = op_j2d * 1280 / 1920
w = 1280
h = 720
reIUV = render_model(sv,
model.f,
w,
h,
cam,
near=0.5,
far=25,
vc=dp_colors,
img=None)
#gaussian_pyramid(input_objective, imshape=None, normalization='SSE', n_levels=3, as_list=False, label=None):
obj_j2d = lambda w, sigma: (w * w_j2d.reshape((-1, 1)) * GMOf(
(op_j2d - cam), sigma))
#input_objective, imshape, normalization, n_levels, as_list
err = tarIUV - reIUV
obj_dense = 100 * gaussian_pyramid(err, n_levels=6, normalization='SSE')
# data term: distance between observed and estimated joints in 2D
# obj_dense = lambda w, sigma: (
# w * GMOf((tarIUV - reIUV).reshape(-1,3), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
if viz:
import matplotlib.pyplot as plt
from time import gmtime, strftime
def on_step(cstep):
"""Create visualization."""
plt.figure(1, figsize=(10, 10))
plt.clf()
plt.subplot(1, 2, 1)
# show optimized joints in 2D
tmp_img = reIUV.r.copy()
tmp_tar = tarIUV.copy()
plt.imshow(tmp_img)
for j1, j2 in zip(cam.r, op_j2d):
plt.plot([j1[0], j2[0]], [j1[1], j2[1]], 'r')
plt.subplot(1, 2, 2)
plt.imshow(tmp_tar)
plt.draw()
plt.savefig(os.path.join(
outPath, '{}.png'.format(strftime("%d_%H_%M_%S", gmtime()))),
bbox_inches='tight')
on_step(stepI)
else:
on_step = None
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78, 4.78, 4.78, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1, 5, 5, 5])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
if stage >= 4:
objs['dense'] = obj_dense
if stage <= 4:
objs['j2d'] = obj_j2d(1, 100)
else:
objs['j2d'] = obj_j2d(0.1, 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
ch.minimize(objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=on_step,
options={
'maxiter': 10000,
'e_3': .001,
'disp': 1
})
t1 = time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if viz:
plt.ioff()
def objFun(vs):
vs = np.array(vs)
res = []
for vs_it, vs_i in enumerate(vs):
changevars(vs_i, free_variables)
import densecrf_model
vis_im = np.array(renderer.indices_image == 1).copy().astype(
np.bool)
bound_im = renderer.boundarybool_image.astype(np.bool)
segmentation, Q = densecrf_model.crfInference(
rendererGT.r, vis_im, bound_im, [0.75, 0.25, 0.01],
resultDir + 'imgs/crf/Q_' + str(test_i) + '_it' + str(vs_it))
vColor = color
if updateColor:
if np.sum(segmentation == 0) > 5:
segmentRegion = segmentation == 0
vColor = np.median(rendererGT.reshape(
[-1, 3])[segmentRegion.ravel()],
axis=0) * 1.4
vColor = vColor / max(np.max(vColor), 1.)
chVColors[:] = vColor
chSHLightCoeffs[:] = lightCoeffs
variances = stds**2
fgProb = ch.exp(-(renderer - rendererGT)**2 /
(2 * variances)) * (1. /
(stds * np.sqrt(2 * np.pi)))
h = renderer.r.shape[0]
w = renderer.r.shape[1]
occProb = np.ones([h, w])
bgProb = np.ones([h, w])
errorFun = -ch.sum(
ch.log(vis_im[:, :, None] *
((Q[0].reshape([h, w, 1]) * fgProb) +
(Q[1].reshape([h, w]) * occProb +
Q[2].reshape([h, w]) * bgProb)[:, :, None]) +
(1 - vis_im[:, :, None]))) / (h * w)
if minAppLight:
options = {'disp': False, 'maxiter': 10}
def cb(_):
print("Error: " + str(errorFun.r))
ch.minimize({'raw': errorFun},
bounds=None,
method=method,
x0=free_variables_app_light,
callback=cb,
options=options)
res = res + [errorFun.r.reshape([1, 1])]
return np.vstack(res)
def optimize_on_joints3D(model, joints3D, opt_shape=False, viz=True):
"""Fit the model to the given set of 3D joints
:param model: initial SMPL model ===> is modified after optimization
:param joints3D: 3D joint locations [16 x 3]
:param opt_shape: boolean, if True optimizes for shape parameter betas
:param viz: boolean, if True enables visualization during optimization
"""
t0 = time()
if joints3D.shape[0] == 16:
obj_joints3D = lambda w, sigma: (w * GMOf(
(joints3D - model.J_transformed[get_indices_16()]), sigma))
elif joints3D.shape[0] == 24:
obj_joints3D = lambda w, sigma: (w * GMOf(
(joints3D - model.J_transformed), sigma))
else:
raise ('How many joints?')
# Create the pose prior (GMM over CMU)
prior = MaxMixtureCompletePrior(n_gaussians=8).get_gmm_prior()
pprior = lambda w: w * prior(model.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
my_exp = lambda x: 10 * chumpy.exp(x)
obj_angle = lambda w: w * chumpy.concatenate([
my_exp(model.pose[55]),
my_exp(-model.pose[58]),
my_exp(-model.pose[12]),
my_exp(-model.pose[15])
])
# Visualization at optimization step
if viz:
def on_step(_):
"""Draw a visualization."""
plt.figure(1, figsize=(5, 5))
renderBody(model)
plt.draw()
plt.pause(1e-3)
else:
on_step = None
# weight configuration (pose and shape: original values as in SMPLify)
# the first list contains the weights for the pose prior,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
print('Initial: error(joints3D) = %.2f' %
(obj_joints3D(100, 100).r**2).sum())
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (wpose, wbetas) in enumerate(opt_weights):
objs = {}
objs['joints3D'] = obj_joints3D(100., 100)
objs['pose'] = pprior(wpose)
objs['pose_exp'] = obj_angle(0.317 * wpose)
if opt_shape:
objs['betas'] = wbetas * model.betas
chumpy.minimize(objs,
x0=[model.pose, model.betas],
method='dogleg',
callback=on_step,
options={
'maxiter': 1000,
'e_3': .0001,
'disp': 0
})
else:
chumpy.minimize(objs,
x0=[model.pose],
method='dogleg',
callback=on_step,
options={
'maxiter': 1000,
'e_3': .0001,
'disp': 0
})
print('Stage %d: error(joints3D) = %.2f' %
(stage, (objs['joints3D'].r**2).sum()))
print('\nElapsed theta fitting (%d joints): %.2f sec.' %
(joints3D.shape[0], (time() - t0)))
def optimize_on_vertices(model,
vertices,
joints3D=np.zeros(1),
weights_corr=np.zeros(1),
vertices_cross_corr=np.zeros(1),
indices_cross_corr=np.zeros(1),
weights_cross_corr=np.zeros(1),
opt_trans=False,
viz=True):
"""Fit the model to the given set of 3D vertices and 3D joints
:param model: initial SMPL model ===> is modified after optimization
:param vertices: 3D vertex locations to fit [num_vertices x 3]
:param joints3D: 3D joint locations to fit [24 x 3]
:param vertices_cross_corr, indices_cross_corr, weights_cross_corr:
:for each point in vertices_cross_corr, we have the index of its corresponding smpl vertex and the weight
:for this correspondence
:param opt_trans: boolean, if True optimizes only translation
:param viz: boolean, if True enables visualization during optimization
"""
t0 = time()
# Optimization term on the joints3D distance
if joints3D.shape[0] > 1:
if joints3D.shape[0] == 16:
obj_joints3d = lambda w, sigma: (w * GMOf(
(joints3D - model.J_transformed[get_indices_16()]), sigma))
elif joints3D.shape[0] == 24:
obj_joints3d = lambda w, sigma: (w * GMOf(
(joints3D - model.J_transformed), sigma))
else:
raise ('How many joints?')
# data term: distance between observed and estimated points in 3D
if (weights_corr.shape[0] == 1):
weights_corr = np.ones((vertices.shape[0]))
obj_vertices = lambda w, sigma: (w * GMOf(
((vertices.T * weights_corr) - (model.T * weights_corr)).T, sigma))
if (vertices_cross_corr.shape[0] > 1):
smplV = model[indices_cross_corr.astype(int), :]
obj_vertices_cross = lambda w, sigma: (w * GMOf(
((vertices_cross_corr.T * weights_cross_corr) -
(smplV.T * weights_cross_corr)).T, sigma))
# Create the pose prior (GMM over CMU)
prior = MaxMixtureCompletePrior(n_gaussians=8).get_gmm_prior()
pprior = lambda w: w * prior(model.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
my_exp = lambda x: 10 * chumpy.exp(x)
obj_angle = lambda w: w * chumpy.concatenate([
my_exp(model.pose[55]),
my_exp(-model.pose[58]),
my_exp(-model.pose[12]),
my_exp(-model.pose[15])
])
# Visualization at optimization step
if viz:
def on_step(_):
"""Draw a visualization."""
plt.figure(1, figsize=(5, 5))
renderBody(model)
plt.draw()
plt.pause(1e-3)
else:
on_step = None
# weight configuration (pose and shape: original values as in SMPLify)
# the first list contains the weights for the pose prior,
# the second list contains the weights for the shape prior
# the third list contains the weights for the joints3D loss
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1], [5, 5, 5, 5])
print('Initial:')
print('\terror(vertices) = %.2f' % (obj_vertices(100, 100).r**2).sum())
if (joints3D.shape[0] > 1):
print('\terror(joints3d) = %.2f' % (obj_joints3d(100, 100).r**2).sum())
if (vertices_cross_corr.shape[0] > 1):
print('\terror(vertices_cross) = %.2f' %
(obj_vertices_cross(100, 100).r**2).sum())
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (wpose, wbetas, wjoints3D) in enumerate(opt_weights):
print('Stage %d' % stage)
objs = {}
if (joints3D.shape[0] > 1):
objs['joints3D'] = wjoints3D * obj_joints3d(100., 100)
objs['vertices'] = obj_vertices(100., 100)
if (vertices_cross_corr.shape[0] > 1):
objs['vertices_cross'] = obj_vertices_cross(100., 100)
objs['pose'] = pprior(wpose)
objs['pose_exp'] = obj_angle(0.317 * wpose)
objs['betas'] = wbetas * model.betas
if opt_trans:
chumpy.minimize(objs,
x0=[model.trans],
method='dogleg',
callback=on_step,
options={
'maxiter': 1000,
'e_3': .0001,
'disp': 0
})
else:
chumpy.minimize(objs,
x0=[model.pose, model.betas],
method='dogleg',
callback=on_step,
options={
'maxiter': 1000,
'e_3': .0001,
'disp': 0
})
print('\terror(vertices) = %.2f' % (objs['vertices'].r**2).sum())
if (joints3D.shape[0] > 1):
print('\terror(joints3D) = %.2f' % (objs['joints3D'].r**2).sum())
if (vertices_cross_corr.shape[0] > 1):
print('\terror(vertices_cross) = %.2f' %
(objs['vertices_cross'].r**2).sum())
print('Elapsed iteration %.2f sec.' % (time() - t0))
def optimize_on_joints(j2d,
model,
cam,
img,
prior,
try_both_orient,
body_orient,
n_betas=10,
regs=None,
conf=None,
viz=False):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit
:param body_orient: 3D vector, initialization for the body orientation
:param n_betas: number of shape coefficients considered during optimization
:param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term
:param conf: 14D vector storing the confidence values from the CNN
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the optimized model, its joints projected on image space, the camera translation
"""
t0 = time()
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
cids = range(12) + [13]
# joint ids for SMPL
# SMPL does not have a joint for head, instead we use a vertex for the head
# and append it later.
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# the vertex id for the joint corresponding to the head
head_id = 411
# weights assigned to each joint during optimization;
# the definition of hips in SMPL and LSP is significantly different so set
# their weights to zero
base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1],
dtype=np.float64)
if try_both_orient:
flipped_orient = cv2.Rodrigues(body_orient)[0].dot(
cv2.Rodrigues(np.array([0., np.pi, 0]))[0])
flipped_orient = cv2.Rodrigues(flipped_orient)[0].ravel()
orientations = [body_orient, flipped_orient]
else:
orientations = [body_orient]
if try_both_orient:
# store here the final error for both orientations,
# and pick the orientation resulting in the lowest error
errors = []
svs = []
cams = []
for o_id, orient in enumerate(orientations):
# initialize the shape to the mean shape in the SMPL training set
betas = ch.zeros(n_betas)
# initialize the pose by using the optimized body orientation and the
# pose prior
init_pose = np.hstack((orient, prior.weights.dot(prior.means)))
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(trans=ch.zeros(3),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# make the SMPL joints depend on betas
Jdirs = np.dstack([
model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))
])
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = global_rigid_transformation(sv.pose,
J_onbetas,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
# add the head joint, corresponding to a vertex...
Jtr = ch.vstack((Jtr, sv[head_id]))
# ... and add the joint id to the list
if o_id == 0:
smpl_ids.append(len(Jtr) - 1)
# update the weights using confidence values
weights = base_weights * conf[
cids] if conf is not None else base_weights
# project SMPL joints on the image plane using the estimated camera
cam.v = Jtr
# data term: distance between observed and estimated joints in 2D
obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf(
(j2d[cids] - cam[smpl_ids]), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
if viz:
import matplotlib.pyplot as plt
plt.ion()
def on_step(_):
"""Create visualization."""
plt.figure(1, figsize=(10, 10))
plt.subplot(1, 2, 1)
# show optimized joints in 2D
tmp_img = img.copy()
for coord, target_coord in zip(
np.around(cam.r[smpl_ids]).astype(int),
np.around(j2d[cids]).astype(int)):
if (coord[0] < tmp_img.shape[1] and coord[0] >= 0
and coord[1] < tmp_img.shape[0] and coord[1] >= 0):
cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255])
if (target_coord[0] < tmp_img.shape[1]
and target_coord[0] >= 0
and target_coord[1] < tmp_img.shape[0]
and target_coord[1] >= 0):
cv2.circle(tmp_img, tuple(target_coord), 3,
[0, 255, 0])
plt.imshow(tmp_img[:, :, ::-1])
plt.draw()
plt.show()
plt.pause(1e-2)
on_step(_)
else:
on_step = None
if regs is not None:
# interpenetration term
sp = SphereCollisions(pose=sv.pose,
betas=sv.betas,
model=model,
regs=regs)
sp.no_hands = True
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
if regs is not None:
objs['sph_coll'] = 1e3 * sp
ch.minimize(objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=on_step,
options={
'maxiter': 100,
'e_3': .0001,
'disp': 0
})
t1 = time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if try_both_orient:
errors.append((objs['j2d'].r**2).sum())
svs.append(sv)
cams.append(cam)
if try_both_orient and errors[0] > errors[1]:
choose_id = 1
else:
choose_id = 0
if viz:
plt.ioff()
return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r, Jtr)
def optimize_on_DensePose(raw_img,
tar_img,
dp_iuv,
op_j2d,
w_j2d,
model,
cam,
cam_old,
prior,
init_trans,
init_pose,
init_betas,
n_betas=10,
viz=False,
imgname='tmp'):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit
:param body_orient: 3D vector, initialization for the body orientation
:param n_betas: number of shape coefficients considered during optimization
:param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term
:param conf: 14D vector storing the confidence values from the CNN
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the optimized model, its joints projected on image space, the camera translation
"""
outPath = os.path.join(ROOT_PATH, 'smplify/{}_stage'.format(imgname))
if not os.path.isdir(outPath):
os.mkdir(outPath)
print outPath
dp_iuv_tar = dp_iuv.copy()
width = dp_iuv_tar.shape[1]
height = dp_iuv_tar.shape[0]
dp_iuv_weight = np.zeros(dp_iuv_tar.shape) + 1
dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 0] = 1
dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 1] = 1
dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 2] = 255.0 / 24
dp_iuv_weight = ch.array(dp_iuv_weight)
t0 = time()
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
tarIUV = ch.array(dp_iuv)
ori_betas = init_betas.copy()
ori_betas_ch = ch.array(ori_betas)
# initialize the shape to the mean shape in the SMPL training set
betas = ch.array(init_betas)
# initialize the pose by using the optimized body orientation and the
# pose prior
pose = ch.array(init_pose)
#init_pose = np.hstack((body_orient, body_init))
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(trans=ch.array(init_trans),
pose=pose,
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=None)
J_tmpx = MatVecMult(J_reg, sv[:, 0])
J_tmpy = MatVecMult(J_reg, sv[:, 1])
J_tmpz = MatVecMult(J_reg, sv[:, 2])
Jtr = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
#Jtr = J_reg.dot(sv)
cam.v = Jtr
reIUV = render_model(sv,
model.f,
width,
height,
cam,
near=0.5,
far=25,
vc=dp_colors,
img=None)
reModel = render_model(sv,
model.f,
width,
height,
cam,
near=0.5,
far=25,
vc=None,
img=None)
fullModel = render_model(sv,
model.f,
raw_img.shape[1],
raw_img.shape[0],
cam_old,
near=0.5,
far=25,
vc=None,
img=None)
#gaussian_pyramid(input_objective, imshape=None, normalization='SSE', n_levels=3, as_list=False, label=None):
def obj_j2d(w, sigma):
return (w * w_j2d.reshape((-1, 1)) * GMOf((op_j2d - cam), sigma))
#input_objective, imshape, normalization, n_levels, as_list
err = (tarIUV - reIUV) * dp_iuv_weight
err_uv = 1 - ch.exp(-err[:, :, 0:2]**2 / 1)
err_inds = 1 - ch.exp(-err[:, :, 2]**2 / 0.001)
err_total = ch.dstack((err_uv, err_inds))
#obj_dense = gaussian_pyramid(err, n_levels=4, normalization='SSE')
obj_dense = gaussian_pyramid(err_total, n_levels=4, normalization=None)
# TODO
# the render error can be further extended to geodistic error.
# data term: distance between observed and estimated joints in 2D
# obj_dense = lambda w, sigma: (
# w * GMOf((tarIUV - reIUV).reshape(-1,3), sigma))
# mixture of gaussians pose prior
def pprior(w):
return w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
def my_exp(x):
return alpha * ch.exp(x)
def obj_angle(w):
return w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
def viz_func(stage_num):
#plt.figure(1, figsize=(10, 10))
# show optimized joints in 2D
tmp_img = reIUV.r.copy()
tmp_tar = tarIUV.copy()
tmp_model = reModel.r.copy()
full_model = fullModel.r.copy()
# w = tmp_tar.shape[1]
# h = tmp_tar.shape[0]
for aa in ax:
for cax in aa:
cax.clear()
ax[0][0].imshow(tar_img)
ax[0][0].imshow(tmp_img, alpha=0.5)
for j1, j2, w_ts in zip(cam.r, op_j2d, w_j2d):
if (w_ts > 0):
ax[0][0].plot([j1[0], j2[0]], [j1[1], j2[1]], 'r')
ax[0][0].set_xlim([0, width])
ax[0][0].set_ylim([height, 0])
ax[0][1].imshow(tar_img)
ax[0][1].imshow(tmp_tar, alpha=0.5)
ax[0][1].set_xlim([0, width])
ax[0][1].set_ylim([height, 0])
ax[0][2].imshow(tar_img)
tmp_model_alpha = np.ones((tmp_model.shape[0], tmp_model.shape[1]))
tmp_model_alpha[tmp_model[:, :, 0] < 1e-2] = 0
tmp_model = np.dstack((tmp_model, tmp_model_alpha))
ax[0][2].imshow(tmp_model)
ax[0][2].set_xlim([0, width])
ax[0][2].set_ylim([height, 0])
tmp_err = err_total.r.copy()
#show error
ax[1][0].imshow(tmp_err[:, :, 0])
ax[1][0].set_xlim([0, width])
ax[1][0].set_ylim([height, 0])
ax[1][1].imshow(tmp_err[:, :, 1])
ax[1][1].set_xlim([0, width])
ax[1][1].set_ylim([height, 0])
ax[1][2].imshow(tmp_err[:, :, 2])
ax[1][2].set_xlim([0, width])
ax[1][2].set_ylim([height, 0])
#show target
ax[2][0].imshow(tmp_tar[:, :, 0])
ax[2][0].set_xlim([0, width])
ax[2][0].set_ylim([height, 0])
ax[2][1].imshow(tmp_tar[:, :, 1])
ax[2][1].set_xlim([0, width])
ax[2][1].set_ylim([height, 0])
ax[2][2].imshow(tmp_tar[:, :, 2])
ax[2][2].set_xlim([0, width])
ax[2][2].set_ylim([height, 0])
#show reproj
ax[3][0].imshow(tmp_img[:, :, 0])
ax[3][0].set_xlim([0, width])
ax[3][0].set_ylim([height, 0])
ax[3][1].imshow(tmp_img[:, :, 1])
ax[3][1].set_xlim([0, width])
ax[3][1].set_ylim([height, 0])
ax[3][2].imshow(tmp_img[:, :, 2])
ax[3][2].set_xlim([0, width])
ax[3][2].set_ylim([height, 0])
for aa in ax:
aa[0].set_xticks(())
aa[0].set_yticks(())
aa[1].set_xticks(())
aa[1].set_yticks(())
aa[2].set_xticks(())
aa[2].set_yticks(())
#plt.tight_layout()
plt.draw()
plt.savefig(os.path.join(outPath, 'stage-{}.png'.format(stage_num)),
bbox_inches='tight')
full_model_int = full_model.copy()
full_model_int *= 255
full_model_int = full_model_int.astype(np.uint8)
cv2.imwrite(os.path.join(outPath, 'model-{}.png'.format(stage_num)),
full_model_int)
out_params = {
'pose': sv.pose.r,
'shape': sv.betas.r,
'trans': sv.trans.r
}
with open(os.path.join(outPath, 'param-{}.pkl'.format(stage_num)),
'w') as fio:
pickle.dump(out_params, fio, pickle.HIGHEST_PROTOCOL)
#if viz:
if viz and False:
def on_step(cstep):
"""Create visualization."""
# TODO this function is in vis_func
#plt.savefig(os.path.join(outPath,'{}.png'.format(strftime("%d_%H_%M_%S", gmtime()))),bbox_inches='tight')
on_step(stepI)
else:
on_step = None
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
# opt_weights = zip([4.78, 3.78, 2.78, 1.78],
# [5, 5, 5, 5],
# [1, 0.25, 0.10, 0])
opt_weights = zip([4.78], [50], [10])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas, w_joints) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
#objs['dense'] = obj_dense
objs['j2d'] = obj_j2d(w_joints, 50)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
#objs['betas'] = wbetas * (sv.betas - ori_betas_ch)
ch.minimize(
objs,
#x0=[sv.betas, sv.pose, sv.trans],
x0=[sv.pose, sv.trans],
method='dogleg',
callback=on_step,
options={
'maxiter': 10000,
'e_3': .001,
'disp': 1
})
viz_func(254)
t1 = time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if viz and False:
plt.ioff()