def run_single_fit(img,
j2d,
conf,
seg,
model,
init_pose,
init_shape,
n_betas=10,
flength=flength):
"""Run the fit for one specific image.
:param img: h x w x 3 image
:param j2d: 14x2 array of CNN joints
:param conf: 14D vector storing the confidence values from the CNN
:param seg: h x w soft silhouette
:param model: SMPL model
:param init_pose: 72D vector, pose prediction results provided by HMR
:param init_shape: 10D vector, shape prediction results provided by HMR
:param n_betas: number of shape coefficients considered during optimization
:param flength: camera focal length (kept fixed during optimization)
:returns: a tuple containing camera/model parameters
"""
# create the pose prior (GMM over CMU)
prior = MaxMixtureCompletePrior(n_gaussians=8).get_gmm_prior()
# get the mean pose as our initial pose
# init_pose = np.hstack((np.zeros(3), prior.weights.dot(prior.means)))
# estimate the camera parameters
print('**** Run camera optimization')
(cam, body_orient) = initialize_camera(model, j2d, img,
init_pose, flength=flength)
# fit
# print('**** Run body model optimization (on joints)')
# (sv, opt_j2d, t) = optimize_on_joints(j2d, model, cam, img, prior,
# init_pose, init_shape,
# n_betas=n_betas, conf=conf)
print('**** Run body model optimization (on silhouette and joints)')
(sv, opt_j2d, t) = optimize_on_joints_and_silhouette(j2d, seg, model, cam, img, prior,
init_pose, init_shape,
n_betas=n_betas, conf=conf)
# return fit parameters
params = {'cam_t': cam.t.r,
'f': cam.f.r,
'pose': sv.pose.r,
'betas': sv.betas.r}
return params
def run_single_fit(img,
j2d,
conf,
model,
regs=None,
n_betas=10,
flength=5000.,
pix_thsh=25.,
scale_factor=1,
viz=False,
do_degrees=None):
"""Run the fit for one specific image.
:param img: h x w x 3 image
:param j2d: 14x2 array of CNN joints
:param conf: 14D vector storing the confidence values from the CNN
:param model: SMPL model
:param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term
:param n_betas: number of shape coefficients considered during optimization
:param flength: camera focal length (kept fixed during optimization)
:param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D
is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both
the estimated one and its flip)
:param scale_factor: int, rescale the image (for LSP, slightly greater images -- 2x -- help obtain better fits)
:param viz: boolean, if True enables visualization during optimization
:param do_degrees: list of degrees in azimuth to render the final fit when saving results
:returns: a tuple containing camera/model parameters and images with rendered fits
"""
if do_degrees is None:
do_degrees = []
# create the pose prior (GMM over CMU)
prior = MaxMixtureCompletePrior(n_gaussians=8).get_gmm_prior()
# get the mean pose as our initial pose
init_pose = np.hstack((np.zeros(3), prior.weights.dot(prior.means)))
if scale_factor != 1:
img = cv2.resize(
img, (img.shape[1] * scale_factor, img.shape[0] * scale_factor))
j2d[:, 0] *= scale_factor
j2d[:, 1] *= scale_factor
# estimate the camera parameters
(cam, try_both_orient, body_orient) = initialize_camera(model,
j2d,
img,
init_pose,
flength=flength,
pix_thsh=pix_thsh,
viz=viz)
# fit
(sv, opt_j2d, t) = optimize_on_joints(
j2d,
model,
cam,
img,
prior,
try_both_orient,
body_orient,
n_betas=n_betas,
conf=conf,
viz=viz,
regs=regs,
)
h = img.shape[0]
w = img.shape[1]
dist = np.abs(cam.t.r[2] - np.mean(sv.r, axis=0)[2])
images = []
orig_v = sv.r
for deg in do_degrees:
if deg != 0:
aroundy = cv2.Rodrigues(np.array([0, np.radians(deg), 0]))[0]
center = orig_v.mean(axis=0)
new_v = np.dot((orig_v - center), aroundy)
verts = new_v + center
else:
verts = orig_v
# now render
im = (render_model(verts, model.f, w, h, cam, far=20 + dist) *
255.).astype('uint8')
images.append(im)
# return fit parameters
params = {
'cam_t': cam.t.r,
'f': cam.f.r,
'pose': sv.pose.r,
'betas': sv.betas.r
}
return params, images
def optimize_on_joints3D(model, joints3D, opt_shape=False, viz=True):
"""Fit the model to the given set of 3D joints
:param model: initial SMPL model ===> is modified after optimization
:param joints3D: 3D joint locations [16 x 3]
:param opt_shape: boolean, if True optimizes for shape parameter betas
:param viz: boolean, if True enables visualization during optimization
"""
t0 = time()
if joints3D.shape[0] == 16:
obj_joints3D = lambda w, sigma: (w * GMOf(
(joints3D - model.J_transformed[get_indices_16()]), sigma))
elif joints3D.shape[0] == 24:
obj_joints3D = lambda w, sigma: (w * GMOf(
(joints3D - model.J_transformed), sigma))
else:
raise ('How many joints?')
# Create the pose prior (GMM over CMU)
prior = MaxMixtureCompletePrior(n_gaussians=8).get_gmm_prior()
pprior = lambda w: w * prior(model.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
my_exp = lambda x: 10 * chumpy.exp(x)
obj_angle = lambda w: w * chumpy.concatenate([
my_exp(model.pose[55]),
my_exp(-model.pose[58]),
my_exp(-model.pose[12]),
my_exp(-model.pose[15])
])
# Visualization at optimization step
if viz:
def on_step(_):
"""Draw a visualization."""
plt.figure(1, figsize=(5, 5))
renderBody(model)
plt.draw()
plt.pause(1e-3)
else:
on_step = None
# weight configuration (pose and shape: original values as in SMPLify)
# the first list contains the weights for the pose prior,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
print('Initial: error(joints3D) = %.2f' %
(obj_joints3D(100, 100).r**2).sum())
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (wpose, wbetas) in enumerate(opt_weights):
objs = {}
objs['joints3D'] = obj_joints3D(100., 100)
objs['pose'] = pprior(wpose)
objs['pose_exp'] = obj_angle(0.317 * wpose)
if opt_shape:
objs['betas'] = wbetas * model.betas
chumpy.minimize(objs,
x0=[model.pose, model.betas],
method='dogleg',
callback=on_step,
options={
'maxiter': 1000,
'e_3': .0001,
'disp': 0
})
else:
chumpy.minimize(objs,
x0=[model.pose],
method='dogleg',
callback=on_step,
options={
'maxiter': 1000,
'e_3': .0001,
'disp': 0
})
print('Stage %d: error(joints3D) = %.2f' %
(stage, (objs['joints3D'].r**2).sum()))
print('\nElapsed theta fitting (%d joints): %.2f sec.' %
(joints3D.shape[0], (time() - t0)))
def optimize_on_vertices(model,
vertices,
joints3D=np.zeros(1),
weights_corr=np.zeros(1),
vertices_cross_corr=np.zeros(1),
indices_cross_corr=np.zeros(1),
weights_cross_corr=np.zeros(1),
opt_trans=False,
viz=True):
"""Fit the model to the given set of 3D vertices and 3D joints
:param model: initial SMPL model ===> is modified after optimization
:param vertices: 3D vertex locations to fit [num_vertices x 3]
:param joints3D: 3D joint locations to fit [24 x 3]
:param vertices_cross_corr, indices_cross_corr, weights_cross_corr:
:for each point in vertices_cross_corr, we have the index of its corresponding smpl vertex and the weight
:for this correspondence
:param opt_trans: boolean, if True optimizes only translation
:param viz: boolean, if True enables visualization during optimization
"""
t0 = time()
# Optimization term on the joints3D distance
if joints3D.shape[0] > 1:
if joints3D.shape[0] == 16:
obj_joints3d = lambda w, sigma: (w * GMOf(
(joints3D - model.J_transformed[get_indices_16()]), sigma))
elif joints3D.shape[0] == 24:
obj_joints3d = lambda w, sigma: (w * GMOf(
(joints3D - model.J_transformed), sigma))
else:
raise ('How many joints?')
# data term: distance between observed and estimated points in 3D
if (weights_corr.shape[0] == 1):
weights_corr = np.ones((vertices.shape[0]))
obj_vertices = lambda w, sigma: (w * GMOf(
((vertices.T * weights_corr) - (model.T * weights_corr)).T, sigma))
if (vertices_cross_corr.shape[0] > 1):
smplV = model[indices_cross_corr.astype(int), :]
obj_vertices_cross = lambda w, sigma: (w * GMOf(
((vertices_cross_corr.T * weights_cross_corr) -
(smplV.T * weights_cross_corr)).T, sigma))
# Create the pose prior (GMM over CMU)
prior = MaxMixtureCompletePrior(n_gaussians=8).get_gmm_prior()
pprior = lambda w: w * prior(model.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
my_exp = lambda x: 10 * chumpy.exp(x)
obj_angle = lambda w: w * chumpy.concatenate([
my_exp(model.pose[55]),
my_exp(-model.pose[58]),
my_exp(-model.pose[12]),
my_exp(-model.pose[15])
])
# Visualization at optimization step
if viz:
def on_step(_):
"""Draw a visualization."""
plt.figure(1, figsize=(5, 5))
renderBody(model)
plt.draw()
plt.pause(1e-3)
else:
on_step = None
# weight configuration (pose and shape: original values as in SMPLify)
# the first list contains the weights for the pose prior,
# the second list contains the weights for the shape prior
# the third list contains the weights for the joints3D loss
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1], [5, 5, 5, 5])
print('Initial:')
print('\terror(vertices) = %.2f' % (obj_vertices(100, 100).r**2).sum())
if (joints3D.shape[0] > 1):
print('\terror(joints3d) = %.2f' % (obj_joints3d(100, 100).r**2).sum())
if (vertices_cross_corr.shape[0] > 1):
print('\terror(vertices_cross) = %.2f' %
(obj_vertices_cross(100, 100).r**2).sum())
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (wpose, wbetas, wjoints3D) in enumerate(opt_weights):
print('Stage %d' % stage)
objs = {}
if (joints3D.shape[0] > 1):
objs['joints3D'] = wjoints3D * obj_joints3d(100., 100)
objs['vertices'] = obj_vertices(100., 100)
if (vertices_cross_corr.shape[0] > 1):
objs['vertices_cross'] = obj_vertices_cross(100., 100)
objs['pose'] = pprior(wpose)
objs['pose_exp'] = obj_angle(0.317 * wpose)
objs['betas'] = wbetas * model.betas
if opt_trans:
chumpy.minimize(objs,
x0=[model.trans],
method='dogleg',
callback=on_step,
options={
'maxiter': 1000,
'e_3': .0001,
'disp': 0
})
else:
chumpy.minimize(objs,
x0=[model.pose, model.betas],
method='dogleg',
callback=on_step,
options={
'maxiter': 1000,
'e_3': .0001,
'disp': 0
})
print('\terror(vertices) = %.2f' % (objs['vertices'].r**2).sum())
if (joints3D.shape[0] > 1):
print('\terror(joints3D) = %.2f' % (objs['joints3D'].r**2).sum())
if (vertices_cross_corr.shape[0] > 1):
print('\terror(vertices_cross) = %.2f' %
(objs['vertices_cross'].r**2).sum())
print('Elapsed iteration %.2f sec.' % (time() - t0))