def unproject_points(self, uvd, camera_space=False):
cam = ProjectPoints3D(
**{k: getattr(self, k)
for k in self.dterms if hasattr(self, k)})
try:
xy_undistorted_camspace = cv2.undistortPoints(
np.asarray(uvd[:, :2].reshape((1, -1, 2)).copy()),
np.asarray(cam.camera_mtx), cam.k.r)
xyz_camera_space = np.hstack(
(xy_undistorted_camspace.squeeze(), col(uvd[:, 2])))
xyz_camera_space[:, :2] *= col(
xyz_camera_space[:, 2]) # scale x,y by z
if camera_space:
return xyz_camera_space
other_answer = xyz_camera_space - row(cam.view_mtx[:,
3]) # translate
result = other_answer.dot(cam.view_mtx[:, :3]) # rotate
except: # slow way, probably not so good. But doesn't require cv2.undistortPoints.
cam.v = np.ones_like(uvd)
ch.minimize(cam - uvd,
x0=[cam.v],
method='dogleg',
options={'disp': 0})
result = cam.v.r
return result
def rigid_scan_2_mesh_alignment(scan, mesh, visualize=False):
options = {'sparse_solver': lambda A, x: cg(A, x, maxiter=2000)[0]}
options['disp'] = 1.0
options['delta_0'] = 0.1
options['e_3'] = 1e-4
s = ch.ones(1)
r = ch.zeros(3)
R = Rodrigues(r)
t = ch.zeros(3)
trafo_mesh = s*(R.dot(mesh.v.T)).T + t
sampler = sample_from_mesh(scan, sample_type='vertices')
s2m = ScanToMesh(scan, trafo_mesh, mesh.f, scan_sampler=sampler, signed=False, normalize=False)
if visualize:
#Visualization code
mv = MeshViewer()
mv.set_static_meshes([scan])
tmp_mesh = Mesh(trafo_mesh.r, mesh.f)
tmp_mesh.set_vertex_colors('light sky blue')
mv.set_dynamic_meshes([tmp_mesh])
def on_show(_):
tmp_mesh = Mesh(trafo_mesh.r, mesh.f)
tmp_mesh.set_vertex_colors('light sky blue')
mv.set_dynamic_meshes([tmp_mesh])
else:
def on_show(_):
pass
ch.minimize(fun={'dist': s2m, 's_reg': 100*(ch.abs(s)-s)}, x0=[s, r, t], callback=on_show, options=options)
return s,Rodrigues(r),t
def fit_pose(frame, last_smpl, frustum, nohands, viz_rn):
if nohands:
faces = faces_no_hands(frame.smpl.f)
else:
faces = frame.smpl.f
dst_type = cv2.cv.CV_DIST_L2 if cv2.__version__[0] == '2' else cv2.DIST_L2
dist_i = cv2.distanceTransform(np.uint8(frame.mask * 255), dst_type, 5) - 1
dist_i[dist_i < 0] = 0
dist_i[dist_i > 50] = 50
dist_o = cv2.distanceTransform(255 - np.uint8(frame.mask * 255), dst_type,
5)
dist_o[dist_o > 50] = 50
rn_m = ColoredRenderer(camera=frame.camera,
v=frame.smpl,
f=faces,
vc=np.ones_like(frame.smpl),
frustum=frustum,
bgcolor=0,
num_channels=1)
E = {
'mask':
gaussian_pyramid(rn_m * dist_o * 100. + (1 - rn_m) * dist_i,
n_levels=4,
normalization='size') * 80.,
'2dpose':
GMOf(frame.pose_obj, 100),
'prior':
frame.pose_prior_obj * 4.,
'sp':
frame.collision_obj * 1e3,
}
if last_smpl is not None:
E['last_pose'] = GMOf(frame.smpl.pose - last_smpl.pose, 0.05) * 50.
E['last_trans'] = GMOf(frame.smpl.trans - last_smpl.trans, 0.05) * 50.
if nohands:
x0 = [
frame.smpl.pose[range(21) + range(27, 30) + range(36, 60)],
frame.smpl.trans
]
else:
x0 = [
frame.smpl.pose[range(21) + range(27, 30) + range(36, 72)],
frame.smpl.trans
]
ch.minimize(E,
x0,
method='dogleg',
options={
'e_3': .01,
},
callback=get_cb(viz_rn, frame))
def compute_FLAME_params(source_path, params_out_fname, flame_model_fname,
template_fname):
'''
Load a template and an existing animation sequence in "zero pose" and compute the FLAME shape, jaw pose, and expression paramters.
Outputs one set of shape paramters for the entire sequence, and pose and expression parameters for each frame.
:param source_path: path of the animation sequence (files must be provided in OBJ file format)
:param params_out_fname output path of the FLAME paramters file
:param flame_model_fname: path of the FLAME model
:param template_fname "zero pose" template used to generate the sequence
'''
if not os.path.exists(os.path.dirname(params_out_fname)):
os.makedirs(os.path.dirname(params_out_fname))
# Load sequence files
sequence_fnames = sorted(glob.glob(os.path.join(source_path, '*.obj')))
num_frames = len(sequence_fnames)
if num_frames == 0:
print('No sequence meshes found')
return
model = load_model(flame_model_fname)
print('Optimize for template identity parameters')
template_mesh = Mesh(filename=template_fname)
ch.minimize(template_mesh.v - model,
x0=[model.betas[:300]],
options={
'sparse_solver': lambda A, x: cg(A, x, maxiter=2000)[0]
})
betas = model.betas.r[:300].copy()
model.betas[:] = 0.
model.v_template[:] = template_mesh.v
model_pose = np.zeros((num_frames, model.pose.shape[0]))
model_exp = np.zeros((num_frames, 100))
for frame_idx in range(num_frames):
print('Process frame %d/%d' % (frame_idx + 1, num_frames))
model.betas[:] = 0.
model.pose[:] = 0.
frame_vertices = Mesh(filename=sequence_fnames[frame_idx]).v
# Optimize for jaw pose and facial expression
ch.minimize(frame_vertices - model,
x0=[model.pose[6:9], model.betas[300:]],
options={
'sparse_solver': lambda A, x: cg(A, x, maxiter=2000)[0]
})
model_pose[frame_idx] = model.pose.r.copy()
model_exp[frame_idx] = model.betas.r[300:].copy()
np.save(params_out_fname, {
'shape': betas,
'pose': model_pose,
'expression': model_exp
})
def fit_joints(both_joints, n_pose_params=15, shape_sigma=10.0,
save_filename=None):
"""
Fits the MANO model to hand joint 3D locations
both_jonts: tuple of length 2, 21 joints per hand, e.g. output of ContactPose.hand_joints()
n_pose_params: number of pose parameters (excluding 3 global rotation params)
shape_sigma: reciprocal of shape regularization strength
save_filename: file where the fitting output will be saved in JSON format
"""
mano_params = []
for hand_idx, joints in enumerate(both_joints):
if joints is None: # hand is not present
mano_params.append(mano_param_dict(n_pose_params)) # dummy
continue
cp_joints = openpose2mano(joints)
# MANO model
m = mutils.load_mano_model(MANOFitter._mano_dicts[hand_idx],
ncomps=n_pose_params, flat_hand_mean=False)
m.betas[:] = np.zeros(m.betas.size)
m.pose[:] = np.zeros(m.pose.size)
mano_joints = mano_joints_with_fingertips(m)
mano_joints_np = np.array([[float(mm) for mm in m] for m in mano_joints])
# align palm
cp_palm = get_palm_joints(np.asarray(cp_joints))
mano_palm = get_palm_joints(np.asarray(mano_joints_np))
mTc = register_pcs(cp_palm, mano_palm)
cp_joints = np.dot(mTc, np.vstack((cp_joints.T, np.ones(len(cp_joints)))))
cp_joints = cp_joints[:3].T
cp_joints = ch.array(cp_joints)
# set up objective
objective = [m-c for m,c in zip(mano_joints, cp_joints)]
mean_betas = ch.array(np.zeros(m.betas.size))
objective.append((m.betas - mean_betas) / shape_sigma)
# optimize
ch.minimize(objective, x0=(m.pose, m.betas, m.trans), method='dogleg')
p = mano_param_dict(n_pose_params)
p['pose'] = np.array(m.pose).tolist()
p['betas'] = np.array(m.betas).tolist()
p['valid'] = True
p['mTc']['translation'] = (mTc[:3, 3] - np.array(m.trans)).tolist()
p['mTc']['rotation'] = txq.mat2quat(mTc[:3, :3]).tolist()
mano_params.append(p)
# # to access hand mesh vertices and faces
# vertices = np.array(m.r)
# vertices = mutils.tform_points(np.linalg.inv(mTc), vertices)
# faces = np.array(m.f)
if save_filename is not None:
with open(save_filename, 'w') as f:
json.dump(mano_params, f, indent=4, separators=(',', ':'))
print('{:s} written'.format(save_filename))
return mano_params
def unpose_garment(smpl, v_free, vert_indices=None):
smpl.v_personal[:] = 0
c = smpl[vert_indices]
E = {'v_personal_high': c - v_free}
ch.minimize(E, x0=[smpl.v_personal], options={'e_3': .00001})
smpl.pose[:] = 0
smpl.trans[:] = 0
return Mesh(smpl.r, smpl.f).keep_vertices(vert_indices), np.copy(
np.array(smpl.v_personal))
def fit_consensus(frames, base_smpl, camera, frustum, model_data, nohands, icp_count, naked, display):
if nohands:
faces = faces_no_hands(base_smpl.f)
else:
faces = base_smpl.f
vis_rn_b = BoundaryRenderer(camera=camera, frustum=frustum, f=faces, num_channels=1)
vis_rn_m = ColoredRenderer(camera=camera, frustum=frustum, f=faces, vc=np.zeros_like(base_smpl), bgcolor=1,
num_channels=1)
model_template = Smpl(model_data)
model_template.betas[:] = base_smpl.betas.r
g_laplace = regularize_laplace()
g_model = regularize_model()
g_symmetry = regularize_symmetry()
face_ids = get_face_vertex_ids()
for step, (w_laplace, w_model, w_symmetry, sigma) in enumerate(zip(
np.linspace(6.5, 4.0, icp_count) if naked else np.linspace(4.0, 2.0, icp_count),
np.linspace(0.9, 0.6, icp_count) if naked else np.linspace(0.6, 0.3, icp_count),
np.linspace(3.6, 1.8, icp_count),
np.linspace(0.06, 0.003, icp_count),
)):
log.info('# Step {}'.format(step))
L = laplacian(model_template.r, base_smpl.f)
delta = L.dot(model_template.r)
w_laplace *= g_laplace.reshape(-1, 1)
w_model *= g_model.reshape(-1, 1)
w_symmetry *= g_symmetry.reshape(-1, 1)
E = {
'laplace': (sp_dot(L, base_smpl.v_shaped_personal) - delta) * w_laplace,
'model': (base_smpl.v_shaped_personal - model_template) * w_model,
'symmetry': (base_smpl.v_personal + np.array([1, -1, -1])
* base_smpl.v_personal[model_data['vert_sym_idxs']]) * w_symmetry,
}
log.info('## Matching rays with contours')
for current, f in enumerate(tqdm(frames)):
E['silh_{}'.format(current)] = ray_objective(f, sigma, base_smpl, camera, vis_rn_b, vis_rn_m)
#paper 2
E['face_{}'.format(current)] = ray_face(f, sigma, base_smpl, camera, face_ids)
log.info('## Run optimization')
ch.minimize(
E,
[base_smpl.v_personal, model_template.betas],
method='dogleg',
options={'maxiter': 15, 'e_3': 0.001},
callback=get_cb(frames[0], base_smpl, camera, frustum) if display else None
)
def _fit_rot_trans(
model, # pylint: disable=too-many-arguments, too-many-locals
j2d,
center,
init_t,
init_pose,
conf,
flength):
"""Find a rotation and translation to minimize the projection error of the pose."""
opt_pose = _ch.array(init_pose) # pylint: disable=no-member
opt_trans = _ch.array(init_t) # pylint: disable=no-member
(_, A_global) = _global_rigid_transformation(opt_pose,
model.J,
model.kintree_table,
xp=_ch)
Jtr = _ch.vstack([g[:3, 3] for g in A_global]) + opt_trans # pylint: disable=no-member
cam = _ProjectPoints(f=_np.array([flength, flength]),
rt=_np.zeros(3),
t=_np.zeros(3),
k=_np.zeros(5),
c=center)
cam_3d = _ProjectPoints3D(f=_np.array([flength, flength]),
rt=_np.zeros(3),
t=_np.zeros(3),
k=_np.zeros(5),
c=center)
cam.v = Jtr
cam_3d.v = Jtr
free_variables = [opt_pose[:3],
opt_trans] # Optimize global rotation and translation.
j2d_ids = range(12)
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
#_CID_TORSO = [2, 3, 8, 9]
#torso_smpl_ids = [2, 1, 17, 16]
_ch.minimize(
{
'j2d': (j2d.T[j2d_ids] - cam[smpl_ids]).T * conf[j2d_ids],
# 'dev^2': 1e2 * (opt_trans[2] - init_t[2])
},
x0=free_variables,
method='dogleg',
callback=None, #on_step_vis,
options={
'maxiter': 100,
'e_3': .0001,
'disp': 0
})
_LOGGER.debug("Global rotation: %s, global translation: %s.",
str(opt_pose[:3].r), str(opt_trans.r))
_LOGGER.debug("Points 3D: %s.", str(cam_3d.r[:10, :]))
_LOGGER.debug("Points 2D: %s.", str(cam.r[:10, :]))
return opt_pose[:3].r, opt_trans.r, cam_3d.r[:,
2].min(), cam_3d.r[:,
2].max()
def test_earth():
m = get_earthmesh(trans=ch.array([0, 0, 0]), rotation=ch.zeros(3))
# Create V, A, U, f: geometry, brightness, camera, renderer
V = ch.array(m.v)
A = SphericalHarmonics(vn=VertNormals(v=V, f=m.f),
components=[3., 2., 0., 0., 0., 0., 0., 0., 0.],
light_color=ch.ones(3))
# camera
U = ProjectPoints(v=V,
f=[w, w],
c=[w / 2., h / 2.],
k=ch.zeros(5),
t=ch.zeros(3),
rt=ch.zeros(3))
f = TexturedRenderer(vc=A,
camera=U,
f=m.f,
bgcolor=[0., 0., 0.],
texture_image=m.texture_image,
vt=m.vt,
ft=m.ft,
frustum={
'width': w,
'height': h,
'near': 1,
'far': 20
})
# Parameterize the vertices
translation, rotation = ch.array([0, 0, 8]), ch.zeros(3)
f.v = translation + V.dot(Rodrigues(rotation))
observed = f.r
np.random.seed(1)
# this is reactive
# in the sense that changes to values will affect function which depend on them.
translation[:] = translation.r + np.random.rand(3)
rotation[:] = rotation.r + np.random.rand(3) * .2
# Create the energy
E_raw = f - observed
E_pyr = gaussian_pyramid(E_raw, n_levels=6, normalization='size')
Image.fromarray((observed * 255).astype(np.uint8)).save(
os.path.join(save_dir, "reference.png"))
step = 0
Image.fromarray((f.r * 255).astype(np.uint8)).save(
os.path.join(save_dir, "step_{:05d}.png".format(step)))
print('OPTIMIZING TRANSLATION, ROTATION, AND LIGHT PARMS')
free_variables = [translation, rotation]
ch.minimize({'pyr': E_pyr}, x0=free_variables, callback=create_callback(f))
ch.minimize({'raw': E_raw}, x0=free_variables, callback=create_callback(f))
def reinit_frame(frame, null_pose, nohands, viz_rn):
mylog("# reinit_frame")
if (np.sum(frame.pose_obj.r ** 2) > 625 or np.sum(frame.pose_prior_obj.r ** 2) > 75)\
and np.sum(frame.keypoints[[0, 2, 5, 8, 11], 2]) > 3.:
log.info('Tracking error too large. Re-init frame...')
x0 = [frame.smpl.pose[:3], frame.smpl.trans]
frame.smpl.pose[3:] = null_pose
if frame.keypoints[2, 0] > frame.keypoints[5, 0]:
frame.smpl.pose[0] = 0
frame.smpl.pose[2] = np.pi
E = {
'init_pose': frame.pose_obj[[0, 2, 5, 8, 11]],
}
ch.minimize(E,
x0,
method='dogleg',
options={
'e_3': .1,
},
callback=get_cb(viz_rn, frame))
E = {
'pose': GMOf(frame.pose_obj, 100),
'prior': frame.pose_prior_obj * 8.,
}
x0 = [frame.smpl.trans]
if nohands:
# x0.append(frame.smpl.pose[range(21) + range(27, 30) + range(36, 60)])
x0.append(frame.smpl.pose[list(range(21)) + list(range(27, 30)) +
list(range(36, 60))])
else:
# x0.append(frame.smpl.pose[range(21) + range(27, 30) + range(36, 72)])
x0.append(frame.smpl.pose[list(range(21)) + list(range(27, 30)) +
list(range(36, 72))])
ch.minimize(E,
x0,
method='dogleg',
options={
'e_3': .01,
},
callback=get_cb(viz_rn, frame))
def initialize_camera(model,
j2d,
img,
init_pose,
flength=flength):
"""Initialize camera translation and body orientation
:param model: SMPL model
:param j2d: 14x2 array of CNN joints
:param img: h x w x 3 image
:param init_pose: 72D vector of pose parameters used for initialization
:param flength: camera focal length (kept fixed)
:param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D
is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on
both the estimated one and its flip)
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the estimated camera,
a boolean deciding if both the optimized body orientation and its flip should be
considered, 3D vector for the body orientation
"""
# optimize camera translation and body orientation based on torso joints
# LSP torso ids:
# 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder
torso_cids = [2, 3, 8, 9]
# corresponding SMPL torso ids
torso_smpl_ids = [2, 1, 17, 16]
center = np.array([img.shape[1] / 2, img.shape[0] / 2])
rt = ch.zeros(3) # initial camera rotation
init_t = guess_init(model, flength, j2d, init_pose)
t = ch.array(init_t) # initial camera translation
opt_pose = ch.array(init_pose)
_, A_global = global_rigid_transformation(opt_pose, model.J, model.kintree_table, xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
# initialize the camera and project SMPL joints
cam = ProjectPoints(f=np.array([flength, flength]), rt=rt, t=t, k=np.zeros(5), c=center)
cam.v = Jtr # project SMPL joints
# optimize for camera translation and body orientation
free_variables = [cam.t, opt_pose[:3]]
ch.minimize(
# data term defined over torso joints...
{'cam': j2d[torso_cids] - cam[torso_smpl_ids],
# ...plus a regularizer for the camera translation
'cam_t': 1e2 * (cam.t[2] - init_t[2])},
x0=free_variables, method='dogleg', callback=None,
options={'maxiter': 100, 'e_3': .0001, 'disp': 0})
return cam, opt_pose[:3].r
def compute_approx_scale(lmk_3d,
model,
lmk_face_idx,
lmk_b_coords,
opt_options=None):
""" function: compute approximate scale to align scan and model
input:
lmk_3d: input landmark 3d, in shape (N,3)
model: FLAME face model
lmk_face_idx, lmk_b_coords: landmark embedding, in face indices and barycentric coordinates
opt_options: optimizaton options
output:
model.r: fitted result vertices
model.f: fitted result triangulations (fixed in this code)
parms: fitted model parameters
"""
scale = ch.ones(1)
scan_lmks = scale * ch.array(lmk_3d)
model_lmks = mesh_points_by_barycentric_coordinates(
model, model.f, lmk_face_idx, lmk_b_coords)
lmk_err = scan_lmks - model_lmks
# options
if opt_options is None:
print("fit_lmk3d(): no 'opt_options' provided, use default settings.")
import scipy.sparse as sp
opt_options = {}
opt_options['disp'] = 1
opt_options['delta_0'] = 0.1
opt_options['e_3'] = 1e-4
opt_options['maxiter'] = 2000
sparse_solver = lambda A, x: sp.linalg.cg(
A, x, maxiter=opt_options['maxiter'])[0]
opt_options['sparse_solver'] = sparse_solver
# on_step callback
def on_step(_):
pass
ch.minimize(fun=lmk_err,
x0=[scale, model.trans, model.pose[:3]],
method='dogleg',
callback=on_step,
options=opt_options)
return scale.r
def unproject_points(self, uvd, camera_space=False):
cam = ProjectPoints3D(**{k: getattr(self, k) for k in self.dterms if hasattr(self, k)})
try:
xy_undistorted_camspace = cv2.undistortPoints(np.asarray(uvd[:,:2].reshape((1,-1,2)).copy()), np.asarray(cam.camera_mtx), cam.k.r)
xyz_camera_space = np.hstack((xy_undistorted_camspace.squeeze(), col(uvd[:,2])))
xyz_camera_space[:,:2] *= col(xyz_camera_space[:,2]) # scale x,y by z
if camera_space:
return xyz_camera_space
other_answer = xyz_camera_space - row(cam.view_mtx[:,3]) # translate
result = other_answer.dot(cam.view_mtx[:,:3]) # rotate
except: # slow way, probably not so good. But doesn't require cv2.undistortPoints.
cam.v = np.ones_like(uvd)
ch.minimize(cam - uvd, x0=[cam.v], method='dogleg', options={'disp': 0})
result = cam.v.r
return result
def lift2Dto3D(projPtsGT,
camMat,
filename,
img,
JVis=np.ones((21, ), dtype=np.float32),
trans=None,
beta=None,
wrist3D=None,
withPoseCoeff=True,
weights=1.0,
relDepGT=None,
rel3DCoordGT=None,
rel3DCoordNormGT=None,
img2DGT=None,
outDir=None,
poseCoeffInit=None,
transInit=None,
betaInit=None):
loss = {}
if withPoseCoeff:
numComp = 30
m, poseCoeffCh, betaCh, transCh, fullposeCh = getHandModelPoseCoeffs(
numComp)
if poseCoeffInit is not None:
poseCoeffCh[:] = poseCoeffInit
if transInit is not None:
transCh[:] = transInit
if betaInit is not None:
betaCh[:] = betaInit
freeVars = [poseCoeffCh]
if beta is None:
freeVars = freeVars + [betaCh]
loss['shape'] = 1e2 * betaCh
else:
betaCh[:] = beta
if trans is None:
freeVars = freeVars + [transCh]
else:
transCh[:] = trans
# loss['pose'] = 0.5e2 * poseCoeffCh[3:]/stdPCACoeff[:numComp]
thetaConstMin, thetaConstMax = Constraints().getHandJointConstraints(
fullposeCh[3:])
loss['constMin'] = 5e2 * thetaConstMin
loss['constMax'] = 5e2 * thetaConstMax
loss['invalidTheta'] = 1e3 * fullposeCh[Constraints().invalidThetaIDs]
else:
m, rotCh, jointsCh, transCh, betaCh = getHandModel()
thetaConstMin, thetaConstMax = Constraints().getHandJointConstraints(
jointsCh)
loss['constMin'] = 5e3 * thetaConstMin
loss['constMax'] = 5e3 * thetaConstMax
validTheta = jointsCh[Constraints().validThetaIDs[3:] - 3]
freeVars = [validTheta, rotCh]
if beta is None:
freeVars = freeVars + [betaCh]
loss['shape'] = 0.5e2 * betaCh
else:
betaCh[:] = beta
if trans is None:
freeVars = freeVars + [transCh]
else:
transCh[:] = trans
if relDepGT is not None:
relDepPred = m.J_transformed[:, 2] - m.J_transformed[0, 2]
loss['relDep'] = (relDepPred - relDepGT) * weights[:, 0] * 5e1
if rel3DCoordGT is not None:
rel3DCoordPred = m.J_transformed - m.J_transformed[0:1, :]
loss['rel3DCoord'] = (rel3DCoordPred - rel3DCoordGT) * np.tile(
weights[:, 0:1], [1, 3]) * 5e1
if rel3DCoordNormGT is not None:
rel3DCoordPred = m.J_transformed[jointsMap][
1:, :] - m.J_transformed[jointsMap][0:1, :]
rel3DCoordPred = rel3DCoordPred / ch.expand_dims(
ch.sqrt(ch.sum(ch.square(rel3DCoordPred), axis=1)), axis=1)
loss['rel3DCoordNorm'] = (
1. - ch.sum(rel3DCoordPred * rel3DCoordNormGT, axis=1)) * 1e4
# loss['rel3DCoordNorm'] = \
# (rel3DCoordNormGT*ch.expand_dims(ch.sum(rel3DCoordPred*rel3DCoordNormGT, axis=1), axis=1) - rel3DCoordPred) * 1e2#5e2
projPts = utilsEval.chProjectPoints(m.J_transformed, camMat,
False)[jointsMap]
JVis = np.tile(np.expand_dims(JVis, 1), [1, 2])
loss['joints2D'] = (projPts - projPtsGT) * JVis * weights * 1e0
loss['joints2DClip'] = clipIden(projPts - projPtsGT) * JVis * weights * 1e1
if wrist3D is not None:
dep = wrist3D[2]
if dep < 0:
dep = -dep
loss['wristDep'] = (m.J_transformed[0, 2] - dep) * 1e2
# vis_mesh(m)
render = False
def cbPass(_):
pass
# print(loss['joints'].r)
print(filename)
warnings.simplefilter('ignore')
loss['joints2D'] = loss[
'joints2D'] * 1e1 / weights # dont want to use confidence now
if True:
ch.minimize({k: loss[k]
for k in loss.keys() if k != 'joints2DClip'},
x0=freeVars,
callback=cbPass if render else cbPass,
method='dogleg',
options={'maxiter': 50})
else:
manoVis.dump3DModel2DKpsHand(img,
m,
filename,
camMat,
est2DJoints=projPtsGT,
gt2DJoints=img2DGT,
outDir=outDir)
freeVars = [poseCoeffCh[:3], transCh]
ch.minimize({k: loss[k]
for k in loss.keys() if k != 'joints2DClip'},
x0=freeVars,
callback=cbPass,
method='dogleg',
options={'maxiter': 20})
manoVis.dump3DModel2DKpsHand(img,
m,
filename,
camMat,
est2DJoints=projPtsGT,
gt2DJoints=img2DGT,
outDir=outDir)
freeVars = [poseCoeffCh[3:]]
ch.minimize({k: loss[k]
for k in loss.keys() if k != 'joints2DClip'},
x0=freeVars,
callback=cb if render else cbPass,
method='dogleg',
options={'maxiter': 20})
manoVis.dump3DModel2DKpsHand(img,
m,
filename,
camMat,
est2DJoints=projPtsGT,
gt2DJoints=img2DGT,
outDir=outDir)
freeVars = [poseCoeffCh, transCh]
if beta is None:
freeVars = freeVars + [betaCh]
ch.minimize({k: loss[k]
for k in loss.keys() if k != 'joints2DClip'},
x0=freeVars,
callback=cb if render else cbPass,
method='dogleg',
options={'maxiter': 20})
if False:
open3dVisualize(m)
else:
manoVis.dump3DModel2DKpsHand(img,
m,
filename,
camMat,
est2DJoints=projPtsGT,
gt2DJoints=img2DGT,
outDir=outDir)
# vis_mesh(m)
joints3D = m.J_transformed.r[jointsMap]
# print(betaCh.r)
# print((relDepPred.r - relDepGT))
return joints3D, poseCoeffCh.r.copy(), betaCh.r.copy(), transCh.r.copy(
), loss['joints2D'].r.copy(), m.r.copy()
def lift2Dto3DMultiFrame(projPtsGT,
camMat,
filename,
JVis=np.ones((21, ), dtype=np.float32),
trans=None,
beta=None,
wrist3D=None,
withPoseCoeff=True,
weights=None,
relDepGT=None,
rel3DCoordGT=None,
isOpenGLCoord=False,
transInit=None,
rotInit=None,
globalPoseCoeffInit=None,
betaInit=None):
'''
:param projPtsGT:
:param camMat:
:param filename:
:param JVis:
:param trans:
:param beta:
:param wrist3D:
:param withPoseCoeff:
:param weights: 21x1 array
:param relDepGT:
:param rel3DCoordGT: always in opencv
:param isOpenGLCoord:
:return: always in opencv
'''
loss = {}
numFrames = projPtsGT.shape[0]
if weights is None:
weights = [np.ones((21, 1), dtype=np.float32)] * numFrames
numComp = 30
mList, chRotList, chGlobalPoseCoeff, chTransList, chGlobalBeta = getHandModelPoseCoeffsMultiFrame(
numComp, numFrames, isOpenGLCoord)
freeVars = [chGlobalPoseCoeff]
if beta is None:
freeVars = freeVars + [chGlobalBeta]
loss['shape'] = 1e2 * chGlobalBeta
else:
chGlobalBeta[:] = beta
if betaInit is not None:
chGlobalBeta[:] = betaInit
if trans is None:
freeVars = freeVars + chTransList
else:
for i, t in enumerate(chTransList):
t[:] = trans[i]
if transInit is not None:
for i in range(len(chTransList)):
chTransList[i][:] = transInit[i]
if rotInit is not None:
for i in range(len(chRotList)):
chRotList[i][:] = rotInit[i]
if globalPoseCoeffInit is not None:
chGlobalPoseCoeff[:] = globalPoseCoeffInit
freeVars = freeVars + chRotList
# loss['pose'] = 0.5e2 * poseCoeffCh[3:]/stdPCACoeff[:numComp]
fullposeCh = mList[0].fullpose ##
thetaConstMin, thetaConstMax = Constraints().getHandJointConstraints(
fullposeCh[3:])
loss['constMin'] = 5e2 * thetaConstMin
loss['constMax'] = 5e2 * thetaConstMax
loss['invalidTheta'] = 5e2 * fullposeCh[Constraints().invalidThetaIDs]
if relDepGT is not None:
for i in range(numFrames):
relDepPred = mList[i].J_transformed[:,
2] - mList[i].J_transformed[0,
2]
loss['relDep_%d' %
(i)] = (relDepPred - relDepGT[i]) * weights[:, 0] * 5e1
coordChangeMat = np.array([[1., 0., 0.], [0, -1., 0.], [0., 0., -1.]],
dtype=np.float32)
if rel3DCoordGT is not None:
for i in range(numFrames):
rel3DCoordPred = mList[i].J_transformed - mList[i].J_transformed[
0:1, :]
if isOpenGLCoord:
rel3DCoordPred = coordChangeMat.dot(coordChangeMat)
loss['rel3DCoord_%d' %
(i)] = (rel3DCoordPred - rel3DCoordGT[i]) * np.tile(
weights[i][:, 0:1], [1, 3]) * 5e1
for i in range(numFrames):
projPts = utilsEval.chProjectPoints(mList[i].J_transformed, camMat,
isOpenGLCoord)[jointsMap]
# JVis = np.tile(np.expand_dims(JVis, 1), [1, 2])
loss['joints2D_%d' %
(i)] = (projPts - projPtsGT[i]
) # * np.tile(weights[i][:,0:1], [1,2])# * 1e1
render = False
def cbPass(_):
pass
# print(loss['joints'].r)
for i in range(numFrames):
loss['joints2D_%d' %
(i)] = loss['joints2D_%d' %
(i)] * np.tile(weights[i][:, 0:1], [1, 2]) * 1e1
ch.minimize(loss,
x0=freeVars,
callback=cbPass,
method='dogleg',
options={'maxiter': 12})
# vis_mesh(mList[0])
# vis_mesh(m)
joints3DList = []
for i in range(numFrames):
joints3D = mList[i].J_transformed.r[jointsMap]
if isOpenGLCoord:
joints3D = joints3D.dot(coordChangeMat)
joints3DList.append(joints3D)
# fullpose list
fullposeList = []
betaList = []
transList = []
for m in mList:
fullposeList.append(m.fullpose.r)
betaList.append(m.betas.r)
transList.append(m.trans.r)
# print(betaCh.r)
# print((relDepPred.r - relDepGT))
return np.stack(
joints3DList, axis=0
), fullposeList, betaList, transList, chGlobalPoseCoeff.r.copy(), mList
def inverseKinematicCh(pts3D, fullposeInit = np.zeros((48,), dtype=np.float32),
transInit = np.array([0., 0., -0.45]), betaInit = np.zeros((10,), dtype=np.float32)):
'''
Performs inverse kinematic optimization when given the 3d points
3D points --> MANO params
:param pts3D:
:param fullposeInit:
:param transInit:
:param betaInit:
:return: fullpose, trans and beta vector
'''
import chumpy as ch
from mano.chumpy.smpl_handpca_wrapper_HAND_only import load_model, load_model_withInputs
from ghope.constraints import Constraints
assert pts3D.shape == (21,3)
constraints = Constraints()
validTheta = ch.zeros((len(constraints.validThetaIDs,)))
fullposeCh = constraints.fullThetaMat.dot(ch.expand_dims(validTheta,1))[:,0]
transCh = ch.array(undo_chumpy(transInit))
betaCh = ch.array(undo_chumpy(betaInit))
pts3DGT = pts3D
m = load_model_withInputs(MANO_MODEL_PATH, fullposeCh, transCh, betaCh, ncomps=6, flat_hand_mean=True)
if fullposeInit.shape == (16,3,3):
fpList = []
for i in range(16):
fpList.append(cv2.Rodrigues(fullposeInit[i])[0][:, 0])
validTheta[:] = np.concatenate(fpList, 0)[constraints.validThetaIDs]
else:
validTheta[:] = undo_chumpy(fullposeInit[constraints.validThetaIDs])
loss = {}
projPts = m.J_transformed
projPtsGT = pts3DGT
loss['joints'] = (projPts - projPtsGT)
# m.fullpose[Constraints().invalidThetaIDs] = 0.
thetaConstMin, thetaConstMax = Constraints().getHandJointConstraintsCh(fullposeCh[3:])
# loss['constMin'] = 1e4 * thetaConstMin
# loss['constMax'] = 1e4 * thetaConstMax
freeVars = [validTheta, transCh]
def cbPass(_):
print(np.max(np.abs(m.J_transformed - pts3DGT)))
pass
ch.minimize(loss, x0=freeVars, callback=cbPass, method='dogleg', options={'maxiter': 10})
if True:
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(m.J_transformed[:, 0], m.J_transformed[:, 1], m.J_transformed[:, 2], c='r')
ax.scatter(pts3DGT[:, 0], pts3DGT[:, 1], pts3DGT[:, 2], c='b')
plt.show()
return undo_chumpy(m.fullpose), undo_chumpy(m.trans), undo_chumpy(m.betas)
def fit_scan(
scan, # input scan
lmk_3d, # input scan landmarks
model, # model
lmk_face_idx,
lmk_b_coords, # landmark embedding
weights, # weights for the objectives
gmo_sigma, # weight of the robustifier
shape_num=300,
expr_num=100,
opt_options=None):
""" function: fit FLAME model to a 3D scan
input:
scan: input scan
lmk_3d: input landmark 3d, in shape (N,3)
model: FLAME face model
lmk_face_idx, lmk_b_coords: landmark embedding, in face indices and barycentric coordinates
weights: weights for each objective
shape_num, expr_num: numbers of shape and expression compoenents used
opt_options: optimizaton options
output:
model.r: fitted result vertices
model.f: fitted result triangulations (fixed in this code)
parms: fitted model parameters
"""
# variables
shape_idx = np.arange(0, min(
300, shape_num)) # valid shape component range in "betas": 0-299
expr_idx = np.arange(300, 300 + min(
100, expr_num)) # valid expression component range in "betas": 300-399
used_idx = np.union1d(shape_idx, expr_idx)
model.betas[:] = np.random.rand(
model.betas.size) * 0.0 # initialized to zero
model.pose[:] = np.random.rand(
model.pose.size) * 0.0 # initialized to zero
free_variables = [model.trans, model.pose, model.betas[used_idx]]
# weights
print("fit_scan(): use the following weights:")
for kk in weights.keys():
print("fit_scan(): weights['%s'] = %f" % (kk, weights[kk]))
# objectives
# landmark error
lmk_err = landmark_error_3d(mesh_verts=model,
mesh_faces=model.f,
lmk_3d=lmk_3d,
lmk_face_idx=lmk_face_idx,
lmk_b_coords=lmk_b_coords)
# scan-to-mesh distance, measuring the distance between scan vertices and the closest points in the model's surface
sampler = sample_from_mesh(scan, sample_type='vertices')
s2m = ScanToMesh(scan,
model,
model.f,
scan_sampler=sampler,
rho=lambda x: GMOf(x, sigma=gmo_sigma))
# regularizer
shape_err = weights['shape'] * model.betas[shape_idx]
expr_err = weights['expr'] * model.betas[expr_idx]
pose_err = weights['pose'] * model.pose[3:] # exclude global rotation
objectives = {
's2m': weights['s2m'] * s2m,
'lmk': weights['lmk'] * lmk_err,
'shape': shape_err,
'expr': expr_err,
'pose': pose_err
}
# options
if opt_options is None:
print("fit_lmk3d(): no 'opt_options' provided, use default settings.")
import scipy.sparse as sp
opt_options = {}
opt_options['disp'] = 1
opt_options['delta_0'] = 0.1
opt_options['e_3'] = 1e-4
opt_options['maxiter'] = 2000
sparse_solver = lambda A, x: sp.linalg.cg(
A, x, maxiter=opt_options['maxiter'])[0]
opt_options['sparse_solver'] = sparse_solver
# on_step callback
def on_step(_):
pass
# optimize
# step 1: rigid alignment
from time import time
timer_start = time()
print("\nstep 1: start rigid fitting...")
ch.minimize(fun=lmk_err,
x0=[model.trans, model.pose[:3]],
method='dogleg',
callback=on_step,
options=opt_options)
timer_end = time()
print("step 1: fitting done, in %f sec\n" % (timer_end - timer_start))
# step 2: non-rigid alignment
timer_start = time()
print("step 2: start non-rigid fitting...")
ch.minimize(fun=objectives,
x0=free_variables,
method='dogleg',
callback=on_step,
options=opt_options)
timer_end = time()
print("step 2: fitting done, in %f sec\n" % (timer_end - timer_start))
# return results
parms = {
'trans': model.trans.r,
'pose': model.pose.r,
'betas': model.betas.r
}
return model.r, model.f, parms
def estimate_global_pose(landmarks,
key_vids,
model,
cam,
img,
fix_t=False,
viz=False,
SOLVE_FLATER=True):
'''
Estimates the global rotation and translation.
only diff in estimate_global_pose from single_frame_ferrari is that all animals have the same kp order.
'''
# Redefining part names..
part_names = [
'leftEye', 'rightEye', 'chin', 'frontLeftFoot', 'frontRightFoot',
'backLeftFoot', 'backRightFoot', 'tailStart', 'frontLeftKnee',
'frontRightKnee', 'backLeftKnee', 'backRightKnee', 'leftShoulder',
'rightShoulder', 'frontLeftAnkle', 'frontRightAnkle', 'backLeftAnkle',
'backRightAnkle', 'neck', 'TailTip'
]
# Use shoulder to "knee"(elbow) distance. also tail to "knee" if available.
use_names = [
'neck', 'leftShoulder', 'rightShoulder', 'backLeftKnee',
'backRightKnee', 'tailStart', 'frontLeftKnee', 'frontRightKnee'
]
use_ids = [part_names.index(name) for name in use_names]
# These might not be visible
visible = landmarks[:, 2].astype(bool)
use_ids = [id for id in use_ids if visible[id]]
if len(use_ids) < 3:
print('Frontal?..')
use_names += [
'frontLeftAnkle', 'frontRightAnkle', 'backLeftAnkle',
'backRightAnkle'
]
model.pose[1] = np.pi / 2
init_t = estimate_translation(landmarks, key_vids, cam.f[0].r, model)
use_ids = [part_names.index(name) for name in use_names]
use_ids = [id for id in use_ids if visible[id]]
# Setup projection error:
all_vids = np.hstack([key_vids[id] for id in use_ids])
cam.v = model[all_vids]
keypoints = landmarks[use_ids, :2].astype(float)
# Duplicate keypoints for the # of vertices for that kp.
num_verts_per_kp = [len(key_vids[row_id]) for row_id in use_ids]
j2d = np.vstack([
np.tile(kp, (num_rep, 1))
for kp, num_rep in zip(keypoints, num_verts_per_kp)
])
assert (cam.r.shape == j2d.shape)
# SLOW but correct method!!
# remember which ones belongs together,,
group = np.hstack([
index * np.ones(len(key_vids[row_id]))
for index, row_id in enumerate(use_ids)
])
assignments = np.vstack([group == i for i in np.arange(group[-1] + 1)])
num_points = len(use_ids)
proj_error = (ch.vstack([
cam[choice] if np.sum(choice) == 1 else cam[choice].mean(axis=0)
for choice in assignments
]) - keypoints) / np.sqrt(num_points)
# Fast but not matching average:
# Normalization weight
j2d_norm_weights = np.sqrt(
1. / len(use_ids) *
np.vstack([1. / num * np.ones((num, 1)) for num in num_verts_per_kp]))
proj_error_fast = j2d_norm_weights * (cam - j2d)
if fix_t:
obj = {'cam': proj_error_fast}
else:
obj = {
'cam': proj_error_fast,
'cam_t': 1e1 * (model.trans[2] - init_t[2])
}
# Only estimate body orientation
if fix_t:
free_variables = [model.pose[:3]]
else:
free_variables = [model.trans, model.pose[:3]]
if not SOLVE_FLATER:
obj['feq'] = 1e3 * (cam.f[0] - cam.f[1])
# So it's under control
obj['freg'] = 1e1 * (cam.f[0] - 3000) / 1000.
# here without this cam.f goes negative.. (asking margin of 500)
obj['fpos'] = ch.maximum(0, 500 - cam.f[0])
# cam t also has to be positive!
obj['cam_t_pos'] = ch.maximum(0, 0.01 - model.trans[2])
del obj['cam_t']
free_variables.append(cam.f)
if viz:
import matplotlib.pyplot as plt
plt.ion()
def on_step(_):
plt.figure(1, figsize=(5, 5))
plt.cla()
plt.imshow(img[:, :, ::-1])
img_here = render_mesh(Mesh(model.r, model.f), img.shape[1],
img.shape[0], cam)
plt.imshow(img_here)
plt.scatter(j2d[:, 0], j2d[:, 1], c='w')
plt.scatter(cam.r[:, 0], cam.r[:, 1])
plt.draw()
plt.pause(1e-3)
if 'feq' in obj:
print('flength %.1f %.1f, z %.f' %
(cam.f[0], cam.f[1], model.trans[2]))
else:
on_step = None
from time import time
t0 = time()
init_angles = [[0, 0, 0]] #, [1.5,0,0], [1.5,-1.,0]]
scores = np.zeros(len(init_angles))
for i, angle in enumerate(init_angles):
# Init translation
model.trans[:] = init_t
model.pose[:3] = angle
ch.minimize(obj,
x0=free_variables,
method='dogleg',
callback=on_step,
options={
'maxiter': 100,
'e_3': .0001
})
scores[i] = np.sum(obj['cam'].r**2.)
j = np.argmin(scores)
model.trans[:] = init_t
model.pose[:3] = init_angles[j]
ch.minimize(obj,
x0=free_variables,
method='dogleg',
callback=on_step,
options={
'maxiter': 100,
'e_3': .0001
})
print('Took %g' % (time() - t0))
#import pdb; pdb.set_trace()
if viz:
dist = np.mean(model.r, axis=0)[2]
img_here = render_mesh(Mesh(model.r, model.f), img.shape[1],
img.shape[0], cam)
plt.imshow(img[:, :, ::-1])
plt.imshow(img_here)
return model.pose[:3].r, model.trans.r
def initialize_camera(model,
j2d,
img,
init_pose,
flength=5000.,
pix_thsh=25.,
viz=False):
"""Initialize camera translation and body orientation
:param model: SMPL model
:param j2d: 14x2 array of CNN joints
:param img: h x w x 3 image
:param init_pose: 72D vector of pose parameters used for initialization
:param flength: camera focal length (kept fixed)
:param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D
is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both
the estimated one and its flip)
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the estimated camera,
a boolean deciding if both the optimized body orientation and its flip should be considered,
3D vector for the body orientation
"""
# optimize camera translation and body orientation based on torso joints
# LSP torso ids:
# 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder
torso_cids = [2, 3, 8, 9]
# corresponding SMPL torso ids
torso_smpl_ids = [2, 1, 17, 16]
center = np.array([img.shape[1] / 2, img.shape[0] / 2])
# initialize camera rotation
rt = ch.zeros(3)
# initialize camera translation
_LOGGER.info('initializing translation via similar triangles')
init_t = guess_init(model, flength, j2d, init_pose)
t = ch.array(init_t)
# check how close the shoulder joints are
try_both_orient = np.linalg.norm(j2d[8] - j2d[9]) < pix_thsh
opt_pose = ch.array(init_pose)
(_, A_global) = global_rigid_transformation(
opt_pose, model.J, model.kintree_table, xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
# initialize the camera
cam = ProjectPoints(
f=np.array([flength, flength]), rt=rt, t=t, k=np.zeros(5), c=center)
# we are going to project the SMPL joints
cam.v = Jtr
if viz:
viz_img = img.copy()
# draw the target (CNN) joints
for coord in np.around(j2d).astype(int):
if (coord[0] < img.shape[1] and coord[0] >= 0 and
coord[1] < img.shape[0] and coord[1] >= 0):
cv2.circle(viz_img, tuple(coord), 3, [0, 255, 0])
import matplotlib.pyplot as plt
plt.ion()
# draw optimized joints at each iteration
def on_step(_):
"""Draw a visualization."""
plt.figure(1, figsize=(5, 5))
plt.subplot(1, 1, 1)
viz_img = img.copy()
for coord in np.around(cam.r[torso_smpl_ids]).astype(int):
if (coord[0] < viz_img.shape[1] and coord[0] >= 0 and
coord[1] < viz_img.shape[0] and coord[1] >= 0):
cv2.circle(viz_img, tuple(coord), 3, [0, 0, 255])
plt.imshow(viz_img[:, :, ::-1])
plt.draw()
plt.show()
plt.pause(1e-3)
else:
on_step = None
# optimize for camera translation and body orientation
free_variables = [cam.t, opt_pose[:3]]
ch.minimize(
# data term defined over torso joints...
{'cam': j2d[torso_cids] - cam[torso_smpl_ids],
# ...plus a regularizer for the camera translation
'cam_t': 1e2 * (cam.t[2] - init_t[2])},
x0=free_variables,
method='dogleg',
callback=on_step,
options={'maxiter': 100,
'e_3': .0001,
# disp set to 1 enables verbose output from the optimizer
'disp': 0})
if viz:
plt.ioff()
return (cam, try_both_orient, opt_pose[:3].r)
def fit_silhouettes_multi_model(objs,
sv,
silhs,
cameras,
w_s2m=10,
w_m2s=20,
max_iter=100,
mv=None,
fix_shape=False,
kp_camera=None,
imgs=None, alpha=None,
fix_trans=False, pyr_scale=1.0, vc=None, symIdx=None, mv2=None, FIX_POSE=False):
nCameras = len(cameras)
# Projected sihlouette
#dist = [np.mean(sv[i].r, axis=0)[2] for i in range(nCameras)]
frustums = [{'near': ch.min(sv[i], axis=0)[2],
'far': ch.max(sv[i], axis=0)[2],
'width': silhs[i].shape[1],
'height': silhs[i].shape[0]} for i,camera in enumerate(cameras)]
rends = [ColoredRenderer(
vc=np.ones_like(sv[i].r),
v=sv[i],
f=sv[i].f,
camera=cameras[i],
frustum=frustums[i],
bgcolor=ch.array([0, 0, 0])) for i in range(nCameras)]
# silhouette error term (model-to-scan)
#obj_m2s, obj_s2m, dist_tsf = setup_silhouette_obj(silhs, rends, sv[0].model.f)
obj_m2s, obj_s2m, dist_tsf = setup_silhouette_obj(silhs, rends, sv[0].f)
if mv is not None:
import matplotlib.pyplot as plt
plt.ion()
def on_step(_):
for i in range(len(rends)):
rend = rends[i]
img = imgs[i]
edges = cv2.Canny(np.uint8(rend[:, :, 0].r * 255), 100, 200)
coords = np.array(np.where(edges > 0)).T
plt.figure(200+i, facecolor='w')
plt.clf()
plt.subplot(2, 2, 1)
plt.imshow(dist_tsf[i])
plt.plot(coords[:, 1], coords[:, 0], 'w.')
plt.axis('off')
plt.subplot(2, 2, 2)
plt.imshow(rend[:, :, 0].r)
plt.title('fitted silhouette')
plt.draw()
plt.axis('off')
if img is not None and kp_camera is not None:
plt.subplot(2, 2, 3)
plt.imshow(img[:, :, ::-1])
plt.scatter(kp_camera[i].r[:, 0], kp_camera[i].r[:, 1])
plt.axis('off')
plt.subplot(2, 2, 4)
plt.imshow((silhs[i]+rend[:, :, 0].r)/2.0)
plt.axis('off')
plt.draw()
plt.show(block=False)
plt.pause(1e-5)
if vc is not None:
vc1 = vc[i].r.copy()
vc1[:,0] = vc[i].r.copy()[:,2]
vc1[:,2] = vc[i].r.copy()[:,0]
mv[i].set_dynamic_meshes([Mesh(sv[i].r, sv[i].model.f, vc=vc1)])
vc2 = vc[i].r.copy()[symIdx,:]
vc2[:,0] = vc[i].r.copy()[symIdx,2]
vc2[:,2] = vc[i].r.copy()[symIdx,0]
mv2[i].set_dynamic_meshes([Mesh(sv[i].r, sv[i].model.f, vc=vc2)])
else:
mv[i].set_dynamic_meshes([Mesh(sv[i].r, sv[i].f)])
else:
on_step = None
new_objs = objs.copy()
for i in range(nCameras):
new_objs['s2m_'+str(i)] = obj_s2m(w_s2m, i)
new_objs['m2s_'+str(i)] = obj_m2s(w_m2s, i)
print('weights: s2m %.2f m2s %.2f' % (w_s2m, w_m2s))
nbetas = len(sv[0].betas.r)
free_variables = []
if fix_trans is False:
for i in range(nCameras):
free_variables.append(sv[i].trans)
else:
free_variables = []
if fix_shape is False:
if alpha is not None:
free_variables.append(alpha)
else:
if not fix_shape:
for i in range(nCameras):
free_variables.append(sv[i].betas)
for i in range(nCameras):
if not FIX_POSE:
free_variables.append(sv[i].pose)
# If objective contains 'feq', then add cam.f to free variables.
if 'feq' in new_objs:
for i in range(len(rends)):
free_variables.append(cameras[i].f)
opt = {'maxiter': max_iter, 'e_3': 1e-2}
if max_iter > 0:
ch.minimize(new_objs, x0=free_variables, method='dogleg', callback=on_step, options=opt)
def render_and_show(sv):
for i in range(len(rends)):
img = imgs[i]
img_res = render_mesh(Mesh(sv[i].r, sv[i].model.f), img.shape[1], img.shape[0], kp_camera[i], near=0.5, far=20)
plt.figure()
plt.imshow(img[:, :, ::-1])
plt.imshow(img_res)
plt.axis('off')
return rends[0].r, new_objs
def optimize_on_joints(j2d,
model,
cam,
img,
prior,
init_pose,
init_shape,
n_betas=10,
conf=None):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param init_pose: 72D vector, pose prediction results provided by HMR
:param init_shape: 10D vector, shape prediction results provided by HMR
:param n_betas: number of shape coefficients considered during optimization
:param conf: 14D vector storing the confidence values from the CNN
:returns: a tuple containing the optimized model, its joints projected on image space, the
camera translation
"""
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
cids = range(12) + [13]
# joint ids for SMPL
# SMPL does not have a joint for head, instead we use a vertex for the head
# and append it later.
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# the vertex id for the joint corresponding to the head
head_id = 411
# weights assigned to each joint during optimization;
# the definition of hips in SMPL and LSP is significantly different so set
# their weights to zero
base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1],
dtype=np.float64)
# initialize the shape to the mean shape in the SMPL training set
betas = ch.array(init_shape)
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(trans=ch.zeros(3),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# make the SMPL joints depend on betas
Jdirs = np.dstack([
model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))
])
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = global_rigid_transformation(sv.pose,
J_onbetas,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
# add the head joint, corresponding to a vertex...
Jtr = ch.vstack((Jtr, sv[head_id]))
# ... and add the joint id to the list
smpl_ids.append(len(Jtr) - 1)
# update the weights using confidence values
weights = base_weights * conf[cids] if conf is not None else base_weights
# project SMPL joints on the image plane using the estimated camera
cam.v = Jtr
# data term: distance between observed and estimated joints in 2D
obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf(
(j2d[cids] - cam[smpl_ids]), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
ch.minimize(objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=None,
options={
'maxiter': 100,
'e_3': .0001,
'disp': 0
})
return sv, cam.r, cam.t.r
result.betas[:]=np.zeros(10);
result.pose[:]= np.zeros(72);
print lista[iters]
Target = loadObj(lista[iters])
#Rigid alignment
scale=Ch(1);
trans=ch.array([0,0,0]);
Tar_shift = Target.vertices+trans;
indexes=a['pF_lb2'].reshape(6890);
distances=Tar_shift[indexes-1]-result*scale;
(t)=ch.minimize(distances, x0 = [trans,result.pose[[0,1,2]]],
method = 'dogleg', callback = None,
options = {'maxiter': 50, 'e_3': .0001, 'disp': 1})
#Non-rigid Alignment
c_pre={};
if (W_Joints):
k=a['Joints_Target'];
j_to_consider = range(0,24)
J_distances = J_reposed[j_to_consider,:] - (k[j_to_consider,:]+trans);
c_pre['Joints']= J_distances*W_Joints;
if(W_FMP2P):
c_pre['FMP2P']= distances*W_FMP2P;
if(W_Norm_B):
c_pre['Norm_B']= ((result.betas)**2)*W_Norm_B;
def initialize_camera(
model, # pylint: disable=too-many-arguments, too-many-locals
j2d,
center,
img,
init_t,
init_pose,
conf,
is_gt,
flength=1000.,
pix_thsh=25.,
viz=False):
"""Initialize the camera."""
# try to optimize camera translation and rotation based on torso joints
# right shoulder, left shoulder, right hip, left hip
torso_cids = _CID_TORSO
torso_smpl_ids = [2, 1, 17, 16]
# initialize camera rotation and translation
rt = _ch.zeros(3) # pylint: disable=no-member
_LOGGER.info('Initializing camera: guessing translation via similarity')
init_t = guess_init(model, flength, j2d, conf, init_pose, is_gt)
t = _ch.array(init_t) # pylint: disable=no-member
# check how close the shoulders are
try_both_orient = _np.linalg.norm(j2d[8] - j2d[9]) < pix_thsh
opt_pose = _ch.array(init_pose) # pylint: disable=no-member
(_, A_global) = _global_rigid_transformation(opt_pose,
model.J,
model.kintree_table,
xp=_ch)
Jtr = _ch.vstack([g[:3, 3] for g in A_global]) # pylint: disable=no-member
# initialize the camera
cam = _ProjectPoints(f=_np.array([flength, flength]),
rt=rt,
t=t,
k=_np.zeros(5),
c=center)
# we are going to project the SMPL joints
cam.v = Jtr
if viz:
viz_img = img.copy()
# draw the target joints
for coord in _np.around(j2d).astype(int):
if (coord[0] < img.shape[1] and coord[0] >= 0
and coord[1] < img.shape[0] and coord[1] >= 0):
_cv2.circle(viz_img, tuple(coord), 3, [0, 255, 0])
import matplotlib.pyplot as plt
plt.ion()
# draw optimized joints at each iteration
def on_step(_):
"""Draw a visualization."""
plt.figure(1, figsize=(5, 5))
plt.subplot(1, 1, 1)
viz_img = img.copy()
for coord in _np.around(cam.r[torso_smpl_ids]).astype(int):
if (coord[0] < viz_img.shape[1] and coord[0] >= 0
and coord[1] < viz_img.shape[0] and coord[1] >= 0):
_cv2.circle(viz_img, tuple(coord), 3, [0, 0, 255])
plt.imshow(viz_img[:, :, ::-1])
plt.draw()
plt.show()
else:
on_step = None
free_variables = [cam.t, opt_pose[:3]] # pylint: disable=no-member
_ch.minimize(
[(j2d[torso_cids] - cam[torso_smpl_ids]).T * conf[torso_cids], 1e2 *
(cam.t[2] - init_t[2])], # pylint: disable=no-member
# The same for verbose output.
#{'cam': (j2d[torso_cids] - cam[torso_smpl_ids]).T * conf[torso_cids],
# # Reduce the weight here to avoid the 'small people' problem.
# 'cam_t': 1e2*(cam.t[2]-init_t[2])}, # pylint: disable=no-member
x0=free_variables,
method='dogleg',
callback=on_step,
options={
'maxiter': 100,
'e_3': .0001,
'disp': 0
})
if viz:
plt.ioff()
return (cam, try_both_orient, opt_pose[:3].r)
def optimize_on_joints(j2d,
model,
cam,
img,
prior,
try_both_orient,
body_orient,
exp_logistic,
n_betas=10,
inner_penetration=False,
silh=None,
conf=None,
viz=False):
"""Run the optimization."""
if silh is not None:
raise NotImplementedError("Silhouette fitting is not supported in "
"this code release due to dependencies on "
"proprietary code for the "
"distance computation.")
t0 = _time()
# define the mapping LSP joints -> SMPL joints
if j2d.shape[0] == 14:
cids = range(12) + [13]
elif j2d.shape[0] == 91:
cids = range(j2d.shape[0])
else:
raise Exception("Unknown number of joints: %d! Mapping not defined!" %
j2d.shape[0])
# joint ids for SMPL
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# weight given to each joint during optimization;
if j2d.shape[0] == 14:
weights = [1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]
else:
weights = [1] * (len(smpl_ids) + len(landmark_mesh_91))
# The non-skeleton vertex ids are added later.
if try_both_orient:
flipped_orient = _cv2.Rodrigues(body_orient)[0].dot(
_cv2.Rodrigues(_np.array([0., _np.pi, 0]))[0])
flipped_orient = _cv2.Rodrigues(flipped_orient)[0].ravel()
orientations = [body_orient, flipped_orient]
else:
orientations = [body_orient]
if try_both_orient:
errors = []
svs = []
cams = []
# rends = []
for o_id, orient in enumerate(orientations):
# initialize betas
betas = _ch.zeros(n_betas) # pylint: disable=no-member
init_pose = _np.hstack((orient, prior.weights.dot(prior.means)))
# 2D joint error term
# make the SMPL joint depend on betas
Jdirs = _np.dstack([
model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))
])
# pylint: disable=no-member
J_onbetas = _ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# instantiate the model
sv = verts_decorated(trans=_ch.zeros(3),
pose=_ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = _global_rigid_transformation(sv.pose,
J_onbetas,
model.kintree_table,
xp=_ch)
Jtr = _ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
if j2d.shape[0] == 14:
# add the "fake" joint for the head
head_id = _HEAD_REGR[0]
Jtr = _ch.vstack((Jtr, sv[head_id]))
if o_id == 0:
smpl_ids.append(len(Jtr) - 1)
else:
# add the plain vertex IDs on the mesh surface.
for vertex_id in landmark_mesh_91.values():
Jtr = _ch.vstack((Jtr, sv[vertex_id]))
# add the joint id
# for SMPL it's the last one added
if o_id == 0:
smpl_ids.append(len(Jtr) - 1)
weights = _np.array(weights, dtype=_np.float64)
if conf is not None:
weights *= conf[cids]
# we'll project the joints on the image plane
cam.v = Jtr
# data term: difference between observed and estimated joints
obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * _GMOf(
(j2d[cids] - cam[smpl_ids]), sigma))
# pose prior
pprior = lambda w: w * prior(sv.pose) # pylint: disable=cell-var-from-loop
# joint angles prior
# 55: left elbow, should bend -np.pi/2
# 58: right elbow, should bend np.pi/2
# 12: left knee, should bend np.pi/2
# 15: right knee, should bend np.pi/2
if exp_logistic:
_LOGGER.info('USING LOGISTIC')
# Skinny Logistic function. as 50-> inf we get a step function at
# 0.1. (0.1) is a margin bc 0 is still ok.
my_exp = lambda x: 1 / (1 + _ch.exp(100 * (0.1 + -x)))
else:
x_0 = 0 #10
alpha = 10
my_exp = lambda x: alpha * _ch.exp((x - x_0)) # pylint: disable=cell-var-from-loop
obj_angle = lambda w: w * _ch.concatenate([
my_exp(sv.pose[55]), # pylint: disable=cell-var-from-loop
my_exp(-sv.pose[58]), # pylint: disable=cell-var-from-loop
my_exp(-sv.pose[12]), # pylint: disable=cell-var-from-loop
my_exp(-sv.pose[15])
]) # pylint: disable=cell-var-from-loop
if viz:
from body.mesh.sphere import Sphere
from body.mesh.meshviewer import MeshViewer
import matplotlib.pyplot as plt
# set up visualization
# openGL window
mv = MeshViewer(window_width=120, window_height=120)
# and ids
show_ids = _np.array(smpl_ids)[weights > 0]
vc = _np.ones((len(Jtr), 3))
vc[show_ids] = [0, 1, 0]
plt.ion()
def on_step(_):
"""Create visualization."""
# show optimized joints in 3D
# pylint: disable=cell-var-from-loop
mv.set_dynamic_meshes([_Mesh(v=sv.r, f=[]),
Sphere(center=cam.t.r,
radius=.1).to_mesh()] \
+ [Sphere(center=jc, radius=.01).to_mesh(vc[ijc])
for ijc, jc in enumerate(Jtr.r)])
plt.figure(1, figsize=(10, 10))
plt.subplot(1, 2, 1)
# show optimized joints in 2D
tmp_img = img.copy()
for coord, target_coord in zip(
_np.around(cam.r[smpl_ids]).astype(int),
_np.around(j2d[cids]).astype(int)):
if (coord[0] < tmp_img.shape[1] and coord[0] >= 0
and coord[1] < tmp_img.shape[0] and coord[1] >= 0):
_cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255])
if (target_coord[0] < tmp_img.shape[1]
and target_coord[0] >= 0
and target_coord[1] < tmp_img.shape[0]
and target_coord[1] >= 0):
_cv2.circle(tmp_img, tuple(target_coord), 3,
[0, 255, 0])
plt.imshow(tmp_img)
plt.draw()
plt.show()
on_step(_)
else:
on_step = None
sp = _SphereCollisions(pose=sv.pose,
betas=sv.betas,
model=model,
regs=_REGRESSORS)
sp.no_hands = True
# configuration used with conf joints
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
#if stage < 2:
objs['j2d'] = obj_j2d(1., 100) # TODO: evaluate.
objs['pose'] = pprior(w)
# WEIGHT FOR ANGLE
if exp_logistic:
# Set to high weight always.
objs['pose_exp'] = obj_angle(5 * 1e3)
else:
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
if inner_penetration:
objs['sph_coll'] = 1e3 * sp
try:
_ch.minimize(objs.values(),
x0=[sv.betas, sv.pose],
method='dogleg',
callback=on_step,
options={
'maxiter': 100,
'e_3': .0001,
'disp': 0
})
except AssertionError:
# Divergence detected.
_LOGGER.warn("Diverging optimization! Breaking!")
break
t1 = _time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if try_both_orient:
errors.append((objs['j2d'].r**2).sum())
svs.append(sv)
cams.append(cam)
# rends.append(rend)
if try_both_orient and errors[0] > errors[1]:
choose_id = 1
else:
choose_id = 0
if viz:
plt.ioff()
return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r,
cams[choose_id].rt.r)
def final_fit(
opt,
part_mesh,
v,
v_offset,
dist_o,
dist_i,
smpl_h_ref,
rn_m,
debug_rn,
dif_mask,
v_ids_template,
faces_template,
v_ids_side,
faces_side,
max_y,
proj_cam,
ref_joint_list_coup,
):
if opt.disp_mesh_side:
mv = MeshViewer()
else:
mv = None
if opt.disp_mesh_whl:
mv2 = MeshViewer()
else:
mv2 = None
import scipy.sparse as sp
sparse_solver = lambda A, x: sp.linalg.cg(A, x, maxiter=500)[0]
tgt_pose = get_pose_prior(init_pose_path=opt.init_pose_path,
gar_type=opt.gar_type)
E = {
'mask':
gaussian_pyramid(rn_m * dist_o * opt.ref_wt_dist_o +
(1 - rn_m) * dist_i,
n_levels=4,
normalization='size') * opt.ref_wt_mask
}
x0 = [v_offset]
if opt.ref_wt_coup:
# x0 = [smpl_h_ref.trans, smpl_h_ref.betas, v_offset]
E['coupling'] = (v + v_offset -
smpl_h_ref[v_ids_template]) * opt.ref_wt_coup
if opt.ref_wt_shp:
E['beta_prior'] = ch.linalg.norm(smpl_h_ref.betas) * opt.ref_wt_shp
if opt.ref_wt_pose:
E['pose'] = (smpl_h_ref.pose - tgt_pose) * opt.ref_wt_pose
if ref_joint_list_coup != None:
range_joint = []
for item in ref_joint_list_coup:
range_joint.append(3 * int(item))
range_joint.append(3 * int(item) + 1)
range_joint.append(3 * int(item) + 2)
x0 = x0 + [smpl_h_ref.pose[range_joint]]
if opt.ref_use_betas:
x0 = x0 + [smpl_h_ref.betas]
if opt.ref_wt_proj:
error_bd = get_rings_error(proj_cam, max_y)
E['proj_bd'] = error_bd * opt.ref_wt_proj
if opt.ref_wt_bd:
gar_rings = compute_boundaries(v + v_offset, faces_template)
error = smooth_rings(gar_rings, v + v_offset)
E['boundary'] = error * opt.ref_wt_bd
if opt.ref_wt_lap:
lap_op = np.asarray(laplacian(part_mesh).todense())
lap_err = ch.dot(lap_op, v + v_offset)
E['laplacian'] = lap_err * opt.ref_wt_lap
ch.minimize(E,
x0,
method='dogleg',
options={
'e_3': .000001,
'sparse_solver': sparse_solver
},
callback=get_callback_ref(rend=debug_rn,
mask=dif_mask,
vertices=v + v_offset,
display=opt.display,
v_ids_sides=v_ids_side,
faces_template=faces_template,
faces_side=faces_side,
disp_mesh_side=opt.disp_mesh_side,
disp_mesh_whl=opt.disp_mesh_whl,
save_dir=opt.save_opt_images,
mv=mv,
mv2=mv2,
show=opt.show))
final_mask = rn_m.r
mask = dif_mask.copy()
mask[dif_mask == 0.5] = 1
final_iou = compute_iou(mask, final_mask)
return v + v_offset, final_iou
def optimize_on_joints(j2d,
model,
cam,
img,
prior,
try_both_orient,
body_orient,
n_betas=10,
regs=None,
conf=None,
viz=False):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit
:param body_orient: 3D vector, initialization for the body orientation
:param n_betas: number of shape coefficients considered during optimization
:param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term
:param conf: 14D vector storing the confidence values from the CNN
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the optimized model, its joints projected on image space, the camera translation
"""
t0 = time()
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
cids = range(12) + [13]
# joint ids for SMPL
# SMPL does not have a joint for head, instead we use a vertex for the head
# and append it later.
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# the vertex id for the joint corresponding to the head
head_id = 411
# weights assigned to each joint during optimization;
# the definition of hips in SMPL and LSP is significantly different so set
# their weights to zero
base_weights = np.array(
[1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=np.float64)
if try_both_orient:
flipped_orient = cv2.Rodrigues(body_orient)[0].dot(
cv2.Rodrigues(np.array([0., np.pi, 0]))[0])
flipped_orient = cv2.Rodrigues(flipped_orient)[0].ravel()
orientations = [body_orient, flipped_orient]
else:
orientations = [body_orient]
if try_both_orient:
# store here the final error for both orientations,
# and pick the orientation resulting in the lowest error
errors = []
svs = []
cams = []
for o_id, orient in enumerate(orientations):
# initialize the shape to the mean shape in the SMPL training set
betas = ch.zeros(n_betas)
# initialize the pose by using the optimized body orientation and the
# pose prior
init_pose = np.hstack((orient, prior.weights.dot(prior.means)))
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(
trans=ch.zeros(3),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# make the SMPL joints depend on betas
Jdirs = np.dstack([model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))])
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = global_rigid_transformation(
sv.pose, J_onbetas, model.kintree_table, xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
# add the head joint, corresponding to a vertex...
Jtr = ch.vstack((Jtr, sv[head_id]))
# ... and add the joint id to the list
if o_id == 0:
smpl_ids.append(len(Jtr) - 1)
# update the weights using confidence values
weights = base_weights * conf[
cids] if conf is not None else base_weights
# project SMPL joints on the image plane using the estimated camera
cam.v = Jtr
# data term: distance between observed and estimated joints in 2D
obj_j2d = lambda w, sigma: (
w * weights.reshape((-1, 1)) * GMOf((j2d[cids] - cam[smpl_ids]), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([my_exp(sv.pose[55]), my_exp(-sv.pose[
58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15])])
if viz:
import matplotlib.pyplot as plt
plt.ion()
def on_step(_):
"""Create visualization."""
plt.figure(1, figsize=(10, 10))
plt.subplot(1, 2, 1)
# show optimized joints in 2D
tmp_img = img.copy()
for coord, target_coord in zip(
np.around(cam.r[smpl_ids]).astype(int),
np.around(j2d[cids]).astype(int)):
if (coord[0] < tmp_img.shape[1] and coord[0] >= 0 and
coord[1] < tmp_img.shape[0] and coord[1] >= 0):
cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255])
if (target_coord[0] < tmp_img.shape[1] and
target_coord[0] >= 0 and
target_coord[1] < tmp_img.shape[0] and
target_coord[1] >= 0):
cv2.circle(tmp_img, tuple(target_coord), 3,
[0, 255, 0])
plt.imshow(tmp_img[:, :, ::-1])
plt.draw()
plt.show()
plt.pause(1e-2)
on_step(_)
else:
on_step = None
if regs is not None:
# interpenetration term
sp = SphereCollisions(
pose=sv.pose, betas=sv.betas, model=model, regs=regs)
sp.no_hands = True
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
if regs is not None:
objs['sph_coll'] = 1e3 * sp
ch.minimize(
objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=on_step,
options={'maxiter': 100,
'e_3': .0001,
'disp': 0})
t1 = time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if try_both_orient:
errors.append((objs['j2d'].r**2).sum())
svs.append(sv)
cams.append(cam)
if try_both_orient and errors[0] > errors[1]:
choose_id = 1
else:
choose_id = 0
if viz:
plt.ioff()
return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r)
def optimize_on_joints_and_silhouette(j2d,
sil,
model,
cam,
img,
prior,
init_pose,
init_shape,
n_betas=10,
conf=None):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param sil: h x w silhouette with soft boundaries (np.float32, range(-1, 1))
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param init_pose: 72D vector, pose prediction results provided by HMR
:param init_shape: 10D vector, shape prediction results provided by HMR
:param n_betas: number of shape coefficients considered during optimization
:param conf: 14D vector storing the confidence values from the CNN
:returns: a tuple containing the optimized model, its joints projected on image space, the
camera translation
"""
# define the mapping LSP joints -> SMPL joints
cids = range(12) + [13]
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
head_id = 411
# weights assigned to each joint during optimization;
# the definition of hips in SMPL and LSP is significantly different so set
# their weights to zero
base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1],
dtype=np.float64)
betas = ch.array(init_shape)
# instantiate the model:
sv = verts_decorated(trans=ch.zeros(3),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# make the SMPL joints depend on betas
Jdirs = np.dstack([
model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))
])
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = global_rigid_transformation(sv.pose,
J_onbetas,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
# add the head joint
Jtr = ch.vstack((Jtr, sv[head_id]))
smpl_ids.append(len(Jtr) - 1)
# update the weights using confidence values
weights = base_weights * conf[cids] if conf is not None else base_weights
# project SMPL joints and vertex on the image plane using the estimated camera
cam.v = ch.vstack([Jtr, sv])
# obtain a gradient map of the soft silhouette
grad_x = cv2.Sobel(sil, cv2.CV_32FC1, 1, 0) * 0.125
grad_y = cv2.Sobel(sil, cv2.CV_32FC1, 0, 1) * 0.125
# data term #1: distance between observed and estimated joints in 2D
obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf(
(j2d[cids] - cam[smpl_ids]), sigma))
# data term #2: distance between the observed and projected boundaries
obj_s2d = lambda w, sigma, flag, target_pose: (w * flag * GMOf(
(target_pose - cam[len(Jtr):(len(Jtr) + 6890)]), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
print('****** Optimization on joints')
curr_pose = sv.pose.r
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
objs['thetas'] = wbetas * (sv.pose - curr_pose
) # constrain theta changes
ch.minimize(objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=None,
options={
'maxiter': 100,
'e_3': .001,
'disp': 0
})
curr_pose = sv.pose.r
# cam.v = ch.vstack([Jtr, sv.r])
# run the optimization in 2 stages, progressively decreasing the
# weights for the priors
print('****** Optimization on silhouette and joints')
opt_weights = zip([57.4, 4.78], [2e2, 1e2])
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
# find the boundary vertices and estimate their expected location
smpl_vs = cam.r[len(Jtr):, :]
boundary_flag = np.zeros((smpl_vs.shape[0], 1))
expected_pos = np.zeros((smpl_vs.shape[0], 2))
for vi, v in enumerate(smpl_vs):
r, c = int(v[1]), int(v[0])
if r < 0 or r >= sil.shape[0] or c < 0 or c >= sil.shape[1]:
continue
sil_v = sil[r, c]
grad = np.array([grad_x[r, c], grad_y[r, c]])
grad_n = np.linalg.norm(grad)
if grad_n > 1e-1 and sil_v < 0.4: # vertex on or out of the boundaries
boundary_flag[vi] = 1.0
step = (grad / grad_n) * (sil_v / grad_n)
expected_pos[vi] = np.array([c - step[0], r - step[1]])
# run optimization
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['s2d'] = obj_s2d(5., 100, boundary_flag, expected_pos)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas # constrain beta changes
objs['thetas'] = wbetas * (sv.pose - curr_pose
) # constrain theta changes
ch.minimize(objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=None,
options={
'maxiter': 100,
'e_3': .001,
'disp': 0
})
return sv, cam.r, cam.t.r
def init(frames, body_height, b2m, viz_rn):
betas = frames[0].smpl.betas
E_height = None
if body_height is not None:
E_height = height_predictor(b2m, betas) - body_height * 1000.
# first get a rough pose for all frames individually
for i, f in enumerate(frames):
if np.sum(f.keypoints[[0, 2, 5, 8, 11], 2]) > 3.:
if f.keypoints[2, 0] > f.keypoints[5, 0]:
f.smpl.pose[0] = 0
f.smpl.pose[2] = np.pi
E_init = {'init_pose_{}'.format(i): f.pose_obj[[0, 2, 5, 8, 11]]}
x0 = [f.smpl.trans, f.smpl.pose[:3]]
if E_height is not None and i == 0:
E_init['height'] = E_height
E_init['betas'] = betas
x0.append(betas)
ch.minimize(E_init,
x0,
method='dogleg',
options={
'e_3': .01,
},
callback=get_cb(viz_rn, f))
weights = zip([5., 4.5, 4.], [5., 4., 3.])
E_betas = betas - betas.r
for w_prior, w_betas in weights:
x0 = [betas]
E = {
'betas': E_betas * w_betas,
}
if E_height is not None:
E['height'] = E_height
for i, f in enumerate(frames):
if np.sum(f.keypoints[[0, 2, 5, 8, 11], 2]) > 3.:
x0.extend([
f.smpl.pose[range(21) + range(27, 30) + range(36, 60)],
f.smpl.trans
])
E['pose_{}'.format(i)] = f.pose_obj
E['prior_{}'.format(i)] = f.pose_prior_obj * w_prior
ch.minimize(E,
x0,
method='dogleg',
options={
'e_3': .01,
},
callback=get_cb(viz_rn, frames[0]))
def optimize_on_joints(j2d,
model,
cam,
img,
prior,
try_both_orient,
body_orient,
n_betas=10,
regs=None,
conf=None,
viz=False):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit
:param body_orient: 3D vector, initialization for the body orientation
:param n_betas: number of shape coefficients considered during optimization
:param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term
:param conf: 14D vector storing the confidence values from the CNN
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the optimized model, its joints projected on image space, the camera translation
"""
t0 = time()
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
cids = range(12) + [13]
# joint ids for SMPL
# SMPL does not have a joint for head, instead we use a vertex for the head
# and append it later.
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# the vertex id for the joint corresponding to the head
head_id = 411
# weights assigned to each joint during optimization;
# the definition of hips in SMPL and LSP is significantly different so set
# their weights to zero
base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1],
dtype=np.float64)
if try_both_orient:
flipped_orient = cv2.Rodrigues(body_orient)[0].dot(
cv2.Rodrigues(np.array([0., np.pi, 0]))[0])
flipped_orient = cv2.Rodrigues(flipped_orient)[0].ravel()
orientations = [body_orient, flipped_orient]
else:
orientations = [body_orient]
if try_both_orient:
# store here the final error for both orientations,
# and pick the orientation resulting in the lowest error
errors = []
svs = []
cams = []
for o_id, orient in enumerate(orientations):
# initialize the shape to the mean shape in the SMPL training set
betas = ch.zeros(n_betas)
# initialize the pose by using the optimized body orientation and the
# pose prior
init_pose = np.hstack((orient, prior.weights.dot(prior.means)))
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(trans=ch.zeros(3),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# make the SMPL joints depend on betas
Jdirs = np.dstack([
model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))
])
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = global_rigid_transformation(sv.pose,
J_onbetas,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
# add the head joint, corresponding to a vertex...
Jtr = ch.vstack((Jtr, sv[head_id]))
# ... and add the joint id to the list
if o_id == 0:
smpl_ids.append(len(Jtr) - 1)
# update the weights using confidence values
weights = base_weights * conf[
cids] if conf is not None else base_weights
# project SMPL joints on the image plane using the estimated camera
cam.v = Jtr
# data term: distance between observed and estimated joints in 2D
obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf(
(j2d[cids] - cam[smpl_ids]), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
if viz:
import matplotlib.pyplot as plt
plt.ion()
def on_step(_):
"""Create visualization."""
plt.figure(1, figsize=(10, 10))
plt.subplot(1, 2, 1)
# show optimized joints in 2D
tmp_img = img.copy()
for coord, target_coord in zip(
np.around(cam.r[smpl_ids]).astype(int),
np.around(j2d[cids]).astype(int)):
if (coord[0] < tmp_img.shape[1] and coord[0] >= 0
and coord[1] < tmp_img.shape[0] and coord[1] >= 0):
cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255])
if (target_coord[0] < tmp_img.shape[1]
and target_coord[0] >= 0
and target_coord[1] < tmp_img.shape[0]
and target_coord[1] >= 0):
cv2.circle(tmp_img, tuple(target_coord), 3,
[0, 255, 0])
plt.imshow(tmp_img[:, :, ::-1])
plt.draw()
plt.show()
plt.pause(1e-2)
on_step(_)
else:
on_step = None
if regs is not None:
# interpenetration term
sp = SphereCollisions(pose=sv.pose,
betas=sv.betas,
model=model,
regs=regs)
sp.no_hands = True
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
if regs is not None:
objs['sph_coll'] = 1e3 * sp
ch.minimize(objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=on_step,
options={
'maxiter': 100,
'e_3': .0001,
'disp': 0
})
t1 = time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if try_both_orient:
errors.append((objs['j2d'].r**2).sum())
svs.append(sv)
cams.append(cam)
if try_both_orient and errors[0] > errors[1]:
choose_id = 1
else:
choose_id = 0
if viz:
plt.ioff()
return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r, Jtr)
def fit_lmk3d(
lmk_3d, # input landmark 3d
model, # model
lmk_face_idx,
lmk_b_coords, # landmark embedding
weights, # weights for the objectives
shape_num=300,
expr_num=100,
opt_options=None):
""" function: fit FLAME model to 3d landmarks
input:
lmk_3d: input landmark 3d, in shape (N,3)
model: FLAME face model
lmk_face_idx, lmk_b_coords: landmark embedding, in face indices and barycentric coordinates
weights: weights for each objective
shape_num, expr_num: numbers of shape and expression compoenents used
opt_options: optimizaton options
output:
model.r: fitted result vertices
model.f: fitted result triangulations (fixed in this code)
parms: fitted model parameters
"""
# variables
shape_idx = np.arange(0, min(
300, shape_num)) # valid shape component range in "betas": 0-299
expr_idx = np.arange(300, 300 + min(
100, expr_num)) # valid expression component range in "betas": 300-399
used_idx = np.union1d(shape_idx, expr_idx)
model.betas[:] = np.random.rand(
model.betas.size) * 0.0 # initialized to zero
model.pose[:] = np.random.rand(
model.pose.size) * 0.0 # initialized to zero
#free_variables = [ model.trans, model.pose, model.betas[used_idx] ]
free_variables = [model.pose, model.betas[used_idx]]
# weights
print "fit_lmk3d(): use the following weights:"
for kk in weights.keys():
print "fit_lmk3d(): weights['%s'] = %f" % (kk, weights[kk])
# objectives
# lmk
scale = ch.array([1])
lmk_err = landmark_error_3d(mesh_verts=scale * model,
mesh_faces=model.f,
lmk_3d=lmk_3d,
lmk_face_idx=lmk_face_idx,
lmk_b_coords=lmk_b_coords,
weight=weights['lmk'])
# regularizer
shape_err = weights['shape'] * model.betas[shape_idx]
expr_err = weights['expr'] * model.betas[expr_idx]
pose_err = weights['pose'] * model.pose[3:] # exclude global rotation
objectives = {}
objectives.update({
'lmk': lmk_err,
'shape': shape_err,
'expr': expr_err,
'pose': pose_err
})
# options
if opt_options is None:
print "fit_lmk3d(): no 'opt_options' provided, use default settings."
import scipy.sparse as sp
opt_options = {}
opt_options['disp'] = 1
opt_options['delta_0'] = 0.1
opt_options['e_3'] = 1e-4
opt_options['maxiter'] = 100
sparse_solver = lambda A, x: sp.linalg.cg(
A, x, maxiter=opt_options['maxiter'])[0]
opt_options['sparse_solver'] = sparse_solver
# on_step callback
def on_step(_):
pass
# optimize
# step 1: rigid alignment
from time import time
timer_start = time()
'''
print "\nstep 1: start rigid fitting..."
ch.minimize( fun = lmk_err,
x0 = [ model.trans, model.pose[0:3],scale ],
method = 'dogleg',
callback = on_step,
options = opt_options )
timer_end = time()
align_v = scale*model.r
align_f = model.f
print "step 1: fitting done, in %f sec\n" % ( timer_end - timer_start )
print "scale： %f\n" % scale
print "model.trans"
print model.trans
print "model.rot"
print model.pose[0:3]
'''
# step 2: non-rigid alignment
timer_start = time()
print "step 2: start non-rigid fitting..."
ch.minimize(fun=objectives,
x0=free_variables,
method='dogleg',
callback=on_step,
options=opt_options)
timer_end = time()
print "step 2: fitting done, in %f sec\n" % (timer_end - timer_start)
# return results
parms = {
'trans': model.trans.r,
'pose': model.pose.r,
'betas': model.betas.r
}
return scale * model.r, model.f, parms #,align_v,align_f
def initialize_camera(model,
j2d,
img,
init_pose,
flength=5000.,
pix_thsh=25.,
viz=False):
"""Initialize camera translation and body orientation
:param model: SMPL model
:param j2d: 14x2 array of CNN joints
:param img: h x w x 3 image
:param init_pose: 72D vector of pose parameters used for initialization
:param flength: camera focal length (kept fixed)
:param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D
is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both
the estimated one and its flip)
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the estimated camera,
a boolean deciding if both the optimized body orientation and its flip should be considered,
3D vector for the body orientation
"""
# optimize camera translation and body orientation based on torso joints
# LSP torso ids:
# 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder
torso_cids = [2, 3, 8, 9]
# corresponding SMPL torso ids
torso_smpl_ids = [2, 1, 17, 16]
center = np.array([img.shape[1] / 2, img.shape[0] / 2])
# initialize camera rotation
rt = ch.zeros(3)
# initialize camera translation
_LOGGER.info('initializing translation via similar triangles')
init_t = guess_init(model, flength, j2d, init_pose)
t = ch.array(init_t)
# check how close the shoulder joints are
try_both_orient = np.linalg.norm(j2d[8] - j2d[9]) < pix_thsh
opt_pose = ch.array(init_pose)
(_, A_global) = global_rigid_transformation(opt_pose,
model.J,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
# initialize the camera
cam = ProjectPoints(f=np.array([flength, flength]),
rt=rt,
t=t,
k=np.zeros(5),
c=center)
# we are going to project the SMPL joints
cam.v = Jtr
if viz:
viz_img = img.copy()
# draw the target (CNN) joints
for coord in np.around(j2d).astype(int):
if (coord[0] < img.shape[1] and coord[0] >= 0
and coord[1] < img.shape[0] and coord[1] >= 0):
cv2.circle(viz_img, tuple(coord), 3, [0, 255, 0])
import matplotlib.pyplot as plt
plt.ion()
# draw optimized joints at each iteration
def on_step(_):
"""Draw a visualization."""
plt.figure(1, figsize=(5, 5))
plt.subplot(1, 1, 1)
viz_img = img.copy()
for coord in np.around(cam.r[torso_smpl_ids]).astype(int):
if (coord[0] < viz_img.shape[1] and coord[0] >= 0
and coord[1] < viz_img.shape[0] and coord[1] >= 0):
cv2.circle(viz_img, tuple(coord), 3, [0, 0, 255])
plt.imshow(viz_img[:, :, ::-1])
plt.draw()
plt.show()
plt.pause(1e-3)
else:
on_step = None
# optimize for camera translation and body orientation
free_variables = [cam.t, opt_pose[:3]]
ch.minimize(
# data term defined over torso joints...
{
'cam': j2d[torso_cids] - cam[torso_smpl_ids],
# ...plus a regularizer for the camera translation
'cam_t': 1e2 * (cam.t[2] - init_t[2])
},
x0=free_variables,
method='dogleg',
callback=on_step,
options={
'maxiter': 100,
'e_3': .0001,
# disp set to 1 enables verbose output from the optimizer
'disp': 0
})
if viz:
plt.ioff()
return (cam, try_both_orient, opt_pose[:3].r)
def init_fit(opt, dist_o, dist_i, dif_mask, rn_m, smpl_h, v_ids_template,
faces_template, debug_rn, v_ids_side, faces_side, joints_list):
range_joint = []
for item in joints_list:
range_joint.append(3 * int(item))
range_joint.append(3 * int(item) + 1)
range_joint.append(3 * int(item) + 2)
tgt_pose = get_pose_prior(init_pose_path=opt.init_pose_path,
gar_type=opt.gar_type)
# ============================================
# FIRST STAGE
# ============================================
from psbody.mesh import MeshViewer
if opt.disp_mesh_side:
mv = MeshViewer()
else:
mv = None
if opt.disp_mesh_whl:
mv2 = MeshViewer()
else:
mv2 = None
if opt.init_first_stage == "Trans":
x0 = [smpl_h.trans]
elif opt.init_first_stage == 'Pose':
x0 = [smpl_h.trans, smpl_h.pose[range_joint]]
# x0 = [smpl_h.trans]
elif opt.init_first_stage == 'Shape':
x0 = [smpl_h.trans, smpl_h.betas]
elif opt.init_first_stage == 'Both':
x0 = [smpl_h.trans, smpl_h.betas, smpl_h.pose[range_joint]]
E = {
'mask':
gaussian_pyramid(rn_m * dist_o * opt.init_fst_wt_dist_o +
(1 - rn_m) * dist_i,
n_levels=4,
normalization='size') * opt.init_fst_wt_mask
}
if opt.init_fst_wt_betas:
E['beta_prior'] = ch.linalg.norm(smpl_h.betas) * opt.init_fst_wt_betas
if opt.init_fst_wt_pose:
E['pose'] = (smpl_h.pose - tgt_pose) * opt.init_fst_wt_pose
ch.minimize(E,
x0,
method='dogleg',
options={
'e_3': .0001,
'disp': True
},
callback=get_callback(rend=debug_rn,
mask=dif_mask,
smpl=smpl_h,
v_ids_template=v_ids_template,
v_ids_sides=v_ids_side,
faces_template=faces_template,
faces_side=faces_side,
display=opt.display,
disp_mesh_side=opt.disp_mesh_side,
disp_mesh_whl=opt.disp_mesh_whl,
mv=mv,
mv2=mv2,
save_dir=opt.save_opt_images,
show=opt.show))
# ===============================================
# SECOND STAGE
# ===============================================
if opt.init_second_stage != "None":
if opt.init_second_stage == 'Pose':
x0 = [smpl_h.trans, smpl_h.pose[range_joint]]
elif opt.init_second_stage == 'Shape':
x0 = [smpl_h.trans, smpl_h.betas]
elif opt.init_second_stage == 'Both':
x0 = [smpl_h.trans, smpl_h.betas, smpl_h.pose[range_joint]]
E = {
'mask':
gaussian_pyramid(rn_m * dist_o * opt.init_sec_wt_dist_o +
(1 - rn_m) * dist_i,
n_levels=4,
normalization='size') * opt.init_sec_wt_mask
}
if opt.init_sec_wt_betas:
E['beta_prior'] = ch.linalg.norm(
smpl_h.betas) * opt.init_sec_wt_betas
if opt.init_sec_wt_pose:
E['pose'] = (smpl_h.pose - tgt_pose) * opt.init_sec_wt_pose
ch.minimize(E,
x0,
method='dogleg',
options={'e_3': .0001},
callback=get_callback(rend=debug_rn,
mask=dif_mask,
smpl=smpl_h,
v_ids_template=v_ids_template,
v_ids_sides=v_ids_side,
faces_template=faces_template,
faces_side=faces_side,
display=opt.display,
disp_mesh_side=opt.disp_mesh_side,
disp_mesh_whl=opt.disp_mesh_whl,
mv=mv,
mv2=mv2,
save_dir=opt.save_opt_images,
show=opt.show))
temp_params = {
'pose': smpl_h.pose.r,
'betas': smpl_h.betas.r,
'trans': smpl_h.trans.r,
'v_personal': smpl_h.v_personal.r
}
part_mesh = Mesh(smpl_h.r[v_ids_template], faces_template)
return part_mesh, temp_params
def initialize_camera(model, j2d, img, init_pose, flength=5000., pix_thsh=25.):
"""Initialize camera translation and body orientation
:param model: SMPL model
:param j2d: 14x2 array of CNN joints
:param img: h x w x 3 image
:param init_pose: 72D vector of pose parameters used for initialization
:param flength: camera focal length (kept fixed)
:param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D
is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both
the estimated one and its flip)
:returns: a tuple containing the estimated camera,
a boolean deciding if both the optimized body orientation and its flip should be considered,
3D vector for the body orientation
"""
# optimize camera translation and body orientation based on torso joints
# LSP torso ids:
# 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder
torso_cids = [2, 3, 8, 9]
# corresponding SMPL torso ids
torso_smpl_ids = [2, 1, 17, 16]
center = np.array([img.shape[1] / 2, img.shape[0] / 2])
# initialize camera rotation
rt = ch.zeros(3)
# initialize camera translation
print 'initializing translation via similar triangles'
init_t = guess_init(model, flength, j2d, init_pose)
t = ch.array(init_t)
# check how close the shoulder joints are
try_both_orient = np.linalg.norm(j2d[8] - j2d[9]) < pix_thsh
opt_pose = ch.array(init_pose)
(_, A_global) = global_rigid_transformation(opt_pose,
model.J,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
# initialize the camera
cam = ProjectPoints(f=np.array([flength, flength]),
rt=rt,
t=t,
k=np.zeros(5),
c=center)
# we are going to project the SMPL joints
cam.v = Jtr
on_step = None
# optimize for camera translation and body orientation
free_variables = [cam.t, opt_pose[:3]]
ch.minimize(
# data term defined over torso joints...
{
'cam': j2d[torso_cids] - cam[torso_smpl_ids],
# ...plus a regularizer for the camera translation
'cam_t': 1e2 * (cam.t[2] - init_t[2])
},
x0=free_variables,
method='dogleg',
callback=on_step,
# maxiter: 100, e_3: .0001
options={
'maxiter': 100,
'e_3': .0001,
# disp set to 1 enables verbose output from the optimizer
'disp': 0
})
return (cam, try_both_orient, opt_pose[:3].r)
