def fit_lmk3d(target_3d_lmks, model_fname, lmk_face_idx, lmk_b_coords, weights, show_fitting=True):
'''
Fit FLAME to 3D landmarks
:param target_3d_lmks: target 3D landmarks provided as (num_lmks x 3) matrix
:param model_fname: saved Tensorflow FLAME model
:param lmk_face_idx: face indices of the landmark embedding in the FLAME topology
:param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology
(i.e. weighting of the three vertices for the trinagle, the landmark is embedded in
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
lmks = tf_get_model_lmks(tf_model, smpl.f, lmk_face_idx, lmk_b_coords)
lmk_dist = tf.reduce_sum(tf.square(1000 * tf.subtract(lmks, target_3d_lmks)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,3:6]))
eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
# Optimize global transformation first
vars = [tf_trans, tf_rot]
loss = weights['lmk'] * lmk_dist
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6})
print('Optimize rigid transformation')
optimizer.minimize(session)
# Optimize for the model parameters
vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp]
loss = weights['lmk'] * lmk_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \
weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6})
print('Optimize model parameters')
optimizer.minimize(session)
print('Fitting done')
if show_fitting:
# Visualize landmark fitting
mv = MeshViewer()
mv.set_static_meshes(create_lmk_spheres(target_3d_lmks, 0.001, [255.0, 0.0, 0.0]))
mv.set_dynamic_meshes([Mesh(session.run(tf_model), smpl.f)] + create_lmk_spheres(session.run(lmks), 0.001, [0.0, 0.0, 255.0]), blocking=True)
six.moves.input('Press key to continue')
return Mesh(session.run(tf_model), smpl.f)
def fit_lmk2d(target_img, target_2d_lmks, template_fname, model_fname,
lmk_face_idx, lmk_b_coords, weights):
'''
Fit FLAME to 2D landmarks
:param target_2d_lmks: target 2D landmarks provided as (num_lmks x 3) matrix
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param model_fname: saved FLAME model
:param lmk_face_idx: face indices of the landmark embedding in the FLAME topology
:param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology
(i.e. weighting of the three vertices for the trinagle, the landmark is embedded in
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
template_mesh = Mesh(filename=template_fname)
tf_trans = tf.Variable(np.zeros((1, 3)),
name="trans",
dtype=tf.float64,
trainable=True)
tf_rot = tf.Variable(np.zeros((1, 3)),
name="pose",
dtype=tf.float64,
trainable=True)
tf_pose = tf.Variable(np.zeros((1, 12)),
name="pose",
dtype=tf.float64,
trainable=True)
tf_shape = tf.Variable(np.zeros((1, 300)),
name="shape",
dtype=tf.float64,
trainable=True)
tf_exp = tf.Variable(np.zeros((1, 100)),
name="expression",
dtype=tf.float64,
trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(
smpl(tf_trans, tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
# Mirror landmark y-coordinates
target_2d_lmks[:, 1] = target_img.shape[0] - target_2d_lmks[:, 1]
lmks_3d = tf_get_model_lmks(tf_model, template_mesh, lmk_face_idx,
lmk_b_coords)
s2d = np.mean(
np.linalg.norm(target_2d_lmks - np.mean(target_2d_lmks, axis=0),
axis=1))
s3d = tf.reduce_mean(
tf.sqrt(
tf.reduce_sum(
tf.square(lmks_3d -
tf.reduce_mean(lmks_3d, axis=0))[:, :2],
axis=1)))
tf_scale = tf.Variable(s2d / s3d, dtype=lmks_3d.dtype)
# trans = 0.5*np.array((target_img.shape[0], target_img.shape[1]))/tf_scale
# trans = 0.5 * s3d * np.array((target_img.shape[0], target_img.shape[1])) / s2d
lmks_proj_2d = tf_project_points(lmks_3d, tf_scale, np.zeros(2))
factor = max(
max(target_2d_lmks[:, 0]) - min(target_2d_lmks[:, 0]),
max(target_2d_lmks[:, 1]) - min(target_2d_lmks[:, 1]))
lmk_dist = weights['lmk'] * tf.reduce_sum(
tf.square(tf.subtract(lmks_proj_2d, target_2d_lmks))) / (factor**2)
neck_pose_reg = weights['neck_pose'] * tf.reduce_sum(
tf.square(tf_pose[:3]))
jaw_pose_reg = weights['jaw_pose'] * tf.reduce_sum(
tf.square(tf_pose[3:6]))
eyeballs_pose_reg = weights['eyeballs_pose'] * tf.reduce_sum(
tf.square(tf_pose[6:]))
shape_reg = weights['shape'] * tf.reduce_sum(tf.square(tf_shape))
exp_reg = weights['expr'] * tf.reduce_sum(tf.square(tf_exp))
session.run(tf.global_variables_initializer())
def on_step(verts,
scale,
faces,
target_img,
target_lmks,
opt_lmks,
lmk_dist=0.0,
shape_reg=0.0,
exp_reg=0.0,
neck_pose_reg=0.0,
jaw_pose_reg=0.0,
eyeballs_pose_reg=0.0):
import cv2
import sys
import numpy as np
from psbody.mesh import Mesh
from utils.render_mesh import render_mesh
if lmk_dist > 0.0 or shape_reg > 0.0 or exp_reg > 0.0 or neck_pose_reg > 0.0 or jaw_pose_reg > 0.0 or eyeballs_pose_reg > 0.0:
print(
'lmk_dist: %f, shape_reg: %f, exp_reg: %f, neck_pose_reg: %f, jaw_pose_reg: %f, eyeballs_pose_reg: %f'
% (lmk_dist, shape_reg, exp_reg, neck_pose_reg,
jaw_pose_reg, eyeballs_pose_reg))
plt_target_lmks = target_lmks.copy()
plt_target_lmks[:, 1] = target_img.shape[0] - plt_target_lmks[:, 1]
for (x, y) in plt_target_lmks:
cv2.circle(target_img, (int(x), int(y)), 4, (0, 0, 255), -1)
plt_opt_lmks = opt_lmks.copy()
plt_opt_lmks[:, 1] = target_img.shape[0] - plt_opt_lmks[:, 1]
for (x, y) in plt_opt_lmks:
cv2.circle(target_img, (int(x), int(y)), 4, (255, 0, 0), -1)
if sys.version_info >= (3, 0):
rendered_img = render_mesh(Mesh(scale * verts, faces),
height=target_img.shape[0],
width=target_img.shape[1])
for (x, y) in plt_opt_lmks:
cv2.circle(rendered_img, (int(x), int(y)), 4, (255, 0, 0),
-1)
target_img = np.hstack((target_img, rendered_img))
cv2.imshow('img', target_img)
cv2.waitKey(10)
print('Optimize rigid transformation')
vars = [tf_scale, tf_trans, tf_rot]
loss = lmk_dist
optimizer = scipy_pt(loss=loss,
var_list=vars,
method='L-BFGS-B',
options={
'disp': 1,
'ftol': 5e-6
})
optimizer.minimize(session,
fetches=[
tf_model, tf_scale,
tf.constant(template_mesh.f),
tf.constant(target_img),
tf.constant(target_2d_lmks), lmks_proj_2d
],
loss_callback=on_step)
print('Optimize model parameters')
vars = [tf_scale, tf_trans[:2], tf_rot, tf_pose, tf_shape, tf_exp]
loss = lmk_dist + shape_reg + exp_reg + neck_pose_reg + jaw_pose_reg + eyeballs_pose_reg
optimizer = scipy_pt(loss=loss,
var_list=vars,
method='L-BFGS-B',
options={
'disp': 0,
'ftol': 1e-7
})
optimizer.minimize(session,
fetches=[
tf_model, tf_scale,
tf.constant(template_mesh.f),
tf.constant(target_img),
tf.constant(target_2d_lmks), lmks_proj_2d,
lmk_dist, shape_reg, exp_reg, neck_pose_reg,
jaw_pose_reg, eyeballs_pose_reg
],
loss_callback=on_step)
print('Fitting done')
np_verts, np_scale = session.run([tf_model, tf_scale])
def _get_verts(expr):
assign_expr = tf.assign(tf_exp, expr[np.newaxis, :])
session.run([assign_expr])
_verts, _scale = session.run([tf_model, tf_scale])
return _verts
expr_basis = []
neutral_verts = _get_verts(np.zeros((100), np.float64))
for i in range(100):
expr_i = np.zeros((100), np.float64)
expr_i[i] = 1
verts_i = _get_verts(expr_i) - neutral_verts
expr_basis.append(verts_i.flatten())
expr_basis = np.asarray(expr_basis, np.float32).transpose(1, 0)
return Mesh(np_verts, template_mesh.f), np_scale, expr_basis
def fit_lmk2d(target_img, target_2d_lmks, template_fname, tf_model_fname,
lmk_face_idx, lmk_b_coords, weights):
'''
Fit FLAME to 2D landmarks
:param target_2d_lmks: target 2D landmarks provided as (num_lmks x 3) matrix
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param tf_model_fname: saved Tensorflow FLAME model
:param lmk_face_idx: face indices of the landmark embedding in the FLAME topology
:param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology
(i.e. weighting of the three vertices for the trinagle, the landmark is embedded in
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
template_mesh = Mesh(filename=template_fname)
saver = tf.train.import_meta_graph(tf_model_fname + '.meta')
graph = tf.get_default_graph()
tf_model = graph.get_tensor_by_name(u'vertices:0')
with tf.Session() as session:
saver.restore(session, tf_model_fname)
# Workaround as existing tf.Variable cannot be retrieved back with tf.get_variable
# tf_v_template = [x for x in tf.trainable_variables() if 'v_template' in x.name][0]
tf_trans = [x for x in tf.trainable_variables()
if 'trans' in x.name][0]
tf_rot = [x for x in tf.trainable_variables() if 'rot' in x.name][0]
tf_pose = [x for x in tf.trainable_variables() if 'pose' in x.name][0]
tf_shape = [x for x in tf.trainable_variables()
if 'shape' in x.name][0]
tf_exp = [x for x in tf.trainable_variables() if 'exp' in x.name][0]
# Mirror landmark y-coordinates
target_2d_lmks[:, 1] = target_img.shape[0] - target_2d_lmks[:, 1]
lmks_3d = tf_get_model_lmks(tf_model, template_mesh, lmk_face_idx,
lmk_b_coords)
s2d = np.mean(
np.linalg.norm(target_2d_lmks - np.mean(target_2d_lmks, axis=0),
axis=1))
s3d = tf.reduce_mean(
tf.sqrt(
tf.reduce_sum(
tf.square(lmks_3d -
tf.reduce_mean(lmks_3d, axis=0))[:, :2],
axis=1)))
tf_scale = tf.Variable(s2d / s3d, dtype=lmks_3d.dtype)
# trans = 0.5*np.array((target_img.shape[0], target_img.shape[1]))/tf_scale
# trans = 0.5 * s3d * np.array((target_img.shape[0], target_img.shape[1])) / s2d
lmks_proj_2d = tf_project_points(lmks_3d, tf_scale, np.zeros(2))
factor = max(
max(target_2d_lmks[:, 0]) - min(target_2d_lmks[:, 0]),
max(target_2d_lmks[:, 1]) - min(target_2d_lmks[:, 1]))
lmk_dist = weights['lmk'] * tf.reduce_sum(
tf.square(tf.subtract(lmks_proj_2d, target_2d_lmks))) / (factor**2)
neck_pose_reg = weights['neck_pose'] * tf.reduce_sum(
tf.square(tf_pose[:3]))
jaw_pose_reg = weights['jaw_pose'] * tf.reduce_sum(
tf.square(tf_pose[3:6]))
eyeballs_pose_reg = weights['eyeballs_pose'] * tf.reduce_sum(
tf.square(tf_pose[6:]))
shape_reg = weights['shape'] * tf.reduce_sum(tf.square(tf_shape))
exp_reg = weights['expr'] * tf.reduce_sum(tf.square(tf_exp))
session.run(tf.global_variables_initializer())
def on_step(verts,
scale,
faces,
target_img,
target_lmks,
opt_lmks,
lmk_dist=0.0,
shape_reg=0.0,
exp_reg=0.0,
neck_pose_reg=0.0,
jaw_pose_reg=0.0,
eyeballs_pose_reg=0.0):
import cv2
import sys
import numpy as np
from psbody.mesh import Mesh
from utils.render_mesh import render_mesh
if lmk_dist > 0.0 or shape_reg > 0.0 or exp_reg > 0.0 or neck_pose_reg > 0.0 or jaw_pose_reg > 0.0 or eyeballs_pose_reg > 0.0:
print(
'lmk_dist: %f, shape_reg: %f, exp_reg: %f, neck_pose_reg: %f, jaw_pose_reg: %f, eyeballs_pose_reg: %f'
% (lmk_dist, shape_reg, exp_reg, neck_pose_reg,
jaw_pose_reg, eyeballs_pose_reg))
plt_target_lmks = target_lmks.copy()
plt_target_lmks[:, 1] = target_img.shape[0] - plt_target_lmks[:, 1]
for (x, y) in plt_target_lmks:
cv2.circle(target_img, (int(x), int(y)), 4, (0, 0, 255), -1)
plt_opt_lmks = opt_lmks.copy()
plt_opt_lmks[:, 1] = target_img.shape[0] - plt_opt_lmks[:, 1]
for (x, y) in plt_opt_lmks:
cv2.circle(target_img, (int(x), int(y)), 4, (255, 0, 0), -1)
if sys.version_info >= (3, 0):
rendered_img = render_mesh(Mesh(scale * verts, faces),
height=target_img.shape[0],
width=target_img.shape[1])
for (x, y) in plt_opt_lmks:
cv2.circle(rendered_img, (int(x), int(y)), 4, (255, 0, 0),
-1)
target_img = np.hstack((target_img, rendered_img))
cv2.imshow('img', target_img)
cv2.waitKey(10)
print('Optimize rigid transformation')
vars = [tf_scale, tf_trans, tf_rot]
loss = lmk_dist
optimizer = scipy_pt(loss=loss,
var_list=vars,
method='L-BFGS-B',
options={
'disp': 1,
'ftol': 5e-6
})
optimizer.minimize(session,
fetches=[
tf_model, tf_scale,
tf.constant(template_mesh.f),
tf.constant(target_img),
tf.constant(target_2d_lmks), lmks_proj_2d
],
loss_callback=on_step)
print('Optimize model parameters')
vars = [tf_scale, tf_trans[:2], tf_rot, tf_pose, tf_shape, tf_exp]
loss = lmk_dist + shape_reg + exp_reg + neck_pose_reg + jaw_pose_reg + eyeballs_pose_reg
optimizer = scipy_pt(loss=loss,
var_list=vars,
method='L-BFGS-B',
options={
'disp': 0,
'ftol': 1e-7
})
optimizer.minimize(session,
fetches=[
tf_model, tf_scale,
tf.constant(template_mesh.f),
tf.constant(target_img),
tf.constant(target_2d_lmks), lmks_proj_2d,
lmk_dist, shape_reg, exp_reg, neck_pose_reg,
jaw_pose_reg, eyeballs_pose_reg
],
loss_callback=on_step)
print('Fitting done')
np_verts, np_scale = session.run([tf_model, tf_scale])
return Mesh(np_verts, template_mesh.f), np_scale
def fit_lmk2d(target_img, target_2d_lmks, model_fname, lmk_face_idx, lmk_b_coords, weights, visualize):
'''
Fit FLAME to 2D landmarks
:param target_2d_lmks target 2D landmarks provided as (num_lmks x 3) matrix
:param model_fname saved FLAME model
:param lmk_face_idx face indices of the landmark embedding in the FLAME topology
:param lmk_b_coords barycentric coordinates of the landmark embedding in the FLAME topology
(i.e. weighting of the three vertices for the trinagle, the landmark is embedded in
:param weights weights of the individual objective functions
:param visualize visualize fitting progress
:return: a mesh with the fitting results
'''
'''
pred_types = {'face': pred_type(slice(0, 17), (0.682, 0.780, 0.909, 0.5)),
'eyebrow1': pred_type(slice(17, 22), (1.0, 0.498, 0.055, 0.4)),
'eyebrow2': pred_type(slice(22, 27), (1.0, 0.498, 0.055, 0.4)),
'nose': pred_type(slice(27, 31), (0.345, 0.239, 0.443, 0.4)),
'nostril': pred_type(slice(31, 36), (0.345, 0.239, 0.443, 0.4)),
'eye1': pred_type(slice(36, 42), (0.596, 0.875, 0.541, 0.3)),
'eye2': pred_type(slice(42, 48), (0.596, 0.875, 0.541, 0.3)),
'lips': pred_type(slice(48, 60), (0.596, 0.875, 0.541, 0.3)),
'teeth': pred_type(slice(60, 68), (0.596, 0.875, 0.541, 0.4))
}
'''
lmks_weights = [[1,1]] * 68
for idx in range(36, 48):
lmks_weights[idx] = [100, 100]
tf_lmks_weights = tf.constant(
lmks_weights,
tf.float64
)
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="rot", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
# session.run(tf.global_variables_initializer())
# Mirror landmark y-coordinates
target_2d_lmks[:,1] = target_img.shape[0]-target_2d_lmks[:,1]
lmks_3d = tf_get_model_lmks(tf_model, smpl.f, lmk_face_idx, lmk_b_coords)
s2d = np.mean(np.linalg.norm(target_2d_lmks-np.mean(target_2d_lmks, axis=0), axis=1))
s3d = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square(lmks_3d-tf.reduce_mean(lmks_3d, axis=0))[:, :2], axis=1)))
tf_scale = tf.Variable(s2d/s3d, dtype=lmks_3d.dtype)
# trans = 0.5*np.array((target_img.shape[0], target_img.shape[1]))/tf_scale
# trans = 0.5 * s3d * np.array((target_img.shape[0], target_img.shape[1])) / s2d
lmks_proj_2d = tf_project_points(lmks_3d, tf_scale, np.zeros(2))
factor = max(max(target_2d_lmks[:,0]) - min(target_2d_lmks[:,0]),max(target_2d_lmks[:,1]) - min(target_2d_lmks[:,1]))
#lmk_dist = weights['lmk']*tf.reduce_sum(tf.square(tf.subtract(lmks_proj_2d, target_2d_lmks))) / (factor ** 2)
lmk_dist = weights['lmk']*tf.reduce_sum(
tf.square(tf.subtract(lmks_proj_2d, target_2d_lmks)) * tf_lmks_weights
) / (factor ** 2)
neck_pose_reg = weights['neck_pose']*tf.reduce_sum(tf.square(tf_pose[:,:3]))
jaw_pose_reg = weights['jaw_pose']*tf.reduce_sum(tf.square(tf_pose[:,3:6]))
eyeballs_pose_reg = weights['eyeballs_pose']*tf.reduce_sum(tf.square(tf_pose[:,6:]))
shape_reg = weights['shape']*tf.reduce_sum(tf.square(tf_shape))
exp_reg = weights['expr']*tf.reduce_sum(tf.square(tf_exp))
session.run(tf.global_variables_initializer())
if visualize:
def on_step(verts, scale, faces, target_img, target_lmks, opt_lmks, lmk_dist=0.0, shape_reg=0.0, exp_reg=0.0, neck_pose_reg=0.0, jaw_pose_reg=0.0, eyeballs_pose_reg=0.0):
import cv2
import sys
import numpy as np
from psbody.mesh import Mesh
from utils.render_mesh import render_mesh
if lmk_dist>0.0 or shape_reg>0.0 or exp_reg>0.0 or neck_pose_reg>0.0 or jaw_pose_reg>0.0 or eyeballs_pose_reg>0.0:
print('lmk_dist: %f, shape_reg: %f, exp_reg: %f, neck_pose_reg: %f, jaw_pose_reg: %f, eyeballs_pose_reg: %f' % (lmk_dist, shape_reg, exp_reg, neck_pose_reg, jaw_pose_reg, eyeballs_pose_reg))
plt_target_lmks = target_lmks.copy()
plt_target_lmks[:, 1] = target_img.shape[0] - plt_target_lmks[:, 1]
for (x, y) in plt_target_lmks:
cv2.circle(target_img, (int(x), int(y)), 4, (0, 0, 255), -1)
plt_opt_lmks = opt_lmks.copy()
plt_opt_lmks[:,1] = target_img.shape[0] - plt_opt_lmks[:,1]
for (x, y) in plt_opt_lmks:
cv2.circle(target_img, (int(x), int(y)), 4, (255, 0, 0), -1)
if sys.version_info >= (3, 0):
rendered_img = render_mesh(Mesh(scale*verts, faces), height=target_img.shape[0], width=target_img.shape[1])
for (x, y) in plt_opt_lmks:
cv2.circle(rendered_img, (int(x), int(y)), 4, (255, 0, 0), -1)
target_img = np.hstack((target_img, rendered_img))
#cv2.imshow('img', target_img)
#cv2.waitKey(10)
else:
def on_step(*_):
pass
print('Optimize rigid transformation')
vars = [tf_scale, tf_trans, tf_rot]
loss = lmk_dist
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6})
optimizer.minimize(session, fetches=[tf_model, tf_scale, tf.constant(smpl.f), tf.constant(target_img), tf.constant(target_2d_lmks), lmks_proj_2d], loss_callback=on_step)
print('Optimize model parameters')
vars = [tf_scale, tf_trans[:2], tf_rot, tf_pose, tf_shape, tf_exp]
loss = lmk_dist + shape_reg + exp_reg + neck_pose_reg + jaw_pose_reg + eyeballs_pose_reg
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 0, 'ftol': 1e-7})
optimizer.minimize(session, fetches=[tf_model, tf_scale, tf.constant(smpl.f), tf.constant(target_img), tf.constant(target_2d_lmks), lmks_proj_2d,
lmk_dist, shape_reg, exp_reg, neck_pose_reg, jaw_pose_reg, eyeballs_pose_reg], loss_callback=on_step)
print('Fitting done')
np_verts, np_scale = session.run([tf_model, tf_scale])
return Mesh(np_verts, smpl.f), np_scale
def fit_lmk3d(target_3d_lmks,
template_fname,
tf_model_fname,
lmk_face_idx,
lmk_b_coords,
weights,
show_fitting=True):
'''
Fit FLAME to 3D landmarks
:param target_3d_lmks: target 3D landmarks provided as (num_lmks x 3) matrix
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param tf_model_fname: saved Tensorflow FLAME model
:param lmk_face_idx: face indices of the landmark embedding in the FLAME topology
:param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology
(i.e. weighting of the three vertices for the trinagle, the landmark is embedded in
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
template_mesh = Mesh(filename=template_fname)
saver = tf.train.import_meta_graph(tf_model_fname + '.meta')
graph = tf.get_default_graph()
tf_model = graph.get_tensor_by_name(u'vertices:0')
with tf.Session() as session:
saver.restore(session, tf_model_fname)
# Workaround as existing tf.Variable cannot be retrieved back with tf.get_variable
# tf_v_template = [x for x in tf.trainable_variables() if 'v_template' in x.name][0]
tf_trans = [x for x in tf.trainable_variables()
if 'trans' in x.name][0]
tf_rot = [x for x in tf.trainable_variables() if 'rot' in x.name][0]
tf_pose = [x for x in tf.trainable_variables() if 'pose' in x.name][0]
tf_shape = [x for x in tf.trainable_variables()
if 'shape' in x.name][0]
tf_exp = [x for x in tf.trainable_variables() if 'exp' in x.name][0]
lmks = tf_get_model_lmks(tf_model, template_mesh, lmk_face_idx,
lmk_b_coords)
lmk_dist = tf.reduce_sum(
tf.square(1000 * tf.subtract(lmks, target_3d_lmks)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[3:6]))
eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
# Optimize global transformation first
vars = [tf_trans, tf_rot]
loss = weights['lmk'] * lmk_dist
optimizer = scipy_pt(loss=loss,
var_list=vars,
method='L-BFGS-B',
options={
'disp': 1,
'ftol': 5e-6
})
print('Optimize rigid transformation')
optimizer.minimize(session)
# Optimize for the model parameters
vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp]
loss = weights['lmk'] * lmk_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \
weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg
optimizer = scipy_pt(loss=loss,
var_list=vars,
method='L-BFGS-B',
options={
'disp': 1,
'ftol': 5e-6
})
print('Optimize model parameters')
optimizer.minimize(session)
print('Fitting done')
if show_fitting:
# Visualize landmark fitting
mv = MeshViewer()
mv.set_static_meshes(
create_lmk_spheres(target_3d_lmks, 0.001, [255.0, 0.0, 0.0]))
mv.set_dynamic_meshes(
[Mesh(session.run(tf_model), template_mesh.f)] +
create_lmk_spheres(session.run(lmks), 0.001,
[0.0, 0.0, 255.0]),
blocking=True)
six.moves.input('Press key to continue')
return Mesh(session.run(tf_model), template_mesh.f)
