def rigid_scan_2_mesh_alignment(scan, mesh, visualize=False):
options = {'sparse_solver': lambda A, x: cg(A, x, maxiter=2000)[0]}
options['disp'] = 1.0
options['delta_0'] = 0.1
options['e_3'] = 1e-4
s = ch.ones(1)
r = ch.zeros(3)
R = Rodrigues(r)
t = ch.zeros(3)
trafo_mesh = s*(R.dot(mesh.v.T)).T + t
sampler = sample_from_mesh(scan, sample_type='vertices')
s2m = ScanToMesh(scan, trafo_mesh, mesh.f, scan_sampler=sampler, signed=False, normalize=False)
if visualize:
#Visualization code
mv = MeshViewer()
mv.set_static_meshes([scan])
tmp_mesh = Mesh(trafo_mesh.r, mesh.f)
tmp_mesh.set_vertex_colors('light sky blue')
mv.set_dynamic_meshes([tmp_mesh])
def on_show(_):
tmp_mesh = Mesh(trafo_mesh.r, mesh.f)
tmp_mesh.set_vertex_colors('light sky blue')
mv.set_dynamic_meshes([tmp_mesh])
else:
def on_show(_):
pass
ch.minimize(fun={'dist': s2m, 's_reg': 100*(ch.abs(s)-s)}, x0=[s, r, t], callback=on_show, options=options)
return s,Rodrigues(r),t
def fit_3D_mesh(target_3d_mesh_fname, model_fname, weights, show_fitting=True):
'''
Fit FLAME to 3D mesh in correspondence to the FLAME mesh (i.e. same number of vertices, same mesh topology)
:param target_3d_mesh_fname: target 3D mesh filename
:param model_fname: saved FLAME model
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
target_mesh = Mesh(filename=target_3d_mesh_fname)
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
mesh_dist = tf.reduce_sum(tf.square(tf.subtract(tf_model, target_mesh.v)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,3:6]))
eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
# Optimize global transformation first
vars = [tf_trans, tf_rot]
loss = mesh_dist
optimizer = scipy_pt(loss=loss, var_list=vars, method='BFGS', options={'disp': 1})
print('Optimize rigid transformation')
optimizer.minimize(session)
# Optimize for the model parameters
vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp]
loss = weights['data'] * mesh_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \
weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg
optimizer = scipy_pt(loss=loss, var_list=vars, method='BFGS', options={'disp': 1})
print('Optimize model parameters')
optimizer.minimize(session)
print('Fitting done')
if show_fitting:
# Visualize fitting
mv = MeshViewer()
fitting_mesh = Mesh(session.run(tf_model), smpl.f)
fitting_mesh.set_vertex_colors('light sky blue')
mv.set_static_meshes([target_mesh, fitting_mesh])
six.moves.input('Press key to continue')
return Mesh(session.run(tf_model), smpl.f)
def fit_lmk3d(target_3d_lmks, model_fname, lmk_face_idx, lmk_b_coords, weights, show_fitting=True):
'''
Fit FLAME to 3D landmarks
:param target_3d_lmks: target 3D landmarks provided as (num_lmks x 3) matrix
:param model_fname: saved Tensorflow FLAME model
:param lmk_face_idx: face indices of the landmark embedding in the FLAME topology
:param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology
(i.e. weighting of the three vertices for the trinagle, the landmark is embedded in
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
lmks = tf_get_model_lmks(tf_model, smpl.f, lmk_face_idx, lmk_b_coords)
lmk_dist = tf.reduce_sum(tf.square(1000 * tf.subtract(lmks, target_3d_lmks)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,3:6]))
eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
# Optimize global transformation first
vars = [tf_trans, tf_rot]
loss = weights['lmk'] * lmk_dist
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6})
print('Optimize rigid transformation')
optimizer.minimize(session)
# Optimize for the model parameters
vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp]
loss = weights['lmk'] * lmk_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \
weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6})
print('Optimize model parameters')
optimizer.minimize(session)
print('Fitting done')
if show_fitting:
# Visualize landmark fitting
mv = MeshViewer()
mv.set_static_meshes(create_lmk_spheres(target_3d_lmks, 0.001, [255.0, 0.0, 0.0]))
mv.set_dynamic_meshes([Mesh(session.run(tf_model), smpl.f)] + create_lmk_spheres(session.run(lmks), 0.001, [0.0, 0.0, 255.0]), blocking=True)
six.moves.input('Press key to continue')
return Mesh(session.run(tf_model), smpl.f)
def sample_texture(model_fname, texture_fname, num_samples, out_path):
'''
Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to
- model identity-dependent shape variations (paramters: shape),
- articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12])
- model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp)
- global translation (paramters: trans)
- global rotation (paramters: rot)
:param model_fname saved FLAME model
:param num_samples number of samples
:param out_path output path to save the generated templates (no templates are saved if path is empty)
'''
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
texture_model = np.load(texture_fname)
tex_dim = texture_model['tex_dir'].shape[-1]
tf_tex_params = tf.Variable(np.zeros((1,tex_dim)), name="pose", dtype=tf.float64, trainable=True)
tf_tex_mean = tf.Variable(np.reshape(texture_model['mean'], (1,-1)), name='tex_mean', dtype=tf.float64, trainable=False)
tf_tex_dir = tf.Variable(np.reshape(texture_model['tex_dir'], (-1, tex_dim)).T, name='tex_dir', dtype=tf.float64, trainable=False)
tf_tex = tf.add(tf_tex_mean, tf.matmul(tf_tex_params, tf_tex_dir)),
tf_tex = tf.reshape(tf_tex, (texture_model['tex_dir'].shape[0], texture_model['tex_dir'].shape[1], texture_model['tex_dir'].shape[2]))
tf_tex = tf.cast(tf.clip_by_value(tf_tex, 0.0, 255.0), tf.int64)
with tf.Session() as session:
session.run(tf.global_variables_initializer())
mv = MeshViewer()
for i in range(num_samples):
assign_tex = tf.assign(tf_tex_params, np.random.randn(tex_dim)[np.newaxis,:])
session.run([assign_tex])
v, tex = session.run([tf_model, tf_tex])
out_mesh = Mesh(v, smpl.f)
out_mesh.vt = texture_model['vt']
out_mesh.ft = texture_model['ft']
mv.set_dynamic_meshes([out_mesh], blocking=True)
key = six.moves.input('Press (s) to save sample, any other key to continue ')
if key == 's':
out_mesh_fname = os.path.join(out_path, 'tex_sample_%02d.obj' % (i+1))
out_tex_fname = out_mesh_fname.replace('obj', 'png')
cv2.imwrite(out_tex_fname, tex)
out_mesh.set_texture_image(out_tex_fname)
out_mesh.write_obj(out_mesh_fname)
def sample_FLAME(model_fname, num_samples, out_path, visualize, sample_VOCA_template=False):
'''
Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to
- model identity-dependent shape variations (paramters: shape),
- articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12])
- model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp)
- global translation (paramters: trans)
- global rotation (paramters: rot)
:param model_fname saved FLAME model
:param num_samples number of samples
:param out_path output path to save the generated templates (no templates are saved if path is empty)
:param visualize visualize samples
:param sample_VOCA_template sample template in 'zero pose' that can be used e.g. for speech-driven animation in VOCA
'''
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
if visualize:
mv = MeshViewer()
for i in range(num_samples):
if sample_VOCA_template:
assign_shape = tf.assign(tf_shape, np.hstack((np.random.randn(100), np.zeros(200)))[np.newaxis,:])
session.run([assign_shape])
out_fname = os.path.join(out_path, 'VOCA_template_%02d.ply' % (i+1))
else:
# assign_trans = tf.assign(tf_trans, np.random.randn(3)[np.newaxis,:])
assign_rot = tf.assign(tf_rot, np.random.randn(3)[np.newaxis,:] * 0.03)
assign_pose = tf.assign(tf_pose, np.random.randn(12)[np.newaxis,:] * 0.02)
assign_shape = tf.assign(tf_shape, np.hstack((np.random.randn(100), np.zeros(200)))[np.newaxis,:])
assign_exp = tf.assign(tf_exp, np.hstack((0.5*np.random.randn(50), np.zeros(50)))[np.newaxis,:])
session.run([assign_rot, assign_pose, assign_shape, assign_exp])
out_fname = os.path.join(out_path, 'FLAME_sample_%02d.ply' % (i+1))
sample_mesh = Mesh(session.run(tf_model), smpl.f)
if visualize:
mv.set_dynamic_meshes([sample_mesh], blocking=True)
key = six.moves.input('Press (s) to save sample, any other key to continue ')
if key == 's':
sample_mesh.write_ply(out_fname)
else:
sample_mesh.write_ply(out_fname)
def visualize(model, self_intersection_ids, default_model, vertices, faces):
radius = 0.01
from psbody.mesh.meshviewer import MeshViewer
# from mesh.mesh.meshviewer import MeshViewer
mv = MeshViewer(window_width=800, window_height=800)
mesh = Mesh(v=default_model.r, f=default_model.f)
mesh.f = []
if self_intersection_ids == []:
return
mv.set_static_meshes([mesh, meanSphere], blocking=True)
else:
m = Mesh(v=model.v, f=model.f, vc='SeaGreen')
m.vc[m.f[faces][self_intersection_ids].flatten()] = ones(1)
mv.set_static_meshes([m], blocking=True)
raw_input('Press Enter to exit')
def sample_FLAME(template_fname, tf_model_fname, num_samples):
'''
Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to
- model identity-dependent shape variations (paramters: shape),
- articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12])
- model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp)
- global translation (paramters: trans)
- global rotation (paramters: rot)
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param tf_model_fname: saved Tensorflow FLAME model
'''
template_mesh = Mesh(filename=template_fname)
saver = tf.train.import_meta_graph(tf_model_fname + '.meta')
graph = tf.get_default_graph()
tf_model = graph.get_tensor_by_name(u'vertices:0')
with tf.Session() as session:
saver.restore(session, tf_model_fname)
# Workaround as existing tf.Variable cannot be retrieved back with tf.get_variable
tf_trans = [x for x in tf.trainable_variables()
if 'trans' in x.name][0]
tf_rot = [x for x in tf.trainable_variables() if 'rot' in x.name][0]
tf_pose = [x for x in tf.trainable_variables() if 'pose' in x.name][0]
tf_shape = [x for x in tf.trainable_variables()
if 'shape' in x.name][0]
tf_exp = [x for x in tf.trainable_variables() if 'exp' in x.name][0]
mv = MeshViewer()
for i in range(num_samples):
assign_trans = tf.assign(tf_trans, np.random.randn(3))
assign_rot = tf.assign(tf_rot, np.random.randn(3) * 0.03)
assign_pose = tf.assign(tf_pose, np.random.randn(12) * 0.03)
assign_shape = tf.assign(tf_shape, np.random.randn(300) * 1.0)
assign_exp = tf.assign(tf_exp, np.random.randn(100) * 0.5)
session.run([
assign_trans, assign_rot, assign_pose, assign_shape, assign_exp
])
mv.set_dynamic_meshes(
[Mesh(session.run(tf_model), template_mesh.f)], blocking=True)
raw_input('Press key to continue')
def sample_FLAME(template_fname, model_fname, num_samples):
'''
Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to
- model identity-dependent shape variations (paramters: shape),
- articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12])
- model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp)
- global translation (paramters: trans)
- global rotation (paramters: rot)
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param model_fname: saved FLAME model
'''
template_mesh = Mesh(filename=template_fname)
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
mv = MeshViewer()
for i in range(num_samples):
assign_trans = tf.assign(tf_trans, np.random.randn(3)[np.newaxis,:])
assign_rot = tf.assign(tf_rot, np.random.randn(3)[np.newaxis,:] * 0.03)
assign_pose = tf.assign(tf_pose, np.random.randn(12)[np.newaxis,:] * 0.03)
assign_shape = tf.assign(tf_shape, np.random.randn(300)[np.newaxis,:] * 1.0)
assign_exp = tf.assign(tf_exp, np.random.randn(100)[np.newaxis,:] * 0.5)
session.run([assign_trans, assign_rot, assign_pose, assign_shape, assign_exp])
mv.set_dynamic_meshes([Mesh(session.run(tf_model), template_mesh.f)], blocking=True)
six.moves.input('Press key to continue')
(pack(results[i].dot(ch.concatenate((self.J[i, :], [0])))))
for i in range(len(results))
]
result = ch.dstack(results2)
return result, results_global
def compute_r(self):
return self.v.r
def compute_dr_wrt(self, wrt):
if wrt is not self.trans and wrt is not self.betas and wrt is not self.pose and wrt is not self.v_personal and wrt is not self.v_template:
return None
return self.v.dr_wrt(wrt)
if __name__ == '__main__':
from utils.smpl_paths import SmplPaths
dp = SmplPaths(gender='neutral')
smpl = Smpl(dp.get_smpl_file())
from psbody.mesh.meshviewer import MeshViewer
from psbody.mesh import Mesh
mv = MeshViewer()
mv.set_static_meshes([Mesh(smpl.r, smpl.f)])
input("Press Enter to continue...")
def fit_3D_mesh(target_3d_mesh_fname, template_fname, tf_model_fname, weights, show_fitting=True):
'''
Fit FLAME to 3D mesh in correspondence to the FLAME mesh (i.e. same number of vertices, same mesh topology)
:param target_3d_mesh_fname: target 3D mesh filename
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param tf_model_fname: saved Tensorflow FLAME model
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
target_mesh = Mesh(filename=target_3d_mesh_fname)
template_mesh = Mesh(filename=template_fname)
if target_mesh.v.shape[0] != template_mesh.v.shape[0]:
print('Target mesh does not have the same number of vertices')
return
saver = tf.train.import_meta_graph(tf_model_fname + '.meta')
graph = tf.get_default_graph()
tf_model = graph.get_tensor_by_name(u'vertices:0')
with tf.Session() as session:
saver.restore(session, tf_model_fname)
# Workaround as existing tf.Variable cannot be retrieved back with tf.get_variable
# tf_v_template = [x for x in tf.trainable_variables() if 'v_template' in x.name][0]
tf_trans = [x for x in tf.trainable_variables() if 'trans' in x.name][0]
tf_rot = [x for x in tf.trainable_variables() if 'rot' in x.name][0]
tf_pose = [x for x in tf.trainable_variables() if 'pose' in x.name][0]
tf_shape = [x for x in tf.trainable_variables() if 'shape' in x.name][0]
tf_exp = [x for x in tf.trainable_variables() if 'exp' in x.name][0]
mesh_dist = tf.reduce_sum(tf.square(tf.subtract(tf_model, target_mesh.v)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[3:6]))
eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
# Optimize global transformation first
vars = [tf_trans, tf_rot]
loss = mesh_dist
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1})
print('Optimize rigid transformation')
optimizer.minimize(session)
# Optimize for the model parameters
vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp]
loss = mesh_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \
weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1})
print('Optimize model parameters')
optimizer.minimize(session)
print('Fitting done')
if show_fitting:
# Visualize fitting
mv = MeshViewer()
fitting_mesh = Mesh(session.run(tf_model), template_mesh.f)
fitting_mesh.set_vertex_colors('light sky blue')
mv.set_static_meshes([target_mesh, fitting_mesh])
raw_input('Press key to continue')
return Mesh(session.run(tf_model), template_mesh.f)
def fit_sources(
dir_tup_list,
tf_model_fname,
template_fname,
weight_reg_shape,
weight_reg_expr,
weight_reg_neck_pos,
weight_reg_jaw_pos,
weight_reg_eye_pos,
showing=False
):
global g_mv
if showing:
g_mv = MeshViewer()
saver = tf.train.import_meta_graph(tf_model_fname + '.meta')
graph = tf.get_default_graph()
tf_model = graph.get_tensor_by_name(u'vertices:0')
with tf.Session() as session:
saver.restore(session, tf_model_fname)
template = Mesh(filename=template_fname)
tf_src = tf.Variable(tf.zeros(template.v.shape, dtype=tf.float64))
# get all params
tf_trans = [x for x in tf.trainable_variables() if 'trans' in x.name][0]
tf_rot = [x for x in tf.trainable_variables() if 'rot' in x.name][0]
tf_pose = [x for x in tf.trainable_variables() if 'pose' in x.name][0]
tf_shape = [x for x in tf.trainable_variables() if 'shape' in x.name][0]
tf_exp = [x for x in tf.trainable_variables() if 'exp' in x.name][0]
def _save_state(*names, **kwargs):
state = dict()
if "trans" in names: state["trans"] = tf_trans.eval()
if "rot" in names: state["rot"] = tf_rot.eval()
if "pose" in names: state["pose"] = tf_pose.eval()
if "shape" in names: state["shape"] = tf_shape.eval()
if "exp" in names: state["exp"] = tf_exp.eval()
if kwargs.get("set_zero", False):
_zero_state(*names)
return state
def _load_state(state):
ops = []
if "trans" in state: ops.append(tf_trans.assign(state["trans"]))
if "rot" in state: ops.append(tf_rot.assign (state["rot"] ))
if "pose" in state: ops.append(tf_pose.assign (state["pose"] ))
if "shape" in state: ops.append(tf_shape.assign(state["shape"]))
if "exp" in state: ops.append(tf_exp.assign (state["exp"] ))
session.run(ops)
def _zero_state(*names):
ops = []
if "trans" in names: ops.append(tf_trans.assign(tf.zeros_like(tf_trans)))
if "rot" in names: ops.append(tf_rot .assign(tf.zeros_like(tf_rot )))
if "pose" in names: ops.append(tf_pose .assign(tf.zeros_like(tf_pose )))
if "shape" in names: ops.append(tf_shape.assign(tf.zeros_like(tf_shape)))
if "exp" in names: ops.append(tf_exp .assign(tf.zeros_like(tf_exp )))
session.run(ops)
mesh_dist = tf.reduce_sum(tf.square(tf.subtract(tf_model, tf_src)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[3:6]))
eye_pose_reg = tf.reduce_sum(tf.square(tf_pose[6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
reg_term = (
weight_reg_neck_pos * neck_pose_reg +
weight_reg_jaw_pos * jaw_pose_reg +
weight_reg_eye_pos * eye_pose_reg +
weight_reg_shape * shape_reg +
weight_reg_expr * exp_reg
)
# optimizers
optim_shared_rigid = scipy_pt(
loss=mesh_dist,
var_list=[tf_trans, tf_rot],
method='L-BFGS-B',
options={'disp': 0}
)
optim_shared_all = scipy_pt(
loss=mesh_dist+reg_term,
var_list=[tf_trans, tf_rot, tf_pose, tf_shape, tf_exp],
method='L-BFGS-B',
options={'disp': 0}
)
optim_seq = scipy_pt(
loss=mesh_dist+reg_term,
var_list=[tf_shape, tf_exp],
method='L-BFGS-B', options={'disp': 0, 'maxiter': 50}
)
def _fit_sentence(src_dir, dst_dir, prm_dir, last_speaker):
_anchor = os.path.join(dst_dir, "_anchor")
if os.path.exists(_anchor):
print("- Skip " + src_dir)
return
if not os.path.exists(src_dir):
print("- Failed to find " + src_dir)
return
if not os.path.exists(dst_dir): os.makedirs(dst_dir)
if not os.path.exists(prm_dir): os.makedirs(prm_dir)
ply_files = []
for root, _, files in os.walk(src_dir):
for f in files:
if os.path.splitext(f)[1] == ".ply":
ply_files.append(os.path.join(root, f))
ply_files = sorted(ply_files)
# get shared
src_mesh = Mesh(filename=ply_files[0])
session.run(tf.assign(tf_src, src_mesh.v))
speaker = os.path.basename(os.path.dirname(src_dir))
if last_speaker != speaker:
print("- clear speaker information")
_zero_state("trans", "rot", "pose", "shape", "exp")
else:
_zero_state("exp")
stt_dir = os.path.join(os.path.dirname(dst_dir), "state")
if os.path.exists(stt_dir):
state_dict = dict(
trans = np.load(os.path.join(stt_dir, "trans.npy")),
rot = np.load(os.path.join(stt_dir, "rot.npy")),
pose = np.load(os.path.join(stt_dir, "pose.npy")),
shape = np.load(os.path.join(stt_dir, "shape.npy")),
)
_load_state(state_dict)
fitting_mesh = Mesh(session.run(tf_model), src_mesh.f)
fitting_mesh.write_ply(os.path.join(stt_dir, "zero.ply"))
fit_zero_dir = os.path.join(os.path.dirname(os.path.dirname(dst_dir)), "zero_exp")
if not os.path.exists(fit_zero_dir): os.makedirs(fit_zero_dir)
print("- " + speaker + " " + os.path.basename(src_dir))
print(" -> fit shared parameters...")
optim_shared_rigid.minimize(session)
optim_shared_all.minimize(session)
state_dict = _save_state("exp", set_zero=True)
fitting_mesh = Mesh(session.run(tf_model), src_mesh.f)
fitting_mesh.write_ply(os.path.join(fit_zero_dir, "{}.ply".format(speaker)))
_load_state(state_dict)
return
if not os.path.exists(stt_dir): os.makedirs(stt_dir)
np.save(os.path.join(stt_dir, "trans.npy"), tf_trans.eval(), allow_pickle=False)
np.save(os.path.join(stt_dir, "rot.npy"), tf_rot.eval(), allow_pickle=False)
np.save(os.path.join(stt_dir, "pose.npy"), tf_pose.eval(), allow_pickle=False)
np.save(os.path.join(stt_dir, "shape.npy"), tf_shape.eval(), allow_pickle=False)
progress = tqdm(ply_files)
for src_fname in progress:
frame = os.path.basename(src_fname)
progress.set_description(" -> " + frame)
dst_fname = os.path.join(dst_dir, frame)
# param filename
prm_fname = os.path.join(prm_dir, frame)
exp_fname = os.path.splitext(prm_fname)[0] + '_exp.npy'
idn_fname = os.path.splitext(prm_fname)[0] + '_idn.npy'
src_mesh = Mesh(filename=src_fname)
session.run(tf.assign(tf_src, src_mesh.v))
optim_seq.minimize(session)
# save expr
np.save(exp_fname, tf_exp.eval())
np.save(idn_fname, tf_shape.eval())
# state_dict = _save_state("trans", "rot", "pose", "shape", set_zero=True)
# save mesh
fitting_mesh = Mesh(session.run(tf_model), src_mesh.f)
fitting_mesh.write_ply(dst_fname)
# _load_state(state_dict)
# print(tf_shape.eval())
if showing:
g_mv.set_static_meshes([fitting_mesh])
os.system("touch {}".format(_anchor))
return speaker
last_speaker = None
for (src, dst, prm) in dir_tup_list:
last_speaker = _fit_sentence(src, dst, prm, last_speaker)
def fit_lmk3d(target_3d_lmks,
template_fname,
tf_model_fname,
lmk_face_idx,
lmk_b_coords,
weights,
show_fitting=True):
'''
Fit FLAME to 3D landmarks
:param target_3d_lmks: target 3D landmarks provided as (num_lmks x 3) matrix
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param tf_model_fname: saved Tensorflow FLAME model
:param lmk_face_idx: face indices of the landmark embedding in the FLAME topology
:param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology
(i.e. weighting of the three vertices for the trinagle, the landmark is embedded in
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
template_mesh = Mesh(filename=template_fname)
saver = tf.train.import_meta_graph(tf_model_fname + '.meta')
graph = tf.get_default_graph()
tf_model = graph.get_tensor_by_name(u'vertices:0')
with tf.Session() as session:
saver.restore(session, tf_model_fname)
# Workaround as existing tf.Variable cannot be retrieved back with tf.get_variable
# tf_v_template = [x for x in tf.trainable_variables() if 'v_template' in x.name][0]
tf_trans = [x for x in tf.trainable_variables()
if 'trans' in x.name][0]
tf_rot = [x for x in tf.trainable_variables() if 'rot' in x.name][0]
tf_pose = [x for x in tf.trainable_variables() if 'pose' in x.name][0]
tf_shape = [x for x in tf.trainable_variables()
if 'shape' in x.name][0]
tf_exp = [x for x in tf.trainable_variables() if 'exp' in x.name][0]
lmks = tf_get_model_lmks(tf_model, template_mesh, lmk_face_idx,
lmk_b_coords)
lmk_dist = tf.reduce_sum(
tf.square(1000 * tf.subtract(lmks, target_3d_lmks)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[3:6]))
eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
# Optimize global transformation first
vars = [tf_trans, tf_rot]
loss = weights['lmk'] * lmk_dist
optimizer = scipy_pt(loss=loss,
var_list=vars,
method='L-BFGS-B',
options={
'disp': 1,
'ftol': 5e-6
})
print('Optimize rigid transformation')
optimizer.minimize(session)
# Optimize for the model parameters
vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp]
loss = weights['lmk'] * lmk_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \
weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg
optimizer = scipy_pt(loss=loss,
var_list=vars,
method='L-BFGS-B',
options={
'disp': 1,
'ftol': 5e-6
})
print('Optimize model parameters')
optimizer.minimize(session)
print('Fitting done')
if show_fitting:
# Visualize landmark fitting
mv = MeshViewer()
mv.set_static_meshes(
create_lmk_spheres(target_3d_lmks, 0.001, [255.0, 0.0, 0.0]))
mv.set_dynamic_meshes(
[Mesh(session.run(tf_model), template_mesh.f)] +
create_lmk_spheres(session.run(lmks), 0.001,
[0.0, 0.0, 255.0]),
blocking=True)
six.moves.input('Press key to continue')
return Mesh(session.run(tf_model), template_mesh.f)