def sample_VOCA_template(template_fname, model_fname, out_mesh_fname):
'''
VOCA animates static templates in FLAME topology. Such templates can be obtained by sampling the FLAME shape space.
This function randomly samples the FLAME identity shape space to generate new templates.
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param model_fname: saved FLAME model
:param out_mesh_fname: filename of the VOCA template
:return:
'''
template_mesh = Mesh(filename=template_fname)
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
assign_shape = tf.assign(tf_shape, np.hstack((np.random.randn(100), np.zeros(200)))[np.newaxis,:])
session.run([assign_shape])
Mesh(session.run(tf_model), template_mesh.f).write_ply(out_mesh_fname)
def fit_3D_mesh(target_3d_mesh_fname, model_fname, weights, show_fitting=True):
'''
Fit FLAME to 3D mesh in correspondence to the FLAME mesh (i.e. same number of vertices, same mesh topology)
:param target_3d_mesh_fname: target 3D mesh filename
:param model_fname: saved FLAME model
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
target_mesh = Mesh(filename=target_3d_mesh_fname)
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
mesh_dist = tf.reduce_sum(tf.square(tf.subtract(tf_model, target_mesh.v)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,3:6]))
eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
# Optimize global transformation first
vars = [tf_trans, tf_rot]
loss = mesh_dist
optimizer = scipy_pt(loss=loss, var_list=vars, method='BFGS', options={'disp': 1})
print('Optimize rigid transformation')
optimizer.minimize(session)
# Optimize for the model parameters
vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp]
loss = weights['data'] * mesh_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \
weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg
optimizer = scipy_pt(loss=loss, var_list=vars, method='BFGS', options={'disp': 1})
print('Optimize model parameters')
optimizer.minimize(session)
print('Fitting done')
if show_fitting:
# Visualize fitting
mv = MeshViewer()
fitting_mesh = Mesh(session.run(tf_model), smpl.f)
fitting_mesh.set_vertex_colors('light sky blue')
mv.set_static_meshes([target_mesh, fitting_mesh])
six.moves.input('Press key to continue')
return Mesh(session.run(tf_model), smpl.f)
def fit_lmk3d(target_3d_lmks, model_fname, lmk_face_idx, lmk_b_coords, weights, show_fitting=True):
'''
Fit FLAME to 3D landmarks
:param target_3d_lmks: target 3D landmarks provided as (num_lmks x 3) matrix
:param model_fname: saved Tensorflow FLAME model
:param lmk_face_idx: face indices of the landmark embedding in the FLAME topology
:param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology
(i.e. weighting of the three vertices for the trinagle, the landmark is embedded in
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
lmks = tf_get_model_lmks(tf_model, smpl.f, lmk_face_idx, lmk_b_coords)
lmk_dist = tf.reduce_sum(tf.square(1000 * tf.subtract(lmks, target_3d_lmks)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,3:6]))
eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
# Optimize global transformation first
vars = [tf_trans, tf_rot]
loss = weights['lmk'] * lmk_dist
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6})
print('Optimize rigid transformation')
optimizer.minimize(session)
# Optimize for the model parameters
vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp]
loss = weights['lmk'] * lmk_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \
weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6})
print('Optimize model parameters')
optimizer.minimize(session)
print('Fitting done')
if show_fitting:
# Visualize landmark fitting
mv = MeshViewer()
mv.set_static_meshes(create_lmk_spheres(target_3d_lmks, 0.001, [255.0, 0.0, 0.0]))
mv.set_dynamic_meshes([Mesh(session.run(tf_model), smpl.f)] + create_lmk_spheres(session.run(lmks), 0.001, [0.0, 0.0, 255.0]), blocking=True)
six.moves.input('Press key to continue')
return Mesh(session.run(tf_model), smpl.f)
def sample_texture(model_fname, texture_fname, num_samples, out_path):
'''
Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to
- model identity-dependent shape variations (paramters: shape),
- articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12])
- model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp)
- global translation (paramters: trans)
- global rotation (paramters: rot)
:param model_fname saved FLAME model
:param num_samples number of samples
:param out_path output path to save the generated templates (no templates are saved if path is empty)
'''
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
texture_model = np.load(texture_fname)
tex_dim = texture_model['tex_dir'].shape[-1]
tf_tex_params = tf.Variable(np.zeros((1,tex_dim)), name="pose", dtype=tf.float64, trainable=True)
tf_tex_mean = tf.Variable(np.reshape(texture_model['mean'], (1,-1)), name='tex_mean', dtype=tf.float64, trainable=False)
tf_tex_dir = tf.Variable(np.reshape(texture_model['tex_dir'], (-1, tex_dim)).T, name='tex_dir', dtype=tf.float64, trainable=False)
tf_tex = tf.add(tf_tex_mean, tf.matmul(tf_tex_params, tf_tex_dir)),
tf_tex = tf.reshape(tf_tex, (texture_model['tex_dir'].shape[0], texture_model['tex_dir'].shape[1], texture_model['tex_dir'].shape[2]))
tf_tex = tf.cast(tf.clip_by_value(tf_tex, 0.0, 255.0), tf.int64)
with tf.Session() as session:
session.run(tf.global_variables_initializer())
mv = MeshViewer()
for i in range(num_samples):
assign_tex = tf.assign(tf_tex_params, np.random.randn(tex_dim)[np.newaxis,:])
session.run([assign_tex])
v, tex = session.run([tf_model, tf_tex])
out_mesh = Mesh(v, smpl.f)
out_mesh.vt = texture_model['vt']
out_mesh.ft = texture_model['ft']
mv.set_dynamic_meshes([out_mesh], blocking=True)
key = six.moves.input('Press (s) to save sample, any other key to continue ')
if key == 's':
out_mesh_fname = os.path.join(out_path, 'tex_sample_%02d.obj' % (i+1))
out_tex_fname = out_mesh_fname.replace('obj', 'png')
cv2.imwrite(out_tex_fname, tex)
out_mesh.set_texture_image(out_tex_fname)
out_mesh.write_obj(out_mesh_fname)
def sample_FLAME(model_fname, num_samples, out_path, visualize, sample_VOCA_template=False):
'''
Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to
- model identity-dependent shape variations (paramters: shape),
- articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12])
- model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp)
- global translation (paramters: trans)
- global rotation (paramters: rot)
:param model_fname saved FLAME model
:param num_samples number of samples
:param out_path output path to save the generated templates (no templates are saved if path is empty)
:param visualize visualize samples
:param sample_VOCA_template sample template in 'zero pose' that can be used e.g. for speech-driven animation in VOCA
'''
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
if visualize:
mv = MeshViewer()
for i in range(num_samples):
if sample_VOCA_template:
assign_shape = tf.assign(tf_shape, np.hstack((np.random.randn(100), np.zeros(200)))[np.newaxis,:])
session.run([assign_shape])
out_fname = os.path.join(out_path, 'VOCA_template_%02d.ply' % (i+1))
else:
# assign_trans = tf.assign(tf_trans, np.random.randn(3)[np.newaxis,:])
assign_rot = tf.assign(tf_rot, np.random.randn(3)[np.newaxis,:] * 0.03)
assign_pose = tf.assign(tf_pose, np.random.randn(12)[np.newaxis,:] * 0.02)
assign_shape = tf.assign(tf_shape, np.hstack((np.random.randn(100), np.zeros(200)))[np.newaxis,:])
assign_exp = tf.assign(tf_exp, np.hstack((0.5*np.random.randn(50), np.zeros(50)))[np.newaxis,:])
session.run([assign_rot, assign_pose, assign_shape, assign_exp])
out_fname = os.path.join(out_path, 'FLAME_sample_%02d.ply' % (i+1))
sample_mesh = Mesh(session.run(tf_model), smpl.f)
if visualize:
mv.set_dynamic_meshes([sample_mesh], blocking=True)
key = six.moves.input('Press (s) to save sample, any other key to continue ')
if key == 's':
sample_mesh.write_ply(out_fname)
else:
sample_mesh.write_ply(out_fname)
def sample_FLAME(template_fname, model_fname, num_samples):
'''
Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to
- model identity-dependent shape variations (paramters: shape),
- articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12])
- model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp)
- global translation (paramters: trans)
- global rotation (paramters: rot)
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param model_fname: saved FLAME model
'''
template_mesh = Mesh(filename=template_fname)
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
mv = MeshViewer()
for i in range(num_samples):
assign_trans = tf.assign(tf_trans, np.random.randn(3)[np.newaxis,:])
assign_rot = tf.assign(tf_rot, np.random.randn(3)[np.newaxis,:] * 0.03)
assign_pose = tf.assign(tf_pose, np.random.randn(12)[np.newaxis,:] * 0.03)
assign_shape = tf.assign(tf_shape, np.random.randn(300)[np.newaxis,:] * 1.0)
assign_exp = tf.assign(tf_exp, np.random.randn(100)[np.newaxis,:] * 0.5)
session.run([assign_trans, assign_rot, assign_pose, assign_shape, assign_exp])
mv.set_dynamic_meshes([Mesh(session.run(tf_model), template_mesh.f)], blocking=True)
six.moves.input('Press key to continue')
def fit_lmk2d(target_img, target_2d_lmks, template_fname, model_fname,
lmk_face_idx, lmk_b_coords, weights):
'''
Fit FLAME to 2D landmarks
:param target_2d_lmks: target 2D landmarks provided as (num_lmks x 3) matrix
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param model_fname: saved FLAME model
:param lmk_face_idx: face indices of the landmark embedding in the FLAME topology
:param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology
(i.e. weighting of the three vertices for the trinagle, the landmark is embedded in
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
template_mesh = Mesh(filename=template_fname)
tf_trans = tf.Variable(np.zeros((1, 3)),
name="trans",
dtype=tf.float64,
trainable=True)
tf_rot = tf.Variable(np.zeros((1, 3)),
name="pose",
dtype=tf.float64,
trainable=True)
tf_pose = tf.Variable(np.zeros((1, 12)),
name="pose",
dtype=tf.float64,
trainable=True)
tf_shape = tf.Variable(np.zeros((1, 300)),
name="shape",
dtype=tf.float64,
trainable=True)
tf_exp = tf.Variable(np.zeros((1, 100)),
name="expression",
dtype=tf.float64,
trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(
smpl(tf_trans, tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
# Mirror landmark y-coordinates
target_2d_lmks[:, 1] = target_img.shape[0] - target_2d_lmks[:, 1]
lmks_3d = tf_get_model_lmks(tf_model, template_mesh, lmk_face_idx,
lmk_b_coords)
s2d = np.mean(
np.linalg.norm(target_2d_lmks - np.mean(target_2d_lmks, axis=0),
axis=1))
s3d = tf.reduce_mean(
tf.sqrt(
tf.reduce_sum(
tf.square(lmks_3d -
tf.reduce_mean(lmks_3d, axis=0))[:, :2],
axis=1)))
tf_scale = tf.Variable(s2d / s3d, dtype=lmks_3d.dtype)
# trans = 0.5*np.array((target_img.shape[0], target_img.shape[1]))/tf_scale
# trans = 0.5 * s3d * np.array((target_img.shape[0], target_img.shape[1])) / s2d
lmks_proj_2d = tf_project_points(lmks_3d, tf_scale, np.zeros(2))
factor = max(
max(target_2d_lmks[:, 0]) - min(target_2d_lmks[:, 0]),
max(target_2d_lmks[:, 1]) - min(target_2d_lmks[:, 1]))
lmk_dist = weights['lmk'] * tf.reduce_sum(
tf.square(tf.subtract(lmks_proj_2d, target_2d_lmks))) / (factor**2)
neck_pose_reg = weights['neck_pose'] * tf.reduce_sum(
tf.square(tf_pose[:3]))
jaw_pose_reg = weights['jaw_pose'] * tf.reduce_sum(
tf.square(tf_pose[3:6]))
eyeballs_pose_reg = weights['eyeballs_pose'] * tf.reduce_sum(
tf.square(tf_pose[6:]))
shape_reg = weights['shape'] * tf.reduce_sum(tf.square(tf_shape))
exp_reg = weights['expr'] * tf.reduce_sum(tf.square(tf_exp))
session.run(tf.global_variables_initializer())
def on_step(verts,
scale,
faces,
target_img,
target_lmks,
opt_lmks,
lmk_dist=0.0,
shape_reg=0.0,
exp_reg=0.0,
neck_pose_reg=0.0,
jaw_pose_reg=0.0,
eyeballs_pose_reg=0.0):
import cv2
import sys
import numpy as np
from psbody.mesh import Mesh
from utils.render_mesh import render_mesh
if lmk_dist > 0.0 or shape_reg > 0.0 or exp_reg > 0.0 or neck_pose_reg > 0.0 or jaw_pose_reg > 0.0 or eyeballs_pose_reg > 0.0:
print(
'lmk_dist: %f, shape_reg: %f, exp_reg: %f, neck_pose_reg: %f, jaw_pose_reg: %f, eyeballs_pose_reg: %f'
% (lmk_dist, shape_reg, exp_reg, neck_pose_reg,
jaw_pose_reg, eyeballs_pose_reg))
plt_target_lmks = target_lmks.copy()
plt_target_lmks[:, 1] = target_img.shape[0] - plt_target_lmks[:, 1]
for (x, y) in plt_target_lmks:
cv2.circle(target_img, (int(x), int(y)), 4, (0, 0, 255), -1)
plt_opt_lmks = opt_lmks.copy()
plt_opt_lmks[:, 1] = target_img.shape[0] - plt_opt_lmks[:, 1]
for (x, y) in plt_opt_lmks:
cv2.circle(target_img, (int(x), int(y)), 4, (255, 0, 0), -1)
if sys.version_info >= (3, 0):
rendered_img = render_mesh(Mesh(scale * verts, faces),
height=target_img.shape[0],
width=target_img.shape[1])
for (x, y) in plt_opt_lmks:
cv2.circle(rendered_img, (int(x), int(y)), 4, (255, 0, 0),
-1)
target_img = np.hstack((target_img, rendered_img))
cv2.imshow('img', target_img)
cv2.waitKey(10)
print('Optimize rigid transformation')
vars = [tf_scale, tf_trans, tf_rot]
loss = lmk_dist
optimizer = scipy_pt(loss=loss,
var_list=vars,
method='L-BFGS-B',
options={
'disp': 1,
'ftol': 5e-6
})
optimizer.minimize(session,
fetches=[
tf_model, tf_scale,
tf.constant(template_mesh.f),
tf.constant(target_img),
tf.constant(target_2d_lmks), lmks_proj_2d
],
loss_callback=on_step)
print('Optimize model parameters')
vars = [tf_scale, tf_trans[:2], tf_rot, tf_pose, tf_shape, tf_exp]
loss = lmk_dist + shape_reg + exp_reg + neck_pose_reg + jaw_pose_reg + eyeballs_pose_reg
optimizer = scipy_pt(loss=loss,
var_list=vars,
method='L-BFGS-B',
options={
'disp': 0,
'ftol': 1e-7
})
optimizer.minimize(session,
fetches=[
tf_model, tf_scale,
tf.constant(template_mesh.f),
tf.constant(target_img),
tf.constant(target_2d_lmks), lmks_proj_2d,
lmk_dist, shape_reg, exp_reg, neck_pose_reg,
jaw_pose_reg, eyeballs_pose_reg
],
loss_callback=on_step)
print('Fitting done')
np_verts, np_scale = session.run([tf_model, tf_scale])
def _get_verts(expr):
assign_expr = tf.assign(tf_exp, expr[np.newaxis, :])
session.run([assign_expr])
_verts, _scale = session.run([tf_model, tf_scale])
return _verts
expr_basis = []
neutral_verts = _get_verts(np.zeros((100), np.float64))
for i in range(100):
expr_i = np.zeros((100), np.float64)
expr_i[i] = 1
verts_i = _get_verts(expr_i) - neutral_verts
expr_basis.append(verts_i.flatten())
expr_basis = np.asarray(expr_basis, np.float32).transpose(1, 0)
return Mesh(np_verts, template_mesh.f), np_scale, expr_basis
import pkg_resources
import tf_smpl
from tf_smpl.batch_smpl import SMPL
from tf_mesh_renderer import mesh_renderer
from pose_3d import utils
if __name__ == '__main__':
if len(sys.argv) != 2:
print("Usage: python3 test_runner.py <path-to-SMPL-model>")
sys.exit()
dirpath = os.path.dirname(os.path.realpath(__file__))
smpl_path = os.path.realpath(sys.argv[1])
smpl_model = SMPL(smpl_path)
betas = tf.random_normal([1, 10], stddev=0.3)
thetas = tf.random_normal([1, 72], stddev=0.15)
verts, _, _ = smpl_model(betas, thetas, get_skin=True)
verts = verts[0]
config = tf.ConfigProto()
config.gpu_options.allow_growth = True # pylint: disable=no-member
sess = tf.Session(config=config)
result = sess.run(verts)
faces_path = pkg_resources.resource_filename(
tf_smpl.__name__, 'smpl_faces.npy')
faces = np.load(faces_path)
def fit_lmk2d(target_img, target_2d_lmks, model_fname, lmk_face_idx, lmk_b_coords, weights, visualize):
'''
Fit FLAME to 2D landmarks
:param target_2d_lmks target 2D landmarks provided as (num_lmks x 3) matrix
:param model_fname saved FLAME model
:param lmk_face_idx face indices of the landmark embedding in the FLAME topology
:param lmk_b_coords barycentric coordinates of the landmark embedding in the FLAME topology
(i.e. weighting of the three vertices for the trinagle, the landmark is embedded in
:param weights weights of the individual objective functions
:param visualize visualize fitting progress
:return: a mesh with the fitting results
'''
'''
pred_types = {'face': pred_type(slice(0, 17), (0.682, 0.780, 0.909, 0.5)),
'eyebrow1': pred_type(slice(17, 22), (1.0, 0.498, 0.055, 0.4)),
'eyebrow2': pred_type(slice(22, 27), (1.0, 0.498, 0.055, 0.4)),
'nose': pred_type(slice(27, 31), (0.345, 0.239, 0.443, 0.4)),
'nostril': pred_type(slice(31, 36), (0.345, 0.239, 0.443, 0.4)),
'eye1': pred_type(slice(36, 42), (0.596, 0.875, 0.541, 0.3)),
'eye2': pred_type(slice(42, 48), (0.596, 0.875, 0.541, 0.3)),
'lips': pred_type(slice(48, 60), (0.596, 0.875, 0.541, 0.3)),
'teeth': pred_type(slice(60, 68), (0.596, 0.875, 0.541, 0.4))
}
'''
lmks_weights = [[1,1]] * 68
for idx in range(36, 48):
lmks_weights[idx] = [100, 100]
tf_lmks_weights = tf.constant(
lmks_weights,
tf.float64
)
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="rot", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
# session.run(tf.global_variables_initializer())
# Mirror landmark y-coordinates
target_2d_lmks[:,1] = target_img.shape[0]-target_2d_lmks[:,1]
lmks_3d = tf_get_model_lmks(tf_model, smpl.f, lmk_face_idx, lmk_b_coords)
s2d = np.mean(np.linalg.norm(target_2d_lmks-np.mean(target_2d_lmks, axis=0), axis=1))
s3d = tf.reduce_mean(tf.sqrt(tf.reduce_sum(tf.square(lmks_3d-tf.reduce_mean(lmks_3d, axis=0))[:, :2], axis=1)))
tf_scale = tf.Variable(s2d/s3d, dtype=lmks_3d.dtype)
# trans = 0.5*np.array((target_img.shape[0], target_img.shape[1]))/tf_scale
# trans = 0.5 * s3d * np.array((target_img.shape[0], target_img.shape[1])) / s2d
lmks_proj_2d = tf_project_points(lmks_3d, tf_scale, np.zeros(2))
factor = max(max(target_2d_lmks[:,0]) - min(target_2d_lmks[:,0]),max(target_2d_lmks[:,1]) - min(target_2d_lmks[:,1]))
#lmk_dist = weights['lmk']*tf.reduce_sum(tf.square(tf.subtract(lmks_proj_2d, target_2d_lmks))) / (factor ** 2)
lmk_dist = weights['lmk']*tf.reduce_sum(
tf.square(tf.subtract(lmks_proj_2d, target_2d_lmks)) * tf_lmks_weights
) / (factor ** 2)
neck_pose_reg = weights['neck_pose']*tf.reduce_sum(tf.square(tf_pose[:,:3]))
jaw_pose_reg = weights['jaw_pose']*tf.reduce_sum(tf.square(tf_pose[:,3:6]))
eyeballs_pose_reg = weights['eyeballs_pose']*tf.reduce_sum(tf.square(tf_pose[:,6:]))
shape_reg = weights['shape']*tf.reduce_sum(tf.square(tf_shape))
exp_reg = weights['expr']*tf.reduce_sum(tf.square(tf_exp))
session.run(tf.global_variables_initializer())
if visualize:
def on_step(verts, scale, faces, target_img, target_lmks, opt_lmks, lmk_dist=0.0, shape_reg=0.0, exp_reg=0.0, neck_pose_reg=0.0, jaw_pose_reg=0.0, eyeballs_pose_reg=0.0):
import cv2
import sys
import numpy as np
from psbody.mesh import Mesh
from utils.render_mesh import render_mesh
if lmk_dist>0.0 or shape_reg>0.0 or exp_reg>0.0 or neck_pose_reg>0.0 or jaw_pose_reg>0.0 or eyeballs_pose_reg>0.0:
print('lmk_dist: %f, shape_reg: %f, exp_reg: %f, neck_pose_reg: %f, jaw_pose_reg: %f, eyeballs_pose_reg: %f' % (lmk_dist, shape_reg, exp_reg, neck_pose_reg, jaw_pose_reg, eyeballs_pose_reg))
plt_target_lmks = target_lmks.copy()
plt_target_lmks[:, 1] = target_img.shape[0] - plt_target_lmks[:, 1]
for (x, y) in plt_target_lmks:
cv2.circle(target_img, (int(x), int(y)), 4, (0, 0, 255), -1)
plt_opt_lmks = opt_lmks.copy()
plt_opt_lmks[:,1] = target_img.shape[0] - plt_opt_lmks[:,1]
for (x, y) in plt_opt_lmks:
cv2.circle(target_img, (int(x), int(y)), 4, (255, 0, 0), -1)
if sys.version_info >= (3, 0):
rendered_img = render_mesh(Mesh(scale*verts, faces), height=target_img.shape[0], width=target_img.shape[1])
for (x, y) in plt_opt_lmks:
cv2.circle(rendered_img, (int(x), int(y)), 4, (255, 0, 0), -1)
target_img = np.hstack((target_img, rendered_img))
#cv2.imshow('img', target_img)
#cv2.waitKey(10)
else:
def on_step(*_):
pass
print('Optimize rigid transformation')
vars = [tf_scale, tf_trans, tf_rot]
loss = lmk_dist
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6})
optimizer.minimize(session, fetches=[tf_model, tf_scale, tf.constant(smpl.f), tf.constant(target_img), tf.constant(target_2d_lmks), lmks_proj_2d], loss_callback=on_step)
print('Optimize model parameters')
vars = [tf_scale, tf_trans[:2], tf_rot, tf_pose, tf_shape, tf_exp]
loss = lmk_dist + shape_reg + exp_reg + neck_pose_reg + jaw_pose_reg + eyeballs_pose_reg
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 0, 'ftol': 1e-7})
optimizer.minimize(session, fetches=[tf_model, tf_scale, tf.constant(smpl.f), tf.constant(target_img), tf.constant(target_2d_lmks), lmks_proj_2d,
lmk_dist, shape_reg, exp_reg, neck_pose_reg, jaw_pose_reg, eyeballs_pose_reg], loss_callback=on_step)
print('Fitting done')
np_verts, np_scale = session.run([tf_model, tf_scale])
return Mesh(np_verts, smpl.f), np_scale
num_stage = 3
num_theta = 72
coeff_num = 85
smpl_neutral_path = "./tf_smpl/neutral_smpl_with_cocoplus_reg.pkl"
joint_type = "cocoplus"
f_len = 366.474487
rend_size = 256.
intrinsic = np.array([[f_len / 480 * 256, 0, rend_size / 2],
[0, f_len / 480 * 256, rend_size / 2], [0, 0, 1]])
# some predefined functions
# ===========================================================
smpl = SMPL(smpl_neutral_path, joint_type=joint_type)
# render predicted coefficients - hmr
def render_hmr_smpl(hmr_coeff, f_len=500., rend_size=224., req_model=False):
# hmr_coeff is a 85-vector, named as theta in hmr
hmr_coeff = np.asarray(hmr_coeff).tolist()
# make scene
scene = pyrender.Scene()
# initialize camera
camera = pyrender.PerspectiveCamera(
yfov=np.arctan(rend_size * 0.5 / f_len) * 2, aspectRatio=1)
camera_pose = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0],
[0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]])
from tf_smpl.batch_smpl import SMPL
from tf_smpl.batch_lbs import batch_rodrigues, batch_global_rigid_transformation
import tensorflow as tf
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
if __name__ == '__main__':
smpl = SMPL('../models/neutral_smpl_with_cocoplus_reg.pkl')
beta = tf.zeros([1, 10], dtype=tf.float32)
npose = np.zeros([1, 72])
pose = tf.constant(npose, dtype=tf.float32)
verts, joints, Rs = smpl(beta, pose, get_skin=True)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
V, J, R = sess.run([verts, joints, Rs])
fig = plt.figure(1)
ax = fig.add_subplot(111, projection='3d')
ax.scatter(J[0, :, 0], J[0, :, 1], J[0, :, 2], c='r')
ax.scatter(J[0, 18, 0], J[0, 18, 1], J[0, 18, 2], c='b')
# ax.scatter(V[0, :, 0], V[0, :, 1], V[0, :, 2], c='b')
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')
ax.set_xlim(-1, 1)
ax.set_ylim(-1, 1)
ax.set_zlim(-1, 1)
plt.show()
def __init__(self, smpl_model_path, gaussians_params_path):
self.smplModel = SMPL(smpl_model_path)
self.gussians_prior = MaxMixtureCompletePrior(gaussians_params_path)
def __init__(self, config, data_loader, mocap_loader):
"""
Args:
config
if no 3D label is available,
data_loader is a dict
else
data_loader is a dict
mocap_loader is a tuple (pose, shape)
"""
# Config + path
self.config = config
self.model_dir = config.model_dir
self.load_path = config.load_path
self.data_format = config.data_format
self.smpl_model_path = config.smpl_model_path
self.pretrained_model_path = config.pretrained_model_path
self.encoder_only = config.encoder_only
self.use_3d_label = config.use_3d_label
# Data size
self.img_size = config.img_size
self.num_stage = config.num_stage
self.batch_size = config.batch_size
self.max_epoch = config.epoch
self.num_cam = 3
self.proj_fn = batch_orth_proj_idrot
self.num_theta = 72 # 24 * 3
self.total_params = self.num_theta + self.num_cam + 10
# Data
num_images = num_examples(config.datasets)
num_mocap = num_examples(config.mocap_datasets)
self.num_itr_per_epoch = num_images / self.batch_size
self.num_mocap_itr_per_epoch = num_mocap / self.batch_size
# First make sure data_format is right
if self.data_format == 'NCHW':
# B x H x W x 3 --> B x 3 x H x W
data_loader['image'] = tf.transpose(data_loader['image'],
[0, 3, 1, 2])
self.image_loader = data_loader['image']
self.kp_loader = data_loader['label']
if self.use_3d_label:
self.poseshape_loader = data_loader['label3d']
# image_loader[3] is N x 2, first column is 3D_joints gt existence,
# second column is 3D_smpl gt existence
self.has_gt3d_joints = data_loader['has3d'][:, 0]
self.has_gt3d_smpl = data_loader['has3d'][:, 1]
self.pose_loader = mocap_loader[0]
self.shape_loader = mocap_loader[1]
self.global_step = tf.Variable(0, name='global_step', trainable=False)
self.log_img_step = config.log_img_step
# For visualization:
num2show = np.minimum(6, self.batch_size)
# Take half from front & back
self.show_these = tf.constant(
np.hstack(
[np.arange(num2show / 2), self.batch_size - np.arange(3) - 1]),
tf.int32)
# Model spec
self.model_type = config.model_type
self.keypoint_loss = keypoint_l1_loss
# Optimizer, learning rate
self.e_lr = config.e_lr
self.d_lr = config.d_lr
# Weight decay
self.e_wd = config.e_wd
self.d_wd = config.d_wd
self.e_loss_weight = config.e_loss_weight
self.d_loss_weight = config.d_loss_weight
self.e_3d_weight = config.e_3d_weight
self.optimizer = tf.train.AdamOptimizer
# Instantiate SMPL
self.smpl = SMPL(self.smpl_model_path)
self.E_var = []
self.build_model()
# Logging
init_fn = None
if self.use_pretrained():
# Make custom init_fn
print("Fine-tuning from %s" % self.pretrained_model_path)
if 'resnet_v2_50' in self.pretrained_model_path:
resnet_vars = [
var for var in self.E_var if 'resnet_v2_50' in var.name
]
self.pre_train_saver = tf.train.Saver(resnet_vars)
elif 'pose-tensorflow' in self.pretrained_model_path:
resnet_vars = [
var for var in self.E_var if 'resnet_v1_101' in var.name
]
self.pre_train_saver = tf.train.Saver(resnet_vars)
else:
self.pre_train_saver = tf.train.Saver()
def load_pretrain(sess):
self.pre_train_saver.restore(sess, self.pretrained_model_path)
init_fn = load_pretrain
self.saver = tf.train.Saver(keep_checkpoint_every_n_hours=5)
self.summary_writer = tf.summary.FileWriter(self.model_dir)
self.sv = tf.train.Supervisor(logdir=self.model_dir,
global_step=self.global_step,
saver=self.saver,
summary_writer=self.summary_writer,
init_fn=init_fn)
gpu_options = tf.GPUOptions(allow_growth=True)
self.sess_config = tf.ConfigProto(allow_soft_placement=False,
log_device_placement=False,
gpu_options=gpu_options)
def main(in_filename):
images_path = os.path.join(__init__.project_path, 'data', 'images')
in_im = cv2.imread(os.path.join(images_path, in_filename))
in_im = cv2.cvtColor(in_im, cv2.COLOR_BGR2RGB)
img_size = config.input_img_size
expect_aspect = img_size[1] / img_size[0]
in_shape = in_im.shape
in_aspect = in_shape[1] / in_shape[0]
if in_aspect != expect_aspect:
if in_im.shape[1] >= in_im.shape[0] * expect_aspect:
diff = int(in_im.shape[1] - in_im.shape[0] * expect_aspect)
pad_im = cv2.copyMakeBorder(in_im, 0, diff, 0, 0,
cv2.BORDER_CONSTANT, None, 0)
else:
diff = int(in_im.shape[0] * expect_aspect - in_im.shape[1])
pad_im = cv2.copyMakeBorder(in_im, 0, 0, 0, diff,
cv2.BORDER_CONSTANT, None, 0)
in_im = pad_im
in_im = cv2.resize(in_im,
dsize=(img_size[1], img_size[0]),
interpolation=cv2.INTER_AREA)
tfconfig = tf.ConfigProto()
tfconfig.gpu_options.allow_growth = True # pylint: disable=no-member
with tf.Graph().as_default():
estimator = OpPoseEstimator(get_graph_path('cmu'),
target_size=(img_size[1] * 2,
img_size[0] * 2),
tf_config=tfconfig)
humans = estimator.inference(in_im,
resize_to_default=True,
upsample_size=4)
heatmaps = estimator.heatMat[:, :, :config.n_joints]
heatmaps = data_helpers.suppress_non_largest_human(humans, heatmaps,
img_size)
heatmaps = heatmaps[np.newaxis] # add "batch" axis
in_im_3d = cv2.normalize(in_im, None, 0, 1, cv2.NORM_MINMAX)
inputs = np.concatenate([heatmaps, in_im_3d[np.newaxis]], axis=3)
# Visualise argmaxs
# input_locs = tf.Session().run(utils.soft_argmax_rescaled(heatmaps))
# input_locs = input_locs[0]
# plt.scatter(input_locs[:, 1], input_locs[:, 0])
# plt.gca().invert_yaxis()
# plt.show()
# for input_loc in input_locs[0]:
# y, x, _ = (input_loc * 240 + np.array([120, 160, 0])).astype(np.int32)
# in_im = cv2.circle(in_im, (x, y), 3, (0, 255, 255))
# plt.imshow(in_im)
# plt.show()
# exit()
# with different graph so checkpoint is restored correctly
pm_graph = tf.Graph()
pm_3d = PoseModel3d((None, 240, 320, 3 + config.n_joints),
pm_graph,
mode='test',
summary_dir=SUMMARY_DIR,
saver_path=SAVER_PATH,
restore_model=True)
out_vals = pm_3d.estimate(inputs)
smpl_dir = os.path.join(__init__.project_path, 'data', 'SMPL_model',
'models_numpy')
smpl_model_path = os.path.join(smpl_dir, 'model_neutral_np.pkl')
faces_path = pkg_resources.resource_filename(tf_smpl.__name__,
'smpl_faces.npy')
faces = np.load(faces_path)
smpl = SMPL(smpl_model_path)
beta = tf.zeros([1, 10])
pose = tf.constant(out_vals[:, :72])
cam_pos = tf.constant(out_vals[:, 72:75])
cam_rot = tf.constant(out_vals[:, 75:78])
cam_f = tf.tile([config.fl], [out_vals.shape[0]])
verts, _, _ = smpl(beta, pose, get_skin=True)
vert_faces = utils.vertex_faces_from_face_verts(faces)
mesh_img = utils.render_mesh_verts_cam(
verts, cam_pos, cam_rot,
tf.atan2(config.ss / 2, cam_f) * 360 / np.pi,
tf.constant(faces, dtype=tf.int32), vert_faces, cam_pos[:,
tf.newaxis, :])
verts_eval, mesh_img_eval = tf.Session().run((verts[0], mesh_img[0]))
dirpath = os.path.dirname(os.path.realpath(__file__))
outmesh_path = os.path.join(dirpath, 'smpl_tf.obj')
with open(outmesh_path, 'w') as fp:
for v in verts_eval:
fp.write('v %f %f %f\n' % (v[0], v[1], v[2]))
for f in faces + 1:
fp.write('f %d %d %d\n' % (f[0], f[1], f[2]))
op_out_im = OpPoseEstimator.draw_humans(in_im, humans, imgcopy=True)
plt.subplot(131)
plt.imshow(np.sum(heatmaps[0][:, :, :config.n_joints], axis=2),
cmap='gray')
plt.subplot(132)
plt.imshow(op_out_im)
plt.subplot(133)
plt.imshow(np.squeeze(mesh_img_eval), cmap='gray')
plt.show()