def rigid_scan_2_mesh_alignment(scan, mesh, visualize=False): options = {'sparse_solver': lambda A, x: cg(A, x, maxiter=2000)[0]} options['disp'] = 1.0 options['delta_0'] = 0.1 options['e_3'] = 1e-4 s = ch.ones(1) r = ch.zeros(3) R = Rodrigues(r) t = ch.zeros(3) trafo_mesh = s*(R.dot(mesh.v.T)).T + t sampler = sample_from_mesh(scan, sample_type='vertices') s2m = ScanToMesh(scan, trafo_mesh, mesh.f, scan_sampler=sampler, signed=False, normalize=False) if visualize: #Visualization code mv = MeshViewer() mv.set_static_meshes([scan]) tmp_mesh = Mesh(trafo_mesh.r, mesh.f) tmp_mesh.set_vertex_colors('light sky blue') mv.set_dynamic_meshes([tmp_mesh]) def on_show(_): tmp_mesh = Mesh(trafo_mesh.r, mesh.f) tmp_mesh.set_vertex_colors('light sky blue') mv.set_dynamic_meshes([tmp_mesh]) else: def on_show(_): pass ch.minimize(fun={'dist': s2m, 's_reg': 100*(ch.abs(s)-s)}, x0=[s, r, t], callback=on_show, options=options) return s,Rodrigues(r),t
def fit_lmk3d(target_3d_lmks, model_fname, lmk_face_idx, lmk_b_coords, weights, show_fitting=True): ''' Fit FLAME to 3D landmarks :param target_3d_lmks: target 3D landmarks provided as (num_lmks x 3) matrix :param model_fname: saved Tensorflow FLAME model :param lmk_face_idx: face indices of the landmark embedding in the FLAME topology :param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology (i.e. weighting of the three vertices for the trinagle, the landmark is embedded in :param weights: weights of the individual objective functions :return: a mesh with the fitting results ''' tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True) tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True) tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True) tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True) tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True) smpl = SMPL(model_fname) tf_model = tf.squeeze(smpl(tf_trans, tf.concat((tf_shape, tf_exp), axis=-1), tf.concat((tf_rot, tf_pose), axis=-1))) with tf.Session() as session: session.run(tf.global_variables_initializer()) lmks = tf_get_model_lmks(tf_model, smpl.f, lmk_face_idx, lmk_b_coords) lmk_dist = tf.reduce_sum(tf.square(1000 * tf.subtract(lmks, target_3d_lmks))) neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,:3])) jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,3:6])) eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,6:])) shape_reg = tf.reduce_sum(tf.square(tf_shape)) exp_reg = tf.reduce_sum(tf.square(tf_exp)) # Optimize global transformation first vars = [tf_trans, tf_rot] loss = weights['lmk'] * lmk_dist optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6}) print('Optimize rigid transformation') optimizer.minimize(session) # Optimize for the model parameters vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp] loss = weights['lmk'] * lmk_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \ weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6}) print('Optimize model parameters') optimizer.minimize(session) print('Fitting done') if show_fitting: # Visualize landmark fitting mv = MeshViewer() mv.set_static_meshes(create_lmk_spheres(target_3d_lmks, 0.001, [255.0, 0.0, 0.0])) mv.set_dynamic_meshes([Mesh(session.run(tf_model), smpl.f)] + create_lmk_spheres(session.run(lmks), 0.001, [0.0, 0.0, 255.0]), blocking=True) six.moves.input('Press key to continue') return Mesh(session.run(tf_model), smpl.f)
def sample_texture(model_fname, texture_fname, num_samples, out_path): ''' Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to - model identity-dependent shape variations (paramters: shape), - articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12]) - model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp) - global translation (paramters: trans) - global rotation (paramters: rot) :param model_fname saved FLAME model :param num_samples number of samples :param out_path output path to save the generated templates (no templates are saved if path is empty) ''' tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True) tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True) tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True) tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True) tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True) smpl = SMPL(model_fname) tf_model = tf.squeeze(smpl(tf_trans, tf.concat((tf_shape, tf_exp), axis=-1), tf.concat((tf_rot, tf_pose), axis=-1))) texture_model = np.load(texture_fname) tex_dim = texture_model['tex_dir'].shape[-1] tf_tex_params = tf.Variable(np.zeros((1,tex_dim)), name="pose", dtype=tf.float64, trainable=True) tf_tex_mean = tf.Variable(np.reshape(texture_model['mean'], (1,-1)), name='tex_mean', dtype=tf.float64, trainable=False) tf_tex_dir = tf.Variable(np.reshape(texture_model['tex_dir'], (-1, tex_dim)).T, name='tex_dir', dtype=tf.float64, trainable=False) tf_tex = tf.add(tf_tex_mean, tf.matmul(tf_tex_params, tf_tex_dir)), tf_tex = tf.reshape(tf_tex, (texture_model['tex_dir'].shape[0], texture_model['tex_dir'].shape[1], texture_model['tex_dir'].shape[2])) tf_tex = tf.cast(tf.clip_by_value(tf_tex, 0.0, 255.0), tf.int64) with tf.Session() as session: session.run(tf.global_variables_initializer()) mv = MeshViewer() for i in range(num_samples): assign_tex = tf.assign(tf_tex_params, np.random.randn(tex_dim)[np.newaxis,:]) session.run([assign_tex]) v, tex = session.run([tf_model, tf_tex]) out_mesh = Mesh(v, smpl.f) out_mesh.vt = texture_model['vt'] out_mesh.ft = texture_model['ft'] mv.set_dynamic_meshes([out_mesh], blocking=True) key = six.moves.input('Press (s) to save sample, any other key to continue ') if key == 's': out_mesh_fname = os.path.join(out_path, 'tex_sample_%02d.obj' % (i+1)) out_tex_fname = out_mesh_fname.replace('obj', 'png') cv2.imwrite(out_tex_fname, tex) out_mesh.set_texture_image(out_tex_fname) out_mesh.write_obj(out_mesh_fname)
def sample_FLAME(model_fname, num_samples, out_path, visualize, sample_VOCA_template=False): ''' Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to - model identity-dependent shape variations (paramters: shape), - articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12]) - model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp) - global translation (paramters: trans) - global rotation (paramters: rot) :param model_fname saved FLAME model :param num_samples number of samples :param out_path output path to save the generated templates (no templates are saved if path is empty) :param visualize visualize samples :param sample_VOCA_template sample template in 'zero pose' that can be used e.g. for speech-driven animation in VOCA ''' tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True) tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True) tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True) tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True) tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True) smpl = SMPL(model_fname) tf_model = tf.squeeze(smpl(tf_trans, tf.concat((tf_shape, tf_exp), axis=-1), tf.concat((tf_rot, tf_pose), axis=-1))) with tf.Session() as session: session.run(tf.global_variables_initializer()) if visualize: mv = MeshViewer() for i in range(num_samples): if sample_VOCA_template: assign_shape = tf.assign(tf_shape, np.hstack((np.random.randn(100), np.zeros(200)))[np.newaxis,:]) session.run([assign_shape]) out_fname = os.path.join(out_path, 'VOCA_template_%02d.ply' % (i+1)) else: # assign_trans = tf.assign(tf_trans, np.random.randn(3)[np.newaxis,:]) assign_rot = tf.assign(tf_rot, np.random.randn(3)[np.newaxis,:] * 0.03) assign_pose = tf.assign(tf_pose, np.random.randn(12)[np.newaxis,:] * 0.02) assign_shape = tf.assign(tf_shape, np.hstack((np.random.randn(100), np.zeros(200)))[np.newaxis,:]) assign_exp = tf.assign(tf_exp, np.hstack((0.5*np.random.randn(50), np.zeros(50)))[np.newaxis,:]) session.run([assign_rot, assign_pose, assign_shape, assign_exp]) out_fname = os.path.join(out_path, 'FLAME_sample_%02d.ply' % (i+1)) sample_mesh = Mesh(session.run(tf_model), smpl.f) if visualize: mv.set_dynamic_meshes([sample_mesh], blocking=True) key = six.moves.input('Press (s) to save sample, any other key to continue ') if key == 's': sample_mesh.write_ply(out_fname) else: sample_mesh.write_ply(out_fname)
def sample_FLAME(template_fname, tf_model_fname, num_samples): ''' Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to - model identity-dependent shape variations (paramters: shape), - articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12]) - model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp) - global translation (paramters: trans) - global rotation (paramters: rot) :param template_fname: template mesh in FLAME topology (only the face information are used) :param tf_model_fname: saved Tensorflow FLAME model ''' template_mesh = Mesh(filename=template_fname) saver = tf.train.import_meta_graph(tf_model_fname + '.meta') graph = tf.get_default_graph() tf_model = graph.get_tensor_by_name(u'vertices:0') with tf.Session() as session: saver.restore(session, tf_model_fname) # Workaround as existing tf.Variable cannot be retrieved back with tf.get_variable tf_trans = [x for x in tf.trainable_variables() if 'trans' in x.name][0] tf_rot = [x for x in tf.trainable_variables() if 'rot' in x.name][0] tf_pose = [x for x in tf.trainable_variables() if 'pose' in x.name][0] tf_shape = [x for x in tf.trainable_variables() if 'shape' in x.name][0] tf_exp = [x for x in tf.trainable_variables() if 'exp' in x.name][0] mv = MeshViewer() for i in range(num_samples): assign_trans = tf.assign(tf_trans, np.random.randn(3)) assign_rot = tf.assign(tf_rot, np.random.randn(3) * 0.03) assign_pose = tf.assign(tf_pose, np.random.randn(12) * 0.03) assign_shape = tf.assign(tf_shape, np.random.randn(300) * 1.0) assign_exp = tf.assign(tf_exp, np.random.randn(100) * 0.5) session.run([ assign_trans, assign_rot, assign_pose, assign_shape, assign_exp ]) mv.set_dynamic_meshes( [Mesh(session.run(tf_model), template_mesh.f)], blocking=True) raw_input('Press key to continue')
def sample_FLAME(template_fname, model_fname, num_samples): ''' Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to - model identity-dependent shape variations (paramters: shape), - articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12]) - model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp) - global translation (paramters: trans) - global rotation (paramters: rot) :param template_fname: template mesh in FLAME topology (only the face information are used) :param model_fname: saved FLAME model ''' template_mesh = Mesh(filename=template_fname) tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True) tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True) tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True) tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True) tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True) smpl = SMPL(model_fname) tf_model = tf.squeeze(smpl(tf_trans, tf.concat((tf_shape, tf_exp), axis=-1), tf.concat((tf_rot, tf_pose), axis=-1))) with tf.Session() as session: session.run(tf.global_variables_initializer()) mv = MeshViewer() for i in range(num_samples): assign_trans = tf.assign(tf_trans, np.random.randn(3)[np.newaxis,:]) assign_rot = tf.assign(tf_rot, np.random.randn(3)[np.newaxis,:] * 0.03) assign_pose = tf.assign(tf_pose, np.random.randn(12)[np.newaxis,:] * 0.03) assign_shape = tf.assign(tf_shape, np.random.randn(300)[np.newaxis,:] * 1.0) assign_exp = tf.assign(tf_exp, np.random.randn(100)[np.newaxis,:] * 0.5) session.run([assign_trans, assign_rot, assign_pose, assign_shape, assign_exp]) mv.set_dynamic_meshes([Mesh(session.run(tf_model), template_mesh.f)], blocking=True) six.moves.input('Press key to continue')
def fit_lmk3d(target_3d_lmks, template_fname, tf_model_fname, lmk_face_idx, lmk_b_coords, weights, show_fitting=True): ''' Fit FLAME to 3D landmarks :param target_3d_lmks: target 3D landmarks provided as (num_lmks x 3) matrix :param template_fname: template mesh in FLAME topology (only the face information are used) :param tf_model_fname: saved Tensorflow FLAME model :param lmk_face_idx: face indices of the landmark embedding in the FLAME topology :param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology (i.e. weighting of the three vertices for the trinagle, the landmark is embedded in :param weights: weights of the individual objective functions :return: a mesh with the fitting results ''' template_mesh = Mesh(filename=template_fname) saver = tf.train.import_meta_graph(tf_model_fname + '.meta') graph = tf.get_default_graph() tf_model = graph.get_tensor_by_name(u'vertices:0') with tf.Session() as session: saver.restore(session, tf_model_fname) # Workaround as existing tf.Variable cannot be retrieved back with tf.get_variable # tf_v_template = [x for x in tf.trainable_variables() if 'v_template' in x.name][0] tf_trans = [x for x in tf.trainable_variables() if 'trans' in x.name][0] tf_rot = [x for x in tf.trainable_variables() if 'rot' in x.name][0] tf_pose = [x for x in tf.trainable_variables() if 'pose' in x.name][0] tf_shape = [x for x in tf.trainable_variables() if 'shape' in x.name][0] tf_exp = [x for x in tf.trainable_variables() if 'exp' in x.name][0] lmks = tf_get_model_lmks(tf_model, template_mesh, lmk_face_idx, lmk_b_coords) lmk_dist = tf.reduce_sum( tf.square(1000 * tf.subtract(lmks, target_3d_lmks))) neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:3])) jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[3:6])) eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[6:])) shape_reg = tf.reduce_sum(tf.square(tf_shape)) exp_reg = tf.reduce_sum(tf.square(tf_exp)) # Optimize global transformation first vars = [tf_trans, tf_rot] loss = weights['lmk'] * lmk_dist optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={ 'disp': 1, 'ftol': 5e-6 }) print('Optimize rigid transformation') optimizer.minimize(session) # Optimize for the model parameters vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp] loss = weights['lmk'] * lmk_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \ weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={ 'disp': 1, 'ftol': 5e-6 }) print('Optimize model parameters') optimizer.minimize(session) print('Fitting done') if show_fitting: # Visualize landmark fitting mv = MeshViewer() mv.set_static_meshes( create_lmk_spheres(target_3d_lmks, 0.001, [255.0, 0.0, 0.0])) mv.set_dynamic_meshes( [Mesh(session.run(tf_model), template_mesh.f)] + create_lmk_spheres(session.run(lmks), 0.001, [0.0, 0.0, 255.0]), blocking=True) six.moves.input('Press key to continue') return Mesh(session.run(tf_model), template_mesh.f)
args = parser.parse_args() sequence_path = args.sequence_path audio_fname = args.audio_fname out_path = args.out_path img_path = os.path.join(out_path, 'img') if not os.path.exists(img_path): os.makedirs(img_path) mv = MeshViewer() sequence_fnames = sorted(glob.glob(os.path.join(sequence_path, '*.obj'))) if len(sequence_fnames) == 0: print('No meshes found') # Render images for frame_idx, mesh_fname in enumerate(sequence_fnames): frame_mesh = Mesh(filename=mesh_fname) mv.set_dynamic_meshes([frame_mesh], blocking=True) img_fname = os.path.join(img_path, '%05d.png' % frame_idx) mv.save_snapshot(img_fname) # Encode images to video cmd_audio = [] if os.path.exists(audio_fname): cmd_audio += ['-i', audio_fname] out_video_fname = os.path.join(out_path, 'video.mp4') cmd = ['ffmpeg', '-framerate', '60', '-pattern_type', 'glob', '-i', os.path.join(img_path, '*.png')] + cmd_audio + [out_video_fname] call(cmd)