def render_sequence_meshes(audio_fname, sequence_vertices, template, out_path, uv_template_fname='', texture_img_fname=''):
if not os.path.exists(out_path):
os.makedirs(out_path)
tmp_video_file = tempfile.NamedTemporaryFile('w', suffix='.mp4', dir=out_path)
if int(cv2.__version__[0]) < 3:
writer = cv2.VideoWriter(tmp_video_file.name, cv2.cv.CV_FOURCC(*'mp4v'), 60, (800, 800), True)
else:
writer = cv2.VideoWriter(tmp_video_file.name, cv2.VideoWriter_fourcc(*'mp4v'), 60, (800, 800), True)
if os.path.exists(uv_template_fname) and os.path.exists(texture_img_fname):
uv_template = Mesh(filename=uv_template_fname)
vt, ft = uv_template.vt, uv_template.ft
tex_img = cv2.imread(texture_img_fname)[:,:,::-1]
else:
vt, ft = None, None
tex_img = None
num_frames = sequence_vertices.shape[0]
center = np.mean(sequence_vertices[0], axis=0)
for i_frame in range(num_frames):
render_mesh = Mesh(sequence_vertices[i_frame], template.f)
if vt is not None and ft is not None:
render_mesh.vt, render_mesh.ft = vt, ft
img = render_mesh_helper(render_mesh, center, tex_img=tex_img)
writer.write(img)
writer.release()
video_fname = os.path.join(out_path, 'video.mp4')
cmd = ('ffmpeg' + ' -i {0} -i {1} -vcodec h264 -ac 2 -channel_layout stereo -pix_fmt yuv420p {2}'.format(
audio_fname, tmp_video_file.name, video_fname)).split()
call(cmd)
def render_sequence_meshes(audio_fname, sequence_vertices, template, out_path):
if not os.path.exists(out_path):
os.makedirs(out_path)
tmp_video_file = tempfile.NamedTemporaryFile('w',
suffix='.mp4',
dir=out_path)
if int(cv2.__version__[0]) < 3:
writer = cv2.VideoWriter(tmp_video_file.name,
cv2.cv.CV_FOURCC(*'mp4v'), 60, (800, 800),
True)
else:
writer = cv2.VideoWriter(tmp_video_file.name,
cv2.VideoWriter_fourcc(*'mp4v'), 60,
(800, 800), True)
num_frames = sequence_vertices.shape[0]
center = np.mean(sequence_vertices[0], axis=0)
for i_frame in range(num_frames):
img = render_mesh_helper(Mesh(sequence_vertices[i_frame], template.f),
center)
writer.write(img)
writer.release()
video_fname = os.path.join(out_path, 'video.mp4')
cmd = (
'ffmpeg' +
' -i {0} -i {1} -vcodec h264 -ac 2 -channel_layout stereo -pix_fmt yuv420p {2}'
.format(audio_fname, tmp_video_file.name, video_fname)).split()
call(cmd)
def _render_sequences_helper(self, video_fname, seq_raw_audio, seq_processed_audio, seq_template, seq_verts, condition_idx):
def add_image_text(img, text):
font = cv2.FONT_HERSHEY_SIMPLEX
textsize = cv2.getTextSize(text, font, 1, 2)[0]
textX = (img.shape[1] - textsize[0]) / 2
textY = textsize[1] + 10
cv2.putText(img, '%s' % (text), (textX, textY), font, 1, (0, 0, 255), 2, cv2.LINE_AA)
num_frames = seq_verts.shape[0]
tmp_audio_file = tempfile.NamedTemporaryFile('w', suffix='.wav', dir=os.path.dirname(video_fname))
wavfile.write(tmp_audio_file.name, seq_raw_audio['sample_rate'], seq_raw_audio['audio'])
tmp_video_file = tempfile.NamedTemporaryFile('w', suffix='.mp4', dir=os.path.dirname(video_fname))
if int(cv2.__version__[0]) < 3:
print('cv2 < 3')
# writer = cv2.VideoWriter(tmp_video_file.name, cv2.cv.CV_FOURCC(*'DIVX'), 60, (4000, 800), True)
writer = cv2.VideoWriter(tmp_video_file.name, cv2.cv.CV_FOURCC(*'mp4v'), 60, (1600, 800), True)
else:
print('cv2 >= 3')
# writer = cv2.VideoWriter(tmp_video_file.name, cv2.VideoWriter_fourcc(*'DIVX'), 60, (4000, 800), True)
writer = cv2.VideoWriter(tmp_video_file.name, cv2.VideoWriter_fourcc(*'mp4v'), 60, (1600, 800), True)
feed_dict = {self.speech_features: np.expand_dims(np.stack(seq_processed_audio), -1),
self.condition_subject_id: np.repeat(condition_idx, num_frames),
self.is_training: False,
self.input_template: np.repeat(seq_template[np.newaxis,:,:,np.newaxis], num_frames, axis=0)}
predicted_vertices, predicted_offset = self.session.run([self.output_decoder, self.expression_offset], feed_dict)
predicted_vertices = np.squeeze(predicted_vertices)
center = np.mean(seq_verts[0], axis=0)
for i_frame in range(num_frames):
gt_img = render_mesh_helper(Mesh(seq_verts[i_frame], self.template_mesh.f), center)
add_image_text(gt_img, 'Captured data')
pred_img = render_mesh_helper(Mesh(predicted_vertices[i_frame], self.template_mesh.f), center)
add_image_text(pred_img, 'VOCA prediction')
img = np.hstack((gt_img, pred_img))
writer.write(img)
writer.release()
cmd = ('ffmpeg' + ' -i {0} -i {1} -vcodec h264 -ac 2 -channel_layout stereo -pix_fmt yuv420p {2}'.format(
tmp_audio_file.name, tmp_video_file.name, video_fname)).split()
call(cmd)
def sample_texture(model_fname, texture_fname, num_samples, out_path):
'''
Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to
- model identity-dependent shape variations (paramters: shape),
- articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12])
- model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp)
- global translation (paramters: trans)
- global rotation (paramters: rot)
:param model_fname saved FLAME model
:param num_samples number of samples
:param out_path output path to save the generated templates (no templates are saved if path is empty)
'''
tf_trans = tf.Variable(np.zeros((1, 3)),
name="trans",
dtype=tf.float64,
trainable=True)
tf_rot = tf.Variable(np.zeros((1, 3)),
name="pose",
dtype=tf.float64,
trainable=True)
tf_pose = tf.Variable(np.zeros((1, 12)),
name="pose",
dtype=tf.float64,
trainable=True)
tf_shape = tf.Variable(np.zeros((1, 300)),
name="shape",
dtype=tf.float64,
trainable=True)
tf_exp = tf.Variable(np.zeros((1, 100)),
name="expression",
dtype=tf.float64,
trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(
smpl(tf_trans, tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
texture_model = np.load(texture_fname)
tex_dim = texture_model['tex_dir'].shape[-1]
tf_tex_params = tf.Variable(np.zeros((1, tex_dim)),
name="pose",
dtype=tf.float64,
trainable=True)
tf_tex_mean = tf.Variable(np.reshape(texture_model['mean'], (1, -1)),
name='tex_mean',
dtype=tf.float64,
trainable=False)
tf_tex_dir = tf.Variable(np.reshape(texture_model['tex_dir'],
(-1, tex_dim)).T,
name='tex_dir',
dtype=tf.float64,
trainable=False)
tf_tex = tf.add(tf_tex_mean, tf.matmul(tf_tex_params, tf_tex_dir)),
tf_tex = tf.reshape(
tf_tex,
(texture_model['tex_dir'].shape[0], texture_model['tex_dir'].shape[1],
texture_model['tex_dir'].shape[2]))
tf_tex = tf.cast(tf.clip_by_value(tf_tex, 0.0, 255.0), tf.int64)
with tf.Session() as session:
session.run(tf.global_variables_initializer())
mv = MeshViewer()
for i in range(num_samples):
assign_tex = tf.assign(tf_tex_params,
np.random.randn(tex_dim)[np.newaxis, :])
session.run([assign_tex])
v, tex = session.run([tf_model, tf_tex])
out_mesh = Mesh(v, smpl.f)
out_mesh.vt = texture_model['vt']
out_mesh.ft = texture_model['ft']
img = render_mesh_helper(out_mesh, None)
cv2.imwrite(os.path.join(out_path, 'test.png'),
cv2.cvtColor(img, cv2.COLOR_RGB2BGR))
mv.set_dynamic_meshes([out_mesh], blocking=True)
key = six.moves.input(
'Press (s) to save sample, any other key to continue ')
if key == 's':
out_mesh_fname = os.path.join(out_path,
'tex_sample_%02d.obj' % (i + 1))
out_tex_fname = out_mesh_fname.replace('obj', 'png')
cv2.imwrite(out_tex_fname, tex)
out_mesh.set_texture_image(out_tex_fname)
out_mesh.write_obj(out_mesh_fname)