def img2tex(self, img, coeffs):
renderer = mesh_renderer.get_mesh_renderer()
head_mesh_warped = face3d.head_mesh(self.pix2face_data.head_mesh)
head_mesh_warped.apply_coefficients(
self.pix2face_data.subject_components,
self.pix2face_data.expression_components, coeffs.subject_coeffs(),
coeffs.expression_coeffs(0))
tex = face3d.image_to_texture_float(img, head_mesh_warped,
coeffs.camera(0), renderer)
return tex
def composite_texture(self, img, subj_coeffs, expr_coeffs, camera, tex):
renderer = mesh_renderer.get_mesh_renderer()
head_mesh_warped = face3d.head_mesh(self.pix2face_data.head_mesh)
head_mesh_warped.apply_coefficients(
self.pix2face_data.subject_components,
self.pix2face_data.expression_components, subj_coeffs, expr_coeffs)
render_walpha = face3d.texture_to_image_float(tex, head_mesh_warped,
camera, renderer)
render_rgb = (render_walpha[:, :, 0:3] * 255).astype(np.uint8)
render_mask = (render_walpha[:, :, 3] * 255).astype(np.uint8)
render_mask[render_mask < 10] = 0
render_mask[render_mask > 0] = 255
render_mask[0, 0] = 255
render_mask[-1, -1] = 255
img_out = cv2.seamlessClone(render_rgb, img, render_mask,
(img.shape[1] // 2, img.shape[0] // 2),
cv2.NORMAL_CLONE)
return img_out
def render_coefficients(coeffs, pix2face_data, img_idx=0):
"""
Returns an image of the face described by coeffs
pix2face_data should be created by calling load_pix2face_data
"""
# lazily create renderer (of which there should only be one.)
renderer = mesh_renderer.get_mesh_renderer()
texture_res = 64 # resolution not important as this will be a solid color texture
green_tex = np.zeros((texture_res, texture_res, 3), np.uint8)
green_tex[:, :, 1] = 255
head_mesh_warped = face3d.head_mesh(pix2face_data.head_mesh)
head_mesh_warped.apply_coefficients(pix2face_data.subject_components,
pix2face_data.expression_components,
coeffs.subject_coeffs(),
coeffs.expression_coeffs(img_idx))
meshes = head_mesh_warped.meshes()
for mesh in meshes:
mesh.set_texture(green_tex)
renderer.set_ambient_weight(0.5)
synth = renderer.render(meshes, coeffs.camera(img_idx))
return synth
def run_pipeline():
# Read pipeline type from command line
parser = argparse.ArgumentParser()
parser.add_argument('--pipeline', help='sets pipeline', default='default')
args = parser.parse_args()
# Read json config
json_file = open('/pix2face/config.json')
data = json.load(json_file)
pipeline_config = data["pipelines"][args.pipeline]
json_file.close()
# Load pretrained model
cuda_device = 0
model = pix2face.test.load_pretrained_model(cuda_device=cuda_device)
this_dir = os.path.dirname(__file__)
pvr_data_dir = os.path.join('/pix2face/lib/face3d/data_3DMM/')
debug_dir = ''
debug_mode = False
num_subject_coeffs = 199 # max 199
num_expression_coeffs = 29 # max 29
# load needed data files
head_mesh = face3d.head_mesh(pvr_data_dir)
subject_components = np.load(os.path.join(pvr_data_dir, 'pca_components_subject.npy'))
expression_components = np.load(os.path.join(pvr_data_dir, 'pca_components_expression.npy'))
subject_ranges = np.load(os.path.join(pvr_data_dir,'pca_coeff_ranges_subject.npy'))
expression_ranges = np.load(os.path.join(pvr_data_dir,'pca_coeff_ranges_expression.npy'))
# keep only the PCA components that we will be estimating
subject_components = vxl.vnl.matrix(subject_components[0:num_subject_coeffs,:])
expression_components = vxl.vnl.matrix(expression_components[0:num_expression_coeffs,:])
subject_ranges = vxl.vnl.matrix(subject_ranges[0:num_subject_coeffs,:])
expression_ranges = vxl.vnl.matrix(expression_ranges[0:num_expression_coeffs,:])
# create rendering object (encapsulates OpenGL context)
renderer = face3d.mesh_renderer()
# create coefficient estimator
coeff_estimator = face3d.media_coefficient_from_PNCC_and_offset_estimator(head_mesh, subject_components, expression_components, subject_ranges, expression_ranges, debug_mode, debug_dir)
data_dir = pipeline_config["inputDir"]
output_dir = pipeline_config["outputDir"]
directories = os.listdir(data_dir)
for directory in directories:
files = os.listdir(os.path.join(data_dir, directory))
for file_item in files:
file_path, ext_path = os.path.splitext(file_item)
img_fname = os.path.join(data_dir, directory, file_item)
print(data_dir, directory, file_item)
img = np.array(Image.open(img_fname))
outputs = pix2face.test.test(model, [img,])
pncc = outputs[0][0]
offsets = outputs[0][1]
pncc_rgb = pncc / 300.0 + 0.5
offsets_rgb = offsets / 60.0 + 0.5
pix2face_data = pix2face_estimation.coefficient_estimation.load_pix2face_data()
# create rendering object (encapsulates OpenGL context)
renderer = face3d.mesh_renderer()
# create coefficient estimator
coeff_estimator = face3d.media_coefficient_from_PNCC_and_offset_estimator(head_mesh, subject_components, expression_components, subject_ranges, expression_ranges, debug_mode, debug_dir)
# Estimate Coefficients from PNCC and Offsets
print('Estimating Coefficients..')
img_ids = ['img0',]
coeffs, result = coeff_estimator.estimate_coefficients_perspective(img_ids, [pncc,], [offsets,])
# Print Yaw, Pitch, Roll of Head
R_cam = np.array(coeffs.camera(0).rotation.as_matrix()) # rotation matrix of estimated camera
R0 = np.diag((1,-1,-1)) # R0 is the rotation matrix of a frontal camera
R_head = np.dot(R0,R_cam)
yaw, pitch, roll = geometry_utils.matrix_to_Euler_angles(R_head, order='YXZ')
print('yaw, pitch, roll = %0.1f, %0.1f, %0.1f (degrees)' % (np.rad2deg(yaw), np.rad2deg(pitch), np.rad2deg(roll)))
# Render 3D-Jittered Images
print('Rendering Jittered Images..')
jitterer = face3d.media_jitterer_perspective([img,], coeffs, head_mesh, subject_components, expression_components, renderer, "")
for entity in pipeline_config["entities"]:
if entity["type"] == "emote":
# manually alter expression
new_expression_coeffs = np.zeros_like(coeffs.expression_coeffs(0))
parameters = entity["parameters"]
new_expression_coeffs[0] = parameters["anger"]
new_expression_coeffs[1] = parameters["disgust"]
new_expression_coeffs[2] = parameters["fear"]
new_expression_coeffs[3] = parameters["happiness"]
new_expression_coeffs[4] = parameters["sadness"]
new_expression_coeffs[5] = parameters["surprise"]
render_img = jitterer.render(coeffs.camera(0), coeffs.subject_coeffs(), new_expression_coeffs, subject_components, expression_components)
elif entity["type"] == "pose":
# manually alter pose
delta_R = vxl.vgl.rotation_3d(geometry_utils.Euler_angles_to_quaternion(np.pi/3, 0, 0, order='YXZ'))
cam = coeffs.camera(0)
new_R = cam.rotation * delta_R
new_cam = face3d.perspective_camera_parameters(cam.focal_len, cam.principal_point, new_R, cam.translation, cam.nx, cam.ny)
render_img = jitterer.render(new_cam, coeffs.subject_coeffs(), coeffs.expression_coeffs(0), subject_components, expression_components)
Image.fromarray(render_img[:,:,0:3]).save(os.path.join(os.path.join(output_dir, directory), file_path + "_" + entity["name"] + "." + ext_path))
Image.fromarray(img).save(os.path.join(os.path.join(output_dir, directory), file_path + "_original." + ext_path))
def load_pix2face_data(pvr_data_dir=None,
num_subject_coeffs=None,
num_expression_coeffs=None,
use_offsets_for_estimation=True):
"""
Load PCA components and ranges.
Returns a structure containing the following matrices loaded as instances of vxl.vnl_matrix
"""
if pvr_data_dir is None:
# guess the pvr_data_dir
this_dir = os.path.dirname(__file__)
pvr_data_dir = os.path.join(this_dir, '../../face3d/data_3DMM')
# load data files as numpy arrays
head_mesh = face3d.head_mesh(pvr_data_dir)
subject_components = np.load(
os.path.join(pvr_data_dir, 'pca_components_subject.npy'))
expression_components = np.load(
os.path.join(pvr_data_dir, 'pca_components_expression.npy'))
subject_ranges = np.load(
os.path.join(pvr_data_dir, 'pca_coeff_ranges_subject.npy'))
expression_ranges = np.load(
os.path.join(pvr_data_dir, 'pca_coeff_ranges_expression.npy'))
if num_subject_coeffs is None:
num_subject_coeffs = subject_components.shape[0]
if num_expression_coeffs is None:
num_expression_coeffs = expression_components.shape[0]
# keep only needed rows of subject and expression matrices and convert to vnl matrices
subject_components = vxl.vnl.matrix(
subject_components[0:num_subject_coeffs, :])
subject_ranges = vxl.vnl.matrix(subject_ranges[0:num_subject_coeffs, :])
expression_components = vxl.vnl.matrix(
expression_components[0:num_expression_coeffs, :])
expression_ranges = vxl.vnl.matrix(
expression_ranges[0:num_expression_coeffs, :])
debug_mode = False
debug_dir = ""
if use_offsets_for_estimation:
coeff_estimator = \
face3d.media_coefficient_from_PNCC_and_offset_estimator(head_mesh,
subject_components, expression_components,
subject_ranges, expression_ranges,
debug_mode, debug_dir)
else:
coeff_estimator = \
face3d.media_coefficient_from_PNCC_estimator(head_mesh,
subject_components, expression_components,
subject_ranges, expression_ranges,
debug_mode, debug_dir)
# return in MMData structure
return Pix2FaceData(head_mesh=head_mesh,
subject_components=subject_components,
subject_ranges=subject_ranges,
expression_components=expression_components,
expression_ranges=expression_ranges,
coeff_estimator=coeff_estimator,
use_offsets=use_offsets_for_estimation)
outputs = pix2face.test.test(model, [
img,
], cuda_device=cuda_device)
pncc = outputs[0][0]
offsets = outputs[0][1]
print('..Done')
pvr_data_dir = os.path.join(this_dir, '../face3d/data_3DMM/')
debug_dir = ''
debug_mode = False
num_subject_coeffs = 199 # max 199
num_expression_coeffs = 29 # max 29
# load needed data files
head_mesh = face3d.head_mesh(pvr_data_dir)
subject_components = np.load(
os.path.join(pvr_data_dir, 'pca_components_subject.npy'))
expression_components = np.load(
os.path.join(pvr_data_dir, 'pca_components_expression.npy'))
subject_ranges = np.load(
os.path.join(pvr_data_dir, 'pca_coeff_ranges_subject.npy'))
expression_ranges = np.load(
os.path.join(pvr_data_dir, 'pca_coeff_ranges_expression.npy'))
# keep only the PCA components that we will be estimating
subject_components = vxl.vnl.matrix(
subject_components[0:num_subject_coeffs, :])
expression_components = vxl.vnl.matrix(
expression_components[0:num_expression_coeffs, :])
subject_ranges = vxl.vnl.matrix(subject_ranges[0:num_subject_coeffs, :])