def visualization(verts,
img,
bfm,
n_sample=None,
MM_base_dir='./external/face3d/examples/Data/BFM'):
bfm_info = load_BFM_info(os.path.join(MM_base_dir, 'Out/BFM_info.mat'))
face_region_mask = bfm.face_region_mask.copy()
face_region_mask[bfm_info['nose_hole'].ravel()] = False
N, V, _, _, _ = img.shape
img_grids = []
for i in range(N):
if n_sample is not None and i >= n_sample:
break
img_list = []
for j in range(V):
cur_img = img[i, j, ...].cpu()
cur_img_np = np.ascontiguousarray(cur_img.numpy().transpose(
(1, 2, 0)))
img_list.append(cur_img)
for k in range(len(verts)):
if k == 0:
vert = verts[k][i, j, ...].detach().cpu().numpy()
else:
vert = verts[k][-1][i, j, ...].detach().cpu().numpy()
geo_vis = visualize_geometry(vert, np.copy(cur_img_np),
bfm.model['tri'],
face_region_mask)
img_list.append(torch.tensor(geo_vis))
img_grid = make_grid(img_list, nrow=1 + len(verts)).detach().cpu()
img_grids.append(img_grid)
return img_grids
def get_tight_face_region(bfm, MM_base_dir='./external/face3d/examples/Data/BFM', front=False):
"""
Get vertices indices of tight face region
:param bfm:
:param MM_base_dir:
:return: mask: True if the vertex is inside tight face region. np.array boolean. (nver, 1)
"""
bfm_info = bfm_load.load_BFM_info(os.path.join(MM_base_dir, 'Out/BFM_info.mat'))
sp = np.zeros((bfm.n_shape_para, 1), dtype=np.float)
ep = np.zeros((bfm.n_exp_para, 1), dtype=np.float)
vertices = bfm.generate_vertices(sp, ep)
nver = vertices.shape[0]
nose_bottom = vertices[bfm.kpt_ind[30], :]
nose_bridge = (vertices[bfm.kpt_ind[39], :] + vertices[bfm.kpt_ind[42], :]) / 2 # between the inner eye corners
face_centre = nose_bottom + 0.3 * (nose_bridge - nose_bottom)
outer_eye_dist = np.linalg.norm(vertices[bfm.kpt_ind[36], :] - vertices[bfm.kpt_ind[45], :])
nose_dist = np.linalg.norm(nose_bridge - nose_bottom)
# Mask lower face
mask_radius = 1.5 * (outer_eye_dist + nose_dist) / 2
# dist = np.linalg.norm(vertices[:nver, :] - face_centre.reshape((1, 3)), axis=1)
# face_region_mask1 = dist <= mask_radius
face_region_mask1 = (np.square((vertices[:nver, 0] - nose_bridge[0]) / (1.2 * mask_radius)) + np.square(
(vertices[:nver, 1] - nose_bridge[1]) / (1.075 * mask_radius)) + np.square(
(vertices[:nver, 2] - nose_bridge[2]) / (0.65 * mask_radius))) <= 1
# Mask forehead
in_ellipse = (np.square((vertices[:nver, 0] - nose_bridge[0]) / mask_radius) + np.square(
(vertices[:nver, 1] - nose_bridge[1]) / (0.5 * mask_radius)) + np.square(
(vertices[:nver, 2] - nose_bridge[2]) / (0.75 * mask_radius))) <= 1
face_region_mask2 = np.logical_or(np.logical_and(in_ellipse, vertices[:nver, 1] > nose_bridge[1]),
vertices[:nver, 1] < nose_bridge[1])
# Mask left & right upper
in_ellipse = (np.abs((vertices[:nver, 0] - nose_bridge[0]) / (1.5 * mask_radius))**4 + np.abs(
(vertices[:nver, 1] + 0.4 * mask_radius - nose_bridge[1]) / (0.85 * mask_radius))**1.2 + np.abs(
(vertices[:nver, 2] - nose_bridge[2]) / (0.6 * mask_radius))**1.) <= 1
face_region_mask3 = np.logical_or(np.logical_and(in_ellipse, vertices[:nver, 2] < nose_bridge[2] + 0.025 * mask_radius),
vertices[:nver, 2] >= nose_bridge[2] + 0.025 * mask_radius)
face_region_mask3 = np.logical_or(np.logical_and(face_region_mask3, vertices[:nver, 1] > nose_bridge[1] - 0.3 * mask_radius),
vertices[:nver, 1] <= nose_bridge[1] - 0.3 * mask_radius)
# face_region_mask3 = in_ellipse
face_region_mask = np.logical_and(
np.logical_and(face_region_mask1, face_region_mask2), face_region_mask3
)
return face_region_mask.reshape(-1, 1)
def load_3DMM(MM_base_dir='./external/face3d/examples/Data/BFM'):
"""
Load BFM 3DMM.
additional attr:
params_mean_3dffa:
params_std_3dffa:
sigma_exp:
face_region_mask: whether a vertex is in the face region. np.array boolean. (nver, 1)
tri_idx: index of adjacent triangles for each vertex. np.array int32. (nver, max_number_tri_per_vert)
updated expPC & expEV: 64 parameters.
uv_coords: y axis point to up, range [0, 1]. np.array (nver, 2)
uv_coords_64: uv_coords mapped to 64x64 image space with zero depths. np.array (nver, 3)
pixel_vert_idx_64: vertices indices of rendered triangle on each pixel in a 64x64 uv map. np.array int. (64, 64, 3)
pixel_vert_weights_64: vertices weights of rendered triangle on each pixel in a 64x64 uv map. np.array (64, 64, 3)
:param MM_base_dir:
:return:
"""
def LoadExpBasis():
n_vertex = 53215
Expbin = open(os.path.join(MM_base_dir, 'Out/Exp_Pca.bin'), 'rb')
exp_dim = array('i')
exp_dim.fromfile(Expbin, 1)
expMU = array('f')
expPC = array('f')
expMU.fromfile(Expbin, 3 * n_vertex)
expPC.fromfile(Expbin, 3 * exp_dim[0] * n_vertex)
expPC = np.array(expPC)
expPC = np.reshape(expPC, [exp_dim[0], -1])
expPC = np.transpose(expPC)
expEV = np.loadtxt(os.path.join(MM_base_dir, 'Out/std_exp.txt')).reshape((exp_dim[0], -1))
expPC = expPC[:, :64].astype(np.float32)
expEV = expEV[:64, :].astype(np.float32)
return expPC, expEV
def process_uv(uv_coords, uv_h=256, uv_w=256):
uv_coords[:, 0] = uv_coords[:, 0] * (uv_w - 1)
uv_coords[:, 1] = uv_coords[:, 1] * (uv_h - 1)
uv_coords[:, 1] = uv_h - uv_coords[:, 1] - 1
uv_coords = np.hstack((uv_coords, np.zeros((uv_coords.shape[0], 1)))) # add z
return uv_coords
bfm = MorphabelModel(os.path.join(MM_base_dir, 'Out/BFM.mat'))
model_info = load_BFM_info(os.path.join(MM_base_dir, 'Out/BFM_info.mat'))
bfm.segbin = model_info['segbin'].astype(np.bool)
# params_attr = sio.loadmat(os.path.join(MM_base_dir, '3ddfa/3DDFA_Release/Matlab/params_attr.mat'))
# bfm.params_mean_3dffa = params_attr['params_mean']
# bfm.params_std_3dffa = params_attr['params_std']
# sigma_exp = sio.loadmat(os.path.join(MM_base_dir, 'sigma_exp.mat'))
# bfm.sigma_exp = sigma_exp['sigma_exp'].reshape((29, 1))
bfm.face_region_mask = get_tight_face_region(bfm, MM_base_dir, True)
vert = np.zeros((int(bfm.nver), 3))
colors = np.zeros((int(bfm.nver), 3))
tri = np.zeros_like(bfm.model['tri'])
tri[:, 0] = bfm.model['tri'][:, 2]
tri[:, 1] = bfm.model['tri'][:, 1]
tri[:, 2] = bfm.model['tri'][:, 0]
_, _, bfm.face_region_tri = filter_non_tight_face_vert(vert, colors, tri, bfm.face_region_mask.copy())
mask = bfm.face_region_mask.copy()
mask[model_info['nose_hole'].ravel()] = False
mask[bfm.kpt_ind] = False
bfm.face_region_tri_mask = get_tight_face_tri_mask(bfm.model['tri'], mask)
bfm.tri_idx = get_adjacent_triangle_idx(int(bfm.nver), bfm.model['tri'])
bfm.neib_vert_idx, bfm.neib_vert_count = get_neighbor_vert_idx(int(bfm.nver), bfm.tri_idx, bfm.model['tri'])
bfm.model['expPC'], bfm.model['expEV'] = LoadExpBasis()
bfm.n_exp_para = 64
bfm.model['shapePC'] = bfm.model['shapePC'][:, :80]
bfm.model['shapeEV'] = bfm.model['shapeEV'][:80, :]
bfm.n_shape_para = 80
bfm.uv_coords = bfm_load.load_uv_coords(os.path.join(MM_base_dir, 'Out/BFM_UV.mat'))
@jit(nopython=True)
def wrap_uv_coords(uv_coords):
for i in range(uv_coords.shape[0]):
if uv_coords[i, 1] < 0.555:
factor = 0.7#min((0.5 - uv_coords[i, 1]) * 2. + 1., 1.6)
factor_l = min((0.555 - uv_coords[i, 1]), 0.5)
x = np.abs(uv_coords[i, 0] - 0.5) * 2. * 0.9#* (1. - (factor - 1.) / 2. * 0.5)
sign = np.sign(uv_coords[i, 0] - 0.5)
uv_coords[i, 0] = sign * np.power(x, factor) / 2. * (0.85 * factor_l + 1.) + 0.5
else:
factor = min((uv_coords[i, 1] - 0.555) * 1.25 + 0.7, 1.25)
factor_l = min((uv_coords[i, 1] - 0.555), 0.5)
x = np.abs(uv_coords[i, 0] - 0.5) * 2. * 0.9
sign = np.sign(uv_coords[i, 0] - 0.5)
uv_coords[i, 0] = sign * np.power(x, factor) / 2. * (0.95 * factor_l + 1.) + 0.5
return uv_coords
bfm.uv_coords = wrap_uv_coords(bfm.uv_coords)
bfm.uv_coords[:, 1] -= 0.105
bfm.uv_coords[:, 0] = (bfm.uv_coords[:, 0] - 0.5) * 12.5 / 10. + 0.5
bfm.uv_coords[:, 1] = (bfm.uv_coords[:, 1] - 0.5) * 4. / 3. + 0.5
bfm.uv_coords_8 = process_uv(bfm.uv_coords.copy(), 8, 8)
bfm.uv_coords_16 = process_uv(bfm.uv_coords.copy(), 16, 16)
bfm.uv_coords_32 = process_uv(bfm.uv_coords.copy(), 32, 32)
bfm.pixel_vert_idx_32, bfm.pixel_vert_weights_32 = \
get_pixel_vert_idx_and_weights(bfm.uv_coords_32, bfm.model['tri'], 32, 32)
bfm.uv_coords_64 = process_uv(bfm.uv_coords.copy(), 64, 64)
bfm.pixel_vert_idx_64, bfm.pixel_vert_weights_64 = \
get_pixel_vert_idx_and_weights(bfm.uv_coords_64, bfm.model['tri'], 64, 64)
bfm.uv_coords_128 = process_uv(bfm.uv_coords.copy(), 128, 128)
bfm.pixel_vert_idx_128, bfm.pixel_vert_weights_128 = \
get_pixel_vert_idx_and_weights(bfm.uv_coords_128, bfm.model['tri'], 128, 128)
bfm.uv_coords_256 = process_uv(bfm.uv_coords.copy(), 256, 256)
bfm.pixel_vert_idx_256, bfm.pixel_vert_weights_256 = \
get_pixel_vert_idx_and_weights(bfm.uv_coords_256, bfm.model['tri'], 256, 256)
return bfm
def load_3DMM(MM_base_dir='./external/face3d/examples/Data/BFM'):
"""
Load BFM 3DMM.
additional attr:
params_mean_3dffa:
params_std_3dffa:
sigma_exp:
face_region_mask: whether a vertex is in the face region. np.array boolean. (nver, 1)
tri_idx: index of adjacent triangles for each vertex. np.array int32. (nver, max_number_tri_per_vert)
updated expPC & expEV: 64 parameters.
uv_coords: y axis point to up, range [0, 1]. np.array (nver, 2)
uv_coords_64: uv_coords mapped to 64x64 image space with zero depths. np.array (nver, 3)
pixel_vert_idx_64: vertices indices of rendered triangle on each pixel in a 64x64 uv map. np.array int. (64, 64, 3)
pixel_vert_weights_64: vertices weights of rendered triangle on each pixel in a 64x64 uv map. np.array (64, 64, 3)
:param MM_base_dir:
:return:
"""
def LoadExpBasis():
n_vertex = 53215
Expbin = open(os.path.join(MM_base_dir, 'Out/Exp_Pca.bin'), 'rb')
exp_dim = array('i')
exp_dim.fromfile(Expbin, 1)
expMU = array('f')
expPC = array('f')
expMU.fromfile(Expbin, 3 * n_vertex)
expPC.fromfile(Expbin, 3 * exp_dim[0] * n_vertex)
expPC = np.array(expPC)
expPC = np.reshape(expPC, [exp_dim[0], -1])
expPC = np.transpose(expPC)
expEV = np.loadtxt(os.path.join(MM_base_dir,
'Out/std_exp.txt')).reshape(
(exp_dim[0], -1))
expPC = expPC[:, :64].astype(np.float32)
expEV = expEV[:64, :].astype(np.float32)
return expPC, expEV
def process_uv(uv_coords, uv_h=256, uv_w=256):
uv_coords[:, 0] = uv_coords[:, 0] * (uv_w - 1)
uv_coords[:, 1] = uv_coords[:, 1] * (uv_h - 1)
uv_coords[:, 1] = uv_h - uv_coords[:, 1] - 1
uv_coords = np.hstack((uv_coords, np.zeros(
(uv_coords.shape[0], 1)))) # add z
return uv_coords
bfm = MorphabelModel(os.path.join(MM_base_dir, 'Out/BFM.mat'))
model_info = load_BFM_info(os.path.join(MM_base_dir, 'Out/BFM_info.mat'))
bfm.segbin = model_info['segbin'].astype(np.bool)
# params_attr = sio.loadmat(os.path.join(MM_base_dir, 'Out/params_attr.mat'))
# bfm.params_mean_3dffa = params_attr['params_mean']
# bfm.params_std_3dffa = params_attr['params_std']
# sigma_exp = sio.loadmat(os.path.join(MM_base_dir, 'Out/sigma_exp.mat'))
# bfm.sigma_exp = sigma_exp['sigma_exp'].reshape((29, 1))
bfm.face_region_mask = get_tight_face_region(bfm, MM_base_dir, True)
bfm.tri_idx = get_adjacent_triangle_idx(int(bfm.nver), bfm.model['tri'])
bfm.neib_vert_idx, bfm.neib_vert_count = get_neighbor_vert_idx(
int(bfm.nver), bfm.tri_idx, bfm.model['tri'])
bfm.model['expPC'], bfm.model['expEV'] = LoadExpBasis()
bfm.n_exp_para = 64
bfm.model['shapePC'] = bfm.model['shapePC'][:, :80]
bfm.model['shapeEV'] = bfm.model['shapeEV'][:80, :]
bfm.n_shape_para = 80
bfm.uv_coords = bfm_load.load_uv_coords(
os.path.join(MM_base_dir, 'Out/BFM_UV.mat'))
bfm.uv_coords[:, 1] -= 0.11
bfm.uv_coords[:, 0] = (bfm.uv_coords[:, 0] - 0.5) * 12. / 11. + 0.5
bfm.uv_coords[:, 1] = (bfm.uv_coords[:, 1] - 0.5) * 4. / 3. + 0.5
bfm.uv_coords_32 = process_uv(bfm.uv_coords.copy(), 32, 32)
bfm.pixel_vert_idx_32, bfm.pixel_vert_weights_32 = \
get_pixel_vert_idx_and_weights(bfm.uv_coords_32, bfm.model['tri'], 32, 32)
bfm.uv_coords_64 = process_uv(bfm.uv_coords.copy(), 64, 64)
bfm.pixel_vert_idx_64, bfm.pixel_vert_weights_64 = \
get_pixel_vert_idx_and_weights(bfm.uv_coords_64, bfm.model['tri'], 64, 64)
bfm.uv_coords_128 = process_uv(bfm.uv_coords.copy(), 128, 128)
bfm.pixel_vert_idx_128, bfm.pixel_vert_weights_128 = \
get_pixel_vert_idx_and_weights(bfm.uv_coords_128, bfm.model['tri'], 128, 128)
return bfm