def align_face(im, src, size):
# Refer to:
# https://github.com/deepinsight/insightface/blob/master/src/common/face_preprocess.py#L46
dst = np.array(
[[30.2946, 51.6963], [65.5318, 51.5014], [48.0252, 71.7366],
[33.5493, 92.3655], [62.7299, 92.2041]],
dtype=np.float32)
dst[:, 0] += 8.0
M = umeyama(src, dst, True)[0:2]
warped = cv2.warpAffine(im, M, (size, size), borderValue=0.0)
return warped
def init_guess(setting, data, use_torso=False, **kwargs):
model = setting['model']
dtype = setting['dtype']
keypoints = data['keypoints']
batch_size = setting['batch_size']
device = setting['device']
est_scale = not setting['fix_scale']
fixed_scale = 1. if setting['fixed_scale'] is None else setting['fixed_scale']
joints3d = recompute3D(setting['extris'], setting['intris'], keypoints)
# reset model
init_t = torch.zeros((1,3), dtype=dtype)
init_r = torch.zeros((1,3), dtype=dtype)
init_s = torch.tensor(fixed_scale, dtype=dtype)
init_shape = torch.zeros((1,10), dtype=dtype)
model.reset_params(transl=init_t, global_orient=init_r, scale=init_s, betas=init_shape)
init_pose = torch.zeros((1,69), dtype=dtype).cuda()
model_output = model(return_verts=True, return_full_pose=True, body_pose=init_pose)
verts = model_output.vertices[0]
if kwargs.get('model_type') == 'smpllsp':
J = torch.matmul(model.joint_regressor, verts)
else:
J = torch.matmul(model.J_regressor, verts)
verts = verts.unsqueeze(0)
J = J.unsqueeze(0)
joints = model.vertex_joint_selector(verts, J)
# Map the joints to the current dataset
if model.joint_mapper is not None:
joints = model.joint_mapper(joints).detach().cpu().numpy()[0]
if use_torso:
joints3d = joints3d[[5,6,11,12]]
joints = joints[[5,6,11,12]]
# get transformation
rot, trans, scale = umeyama(joints, joints3d, est_scale)
rot = cv2.Rodrigues(rot)[0]
# apply to model
if est_scale:
init_s = torch.tensor(scale, dtype=dtype)
else:
init_s = torch.tensor(fixed_scale, dtype=dtype)
init_t = torch.tensor(trans, dtype=dtype)
init_r = torch.tensor(rot, dtype=dtype).reshape(1,3)
model.reset_params(transl=init_t, global_orient=init_r, scale=init_s)
if kwargs.get('use_vposer'):
with torch.no_grad():
setting['pose_embedding'].fill_(0)
def random_warp_64(image):
assert image.shape == (256, 256, 3)
range_ = np.linspace(128 - 120, 128 + 120, 5)
mapx = np.broadcast_to(range_, (5, 5))
mapy = mapx.T
mapx = mapx + np.random.normal(size=(5, 5), scale=5)
mapy = mapy + np.random.normal(size=(5, 5), scale=5)
interp_mapx = cv2.resize(mapx, (80, 80))[8:72, 8:72].astype('float32')
interp_mapy = cv2.resize(mapy, (80, 80))[8:72, 8:72].astype('float32')
warped_image = cv2.remap(image, interp_mapx, interp_mapy, cv2.INTER_LINEAR)
src_points = np.stack([mapx.ravel(), mapy.ravel()], axis=-1)
dst_points = np.mgrid[0:65:16, 0:65:16].T.reshape(-1, 2)
mat = umeyama(src_points, dst_points, True)[0:2]
target_image = cv2.warpAffine(image, mat, (64, 64))
return warped_image, target_image
def random_warp_128(image):
assert image.shape == (256, 256, 3), 'resize image to 256 256 first'
range_ = np.linspace(128 - 120, 128 + 120, 9)
mapx = np.broadcast_to(range_, (9, 9))
mapy = mapx.T
mapx = mapx + np.random.normal(size=(9, 9), scale=5)
mapy = mapy + np.random.normal(size=(9, 9), scale=5)
interp_mapx = cv2.resize(mapx, (144, 144))[8:136, 8:136].astype('float32')
interp_mapy = cv2.resize(mapy, (144, 144))[8:136, 8:136].astype('float32')
warped_image = cv2.remap(image, interp_mapx, interp_mapy, cv2.INTER_LINEAR)
src_points = np.stack([mapx.ravel(), mapy.ravel()], axis=-1)
dst_points = np.mgrid[0:129:16, 0:129:16].T.reshape(-1, 2)
mat = umeyama(src_points, dst_points, True)[0:2]
target_image = cv2.warpAffine(image, mat, (128, 128))
return warped_image, target_image
def random_warp(image):
assert image.shape == (256, 256, 3)
range_ = numpy.linspace(128 - 80, 128 + 80, 5)
mapx = numpy.broadcast_to(range_, (5, 5))
mapy = mapx.T
mapx = mapx + numpy.random.normal(size=(5, 5), scale=5)
mapy = mapy + numpy.random.normal(size=(5, 5), scale=5)
interp_mapx = cv2.resize(mapx, (80, 80))[8:72, 8:72].astype('float32')
interp_mapy = cv2.resize(mapy, (80, 80))[8:72, 8:72].astype('float32')
# just crop the image, remove the top left bottom right 8 pixels (in order to get the pure face)
warped_image = cv2.remap(image, interp_mapx, interp_mapy, cv2.INTER_LINEAR)
src_points = numpy.stack([mapx.ravel(), mapy.ravel()], axis=-1)
dst_points = numpy.mgrid[0:65:16, 0:65:16].T.reshape(-1, 2)
mat = umeyama(src_points, dst_points, True)[0:2]
target_image = cv2.warpAffine(image, mat, (64, 64))
return warped_image, target_image
def init_guess(setting, data, use_torso=False, **kwargs):
model = setting['model']
dtype = setting['dtype']
keypoints = data['keypoints']
batch_size = setting['batch_size']
device = setting['device']
est_scale = not setting['fix_scale']
fixed_scale = 1. if setting['fixed_scale'] is None else setting[
'fixed_scale']
# reset model
init_t = torch.zeros((1, 3), dtype=dtype)
init_r = torch.zeros((1, 3), dtype=dtype)
init_s = torch.tensor(fixed_scale, dtype=dtype)
init_shape = torch.zeros((1, 10), dtype=dtype)
model.reset_params(transl=init_t,
global_orient=init_r,
scale=init_s,
betas=init_shape)
init_pose = torch.zeros((1, 69), dtype=dtype).cuda()
model_output = model(return_verts=True,
return_full_pose=True,
body_pose=init_pose)
verts = model_output.vertices[0]
if kwargs.get('model_type') == 'smpllsp':
J = torch.matmul(model.joint_regressor, verts)
else:
J = torch.matmul(model.J_regressor, verts)
verts = verts.unsqueeze(0)
J = J.unsqueeze(0)
joints = model.vertex_joint_selector(verts, J)
# Map the joints to the current dataset
if model.joint_mapper is not None:
joints = model.joint_mapper(joints).detach().cpu().numpy()[0]
if len(keypoints) == 1:
# guess depth for single-view input
# 5 is L shoulder, 11 is L hip
# 6 is R shoulder, 12 is R hip
torso3d = joints[[5, 6, 11, 12]]
torso2d = keypoints[0][0][[5, 6, 11, 12]]
torso3d = np.insert(torso3d, 3, 1, axis=1).T
torso3d = (np.dot(setting['extris'][0], torso3d).T)[:, :3]
diff3d = np.array([torso3d[0] - torso3d[2], torso3d[1] - torso3d[3]])
mean_height3d = np.mean(np.sqrt(np.sum(diff3d**2, axis=1)))
diff2d = np.array([torso2d[0] - torso2d[2], torso2d[0] - torso2d[2]])
mean_height2d = np.mean(np.sqrt(np.sum(diff2d**2, axis=1)))
est_d = setting['intris'][0][0][0] * (mean_height3d / mean_height2d)
# just set the z value
cam_joints = np.dot(setting['extris'][0],
np.insert(joints.copy(), 3, 1, axis=1).T)
cam_joints[2, :] += est_d
joints3d = (np.dot(np.linalg.inv(setting['extris'][0]),
cam_joints).T)[:, :3]
# trans = cal_trans(camcoord, keypoints[0][0][[5,6,11,12]], setting['intris'][0])
# trans = np.dot(np.linalg.inv(setting['extris'][0]), np.insert(trans.reshape(3,1), 3, 1, axis=0)).reshape(1,-1)[:,:3]
# joints3d = joints + trans
else:
joints3d = recompute3D(setting['extris'], setting['intris'], keypoints)
if kwargs.get('use_3d') and data['3d_joint'] is not None:
joints3d = data['3d_joint'][0][:, :3]
if use_torso:
joints3d = joints3d[[5, 6, 11, 12]]
joints = joints[[5, 6, 11, 12]]
# get transformation
rot, trans, scale = umeyama(joints, joints3d, est_scale)
rot = cv2.Rodrigues(rot)[0]
# apply to model
if est_scale:
init_s = torch.tensor(scale, dtype=dtype)
else:
init_s = torch.tensor(fixed_scale, dtype=dtype)
init_t = torch.tensor(trans, dtype=dtype)
init_r = torch.tensor(rot, dtype=dtype).reshape(1, 3)
model.reset_params(transl=init_t, global_orient=init_r, scale=init_s)
if kwargs.get('use_vposer'):
with torch.no_grad():
setting['pose_embedding'].fill_(0)
# # load fixed parameters
# init_s = torch.tensor(7., dtype=dtype)
# init_shape = torch.tensor([2.39806, 0.678491, -1.38193, -0.966748, -1.29383,-0.795755, -0.303195, -1.1032, -0.197056, -0.102728 ], dtype=dtype)
# model.reset_params(transl=init_t, global_orient=init_r, scale=init_s)
# model.betas.requires_grad = False
# model.scale.requires_grad = False
# visualize
if False:
if kwargs.get('use_vposer'):
vposer = setting['vposer']
init_pose = vposer.decode(setting['pose_embedding'],
output_type='aa').view(1, -1)
else:
init_pose = torch.zeros((1, 69), dtype=dtype).cuda()
model_output = model(return_verts=True,
return_full_pose=True,
body_pose=init_pose)
joints = model_output.joints.detach().cpu().numpy()[0]
verts = model_output.vertices.detach().cpu().numpy()[0]
from utils.utils import joint_projection, surface_projection
for i in range(1):
joint_projection(joints3d, setting['extris'][i],
setting['intris'][i], data['img'][i][:, :, ::-1],
True)
surface_projection(verts, model.faces, joints,
setting['extris'][i], setting['intris'][i],
data['img'][i][:, :, ::-1], 5)