def align_face(im, src, size): # Refer to: # https://github.com/deepinsight/insightface/blob/master/src/common/face_preprocess.py#L46 dst = np.array( [[30.2946, 51.6963], [65.5318, 51.5014], [48.0252, 71.7366], [33.5493, 92.3655], [62.7299, 92.2041]], dtype=np.float32) dst[:, 0] += 8.0 M = umeyama(src, dst, True)[0:2] warped = cv2.warpAffine(im, M, (size, size), borderValue=0.0) return warped
def init_guess(setting, data, use_torso=False, **kwargs): model = setting['model'] dtype = setting['dtype'] keypoints = data['keypoints'] batch_size = setting['batch_size'] device = setting['device'] est_scale = not setting['fix_scale'] fixed_scale = 1. if setting['fixed_scale'] is None else setting['fixed_scale'] joints3d = recompute3D(setting['extris'], setting['intris'], keypoints) # reset model init_t = torch.zeros((1,3), dtype=dtype) init_r = torch.zeros((1,3), dtype=dtype) init_s = torch.tensor(fixed_scale, dtype=dtype) init_shape = torch.zeros((1,10), dtype=dtype) model.reset_params(transl=init_t, global_orient=init_r, scale=init_s, betas=init_shape) init_pose = torch.zeros((1,69), dtype=dtype).cuda() model_output = model(return_verts=True, return_full_pose=True, body_pose=init_pose) verts = model_output.vertices[0] if kwargs.get('model_type') == 'smpllsp': J = torch.matmul(model.joint_regressor, verts) else: J = torch.matmul(model.J_regressor, verts) verts = verts.unsqueeze(0) J = J.unsqueeze(0) joints = model.vertex_joint_selector(verts, J) # Map the joints to the current dataset if model.joint_mapper is not None: joints = model.joint_mapper(joints).detach().cpu().numpy()[0] if use_torso: joints3d = joints3d[[5,6,11,12]] joints = joints[[5,6,11,12]] # get transformation rot, trans, scale = umeyama(joints, joints3d, est_scale) rot = cv2.Rodrigues(rot)[0] # apply to model if est_scale: init_s = torch.tensor(scale, dtype=dtype) else: init_s = torch.tensor(fixed_scale, dtype=dtype) init_t = torch.tensor(trans, dtype=dtype) init_r = torch.tensor(rot, dtype=dtype).reshape(1,3) model.reset_params(transl=init_t, global_orient=init_r, scale=init_s) if kwargs.get('use_vposer'): with torch.no_grad(): setting['pose_embedding'].fill_(0)
def random_warp_64(image): assert image.shape == (256, 256, 3) range_ = np.linspace(128 - 120, 128 + 120, 5) mapx = np.broadcast_to(range_, (5, 5)) mapy = mapx.T mapx = mapx + np.random.normal(size=(5, 5), scale=5) mapy = mapy + np.random.normal(size=(5, 5), scale=5) interp_mapx = cv2.resize(mapx, (80, 80))[8:72, 8:72].astype('float32') interp_mapy = cv2.resize(mapy, (80, 80))[8:72, 8:72].astype('float32') warped_image = cv2.remap(image, interp_mapx, interp_mapy, cv2.INTER_LINEAR) src_points = np.stack([mapx.ravel(), mapy.ravel()], axis=-1) dst_points = np.mgrid[0:65:16, 0:65:16].T.reshape(-1, 2) mat = umeyama(src_points, dst_points, True)[0:2] target_image = cv2.warpAffine(image, mat, (64, 64)) return warped_image, target_image
def random_warp_128(image): assert image.shape == (256, 256, 3), 'resize image to 256 256 first' range_ = np.linspace(128 - 120, 128 + 120, 9) mapx = np.broadcast_to(range_, (9, 9)) mapy = mapx.T mapx = mapx + np.random.normal(size=(9, 9), scale=5) mapy = mapy + np.random.normal(size=(9, 9), scale=5) interp_mapx = cv2.resize(mapx, (144, 144))[8:136, 8:136].astype('float32') interp_mapy = cv2.resize(mapy, (144, 144))[8:136, 8:136].astype('float32') warped_image = cv2.remap(image, interp_mapx, interp_mapy, cv2.INTER_LINEAR) src_points = np.stack([mapx.ravel(), mapy.ravel()], axis=-1) dst_points = np.mgrid[0:129:16, 0:129:16].T.reshape(-1, 2) mat = umeyama(src_points, dst_points, True)[0:2] target_image = cv2.warpAffine(image, mat, (128, 128)) return warped_image, target_image
def random_warp(image): assert image.shape == (256, 256, 3) range_ = numpy.linspace(128 - 80, 128 + 80, 5) mapx = numpy.broadcast_to(range_, (5, 5)) mapy = mapx.T mapx = mapx + numpy.random.normal(size=(5, 5), scale=5) mapy = mapy + numpy.random.normal(size=(5, 5), scale=5) interp_mapx = cv2.resize(mapx, (80, 80))[8:72, 8:72].astype('float32') interp_mapy = cv2.resize(mapy, (80, 80))[8:72, 8:72].astype('float32') # just crop the image, remove the top left bottom right 8 pixels (in order to get the pure face) warped_image = cv2.remap(image, interp_mapx, interp_mapy, cv2.INTER_LINEAR) src_points = numpy.stack([mapx.ravel(), mapy.ravel()], axis=-1) dst_points = numpy.mgrid[0:65:16, 0:65:16].T.reshape(-1, 2) mat = umeyama(src_points, dst_points, True)[0:2] target_image = cv2.warpAffine(image, mat, (64, 64)) return warped_image, target_image
def init_guess(setting, data, use_torso=False, **kwargs): model = setting['model'] dtype = setting['dtype'] keypoints = data['keypoints'] batch_size = setting['batch_size'] device = setting['device'] est_scale = not setting['fix_scale'] fixed_scale = 1. if setting['fixed_scale'] is None else setting[ 'fixed_scale'] # reset model init_t = torch.zeros((1, 3), dtype=dtype) init_r = torch.zeros((1, 3), dtype=dtype) init_s = torch.tensor(fixed_scale, dtype=dtype) init_shape = torch.zeros((1, 10), dtype=dtype) model.reset_params(transl=init_t, global_orient=init_r, scale=init_s, betas=init_shape) init_pose = torch.zeros((1, 69), dtype=dtype).cuda() model_output = model(return_verts=True, return_full_pose=True, body_pose=init_pose) verts = model_output.vertices[0] if kwargs.get('model_type') == 'smpllsp': J = torch.matmul(model.joint_regressor, verts) else: J = torch.matmul(model.J_regressor, verts) verts = verts.unsqueeze(0) J = J.unsqueeze(0) joints = model.vertex_joint_selector(verts, J) # Map the joints to the current dataset if model.joint_mapper is not None: joints = model.joint_mapper(joints).detach().cpu().numpy()[0] if len(keypoints) == 1: # guess depth for single-view input # 5 is L shoulder, 11 is L hip # 6 is R shoulder, 12 is R hip torso3d = joints[[5, 6, 11, 12]] torso2d = keypoints[0][0][[5, 6, 11, 12]] torso3d = np.insert(torso3d, 3, 1, axis=1).T torso3d = (np.dot(setting['extris'][0], torso3d).T)[:, :3] diff3d = np.array([torso3d[0] - torso3d[2], torso3d[1] - torso3d[3]]) mean_height3d = np.mean(np.sqrt(np.sum(diff3d**2, axis=1))) diff2d = np.array([torso2d[0] - torso2d[2], torso2d[0] - torso2d[2]]) mean_height2d = np.mean(np.sqrt(np.sum(diff2d**2, axis=1))) est_d = setting['intris'][0][0][0] * (mean_height3d / mean_height2d) # just set the z value cam_joints = np.dot(setting['extris'][0], np.insert(joints.copy(), 3, 1, axis=1).T) cam_joints[2, :] += est_d joints3d = (np.dot(np.linalg.inv(setting['extris'][0]), cam_joints).T)[:, :3] # trans = cal_trans(camcoord, keypoints[0][0][[5,6,11,12]], setting['intris'][0]) # trans = np.dot(np.linalg.inv(setting['extris'][0]), np.insert(trans.reshape(3,1), 3, 1, axis=0)).reshape(1,-1)[:,:3] # joints3d = joints + trans else: joints3d = recompute3D(setting['extris'], setting['intris'], keypoints) if kwargs.get('use_3d') and data['3d_joint'] is not None: joints3d = data['3d_joint'][0][:, :3] if use_torso: joints3d = joints3d[[5, 6, 11, 12]] joints = joints[[5, 6, 11, 12]] # get transformation rot, trans, scale = umeyama(joints, joints3d, est_scale) rot = cv2.Rodrigues(rot)[0] # apply to model if est_scale: init_s = torch.tensor(scale, dtype=dtype) else: init_s = torch.tensor(fixed_scale, dtype=dtype) init_t = torch.tensor(trans, dtype=dtype) init_r = torch.tensor(rot, dtype=dtype).reshape(1, 3) model.reset_params(transl=init_t, global_orient=init_r, scale=init_s) if kwargs.get('use_vposer'): with torch.no_grad(): setting['pose_embedding'].fill_(0) # # load fixed parameters # init_s = torch.tensor(7., dtype=dtype) # init_shape = torch.tensor([2.39806, 0.678491, -1.38193, -0.966748, -1.29383,-0.795755, -0.303195, -1.1032, -0.197056, -0.102728 ], dtype=dtype) # model.reset_params(transl=init_t, global_orient=init_r, scale=init_s) # model.betas.requires_grad = False # model.scale.requires_grad = False # visualize if False: if kwargs.get('use_vposer'): vposer = setting['vposer'] init_pose = vposer.decode(setting['pose_embedding'], output_type='aa').view(1, -1) else: init_pose = torch.zeros((1, 69), dtype=dtype).cuda() model_output = model(return_verts=True, return_full_pose=True, body_pose=init_pose) joints = model_output.joints.detach().cpu().numpy()[0] verts = model_output.vertices.detach().cpu().numpy()[0] from utils.utils import joint_projection, surface_projection for i in range(1): joint_projection(joints3d, setting['extris'][i], setting['intris'][i], data['img'][i][:, :, ::-1], True) surface_projection(verts, model.faces, joints, setting['extris'][i], setting['intris'][i], data['img'][i][:, :, ::-1], 5)