def _load_tri(bfm_fp): if osp.split(bfm_fp)[-1] == 'bfm_noneck_v3.pkl': tri = _load(make_abs_path('../configs/tri.pkl')) # this tri/face is re-built for bfm_noneck_v3 else: tri = _load(bfm_fp).get('tri') tri = _to_ctype(tri.T).astype(np.int32) return tri
def __init__(self, **kvs): # torch.set_grad_enabled(False) # config self.gpu_mode = kvs.get('gpu_mode', False) self.gpu_id = kvs.get('gpu_id', 0) self.size = kvs.get('size', 120) param_mean_std_fp = kvs.get( 'param_mean_std_fp', make_abs_path( f'configs/param_mean_std_62d_{self.size}x{self.size}.pkl')) onnx_fp = kvs.get('onnx_fp', kvs.get('checkpoint_fp').replace('.pth', '.onnx')) # convert to onnx online if not existed if onnx_fp is None or not osp.exists(onnx_fp): print( f'{onnx_fp} does not exist, try to convert the `.pth` version to `.onnx` online' ) onnx_fp = convert_to_onnx(**kvs) self.session = onnxruntime.InferenceSession(onnx_fp, None) # params normalization config r = _load(param_mean_std_fp) self.param_mean = r.get('mean') self.param_std = r.get('std')
def __init__(self, bfm_fp, shape_dim=40, exp_dim=10): bfm = _load(bfm_fp) self.u = bfm.get('u').astype(np.float32) # fix bug self.w_shp = bfm.get('w_shp').astype(np.float32)[..., :shape_dim] self.w_exp = bfm.get('w_exp').astype(np.float32)[..., :exp_dim] if osp.split(bfm_fp)[-1] == 'bfm_noneck_v3.pkl': self.tri = _load(make_abs_path('../configs/tri.pkl')) # this tri/face is re-built for bfm_noneck_v3 else: self.tri = bfm.get('tri') self.tri = _to_ctype(self.tri.T).astype(np.int32) self.keypoints = bfm.get('keypoints').astype(np.long) # fix bug w = np.concatenate((self.w_shp, self.w_exp), axis=1) self.w_norm = np.linalg.norm(w, axis=0) self.u_base = self.u[self.keypoints].reshape(-1, 1) self.w_shp_base = self.w_shp[self.keypoints] self.w_exp_base = self.w_exp[self.keypoints]
def __init__(self, bfm_fp): bfm = _load(bfm_fp) self.u = bfm.get('u').astype(np.float32) # fix bug self.w_shp = bfm.get('w_shp').astype(np.float32) self.w_exp = bfm.get('w_exp').astype(np.float32) self.tri = bfm.get('tri') self.keypoints = bfm.get('keypoints').astype(np.long) # fix bug w = np.concatenate((self.w_shp, self.w_exp), axis=1) self.w_norm = np.linalg.norm(w, axis=0) self.u_base = self.u[self.keypoints].reshape(-1, 1) self.w_shp_base = self.w_shp[self.keypoints] self.w_exp_base = self.w_exp[self.keypoints]
def __init__(self, bfm_fp, shape_dim=40, exp_dim=10): super(BFMModel_ONNX, self).__init__() _to_tensor = _numpy_to_tensor # load bfm bfm = _load(bfm_fp) u = _to_tensor(bfm.get('u').astype(np.float32)) self.u = u.view(-1, 3).transpose(1, 0) w_shp = _to_tensor(bfm.get('w_shp').astype(np.float32)[..., :shape_dim]) w_exp = _to_tensor(bfm.get('w_exp').astype(np.float32)[..., :exp_dim]) w = torch.cat((w_shp, w_exp), dim=1) self.w = w.view(-1, 3, w.shape[-1]).contiguous().permute(1, 0, 2)
def __init__(self, **kvs): torch.set_grad_enabled(False) # load BFM self.bfm = BFMModel(bfm_fp=kvs.get( 'bfm_fp', make_abs_path('configs/bfm_noneck_v3.pkl')), shape_dim=kvs.get('shape_dim', 40), exp_dim=kvs.get('exp_dim', 10)) self.tri = self.bfm.tri # config self.gpu_mode = kvs.get('gpu_mode', False) self.gpu_id = kvs.get('gpu_id', 0) self.size = kvs.get('size', 120) if self.gpu_mode: print(f"Loading 3DDFA_v2 to GPU (id={self.gpu_id})...") param_mean_std_fp = kvs.get( 'param_mean_std_fp', make_abs_path( f'configs/param_mean_std_62d_{self.size}x{self.size}.pkl')) # load model, default output is dimension with length 62 = 12(pose) + 40(shape) +10(expression) model = getattr(models, kvs.get('arch'))(num_classes=kvs.get('num_params', 62), widen_factor=kvs.get( 'widen_factor', 1), size=self.size, mode=kvs.get('mode', 'small')) model = load_model(model, kvs.get('checkpoint_fp')) if self.gpu_mode: print("Loading 3DDFA_v2 model to GPU...") cudnn.benchmark = True model = model.cuda(device=self.gpu_id) self.model = model self.model.eval() # eval mode, fix BN # data normalization transform_normalize = NormalizeGjz(mean=127.5, std=128) transform_to_tensor = ToTensorGjz() transform = Compose([transform_to_tensor, transform_normalize]) self.transform = transform # params normalization config r = _load(param_mean_std_fp) self.param_mean = r.get('mean') self.param_std = r.get('std')
def gen_3d_vertex(): filelists = 'test.data/AFLW2000-3D_crop.list' root = 'AFLW-2000-3D/' fns = open(filelists).read().strip().split('\n') params = _load('res/params_aflw2000.npy') sel = ['00427', '00439', '00475', '00477', '00497', '00514', '00562', '00623', '01045', '01095', '01104', '01506', '01621', '02214', '02244', '03906', '04157'] sel = list(map(lambda x: f'image{x}.jpg', sel)) for i in range(2000): fn = fns[i] if fn in sel: vertex = reconstruct_vertex(params[i], dense=True) wfp = osp.join('res/AFLW-2000-3D_vertex/', fn.replace('.jpg', '.mat')) print(wfp) sio.savemat(wfp, {'vertex': vertex})
def __init__(self, **kvs): # torch.set_grad_enabled(False) # load onnx version of BFM bfm_fp = kvs.get('bfm_fp', make_abs_path(__file__, 'configs/bfm_noneck_v3.pkl')) bfm_onnx_fp = bfm_fp.replace('.pkl', '.onnx') if not osp.exists(bfm_onnx_fp): convert_bfm_to_onnx(bfm_onnx_fp, shape_dim=kvs.get('shape_dim', 40), exp_dim=kvs.get('exp_dim', 10)) self.bfm_session = onnxruntime.InferenceSession(bfm_onnx_fp, None) # load for optimization bfm = BFMModel(bfm_fp, shape_dim=kvs.get('shape_dim', 40), exp_dim=kvs.get('exp_dim', 10)) self.tri = bfm.tri self.u_base, self.w_shp_base, self.w_exp_base = bfm.u_base, bfm.w_shp_base, bfm.w_exp_base # config self.gpu_mode = kvs.get('gpu_mode', False) self.gpu_id = kvs.get('gpu_id', 0) self.size = kvs.get('size', 120) param_mean_std_fp = kvs.get( 'param_mean_std_fp', make_abs_path( __file__, f'configs/param_mean_std_62d_{self.size}x{self.size}.pkl')) onnx_fp = kvs.get('onnx_fp', kvs.get('checkpoint_fp').replace('.pth', '.onnx')) # convert to onnx online if not existed if onnx_fp is None or not osp.exists(onnx_fp): print( f'{onnx_fp} does not exist, try to convert the `.pth` version to `.onnx` online' ) onnx_fp = convert_to_onnx(**kvs) self.session = onnxruntime.InferenceSession(onnx_fp, None) # params normalization config r = _load(param_mean_std_fp) self.param_mean = r.get('mean') self.param_std = r.get('std')
def pncc(img, ver_lst, show_flag=False, wfp=None, with_bg_flag=True): ncc_code = _load(make_abs_path('../configs/ncc_code.npy')) if with_bg_flag: overlap = img.copy() else: overlap = np.zeros_like(img) # rendering pncc for ver_ in ver_lst: ver = _to_ctype(ver_.T) # transpose overlap = rasterize(ver, tri, ncc_code.T, bg=overlap) # m x 3 if wfp is not None: cv2.imwrite(wfp, overlap) print(f'Save visualization result to {wfp}') if show_flag: plot_image(overlap) return overlap
def __init__(self, **kvs): torch.set_grad_enabled(False) # config self.gpu_mode = kvs.get('gpu_mode', False) self.gpu_id = kvs.get('gpu_id', 0) self.size = kvs.get('size', 120) param_mean_std_fp = kvs.get( 'param_mean_std_fp', make_abs_path( f'configs/param_mean_std_62d_{self.size}x{self.size}.pkl')) # load model, 62 = 12(pose) + 40(shape) +10(expression) model = getattr(models, kvs.get('arch'))(num_classes=kvs.get('num_params', 62), widen_factor=kvs.get( 'widen_factor', 1), size=self.size, mode=kvs.get('mode', 'small')) model = load_model(model, kvs.get('checkpoint_fp')) if self.gpu_mode: cudnn.benchmark = True model = model.cuda(device=self.gpu_id) self.model = model self.model.eval() # eval mode, fix BN # data normalization transform_normalize = NormalizeGjz(mean=127.5, std=128) transform_to_tensor = ToTensorGjz() transform = Compose([transform_to_tensor, transform_normalize]) self.transform = transform # params normalization config r = _load(param_mean_std_fp) self.param_mean = r.get('mean') self.param_std = r.get('std')
uv_coords = C['UV'].copy(order='C').astype(np.float32) return uv_coords def process_uv(uv_coords, uv_h=256, uv_w=256): uv_coords[:, 0] = uv_coords[:, 0] * (uv_w - 1) uv_coords[:, 1] = uv_coords[:, 1] * (uv_h - 1) uv_coords[:, 1] = uv_h - uv_coords[:, 1] - 1 uv_coords = np.hstack( (uv_coords, np.zeros((uv_coords.shape[0], 1), dtype=np.float32))) # add z return uv_coords g_uv_coords = load_uv_coords(make_abs_path('../configs/BFM_UV.mat')) indices = _load(make_abs_path('../configs/indices.npy')) g_uv_coords = g_uv_coords[indices, :] def get_colors(img, ver): # nearest-neighbor sampling [h, w, _] = img.shape ver[0, :] = np.minimum(np.maximum(ver[0, :], 0), w - 1) # x ver[1, :] = np.minimum(np.maximum(ver[1, :], 0), h - 1) # y ind = np.round(ver).astype(np.int32) colors = img[ind[1, :], ind[0, :], :] # n x 3 return colors def bilinear_interpolate(img, x, y):
""" Notation (2019.09.15): two versions of spliting AFLW2000-3D: 1) AFLW2000-3D.pose.npy: according to the fitted pose 2) AFLW2000-3D-new.pose: according to AFLW labels There is no obvious difference between these two splits. """ import os.path as osp import numpy as np from math import sqrt from utils.io import _load d = 'test.configs' # [1312, 383, 305], current version yaws_list = _load(osp.join(d, 'AFLW2000-3D.pose.npy')) # [1306, 462, 232], same as paper # yaws_list = _load(osp.join(d, 'AFLW2000-3D-new.pose.npy')) # origin pts68_all_ori = _load(osp.join(d, 'AFLW2000-3D.pts68.npy')) # reannonated pts68_all_re = _load(osp.join(d, 'AFLW2000-3D-Reannotated.pts68.npy')) roi_boxs = _load(osp.join(d, 'AFLW2000-3D_crop.roi_box.npy')) def ana(nme_list): yaw_list_abs = np.abs(yaws_list) ind_yaw_1 = yaw_list_abs <= 30
#!/usr/bin/env python3 # coding: utf-8 import os.path as osp import numpy as np from math import sqrt from utils.io import _load d = 'test.configs' yaw_list = _load(osp.join(d, 'AFLW_GT_crop_yaws.npy')) roi_boxs = _load(osp.join(d, 'AFLW_GT_crop_roi_box.npy')) pts68_all = _load(osp.join(d, 'AFLW_GT_pts68.npy')) pts21_all = _load(osp.join(d, 'AFLW_GT_pts21.npy')) def ana(nme_list): yaw_list_abs = np.abs(yaw_list) ind_yaw_1 = yaw_list_abs <= 30 ind_yaw_2 = np.bitwise_and(yaw_list_abs > 30, yaw_list_abs <= 60) ind_yaw_3 = yaw_list_abs > 60 nme_1 = nme_list[ind_yaw_1] nme_2 = nme_list[ind_yaw_2] nme_3 = nme_list[ind_yaw_3] mean_nme_1 = np.mean(nme_1) * 100 mean_nme_2 = np.mean(nme_2) * 100 mean_nme_3 = np.mean(nme_3) * 100 # mean_nme_all = np.mean(nme_list) * 100 std_nme_1 = np.std(nme_1) * 100
uv_coords = C['UV'].copy(order='C').astype(np.float32) return uv_coords def process_uv(uv_coords, uv_h=256, uv_w=256): uv_coords[:, 0] = uv_coords[:, 0] * (uv_w - 1) uv_coords[:, 1] = uv_coords[:, 1] * (uv_h - 1) uv_coords[:, 1] = uv_h - uv_coords[:, 1] - 1 uv_coords = np.hstack( (uv_coords, np.zeros((uv_coords.shape[0], 1), dtype=np.float32))) # add z return uv_coords g_uv_coords = load_uv_coords(make_abs_path('../configs/BFM_UV.mat')) indices = _load( make_abs_path('../configs/indices.npy')) # todo: handle bfm_slim g_uv_coords = g_uv_coords[indices, :] def get_colors(img, ver): # nearest-neighbor sampling [h, w, _] = img.shape ver[0, :] = np.minimum(np.maximum(ver[0, :], 0), w - 1) # x ver[1, :] = np.minimum(np.maximum(ver[1, :], 0), h - 1) # y ind = np.round(ver).astype(np.int32) colors = img[ind[1, :], ind[0, :], :] # n x 3 return colors def bilinear_interpolate(img, x, y):
import sys sys.path.append('..') import os.path as osp import numpy as np from utils.io import _load make_abs_path = lambda fn: osp.join(osp.dirname(osp.realpath(__file__)), fn) class BFMModel(object): def __init__(self, bfm_fp): bfm = _load(bfm_fp) self.u = bfm.get('u').astype(np.float32) # fix bug self.w_shp = bfm.get('w_shp').astype(np.float32) self.w_exp = bfm.get('w_exp').astype(np.float32) self.tri = bfm.get('tri') self.keypoints = bfm.get('keypoints').astype(np.long) # fix bug w = np.concatenate((self.w_shp, self.w_exp), axis=1) self.w_norm = np.linalg.norm(w, axis=0) self.u_base = self.u[self.keypoints].reshape(-1, 1) self.w_shp_base = self.w_shp[self.keypoints] self.w_exp_base = self.w_exp[self.keypoints] cfg_path = make_abs_path('../configs') bfm = BFMModel(osp.join(cfg_path, 'bfm_noneck_v3.pkl')) tri = _load(osp.join(cfg_path, 'tri.pkl'))
sys.path.append('..') import os.path as osp import numpy as np from utils.io import _load make_abs_path = lambda fn: osp.join(osp.dirname(osp.realpath(__file__)), fn) class BFMModel(object): def __init__(self, bfm_fp): bfm = _load(bfm_fp) self.u = bfm.get('u').astype(np.float32) # fix bug self.w_shp = bfm.get('w_shp').astype(np.float32) self.w_exp = bfm.get('w_exp').astype(np.float32) self.tri = bfm.get('tri') self.keypoints = bfm.get('keypoints').astype(np.long) # fix bug w = np.concatenate((self.w_shp, self.w_exp), axis=1) self.w_norm = np.linalg.norm(w, axis=0) self.u_base = self.u[self.keypoints].reshape(-1, 1) self.w_shp_base = self.w_shp[self.keypoints] self.w_exp_base = self.w_exp[self.keypoints] cfg_path = make_abs_path('../configs') bfm = BFMModel(osp.join( cfg_path, 'bfm_noneck_v3.pkl')) # you can change the bfm pkl path tri = _load(osp.join(cfg_path, 'tri.pkl')) # this tri is re-built for bfm_noneck_v3
def draw_landmarks(): filelists = 'test.data/AFLW2000-3D_crop.list' root = 'AFLW-2000-3D/' fns = open(filelists).read().strip().split('\n') params = _load('res/params_aflw2000.npy') for i in range(2000): plt.close() img_fp = osp.join(root, fns[i]) img = io.imread(img_fp) lms = reconstruct_vertex(params[i], dense=False) lms = convert_to_ori(lms, i) # print(lms.shape) fig = plt.figure(figsize=plt.figaspect(.5)) # fig = plt.figure(figsize=(8, 4)) ax = fig.add_subplot(1, 2, 1) ax.imshow(img) alpha = 0.8 markersize = 4 lw = 1.5 color = 'w' markeredgecolor = 'black' nums = [0, 17, 22, 27, 31, 36, 42, 48, 60, 68] for ind in range(len(nums) - 1): l, r = nums[ind], nums[ind + 1] ax.plot(lms[0, l:r], lms[1, l:r], color=color, lw=lw, alpha=alpha - 0.1) ax.plot(lms[0, l:r], lms[1, l:r], marker='o', linestyle='None', markersize=markersize, color=color, markeredgecolor=markeredgecolor, alpha=alpha) ax.axis('off') # 3D ax = fig.add_subplot(1, 2, 2, projection='3d') lms[1] = img.shape[1] - lms[1] lms[2] = -lms[2] # print(lms) ax.scatter(lms[0], lms[2], lms[1], c="cyan", alpha=1.0, edgecolor='b') for ind in range(len(nums) - 1): l, r = nums[ind], nums[ind + 1] ax.plot3D(lms[0, l:r], lms[2, l:r], lms[1, l:r], color='blue') ax.view_init(elev=5., azim=-95) # ax.set_xlabel('x') # ax.set_ylabel('y') # ax.set_zlabel('z') ax.set_xticklabels([]) ax.set_yticklabels([]) ax.set_zticklabels([]) plt.tight_layout() # plt.show() wfp = f'res/AFLW-2000-3D/{osp.basename(img_fp)}' plt.savefig(wfp, dpi=200)