def load_vertices_with_colors(fp, face_boxes, tddfa): img_orig = cv2.imread(fp) box = face_boxes(img_orig)[0] param_lst, roi_box_lst = tddfa(img_orig, [box]) param = param_lst[0] roi_box = roi_box_lst[0] _, _, alpha_shp, alpha_exp = _parse_param(param) ver = recon_dense(param, roi_box, tddfa.size) color = get_colors(img_orig, ver) return ver, color, param
def recon_vers(self, param_lst, roi_box_lst, **kvs): dense_flag = kvs.get('dense_flag', False) size = self.size ver_lst = [] for param, roi_box in zip(param_lst, roi_box_lst): if dense_flag: R, offset, alpha_shp, alpha_exp = _parse_param(param) pts3d = R @ (self.bfm.u + self.bfm.w_shp @ alpha_shp + self.bfm.w_exp @ alpha_exp). \ reshape(3, -1, order='F') + offset pts3d = similar_transform(pts3d, roi_box, size) else: R, offset, alpha_shp, alpha_exp = _parse_param(param) pts3d = R @ (self.bfm.u_base + self.bfm.w_shp_base @ alpha_shp + self.bfm.w_exp_base @ alpha_exp). \ reshape(3, -1, order='F') + offset pts3d = similar_transform(pts3d, roi_box, size) ver_lst.append(pts3d) return ver_lst
def recon_vers(self, param_lst, roi_box_lst, **kvs): dense_flag = kvs.get('dense_flag', False) size = self.size ver_lst = [] for param, roi_box in zip(param_lst, roi_box_lst): R, offset, alpha_shp, alpha_exp = _parse_param(param) if dense_flag: inp_dct = { 'R': R, 'offset': offset, 'alpha_shp': alpha_shp, 'alpha_exp': alpha_exp } pts3d = self.bfm_session.run(None, inp_dct)[0] pts3d = similar_transform(pts3d, roi_box, size) else: pts3d = R @ (self.u_base + self.w_shp_base @ alpha_shp + self.w_exp_base @ alpha_exp). \ reshape(3, -1, order='F') + offset pts3d = similar_transform(pts3d, roi_box, size) ver_lst.append(pts3d) return ver_lst
def main(args): # Init TDDFA or TDDFA_ONNX cfg = yaml.load(open(args.config), Loader=yaml.SafeLoader) if args.onnx: from TDDFA_ONNX import TDDFA_ONNX tddfa = TDDFA_ONNX(**cfg) else: gpu_mode = args.mode == 'gpu' tddfa = TDDFA(gpu_mode=gpu_mode, **cfg) # Initialize FaceBoxes face_boxes = FaceBoxes() # Given a video path fn = args.video_fp.split('/')[-1] reader = imageio.get_reader(args.video_fp) fps = reader.get_meta_data()['fps'] dense_flag = args.opt in ('3d', ) nick_orig = cv2.imread('Assets4FacePaper/nick.bmp') tv_neutral, tc_neutral, tv_set, tc_set, nick_param = load_vertices_set( 'Assets4FacePaper', [ 'nick.bmp', 'nick_crop_1.jpg', 'nick_crop_2.jpg', 'nick_crop_3.jpg', 'nick_crop_4.jpg', 'nick_crop_5.jpg', 'nick_crop_6.png' ], face_boxes, tddfa) sv_neutral, sc_neutral, sv_set, sc_set, thanh_param = load_vertices_set( 'Assets4FacePaper', [ 'Thanh.jpg', 'Thanh1.jpg', 'Thanh2.jpg', 'Thanh3.jpg', 'Thanh4.jpg', 'Thanh5.jpg', 'Thanh6.jpg' ], face_boxes, tddfa) _, _, nick_alpha_shp, nick_alpha_exp = _parse_param(nick_param) nick_act_masks = calc_activation_masks(tv_neutral, tv_set) delta_set_nick = get_delta_vertices_set(tv_neutral, tv_set) _, _, thanh_alpha_shp, thanh_alpha_exp = _parse_param(thanh_param) thanh_act_masks = calc_activation_masks(sv_neutral, sv_set) delta_set_thanh = get_delta_vertices_set(sv_neutral, sv_set) nick_ver = tc_neutral nick_color = tc_neutral suffix = get_suffix(args.video_fp) video_wfp = f'examples/results/videos/{fn.replace(suffix, "")}_{args.opt}.mp4' N = list(tv_neutral.size())[0] E = list(tv_set.size())[0] # run pre_ver = None total_timer = Timer() optimizing_timer = Timer() for i, frame in tqdm(enumerate(reader)): total_timer.tic() frame_bgr = frame[..., ::-1] # RGB->BGR if i == 0: # the first frame, detect face, here we only use the first face, you can change depending on your need boxes = face_boxes(frame_bgr) boxes = [boxes[0]] param_lst, roi_box_lst = tddfa(frame_bgr, boxes) ver = tddfa.recon_vers(param_lst, roi_box_lst, dense_flag=dense_flag)[0] # refine param_lst, roi_box_lst = tddfa(frame_bgr, [ver], crop_policy='landmark') R, offset, alpha_shp, alpha_exp = _parse_param(param_lst[0]) roi_box = roi_box_lst[0] ver = recon_dense_explicit(R, offset, nick_alpha_shp, alpha_exp, roi_box, tddfa.size) ver_base = recon_dense_base(nick_alpha_shp, alpha_exp) else: param_lst, roi_box_lst = tddfa(frame_bgr, [pre_ver], crop_policy='landmark') roi_box = roi_box_lst[0] # todo: add confidence threshold to judge the tracking is failed if abs(roi_box[2] - roi_box[0]) * abs(roi_box[3] - roi_box[1]) < 2020: boxes = face_boxes(frame_bgr) boxes = [boxes[0]] param_lst, roi_box_lst = tddfa(frame_bgr, boxes) R, offset, alpha_shp, alpha_exp = _parse_param(param_lst[0]) ver_base = recon_dense_base(nick_alpha_shp, alpha_exp) ver = recon_dense_explicit(R, offset, nick_alpha_shp, alpha_exp, roi_box, tddfa.size) # Write object pre_ver = ver # for tracking ver = torch.from_numpy(ver_base.transpose()).float().to(device) optimizing_timer.tic() a_set = fit_coeffs_aio(ver, tv_neutral, delta_set_nick, 100) print(a_set) tver = torch.matmul(delta_set_nick.view(E, -1).transpose(0, 1), a_set).view(N, 3) + tv_neutral tver = transform_vertices(R, offset, tver.cpu().numpy().transpose(), roi_box, tddfa.size) # 3 N cver = calc_colors_w_act_mask(a_set, nick_act_masks, tc_neutral, tc_set).cpu().numpy() optimizing_time = optimizing_timer.toc() execution_time = total_timer.toc() print("Execution time: {}, Total time: {}".format( optimizing_time, execution_time)) ''' tver = torch.matmul(delta_set_thanh.view(E, -1).transpose(0, 1), a_set).view( N, 3) + sv_neutral tver = transform_vertices(R, offset, tver.numpy().transpose(), roi_box, tddfa.size) # 3 N cver = calc_colors_w_act_mask(a_set, thanh_act_masks, sc_neutral, sc_set).numpy() print(cver.shape) ''' ser_to_obj_multiple_colors(cver, [tver], height=int(nick_orig.shape[0] * 1.5), wfp='nick_' + str(i) + '.obj') print(f'Dump to nick_{str(i)}.obj') if i > 1000: break
def main(args): # Init TDDFA or TDDFA_ONNX cfg = yaml.load(open(args.config), Loader=yaml.SafeLoader) if args.onnx: from TDDFA_ONNX import TDDFA_ONNX tddfa = TDDFA_ONNX(**cfg) else: gpu_mode = args.mode == 'gpu' tddfa = TDDFA(gpu_mode=gpu_mode, **cfg) # Initialize FaceBoxes face_boxes = FaceBoxes() # Given a video path fn = args.video_fp.split('/')[-1] reader = imageio.get_reader(args.video_fp) fps = reader.get_meta_data()['fps'] dense_flag = args.opt in ('3d',) nick_orig = cv2.imread('Assets4FacePaper/nick.bmp') tv_neutral, tc_neutral, tv_set, tc_set, nick_param = load_vertices_set( 'Assets4FacePaper', ['sam_0_0.jpg', 'sam_0_1.jpg', 'sam_0_2.jpg', 'sam_0_3.jpg', 'sam_0_4.jpg', 'sam_0_5.jpg', 'sam_0_6.jpg'], face_boxes, tddfa ) sv_neutral, sc_neutral, sv_set, sc_set, thanh_param = load_vertices_set( 'Assets4FacePaper', ['Thanh.jpg', 'Thanh1.jpg', 'Thanh2.jpg', 'Thanh3.jpg', 'Thanh4.jpg', 'Thanh5.jpg', 'Thanh6.jpg'], face_boxes, tddfa ) _, _, nick_alpha_shp, nick_alpha_exp = _parse_param(nick_param) nick_act_masks = calc_activation_masks(tv_neutral, tv_set) delta_set_nick = get_delta_vertices_set(tv_neutral, tv_set) _, _, thanh_alpha_shp, thanh_alpha_exp = _parse_param(thanh_param) thanh_act_masks = calc_activation_masks(sv_neutral, sv_set) delta_set_thanh = get_delta_vertices_set(sv_neutral, sv_set) nick_ver = tc_neutral nick_color = tc_neutral suffix = get_suffix(args.video_fp) video_wfp = f'examples/results/videos/{fn.replace(suffix, "")}_{args.opt}.mp4' # run pre_ver = None for i, frame in tqdm(enumerate(reader)): frame_bgr = frame[..., ::-1] # RGB->BGR if i == 0: # the first frame, detect face, here we only use the first face, you can change depending on your need boxes = face_boxes(frame_bgr) boxes = [boxes[0]] param_lst, roi_box_lst = tddfa(frame_bgr, boxes) ver = tddfa.recon_vers(param_lst, roi_box_lst, dense_flag=dense_flag)[0] # refine param_lst, roi_box_lst = tddfa(frame_bgr, [ver], crop_policy='landmark') R, offset, _ , alpha_exp = _parse_param(param_lst[0]) ver = recon_dense_explicit(R, offset, nick_alpha_shp, alpha_exp, roi_box_lst[0], tddfa.size) else: param_lst, roi_box_lst = tddfa(frame_bgr, [pre_ver], crop_policy='landmark') roi_box = roi_box_lst[0] # todo: add confidence threshold to judge the tracking is failed if abs(roi_box[2] - roi_box[0]) * abs(roi_box[3] - roi_box[1]) < 2020: boxes = face_boxes(frame_bgr) boxes = [boxes[0]] param_lst, roi_box_lst = tddfa(frame_bgr, boxes) R, offset, _ , alpha_exp = _parse_param(param_lst[0]) ver = recon_dense_explicit(R, offset, nick_alpha_shp, alpha_exp, roi_box_lst[0], tddfa.size) # Write object pre_ver = ver # for tracking ser_to_obj_multiple_colors(nick_color, [ver], height=int(nick_orig.shape[0]*1.5), wfp='./sam_exp_only_obj/' + str(i) +'.obj') print(f'Dump to sam_exp_only_obj/{str(i)}.obj') if i > 1000: break