def load_vertices_with_colors(fp, face_boxes, tddfa):
img_orig = cv2.imread(fp)
box = face_boxes(img_orig)[0]
param_lst, roi_box_lst = tddfa(img_orig, [box])
param = param_lst[0]
roi_box = roi_box_lst[0]
_, _, alpha_shp, alpha_exp = _parse_param(param)
ver = recon_dense(param, roi_box, tddfa.size)
color = get_colors(img_orig, ver)
return ver, color, param
def recon_vers(self, param_lst, roi_box_lst, **kvs):
dense_flag = kvs.get('dense_flag', False)
size = self.size
ver_lst = []
for param, roi_box in zip(param_lst, roi_box_lst):
if dense_flag:
R, offset, alpha_shp, alpha_exp = _parse_param(param)
pts3d = R @ (self.bfm.u + self.bfm.w_shp @ alpha_shp + self.bfm.w_exp @ alpha_exp). \
reshape(3, -1, order='F') + offset
pts3d = similar_transform(pts3d, roi_box, size)
else:
R, offset, alpha_shp, alpha_exp = _parse_param(param)
pts3d = R @ (self.bfm.u_base + self.bfm.w_shp_base @ alpha_shp + self.bfm.w_exp_base @ alpha_exp). \
reshape(3, -1, order='F') + offset
pts3d = similar_transform(pts3d, roi_box, size)
ver_lst.append(pts3d)
return ver_lst
def recon_vers(self, param_lst, roi_box_lst, **kvs):
dense_flag = kvs.get('dense_flag', False)
size = self.size
ver_lst = []
for param, roi_box in zip(param_lst, roi_box_lst):
R, offset, alpha_shp, alpha_exp = _parse_param(param)
if dense_flag:
inp_dct = {
'R': R,
'offset': offset,
'alpha_shp': alpha_shp,
'alpha_exp': alpha_exp
}
pts3d = self.bfm_session.run(None, inp_dct)[0]
pts3d = similar_transform(pts3d, roi_box, size)
else:
pts3d = R @ (self.u_base + self.w_shp_base @ alpha_shp + self.w_exp_base @ alpha_exp). \
reshape(3, -1, order='F') + offset
pts3d = similar_transform(pts3d, roi_box, size)
ver_lst.append(pts3d)
return ver_lst
def main(args):
# Init TDDFA or TDDFA_ONNX
cfg = yaml.load(open(args.config), Loader=yaml.SafeLoader)
if args.onnx:
from TDDFA_ONNX import TDDFA_ONNX
tddfa = TDDFA_ONNX(**cfg)
else:
gpu_mode = args.mode == 'gpu'
tddfa = TDDFA(gpu_mode=gpu_mode, **cfg)
# Initialize FaceBoxes
face_boxes = FaceBoxes()
# Given a video path
fn = args.video_fp.split('/')[-1]
reader = imageio.get_reader(args.video_fp)
fps = reader.get_meta_data()['fps']
dense_flag = args.opt in ('3d', )
nick_orig = cv2.imread('Assets4FacePaper/nick.bmp')
tv_neutral, tc_neutral, tv_set, tc_set, nick_param = load_vertices_set(
'Assets4FacePaper', [
'nick.bmp', 'nick_crop_1.jpg', 'nick_crop_2.jpg',
'nick_crop_3.jpg', 'nick_crop_4.jpg', 'nick_crop_5.jpg',
'nick_crop_6.png'
], face_boxes, tddfa)
sv_neutral, sc_neutral, sv_set, sc_set, thanh_param = load_vertices_set(
'Assets4FacePaper', [
'Thanh.jpg', 'Thanh1.jpg', 'Thanh2.jpg', 'Thanh3.jpg',
'Thanh4.jpg', 'Thanh5.jpg', 'Thanh6.jpg'
], face_boxes, tddfa)
_, _, nick_alpha_shp, nick_alpha_exp = _parse_param(nick_param)
nick_act_masks = calc_activation_masks(tv_neutral, tv_set)
delta_set_nick = get_delta_vertices_set(tv_neutral, tv_set)
_, _, thanh_alpha_shp, thanh_alpha_exp = _parse_param(thanh_param)
thanh_act_masks = calc_activation_masks(sv_neutral, sv_set)
delta_set_thanh = get_delta_vertices_set(sv_neutral, sv_set)
nick_ver = tc_neutral
nick_color = tc_neutral
suffix = get_suffix(args.video_fp)
video_wfp = f'examples/results/videos/{fn.replace(suffix, "")}_{args.opt}.mp4'
N = list(tv_neutral.size())[0]
E = list(tv_set.size())[0]
# run
pre_ver = None
total_timer = Timer()
optimizing_timer = Timer()
for i, frame in tqdm(enumerate(reader)):
total_timer.tic()
frame_bgr = frame[..., ::-1] # RGB->BGR
if i == 0:
# the first frame, detect face, here we only use the first face, you can change depending on your need
boxes = face_boxes(frame_bgr)
boxes = [boxes[0]]
param_lst, roi_box_lst = tddfa(frame_bgr, boxes)
ver = tddfa.recon_vers(param_lst,
roi_box_lst,
dense_flag=dense_flag)[0]
# refine
param_lst, roi_box_lst = tddfa(frame_bgr, [ver],
crop_policy='landmark')
R, offset, alpha_shp, alpha_exp = _parse_param(param_lst[0])
roi_box = roi_box_lst[0]
ver = recon_dense_explicit(R, offset, nick_alpha_shp, alpha_exp,
roi_box, tddfa.size)
ver_base = recon_dense_base(nick_alpha_shp, alpha_exp)
else:
param_lst, roi_box_lst = tddfa(frame_bgr, [pre_ver],
crop_policy='landmark')
roi_box = roi_box_lst[0]
# todo: add confidence threshold to judge the tracking is failed
if abs(roi_box[2] - roi_box[0]) * abs(roi_box[3] -
roi_box[1]) < 2020:
boxes = face_boxes(frame_bgr)
boxes = [boxes[0]]
param_lst, roi_box_lst = tddfa(frame_bgr, boxes)
R, offset, alpha_shp, alpha_exp = _parse_param(param_lst[0])
ver_base = recon_dense_base(nick_alpha_shp, alpha_exp)
ver = recon_dense_explicit(R, offset, nick_alpha_shp, alpha_exp,
roi_box, tddfa.size)
# Write object
pre_ver = ver # for tracking
ver = torch.from_numpy(ver_base.transpose()).float().to(device)
optimizing_timer.tic()
a_set = fit_coeffs_aio(ver, tv_neutral, delta_set_nick, 100)
print(a_set)
tver = torch.matmul(delta_set_nick.view(E, -1).transpose(0, 1),
a_set).view(N, 3) + tv_neutral
tver = transform_vertices(R, offset,
tver.cpu().numpy().transpose(), roi_box,
tddfa.size) # 3 N
cver = calc_colors_w_act_mask(a_set, nick_act_masks, tc_neutral,
tc_set).cpu().numpy()
optimizing_time = optimizing_timer.toc()
execution_time = total_timer.toc()
print("Execution time: {}, Total time: {}".format(
optimizing_time, execution_time))
'''
tver = torch.matmul(delta_set_thanh.view(E, -1).transpose(0, 1), a_set).view(
N, 3) + sv_neutral
tver = transform_vertices(R, offset, tver.numpy().transpose(),
roi_box, tddfa.size) # 3 N
cver = calc_colors_w_act_mask(a_set, thanh_act_masks, sc_neutral,
sc_set).numpy()
print(cver.shape)
'''
ser_to_obj_multiple_colors(cver, [tver],
height=int(nick_orig.shape[0] * 1.5),
wfp='nick_' + str(i) + '.obj')
print(f'Dump to nick_{str(i)}.obj')
if i > 1000:
break
def main(args):
# Init TDDFA or TDDFA_ONNX
cfg = yaml.load(open(args.config), Loader=yaml.SafeLoader)
if args.onnx:
from TDDFA_ONNX import TDDFA_ONNX
tddfa = TDDFA_ONNX(**cfg)
else:
gpu_mode = args.mode == 'gpu'
tddfa = TDDFA(gpu_mode=gpu_mode, **cfg)
# Initialize FaceBoxes
face_boxes = FaceBoxes()
# Given a video path
fn = args.video_fp.split('/')[-1]
reader = imageio.get_reader(args.video_fp)
fps = reader.get_meta_data()['fps']
dense_flag = args.opt in ('3d',)
nick_orig = cv2.imread('Assets4FacePaper/nick.bmp')
tv_neutral, tc_neutral, tv_set, tc_set, nick_param = load_vertices_set(
'Assets4FacePaper',
['sam_0_0.jpg', 'sam_0_1.jpg', 'sam_0_2.jpg',
'sam_0_3.jpg', 'sam_0_4.jpg', 'sam_0_5.jpg',
'sam_0_6.jpg'], face_boxes, tddfa
)
sv_neutral, sc_neutral, sv_set, sc_set, thanh_param = load_vertices_set(
'Assets4FacePaper',
['Thanh.jpg', 'Thanh1.jpg', 'Thanh2.jpg',
'Thanh3.jpg', 'Thanh4.jpg', 'Thanh5.jpg',
'Thanh6.jpg'], face_boxes, tddfa
)
_, _, nick_alpha_shp, nick_alpha_exp = _parse_param(nick_param)
nick_act_masks = calc_activation_masks(tv_neutral, tv_set)
delta_set_nick = get_delta_vertices_set(tv_neutral, tv_set)
_, _, thanh_alpha_shp, thanh_alpha_exp = _parse_param(thanh_param)
thanh_act_masks = calc_activation_masks(sv_neutral, sv_set)
delta_set_thanh = get_delta_vertices_set(sv_neutral, sv_set)
nick_ver = tc_neutral
nick_color = tc_neutral
suffix = get_suffix(args.video_fp)
video_wfp = f'examples/results/videos/{fn.replace(suffix, "")}_{args.opt}.mp4'
# run
pre_ver = None
for i, frame in tqdm(enumerate(reader)):
frame_bgr = frame[..., ::-1] # RGB->BGR
if i == 0:
# the first frame, detect face, here we only use the first face, you can change depending on your need
boxes = face_boxes(frame_bgr)
boxes = [boxes[0]]
param_lst, roi_box_lst = tddfa(frame_bgr, boxes)
ver = tddfa.recon_vers(param_lst, roi_box_lst, dense_flag=dense_flag)[0]
# refine
param_lst, roi_box_lst = tddfa(frame_bgr, [ver],
crop_policy='landmark')
R, offset, _ , alpha_exp = _parse_param(param_lst[0])
ver = recon_dense_explicit(R, offset, nick_alpha_shp, alpha_exp,
roi_box_lst[0], tddfa.size)
else:
param_lst, roi_box_lst = tddfa(frame_bgr, [pre_ver],
crop_policy='landmark')
roi_box = roi_box_lst[0]
# todo: add confidence threshold to judge the tracking is failed
if abs(roi_box[2] - roi_box[0]) * abs(roi_box[3] - roi_box[1]) < 2020:
boxes = face_boxes(frame_bgr)
boxes = [boxes[0]]
param_lst, roi_box_lst = tddfa(frame_bgr, boxes)
R, offset, _ , alpha_exp = _parse_param(param_lst[0])
ver = recon_dense_explicit(R, offset, nick_alpha_shp, alpha_exp,
roi_box_lst[0], tddfa.size)
# Write object
pre_ver = ver # for tracking
ser_to_obj_multiple_colors(nick_color, [ver],
height=int(nick_orig.shape[0]*1.5),
wfp='./sam_exp_only_obj/' + str(i) +'.obj')
print(f'Dump to sam_exp_only_obj/{str(i)}.obj')
if i > 1000:
break