def main(args):
_t = {'det': Timer(), 'reg': Timer(), 'recon': Timer()}
cfg = read_config_file(args.config)
# Init FaceBoxes and TDDFA
if args.onnx:
import os
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
os.environ['OMP_NUM_THREADS'] = '4'
from FaceBoxes.FaceBoxes_ONNX import FaceBoxes_ONNX
from TDDFA_ONNX import TDDFA_ONNX
face_boxes = FaceBoxes_ONNX()
tddfa = TDDFA_ONNX(**cfg)
else:
tddfa = TDDFA(**cfg)
face_boxes = FaceBoxes()
# Given a still image path and load to BGR channel
img = cv2.imread(args.img_fp)
print(f'Input image: {args.img_fp}')
# Detect faces, get 3DMM params and roi boxes
print(f'Input shape: {img.shape}')
if args.warmup:
print('Warmup by once')
boxes = face_boxes(img)
param_lst, roi_box_lst = tddfa(img, boxes)
ver_lst = tddfa.recon_vers(param_lst,
roi_box_lst,
dense_flag=args.dense_flag)
for _ in range(args.repeated):
img = cv2.imread(args.img_fp)
_t['det'].tic()
boxes = face_boxes(img)
_t['det'].toc()
n = len(boxes)
if n == 0:
print(f'No face detected, exit')
sys.exit(-1)
_t['reg'].tic()
param_lst, roi_box_lst = tddfa(img, boxes)
_t['reg'].toc()
_t['recon'].tic()
ver_lst = tddfa.recon_vers(param_lst,
roi_box_lst,
dense_flag=args.dense_flag)
_t['recon'].toc()
mode = 'Dense' if args.dense_flag else 'Sparse'
print(f"Face detection: {_t['det'].average_time * 1000:.2f}ms, "
f"3DMM regression: {_t['reg'].average_time * 1000:.2f}ms, "
f"{mode} reconstruction: {_t['recon'].average_time * 1000:.2f}ms")
def evaluate_Blender():
ds = load_model_ds('BlenderFiles/model_frames')
cfg = yaml.load(open('configs/mb1_120x120.yml'), Loader=yaml.SafeLoader)
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
os.environ['OMP_NUM_THREADS'] = '4'
tddfa = TDDFA_ONNX(**cfg)
face_boxes = FaceBoxes_ONNX()
for artifact_id in ds:
for img, id in zip(ds[artifact_id]['img'], ds[artifact_id]['ids']):
wfp = f'examples/results/' + artifact_id + '_' + str(
id) + '_2d_sparse.jpg'
boxes = face_boxes(img)
print(boxes)
n = len(boxes)
if n == 0:
print(f'No face detected, exit')
sys.exit(-1)
print(f'Detect {n} faces')
param_lst, roi_box_lst = tddfa(img, boxes)
ver_lst = tddfa.recon_vers(param_lst,
roi_box_lst,
dense_flag=False)
draw_landmarks(img, ver_lst, dense_flag=False, wfp=wfp)
def main(args):
cfg = yaml.load(open(args.config), Loader=yaml.SafeLoader)
# Init FaceBoxes and TDDFA, recommend using onnx flag
if args.onnx:
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
os.environ['OMP_NUM_THREADS'] = '4'
from FaceBoxes.FaceBoxes_ONNX import FaceBoxes_ONNX
from TDDFA_ONNX import TDDFA_ONNX
face_boxes = FaceBoxes_ONNX()
tddfa = TDDFA_ONNX(**cfg)
else:
gpu_mode = args.mode == 'gpu'
tddfa = TDDFA(gpu_mode=gpu_mode, **cfg)
face_boxes = FaceBoxes()
# run
dense_flag = args.opt in ('2d_dense', '3d')
pre_ver = None
if not os.path.isdir(args.save_dataset_path):
os.makedirs(args.save_dataset_path)
data = glob.glob(args.training_dataset_path + '/*.jpg')
# print(args.training_dataset_path + '/*.jpg')
for img_file in tqdm(data):
label_file = img_file.split('\\')[-1].replace('.jpg', '.txt')
label_path = os.path.join(args.save_dataset_path, label_file)
label_f = open(label_path, mode='wt', encoding='utf-8')
img = cv2.imread(img_file)
img_save = cv2.resize(img, (640, 480), interpolation=cv2.INTER_LINEAR)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
boxes = face_boxes(img)
# print(boxes)
if len(boxes) == 0:
boxes = [[0, 0, img.shape[1], img.shape[0]]]
param_lst, roi_box_lst = tddfa(img, boxes)
ver = tddfa.recon_vers(param_lst, roi_box_lst,
dense_flag=dense_flag)[0]
# refine
param_lst, roi_box_lst = tddfa(img, [ver], crop_policy='landmark')
ver = tddfa.recon_vers(param_lst, roi_box_lst,
dense_flag=dense_flag)[0]
# write img
# cv2.imwrite(label_path.replace('.txt','.jpg'), img_save)
# # write label
label = ''
for pt in range(ver.shape[1]):
label += str(ver[0, pt] / img.shape[1]) + ' ' + str(
ver[1, pt] / img.shape[0]) + ' '
label += '\n'
label_f.write(label)
label_f.close()
def __init__(self, root, config):
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
os.environ['OMP_NUM_THREADS'] = '4'
cfg = yaml.load(open(os.path.join(root, config)),
Loader=yaml.SafeLoader)
cfg["checkpoint_fp"] = os.path.join(root, cfg["checkpoint_fp"])
cfg["bfm_fp"] = os.path.join(root, cfg["bfm_fp"])
self.face_boxes = FaceBoxes_ONNX()
self.tddfa = TDDFA_ONNX(**cfg)
reader = imageio.get_reader("<video0>")
self.frames = iter(tqdm(enumerate(reader)))
self.pre_ver = None
self.has_boxes = False
def main(args):
cfg = yaml.load(open(args.config), Loader=yaml.SafeLoader)
# Init FaceBoxes and TDDFA, recommend using onnx flag
if args.onnx:
import os
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
os.environ['OMP_NUM_THREADS'] = '4'
from FaceBoxes.FaceBoxes_ONNX import FaceBoxes_ONNX
from TDDFA_ONNX import TDDFA_ONNX
face_boxes = FaceBoxes_ONNX()
tddfa = TDDFA_ONNX(**cfg)
else:
gpu_mode = args.mode == 'gpu'
tddfa = TDDFA(gpu_mode=gpu_mode, **cfg)
face_boxes = FaceBoxes()
# Given a camera
# before run this line, make sure you have installed `imageio-ffmpeg`
reader = imageio.get_reader("<video0>")
# the simple implementation of average smoothing by looking ahead by n_next frames
# assert the frames of the video >= n
n_pre, n_next = args.n_pre, args.n_next
n = n_pre + n_next + 1
queue_ver = deque()
queue_frame = deque()
# run
dense_flag = args.opt in ('2d_dense', '3d')
pre_ver = None
for i, frame in tqdm(enumerate(reader)):
frame_bgr = frame[..., ::-1] # RGB->BGR
if i == 0:
# the first frame, detect face, here we only use the first face, you can change depending on your need
boxes = face_boxes(frame_bgr)
boxes = [boxes[0]]
param_lst, roi_box_lst = tddfa(frame_bgr, boxes)
ver = tddfa.recon_vers(param_lst, roi_box_lst, dense_flag=dense_flag)[0]
# refine
param_lst, roi_box_lst = tddfa(frame_bgr, [ver], crop_policy='landmark')
ver = tddfa.recon_vers(param_lst, roi_box_lst, dense_flag=dense_flag)[0]
# padding queue
for _ in range(n_pre):
queue_ver.append(ver.copy())
queue_ver.append(ver.copy())
for _ in range(n_pre):
queue_frame.append(frame_bgr.copy())
queue_frame.append(frame_bgr.copy())
else:
param_lst, roi_box_lst = tddfa(frame_bgr, [pre_ver], crop_policy='landmark')
roi_box = roi_box_lst[0]
# todo: add confidence threshold to judge the tracking is failed
if abs(roi_box[2] - roi_box[0]) * abs(roi_box[3] - roi_box[1]) < 2020:
boxes = face_boxes(frame_bgr)
boxes = [boxes[0]]
param_lst, roi_box_lst = tddfa(frame_bgr, boxes)
ver = tddfa.recon_vers(param_lst, roi_box_lst, dense_flag=dense_flag)[0]
queue_ver.append(ver.copy())
queue_frame.append(frame_bgr.copy())
pre_ver = ver # for tracking
# smoothing: enqueue and dequeue ops
if len(queue_ver) >= n:
ver_ave = np.mean(queue_ver, axis=0)
if args.opt == '2d_sparse':
img_draw = cv_draw_landmark(queue_frame[n_pre], ver_ave) # since we use padding
elif args.opt == '2d_dense':
img_draw = cv_draw_landmark(queue_frame[n_pre], ver_ave, size=1)
elif args.opt == '3d':
img_draw = render(queue_frame[n_pre], [ver_ave], tddfa.tri, alpha=0.7)
else:
raise ValueError(f'Unknown opt {args.opt}')
cv2.imshow('image', img_draw)
k = cv2.waitKey(20)
if (k & 0xff == ord('q')):
break
queue_ver.popleft()
queue_frame.popleft()
def main(args):
cfg = yaml.load(open(args.config), Loader=yaml.SafeLoader)
# Init FaceBoxes and TDDFA, recommend using onnx flag
if args.onnx:
import os
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
os.environ['OMP_NUM_THREADS'] = '4'
from FaceBoxes.FaceBoxes_ONNX import FaceBoxes_ONNX
from TDDFA_ONNX import TDDFA_ONNX
face_boxes = FaceBoxes_ONNX()
tddfa = TDDFA_ONNX(**cfg)
else:
gpu_mode = args.mode == 'gpu'
tddfa = TDDFA(gpu_mode=gpu_mode, **cfg)
face_boxes = FaceBoxes()
# Given a video path
fn = args.video_fp.split('/')[-1]
reader = imageio.get_reader(args.video_fp)
fps = reader.get_meta_data()['fps']
suffix = get_suffix(args.video_fp)
video_wfp = f'examples/results/videos/{fn.replace(suffix, "")}_{args.opt}.mp4'
writer = imageio.get_writer(video_wfp, fps=fps)
# run
dense_flag = args.opt in ('3d', )
pre_ver = None
for i, frame in tqdm(enumerate(reader)):
frame_bgr = frame[..., ::-1] # RGB->BGR
if i == 0:
# the first frame, detect face, here we only use the first face, you can change depending on your need
boxes = face_boxes(frame_bgr)
boxes = [boxes[0]]
param_lst, roi_box_lst = tddfa(frame_bgr, boxes)
ver = tddfa.recon_vers(param_lst,
roi_box_lst,
dense_flag=dense_flag)[0]
# refine
param_lst, roi_box_lst = tddfa(frame_bgr, [ver],
crop_policy='landmark')
ver = tddfa.recon_vers(param_lst,
roi_box_lst,
dense_flag=dense_flag)[0]
else:
param_lst, roi_box_lst = tddfa(frame_bgr, [pre_ver],
crop_policy='landmark')
roi_box = roi_box_lst[0]
# todo: add confidence threshold to judge the tracking is failed
if abs(roi_box[2] - roi_box[0]) * abs(roi_box[3] -
roi_box[1]) < 2020:
boxes = face_boxes(frame_bgr)
boxes = [boxes[0]]
param_lst, roi_box_lst = tddfa(frame_bgr, boxes)
ver = tddfa.recon_vers(param_lst,
roi_box_lst,
dense_flag=dense_flag)[0]
pre_ver = ver # for tracking
if args.opt == '2d_sparse':
res = cv_draw_landmark(frame_bgr, ver)
elif args.opt == '3d':
res = render(frame_bgr, [ver], tddfa.tri)
else:
raise ValueError(f'Unknown opt {args.opt}')
writer.append_data(res[..., ::-1]) # BGR->RGB
writer.close()
print(f'Dump to {video_wfp}')
def main(args):
cfg = yaml.load(open(args.config), Loader=yaml.SafeLoader)
# Init FaceBoxes and TDDFA, recommend using onnx flag
if args.onnx:
import os
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
os.environ['OMP_NUM_THREADS'] = '4'
from FaceBoxes.FaceBoxes_ONNX import FaceBoxes_ONNX
from TDDFA_ONNX import TDDFA_ONNX
face_boxes = FaceBoxes_ONNX()
tddfa = TDDFA_ONNX(**cfg)
else:
gpu_mode = args.mode == 'gpu'
tddfa = TDDFA(gpu_mode=gpu_mode, **cfg)
face_boxes = FaceBoxes()
# Given a still image path and load to BGR channel
print(args.img_fp)
if not os.path.isfile(args.img_fp):
raise ValueError(
'--face argument must be a valid path to video/image file')
elif args.img_fp.split('.')[1] in ['jpg', 'png', 'jpeg']:
full_frames = [cv2.imread(args.img_fp)]
else:
video_stream = cv2.VideoCapture(args.img_fp)
fps = video_stream.get(cv2.CAP_PROP_FPS)
print('Reading video frames...')
full_frames = []
while 1:
still_reading, frame = video_stream.read()
if not still_reading:
video_stream.release()
break
full_frames.append(frame)
print("num of frames : ", len(full_frames))
for i, img in enumerate(full_frames):
# Detect faces, get 3DMM params and roi boxes
boxes = face_boxes(img)
n = len(boxes)
if n == 0:
print(f'No face detected, exit')
sys.exit(-1)
print(f'Detect {n} faces')
param_lst, roi_box_lst = tddfa(img, boxes)
# Visualization and serialization
dense_flag = args.opt in ('2d_dense', '3d', 'depth', 'pncc', 'uv_tex',
'ply', 'obj')
old_suffix = get_suffix(args.img_fp)
new_suffix = f'.{args.opt}' if args.opt in ('ply', 'obj') else '.jpg'
idx = str(i)
while (len(idx) != 4):
idx = "0" + idx
wfp = f'examples/results/obj/{args.img_fp.split("/")[-1].replace(old_suffix, "")}_{args.opt}' + idx + new_suffix
ver_lst = tddfa.recon_vers(param_lst,
roi_box_lst,
dense_flag=dense_flag)
if args.opt == 'obj':
ser_to_obj(img, ver_lst, tddfa.tri, height=img.shape[0], wfp=wfp)
# elif args.opt == '2d_sparse':
# draw_landmarks(img, ver_lst, show_flag=args.show_flag, dense_flag=dense_flag, wfp=wfp)
# elif args.opt == '2d_dense':
# draw_landmarks(img, ver_lst, show_flag=args.show_flag, dense_flag=dense_flag, wfp=wfp)
# elif args.opt == '3d':
# render(img, ver_lst, tddfa.tri, alpha=0.6, show_flag=args.show_flag, wfp=wfp)
# elif args.opt == 'depth':
# # if `with_bf_flag` is False, the background is black
# depth(img, ver_lst, tddfa.tri, show_flag=args.show_flag, wfp=wfp, with_bg_flag=True)
elif args.opt == 'pncc':
pncc(img,
ver_lst,
tddfa.tri,
show_flag=args.show_flag,
wfp=wfp,
with_bg_flag=True)
elif args.opt == 'uv_tex':
uv_tex(img, ver_lst, tddfa.tri, show_flag=args.show_flag, wfp=wfp)
# elif args.opt == 'pose':
# viz_pose(img, param_lst, ver_lst, show_flag=args.show_flag, wfp=wfp)
# elif args.opt == 'ply':
# ser_to_ply(ver_lst, tddfa.tri, height=img.shape[0], wfp=wfp)
else:
raise ValueError(f'Unknown opt {args.opt}')
def main(args):
cfg = yaml.load(open(args.config), Loader=yaml.SafeLoader)
# Init FaceBoxes and TDDFA, recommend using onnx flag
if args.onnx:
import os
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
os.environ['OMP_NUM_THREADS'] = '4'
from FaceBoxes.FaceBoxes_ONNX import FaceBoxes_ONNX
from TDDFA_ONNX import TDDFA_ONNX
face_boxes = FaceBoxes_ONNX()
tddfa = TDDFA_ONNX(**cfg)
else:
gpu_mode = args.mode == 'gpu'
tddfa = TDDFA(gpu_mode=gpu_mode, **cfg)
face_boxes = FaceBoxes()
# Given a still image path and load to BGR channel
img = cv2.imread(args.img_fp)
# Detect faces, get 3DMM params and roi boxes
boxes = face_boxes(img)
n = len(boxes)
if n == 0:
print(f'No face detected, exit')
sys.exit(-1)
print(f'Detect {n} faces')
param_lst, roi_box_lst = tddfa(img, boxes)
# Visualization and serialization
dense_flag = args.opt in ('2d_dense', '3d', 'depth', 'pncc', 'uv_tex', 'ply', 'obj')
old_suffix = get_suffix(args.img_fp)
new_suffix = f'.{args.opt}' if args.opt in ('ply', 'obj') else '.jpg'
wfp = f'examples/results/{args.img_fp.split("/")[-1].replace(old_suffix, "")}_{args.opt}' + new_suffix
ver_lst = tddfa.recon_vers(param_lst, roi_box_lst, dense_flag=dense_flag)
if args.opt == '2d_sparse':
draw_landmarks(img, ver_lst, show_flag=args.show_flag, dense_flag=dense_flag, wfp=wfp)
elif args.opt == '2d_dense':
draw_landmarks(img, ver_lst, show_flag=args.show_flag, dense_flag=dense_flag, wfp=wfp)
elif args.opt == '3d':
render(img, ver_lst, tddfa.tri, alpha=0.6, show_flag=args.show_flag, wfp=wfp)
elif args.opt == 'depth':
# if `with_bf_flag` is False, the background is black
depth(img, ver_lst, tddfa.tri, show_flag=args.show_flag, wfp=wfp, with_bg_flag=True)
elif args.opt == 'pncc':
pncc(img, ver_lst, tddfa.tri, show_flag=args.show_flag, wfp=wfp, with_bg_flag=True)
elif args.opt == 'uv_tex':
uv_tex(img, ver_lst, tddfa.tri, show_flag=args.show_flag, wfp=wfp)
elif args.opt == 'pose':
viz_pose(img, param_lst, ver_lst, show_flag=args.show_flag, wfp=wfp)
elif args.opt == 'ply':
ser_to_ply(ver_lst, tddfa.tri, height=img.shape[0], wfp=wfp)
elif args.opt == 'obj':
ser_to_obj(img, ver_lst, tddfa.tri, height=img.shape[0], wfp=wfp)
else:
raise ValueError(f'Unknown opt {args.opt}')
from skimage import io
import gradio as gr
# load config
cfg = yaml.load(open('configs/mb1_120x120.yml'), Loader=yaml.SafeLoader)
# Init FaceBoxes and TDDFA, recommend using onnx flag
onnx_flag = True # or True to use ONNX to speed up
if onnx_flag:
import os
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
os.environ['OMP_NUM_THREADS'] = '4'
from FaceBoxes.FaceBoxes_ONNX import FaceBoxes_ONNX
from TDDFA_ONNX import TDDFA_ONNX
face_boxes = FaceBoxes_ONNX()
tddfa = TDDFA_ONNX(**cfg)
else:
face_boxes = FaceBoxes()
tddfa = TDDFA(gpu_mode=False, **cfg)
def inference(img):
# face detection
boxes = face_boxes(img)
# regress 3DMM params
param_lst, roi_box_lst = tddfa(img, boxes)
# reconstruct vertices and render
ver_lst = tddfa.recon_vers(param_lst, roi_box_lst, dense_flag=True)
return render(img, ver_lst, tddfa.tri, alpha=0.6, show_flag=False)
def main(args):
cfg = yaml.load(open(args.config), Loader=yaml.SafeLoader)
# Init FaceBoxes and TDDFA, recommend using onnx flag
if args.onnx:
import os
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
os.environ['OMP_NUM_THREADS'] = '4'
from FaceBoxes.FaceBoxes_ONNX import FaceBoxes_ONNX
from TDDFA_ONNX import TDDFA_ONNX
face_boxes = FaceBoxes_ONNX()
tddfa = TDDFA_ONNX(**cfg)
else:
gpu_mode = args.mode == 'gpu'
tddfa = TDDFA(gpu_mode=gpu_mode, **cfg)
face_boxes = FaceBoxes()
# Given a video path
fn = args.video_fp.split('/')[-1]
reader = imageio.get_reader(args.video_fp)
fps = reader.get_meta_data()['fps']
suffix = get_suffix(args.video_fp)
video_wfp = f'examples/results/videos/{fn.replace(suffix, "")}_{args.opt}_smooth.mp4'
writer = imageio.get_writer(video_wfp, fps=fps)
# the simple implementation of average smoothing by looking ahead by n_next frames
# assert the frames of the video >= n
n_pre, n_next = args.n_pre, args.n_next
n = n_pre + n_next + 1
queue_ver = deque()
queue_frame = deque()
# run
dense_flag = args.opt in ('2d_dense', '3d',)
pre_ver = None
for i, frame in tqdm(enumerate(reader)):
if args.start > 0 and i < args.start:
continue
if args.end > 0 and i > args.end:
break
frame_bgr = frame[..., ::-1] # RGB->BGR
if i == 0:
# detect
boxes = face_boxes(frame_bgr)
#boxes = [boxes[0]]
param_lst, roi_box_lst = tddfa(frame_bgr, boxes)
ver = tddfa.recon_vers(param_lst, roi_box_lst, dense_flag=dense_flag)
# refine
param_lst, roi_box_lst = tddfa(frame_bgr, ver, crop_policy='landmark')
ver = tddfa.recon_vers(param_lst, roi_box_lst, dense_flag=dense_flag)
# padding queue
for _ in range(n_pre):
queue_ver.append(ver.copy())
queue_ver.append(ver.copy())
for _ in range(n_pre):
queue_frame.append(frame_bgr.copy())
queue_frame.append(frame_bgr.copy())
else:
param_lst, roi_box_lst = tddfa(frame_bgr, pre_ver, crop_policy='landmark')
roi_box = roi_box_lst[0]
# todo: add confidence threshold to judge the tracking is failed
if abs(roi_box[2] - roi_box[0]) * abs(roi_box[3] - roi_box[1]) < 2020:
print("xxx")
boxes = face_boxes(frame_bgr)
#boxes = [boxes[0]]
param_lst, roi_box_lst = tddfa(frame_bgr, boxes)
ver = tddfa.recon_vers(param_lst, roi_box_lst, dense_flag=dense_flag)
queue_ver.append(ver.copy())
queue_frame.append(frame_bgr.copy())
pre_ver = ver # for tracking
# smoothing: enqueue and dequeue ops
if len(queue_ver) >= n:
ver_ave = np.mean(queue_ver, axis=0)
if args.opt == '2d_sparse':
img_draw = cv_draw_landmark2(queue_frame[n_pre], ver_ave) # since we use padding
elif args.opt == '2d_dense':
img_draw = cv_draw_landmark2(queue_frame[n_pre], ver_ave, size=1)
elif args.opt == '3d':
img_draw = render(queue_frame[n_pre], ver_ave, tddfa.tri, alpha=0.7)
else:
raise ValueError(f'Unknown opt {args.opt}')
writer.append_data(img_draw[:, :, ::-1]) # BGR->RGB
queue_ver.popleft()
queue_frame.popleft()
# we will lost the last n_next frames, still padding
for _ in range(n_next):
queue_ver.append(ver.copy())
queue_frame.append(frame_bgr.copy()) # the last frame
ver_ave = np.mean(queue_ver, axis=0)
if args.opt == '2d_sparse':
img_draw = cv_draw_landmark2(queue_frame[n_pre], ver_ave) # since we use padding
elif args.opt == '2d_dense':
img_draw = cv_draw_landmark2(queue_frame[n_pre], ver_ave, size=1)
elif args.opt == '3d':
img_draw = render(queue_frame[n_pre], ver_ave, tddfa.tri, alpha=0.7)
else:
raise ValueError(f'Unknown opt {args.opt}')
writer.append_data(img_draw[..., ::-1]) # BGR->RGB
queue_ver.popleft()
queue_frame.popleft()
writer.close()
print(f'Dump to {video_wfp}')