def getindices(ftl_face, sq, padw=padw, detw=detw):
# get mask region using boundary, chin landmarks and nose landmarks
# boundary region: left -> right, upper -> lower
WH = ftl_face.shape[0]
boundary = sq.align(detw)
left, right, upper, lower = np.array(boundary) + padw
indices = np.array([(x, y) for x in range(left, right)
for y in range(upper, lower)])
# get landmarks of frontalized face
det = detector(ftl_face, 1)[0]
shape = predictor(ftl_face, det)
ldmk = np.asarray([(
shape.part(n).x,
shape.part(n).y,
) for n in range(shape.num_parts)], np.float32)
chin_xp, chin_fp = ldmk[3:14, 0], ldmk[3:14, 1]
chin_line = np.interp(np.arange(WH), chin_xp, chin_fp)
nose_xp, nose_fp = ldmk[31:36, 0], ldmk[31:36, 1]
nose_line = np.interp(np.arange(WH), nose_xp, nose_fp)
# filter the position which is out of chin line and nose line
check = np.logical_and(indices[:, 1] < chin_line[indices[:, 0]],
indices[:, 1] > nose_line[indices[:, 0]])
return indices[check.nonzero()]
def process_proxy(rsize, ksize=(17, 17), sigma=1e2, k=1):
# process teeth proxies to get their landmarks and high-pass filters
F, S = {}, {}
for mode in ('upper', 'lower'):
pxyfile, = glob.glob('reference/proxy_%s_*.png' % mode)
img = cv2.imread(pxyfile)
# detect face and landmarks
det = detector(img, 1)[0]
shape = predictor(img, det)
ldmk = np.asarray([(shape.part(n).x, shape.part(n).y)
for n in range(48, shape.num_parts)], np.float32)
# normalize landmarks
origin = np.array([det.left(), det.top()])
size = np.array([det.width(), det.height()])
ldmk = (ldmk - origin) / size # restrained in [0, 0] ~ [1, 1]
# resize texture
txtr = img[origin[1]:origin[1] + size[1],
origin[0]:origin[0] + size[0]]
txtr = cv2.resize(txtr, (rsize, rsize))
# generate hgih-pass filter (only one channel)
norm_txtr = txtr.astype(np.float) / 255
smooth_txtr = cv2.GaussianBlur(txtr, ksize, sigma) / 255
filt = (norm_txtr - smooth_txtr) * k + 0.5
filt[filt < 0] = 0
filt[filt > 1] = 1
# add landmarks and filter into dict S and F respectively
F[mode] = filt
S[mode] = ldmk
return F, S
def util2(mp4_path, save_path, startfr=0, endfr=None):
# frontalize every frame from a video and save to somewhere
from new_facefrontal import facefrontal
cap = cv2.VideoCapture(mp4_path)
cap.set(cv2.CAP_PROP_POS_FRAMES, startfr)
cnt = startfr
endfr = cap.get(cv2.CAP_PROP_FRAME_COUNT) if endfr is None else endfr
print('Start preprocessing...')
while cap.isOpened():
if cnt == endfr:
break
print("%s: %04d/%04d" % (save_path, cnt + 1, endfr))
cnt += 1
_, img = cap.read()
img = facefrontal(img, detector, predictor)
if img is None:
continue
dets = detector(img, 1)
if len(dets) != 1:
continue
cv2.imwrite('%s%04d.png' % (save_path, cnt), img)
print('Done')
def facefrontal(img, detector, predictor):
'''
### parameters
img: original image to be frontalized \\
detector: face detector generated by dlib.get_frontal_face_detector() \\
predictor: landmark extractor generated by dlib.shape_predictor(...)
### retval
newimg: (320, 320, 3), frontalized image
'''
dets = detector(img, 1) # only 0 or 1 face in each frame
if (len(dets) == 0):
return None
det = dets[0]
shape = predictor(img, det)
p2d = np.asarray([(
shape.part(n).x,
shape.part(n).y,
) for n in range(shape.num_parts)], np.float32)
rawfront, symfront = fronter.frontalization(img, det, p2d)
newimg = symfront.astype('uint8')
# cv2.imshow('newimg', newimg)
# cv2.waitKey(0)
return newimg
def preprocess(mp4_path, save_path, rsize, startfr=0, endfr=None):
'''
### parameters
mp4_path: path of mp4 file \\
sq: Squre instance which defines the boundary of lower face texture
rsize: width (height) of clipped texture in every target video frame
### retval
savepath: path that saves landmarks and textures
'''
landmarks = []
textures = []
cap = cv2.VideoCapture(mp4_path)
cap.set(cv2.CAP_PROP_POS_FRAMES, startfr)
cnt = startfr
endfr = cap.get(cv2.CAP_PROP_FRAME_COUNT) if endfr is None else endfr
print('Start preprocessing...')
while cap.isOpened():
if cnt == endfr:
break
print("%04d/%04d" % (cnt, endfr - 1))
cnt += 1
ret, img_ = cap.read()
img = facefrontal(img_, detector, predictor)
if img is None:
continue
dets = detector(img, 1)
if len(dets) != 1:
continue
det = dets[0]
shape = predictor(img, det)
ldmk = np.asarray([(shape.part(n).x, shape.part(n).y)
for n in range(48, shape.num_parts)], np.float32)
# normalization according to det.shape & reshape into 40-D features
origin = np.array([det.left(), det.top()])
size = np.array([det.width(), det.height()])
ldmk = (ldmk - origin) / size # restrained in [0, 0] ~ [1, 1]
# validate landmarks using statistics in the dataset
if np.sum(np.logical_or(ldmk < boundL, ldmk > boundU)) > 0:
continue
landmarks.append(ldmk.flatten())
# resize texture into a square
txtr = img[origin[1]:origin[1] + size[1],
origin[0]:origin[0] + size[0]]
txtr = cv2.resize(txtr, (rsize, rsize))
# mask & inpaint for clothes region
txtr = mask_inpaint(txtr)
textures.append(txtr)
landmarks = np.array(landmarks)
textures = np.array(textures)
# filter frames which are not locally smooth
approx = (landmarks[2:, :] + landmarks[:-2, :]) / 2
L2 = np.linalg.norm(landmarks[1:-1, :] - approx, ord=2, axis=1)
check = (L2 <= 0.1).nonzero()
landmarks = landmarks[1:-1][check].reshape((-1, 20, 2))
textures = textures[1:-1][check]
np.savez(save_path, landmarks=landmarks, textures=textures)