def __init__(self, style_img, input_img, style_mask, input_mask, save=False):
style_name = os.path.basename(style_img).split('.')[0]
input_name = os.path.basename(input_img).split('.')[0]
self.style_img = np.float32(imread(style_img))
self.input_img = np.float32(imread(input_img))
self.style_mask = np.float32(imread(style_mask))
self.input_mask = np.float32(imread(input_mask))
# Fetch Facial Landmarks
if os.path.exists('input/%s_%s_lm.pkl' % (style_name, input_name)):
with open('input/%s_%s_lm.pkl' % (style_name, input_name), 'rb') as f:
pkl = pickle.load(f)
self.style_lm = pkl['style']
self.input_lm = pkl['input']
else:
fa = FaceAlignment(LandmarksType._2D, device='cpu', flip_input=False)
self.style_lm = fa.get_landmarks(self.style_img)[0]
self.input_lm = fa.get_landmarks(self.input_img)[0]
with open('input/%s_%s_lm.pkl' % (style_name, input_name),
'wb') as f:
pickle.dump({
'style': self.style_lm,
'input': self.input_lm
}, f, protocol=2)
self.output_filename = '_'.join({input_name, style_name})
self.save = save
def encode_filter(filter_files):
images = []
faces = []
FACE_ALIGNMENT = FaceAlignment(LandmarksType._2D,
enable_cuda=True,
flip_input=False)
for i, filter_file in enumerate(filter_files):
images.append(skimage.io.imread(str(filter_file)))
faces.append(FACE_ALIGNMENT.get_landmarks(images[i]))
FACE_ALIGNMENT = None
face_recognition_model = face_recognition_models.face_recognition_model_location(
)
face_encoder = dlib.face_recognition_model_v1(face_recognition_model)
for i, face in enumerate(faces):
if face is None:
print('Warning: {} has no face.'.format(filter_files[i]))
continue
if len(face) > 1:
print('Warning: {} has more than one face.'.format(
filter_files[i]))
parts = []
for p in face[0]:
parts.append(dlib.point(p[0], p[1]))
raw_landmark_set = dlib.full_object_detection(rect, parts)
yield numpy.array(
face_encoder.compute_face_descriptor(images[i],
raw_landmark_set, 1))
def __getitem__(self, idx):
real_idx = self.indexes[idx]
path = self.files[real_idx]
print("image file path=", path)
fa = FaceAlignment(LandmarksType._2D, device=self.device)
imgUMat = cv2.imread(path)
x_temp = cv2.cvtColor(imgUMat, cv2.COLOR_BGR2RGB)
y_temp = fa.get_landmarks(x_temp)[0]
out = []
x = PIL.Image.fromarray(x_temp, 'RGB')
y = plot_landmarks(x_temp, y_temp)
if self.transform:
x = self.transform(x)
y = self.transform(y)
out.append({'frame': x, 'landmarks': y})
return real_idx, out
def evaluate(respth='./results/data_src', dspth='../data'):
respth = osp.join(os.path.abspath(os.path.dirname(__file__)), respth)
if not os.path.exists(respth):
os.makedirs(respth)
face_model = FaceAlignment(LandmarksType._2D, device="cuda")
data_path = osp.join(os.path.abspath(os.path.dirname(__file__)), dspth)
for image_path in os.listdir(data_path):
image = cv2.imread(osp.join(data_path, image_path))
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
landmark = face_model.get_landmarks(image)[-1]
# print(landmark)
mask = get_image_hull_mask(np.shape(image), landmark).astype(np.uint8)
# cv2.imshow("mask", (mask*255).astype(np.uint8))
image_bgra = merge(image, mask)
# cv2.imshow("image_bgra", image_bgra)
# cv2.waitKey(1)
save_path = osp.join(respth, image_path)
cv2.imwrite(save_path[:-4] + '.png', image_bgra)
class FaceModel:
def __init__(self, args):
self.args = args
model = edict()
self.threshold = args.threshold
self.det_minsize = 50
self.det_threshold = [0.4, 0.6, 0.6]
self.det_factor = 0.9
_vec = args.image_size.split(',')
assert len(_vec) == 2
image_size = (int(_vec[0]), int(_vec[1]))
self.image_size = image_size
_vec = args.model.split(',')
assert len(_vec) == 2
prefix = _vec[0]
epoch = int(_vec[1])
print('loading', prefix, epoch)
ctx = mx.gpu(args.gpu)
sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch)
all_layers = sym.get_internals()
sym = all_layers['fc1_output']
model = mx.mod.Module(symbol=sym, context=ctx, label_names=None)
#model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))])
model.bind(data_shapes=[('data', (1, 3, image_size[0],
image_size[1]))])
model.set_params(arg_params, aux_params)
self.model = model
# mtcnn_path = os.path.join(os.path.dirname(__file__), 'mtcnn-model')
mtcnn_path = os.path.join('deploy', 'mtcnn-model')
detector = MtcnnDetector(model_folder=mtcnn_path,
ctx=ctx,
num_worker=1,
accurate_landmark=True,
threshold=[0.0, 0.0, 0.2])
self.detector = detector
self.FACE_ALIGNMENT = FaceAlignment(LandmarksType._3D,
device='cuda',
flip_input=False)
def get_feature(self, face_img):
detected = True
#face_img is bgr image
def mtcnn_align(img):
ret = self.detector.detect_face_limited(face_img,
det_type=self.args.det)
if ret is None:
# detected = False
bbox, points = None, None
else:
bbox, points = ret
if bbox.shape[0] == 0:
# detected = False
bbox, points = None, None
else:
bbox = bbox[0, 0:4]
points = points[0, :].reshape((2, 5)).T
# print(bbox)
# print(points)
nimg = face_preprocess.preprocess(face_img,
bbox,
points,
image_size='112,112')
return nimg
# skimage.io.imread( str(fn) )
faces = self.FACE_ALIGNMENT.get_landmarks(face_img)
if faces is not None:
if len(faces) > 1:
faces = faces[0:1]
# pdb.set_trace()
points = faces[0]
alignment = umeyama(points[17:], landmarks_2D, True)[0:2]
nimg = _aligned_image = transform(face_img, alignment, 112, 0)
else:
nimg = mtcnn_align(face_img)
# nimg = face_preprocess.preprocess(face_img, image_size='112,112')
# pdb.set_trace()
nimg = cv2.cvtColor(nimg, cv2.COLOR_BGR2RGB)
aligned = np.transpose(nimg, (2, 0, 1))
#print(nimg.shape)
embedding = None
for flipid in [0, 1]:
if flipid == 1:
if self.args.flip == 0:
break
do_flip(aligned)
input_blob = np.expand_dims(aligned, axis=0)
data = mx.nd.array(input_blob)
db = mx.io.DataBatch(data=(data, ))
self.model.forward(db, is_train=False)
_embedding = self.model.get_outputs()[0].asnumpy()
#print(_embedding.shape)
if embedding is None:
embedding = _embedding
else:
embedding += _embedding
embedding = sklearn.preprocessing.normalize(embedding).flatten()
return detected, embedding
class FaceTools:
""" Face Toolkit
"""
def __init__(self, dimensions='2d'):
landmarkType = LandmarksType._2D if dimensions == '2d' else LandmarksType._3D
self.faceAlignment = FaceAlignment(landmarkType,
flip_input=False,
device='cpu',
verbose=False)
@staticmethod
def _polyArea(points):
""" calculate the area of a polygon (Gauss equation)
"""
x = [point[0] for point in points]
y = [point[1] for point in points]
return 0.5 * np.abs(
np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1)))
@staticmethod
def _slope(point1, point2):
""" compute the slope of the line passing through two points
"""
return (point2[1] - point1[1]) / (point2[0] - point1[0])
def landmarks(self, img):
""" get landmarks
"""
return self.faceAlignment.get_landmarks(img)[0]
def geometricFeatures1(self, img, landmarks=None):
"""Improved Performance in Facial Expression Recognition Using 32 Geometric Features
Linear features(15):
– 3 for left eyebrow
– 2 for left eye
– 1 for cheeks
– 1 for nose
– 8 for mouth
Polygonal features(3):
– 1 for the left eye
– 1 between corners of left eye and left corner of mouth
– 1 for mouth
Elliptical features(7):
– 1 for left eyebrow
– 3 for the left eye: eye, upper and lower eyelids
– 3 for mouth: upper and lower lips
Slope features(7):
– 1 for left eyebrow
– 6 for mouth corners
"""
pts = self.faceAlignment.get_landmarks(
img)[0] if landmarks is None else landmarks
result = []
# Linear features
#eyebrow
result.append(d(pts[21], pts[22]))
result.append(d(pts[22], pts[42]))
result.append(d(pts[26], pts[45]))
#left eye
result.append(d(pts[42], pts[45]))
#d2=43-47 || 44-46
result.append(d(pts[43], pts[47]))
#cheeks
result.append(d(pts[1], pts[15]))
#nose
result.append(d(pts[31], pts[35]))
#mouth
result.append(d(pts[48], pts[51]))
result.append(d(pts[51], pts[54]))
result.append(d(pts[62], pts[66]))
result.append(d(pts[51], pts[57]))
result.append(d(pts[50], pts[33]))
result.append(d(pts[52], pts[33]))
result.append(d(pts[48], pts[31]))
result.append(d(pts[54], pts[35]))
#Polygonal features:
result.append(FaceTools._polyArea([pts[48], pts[54], pts[57]]))
result.append(FaceTools._polyArea([pts[54], pts[42], pts[45]]))
result.append(FaceTools._polyArea([pts[42], pts[22], pts[26],
pts[45]]))
#Slope features:
result.append(FaceTools._slope(pts[22], pts[26]))
result.append(FaceTools._slope(pts[48], pts[31]))
result.append(FaceTools._slope(pts[54], pts[35]))
result.append(FaceTools._slope(pts[48], pts[51]))
result.append(FaceTools._slope(pts[51], pts[54]))
result.append(FaceTools._slope(pts[54], pts[57]))
result.append(FaceTools._slope(pts[48], pts[57]))
return result
def faceDistances(self, img, landmarks=None):
""" calculate distances as described in
'Automatic Facial Expression Recognition Using Combined Geometric Features':
D1 Left eyebrow length
D2 Right eyebrow length
D3 Distance between left and right eyebrow
D4 Left eye height
D5 Left eye width
D6 Right eye height
D7 Right eye width
D8 Distance between left eyebrow and left eye
D9 Distance between right eyebrow and right eye
D10 Distance between nose tip and upper lip
D11 Lip width
D12 Lip height
D13 Inner lip distance
D14 Distance between left eye corner and lip left corner
D15 Distance between right eye corner and lip right corner
"""
pts = self.faceAlignment.get_landmarks(
img)[0] if landmarks is None else landmarks
distances = [
d(pts[17], pts[18]) + d(pts[18], pts[19]) + d(pts[19], pts[20]) + \
d(pts[20], pts[21]),
d(pts[22], pts[23]) + d(pts[23], pts[24]) + d(pts[24], pts[25]) + \
d(pts[25], pts[26]),
d(pts[21], pts[22]),
d(pts[40], pts[38]),
d(pts[36], pts[39]),
d(pts[43], pts[47]),
d(pts[42], pts[45]),
d(pts[19], pts[37]),
d(pts[24], pts[44]),
d(pts[33], pts[51]),
d(pts[48], pts[54]),
d(pts[51], pts[57]),
d(pts[62], pts[66]),
d(pts[36], pts[48]),
d(pts[45], pts[54])
]
return distances