def __init__(self, style_img, input_img, style_mask, input_mask, save=False): style_name = os.path.basename(style_img).split('.')[0] input_name = os.path.basename(input_img).split('.')[0] self.style_img = np.float32(imread(style_img)) self.input_img = np.float32(imread(input_img)) self.style_mask = np.float32(imread(style_mask)) self.input_mask = np.float32(imread(input_mask)) # Fetch Facial Landmarks if os.path.exists('input/%s_%s_lm.pkl' % (style_name, input_name)): with open('input/%s_%s_lm.pkl' % (style_name, input_name), 'rb') as f: pkl = pickle.load(f) self.style_lm = pkl['style'] self.input_lm = pkl['input'] else: fa = FaceAlignment(LandmarksType._2D, device='cpu', flip_input=False) self.style_lm = fa.get_landmarks(self.style_img)[0] self.input_lm = fa.get_landmarks(self.input_img)[0] with open('input/%s_%s_lm.pkl' % (style_name, input_name), 'wb') as f: pickle.dump({ 'style': self.style_lm, 'input': self.input_lm }, f, protocol=2) self.output_filename = '_'.join({input_name, style_name}) self.save = save
def encode_filter(filter_files): images = [] faces = [] FACE_ALIGNMENT = FaceAlignment(LandmarksType._2D, enable_cuda=True, flip_input=False) for i, filter_file in enumerate(filter_files): images.append(skimage.io.imread(str(filter_file))) faces.append(FACE_ALIGNMENT.get_landmarks(images[i])) FACE_ALIGNMENT = None face_recognition_model = face_recognition_models.face_recognition_model_location( ) face_encoder = dlib.face_recognition_model_v1(face_recognition_model) for i, face in enumerate(faces): if face is None: print('Warning: {} has no face.'.format(filter_files[i])) continue if len(face) > 1: print('Warning: {} has more than one face.'.format( filter_files[i])) parts = [] for p in face[0]: parts.append(dlib.point(p[0], p[1])) raw_landmark_set = dlib.full_object_detection(rect, parts) yield numpy.array( face_encoder.compute_face_descriptor(images[i], raw_landmark_set, 1))
def __getitem__(self, idx): real_idx = self.indexes[idx] path = self.files[real_idx] print("image file path=", path) fa = FaceAlignment(LandmarksType._2D, device=self.device) imgUMat = cv2.imread(path) x_temp = cv2.cvtColor(imgUMat, cv2.COLOR_BGR2RGB) y_temp = fa.get_landmarks(x_temp)[0] out = [] x = PIL.Image.fromarray(x_temp, 'RGB') y = plot_landmarks(x_temp, y_temp) if self.transform: x = self.transform(x) y = self.transform(y) out.append({'frame': x, 'landmarks': y}) return real_idx, out
def evaluate(respth='./results/data_src', dspth='../data'): respth = osp.join(os.path.abspath(os.path.dirname(__file__)), respth) if not os.path.exists(respth): os.makedirs(respth) face_model = FaceAlignment(LandmarksType._2D, device="cuda") data_path = osp.join(os.path.abspath(os.path.dirname(__file__)), dspth) for image_path in os.listdir(data_path): image = cv2.imread(osp.join(data_path, image_path)) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) landmark = face_model.get_landmarks(image)[-1] # print(landmark) mask = get_image_hull_mask(np.shape(image), landmark).astype(np.uint8) # cv2.imshow("mask", (mask*255).astype(np.uint8)) image_bgra = merge(image, mask) # cv2.imshow("image_bgra", image_bgra) # cv2.waitKey(1) save_path = osp.join(respth, image_path) cv2.imwrite(save_path[:-4] + '.png', image_bgra)
class FaceModel: def __init__(self, args): self.args = args model = edict() self.threshold = args.threshold self.det_minsize = 50 self.det_threshold = [0.4, 0.6, 0.6] self.det_factor = 0.9 _vec = args.image_size.split(',') assert len(_vec) == 2 image_size = (int(_vec[0]), int(_vec[1])) self.image_size = image_size _vec = args.model.split(',') assert len(_vec) == 2 prefix = _vec[0] epoch = int(_vec[1]) print('loading', prefix, epoch) ctx = mx.gpu(args.gpu) sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) all_layers = sym.get_internals() sym = all_layers['fc1_output'] model = mx.mod.Module(symbol=sym, context=ctx, label_names=None) #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))]) model.set_params(arg_params, aux_params) self.model = model # mtcnn_path = os.path.join(os.path.dirname(__file__), 'mtcnn-model') mtcnn_path = os.path.join('deploy', 'mtcnn-model') detector = MtcnnDetector(model_folder=mtcnn_path, ctx=ctx, num_worker=1, accurate_landmark=True, threshold=[0.0, 0.0, 0.2]) self.detector = detector self.FACE_ALIGNMENT = FaceAlignment(LandmarksType._3D, device='cuda', flip_input=False) def get_feature(self, face_img): detected = True #face_img is bgr image def mtcnn_align(img): ret = self.detector.detect_face_limited(face_img, det_type=self.args.det) if ret is None: # detected = False bbox, points = None, None else: bbox, points = ret if bbox.shape[0] == 0: # detected = False bbox, points = None, None else: bbox = bbox[0, 0:4] points = points[0, :].reshape((2, 5)).T # print(bbox) # print(points) nimg = face_preprocess.preprocess(face_img, bbox, points, image_size='112,112') return nimg # skimage.io.imread( str(fn) ) faces = self.FACE_ALIGNMENT.get_landmarks(face_img) if faces is not None: if len(faces) > 1: faces = faces[0:1] # pdb.set_trace() points = faces[0] alignment = umeyama(points[17:], landmarks_2D, True)[0:2] nimg = _aligned_image = transform(face_img, alignment, 112, 0) else: nimg = mtcnn_align(face_img) # nimg = face_preprocess.preprocess(face_img, image_size='112,112') # pdb.set_trace() nimg = cv2.cvtColor(nimg, cv2.COLOR_BGR2RGB) aligned = np.transpose(nimg, (2, 0, 1)) #print(nimg.shape) embedding = None for flipid in [0, 1]: if flipid == 1: if self.args.flip == 0: break do_flip(aligned) input_blob = np.expand_dims(aligned, axis=0) data = mx.nd.array(input_blob) db = mx.io.DataBatch(data=(data, )) self.model.forward(db, is_train=False) _embedding = self.model.get_outputs()[0].asnumpy() #print(_embedding.shape) if embedding is None: embedding = _embedding else: embedding += _embedding embedding = sklearn.preprocessing.normalize(embedding).flatten() return detected, embedding
class FaceTools: """ Face Toolkit """ def __init__(self, dimensions='2d'): landmarkType = LandmarksType._2D if dimensions == '2d' else LandmarksType._3D self.faceAlignment = FaceAlignment(landmarkType, flip_input=False, device='cpu', verbose=False) @staticmethod def _polyArea(points): """ calculate the area of a polygon (Gauss equation) """ x = [point[0] for point in points] y = [point[1] for point in points] return 0.5 * np.abs( np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1))) @staticmethod def _slope(point1, point2): """ compute the slope of the line passing through two points """ return (point2[1] - point1[1]) / (point2[0] - point1[0]) def landmarks(self, img): """ get landmarks """ return self.faceAlignment.get_landmarks(img)[0] def geometricFeatures1(self, img, landmarks=None): """Improved Performance in Facial Expression Recognition Using 32 Geometric Features Linear features(15): – 3 for left eyebrow – 2 for left eye – 1 for cheeks – 1 for nose – 8 for mouth Polygonal features(3): – 1 for the left eye – 1 between corners of left eye and left corner of mouth – 1 for mouth Elliptical features(7): – 1 for left eyebrow – 3 for the left eye: eye, upper and lower eyelids – 3 for mouth: upper and lower lips Slope features(7): – 1 for left eyebrow – 6 for mouth corners """ pts = self.faceAlignment.get_landmarks( img)[0] if landmarks is None else landmarks result = [] # Linear features #eyebrow result.append(d(pts[21], pts[22])) result.append(d(pts[22], pts[42])) result.append(d(pts[26], pts[45])) #left eye result.append(d(pts[42], pts[45])) #d2=43-47 || 44-46 result.append(d(pts[43], pts[47])) #cheeks result.append(d(pts[1], pts[15])) #nose result.append(d(pts[31], pts[35])) #mouth result.append(d(pts[48], pts[51])) result.append(d(pts[51], pts[54])) result.append(d(pts[62], pts[66])) result.append(d(pts[51], pts[57])) result.append(d(pts[50], pts[33])) result.append(d(pts[52], pts[33])) result.append(d(pts[48], pts[31])) result.append(d(pts[54], pts[35])) #Polygonal features: result.append(FaceTools._polyArea([pts[48], pts[54], pts[57]])) result.append(FaceTools._polyArea([pts[54], pts[42], pts[45]])) result.append(FaceTools._polyArea([pts[42], pts[22], pts[26], pts[45]])) #Slope features: result.append(FaceTools._slope(pts[22], pts[26])) result.append(FaceTools._slope(pts[48], pts[31])) result.append(FaceTools._slope(pts[54], pts[35])) result.append(FaceTools._slope(pts[48], pts[51])) result.append(FaceTools._slope(pts[51], pts[54])) result.append(FaceTools._slope(pts[54], pts[57])) result.append(FaceTools._slope(pts[48], pts[57])) return result def faceDistances(self, img, landmarks=None): """ calculate distances as described in 'Automatic Facial Expression Recognition Using Combined Geometric Features': D1 Left eyebrow length D2 Right eyebrow length D3 Distance between left and right eyebrow D4 Left eye height D5 Left eye width D6 Right eye height D7 Right eye width D8 Distance between left eyebrow and left eye D9 Distance between right eyebrow and right eye D10 Distance between nose tip and upper lip D11 Lip width D12 Lip height D13 Inner lip distance D14 Distance between left eye corner and lip left corner D15 Distance between right eye corner and lip right corner """ pts = self.faceAlignment.get_landmarks( img)[0] if landmarks is None else landmarks distances = [ d(pts[17], pts[18]) + d(pts[18], pts[19]) + d(pts[19], pts[20]) + \ d(pts[20], pts[21]), d(pts[22], pts[23]) + d(pts[23], pts[24]) + d(pts[24], pts[25]) + \ d(pts[25], pts[26]), d(pts[21], pts[22]), d(pts[40], pts[38]), d(pts[36], pts[39]), d(pts[43], pts[47]), d(pts[42], pts[45]), d(pts[19], pts[37]), d(pts[24], pts[44]), d(pts[33], pts[51]), d(pts[48], pts[54]), d(pts[51], pts[57]), d(pts[62], pts[66]), d(pts[36], pts[48]), d(pts[45], pts[54]) ] return distances