def __init__(self, model = 'PLFD', checkpoint = os.path.join(BASE_DIR, 'thirdparty', \ 'pytorch_face_landmark','checkpoint','pfld_model_best.pth.tar'), \ detector = 'MTCNN'): import torch sys.path.append(os.path.join(BASE_DIR,'thirdparty','pytorch_face_landmark')) if model == 'MobileNet': from models.basenet import MobileNet_GDConv self.model = MobileNet_GDConv(136) self.model = torch.nn.DataParallel(self.model) self.model.load_state_dict(torch.load(checkpoint)['state_dict']) self.model.eval() self.size = 224 elif model == 'MobileFaceNet': from models.mobilefacenet import MobileFaceNet self.model = MobileFaceNet([112, 112], 136) self.model.load_state_dict(torch.load(checkpoint)['state_dict']) self.model.eval() self.size = 112 elif model == 'PLFD': from models.pfld_compressed import PFLDInference self.model = PFLDInference() self.model.load_state_dict(torch.load(checkpoint)['state_dict']) self.model.eval() self.size = 112 if detector == 'MTCNN': from MTCNN import detect_faces self.detect_fun = lambda x: detect_faces(x[:,:,::-1]) elif detector == 'FaceBoxes': from FaceBoxes import FaceBoxes self.detector = FaceBoxes() self.detect_fun = lambda x: self.detector.face_boxex(x) elif detector == 'Retinaface': from Retinaface import Retinaface self.detector = Retinaface.Retinaface() self.detect_fun = lambda x: self.detector(x) else: import dlib self.detector = dlib.get_frontal_face_detector() self.detect_fun = lambda x: self.detector(cv2.cvtColor(x,cv2.COLOR_BGR2GRAY))
def face_swap(img, swap_area): img = cv2.resize(img, dsize=(0, 0), fx=2, fy=2, interpolation=cv2.INTER_LINEAR) height, width, channels = img.shape img_new_face = np.zeros((height, width, channels), np.uint8) img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) out_size = 112 model = PFLDInference() checkpoint = torch.load('checkpoint/pfld_model_best.pth.tar', map_location='cpu') model.load_state_dict(checkpoint['state_dict']) model = model.eval() for k in range(0, len(swap_area)): #face swap할 이미지 분석 x1 = swap_area[k][0]*2 y1 = swap_area[k][1]*2 x2 = swap_area[k][2]*2 y2 = swap_area[k][3]*2 dx = max(0, -x1) dy = max(0, -y1) x1 = max(0, x1) y1 = max(0, y1) edx = max(0, x2 - width) edy = max(0, y2 - height) new_bbox = list(map(int, [x1, x2, y1, y2])) new_bbox = BBox(new_bbox) cropped = img[new_bbox.top:new_bbox.bottom, new_bbox.left:new_bbox.right] if (dx > 0 or dy > 0 or edx > 0 or edy > 0): cropped = cv2.copyMakeBorder(cropped, int(dy), int(edy), int(dx), int(edx), cv2.BORDER_CONSTANT, 0) cropped_face = cv2.resize(cropped, (out_size, out_size)) if cropped_face.shape[0] <= 0 or cropped_face.shape[1] <= 0: continue test_face = cropped_face.copy() test_face = test_face / 255.0 test_face = test_face.transpose((2, 0, 1)) test_face = test_face.reshape((1,) + test_face.shape) input = torch.from_numpy(test_face).float() input = torch.autograd.Variable(input) landmark = model(input).cpu().data.numpy() landmark = landmark.reshape(-1, 2) landmark = new_bbox.reprojectLandmark(landmark) points = np.array(landmark, np.int32) convexhull = cv2.convexHull(points) landmarks_points = [] for x, y in landmark: landmarks_points.append(( int(x), int(y) )) img2 = cv2.imread("samples/12--Group/newface2.jpg") height2, width2, _ = img2.shape img2 = cv2.resize(img2, dsize=(0, 0), fx=2, fy=2, interpolation=cv2.INTER_LINEAR) img_gray2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY) mask = np.zeros_like(img_gray2) new_face = cv2.cvtColor(img2, cv2.COLOR_BGR2RGB) new_face = Image.fromarray(new_face) face2 = list(map(int, detect_faces(new_face)[0])) new_bbox2 = list(map(int, [face2[0], face2[2], face2[1], face2[3]])) new_bbox2 = BBox(new_bbox2) cropped2 = img2[new_bbox2.top:new_bbox2.bottom, new_bbox2.left:new_bbox2.right] cropped2_face = cv2.resize(cropped2, (out_size, out_size)) test_face2 = cropped2_face.copy() test_face2 = test_face2 / 255.0 test_face2 = test_face2.transpose((2, 0, 1)) test_face2 = test_face2.reshape((1,) + test_face2.shape) input2 = torch.from_numpy(test_face2).float() input2 = torch.autograd.Variable(input2) landmark2 = model(input2).cpu().data.numpy() landmark2 = landmark2.reshape(-1, 2) landmark2 = new_bbox2.reprojectLandmark(landmark2) points2 = np.array(landmark2, np.int32) convexhull2 = cv2.convexHull(points2) cv2.fillConvexPoly(mask, convexhull2, 255) rect2 = cv2.boundingRect(convexhull2) subdiv = cv2.Subdiv2D(rect2) landmarks_points2 = [] for x, y in landmark2: landmarks_points2.append((int(x), int(y))) subdiv.insert(landmarks_points2) triangles = subdiv.getTriangleList() triangles = np.array(triangles, dtype=np.int32) indexes_triangles = [] for t in triangles: pt1 = (t[0], t[1]) pt2 = (t[2], t[3]) pt3 = (t[4], t[5]) index_pt1 = np.where((points2 == pt1).all(axis=1)) index_pt1 = extract_index_nparray(index_pt1) index_pt2 = np.where((points2 == pt2).all(axis=1)) index_pt2 = extract_index_nparray(index_pt2) index_pt3 = np.where((points2 == pt3).all(axis=1)) index_pt3 = extract_index_nparray(index_pt3) if index_pt1 is not None and index_pt2 is not None and index_pt3 is not None: triangle = [index_pt1, index_pt2, index_pt3] indexes_triangles.append(triangle) img_face_mask = np.zeros_like(img_gray) img_head_mask = cv2.fillConvexPoly(img_face_mask, convexhull, 255) for triangle_index in indexes_triangles: # Triangulation of the first face2 tr2_pt1 = landmarks_points2[triangle_index[0]] tr2_pt2 = landmarks_points2[triangle_index[1]] tr2_pt3 = landmarks_points2[triangle_index[2]] triangle2 = np.array([tr2_pt1, tr2_pt2, tr2_pt3], np.int32) rect2 = cv2.boundingRect(triangle2) (x, y, w, h) = rect2 cropped_triangle = img2[y: y + h, x: x + w] cropped_tr2_mask = np.zeros((h, w), np.uint8) points2 = np.array([[tr2_pt1[0] - x, tr2_pt1[1] - y], [tr2_pt2[0] - x, tr2_pt2[1] - y], [tr2_pt3[0] - x, tr2_pt3[1] - y]], np.int32) cv2.fillConvexPoly(cropped_tr2_mask, points2, 255) # Triangulation of second face2 tr1_pt1 = landmarks_points[triangle_index[0]] tr1_pt2 = landmarks_points[triangle_index[1]] tr1_pt3 = landmarks_points[triangle_index[2]] triangle = np.array([tr1_pt1, tr1_pt2, tr1_pt3], np.int32) rect1 = cv2.boundingRect(triangle) (x, y, w, h) = rect1 cropped_tr1_mask = np.zeros((h, w), np.uint8) points = np.array([[tr1_pt1[0] - x, tr1_pt1[1] - y], [tr1_pt2[0] - x, tr1_pt2[1] - y], [tr1_pt3[0] - x, tr1_pt3[1] - y]], np.int32) cv2.fillConvexPoly(cropped_tr1_mask, points, 255) # Warp triangles points2 = np.float32(points2) points = np.float32(points) M = cv2.getAffineTransform(points2, points) warped_triangle = cv2.warpAffine(cropped_triangle, M, (w, h)) warped_triangle = cv2.bitwise_and(warped_triangle, warped_triangle, mask=cropped_tr1_mask) # Reconstructing destination face2 img_new_face_rect_area = img_new_face[y: y + h, x: x + w] img_new_face_rect_area_gray = cv2.cvtColor(img_new_face_rect_area, cv2.COLOR_BGR2GRAY) _, mask_triangles_designed = cv2.threshold(img_new_face_rect_area_gray, 1, 255, cv2.THRESH_BINARY_INV) warped_triangle = cv2.bitwise_and(warped_triangle, warped_triangle, mask=mask_triangles_designed) img_new_face_rect_area = cv2.add(img_new_face_rect_area, warped_triangle) img_new_face[y: y + h, x: x + w] = img_new_face_rect_area img_face_mask = cv2.bitwise_not(img_head_mask) img_head_noface = cv2.bitwise_and(img, img, mask=img_face_mask) img_new_face = cv2.medianBlur(img_new_face, 3) result = cv2.add(img_head_noface, img_new_face) (x, y, w, h) = cv2.boundingRect(convexhull) center_face = (int((x + x + w) / 2), int((y + y + h) / 2)) img = cv2.seamlessClone(result, img, img_head_mask, center_face, cv2.MIXED_CLONE) img = cv2.resize(img, dsize=(0, 0), fx=0.5, fy=0.5, interpolation=cv2.INTER_LINEAR) # cv2.imwrite(os.path.join('results', "6.jpg"), img) # cv2.imshow("a", img) # cv2.waitKey(0) return img
def face_detection(img, friends): faces_area=[] faces_crop=[] friend_dic={} friend_list=[] friend_crop_img_list=[] for friend in friends: friend_crop_img_list.append(fr.face_encodings(cv2.imread(change_url_to_absolute(friend['img_url'])))) height,width,_=img.shape # perform face detection using MTCNN image = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) image = Image.fromarray(image) faces = detect_faces(image) if len(faces)==0: print('NO face is detected!') return for k, face in enumerate(faces): x1=face[0] y1=face[1] x2=face[2] y2=face[3] w = x2 - x1 + 1 h = y2 - y1 + 1 size = int(min([w, h])*1.2) cx = x1 + w//2 cy = y1 + h//2 x1 = cx - size//2 x2 = x1 + size y1 = cy - size//2 y2 = y1 + size x1 = int(max(0, x1)) y1 = int(max(0, y1)) x2 = int(min(width, x2)) y2 = int(min(height, y2)) img_crop=img[y1:y2, x1:x2] faces_crop.append(img_crop) faces_area.append((x1, y1, x2, y2)) # 등록되지 않은 얼굴을 128-dimensional face 인코딩 enc_unknown_face = fr.face_encodings(img_crop, known_face_locations=[(0, x2-x1, y2-y1, 0)]) for idx, friend in enumerate(friends): # print(friend) # 등록된 얼굴을 128-dimensional face 인코딩 # enc_known_face = fr.face_encodings(friend_list[0]) enc_known_face = friend_crop_img_list[idx] # 등록된 얼굴과 새로운 얼굴의 distance를 얻기 (유사도) # 1에 가까울수록 다른 사람 얼굴 이며, 보통 0.5이상이면 다른사람으로 인식한다. distance = fr.face_distance(enc_known_face, enc_unknown_face[0]) # print(distance) # 제약을 0.35로 하여 친구 얼굴이 확실 할때만 체크한다. if distance < 0.35: if friend['fname'] in friend_dic: friend_dic[friend['fname']] = {'distance': min(friend_dic[friend['fname']]['distance'], distance), 'square':list(map(int, [x1, y1, x2, y2])), 'idx':k} else: friend_dic[friend['fname']] = {'distance': distance, 'square':list(map(int, [x1, y1, x2, y2])), 'idx':k} # print(distance) # 친구라고 인식된 얼굴 보기 # for friend in friend_dic: # print(friend) # plt.imshow(cv2.cvtColor(img[ceil(friend_dic[friend]['square'][1]):ceil(friend_dic[friend]['square'][3]),ceil(friend_dic[friend]['square'][0]):ceil(friend_dic[friend]['square'][2])], cv2.COLOR_RGB2BGR)) # plt.show() for friend in friend_dic: friend_list.append({'fname': friend, 'square': friend_dic[friend]['square'], 'idx':friend_dic[friend]['idx']}) return faces_area, faces_crop, friend_list
if args.backbone == 'MobileNet': out_size = 224 else: out_size = 112 model = load_model() model = model.eval() filenames = glob.glob("samples/12--Group/*.jpg") for imgname in filenames: print(imgname) img = cv2.imread(imgname) org_img = Image.open(imgname) height, width, _ = img.shape if args.detector == 'MTCNN': # perform face detection using MTCNN image = Image.open(imgname) faces, landmarks = detect_faces(image) elif args.detector == 'FaceBoxes': face_boxes = FaceBoxes() faces = face_boxes(img) elif args.detector == 'Retinaface': retinaface = Retinaface.Retinaface() faces = retinaface(img) else: print('Error: not suppored detector') ratio = 0 if len(faces) == 0: print('NO face is detected!') continue for k, face in enumerate(faces): if face[4] < 0.9: # remove low confidence detection continue