def __init__(self, model = 'PLFD', checkpoint = os.path.join(BASE_DIR, 'thirdparty', \ 'pytorch_face_landmark','checkpoint','pfld_model_best.pth.tar'), \ detector = 'MTCNN'): import torch sys.path.append(os.path.join(BASE_DIR,'thirdparty','pytorch_face_landmark')) if model == 'MobileNet': from models.basenet import MobileNet_GDConv self.model = MobileNet_GDConv(136) self.model = torch.nn.DataParallel(self.model) self.model.load_state_dict(torch.load(checkpoint)['state_dict']) self.model.eval() self.size = 224 elif model == 'MobileFaceNet': from models.mobilefacenet import MobileFaceNet self.model = MobileFaceNet([112, 112], 136) self.model.load_state_dict(torch.load(checkpoint)['state_dict']) self.model.eval() self.size = 112 elif model == 'PLFD': from models.pfld_compressed import PFLDInference self.model = PFLDInference() self.model.load_state_dict(torch.load(checkpoint)['state_dict']) self.model.eval() self.size = 112 if detector == 'MTCNN': from MTCNN import detect_faces self.detect_fun = lambda x: detect_faces(x[:,:,::-1]) elif detector == 'FaceBoxes': from FaceBoxes import FaceBoxes self.detector = FaceBoxes() self.detect_fun = lambda x: self.detector.face_boxex(x) elif detector == 'Retinaface': from Retinaface import Retinaface self.detector = Retinaface.Retinaface() self.detect_fun = lambda x: self.detector(x) else: import dlib self.detector = dlib.get_frontal_face_detector() self.detect_fun = lambda x: self.detector(cv2.cvtColor(x,cv2.COLOR_BGR2GRAY))
def face_swap(img, swap_area):
img = cv2.resize(img, dsize=(0, 0), fx=2, fy=2, interpolation=cv2.INTER_LINEAR)
height, width, channels = img.shape
img_new_face = np.zeros((height, width, channels), np.uint8)
img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
out_size = 112
model = PFLDInference()
checkpoint = torch.load('checkpoint/pfld_model_best.pth.tar', map_location='cpu')
model.load_state_dict(checkpoint['state_dict'])
model = model.eval()
for k in range(0, len(swap_area)):
#face swap할 이미지 분석
x1 = swap_area[k][0]*2
y1 = swap_area[k][1]*2
x2 = swap_area[k][2]*2
y2 = swap_area[k][3]*2
dx = max(0, -x1)
dy = max(0, -y1)
x1 = max(0, x1)
y1 = max(0, y1)
edx = max(0, x2 - width)
edy = max(0, y2 - height)
new_bbox = list(map(int, [x1, x2, y1, y2]))
new_bbox = BBox(new_bbox)
cropped = img[new_bbox.top:new_bbox.bottom, new_bbox.left:new_bbox.right]
if (dx > 0 or dy > 0 or edx > 0 or edy > 0):
cropped = cv2.copyMakeBorder(cropped, int(dy), int(edy), int(dx), int(edx), cv2.BORDER_CONSTANT, 0)
cropped_face = cv2.resize(cropped, (out_size, out_size))
if cropped_face.shape[0] <= 0 or cropped_face.shape[1] <= 0:
continue
test_face = cropped_face.copy()
test_face = test_face / 255.0
test_face = test_face.transpose((2, 0, 1))
test_face = test_face.reshape((1,) + test_face.shape)
input = torch.from_numpy(test_face).float()
input = torch.autograd.Variable(input)
landmark = model(input).cpu().data.numpy()
landmark = landmark.reshape(-1, 2)
landmark = new_bbox.reprojectLandmark(landmark)
points = np.array(landmark, np.int32)
convexhull = cv2.convexHull(points)
landmarks_points = []
for x, y in landmark:
landmarks_points.append(( int(x), int(y) ))
img2 = cv2.imread("samples/12--Group/newface2.jpg")
height2, width2, _ = img2.shape
img2 = cv2.resize(img2, dsize=(0, 0), fx=2, fy=2, interpolation=cv2.INTER_LINEAR)
img_gray2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)
mask = np.zeros_like(img_gray2)
new_face = cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)
new_face = Image.fromarray(new_face)
face2 = list(map(int, detect_faces(new_face)[0]))
new_bbox2 = list(map(int, [face2[0], face2[2], face2[1], face2[3]]))
new_bbox2 = BBox(new_bbox2)
cropped2 = img2[new_bbox2.top:new_bbox2.bottom, new_bbox2.left:new_bbox2.right]
cropped2_face = cv2.resize(cropped2, (out_size, out_size))
test_face2 = cropped2_face.copy()
test_face2 = test_face2 / 255.0
test_face2 = test_face2.transpose((2, 0, 1))
test_face2 = test_face2.reshape((1,) + test_face2.shape)
input2 = torch.from_numpy(test_face2).float()
input2 = torch.autograd.Variable(input2)
landmark2 = model(input2).cpu().data.numpy()
landmark2 = landmark2.reshape(-1, 2)
landmark2 = new_bbox2.reprojectLandmark(landmark2)
points2 = np.array(landmark2, np.int32)
convexhull2 = cv2.convexHull(points2)
cv2.fillConvexPoly(mask, convexhull2, 255)
rect2 = cv2.boundingRect(convexhull2)
subdiv = cv2.Subdiv2D(rect2)
landmarks_points2 = []
for x, y in landmark2:
landmarks_points2.append((int(x), int(y)))
subdiv.insert(landmarks_points2)
triangles = subdiv.getTriangleList()
triangles = np.array(triangles, dtype=np.int32)
indexes_triangles = []
for t in triangles:
pt1 = (t[0], t[1])
pt2 = (t[2], t[3])
pt3 = (t[4], t[5])
index_pt1 = np.where((points2 == pt1).all(axis=1))
index_pt1 = extract_index_nparray(index_pt1)
index_pt2 = np.where((points2 == pt2).all(axis=1))
index_pt2 = extract_index_nparray(index_pt2)
index_pt3 = np.where((points2 == pt3).all(axis=1))
index_pt3 = extract_index_nparray(index_pt3)
if index_pt1 is not None and index_pt2 is not None and index_pt3 is not None:
triangle = [index_pt1, index_pt2, index_pt3]
indexes_triangles.append(triangle)
img_face_mask = np.zeros_like(img_gray)
img_head_mask = cv2.fillConvexPoly(img_face_mask, convexhull, 255)
for triangle_index in indexes_triangles:
# Triangulation of the first face2
tr2_pt1 = landmarks_points2[triangle_index[0]]
tr2_pt2 = landmarks_points2[triangle_index[1]]
tr2_pt3 = landmarks_points2[triangle_index[2]]
triangle2 = np.array([tr2_pt1, tr2_pt2, tr2_pt3], np.int32)
rect2 = cv2.boundingRect(triangle2)
(x, y, w, h) = rect2
cropped_triangle = img2[y: y + h, x: x + w]
cropped_tr2_mask = np.zeros((h, w), np.uint8)
points2 = np.array([[tr2_pt1[0] - x, tr2_pt1[1] - y],
[tr2_pt2[0] - x, tr2_pt2[1] - y],
[tr2_pt3[0] - x, tr2_pt3[1] - y]], np.int32)
cv2.fillConvexPoly(cropped_tr2_mask, points2, 255)
# Triangulation of second face2
tr1_pt1 = landmarks_points[triangle_index[0]]
tr1_pt2 = landmarks_points[triangle_index[1]]
tr1_pt3 = landmarks_points[triangle_index[2]]
triangle = np.array([tr1_pt1, tr1_pt2, tr1_pt3], np.int32)
rect1 = cv2.boundingRect(triangle)
(x, y, w, h) = rect1
cropped_tr1_mask = np.zeros((h, w), np.uint8)
points = np.array([[tr1_pt1[0] - x, tr1_pt1[1] - y],
[tr1_pt2[0] - x, tr1_pt2[1] - y],
[tr1_pt3[0] - x, tr1_pt3[1] - y]], np.int32)
cv2.fillConvexPoly(cropped_tr1_mask, points, 255)
# Warp triangles
points2 = np.float32(points2)
points = np.float32(points)
M = cv2.getAffineTransform(points2, points)
warped_triangle = cv2.warpAffine(cropped_triangle, M, (w, h))
warped_triangle = cv2.bitwise_and(warped_triangle, warped_triangle, mask=cropped_tr1_mask)
# Reconstructing destination face2
img_new_face_rect_area = img_new_face[y: y + h, x: x + w]
img_new_face_rect_area_gray = cv2.cvtColor(img_new_face_rect_area, cv2.COLOR_BGR2GRAY)
_, mask_triangles_designed = cv2.threshold(img_new_face_rect_area_gray, 1, 255, cv2.THRESH_BINARY_INV)
warped_triangle = cv2.bitwise_and(warped_triangle, warped_triangle, mask=mask_triangles_designed)
img_new_face_rect_area = cv2.add(img_new_face_rect_area, warped_triangle)
img_new_face[y: y + h, x: x + w] = img_new_face_rect_area
img_face_mask = cv2.bitwise_not(img_head_mask)
img_head_noface = cv2.bitwise_and(img, img, mask=img_face_mask)
img_new_face = cv2.medianBlur(img_new_face, 3)
result = cv2.add(img_head_noface, img_new_face)
(x, y, w, h) = cv2.boundingRect(convexhull)
center_face = (int((x + x + w) / 2), int((y + y + h) / 2))
img = cv2.seamlessClone(result, img, img_head_mask, center_face, cv2.MIXED_CLONE)
img = cv2.resize(img, dsize=(0, 0), fx=0.5, fy=0.5, interpolation=cv2.INTER_LINEAR)
# cv2.imwrite(os.path.join('results', "6.jpg"), img)
# cv2.imshow("a", img)
# cv2.waitKey(0)
return img
def face_detection(img, friends):
faces_area=[]
faces_crop=[]
friend_dic={}
friend_list=[]
friend_crop_img_list=[]
for friend in friends:
friend_crop_img_list.append(fr.face_encodings(cv2.imread(change_url_to_absolute(friend['img_url']))))
height,width,_=img.shape
# perform face detection using MTCNN
image = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
image = Image.fromarray(image)
faces = detect_faces(image)
if len(faces)==0:
print('NO face is detected!')
return
for k, face in enumerate(faces):
x1=face[0]
y1=face[1]
x2=face[2]
y2=face[3]
w = x2 - x1 + 1
h = y2 - y1 + 1
size = int(min([w, h])*1.2)
cx = x1 + w//2
cy = y1 + h//2
x1 = cx - size//2
x2 = x1 + size
y1 = cy - size//2
y2 = y1 + size
x1 = int(max(0, x1))
y1 = int(max(0, y1))
x2 = int(min(width, x2))
y2 = int(min(height, y2))
img_crop=img[y1:y2, x1:x2]
faces_crop.append(img_crop)
faces_area.append((x1, y1, x2, y2))
# 등록되지 않은 얼굴을 128-dimensional face 인코딩
enc_unknown_face = fr.face_encodings(img_crop, known_face_locations=[(0, x2-x1, y2-y1, 0)])
for idx, friend in enumerate(friends):
# print(friend)
# 등록된 얼굴을 128-dimensional face 인코딩
# enc_known_face = fr.face_encodings(friend_list[0])
enc_known_face = friend_crop_img_list[idx]
# 등록된 얼굴과 새로운 얼굴의 distance를 얻기 (유사도)
# 1에 가까울수록 다른 사람 얼굴 이며, 보통 0.5이상이면 다른사람으로 인식한다.
distance = fr.face_distance(enc_known_face, enc_unknown_face[0])
# print(distance)
# 제약을 0.35로 하여 친구 얼굴이 확실 할때만 체크한다.
if distance < 0.35:
if friend['fname'] in friend_dic:
friend_dic[friend['fname']] = {'distance': min(friend_dic[friend['fname']]['distance'], distance), 'square':list(map(int, [x1, y1, x2, y2])), 'idx':k}
else:
friend_dic[friend['fname']] = {'distance': distance, 'square':list(map(int, [x1, y1, x2, y2])), 'idx':k}
# print(distance)
# 친구라고 인식된 얼굴 보기
# for friend in friend_dic:
# print(friend)
# plt.imshow(cv2.cvtColor(img[ceil(friend_dic[friend]['square'][1]):ceil(friend_dic[friend]['square'][3]),ceil(friend_dic[friend]['square'][0]):ceil(friend_dic[friend]['square'][2])], cv2.COLOR_RGB2BGR))
# plt.show()
for friend in friend_dic:
friend_list.append({'fname': friend, 'square': friend_dic[friend]['square'], 'idx':friend_dic[friend]['idx']})
return faces_area, faces_crop, friend_list
if args.backbone == 'MobileNet':
out_size = 224
else:
out_size = 112
model = load_model()
model = model.eval()
filenames = glob.glob("samples/12--Group/*.jpg")
for imgname in filenames:
print(imgname)
img = cv2.imread(imgname)
org_img = Image.open(imgname)
height, width, _ = img.shape
if args.detector == 'MTCNN':
# perform face detection using MTCNN
image = Image.open(imgname)
faces, landmarks = detect_faces(image)
elif args.detector == 'FaceBoxes':
face_boxes = FaceBoxes()
faces = face_boxes(img)
elif args.detector == 'Retinaface':
retinaface = Retinaface.Retinaface()
faces = retinaface(img)
else:
print('Error: not suppored detector')
ratio = 0
if len(faces) == 0:
print('NO face is detected!')
continue
for k, face in enumerate(faces):
if face[4] < 0.9: # remove low confidence detection
continue