def initialize_face_bank_retinaface(img, retinaface_res_path):
embeddings = []
names = ["Unknown"]
emb_counts = []
# get retinaface detections
refrence = get_reference_facial_points(default_square=True)
retinaface_res = np.loadtxt(retinaface_res_path)
info = retinaface_res[retinaface_res[:, 0] == 1]
if info.shape[0] > 0:
bboxes = info[:, 1:5]
landmarks = info[:, 5:]
# Warp faces
faces = []
for i, landmark in enumerate(landmarks):
facial5points = [[landmark[j], landmark[j + 1]]
for j in range(0, 10, 2)]
warped_face = warp_and_crop_face(np.array(img),
facial5points,
refrence,
crop_size=(112, 112))
faces.append(Image.fromarray(warped_face))
# Extracting face embeddings
for i, img in enumerate(faces):
emb = learner.model(
conf.test_transform(img).to(conf.device).unsqueeze(0))
embeddings.append(emb)
names.append("{:02d}".format(i))
emb_counts.append(1)
embeddings = torch.cat(embeddings)
return embeddings, names, emb_counts, bboxes
return None
def find_retinaface_detections(img, retinaface_res_path, im_path):
print("loading from {:s}".format(retinaface_res_path))
refrence = get_reference_facial_points(default_square=True)
frame = int(im_path.replace(".jpg", ""))
retinaface_res = np.loadtxt(retinaface_res_path)
info = retinaface_res[retinaface_res[:, 0] == frame]
if info.shape[0] > 0:
bboxes = info[:, 1:5]
landmarks = info[:, 5:]
faces = []
for i, landmark in enumerate(landmarks):
facial5points = [[landmark[j], landmark[j + 1]]
for j in range(0, 10, 2)]
warped_face = warp_and_crop_face(np.array(img),
facial5points,
refrence,
crop_size=(112, 112))
im_warped = Image.fromarray(warped_face)
os.makedirs(osp.join(image_root, "debug"), exist_ok=True)
im_warped.save(
osp.join(image_root, "debug",
"face_warped_{}_{}.jpg".format(frame, i)))
faces.append(Image.fromarray(warped_face))
else:
bboxes = []
faces = []
landmarks = []
print(bboxes)
return bboxes, faces, landmarks
def align(self, img):
_, landmarks = self.detect_faces(img)
if landmarks is not None and len(landmarks)==1:
facial5points = [[landmarks[0][j],landmarks[0][j+5]] for j in range(5)]
warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=(112,112))
return Image.fromarray(warped_face)
else: return None
def align_best(self, img, limit=None, min_face_size=20., **kwargs):
try:
boxes, landmarks = self.detect_faces(img, min_face_size,)
img = to_numpy(img)
if limit:
boxes = boxes[:limit]
landmarks = landmarks[:limit]
nrof_faces = len(boxes)
boxes = np.asarray(boxes)
if nrof_faces > 0:
det = boxes[:, 0:4]
img_size = np.asarray(img.shape)[0:2]
bindex = 0
if nrof_faces > 1:
bounding_box_size = (det[:, 2] - det[:, 0]) * (det[:, 3] - det[:, 1])
img_center = img_size / 2
offsets = np.vstack(
[(det[:, 0] + det[:, 2]) / 2 - img_center[1], (det[:, 1] + det[:, 3]) / 2 - img_center[0]])
offset_dist_squared = np.sum(np.power(offsets, 2.0), 0)
bindex = np.argmax(
bounding_box_size - offset_dist_squared * 2.0) # some extra weight on the centering
boxes = boxes[bindex, 0:4]
landmarks = landmarks[bindex, :]
facial5points = [[landmarks[j], landmarks[j + 5]] for j in range(5)]
warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=(112, 112))
return to_image(warped_face)
else:
logging.warning(f'no face detected, {kwargs} ')
return to_image(img).resize((112, 112), Image.BILINEAR)
except Exception as e:
logging.warning(f'face detect fail, err {e}')
return to_image(img).resize((112, 112), Image.BILINEAR)
def align_fully(self,
img,
crop_size=(112, 112),
return_trans_inv=False,
ori=[0, 1, 3],
fast_mode=True):
ori_size = img.copy()
h = img.size[1]
w = img.size[0]
sw = 320. if fast_mode else w
scale = sw / w
img = img.resize((int(w * scale), int(h * scale)))
candi = []
for i in ori:
if len(candi) > 0:
break
if i > 0:
rimg = img.transpose(i + 1)
else:
rimg = img
box, landmarks = self.detect_faces(rimg,
min_face_size=sw / 10,
thresholds=[0.6, 0.7, 0.7])
box, landmarks = self.detect_faces(rimg,
min_face_size=sw / 10,
thresholds=[0.61, 0.71, 0.75])
landmarks /= scale
if len(landmarks) == 0:
continue
if i == 0:
f5p = [[landmarks[0][j], landmarks[0][j + 5]]
for j in range(5)]
elif i == 1:
f5p = [[w - 1 - landmarks[0][j + 5], landmarks[0][j]]
for j in range(5)]
elif i == 2:
f5p = [[w - 1 - landmarks[0][j], h - 1 - landmarks[0][j + 5]]
for j in range(5)]
elif i == 3:
f5p = [[landmarks[0][j + 5], h - 1 - landmarks[0][j]]
for j in range(5)]
candi.append((box[0][4], f5p))
if len(candi) == 0:
return None if not return_trans_inv else (None, None)
while len(candi) > 1:
if candi[0][0] > candi[1][0]:
del candi[1]
else:
del candi[0]
facial5points = candi[0][1]
warped_face = warp_and_crop_face(np.array(ori_size),
facial5points,
self.refrence,
crop_size=crop_size,
return_trans_inv=return_trans_inv)
if return_trans_inv:
return Image.fromarray(warped_face[0]), warped_face[1]
else:
return Image.fromarray(warped_face)
def align(self, img):
bounding_boxes, landmarks = self.detect_faces(img)
facial5points = [[landmarks[0][j], landmarks[0][j + 5]]
for j in range(5)]
warped_face = warp_and_crop_face(np.array(img),
facial5points,
self.refrence,
crop_size=(112, 112))
return Image.fromarray(warped_face), bounding_boxes[0]
def align_multi_with_landmarks(self, img, limit=None, min_face_size=30.0, thresholds=[0.6, 0.7, 0.8]):
boxes, landmarks = self.detect_faces(img, min_face_size, thresholds=thresholds)
if limit:
boxes = boxes[:limit]
landmarks = landmarks[:limit]
faces = []
for landmark in landmarks:
facial5points = [[landmark[j],landmark[j+5]] for j in range(5)]
warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=(112,112))
faces.append(Image.fromarray(warped_face))
return boxes, faces, landmarks
def align(self, img, crop_size=(112, 112), return_trans_inv=False):
_, landmarks = self.detect_faces(img)
if len(landmarks) == 0:
return None if not return_trans_inv else (None, None)
facial5points = [[landmarks[0][j],landmarks[0][j+5]] for j in range(5)]
warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=crop_size,
return_trans_inv=return_trans_inv)
if return_trans_inv:
return Image.fromarray(warped_face[0]), warped_face[1]
else:
return Image.fromarray(warped_face)
def align_multi(self, img, limit=None, min_face_size=30.0):
boxes, landmarks = self.detect_faces(img, min_face_size)
if limit:
boxes = boxes[:limit]
landmarks = landmarks[:limit]
faces = []
target_faces = []
for landmark in landmarks:
facial5points = [[landmark[j], landmark[j + 5]] for j in range(5)]
warped_face = warp_and_crop_face(np.array(img),
facial5points,
self.refrence,
crop_size=(112, 112))
target_face = warp_and_crop_face(np.array(img),
facial5points,
self.refrence,
crop_size=(256, 256))
faces.append(Image.fromarray(warped_face))
target_faces.append(Image.fromarray(target_face))
return boxes, faces, target_faces
def get_landmarks(self, img, min_face_size=32, crop_size=(256, 256), fast_mode=False, ori=[0,1,3]):
ori_size = img.copy()
h = img.size[1]
w = img.size[0]
sw = 640. if fast_mode else w
scale = sw / w
img = img.resize((int(w*scale), int(h*scale)))
min_face_size = min_face_size if not fast_mode else sw/20
candi = []
boxes = np.zeros([0, 5])
for i in ori:
if i > 0:
rimg = img.transpose(i+1)
else:
rimg = img
box, landmarks = self.detect_faces(rimg, min_face_size=min_face_size, thresholds=[0.6, 0.7, 0.7])
landmarks /= scale
if len(landmarks) == 0:
continue
if i == 0:
f5p = [[landmarks[0][j], landmarks[0][j + 5]] for j in range(5)]
elif i == 1:
f5p = [[w-1-landmarks[0][j+5], landmarks[0][j]] for j in range(5)]
x1 = w-1-box[:, 1]
y1 = box[:, 0]
x2 = w-1-box[:, 3]
y2 = box[:, 2]
box[:, :4] = np.stack((x2, y1, x1, y2), axis=1)
elif i == 2:
f5p = [[w-1-landmarks[0][j], h-1-landmarks[0][j+5]] for j in range(5)]
x1 = w-1-box[:, 0]
y1 = h-1-box[:, 1]
x2 = w-1-box[:, 2]
y2 = h-1-box[:, 3]
box[:, :4] = np.stack((x2, y2, x1, y1), axis=1)
elif i == 3:
f5p = [[landmarks[0][j + 5], h-1-landmarks[0][j]] for j in range(5)]
x1 = box[:, 1]
y1 = h-1-box[:, 0]
x2 = box[:, 3]
y2 = h-1-box[:, 2]
box[:, :4] = np.stack((x1, y2, x2, y1), axis=1)
candi.append(f5p)
boxes = np.concatenate((boxes, box), axis=0)
# pick = nms(boxes)
faces = []
for idx, facial5points in enumerate(candi):
# if idx not in pick:
# continue
warped_face = warp_and_crop_face(np.array(ori_size), facial5points, self.refrence, crop_size=crop_size,
return_trans_inv=False)
faces.append((warped_face, facial5points))
return faces
def align_and_take_one(self, img, limit=None, min_face_size=30.0):
boxes, landmarks = self.detect_faces(img, min_face_size)
if len(landmarks) == 0:
return None
if len(boxes) != 1:
return None
facial5points = [[landmarks[0][j], landmarks[0][j + 5]]
for j in range(5)]
warped_face = warp_and_crop_face(np.array(img),
facial5points,
self.refrence,
crop_size=(112, 112))
return Image.fromarray(warped_face)
def align_multi(self, img, limit=None, min_face_size=30.0):
boxes, landmarks = self.detect_faces(img, min_face_size)
if limit:
boxes = boxes[:limit]
landmarks = landmarks[:limit]
faces = []
for landmark in landmarks:
facial5points = [[landmark[j], landmark[j + 1]]
for j in range(0, 10, 2)]
warped_face = warp_and_crop_face(np.array(img),
facial5points,
self.refrence,
crop_size=(112, 112))
#cv2.imshow("faceCrop",warped_face)
faces.append(Image.fromarray(warped_face))
return boxes, faces
def align(self, frame):
img = Image.fromarray(frame[..., ::-1])
boxes, landmarks = self.detect_faces(img)
# Early return for no faces case
if landmarks is None:
return None, None
elif len(landmarks) == 0:
return None, None
facial5points = [[landmarks[0][j], landmarks[0][j + 5]]
for j in range(5)]
warped_face = warp_and_crop_face(np.array(img),
facial5points,
self.refrence,
crop_size=(112, 112))
return boxes, Image.fromarray(warped_face)
def align_multi(self, img, limit=None, min_face_size=64.0, crop_size=(112, 112)):
boxes, landmarks = self.detect_faces(img, min_face_size)
# print('test')
if len(landmarks) == 0:
# print('f**k')
return None
if limit:
boxes = boxes[:limit]
landmarks = landmarks[:limit]
faces = []
for landmark in landmarks:
facial5points = [[landmark[j],landmark[j+5]] for j in range(5)]
warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=crop_size)
faces.append(Image.fromarray(warped_face))
# print('test')
# return boxes, faces
return faces
def align_multi(self, img, limit=None, min_face_size=30.0):
boxes, landmarks = self.detect_faces(img, min_face_size)
img_bounding = show_bboxes(img, boxes, landmarks)
#img_bounding.show()
#input("image")
if limit:
boxes = boxes[:limit]
landmarks = landmarks[:limit]
faces = []
for landmark in landmarks:
facial5points = [[landmark[j], landmark[j + 5]] for j in range(5)]
warped_face = warp_and_crop_face(np.array(img),
facial5points,
self.refrence,
crop_size=(112, 112))
faces.append(Image.fromarray(warped_face))
return boxes, faces
def align_multi(self, img, limit=None, min_face_size=30.0):
#print('before detect_faces')
boxes, landmarks = self.detect_faces(img, min_face_size)
#print('after detect_faces')
print(boxes)
if limit:
boxes = boxes[:limit]
landmarks = landmarks[:limit]
faces = []
for landmark in landmarks:
facial5points = [[landmark[j], landmark[j + 5]] for j in range(5)]
#print('before warp_and_crop_face')
warped_face = warp_and_crop_face(np.array(img),
facial5points,
self.refrence,
crop_size=(112, 112))
#print('after warp_and_crop_face')
faces.append(Image.fromarray(warped_face))
return boxes, faces
def construct_face_bank(seq: str, retina_face_root: str, image_root: str, data_root: str, learner: face_learner, verbose: bool):
"""
loop through all images under the seq folder and count the number of distinct face embeddings
:param seq: sequence id
:param retina_face_root: root dir for retinaface detections. (frame, face_bbox (4), landmarks (10))
:param image_root: root dir for images
:param openface_root: root dir for openface results
:param learner: a face verification network
:return:
"""
seq_images = sorted(os.listdir(image_root))
seq_retina_face_result = np.loadtxt(retina_face_root, dtype=str)
embeddings = []
names = ["Unknown"]
emb_counts = []
# get default reference face points
reference = get_reference_facial_points(default_square=True)
for frame_idx, frame in enumerate(seq_images):
if verbose:
print("Processing {} {}".format(frame_idx, frame))
print("Names: {}".format(names))
print("emb_counts: {}".format(emb_counts))
# 1. load data
# 1.1. read image
img = Image.open(osp.join(image_root, frame))
# 1.2. get the retinaface detections
retinaface_result = seq_retina_face_result[seq_retina_face_result[:, 0] == frame]
# skip if no detection
if len(retinaface_result) == 0:
print("No retinaface")
continue
retinaface_bboxes = retinaface_result[:, 1:5].astype(np.int)
landmarks = retinaface_result[:, 5:].astype(np.float32)
retinaface_bboxes = retinaface_bboxes
# Warp faces: preparing input for learner
faces = []
for i, landmark in enumerate(landmarks):
facial5points = [[landmark[j], landmark[j + 1]] for j in range(0, 10, 2)]
warped_face = warp_and_crop_face(np.array(img), facial5points, reference, crop_size=(112, 112))
faces.append(Image.fromarray(warped_face))
# if the very first frame, we initialize face bank
if len(embeddings) == 0:
# Extracting face embeddings
for i, img in enumerate(faces):
with torch.no_grad():
emb = learner.model(conf.test_transform(img).to(conf.device).unsqueeze(0))
embeddings.append(emb)
names.append("{:02d}".format(i))
emb_counts.append(1)
embeddings = torch.cat(embeddings)
else: # otherwise we try to match, and update the face bank
with torch.no_grad():
print(len(faces))
results, score, source_embs = learner.infer_embeddings(conf, faces, embeddings, True)
# udpate facebank
for i, idx in enumerate(results):
if idx != -1: # we find a match, smooth the previous embeddings
embeddings[idx] = (emb_counts[idx] * embeddings[idx] + source_embs[i]) / (emb_counts[idx] + 1)
emb_counts[idx] += 1
else: # otherwise this is a new face
embeddings = torch.cat((embeddings, source_embs[i].unsqueeze(0)), dim=0)
emb_counts.append(1)
names.append("{:02d}".format(int(names[-1]) + 1))
results[i] = len(embeddings) - 1
np.savez(osp.join(data_root, "{}.npz".format(seq)),
embeddings=embeddings.cpu().numpy(), names=names, emb_counts=emb_counts)
return embeddings, names, emb_counts
