def initialize_face_bank_retinaface(img, retinaface_res_path): embeddings = [] names = ["Unknown"] emb_counts = [] # get retinaface detections refrence = get_reference_facial_points(default_square=True) retinaface_res = np.loadtxt(retinaface_res_path) info = retinaface_res[retinaface_res[:, 0] == 1] if info.shape[0] > 0: bboxes = info[:, 1:5] landmarks = info[:, 5:] # Warp faces faces = [] for i, landmark in enumerate(landmarks): facial5points = [[landmark[j], landmark[j + 1]] for j in range(0, 10, 2)] warped_face = warp_and_crop_face(np.array(img), facial5points, refrence, crop_size=(112, 112)) faces.append(Image.fromarray(warped_face)) # Extracting face embeddings for i, img in enumerate(faces): emb = learner.model( conf.test_transform(img).to(conf.device).unsqueeze(0)) embeddings.append(emb) names.append("{:02d}".format(i)) emb_counts.append(1) embeddings = torch.cat(embeddings) return embeddings, names, emb_counts, bboxes return None
def find_retinaface_detections(img, retinaface_res_path, im_path): print("loading from {:s}".format(retinaface_res_path)) refrence = get_reference_facial_points(default_square=True) frame = int(im_path.replace(".jpg", "")) retinaface_res = np.loadtxt(retinaface_res_path) info = retinaface_res[retinaface_res[:, 0] == frame] if info.shape[0] > 0: bboxes = info[:, 1:5] landmarks = info[:, 5:] faces = [] for i, landmark in enumerate(landmarks): facial5points = [[landmark[j], landmark[j + 1]] for j in range(0, 10, 2)] warped_face = warp_and_crop_face(np.array(img), facial5points, refrence, crop_size=(112, 112)) im_warped = Image.fromarray(warped_face) os.makedirs(osp.join(image_root, "debug"), exist_ok=True) im_warped.save( osp.join(image_root, "debug", "face_warped_{}_{}.jpg".format(frame, i))) faces.append(Image.fromarray(warped_face)) else: bboxes = [] faces = [] landmarks = [] print(bboxes) return bboxes, faces, landmarks
def align(self, img): _, landmarks = self.detect_faces(img) if landmarks is not None and len(landmarks)==1: facial5points = [[landmarks[0][j],landmarks[0][j+5]] for j in range(5)] warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=(112,112)) return Image.fromarray(warped_face) else: return None
def align_best(self, img, limit=None, min_face_size=20., **kwargs): try: boxes, landmarks = self.detect_faces(img, min_face_size,) img = to_numpy(img) if limit: boxes = boxes[:limit] landmarks = landmarks[:limit] nrof_faces = len(boxes) boxes = np.asarray(boxes) if nrof_faces > 0: det = boxes[:, 0:4] img_size = np.asarray(img.shape)[0:2] bindex = 0 if nrof_faces > 1: bounding_box_size = (det[:, 2] - det[:, 0]) * (det[:, 3] - det[:, 1]) img_center = img_size / 2 offsets = np.vstack( [(det[:, 0] + det[:, 2]) / 2 - img_center[1], (det[:, 1] + det[:, 3]) / 2 - img_center[0]]) offset_dist_squared = np.sum(np.power(offsets, 2.0), 0) bindex = np.argmax( bounding_box_size - offset_dist_squared * 2.0) # some extra weight on the centering boxes = boxes[bindex, 0:4] landmarks = landmarks[bindex, :] facial5points = [[landmarks[j], landmarks[j + 5]] for j in range(5)] warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=(112, 112)) return to_image(warped_face) else: logging.warning(f'no face detected, {kwargs} ') return to_image(img).resize((112, 112), Image.BILINEAR) except Exception as e: logging.warning(f'face detect fail, err {e}') return to_image(img).resize((112, 112), Image.BILINEAR)
def align_fully(self, img, crop_size=(112, 112), return_trans_inv=False, ori=[0, 1, 3], fast_mode=True): ori_size = img.copy() h = img.size[1] w = img.size[0] sw = 320. if fast_mode else w scale = sw / w img = img.resize((int(w * scale), int(h * scale))) candi = [] for i in ori: if len(candi) > 0: break if i > 0: rimg = img.transpose(i + 1) else: rimg = img box, landmarks = self.detect_faces(rimg, min_face_size=sw / 10, thresholds=[0.6, 0.7, 0.7]) box, landmarks = self.detect_faces(rimg, min_face_size=sw / 10, thresholds=[0.61, 0.71, 0.75]) landmarks /= scale if len(landmarks) == 0: continue if i == 0: f5p = [[landmarks[0][j], landmarks[0][j + 5]] for j in range(5)] elif i == 1: f5p = [[w - 1 - landmarks[0][j + 5], landmarks[0][j]] for j in range(5)] elif i == 2: f5p = [[w - 1 - landmarks[0][j], h - 1 - landmarks[0][j + 5]] for j in range(5)] elif i == 3: f5p = [[landmarks[0][j + 5], h - 1 - landmarks[0][j]] for j in range(5)] candi.append((box[0][4], f5p)) if len(candi) == 0: return None if not return_trans_inv else (None, None) while len(candi) > 1: if candi[0][0] > candi[1][0]: del candi[1] else: del candi[0] facial5points = candi[0][1] warped_face = warp_and_crop_face(np.array(ori_size), facial5points, self.refrence, crop_size=crop_size, return_trans_inv=return_trans_inv) if return_trans_inv: return Image.fromarray(warped_face[0]), warped_face[1] else: return Image.fromarray(warped_face)
def align(self, img): bounding_boxes, landmarks = self.detect_faces(img) facial5points = [[landmarks[0][j], landmarks[0][j + 5]] for j in range(5)] warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=(112, 112)) return Image.fromarray(warped_face), bounding_boxes[0]
def align_multi_with_landmarks(self, img, limit=None, min_face_size=30.0, thresholds=[0.6, 0.7, 0.8]): boxes, landmarks = self.detect_faces(img, min_face_size, thresholds=thresholds) if limit: boxes = boxes[:limit] landmarks = landmarks[:limit] faces = [] for landmark in landmarks: facial5points = [[landmark[j],landmark[j+5]] for j in range(5)] warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=(112,112)) faces.append(Image.fromarray(warped_face)) return boxes, faces, landmarks
def align(self, img, crop_size=(112, 112), return_trans_inv=False): _, landmarks = self.detect_faces(img) if len(landmarks) == 0: return None if not return_trans_inv else (None, None) facial5points = [[landmarks[0][j],landmarks[0][j+5]] for j in range(5)] warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=crop_size, return_trans_inv=return_trans_inv) if return_trans_inv: return Image.fromarray(warped_face[0]), warped_face[1] else: return Image.fromarray(warped_face)
def align_multi(self, img, limit=None, min_face_size=30.0): boxes, landmarks = self.detect_faces(img, min_face_size) if limit: boxes = boxes[:limit] landmarks = landmarks[:limit] faces = [] target_faces = [] for landmark in landmarks: facial5points = [[landmark[j], landmark[j + 5]] for j in range(5)] warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=(112, 112)) target_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=(256, 256)) faces.append(Image.fromarray(warped_face)) target_faces.append(Image.fromarray(target_face)) return boxes, faces, target_faces
def get_landmarks(self, img, min_face_size=32, crop_size=(256, 256), fast_mode=False, ori=[0,1,3]): ori_size = img.copy() h = img.size[1] w = img.size[0] sw = 640. if fast_mode else w scale = sw / w img = img.resize((int(w*scale), int(h*scale))) min_face_size = min_face_size if not fast_mode else sw/20 candi = [] boxes = np.zeros([0, 5]) for i in ori: if i > 0: rimg = img.transpose(i+1) else: rimg = img box, landmarks = self.detect_faces(rimg, min_face_size=min_face_size, thresholds=[0.6, 0.7, 0.7]) landmarks /= scale if len(landmarks) == 0: continue if i == 0: f5p = [[landmarks[0][j], landmarks[0][j + 5]] for j in range(5)] elif i == 1: f5p = [[w-1-landmarks[0][j+5], landmarks[0][j]] for j in range(5)] x1 = w-1-box[:, 1] y1 = box[:, 0] x2 = w-1-box[:, 3] y2 = box[:, 2] box[:, :4] = np.stack((x2, y1, x1, y2), axis=1) elif i == 2: f5p = [[w-1-landmarks[0][j], h-1-landmarks[0][j+5]] for j in range(5)] x1 = w-1-box[:, 0] y1 = h-1-box[:, 1] x2 = w-1-box[:, 2] y2 = h-1-box[:, 3] box[:, :4] = np.stack((x2, y2, x1, y1), axis=1) elif i == 3: f5p = [[landmarks[0][j + 5], h-1-landmarks[0][j]] for j in range(5)] x1 = box[:, 1] y1 = h-1-box[:, 0] x2 = box[:, 3] y2 = h-1-box[:, 2] box[:, :4] = np.stack((x1, y2, x2, y1), axis=1) candi.append(f5p) boxes = np.concatenate((boxes, box), axis=0) # pick = nms(boxes) faces = [] for idx, facial5points in enumerate(candi): # if idx not in pick: # continue warped_face = warp_and_crop_face(np.array(ori_size), facial5points, self.refrence, crop_size=crop_size, return_trans_inv=False) faces.append((warped_face, facial5points)) return faces
def align_and_take_one(self, img, limit=None, min_face_size=30.0): boxes, landmarks = self.detect_faces(img, min_face_size) if len(landmarks) == 0: return None if len(boxes) != 1: return None facial5points = [[landmarks[0][j], landmarks[0][j + 5]] for j in range(5)] warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=(112, 112)) return Image.fromarray(warped_face)
def align_multi(self, img, limit=None, min_face_size=30.0): boxes, landmarks = self.detect_faces(img, min_face_size) if limit: boxes = boxes[:limit] landmarks = landmarks[:limit] faces = [] for landmark in landmarks: facial5points = [[landmark[j], landmark[j + 1]] for j in range(0, 10, 2)] warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=(112, 112)) #cv2.imshow("faceCrop",warped_face) faces.append(Image.fromarray(warped_face)) return boxes, faces
def align(self, frame): img = Image.fromarray(frame[..., ::-1]) boxes, landmarks = self.detect_faces(img) # Early return for no faces case if landmarks is None: return None, None elif len(landmarks) == 0: return None, None facial5points = [[landmarks[0][j], landmarks[0][j + 5]] for j in range(5)] warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=(112, 112)) return boxes, Image.fromarray(warped_face)
def align_multi(self, img, limit=None, min_face_size=64.0, crop_size=(112, 112)): boxes, landmarks = self.detect_faces(img, min_face_size) # print('test') if len(landmarks) == 0: # print('f**k') return None if limit: boxes = boxes[:limit] landmarks = landmarks[:limit] faces = [] for landmark in landmarks: facial5points = [[landmark[j],landmark[j+5]] for j in range(5)] warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=crop_size) faces.append(Image.fromarray(warped_face)) # print('test') # return boxes, faces return faces
def align_multi(self, img, limit=None, min_face_size=30.0): boxes, landmarks = self.detect_faces(img, min_face_size) img_bounding = show_bboxes(img, boxes, landmarks) #img_bounding.show() #input("image") if limit: boxes = boxes[:limit] landmarks = landmarks[:limit] faces = [] for landmark in landmarks: facial5points = [[landmark[j], landmark[j + 5]] for j in range(5)] warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=(112, 112)) faces.append(Image.fromarray(warped_face)) return boxes, faces
def align_multi(self, img, limit=None, min_face_size=30.0): #print('before detect_faces') boxes, landmarks = self.detect_faces(img, min_face_size) #print('after detect_faces') print(boxes) if limit: boxes = boxes[:limit] landmarks = landmarks[:limit] faces = [] for landmark in landmarks: facial5points = [[landmark[j], landmark[j + 5]] for j in range(5)] #print('before warp_and_crop_face') warped_face = warp_and_crop_face(np.array(img), facial5points, self.refrence, crop_size=(112, 112)) #print('after warp_and_crop_face') faces.append(Image.fromarray(warped_face)) return boxes, faces
def construct_face_bank(seq: str, retina_face_root: str, image_root: str, data_root: str, learner: face_learner, verbose: bool): """ loop through all images under the seq folder and count the number of distinct face embeddings :param seq: sequence id :param retina_face_root: root dir for retinaface detections. (frame, face_bbox (4), landmarks (10)) :param image_root: root dir for images :param openface_root: root dir for openface results :param learner: a face verification network :return: """ seq_images = sorted(os.listdir(image_root)) seq_retina_face_result = np.loadtxt(retina_face_root, dtype=str) embeddings = [] names = ["Unknown"] emb_counts = [] # get default reference face points reference = get_reference_facial_points(default_square=True) for frame_idx, frame in enumerate(seq_images): if verbose: print("Processing {} {}".format(frame_idx, frame)) print("Names: {}".format(names)) print("emb_counts: {}".format(emb_counts)) # 1. load data # 1.1. read image img = Image.open(osp.join(image_root, frame)) # 1.2. get the retinaface detections retinaface_result = seq_retina_face_result[seq_retina_face_result[:, 0] == frame] # skip if no detection if len(retinaface_result) == 0: print("No retinaface") continue retinaface_bboxes = retinaface_result[:, 1:5].astype(np.int) landmarks = retinaface_result[:, 5:].astype(np.float32) retinaface_bboxes = retinaface_bboxes # Warp faces: preparing input for learner faces = [] for i, landmark in enumerate(landmarks): facial5points = [[landmark[j], landmark[j + 1]] for j in range(0, 10, 2)] warped_face = warp_and_crop_face(np.array(img), facial5points, reference, crop_size=(112, 112)) faces.append(Image.fromarray(warped_face)) # if the very first frame, we initialize face bank if len(embeddings) == 0: # Extracting face embeddings for i, img in enumerate(faces): with torch.no_grad(): emb = learner.model(conf.test_transform(img).to(conf.device).unsqueeze(0)) embeddings.append(emb) names.append("{:02d}".format(i)) emb_counts.append(1) embeddings = torch.cat(embeddings) else: # otherwise we try to match, and update the face bank with torch.no_grad(): print(len(faces)) results, score, source_embs = learner.infer_embeddings(conf, faces, embeddings, True) # udpate facebank for i, idx in enumerate(results): if idx != -1: # we find a match, smooth the previous embeddings embeddings[idx] = (emb_counts[idx] * embeddings[idx] + source_embs[i]) / (emb_counts[idx] + 1) emb_counts[idx] += 1 else: # otherwise this is a new face embeddings = torch.cat((embeddings, source_embs[i].unsqueeze(0)), dim=0) emb_counts.append(1) names.append("{:02d}".format(int(names[-1]) + 1)) results[i] = len(embeddings) - 1 np.savez(osp.join(data_root, "{}.npz".format(seq)), embeddings=embeddings.cpu().numpy(), names=names, emb_counts=emb_counts) return embeddings, names, emb_counts