def align(self, image, gray, rect): # convert the landmark (x, y)-coordinates to a NumPy array shape = self.predictor(gray, rect) shape = shape_to_np(shape) # simple hack ;) if (len(shape) == 68): # extract the left and right eye (x, y)-coordinates (lStart, lEnd) = FACIAL_LANDMARKS_68_IDXS["left_eye"] (rStart, rEnd) = FACIAL_LANDMARKS_68_IDXS["right_eye"] else: (lStart, lEnd) = FACIAL_LANDMARKS_5_IDXS["left_eye"] (rStart, rEnd) = FACIAL_LANDMARKS_5_IDXS["right_eye"] leftEyePts = shape[lStart:lEnd] rightEyePts = shape[rStart:rEnd] # compute the center of mass for each eye leftEyeCenter = leftEyePts.mean(axis=0).astype("int") rightEyeCenter = rightEyePts.mean(axis=0).astype("int") # compute the angle between the eye centroids dY = rightEyeCenter[1] - leftEyeCenter[1] dX = rightEyeCenter[0] - leftEyeCenter[0] angle = np.degrees(np.arctan2(dY, dX)) - 180 # compute the desired right eye x-coordinate based on the # desired x-coordinate of the left eye desiredRightEyeX = 1.0 - self.desiredLeftEye[0] # determine the scale of the new resulting image by taking # the ratio of the distance between eyes in the *current* # image to the ratio of distance between eyes in the # *desired* image dist = np.sqrt((dX ** 2) + (dY ** 2)) desiredDist = (desiredRightEyeX - self.desiredLeftEye[0]) desiredDist *= self.desiredFaceWidth scale = desiredDist / dist # compute center (x, y)-coordinates (i.e., the median point) # between the two eyes in the input image eyesCenter = ((leftEyeCenter[0] + rightEyeCenter[0]) // 2, (leftEyeCenter[1] + rightEyeCenter[1]) // 2) # grab the rotation matrix for rotating and scaling the face M = cv2.getRotationMatrix2D(eyesCenter, angle, scale) # update the translation component of the matrix tX = self.desiredFaceWidth * 0.5 tY = self.desiredFaceHeight * self.desiredLeftEye[1] M[0, 2] += (tX - eyesCenter[0]) M[1, 2] += (tY - eyesCenter[1]) # apply the affine transformation (w, h) = (self.desiredFaceWidth, self.desiredFaceHeight) output = cv2.warpAffine(image, M, (w, h), flags=cv2.INTER_CUBIC) # return the aligned face return output
def align(self, image, gray, rect): # convert the landmark (x, y)-coordinates to a NumPy array shape = self.predictor(gray, rect) shape = shape_to_np(shape) # extract the left and right eye (x, y)-coordinates (lStart, lEnd) = FACIAL_LANDMARKS_IDXS["left_eye"] (rStart, rEnd) = FACIAL_LANDMARKS_IDXS["right_eye"] leftEyePts = shape[lStart:lEnd] rightEyePts = shape[rStart:rEnd] # compute the center of mass for each eye leftEyeCenter = leftEyePts.mean(axis=0).astype("int") rightEyeCenter = rightEyePts.mean(axis=0).astype("int") # compute the angle between the eye centroids dY = rightEyeCenter[1] - leftEyeCenter[1] dX = rightEyeCenter[0] - leftEyeCenter[0] angle = np.degrees(np.arctan2(dY, dX)) - 180 # compute the desired right eye x-coordinate based on the # desired x-coordinate of the left eye desiredRightEyeX = 1.0 - self.desiredLeftEye[0] # determine the scale of the new resulting image by taking # the ratio of the distance between eyes in the *current* # image to the ratio of distance between eyes in the # *desired* image dist = np.sqrt((dX ** 2) + (dY ** 2)) desiredDist = (desiredRightEyeX - self.desiredLeftEye[0]) desiredDist *= self.desiredFaceWidth scale = desiredDist / dist # compute center (x, y)-coordinates (i.e., the median point) # between the two eyes in the input image eyesCenter = ((leftEyeCenter[0] + rightEyeCenter[0]) // 2, (leftEyeCenter[1] + rightEyeCenter[1]) // 2) # grab the rotation matrix for rotating and scaling the face M = cv2.getRotationMatrix2D(eyesCenter, angle, scale/1.5) # update the translation component of the matrix tX = self.desiredFaceWidth * 0.5 tY = self.desiredFaceHeight * self.desiredLeftEye[1] M[0, 2] += (tX - eyesCenter[0]) M[1, 2] += (tY - eyesCenter[1]) # apply the affine transformation (w, h) = (self.desiredFaceWidth, self.desiredFaceHeight) output = cv2.warpAffine(image, M, (w, h), flags=cv2.INTER_CUBIC) # return the aligned face return output
def align(self, image, gray, rect, number): # convert the landmark (x, y)-coordinates to a NumPy array shape = self.predictor(gray, rect) shape = shape_to_np(shape) # extract the left and right eye (x, y)-coordinates (lStart, lEnd) = FACIAL_LANDMARKS_IDXS["left_eye"] (rStart, rEnd) = FACIAL_LANDMARKS_IDXS["right_eye"] leftEyePts = shape[lStart:lEnd] rightEyePts = shape[rStart:rEnd] # compute the center of mass for each eye leftEyeCenter = leftEyePts.mean(axis=0).astype("int") #print(leftEyeCenter) leftEyes[number] = leftEyeCenter.tolist() rightEyeCenter = rightEyePts.mean(axis=0).astype("int") #print(rightEyeCenter) rightEyes[number] = rightEyeCenter.tolist()
def analyze_single(self, img, path): boxes = self.detector(img) if not boxes: raise AlignFaceException('No face detected') gray = np.asarray(Image.fromarray(img, mode='RGB').convert('L')) shape = self.predictor(gray, boxes[0]) shape = shape_to_np(shape) # extract the left and right eye (x, y)-coordinates (lStart, lEnd) = FACIAL_LANDMARKS_68_IDXS["left_eye"] (rStart, rEnd) = FACIAL_LANDMARKS_68_IDXS["right_eye"] leftEyePts = shape[lStart:lEnd] rightEyePts = shape[rStart:rEnd] # compute the center of mass for each eye leftEyeCenter = leftEyePts.mean(axis=0) rightEyeCenter = rightEyePts.mean(axis=0) M = self.face_aligner.analyze((rightEyeCenter, leftEyeCenter)) return M
def align(self, image, gray, rect): # convert the landmark (x, y)-coordinates to a NumPy array shape = self.predictor(gray, rect) shape = shape_to_np(shape) # extract the left and right eye (x, y)-coordinates (lStart, lEnd) = FACIAL_LANDMARKS_IDXS["left_eye"] (rStart, rEnd) = FACIAL_LANDMARKS_IDXS["right_eye"] leftEyePts = shape[lStart:lEnd] rightEyePts = shape[rStart:rEnd] # compute the center of mass for each eye leftEyeCenter = leftEyePts.mean(axis=0).astype("int") rightEyeCenter = rightEyePts.mean(axis=0).astype("int") # compute the angle between the eye centroids dY = rightEyeCenter[1] - leftEyeCenter[1] dX = rightEyeCenter[0] - leftEyeCenter[0] angle = np.degrees(np.arctan2(dY, dX)) - 180 # compute the desired right eye x-coordinate based on the # desired x-coordinate of the left eye desiredRightEyeX = 1.0 - self.desiredLeftEye[0] # determine the scale of the new resulting image by taking # the ratio of the distance between eyes in the *current* # image to the ratio of distance between eyes in the # *desired* image dist = np.sqrt((dX**2) + (dY**2)) desiredDist = (desiredRightEyeX - self.desiredLeftEye[0]) desiredDist *= self.desiredFaceWidth scale = desiredDist / dist # compute center (x, y)-coordinates (i.e., the median point) # between the two eyes in the input image eyesCenter = ((leftEyeCenter[0] + rightEyeCenter[0]) // 2, (leftEyeCenter[1] + rightEyeCenter[1]) // 2) # grab the rotation matrix for rotating and scaling the face return -angle
def align(self, image, gray, rect, z_addition): # convert the landmark (x, y)-coordinates to a NumPy h1, w1 = image.shape[:2] shape = self.predictor(gray, rect) shape = shape_to_np(shape) #simple hack ;) if (len(shape) == 68): # extract the left and right eye (x, y)-coordinates (lStart, lEnd) = FACIAL_LANDMARKS_68_IDXS["left_eye"] (rStart, rEnd) = FACIAL_LANDMARKS_68_IDXS["right_eye"] else: (lStart, lEnd) = FACIAL_LANDMARKS_5_IDXS["left_eye"] (rStart, rEnd) = FACIAL_LANDMARKS_5_IDXS["right_eye"] leftEyePts = shape[lStart:lEnd] rightEyePts = shape[rStart:rEnd] # compute the center of mass for each eye leftEyeCenter = leftEyePts.mean(axis=0).astype("int") rightEyeCenter = rightEyePts.mean(axis=0).astype("int") # compute the angle between the eye centroids dY = rightEyeCenter[1] - leftEyeCenter[1] dX = rightEyeCenter[0] - leftEyeCenter[0] angle = np.degrees(np.arctan2(dY, dX)) - 180 # compute the desired right eye x-coordinate based on the # desired x-coordinate of the left eye desiredRightEyeX = 1.0 - self.desiredLeftEye[0] # determine the scale of the new resulting image by taking # the ratio of the distance between eyes in the *current* # image to the ratio of distance between eyes in the # *desired* image dist = np.sqrt((dX**2) + (dY**2)) desiredDist = (desiredRightEyeX - self.desiredLeftEye[0]) desiredDist *= self.desiredFaceWidth scale = desiredDist / dist # compute center (x, y)-coordinates (i.e., the median point) # between the two eyes in the input image eyesCenter = ((leftEyeCenter[0] + rightEyeCenter[0]) // 2, (leftEyeCenter[1] + rightEyeCenter[1]) // 2) # grab the rotation matrix for rotating and scaling the face M = cv2.getRotationMatrix2D(eyesCenter, angle, scale) # update the translation component of the matrix tX = self.desiredFaceWidth * 0.5 tY = self.desiredFaceHeight * self.desiredLeftEye[1] M[0, 2] += (tX - eyesCenter[0]) M[1, 2] += (tY - eyesCenter[1]) # apply the affine transformation (w, h) = (self.desiredFaceWidth, self.desiredFaceHeight) output = cv2.warpAffine(image, M, (w, h), flags=cv2.INTER_CUBIC) # invert the previous affine transformation for later Mi = cv2.invertAffineTransform(M) # BGR -> RGB output = output[:, :, ::-1] # encode with GLOW, do operations on z z = encode(output) z[0] += z_addition # decode back to image and back to BGR output = decode(z)[0] output = output[:, :, ::-1] # invert the affine transformation on output output = cv2.warpAffine(output, Mi, (w1, h1), flags=cv2.INTER_CUBIC) # overwrite original image with masked output mask = np.sum(output, axis=2) == 0.0 image = np.multiply(mask.reshape((h1, w1, 1)), image) image += output return image
count += 1 identityStr = os.path.basename(identity).split('\\')[-1] pics = glob(identity + "\\*") print("Creating patches for " + identityStr + "'s " + str(len(pics)) + " picture(s)") inputPicTotal += len(pics) for pic in pics: picID = os.path.basename(pic).split('\\')[-1] picID = picID.split('.')[0] image = cv2.imread(pic) gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) rect = dlib.rectangle( 0, 0, 64, 64 ) #sketchy manually defined bounding box. saves time over using dnnfacedetector again because images are assured to be 64x64 shape = predictor(gray, rect) shape = shape_to_np(shape) #landmarks are 42, 47, 20, 25, 34, and 67 for right eye, left eye, right eyebrow, left eyebrow, nose, and mouth. #decrement by 1 to account for starting at 0 not 1 -> 41, 46, 19, 24, 33, 66 centerpts = [41, 46, 19, 24, 33, 66] #putting numbers before names to make sure theyre in the right order in the folders areanames = [ "right_eyes\\", "left_eyes\\", "right_eyebrows\\", "left_eyebrows\\", "noses\\", "mouths\\" ] for i, point in enumerate(centerpts): x, y = shape[point] if i == 0: reyeminy = min(y, reyeminy) reyemaxy = max(y, reyemaxy) reyeminx = min(x, reyeminx)
def realtime_cropper(): capture = cv2.VideoCapture(0) capture.set(cv2.CAP_PROP_BUFFERSIZE, 2) assert capture.isOpened(), IOError("Cannot open webcam") # Load the detector for detecting the face detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor(config.PREDICTOR) mean_face = np.load(config.MEAN_FACE) frame_idx = 0 while True: (status, frame) = capture.read() if frame_idx == 0: q_frame, q_landmarks = col.deque(), col.deque() sequence = [] # Convert frames into grayscale gray = cv2.cvtColor(src=frame, code=cv2.COLOR_BGR2GRAY) # Use detector to find landmarks (as rectangle) rects = detector(gray) if len(rects) != 0: # Determine the facial landmarks for the face region, then # convert the landmark (x, y)-coordinates to a NumPy array landmarks = predictor(gray, rects[0]) landmarks = shape_to_np(landmarks) q_landmarks.append(landmarks) q_frame.append(frame) if len(q_frame) == config.WINDOW_MARGIN: smoothed_landmarks = np.mean(q_landmarks, axis=0) cur_landmarks = q_landmarks.popleft() cur_frame = q_frame.popleft() # After transformation trans_frame, tform = transform.warp_img( smoothed_landmarks[config.STABLE_PNTS_IDS, :], mean_face[config.STABLE_PNTS_IDS, :], cur_frame, config.STD_SIZE) trans_landmarks = tform(cur_landmarks) # Crop mouth patch sequence.append(transform.crop_patch( trans_frame, trans_landmarks[config.START_IDX:config.END_IDX], config.CROP_WIDTH // 2, config.CROP_HEIGHT // 2 )) cv2.imshow("Transformed", transform.crop_patch( trans_frame, trans_landmarks[config.START_IDX:config.END_IDX], config.CROP_WIDTH // 2, config.CROP_HEIGHT // 2 )) if frame_idx == config.FRAME_AMOUNT - 1: while q_frame: cur_frame = q_frame.popleft() # Transform frame trans_frame = transform.apply_transform( tform, cur_frame, config.STD_SIZE) # Transform landmarks trans_landmarks = tform(q_landmarks.popleft()) # Crop mouth patch sequence.append(transform.crop_patch( trans_frame, trans_landmarks[config.START_IDX:config.END_IDX], config.CROP_WIDTH // 2, config.CROP_HEIGHT // 2 )) return np.array(sequence) frame_idx += 1 # cv2.imshow("Original", frame) if cv2.waitKey(delay=1) == 27: break capture.release() cv2.destroyAllWindows()
def detect_face_from_file(filename, verbose=False): frame_idx = 0 # Generator object frame_gen = utils.load_video(filename, verbose) # Load the detector for detecting the face detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor(config.PREDICTOR) mean_face = np.load(config.MEAN_FACE) while True: try: # BGR, numpy array frame, frame_amount = frame_gen.__next__() except StopIteration: break if frame_idx == 0: q_frame, q_landmarks = col.deque(), col.deque() sequence = [] # Convert frames into grayscale gray = cv2.cvtColor(src=frame, code=cv2.COLOR_BGR2GRAY) # Use detector to find landmarks (as rectangle) rects = detector(gray) # Determine the facial landmarks for the face region, then # convert the landmark (x, y)-coordinates to a NumPy array landmarks = predictor(gray, rects[0]) landmarks = shape_to_np(landmarks) q_landmarks.append(landmarks) q_frame.append(frame) if len(q_frame) == config.WINDOW_MARGIN: smoothed_landmarks = np.mean(q_landmarks, axis=0) cur_landmarks = q_landmarks.popleft() cur_frame = q_frame.popleft() # After transformation trans_frame, tform = transform.warp_img( smoothed_landmarks[config.STABLE_PNTS_IDS, :], mean_face[config.STABLE_PNTS_IDS, :], cur_frame, config.STD_SIZE) trans_landmarks = tform(cur_landmarks) # Crop mouth patch sequence.append(transform.crop_patch( trans_frame, trans_landmarks[config.START_IDX:config.END_IDX], config.CROP_WIDTH // 2, config.CROP_HEIGHT // 2 )) if frame_idx == frame_amount - 1: while q_frame: cur_frame = q_frame.popleft() # Transform frame trans_frame = transform.apply_transform( tform, cur_frame, config.STD_SIZE) # Transform landmarks trans_landmarks = tform(q_landmarks.popleft()) # Crop mouth patch sequence.append(transform.crop_patch( trans_frame, trans_landmarks[config.START_IDX:config.END_IDX], config.CROP_WIDTH // 2, config.CROP_HEIGHT // 2 )) return np.array(sequence) frame_idx += 1 return None