def align(self, image, gray, rect):
# convert the landmark (x, y)-coordinates to a NumPy array
shape = self.predictor(gray, rect)
shape = shape_to_np(shape)
# simple hack ;)
if (len(shape) == 68):
# extract the left and right eye (x, y)-coordinates
(lStart, lEnd) = FACIAL_LANDMARKS_68_IDXS["left_eye"]
(rStart, rEnd) = FACIAL_LANDMARKS_68_IDXS["right_eye"]
else:
(lStart, lEnd) = FACIAL_LANDMARKS_5_IDXS["left_eye"]
(rStart, rEnd) = FACIAL_LANDMARKS_5_IDXS["right_eye"]
leftEyePts = shape[lStart:lEnd]
rightEyePts = shape[rStart:rEnd]
# compute the center of mass for each eye
leftEyeCenter = leftEyePts.mean(axis=0).astype("int")
rightEyeCenter = rightEyePts.mean(axis=0).astype("int")
# compute the angle between the eye centroids
dY = rightEyeCenter[1] - leftEyeCenter[1]
dX = rightEyeCenter[0] - leftEyeCenter[0]
angle = np.degrees(np.arctan2(dY, dX)) - 180
# compute the desired right eye x-coordinate based on the
# desired x-coordinate of the left eye
desiredRightEyeX = 1.0 - self.desiredLeftEye[0]
# determine the scale of the new resulting image by taking
# the ratio of the distance between eyes in the *current*
# image to the ratio of distance between eyes in the
# *desired* image
dist = np.sqrt((dX ** 2) + (dY ** 2))
desiredDist = (desiredRightEyeX - self.desiredLeftEye[0])
desiredDist *= self.desiredFaceWidth
scale = desiredDist / dist
# compute center (x, y)-coordinates (i.e., the median point)
# between the two eyes in the input image
eyesCenter = ((leftEyeCenter[0] + rightEyeCenter[0]) // 2,
(leftEyeCenter[1] + rightEyeCenter[1]) // 2)
# grab the rotation matrix for rotating and scaling the face
M = cv2.getRotationMatrix2D(eyesCenter, angle, scale)
# update the translation component of the matrix
tX = self.desiredFaceWidth * 0.5
tY = self.desiredFaceHeight * self.desiredLeftEye[1]
M[0, 2] += (tX - eyesCenter[0])
M[1, 2] += (tY - eyesCenter[1])
# apply the affine transformation
(w, h) = (self.desiredFaceWidth, self.desiredFaceHeight)
output = cv2.warpAffine(image, M, (w, h),
flags=cv2.INTER_CUBIC)
# return the aligned face
return output
def align(self, image, gray, rect):
# convert the landmark (x, y)-coordinates to a NumPy array
shape = self.predictor(gray, rect)
shape = shape_to_np(shape)
# extract the left and right eye (x, y)-coordinates
(lStart, lEnd) = FACIAL_LANDMARKS_IDXS["left_eye"]
(rStart, rEnd) = FACIAL_LANDMARKS_IDXS["right_eye"]
leftEyePts = shape[lStart:lEnd]
rightEyePts = shape[rStart:rEnd]
# compute the center of mass for each eye
leftEyeCenter = leftEyePts.mean(axis=0).astype("int")
rightEyeCenter = rightEyePts.mean(axis=0).astype("int")
# compute the angle between the eye centroids
dY = rightEyeCenter[1] - leftEyeCenter[1]
dX = rightEyeCenter[0] - leftEyeCenter[0]
angle = np.degrees(np.arctan2(dY, dX)) - 180
# compute the desired right eye x-coordinate based on the
# desired x-coordinate of the left eye
desiredRightEyeX = 1.0 - self.desiredLeftEye[0]
# determine the scale of the new resulting image by taking
# the ratio of the distance between eyes in the *current*
# image to the ratio of distance between eyes in the
# *desired* image
dist = np.sqrt((dX ** 2) + (dY ** 2))
desiredDist = (desiredRightEyeX - self.desiredLeftEye[0])
desiredDist *= self.desiredFaceWidth
scale = desiredDist / dist
# compute center (x, y)-coordinates (i.e., the median point)
# between the two eyes in the input image
eyesCenter = ((leftEyeCenter[0] + rightEyeCenter[0]) // 2,
(leftEyeCenter[1] + rightEyeCenter[1]) // 2)
# grab the rotation matrix for rotating and scaling the face
M = cv2.getRotationMatrix2D(eyesCenter, angle, scale/1.5)
# update the translation component of the matrix
tX = self.desiredFaceWidth * 0.5
tY = self.desiredFaceHeight * self.desiredLeftEye[1]
M[0, 2] += (tX - eyesCenter[0])
M[1, 2] += (tY - eyesCenter[1])
# apply the affine transformation
(w, h) = (self.desiredFaceWidth, self.desiredFaceHeight)
output = cv2.warpAffine(image, M, (w, h),
flags=cv2.INTER_CUBIC)
# return the aligned face
return output
def align(self, image, gray, rect, number):
# convert the landmark (x, y)-coordinates to a NumPy array
shape = self.predictor(gray, rect)
shape = shape_to_np(shape)
# extract the left and right eye (x, y)-coordinates
(lStart, lEnd) = FACIAL_LANDMARKS_IDXS["left_eye"]
(rStart, rEnd) = FACIAL_LANDMARKS_IDXS["right_eye"]
leftEyePts = shape[lStart:lEnd]
rightEyePts = shape[rStart:rEnd]
# compute the center of mass for each eye
leftEyeCenter = leftEyePts.mean(axis=0).astype("int")
#print(leftEyeCenter)
leftEyes[number] = leftEyeCenter.tolist()
rightEyeCenter = rightEyePts.mean(axis=0).astype("int")
#print(rightEyeCenter)
rightEyes[number] = rightEyeCenter.tolist()
def analyze_single(self, img, path):
boxes = self.detector(img)
if not boxes:
raise AlignFaceException('No face detected')
gray = np.asarray(Image.fromarray(img, mode='RGB').convert('L'))
shape = self.predictor(gray, boxes[0])
shape = shape_to_np(shape)
# extract the left and right eye (x, y)-coordinates
(lStart, lEnd) = FACIAL_LANDMARKS_68_IDXS["left_eye"]
(rStart, rEnd) = FACIAL_LANDMARKS_68_IDXS["right_eye"]
leftEyePts = shape[lStart:lEnd]
rightEyePts = shape[rStart:rEnd]
# compute the center of mass for each eye
leftEyeCenter = leftEyePts.mean(axis=0)
rightEyeCenter = rightEyePts.mean(axis=0)
M = self.face_aligner.analyze((rightEyeCenter, leftEyeCenter))
return M
def align(self, image, gray, rect):
# convert the landmark (x, y)-coordinates to a NumPy array
shape = self.predictor(gray, rect)
shape = shape_to_np(shape)
# extract the left and right eye (x, y)-coordinates
(lStart, lEnd) = FACIAL_LANDMARKS_IDXS["left_eye"]
(rStart, rEnd) = FACIAL_LANDMARKS_IDXS["right_eye"]
leftEyePts = shape[lStart:lEnd]
rightEyePts = shape[rStart:rEnd]
# compute the center of mass for each eye
leftEyeCenter = leftEyePts.mean(axis=0).astype("int")
rightEyeCenter = rightEyePts.mean(axis=0).astype("int")
# compute the angle between the eye centroids
dY = rightEyeCenter[1] - leftEyeCenter[1]
dX = rightEyeCenter[0] - leftEyeCenter[0]
angle = np.degrees(np.arctan2(dY, dX)) - 180
# compute the desired right eye x-coordinate based on the
# desired x-coordinate of the left eye
desiredRightEyeX = 1.0 - self.desiredLeftEye[0]
# determine the scale of the new resulting image by taking
# the ratio of the distance between eyes in the *current*
# image to the ratio of distance between eyes in the
# *desired* image
dist = np.sqrt((dX**2) + (dY**2))
desiredDist = (desiredRightEyeX - self.desiredLeftEye[0])
desiredDist *= self.desiredFaceWidth
scale = desiredDist / dist
# compute center (x, y)-coordinates (i.e., the median point)
# between the two eyes in the input image
eyesCenter = ((leftEyeCenter[0] + rightEyeCenter[0]) // 2,
(leftEyeCenter[1] + rightEyeCenter[1]) // 2)
# grab the rotation matrix for rotating and scaling the face
return -angle
def align(self, image, gray, rect, z_addition):
# convert the landmark (x, y)-coordinates to a NumPy
h1, w1 = image.shape[:2]
shape = self.predictor(gray, rect)
shape = shape_to_np(shape)
#simple hack ;)
if (len(shape) == 68):
# extract the left and right eye (x, y)-coordinates
(lStart, lEnd) = FACIAL_LANDMARKS_68_IDXS["left_eye"]
(rStart, rEnd) = FACIAL_LANDMARKS_68_IDXS["right_eye"]
else:
(lStart, lEnd) = FACIAL_LANDMARKS_5_IDXS["left_eye"]
(rStart, rEnd) = FACIAL_LANDMARKS_5_IDXS["right_eye"]
leftEyePts = shape[lStart:lEnd]
rightEyePts = shape[rStart:rEnd]
# compute the center of mass for each eye
leftEyeCenter = leftEyePts.mean(axis=0).astype("int")
rightEyeCenter = rightEyePts.mean(axis=0).astype("int")
# compute the angle between the eye centroids
dY = rightEyeCenter[1] - leftEyeCenter[1]
dX = rightEyeCenter[0] - leftEyeCenter[0]
angle = np.degrees(np.arctan2(dY, dX)) - 180
# compute the desired right eye x-coordinate based on the
# desired x-coordinate of the left eye
desiredRightEyeX = 1.0 - self.desiredLeftEye[0]
# determine the scale of the new resulting image by taking
# the ratio of the distance between eyes in the *current*
# image to the ratio of distance between eyes in the
# *desired* image
dist = np.sqrt((dX**2) + (dY**2))
desiredDist = (desiredRightEyeX - self.desiredLeftEye[0])
desiredDist *= self.desiredFaceWidth
scale = desiredDist / dist
# compute center (x, y)-coordinates (i.e., the median point)
# between the two eyes in the input image
eyesCenter = ((leftEyeCenter[0] + rightEyeCenter[0]) // 2,
(leftEyeCenter[1] + rightEyeCenter[1]) // 2)
# grab the rotation matrix for rotating and scaling the face
M = cv2.getRotationMatrix2D(eyesCenter, angle, scale)
# update the translation component of the matrix
tX = self.desiredFaceWidth * 0.5
tY = self.desiredFaceHeight * self.desiredLeftEye[1]
M[0, 2] += (tX - eyesCenter[0])
M[1, 2] += (tY - eyesCenter[1])
# apply the affine transformation
(w, h) = (self.desiredFaceWidth, self.desiredFaceHeight)
output = cv2.warpAffine(image, M, (w, h), flags=cv2.INTER_CUBIC)
# invert the previous affine transformation for later
Mi = cv2.invertAffineTransform(M)
# BGR -> RGB
output = output[:, :, ::-1]
# encode with GLOW, do operations on z
z = encode(output)
z[0] += z_addition
# decode back to image and back to BGR
output = decode(z)[0]
output = output[:, :, ::-1]
# invert the affine transformation on output
output = cv2.warpAffine(output, Mi, (w1, h1), flags=cv2.INTER_CUBIC)
# overwrite original image with masked output
mask = np.sum(output, axis=2) == 0.0
image = np.multiply(mask.reshape((h1, w1, 1)), image)
image += output
return image
count += 1
identityStr = os.path.basename(identity).split('\\')[-1]
pics = glob(identity + "\\*")
print("Creating patches for " + identityStr + "'s " + str(len(pics)) +
" picture(s)")
inputPicTotal += len(pics)
for pic in pics:
picID = os.path.basename(pic).split('\\')[-1]
picID = picID.split('.')[0]
image = cv2.imread(pic)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
rect = dlib.rectangle(
0, 0, 64, 64
) #sketchy manually defined bounding box. saves time over using dnnfacedetector again because images are assured to be 64x64
shape = predictor(gray, rect)
shape = shape_to_np(shape)
#landmarks are 42, 47, 20, 25, 34, and 67 for right eye, left eye, right eyebrow, left eyebrow, nose, and mouth.
#decrement by 1 to account for starting at 0 not 1 -> 41, 46, 19, 24, 33, 66
centerpts = [41, 46, 19, 24, 33, 66]
#putting numbers before names to make sure theyre in the right order in the folders
areanames = [
"right_eyes\\", "left_eyes\\", "right_eyebrows\\",
"left_eyebrows\\", "noses\\", "mouths\\"
]
for i, point in enumerate(centerpts):
x, y = shape[point]
if i == 0:
reyeminy = min(y, reyeminy)
reyemaxy = max(y, reyemaxy)
reyeminx = min(x, reyeminx)
def realtime_cropper():
capture = cv2.VideoCapture(0)
capture.set(cv2.CAP_PROP_BUFFERSIZE, 2)
assert capture.isOpened(), IOError("Cannot open webcam")
# Load the detector for detecting the face
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(config.PREDICTOR)
mean_face = np.load(config.MEAN_FACE)
frame_idx = 0
while True:
(status, frame) = capture.read()
if frame_idx == 0:
q_frame, q_landmarks = col.deque(), col.deque()
sequence = []
# Convert frames into grayscale
gray = cv2.cvtColor(src=frame, code=cv2.COLOR_BGR2GRAY)
# Use detector to find landmarks (as rectangle)
rects = detector(gray)
if len(rects) != 0:
# Determine the facial landmarks for the face region, then
# convert the landmark (x, y)-coordinates to a NumPy array
landmarks = predictor(gray, rects[0])
landmarks = shape_to_np(landmarks)
q_landmarks.append(landmarks)
q_frame.append(frame)
if len(q_frame) == config.WINDOW_MARGIN:
smoothed_landmarks = np.mean(q_landmarks, axis=0)
cur_landmarks = q_landmarks.popleft()
cur_frame = q_frame.popleft()
# After transformation
trans_frame, tform = transform.warp_img(
smoothed_landmarks[config.STABLE_PNTS_IDS, :],
mean_face[config.STABLE_PNTS_IDS, :],
cur_frame,
config.STD_SIZE)
trans_landmarks = tform(cur_landmarks)
# Crop mouth patch
sequence.append(transform.crop_patch(
trans_frame,
trans_landmarks[config.START_IDX:config.END_IDX],
config.CROP_WIDTH // 2,
config.CROP_HEIGHT // 2
))
cv2.imshow("Transformed", transform.crop_patch(
trans_frame,
trans_landmarks[config.START_IDX:config.END_IDX],
config.CROP_WIDTH // 2,
config.CROP_HEIGHT // 2
))
if frame_idx == config.FRAME_AMOUNT - 1:
while q_frame:
cur_frame = q_frame.popleft()
# Transform frame
trans_frame = transform.apply_transform(
tform, cur_frame, config.STD_SIZE)
# Transform landmarks
trans_landmarks = tform(q_landmarks.popleft())
# Crop mouth patch
sequence.append(transform.crop_patch(
trans_frame,
trans_landmarks[config.START_IDX:config.END_IDX],
config.CROP_WIDTH // 2,
config.CROP_HEIGHT // 2
))
return np.array(sequence)
frame_idx += 1
# cv2.imshow("Original", frame)
if cv2.waitKey(delay=1) == 27:
break
capture.release()
cv2.destroyAllWindows()
def detect_face_from_file(filename, verbose=False):
frame_idx = 0
# Generator object
frame_gen = utils.load_video(filename, verbose)
# Load the detector for detecting the face
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(config.PREDICTOR)
mean_face = np.load(config.MEAN_FACE)
while True:
try:
# BGR, numpy array
frame, frame_amount = frame_gen.__next__()
except StopIteration:
break
if frame_idx == 0:
q_frame, q_landmarks = col.deque(), col.deque()
sequence = []
# Convert frames into grayscale
gray = cv2.cvtColor(src=frame, code=cv2.COLOR_BGR2GRAY)
# Use detector to find landmarks (as rectangle)
rects = detector(gray)
# Determine the facial landmarks for the face region, then
# convert the landmark (x, y)-coordinates to a NumPy array
landmarks = predictor(gray, rects[0])
landmarks = shape_to_np(landmarks)
q_landmarks.append(landmarks)
q_frame.append(frame)
if len(q_frame) == config.WINDOW_MARGIN:
smoothed_landmarks = np.mean(q_landmarks, axis=0)
cur_landmarks = q_landmarks.popleft()
cur_frame = q_frame.popleft()
# After transformation
trans_frame, tform = transform.warp_img(
smoothed_landmarks[config.STABLE_PNTS_IDS, :],
mean_face[config.STABLE_PNTS_IDS, :],
cur_frame,
config.STD_SIZE)
trans_landmarks = tform(cur_landmarks)
# Crop mouth patch
sequence.append(transform.crop_patch(
trans_frame,
trans_landmarks[config.START_IDX:config.END_IDX],
config.CROP_WIDTH // 2,
config.CROP_HEIGHT // 2
))
if frame_idx == frame_amount - 1:
while q_frame:
cur_frame = q_frame.popleft()
# Transform frame
trans_frame = transform.apply_transform(
tform, cur_frame, config.STD_SIZE)
# Transform landmarks
trans_landmarks = tform(q_landmarks.popleft())
# Crop mouth patch
sequence.append(transform.crop_patch(
trans_frame,
trans_landmarks[config.START_IDX:config.END_IDX],
config.CROP_WIDTH // 2,
config.CROP_HEIGHT // 2
))
return np.array(sequence)
frame_idx += 1
return None
