def facial_pt_extractor(pic_path):
## Since we have the helper methods in helpers.py, we just need to import them
# def rect_to_bb(rect):
# x = rect.left()
# y = rect.top()
# w = rect.right() - x
# h = rect.bottom() - y
# return (x, y, w, h)
# def shape_to_np(shape, dtype="int"):
# coords = np.zeros((68, 2), dtype=dtype)
# for i in range(0, 68):
# coords[i] = (shape.part(i).x, shape.part(i).y)
#
# return coords
#
# def resize(image, width=1200):
# r = width * 1.0 / image.shape[1]
# dim = (width, int(image.shape[0] * r))
# resized = cv2.resize(image, dim, interpolation=cv2.INTER_AREA)
# return resized
image = cv2.imread(pic_path)
image = resize(image, width=1200)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
rects = detector(gray, 1)
coordinates = []
for (i, rect) in enumerate(rects):
shape = predictor(gray, rect)
shape = shape_to_np(shape)
for (x, y) in shape:
coordinates.append(x)
coordinates.append(y)
return coordinates
def main():
'''
This file is used to transform/adjust the training image for emotion
detection in order to get a better performance in eigenface algorithm.
'''
expression_files = os.listdir(expression_path)
data_as_array = np.zeros((len(expression_files), img_row, img_col))
for i in range(len(expression_files)):
image = Image.open(expression_path + expression_files[i])
image = np.array(image)
gray, faces = detect(image)
result = np.zeros((img_row, img_col))
for face in faces:
shape = DLIB_PREDICTOR(gray, face)
shape = shape_to_np(shape)
for point in shape:
if point[0] >= 50:
point[0] = 49
if point[1] >= 50:
point[1] = 49
result[point[1], point[0]] = 255
filename = expression_files[i]
base = filename[0:(len(filename) - 4)]
imageio.imwrite(output_path + base + ".png", result)
def align_to_template_similarity(self, image, gray, rect):
template_landmarks = get_template_landmark()
detected_landmarks = shape_to_np(self.predictor(gray, rect))
tf = transform.estimate_transform('similarity', detected_landmarks,
template_landmarks)
result = img_as_ubyte(
transform.warp(image,
inverse_map=tf.inverse,
output_shape=(self.desiredFaceWidth,
self.desiredFaceWidth, 3)))
# imshow(result)
# overlay template landmarks on result -- successful
# canvas = result
# for p in template_landmarks:
# x, y = p
# result[y, x] = [0, 255, 0] # OG
# surround
# result[y-1, x] = [0, 255, 0]
# result[y-1, x+1] = [0, 255, 0]
# result[y, x+1] = [0, 255, 0]
# result[y+1, x+1] = [0, 255, 0]
# result[y+1, x] = [0, 255, 0]
# result[y+1, x-1] = [0, 255, 0]
# result[y, x-1] = [0, 255, 0]
# result[y-1, x-1] = [0, 255, 0]
#
# imshow(canvas)
# #
# # save image as jpg -- successful
# result = Image.fromarray(result, mode='RGB')
# result.save('testing123_2.jpg')
return result
def align(self, image, gray, rect):
# convert the landmark (x, y)-coordinates to a NumPy array
shape = self.predictor(gray, rect)
shape = shape_to_np(shape)
#simple hack ;)
if (len(shape) == 68):
# extract the left and right eye (x, y)-coordinates
(lStart, lEnd) = FACIAL_LANDMARKS_68_IDXS["left_eye"]
(rStart, rEnd) = FACIAL_LANDMARKS_68_IDXS["right_eye"]
else:
(lStart, lEnd) = FACIAL_LANDMARKS_5_IDXS["left_eye"]
(rStart, rEnd) = FACIAL_LANDMARKS_5_IDXS["right_eye"]
leftEyePts = shape[lStart:lEnd]
rightEyePts = shape[rStart:rEnd]
# compute the center of mass for each eye
leftEyeCenter = leftEyePts.mean(axis=0).astype("int")
rightEyeCenter = rightEyePts.mean(axis=0).astype("int")
# compute the angle between the eye centroids
dY = rightEyeCenter[1] - leftEyeCenter[1]
dX = rightEyeCenter[0] - leftEyeCenter[0]
angle = np.degrees(np.arctan2(dY, dX)) - 180
# compute the desired right eye x-coordinate based on the
# desired x-coordinate of the left eye
desiredRightEyeX = 1.0 - self.desiredLeftEye[0]
# determine the scale of the new resulting image by taking
# the ratio of the distance between eyes in the *current*
# image to the ratio of distance between eyes in the
# *desired* image
dist = np.sqrt((dX**2) + (dY**2))
desiredDist = (desiredRightEyeX - self.desiredLeftEye[0])
desiredDist *= self.desiredFaceWidth
scale = desiredDist / dist
# compute center (x, y)-coordinates (i.e., the median point)
# between the two eyes in the input image
eyesCenter = ((leftEyeCenter[0] + rightEyeCenter[0]) // 2,
(leftEyeCenter[1] + rightEyeCenter[1]) // 2)
# grab the rotation matrix for rotating and scaling the face
M = cv2.getRotationMatrix2D(eyesCenter, angle, scale)
# update the translation component of the matrix
tX = self.desiredFaceWidth * 0.5
tY = self.desiredFaceHeight * self.desiredLeftEye[1]
M[0, 2] += (tX - eyesCenter[0])
M[1, 2] += (tY - eyesCenter[1])
# apply the affine transformation
(w, h) = (self.desiredFaceWidth, self.desiredFaceHeight)
output = cv2.warpAffine(image, M, (w, h), flags=cv2.INTER_CUBIC)
# return the aligned face
return output
def align(self, image, gray, rect):
# convert the landmark (x, y)-coordinates to a NumPy array
shape = self.predictor(gray, rect)
shape = shape_to_np(shape)
# extract the left and right eye (x, y)-coordinates
(lStart, lEnd) = FACIAL_LANDMARKS_IDXS["left_eye"]
(rStart, rEnd) = FACIAL_LANDMARKS_IDXS["right_eye"]
leftEyePts = shape[lStart:lEnd]
rightEyePts = shape[rStart:rEnd]
# compute the center of mass for each eye
leftEyeCenter = leftEyePts.mean(axis=0).astype("int")
rightEyeCenter = rightEyePts.mean(axis=0).astype("int")
# compute the angle between the eye centroids
dY = rightEyeCenter[1] - leftEyeCenter[1]
dX = rightEyeCenter[0] - leftEyeCenter[0]
angle = np.degrees(np.arctan2(dY, dX)) - 180
# compute the center of mass for each eye
leftEyeCenter = leftEyePts.mean(axis=0).astype("int")
rightEyeCenter = rightEyePts.mean(axis=0).astype("int")
# compute the angle between the eye centroids
dY = rightEyeCenter[1] - leftEyeCenter[1]
dX = rightEyeCenter[0] - leftEyeCenter[0]
angle = np.degrees(np.arctan2(dY, dX)) - 180
# compute center (x, y)-coordinates (i.e., the median point)
# between the two eyes in the input image
eyesCenter = ((leftEyeCenter[0] + rightEyeCenter[0]) // 2,
(leftEyeCenter[1] + rightEyeCenter[1]) // 2)
# grab the rotation matrix for rotating and scaling the face
M = cv2.getRotationMatrix2D(eyesCenter, angle, scale)
# update the translation component of the matrix
tX = self.desiredFaceWidth * 0.5
tY = self.desiredFaceHeight * self.desiredLeftEye[1]
M[0, 2] += (tX - eyesCenter[0])
M[1, 2] += (tY - eyesCenter[1] )
# apply the affine transformation
(w, h) = (self.desiredFaceWidth, self.desiredFaceHeight)
output = cv2.warpAffine(image, M, (w, h),flags=cv2.INTER_CUBIC)
# return the aligned face
return output
def align_center(self, image, gray, rect, radius):
# convert the landmark (x, y)-coordinates to a NumPy array
shape = self.predictor(gray, rect)
shape = shape_to_np(shape)
# extract the left and right eye (x, y)-coordinates
(l_start, l_end) = FACIAL_LANDMARKS_IDXS["left_eye"]
(r_start, r_end) = FACIAL_LANDMARKS_IDXS["right_eye"]
(m_start, m_end) = FACIAL_LANDMARKS_IDXS['mouth']
(j_start, j_end) = FACIAL_LANDMARKS_IDXS['jaw']
left_eye_points = shape[l_start:l_end]
right_eye_points = shape[r_start:r_end]
mouth_points = shape[m_start:m_end]
jaw_points = shape[j_start:j_end]
# compute the geometrical center of each eye
left_eye_bbox = get_bbox(left_eye_points)
left_eye_center = np.array([
(left_eye_bbox[0][0] + left_eye_bbox[1][0]) / 2,
(left_eye_bbox[2][1] + left_eye_bbox[3][1]) / 2
])
right_eye_bbox = get_bbox(right_eye_points)
right_eye_center = np.array([
(right_eye_bbox[0][0] + right_eye_bbox[1][0]) / 2,
(right_eye_bbox[2][1] + right_eye_bbox[3][1]) / 2
])
mouth_bbox = get_bbox(mouth_points)
mouth_center = np.array([(mouth_bbox[0][0] + mouth_bbox[1][0]) / 2,
(mouth_bbox[2][1] + mouth_bbox[3][1]) / 2])
center_pixel = np.mean(
[left_eye_center, right_eye_center, mouth_center],
axis=0,
dtype=int)
jaw_bbox = get_bbox(jaw_points)
lowest_face_y = jaw_bbox[3][1]
if radius == 0:
radius = lowest_face_y - center_pixel[1]
top_left = [center_pixel[0] - radius, center_pixel[1] - radius]
bottom_right = [center_pixel[0] + radius, center_pixel[1] + radius]
img = Image.fromarray(image)
cropped_image = img.crop(
(top_left[0], top_left[1], bottom_right[0], bottom_right[1]))
resized_image = cropped_image.resize(
(self.desiredFaceWidth, self.desiredFaceHeight))
return np.array(resized_image, dtype='uint8'), radius
def get_facial_landmarks(self):
'''
Adds facial landmarks to the data using DLIB's facial landmark predictor
Example: http://dlib.net/face_landmark_detection.py.html
'''
self.facial_landmarks = helpers.shape_to_np(
predictor(self.grey_image, self._bounding_box))
self.PointChin = self.facial_landmarks[8]
self.PointNose = self.facial_landmarks[30]
self.PointLeftEyeLeft = self.facial_landmarks[36]
self.PointRightEyeRight = self.facial_landmarks[45]
self.PointMouthLeft = self.facial_landmarks[48]
self.PointMouthRight = self.facial_landmarks[54]
self.PointCheekLeft = self.facial_landmarks[0]
self.PointCheekRight = self.facial_landmarks[16]
def align_to_template_affine(self, image, gray, rect):
# align by affine warping transform image to hard set landmark locations on a template
# example template. Just something I came up with
template = {
'mouth': [48, 66],
'left_eye': [32, 33],
'right_eye': [64, 33]
}
shape = shape_to_np(self.predictor(gray, rect))
(l_start, l_end) = FACIAL_LANDMARKS_IDXS["left_eye"]
(r_start, r_end) = FACIAL_LANDMARKS_IDXS["right_eye"]
(m_start, m_end) = FACIAL_LANDMARKS_IDXS['mouth']
left_eye_points = shape[l_start:l_end]
right_eye_points = shape[r_start:r_end]
mouth_points = shape[m_start:m_end]
left_eye_bbox = get_bbox(left_eye_points)
left_eye_center = [(left_eye_bbox[0][0] + left_eye_bbox[1][0]) / 2,
(left_eye_bbox[2][1] + left_eye_bbox[3][1]) / 2]
right_eye_bbox = get_bbox(right_eye_points)
right_eye_center = [(right_eye_bbox[0][0] + right_eye_bbox[1][0]) / 2,
(right_eye_bbox[2][1] + right_eye_bbox[3][1]) / 2]
mouth_bbox = get_bbox(mouth_points)
mouth_center = [(mouth_bbox[0][0] + mouth_bbox[1][0]) / 2,
(mouth_bbox[2][1] + mouth_bbox[3][1]) / 2]
pts1 = np.float32([left_eye_center, right_eye_center, mouth_center])
pts2 = np.float32(
[template['left_eye'], template['right_eye'], template['mouth']])
affine_transform_matrix = cv2.getAffineTransform(pts1, pts2)
result = cv2.warpAffine(
image, affine_transform_matrix,
(self.desiredFaceWidth, self.desiredFaceHeight))
return result
def get_face_info(gray, face):
'''
This function follows func detect_dlib()
Return the center pos/radius/2D orientation of the given face
INPUT:
gray - the gray scaled image used for prediction
face - the given face
OUTPUT:
angle - the orientation of the face in 2D
radius - the radius of the face range, unit in pixels
center - the position/center of the face, set as the middle point between eyes
'''
# extract the ROI of the *original* face, then align the face
# using facial landmarks
shape = DLIB_PREDICTOR(gray, face)
shape = shape_to_np(shape)
# Left eye
avg_x_left, avg_y_left = get_feature_pos("left_eye", shape)
cv2.circle(gray, (int(avg_x_left), int(avg_y_left)), 1, (128, 128, 128),
-1)
# Right eye
avg_x_right, avg_y_right = get_feature_pos("right_eye", shape)
cv2.circle(gray, (int(avg_x_right), int(avg_y_right)), 1, (128, 128, 128),
-1)
# Calculate the angle (2D orientation)
angle = np.arctan((avg_y_right - avg_y_left) / (avg_x_right - avg_x_left))
angle = angle / np.pi * 180
# Calculate the radius
width = (face.height() + face.width()) / 2.0
# Calulate center
center = (int(
(avg_x_left + avg_x_right) / 2), int((avg_y_left + avg_y_right) / 2))
return angle, width, center
def get_face_features(gray, face):
'''
Used to judge if two faces are the same person
INPUT:
gray - the gray scaled image used for prediction
face - the given face
OUTPUT:
features - a vector recording the distances between face features
Here I take the person's nose as the reference point, thus in the
following expression dist[<feature>] means the distance between
his/her nose and the feature point.
The order of features np array:
[dist["mouth"], dist["right_eyebrow"], dist["left_eyebrow"],
dist["right_eye"], dist["left_eye"], dist["jaw"]]
'''
shape = DLIB_PREDICTOR(gray, face)
shape = shape_to_np(shape)
positions = []
nose = get_feature_pos("nose", shape)
positions.append(get_feature_pos("mouth", shape))
positions.append(get_feature_pos("right_eyebrow", shape))
positions.append(get_feature_pos("left_eyebrow", shape))
positions.append(get_feature_pos("right_eye", shape))
positions.append(get_feature_pos("left_eye", shape))
positions.append(get_feature_pos("jaw", shape))
# Get the vector
features = np.zeros((1, 6))
for i in range(6):
features[0, i] = get_distance(nose, positions[i])
# Normalize
features = features / np.sqrt(np.sum(features**2))
return features
def run():
print("Reach Position 1")
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
fishface = Loader.load_model_fish("googleCKPlus.xml")
video = cv2.VideoCapture(0)
current = 0
model = "20180402-114759"
print("Reach Position 2")
with tf.Graph().as_default():
# print(tf.get_default_graph())
print("Reach Position 3")
with tf.Session() as sess:
# Load the model
## we need to load the model first, then load each layer
Loader.load_model(model)
images_placeholder = tf.get_default_graph().get_tensor_by_name(
"input:0")
embeddings = tf.get_default_graph().get_tensor_by_name(
"embeddings:0")
phase_train_placeholder = tf.get_default_graph(
).get_tensor_by_name("phase_train:0")
print("Reach Position 4")
while (True):
ret, frame = video.read()
frame = helpers.resize(frame, width=1200)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# this is the right place to put the copy,
# otherwise it will have empty when the face is too big
temp = copy.deepcopy(frame)
rects = detector(gray, 1)
print("Running.....")
for (i, rect) in enumerate(rects):
shape = predictor(gray, rect)
shape = helpers.shape_to_np(shape)
(x, y, w, h) = helpers.rect_to_coordinate(rect)
# draw rectangle
cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0),
2)
# draw circle
for (x1, y1) in shape:
cv2.circle(frame, (x1, y1), 2, (0, 0, 255), -1)
# to prevent empty frame
try:
temp = temp[y:y + h, x:x + w]
temp_160 = misc.imresize(temp, (160, 160),
interp='bilinear')
# Snap by the camera save by the time stamp
# cv2.imwrite("./camera_photo/{}.png".format(datetime.fromtimestamp(time.time())), temp)
# print("SNAP!!!!!!!!!!!!! GIVE A SMILE")
if temp_160.ndim == 2:
temp_160 = helpers.to_rgb_from2(temp_160)
x1, y1, a1 = temp_160.shape
temp_re = temp_160.reshape([1, x1, y1, a1])
# we put the cropped image to the FaceNet, input shape(1,160,160,3)
feed_dict = {
images_placeholder: temp_re,
phase_train_placeholder: False
}
# emb return the facial feature of shape (1,512)
emb = sess.run(embeddings, feed_dict=feed_dict)
print("Network running....")
except ValueError:
pass
try:
tag = helpers.calculation(emb[0])
cv2.putText(frame, "{}".format(tag), (x - 10, y - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
# out = cv2.resize(temp, (350, 350))
gray = cv2.cvtColor(
temp, cv2.COLOR_BGR2GRAY) # Convert image to grayscale
# out=misc.imresize(gray, (350, 350), interp='bilinear')
out = cv2.resize(gray, (350, 350))
pred, conf = fishface.predict(out)
# write on img
info1 = 'Guessed emotion: ' + emotions[pred]
cv2.putText(frame, info1, (10, 20),
cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255, 100, 0))
print(tag)
print(emotions[pred])
except UnboundLocalError:
pass
# we put the processed frame back to the camera
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2BGRA)
cv2.imshow('frame', rgb)
# press the key 'q' to quit the program
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# press the key 'p' to snap a picture
if cv2.waitKey(1) & 0xFF == ord('p'):
cv2.imwrite('capture{}.jpg'.format(current), frame)
current += 1
video.release()
cv2.destroyAllWindows()
def run(video, name):
print("Reach Position 1")
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
fishface = Loader.load_model_fish("googleCKPlus.xml")
video = cv2.VideoCapture(video)
# Check if camera opened successfully
if (video.isOpened() == False):
print("Error opening video stream or file")
current = 0
model = "20180402-114759"
print("Reach Position 2")
rectangles = []
kkkk = 0
with tf.Graph().as_default():
# print(tf.get_default_graph())
print("Reach Position 3")
with tf.Session() as sess:
# Load the model
## we need to load the model first, then load each layer
Loader.load_model(model)
images_placeholder = tf.get_default_graph().get_tensor_by_name(
"input:0")
embeddings = tf.get_default_graph().get_tensor_by_name(
"embeddings:0")
phase_train_placeholder = tf.get_default_graph(
).get_tensor_by_name("phase_train:0")
print("Reach Position 4")
while (video.isOpened()):
# videoture frame-by-frame
try:
print("processing frame")
print(kkkk)
kkkk += 1
ret, frame = video.read()
data_landmark = []
height, width, layers = frame.shape
except AttributeError:
break
size = (width, height)
# frame = helpers.resize(frame, width=1200)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# this is the right place to put the copy,
# otherwise it will have empty when the face is too big
temp = copy.deepcopy(frame)
rects = detector(gray, 1)
print("Running.....")
for (i, rect) in enumerate(rects):
shape = predictor(gray, rect)
shape = helpers.shape_to_np(shape)
(x, y, w, h) = helpers.rect_to_coordinate(rect)
# draw rectangle
cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0),
2)
# draw circle
for (x1, y1) in shape:
cv2.circle(frame, (x1, y1), 2, (0, 0, 255), -1)
data_landmark.append(x1)
data_landmark.append(y1)
# to prevent empty frame
try:
temp = temp[y:y + h, x:x + w]
print(
temp.shape
) #this is to trigger error then the temp is out of scale and become empty, we skip it
temp_160 = misc.imresize(temp, (160, 160),
interp='bilinear')
# Snap by the camera save by the time stamp
# cv2.imwrite("./camera_photo/{}.png".format(datetime.fromtimestamp(time.time())), temp)
# print("SNAP!!!!!!!!!!!!! GIVE A SMILE")
if temp_160.ndim == 2:
temp_160 = helpers.to_rgb_from2(temp_160)
x1, y1, a1 = temp_160.shape
temp_re = temp_160.reshape([1, x1, y1, a1])
# we put the cropped image to the FaceNet, input shape(1,160,160,3)
feed_dict = {
images_placeholder: temp_re,
phase_train_placeholder: False
}
# emb return the facial feature of shape (1,512)
emb = sess.run(embeddings, feed_dict=feed_dict)
print("Network running....")
except ValueError:
continue
try:
tag = helpers.calculation(emb[0])
cv2.putText(frame, "{}".format(tag), (x - 10, y - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
print("success!!!!!!1")
# out = cv2.resize(temp, (350, 350))
info1 = emotion_predictor.output(data_landmark)
cv2.putText(frame, info1, (10, 20),
cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 255, 0))
print(tag)
rectangles.append(frame)
except UnboundLocalError:
pass
out = cv2.VideoWriter('{}.avi'.format(name),
cv2.VideoWriter_fourcc(*'DIVX'), 15, size)
for img in rectangles:
# write to video
out.write(img)
out.release()
print("herer!!!!!!!")
def run(pic, name):
print("Reach Position 1")
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
# Check if camera opened successfully
current = 0
model = "20180402-114759"
print("Reach Position 2")
rectangles = []
kkkk = 0
with tf.Graph().as_default():
# print(tf.get_default_graph())
print("Reach Position 3")
with tf.Session() as sess:
# Load the model
## we need to load the model first, then load each layer
Loader.load_model(model)
images_placeholder = tf.get_default_graph().get_tensor_by_name(
"input:0")
embeddings = tf.get_default_graph().get_tensor_by_name(
"embeddings:0")
phase_train_placeholder = tf.get_default_graph(
).get_tensor_by_name("phase_train:0")
print("Reach Position 4")
frame = cv2.imread(pic)
# frame = helpers.resize(frame, width=1200)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# this is the right place to put the copy,
# otherwise it will have empty when the face is too big
rects = detector(gray, 1)
for (i, rect) in enumerate(rects):
temp = copy.deepcopy(frame)
data_landmark = []
shape = predictor(gray, rect)
shape = helpers.shape_to_np(shape)
(x, y, w, h) = helpers.rect_to_coordinate(rect)
# draw rectangle
cv2.rectangle(frame, (x, y), (x + w, y + h), (87, 192, 56), 2)
# draw circle
for (x1, y1) in shape:
cv2.circle(frame, (x1, y1), 4, (0, 0, 255), -1)
data_landmark.append(x1)
data_landmark.append(y1)
# to prevent empty frame
try:
temp = temp[y:y + h, x:x + w]
print(
temp.shape
) #this is to trigger error then the temp is out of scale and become empty, we skip it
cv2.imwrite("?????{}.jpg".format(name), temp)
temp_160 = misc.imresize(temp, (160, 160),
interp='bilinear')
# Snap by the camera save by the time stamp
if temp_160.ndim == 2:
temp_160 = helpers.to_rgb_from2(temp_160)
x1, y1, a1 = temp_160.shape
temp_re = temp_160.reshape([1, x1, y1, a1])
# we put the cropped image to the FaceNet, input shape(1,160,160,3)
feed_dict = {
images_placeholder: temp_re,
phase_train_placeholder: False
}
# emb return the facial feature of shape (1,512)
emb = sess.run(embeddings, feed_dict=feed_dict)
print("Network running....")
except ValueError:
continue
try:
tag = helpers.calculation(emb[0])
cv2.putText(frame, "{}".format(tag), (x - 10, y - 10),
cv2.FONT_HERSHEY_SIMPLEX, 2, (87, 192, 56), 2)
print("success!!!!!!1")
# out = cv2.resize(temp, (350, 350))
info1 = emotion_predictor.output(data_landmark)
cv2.putText(frame, info1, (x - 10, y - 80),
cv2.FONT_HERSHEY_SIMPLEX, 2, (87, 192, 56), 2)
print(tag)
except UnboundLocalError:
pass
out = cv2.imwrite("{}.jpg".format(name), frame)
print("herer!!!!!!!")
def calculate_feature(names):
# output log file
log_file = open("Uploader_info.log", "w")
old_stdout = sys.stdout
sys.stdout = log_file
img_list = []
ppp = -1
error_list = []
with tf.Graph().as_default():
with tf.Session() as sess:
# we need to load the model first, then load each layer
Loader.load_model(model)
images_placeholder = tf.get_default_graph().get_tensor_by_name(
"input:0")
embeddings = tf.get_default_graph().get_tensor_by_name(
"embeddings:0")
phase_train_placeholder = tf.get_default_graph(
).get_tensor_by_name("phase_train:0")
for name in names:
ppp += 1
print("\n" + name)
# sometime read image may have null, thus nullpointer exception
try:
# name=os.path.join('./Team/', name)
img = cv2.imread(name)
img = helpers.resize(img, width=1200)
except AttributeError:
print("error, cannot find {} ".format(name))
error_list.append(ppp)
continue
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# this is the right place to put the copy,
# otherwise it will have empty when the face is too big
rects = detector(gray, 1)
if (len(rects)) == 0:
print("error, cannot detect face in {} ".format(name))
error_list.append(ppp)
continue
print("Running.....on {}".format(name))
for (i, rect) in enumerate(rects):
try:
if (len(rects)) > 1:
raise ValueError
shape = predictor(gray, rect)
shape = helpers.shape_to_np(shape)
(x, y, w, h) = helpers.rect_to_coordinate(rect)
img = img[y:y + h, x:x + w]
img = misc.imresize(img, (160, 160), interp='bilinear')
cv2.imwrite("./database_snap/{}".format(name[7:]), img)
img_list.append(img)
print(name + " Success!!!!!!!!!!!!!!!")
# if there are one more one face, we add it to the error list
except ValueError:
print("error, {} have more than one faces!!!".format(
name))
error_list.append(ppp)
break
all_img = np.stack(img_list, axis=0)
# we put the cropped image to the FaceNet, input shape(1,160,160,3)
feed_dict = {
images_placeholder: all_img,
phase_train_placeholder: False
}
# emb return the facial feature of shape (1,512)
embs = sess.run(embeddings, feed_dict=feed_dict)
#log infor
sys.stdout = old_stdout
log_file.close()
return all_img.tolist(), embs.tolist(), error_list
threading.Thread(target=helpers.sendData).start()
# loop over the frames from the video stream
while True:
frame = capture.read()[1]
frame = cv2.resize(frame, (data.cap_width, data.cap_height))
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# detect faces in the grayscale frame
rects = detector(gray, 0)
face_cnt = len(rects)
# always use the first face
if face_cnt > 0:
points = predictor(gray, rects[0])
points = helpers.shape_to_np(points)
cv2.rectangle(frame, (rects[0].left(), rects[0].top()),
(rects[0].right(), rects[0].bottom()), (255, 0, 0))
for (x, y) in points:
cv2.circle(frame, (x, y), 3, (0, 0, 255), -1)
cv2.polylines(frame, numpy.int32([points[:17]]), False, (0, 0, 255))
cv2.polylines(frame, numpy.int32([points[17:22]]), False, (0, 0, 255))
cv2.polylines(frame, numpy.int32([points[22:27]]), False, (0, 0, 255))
cv2.polylines(frame, numpy.int32([points[27:31]]), False, (0, 0, 255))
cv2.polylines(frame, numpy.int32([points[31:36]]), False, (0, 0, 255))
cv2.polylines(frame, numpy.int32([points[36:42]]), True, (0, 0, 255))
cv2.polylines(frame, numpy.int32([points[42:48]]), True, (0, 0, 255))
cv2.polylines(frame, numpy.int32([points[48:60]]), True, (0, 0, 255))
def align_geometric_eyes(self, image, gray, rect):
# convert the landmark (x, y)-coordinates to a NumPy array
shape = self.predictor(gray, rect)
shape = shape_to_np(shape)
the_real_shape = np.shape(image)
# extract the left and right eye (x, y)-coordinates
(l_start, l_end) = FACIAL_LANDMARKS_IDXS["left_eye"]
(r_start, r_end) = FACIAL_LANDMARKS_IDXS["right_eye"]
(m_start, m_end) = FACIAL_LANDMARKS_IDXS['mouth']
left_eye_points = shape[l_start:l_end]
right_eye_points = shape[r_start:r_end]
mouth_points = shape[m_start:m_end]
# print('lefT_eye_point: %s' % str(left_eye_points))
# print('right_eye_point: %s' % str(right_eye_points))
# print('mouth_point: %s' % str(mouth_points))
# compute the center of mass for each eye
# left_eye_center = left_eye_points.mean(axis=0).astype("int")
# right_eye_center = right_eye_points.mean(axis=0).astype("int")
# compute the geometrical center of each eye
left_eye_bbox = get_bbox(left_eye_points)
left_eye_center = np.array([
(left_eye_bbox[0][0] + left_eye_bbox[1][0]) / 2,
(left_eye_bbox[2][1] + left_eye_bbox[3][1]) / 2
])
right_eye_bbox = get_bbox(right_eye_points)
right_eye_center = np.array([
(right_eye_bbox[0][0] + right_eye_bbox[1][0]) / 2,
(right_eye_bbox[2][1] + right_eye_bbox[3][1]) / 2
])
# compute the geometrical center of the mouth
mouth_bbox = get_bbox(mouth_points)
mouth_center = np.array([(mouth_bbox[0][0] + mouth_bbox[1][0]) / 2,
(mouth_bbox[2][1] + mouth_bbox[3][1]) / 2])
# compute the angle between the eye centroids
dY = right_eye_center[1] - left_eye_center[1]
dX = right_eye_center[0] - left_eye_center[0]
angle = np.degrees(np.arctan2(dY, dX)) - 180
# compute the desired right eye x-coordinate based on the
# desired x-coordinate of the left eye
desiredRightEyeX = 1.0 - self.desiredLeftEye[0]
# determine the scale of the new resulting image by taking
# the ratio of the distance between eyes in the *current*
# image to the ratio of distance between eyes in the
# *desired* image
dist = np.sqrt((dX**2) + (dY**2))
desiredDist = (desiredRightEyeX - self.desiredLeftEye[0])
desiredDist *= self.desiredFaceWidth
scale = desiredDist / dist
# compute center (x, y)-coordinates (i.e., the median point)
# between the two eyes in the input image
eyesCenter = ((left_eye_center[0] + right_eye_center[0]) // 2,
(left_eye_center[1] + right_eye_center[1]) // 2)
# grab the rotation matrix for rotating and scaling the face
M = cv2.getRotationMatrix2D(eyesCenter, angle, scale)
# update the translation component of the matrix
tX = self.desiredFaceWidth * 0.5
tY = self.desiredFaceHeight * self.desiredLeftEye[1]
M[0, 2] += (tX - eyesCenter[0])
M[1, 2] += (tY - eyesCenter[1])
def pretend_image(ship, h, w):
new_array = np.zeros((h, w, 3))
for pixel in ship:
new_array[pixel[1]][pixel[0]] = 255
return new_array
# apply the affine transformation
(w, h) = (self.desiredFaceWidth, self.desiredFaceHeight)
output = cv2.warpAffine(image, M, (w, h), flags=cv2.INTER_CUBIC)
shape = pretend_image(shape, the_real_shape[0], the_real_shape[1])
# shape = np.array(shape, dtype='uint8')
# img = Image.fromarray(shape, mode='RGB')
# img.save('/home/gabi/PycharmProjects/imutils/testing/just_landmarks.jpg')
warped_landmarks = cv2.warpAffine(shape,
M, (w, h),
flags=cv2.INTER_CUBIC)
# new_shape = get_shape_from_pretend()
original_shape = np.shape(warped_landmarks)
warped_landmarks = np.ndarray.flatten(warped_landmarks)
length = len(warped_landmarks)
warped_landmarks = np.array(
[255 if warped_landmarks[i] > 0 else 0 for i in range(length)],
dtype='uint8')
warped_landmarks = np.reshape(warped_landmarks, original_shape)
# warped_landmarks = np.array(warped_landmarks, dtype='uint8')
# img = Image.fromarray(warped_landmarks, mode='RGB')
# img.save('/home/gabi/PycharmProjects/imutils/testing/warped_landmarks.jpg')
new_list = []
warped_landmarks = warped_landmarks[:, :, 0]
for x in range(np.shape(warped_landmarks)[0]):
for y in range(np.shape(warped_landmarks)[1]):
if warped_landmarks[x][y] > 0:
new_list.append([x, y])
warped_landmarks = np.array(new_list)
# return the aligned face
return output, [left_eye_points, right_eye_points,
mouth_points], warped_landmarks
