def demo():
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
# load query image
img = cv2.imread("test.jpg", 1)
plt.title('Query Image')
plt.imshow(img[:, :, ::-1])
# extract landmarks from the query image
# list containing a 2D array with points (x, y) for each face detected in the query image
lmarks = feature_detection.get_landmarks(img)
plt.figure()
plt.title('Landmarks Detected')
plt.imshow(img[:, :, ::-1])
plt.scatter(lmarks[0][:, 0], lmarks[0][:, 1])
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
# load mask to exclude eyes from symmetry
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
# perform frontalization
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix,
model3D.ref_U, eyemask)
plt.figure()
plt.title('Frontalized no symmetry')
plt.imshow(frontal_raw[:, :, ::-1])
plt.figure()
plt.title('Frontalized with soft symmetry')
plt.imshow(frontal_sym[:, :, ::-1])
plt.show()
def demo():
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
# load query image
img = cv2.imread("test.jpg", 1)
plt.title('Query Image')
plt.imshow(img[:, :, ::-1])
# extract landmarks from the query image
# list containing a 2D array with points (x, y) for each face detected in the query image
lmarks = feature_detection.get_landmarks(img)
plt.figure()
plt.title('Landmarks Detected')
plt.imshow(img[:, :, ::-1])
plt.scatter(lmarks[0][:, 0], lmarks[0][:, 1])
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(model3D, lmarks[0])
# load mask to exclude eyes from symmetry
eyemask = np.asarray(io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
# perform frontalization
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix, model3D.ref_U, eyemask)
plt.figure()
plt.title('Frontalized no symmetry')
plt.imshow(frontal_raw[:, :, ::-1])
plt.figure()
plt.title('Frontalized with soft symmetry')
plt.imshow(frontal_sym[:, :, ::-1])
plt.show()
def __init__(self):
this_path = os.path.dirname(os.path.abspath(__file__))
check.check_dlib_landmark_weights()
self.model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
self.eyemask = np.asarray(
io.loadmat(this_path +
'/frontalization_models/eyemask.mat')['eyemask'])
def demo():
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
# load query image
# img2 = cv2.imread("test.jpg", 1)
img = cv2.imread(
"/home/jordan/PycharmProjects/Emotion_Analysis/Facial_Expression_recog/data/test_photo.jpg",
1)
img = cv2.resize(img, (int(img.shape[0] / 3), int(img.shape[1] / 9)))
# detected_object = mouthdetection.findmouth(img, haar_face, haar_mouth)[0]
# img = smile_detection.crop(detected_object, img, extended = True)
cv2.imshow("", img)
cv2.waitKey(5000)
img = img.astype(np.uint8)
# print img.shape
plt.title('Query Image')
plt.imshow(img[:, :, ::-1])
# extract landmarks from the query image
# list containing a 2D array with points (x, y) for each face detected in the query image
lmarks = feature_detection.get_landmarks(img)
plt.figure()
plt.title('Landmarks Detected')
plt.imshow(img[:, :, ::-1])
plt.scatter(lmarks[0][:, 0], lmarks[0][:, 1])
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
# load mask to exclude eyes from symmetry
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
# perform frontalization
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix,
model3D.ref_U, eyemask)
plt.figure()
plt.title('Frontalized no symmetry')
plt.imshow(frontal_raw[:, :, ::-1])
plt.figure()
plt.title('Frontalized with soft symmetry')
plt.imshow(frontal_sym[:, :, ::-1])
plt.show()
def demo(image):
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
img = cv2.resize(image, (250, 250), interpolation=cv2.INTER_LINEAR)
lmarks, f = feature_detection.get_landmarks(img)
if not f:
return False, 1
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
print(proj_matrix)
# load mask to exclude eyes from symmetry
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
# perform frontalization
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix,
model3D.ref_U, eyemask)
'''
plt.figure()
plt.title('Image frontalized(Before symmetry)')
plt.imshow(frontal_raw[:,:,::-1].astype('uint8'))
plt.figure()
plt.title('Image frontalized(After symmetry)')
plt.imshow(frontal_sym[:,:,::-1].astype('uint8'))
'''
x, y, z = grammer.get_angle(rmat)
print(('旋转的角度： x: {}, y: {}, z: {}').format(x, y, z)) #估算大概的旋转角度
image_output = frontal_sym.astype('uint8')
return True, image_output
def load_face_frontal_elements(path_to_face_front_utilyties):
'''
Routine to prepare elements for face-frontalization module by dougsouza in:
https://github.com/dougsouza/face-frontalization
Tal Hassner, Shai Harel, Eran Paz and Roee Enbar, Effective Face Frontalization in Unconstrained Images,
IEEE Conf. on Computer Vision and Pattern Recognition (CVPR), Boston, June 2015
'''
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(
path_to_face_front_utilyties +
'/frontalization_models/model3Ddlib.mat', 'model_dlib')
# load mask to exclude eyes from symmetry
eyemask = np.asarray(
io.loadmat(path_to_face_front_utilyties +
'/frontalization_models/eyemask.mat')['eyemask'])
return model3D, eyemask
def demo():
nSub = opts.getint('general', 'nTotSub')
fileList, outputFolder = myutil.parse(sys.argv)
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/d.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
## Preloading all the models for speed
allModels = myutil.preload(this_path, pose_models_folder, pose_models,
nSub)
for f in fileList:
if '#' in f: #skipping comments
continue
splitted = f.split(',')
image_key = splitted[0]
image_path = splitted[1]
image_landmarks = splitted[2]
img = cv2.imread(image_path, 1)
if image_landmarks != "None":
lmark = np.loadtxt(image_landmarks)
lmarks = []
lmarks.append(lmark)
else:
print('> Detecting landmarks')
lmarks = feature_detection.get_landmarks(img, this_path)
if len(lmarks) != 0:
## Copy back original image and flipping image in case we need
## This flipping is performed using all the model or all the poses
## To refine the estimation of yaw. Yaw can change from model to model...
img_display = img.copy()
img, lmarks, yaw = myutil.flipInCase(img, lmarks, allModels)
listPose = myutil.decidePose(yaw, opts, newModels)
## Looping over the poses
for poseId in listPose:
posee = pose_models[poseId]
## Looping over the subjects
for subj in range(1, nSub + 1):
pose = posee + '_' + str(subj).zfill(2) + '.mat'
print('> Looking at file: ' + image_path + ' with ' + pose)
# load detections performed by dlib library on 3D model and Reference Image
print("> Using pose model in " + pose)
## Indexing the right model instead of loading it each time from memory.
model3D = allModels[pose]
eyemask = model3D.eyemask
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
## We use eyemask only for frontal
if not myutil.isFrontal(pose):
eyemask = None
##### Main part of the code: doing the rendering #############
rendered_raw, rendered_sym, face_proj, background_proj, temp_proj2_out_2, sym_weight = renderer.render(img, proj_matrix,\
model3D.ref_U, eyemask, model3D.facemask, opts)
########################################################
if myutil.isFrontal(pose):
rendered_raw = rendered_sym
## Cropping if required by crop_models
rendered_raw = myutil.cropFunc(pose, rendered_raw,
crop_models[poseId])
## Resizing if required
if resizeCNN:
rendered_raw = cv2.resize(
rendered_raw, (cnnSize, cnnSize),
interpolation=cv2.INTER_CUBIC)
## Saving if required
if opts.getboolean('general', 'saveON'):
subjFolder = outputFolder + '/' + image_key.split(
'_')[0]
myutil.mymkdir(subjFolder)
savingString = subjFolder + '/' + image_key + '_rendered_' + pose[
8:-7] + '_' + str(subj).zfill(2) + '.jpg'
cv2.imwrite(savingString, rendered_raw)
## Plotting if required
if opts.getboolean('general', 'plotON'):
myutil.show(img_display, img, lmarks, rendered_raw, \
face_proj, background_proj, temp_proj2_out_2, sym_weight)
else:
print('> Landmark not detected for this image...')
def myfrontalize(X, limit=0):
count = 0
if (limit == 0):
limit = X.shape[0]
print("Total Images: ", limit)
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
# load mask to exclude eyes from symmetry
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
for i in range(0, limit):
print("\r", end='')
print("Images Completed: {0}".format(i), end='', flush=True)
# cast img to type int for cv2
img = X[i, :, :, 0].astype(np.uint8)
# create a color version for frontalizer stuffs
c_img = np.copy(img)
c_img = cv2.cvtColor(c_img, cv2.COLOR_GRAY2BGR)
# extract landmarks from the query image
# list containing a 2D array with points (x, y) for each face detected in the query image
lmarks = feature_detection.get_landmarks(c_img)
if type(lmarks) is np.ndarray:
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
# perform frontalization and convert result to grayscale
frontal_raw, frontal_sym = frontalize.frontalize(
c_img, proj_matrix, model3D.ref_U, eyemask)
temp = cv2.cvtColor(frontal_raw, cv2.COLOR_BGR2GRAY)
# find nonzero bbox and crop image to remove uncessesary black space from edges
temp_mask = cv2.findNonZero(temp)
t_x, t_y, t_w, t_h = cv2.boundingRect(temp_mask)
t_bbox = temp[t_y:t_y + t_h, t_x:t_x + t_w]
# resize the cropped image to the appropriate dimensions for network
t_bbox = cv2.resize(t_bbox, dsize=(48, 48))
t_bbox = np.resize(t_bbox, (48, 48, 1))
X[i] = t_bbox.astype(np.float32)
plt.show()
count += 1
print()
print('{} images out of {} were frontalized.'.format(count, limit))
__author__ = 'Merle'
import math
import os
import time
import numpy as np
from skimage import io
import time
import check_resources as check
import dlib
import facial_feature_detector as feature_detection
import PoseEstimation as PE
this_path = os.path.dirname(os.path.abspath(__file__))
check.check_dlib_landmark_weights() # 检测dlib参数是否下载，没下载的话下载
image_name = 'zjl.jpg'
image_path = this_path + '/input/' + image_name
time1 = time.time()
print(this_path)
predictor_path = this_path + "/dlib_models/shape_predictor_68_face_landmarks.dat"
print(predictor_path)
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(predictor_path)
time2 = time.time()
def get_angle(image, pt3d, sess):
lmarks = feature_detection.get_landmarks(image, detector, predictor)
Pose_Para = []
video_out = cv2.VideoWriter('./rotated/' + video_name + '.mp4', fourcc,
fps, (320, 320)) #3d_output
leye_out = cv2.VideoWriter('./rotated/' + video_name + '_leye.mp4', fourcc,
fps, (EYE_SIZE[0], EYE_SIZE[1])) #leye
reye_out = cv2.VideoWriter('./rotated/' + video_name + '_reye.mp4', fourcc,
fps, (EYE_SIZE[0], EYE_SIZE[1])) #reye
mouth_out = cv2.VideoWriter('./rotated/' + video_name + '_mouth.mp4',
fourcc, fps,
(MOUTH_SIZE[0], MOUTH_SIZE[1])) #mouth
print('=== loading landmarks ===')
lmarks = np.load(sys.argv[2])
#lmarks = np.load('./detected_3d/' + video_name + '_fixed.npy')
print('===loading model === ')
check.check_dlib_landmark_weights()
black_img = np.zeros((320, 320, 3), dtype='uint8')
i = -1
while (video.isOpened() and i + 1 < len(lmarks)):
ret, frame = video.read()
i += 1
print('reading : frame ' + str(i))
if ret == True and len(lmarks[i]) > 0:
lmark = copy.copy(lmarks[i][0][:, :2])
frontal, reye, leye, mouth, _ = img_normalization(frame, lmark)
video_out.write(frontal)
reye_out.write(reye)
leye_out.write(leye)
def face_detect(image_file_path, image_save_path):
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
# load detections performed by dlib library on 3D model and Reference Image
# load query image
#print image_file_path
img = cv2.imread(image_file_path, 1)
(ori_w, org_h, org_channel) = img.shape
check.check_dlib_landmark_weights()
lmarks = feature_detection.get_landmarks(img)
if lmarks.shape[0] == 0:
return 0
else:
print image_save_path
if not os.path.exists(image_save_path):
os.makedirs(image_save_path)
for num_face in range(0, lmarks.shape[0]):
face_landmarks = lmarks[num_face, :, :]
print "Detect face %d out of total %d faces" % (
(num_face + 1), lmarks.shape[0])
print "start to align face"
w = 256
h = 256
eyecornerDst = [(np.int(0.25 * w), np.int(h / 3.5)),
(np.int(0.75 * w), np.int(h / 3.5)),
(np.int(0.5 * w), np.int(h / 2))]
eyecornerSrc = [
(np.int(face_landmarks[36][0]), np.int(face_landmarks[36][1])),
(np.int(face_landmarks[45][0]), np.int(face_landmarks[45][1])),
(np.int(face_landmarks[30][0]), np.int(face_landmarks[30][1]))
]
# Apply similarity transformation
tform = similarityTransform(eyecornerSrc, eyecornerDst)
img_face_aligned = cv2.warpAffine(img, tform, (w, h))
#note that x is horizontal, is width, while y is height, is vertical
#print tform
width = np.int(
(face_landmarks[16][0] - face_landmarks[0][0]) * 1.3)
height = np.int(
(face_landmarks[8][1] - face_landmarks[19][1]) * 1.3)
if width < 0 | height < 0:
print("face crooked")
sys.exit()
left_top_x = np.int(face_landmarks[0][0]) - np.int(
(np.int(face_landmarks[16][0]) - np.int(face_landmarks[0][0]))
* 0.15)
left_top_y = np.int(face_landmarks[19][1]) - np.int(
(np.int(face_landmarks[8][1]) - np.int(face_landmarks[19][1]))
* 0.15)
if left_top_x < 0:
left_top_x = 0
if left_top_y < 0:
left_top_y = 0
image_new_name_affined = image_save_path + '/A_' + str(
left_top_x) + '_' + str(left_top_y) + '_' + str(
width) + '_' + str(height) + '.jpg'
resized_image = cv2.resize(img_face_aligned, (256, 256))
cv2.imwrite(image_new_name_affined, resized_image)
return 1
