def get_frontal_image(image):
predictor_path = 'dlib_models/shape_predictor_68_face_landmarks.dat'
model3D_path = 'frontalization_models/model3Ddlib.mat'
eyemask_path = 'frontalization_models/eyemask.mat'
model3D = frontalize.ThreeD_Model(model3D_path, 'model_dlib')
eyemask = np.asarray(io.loadmat(eyemask_path)['eyemask'])
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(predictor_path)
img = cv2.imread(image, 1)
height, width, layers = img.shape
new_h = round(height / 3)
new_w = round(width / 3)
img = cv2.resize(img, (new_w, new_h))
model3D = frontalize.ThreeD_Model('frontalization_models/model3Ddlib.mat',
'model_dlib')
lmarks = feature_detection.get_landmarks(img, detector, predictor)
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix,
model3D.ref_U, eyemask)
cv2.imwrite('frontal_image.png', frontal_sym)
def demo():
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
# load query image
img = cv2.imread("test.jpg", 1)
plt.title('Query Image')
plt.imshow(img[:, :, ::-1])
# extract landmarks from the query image
# list containing a 2D array with points (x, y) for each face detected in the query image
lmarks = feature_detection.get_landmarks(img)
plt.figure()
plt.title('Landmarks Detected')
plt.imshow(img[:, :, ::-1])
plt.scatter(lmarks[0][:, 0], lmarks[0][:, 1])
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
# load mask to exclude eyes from symmetry
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
# perform frontalization
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix,
model3D.ref_U, eyemask)
plt.figure()
plt.title('Frontalized no symmetry')
plt.imshow(frontal_raw[:, :, ::-1])
plt.figure()
plt.title('Frontalized with soft symmetry')
plt.imshow(frontal_sym[:, :, ::-1])
plt.show()
def demo():
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
# load query image
img = cv2.imread("test.jpg", 1)
plt.title('Query Image')
plt.imshow(img[:, :, ::-1])
# extract landmarks from the query image
# list containing a 2D array with points (x, y) for each face detected in the query image
lmarks = feature_detection.get_landmarks(img)
plt.figure()
plt.title('Landmarks Detected')
plt.imshow(img[:, :, ::-1])
plt.scatter(lmarks[0][:, 0], lmarks[0][:, 1])
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(model3D, lmarks[0])
# load mask to exclude eyes from symmetry
eyemask = np.asarray(io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
# perform frontalization
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix, model3D.ref_U, eyemask)
plt.figure()
plt.title('Frontalized no symmetry')
plt.imshow(frontal_raw[:, :, ::-1])
plt.figure()
plt.title('Frontalized with soft symmetry')
plt.imshow(frontal_sym[:, :, ::-1])
plt.show()
def get_angle(image, pt3d, sess):
lmarks = feature_detection.get_landmarks(image, detector, predictor)
Pose_Para = []
if len(lmarks):
start = time.time()
pt3d = np.load('pt3d.npy')
Pose_Para = PE.poseEstimation(image, lmarks, pt3d, sess)
print("calauate pose cost : ", time.time() - start)
return np.array(Pose_Para) * 180 / math.pi, lmarks
else:
print('can not find landmark!')
return None, None
def frontalized(self, image):
landmarks = facial.get_landmarks(image, self.args.resource_dir,
self.args)
if len(landmarks) > 0:
proj_matrix, camera_matrix, rmat, tvec = camera.estimate_camera(
self.model3D, landmarks[0])
_, front_image = front.frontalize(image, proj_matrix,
self.model3D.ref_U, self.eyemask)
detection = self.detector(front_image, 1)
for _, detection in enumerate(detection):
return front_image[detection.top():detection.bottom(),
detection.left():detection.right()]
return
def demo():
img = io.imread(image_path)
# 检测特征点
time3 = time.time()
lmarks = feature_detection.get_landmarks(img, detector, predictor)
time4 = time.time()
if len(lmarks):
Pose_Para = PE.poseEstimation(img, lmarks)
time5 = time.time()
else:
print('NO face detected!')
return 0
print('Dlib Model Load time is:', time2-time1, 'Lmarks Detect time is:', time4-time3, 'Pose estimation time is:', time5-time4)
print(np.array(Pose_Para)*180/math.pi)
def front_database_build(name, data_path, work_path, eyemask):
'''
Given the name of the subject in the variable -name-, the path from where to obtain the images,
the path to the destiny folder in which to save the database and the eyemask parameter used for
frontalization of the faces, this routine takes the images from data_path and saves the
frontalized face obtained in the work_path separating pictures in folders for each individual.
'''
for x in name[:]:
pathdir = os.path.join(work_path, x)
#print pathdir #for debug purposes
make_sure_path_exists(pathdir)
search_filename = x + "*.pgm"
search_location = os.path.join(data_path, search_filename)
#print search_location #for debug purposes
for filepath in glob.glob(search_location):
# Extract filename from the filepath found at search location
tail = os.path.basename(filepath) #os.path.split(search_location)
#print tail #for debug purposes
out_name = tail + 'fr.pgm'
# For MIT-CBCL dataset file naming, break name into the image data provided in filename
# 'subject' - 'face rotation' - etc
file_data = re.split(r"[_]", tail)
#print file_data #for debug purposes
# angular limit for rotated face frontalization
face_rot = -26
#Copy only the images that have less than face_rot rotation angle
if os.path.isfile(filepath) and face_rot < float(file_data[1]):
img = cv2.imread(filepath, 1)
# extract landmarks from the query image
# list containing a 2D array with points (x, y) for each face detected in the query image
lmarks = feature_detection.get_landmarks(img)
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
# perform frontalization
frontal_raw, frontal_sym = frontalize.frontalize(
img, proj_matrix, model3D.ref_U, eyemask)
#shutil.copy(filepath, pathdir)
#write frontal_sym to image file in pathdir location
cv2.imwrite(os.path.join(pathdir, out_name), frontal_sym)
def demo():
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
# load query image
# img2 = cv2.imread("test.jpg", 1)
img = cv2.imread(
"/home/jordan/PycharmProjects/Emotion_Analysis/Facial_Expression_recog/data/test_photo.jpg",
1)
img = cv2.resize(img, (int(img.shape[0] / 3), int(img.shape[1] / 9)))
# detected_object = mouthdetection.findmouth(img, haar_face, haar_mouth)[0]
# img = smile_detection.crop(detected_object, img, extended = True)
cv2.imshow("", img)
cv2.waitKey(5000)
img = img.astype(np.uint8)
# print img.shape
plt.title('Query Image')
plt.imshow(img[:, :, ::-1])
# extract landmarks from the query image
# list containing a 2D array with points (x, y) for each face detected in the query image
lmarks = feature_detection.get_landmarks(img)
plt.figure()
plt.title('Landmarks Detected')
plt.imshow(img[:, :, ::-1])
plt.scatter(lmarks[0][:, 0], lmarks[0][:, 1])
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
# load mask to exclude eyes from symmetry
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
# perform frontalization
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix,
model3D.ref_U, eyemask)
plt.figure()
plt.title('Frontalized no symmetry')
plt.imshow(frontal_raw[:, :, ::-1])
plt.figure()
plt.title('Frontalized with soft symmetry')
plt.imshow(frontal_sym[:, :, ::-1])
plt.show()
def demo(path):
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
img = cv2.imread(path, 1)
lmarks = feature_detection.get_landmarks(img)
if (len(lmarks) > 0):
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
frontal_raw, frontal_sym = frontalize.frontalize(
img, proj_matrix, model3D.ref_U, eyemask)
cv2.imwrite(path, frontal_raw)
def demo(image):
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
img = cv2.resize(image, (250, 250), interpolation=cv2.INTER_LINEAR)
lmarks, f = feature_detection.get_landmarks(img)
if not f:
return False, 1
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
print(proj_matrix)
# load mask to exclude eyes from symmetry
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
# perform frontalization
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix,
model3D.ref_U, eyemask)
'''
plt.figure()
plt.title('Image frontalized(Before symmetry)')
plt.imshow(frontal_raw[:,:,::-1].astype('uint8'))
plt.figure()
plt.title('Image frontalized(After symmetry)')
plt.imshow(frontal_sym[:,:,::-1].astype('uint8'))
'''
x, y, z = grammer.get_angle(rmat)
print(('旋转的角度： x: {}, y: {}, z: {}').format(x, y, z)) #估算大概的旋转角度
image_output = frontal_sym.astype('uint8')
return True, image_output
def get_def_pts(
image,
predictor_path='dlib_models/shape_predictor_68_face_landmarks.dat',
model3D_path='frontalization_models/model3Ddlib.mat',
eyemask_path='frontalization_models/eyemask.mat',
resize=False):
list_def_face = []
for f_name in sorted(os.listdir('data')):
if f_name.startswith('def_face'):
list_def_face.append(f_name)
points_list = []
print(image)
img = cv2.imread(image, 1)
height, width, layers = img.shape
if resize == True:
new_h = round(height / 3)
new_w = round(width / 3)
img = cv2.resize(img, (new_w, new_h))
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(predictor_path)
lmarks = feature_detection.get_landmarks(img, detector, predictor)
points_list = [[x, y] for x, y in lmarks[0]]
for num, file in enumerate(list_def_face):
with shelve.open('extracted_data/default_points_{}.txt'.format(num),
'c') as db:
for i, x in enumerate(points_list):
db['default_face'] = points_list
return (points_list)
def demo():
nSub = opts.getint('general', 'nTotSub')
fileList, outputFolder = myutil.parse(sys.argv)
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/d.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
## Preloading all the models for speed
allModels = myutil.preload(this_path, pose_models_folder, pose_models,
nSub)
for f in fileList:
if '#' in f: #skipping comments
continue
splitted = f.split(',')
image_key = splitted[0]
image_path = splitted[1]
image_landmarks = splitted[2]
img = cv2.imread(image_path, 1)
if image_landmarks != "None":
lmark = np.loadtxt(image_landmarks)
lmarks = []
lmarks.append(lmark)
else:
print('> Detecting landmarks')
lmarks = feature_detection.get_landmarks(img, this_path)
if len(lmarks) != 0:
## Copy back original image and flipping image in case we need
## This flipping is performed using all the model or all the poses
## To refine the estimation of yaw. Yaw can change from model to model...
img_display = img.copy()
img, lmarks, yaw = myutil.flipInCase(img, lmarks, allModels)
listPose = myutil.decidePose(yaw, opts, newModels)
## Looping over the poses
for poseId in listPose:
posee = pose_models[poseId]
## Looping over the subjects
for subj in range(1, nSub + 1):
pose = posee + '_' + str(subj).zfill(2) + '.mat'
print('> Looking at file: ' + image_path + ' with ' + pose)
# load detections performed by dlib library on 3D model and Reference Image
print("> Using pose model in " + pose)
## Indexing the right model instead of loading it each time from memory.
model3D = allModels[pose]
eyemask = model3D.eyemask
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
## We use eyemask only for frontal
if not myutil.isFrontal(pose):
eyemask = None
##### Main part of the code: doing the rendering #############
rendered_raw, rendered_sym, face_proj, background_proj, temp_proj2_out_2, sym_weight = renderer.render(img, proj_matrix,\
model3D.ref_U, eyemask, model3D.facemask, opts)
########################################################
if myutil.isFrontal(pose):
rendered_raw = rendered_sym
## Cropping if required by crop_models
rendered_raw = myutil.cropFunc(pose, rendered_raw,
crop_models[poseId])
## Resizing if required
if resizeCNN:
rendered_raw = cv2.resize(
rendered_raw, (cnnSize, cnnSize),
interpolation=cv2.INTER_CUBIC)
## Saving if required
if opts.getboolean('general', 'saveON'):
subjFolder = outputFolder + '/' + image_key.split(
'_')[0]
myutil.mymkdir(subjFolder)
savingString = subjFolder + '/' + image_key + '_rendered_' + pose[
8:-7] + '_' + str(subj).zfill(2) + '.jpg'
cv2.imwrite(savingString, rendered_raw)
## Plotting if required
if opts.getboolean('general', 'plotON'):
myutil.show(img_display, img, lmarks, rendered_raw, \
face_proj, background_proj, temp_proj2_out_2, sym_weight)
else:
print('> Landmark not detected for this image...')
def get_video_points(
video_file,
predictor_path='dlib_models/shape_predictor_68_face_landmarks.dat',
model3D_path='frontalization_models/model3Ddlib.mat',
eyemask_path='frontalization_models/eyemask.mat',
mode='face',
resize=False):
time_list = []
points_list = []
points_list_sym = []
points_list_raw = []
model3D = frontalize.ThreeD_Model(model3D_path, 'model_dlib')
eyemask = np.asarray(io.loadmat(eyemask_path)['eyemask'])
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(predictor_path)
cap = cv2.VideoCapture(video_file)
fps = cap.get(cv2.CAP_PROP_FPS)
print('fps=', fps)
# width = 320
# height = 320
# fourcc = cv2.VideoWriter_fourcc(*'XVID')
# video_sym = cv2.VideoWriter('video_sym.avi', fourcc, fps, (width, height))
# video_raw = cv2.VideoWriter('video_raw.avi', fourcc, fps, (width, height))
while True:
try:
ret, frame = cap.read()
if not ret:
print("no ret")
break
msec = cap.get(cv2.CAP_PROP_POS_MSEC)
print(msec / 1000)
img = frame
iheight, iwidth, layers = img.shape
#resize
if resize == True:
new_h = round(iheight / 3)
new_w = round(iwidth / 3)
img = cv2.resize(img, (new_w, new_h))
lmarks = feature_detection.get_landmarks(img, detector, predictor)
if len(lmarks) != 1:
continue
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
frontal_raw, frontal_sym = frontalize.frontalize(
img, proj_matrix, model3D.ref_U, eyemask)
time_list.append(msec)
new_img_sym = np.round(frontal_sym).astype(np.uint8)
new_img_raw = np.round(frontal_raw).astype(np.uint8)
temp_pts = []
lmarks_sym = feature_detection.get_landmarks(
new_img_sym, detector, predictor)
temp_pts = [[x, y] for x, y in lmarks_sym[0]]
if mode == 'mouth':
temp_pts = temp_pts[33:34] + temp_pts[48:]
points_list_sym.append(temp_pts)
# video_sym.write(new_img_sym)
# video_raw.write(new_img_raw)
except ValueError as e:
print(e)
break
cap.release()
points_list = list(zip(time_list, points_list_sym))
with shelve.open(
'extracted_data/' + video_file.split('/')[-1].split('.')[0] +
'.txt', 'c') as db:
for i, x in enumerate(points_list):
db[str(i)] = x
return (points_list)
def myfrontalize(X, limit=0):
count = 0
if (limit == 0):
limit = X.shape[0]
print("Total Images: ", limit)
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
# load mask to exclude eyes from symmetry
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
for i in range(0, limit):
print("\r", end='')
print("Images Completed: {0}".format(i), end='', flush=True)
# cast img to type int for cv2
img = X[i, :, :, 0].astype(np.uint8)
# create a color version for frontalizer stuffs
c_img = np.copy(img)
c_img = cv2.cvtColor(c_img, cv2.COLOR_GRAY2BGR)
# extract landmarks from the query image
# list containing a 2D array with points (x, y) for each face detected in the query image
lmarks = feature_detection.get_landmarks(c_img)
if type(lmarks) is np.ndarray:
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
# perform frontalization and convert result to grayscale
frontal_raw, frontal_sym = frontalize.frontalize(
c_img, proj_matrix, model3D.ref_U, eyemask)
temp = cv2.cvtColor(frontal_raw, cv2.COLOR_BGR2GRAY)
# find nonzero bbox and crop image to remove uncessesary black space from edges
temp_mask = cv2.findNonZero(temp)
t_x, t_y, t_w, t_h = cv2.boundingRect(temp_mask)
t_bbox = temp[t_y:t_y + t_h, t_x:t_x + t_w]
# resize the cropped image to the appropriate dimensions for network
t_bbox = cv2.resize(t_bbox, dsize=(48, 48))
t_bbox = np.resize(t_bbox, (48, 48, 1))
X[i] = t_bbox.astype(np.float32)
plt.show()
count += 1
print()
print('{} images out of {} were frontalized.'.format(count, limit))
def face_detect(image_file_path, image_save_path):
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
# load detections performed by dlib library on 3D model and Reference Image
# load query image
#print image_file_path
img = cv2.imread(image_file_path, 1)
(ori_w, org_h, org_channel) = img.shape
check.check_dlib_landmark_weights()
lmarks = feature_detection.get_landmarks(img)
if lmarks.shape[0] == 0:
return 0
else:
print image_save_path
if not os.path.exists(image_save_path):
os.makedirs(image_save_path)
for num_face in range(0, lmarks.shape[0]):
face_landmarks = lmarks[num_face, :, :]
print "Detect face %d out of total %d faces" % (
(num_face + 1), lmarks.shape[0])
print "start to align face"
w = 256
h = 256
eyecornerDst = [(np.int(0.25 * w), np.int(h / 3.5)),
(np.int(0.75 * w), np.int(h / 3.5)),
(np.int(0.5 * w), np.int(h / 2))]
eyecornerSrc = [
(np.int(face_landmarks[36][0]), np.int(face_landmarks[36][1])),
(np.int(face_landmarks[45][0]), np.int(face_landmarks[45][1])),
(np.int(face_landmarks[30][0]), np.int(face_landmarks[30][1]))
]
# Apply similarity transformation
tform = similarityTransform(eyecornerSrc, eyecornerDst)
img_face_aligned = cv2.warpAffine(img, tform, (w, h))
#note that x is horizontal, is width, while y is height, is vertical
#print tform
width = np.int(
(face_landmarks[16][0] - face_landmarks[0][0]) * 1.3)
height = np.int(
(face_landmarks[8][1] - face_landmarks[19][1]) * 1.3)
if width < 0 | height < 0:
print("face crooked")
sys.exit()
left_top_x = np.int(face_landmarks[0][0]) - np.int(
(np.int(face_landmarks[16][0]) - np.int(face_landmarks[0][0]))
* 0.15)
left_top_y = np.int(face_landmarks[19][1]) - np.int(
(np.int(face_landmarks[8][1]) - np.int(face_landmarks[19][1]))
* 0.15)
if left_top_x < 0:
left_top_x = 0
if left_top_y < 0:
left_top_y = 0
image_new_name_affined = image_save_path + '/A_' + str(
left_top_x) + '_' + str(left_top_y) + '_' + str(
width) + '_' + str(height) + '.jpg'
resized_image = cv2.resize(img_face_aligned, (256, 256))
cv2.imwrite(image_new_name_affined, resized_image)
return 1
def get_landmark_2d(self, im):
return feature_detection.get_landmarks(im, detector, predictor)
