def demo(): # check for dlib saved weights for face landmark detection # if it fails, dowload and extract it manually from # http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2 check.check_dlib_landmark_weights() # load detections performed by dlib library on 3D model and Reference Image model3D = frontalize.ThreeD_Model( this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib') # load query image img = cv2.imread("test.jpg", 1) plt.title('Query Image') plt.imshow(img[:, :, ::-1]) # extract landmarks from the query image # list containing a 2D array with points (x, y) for each face detected in the query image lmarks = feature_detection.get_landmarks(img) plt.figure() plt.title('Landmarks Detected') plt.imshow(img[:, :, ::-1]) plt.scatter(lmarks[0][:, 0], lmarks[0][:, 1]) # perform camera calibration according to the first face detected proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera( model3D, lmarks[0]) # load mask to exclude eyes from symmetry eyemask = np.asarray( io.loadmat('frontalization_models/eyemask.mat')['eyemask']) # perform frontalization frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix, model3D.ref_U, eyemask) plt.figure() plt.title('Frontalized no symmetry') plt.imshow(frontal_raw[:, :, ::-1]) plt.figure() plt.title('Frontalized with soft symmetry') plt.imshow(frontal_sym[:, :, ::-1]) plt.show()
def demo(): # check for dlib saved weights for face landmark detection # if it fails, dowload and extract it manually from # http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2 check.check_dlib_landmark_weights() # load detections performed by dlib library on 3D model and Reference Image model3D = frontalize.ThreeD_Model(this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib') # load query image img = cv2.imread("test.jpg", 1) plt.title('Query Image') plt.imshow(img[:, :, ::-1]) # extract landmarks from the query image # list containing a 2D array with points (x, y) for each face detected in the query image lmarks = feature_detection.get_landmarks(img) plt.figure() plt.title('Landmarks Detected') plt.imshow(img[:, :, ::-1]) plt.scatter(lmarks[0][:, 0], lmarks[0][:, 1]) # perform camera calibration according to the first face detected proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(model3D, lmarks[0]) # load mask to exclude eyes from symmetry eyemask = np.asarray(io.loadmat('frontalization_models/eyemask.mat')['eyemask']) # perform frontalization frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix, model3D.ref_U, eyemask) plt.figure() plt.title('Frontalized no symmetry') plt.imshow(frontal_raw[:, :, ::-1]) plt.figure() plt.title('Frontalized with soft symmetry') plt.imshow(frontal_sym[:, :, ::-1]) plt.show()
def __init__(self): this_path = os.path.dirname(os.path.abspath(__file__)) check.check_dlib_landmark_weights() self.model3D = frontalize.ThreeD_Model( this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib') self.eyemask = np.asarray( io.loadmat(this_path + '/frontalization_models/eyemask.mat')['eyemask'])
def demo(): # check for dlib saved weights for face landmark detection # if it fails, dowload and extract it manually from # http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2 check.check_dlib_landmark_weights() # load detections performed by dlib library on 3D model and Reference Image model3D = frontalize.ThreeD_Model( this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib') # load query image # img2 = cv2.imread("test.jpg", 1) img = cv2.imread( "/home/jordan/PycharmProjects/Emotion_Analysis/Facial_Expression_recog/data/test_photo.jpg", 1) img = cv2.resize(img, (int(img.shape[0] / 3), int(img.shape[1] / 9))) # detected_object = mouthdetection.findmouth(img, haar_face, haar_mouth)[0] # img = smile_detection.crop(detected_object, img, extended = True) cv2.imshow("", img) cv2.waitKey(5000) img = img.astype(np.uint8) # print img.shape plt.title('Query Image') plt.imshow(img[:, :, ::-1]) # extract landmarks from the query image # list containing a 2D array with points (x, y) for each face detected in the query image lmarks = feature_detection.get_landmarks(img) plt.figure() plt.title('Landmarks Detected') plt.imshow(img[:, :, ::-1]) plt.scatter(lmarks[0][:, 0], lmarks[0][:, 1]) # perform camera calibration according to the first face detected proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera( model3D, lmarks[0]) # load mask to exclude eyes from symmetry eyemask = np.asarray( io.loadmat('frontalization_models/eyemask.mat')['eyemask']) # perform frontalization frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix, model3D.ref_U, eyemask) plt.figure() plt.title('Frontalized no symmetry') plt.imshow(frontal_raw[:, :, ::-1]) plt.figure() plt.title('Frontalized with soft symmetry') plt.imshow(frontal_sym[:, :, ::-1]) plt.show()
def demo(image): # check for dlib saved weights for face landmark detection # if it fails, dowload and extract it manually from # http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2 check.check_dlib_landmark_weights() # load detections performed by dlib library on 3D model and Reference Image model3D = frontalize.ThreeD_Model( this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib') img = cv2.resize(image, (250, 250), interpolation=cv2.INTER_LINEAR) lmarks, f = feature_detection.get_landmarks(img) if not f: return False, 1 # perform camera calibration according to the first face detected proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera( model3D, lmarks[0]) print(proj_matrix) # load mask to exclude eyes from symmetry eyemask = np.asarray( io.loadmat('frontalization_models/eyemask.mat')['eyemask']) # perform frontalization frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix, model3D.ref_U, eyemask) ''' plt.figure() plt.title('Image frontalized(Before symmetry)') plt.imshow(frontal_raw[:,:,::-1].astype('uint8')) plt.figure() plt.title('Image frontalized(After symmetry)') plt.imshow(frontal_sym[:,:,::-1].astype('uint8')) ''' x, y, z = grammer.get_angle(rmat) print(('旋转的角度: x: {}, y: {}, z: {}').format(x, y, z)) #估算大概的旋转角度 image_output = frontal_sym.astype('uint8') return True, image_output
def load_face_frontal_elements(path_to_face_front_utilyties): ''' Routine to prepare elements for face-frontalization module by dougsouza in: https://github.com/dougsouza/face-frontalization Tal Hassner, Shai Harel, Eran Paz and Roee Enbar, Effective Face Frontalization in Unconstrained Images, IEEE Conf. on Computer Vision and Pattern Recognition (CVPR), Boston, June 2015 ''' # check for dlib saved weights for face landmark detection # if it fails, dowload and extract it manually from # http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2 check.check_dlib_landmark_weights() # load detections performed by dlib library on 3D model and Reference Image model3D = frontalize.ThreeD_Model( path_to_face_front_utilyties + '/frontalization_models/model3Ddlib.mat', 'model_dlib') # load mask to exclude eyes from symmetry eyemask = np.asarray( io.loadmat(path_to_face_front_utilyties + '/frontalization_models/eyemask.mat')['eyemask']) return model3D, eyemask
def demo(): nSub = opts.getint('general', 'nTotSub') fileList, outputFolder = myutil.parse(sys.argv) # check for dlib saved weights for face landmark detection # if it fails, dowload and extract it manually from # http://sourceforge.net/projects/dclib/files/d.10/shape_predictor_68_face_landmarks.dat.bz2 check.check_dlib_landmark_weights() ## Preloading all the models for speed allModels = myutil.preload(this_path, pose_models_folder, pose_models, nSub) for f in fileList: if '#' in f: #skipping comments continue splitted = f.split(',') image_key = splitted[0] image_path = splitted[1] image_landmarks = splitted[2] img = cv2.imread(image_path, 1) if image_landmarks != "None": lmark = np.loadtxt(image_landmarks) lmarks = [] lmarks.append(lmark) else: print('> Detecting landmarks') lmarks = feature_detection.get_landmarks(img, this_path) if len(lmarks) != 0: ## Copy back original image and flipping image in case we need ## This flipping is performed using all the model or all the poses ## To refine the estimation of yaw. Yaw can change from model to model... img_display = img.copy() img, lmarks, yaw = myutil.flipInCase(img, lmarks, allModels) listPose = myutil.decidePose(yaw, opts, newModels) ## Looping over the poses for poseId in listPose: posee = pose_models[poseId] ## Looping over the subjects for subj in range(1, nSub + 1): pose = posee + '_' + str(subj).zfill(2) + '.mat' print('> Looking at file: ' + image_path + ' with ' + pose) # load detections performed by dlib library on 3D model and Reference Image print("> Using pose model in " + pose) ## Indexing the right model instead of loading it each time from memory. model3D = allModels[pose] eyemask = model3D.eyemask # perform camera calibration according to the first face detected proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera( model3D, lmarks[0]) ## We use eyemask only for frontal if not myutil.isFrontal(pose): eyemask = None ##### Main part of the code: doing the rendering ############# rendered_raw, rendered_sym, face_proj, background_proj, temp_proj2_out_2, sym_weight = renderer.render(img, proj_matrix,\ model3D.ref_U, eyemask, model3D.facemask, opts) ######################################################## if myutil.isFrontal(pose): rendered_raw = rendered_sym ## Cropping if required by crop_models rendered_raw = myutil.cropFunc(pose, rendered_raw, crop_models[poseId]) ## Resizing if required if resizeCNN: rendered_raw = cv2.resize( rendered_raw, (cnnSize, cnnSize), interpolation=cv2.INTER_CUBIC) ## Saving if required if opts.getboolean('general', 'saveON'): subjFolder = outputFolder + '/' + image_key.split( '_')[0] myutil.mymkdir(subjFolder) savingString = subjFolder + '/' + image_key + '_rendered_' + pose[ 8:-7] + '_' + str(subj).zfill(2) + '.jpg' cv2.imwrite(savingString, rendered_raw) ## Plotting if required if opts.getboolean('general', 'plotON'): myutil.show(img_display, img, lmarks, rendered_raw, \ face_proj, background_proj, temp_proj2_out_2, sym_weight) else: print('> Landmark not detected for this image...')
def myfrontalize(X, limit=0): count = 0 if (limit == 0): limit = X.shape[0] print("Total Images: ", limit) # check for dlib saved weights for face landmark detection # if it fails, dowload and extract it manually from # http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2 check.check_dlib_landmark_weights() # load detections performed by dlib library on 3D model and Reference Image model3D = frontalize.ThreeD_Model( this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib') # load mask to exclude eyes from symmetry eyemask = np.asarray( io.loadmat('frontalization_models/eyemask.mat')['eyemask']) for i in range(0, limit): print("\r", end='') print("Images Completed: {0}".format(i), end='', flush=True) # cast img to type int for cv2 img = X[i, :, :, 0].astype(np.uint8) # create a color version for frontalizer stuffs c_img = np.copy(img) c_img = cv2.cvtColor(c_img, cv2.COLOR_GRAY2BGR) # extract landmarks from the query image # list containing a 2D array with points (x, y) for each face detected in the query image lmarks = feature_detection.get_landmarks(c_img) if type(lmarks) is np.ndarray: # perform camera calibration according to the first face detected proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera( model3D, lmarks[0]) # perform frontalization and convert result to grayscale frontal_raw, frontal_sym = frontalize.frontalize( c_img, proj_matrix, model3D.ref_U, eyemask) temp = cv2.cvtColor(frontal_raw, cv2.COLOR_BGR2GRAY) # find nonzero bbox and crop image to remove uncessesary black space from edges temp_mask = cv2.findNonZero(temp) t_x, t_y, t_w, t_h = cv2.boundingRect(temp_mask) t_bbox = temp[t_y:t_y + t_h, t_x:t_x + t_w] # resize the cropped image to the appropriate dimensions for network t_bbox = cv2.resize(t_bbox, dsize=(48, 48)) t_bbox = np.resize(t_bbox, (48, 48, 1)) X[i] = t_bbox.astype(np.float32) plt.show() count += 1 print() print('{} images out of {} were frontalized.'.format(count, limit))
__author__ = 'Merle' import math import os import time import numpy as np from skimage import io import time import check_resources as check import dlib import facial_feature_detector as feature_detection import PoseEstimation as PE this_path = os.path.dirname(os.path.abspath(__file__)) check.check_dlib_landmark_weights() # 检测dlib参数是否下载,没下载的话下载 image_name = 'zjl.jpg' image_path = this_path + '/input/' + image_name time1 = time.time() print(this_path) predictor_path = this_path + "/dlib_models/shape_predictor_68_face_landmarks.dat" print(predictor_path) detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor(predictor_path) time2 = time.time() def get_angle(image, pt3d, sess): lmarks = feature_detection.get_landmarks(image, detector, predictor) Pose_Para = []
video_out = cv2.VideoWriter('./rotated/' + video_name + '.mp4', fourcc, fps, (320, 320)) #3d_output leye_out = cv2.VideoWriter('./rotated/' + video_name + '_leye.mp4', fourcc, fps, (EYE_SIZE[0], EYE_SIZE[1])) #leye reye_out = cv2.VideoWriter('./rotated/' + video_name + '_reye.mp4', fourcc, fps, (EYE_SIZE[0], EYE_SIZE[1])) #reye mouth_out = cv2.VideoWriter('./rotated/' + video_name + '_mouth.mp4', fourcc, fps, (MOUTH_SIZE[0], MOUTH_SIZE[1])) #mouth print('=== loading landmarks ===') lmarks = np.load(sys.argv[2]) #lmarks = np.load('./detected_3d/' + video_name + '_fixed.npy') print('===loading model === ') check.check_dlib_landmark_weights() black_img = np.zeros((320, 320, 3), dtype='uint8') i = -1 while (video.isOpened() and i + 1 < len(lmarks)): ret, frame = video.read() i += 1 print('reading : frame ' + str(i)) if ret == True and len(lmarks[i]) > 0: lmark = copy.copy(lmarks[i][0][:, :2]) frontal, reye, leye, mouth, _ = img_normalization(frame, lmark) video_out.write(frontal) reye_out.write(reye) leye_out.write(leye)
def face_detect(image_file_path, image_save_path): # check for dlib saved weights for face landmark detection # if it fails, dowload and extract it manually from # http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2 # load detections performed by dlib library on 3D model and Reference Image # load query image #print image_file_path img = cv2.imread(image_file_path, 1) (ori_w, org_h, org_channel) = img.shape check.check_dlib_landmark_weights() lmarks = feature_detection.get_landmarks(img) if lmarks.shape[0] == 0: return 0 else: print image_save_path if not os.path.exists(image_save_path): os.makedirs(image_save_path) for num_face in range(0, lmarks.shape[0]): face_landmarks = lmarks[num_face, :, :] print "Detect face %d out of total %d faces" % ( (num_face + 1), lmarks.shape[0]) print "start to align face" w = 256 h = 256 eyecornerDst = [(np.int(0.25 * w), np.int(h / 3.5)), (np.int(0.75 * w), np.int(h / 3.5)), (np.int(0.5 * w), np.int(h / 2))] eyecornerSrc = [ (np.int(face_landmarks[36][0]), np.int(face_landmarks[36][1])), (np.int(face_landmarks[45][0]), np.int(face_landmarks[45][1])), (np.int(face_landmarks[30][0]), np.int(face_landmarks[30][1])) ] # Apply similarity transformation tform = similarityTransform(eyecornerSrc, eyecornerDst) img_face_aligned = cv2.warpAffine(img, tform, (w, h)) #note that x is horizontal, is width, while y is height, is vertical #print tform width = np.int( (face_landmarks[16][0] - face_landmarks[0][0]) * 1.3) height = np.int( (face_landmarks[8][1] - face_landmarks[19][1]) * 1.3) if width < 0 | height < 0: print("face crooked") sys.exit() left_top_x = np.int(face_landmarks[0][0]) - np.int( (np.int(face_landmarks[16][0]) - np.int(face_landmarks[0][0])) * 0.15) left_top_y = np.int(face_landmarks[19][1]) - np.int( (np.int(face_landmarks[8][1]) - np.int(face_landmarks[19][1])) * 0.15) if left_top_x < 0: left_top_x = 0 if left_top_y < 0: left_top_y = 0 image_new_name_affined = image_save_path + '/A_' + str( left_top_x) + '_' + str(left_top_y) + '_' + str( width) + '_' + str(height) + '.jpg' resized_image = cv2.resize(img_face_aligned, (256, 256)) cv2.imwrite(image_new_name_affined, resized_image) return 1