def get_frontal_image(image):
predictor_path = 'dlib_models/shape_predictor_68_face_landmarks.dat'
model3D_path = 'frontalization_models/model3Ddlib.mat'
eyemask_path = 'frontalization_models/eyemask.mat'
model3D = frontalize.ThreeD_Model(model3D_path, 'model_dlib')
eyemask = np.asarray(io.loadmat(eyemask_path)['eyemask'])
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(predictor_path)
img = cv2.imread(image, 1)
height, width, layers = img.shape
new_h = round(height / 3)
new_w = round(width / 3)
img = cv2.resize(img, (new_w, new_h))
model3D = frontalize.ThreeD_Model('frontalization_models/model3Ddlib.mat',
'model_dlib')
lmarks = feature_detection.get_landmarks(img, detector, predictor)
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix,
model3D.ref_U, eyemask)
cv2.imwrite('frontal_image.png', frontal_sym)
def demo():
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
# load query image
img = cv2.imread("test.jpg", 1)
plt.title('Query Image')
plt.imshow(img[:, :, ::-1])
# extract landmarks from the query image
# list containing a 2D array with points (x, y) for each face detected in the query image
lmarks = feature_detection.get_landmarks(img)
plt.figure()
plt.title('Landmarks Detected')
plt.imshow(img[:, :, ::-1])
plt.scatter(lmarks[0][:, 0], lmarks[0][:, 1])
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(model3D, lmarks[0])
# load mask to exclude eyes from symmetry
eyemask = np.asarray(io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
# perform frontalization
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix, model3D.ref_U, eyemask)
plt.figure()
plt.title('Frontalized no symmetry')
plt.imshow(frontal_raw[:, :, ::-1])
plt.figure()
plt.title('Frontalized with soft symmetry')
plt.imshow(frontal_sym[:, :, ::-1])
plt.show()
def demo():
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
# load query image
img = cv2.imread("test.jpg", 1)
plt.title('Query Image')
plt.imshow(img[:, :, ::-1])
# extract landmarks from the query image
# list containing a 2D array with points (x, y) for each face detected in the query image
lmarks = feature_detection.get_landmarks(img)
plt.figure()
plt.title('Landmarks Detected')
plt.imshow(img[:, :, ::-1])
plt.scatter(lmarks[0][:, 0], lmarks[0][:, 1])
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
# load mask to exclude eyes from symmetry
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
# perform frontalization
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix,
model3D.ref_U, eyemask)
plt.figure()
plt.title('Frontalized no symmetry')
plt.imshow(frontal_raw[:, :, ::-1])
plt.figure()
plt.title('Frontalized with soft symmetry')
plt.imshow(frontal_sym[:, :, ::-1])
plt.show()
def flipInCase(img, lmarks, allModels):
## Check if we need to flip the image
# yaw（摇摆）表示头部绕垂直轴旋转
yaws = [] #np.zeros(1,len(allModels))
## Getting yaw estimate over poses and subjects
# 迭代每个FaceModel对象
for mmm in allModels.itervalues():
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
mmm, lmarks[0])
yaws.append(calib.get_yaw(rmat))
yaws = np.asarray(yaws)
# 计算 yaws 所有项的平均值
yaw = yaws.mean()
print '> Yaw value mean: ', yaw
if yaw < 0:
print '> Positive yaw detected, flipping the image'
# 水平翻转图像
img = cv2.flip(img, 1)
# Flipping X values for landmarks
lmarks[0][:, 0] = img.shape[1] - lmarks[0][:, 0]
# Creating flipped landmarks with new indexing
lmarks3 = np.zeros((1, 68, 2))
for i in range(len(repLand)):
lmarks3[0][i, :] = lmarks[0][repLand[i] - 1, :]
lmarks = lmarks3
return img, lmarks, yaw
def img_normalization(img, lmark):
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
height, width, _ = img.shape
#face_lu = np.array([min(lmark[:,0]), min(lmark[:,1])])
#face_rd = np.array([max(lmark[:,0]), max(lmark[:,1])])
#face_lu = face_lu - 20
#face_rd = face_rd + 20
#if face_lu[0]<0:
# face_lu[0] = 0
#if face_lu[1]<0:
# face_lu[1] = 0
#if face_rd[0] > width:
# face_rd[0] = width
#if face_rd[1] > height:
# face_rd[1] = height
#face_lu = face_lu.astype('uint64')
#face_rd = face_rd.astype('uint64')
#new_img = img[face_lu[1]:face_rd[1], face_lu[0]:face_rd[0]]
#new_img = cv2.resize(img, (320, 320))
#lmark = lmark - [face_lu[0], face_lu[1]]
#w_res = 320/(face_rd[0]-face_lu[0])
#h_res = 320/(face_rd[1]-face_lu[1])
#lmark = lmark*[w_res, h_res]
#new_lm = copy.copy(lmark)
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmark)
frontal_raw, frontal_sym, proj_map = frontalize.frontalize(
img, proj_matrix, model3D.ref_U, eyemask)
rot_lmark = np.zeros((lmark.shape[0], 2))
for i in range(lmark.shape[0]):
rot_lmark[i][1], rot_lmark[i][0] = find_nearest(proj_map, lmark[i])
reye_ul, reye_dr = lmark_bbox(rot_lmark[36:42])
leye_ul, leye_dr = lmark_bbox(rot_lmark[42:48])
mouth_ul, mouth_dr = lmark_bbox(rot_lmark[48:68], pos='mouth')
int_frontal = frontal_sym.astype('uint8')
reye = int_frontal[reye_ul[0]:reye_dr[0], reye_ul[1]:reye_dr[1]]
leye = int_frontal[leye_ul[0]:leye_dr[0], leye_ul[1]:leye_dr[1]]
mouth = int_frontal[mouth_ul[0]:mouth_dr[0], mouth_ul[1]:mouth_dr[1]]
frontal_lmark = np.copy(int_frontal)
for i in range(len(rot_lmark)):
cv2.circle(frontal_lmark, (int(rot_lmark[i][0]), int(rot_lmark[i][1])),
2, (0, 0, 255),
thickness=-1)
return int_frontal, reye, leye, mouth, frontal_lmark
def frontalize(self, img):
lmarks = feature_detection.get_landmarks1(img)
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
self.model3D, lmarks[0])
frontal_raw, frontal_sym = frontalize.frontalize(
img, proj_matrix, self.model3D.ref_U, self.eyemask)
return frontal_raw
def decideSide_from_db(img, pose_Rt, allModels):
## Check if we need to flip the image
#model3D = ThreeD_Model.FaceModel(this_path + "/models3d/" + pose_models[0] +'_01.mat', 'model3D')
## Getting yaw estimate over poses and subjects
mm = allModels.values()[0]
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
mm, pose_Rt, pose_db_on=True)
yaw = calib.get_yaw(rmat)
print('> Yaw value mean: ', yaw)
return yaw
def frontalized(self, image):
landmarks = facial.get_landmarks(image, self.args.resource_dir,
self.args)
if len(landmarks) > 0:
proj_matrix, camera_matrix, rmat, tvec = camera.estimate_camera(
self.model3D, landmarks[0])
_, front_image = front.frontalize(image, proj_matrix,
self.model3D.ref_U, self.eyemask)
detection = self.detector(front_image, 1)
for _, detection in enumerate(detection):
return front_image[detection.top():detection.bottom(),
detection.left():detection.right()]
return
def front_database_build(name, data_path, work_path, eyemask):
'''
Given the name of the subject in the variable -name-, the path from where to obtain the images,
the path to the destiny folder in which to save the database and the eyemask parameter used for
frontalization of the faces, this routine takes the images from data_path and saves the
frontalized face obtained in the work_path separating pictures in folders for each individual.
'''
for x in name[:]:
pathdir = os.path.join(work_path, x)
#print pathdir #for debug purposes
make_sure_path_exists(pathdir)
search_filename = x + "*.pgm"
search_location = os.path.join(data_path, search_filename)
#print search_location #for debug purposes
for filepath in glob.glob(search_location):
# Extract filename from the filepath found at search location
tail = os.path.basename(filepath) #os.path.split(search_location)
#print tail #for debug purposes
out_name = tail + 'fr.pgm'
# For MIT-CBCL dataset file naming, break name into the image data provided in filename
# 'subject' - 'face rotation' - etc
file_data = re.split(r"[_]", tail)
#print file_data #for debug purposes
# angular limit for rotated face frontalization
face_rot = -26
#Copy only the images that have less than face_rot rotation angle
if os.path.isfile(filepath) and face_rot < float(file_data[1]):
img = cv2.imread(filepath, 1)
# extract landmarks from the query image
# list containing a 2D array with points (x, y) for each face detected in the query image
lmarks = feature_detection.get_landmarks(img)
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
# perform frontalization
frontal_raw, frontal_sym = frontalize.frontalize(
img, proj_matrix, model3D.ref_U, eyemask)
#shutil.copy(filepath, pathdir)
#write frontal_sym to image file in pathdir location
cv2.imwrite(os.path.join(pathdir, out_name), frontal_sym)
def demo():
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
# load query image
# img2 = cv2.imread("test.jpg", 1)
img = cv2.imread(
"/home/jordan/PycharmProjects/Emotion_Analysis/Facial_Expression_recog/data/test_photo.jpg",
1)
img = cv2.resize(img, (int(img.shape[0] / 3), int(img.shape[1] / 9)))
# detected_object = mouthdetection.findmouth(img, haar_face, haar_mouth)[0]
# img = smile_detection.crop(detected_object, img, extended = True)
cv2.imshow("", img)
cv2.waitKey(5000)
img = img.astype(np.uint8)
# print img.shape
plt.title('Query Image')
plt.imshow(img[:, :, ::-1])
# extract landmarks from the query image
# list containing a 2D array with points (x, y) for each face detected in the query image
lmarks = feature_detection.get_landmarks(img)
plt.figure()
plt.title('Landmarks Detected')
plt.imshow(img[:, :, ::-1])
plt.scatter(lmarks[0][:, 0], lmarks[0][:, 1])
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
# load mask to exclude eyes from symmetry
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
# perform frontalization
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix,
model3D.ref_U, eyemask)
plt.figure()
plt.title('Frontalized no symmetry')
plt.imshow(frontal_raw[:, :, ::-1])
plt.figure()
plt.title('Frontalized with soft symmetry')
plt.imshow(frontal_sym[:, :, ::-1])
plt.show()
def demo(path):
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
img = cv2.imread(path, 1)
lmarks = feature_detection.get_landmarks(img)
if (len(lmarks) > 0):
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
frontal_raw, frontal_sym = frontalize.frontalize(
img, proj_matrix, model3D.ref_U, eyemask)
cv2.imwrite(path, frontal_raw)
def frontalize(self, img):
lmarks = feature_detection.get_landmarks1(img)
'''
plt.figure()
plt.title('Landmarks Detected')
plt.imshow(img[:, :, ::-1])
plt.scatter(lmarks[0][:, 0], lmarks[0][:, 1])
'''
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(self.model3D, lmarks[0])
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix, self.model3D.ref_U, self.eyemask)
'''
plt.figure()
plt.title('Frontalized no symmetry')
plt.imshow(frontal_raw[:, :, ::-1])
plt.figure()
plt.title('Frontalized with soft symmetry')
plt.imshow(frontal_sym[:, :, ::-1])
plt.show()
'''
return frontal_raw
def demo(image):
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
img = cv2.resize(image, (250, 250), interpolation=cv2.INTER_LINEAR)
lmarks, f = feature_detection.get_landmarks(img)
if not f:
return False, 1
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
print(proj_matrix)
# load mask to exclude eyes from symmetry
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
# perform frontalization
frontal_raw, frontal_sym = frontalize.frontalize(img, proj_matrix,
model3D.ref_U, eyemask)
'''
plt.figure()
plt.title('Image frontalized(Before symmetry)')
plt.imshow(frontal_raw[:,:,::-1].astype('uint8'))
plt.figure()
plt.title('Image frontalized(After symmetry)')
plt.imshow(frontal_sym[:,:,::-1].astype('uint8'))
'''
x, y, z = grammer.get_angle(rmat)
print(('旋转的角度： x: {}, y: {}, z: {}').format(x, y, z)) #估算大概的旋转角度
image_output = frontal_sym.astype('uint8')
return True, image_output
def demo():
nSub = opts.getint('general', 'nTotSub')
fileList, outputFolder = myutil.parse(sys.argv)
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/d.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
## Preloading all the models for speed
allModels = myutil.preload(this_path, pose_models_folder, pose_models,
nSub)
for f in fileList:
if '#' in f: #skipping comments
continue
splitted = f.split(',')
image_key = splitted[0]
image_path = splitted[1]
image_landmarks = splitted[2]
img = cv2.imread(image_path, 1)
if image_landmarks != "None":
lmark = np.loadtxt(image_landmarks)
lmarks = []
lmarks.append(lmark)
else:
print('> Detecting landmarks')
lmarks = feature_detection.get_landmarks(img, this_path)
if len(lmarks) != 0:
## Copy back original image and flipping image in case we need
## This flipping is performed using all the model or all the poses
## To refine the estimation of yaw. Yaw can change from model to model...
img_display = img.copy()
img, lmarks, yaw = myutil.flipInCase(img, lmarks, allModels)
listPose = myutil.decidePose(yaw, opts, newModels)
## Looping over the poses
for poseId in listPose:
posee = pose_models[poseId]
## Looping over the subjects
for subj in range(1, nSub + 1):
pose = posee + '_' + str(subj).zfill(2) + '.mat'
print('> Looking at file: ' + image_path + ' with ' + pose)
# load detections performed by dlib library on 3D model and Reference Image
print("> Using pose model in " + pose)
## Indexing the right model instead of loading it each time from memory.
model3D = allModels[pose]
eyemask = model3D.eyemask
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
## We use eyemask only for frontal
if not myutil.isFrontal(pose):
eyemask = None
##### Main part of the code: doing the rendering #############
rendered_raw, rendered_sym, face_proj, background_proj, temp_proj2_out_2, sym_weight = renderer.render(img, proj_matrix,\
model3D.ref_U, eyemask, model3D.facemask, opts)
########################################################
if myutil.isFrontal(pose):
rendered_raw = rendered_sym
## Cropping if required by crop_models
rendered_raw = myutil.cropFunc(pose, rendered_raw,
crop_models[poseId])
## Resizing if required
if resizeCNN:
rendered_raw = cv2.resize(
rendered_raw, (cnnSize, cnnSize),
interpolation=cv2.INTER_CUBIC)
## Saving if required
if opts.getboolean('general', 'saveON'):
subjFolder = outputFolder + '/' + image_key.split(
'_')[0]
myutil.mymkdir(subjFolder)
savingString = subjFolder + '/' + image_key + '_rendered_' + pose[
8:-7] + '_' + str(subj).zfill(2) + '.jpg'
cv2.imwrite(savingString, rendered_raw)
## Plotting if required
if opts.getboolean('general', 'plotON'):
myutil.show(img_display, img, lmarks, rendered_raw, \
face_proj, background_proj, temp_proj2_out_2, sym_weight)
else:
print('> Landmark not detected for this image...')
def readVideosAndLabels(directory, alignment="2D"):
namesUnique = os.listdir(directory)
labels = []
faceMatrix = None
outPath = directory + "_aligned_" + alignment
createDirectory(outPath)
if (alignment != "2D"):
model3D = frontalize.ThreeD_Model(
"./frontalization_models/model3Ddlib.mat", 'model_dlib')
eyemask = np.asarray(
sio.loadmat('frontalization_models/eyemask.mat')['eyemask'])
eyemask = imutils.resize(eyemask[52:250, 91:225], height=60)
model3D.ref_U = imutils.resize(model3D.ref_U[52:250, 91:225, :],
height=60)
model3D.out_A = np.asmatrix(np.array(
[[0.5 * 506.696672, 0, 0.5 * 324.202],
[0, 0.5 * 506.3752, 0.5 * 245.7785096], [0, 0, 1]]),
dtype='float32') #3x3
model3D.distCoeff = None
count = 0
for ind, folder in enumerate(namesUnique):
createDirectory(os.path.join(outPath, folder))
for path in os.listdir(os.path.join(directory, folder)):
path = os.path.join(directory, folder, path)
cap = cv2.VideoCapture(path)
trackingLost = True
while (True):
ret, imageO = cap.read()
if (not ret):
break
landmarks, faceROI, trackingLost, image = trackFaceInANeighborhoodAndDetectLandmarks(
np.copy(imageO),
faceROI=[0, 0, imageO.shape[0] - 1, imageO.shape[1] - 1],
drawBoundingBoxes=True)
if (trackingLost):
continue
for k, landmark in enumerate(landmarks):
if (alignment == "2D"):
registeredFace = performFaceAlignment(imageO,
landmark,
cols=600,
rows=600)
else:
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, landmark)
frontal_raw, registeredFace = frontalize.frontalize(
imageO, proj_matrix, model3D.ref_U, eyemask)
xdir = 2 * np.array([-49.6694, -0.3201, 1.0163])
ydir = 4 * np.array([-0.9852, -3.1128, 15.0628])
zdir = -np.array([-1.658, 747.159, 154.29]) / 5.0
origin = np.array([-0.0845, -74.7281, 27.2774])
image, _ = model3D.drawCoordinateSystems(
np.hstack((rmat, tvec)),
image,
_3Dpoints=np.array([
origin, origin + xdir, origin + ydir,
origin + zdir
]))
#image=model3D.drawCandideMesh(np.hstack((rmat,tvec)),image)
if (registeredFace is not None):
cv2.polylines(image,
np.int32(landmark.reshape((-1, 1, 2))),
True, (0, 0, 255), 3)
box = goodAlignement_cascade.detectMultiScale(
registeredFace, 1.1, 1, minSize=(32, 32))
try:
"""count=count+1
cv2.imwrite(os.path.join(outPath,folder,str(count)+'.jpg'), registeredFace)
"""
if (len(box) > 0):
cv2.imwrite(
os.path.join(outPath, folder,
str(count) + '.jpg'),
imutils.resize(registeredFace, height=48))
cv2.rectangle(registeredFace,
(box[0][0], box[0][1]),
(box[0][0] + box[0][2],
box[0][1] + box[0][3]),
(255, 0, 255), 2)
count = count + 1
except Exception as e:
print(e)
cap.release()
return faceMatrix, labels
def performFaceRecognitionWithFrontalisationV2(image, recognizer, model3D,
eyemask, names):
start = time.time()
imageWithLandmarks, landmarks, faceROIs = searchForFaceInTheWholeImage(
np.copy(image))
result = []
registeredFaceColor = 255
raw = 255
for landmark, faceROI in zip(landmarks, faceROIs):
goodAlignment = False
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, landmark)
raw, registeredFaceColor = frontalize.frontalize(
image, proj_matrix, model3D.ref_U, eyemask)
registeredFaceGray = cv2.cvtColor(registeredFaceColor,
cv2.COLOR_BGR2GRAY)
box = goodAlignement_cascade.detectMultiScale(registeredFaceGray,
1.1,
1,
minSize=(32, 32))
###############
registeredFace = imutils.resize(registeredFaceGray, height=48)
try:
cv2.rectangle(registeredFace, (box[0][1], box[0][0]),
(box[0][1] + box[0][3], box[0][0] + box[0][2]),
(255, 0, 255), 2)
goodAlignment = True
except:
pass
if (goodAlignment):
pred, conf = recognizer.predict(registeredFace)
xdir = 2 * np.array([-49.6694, -0.3201, 1.0163])
ydir = 4 * np.array([-0.9852, -3.1128, 15.0628])
zdir = -np.array([-1.658, 747.159, 154.29]) / 5.0
origin = np.array([-0.0845, -74.7281, 27.2774])
try:
image, _ = model3D.drawCoordinateSystems(np.hstack(
(rmat, tvec)),
imageWithLandmarks,
_3Dpoints=np.array([
origin,
origin + xdir,
origin + ydir,
origin + zdir
]))
except:
pass
identity = list(names.keys())[list(names.values()).index(pred)]
cv2.rectangle(image, (faceROI[1], faceROI[0]),
(faceROI[1] + faceROI[3], faceROI[0] + faceROI[2]),
(255, 0, 255), 2)
cv2.putText(image,
identity, (faceROI[1], faceROI[0]),
cv2.FONT_HERSHEY_SIMPLEX,
1, (0, 0, 255),
thickness=1)
end = time.time()
cv2.putText(image,
"FPS: " + "{0:.2f}".format(round(1.0 / (end - start), 2)),
(15, 15),
cv2.FONT_HERSHEY_SIMPLEX,
0.5, (0, 0, 255),
thickness=1)
return image, registeredFaceColor
def render_fpn(inputFile, output_pose_db, outputFolder):
## Opening FPN pose db
pose_env = lmdb.open(output_pose_db, readonly=True)
pose_cnn_lmdb = pose_env.begin()
## looping over images
with open(inputFile, 'r') as csvfile:
csvreader = csv.reader(csvfile, delimiter=',')
lines = csvfile.readlines()
for lin in lines:
### key1, image_path_key_1
image_key = lin.split(',')[0]
if 'flip' in image_key:
continue
image_path = lin.split(',')[-1].rstrip('\n')
img = cv2.imread(image_path, 1)
pose_Rt_raw = pose_cnn_lmdb.get(image_key)
pose_Rt_flip_raw = pose_cnn_lmdb.get(image_key + '_flip')
if pose_Rt_raw is not None:
pose_Rt = np.frombuffer(pose_Rt_raw, np.float32)
pose_Rt_flip = np.frombuffer(pose_Rt_flip_raw, np.float32)
yaw = myutil.decideSide_from_db(img, pose_Rt, allModels)
if yaw < 0: # Flip image and get the corresponsidng pose
img = cv2.flip(img, 1)
pose_Rt = pose_Rt_flip
listPose = myutil.decidePose(yaw, opts, newModels)
## Looping over the poses
for poseId in listPose:
posee = pose_models[poseId]
## Looping over the subjects
for subj in [10]:
pose = posee + '_' + str(subj).zfill(2) + '.mat'
print '> Looking at file: ' + image_path + ' with ' + pose
# load detections performed by dlib library on 3D model and Reference Image
print "> Using pose model in " + pose
## Indexing the right model instead of loading it each time from memory.
model3D = allModels[pose]
eyemask = model3D.eyemask
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, pose_Rt, pose_db_on=True)
## We use eyemask only for frontal
if not myutil.isFrontal(pose):
eyemask = None
##### Main part of the code: doing the rendering #############
rendered_raw, rendered_sym, face_proj, background_proj, temp_proj2_out_2, sym_weight = renderer_core.render(img, proj_matrix,\
model3D.ref_U, eyemask, model3D.facemask, opts)
########################################################
if myutil.isFrontal(pose):
rendered_raw = rendered_sym
## Cropping if required by crop_models
#rendered_raw = myutil.cropFunc(pose,rendered_raw,crop_models[poseId])
## Resizing if required
#if resizeCNN:
# rendered_raw = cv2.resize(rendered_raw, ( cnnSize, cnnSize ), interpolation=cv2.INTER_CUBIC )
## Saving if required
if opts.getboolean('general', 'saveON'):
subjFolder = outputFolder + '/' + image_key.split(
'_')[0]
myutil.mymkdir(subjFolder)
savingString = subjFolder + '/' + image_key + '_rendered_' + pose[
8:-7] + '_' + str(subj).zfill(2) + '.jpg'
cv2.imwrite(savingString, rendered_raw)
def performFaceRecognitionWithFrontalisation(image, recognizer, names, model3D,
eyemask):
#indiceToSymmetry=[17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1, 27,26,25,24,23, 22,21,20,19,18, 28,29,30,31, 36,35,35,33,32, 46,45,44,43,48,47, 40,39,38,37,42,41, 55,54,53,52,51,50,49,60,59,58,57,56, 65,64,63,62,61,68,67,66]
#indiceToSymmetry=list(np.array(indiceToSymmetry)-1)
imageWithLandmarks, landmarks, faceROIs = searchForFaceInTheWholeImage(
np.copy(image))
result = []
for landmark, faceROI in zip(landmarks, faceROIs):
#registeredFace=performFaceAlignment(image,landmark,cols=600,rows=600)
#landmark[range(0,68),:]=landmark[indiceToSymmetry,:]
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, landmark)
# load mask to exclude eyes from symmetry
# perform frontalization
frontal_raw, registeredFace = frontalize.frontalize(
image, proj_matrix, model3D.ref_U, eyemask)
registeredFace = imutils.resize(registeredFace, height=100)
cv2.imshow('el3ab', cv2.cvtColor(registeredFace, cv2.COLOR_BGR2GRAY))
cv2.waitKey(1)
pred, conf = recognizer.predict(
cv2.cvtColor(registeredFace, cv2.COLOR_BGR2GRAY))
xdir = 2 * np.array([-49.6694, -0.3201, 1.0163])
ydir = 4 * np.array([-0.9852, -3.1128, 15.0628])
zdir = -np.array([-1.658, 747.159, 154.29]) / 5.0
origin = np.array([-0.0845, -74.7281, 27.2774])
image, _ = model3D.drawCoordinateSystems(np.hstack((rmat, tvec)),
image,
_3Dpoints=np.array([
origin, origin + xdir,
origin + ydir,
origin + zdir
]))
image = model3D.drawCandideMesh(np.hstack((rmat, tvec)), image)
#
#distance, indice = svm.kneighbors(faceVector.reshape(1,-1))
#indice=classifier.predict(embeddedFace.reshape(1,-1))
#pred=svm_Tuned.predict(faceVector.reshape(1,-1))[0]
###distance, indice = nbrs.kneighbors(embeddedFace.reshape(1,-1))
#indice=classifier.predict(embeddedFace.reshape(1,-1))
identity = list(names.keys())[list(names.values()).index(pred)]
print(pred, conf)
##(#cv2.rectangle(image,(faceROI[1],faceROI[0]),(faceROI[1]+faceROI[3],faceROI[0]+faceROI[2]),(255,0,255),2)
###cv2.putText(image,indice[0],(faceROI[1],faceROI[0]),cv2.FONT_HERSHEY_SIMPLEX,1,(255,0,0))
cv2.rectangle(image, (faceROI[1], faceROI[0]),
(faceROI[1] + faceROI[3], faceROI[0] + faceROI[2]),
(255, 0, 255), 2)
cv2.putText(image,
identity, (faceROI[1], faceROI[0]),
cv2.FONT_HERSHEY_SIMPLEX,
1, (255, 0, 0),
thickness=2)
###result.append(indice[0])
#print (distance,labels[indice[0][0]])
"""if(distance<threshold):
cv2.rectangle(image,(faceROI[1],faceROI[0]),(faceROI[1]+faceROI[3],faceROI[0]+faceROI[2]),(255,0,255),2)
cv2.putText(image,labels[indice[0][0]] ,(faceROI[1],faceROI[0]),cv2.FONT_HERSHEY_SIMPLEX,1,(255,0,0))
result.append(labels[indice[0][0]])
else:
cv2.rectangle(image,(faceROI[1],faceROI[0]),(faceROI[1]+faceROI[3],faceROI[0]+faceROI[2]),(255,0,255),2)
cv2.putText(image,"unknown" ,(faceROI[1],faceROI[0]),cv2.FONT_HERSHEY_SIMPLEX,1,(255,0,0))
result.append("unknown")
"""
cv2.imshow('image', image)
cv2.waitKey(1)
return result, faceROIs
def get_video_points(
video_file,
predictor_path='dlib_models/shape_predictor_68_face_landmarks.dat',
model3D_path='frontalization_models/model3Ddlib.mat',
eyemask_path='frontalization_models/eyemask.mat',
mode='face',
resize=False):
time_list = []
points_list = []
points_list_sym = []
points_list_raw = []
model3D = frontalize.ThreeD_Model(model3D_path, 'model_dlib')
eyemask = np.asarray(io.loadmat(eyemask_path)['eyemask'])
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(predictor_path)
cap = cv2.VideoCapture(video_file)
fps = cap.get(cv2.CAP_PROP_FPS)
print('fps=', fps)
# width = 320
# height = 320
# fourcc = cv2.VideoWriter_fourcc(*'XVID')
# video_sym = cv2.VideoWriter('video_sym.avi', fourcc, fps, (width, height))
# video_raw = cv2.VideoWriter('video_raw.avi', fourcc, fps, (width, height))
while True:
try:
ret, frame = cap.read()
if not ret:
print("no ret")
break
msec = cap.get(cv2.CAP_PROP_POS_MSEC)
print(msec / 1000)
img = frame
iheight, iwidth, layers = img.shape
#resize
if resize == True:
new_h = round(iheight / 3)
new_w = round(iwidth / 3)
img = cv2.resize(img, (new_w, new_h))
lmarks = feature_detection.get_landmarks(img, detector, predictor)
if len(lmarks) != 1:
continue
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
frontal_raw, frontal_sym = frontalize.frontalize(
img, proj_matrix, model3D.ref_U, eyemask)
time_list.append(msec)
new_img_sym = np.round(frontal_sym).astype(np.uint8)
new_img_raw = np.round(frontal_raw).astype(np.uint8)
temp_pts = []
lmarks_sym = feature_detection.get_landmarks(
new_img_sym, detector, predictor)
temp_pts = [[x, y] for x, y in lmarks_sym[0]]
if mode == 'mouth':
temp_pts = temp_pts[33:34] + temp_pts[48:]
points_list_sym.append(temp_pts)
# video_sym.write(new_img_sym)
# video_raw.write(new_img_raw)
except ValueError as e:
print(e)
break
cap.release()
points_list = list(zip(time_list, points_list_sym))
with shelve.open(
'extracted_data/' + video_file.split('/')[-1].split('.')[0] +
'.txt', 'c') as db:
for i, x in enumerate(points_list):
db[str(i)] = x
return (points_list)
def myfrontalize(X, limit=0):
count = 0
if (limit == 0):
limit = X.shape[0]
print("Total Images: ", limit)
# check for dlib saved weights for face landmark detection
# if it fails, dowload and extract it manually from
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2
check.check_dlib_landmark_weights()
# load detections performed by dlib library on 3D model and Reference Image
model3D = frontalize.ThreeD_Model(
this_path + "/frontalization_models/model3Ddlib.mat", 'model_dlib')
# load mask to exclude eyes from symmetry
eyemask = np.asarray(
io.loadmat('frontalization_models/eyemask.mat')['eyemask'])
for i in range(0, limit):
print("\r", end='')
print("Images Completed: {0}".format(i), end='', flush=True)
# cast img to type int for cv2
img = X[i, :, :, 0].astype(np.uint8)
# create a color version for frontalizer stuffs
c_img = np.copy(img)
c_img = cv2.cvtColor(c_img, cv2.COLOR_GRAY2BGR)
# extract landmarks from the query image
# list containing a 2D array with points (x, y) for each face detected in the query image
lmarks = feature_detection.get_landmarks(c_img)
if type(lmarks) is np.ndarray:
# perform camera calibration according to the first face detected
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, lmarks[0])
# perform frontalization and convert result to grayscale
frontal_raw, frontal_sym = frontalize.frontalize(
c_img, proj_matrix, model3D.ref_U, eyemask)
temp = cv2.cvtColor(frontal_raw, cv2.COLOR_BGR2GRAY)
# find nonzero bbox and crop image to remove uncessesary black space from edges
temp_mask = cv2.findNonZero(temp)
t_x, t_y, t_w, t_h = cv2.boundingRect(temp_mask)
t_bbox = temp[t_y:t_y + t_h, t_x:t_x + t_w]
# resize the cropped image to the appropriate dimensions for network
t_bbox = cv2.resize(t_bbox, dsize=(48, 48))
t_bbox = np.resize(t_bbox, (48, 48, 1))
X[i] = t_bbox.astype(np.float32)
plt.show()
count += 1
print()
print('{} images out of {} were frontalized.'.format(count, limit))
def readVideosAndLabels(directory):
namesUnique = os.listdir(directory)
labels = []
faceMatrix = None
print(namesUnique)
indiceToSymmetry = [
17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 27, 26, 25,
24, 23, 22, 21, 20, 19, 18, 28, 29, 30, 31, 36, 35, 35, 33, 32, 46, 45,
44, 43, 48, 47, 40, 39, 38, 37, 42, 41, 55, 54, 53, 52, 51, 50, 49, 60,
59, 58, 57, 56, 65, 64, 63, 62, 61, 68, 67, 66
]
indiceToSymmetry = list(np.array(indiceToSymmetry) - 1)
model3D = frontalize.ThreeD_Model(
"./frontalization_models/model3Ddlib.mat", 'model_dlib')
eyemask = np.asarray(
sio.loadmat('frontalization_models/eyemask.mat')['eyemask'])
for ind, folder in enumerate(namesUnique):
for path in os.listdir(os.path.join(directory, folder)):
path = os.path.join(directory, folder, path)
cap = cv2.VideoCapture(path)
count = 0
trackingLost = True
print(folder)
"""
if(folder!="Romuald"):
print("a")
if(folder=="Stephane"):
pass
else:
break
"""
if (folder != "Florian"):
continue
while (True):
ret, imageO = cap.read()
if (count % 2 == 1):
if (not ret):
break
#print(imageO.shape)
#image = cv2.resize(image,(700,700))
if (folder == "Faddwzi"):
#image=imutils.rotate_bound(image, -90)
break
#pass
else:
#image=imutils.rotate_bound(image, 90)
image = np.ascontiguousarray(np.transpose(
np.copy(imageO), (1, 0, 2)),
dtype=np.uint8)
#image=np.ascontiguousarray(rotateImage(imageO),dtype=np.uint8)
#image=imutils.rotate_bound(image, 90)
#image=imutils.resize(image, width=370)
#imageWithLandmarks,landmarks,_=searchForFaceInTheWholeImage(np.copy(image))
if (trackingLost):
landmarks, faceROI, trackingLost, image = trackFaceInANeighborhoodAndDetectLandmarks(
np.copy(image),
faceROI=[
0, 0, image.shape[0] - 1, image.shape[1] - 1
],
drawBoundingBoxes=True)
else:
landmarks, faceROI, trackingLost, image = trackFaceInANeighborhoodAndDetectLandmarks(
np.copy(image), faceROI[0], drawBoundingBoxes=True)
#print(trackingLost,faceROI)
cv2.imshow('image', image)
cv2.waitKey(1)
if (trackingLost):
continue
#pdb.set_trace()
#landmarks[0][:,1]=landmarks[0][:,1]+np.sign(image.shape[]landmarks[0][:,1])
landmarks[0][:, [0, 1]] = landmarks[0][:, [1, 0]]
#landmarks[0][range(0,68)]=landmarks[0][indiceToSymmetry,:]
#pdb.set_trace()
#######registeredFace=performFaceAlignment(image,landmarks[0],cols=600,rows=600)
proj_matrix, camera_matrix, rmat, tvec = calib.estimate_camera(
model3D, landmarks[0])
# load mask to exclude eyes from symmetry
# perform frontalization
frontal_raw, registeredFace = frontalize.frontalize(
imageO, proj_matrix, model3D.ref_U, eyemask)
if (registeredFace is not None):
faceVector = ((registeredFace) / 255.0).reshape(-1)
if (faceMatrix is None):
faceMatrix = faceVector
else:
faceMatrix = np.vstack((faceMatrix, faceVector))
labels.append(folder)
for landmark in landmarks:
cv2.polylines(
image, np.int32(landmark.reshape((-1, 1, 2))),
True, (0, 0, 255), 3)
cv2.imshow('image333', frontal_raw)
cv2.imshow('aligned face', registeredFace)
cv2.waitKey(1)
cv2.imwrite(
os.path.join(directory, folder,
str(count) + '.jpg'), registeredFace)
count = count + 1
cap.release()
return faceMatrix, labels
