class PRN(object):
"""Joint 3D Face Reconstruction and Dense Alighment with Position Map Regression Network
"""
def __init__(self, is_Dlib=False, prefix='.'):
"""
Args:
is_dlib(bool, optional): if True, dlib is used for detecting faces.
prefix(str, optional): If run at another folder, the absolute path is needed to load the data.
"""
# Resolution of input and output image size.
self.resolution_input = 256
self.resolution_output = 256
# Load detectors
if is_Dlib:
import dlib
detector_path = os.path.join(
prefix, 'Data/net-data/mmod_human_face_detector.dat')
self.face_detector = dlib.cnn_face_detection_model_v1(
detector_path) # TODO Invest more and try other face detector.
# Load PRN
self.pos_predictor = PosPrediction(self.resolution_input,
self.resolution_output)
prn_path = os.path.join(prefix, 'Data/net-data/256_256_resfcn256_weight')
if not os.path.isfile(prn_path + '.data-00000-of-00001'):
print('Please download PRN trained model first.')
exit()
self.pos_predictor.restore(prn_path)
class PRN:
''' Joint 3D Face Reconstruction and Dense Alignment with Position Map Regression Network
Args:
is_dlib(bool, optional): If true, dlib is used for detecting faces.
prefix(str, optional): If run at another folder, the absolute path is needed to load the data.
'''
def __init__(self, is_dlib = False, prefix = '.'):
# resolution of input and output image size.
self.resolution_inp = 256
self.resolution_op = 256
#---- load PRN
self.pos_predictor = PosPrediction(self.resolution_inp, self.resolution_op)
prn_path = os.path.join(prefix, 'Data/net-data/256_256_resfcn256_weight')
if not os.path.isfile(prn_path + '.data-00000-of-00001'):
print("please download PRN trained model first.")
exit()
self.pos_predictor.restore(prn_path)
def net_forward(self, image):
''' The core of out method: regress the position map of a given image.
Args:
image: (256,256,3) array. value range: 0~1
Returns:
pos: the 3D position map. (256, 256, 3) array.
'''
return self.pos_predictor.predict(image)
class faceLandmark(object):
def __init__(self, model, gpu, resolution=256):
self.resa = self.resb = 256
self.pos_preditor = PosPrediction(gpu, self.resa, self.resb)
self.pos_preditor.restore(model)
self.faceindex = np.array([
24591, 30230, 36122, 42272, 46893, 48707, 48219, 47984, 49536,
48015, 48292, 48828, 47058, 42463, 36325, 30441, 24816, 12602,
10823, 10069, 10337, 10858, 10901, 10398, 10154, 10936, 12741,
15232, 18816, 22144, 24704, 28533, 29050, 29568, 29061, 28554,
17230, 15446, 15711, 16742, 17504, 17751, 16793, 15776, 15529,
17329, 17832, 17567, 36460, 33652, 32636, 32896, 32643, 33675,
36498, 38025, 38532, 38528, 38523, 38006, 36206, 34682, 34432,
34693, 36497, 36740, 36480, 36731
],
dtype=np.int32)
def circlebox(self, img, box):
x1, y1, x2, y2 = box
oldsize = (x2 + y2 - x1 - y1) / 2
center = np.array([(x1 + x2) / 2, (y1 + y2) / 2 + int(0.1 * oldsize)])
size = int(oldsize * 1.5)
SRC_PTS = np.array([[center[0] - size / 2, center[1] - size / 2],
[center[0] - size / 2, center[1] + size / 2],
[center[0] + size / 2, center[1] - size / 2]])
DST_PTS = np.array([[0, 0], [0, 256 - 1], [256 - 1, 0]])
tform = cv2.estimateRigidTransform(SRC_PTS, DST_PTS, False)
image = cv2.warpAffine(img, tform, (self.resa, self.resa))
return image, tform
def predict(self, img, box):
h, w, _ = img.shape
image, tform = self.circlebox(img, box)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
image = image / 255.
pos = np.reshape(self.pos_preditor.predict(image), (-1, 3)).T
pos[2, :] = 1
tform = np.vstack((tform, np.array([0, 0, 1])))
pos = np.dot(np.linalg.inv(tform), pos).T.reshape(-1, 3)
landmarks = pos[self.faceindex, :2].astype(np.float32)
return landmarks
class PRN:
''' Joint 3D Face Reconstruction and Dense Alignment with Position Map Regression Network
Args:
is_dlib(bool, optional): If true, dlib is used for detecting faces.
is_opencv(bool, optional): If true, opencv is used for extracting texture.
prefix(str, optional): If run at another folder, the absolute path is needed to load the data.
'''
def __init__(self, is_dlib=False, is_opencv=False, prefix='.'):
# resolution of input and output image size.
self.resolution_inp = 256
self.resolution_op = 256
#---- load detectors
if is_dlib:
import dlib
detector_path = os.path.join(
prefix, 'Data/net-data/mmod_human_face_detector.dat')
self.face_detector = dlib.cnn_face_detection_model_v1(
detector_path)
if is_opencv:
import cv2
#---- load PRN
self.pos_predictor = PosPrediction(self.resolution_inp,
self.resolution_op)
prn_path = os.path.join(prefix,
'Data/net-data/256_256_resfcn256_weight')
if not os.path.isfile(prn_path + '.data-00000-of-00001'):
print("please download PRN trained model first.")
exit()
self.pos_predictor.restore(prn_path)
# uv file
self.uv_kpt_ind = np.loadtxt(prefix +
'/Data/uv-data/uv_kpt_ind.txt').astype(
np.int32) # 2 x 68 get kpt
self.face_ind = np.loadtxt(
prefix + '/Data/uv-data/face_ind.txt').astype(
np.int32) # get valid vertices in the pos map
self.triangles = np.loadtxt(prefix +
'/Data/uv-data/triangles.txt').astype(
np.int32) # ntri x 3
def dlib_detect(self, image):
return self.face_detector(image, 1)
def net_forward(self, image):
''' The core of out method: regress the position map of a given image.
Args:
image: (256,256,3) array. value range: 0~1
Returns:
pos: the 3D position map. (256, 256, 3) array.
'''
return self.pos_predictor.predict(image)
def process(self, input, image_info=None):
''' process image with crop operation.
Args:
input: (h,w,3) array or str(image path). image value range:1~255.
image_info(optional): the bounding box information of faces. if None, will use dlib to detect face.
Returns:
pos: the 3D position map. (256, 256, 3).
'''
if isinstance(input, str):
try:
image = imread(input)
except IOError:
print("error opening file: ", input)
return None
else:
image = input
if image.ndim < 3:
image = np.tile(image[:, :, np.newaxis], [1, 1, 3])
if image_info is not None:
if np.max(image_info.shape) > 4: # key points to get bounding box
kpt = image_info
if kpt.shape[0] > 3:
kpt = kpt.T
left = np.min(kpt[0, :])
right = np.max(kpt[0, :])
top = np.min(kpt[1, :])
bottom = np.max(kpt[1, :])
else: # bounding box
bbox = image_info
left = bbox[0]
right = bbox[1]
top = bbox[2]
bottom = bbox[3]
old_size = (right - left + bottom - top) / 2
center = np.array(
[right - (right - left) / 2.0, bottom - (bottom - top) / 2.0])
size = int(old_size * 1.6)
else:
detected_faces = self.dlib_detect(image)
if len(detected_faces) == 0:
print('warning: no detected face')
return None
d = detected_faces[
0].rect ## only use the first detected face (assume that each input image only contains one face)
left = d.left()
right = d.right()
top = d.top()
bottom = d.bottom()
old_size = (right - left + bottom - top) / 2
center = np.array([
right - (right - left) / 2.0,
bottom - (bottom - top) / 2.0 + old_size * 0.14
])
size = int(old_size * 1.58)
# crop image
src_pts = np.array([[center[0] - size / 2, center[1] - size / 2],
[center[0] - size / 2, center[1] + size / 2],
[center[0] + size / 2, center[1] - size / 2]])
DST_PTS = np.array([[0, 0], [0, self.resolution_inp - 1],
[self.resolution_inp - 1, 0]])
tform = estimate_transform('similarity', src_pts, DST_PTS)
image = image / 255.
cropped_image = warp(image,
tform.inverse,
output_shape=(self.resolution_inp,
self.resolution_inp))
# run our net
#st = time()
cropped_pos = self.net_forward(cropped_image)
#print 'net time:', time() - st
# restore
cropped_pos = np.squeeze(cropped_pos)
cropped_vertices = np.reshape(cropped_pos, [-1, 3]).T
z = cropped_vertices[2, :].copy() / tform.params[0, 0]
cropped_vertices[2, :] = 1
vertices = np.dot(np.linalg.inv(tform.params), cropped_vertices)
vertices = np.vstack((vertices[:2, :], z))
pos = np.reshape(vertices.T,
[self.resolution_op, self.resolution_op, 3])
return pos
def get_landmarks(self, pos):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
kpt: 68 3D landmarks. shape = (68, 3).
'''
kpt = pos[self.uv_kpt_ind[1, :], self.uv_kpt_ind[0, :], :]
return kpt
def get_vertices(self, pos):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
vertices: the vertices(point cloud). shape = (num of points, 3). n is about 40K here.
'''
all_vertices = np.reshape(pos, [self.resolution_op**2, -1])
vertices = all_vertices[self.face_ind, :]
return vertices
def get_texture(self, image, pos):
''' extract uv texture from image. opencv is needed here.
Args:
image: input image.
pos: the 3D position map. shape = (256, 256, 3).
Returns:
texture: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here.
'''
texture = cv2.remap(image,
pos[:, :, :2].astype(np.float32),
None,
interpolation=cv2.INTER_NEAREST,
borderMode=cv2.BORDER_CONSTANT,
borderValue=(0))
return texture
def get_colors(self, image, vertices):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here.
'''
[h, w, _] = image.shape
vertices[:, 0] = np.minimum(np.maximum(vertices[:, 0], 0), w - 1) # x
vertices[:, 1] = np.minimum(np.maximum(vertices[:, 1], 0), h - 1) # y
ind = np.round(vertices).astype(np.int32)
colors = image[ind[:, 1], ind[:, 0], :] # n x 3
return colors
class PRN:
''' Joint 3D Face Reconstruction and Dense Alignment with Position Map Regression Network
Args:
is_dlib(bool, optional): If true, dlib is used for detecting faces.
prefix(str, optional): If run at another folder, the absolute path is needed to load the data.
'''
def __init__(self, is_dlib=False, prefix='.'):
# resolution of input and output image size.
self.resolution_inp = 256
self.resolution_op = 256
#---- load detectors
if is_dlib:
import dlib
detector_path = os.path.join(
prefix, 'Data/net-data/mmod_human_face_detector.dat')
self.face_detector = dlib.cnn_face_detection_model_v1(
detector_path)
#---- load PRN
self.pos_predictor = PosPrediction(self.resolution_inp,
self.resolution_op)
prn_path = os.path.join(prefix,
'Data/net-data/256_256_resfcn256_weight')
if not os.path.isfile(prn_path + '.data-00000-of-00001'):
print("please download PRN trained model first.")
exit()
self.pos_predictor.restore(prn_path)
# uv file
self.uv_kpt_ind = np.loadtxt(prefix +
'/Data/uv-data/uv_kpt_ind.txt').astype(
np.int32) # 2 x 68 get kpt
self.face_ind = np.loadtxt(
prefix + '/Data/uv-data/face_ind.txt').astype(
np.int32) # get valid vertices in the pos map
self.triangles = np.loadtxt(prefix +
'/Data/uv-data/triangles.txt').astype(
np.int32) # ntri x 3
self.uv_coords = self.generate_uv_coords()
def generate_uv_coords(self):
resolution = self.resolution_op
uv_coords = np.meshgrid(range(resolution), range(resolution))
uv_coords = np.transpose(np.array(uv_coords), [1, 2, 0])
uv_coords = np.reshape(uv_coords, [resolution**2, -1])
uv_coords = uv_coords[self.face_ind, :]
uv_coords = np.hstack(
(uv_coords[:, :2], np.zeros([uv_coords.shape[0], 1])))
return uv_coords
def dlib_detect(self, image):
return self.face_detector(image, 1)
def net_forward(self, image):
''' The core of out method: regress the position map of a given image.
Args:
image: (256,256,3) array. value range: 0~1
Returns:
pos: the 3D position map. (256, 256, 3) array.
'''
return self.pos_predictor.predict(image)
def process(self, input, prev, image_info=None):
''' process image with crop operation.
Args:
input: (h,w,3) array or str(image path). image value range:1~255.
image_info(optional): the bounding box information of faces. if None, will use dlib to detect face.
Returns:
pos: the 3D position map. (256, 256, 3).
'''
if isinstance(input, str):
try:
image = imread(input)
except IOError:
print("error opening file: ", input)
return None
else:
image = input
if image.ndim < 3:
image = np.tile(image[:, :, np.newaxis], [1, 1, 3])
if image_info is not None:
if np.max(image_info.shape) > 4: # key points to get bounding box
kpt = image_info
if kpt.shape[0] > 3:
kpt = kpt.T
left = np.min(kpt[0, :])
right = np.max(kpt[0, :])
top = np.min(kpt[1, :])
bottom = np.max(kpt[1, :])
else: # bounding box
bbox = image_info
left = bbox[0]
right = bbox[1]
top = bbox[2]
bottom = bbox[3]
old_size = (right - left + bottom - top) / 2
center = np.array(
[right - (right - left) / 2.0, bottom - (bottom - top) / 2.0])
size = int(old_size * 1.6)
else:
detected_faces = self.dlib_detect(image)
if len(detected_faces) == 0:
print('warning: no detected face, use the previous valid one.')
#return None
detected_faces = self.dlib_detect(prev)
if len(detected_faces) == 0:
return None, prev, 0
else:
prev = image
d = detected_faces[
0].rect ## only use the first detected face (assume that each input image only contains one face)
left = d.left()
right = d.right()
top = d.top()
bottom = d.bottom()
old_size = (right - left + bottom - top) / 2
center = np.array([
right - (right - left) / 2.0,
bottom - (bottom - top) / 2.0 + old_size * 0.14
])
size = int(old_size * 1.58)
# crop image
src_pts = np.array([[center[0] - size / 2, center[1] - size / 2],
[center[0] - size / 2, center[1] + size / 2],
[center[0] + size / 2, center[1] - size / 2]])
DST_PTS = np.array([[0, 0], [0, self.resolution_inp - 1],
[self.resolution_inp - 1, 0]])
tform = estimate_transform('similarity', src_pts, DST_PTS)
image = image / 255.
cropped_image = warp(image,
tform.inverse,
output_shape=(self.resolution_inp,
self.resolution_inp))
# run our net
cropped_pos = self.net_forward(cropped_image)
# restore
cropped_vertices = np.reshape(cropped_pos, [-1, 3]).T
z = cropped_vertices[2, :].copy() / tform.params[0, 0]
cropped_vertices[2, :] = 1
vertices = np.dot(np.linalg.inv(tform.params), cropped_vertices)
vertices = np.vstack((vertices[:2, :], z))
pos = np.reshape(vertices.T,
[self.resolution_op, self.resolution_op, 3])
return pos, prev, 1
def get_landmarks(self, pos):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
kpt: 68 3D landmarks. shape = (68, 3).
'''
kpt = pos[self.uv_kpt_ind[1, :], self.uv_kpt_ind[0, :], :]
return kpt
def get_vertices(self, pos):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
vertices: the vertices(point cloud). shape = (num of points, 3). n is about 40K here.
'''
all_vertices = np.reshape(pos, [self.resolution_op**2, -1])
vertices = all_vertices[self.face_ind, :]
return vertices
def get_colors_from_texture(self, texture):
'''
Args:
texture: the texture map. shape = (256, 256, 3).
Returns:
colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here.
'''
all_colors = np.reshape(texture, [self.resolution_op**2, -1])
colors = all_colors[self.face_ind, :]
return colors
def get_colors(self, image, vertices):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here.
'''
[h, w, _] = image.shape
vertices[:, 0] = np.minimum(np.maximum(vertices[:, 0], 0), w - 1) # x
vertices[:, 1] = np.minimum(np.maximum(vertices[:, 1], 0), h - 1) # y
ind = np.round(vertices).astype(np.int32)
colors = image[ind[:, 1], ind[:, 0], :] # n x 3
return colors
def softmax_smooth2(self, arr_2d, steep):
# matrix: numpy matrix
# steep: close to 0 means taking average, high steep means only looking at new data
n_array = np.arange(1, arr_2d.shape[0] + 1)
numerator = np.apply_along_axis(self.softmax_numerator, 0, arr_2d,
n_array, steep)
denominator = np.sum(np.exp(-steep * n_array))
smooth_array = numerator / denominator
return smooth_array
def softmax_numerator(self, row, n_array, steep=1):
numerator = np.sum(np.exp(-steep * n_array) * row)
return numerator
class PRN:
''' Joint 3D Face Reconstruction and Dense Alignment with Position Map Regression Network
Args:
is_dlib(bool, optional): If true, dlib is used for detecting faces.
prefix(str, optional): If run at another folder, the absolute path is needed to load the data.
'''
def __init__(self, is_dlib = False, prefix = '.'):
# resolution of input and output image size.
self.resolution_inp = 256
self.resolution_op = 256
#---- load detectors
if is_dlib:
import dlib
detector_path = os.path.join(prefix, 'Data/net-data/mmod_human_face_detector.dat')
self.face_detector = dlib.cnn_face_detection_model_v1(
detector_path)
#---- load PRN
self.pos_predictor = PosPrediction(self.resolution_inp, self.resolution_op)
prn_path = os.path.join(prefix, 'Data/net-data/256_256_resfcn256_weight')
if not os.path.isfile(prn_path + '.data-00000-of-00001'):
print("please download PRN trained model first.")
exit()
self.pos_predictor.restore(prn_path)
# uv file
self.uv_kpt_ind = np.loadtxt(prefix + '/Data/uv-data/uv_kpt_ind.txt').astype(np.int32) # 2 x 68 get kpt
self.face_ind = np.loadtxt(prefix + '/Data/uv-data/face_ind.txt').astype(np.int32) # get valid vertices in the pos map
self.triangles = np.loadtxt(prefix + '/Data/uv-data/triangles.txt').astype(np.int32) # ntri x 3
self.uv_coords = self.generate_uv_coords()
def generate_uv_coords(self):
resolution = self.resolution_op
uv_coords = np.meshgrid(range(resolution),range(resolution))
uv_coords = np.transpose(np.array(uv_coords), [1,2,0])
uv_coords = np.reshape(uv_coords, [resolution**2, -1]);
uv_coords = uv_coords[self.face_ind, :]
uv_coords = np.hstack((uv_coords[:,:2], np.zeros([uv_coords.shape[0], 1])))
return uv_coords
def dlib_detect(self, image):
return self.face_detector(image, 1)
def net_forward(self, image):
''' The core of out method: regress the position map of a given image.
Args:
image: (256,256,3) array. value range: 0~1
Returns:
pos: the 3D position map. (256, 256, 3) array.
'''
return self.pos_predictor.predict(image)
def process(self, input, image_info = None):
''' process image with crop operation.
Args:
input: (h,w,3) array or str(image path). image value range:1~255.
image_info(optional): the bounding box information of faces. if None, will use dlib to detect face.
Returns:
pos: the 3D position map. (256, 256, 3).
'''
if isinstance(input, str):
try:
image = imread(input)
except IOError:
print("error opening file: ", input)
return None
else:
image = input
if image.ndim < 3:
image = np.tile(image[:,:,np.newaxis], [1,1,3])
if image_info is not None:
if np.max(image_info.shape) > 4: # key points to get bounding box
kpt = image_info
if kpt.shape[0] > 3:
kpt = kpt.T
left = np.min(kpt[0, :]); right = np.max(kpt[0, :]);
top = np.min(kpt[1,:]); bottom = np.max(kpt[1,:])
else: # bounding box
bbox = image_info
left = bbox[0]; right = bbox[1]; top = bbox[2]; bottom = bbox[3]
old_size = (right - left + bottom - top)/2
center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0])
size = int(old_size*1.6)
else:
detected_faces = self.dlib_detect(image)
if len(detected_faces) == 0:
print('warning: no detected face')
return None
d = detected_faces[0].rect ## only use the first detected face (assume that each input image only contains one face)
left = d.left(); right = d.right(); top = d.top(); bottom = d.bottom()
old_size = (right - left + bottom - top)/2
center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0 + old_size*0.14])
size = int(old_size*1.58)
# crop image
src_pts = np.array([[center[0]-size/2, center[1]-size/2], [center[0] - size/2, center[1]+size/2], [center[0]+size/2, center[1]-size/2]])
DST_PTS = np.array([[0,0], [0,self.resolution_inp - 1], [self.resolution_inp - 1, 0]])
tform = estimate_transform('similarity', src_pts, DST_PTS)
image = image/255.
cropped_image = warp(image, tform.inverse, output_shape=(self.resolution_inp, self.resolution_inp))
# run our net
#st = time()
cropped_pos = self.net_forward(cropped_image)
#print 'net time:', time() - st
# restore
cropped_vertices = np.reshape(cropped_pos, [-1, 3]).T
z = cropped_vertices[2,:].copy()/tform.params[0,0]
cropped_vertices[2,:] = 1
vertices = np.dot(np.linalg.inv(tform.params), cropped_vertices)
vertices = np.vstack((vertices[:2,:], z))
pos = np.reshape(vertices.T, [self.resolution_op, self.resolution_op, 3])
return pos
def get_landmarks(self, pos):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
kpt: 68 3D landmarks. shape = (68, 3).
'''
kpt = pos[self.uv_kpt_ind[1,:], self.uv_kpt_ind[0,:], :]
return kpt
def get_vertices(self, pos):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
vertices: the vertices(point cloud). shape = (num of points, 3). n is about 40K here.
'''
all_vertices = np.reshape(pos, [self.resolution_op**2, -1])
vertices = all_vertices[self.face_ind, :]
return vertices
def get_colors_from_texture(self, texture):
'''
Args:
texture: the texture map. shape = (256, 256, 3).
Returns:
colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here.
'''
all_colors = np.reshape(texture, [self.resolution_op**2, -1])
colors = all_colors[self.face_ind, :]
return colors
def get_colors(self, image, vertices):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here.
'''
[h, w, _] = image.shape
vertices[:,0] = np.minimum(np.maximum(vertices[:,0], 0), w - 1) # x
vertices[:,1] = np.minimum(np.maximum(vertices[:,1], 0), h - 1) # y
ind = np.round(vertices).astype(np.int32)
colors = image[ind[:,1], ind[:,0], :] # n x 3
return colors
class PRN:
''' Joint 3D Face Reconstruction and Dense Alignment with Position Map Regression Network
Args:
is_dlib(bool, optional): If true, dlib is used for detecting faces.
prefix(str, optional): If run at another folder, the absolute path is needed to load the data.
'''
def __init__(self, is_dlib = False, is_faceboxes = True, prefix = '.'):
# resolution of input and output image size.
self.resolution_inp = 256
self.resolution_op = 256
#---- load detectors
if is_dlib:
import dlib
self.isDilb = True
detector_path = os.path.join(prefix, 'Data/net-data/mmod_human_face_detector.dat')
self.face_detector = dlib.cnn_face_detection_model_v1(
detector_path)
if is_faceboxes:
from faceboxes.face_detector import FaceDetector
self.isFaceBoxes = True
MODEL_PATH = './faceboxes/model/adas_model.pb'
self.face_detector = FaceDetector(MODEL_PATH, gpu_memory_fraction=0.25, visible_device_list='0')
#---- load PRN
self.pos_predictor = PosPrediction(self.resolution_inp, self.resolution_op)
prn_path = os.path.join(prefix, 'Data/net-data/256_256_resfcn256_weight')
if not os.path.isfile(prn_path + '.data-00000-of-00001'):
print("please download PRN trained model first.")
exit()
self.pos_predictor.restore(prn_path)
# uv file
self.uv_kpt_ind = np.loadtxt(prefix + '/Data/uv-data/uv_kpt_ind.txt').astype(np.int32) # 2 x 68 get kpt
self.face_ind = np.loadtxt(prefix + '/Data/uv-data/face_ind.txt').astype(np.int32) # get valid vertices in the pos map
self.triangles = np.loadtxt(prefix + '/Data/uv-data/triangles.txt').astype(np.int32) # ntri x 3
self.uv_coords = self.generate_uv_coords()
def generate_uv_coords(self):
resolution = self.resolution_op
uv_coords = np.meshgrid(range(resolution),range(resolution))
uv_coords = np.transpose(np.array(uv_coords), [1,2,0])
uv_coords = np.reshape(uv_coords, [resolution**2, -1])
uv_coords = uv_coords[self.face_ind, :]
uv_coords = np.hstack((uv_coords[:,:2], np.zeros([uv_coords.shape[0], 1])))
return uv_coords
def dlib_detect(self, image):
return self.face_detector(image, 1)
def faceboxes_detect(self, image):
# image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
boxes, scores = self.face_detector(image, score_threshold=0.3)
# 按照离中心点最近，输入检测框
return find_face(boxes, scores)
def net_forward(self, image):
''' The core of out method: regress the position map of a given image.
Args:
image: (256,256,3) array. value range: 0~1
Returns:
pos: the 3D position map. (256, 256, 3) array.
'''
return self.pos_predictor.predict(image)
def process(self, input, image_info = None):
''' process image with crop operation.
Args:
input: (h,w,3) array or str(image path). image value range:1~255.
image_info(optional): the bounding box information of faces. if None, will use dlib to detect face.
Returns:
pos: the 3D position map. (256, 256, 3).
'''
if isinstance(input, str):
try:
image = imread(input)
except IOError:
print("error opening file: ", input)
return None
else:
image = input
if image.ndim < 3:
image = np.tile(image[:,:,np.newaxis], [1,1,3])
if image_info is not None:
if np.max(image_info.shape) > 4: # key points to get bounding box
kpt = image_info
if kpt.shape[0] > 3:
kpt = kpt.T
left = np.min(kpt[0, :]); right = np.max(kpt[0, :])
top = np.min(kpt[1,:]); bottom = np.max(kpt[1,:])
else: # bounding box
bbox = image_info
left = bbox[0]; right = bbox[1]; top = bbox[2]; bottom = bbox[3]
old_size = (right - left + bottom - top)/2
center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0])
size = int(old_size*1.6) # 1.6倍的人脸区域
elif self.isFaceBoxes:
boxes, scores = self.faceboxes_detect(image)
if len(scores) == 0:
print('warning: no detected face')
return None
# cv2.imshow('face detection', draw_rect_on_image(image, boxes, scores))
# cv2.waitKey(1)
d = boxes[0]
left = d[1]; right = d[3]; top = d[0]; bottom = d[2]
old_size = (right - left + bottom - top)/2
center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0])
size = int(old_size*1.6)
elif self.isDilb:
detected_faces = self.dlib_detect(image)
if len(detected_faces) == 0:
print('warning: no detected face')
return None
d = detected_faces[0].rect ## only use the first detected face (assume that each input image only contains one face)
left = d.left(); right = d.right(); top = d.top(); bottom = d.bottom()
old_size = (right - left + bottom - top)/2
center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0 + old_size*0.14])
size = int(old_size*1.58)
# crop image
src_pts = np.array([[center[0]-size/2, center[1]-size/2],
[center[0]-size/2, center[1]+size/2],
[center[0]+size/2, center[1]-size/2]])
DST_PTS = np.array([[0,0], [0,self.resolution_inp - 1], [self.resolution_inp - 1, 0]])
# print(src_pts)
# print('---------------------')
# print(DST_PTS)
tform = estimate_transform('similarity', src_pts, DST_PTS) # 人脸的检测框到256256框的相似变换矩阵
image = image/255.
# 对输入图像按照相似变换从输入图像中的到RPNet模型需要的输入
cropped_image = warp(image, tform.inverse, output_shape=(self.resolution_inp, self.resolution_inp))
# run our net
# st = time()
cropped_pos = self.net_forward(cropped_image)
# print 'net time:', time() - st
# restore
cropped_vertices = np.reshape(cropped_pos, [-1, 3]).T
z = cropped_vertices[2,:].copy()/tform.params[0,0]
cropped_vertices[2,:] = 1
vertices = np.dot(np.linalg.inv(tform.params), cropped_vertices)
vertices = np.vstack((vertices[:2,:], z))
pos = np.reshape(vertices.T, [self.resolution_op, self.resolution_op, 3])
# 同时返回检测到的pos参数以及对应的人脸区域
return pos, cropped_image*255
def get_landmarks(self, pos):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
kpt: 68 3D landmarks. shape = (68, 3).
'''
kpt = pos[self.uv_kpt_ind[1,:], self.uv_kpt_ind[0,:], :]
return kpt
def get_landmarks_2d(self, pos):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
kpt: 68 2D landmarks. shape = (68, 2).
'''
kpt = pos[self.uv_kpt_ind[1,:], self.uv_kpt_ind[0,:], :2]
return kpt
def get_vertices(self, pos):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
vertices: the vertices(point cloud). shape = (num of points, 3). n is about 40K here.
'''
all_vertices = np.reshape(pos, [self.resolution_op**2, -1])
vertices = all_vertices[self.face_ind, :]
return vertices
def get_colors_from_texture(self, texture):
'''
Args:
texture: the texture map. shape = (256, 256, 3).
Returns:
colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here.
'''
all_colors = np.reshape(texture, [self.resolution_op**2, -1])
colors = all_colors[self.face_ind, :]
return colors
def get_colors(self, image, vertices):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here.
'''
[h, w, _] = image.shape
vertices[:,0] = np.minimum(np.maximum(vertices[:,0], 0), w - 1) # x
vertices[:,1] = np.minimum(np.maximum(vertices[:,1], 0), h - 1) # y
ind = np.round(vertices).astype(np.int32)
colors = image[ind[:,1], ind[:,0], :] # n x 3
return colors
class PRN:
''' Joint 3D Face Reconstruction and Dense Alignment with Position Map Regression Network
Args:
is_dlib(bool, optional): If true, dlib is used for detecting faces.
prefix(str, optional): If run at another folder, the absolute path is needed to load the data.
'''
def __init__(self, is_dlib = False, prefix = '.'):
# resolution of input and output image size.
self.resolution_inp = 256
self.resolution_op = 256
#---- load detectors
#is_dlib = False
if is_dlib:
import dlib
#detector_path = os.path.join(prefix,'Data/net-data/shape_predictor_68_face_landmarks.dat')
self.face_detector = dlib.get_frontal_face_detector()
#detector_path = os.path.join(prefix, 'Data/net-data/mmod_human_face_detector.dat')
#self.face_detector = dlib.cnn_face_detection_model_v1( detector_path)
else:
self.face_detector = face_alignment.FaceAlignment(face_alignment.LandmarksType._2D, device='cuda:0')
#---- load PRN
self.pos_predictor = PosPrediction(self.resolution_inp, self.resolution_op)
prn_path = os.path.join(prefix, 'Data/net-data/256_256_resfcn256_weight')
if not os.path.isfile(prn_path + '.data-00000-of-00001'):
print("please download PRN trained model first.")
#exit()
self.pos_predictor.restore(prn_path)
# uv file
self.uv_kpt_ind = np.loadtxt(prefix + '/Data/uv-data/uv_kpt_ind.txt').astype(np.int32) # 2 x 68 get kpt
self.face_ind = np.loadtxt(prefix + '/Data/uv-data/face_ind.txt').astype(np.int32) # get valid vertices in the pos map
self.triangles = np.loadtxt(prefix + '/Data/uv-data/triangles.txt').astype(np.int32) # ntri x 3
self.uv_coords = self.generate_uv_coords()
def generate_uv_coords(self):
resolution = self.resolution_op
uv_coords = np.meshgrid(range(resolution),range(resolution))
uv_coords = np.transpose(np.array(uv_coords), [1,2,0])
uv_coords = np.reshape(uv_coords, [resolution**2, -1])
uv_coords = uv_coords[self.face_ind, :]
uv_coords = np.hstack((uv_coords[:,:2], np.zeros([uv_coords.shape[0], 1])))
return uv_coords
def dlib_detect(self, image):
print ('++++++++++',image.shape)
#return self.face_detector.get_landmarks(image)
return self.face_detector(image,1)
def net_forward(self, image):
''' The core of out method: regress the position map of a given image.
Args:
image: (256,256,3) array. value range: 0~1
Returns:
pos: the 3D position map. (256, 256, 3) array.
'''
return self.pos_predictor.predict(image)
def process(self, input, image_info = None):
''' process image with crop operation.
Args:
input: (h,w,3) array or str(image path). image value range:1~255.
image_info(optional): the bounding box information of faces. if None, will use dlib to detect face.
Returns:
pos: the 3D position map. (256, 256, 3).
'''
if isinstance(input, str):
#try:
image = imread(input)
#except IOError:
# print("error opening file: ", input)
# return None
else:
image = input
if image.ndim < 3:
image = np.tile(image[:,:,np.newaxis], [1,1,3])
detected_faces = self.dlib_detect( cv2.cvtColor(image, cv2.COLOR_BGR2GRAY))
if len(detected_faces) == 0:
# print ('+++++++++++++++++++++++++++++++++++++')
print('warning: no detected face')
return None
d = detected_faces[0]#.rect#.rect ## only use the first detected face (assume that each input image only contains one face)
print (type(d))
left = d.left(); right = d.right(); top = d.top(); bottom = d.bottom()
old_size = (right - left + bottom - top)/2
center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0 + old_size*0.14])
size = int(old_size*1.58)
# crop image
src_pts = np.array([[center[0]-size/2, center[1]-size/2], [center[0] - size/2, center[1]+size/2], [center[0]+size/2, center[1]-size/2]])
DST_PTS = np.array([[0,0], [0,self.resolution_inp - 1], [self.resolution_inp - 1, 0]])
tform = estimate_transform('similarity', src_pts, DST_PTS)
image = image/255.
cropped_image = warp(image, tform.inverse, output_shape=(self.resolution_inp, self.resolution_inp))
# run our net
#st = time()
cropped_pos = self.net_forward(cropped_image)
#print 'net time:', time() - st
# restore
cropped_vertices = np.reshape(cropped_pos, [-1, 3]).T
z = cropped_vertices[2,:].copy()/tform.params[0,0]
cropped_vertices[2,:] = 1
vertices = np.dot(np.linalg.inv(tform.params), cropped_vertices)
vertices = np.vstack((vertices[:2,:], z))
pos = np.reshape(vertices.T, [self.resolution_op, self.resolution_op, 3])
# print ('==================')
return pos
def get_landmarks(self, pos):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
kpt: 68 3D landmarks. shape = (68, 3).
'''
kpt = pos[self.uv_kpt_ind[1,:], self.uv_kpt_ind[0,:], :]
return kpt
def get_vertices(self, pos):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
vertices: the vertices(point cloud). shape = (num of points, 3). n is about 40K here.
'''
all_vertices = np.reshape(pos, [self.resolution_op**2, -1])
vertices = all_vertices[self.face_ind, :]
return vertices
def get_colors_from_texture(self, texture):
'''
Args:
texture: the texture map. shape = (256, 256, 3).
Returns:
colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here.
'''
all_colors = np.reshape(texture, [self.resolution_op**2, -1])
colors = all_colors[self.face_ind, :]
return colors
def get_colors(self, image, vertices):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here.
'''
[h, w, _] = image.shape
vertices[:,0] = np.minimum(np.maximum(vertices[:,0], 0), w - 1) # x
vertices[:,1] = np.minimum(np.maximum(vertices[:,1], 0), h - 1) # y
ind = np.round(vertices).astype(np.int32)
colors = image[ind[:,1], ind[:,0], :] # n x 3
return colors
class PRN:
''' Joint 3D Face Reconstruction and Dense Alignment with Position Map Regression Network
Args:
is_dlib(bool, optional): If true, dlib is used for detecting faces.
prefix(str, optional): If run at another folder, the absolute path is needed to load the data.
'''
def __init__(self, is_dlib = False, prefix = '.'):
# resolution of input and output image size.
self.resolution_inp = 256
self.resolution_op = 256
#---- load detectors
if is_dlib:
detector_path = os.path.join(prefix, 'Data/net-data/mmod_human_face_detector.dat')
self.face_regressor = dlib.shape_predictor('Data/net-data/shape_predictor_68_face_landmarks.dat')
self.face_detector = dlib.cnn_face_detection_model_v1(
detector_path)
#---- load PRN
self.pos_predictor = PosPrediction(self.resolution_inp, self.resolution_op)
prn_path = os.path.join(prefix, 'Data/net-data/256_256_resfcn256_weight')
if not os.path.isfile(prn_path + '.data-00000-of-00001'):
print("please download PRN trained model first.")
exit()
self.pos_predictor.restore(prn_path)
# uv file
self.uv_kpt_ind = np.loadtxt(prefix + '/Data/uv-data/uv_kpt_ind.txt').astype(np.int32) # 2 x 68 get kpt
self.face_ind = np.loadtxt(prefix + '/Data/uv-data/face_ind.txt').astype(np.int32) # get valid vertices in the pos map
self.triangles = np.loadtxt(prefix + '/Data/uv-data/triangles.txt').astype(np.int32) # ntri x 3
self.uv_coords = self.generate_uv_coords()
def generate_uv_coords(self):
resolution = self.resolution_op
uv_coords = np.meshgrid(range(resolution),range(resolution))
uv_coords = np.transpose(np.array(uv_coords), [1,2,0])
uv_coords = np.reshape(uv_coords, [resolution**2, -1]);
uv_coords = uv_coords[self.face_ind, :]
uv_coords = np.hstack((uv_coords[:,:2], np.zeros([uv_coords.shape[0], 1])))
return uv_coords
def dlib_detect(self, image):
return self.face_detector(image, 1)
def net_forward(self, image):
''' The core of out method: regress the position map of a given image.
Args:
image: (256,256,3) array. value range: 0~1
Returns:
pos: the 3D position map. (256, 256, 3) array.
'''
return self.pos_predictor.predict(image)
def affine_align(self, img, landmark):
M = None
src = np.array([
[38.2946, 51.6963],
[73.5318, 51.5014],
[56.0252, 71.7366],
[41.5493, 92.3655],
[70.7299, 92.2041] ], dtype=np.float32 )
src=src * 256 / 112
dst = landmark.astype(np.float32)
tform = estimate_transform('similarity', src, dst)
M = tform.params[0:2,:]
return M,tform
def landmark_68_to_5(self,t68):
le = t68[36:42, :].mean(axis=0, keepdims=True)
re = t68[42:48, :].mean(axis=0, keepdims=True)
no = t68[31:32, :]
lm = t68[48:49, :]
rm = t68[54:55, :]
t5 = np.concatenate([le, re, no, lm, rm], axis=0)
t5 = t5.reshape(10)
return t5
def process(self, input, image_info = None):
''' process image with crop operation.
Args:
input: (h,w,3) array or str(image path). image value range:1~255.
image_info(optional): the bounding box information of faces. if None, will use dlib to detect face.
Returns:
pos: the 3D position map. (256, 256, 3).
'''
if isinstance(input, str):
try:
image = imread(input)
except IOError:
print("error opening file: ", input)
return None
else:
image = input
if image.ndim < 3:
image = np.tile(image[:,:,np.newaxis], [1,1,3])
if image_info is not None:
if np.max(image_info.shape) > 4: # key points to get bounding box
kpt = image_info
if kpt.shape[0] > 3:
kpt = kpt.T
left = np.min(kpt[0, :]); right = np.max(kpt[0, :]);
top = np.min(kpt[1,:]); bottom = np.max(kpt[1,:])
else: # bounding box
bbox = image_info
left = bbox[0]; right = bbox[1]; top = bbox[2]; bottom = bbox[3]
old_size = (right - left + bottom - top)/2
center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0])
size = int(old_size*1.6)
else:
detected_faces = self.dlib_detect(image)
if len(detected_faces) == 0:
print('warning: no detected face')
return None
face_detector = dlib.get_frontal_face_detector()
rects = face_detector(image, 1)
pts = self.face_regressor(image, rects[0]).parts()
pts = np.array([[pt.x, pt.y] for pt in pts]).T
x = pts.T
# d = detected_faces[0].rect ## only use the first detected face (assume that each input image only contains one face)
# left = d.left(); right = d.right(); top = d.top(); bottom = d.bottom()
# old_size = (right - left + bottom - top)/2
# center = np.array([right - (right - left) / 2.0, bottom - (bottom - top) / 2.0 + old_size*0.14])
# size = int(old_size*1.58)
# crop image
# src_pts = np.array([[center[0]-size/2, center[1]-size/2], [center[0] - size/2, center[1]+size/2], [center[0]+size/2, center[1]-size/2]])
# DST_PTS = np.array([[0,0], [0,self.resolution_inp - 1], [self.resolution_inp - 1, 0]])
# tform = estimate_transform('similarity', src_pts, DST_PTS)
# image = image/255.
# cropped_image = warp(image, tform.inverse, output_shape=(self.resolution_inp, self.resolution_inp))
x_5 = self.landmark_68_to_5(x)
M,tform = self.affine_align(image,x_5.reshape(5,2))
warped = cv.warpAffine(image, M, (256, 256), borderValue = 0.0)
# cv.imshow('img',warped)
# cv.waitKey(0)
# run our net
#st = time()
cropped_pos = self.net_forward(warped)
#print 'net time:', time() - st
# restore
cropped_vertices = np.reshape(cropped_pos, [-1, 3]).T
z = cropped_vertices[2,:].copy()/tform.params[0,0]
cropped_vertices[2,:] = 1
vertices = np.dot(np.linalg.inv(tform.params), cropped_vertices)
vertices = np.vstack((vertices[:2,:], z))
pos = np.reshape(vertices.T, [self.resolution_op, self.resolution_op, 3])
return pos
def get_landmarks(self, pos):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
kpt: 68 3D landmarks. shape = (68, 3).
'''
kpt = pos[self.uv_kpt_ind[1,:], self.uv_kpt_ind[0,:], :]
return kpt
def get_vertices(self, pos):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
vertices: the vertices(point cloud). shape = (num of points, 3). n is about 40K here.
'''
all_vertices = np.reshape(pos, [self.resolution_op**2, -1])
vertices = all_vertices[self.face_ind, :]
return vertices
def get_colors_from_texture(self, texture):
'''
Args:
texture: the texture map. shape = (256, 256, 3).
Returns:
colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here.
'''
all_colors = np.reshape(texture, [self.resolution_op**2, -1])
colors = all_colors[self.face_ind, :]
return colors
def get_colors(self, image, vertices):
'''
Args:
pos: the 3D position map. shape = (256, 256, 3).
Returns:
colors: the corresponding colors of vertices. shape = (num of points, 3). n is 45128 here.
'''
[h, w, _] = image.shape
vertices[:,0] = np.minimum(np.maximum(vertices[:,0], 0), w - 1) # x
vertices[:,1] = np.minimum(np.maximum(vertices[:,1], 0), h - 1) # y
ind = np.round(vertices).astype(np.int32)
colors = image[ind[:,1], ind[:,0], :] # n x 3
return colors
