def main(args):
if args.isShow or args.isTexture:
import cv2
from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box
# ---- init PRN
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU
prn = PRN(is_dlib=args.isDlib)
# ------------- load data
image_folder = args.inputDir
save_folder = args.outputDir
if not os.path.exists(save_folder):
os.mkdir(save_folder)
types = ('*.jpg', '*.png')
image_path_list = []
if os.path.isfile(image_folder):
image_path_list.append(image_folder)
for files in types:
image_path_list.extend(glob(os.path.join(image_folder, files)))
total_num = len(image_path_list)
for i, image_path in enumerate(image_path_list):
name = image_path.strip().split('/')[-1][:-4]
# read image
image = imread(image_path)
[h, w, _] = image.shape
# the core: regress position map
if args.isDlib:
max_size = max(image.shape[0], image.shape[1])
if max_size > 1000:
image = rescale(image, 1000. / max_size)
image = (image * 255).astype(np.uint8)
pos, crop_image = prn.process(image) # use dlib to detect face
else:
if image.shape[1] == image.shape[2]:
image = resize(image, (256, 256))
pos = prn.net_forward(
image / 255.) # input image has been cropped to 256x256
crop_image = None
else:
box = np.array([0, image.shape[1] - 1, 0, image.shape[0] - 1
]) # cropped with bounding box
pos, crop_image = prn.process(image, box)
image = image / 255.
if pos is None:
continue
if args.is3d or args.isMat or args.isPose or args.isShow:
# 3D vertices
vertices = prn.get_vertices(pos)
if args.isFront:
save_vertices = frontalize(vertices)
else:
save_vertices = vertices.copy()
save_vertices[:, 1] = h - 1 - save_vertices[:, 1]
if args.isImage and crop_image is not None:
imsave(os.path.join(save_folder, name + '_crop.jpg'), crop_image)
imsave(os.path.join(save_folder, name + '_orig.jpg'), image)
if args.is3d:
# corresponding colors
colors = prn.get_colors(image, vertices)
if args.isTexture:
texture = cv2.remap(image,
pos[:, :, :2].astype(np.float32),
None,
interpolation=cv2.INTER_NEAREST,
borderMode=cv2.BORDER_CONSTANT,
borderValue=(0))
if args.isMask:
vertices_vis = get_visibility(vertices, prn.triangles, h,
w)
uv_mask = get_uv_mask(vertices_vis, prn.triangles,
prn.uv_coords, h, w,
prn.resolution_op)
texture = texture * uv_mask[:, :, np.newaxis]
write_obj_with_texture(
os.path.join(save_folder,
name + '.obj'), save_vertices, colors,
prn.triangles, texture, prn.uv_coords / prn.resolution_op
) #save 3d face with texture(can open with meshlab)
else:
write_obj(os.path.join(save_folder,
name + '.obj'), save_vertices, colors,
prn.triangles) #save 3d face(can open with meshlab)
if args.isDepth:
depth_image = get_depth_image(vertices, prn.triangles, h, w, True)
depth = get_depth_image(vertices, prn.triangles, h, w)
imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image)
sio.savemat(os.path.join(save_folder, name + '_depth.mat'),
{'depth': depth})
if args.isMat:
sio.savemat(os.path.join(save_folder, name + '_mesh.mat'), {
'vertices': vertices,
'colors': colors,
'triangles': prn.triangles
})
if args.isKpt or args.isShow:
# get landmarks
kpt = prn.get_landmarks(pos)
np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt)
if args.isPose or args.isShow:
# estimate pose
camera_matrix, pose = estimate_pose(vertices)
np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose)
np.savetxt(os.path.join(save_folder, name + '_camera_matrix.txt'),
camera_matrix)
np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose)
if args.isShow:
# ---------- Plot
image_pose = plot_pose_box(image, camera_matrix, kpt)
cv2.imshow('sparse alignment', plot_kpt(image, kpt))
cv2.imshow('dense alignment', plot_vertices(image, vertices))
cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt))
if crop_image is not None:
cv2.imshow('crop', crop_image)
cv2.waitKey(0)
image = img
pos = prn.process(image) # use dlib to detect face
if pos is not None:
# -- Basic Applications
# get landmarks
kpt = prn.get_landmarks(pos)
# # 3D vertices
# vertices = prn.get_vertices(pos)
# # corresponding colors
# colors = prn.get_colors(image, vertices)
# cv2.imwrite(os.path.join(save_folder, '{}_nn_kpt.jpg'.format(ind)), plot_kpt(image, kpt))
out.write(plot_kpt(image, kpt))
print("capture frame count: {}".format(ind))
# -- save
# np.savetxt(os.path.join(save_folder, name + '.txt'), kpt)
# write_obj_with_colors(os.path.join(save_folder, name + '.obj'), vertices, prn.triangles, colors) #save 3d face(can open with meshlab)
# sio.savemat(os.path.join(save_folder, name + '_mesh.mat'), {'vertices': vertices, 'colors': colors, 'triangles': prn.triangles})
# Press Q on keyboard to stop recording
if cv2.waitKey(1) & 0xFF == ord('q'):
break
ind = ind + 1
if ind > 1000:
break
if args.isMat:
# sio.savemat(os.path.join(save_folder, name + '_mesh.mat'), {'vertices': vertices, 'colors': colors, 'triangles': prn.triangles})
pass
if args.isKpt or args.isShow:
# get landmarks
kpt = prn.get_landmarks(pos)
# np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt)
if args.isPose or args.isShow:
# estimate pose
camera_matrix, pose = estimate_pose(vertices)
# np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose)
# np.savetxt(os.path.join(save_folder, name + '_camera_matrix.txt'), camera_matrix)
# np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose)
if args.isShow:
# ---------- Plot
image_pose = plot_pose_box(image, camera_matrix, kpt)
cv2.imshow('sparse alignment', plot_kpt(image, kpt))
cv2.imshow('dense alignment', plot_vertices(image, vertices))
cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt))
#cv2.waitKey(0)
frame_count += 1
if cv2.waitKey(1) & 0xFF == ord('q'):
break
print('\nAverage FPS: ', frame_count / (time.time() - start))
def main(args):
if args.isShow or args.isTexture:
import cv2
from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box
# ---- init PRN
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU
prn = PRN(is_dlib=args.isDlib, is_faceboxes=args.isFaceBoxes)
# ---- load data
image_folder = args.inputDir
save_folder = args.outputDir
if not os.path.exists(save_folder):
os.mkdir(save_folder)
types = ('*.jpg', '*.png')
image_path_list = []
for files in types:
image_path_list.extend(glob(os.path.join(image_folder, files)))
total_num = len(image_path_list)
for i, image_path in enumerate(image_path_list):
name = image_path.strip().split('/')[-1][:-4]
# read image
image = imread(image_path)
[h, w, c] = image.shape
if c > 3: image = image[:, :, :3] # RGBA图中，去除A通道
# the core: regress position map
if args.isDlib:
max_size = max(image.shape[0], image.shape[1])
if max_size > 1000:
image = rescale(image, 1000. / max_size)
image = (image * 255).astype(np.uint8)
pos = prn.process(image) # use dlib to detect face
elif args.isFaceBoxes:
pos, cropped_img = prn.process(
image) # use faceboxes to detect face
else:
if image.shape[0] == image.shape[1]:
image = resize(image, (256, 256))
pos = prn.net_forward(
image / 255.) # input image has been cropped to 256x256
else:
box = np.array([0, image.shape[1] - 1, 0, image.shape[0] - 1
]) # cropped with bounding box
pos = prn.process(image, box)
image = image / 255.
if pos is None: continue
if args.is3d or args.isMat or args.isPose or args.isShow:
# 3D vertices
vertices = prn.get_vertices(pos)
if args.isFront:
save_vertices = frontalize(vertices)
else:
save_vertices = vertices.copy()
save_vertices[:, 1] = h - 1 - save_vertices[:, 1]
# 三维人脸旋转对齐方法
# if args.isImage:
# vertices = prn.get_vertices(pos)
# scale_init = 180 / (np.max(vertices[:, 1]) - np.min(vertices[:, 1]))
# colors = prn.get_colors(image, vertices)
# triangles = prn.triangles
# camera_matrix, pose = estimate_pose(vertices)
# yaw, pitch, roll = pos * ANGULAR
# vertices1 = vertices - np.mean(vertices, 0)[np.newaxis, :]
#
# obj = {'s': scale_init, 'angles': [-pitch, yaw, -roll + 180], 't': [0, 0, 0]}
# camera = {'eye':[0, 0, 256], 'proj_type':'perspective', 'at':[0, 0, 0],
# 'near': 1000, 'far':-100, 'fovy':30, 'up':[0,1,0]}
#
# image1 = transform_test(vertices1, obj, camera, triangles, colors, h=256, w=256) * 255
# image1 = image1.astype(np.uint8)
# imsave(os.path.join(save_folder, name + '.jpg'), image1)
if args.is3d:
# corresponding colors
colors = prn.get_colors(image, vertices)
if args.isTexture:
if args.texture_size != 256:
pos_interpolated = resize(
pos, (args.texture_size, args.texture_size),
preserve_range=True)
else:
pos_interpolated = pos.copy()
texture = cv2.remap(image,
pos_interpolated[:, :, :2].astype(
np.float32),
None,
interpolation=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_CONSTANT,
borderValue=(0))
if args.isMask:
vertices_vis = get_visibility(vertices, prn.triangles, h,
w)
uv_mask = get_uv_mask(vertices_vis, prn.triangles,
prn.uv_coords, h, w,
prn.resolution_op)
uv_mask = resize(uv_mask,
(args.texture_size, args.texture_size),
preserve_range=True)
texture = texture * uv_mask[:, :, np.newaxis]
write_obj_with_texture(
os.path.join(save_folder, name + '.obj'), save_vertices,
prn.triangles, texture, prn.uv_coords / prn.resolution_op
) #save 3d face with texture(can open with meshlab)
else:
write_obj_with_colors(
os.path.join(save_folder,
name + '.obj'), save_vertices, prn.triangles,
colors) #save 3d face(can open with meshlab)
if args.isDepth:
depth_image = get_depth_image(vertices, prn.triangles, h, w, True)
depth = get_depth_image(vertices, prn.triangles, h, w)
imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image)
sio.savemat(os.path.join(save_folder, name + '_depth.mat'),
{'depth': depth})
if args.isMat:
sio.savemat(os.path.join(save_folder, name + '_mesh.mat'), {
'vertices': vertices,
'colors': colors,
'triangles': prn.triangles
})
if args.isKpt:
# get landmarks
kpt = prn.get_landmarks(pos)
np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt)
if args.is2dKpt and args.is68Align:
ori_kpt = prn.get_landmarks_2d(pos)
dlib_aligner = DlibAlign()
dst_img = dlib_aligner.dlib_68_align(image, ori_kpt, 256, 0.5)
imsave(os.path.join(save_folder, name + '.jpg'), dst_img)
if args.isPose:
# estimate pose
camera_matrix, pose, rot = estimate_pose(vertices)
np.savetxt(os.path.join(save_folder, name + '_pose.txt'),
np.array(pose) * ANGULAR)
np.savetxt(os.path.join(save_folder, name + '_camera_matrix.txt'),
camera_matrix)
if args.isShow:
kpt = prn.get_landmarks(pos)
cv2.imshow('sparse alignment', plot_kpt(image, kpt))
# cv2.imshow('dense alignment', plot_vertices(image, vertices))
# cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt))
cv2.waitKey(1)
def main(args):
if args.isShow:
args.isOpencv = True
from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box
if args.isObj:
from utils.write import write_obj
if args.isPose:
from utils.estimate_pose import estimate_pose
# ---- init PRN
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU
prn = PRN(is_dlib=args.isDlib, is_opencv=args.isOpencv)
# ------------- load data
image_folder = args.inputFolder
save_folder = args.outputFolder
if not os.path.exists(save_folder):
os.mkdir(save_folder)
types = ('*.jpg', '*.png')
image_path_list = []
for files in types:
image_path_list.extend(glob(os.path.join(image_folder, files)))
total_num = len(image_path_list)
for i, image_path in enumerate(image_path_list):
name = image_path.strip().split('/')[-1][:-4]
# read image
image = imread(image_path)
# the core: regress position map
pos = prn.process(image) # use dlib to detect face
if args.isObj or args.isShow:
# 3D vertices
vertices = prn.get_vertices(pos)
# corresponding colors
colors = prn.get_colors(image, vertices)
write_obj(os.path.join(save_folder,
name + '.obj'), vertices, colors,
prn.triangles) #save 3d face(can open with meshlab)
if args.isKpt or args.isShow:
# get landmarks
kpt = prn.get_landmarks(pos)
np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt)
if args.isPose or args.isShow:
# estimate pose
camera_matrix, pose = estimate_pose(vertices)
np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose)
if args.isShow:
# ---------- Plot
image_pose = plot_pose_box(image, camera_matrix, kpt)
cv2.imshow('sparse alignment', plot_kpt(image, kpt))
cv2.imshow('dense alignment', plot_vertices(image, vertices))
cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt))
cv2.waitKey(0)
def main(args):
if args.isShow or args.isTexture:
import cv2
from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box
# ---- transform
transform_img = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(FLAGS["normalize_mean"], FLAGS["normalize_std"])
])
# ---- init PRN
prn = PRN(args.model)
# ------------- load data
image_folder = args.inputDir
save_folder = args.outputDir
if not os.path.exists(save_folder):
os.mkdir(save_folder)
types = ('*.jpg', '*.png')
image_path_list = []
for files in types:
image_path_list.extend(glob(os.path.join(image_folder, files)))
total_num = len(image_path_list)
print("#" * 25)
print("[PRNet Inference] {} picture were under processing~".format(
total_num))
print("#" * 25)
for i, image_path in enumerate(image_path_list):
name = image_path.strip().split('/')[-1][:-4]
# read image
image = cv2.imread(image_path)
[h, w, c] = image.shape
# the core: regress position map
image = cv2.resize(image, (256, 256))
image_t = transform_img(image)
image_t = image_t.unsqueeze(0)
pos = prn.net_forward(
image_t) # input image has been cropped to 256x256
out = pos.cpu().detach().numpy()
pos = np.squeeze(out)
cropped_pos = pos * 255
pos = cropped_pos.transpose(1, 2, 0)
if pos is None:
continue
if args.is3d or args.isMat or args.isPose or args.isShow:
# 3D vertices
vertices = prn.get_vertices(pos)
if args.isFront:
save_vertices = frontalize(vertices)
else:
save_vertices = vertices.copy()
save_vertices[:, 1] = h - 1 - save_vertices[:, 1]
if args.isImage:
cv2.imwrite(os.path.join(save_folder, name + '.jpg'), image)
if args.is3d:
# corresponding colors
colors = prn.get_colors(image, vertices)
if args.isTexture:
if args.texture_size != 256:
pos_interpolated = cv2.resize(
pos, (args.texture_size, args.texture_size),
preserve_range=True)
else:
pos_interpolated = pos.copy()
texture = cv2.remap(image,
pos_interpolated[:, :, :2].astype(
np.float32),
None,
interpolation=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_CONSTANT,
borderValue=(0))
if args.isMask:
vertices_vis = get_visibility(vertices, prn.triangles, h,
w)
uv_mask = get_uv_mask(vertices_vis, prn.triangles,
prn.uv_coords, h, w,
prn.resolution_op)
uv_mask = cv2.resize(
uv_mask, (args.texture_size, args.texture_size),
preserve_range=True)
texture = texture * uv_mask[:, :, np.newaxis]
write_obj_with_texture(
os.path.join(save_folder, name + '.obj'), save_vertices,
prn.triangles, texture, prn.uv_coords / prn.resolution_op
) # save 3d face with texture(can open with meshlab)
else:
write_obj_with_colors(
os.path.join(save_folder,
name + '.obj'), save_vertices, prn.triangles,
colors) # save 3d face(can open with meshlab)
# if args.isDepth:
# depth_image = get_depth_image(vertices, prn.triangles, h, w, True)
# depth = get_depth_image(vertices, prn.triangles, h, w)
# cv2.imwrite(os.path.join(save_folder, name + '_depth.jpg'), depth_image)
# sio.savemat(os.path.join(save_folder, name + '_depth.mat'), {'depth': depth})
if args.isKpt or args.isShow:
# get landmarks
kpt = prn.get_landmarks(pos)
np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt)
if args.isPose or args.isShow:
# estimate pose
camera_matrix, pose = estimate_pose(vertices)
np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose)
np.savetxt(os.path.join(save_folder, name + '_camera_matrix.txt'),
camera_matrix)
np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose)
if args.isShow:
# ---------- Plot
image_pose = plot_pose_box(image, camera_matrix, kpt)
cv2.imshow('sparse alignment', plot_kpt(image, kpt))
cv2.imshow('dense alignment', plot_vertices(image, vertices))
cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt))
cv2.waitKey(0)
def run_one_image(uv_kpt_ind,
face_ind,
triangles,
s_uv_coords,
image_path,
npy_path,
save_folder,
name,
uv_h=256,
uv_w=256,
image_h=256,
image_w=256):
# 1. load image
cropped_image = imread(image_path) / 255.
print('input image is ok!')
# 2. load uv position map
pos = np.load(npy_path)
print('uv map is ok!')
# 3. deal uv map
# run model to get uv_map
max_pos = image_h
pos = pos * max_pos
# 4. get useful vertices
vertices = get_vertices(pos, face_ind, uv_h)
save_vertices = vertices.copy()
save_vertices[:, 1] = image_h - 1 - save_vertices[:, 1]
# 5. get colors
colors = get_colors(cropped_image, vertices)
write_obj_with_colors(os.path.join(save_folder, name + '_c.obj'),
save_vertices, triangles, colors)
print('color 3d face is ok!')
# 6. get texture
pos_interpolated = pos.copy()
texture = cv2.remap(cropped_image,
pos_interpolated[:, :, :2].astype(np.float32),
None,
interpolation=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_CONSTANT,
borderValue=(0))
vertices_vis = get_visibility(vertices, triangles, image_h, image_w)
uv_mask = get_uv_mask(vertices_vis, triangles, s_uv_coords, image_h,
image_w, uv_h)
uv_mask = resize(uv_mask, (256, 256), preserve_range=True)
texture = texture * uv_mask[:, :, np.newaxis]
write_obj_with_texture(os.path.join(save_folder,
name + '.obj'), save_vertices,
triangles, texture, s_uv_coords / uv_h)
print('texture 3d face is ok!')
# 7. get landmarks
t_image = (cropped_image * 255.).astype(np.uint8)
kpt = get_landmarks(pos, uv_kpt_ind)
kpt_origin = plot_kpt(cropped_image, kpt).astype(np.uint8)
kpt_gray = cv2.cvtColor(kpt_origin, cv2.COLOR_RGB2GRAY)
ret, kpt_mask = cv2.threshold(kpt_gray, 127, 255, cv2.THRESH_BINARY)
kpt_mask = cv2.bitwise_not(kpt_mask)
kpt_and = cv2.bitwise_and(t_image, t_image, mask=kpt_mask)
kpt_image = cv2.add(kpt_and, kpt_origin)
imsave(os.path.join(save_folder, name + '_kpt.jpg'), kpt_image / 255.)
print('kpt image is ok!')
def main(args):
if args.isShow or args.isTexture:
import cv2
from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box
# ---- init PRN
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU
prn = PRN(is_dlib = args.isDlib)
# ------------- load data
image_folder = args.inputDir
save_folder = args.outputDir
if not os.path.exists(save_folder):
os.mkdir(save_folder)
# types = ('*.jpg', '*.png')
# image_path_list= []
# for files in types:
# image_path_list.extend(glob(os.path.join(image_folder, files)))
# total_num = len(image_path_list)
for dir, dirs, files in sorted(os.walk(image_folder)):
for file in files:
image_path = os.path.join(dir, file)
dir = dir.replace("\\", "/")
new_dir = dir.replace(image_folder, save_folder)
if not os.path.isdir(new_dir):
os.mkdir(new_dir)
name = image_path.replace(image_folder, save_folder)
print('data path:', name)
# read image
image = imread(image_path)
[h, w, c] = image.shape
if c>3:
image = image[:,:,:3]
# the core: regress position map
if args.isDlib:
max_size = max(image.shape[0], image.shape[1])
if max_size> 1000:
image = rescale(image, 1000./max_size)
image = (image*255).astype(np.uint8)
pos = prn.process(image) # use dlib to detect face
else:
# if image.shape[0] == image.shape[1]:
# image = resize(image, (256,256))
# pos = prn.net_forward(image/255.) # input image has been cropped to 256x256
# else:
box = np.array([0, image.shape[1]-1, 0, image.shape[0]-1]) # cropped with bounding box
pos = prn.process(image, box)
image = image/255.
if pos is None:
continue
if args.is3d or args.isMat or args.isPose or args.isShow:
# 3D vertices
vertices = prn.get_vertices(pos)
if args.isFront:
save_vertices = frontalize(vertices)
else:
save_vertices = vertices.copy()
save_vertices[:,1] = h - 1 - save_vertices[:,1]
if args.isImage:
imsave(name, image)
if args.is3d:
# corresponding colors
colors = prn.get_colors(image, vertices)
if args.isTexture:
if args.texture_size != 256:
pos_interpolated = resize(pos, (args.texture_size, args.texture_size), preserve_range = True)
else:
pos_interpolated = pos.copy()
texture = cv2.remap(image, pos_interpolated[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT,borderValue=(0))
if args.isMask:
vertices_vis = get_visibility(vertices, prn.triangles, h, w)
uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op)
uv_mask = resize(uv_mask, (args.texture_size, args.texture_size), preserve_range = True)
texture = texture*uv_mask[:,:,np.newaxis]
write_obj_with_texture(name.replace('.jpg', '.obj'), save_vertices, prn.triangles, texture, prn.uv_coords/prn.resolution_op)#save 3d face with texture(can open with meshlab)
else:
write_obj_with_colors(name.replace('.jpg', '.obj'), save_vertices, prn.triangles, colors) #save 3d face(can open with meshlab)
filepath = name.replace('.jpg', '.obj')
filepath = filepath.replace("\\", "/")
print('filepath:', filepath)
new_dir = dir.replace(args.inputDir, args.renderDir)
# print(new_dir + '/' + file)
if not os.path.isdir(new_dir):
os.mkdir(new_dir)
color_image1, _ = render_scene(filepath, 4.0, 0.0, 3.0)
color_image2, _ = render_scene(filepath, 4.0, np.pi / 18.0, 3.0)
color_image3, _ = render_scene(filepath, 4.0, np.pi / 9.0, 3.0)
if color_image1 is None or color_image2 is None:
continue
new_path = filepath.replace(args.outputDir, args.renderDir)
# print('new_path:', new_path)
save_image(new_path, '_40_', color_image1)
save_image(new_path, '_50_', color_image2)
save_image(new_path, '_60_', color_image3)
os.remove(name.replace('.jpg', '.obj'))
if args.isDepth:
depth_image = get_depth_image(vertices, prn.triangles, h, w, True)
depth = get_depth_image(vertices, prn.triangles, h, w)
imsave(os.path.join(name.replace('.jpg', '_depth.jpg')), depth_image)
sio.savemat(name.replace('.jpg', '_depth.mat'), {'depth': depth})
if args.isMat:
sio.savemat(name.replace('.jpg', '_mesh.mat'),
{'vertices': vertices, 'colors': colors, 'triangles': prn.triangles})
if args.isKpt or args.isShow:
# get landmarks
kpt = prn.get_landmarks(pos)
np.savetxt(name.replace('.jpg', '_kpt.txt'), kpt)
if args.isPose or args.isShow:
# estimate pose
camera_matrix, pose = estimate_pose(vertices)
np.savetxt(name.replace('.jpg', '_pose.txt'), pose)
np.savetxt(name.replace('.jpg', '_camera_matrix.txt'), camera_matrix)
np.savetxt(name.replace('.jpg', '_pose.txt'), pose)
if args.isShow:
# ---------- Plot
image_pose = plot_pose_box(image, camera_matrix, kpt)
cv2.imshow('sparse alignment', plot_kpt(image, kpt))
cv2.imshow('dense alignment', plot_vertices(image, vertices))
cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt))
cv2.waitKey(0)
def main(args):
if args.isShow or args.isTexture:
import cv2
from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box
# ---- init PRN
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU
prn = PRN(is_dlib = args.isDlib)
# ------------- load data
image_folder = args.inputDir
save_folder = args.outputDir
if not os.path.exists(save_folder):
os.mkdir(save_folder)
types = ('*.jpg', '*.png')
image_path_list= []
for files in types:
image_path_list.extend(glob(os.path.join(image_folder, files)))
total_num = len(image_path_list)
for i, image_path in enumerate(image_path_list):
name = image_path.strip().split('/')[-1][:-4]
# read image
image = imread(image_path)
[h, w, _] = image.shape
# the core: regress position map
if args.isDlib:
max_size = max(image.shape[0], image.shape[1])
if max_size> 1000:
image = rescale(image, 1000./max_size)
pos = prn.process(image) # use dlib to detect face
else:
if image.shape[1] == image.shape[2]:
image = resize(image, (256,256))
pos = prn.net_forward(image/255.) # input image has been cropped to 256x256
else:
box = np.array([0, image.shape[1]-1, 0, image.shape[0]-1]) # cropped with bounding box
pos = prn.process(image, box)
image = image/255.
if pos is None:
continue
if args.is3d or args.isMat or args.isPose or args.isShow:
# 3D vertices
vertices = prn.get_vertices(pos)
if args.isFront:
save_vertices = frontalize(vertices)
else:
save_vertices = vertices
if args.isImage:
imsave(os.path.join(save_folder, name + '.jpg'), image)
if args.is3d:
# corresponding colors
colors = prn.get_colors(image, vertices)
if args.isTexture:
texture = cv2.remap(image, pos[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT,borderValue=(0))
if args.isMask:
vertices_vis = get_visibility(vertices, prn.triangles, h, w)
uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op)
texture = texture*uv_mask[:,:,np.newaxis]
write_obj_with_texture(os.path.join(save_folder, name + '.obj'), save_vertices, colors, prn.triangles, texture, prn.uv_coords/prn.resolution_op)#save 3d face with texture(can open with meshlab)
else:
write_obj(os.path.join(save_folder, name + '.obj'), save_vertices, colors, prn.triangles) #save 3d face(can open with meshlab)
if args.isDepth:
depth_image = get_depth_image(vertices, prn.triangles, h, w)
imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image)
if args.isMat:
sio.savemat(os.path.join(save_folder, name + '_mesh.mat'), {'vertices': save_vertices, 'colors': colors, 'triangles': prn.triangles})
if args.isKpt or args.isShow:
# get landmarks
kpt = prn.get_landmarks(pos)
np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt)
if args.isPose or args.isShow:
# estimate pose
camera_matrix, pose = estimate_pose(vertices)
np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose)
if args.isShow:
# ---------- Plot
image_pose = plot_pose_box(image, camera_matrix, kpt)
cv2.imshow('sparse alignment', plot_kpt(image, kpt))
cv2.imshow('dense alignment', plot_vertices(image, vertices))
cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt))
cv2.waitKey(0)
def run_one_image(image_path,
uv_kpt_ind,
face_ind,
triangles,
s_uv_coords,
pnet,
rnet,
onet,
x,
y,
Tsess,
minsize=30,
threshold=[0.6, 0.7, 0.7],
factor=0.709,
best_score=0.7,
uv_h=256,
uv_w=256,
image_h=256,
image_w=256):
input_image = cv2.imread(image_path, 1)
output_image = input_image.copy()
boxes, pnts = face_detect.detect_face(input_image, minsize, pnet, rnet,
onet, threshold, factor)
faces = process_bbox(boxes, input_image.shape)
for idx, (x0, y1, x1, y0, conf_score) in enumerate(faces):
if conf_score > best_score:
det_face = input_image[int(x0):int(x1), int(y0):int(y1), :]
face_shape = (int(y1) - int(y0), int(x1) - int(x0))
det_face = cv2.resize(det_face, (256, 256)) / 255.
pos = Tsess.run(y, feed_dict={x: det_face[np.newaxis, :, :, :]})
pos = np.squeeze(pos)
max_pos = image_h
pos = pos * max_pos
vertices = get_vertices(pos, face_ind, uv_h)
from utils.write import write_obj_with_colors
save_vertices = vertices.copy()
save_vertices[:, 1] = image_h - 1 - save_vertices[:, 1]
colors = get_colors(det_face, vertices)
write_obj_with_colors(os.path.join('images', 'test' + '_c.obj'),
save_vertices, triangles, colors)
t_image = (det_face * 255.).astype(np.uint8)
kpt = get_landmarks(pos, uv_kpt_ind)
kpt_origin = plot_kpt(det_face, kpt).astype(np.uint8)
kpt_gray = cv2.cvtColor(kpt_origin, cv2.COLOR_RGB2GRAY)
ret, kpt_mask = cv2.threshold(kpt_gray, 127, 255,
cv2.THRESH_BINARY)
kpt_mask = cv2.bitwise_not(kpt_mask)
kpt_and = cv2.bitwise_and(t_image, t_image, mask=kpt_mask)
kpt_image = cv2.add(kpt_and, kpt_origin)
imsave(os.path.join('images', 'test' + '_kpt.jpg'),
kpt_image / 255.)
t_image = (det_face * 255.).astype(np.uint8)
ver_origin = plot_vertices(det_face, vertices).astype(np.uint8)
ver_gray = cv2.cvtColor(ver_origin, cv2.COLOR_RGB2GRAY)
ret, ver_mask = cv2.threshold(ver_gray, 127, 255,
cv2.THRESH_BINARY)
ver_mask = cv2.bitwise_not(ver_mask)
ver_and = cv2.bitwise_and(t_image, t_image, mask=ver_mask)
ver_image = cv2.add(ver_and, ver_origin)
imsave(os.path.join('images', 'test' + '_ver.jpg'),
ver_image / 255.)
resize_ver_image = cv2.resize(ver_image, face_shape)
output_image[int(x0):int(x1), int(y0):int(y1)] = resize_ver_image
return output_image / 255.
def main(args):
if args.isShow or args.isTexture or args.isCamera:
import cv2
from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box
# ---- init PRN
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU
prn = PRN(is_dlib=args.isDlib)
# ------------- load data
image_folder = args.inputDir
save_folder = args.outputDir
if not os.path.exists(save_folder):
os.mkdir(save_folder)
types = ('*.jpg', '*.png')
image_path_list = []
for files in types:
image_path_list.extend(glob(os.path.join(image_folder, files)))
total_num = len(image_path_list)
if args.isCamera:
# Create a VideoCapture object and read from input file
# If the input is the camera, pass 0 instead of the video file name
cap = cv2.VideoCapture(0)
# Check if camera opened successfully
if (cap.isOpened() == False):
print("Error opening video stream or file")
# Read until video is completed
while (cap.isOpened()):
# Capture frame-by-frame
ret, frame = cap.read()
if ret == True:
if args.isDlib:
max_size = max(frame.shape[0], frame.shape[1])
if max_size > 1000:
frame = rescale(frame, 1000. / max_size)
frame = (frame * 255).astype(np.uint8)
pos = prn.process(frame) # use dlib to detect face
else:
if frame.shape[0] == frame.shape[1]:
frame = resize(frame, (256, 256))
pos = prn.net_forward(
frame /
255.) # input frame has been cropped to 256x256
else:
box = np.array(
[0, frame.shape[1] - 1, 0,
frame.shape[0] - 1]) # cropped with bounding box
pos = prn.process(frame, box)
# Normalizing the frame and skiping if there was no one in the frame
frame = frame / 255.
if pos is None:
continue
# Get landmarks in frame
kpt = prn.get_landmarks(pos)
# Display the resulting frame
cv2.imshow('sparse alignment', plot_kpt(frame, kpt))
# Press Q on keyboard to exit
if cv2.waitKey(25) & 0xFF == ord('q'):
break
# Break the loop
else:
break
# When everything done, release the video capture object
cap.release()
# Closes all the frames
cv2.destroyAllWindows()
else:
for i, image_path in enumerate(image_path_list):
name = image_path.strip().split('/')[-1][:-4]
# read image
image = imread(image_path)
[h, w, c] = image.shape
if c > 3:
image = image[:, :, :3]
# the core: regress position map
if args.isDlib:
max_size = max(image.shape[0], image.shape[1])
if max_size > 1000:
image = rescale(image, 1000. / max_size)
image = (image * 255).astype(np.uint8)
pos = prn.process(image) # use dlib to detect face
else:
if image.shape[0] == image.shape[1]:
image = resize(image, (256, 256))
pos = prn.net_forward(
image /
255.) # input image has been cropped to 256x256
else:
box = np.array(
[0, image.shape[1] - 1, 0,
image.shape[0] - 1]) # cropped with bounding box
pos = prn.process(image, box)
image = image / 255.
if pos is None:
continue
if args.is3d or args.isMat or args.isPose or args.isShow:
# 3D vertices
vertices = prn.get_vertices(pos)
if args.isFront:
save_vertices = frontalize(vertices)
else:
save_vertices = vertices.copy()
save_vertices[:, 1] = h - 1 - save_vertices[:, 1]
if args.isImage:
imsave(os.path.join(save_folder, name + '.jpg'), image)
if args.is3d:
# corresponding colors
colors = prn.get_colors(image, vertices)
if args.isTexture:
if args.texture_size != 256:
pos_interpolated = resize(
pos, (args.texture_size, args.texture_size),
preserve_range=True)
else:
pos_interpolated = pos.copy()
texture = cv2.remap(image,
pos_interpolated[:, :, :2].astype(
np.float32),
None,
interpolation=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_CONSTANT,
borderValue=(0))
if args.isMask:
vertices_vis = get_visibility(vertices, prn.triangles,
h, w)
uv_mask = get_uv_mask(vertices_vis, prn.triangles,
prn.uv_coords, h, w,
prn.resolution_op)
uv_mask = resize(
uv_mask, (args.texture_size, args.texture_size),
preserve_range=True)
texture = texture * uv_mask[:, :, np.newaxis]
write_obj_with_texture(
os.path.join(save_folder, name + '.obj'),
save_vertices, prn.triangles, texture,
prn.uv_coords / prn.resolution_op
) #save 3d face with texture(can open with meshlab)
else:
write_obj_with_colors(
os.path.join(save_folder, name + '.obj'),
save_vertices, prn.triangles,
colors) #save 3d face(can open with meshlab)
if args.isDepth:
depth_image = get_depth_image(vertices, prn.triangles, h, w,
True)
depth = get_depth_image(vertices, prn.triangles, h, w)
imsave(os.path.join(save_folder, name + '_depth.jpg'),
depth_image)
sio.savemat(os.path.join(save_folder, name + '_depth.mat'),
{'depth': depth})
if args.isMat:
sio.savemat(
os.path.join(save_folder, name + '_mesh.mat'), {
'vertices': vertices,
'colors': colors,
'triangles': prn.triangles
})
if args.isKpt or args.isShow:
# get landmarks
kpt = prn.get_landmarks(pos)
np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt)
if args.isPose or args.isShow:
# estimate pose
camera_matrix, pose = estimate_pose(vertices)
np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose)
np.savetxt(
os.path.join(save_folder, name + '_camera_matrix.txt'),
camera_matrix)
np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose)
if args.isShow:
# ---------- Plot
image_pose = plot_pose_box(image, camera_matrix, kpt)
cv2.imshow(
'sparse alignment',
cv2.cvtColor(np.float32(plot_kpt(image, kpt)),
cv2.COLOR_RGB2BGR))
cv2.imshow(
'dense alignment',
cv2.cvtColor(np.float32(plot_vertices(image, vertices)),
cv2.COLOR_RGB2BGR))
cv2.imshow(
'pose',
cv2.cvtColor(
np.float32(plot_pose_box(image, camera_matrix, kpt)),
cv2.COLOR_RGB2BGR))
cv2.waitKey(0)
def getFacialLandmarks(isDlib, img_, numFaces=1):
img = copy.deepcopy(img_)
# use dlib or PrNetfor prediction of facial landmarks
if isDlib == "True":
# load shape predictor model
model_path = 'Code/dlib_model/shape_predictor_68_face_landmarks.dat'
# load the detector and the predictor.
# predictor accepts pre-trained model as input
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(model_path)
img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
rects = detector(img_gray, 1)
# store landmark locations of both faces
landmarkCoordAll = []
# iterate through the points in both faces
for r, rect in enumerate(rects):
landmarks = predictor(img_gray, rect)
# reshape landmarks to (68X2)
landmarkCoord = np.zeros((68, 2), dtype='int')
for i in range(68):
landmarkCoord[i] = (landmarks.part(i).x, landmarks.part(i).y)
landmarkCoordAll.append(landmarkCoord)
# draw bounding box on face
cv2.rectangle(img, (rect.left(), rect.top()), (rect.right(), rect.bottom()), (0, 255, 255), 0)
# draw facial landmarks
img_ = drawFacialLandmarks(img, landmarkCoord)
if isDlib == "False":
# prn uses dlib for face detection and its own trained model for prediction of facial landmarks
prn = PRN(is_dlib = True, prefix='Code/prnet/')
[h, w, c] = img.shape
if c>3:
img = img[:,:,:3]
if img.shape[0] == img.shape[1]:
img = resize(img, (256,256))
pos = prn.net_forward(img/255.) # input image has been cropped to 256x256
else:
posList = []
for i in range(numFaces):
pos = prn.process(img, i)
posList.append(pos)
landmarkCoordAll = []
for i, pos in enumerate(posList):
if pos is None:
return img_, landmarkCoordAll
# get landmark points of face
landmarkCoord = prn.get_landmarks(pos)
img_ = plot_kpt(img_, landmarkCoord)
landmarkCoord = landmarkCoord[:, 0:2]
landmarkCoordAll.append(landmarkCoord)
return img_, landmarkCoordAll
