def main(args): image_folder = args.inputDir save_folder = args.outputDir if not os.path.exists(save_folder): os.mkdir(save_folder) types = ('*.jpg', '*.png') image_path_list = [] for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) resolution = args.resolution face_ind = np.loadtxt('./Data/uv-data/face_ind.txt').astype( np.int32) # get valid vertices in the pos map uv_kpt_ind = np.loadtxt('./Data/uv-data/uv_kpt_ind.txt').astype( np.int32) # 2 x 68 get kpt for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] # read image image = imread(image_path) [h, w, _] = image.shape tform = np.loadtxt(os.path.join(save_folder, 'tform.txt'), delimiter=',') pos = get_pos(image, tform, resolution) if args.isPose: # 3D vertices vertices = get_vertices(pos, resolution, face_ind) # get valid points if args.isFront: save_vertices = frontalize( vertices) # affine matrix to frontalize else: save_vertices = vertices.copy() # 68 * 3 save_vertices[:, 1] = h - 1 - save_vertices[:, 1] # control the scope if args.isKpt or args.isShow: # get landmarks kpt = get_landmarks(pos, uv_kpt_ind) np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) if args.isPose: # camera matrix # estimate pose camera_matrix, pose = estimate_pose(vertices) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) np.savetxt(os.path.join(save_folder, name + '_camera_matrix.txt'), camera_matrix)
def generate_depth_image(inputDir, prn): depth_images = [] # os.environ['CUDA_VISIBLE_DEVICES'] = "-1" # only cpu for now # prn = PRN(is_dlib = True) types = ('*.jpg', '*.png') image_path_list= [] for files in types: image_path_list.extend(glob(os.path.join(inputDir, files))) total_num = len(image_path_list) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] # read image image = imread(image_path) [h, w, c] = image.shape if c>3: image = image[:,:,:3] # the core: regress position map max_size = max(image.shape[0], image.shape[1]) if max_size> 1000: image = rescale(image, 1000./max_size) image = (image*255).astype(np.uint8) pos = prn.process(image) # use dlib to detect face image = image/255. if pos is None: continue vertices = prn.get_vertices(pos) save_vertices = frontalize(vertices) save_vertices[:,1] = h - 1 - save_vertices[:,1] depth = get_depth_image(vertices, prn.triangles, h, w) # save_depth_image(depth) depth_images.append(depth) return depth exit(1) print(len(depth_images)) return depth_images[0]
def main(args): if args.isShow or args.isTexture: import cv2 from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib=args.isDlib) # ------------- load data image_folder = args.inputDir save_folder = args.outputDir if not os.path.exists(save_folder): os.mkdir(save_folder) types = ('*.jpg', '*.png') image_path_list = [] if os.path.isfile(image_folder): image_path_list.append(image_folder) for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] # read image image = imread(image_path) [h, w, _] = image.shape # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size > 1000: image = rescale(image, 1000. / max_size) image = (image * 255).astype(np.uint8) pos, crop_image = prn.process(image) # use dlib to detect face else: if image.shape[1] == image.shape[2]: image = resize(image, (256, 256)) pos = prn.net_forward( image / 255.) # input image has been cropped to 256x256 crop_image = None else: box = np.array([0, image.shape[1] - 1, 0, image.shape[0] - 1 ]) # cropped with bounding box pos, crop_image = prn.process(image, box) image = image / 255. if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy() save_vertices[:, 1] = h - 1 - save_vertices[:, 1] if args.isImage and crop_image is not None: imsave(os.path.join(save_folder, name + '_crop.jpg'), crop_image) imsave(os.path.join(save_folder, name + '_orig.jpg'), image) if args.is3d: # corresponding colors colors = prn.get_colors(image, vertices) if args.isTexture: texture = cv2.remap(image, pos[:, :, :2].astype(np.float32), None, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) if args.isMask: vertices_vis = get_visibility(vertices, prn.triangles, h, w) uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) texture = texture * uv_mask[:, :, np.newaxis] write_obj_with_texture( os.path.join(save_folder, name + '.obj'), save_vertices, colors, prn.triangles, texture, prn.uv_coords / prn.resolution_op ) #save 3d face with texture(can open with meshlab) else: write_obj(os.path.join(save_folder, name + '.obj'), save_vertices, colors, prn.triangles) #save 3d face(can open with meshlab) if args.isDepth: depth_image = get_depth_image(vertices, prn.triangles, h, w, True) depth = get_depth_image(vertices, prn.triangles, h, w) imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image) sio.savemat(os.path.join(save_folder, name + '_depth.mat'), {'depth': depth}) if args.isMat: sio.savemat(os.path.join(save_folder, name + '_mesh.mat'), { 'vertices': vertices, 'colors': colors, 'triangles': prn.triangles }) if args.isKpt or args.isShow: # get landmarks kpt = prn.get_landmarks(pos) np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) if args.isPose or args.isShow: # estimate pose camera_matrix, pose = estimate_pose(vertices) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) np.savetxt(os.path.join(save_folder, name + '_camera_matrix.txt'), camera_matrix) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) if args.isShow: # ---------- Plot image_pose = plot_pose_box(image, camera_matrix, kpt) cv2.imshow('sparse alignment', plot_kpt(image, kpt)) cv2.imshow('dense alignment', plot_vertices(image, vertices)) cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt)) if crop_image is not None: cv2.imshow('crop', crop_image) cv2.waitKey(0)
if image.shape[0] == image.shape[1]: image = resize(image, (256,256)) pos = prn.net_forward(image/255.) # input image has been cropped to 256x256 else: box = np.array([0, image.shape[1]-1, 0, image.shape[0]-1]) # cropped with bounding box pos = prn.process(image, box) image = image/255. if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy() save_vertices[:,1] = h - 1 - save_vertices[:,1] if args.isImage: # imsave(os.path.join(save_folder, name + '.jpg'), image) pass if args.is3d: # corresponding colors colors = prn.get_colors(image, vertices) if args.isTexture: if args.texture_size != 256: pos_interpolated = resize(pos, (args.texture_size, args.texture_size), preserve_range = True)
def main(): # OpenCV #cap = cv2.VideoCapture(args.video_source) cap = cv2.VideoCapture('b.mov') fps = video.FPS().start() # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib=args.isDlib) #while True: while cap.isOpened(): ret, frame = cap.read() # resize image and detect face frame_resize = cv2.resize(frame, None, fx=1 / DOWNSAMPLE_RATIO, fy=1 / DOWNSAMPLE_RATIO) # read image image = frame_resize image = resize(image) [h, w, c] = image.shape if c > 3: image = image[:, :, :3] # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size > 1000: image = rescale(image, 1000. / max_size) image = (image * 255).astype(np.uint8) st = time() pos = prn.process(image) # use dlib to detect face print('process', time() - st) else: if image.shape[0] == image.shape[1]: image = resize(image, (256, 256)) pos = prn.net_forward( image / 255.) # input image has been cropped to 256x256 else: box = np.array([0, image.shape[1] - 1, 0, image.shape[0] - 1 ]) # cropped with bounding box pos = prn.process(image, box) image = image / 255. if pos is None: cv2.imshow('a', frame_resize) fps.update() if cv2.waitKey(1) & 0xFF == ord('q'): break continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy() save_vertices[:, 1] = h - 1 - save_vertices[:, 1] #colors = prn.get_colors(image, vertices) #write_obj_with_colors(os.path.join('', 'webcam' + '.obj'), save_vertices, prn.triangles, colors) #if args.is3d: # # corresponding colors # colors = prn.get_colors(image, vertices) # # if args.isTexture: # if args.texture_size != 256: # pos_interpolated = resize(pos, (args.texture_size, args.texture_size), preserve_range = True) # else: # pos_interpolated = pos.copy() # texture = cv2.remap(image, pos_interpolated[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT,borderValue=(0)) # if args.isMask: # vertices_vis = get_visibility(vertices, prn.triangles, h, w) # uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) # uv_mask = resize(uv_mask, (args.texture_size, args.texture_size), preserve_range = True) # texture = texture*uv_mask[:,:,np.newaxis] # #write_obj_with_texture(os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, texture, prn.uv_coords/prn.resolution_op)#save 3d face with texture(can open with meshlab) # else: # True # #write_obj_with_colors(os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, colors) #save 3d face(can open with meshlab) # #if args.isDepth: # depth_image = get_depth_image(vertices, prn.triangles, h, w, True) # depth = get_depth_image(vertices, prn.triangles, h, w) # #imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image) # #sio.savemat(os.path.join(save_folder, name + '_depth.mat'), {'depth':depth}) # #if args.isKpt or args.isShow: # # get landmarks # kpt = prn.get_landmarks(pos) # #np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) # #if args.isPose or args.isShow: # # estimate pose # camera_matrix, pose = estimate_pose(vertices) #write_obj_with_colors(os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, colors) rendering_cc = mesh.render.render_grid(save_vertices, prn.triangles, 900, 900) a = np.transpose(rendering_cc, axes=[1, 0, 2]) dim = rendering_cc.shape[0] i_t = np.ones([dim, dim, 3], dtype=np.float32) for i in range(dim): i_t[i] = a[dim - 1 - i] i_t = i_t / 255 #imsave('webcam.png', i_t) #kpt = prn.get_landmarks(pos) #cv2.imshow('frame', image) #cv2.imshow('a',i_t/255) #cv2.imshow('sparse alignment', np.concatenate([image, i_t], axis=1)) cv2.imshow('sparse alignment', i_t) cv2.imshow('vedio', image) #cv2.imshow('sparse alignment', np.concatenate([plot_kpt(image, kpt), i_t], axis=1)) #cv2.imshow('dense alignment', plot_vertices(image, vertices)) #cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt)) fps.update() if cv2.waitKey(1) & 0xFF == ord('q'): break fps.stop() print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed())) print('[INFO] approx. FPS: {:.2f}'.format(fps.fps())) cap.release() cv2.destroyAllWindows()
def main(args): if args.isShow or args.isTexture: import cv2 from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib=args.isDlib, is_faceboxes=args.isFaceBoxes) # ---- load data image_folder = args.inputDir save_folder = args.outputDir if not os.path.exists(save_folder): os.mkdir(save_folder) types = ('*.jpg', '*.png') image_path_list = [] for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] # read image image = imread(image_path) [h, w, c] = image.shape if c > 3: image = image[:, :, :3] # RGBA图中,去除A通道 # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size > 1000: image = rescale(image, 1000. / max_size) image = (image * 255).astype(np.uint8) pos = prn.process(image) # use dlib to detect face elif args.isFaceBoxes: pos, cropped_img = prn.process( image) # use faceboxes to detect face else: if image.shape[0] == image.shape[1]: image = resize(image, (256, 256)) pos = prn.net_forward( image / 255.) # input image has been cropped to 256x256 else: box = np.array([0, image.shape[1] - 1, 0, image.shape[0] - 1 ]) # cropped with bounding box pos = prn.process(image, box) image = image / 255. if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy() save_vertices[:, 1] = h - 1 - save_vertices[:, 1] # 三维人脸旋转对齐方法 # if args.isImage: # vertices = prn.get_vertices(pos) # scale_init = 180 / (np.max(vertices[:, 1]) - np.min(vertices[:, 1])) # colors = prn.get_colors(image, vertices) # triangles = prn.triangles # camera_matrix, pose = estimate_pose(vertices) # yaw, pitch, roll = pos * ANGULAR # vertices1 = vertices - np.mean(vertices, 0)[np.newaxis, :] # # obj = {'s': scale_init, 'angles': [-pitch, yaw, -roll + 180], 't': [0, 0, 0]} # camera = {'eye':[0, 0, 256], 'proj_type':'perspective', 'at':[0, 0, 0], # 'near': 1000, 'far':-100, 'fovy':30, 'up':[0,1,0]} # # image1 = transform_test(vertices1, obj, camera, triangles, colors, h=256, w=256) * 255 # image1 = image1.astype(np.uint8) # imsave(os.path.join(save_folder, name + '.jpg'), image1) if args.is3d: # corresponding colors colors = prn.get_colors(image, vertices) if args.isTexture: if args.texture_size != 256: pos_interpolated = resize( pos, (args.texture_size, args.texture_size), preserve_range=True) else: pos_interpolated = pos.copy() texture = cv2.remap(image, pos_interpolated[:, :, :2].astype( np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) if args.isMask: vertices_vis = get_visibility(vertices, prn.triangles, h, w) uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) uv_mask = resize(uv_mask, (args.texture_size, args.texture_size), preserve_range=True) texture = texture * uv_mask[:, :, np.newaxis] write_obj_with_texture( os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, texture, prn.uv_coords / prn.resolution_op ) #save 3d face with texture(can open with meshlab) else: write_obj_with_colors( os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, colors) #save 3d face(can open with meshlab) if args.isDepth: depth_image = get_depth_image(vertices, prn.triangles, h, w, True) depth = get_depth_image(vertices, prn.triangles, h, w) imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image) sio.savemat(os.path.join(save_folder, name + '_depth.mat'), {'depth': depth}) if args.isMat: sio.savemat(os.path.join(save_folder, name + '_mesh.mat'), { 'vertices': vertices, 'colors': colors, 'triangles': prn.triangles }) if args.isKpt: # get landmarks kpt = prn.get_landmarks(pos) np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) if args.is2dKpt and args.is68Align: ori_kpt = prn.get_landmarks_2d(pos) dlib_aligner = DlibAlign() dst_img = dlib_aligner.dlib_68_align(image, ori_kpt, 256, 0.5) imsave(os.path.join(save_folder, name + '.jpg'), dst_img) if args.isPose: # estimate pose camera_matrix, pose, rot = estimate_pose(vertices) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), np.array(pose) * ANGULAR) np.savetxt(os.path.join(save_folder, name + '_camera_matrix.txt'), camera_matrix) if args.isShow: kpt = prn.get_landmarks(pos) cv2.imshow('sparse alignment', plot_kpt(image, kpt)) # cv2.imshow('dense alignment', plot_vertices(image, vertices)) # cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt)) cv2.waitKey(1)
def texture_editing(prn, args): # read image image = imread(args.image_path) if len(image.shape) > 2: [h, w, c] = image.shape if c > 3: image = image[:, :, :3] else: [h, w] = image.shape image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR) #[h, w, _] = image.shape #-- 1. 3d reconstruction -> get texture. pos = prn.process(image) print("Pose shape: ", pos.shape) vertices = prn.get_vertices(pos) print("Vertice shape: ", vertices.shape) image = image / 255. texture = cv2.remap(image, pos[:, :, :2].astype(np.float32), None, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) #filename, _ = os.path.splitext(args.image_path) #cv2.imwrite(filename + '_tex.jpg', texture[:, :, ::-1] * 255) #-- 2. Texture Editing Mode = args.mode # change part of texture(for data augumentation/selfie editing. Here modify eyes for example) if Mode == 0: # load eye mask uv_face_eye = imread('Data/uv-data/uv_face_eyes.png', as_grey=True) / 255. uv_face = imread('Data/uv-data/uv_face.png', as_grey=True) / 255. eye_mask = (abs(uv_face_eye - uv_face) > 0).astype(np.float32) # texture from another image or a processed texture ref_image = imread(args.ref_path) ref_pos = prn.process(ref_image) ref_image = ref_image / 255. ref_texture = cv2.remap(ref_image, ref_pos[:, :, :2].astype(np.float32), None, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) # modify texture new_texture = texture * (1 - eye_mask[:, :, np.newaxis] ) + ref_texture * eye_mask[:, :, np.newaxis] # change whole face(face swap) elif Mode == 1: # texture from another image or a processed texture ref_image = imread(args.ref_path) if len(ref_image.shape) > 2: #[h, w, c] = ref_image.shape if c > 3: ref_image = ref_image[:, :, :3] else: #[h, w] = ref_image.shape ref_image = cv2.cvtColor(ref_image, cv2.COLOR_GRAY2BGR) ref_pos = prn.process(ref_image) ref_image = ref_image / 255. ref_texture = cv2.remap(ref_image, ref_pos[:, :, :2].astype(np.float32), None, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) filename, _ = os.path.splitext(args.ref_path) cv2.imwrite(filename + '_tex.jpg', ref_texture[:, :, ::-1] * 255) ref_vertices = prn.get_vertices(ref_pos) new_texture = ref_texture #(texture + ref_texture)/2. else: print('Wrong Mode! Mode should be 0 or 1.') exit() #-- 3. remap to input image.(render) vis_colors = np.ones((vertices.shape[0], 1)) face_mask = render_texture(vertices.T, vis_colors.T, prn.triangles.T, h, w, c=1) face_mask = np.squeeze(face_mask > 0).astype(np.float32) new_colors = prn.get_colors_from_texture(new_texture) new_image = render_texture(vertices.T, new_colors.T, prn.triangles.T, h, w, c=3) new_image = image * (1 - face_mask[:, :, np.newaxis] ) + new_image * face_mask[:, :, np.newaxis] # Possion Editing for blending image vis_ind = np.argwhere(face_mask > 0) vis_min = np.min(vis_ind, 0) vis_max = np.max(vis_ind, 0) center = (int((vis_min[1] + vis_max[1]) / 2 + 0.5), int((vis_min[0] + vis_max[0]) / 2 + 0.5)) output = cv2.seamlessClone( (new_image * 255).astype(np.uint8), (image * 255).astype(np.uint8), (face_mask * 255).astype(np.uint8), center, cv2.NORMAL_CLONE) # save output print("Vertice:", vertices) camera_matrix, pose, rotation_matrix = estimate_pose(vertices) # pose, rotation_matrix = estimate_pose(vertices) center_pt = np.mean(vertices, axis=0) vertices_trans = vertices - center_pt save_vertices = frontalize(vertices_trans, rotation_matrix) save_vertices = save_vertices + center_pt if not os.path.exists(os.path.dirname(args.output_path)): os.makedirs(os.path.dirname(args.output_path)) sio.savemat( os.path.join(os.path.dirname(args.output_path), os.path.basename(args.output_path) + '_mesh.mat'), { 'vertices': save_vertices, 'colors': new_colors, 'triangles': prn.triangles }) imsave(args.output_path, output) print('Done.')
def main(args): if args.isShow or args.isTexture: import cv2 from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box # ---- transform transform_img = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(FLAGS["normalize_mean"], FLAGS["normalize_std"]) ]) # ---- init PRN prn = PRN(args.model) # ------------- load data image_folder = args.inputDir save_folder = args.outputDir if not os.path.exists(save_folder): os.mkdir(save_folder) types = ('*.jpg', '*.png') image_path_list = [] for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) print("#" * 25) print("[PRNet Inference] {} picture were under processing~".format( total_num)) print("#" * 25) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] # read image image = cv2.imread(image_path) [h, w, c] = image.shape # the core: regress position map image = cv2.resize(image, (256, 256)) image_t = transform_img(image) image_t = image_t.unsqueeze(0) pos = prn.net_forward( image_t) # input image has been cropped to 256x256 out = pos.cpu().detach().numpy() pos = np.squeeze(out) cropped_pos = pos * 255 pos = cropped_pos.transpose(1, 2, 0) if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy() save_vertices[:, 1] = h - 1 - save_vertices[:, 1] if args.isImage: cv2.imwrite(os.path.join(save_folder, name + '.jpg'), image) if args.is3d: # corresponding colors colors = prn.get_colors(image, vertices) if args.isTexture: if args.texture_size != 256: pos_interpolated = cv2.resize( pos, (args.texture_size, args.texture_size), preserve_range=True) else: pos_interpolated = pos.copy() texture = cv2.remap(image, pos_interpolated[:, :, :2].astype( np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) if args.isMask: vertices_vis = get_visibility(vertices, prn.triangles, h, w) uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) uv_mask = cv2.resize( uv_mask, (args.texture_size, args.texture_size), preserve_range=True) texture = texture * uv_mask[:, :, np.newaxis] write_obj_with_texture( os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, texture, prn.uv_coords / prn.resolution_op ) # save 3d face with texture(can open with meshlab) else: write_obj_with_colors( os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, colors) # save 3d face(can open with meshlab) # if args.isDepth: # depth_image = get_depth_image(vertices, prn.triangles, h, w, True) # depth = get_depth_image(vertices, prn.triangles, h, w) # cv2.imwrite(os.path.join(save_folder, name + '_depth.jpg'), depth_image) # sio.savemat(os.path.join(save_folder, name + '_depth.mat'), {'depth': depth}) if args.isKpt or args.isShow: # get landmarks kpt = prn.get_landmarks(pos) np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) if args.isPose or args.isShow: # estimate pose camera_matrix, pose = estimate_pose(vertices) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) np.savetxt(os.path.join(save_folder, name + '_camera_matrix.txt'), camera_matrix) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) if args.isShow: # ---------- Plot image_pose = plot_pose_box(image, camera_matrix, kpt) cv2.imshow('sparse alignment', plot_kpt(image, kpt)) cv2.imshow('dense alignment', plot_vertices(image, vertices)) cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt)) cv2.waitKey(0)
def main(args): if args.isShow or args.isTexture: import cv2 from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib = args.isDlib) # ------------- load data image_folder = args.inputDir save_folder = args.outputDir if not os.path.exists(save_folder): os.mkdir(save_folder) # types = ('*.jpg', '*.png') # image_path_list= [] # for files in types: # image_path_list.extend(glob(os.path.join(image_folder, files))) # total_num = len(image_path_list) for dir, dirs, files in sorted(os.walk(image_folder)): for file in files: image_path = os.path.join(dir, file) dir = dir.replace("\\", "/") new_dir = dir.replace(image_folder, save_folder) if not os.path.isdir(new_dir): os.mkdir(new_dir) name = image_path.replace(image_folder, save_folder) print('data path:', name) # read image image = imread(image_path) [h, w, c] = image.shape if c>3: image = image[:,:,:3] # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size> 1000: image = rescale(image, 1000./max_size) image = (image*255).astype(np.uint8) pos = prn.process(image) # use dlib to detect face else: # if image.shape[0] == image.shape[1]: # image = resize(image, (256,256)) # pos = prn.net_forward(image/255.) # input image has been cropped to 256x256 # else: box = np.array([0, image.shape[1]-1, 0, image.shape[0]-1]) # cropped with bounding box pos = prn.process(image, box) image = image/255. if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy() save_vertices[:,1] = h - 1 - save_vertices[:,1] if args.isImage: imsave(name, image) if args.is3d: # corresponding colors colors = prn.get_colors(image, vertices) if args.isTexture: if args.texture_size != 256: pos_interpolated = resize(pos, (args.texture_size, args.texture_size), preserve_range = True) else: pos_interpolated = pos.copy() texture = cv2.remap(image, pos_interpolated[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT,borderValue=(0)) if args.isMask: vertices_vis = get_visibility(vertices, prn.triangles, h, w) uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) uv_mask = resize(uv_mask, (args.texture_size, args.texture_size), preserve_range = True) texture = texture*uv_mask[:,:,np.newaxis] write_obj_with_texture(name.replace('.jpg', '.obj'), save_vertices, prn.triangles, texture, prn.uv_coords/prn.resolution_op)#save 3d face with texture(can open with meshlab) else: write_obj_with_colors(name.replace('.jpg', '.obj'), save_vertices, prn.triangles, colors) #save 3d face(can open with meshlab) filepath = name.replace('.jpg', '.obj') filepath = filepath.replace("\\", "/") print('filepath:', filepath) new_dir = dir.replace(args.inputDir, args.renderDir) # print(new_dir + '/' + file) if not os.path.isdir(new_dir): os.mkdir(new_dir) color_image1, _ = render_scene(filepath, 4.0, 0.0, 3.0) color_image2, _ = render_scene(filepath, 4.0, np.pi / 18.0, 3.0) color_image3, _ = render_scene(filepath, 4.0, np.pi / 9.0, 3.0) if color_image1 is None or color_image2 is None: continue new_path = filepath.replace(args.outputDir, args.renderDir) # print('new_path:', new_path) save_image(new_path, '_40_', color_image1) save_image(new_path, '_50_', color_image2) save_image(new_path, '_60_', color_image3) os.remove(name.replace('.jpg', '.obj')) if args.isDepth: depth_image = get_depth_image(vertices, prn.triangles, h, w, True) depth = get_depth_image(vertices, prn.triangles, h, w) imsave(os.path.join(name.replace('.jpg', '_depth.jpg')), depth_image) sio.savemat(name.replace('.jpg', '_depth.mat'), {'depth': depth}) if args.isMat: sio.savemat(name.replace('.jpg', '_mesh.mat'), {'vertices': vertices, 'colors': colors, 'triangles': prn.triangles}) if args.isKpt or args.isShow: # get landmarks kpt = prn.get_landmarks(pos) np.savetxt(name.replace('.jpg', '_kpt.txt'), kpt) if args.isPose or args.isShow: # estimate pose camera_matrix, pose = estimate_pose(vertices) np.savetxt(name.replace('.jpg', '_pose.txt'), pose) np.savetxt(name.replace('.jpg', '_camera_matrix.txt'), camera_matrix) np.savetxt(name.replace('.jpg', '_pose.txt'), pose) if args.isShow: # ---------- Plot image_pose = plot_pose_box(image, camera_matrix, kpt) cv2.imshow('sparse alignment', plot_kpt(image, kpt)) cv2.imshow('dense alignment', plot_vertices(image, vertices)) cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt)) cv2.waitKey(0)
import scipy.io as sio from utils.rotate_vertices import frontalize import matplotlib.pyplot as plt from mpl_toolkits.mplot3d import Axes3D from utils.render_app import get_depth_image from skimage.io import imread, imsave data = sio.loadmat('image00002_mesh.mat') vertices = data['vertices'] triangles = data['triangles'] front_v = frontalize(vertices) print(vertices) print(front_v)
def main(args): if args.isShow or args.isTexture: import cv2 from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib = args.isDlib) # ------------- load data image_folder = args.inputDir save_folder = args.outputDir if not os.path.exists(save_folder): os.mkdir(save_folder) types = ('*.jpg', '*.png') image_path_list= [] for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] # read image image = imread(image_path) [h, w, _] = image.shape # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size> 1000: image = rescale(image, 1000./max_size) pos = prn.process(image) # use dlib to detect face else: if image.shape[1] == image.shape[2]: image = resize(image, (256,256)) pos = prn.net_forward(image/255.) # input image has been cropped to 256x256 else: box = np.array([0, image.shape[1]-1, 0, image.shape[0]-1]) # cropped with bounding box pos = prn.process(image, box) image = image/255. if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices if args.isImage: imsave(os.path.join(save_folder, name + '.jpg'), image) if args.is3d: # corresponding colors colors = prn.get_colors(image, vertices) if args.isTexture: texture = cv2.remap(image, pos[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT,borderValue=(0)) if args.isMask: vertices_vis = get_visibility(vertices, prn.triangles, h, w) uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) texture = texture*uv_mask[:,:,np.newaxis] write_obj_with_texture(os.path.join(save_folder, name + '.obj'), save_vertices, colors, prn.triangles, texture, prn.uv_coords/prn.resolution_op)#save 3d face with texture(can open with meshlab) else: write_obj(os.path.join(save_folder, name + '.obj'), save_vertices, colors, prn.triangles) #save 3d face(can open with meshlab) if args.isDepth: depth_image = get_depth_image(vertices, prn.triangles, h, w) imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image) if args.isMat: sio.savemat(os.path.join(save_folder, name + '_mesh.mat'), {'vertices': save_vertices, 'colors': colors, 'triangles': prn.triangles}) if args.isKpt or args.isShow: # get landmarks kpt = prn.get_landmarks(pos) np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) if args.isPose or args.isShow: # estimate pose camera_matrix, pose = estimate_pose(vertices) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) if args.isShow: # ---------- Plot image_pose = plot_pose_box(image, camera_matrix, kpt) cv2.imshow('sparse alignment', plot_kpt(image, kpt)) cv2.imshow('dense alignment', plot_vertices(image, vertices)) cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt)) cv2.waitKey(0)
def main(args): if args.isShow or args.isTexture: import cv2 from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib = args.isDlib) # ------------- load data image_folder = args.inputDir print(image_folder) save_folder = args.outputDir print(save_folder) if not os.path.exists(save_folder): os.mkdir(save_folder) meta_save_folder = os.path.join(save_folder, 'meta') if not os.path.exists(meta_save_folder): os.mkdir(meta_save_folder) types = ('*.jpg', '*.png', '*,JPG') image_path_list= find_files(image_folder, ('.jpg', '.png', '.JPG')) total_num = len(image_path_list) print(image_path_list) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] print(image_path) # read image image = imread(image_path) [h, w, c] = image.shape if c>3: image = image[:,:,:3] # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size> 1000: image = rescale(image, 1000./max_size) image = (image*255).astype(np.uint8) pos = prn.process(image) # use dlib to detect face else: if image.shape[1] == image.shape[2]: image = resize(image, (256,256)) pos = prn.net_forward(image/255.) # input image has been cropped to 256x256 else: box = np.array([0, image.shape[1]-1, 0, image.shape[0]-1]) # cropped with bounding box pos = prn.process(image, box) image = image/255. if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy() save_vertices[:,1] = h - 1 - save_vertices[:,1] if args.isImage: imsave(os.path.join(save_folder, name + '.jpg'), image) if args.is3d: # corresponding colors colors = prn.get_colors(image, vertices) if args.isTexture: if args.texture_size != 256: pos_interpolated = resize(pos, (args.texture_size, args.texture_size), preserve_range = True) else: pos_interpolated = pos.copy() texture = cv2.remap(image, pos_interpolated[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT,borderValue=(0)) if args.isMask: vertices_vis = get_visibility(vertices, prn.triangles, h, w) uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) uv_mask = resize(uv_mask, (args.texture_size, args.texture_size), preserve_range = True) texture = texture*uv_mask[:,:,np.newaxis] write_obj_with_texture(os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, texture, prn.uv_coords/prn.resolution_op)#save 3d face with texture(can open with meshlab) else: write_obj_with_colors(os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, colors) #save 3d face(can open with meshlab) if args.isDepth: depth_image = get_depth_image(vertices, prn.triangles, h, w, True) depth = get_depth_image(vertices, prn.triangles, h, w) imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image) sio.savemat(os.path.join(meta_save_folder, name + '_depth.mat'), {'depth':depth}) if args.isMat: sio.savemat(os.path.join(meta_save_folder, name + '_mesh.mat'), {'vertices': vertices, 'colors': colors, 'triangles': prn.triangles}) if args.isKpt or args.isShow: # get landmarks kpt = prn.get_landmarks(pos) # pdb.set_trace() np.save(os.path.join(meta_save_folder, name + '_kpt.npy'), kpt) # cv2.imwrite(os.path.join(save_folder, name + '_skpt.jpg'), plot_kpt(image, kpt)) if args.isPose or args.isShow: # estimate pose camera_matrix, pose = estimate_pose(vertices) np.savetxt(os.path.join(meta_save_folder, name + '_pose.txt'), pose) np.savetxt(os.path.join(meta_save_folder, name + '_camera_matrix.txt'), camera_matrix) if args.isShow: # ---------- Plot image = imread(os.path.join(save_folder, name + '.jpg')) image_pose = plot_pose_box(image, camera_matrix, kpt) #cv2.imwrite(os.path.join(save_folder, name + '_pose.jpg'), plot_kpt(image, kpt)) #cv2.imwrite(os.path.join(save_folder, name + '_camera_matrix.jpg'), plot_vertices(image, vertices)) #cv2.imwrite(os.path.join(save_folder, name + '_pose.jpg'), plot_pose_box(image, camera_matrix, kpt)) image = imread(os.path.join(save_folder, name + '.jpg')) b, g, r = cv2.split(image) image = cv2.merge([r,g,b])
def main(args): if args.isShow or args.isTexture or args.isCamera: import cv2 from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib=args.isDlib) # ------------- load data image_folder = args.inputDir save_folder = args.outputDir if not os.path.exists(save_folder): os.mkdir(save_folder) types = ('*.jpg', '*.png') image_path_list = [] for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) if args.isCamera: # Create a VideoCapture object and read from input file # If the input is the camera, pass 0 instead of the video file name cap = cv2.VideoCapture(0) # Check if camera opened successfully if (cap.isOpened() == False): print("Error opening video stream or file") # Read until video is completed while (cap.isOpened()): # Capture frame-by-frame ret, frame = cap.read() if ret == True: if args.isDlib: max_size = max(frame.shape[0], frame.shape[1]) if max_size > 1000: frame = rescale(frame, 1000. / max_size) frame = (frame * 255).astype(np.uint8) pos = prn.process(frame) # use dlib to detect face else: if frame.shape[0] == frame.shape[1]: frame = resize(frame, (256, 256)) pos = prn.net_forward( frame / 255.) # input frame has been cropped to 256x256 else: box = np.array( [0, frame.shape[1] - 1, 0, frame.shape[0] - 1]) # cropped with bounding box pos = prn.process(frame, box) # Normalizing the frame and skiping if there was no one in the frame frame = frame / 255. if pos is None: continue # Get landmarks in frame kpt = prn.get_landmarks(pos) # Display the resulting frame cv2.imshow('sparse alignment', plot_kpt(frame, kpt)) # Press Q on keyboard to exit if cv2.waitKey(25) & 0xFF == ord('q'): break # Break the loop else: break # When everything done, release the video capture object cap.release() # Closes all the frames cv2.destroyAllWindows() else: for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] # read image image = imread(image_path) [h, w, c] = image.shape if c > 3: image = image[:, :, :3] # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size > 1000: image = rescale(image, 1000. / max_size) image = (image * 255).astype(np.uint8) pos = prn.process(image) # use dlib to detect face else: if image.shape[0] == image.shape[1]: image = resize(image, (256, 256)) pos = prn.net_forward( image / 255.) # input image has been cropped to 256x256 else: box = np.array( [0, image.shape[1] - 1, 0, image.shape[0] - 1]) # cropped with bounding box pos = prn.process(image, box) image = image / 255. if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy() save_vertices[:, 1] = h - 1 - save_vertices[:, 1] if args.isImage: imsave(os.path.join(save_folder, name + '.jpg'), image) if args.is3d: # corresponding colors colors = prn.get_colors(image, vertices) if args.isTexture: if args.texture_size != 256: pos_interpolated = resize( pos, (args.texture_size, args.texture_size), preserve_range=True) else: pos_interpolated = pos.copy() texture = cv2.remap(image, pos_interpolated[:, :, :2].astype( np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) if args.isMask: vertices_vis = get_visibility(vertices, prn.triangles, h, w) uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) uv_mask = resize( uv_mask, (args.texture_size, args.texture_size), preserve_range=True) texture = texture * uv_mask[:, :, np.newaxis] write_obj_with_texture( os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, texture, prn.uv_coords / prn.resolution_op ) #save 3d face with texture(can open with meshlab) else: write_obj_with_colors( os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, colors) #save 3d face(can open with meshlab) if args.isDepth: depth_image = get_depth_image(vertices, prn.triangles, h, w, True) depth = get_depth_image(vertices, prn.triangles, h, w) imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image) sio.savemat(os.path.join(save_folder, name + '_depth.mat'), {'depth': depth}) if args.isMat: sio.savemat( os.path.join(save_folder, name + '_mesh.mat'), { 'vertices': vertices, 'colors': colors, 'triangles': prn.triangles }) if args.isKpt or args.isShow: # get landmarks kpt = prn.get_landmarks(pos) np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) if args.isPose or args.isShow: # estimate pose camera_matrix, pose = estimate_pose(vertices) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) np.savetxt( os.path.join(save_folder, name + '_camera_matrix.txt'), camera_matrix) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) if args.isShow: # ---------- Plot image_pose = plot_pose_box(image, camera_matrix, kpt) cv2.imshow( 'sparse alignment', cv2.cvtColor(np.float32(plot_kpt(image, kpt)), cv2.COLOR_RGB2BGR)) cv2.imshow( 'dense alignment', cv2.cvtColor(np.float32(plot_vertices(image, vertices)), cv2.COLOR_RGB2BGR)) cv2.imshow( 'pose', cv2.cvtColor( np.float32(plot_pose_box(image, camera_matrix, kpt)), cv2.COLOR_RGB2BGR)) cv2.waitKey(0)