def main(args): # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN() # ------------- load data image_folder = args.inputDir save_folder = args.outputDir if not os.path.exists(save_folder): os.mkdir(save_folder) types = ('*.jpg', '*.png') image_path_list= [] for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] # read image image = imread(image_path) [h, w, _] = image.shape pos = prn.net_forward(image/255.) # input image has been cropped to 256x256 imsave(os.path.join(save_folder, name + '.jpg'), image)
def extract_param(checkpoint_fp, root='', filelists=None, num_classes=62, device_ids=[0], batch_size=1, num_workers=0): map_location = {f'cuda:{i}': 'cuda:0' for i in range(8)} # checkpoint = torch.load(checkpoint_fp, map_location=map_location)['state_dict'] torch.cuda.set_device(device_ids[0]) model = PRN(checkpoint_fp) # model = nn.DataParallel(model, device_ids=device_ids).cuda() # model.load_state_dict(checkpoint) dataset = DDFATestDataset(filelists=filelists, root=root, transform=transforms.Compose([ToTensorGjz(), NormalizeGjz(mean=127.5, std=128)])) data_loader = data.DataLoader(dataset, batch_size=batch_size, num_workers=num_workers) cudnn.benchmark = True # model.eval() end = time.time() outputs = [] with torch.no_grad(): for _, inputs in enumerate(data_loader): inputs = inputs.cuda() # Get the output landmarks pos = model.net_forward(inputs) out = pos.cpu().detach().numpy() pos = np.squeeze(out) cropped_pos = pos * 255 pos = cropped_pos.transpose(1, 2, 0) if pos is None: continue # print(pos.shape) output = model.get_landmarks(pos) # print(output.shape) outputs.append(output) outputs = np.array(outputs, dtype=np.float32) print("outputs",outputs.shape) print(f'Extracting params take {time.time() - end: .3f}s') return outputs
def main(args): if args.isShow or args.isTexture: import cv2 from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib=args.isDlib) # ------------- load data image_folder = args.inputDir save_folder = args.outputDir if not os.path.exists(save_folder): os.mkdir(save_folder) types = ('*.jpg', '*.png') image_path_list = [] if os.path.isfile(image_folder): image_path_list.append(image_folder) for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] # read image image = imread(image_path) [h, w, _] = image.shape # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size > 1000: image = rescale(image, 1000. / max_size) image = (image * 255).astype(np.uint8) pos, crop_image = prn.process(image) # use dlib to detect face else: if image.shape[1] == image.shape[2]: image = resize(image, (256, 256)) pos = prn.net_forward( image / 255.) # input image has been cropped to 256x256 crop_image = None else: box = np.array([0, image.shape[1] - 1, 0, image.shape[0] - 1 ]) # cropped with bounding box pos, crop_image = prn.process(image, box) image = image / 255. if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy() save_vertices[:, 1] = h - 1 - save_vertices[:, 1] if args.isImage and crop_image is not None: imsave(os.path.join(save_folder, name + '_crop.jpg'), crop_image) imsave(os.path.join(save_folder, name + '_orig.jpg'), image) if args.is3d: # corresponding colors colors = prn.get_colors(image, vertices) if args.isTexture: texture = cv2.remap(image, pos[:, :, :2].astype(np.float32), None, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) if args.isMask: vertices_vis = get_visibility(vertices, prn.triangles, h, w) uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) texture = texture * uv_mask[:, :, np.newaxis] write_obj_with_texture( os.path.join(save_folder, name + '.obj'), save_vertices, colors, prn.triangles, texture, prn.uv_coords / prn.resolution_op ) #save 3d face with texture(can open with meshlab) else: write_obj(os.path.join(save_folder, name + '.obj'), save_vertices, colors, prn.triangles) #save 3d face(can open with meshlab) if args.isDepth: depth_image = get_depth_image(vertices, prn.triangles, h, w, True) depth = get_depth_image(vertices, prn.triangles, h, w) imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image) sio.savemat(os.path.join(save_folder, name + '_depth.mat'), {'depth': depth}) if args.isMat: sio.savemat(os.path.join(save_folder, name + '_mesh.mat'), { 'vertices': vertices, 'colors': colors, 'triangles': prn.triangles }) if args.isKpt or args.isShow: # get landmarks kpt = prn.get_landmarks(pos) np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) if args.isPose or args.isShow: # estimate pose camera_matrix, pose = estimate_pose(vertices) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) np.savetxt(os.path.join(save_folder, name + '_camera_matrix.txt'), camera_matrix) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) if args.isShow: # ---------- Plot image_pose = plot_pose_box(image, camera_matrix, kpt) cv2.imshow('sparse alignment', plot_kpt(image, kpt)) cv2.imshow('dense alignment', plot_vertices(image, vertices)) cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt)) if crop_image is not None: cv2.imshow('crop', crop_image) cv2.waitKey(0)
def main(args): #---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib = args.isDlib) if mode == FAKE: dataset_folder_path = "/home/wukong/librealsense/examples/realsense-dataset/attack_dataset" elif mode == REAL: dataset_folder_path = "/home/wukong/librealsense/examples/realsense-dataset/all_dataset" elif mode == PRINT: dataset_folder_path = "/home/wukong/librealsense/examples/realsense-dataset/all_dataset" dataset_folder_list = os.walk(dataset_folder_path).next()[1] train_folder_path = "/home/wukong/anaconda3/dataset/phase7/train" train_folder_list = os.walk(train_folder_path).next()[1] for fname in dataset_folder_list: source_path = os.path.join(dataset_folder_path, fname) hs_warp_folder = os.path.join(source_path, "hs_raw_warp") rs_color_folder = os.path.join(source_path, "rs_raw_color") rs_depth_folder = os.path.join(source_path, "rs_raw_depth") hs_warp_list = sorted(glob(os.path.join(hs_warp_folder, '*.jpg'))) rs_color_list = sorted(glob(os.path.join(rs_color_folder, '*.jpg'))) rs_depth_list = sorted(glob(os.path.join(rs_depth_folder, '*.jpg'))) if not hs_warp_list or not rs_color_list or not rs_depth_list: print("skip ", source_path) continue elif len(hs_warp_list) != len(rs_color_list) or len(hs_warp_list) != len(rs_depth_list): print("skip ", source_path) continue else: pass if mode == FAKE: new_fname = "1_" + fname[1:] + "_3_1_4" elif mode == REAL: new_fname = "1_" + fname[1:] + "_1_1_1" elif mode == PRINT: new_fname = "1_" + fname[1:] + "_0_0_0" if new_fname not in train_folder_list: new_path = os.path.join(train_folder_path, new_fname) os.mkdir(new_path) new_depth_path = os.path.join(new_path, "depth") os.mkdir(new_depth_path) new_profile_path = os.path.join(new_path, "profile") os.mkdir(new_profile_path) new_rs_path = os.path.join(new_path, "rs") os.mkdir(new_rs_path) print("create new folder: {}".format(fname)) else: print("skip {}".format(new_fname)) continue spatial_coordinate_idx = index.Index(properties=p) count_num = 1 total_num = len(hs_warp_list) for j in range(total_num): if j % 10 == 0: print("has processed {} of {} images".format(j, total_num)) hs_warp_image = imread(hs_warp_list[j]) [h, w, c] = hs_warp_image.shape if c>3: hs_warp_image = hs_warp_image[:,:,:3] # the core: regress position map if args.isDlib: max_size = max(hs_warp_image.shape[0], hs_warp_image.shape[1]) if max_size> 1000: hs_warp_image = rescale(hs_warp_image, 1000./max_size) hs_warp_image = (hs_warp_image*255).astype(np.uint8) hs_pos = prn.process(hs_warp_image) # use dlib to detect face else: if hs_warp_image.shape[0] == hs_warp_image.shape[1]: hs_warp_image = resize(hs_warp_image, (256,256)) hs_pos = prn.net_forward(hs_warp_image/255.) # input hs_warp_image has been cropped to 256x256 else: box = np.array([0, hs_warp_image.shape[1]-1, 0, hs_warp_image.shape[0]-1]) # cropped with bounding box hs_pos = prn.process(hs_warp_image, box) hs_warp_image = hs_warp_image/255. if hs_pos is None: continue hs_vertices = prn.get_vertices(hs_pos) camera_matrix, euler_pose = estimate_pose(hs_vertices) # check similarity with previous pose hit = spatial_coordinate_idx.nearest((euler_pose[0], euler_pose[1], euler_pose[2], euler_pose[0], euler_pose[1], euler_pose[2]), 1, objects=True) hit = [i for i in hit] if hit: nearest_euler_pose = np.array(hit[0].bbox[:3]) current_euler_pose = np.array(euler_pose) dist = np.linalg.norm(current_euler_pose - nearest_euler_pose) if dist > SPATIAL_THRESHOLD_DEGREE: print("Get a new euler pose {}".format(euler_pose)) spatial_coordinate_idx.insert(0,(euler_pose[0], euler_pose[1], euler_pose[2], euler_pose[0], euler_pose[1], euler_pose[2])) else: continue else: print("First euler_pose: {}".format(euler_pose)) spatial_coordinate_idx.insert(0,(euler_pose[0], euler_pose[1], euler_pose[2], euler_pose[0], euler_pose[1], euler_pose[2])) ############################################## # ############################################## if mode == FAKE: imsave(os.path.join(new_profile_path, ('%04d' % count_num) + '.jpg'), plot_crop(hs_warp_image, hs_vertices)) rs_depth_image = imread(rs_depth_list[j]) imsave(os.path.join(new_depth_path, ('%04d' % count_num) + '.jpg'), plot_crop(rs_depth_image, hs_vertices)) elif mode == PRINT: rs_color_image = imread(rs_color_list[j]) imsave(os.path.join(new_rs_path, ('%04d' % count_num) + '.jpg'), rs_color_image) elif mode == REAL: rs_color_image = imread(rs_color_list[j]) [h, w, c] = rs_color_image.shape if c>3: rs_color_image = rs_color_image[:,:,:3] # the core: regress position map if args.isDlib: max_size = max(rs_color_image.shape[0], rs_color_image.shape[1]) if max_size> 1000: rs_color_image = rescale(rs_color_image, 1000./max_size) rs_color_image = (rs_color_image*255).astype(np.uint8) rs_pos = prn.process(rs_color_image) # use dlib to detect face else: if rs_color_image.shape[0] == rs_color_image.shape[1]: rs_color_image = resize(rs_color_image, (256,256)) rs_pos = prn.net_forward(rs_color_image/255.) # input rs_color_image has been cropped to 256x256 else: box = np.array([0, rs_color_image.shape[1]-1, 0, rs_color_image.shape[0]-1]) # cropped with bounding box rs_pos = prn.process(rs_color_image, box) rs_color_image = rs_color_image/255. if rs_pos is None: continue rs_vertices = prn.get_vertices(rs_pos) rs_depth_image = imread(rs_depth_list[j]) imsave(os.path.join(new_profile_path, ('%04d' % count_num) + '.jpg'), plot_crop(hs_warp_image, rs_vertices)) imsave(os.path.join(new_depth_path, ('%04d' % count_num) + '.jpg'), plot_crop(rs_depth_image, rs_vertices)) imsave(os.path.join(new_rs_path, ('%04d' % count_num) + '.jpg'), plot_crop(rs_color_image, rs_vertices)) count_num += 1
ret, image = cap.read() [h, w, c] = image.shape if c>3: image = image[:,:,:3] # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size> 1000: image = rescale(image, 1000./max_size) image = (image*255).astype(np.uint8) pos = prn.process(image) # use dlib to detect face else: if image.shape[0] == image.shape[1]: image = resize(image, (256,256)) pos = prn.net_forward(image/255.) # input image has been cropped to 256x256 else: box = np.array([0, image.shape[1]-1, 0, image.shape[0]-1]) # cropped with bounding box pos = prn.process(image, box) image = image/255. if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy()
def out_vert(args): # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib=args.isDlib) # ------------- load data # image_folder = args.inputDir # print (image_folder) #save_folder = args.outputDir base_dir = args.baseDir character = args.characterDir target_num = args.targNum #e.g. d:\characters\richardson\face\richardson_t10 image_folder = "%s\\%s\\face\\%s_t%s" % (base_dir, character, character, target_num) print(image_folder) #e.g. d:\characters\richardson\vertices\richardson_t10 save_folder = "%s\\%s\\vertices\\%s_t%s" % (base_dir, character, character, target_num) print(save_folder) if not os.path.exists(save_folder): os.makedirs(save_folder) types = ('*.jpg', '*.png') image_path_list = [] for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) print(total_num) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('\\')[-1][:-4] print(image_path) print(name) # read image image = imread(image_path) [h, w, _] = image.shape # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size > 1000: image = rescale(image, 1000. / max_size) image = (image * 255).astype(np.uint8) pos = prn.process(image) # use dlib to detect face else: if image.shape[1] == image.shape[2]: image = resize(image, (256, 256)) pos = prn.net_forward( image / 255.) # input image has been cropped to 256x256 else: box = np.array([0, image.shape[1] - 1, 0, image.shape[0] - 1 ]) # cropped with bounding box pos = prn.process(image, box) image = image / 255. if pos is None: continue vertices = prn.get_vertices(pos) np.save("%s/%s" % (save_folder, name), vertices) save_vertices = vertices.copy() save_vertices[:, 1] = h - 1 - save_vertices[:, 1]
def main(args): # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib=args.isDlib) # ------------- load data # image_folder = args.inputDir # save_folder = args.outputDir # vertices_dir = args.vertDir #i.e. d:\source base_dir = args.baseDir #i.e. d:\characters base_save_dir = args.baseSavedir #i.e. source\raupach scene = args.sceneDir #i.e source\raupach\richardson (the target character) character = args.characterDir #i.e. source\rauapch\richardson\richardson_001 source_num = args.sourceNum # targ_character = args.targChar #i.e. richardson_targ_10 targ_num = args.targNum # something like D:\source\raupach\richardson\raupach_richardson_001 image_folder = "%s\\%s\\%s\\%s_%s_%s" % (base_dir, scene, character, scene, character, source_num) print(image_folder) #something like d:\character\richardson\vertices\richards_t10 vertices_dir = "%s\\%s\\vertices\\%s_t%s" % (base_save_dir, character, character, targ_num) print(vertices_dir) #something like d:\character\raupach\src\align\raupach_richardson_t10_s001\\obj save_folder = "%s\\%s\\src\\align\\%s_%s_s%s_t%s\\obj" % ( base_save_dir, character, scene, character, source_num, targ_num) print(save_folder) if not os.path.exists(save_folder): os.makedirs(save_folder) # image_path_list= [] # for root, dirs, files in os.walk('%s' % image_folder): # for file in files: # if file.endswith('.jpg'): # image_path_list.append(file) # print (image_path_list) types = ('*.jpg', '*.png') image_path_list = [] for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) image_path_list = sorted(image_path_list) #print (image_path_list) # #repeating the above logic for a vertices directory. types = ('*.npy', '*.jpg') vert_path_list = [] for files in types: vert_path_list.extend(glob(os.path.join(vertices_dir, files))) total_num_vert = len(vert_path_list) # vert_path_list.reverse() vert_path_list = sorted(vert_path_list) #print (vert_path_list) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('\\')[-1][:-4] print("%s aligned with %s" % (image_path_list[i], vert_path_list[i])) # read image image = imread(image_path) [h, w, _] = image.shape # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size > 1000: image = rescale(image, 1000. / max_size) image = (image * 255).astype(np.uint8) pos = prn.process(image) # use dlib to detect face else: if image.shape[1] == image.shape[2]: image = resize(image, (256, 256)) pos = prn.net_forward( image / 255.) # input image has been cropped to 256x256 else: box = np.array([0, image.shape[1] - 1, 0, image.shape[0] - 1 ]) # cropped with bounding box pos = prn.process(image, box) image = image / 255. if pos is None: continue vertices = prn.get_vertices(pos) #takes the nth file in the directory of the vertices to "frontalize" the source image. can_vert = vert_path_list[i] print(can_vert) save_vertices = align(vertices, can_vert) save_vertices[:, 1] = h - 1 - save_vertices[:, 1] colors = prn.get_colors(image, vertices) if args.isTexture: texture = cv2.remap(image, pos[:, :, :2].astype(np.float32), None, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) if args.isMask: vertices_vis = get_visibility(vertices, prn.triangles, h, w) uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) texture = texture * uv_mask[:, :, np.newaxis] write_obj_with_texture( os.path.join(save_folder, name + '.obj'), save_vertices, colors, prn.triangles, texture, prn.uv_coords / prn.resolution_op ) #save 3d face with texture(can open with meshlab) else: write_obj(os.path.join(save_folder, name + '.obj'), save_vertices, colors, prn.triangles) #save 3d face(can open with meshlab)
def main(): # OpenCV #cap = cv2.VideoCapture(args.video_source) cap = cv2.VideoCapture('b.mov') fps = video.FPS().start() # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib=args.isDlib) #while True: while cap.isOpened(): ret, frame = cap.read() # resize image and detect face frame_resize = cv2.resize(frame, None, fx=1 / DOWNSAMPLE_RATIO, fy=1 / DOWNSAMPLE_RATIO) # read image image = frame_resize image = resize(image) [h, w, c] = image.shape if c > 3: image = image[:, :, :3] # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size > 1000: image = rescale(image, 1000. / max_size) image = (image * 255).astype(np.uint8) st = time() pos = prn.process(image) # use dlib to detect face print('process', time() - st) else: if image.shape[0] == image.shape[1]: image = resize(image, (256, 256)) pos = prn.net_forward( image / 255.) # input image has been cropped to 256x256 else: box = np.array([0, image.shape[1] - 1, 0, image.shape[0] - 1 ]) # cropped with bounding box pos = prn.process(image, box) image = image / 255. if pos is None: cv2.imshow('a', frame_resize) fps.update() if cv2.waitKey(1) & 0xFF == ord('q'): break continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy() save_vertices[:, 1] = h - 1 - save_vertices[:, 1] #colors = prn.get_colors(image, vertices) #write_obj_with_colors(os.path.join('', 'webcam' + '.obj'), save_vertices, prn.triangles, colors) #if args.is3d: # # corresponding colors # colors = prn.get_colors(image, vertices) # # if args.isTexture: # if args.texture_size != 256: # pos_interpolated = resize(pos, (args.texture_size, args.texture_size), preserve_range = True) # else: # pos_interpolated = pos.copy() # texture = cv2.remap(image, pos_interpolated[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT,borderValue=(0)) # if args.isMask: # vertices_vis = get_visibility(vertices, prn.triangles, h, w) # uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) # uv_mask = resize(uv_mask, (args.texture_size, args.texture_size), preserve_range = True) # texture = texture*uv_mask[:,:,np.newaxis] # #write_obj_with_texture(os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, texture, prn.uv_coords/prn.resolution_op)#save 3d face with texture(can open with meshlab) # else: # True # #write_obj_with_colors(os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, colors) #save 3d face(can open with meshlab) # #if args.isDepth: # depth_image = get_depth_image(vertices, prn.triangles, h, w, True) # depth = get_depth_image(vertices, prn.triangles, h, w) # #imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image) # #sio.savemat(os.path.join(save_folder, name + '_depth.mat'), {'depth':depth}) # #if args.isKpt or args.isShow: # # get landmarks # kpt = prn.get_landmarks(pos) # #np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) # #if args.isPose or args.isShow: # # estimate pose # camera_matrix, pose = estimate_pose(vertices) #write_obj_with_colors(os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, colors) rendering_cc = mesh.render.render_grid(save_vertices, prn.triangles, 900, 900) a = np.transpose(rendering_cc, axes=[1, 0, 2]) dim = rendering_cc.shape[0] i_t = np.ones([dim, dim, 3], dtype=np.float32) for i in range(dim): i_t[i] = a[dim - 1 - i] i_t = i_t / 255 #imsave('webcam.png', i_t) #kpt = prn.get_landmarks(pos) #cv2.imshow('frame', image) #cv2.imshow('a',i_t/255) #cv2.imshow('sparse alignment', np.concatenate([image, i_t], axis=1)) cv2.imshow('sparse alignment', i_t) cv2.imshow('vedio', image) #cv2.imshow('sparse alignment', np.concatenate([plot_kpt(image, kpt), i_t], axis=1)) #cv2.imshow('dense alignment', plot_vertices(image, vertices)) #cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt)) fps.update() if cv2.waitKey(1) & 0xFF == ord('q'): break fps.stop() print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed())) print('[INFO] approx. FPS: {:.2f}'.format(fps.fps())) cap.release() cv2.destroyAllWindows()
def main(data_dir): # 1) Create Dataset of 300_WLP & Dataloader. wlp300 = PRNetDataset(root_dir=data_dir, transform=transforms.Compose([ToTensor(), ToNormalize(FLAGS["normalize_mean"], FLAGS["normalize_std"])])) wlp300_dataloader = DataLoader(dataset=wlp300, batch_size=FLAGS['batch_size'], shuffle=True, num_workers=0) # 2) Intermediate Processing. transform_img = transforms.Compose([ # transforms.ToTensor(), transforms.Normalize(FLAGS["normalize_mean"], FLAGS["normalize_std"]) ]) # # 3) Create PRNet model. # start_epoch, target_epoch = FLAGS['start_epoch'], FLAGS['target_epoch'] # model = ResFCN256() # # # Load the pre-trained weight # if FLAGS['resume'] and os.path.exists(os.path.join(FLAGS['images'], "3channels.pth")): # state = torch.load(os.path.join(FLAGS['images'], "3channels.pth")) # model.load_state_dict(state['prnet']) # start_epoch = state['start_epoch'] # INFO("Load the pre-trained weight! Start from Epoch", start_epoch) # # model.to("cuda") prn = PRN(os.path.join(FLAGS['images'], "3channels.pth")) bar = tqdm(wlp300_dataloader) nme_list = [] for i, sample in enumerate(bar): uv_map, origin = sample['uv_map'].to(FLAGS['device']), sample['origin'].to(FLAGS['device']) # print(origin.shape) # Inference. # origin = cv2.resize(origin, (256, 256)) # origin = transform_img(origin) # origin = origin.unsqueeze(0) uv_map_result = prn.net_forward(origin.cuda()) out = uv_map_result.cpu().detach().numpy() uv_map_result = np.squeeze(out) cropped_pos = uv_map_result * 255 uv_map_result = cropped_pos.transpose(1, 2, 0) out = uv_map.cpu().detach().numpy() uv_map = np.squeeze(out) cropped_pos = uv_map * 255 uv_map = cropped_pos.transpose(1, 2, 0) kpt_predicted = prn.get_landmarks(uv_map_result)[:, :2] kpt_gt = prn.get_landmarks(uv_map)[:, :2] nme_sum = 0 for j in range(kpt_gt.shape[0]): x = kpt_gt[j][0] - kpt_predicted[j][0] y = kpt_gt[j][1] - kpt_predicted[j][1] L2_norm = math.sqrt(math.pow(x, 2) + math.pow(y, 2)) # bounding box size has been fixed to 256x256 d = 256*256 error = L2_norm/d nme_sum += error nme_list.append(nme_sum/68) print(np.mean(nme_list))
def main(args): if args.isShow or args.isTexture: import cv2 from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib=args.isDlib, is_faceboxes=args.isFaceBoxes) # ---- load data image_folder = args.inputDir save_folder = args.outputDir if not os.path.exists(save_folder): os.mkdir(save_folder) types = ('*.jpg', '*.png') image_path_list = [] for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] # read image image = imread(image_path) [h, w, c] = image.shape if c > 3: image = image[:, :, :3] # RGBA图中,去除A通道 # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size > 1000: image = rescale(image, 1000. / max_size) image = (image * 255).astype(np.uint8) pos = prn.process(image) # use dlib to detect face elif args.isFaceBoxes: pos, cropped_img = prn.process( image) # use faceboxes to detect face else: if image.shape[0] == image.shape[1]: image = resize(image, (256, 256)) pos = prn.net_forward( image / 255.) # input image has been cropped to 256x256 else: box = np.array([0, image.shape[1] - 1, 0, image.shape[0] - 1 ]) # cropped with bounding box pos = prn.process(image, box) image = image / 255. if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy() save_vertices[:, 1] = h - 1 - save_vertices[:, 1] # 三维人脸旋转对齐方法 # if args.isImage: # vertices = prn.get_vertices(pos) # scale_init = 180 / (np.max(vertices[:, 1]) - np.min(vertices[:, 1])) # colors = prn.get_colors(image, vertices) # triangles = prn.triangles # camera_matrix, pose = estimate_pose(vertices) # yaw, pitch, roll = pos * ANGULAR # vertices1 = vertices - np.mean(vertices, 0)[np.newaxis, :] # # obj = {'s': scale_init, 'angles': [-pitch, yaw, -roll + 180], 't': [0, 0, 0]} # camera = {'eye':[0, 0, 256], 'proj_type':'perspective', 'at':[0, 0, 0], # 'near': 1000, 'far':-100, 'fovy':30, 'up':[0,1,0]} # # image1 = transform_test(vertices1, obj, camera, triangles, colors, h=256, w=256) * 255 # image1 = image1.astype(np.uint8) # imsave(os.path.join(save_folder, name + '.jpg'), image1) if args.is3d: # corresponding colors colors = prn.get_colors(image, vertices) if args.isTexture: if args.texture_size != 256: pos_interpolated = resize( pos, (args.texture_size, args.texture_size), preserve_range=True) else: pos_interpolated = pos.copy() texture = cv2.remap(image, pos_interpolated[:, :, :2].astype( np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) if args.isMask: vertices_vis = get_visibility(vertices, prn.triangles, h, w) uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) uv_mask = resize(uv_mask, (args.texture_size, args.texture_size), preserve_range=True) texture = texture * uv_mask[:, :, np.newaxis] write_obj_with_texture( os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, texture, prn.uv_coords / prn.resolution_op ) #save 3d face with texture(can open with meshlab) else: write_obj_with_colors( os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, colors) #save 3d face(can open with meshlab) if args.isDepth: depth_image = get_depth_image(vertices, prn.triangles, h, w, True) depth = get_depth_image(vertices, prn.triangles, h, w) imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image) sio.savemat(os.path.join(save_folder, name + '_depth.mat'), {'depth': depth}) if args.isMat: sio.savemat(os.path.join(save_folder, name + '_mesh.mat'), { 'vertices': vertices, 'colors': colors, 'triangles': prn.triangles }) if args.isKpt: # get landmarks kpt = prn.get_landmarks(pos) np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) if args.is2dKpt and args.is68Align: ori_kpt = prn.get_landmarks_2d(pos) dlib_aligner = DlibAlign() dst_img = dlib_aligner.dlib_68_align(image, ori_kpt, 256, 0.5) imsave(os.path.join(save_folder, name + '.jpg'), dst_img) if args.isPose: # estimate pose camera_matrix, pose, rot = estimate_pose(vertices) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), np.array(pose) * ANGULAR) np.savetxt(os.path.join(save_folder, name + '_camera_matrix.txt'), camera_matrix) if args.isShow: kpt = prn.get_landmarks(pos) cv2.imshow('sparse alignment', plot_kpt(image, kpt)) # cv2.imshow('dense alignment', plot_vertices(image, vertices)) # cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt)) cv2.waitKey(1)
def main(args): if args.isShow or args.isTexture: import cv2 from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box # ---- transform transform_img = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(FLAGS["normalize_mean"], FLAGS["normalize_std"]) ]) # ---- init PRN prn = PRN(args.model) # ------------- load data image_folder = args.inputDir save_folder = args.outputDir if not os.path.exists(save_folder): os.mkdir(save_folder) types = ('*.jpg', '*.png') image_path_list = [] for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) print("#" * 25) print("[PRNet Inference] {} picture were under processing~".format( total_num)) print("#" * 25) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] # read image image = cv2.imread(image_path) [h, w, c] = image.shape # the core: regress position map image = cv2.resize(image, (256, 256)) image_t = transform_img(image) image_t = image_t.unsqueeze(0) pos = prn.net_forward( image_t) # input image has been cropped to 256x256 out = pos.cpu().detach().numpy() pos = np.squeeze(out) cropped_pos = pos * 255 pos = cropped_pos.transpose(1, 2, 0) if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy() save_vertices[:, 1] = h - 1 - save_vertices[:, 1] if args.isImage: cv2.imwrite(os.path.join(save_folder, name + '.jpg'), image) if args.is3d: # corresponding colors colors = prn.get_colors(image, vertices) if args.isTexture: if args.texture_size != 256: pos_interpolated = cv2.resize( pos, (args.texture_size, args.texture_size), preserve_range=True) else: pos_interpolated = pos.copy() texture = cv2.remap(image, pos_interpolated[:, :, :2].astype( np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) if args.isMask: vertices_vis = get_visibility(vertices, prn.triangles, h, w) uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) uv_mask = cv2.resize( uv_mask, (args.texture_size, args.texture_size), preserve_range=True) texture = texture * uv_mask[:, :, np.newaxis] write_obj_with_texture( os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, texture, prn.uv_coords / prn.resolution_op ) # save 3d face with texture(can open with meshlab) else: write_obj_with_colors( os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, colors) # save 3d face(can open with meshlab) # if args.isDepth: # depth_image = get_depth_image(vertices, prn.triangles, h, w, True) # depth = get_depth_image(vertices, prn.triangles, h, w) # cv2.imwrite(os.path.join(save_folder, name + '_depth.jpg'), depth_image) # sio.savemat(os.path.join(save_folder, name + '_depth.mat'), {'depth': depth}) if args.isKpt or args.isShow: # get landmarks kpt = prn.get_landmarks(pos) np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) if args.isPose or args.isShow: # estimate pose camera_matrix, pose = estimate_pose(vertices) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) np.savetxt(os.path.join(save_folder, name + '_camera_matrix.txt'), camera_matrix) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) if args.isShow: # ---------- Plot image_pose = plot_pose_box(image, camera_matrix, kpt) cv2.imshow('sparse alignment', plot_kpt(image, kpt)) cv2.imshow('dense alignment', plot_vertices(image, vertices)) cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt)) cv2.waitKey(0)
def main(args): if args.isShow or args.isTexture: import cv2 from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib = args.isDlib) # ------------- load data image_folder = args.inputDir save_folder = args.outputDir if not os.path.exists(save_folder): os.mkdir(save_folder) types = ('*.jpg', '*.png') image_path_list= [] for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] # read image image = imread(image_path) [h, w, _] = image.shape # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size> 1000: image = rescale(image, 1000./max_size) pos = prn.process(image) # use dlib to detect face else: if image.shape[1] == image.shape[2]: image = resize(image, (256,256)) pos = prn.net_forward(image/255.) # input image has been cropped to 256x256 else: box = np.array([0, image.shape[1]-1, 0, image.shape[0]-1]) # cropped with bounding box pos = prn.process(image, box) image = image/255. if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices if args.isImage: imsave(os.path.join(save_folder, name + '.jpg'), image) if args.is3d: # corresponding colors colors = prn.get_colors(image, vertices) if args.isTexture: texture = cv2.remap(image, pos[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT,borderValue=(0)) if args.isMask: vertices_vis = get_visibility(vertices, prn.triangles, h, w) uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) texture = texture*uv_mask[:,:,np.newaxis] write_obj_with_texture(os.path.join(save_folder, name + '.obj'), save_vertices, colors, prn.triangles, texture, prn.uv_coords/prn.resolution_op)#save 3d face with texture(can open with meshlab) else: write_obj(os.path.join(save_folder, name + '.obj'), save_vertices, colors, prn.triangles) #save 3d face(can open with meshlab) if args.isDepth: depth_image = get_depth_image(vertices, prn.triangles, h, w) imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image) if args.isMat: sio.savemat(os.path.join(save_folder, name + '_mesh.mat'), {'vertices': save_vertices, 'colors': colors, 'triangles': prn.triangles}) if args.isKpt or args.isShow: # get landmarks kpt = prn.get_landmarks(pos) np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) if args.isPose or args.isShow: # estimate pose camera_matrix, pose = estimate_pose(vertices) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) if args.isShow: # ---------- Plot image_pose = plot_pose_box(image, camera_matrix, kpt) cv2.imshow('sparse alignment', plot_kpt(image, kpt)) cv2.imshow('dense alignment', plot_vertices(image, vertices)) cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt)) cv2.waitKey(0)
def main(args): print args.isDlib if args.isShow: args.isOpencv = True import cv2 from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box from utils.write import write_obj from utils.estimate_pose import estimate_pose elif args.is3d: from utils.write import write_obj elif args.isPose: from utils.estimate_pose import estimate_pose # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib=args.isDlib, is_opencv=args.isOpencv) # ------------- load data image_folder = args.inputDir save_folder = args.outputDir if not os.path.exists(save_folder): os.mkdir(save_folder) types = ('*.jpg', '*.png') image_path_list = [] for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] # read image image = imread(image_path) # the core: regress position map if args.isDlib: pos = prn.process(image) # use dlib to detect face else: if image.shape[1] == 256: pos = prn.net_forward( image / 255.) # input image has been cropped to 256x256 else: print('please make sure the image has been cropped') exit() if pos is None: continue if args.is3d or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) # corresponding colors colors = prn.get_colors(image, vertices) write_obj(os.path.join(save_folder, name + '.obj'), vertices, colors, prn.triangles) #save 3d face(can open with meshlab) if args.isKpt or args.isShow: # get landmarks kpt = prn.get_landmarks(pos) np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) if args.isPose or args.isShow: # estimate pose camera_matrix, pose = estimate_pose(vertices) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) if args.isShow: # ---------- Plot image_pose = plot_pose_box(image, camera_matrix, kpt) cv2.imshow('sparse alignment', plot_kpt(image, kpt)) cv2.imshow('dense alignment', plot_vertices(image, vertices)) cv2.imshow('pose', plot_pose_box(image, camera_matrix, kpt)) cv2.waitKey(0)
def main(args): if args.isShow or args.isTexture: import cv2 from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib = args.isDlib) # ------------- load data image_folder = args.inputDir print(image_folder) save_folder = args.outputDir print(save_folder) if not os.path.exists(save_folder): os.mkdir(save_folder) meta_save_folder = os.path.join(save_folder, 'meta') if not os.path.exists(meta_save_folder): os.mkdir(meta_save_folder) types = ('*.jpg', '*.png', '*,JPG') image_path_list= find_files(image_folder, ('.jpg', '.png', '.JPG')) total_num = len(image_path_list) print(image_path_list) for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] print(image_path) # read image image = imread(image_path) [h, w, c] = image.shape if c>3: image = image[:,:,:3] # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size> 1000: image = rescale(image, 1000./max_size) image = (image*255).astype(np.uint8) pos = prn.process(image) # use dlib to detect face else: if image.shape[1] == image.shape[2]: image = resize(image, (256,256)) pos = prn.net_forward(image/255.) # input image has been cropped to 256x256 else: box = np.array([0, image.shape[1]-1, 0, image.shape[0]-1]) # cropped with bounding box pos = prn.process(image, box) image = image/255. if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy() save_vertices[:,1] = h - 1 - save_vertices[:,1] if args.isImage: imsave(os.path.join(save_folder, name + '.jpg'), image) if args.is3d: # corresponding colors colors = prn.get_colors(image, vertices) if args.isTexture: if args.texture_size != 256: pos_interpolated = resize(pos, (args.texture_size, args.texture_size), preserve_range = True) else: pos_interpolated = pos.copy() texture = cv2.remap(image, pos_interpolated[:,:,:2].astype(np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT,borderValue=(0)) if args.isMask: vertices_vis = get_visibility(vertices, prn.triangles, h, w) uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) uv_mask = resize(uv_mask, (args.texture_size, args.texture_size), preserve_range = True) texture = texture*uv_mask[:,:,np.newaxis] write_obj_with_texture(os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, texture, prn.uv_coords/prn.resolution_op)#save 3d face with texture(can open with meshlab) else: write_obj_with_colors(os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, colors) #save 3d face(can open with meshlab) if args.isDepth: depth_image = get_depth_image(vertices, prn.triangles, h, w, True) depth = get_depth_image(vertices, prn.triangles, h, w) imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image) sio.savemat(os.path.join(meta_save_folder, name + '_depth.mat'), {'depth':depth}) if args.isMat: sio.savemat(os.path.join(meta_save_folder, name + '_mesh.mat'), {'vertices': vertices, 'colors': colors, 'triangles': prn.triangles}) if args.isKpt or args.isShow: # get landmarks kpt = prn.get_landmarks(pos) # pdb.set_trace() np.save(os.path.join(meta_save_folder, name + '_kpt.npy'), kpt) # cv2.imwrite(os.path.join(save_folder, name + '_skpt.jpg'), plot_kpt(image, kpt)) if args.isPose or args.isShow: # estimate pose camera_matrix, pose = estimate_pose(vertices) np.savetxt(os.path.join(meta_save_folder, name + '_pose.txt'), pose) np.savetxt(os.path.join(meta_save_folder, name + '_camera_matrix.txt'), camera_matrix) if args.isShow: # ---------- Plot image = imread(os.path.join(save_folder, name + '.jpg')) image_pose = plot_pose_box(image, camera_matrix, kpt) #cv2.imwrite(os.path.join(save_folder, name + '_pose.jpg'), plot_kpt(image, kpt)) #cv2.imwrite(os.path.join(save_folder, name + '_camera_matrix.jpg'), plot_vertices(image, vertices)) #cv2.imwrite(os.path.join(save_folder, name + '_pose.jpg'), plot_pose_box(image, camera_matrix, kpt)) image = imread(os.path.join(save_folder, name + '.jpg')) b, g, r = cv2.split(image) image = cv2.merge([r,g,b])
def main(args): if args.isShow or args.isTexture or args.isCamera: import cv2 from utils.cv_plot import plot_kpt, plot_vertices, plot_pose_box # ---- init PRN os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # GPU number, -1 for CPU prn = PRN(is_dlib=args.isDlib) # ------------- load data image_folder = args.inputDir save_folder = args.outputDir if not os.path.exists(save_folder): os.mkdir(save_folder) types = ('*.jpg', '*.png') image_path_list = [] for files in types: image_path_list.extend(glob(os.path.join(image_folder, files))) total_num = len(image_path_list) if args.isCamera: # Create a VideoCapture object and read from input file # If the input is the camera, pass 0 instead of the video file name cap = cv2.VideoCapture(0) # Check if camera opened successfully if (cap.isOpened() == False): print("Error opening video stream or file") # Read until video is completed while (cap.isOpened()): # Capture frame-by-frame ret, frame = cap.read() if ret == True: if args.isDlib: max_size = max(frame.shape[0], frame.shape[1]) if max_size > 1000: frame = rescale(frame, 1000. / max_size) frame = (frame * 255).astype(np.uint8) pos = prn.process(frame) # use dlib to detect face else: if frame.shape[0] == frame.shape[1]: frame = resize(frame, (256, 256)) pos = prn.net_forward( frame / 255.) # input frame has been cropped to 256x256 else: box = np.array( [0, frame.shape[1] - 1, 0, frame.shape[0] - 1]) # cropped with bounding box pos = prn.process(frame, box) # Normalizing the frame and skiping if there was no one in the frame frame = frame / 255. if pos is None: continue # Get landmarks in frame kpt = prn.get_landmarks(pos) # Display the resulting frame cv2.imshow('sparse alignment', plot_kpt(frame, kpt)) # Press Q on keyboard to exit if cv2.waitKey(25) & 0xFF == ord('q'): break # Break the loop else: break # When everything done, release the video capture object cap.release() # Closes all the frames cv2.destroyAllWindows() else: for i, image_path in enumerate(image_path_list): name = image_path.strip().split('/')[-1][:-4] # read image image = imread(image_path) [h, w, c] = image.shape if c > 3: image = image[:, :, :3] # the core: regress position map if args.isDlib: max_size = max(image.shape[0], image.shape[1]) if max_size > 1000: image = rescale(image, 1000. / max_size) image = (image * 255).astype(np.uint8) pos = prn.process(image) # use dlib to detect face else: if image.shape[0] == image.shape[1]: image = resize(image, (256, 256)) pos = prn.net_forward( image / 255.) # input image has been cropped to 256x256 else: box = np.array( [0, image.shape[1] - 1, 0, image.shape[0] - 1]) # cropped with bounding box pos = prn.process(image, box) image = image / 255. if pos is None: continue if args.is3d or args.isMat or args.isPose or args.isShow: # 3D vertices vertices = prn.get_vertices(pos) if args.isFront: save_vertices = frontalize(vertices) else: save_vertices = vertices.copy() save_vertices[:, 1] = h - 1 - save_vertices[:, 1] if args.isImage: imsave(os.path.join(save_folder, name + '.jpg'), image) if args.is3d: # corresponding colors colors = prn.get_colors(image, vertices) if args.isTexture: if args.texture_size != 256: pos_interpolated = resize( pos, (args.texture_size, args.texture_size), preserve_range=True) else: pos_interpolated = pos.copy() texture = cv2.remap(image, pos_interpolated[:, :, :2].astype( np.float32), None, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) if args.isMask: vertices_vis = get_visibility(vertices, prn.triangles, h, w) uv_mask = get_uv_mask(vertices_vis, prn.triangles, prn.uv_coords, h, w, prn.resolution_op) uv_mask = resize( uv_mask, (args.texture_size, args.texture_size), preserve_range=True) texture = texture * uv_mask[:, :, np.newaxis] write_obj_with_texture( os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, texture, prn.uv_coords / prn.resolution_op ) #save 3d face with texture(can open with meshlab) else: write_obj_with_colors( os.path.join(save_folder, name + '.obj'), save_vertices, prn.triangles, colors) #save 3d face(can open with meshlab) if args.isDepth: depth_image = get_depth_image(vertices, prn.triangles, h, w, True) depth = get_depth_image(vertices, prn.triangles, h, w) imsave(os.path.join(save_folder, name + '_depth.jpg'), depth_image) sio.savemat(os.path.join(save_folder, name + '_depth.mat'), {'depth': depth}) if args.isMat: sio.savemat( os.path.join(save_folder, name + '_mesh.mat'), { 'vertices': vertices, 'colors': colors, 'triangles': prn.triangles }) if args.isKpt or args.isShow: # get landmarks kpt = prn.get_landmarks(pos) np.savetxt(os.path.join(save_folder, name + '_kpt.txt'), kpt) if args.isPose or args.isShow: # estimate pose camera_matrix, pose = estimate_pose(vertices) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) np.savetxt( os.path.join(save_folder, name + '_camera_matrix.txt'), camera_matrix) np.savetxt(os.path.join(save_folder, name + '_pose.txt'), pose) if args.isShow: # ---------- Plot image_pose = plot_pose_box(image, camera_matrix, kpt) cv2.imshow( 'sparse alignment', cv2.cvtColor(np.float32(plot_kpt(image, kpt)), cv2.COLOR_RGB2BGR)) cv2.imshow( 'dense alignment', cv2.cvtColor(np.float32(plot_vertices(image, vertices)), cv2.COLOR_RGB2BGR)) cv2.imshow( 'pose', cv2.cvtColor( np.float32(plot_pose_box(image, camera_matrix, kpt)), cv2.COLOR_RGB2BGR)) cv2.waitKey(0)
def getFacialLandmarks(isDlib, img_, numFaces=1): img = copy.deepcopy(img_) # use dlib or PrNetfor prediction of facial landmarks if isDlib == "True": # load shape predictor model model_path = 'Code/dlib_model/shape_predictor_68_face_landmarks.dat' # load the detector and the predictor. # predictor accepts pre-trained model as input detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor(model_path) img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) rects = detector(img_gray, 1) # store landmark locations of both faces landmarkCoordAll = [] # iterate through the points in both faces for r, rect in enumerate(rects): landmarks = predictor(img_gray, rect) # reshape landmarks to (68X2) landmarkCoord = np.zeros((68, 2), dtype='int') for i in range(68): landmarkCoord[i] = (landmarks.part(i).x, landmarks.part(i).y) landmarkCoordAll.append(landmarkCoord) # draw bounding box on face cv2.rectangle(img, (rect.left(), rect.top()), (rect.right(), rect.bottom()), (0, 255, 255), 0) # draw facial landmarks img_ = drawFacialLandmarks(img, landmarkCoord) if isDlib == "False": # prn uses dlib for face detection and its own trained model for prediction of facial landmarks prn = PRN(is_dlib = True, prefix='Code/prnet/') [h, w, c] = img.shape if c>3: img = img[:,:,:3] if img.shape[0] == img.shape[1]: img = resize(img, (256,256)) pos = prn.net_forward(img/255.) # input image has been cropped to 256x256 else: posList = [] for i in range(numFaces): pos = prn.process(img, i) posList.append(pos) landmarkCoordAll = [] for i, pos in enumerate(posList): if pos is None: return img_, landmarkCoordAll # get landmark points of face landmarkCoord = prn.get_landmarks(pos) img_ = plot_kpt(img_, landmarkCoord) landmarkCoord = landmarkCoord[:, 0:2] landmarkCoordAll.append(landmarkCoord) return img_, landmarkCoordAll