def render_hmr_smpl(hmr_coeff, f_len=500., rend_size=224., req_model=False): # hmr_coeff is a 85-vector, named as theta in hmr hmr_coeff = np.asarray(hmr_coeff).tolist() # make scene scene = pyrender.Scene() # initialize camera camera = pyrender.PerspectiveCamera( yfov=np.arctan(rend_size * 0.5 / f_len) * 2, aspectRatio=1) camera_pose = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0]]) scene.add(camera, pose=camera_pose) # initialize light light_posi1 = np.array([[1.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, -1.0], [0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]]) light_posi2 = np.array([[1.0, 0.0, 0.0, -1.0], [0.0, 0.0, 1.0, -1.0], [0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]]) light_posi3 = np.array([[1.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, 1.0], [0.0, 1.0, 0.0, -2.0], [0.0, 0.0, 0.0, 1.0]]) light = pyrender.SpotLight(color=np.array( [0.65098039, 0.74117647, 0.85882353]), intensity=100, innerConeAngle=np.pi / 16.0, outerConeAngle=np.pi / 6.0) scene.add(light, pose=light_posi1) scene.add(light, pose=light_posi2) scene.add(light, pose=light_posi3) # get renderer r = pyrender.OffscreenRenderer(viewport_width=rend_size, viewport_height=rend_size) # get verts from smpl coefficients smpl_op = load_model("./tf_smpl/neutral_smpl_with_cocoplus_reg.pkl") smpl_op.pose[:] = np.asarray(hmr_coeff[3:75]) smpl_op.betas[:] = np.array(hmr_coeff[75:85]) verts = np.array(smpl_op) global_t = np.array( [hmr_coeff[1], hmr_coeff[2], f_len / (0.5 * rend_size * hmr_coeff[0])]) verts = verts + global_t faces = np.load("./tf_smpl/smpl_faces.npy").astype(np.int32) # smooth and expand om_mesh = make_trimesh(verts, faces) om_mesh = smooth_mesh(om_mesh, 4) om_mesh = expand_mesh(om_mesh, 0.026) this_trimesh = trimesh.Trimesh(vertices=om_mesh.points(), faces=om_mesh.face_vertex_indices()) this_mesh = pyrender.Mesh.from_trimesh(this_trimesh) scene.add(this_mesh) rend_img, depth = r.render(scene) if req_model is True: return rend_img, verts, faces, depth else: return rend_img
def proc_coco(train_dir, test_dir, train_id, test_id, coco_dataset): # read dataset coco = COCO(coco_dataset) tupleIds = coco.getImgIds(catIds=1) # id = 1 means person faces = np.load("../predef/smpl_faces.npy") face_num = len(faces) hmr_pred = hmr_predictor() renderer = rd.SMPLRenderer(face_path="../predef/smpl_faces.npy") with open('../predef/mesh_joint_list.pkl', 'rb') as fp: mesh_joint = pickle.load(fp) count_all = 0. count_work = 0. total_num = len(tupleIds) train_num = int(np.floor(total_num * 0.8)) # make train set tr = trange(3584, train_num, desc='Bar desc', leave=True) # for i in tr: # tr.set_description("COCO - train part") # tr.refresh() # to show immediately the update # sleep(0.01) # count_all += 1 # # get tuple # one_tuple = coco.loadImgs(tupleIds[i])[0] # img_size = (one_tuple['height'], one_tuple['width']) # crt_id = one_tuple['id'] # # get anns # annIds = coco.getAnnIds(imgIds=one_tuple['id'], catIds=1, iscrowd=None) # anns = coco.loadAnns(annIds) # # RULE 1: objects < 5 # if len(anns)>4: # #print("filter out by too many objects") # take_notes("COCO %05d BAN -1\n" % (i), "./data_log.txt") # continue # for j in range(len(anns)): # # get sil points # seg_points = anns[j]['segmentation'][0] # # RULE 2: seg_points number >= 80 # if len(seg_points)<80: # take_notes("COCO %05d%03d BAN -1\n" % (i, j), "./data_log.txt") # #print("filter out by too few seg_points number") # continue # # get key points # key_points = anns[j]['keypoints'] # key_points = np.resize(key_points,(17,3)) # # draw sil # sil = points2sil(seg_points, img_size) # result = coco_filter(key_points, sil) # if result is False: # take_notes("COCO %05d BAN -1\n" % (i), "./data_log.txt") # continue # # Here we finally decide to use it # if result is True: # # read image # print(one_tuple['coco_url']) # ori_img = io.imread(one_tuple['coco_url']) # # read sil # src_gt_sil = sil # # hmr predict # verts, cam, proc_para, std_img = hmr_pred.predict(ori_img, # True, # src_gt_sil) # # unnormalize std_img # src_img = ((std_img+1)/2.0*255).astype(np.uint8) # # save img # img_file = "img/COCO_%08d%02d.png" % (crt_id, j) # PIL.Image.fromarray(src_img).save(train_dir + img_file) # # process sil # gt_sil = proc_sil(src_gt_sil, proc_para) # # get proj sil # proj_sil = renderer.silhouette(verts = verts, # cam = cam, # img_size = src_img.shape, # norm = False) # # make TriMesh # mesh = make_trimesh(verts, faces, compute_vn = True) # vert_norms = mesh.vertex_normals() # # get joint move # coco_joints_t = transform_coco_joints(key_points) # new_jv, _, joint_move, joint_posi = get_joint_move(verts, # coco_joints_t, # proc_para, # mesh_joint, # unseen_mode = True, # ) # joint_move = joint_move.flatten() # # joint deform # fd_ja = fast_deform_dja(weight = 10.0) # ja_verts = fd_ja.deform(np.asarray(verts), new_jv) # # get achr move # proj_sil_ja = renderer.silhouette(verts = ja_verts, # norm = False) # _, achr_verts, achr_move = get_achr_move(gt_sil, # ja_verts, # vert_norms, # proj_sil_ja) # achr_posi = get_anchor_posi(achr_verts) # # save sil # compose_sil = np.stack((gt_sil, proj_sil, proj_sil_ja)) # compose_sil = np.moveaxis(compose_sil, 0, 2) # compose_sil = PIL.Image.fromarray(compose_sil.astype(np.uint8)) # compose_sil.save(train_dir + "sil/%08d.png" % train_id) # # save para # proc_para['end_pt'] = proc_para['end_pt'].tolist() # proc_para['start_pt'] = proc_para['start_pt'].tolist() # para = {"verts": verts.tolist(), # "vert_norms": vert_norms.tolist(), # "proc_para": proc_para, # "joint_move": joint_move.tolist(), # "joint_posi": joint_posi.tolist(), # "achr_move": achr_move.tolist(), # "achr_posi": achr_posi.tolist(), # "img_file": img_file, # } # with open(train_dir + "para/%08d.json" % train_id, 'wb') as fp: # json.dump(para, fp) # take_notes("COCO %05d%03d TRAIN %08d\n" % (i, j, train_id), # "./data_log.txt") # train_id += 1 # count_work += 1 # make test set test_num = total_num - train_num tr = trange(1855, test_num, desc='Bar desc', leave=True) for i in tr: tr.set_description("COCO - test part") tr.refresh() # to show immediately the update sleep(0.01) count_all += 1 # get tuple one_tuple = coco.loadImgs(tupleIds[i + train_num])[0] img_size = (one_tuple['height'], one_tuple['width']) crt_id = one_tuple['id'] # get anns annIds = coco.getAnnIds(imgIds=one_tuple['id'], catIds=1, iscrowd=None) anns = coco.loadAnns(annIds) # RULE 1: objects < 4 if len(anns) > 3: #print("filter out by too many objects") take_notes("COCO %05d BAN -1\n" % (i + train_num), "./data_log.txt") continue for j in range(len(anns)): # get sil points seg_points = anns[j]['segmentation'][0] # RULE 2: seg_points number >= 100 if len(seg_points) < 100: take_notes("COCO %05d%03d BAN -1\n" % (i + train_num, j), "./data_log.txt") #print("filter out by too few seg_points number") continue # get key points key_points = anns[j]['keypoints'] key_points = np.resize(key_points, (17, 3)) # draw sil sil = points2sil(seg_points, img_size) result = coco_filter(key_points, sil) if result is False: take_notes("COCO %05d BAN -1\n" % (i + train_num), "./data_log.txt") continue # Here we finally decide to use it if result is True: # read image ori_img = io.imread(one_tuple['coco_url']) # read sil src_gt_sil = sil # hmr predict verts, cam, proc_para, std_img = hmr_pred.predict( ori_img, True, src_gt_sil) # unnormalize std_img src_img = ((std_img + 1) / 2.0 * 255).astype(np.uint8) # save img img_file = "img/COCO_%08d%02d.png" % (crt_id, j) PIL.Image.fromarray(src_img).save(test_dir + img_file) # process sil gt_sil = proc_sil(src_gt_sil, proc_para) # get proj sil proj_sil = renderer.silhouette(verts=verts, cam=cam, img_size=src_img.shape, norm=False) # make TriMesh mesh = make_trimesh(verts, faces, compute_vn=True) vert_norms = mesh.vertex_normals() # get joint move coco_joints_t = transform_coco_joints(key_points) new_jv, _, joint_move, joint_posi = get_joint_move( verts, coco_joints_t, proc_para, mesh_joint, unseen_mode=True, ) joint_move = joint_move.flatten() # joint deform fd_ja = fast_deform_dja(weight=10.0) ja_verts = fd_ja.deform(np.asarray(verts), new_jv) # get achr move proj_sil_ja = renderer.silhouette(verts=ja_verts, norm=False) _, achr_verts, achr_move = get_achr_move( gt_sil, ja_verts, vert_norms, proj_sil_ja) achr_posi = get_anchor_posi(achr_verts) # save sil compose_sil = np.stack((gt_sil, proj_sil, proj_sil_ja)) compose_sil = np.moveaxis(compose_sil, 0, 2) compose_sil = PIL.Image.fromarray(compose_sil.astype(np.uint8)) compose_sil.save(test_dir + "sil/%08d.png" % test_id) # save para proc_para['end_pt'] = proc_para['end_pt'].tolist() proc_para['start_pt'] = proc_para['start_pt'].tolist() para = { "verts": verts.tolist(), "vert_norms": vert_norms.tolist(), "proc_para": proc_para, "joint_move": joint_move.tolist(), "joint_posi": joint_posi.tolist(), "achr_move": achr_move.tolist(), "achr_posi": achr_posi.tolist(), "img_file": img_file, } with open(test_dir + "para/%08d.json" % test_id, 'wb') as fp: json.dump(para, fp) take_notes( "COCO %05d%03d TEST %08d\n" % (i + train_num, j, test_id), "./data_log.txt") test_id += 1 count_work += 1 print("work ratio = %f, (%d / %d)" % (count_work / count_all, count_work, count_all)) return train_id, test_id
def proc_mpii(train_dir, test_dir, train_id, test_id, upi_dir): mpii_dir = upi_dir + "data/mpii/" faces = np.load("../predef/smpl_faces.npy") face_num = len(faces) hmr_pred = hmr_predictor() renderer = rd.SMPLRenderer(face_path = "../predef/smpl_faces.npy") mpii_joints_o = np.load(mpii_dir + "poses.npz")['poses'] mpii_joints = transform_mpii_joints(mpii_joints_o) with open ('../predef/mesh_joint_list.pkl', 'rb') as fp: mesh_joint = pickle.load(fp) count_all = 0. count_work = 0. # make train set tr = trange(10424, desc='Bar desc', leave=True) for i in tr: tr.set_description("MPII - train part") tr.refresh() # to show immediately the update sleep(0.01) count_all += 1 # read sil src_gt_sil = np.array(PIL.Image.open(mpii_dir + \ "images/%05d_segmentation.png"%(i+1)))[:,:,0] # judge using filter result = mpii_filter(mpii_joints[:,:,i], src_gt_sil) if result is False: take_notes("MPII %05d BAN -1\n" % (i+1), "./data_log.txt") continue # read ori img ori_img = np.array(PIL.Image.open( mpii_dir + "images/%05d.png"%(i+1))) # hmr predict verts, cam, proc_para, std_img = hmr_pred.predict(ori_img, True, src_gt_sil) # unnormalize std_img src_img = ((std_img+1).astype(np.float)/2.0*255).astype(np.uint8) # save img img_file = "img/MPII_%08d.png" % (i + 1) PIL.Image.fromarray(src_img).save(train_dir + img_file) # process sil gt_sil = proc_sil(src_gt_sil, proc_para) # get proj sil proj_sil = renderer.silhouette(verts = verts, cam = cam, img_size = src_img.shape, norm = False) # make TriMesh mesh = make_trimesh(verts, faces, compute_vn = True) vert_norms = mesh.vertex_normals() # get joint move new_jv, _, joint_move, joint_posi = get_joint_move(verts, mpii_joints[:,:,i], proc_para, mesh_joint) joint_move = joint_move.flatten() # joint deform fd_ja = fast_deform_dja(weight = 10.0) ja_verts = fd_ja.deform(np.asarray(verts), new_jv) # get achr move proj_sil_ja = renderer.silhouette(verts = ja_verts, norm = False) _, achr_verts, achr_move = get_achr_move(gt_sil, ja_verts, vert_norms, proj_sil_ja) achr_posi = get_anchor_posi(achr_verts) # save sil compose_sil = np.stack((gt_sil, proj_sil, proj_sil_ja)) compose_sil = np.moveaxis(compose_sil, 0, 2) compose_sil = PIL.Image.fromarray(compose_sil.astype(np.uint8)) compose_sil.save(train_dir + "sil/%08d.png" % train_id) # save para proc_para['end_pt'] = proc_para['end_pt'].tolist() proc_para['start_pt'] = proc_para['start_pt'].tolist() para = {"verts": verts.tolist(), "vert_norms": vert_norms.tolist(), "proc_para": proc_para, "joint_move": joint_move.tolist(), "joint_posi": joint_posi.tolist(), "achr_move": achr_move.tolist(), "achr_posi": achr_posi.tolist(), "img_file": img_file, } with open(train_dir + "para/%08d.json" % train_id, 'wb') as fp: json.dump(para, fp) take_notes("MPII %05d TRAIN %08d\n" % (i+1, train_id), "./data_log.txt") train_id += 1 count_work += 1 #make test set tr = trange(2606, desc='Bar desc', leave=True) for i in tr: tr.set_description("MPII - test part") tr.refresh() # to show immediately the update sleep(0.01) count_all += 1 # read sil src_gt_sil = np.array(PIL.Image.open(mpii_dir + \ "images/%05d_segmentation.png"%(i+10425)))[:,:,0] # judge using filter result = mpii_filter(mpii_joints[:,:,i+10424], src_gt_sil) if result is False: take_notes("MPII %05d BAN -1\n" % (i+10425), "./data_log.txt") continue # read ori img ori_img = np.array(PIL.Image.open( mpii_dir + "images/%05d.png"%(i+10425))) # hmr predict verts, cam, proc_para, std_img = hmr_pred.predict(ori_img, True, src_gt_sil) # unnormalize std_img src_img = ((std_img+1).astype(np.float)/2.0*255).astype(np.uint8) # save img img_file = "img/MPII_%08d.png" % (i+10425) PIL.Image.fromarray(src_img).save(test_dir + img_file) # process sil gt_sil = proc_sil(src_gt_sil, proc_para) # get proj sil proj_sil = renderer.silhouette(verts = verts, cam = cam, img_size = src_img.shape, norm = False) # make TriMesh mesh = make_trimesh(verts, faces, compute_vn = True) vert_norms = mesh.vertex_normals() # get joint move new_jv, _, joint_move, joint_posi = get_joint_move(verts, mpii_joints[:,:,i+10424], proc_para, mesh_joint) joint_move = joint_move.flatten() # joint deform fd_ja = fast_deform_dja(weight = 10.0) ja_verts = fd_ja.deform(np.asarray(verts), new_jv) # get achr move proj_sil_ja = renderer.silhouette(verts = ja_verts, norm = False) _, achr_verts, achr_move = get_achr_move(gt_sil, ja_verts, vert_norms, proj_sil_ja) achr_posi = get_anchor_posi(achr_verts) # save sil compose_sil = np.stack((gt_sil, proj_sil, proj_sil_ja)) compose_sil = np.moveaxis(compose_sil, 0, 2) compose_sil = PIL.Image.fromarray(compose_sil.astype(np.uint8)) compose_sil.save(test_dir + "sil/%08d.png" % test_id) # save para proc_para['end_pt'] = proc_para['end_pt'].tolist() proc_para['start_pt'] = proc_para['start_pt'].tolist() para = {"verts": verts.tolist(), "vert_norms": vert_norms.tolist(), "proc_para": proc_para, "joint_move": joint_move.tolist(), "joint_posi": joint_posi.tolist(), "achr_move": achr_move.tolist(), "achr_posi": achr_posi.tolist(), "img_file": img_file, } with open(test_dir + "para/%08d.json" % test_id, 'wb') as fp: json.dump(para, fp) take_notes("MPII %05d TEST %08d\n" % (i+10425, test_id), "./data_log.txt") test_id += 1 count_work += 1 print("work ratio = %f, (%d / %d)" % (count_work/count_all, count_work, count_all)) return train_id, test_id
if opt.gpu is True: torch.cuda.empty_cache() # prepare shading refine device = torch.device("cuda:0" if opt.gpu else "cpu") net_shading = shading_net(init_weights=True).eval().to(device) if opt.gpu is True: net_shading.load_state_dict( torch.load(shading_model, map_location='cuda:0')) # load pretrained model else: net_shading.load_state_dict(torch.load(shading_model, map_location='cpu')) faces_smpl = np.load("../predef/smpl_faces.npy") mesh = make_trimesh(sa_verts, faces_smpl) # mesh after adjust joint and anchor proj_sil = my_renderer.silhouette(verts=mesh.points()) proj_sil_l = cv2.resize(proj_sil, dsize=(448, 448)) proj_sil_l[proj_sil_l < 0.5] = 0 proj_sil_l[proj_sil_l >= 0.5] = 1 # img = PIL.Image.fromarray(proj_sil_l) # img.show() # load data src_img_l = cv2.resize(src_img, dsize=(448, 448)) # img = PIL.Image.fromarray(src_img_l) # img.show() input_arr = np.rollaxis(src_img_l, 2, 0) input_arr = np.expand_dims(input_arr, 0) input_arr = input_arr / 255.0
def proc_lspet(train_dir, train_id, lspet_dir, upi_dir): faces = np.load("../predef/smpl_faces.npy") face_num = len(faces) hmr_pred = hmr_predictor() renderer = rd.SMPLRenderer(face_path = "../predef/smpl_faces.npy") lspet_joints = loadmat(lspet_dir + "joints.mat")['joints'] # roll axis because the definition order of lspet is different from lsp lspet_joints = np.rollaxis(lspet_joints,1,0) with open ('../predef/mesh_joint_list.pkl', 'rb') as fp: mesh_joint = pickle.load(fp) count_all = 0. count_work = 0. # make train set tr = trange(10000, desc='Bar desc', leave=True) for i in tr: tr.set_description("LSPET - train part") tr.refresh() # to show immediately the update sleep(0.01) count_all += 1 # judge if sil file exists, if not, skip it if not os.path.isfile(upi_dir + \ "lsp_extended/im%05d_segmentation.png"%(i+1)): take_notes("LSPET %05d BAN -1\n" % (i+1), "./data_log.txt") continue # read sil src_gt_sil = np.array(PIL.Image.open(upi_dir + \ "lsp_extended/im%05d_segmentation.png"%(i+1)))[:,:,0] # judge using filter result = lspet_filter(lspet_joints[:,:,i], src_gt_sil) if result is False: take_notes("LSPET %05d BAN -1\n" % (i+1), "./data_log.txt") continue # read ori img ori_img = np.array(PIL.Image.open( lspet_dir + "images/im%05d.jpg"%(i+1))) # hmr predict verts, cam, proc_para, std_img = hmr_pred.predict(ori_img, True, src_gt_sil) # unnormalize std_img src_img = ((std_img+1).astype(np.float)/2.0*255).astype(np.uint8) # save img img_file = "img/LSPET_%08d.png" % (i + 1) PIL.Image.fromarray(src_img).save(train_dir + img_file) # process sil gt_sil = proc_sil(src_gt_sil, proc_para) # get proj sil proj_sil = renderer.silhouette(verts = verts, cam = cam, img_size = src_img.shape, norm = False) # make TriMesh mesh = make_trimesh(verts, faces, compute_vn = True) vert_norms = mesh.vertex_normals() # get joint move new_jv, _, joint_move, joint_posi = get_joint_move(verts, lspet_joints[:,:,i], proc_para, mesh_joint, unseen_mode = True) joint_move = joint_move.flatten() # joint deform fd_ja = fast_deform_dja(weight = 10.0) ja_verts = fd_ja.deform(np.asarray(verts), new_jv) # get achr move proj_sil_ja = renderer.silhouette(verts = ja_verts, norm = False) _, achr_verts, achr_move = get_achr_move(gt_sil, ja_verts, vert_norms, proj_sil_ja) achr_posi = get_anchor_posi(achr_verts) # save sil compose_sil = np.stack((gt_sil, proj_sil, proj_sil_ja)) compose_sil = np.moveaxis(compose_sil, 0, 2) compose_sil = PIL.Image.fromarray(compose_sil.astype(np.uint8)) compose_sil.save(train_dir + "sil/%08d.png" % train_id) # save para proc_para['end_pt'] = proc_para['end_pt'].tolist() proc_para['start_pt'] = proc_para['start_pt'].tolist() para = {"verts": verts.tolist(), "vert_norms": vert_norms.tolist(), "proc_para": proc_para, "joint_move": joint_move.tolist(), "joint_posi": joint_posi.tolist(), "achr_move": achr_move.tolist(), "achr_posi": achr_posi.tolist(), "img_file": img_file, } with open(train_dir + "para/%08d.json" % train_id, 'wb') as fp: json.dump(para, fp) take_notes("LSPET %05d TRAIN %08d\n" % (i+1, train_id), "./data_log.txt") train_id += 1 count_work += 1 print("work ratio = %f, (%d / %d)" % (count_work/count_all, count_work, count_all)) return train_id
# Shading Refine Start if opt.gpu is True: torch.cuda.empty_cache() # prepare shading refine device = torch.device("cuda:0" if opt.gpu else "cpu") net_shading = shading_net(init_weights=True).eval().to(device) if opt.gpu is True: net_shading.load_state_dict( torch.load(shading_model, map_location='cuda:0')) else: net_shading.load_state_dict(torch.load(shading_model, map_location='cpu')) faces_smpl = np.load("../predef/smpl_faces.npy") mesh = make_trimesh(sa_verts, faces_smpl) proj_sil = my_renderer.silhouette(verts=mesh.points()) proj_sil_l = cv2.resize(proj_sil, dsize=(448, 448)) proj_sil_l[proj_sil_l < 0.5] = 0 proj_sil_l[proj_sil_l >= 0.5] = 1 # load data src_img_l = cv2.resize(src_img, dsize=(448, 448)) input_arr = np.rollaxis(src_img_l, 2, 0) input_arr = np.expand_dims(input_arr, 0) input_arr = torch.tensor(input_arr).float().to(device) input_arr = input_arr / 255.0 proj_sil_l = np.expand_dims(proj_sil_l, 0)
visi_vert_inds.append(fv[0]) visi_vert_inds.append(fv[1]) visi_vert_inds.append(fv[2]) visi_vert_inds = set(visi_vert_inds) # filter out exempt version visi_vert_inds = list(set(visi_vert_inds).difference(exempt_vert_list)) visi_vert_inds_m = [] for i in visi_vert_inds: xy = cam_para.project(verts[i]) x = int(round(xy[1])) y = int(round(xy[0])) if x < 0 or y < 0 or x >= 448 or y >= 448: continue if np.absolute(proj_depth[x, y] - verts[i, 2]) < 0.01: visi_vert_inds_m.append(i) for i in visi_vert_inds_m: xy = cam_para.project(verts[i]) x = int(round(xy[1])) y = int(round(xy[0])) depth = proj_depth[x, y] + pred_depth[x, y] #print(depth, verts[i]) if depth > 8.: continue verts[i][2] = depth deformed_mesh = make_trimesh(verts, faces) om.write_mesh("./eval_data/%s_set/pred_save/s_%03d.obj" % \ (opt.set, test_num), deformed_mesh)
assert opt.set in ["recon", "syn"], \ "set must be one of [recon, syn]" data_num = int(opt.num) output_dir = "./eval_data/%s_set/pred_save/" % opt.set if not os.path.exists(output_dir): os.makedirs(output_dir) hmr_pred = hmr_predictor() faces = np.load("../predef/smpl_faces.npy") tr = trange(data_num, desc='Bar desc', leave=True) for i in tr: src_img = np.array(PIL.Image.open("./eval_data/%s_set/input_masked/%03d_img.png" % \ (opt.set, i)))[:,:,:3] verts, cam, proc_para, std_img = hmr_pred.predict(src_img) with open(output_dir + "pre_%03d.pkl" % i, 'wb') as fp: pickle.dump( { "verts": verts, "cam": cam, "proc_para": proc_para, "std_img": std_img, }, fp) mesh = make_trimesh(verts, faces) om.write_mesh("./eval_data/%s_set/pred_save/hmr_%03d.obj" % (opt.set, i), mesh)
tr = trange(data_num, desc='Bar desc', leave=True) for test_ind in tr: src_img = np.array(PIL.Image.open("./eval_data/%s_set/input_masked/%03d_img.png" \ % (opt.set, test_ind))) #verts, cam, proc_para, std_img = hmr_pred.predict(src_img) with open ('./eval_data/%s_set/pred_save/pre_%03d.pkl' % \ (opt.set, test_ind), 'rb') as fp: hmr_data = pickle.load(fp) verts = hmr_data['verts'] cam = hmr_data['cam'] proc_para = hmr_data['proc_para'] std_img = hmr_data['std_img'] mesh = make_trimesh(verts, faces, compute_vn=True) vert_norms = mesh.vertex_normals() # get proj sil proj_sil = renderer.silhouette(verts=verts, cam=cam, img_size=src_img.shape, norm=True) # make joint posi joint_posi = get_joint_posi(verts, point_list) sil_j = np.expand_dims(proj_sil, 2) src_j = np.zeros((10, 4, 64, 64)) for i in range(len(joint_posi)): crop_sil = center_crop(sil_j, joint_posi[i], 64)
if opt.crop_y != -1: if len(opt.crop_y) != 2: print("ERROR: crop_y must be a list with 2 elements") crop_img = crop_img[opt.crop_y[0]:-opt.crop_y[1], :] if opt.pad>0: crop_img = pad_arr(crop_img, 50) std_img, proc_para = preproc_img(crop_img, img_size = 224, margin = 30, normalize = True) # initialize hmr_pred = hmr_predictor() renderer = SMPLRenderer() faces = np.load("../predef/smpl_faces.npy") # hmr predict verts, cam, proc_para, std_img = hmr_pred.predict(std_img, normalize=False) # build output folder if not exist if not os.path.exists(opt.outf): os.makedirs(opt.outf) # write results result_mesh = make_trimesh(verts, faces) om.write_mesh(opt.outf + "hmr_mesh.obj", result_mesh) final_img = ((std_img.copy()+1)*127).astype(np.uint8) PIL.Image.fromarray(final_img).save(opt.outf + "std_img.jpg") print("%s - finished, results are saved to [%s]" % (opt.img, opt.outf)) print("hmr done")
def proc_h36m(train_dir, test_dir, train_id, test_id, h36m_dir): sample_interval = 10 faces = np.load("../predef/smpl_faces.npy") face_num = len(faces) with open('../predef/mesh_joint_list.pkl', 'rb') as fp: mesh_joint = pickle.load(fp) hmr_pred = hmr_predictor() renderer = rd.SMPLRenderer(face_path="../predef/smpl_faces.npy") # open available video list with open("./h36m_list.txt") as f: h36m_list = f.read().split("\r\n") vid_num = int(h36m_list[0]) h36m_list = [[ h36m_list[i * 3 + 1], h36m_list[i * 3 + 2], h36m_list[i * 3 + 3] ] for i in range(vid_num)] # compute data number for training and testing train_num = int(vid_num * 0.8) test_num = vid_num - train_num count_all = 0. count_work = 0. # make test set tr = trange(test_num, desc='Bar desc', leave=True) for i in tr: tr.set_description("H36M - test part") tr.refresh() # to show immediately the update sleep(0.01) vid_idx = i + train_num # read video of image, silhouette and pose vid_img = cv2.VideoCapture(h36m_dir + h36m_list[vid_idx][0]) vid_sil = cv2.VideoCapture(h36m_dir + h36m_list[vid_idx][1]) pose_list = loadmat(h36m_dir + h36m_list[vid_idx][2])['pose'] vid_len = min(int(vid_img.get(cv2.CAP_PROP_FRAME_COUNT)), int(vid_sil.get(cv2.CAP_PROP_FRAME_COUNT)), len(pose_list)) for frm_idx in range(0, vid_len, sample_interval): count_all += 1 # read sil vid_sil.set(1, frm_idx) _, src_gt_sil = vid_sil.read() src_gt_sil[src_gt_sil < 128] = 0 src_gt_sil[src_gt_sil >= 128] = 255 src_gt_sil = refine_sil(src_gt_sil, 100) src_gt_sil = src_gt_sil[:, :, 0] # read ori img vid_img.set(1, frm_idx) _, ori_img = vid_img.read() # BGR to RGB ori_img = np.stack( (ori_img[:, :, 2], ori_img[:, :, 1], ori_img[:, :, 0]), axis=2) # hmr predict verts, cam, proc_para, std_img = hmr_pred.predict( ori_img, True, src_gt_sil) # unnormalize std_img src_img = ((std_img + 1).astype(np.float) / 2.0 * 255).astype( np.uint8) # save img img_file = "img/H36M_%04d%04d.png" % (vid_idx, frm_idx) PIL.Image.fromarray(src_img).save(test_dir + img_file) # process sil gt_sil = proc_sil(src_gt_sil, proc_para) # get proj sil proj_sil = renderer.silhouette(verts=verts, cam=cam, img_size=src_img.shape, norm=False) # make TriMesh mesh = make_trimesh(verts, faces, compute_vn=True) vert_norms = mesh.vertex_normals() h36m_joint = transform_h36m_joints(pose_list[frm_idx]) # get joint move new_jv, _, joint_move, joint_posi = get_joint_move( verts, h36m_joint, proc_para, mesh_joint) joint_move = joint_move.flatten() # joint deform fd_ja = fast_deform_dja(weight=10.0) ja_verts = fd_ja.deform(np.asarray(verts), new_jv) # get achr move proj_sil_ja = renderer.silhouette(verts=ja_verts, norm=False) _, achr_verts, achr_move = get_achr_move(gt_sil, ja_verts, vert_norms, proj_sil_ja) achr_posi = get_anchor_posi(achr_verts) # save sil compose_sil = np.stack((gt_sil, proj_sil, proj_sil_ja)) compose_sil = np.moveaxis(compose_sil, 0, 2) compose_sil = PIL.Image.fromarray(compose_sil.astype(np.uint8)) compose_sil.save(test_dir + "sil/%08d.png" % test_id) # save para proc_para['end_pt'] = proc_para['end_pt'].tolist() proc_para['start_pt'] = proc_para['start_pt'].tolist() para = { "verts": verts.tolist(), "vert_norms": vert_norms.tolist(), "proc_para": proc_para, "joint_move": joint_move.tolist(), "joint_posi": joint_posi.tolist(), "achr_move": achr_move.tolist(), "achr_posi": achr_posi.tolist(), "img_file": img_file, } with open(test_dir + "para/%08d.json" % test_id, 'wb') as fp: json.dump(para, fp) take_notes( "H36M %04d%04d TEST %08d\n" % (vid_idx, frm_idx, test_id), "./data_log.txt") test_id += 1 count_work += 1 print("work ratio = %f, (%d / %d)" % (count_work / count_all, count_work, count_all)) return train_id, test_id