def pw3d_extract(dataset_path, out_path):
# scale factor
scaleFactor = 1.2
# structs we use
imgnames_, scales_, centers_, parts_ = [], [], [], []
poses_, shapes_, genders_ = [], [], []
subjectIds_ = []
# get a list of .pkl files in the directory
dataset_path = os.path.join(dataset_path, 'sequenceFiles', g_traintest)
files = [
os.path.join(dataset_path, f) for f in os.listdir(dataset_path)
if f.endswith('.pkl')
]
reject_cnt = 0
# go through all the .pkl files
for filename in files:
with open(filename, 'rb') as f:
data = pickle.load(f, encoding='latin1')
smpl_pose = data['poses']
smpl_betas = data['betas']
poses2d = data['poses2d'] #(N, 3, 18)
global_poses = data['cam_poses']
genders = data['genders']
valid = np.array(data['campose_valid']).astype(np.bool)
num_people = len(smpl_pose)
num_frames = len(smpl_pose[0])
seq_name = str(data['sequence'])
img_names = np.array([
'imageFiles/' + seq_name + '/image_%s.jpg' % str(i).zfill(5)
for i in range(num_frames)
])
# get through all the people in the sequence
for i in range(num_people):
p_id = i
valid_pose = smpl_pose[i][valid[i]]
valid_betas = np.tile(smpl_betas[i][:10].reshape(1, -1),
(num_frames, 1))
valid_betas = valid_betas[valid[i]]
valid_keypoints_2d = poses2d[i][valid[i]]
valid_img_names = img_names[valid[i]]
valid_global_poses = global_poses[valid[i]]
gender = genders[i]
# consider only valid frames
for valid_i in range(valid_pose.shape[0]):
part = valid_keypoints_2d[valid_i, :, :].T
target_joint = [
10, 9, 8, 11, 12, 13, 4, 3, 2, 5, 6, 7, 0, 0
] #From VIBE
valid_joint_cnt = part[target_joint, 2] > 0.3
valid_joint_cnt[12:] = 0
if sum(valid_joint_cnt) <= 6: #Following VIBE's prop
reject_cnt += 1
continue
part = part[part[:, 2] > 0, :]
bbox = [
min(part[:, 0]),
min(part[:, 1]),
max(part[:, 0]),
max(part[:, 1])
]
##Temporary. Export bbox
if True:
bbox_out_path_root = '/run/media/hjoo/disk/data/3dpw/bbox'
img_name = valid_img_names[valid_i]
image_full_path = os.path.join(
'/run/media/hjoo/disk/data/3dpw', img_name)
bbox_xywh = [
bbox[0], bbox[1], bbox[2] - bbox[0],
bbox[3] - bbox[1]
]
bbox_element = {
"image_path": image_full_path,
"body_bbox_list": [bbox_xywh],
"id_list": [p_id]
}
json_out_path = os.path.join(
bbox_out_path_root,
img_name[11:].replace('/', '_')[:-4] +
f'_pid{p_id}' + '.json')
print(f"Saved to {json_out_path}")
with open(json_out_path, 'w') as f:
json.dump(bbox_element, f)
center = [(bbox[2] + bbox[0]) / 2, (bbox[3] + bbox[1]) / 2]
scale = scaleFactor * max(bbox[2] - bbox[0],
bbox[3] - bbox[1]) / 200
# transform global pose
pose = valid_pose[valid_i] #(72,)
extrinsics = valid_global_poses[valid_i][:3, :3]
pose[:3] = cv2.Rodrigues(
np.dot(extrinsics,
cv2.Rodrigues(pose[:3])[0]))[0].T[0]
imgnames_.append(valid_img_names[valid_i])
centers_.append(center) #BBox center
scales_.append(
scale) #bbox scale (from tight bbox w.r.t 200)
poses_.append(pose) #,72
shapes_.append(valid_betas[valid_i])
genders_.append(gender)
subjectIds_.append(p_id)
poseidx_spin24 = [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 19, 20, 21, 22,
23
]
poseidx_openpose18 = [
10, 9, 8, 11, 12, 13, 4, 3, 2, 5, 6, 7, 0, 15, 14, 17,
16
]
part = np.zeros([24, 3])
openpose_pt2d = valid_keypoints_2d[valid_i, :, :].T #18,3
part[poseidx_spin24, :] = openpose_pt2d[
poseidx_openpose18, :]
part[poseidx_spin24,
2] = 1 * (part[poseidx_spin24, 2] > 0.3)
parts_.append(part)
#2D keypoints (total26 -> SPIN24)
if False:
from renderer import viewer2D
imgName = os.path.join(
'/run/media/hjoo/disk/data/3dpw',
valid_img_names[valid_i])
rawImg = cv2.imread(imgName)
# viewer2D.ImShow(rawImg)
rawImg = viewer2D.Vis_Skeleton_2D_Openpose18(
openpose_pt2d[:, :2].ravel(),
image=rawImg,
pt2d_visibility=openpose_pt2d[:, 2] > 0.2)
# rawImg = viewer2D.Vis_Skeleton_2D_SPIN49(openpose_pt2d[:,:2], image= rawImg, pt2d_visibility=openpose_pt2d[:,2]>0.2)
viewer2D.ImShow(rawImg)
print("reject_cnt: {}, valid_cnt:{}".format(reject_cnt, len(imgnames_)))
sampleNum = len(imgnames_)
# store data
if not os.path.isdir(out_path):
os.makedirs(out_path)
out_file = os.path.join(out_path,
f'3dpw_{g_traintest}_{sampleNum}_subjId.npz')
np.savez(out_file,
imgname=imgnames_,
center=centers_,
scale=scales_,
pose=poses_,
shape=shapes_,
gender=genders_,
subjectIds=subjectIds_,
part=parts_)
def visualize(self, image, instances, mesh_fname=""):
"""
"""
pred_camera = self.cam_all[0]
camParam_scale = pred_camera[0]
camParam_trans = pred_camera[1:]
pred_vert_vis = self.vert_all[0]
# smpl_joints_3d_vis = smpl_joints_3d
draw_onbbox = True
draw_rawimg = True
if draw_onbbox:
pred_vert_vis = convert_smpl_to_bbox(pred_vert_vis, camParam_scale,
camParam_trans)
smpl_joints_3d_vis = convert_smpl_to_bbox(smpl_joints_3d_vis,
camParam_scale,
camParam_trans)
renderer.setBackgroundTexture(croppedImg)
renderer.setViewportSize(croppedImg.shape[1], croppedImg.shape[0])
pred_vert_vis *= MAGNIFY_RATIO
if draw_rawimg:
#Covert SMPL to BBox first
pred_vert_vis = convert_smpl_to_bbox(pred_vert_vis, camParam_scale,
camParam_trans)
smpl_joints_3d_vis = convert_smpl_to_bbox(smpl_joints_3d_vis,
camParam_scale,
camParam_trans)
#From cropped space to original
pred_vert_vis = convert_bbox_to_oriIm(pred_vert_vis, boxScale_o2n,
bboxTopLeft, rawImg.shape[1],
rawImg.shape[0])
smpl_joints_3d_vis = convert_bbox_to_oriIm(smpl_joints_3d_vis,
boxScale_o2n,
bboxTopLeft,
rawImg.shape[1],
rawImg.shape[0])
renderer.setBackgroundTexture(rawImg)
renderer.setViewportSize(rawImg.shape[1], rawImg.shape[0])
#In orthographic model. XY of 3D is just 2D projection
smpl_joints_2d_vis = conv_3djoint_2djoint(smpl_joints_3d_vis,
rawImg.shape)
# image_2dkeypoint_pred = viewer2D.Vis_Skeleton_2D_smpl45(smpl_joints_2d_vis, image=rawImg.copy(),color=(0,255,255))
image_2dkeypoint_pred = viewer2D.Vis_Skeleton_2D_Openpose18(
smpl_joints_2d_vis, image=rawImg.copy(),
color=(255, 0, 0)) #All 2D joint
image_2dkeypoint_pred = viewer2D.Vis_Skeleton_2D_Openpose18(
smpl_joints_2d_vis,
pt2d_visibility=keypoint_2d_validity,
image=image_2dkeypoint_pred,
color=(0, 255, 255)) #Only valid
viewer2D.ImShow(image_2dkeypoint_pred,
name='keypoint_2d_pred',
waitTime=1)
pred_meshes = {'ver': pred_vert_vis, 'f': smpl.faces}
v = pred_meshes['ver']
f = pred_meshes['f']
#Visualize in the original image space
renderer.set_mesh(v, f)
renderer.showBackground(True)
renderer.setWorldCenterBySceneCenter()
renderer.setCameraViewMode("cam")
#Set image size for rendering
if args.onbbox:
renderer.setViewportSize(croppedImg.shape[1], croppedImg.shape[0])
else:
renderer.setViewportSize(rawImg.shape[1], rawImg.shape[0])
renderer.display()
renderImg = renderer.get_screen_color_ibgr()
viewer2D.ImShow(renderImg, waitTime=1)
def visEFT_singleSubject(renderer):
MAGNIFY_RATIO = 3 #onbbox only. To magnify the rendered image size
bStopForEachSample = args.waitforkeys #if True, it will wait for any key pressed to move to the next sample
bShowTurnTable = args.turntable
inputData = args.fit_data
imgDir = args.img_dir
#Load SMPL model
smplModelPath = args.smpl_dir + '/basicModel_neutral_lbs_10_207_0_v1.0.0.pkl'
smpl = SMPL_19(smplModelPath, batch_size=1, create_transl=False)
#Load EFT fitting data
print(f"Loading EFT data from {inputData}")
if os.path.exists(inputData):
with open(inputData,'r') as f:
eft_data = json.load(f)
print("EFT data: ver {}".format(eft_data['ver']))
eft_data_all = eft_data['data']
else:
print(f"ERROR:: Cannot find EFT data: {inputData}")
assert False
#Visualize each EFT Fitting output
for idx, eft_data in enumerate(tqdm(eft_data_all)):
#Get raw image path
imgFullPath = eft_data['imageName']
# imgName = os.path.basename(imgFullPath)
imgName = imgFullPath
imgFullPath =os.path.join(imgDir, imgName)
if os.path.exists(imgFullPath) ==False:
print(f"Img path is not valid: {imgFullPath}")
assert False
rawImg = cv2.imread(imgFullPath)
print(f'Input image: {imgFullPath}')
#EFT data
bbox_scale = eft_data['bbox_scale']
bbox_center = eft_data['bbox_center']
pred_camera = np.array(eft_data['parm_cam'])
pred_betas = np.reshape(np.array( eft_data['parm_shape'], dtype=np.float32), (1,10) ) #(10,)
pred_betas = torch.from_numpy(pred_betas)
pred_pose_rotmat = np.reshape( np.array( eft_data['parm_pose'], dtype=np.float32), (1,24,3,3) ) #(24,3,3)
pred_pose_rotmat = torch.from_numpy(pred_pose_rotmat)
keypoint_2d_validity = eft_data['joint_validity_openpose18']
#COCO only. Annotation index
if 'annotId' in eft_data.keys():
print("COCO annotId: {}".format(eft_data['annotId']))
#Get SMPL mesh and joints from SMPL parameters
smpl_output = smpl(betas=pred_betas, body_pose=pred_pose_rotmat[:,1:], global_orient=pred_pose_rotmat[:,[0]], pose2rot=False)
smpl_vertices = smpl_output.vertices.detach().cpu().numpy()[0]
smpl_joints_3d = smpl_output.joints.detach().cpu().numpy()[0]
#Crop image using cropping information
croppedImg, boxScale_o2n, bboxTopLeft = crop_bboxInfo(rawImg, bbox_center, bbox_scale, (BBOX_IMG_RES, BBOX_IMG_RES) )
if MAGNIFY_RATIO>1:
croppedImg = cv2.resize(croppedImg, (croppedImg.shape[1]*MAGNIFY_RATIO, croppedImg.shape[0]*MAGNIFY_RATIO) )
########################
# Visualization
########################
# Visualize 2D image
if True:
viewer2D.ImShow(rawImg, name='rawImg', waitTime=1) #You should press any key
viewer2D.ImShow(croppedImg, name='croppedImg', waitTime=1)
#Convert bbox_center, bbox_scale --> bbox_xyxy
bbox_xyxy = conv_bboxinfo_bboxXYXY(bbox_scale,bbox_center)
img_bbox = viewer2D.Vis_Bbox_minmaxPt(rawImg.copy(),bbox_xyxy[:2], bbox_xyxy[2:])
viewer2D.ImShow(img_bbox, name='img_bbox', waitTime=1)
# Visualization Mesh
if True:
camParam_scale = pred_camera[0]
camParam_trans = pred_camera[1:]
pred_vert_vis = smpl_vertices
smpl_joints_3d_vis = smpl_joints_3d
if args.onbbox:
pred_vert_vis = convert_smpl_to_bbox(pred_vert_vis, camParam_scale, camParam_trans)
smpl_joints_3d_vis = convert_smpl_to_bbox(smpl_joints_3d_vis, camParam_scale, camParam_trans)
renderer.setBackgroundTexture(croppedImg)
renderer.setViewportSize(croppedImg.shape[1], croppedImg.shape[0])
pred_vert_vis *=MAGNIFY_RATIO
else:
#Covert SMPL to BBox first
pred_vert_vis = convert_smpl_to_bbox(pred_vert_vis, camParam_scale, camParam_trans)
smpl_joints_3d_vis = convert_smpl_to_bbox(smpl_joints_3d_vis, camParam_scale, camParam_trans)
#From cropped space to original
pred_vert_vis = convert_bbox_to_oriIm(pred_vert_vis, boxScale_o2n, bboxTopLeft, rawImg.shape[1], rawImg.shape[0])
smpl_joints_3d_vis = convert_bbox_to_oriIm(smpl_joints_3d_vis, boxScale_o2n, bboxTopLeft, rawImg.shape[1], rawImg.shape[0])
renderer.setBackgroundTexture(rawImg)
renderer.setViewportSize(rawImg.shape[1], rawImg.shape[0])
#In orthographic model. XY of 3D is just 2D projection
smpl_joints_2d_vis = conv_3djoint_2djoint(smpl_joints_3d_vis,rawImg.shape )
# image_2dkeypoint_pred = viewer2D.Vis_Skeleton_2D_smpl45(smpl_joints_2d_vis, image=rawImg.copy(),color=(0,255,255))
image_2dkeypoint_pred = viewer2D.Vis_Skeleton_2D_Openpose18(smpl_joints_2d_vis, image=rawImg.copy(),color=(255,0,0)) #All 2D joint
image_2dkeypoint_pred = viewer2D.Vis_Skeleton_2D_Openpose18(smpl_joints_2d_vis, pt2d_visibility=keypoint_2d_validity, image=image_2dkeypoint_pred,color=(0,255,255)) #Only valid
viewer2D.ImShow(image_2dkeypoint_pred, name='keypoint_2d_pred', waitTime=1)
pred_meshes = {'ver': pred_vert_vis, 'f': smpl.faces}
v = pred_meshes['ver']
f = pred_meshes['f']
#Visualize in the original image space
renderer.set_mesh(v,f)
renderer.showBackground(True)
renderer.setWorldCenterBySceneCenter()
renderer.setCameraViewMode("cam")
#Set image size for rendering
if args.onbbox:
renderer.setViewportSize(croppedImg.shape[1], croppedImg.shape[0])
else:
renderer.setViewportSize(rawImg.shape[1], rawImg.shape[0])
renderer.display()
renderImg = renderer.get_screen_color_ibgr()
viewer2D.ImShow(renderImg,waitTime=1)
# Visualize multi-level cropped bbox
if args.multi_bbox:
from demo.multi_bbox_gen import multilvel_bbox_crop_gen
bbox_list = multilvel_bbox_crop_gen(rawImg, pred_vert_vis, bbox_center, bbox_scale)
#Visualize BBox
for b_idx, b in enumerate(bbox_list):
# bbox_xyxy= conv_bboxinfo_centerscale_to_bboxXYXY(b['center'], b['scale'])
bbox_xyxy= b['bbox_xyxy']
if b_idx==0:
img_multi_bbox = viewer2D.Vis_Bbox_minmaxPt(rawImg, bbox_xyxy[:2], bbox_xyxy[2:] ,color=(0,255,0))
else:
img_multi_bbox = viewer2D.Vis_Bbox_minmaxPt(rawImg, bbox_xyxy[:2], bbox_xyxy[2:] ,color=(0,255,255))
viewer2D.ImShow(img_multi_bbox, name='multi_bbox', waitTime=1)
# for bbox in bbox_list:
# Visualization Mesh on side view
if True:
renderer.showBackground(False)
renderer.setWorldCenterBySceneCenter()
# renderer.setCameraViewMode("side") #To show the object in side vie
renderer.setCameraViewMode("free")
renderer.setViewAngle(90,20)
#Set image size for rendering
if args.onbbox:
renderer.setViewportSize(croppedImg.shape[1], croppedImg.shape[0])
else:
renderer.setViewportSize(rawImg.shape[1], rawImg.shape[0])
renderer.display()
sideImg = renderer.get_screen_color_ibgr() #Overwite on rawImg
viewer2D.ImShow(sideImg,waitTime=1)
sideImg = cv2.resize(sideImg, (renderImg.shape[1], renderImg.shape[0]) )
# renderImg = cv2.resize(renderImg, (sideImg.shape[1], sideImg.shape[0]) )
# Visualization Mesh on side view
if True:
renderer.showBackground(False)
renderer.setWorldCenterBySceneCenter()
# renderer.setCameraViewMode("side") #To show the object in side vie
renderer.setCameraViewMode("free")
renderer.setViewAngle(-60,50)
#Set image size for rendering
if args.onbbox:
renderer.setViewportSize(croppedImg.shape[1], croppedImg.shape[0])
else:
renderer.setViewportSize(rawImg.shape[1], rawImg.shape[0])
renderer.display()
sideImg_2 = renderer.get_screen_color_ibgr() #Overwite on rawImg
viewer2D.ImShow(sideImg_2,waitTime=1)
sideImg_2 = cv2.resize(sideImg_2, (renderImg.shape[1], renderImg.shape[0]) )
# renderImg = cv2.resize(renderImg, (sideImg.shape[1], sideImg.shape[0]) )
#Visualize camera view and side view
saveImg = np.concatenate( (renderImg,sideImg), axis =1)
# saveImg = np.concatenate( (croppedImg, renderImg,sideImg, sideImg_2), axis =1)
if bStopForEachSample:
viewer2D.ImShow(saveImg,waitTime=0) #waitTime=0 means that it will wait for any key pressed
else:
viewer2D.ImShow(saveImg,waitTime=1)
#Render Mesh on the rotating view
if bShowTurnTable:
renderer.showBackground(False)
renderer.setWorldCenterBySceneCenter()
renderer.setCameraViewMode("free")
for i in range(90):
renderer.setViewAngle(i*4,0)
renderer.display()
sideImg = renderer.get_screen_color_ibgr() #Overwite on rawImg
viewer2D.ImShow(sideImg,waitTime=1,name="turn_table")
if False: #If you want to save this into files
render_output_path = args.render_dir + '/turntable_{}_{:08d}.jpg'.format(os.path.basename(imgName),i)
cv2.imwrite(render_output_path, sideImg)
#Save the rendered image to files
if True:
if os.path.exists(args.render_dir) == False:
os.mkdir(args.render_dir)
render_output_path = args.render_dir + '/render_{}_eft{:08d}.jpg'.format(imgName[:-4],idx)
print(f"Save to {render_output_path}")
cv2.imwrite(render_output_path, saveImg)