def regress(self, img_original, bbox_XYWH, bExport=True):
"""
args:
img_original: original raw image (BGR order by using cv2.imread)
bbox_XYWH: bounding box around the target: (minX,minY,width, height)
outputs:
Default output:
pred_vertices_img:
pred_joints_vis_img:
if bExport==True
pred_rotmat
pred_betas
pred_camera
bbox: [bbr[0], bbr[1],bbr[0]+bbr[2], bbr[1]+bbr[3]])
bboxTopLeft: bbox top left (redundant)
boxScale_o2n: bbox scaling factor (redundant)
"""
img, norm_img, boxScale_o2n, bboxTopLeft, bbox = process_image_bbox(
img_original, bbox_XYWH, input_res=constants.IMG_RES)
if img is None:
return None
with torch.no_grad():
pred_rotmat, pred_betas, pred_camera = self.model_regressor(
norm_img.to(self.device))
pred_output = self.smpl(betas=pred_betas,
body_pose=pred_rotmat[:, 1:],
global_orient=pred_rotmat[:,
0].unsqueeze(1),
pose2rot=False)
pred_vertices = pred_output.vertices
pred_joints_3d = pred_output.joints
# img_original = img
if False:
#show cropped image
# curImgVis = img
curImgVis = self.de_normalize_img(img).cpu().numpy()
curImgVis = np.transpose(curImgVis, (1, 2, 0)) * 255.0
curImgVis = curImgVis[:, :, [2, 1, 0]]
curImgVis = np.ascontiguousarray(curImgVis, dtype=np.uint8)
viewer2D.ImShow(curImgVis, name='input_{}'.format(idx))
pred_vertices = pred_vertices[0].cpu().numpy()
img = img[:, :, [2, 1, 0]]
img = np.ascontiguousarray(img * 255, dtype=np.uint8)
pred_camera = pred_camera.cpu().numpy().ravel()
camScale = pred_camera[0] # *1.15
camTrans = pred_camera[1:]
#Convert mesh to original image space (X,Y are aligned to image)
pred_vertices_bbox = convert_smpl_to_bbox(
pred_vertices, camScale, camTrans) #SMPL -> 2D bbox
pred_vertices_img = convert_bbox_to_oriIm(
pred_vertices_bbox, boxScale_o2n, bboxTopLeft,
img_original.shape[1],
img_original.shape[0]) #2D bbox -> original 2D image
#Convert joint to original image space (X,Y are aligned to image)
pred_joints_3d = pred_joints_3d[0].cpu().numpy() #(1,49,3)
pred_joints_vis = pred_joints_3d[:, :3] #(49,3)
pred_joints_vis_bbox = convert_smpl_to_bbox(
pred_joints_vis, camScale, camTrans) #SMPL -> 2D bbox
pred_joints_vis_img = convert_bbox_to_oriIm(
pred_joints_vis_bbox, boxScale_o2n, bboxTopLeft,
img_original.shape[1],
img_original.shape[0]) #2D bbox -> original 2D image
##Output
predoutput = {}
predoutput[
'pred_vertices_img'] = pred_vertices_img #SMPL vertex in image space
predoutput[
'pred_joints_img'] = pred_joints_vis_img #SMPL joints in image space
if bExport:
predoutput['pred_rotmat'] = pred_rotmat.detach().cpu().numpy()
predoutput['pred_betas'] = pred_betas.detach().cpu().numpy()
predoutput['pred_camera'] = pred_camera
predoutput['bbox_xyxy'] = [
bbox_XYWH[0], bbox_XYWH[1], bbox_XYWH[0] + bbox_XYWH[2],
bbox_XYWH[1] + bbox_XYWH[3]
]
predoutput['bboxTopLeft'] = bboxTopLeft
predoutput['boxScale_o2n'] = boxScale_o2n
return predoutput
def regress(self, img_original, body_bbox_list):
"""
args:
img_original: original raw image (BGR order by using cv2.imread)
body_bbox: bounding box around the target: (minX, minY, width, height)
outputs:
pred_vertices_img:
pred_joints_vis_img:
pred_rotmat
pred_betas
pred_camera
bbox: [bbr[0], bbr[1],bbr[0]+bbr[2], bbr[1]+bbr[3]])
bboxTopLeft: bbox top left (redundant)
boxScale_o2n: bbox scaling factor (redundant)
"""
pred_output_list = list()
for body_bbox in body_bbox_list:
img, norm_img, boxScale_o2n, bboxTopLeft, bbox = process_image_bbox(
img_original, body_bbox, input_res=constants.IMG_RES)
# bboxTopLeft = bbox['bboxXYWH'][:2]
'''
print("bboxTopLeft", bboxTopLeft)
print('img', img.shape)
cv2.imwrite("img.png", img)
print("here !!!!", )
sys.exit(0)
'''
if img is None:
pred_output_list.append(None)
continue
with torch.no_grad():
# model forward
pred_rotmat, pred_betas, pred_camera = self.model_regressor(
norm_img.to(self.device))
#Convert rot_mat to aa since hands are always in aa
pred_aa = rotmat3x3_to_angleaxis(pred_rotmat)
pred_aa = pred_aa.view(pred_aa.shape[0], -1)
smpl_output = self.smpl(betas=pred_betas,
body_pose=pred_aa[:, 3:],
global_orient=pred_aa[:, :3],
pose2rot=True)
pred_vertices = smpl_output.vertices
pred_joints_3d = smpl_output.joints
pred_vertices = pred_vertices[0].cpu().numpy()
pred_camera = pred_camera.cpu().numpy().ravel()
camScale = pred_camera[0] # *1.15
camTrans = pred_camera[1:]
pred_output = dict()
# Convert mesh to original image space (X,Y are aligned to image)
# 1. SMPL -> 2D bbox
# 2. 2D bbox -> original 2D image
pred_vertices_bbox = convert_smpl_to_bbox(
pred_vertices, camScale, camTrans)
pred_vertices_img = convert_bbox_to_oriIm(
pred_vertices_bbox, boxScale_o2n, bboxTopLeft,
img_original.shape[1], img_original.shape[0])
# Convert joint to original image space (X,Y are aligned to image)
pred_joints_3d = pred_joints_3d[0].cpu().numpy() # (1,49,3)
pred_joints_vis = pred_joints_3d[:, :3] # (49,3)
pred_joints_vis_bbox = convert_smpl_to_bbox(
pred_joints_vis, camScale, camTrans)
pred_joints_vis_img = convert_bbox_to_oriIm(
pred_joints_vis_bbox, boxScale_o2n, bboxTopLeft,
img_original.shape[1], img_original.shape[0])
# Output
pred_output['img_cropped'] = img[:, :, ::-1]
pred_output['pred_vertices_smpl'] = smpl_output.vertices[
0].cpu().numpy() # SMPL vertex in original smpl space
pred_output[
'pred_vertices_img'] = pred_vertices_img # SMPL vertex in image space
pred_output[
'pred_joints_img'] = pred_joints_vis_img # SMPL joints in image space
pred_rotmat_tensor = torch.zeros((1, 24, 3, 4),
dtype=torch.float32)
pred_rotmat_tensor[:, :, :, :3] = pred_rotmat.detach().cpu()
pred_aa_tensor = gu.rotation_matrix_to_angle_axis(
pred_rotmat_tensor.squeeze())
pred_output['pred_body_pose'] = pred_aa_tensor.cpu().numpy(
).reshape(1, 72)
pred_output['pred_rotmat'] = pred_rotmat.detach().cpu().numpy(
) # (1, 24, 3, 3)
pred_output['pred_betas'] = pred_betas.detach().cpu().numpy(
) # (1, 10)
pred_output['pred_camera'] = pred_camera
pred_output['bbox_top_left'] = bboxTopLeft
pred_output['bbox_scale_ratio'] = boxScale_o2n
pred_output['faces'] = self.smpl.faces
if self.use_smplx:
img_center = np.array(
(img_original.shape[1], img_original.shape[0])) * 0.5
# right hand
pred_joints = smpl_output.right_hand_joints[0].cpu().numpy(
)
pred_joints_bbox = convert_smpl_to_bbox(
pred_joints, camScale, camTrans)
pred_joints_img = convert_bbox_to_oriIm(
pred_joints_bbox, boxScale_o2n, bboxTopLeft,
img_original.shape[1], img_original.shape[0])
pred_output[
'right_hand_joints_img_coord'] = pred_joints_img
# left hand
pred_joints = smpl_output.left_hand_joints[0].cpu().numpy()
pred_joints_bbox = convert_smpl_to_bbox(
pred_joints, camScale, camTrans)
pred_joints_img = convert_bbox_to_oriIm(
pred_joints_bbox, boxScale_o2n, bboxTopLeft,
img_original.shape[1], img_original.shape[0])
pred_output['left_hand_joints_img_coord'] = pred_joints_img
pred_output_list.append(pred_output)
return pred_output_list
def body_regress(self, img_original, body_bbox_list):
"""
args:
img_original: original raw image (BGR order by using cv2.imread)
body_bbox: bounding box around the target: (minX, minY, width, height)
outputs:
pred_vertices_img:
pred_joints_vis_img:
pred_rotmat
pred_betas
pred_camera
bbox: [bbr[0], bbr[1],bbr[0]+bbr[2], bbr[1]+bbr[3]])
bboxTopLeft: bbox top left (redundant)
boxScale_o2n: bbox scaling factor (redundant)
"""
pred_output_list = list()
for body_bbox in body_bbox_list:
img, norm_img, boxScale_o2n, bboxTopLeft, bbox = process_image_bbox(
img_original, body_bbox, input_res=constants.IMG_RES)
bboxTopLeft = np.array(bboxTopLeft)
# bboxTopLeft = bbox['bboxXYWH'][:2]
if img is None:
pred_output_list.append(None)
continue
with torch.no_grad():
# model forward
pred_rotmat, pred_betas, pred_camera = self.model_regressor(norm_img.to(self.device))
#Convert rot_mat to aa since hands are always in aa
# pred_aa = rotmat3x3_to_angle_axis(pred_rotmat)
pred_aa = gu.rotation_matrix_to_angle_axis(pred_rotmat).cuda()
pred_aa = pred_aa.reshape(pred_aa.shape[0], 72)
smpl_output = self.smpl(
betas=pred_betas,
body_pose=pred_aa[:,3:],
global_orient=pred_aa[:,:3],
pose2rot=True)
pred_vertices = smpl_output.vertices
pred_joints_3d = smpl_output.joints
pred_vertices = pred_vertices[0].cpu().numpy()
pred_camera = pred_camera.cpu().numpy().ravel()
camScale = pred_camera[0] # *1.15
camTrans = pred_camera[1:]
pred_output = dict()
# Convert mesh to original image space (X,Y are aligned to image)
# 1. SMPL -> 2D bbox
# 2. 2D bbox -> original 2D image
pred_vertices_bbox = convert_smpl_to_bbox(pred_vertices, camScale, camTrans)
pred_vertices_img = convert_bbox_to_oriIm(
pred_vertices_bbox, boxScale_o2n, bboxTopLeft, img_original.shape[1], img_original.shape[0])
# Convert joint to original image space (X,Y are aligned to image)
pred_joints_3d = pred_joints_3d[0].cpu().numpy() # (1,49,3)
pred_joints_vis = pred_joints_3d[:,:3] # (49,3)
pred_joints_vis_bbox = convert_smpl_to_bbox(pred_joints_vis, camScale, camTrans)
pred_joints_vis_img = convert_bbox_to_oriIm(
pred_joints_vis_bbox, boxScale_o2n, bboxTopLeft, img_original.shape[1], img_original.shape[0])
# Output
pred_output['img_cropped'] = img[:, :, ::-1]
pred_output['pred_vertices_smpl'] = smpl_output.vertices[0].cpu().numpy() # SMPL vertex in original smpl space
pred_output['pred_vertices_img'] = pred_vertices_img # SMPL vertex in image space
pred_output['pred_joints_img'] = pred_joints_vis_img # SMPL joints in image space
pred_aa_tensor = gu.rotation_matrix_to_angle_axis(pred_rotmat.detach().cpu()[0])
pred_output['pred_body_pose'] = pred_aa_tensor.cpu().numpy().reshape(1, 72)
pred_body_pose = pred_output['pred_body_pose']
pred_output['pred_rotmat'] = pred_rotmat.detach().cpu().numpy() # (1, 24, 3, 3)
pred_output['pred_betas'] = pred_betas.detach().cpu().numpy() # (1, 10)
pred_output['pred_camera'] = pred_camera
pred_output['bbox_top_left'] = bboxTopLeft
pred_output['bbox_scale_ratio'] = boxScale_o2n
pred_output['faces'] = self.smpl.faces
if self.use_smplx:
img_center = np.array((img_original.shape[1], img_original.shape[0]) ) * 0.5
# right hand
pred_joints = smpl_output.right_hand_joints[0].cpu().numpy()
pred_joints_bbox = convert_smpl_to_bbox(pred_joints, camScale, camTrans)
pred_joints_img = convert_bbox_to_oriIm(
pred_joints_bbox, boxScale_o2n, bboxTopLeft, img_original.shape[1], img_original.shape[0])
pred_output['right_hand_joints_img_coord'] = pred_joints_img
# left hand
pred_joints = smpl_output.left_hand_joints[0].cpu().numpy()
pred_joints_bbox = convert_smpl_to_bbox(pred_joints, camScale, camTrans)
pred_joints_img = convert_bbox_to_oriIm(
pred_joints_bbox, boxScale_o2n, bboxTopLeft, img_original.shape[1], img_original.shape[0])
pred_output['left_hand_joints_img_coord'] = pred_joints_img
pred_output_list.append(pred_output)
if self.body_only is True:
with open('/home/student/bodyonly_side/' + 'results.txt', 'a') as file_handle:
file_handle.write('\nframe_id:')
file_handle.write(str(self.frame_id))
file_handle.write('\npred_body_pose:')
file_handle.write(str(pred_body_pose))
file_handle.write('\npred_joints_vis_img:')
file_handle.write(str(pred_joints_vis_img))
#save images(only body_module has visualizer)
# extract mesh for rendering (vertices in image space and faces) from pred_output_list
pred_mesh_list = demo_utils.extract_mesh_from_output(pred_output_list)
# visualization
res_img = self.visualizer.visualize(img_original,pred_mesh_list = pred_mesh_list, body_bbox_list = body_bbox_list)
# show result in the screen
res_img = res_img.astype(np.uint8)
# ImShow(res_img)
cv2.imwrite('/home/student/bodyonly_side/' +str(self.frame_id)+'.jpg',res_img)
return pred_output_list
