def update_figure():
# images
im_input.set_array(tensor_to_img(input_dict['img_crop'][0]))
im_pred.set_array(tensor_to_img(output_dict['img_crop'][0]))
# gt 3D poses
gt_pose = label_dict['3D'][0]
R_cam_2_world = label_dict['extrinsic_rot_inv'][0].numpy()
R_world_in_cam = la.inv(R_cam_2_world) @ input_dict[
'external_rotation_global'] @ R_cam_2_world
pose_rotated = R_world_in_cam @ gt_pose.numpy().reshape([-1, 3]).T
utils_plt.plot_3Dpose_simple(ax_gt_skel,
pose_rotated,
bones=utils_skel.bones_h36m,
plot_handles=handle_gt_skel)
# prediction 3D poses
pose_mean = label_dict['pose_mean'][0].numpy()
pose_std = label_dict['pose_std'][0].numpy()
pred_pose = (output_dict['3D'][0].numpy().reshape(pose_mean.shape)
* pose_std) + pose_mean
pose_rotated = R_world_in_cam @ pred_pose.reshape([-1, 3]).T
utils_plt.plot_3Dpose_simple(ax_pred_skel,
pose_rotated,
bones=utils_skel.bones_h36m,
plot_handles=handle_pred_skel)
# flush drawings
fig.canvas.draw_idle()
def update_figure():
# images
im_input.set_array(tensor_to_img(input_dict['img'][image_index]))
im_input2.set_array(tensor_to_img(input_dict['img'][image_index]))
self.plotTransformer(ax_in_img2, image_index, output_dict['spatial_transformer'], width, height, rect)
key = 'img'
if key in output_dict:
handle_im_out.set_array(tensor_to_img(output_dict[key][image_index]))
handle_seg_out.set_array(output_dict['blend_mask'][image_index].cpu().data.permute(1, 2, 0))
handle_depth_out.set_array(tensor_to_img(output_dict['depth_map'][image_index]))
key = '3D'
if key in output_dict:
# TODO: why ordered right to left here?
# gt 3D poses
gt_pose = label_dict[key][image_index]
#IPython.embed()
R_cam_2_world = input_dict['R_cam_2_world'][image_index].numpy()
R_world_in_cam = la.inv(R_cam_2_world) @ input_dict['external_rotation_global'].cpu().numpy() @ R_cam_2_world
pose_rotated1 = R_world_in_cam @ gt_pose[1].numpy().reshape([-1, 3]).T
utils_plt.plot_3Dpose_simple(ax_gt_skel1, pose_rotated1, bones=utils_skel.bones_h36m,
plot_handles=handle_gt_skel1)
pose_rotated2 = R_world_in_cam @ gt_pose[0].numpy().reshape([-1, 3]).T
utils_plt.plot_3Dpose_simple(ax_gt_skel2, pose_rotated2, bones=utils_skel.bones_h36m,
plot_handles=handle_gt_skel2)
# prediction 3D poses
pose_mean = label_dict['pose_mean'][image_index].numpy()
pose_std = label_dict['pose_std'][image_index].numpy()
pred_pose1 = (output_dict[key][image_index].numpy().reshape(pose_mean.shape) * pose_std) + pose_mean
pose_rotated1 = R_world_in_cam @ pred_pose1[0].reshape([-1, 3]).T
utils_plt.plot_3Dpose_simple(ax_pred_skel1, pose_rotated1, bones=utils_skel.bones_h36m,
plot_handles=handle_pred_skel1)
pred_pose2 = (output_dict[key][image_index].numpy().reshape(pose_mean.shape) * pose_std) + pose_mean
pose_rotated2 = R_world_in_cam @ pred_pose2[1].reshape([-1, 3]).T
utils_plt.plot_3Dpose_simple(ax_pred_skel2, pose_rotated2, bones=utils_skel.bones_h36m,
plot_handles=handle_pred_skel2)
# flush drawings
fig.canvas.draw_idle()
print("using",device,"device")
def run(self, config_dict_file, config_dict):
config_dict['n_hidden_to3Dpose'] = config_dict.get(
'n_hidden_to3Dpose', 2)
# load data
device = 'cuda'
if 1: # load small example data
import pickle
data_loader = pickle.load(open('examples/test_set.pickl', "rb"))
else:
data_loader = self.load_data_test(config_dict)
# save example data
if 0:
import pickle
IPython.embed()
data_iterator = iter(data_loader)
data_cach = [next(data_iterator) for i in range(10)]
data_cach = tuple(data_cach)
pickle.dump(data_cach, open('examples/test_set.pickl', "wb"))
# load model
model = self.load_network(config_dict)
model = model.to(device)
def tensor_to_npimg(torch_array):
return np.swapaxes(np.swapaxes(torch_array.numpy(), 0, 2), 0, 1)
def denormalize(np_array):
return np_array * np.array(config_dict['img_std']) + np.array(
config_dict['img_mean'])
# extract image
def tensor_to_img(output_tensor):
output_img = tensor_to_npimg(output_tensor)
output_img = denormalize(output_img)
output_img = np.clip(output_img, 0, 1)
return output_img
def rotationMatrixXZY(theta, phi, psi):
Ax = np.matrix([[1, 0, 0], [0, np.cos(theta), -np.sin(theta)],
[0, np.sin(theta), np.cos(theta)]])
Ay = np.matrix([[np.cos(phi), 0, -np.sin(phi)], [0, 1, 0],
[np.sin(phi), 0, np.cos(phi)]])
Az = np.matrix([
[np.cos(psi), -np.sin(psi), 0],
[np.sin(psi), np.cos(psi), 0],
[0, 0, 1],
])
return Az * Ay * Ax
# get next image
input_dict, label_dict = None, None
data_iterator = iter(data_loader)
def nextImage():
nonlocal input_dict, label_dict
input_dict, label_dict = next(data_iterator)
print(np.shape(input_dict['extrinsic_rot_inv']))
input_dict['external_rotation_global'] = torch.from_numpy(
np.eye(3)).float().cuda()
nextImage()
# apply model on images
output_dict = None
def predict():
nonlocal output_dict
model.eval()
with torch.no_grad():
input_dict_cuda, label_dict_cuda = utils_data.nestedDictToDevice(
(input_dict, label_dict), device=device)
output_dict_cuda = model(input_dict_cuda)
output_dict = utils_data.nestedDictToDevice(output_dict_cuda,
device='cpu')
predict()
# init figure
my_dpi = 400
fig, ax_blank = plt.subplots(figsize=(5 * 800 / my_dpi,
5 * 300 / my_dpi))
plt.axis('off')
# gt skeleton
ax_gt_skel = fig.add_subplot(111, projection='3d')
ax_gt_skel.set_position([0.8, 0.0, 0.2, 0.98])
handle_gt_skel = utils_plt.plot_3Dpose_simple(
ax_gt_skel,
label_dict['3D'][0].numpy().reshape([-1, 3]).T,
bones=utils_skel.bones_h36m,
linewidth=5,
plot_handles=None) # , colormap='Greys')
ax_gt_skel.invert_zaxis()
ax_gt_skel.grid(False)
ax_gt_skel.set_axis_off()
ax_gt_skel.set_title("GT pose")
# output skeleton
ax_pred_skel = fig.add_subplot(111, projection='3d')
ax_pred_skel.set_position([0.65, 0.0, 0.2, 0.98])
handle_pred_skel = utils_plt.plot_3Dpose_simple(
ax_pred_skel,
label_dict['3D'][0].numpy().reshape([-1, 3]).T,
bones=utils_skel.bones_h36m,
linewidth=5,
plot_handles=None) # , colormap='Greys')
ax_pred_skel.invert_zaxis()
ax_pred_skel.grid(False)
ax_pred_skel.set_axis_off()
ax_pred_skel.set_title("Pred. pose")
# input image
ax_in_img = plt.axes([-0.16, 0.2, 0.7, 0.7])
ax_in_img.axis('off')
im_input = plt.imshow(tensor_to_img(input_dict['img_crop'][0]),
animated=True)
ax_in_img.set_title("Input img")
# output image
ax_out_img = plt.axes([0.15, 0.2, 0.7, 0.7])
ax_out_img.axis('off')
im_pred = plt.imshow(tensor_to_img(output_dict['img_crop'][0]),
animated=True)
ax_out_img.set_title("Output img")
# update figure with new data
def update_figure():
# images
im_input.set_array(tensor_to_img(input_dict['img_crop'][0]))
im_pred.set_array(tensor_to_img(output_dict['img_crop'][0]))
# gt 3D poses
gt_pose = label_dict['3D'][0]
R_cam_2_world = label_dict['extrinsic_rot_inv'][0].numpy()
R_world_in_cam = la.inv(R_cam_2_world) @ input_dict[
'external_rotation_global'] @ R_cam_2_world
pose_rotated = R_world_in_cam @ gt_pose.numpy().reshape([-1, 3]).T
utils_plt.plot_3Dpose_simple(ax_gt_skel,
pose_rotated,
bones=utils_skel.bones_h36m,
plot_handles=handle_gt_skel)
# prediction 3D poses
pose_mean = label_dict['pose_mean'][0].numpy()
pose_std = label_dict['pose_std'][0].numpy()
pred_pose = (output_dict['3D'][0].numpy().reshape(pose_mean.shape)
* pose_std) + pose_mean
pose_rotated = R_world_in_cam @ pred_pose.reshape([-1, 3]).T
utils_plt.plot_3Dpose_simple(ax_pred_skel,
pose_rotated,
bones=utils_skel.bones_h36m,
plot_handles=handle_pred_skel)
# flush drawings
fig.canvas.draw_idle()
def update_rotation(event):
rot = slider_yaw_glob.val
print("Rotationg ", rot)
batch_size = input_dict['img_crop'].size()[0]
input_dict['external_rotation_global'] = torch.from_numpy(
rotationMatrixXZY(theta=0, phi=0, psi=rot)).float().cuda()
input_dict['external_rotation_cam'] = torch.from_numpy(
np.eye(3)
).float().cuda(
) # torch.from_numpy(rotationMatrixXZY(theta=0, phi=rot, psi=0)).float().cuda()
predict()
update_figure()
ax_next = plt.axes([0.05, 0.1, 0.15, 0.04])
button_next = Button(ax_next,
'Next image',
color='lightgray',
hovercolor='0.975')
def nextButtonPressed(event):
nextImage()
predict()
update_figure()
button_next.on_clicked(nextButtonPressed)
ax_yaw_glob = plt.axes([0.25, 0.1, 0.65, 0.015], facecolor='lightgray')
slider_range = 2 * np.pi
slider_yaw_glob = Slider(ax_yaw_glob,
'Yaw',
-slider_range,
slider_range,
valinit=0)
slider_yaw_glob.on_changed(update_rotation)
plt.show()
model.eval()
with torch.no_grad():
input_dict_cuda, label_dict_cuda = utils_data.nestedDictToDevice((input_dict, label_dict), device=device)
output_dict_cuda = model(input_dict_cuda)
output_dict = utils_data.nestedDictToDevice(output_dict_cuda, device='cpu')
predict()
# init figure
my_dpi = 400
fig, ax_blank = plt.subplots(figsize=(5 * 800 / my_dpi, 5 * 300 / my_dpi))
plt.axis('off')
# gt skeleton
ax_gt_skel = fig.add_subplot(111, projection='3d')
ax_gt_skel.set_position([0.8, 0.0, 0.2, 0.98])
handle_gt_skel = utils_plt.plot_3Dpose_simple(ax_gt_skel, label_dict['3D'][0].numpy().reshape([-1, 3]).T,
bones=utils_skel.bones_h36m, linewidth=5,
plot_handles=None) # , colormap='Greys')
ax_gt_skel.invert_zaxis()
ax_gt_skel.grid(False)
ax_gt_skel.set_axis_off()
ax_gt_skel.set_title("GT pose")
# output skeleton
ax_pred_skel = fig.add_subplot(111, projection='3d')
ax_pred_skel.set_position([0.65, 0.0, 0.2, 0.98])
handle_pred_skel = utils_plt.plot_3Dpose_simple(ax_pred_skel, label_dict['3D'][0].numpy().reshape([-1, 3]).T,
bones=utils_skel.bones_h36m, linewidth=5,
plot_handles=None) # , colormap='Greys')
ax_pred_skel.invert_zaxis()
ax_pred_skel.grid(False)
ax_pred_skel.set_axis_off()
ax_pred_skel.set_title("Pred. pose")
def run(self, config_dict_file, config_dict):
if 1: # load small example data
import pickle
data = pickle.load(open('../examples/test_set.pickl', "rb"))
data = [{k: torch.FloatTensor(v)
for k, v in d.items()} for d in data] # numpy to torch
else:
data_loader = self.load_data_test(config_dict)
# save example data
if 0:
import pickle
IPython.embed()
data_iterator = iter(data_loader)
data_cach = [next(data_iterator) for i in range(3)]
data_cach = tuple(data_cach)
pickle.dump(data_cach, open('../examples/test_set.pickl',
"wb"))
# load model
model = self.load_network(config_dict)
model = model.to(device)
def tensor_to_npimg(torch_array):
return np.swapaxes(np.swapaxes(torch_array.numpy(), 0, 2), 0, 1)
def denormalize(np_array):
return np_array * np.array(config_dict['img_std']) + np.array(
config_dict['img_mean'])
# extract image
def tensor_to_img(output_tensor):
output_img = tensor_to_npimg(output_tensor)
output_img = denormalize(output_img)
output_img = np.clip(output_img, 0, 1)
return output_img
def rotationMatrixXZY(theta, phi, psi):
Ax = np.matrix([[1, 0, 0], [0, np.cos(theta), -np.sin(theta)],
[0, np.sin(theta), np.cos(theta)]])
Ay = np.matrix([[np.cos(phi), 0, -np.sin(phi)], [0, 1, 0],
[np.sin(phi), 0, np.cos(phi)]])
Az = np.matrix([
[np.cos(psi), -np.sin(psi), 0],
[np.sin(psi), np.cos(psi), 0],
[0, 0, 1],
])
return Az * Ay * Ax
# get next image
input_dict, label_dict = None, None
image_index = -1
input_dict, label_dict = data
def nextImage():
nonlocal image_index
image_index += 1
#nonlocal input_dict, label_dict
#input_dict, label_dict = next(data_iterator)
input_dict['external_rotation_global'] = torch.from_numpy(
np.eye(3)).float().to(device)
nextImage()
# apply model on images
output_dict = None
def predict():
nonlocal output_dict
model.eval()
with torch.no_grad():
input_dict_cuda, label_dict_cuda = utils_data.nestedDictToDevice(
(input_dict, label_dict), device=device)
output_dict_cuda = model(input_dict_cuda)
output_dict = utils_data.nestedDictToDevice(output_dict_cuda,
device='cpu')
predict()
# init figure
my_dpi = 400
fig, ax_blank = plt.subplots(figsize=(5 * 800 / my_dpi,
5 * 300 / my_dpi))
plt.axis('off')
title_font_size = 5
# input image
ax_in_img = plt.axes([-0.05, 0.55, 0.4, 0.4])
ax_in_img.axis('off')
im_input = plt.imshow(tensor_to_img(input_dict['img'][image_index]),
animated=True)
ax_in_img.set_title("Input img", size=title_font_size)
# input image with bounding box
ax_in_img2 = plt.axes([0.25, 0.55, 0.4, 0.4])
ax_in_img2.axis('off')
im_input2 = plt.imshow(tensor_to_img(input_dict['img'][image_index]),
animated=True)
ax_in_img2.set_title("Bounding box", size=title_font_size)
height, width = input_dict['img'][image_index].shape[1:3]
rect = self.plotTransformer(ax_in_img2, image_index,
output_dict['spatial_transformer'], width,
height)
# output image
key = 'img'
if key in output_dict:
ax_out_img = plt.axes([0.55, 0.55, 0.4, 0.4])
ax_out_img.axis('off')
img_out = tensor_to_img(output_dict[key][image_index])
handle_im_out = plt.imshow(img_out, animated=True)
ax_out_img.set_title("Output img", size=title_font_size)
# output seg
ax_out_seg = plt.axes([0.25, 0.08, 0.4, 0.4])
ax_out_seg.axis('off')
color = output_dict['blend_mask'][0].cpu().data.permute(1, 2, 0)
handle_seg_out = plt.imshow(color, animated=True)
ax_out_seg.set_title("Intance segmentation", size=title_font_size)
# output depth
ax_out_depth = plt.axes([0.55, 0.08, 0.4, 0.4])
ax_out_depth.axis('off')
img_out = tensor_to_img(output_dict['depth_map'][image_index])
handle_depth_out = plt.imshow(img_out, animated=True)
ax_out_depth.set_title("Depth map", size=title_font_size)
key = '3D'
if key in output_dict:
# output skeleton 1
ax_pred_skel1 = fig.add_subplot(111, projection='3d')
ax_pred_skel1.set_position([0.6, 0.5, 0.1, 0.4])
handle_pred_skel1 = utils_plt.plot_3Dpose_simple(
ax_pred_skel1,
label_dict['3D'][image_index].numpy().reshape([-1, 3]).T,
bones=utils_skel.bones_h36m,
linewidth=5,
plot_handles=None) # , colormap='Greys')
ax_pred_skel1.invert_zaxis()
ax_pred_skel1.grid(False)
ax_pred_skel1.set_axis_off()
ax_pred_skel1.set_title("Pred. (left)", size=title_font_size)
# output skeleton 2
ax_pred_skel2 = fig.add_subplot(111, projection='3d')
ax_pred_skel2.set_position([0.7, 0.5, 0.1, 0.4])
handle_pred_skel2 = utils_plt.plot_3Dpose_simple(
ax_pred_skel2,
label_dict['3D'][image_index].numpy().reshape([-1, 3]).T,
bones=utils_skel.bones_h36m,
linewidth=5,
plot_handles=None) # , colormap='Greys')
ax_pred_skel2.invert_zaxis()
ax_pred_skel2.grid(False)
ax_pred_skel2.set_axis_off()
ax_pred_skel2.set_title("(right)", size=title_font_size)
# gt skeleton 1
ax_gt_skel2 = fig.add_subplot(111, projection='3d')
ax_gt_skel2.set_position([0.8, 0.5, 0.1, 0.4])
handle_gt_skel2 = utils_plt.plot_3Dpose_simple(
ax_gt_skel2,
label_dict['3D'][image_index].numpy().reshape([-1, 3]).T,
bones=utils_skel.bones_h36m,
linewidth=5,
plot_handles=None) # , colormap='Greys')
ax_gt_skel2.invert_zaxis()
ax_gt_skel2.grid(False)
ax_gt_skel2.set_axis_off()
ax_gt_skel2.set_title("GT (left)", size=title_font_size)
# gt skeleton 2
ax_gt_skel1 = fig.add_subplot(111, projection='3d')
ax_gt_skel1.set_position([0.9, 0.5, 0.1, 0.4])
handle_gt_skel1 = utils_plt.plot_3Dpose_simple(
ax_gt_skel1,
label_dict['3D'][image_index].numpy().reshape([-1, 3]).T,
bones=utils_skel.bones_h36m,
linewidth=5,
plot_handles=None) # , colormap='Greys')
ax_gt_skel1.invert_zaxis()
ax_gt_skel1.grid(False)
ax_gt_skel1.set_axis_off()
ax_gt_skel1.set_title("(right)", size=title_font_size)
# update figure with new data
def update_figure():
# images
im_input.set_array(tensor_to_img(input_dict['img'][image_index]))
im_input2.set_array(tensor_to_img(input_dict['img'][image_index]))
self.plotTransformer(ax_in_img2, image_index,
output_dict['spatial_transformer'], width,
height, rect)
key = 'img'
if key in output_dict:
handle_im_out.set_array(
tensor_to_img(output_dict[key][image_index]))
handle_seg_out.set_array(
output_dict['blend_mask'][image_index].cpu().data.permute(
1, 2, 0))
handle_depth_out.set_array(
tensor_to_img(output_dict['depth_map'][image_index]))
key = '3D'
if key in output_dict:
# TODO: why ordered right to left here?
# gt 3D poses
gt_pose = label_dict[key][image_index]
#IPython.embed()
R_cam_2_world = input_dict['R_cam_2_world'][image_index].numpy(
)
R_world_in_cam = la.inv(R_cam_2_world) @ input_dict[
'external_rotation_global'].cpu().numpy() @ R_cam_2_world
pose_rotated1 = R_world_in_cam @ gt_pose[1].numpy().reshape(
[-1, 3]).T
utils_plt.plot_3Dpose_simple(ax_gt_skel1,
pose_rotated1,
bones=utils_skel.bones_h36m,
plot_handles=handle_gt_skel1)
pose_rotated2 = R_world_in_cam @ gt_pose[0].numpy().reshape(
[-1, 3]).T
utils_plt.plot_3Dpose_simple(ax_gt_skel2,
pose_rotated2,
bones=utils_skel.bones_h36m,
plot_handles=handle_gt_skel2)
# prediction 3D poses
pose_mean = label_dict['pose_mean'][image_index].numpy()
pose_std = label_dict['pose_std'][image_index].numpy()
pred_pose1 = (output_dict[key][image_index].numpy().reshape(
pose_mean.shape) * pose_std) + pose_mean
pose_rotated1 = R_world_in_cam @ pred_pose1[0].reshape([-1, 3
]).T
utils_plt.plot_3Dpose_simple(ax_pred_skel1,
pose_rotated1,
bones=utils_skel.bones_h36m,
plot_handles=handle_pred_skel1)
pred_pose2 = (output_dict[key][image_index].numpy().reshape(
pose_mean.shape) * pose_std) + pose_mean
pose_rotated2 = R_world_in_cam @ pred_pose2[1].reshape([-1, 3
]).T
utils_plt.plot_3Dpose_simple(ax_pred_skel2,
pose_rotated2,
bones=utils_skel.bones_h36m,
plot_handles=handle_pred_skel2)
# flush drawings
fig.canvas.draw_idle()
print("using", device, "device")
update_figure()
def update_rotation(event):
rot = slider_yaw_glob.val
print("Rotationg ", rot)
batch_size = input_dict['img'].size()[0]
input_dict['external_rotation_global'] = torch.from_numpy(
rotationMatrixXZY(theta=0, phi=rot,
psi=0)).float().to(device) # boxing coords
#input_dict['external_rotation_global'] = torch.from_numpy(rotationMatrixXZY(theta=0, phi=0, psi=rot)).float().to(device) # H36m
input_dict['external_rotation_cam'] = torch.from_numpy(
np.eye(3)
).float().to(
device
) # torch.from_numpy(rotationMatrixXZY(theta=0, phi=rot, psi=0)).float().cuda()
predict()
update_figure()
ax_next = plt.axes([0.05, 0.03, 0.15, 0.04])
button_next = Button(ax_next,
'Next image',
color='lightgray',
hovercolor='0.975')
def nextButtonPressed(event):
nextImage()
predict()
update_figure()
button_next.on_clicked(nextButtonPressed)
ax_yaw_glob = plt.axes([0.25, 0.03, 0.65, 0.015],
facecolor='lightgray')
slider_range = np.pi
slider_yaw_glob = Slider(ax_yaw_glob,
'Yaw',
-slider_range,
slider_range,
valinit=0)
slider_yaw_glob.on_changed(update_rotation)
plt.show()
