def __getitem__(self, idx):
subpath, file_num = self.get_subpath_and_file_num(idx)
depth_filepath = self.root_folder + self.video_folder + subpath + \
self.depth_folder + 'depth_' + \
file_num + '.' + self.depth_fileext
depth_image = fpa_io.read_depth_img(depth_filepath)
joints_filepath = self.root_folder + self.hand_pose_folder + \
subpath + 'skeleton.txt'
hand_joints = fpa_io.read_action_joints_sequence(joints_filepath)[int(
file_num)]
_, crop_coords = self.get_cropped_depth_img(depth_image, hand_joints)
crop_coords_numpy = np.zeros((2, 2))
crop_coords_numpy[0, 0] = crop_coords[0]
crop_coords_numpy[0, 1] = crop_coords[1]
crop_coords_numpy[1, 0] = crop_coords[2]
crop_coords_numpy[1, 1] = crop_coords[3]
corner_heatmap1 = conv.color_space_label_to_heatmap(
crop_coords_numpy[0, :],
heatmap_res=self.orig_img_res,
orig_img_res=self.orig_img_res)
#print(np.unravel_index(np.argmax(corner_heatmap1), corner_heatmap1.shape))
corner_heatmap2 = conv.color_space_label_to_heatmap(
crop_coords_numpy[1, :],
heatmap_res=self.orig_img_res,
orig_img_res=self.orig_img_res)
corner_heatmaps = np.stack((corner_heatmap1, corner_heatmap2))
corner_heatmaps = torch.from_numpy(corner_heatmaps).float()
return data_image, corner_heatmaps
def read_fpa_skeleton(root_folder, subpath, frame_idx):
#root_idx, thumb_idxs, index_idxs, middle_idxs, ring_idxs, little_idxs = get_fpa_finger_idx()
hand_pose_folder = 'Hand_pose_annotation_v1/'
joints_filepath = root_folder + hand_pose_folder + subpath + 'skeleton.txt'
hand_joints = fpa_io.read_action_joints_sequence(joints_filepath)[int(
frame_idx)]
hand_joints = hand_joints.reshape((21, 3))
skeleton = np.copy(hand_joints)
#skeleton[0, :] = hand_joints[root_idx, :]
#skeleton[1:5, :] = hand_joints[thumb_idxs, :]
#skeleton[5:9, :] = hand_joints[index_idxs, :]
#skeleton[9:13, :] = hand_joints[middle_idxs, :]
#skeleton[13:17, :] = hand_joints[ring_idxs, :]
#skeleton[17:, :] = hand_joints[little_idxs, :]
return skeleton
def __getitem__(self, idx):
subpath, file_num = self.get_subpath_and_file_num(idx)
depth_img = self.read_depth_img(subpath, file_num).astype(float)
#depth_obj_img_path = self.root_folder + self.gen_obj_folder + subpath + \
# str(int(file_num)) + '_depth.jpg'
#depth_obj_img = fpa_io.read_depth_img(depth_obj_img_path)
depth_obj_csv_path = self.root_folder + self.gen_obj_folder + subpath + \
str(int(file_num)) + '_depth.csv'
img2_depth_array = np.loadtxt(open(depth_obj_csv_path, "rb"),
delimiter=",")
depth_obj_img = img2_depth_array.T
joints_filepath = self.root_folder + self.hand_pose_folder + \
subpath + 'skeleton.txt'
hand_joints = fpa_io.read_action_joints_sequence(joints_filepath)[int(
file_num)]
depth_img, _ = self.get_cropped_depth_img(depth_img, hand_joints)
depth_obj_img, _ = self.get_cropped_depth_img(depth_obj_img,
hand_joints)
depth_img /= self.normalise_const_max_depth
depth_obj_img /= self.normalise_const_max_depth
#vis.plot_image(depth_img)
#vis.show()
#vis.plot_image(depth_obj_img)
#vis.show()
depth_img = depth_img.reshape(
(1, depth_img.shape[0], depth_img.shape[1]))
depth_img_torch = torch.from_numpy(depth_img).float()
depth_img_torch = self.resize2d(depth_img_torch, self.crop_res)
depth_obj_img = depth_obj_img.reshape(
(1, depth_obj_img.shape[0], depth_obj_img.shape[1]))
depth_obj_img_torch = torch.from_numpy(depth_obj_img).float()
depth_obj_img_torch = self.resize2d(depth_obj_img_torch, self.crop_res)
return depth_img_torch, depth_obj_img_torch
def get_hand_joints(self, idx):
subpath, file_num = self.get_subpath_and_file_num(idx)
joints_filepath = self.root_folder + self.hand_pose_folder + \
subpath + 'skeleton.txt'
return fpa_io.read_action_joints_sequence(joints_filepath)[int(
file_num)]
import fpa_io
import visualize
import camera as cam
dataset_root_folder = 'C:/Users/Administrator/Documents/Datasets/fpa_benchmark/'
gt_folder = 'Hand_pose_annotation_v1'
data_folder = 'video_files'
subject = 'Subject_1'
action = 'put_tea_bag'
sequence = '4'
gt_filepath = '/'.join([dataset_root_folder, gt_folder, subject, action, sequence, 'skeleton.txt'])
curr_data_folder = '/'.join([dataset_root_folder, data_folder, subject, action, sequence])
gt_skeletons = fpa_io.read_action_joints_sequence(gt_filepath)
fig = visualize.create_fig()
for i in range(99):
if i < 10:
frame_num = '000' + str(i)
else:
frame_num = '00' + str(i)
joints = gt_skeletons[i, :].reshape((21, 3))
color_filepath = '/'.join([curr_data_folder, 'color', 'color_' + frame_num + '.jpeg'])
depth_filepath = '/'.join([curr_data_folder, 'depth', 'depth_' + frame_num + '.png'])
depth_img = fpa_io.read_depth_img(depth_filepath)
joints_uv = cam.joints_depth2color(joints, cam.fpa_depth_intrinsics)
visualize.plot_joints_from_colorspace(joints_colorspace=joints_uv, data=depth_img,
fig=fig, title='/'.join([subject, action, sequence]))
visualize.pause(0.001)
def __getitem__(self, idx):
subpath, file_num = self.get_subpath_and_file_num(idx)
depth_img = self.read_depth_img(subpath, file_num).astype(float)
# depth_obj_img_path = self.root_folder + self.gen_obj_folder + subpath + \
# str(int(file_num)) + '_depth.jpg'
# depth_obj_img = fpa_io.read_depth_img(depth_obj_img_path)
# read obj img
depth_obj_csv_path = self.root_folder + self.gen_obj_folder + subpath + \
str(int(file_num)) + '_depth.csv'
img2_depth_array = np.loadtxt(open(depth_obj_csv_path, "rb"),
delimiter=",")
depth_obj_img = img2_depth_array.T
# read hand image
hand_img_path = self.root_folder + self.gen_hands_folder + subpath + \
'depth_' + str(int(file_num)) + '.bmp'
hand_img = self.read_unreal_depth_img(
hand_img_path, unreal_max_depth=self.unreal_max_depth)
joints_filepath = self.root_folder + self.hand_pose_folder + \
subpath + 'skeleton.txt'
hand_joints = fpa_io.read_action_joints_sequence(joints_filepath)[int(
file_num)]
depth_img, _ = self.get_cropped_depth_img(depth_img, hand_joints)
depth_obj_img, _ = self.get_cropped_depth_img(depth_obj_img,
hand_joints)
hand_img, _ = self.get_cropped_depth_img(hand_img, hand_joints)
depth_img /= self.normalise_const_max_depth
depth_obj_img /= self.normalise_const_max_depth
hand_img /= self.normalise_const_max_depth
#print(np.max(depth_img))
#print(np.max(depth_obj_img))
#print(np.max(hand_img))
#vis.plot_image(depth_img)
#vis.show()
#vis.plot_image(depth_obj_img)
#vis.show()
#vis.plot_image(hand_img)
#vis.show()
depth_img = depth_img.reshape(
(1, depth_img.shape[0], depth_img.shape[1]))
depth_img_torch = torch.from_numpy(depth_img).float()
depth_img_torch = self.resize2d(depth_img_torch, self.crop_res)
depth_obj_img = depth_obj_img.reshape(
(1, depth_obj_img.shape[0], depth_obj_img.shape[1]))
depth_obj_img_torch = torch.from_numpy(depth_obj_img).float()
depth_obj_img_torch = self.resize2d(depth_obj_img_torch, self.crop_res)
#obj_img_numpy = depth_obj_img_torch.numpy().astype(float).reshape(self.crop_res)
#vis.plot_image(obj_img_numpy)
#vis.show()
hand_img = hand_img.reshape((1, hand_img.shape[0], hand_img.shape[1]))
hand_img_torch = torch.from_numpy(hand_img).float()
hand_img_torch = self.resize2d(hand_img_torch, self.crop_res)
#hand_img_numpy = hand_img_torch.numpy().astype(float).reshape(self.crop_res)
#vis.plot_image(hand_img_numpy)
#vis.show()
obj_hand_img = torch.cat((hand_img_torch, depth_obj_img_torch))
return depth_img_torch, obj_hand_img
frame = 21
unreal_bone_lengths_filepath = dataset_root + 'bonelengths/' + 'UnrealMaleRightHands.txt'
unreal_bone_lengths = genfromtxt(unreal_bone_lengths_filepath, delimiter=',')
unreal_bone_lengths = unreal_bone_lengths[:, 1]
subject_bone_lengths_filepath = dataset_root + 'bonelengths/' + subject + '.txt'
subject_bone_lengths = genfromtxt(subject_bone_lengths_filepath, delimiter=',')
subject_bone_lengths = subject_bone_lengths[:, 1]
bone_prop = (unreal_bone_lengths / subject_bone_lengths).reshape((20, 1))
bone_prop[bone_prop < 0] = 1.0
hand_gt_filepath = dataset_root + hand_gt_folder + subject + '/' + action + '/' + seq + 'skeleton.txt'
hand_joints = fpa_io.read_action_joints_sequence(hand_gt_filepath)[int(31)]
hand_joints = hand_joints.reshape((21, 3))
hand_joints -= hand_joints[0, :]
hand_joints_unreal = np.copy(hand_joints)
i = 0
for finger_idx in range(5):
finger_start_joint_idx = (finger_idx * 4) + 1
for j in range(3):
parent_joint_idx = finger_start_joint_idx + j
parent_joint_before = np.copy(hand_joints_unreal[parent_joint_idx, :])
curr_bone_prop = bone_prop[parent_joint_idx - 1]
hand_joints_unreal[parent_joint_idx, :] *= curr_bone_prop
parent_joint_transl = hand_joints_unreal[
parent_joint_idx, :] - parent_joint_before
print(