def __getitem__(self, index):
# get data
# input: C, T, V, M
data1_numpy = self.data1[index]
data2_numpy = self.data2[index]
# if self.mmap = True, the loaded data_numpy is read-only, and torch.utils.data.DataLoader could load type 'numpy.core.memmap.memmap'
if self.mmap:
data1_numpy = np.array(data1_numpy) # convert numpy.core.memmap.memmap to numpy
date2_numpy = np.array(data2_numpy)
label = self.label[index]
valid_frame_num = self.valid_frame_num[index]
# normalization
if self.normalization:
# data_numpy = (data_numpy - self.mean_map) / self.std_map
# be careful the value is for NTU_RGB-D, for other dataset, please replace with value from function 'get_mean_map'
if self.origin_transfer == 0:
min_map, max_map = np.array([-4.9881773, -2.939787, -4.728529]), np.array(
[5.826573, 2.391671, 4.824233])
elif self.origin_transfer == 1:
min_map, max_map = np.array([-5.836631, -2.793758, -4.574943]), np.array([5.2021008, 2.362596, 5.1607])
elif self.origin_transfer == 2:
min_map, max_map = np.array([-2.965678, -1.8587272, -4.574943]), np.array(
[2.908885, 2.0095677, 4.843938])
else:
min_map, max_map = np.array([-3.602826, -2.716611, 0.]), np.array([3.635367, 1.888282, 5.209939])
data1_numpy = np.floor(255 * (data1_numpy - min_map[:, None, None, None]) / \
(max_map[:, None, None, None] - min_map[:, None, None, None])) / 255
# processing
if self.crop_resize:
data1_numpy = tools.valid_crop_resize(data1_numpy, valid_frame_num, self.p_interval, self.window_size)
data2_numpy = tools.valid_crop_resize(data2_numpy, valid_frame_num, self.p_interval, self.window_size)
if self.rand_rotate > 0:
data1_numpy = tools.rand_rotate(data1_numpy, self.rand_rotate)
data2_numpy = tools.rand_rotate(data2_numpy, self.rand_rotate)
if self.random_choose:
data1_numpy = tools.random_choose(data1_numpy, self.window_size, auto_pad=True)
data2_numpy = tools.random_choose(data2_numpy, self.window_size, auto_pad=True)
if self.random_valid_choose:
data1_numpy = tools.valid_choose(data1_numpy, self.window_size, random_pad=True)
data2_numpy = tools.valid_choose(data2_numpy, self.window_size, random_pad=True)
elif self.window_size > 0 and (not self.crop_resize) and (not self.random_choose):
data1_numpy = tools.valid_choose(data1_numpy, self.window_size, random_pad=False)
data2_numpy = tools.valid_choose(data2_numpy, self.window_size, random_pad=False)
if self.random_shift:
data1_numpy = tools.random_shift(data1_numpy)
data2_numpy = tools.random_shift(data2_numpy)
if self.random_move:
data1_numpy = tools.random_move(data1_numpy)
data2_numpy = tools.random_move(data2_numpy)
return data1_numpy, data2_numpy, label
def __getitem__(self, index):
# output shape (C, T, V, M)
# get data
sample_name = self.sample_name[index]
sample_path = os.path.join(self.data_path, sample_name)
with open(sample_path, 'r') as f:
video_info = json.load(f)
# fill data_numpy
data = video_info['data']
if self.repeat_frames:
data = self.repeat_frames_in_data(data)
data_numpy = np.zeros((self.C, self.T, self.V, self.num_person_in))
for frame_info in data:
frame_index = frame_info['frame_index']
for m, skeleton_info in enumerate(frame_info["skeleton"]):
if m >= self.num_person_in:
break
pose = skeleton_info['pose']
score = skeleton_info['score']
data_numpy[0, frame_index, :, m] = pose[0::2]
data_numpy[1, frame_index, :, m] = pose[1::2]
data_numpy[2, frame_index, :, m] = score
# centralization
data_numpy[0:2] = data_numpy[0:2] - 0.5
data_numpy[0][data_numpy[2] == 0] = 0
data_numpy[1][data_numpy[2] == 0] = 0
# get & check label index
label = video_info['label_index']
assert (self.label[index] == label)
# data augmentation
if self.random_shift:
data_numpy = tools.random_shift(data_numpy)
if self.random_choose:
data_numpy = tools.random_choose(data_numpy, self.window_size)
elif self.window_size > 0:
data_numpy = tools.auto_pading(data_numpy, self.window_size)
if self.random_move:
data_numpy = tools.random_move(data_numpy)
# sort by score
sort_index = (-data_numpy[2, :, :, :].sum(axis=1)).argsort(axis=1)
for t, s in enumerate(sort_index):
data_numpy[:, t, :, :] = data_numpy[:, t, :, s].transpose(
(1, 2, 0))
data_numpy = data_numpy[:, :, :, 0:self.num_person_out]
# match poses between 2 frames
if self.pose_matching:
data_numpy = tools.openpose_match(data_numpy)
return data_numpy, label
def __getitem__(self, index):
# get data
# input: C, T, V, M
data_numpy = self.data[index]
#print(data_numpy.shape)
# if self.mmap = True, the loaded data_numpy is read-only, and torch.utils.data.DataLoader could load type 'numpy.core.memmap.memmap'
if self.mmap:
data_numpy = np.array(data_numpy) # convert numpy.core.memmap.memmap to numpy
label = self.label[index]
valid_frame_num = self.valid_frame_num[index]
# normalization
if self.normalization:
# data_numpy = (data_numpy - self.mean_map) / self.std_map
# be careful the value is for kinetics, for other dataset, please replace with value from function 'get_mean_map'
if self.origin_transfer == 0:
min_map, max_map = np.array([-1.041, -0.984, -1.078]), np.array(
[1.072, 0.985, 0.954])
elif self.origin_transfer == 1:
min_map, max_map = np.array([-1.08, -0.984, -1.036]), np.array([1.15, 0.986, 1.08 ])
elif self.origin_transfer == 2:
min_map, max_map = np.array([-0.745, -0.837, -1.036]), np.array(
[0.747, 0.985, 1.004])
else:
min_map, max_map = np.array([-0.495, -0.493, 0. ]), np.array([0.681, 0.5, 1.298])
data_numpy = np.floor(255 * (data_numpy - min_map[:, None, None, None]) / \
(max_map[:, None, None, None] - min_map[:, None, None, None])) / 255
# processing
if self.crop_resize:
#print(data_numpy.shape) (3,300,18,2)
data_numpy = tools.valid_crop_resize(data_numpy, valid_frame_num, self.p_interval, self.window_size)
if self.rand_rotate > 0:
data_numpy = tools.rand_rotate(data_numpy, self.rand_rotate)
if self.random_choose:
data_numpy = tools.random_choose(data_numpy, self.window_size, auto_pad=True)
if self.random_valid_choose:
data_numpy = tools.valid_choose(data_numpy, self.window_size, random_pad = True)
elif self.window_size > 0 and (not self.crop_resize) and (not self.random_choose):
data_numpy = tools.valid_choose(data_numpy, self.window_size, random_pad=False)
if self.random_shift:
data_numpy = tools.random_shift(data_numpy)
if self.random_move:
data_numpy = tools.random_move(data_numpy)
return data_numpy, label
def __getitem__(self, index):
# get data
# input: C, T, V, M
data_numpy = self.data[index]
# if self.mmap = True, the loaded data_numpy is read-only, and torch.utils.data.DataLoader could load type 'numpy.core.memmap.memmap'
if self.mmap:
data_numpy = np.array(
data_numpy) # convert numpy.core.memmap.memmap to numpy
label = self.label[index]
valid_frame_num = self.valid_frame_num[index]
## preprocessing
if self.crop_resize:
data_numpy = tools.valid_crop_resize(data_numpy, valid_frame_num,
self.p_interval,
self.window_size)
if self.rand_rotate > 0:
data_numpy = tools.rand_rotate(data_numpy, self.rand_rotate)
if self.random_choose:
data_numpy = tools.random_choose(data_numpy,
self.window_size,
auto_pad=True)
if self.random_valid_choose:
data_numpy = tools.valid_choose(data_numpy,
self.window_size,
random_pad=True)
elif self.window_size > 0 and (not self.crop_resize) and (
not self.random_choose):
data_numpy = tools.valid_choose(data_numpy,
self.window_size,
random_pad=False)
if self.random_shift:
data_numpy = tools.random_shift(data_numpy)
if self.random_move:
data_numpy = tools.random_move(data_numpy)
return data_numpy, label
def __getitem__(self, index):
# get data
# input: C, T, V, M
data_numpy = self.data[index]
rl_label = self.rl_label[index]
#print(data_numpy.shape)
# if self.mmap = True, the loaded data_numpy is read-only, and torch.utils.data.DataLoader could load type 'numpy.core.memmap.memmap'
if self.mmap:
data_numpy = np.array(
data_numpy) # convert numpy.core.memmap.memmap to numpy
label = self.label[index]
valid_frame_num_source = self.valid_frame_num_source[index]
if self.data_path_target != 'None':
valid_frame_num_target = self.valid_frame_num_target[index]
# normalization
if self.normalization:
# data_numpy = (data_numpy - self.mean_map) / self.std_map
# be careful the value is for NTU_RGB-D, for other dataset, please replace with value from function 'get_mean_map'
if self.origin_transfer == 0:
min_map, max_map = np.array(
[-4.9881773, -2.939787,
-4.728529]), np.array([5.826573, 2.391671, 4.824233])
elif self.origin_transfer == 1:
min_map, max_map = np.array([-5.836631, -2.793758,
-4.574943]), np.array(
[5.2021008, 2.362596, 5.1607])
elif self.origin_transfer == 2:
min_map, max_map = np.array(
[-2.965678, -1.8587272,
-4.574943]), np.array([2.908885, 2.0095677, 4.843938])
else:
min_map, max_map = np.array(
[-3.602826, -2.716611,
0.]), np.array([3.635367, 1.888282, 5.209939])
data_numpy = np.floor(255 * (data_numpy - min_map[:, None, None, None]) / \
(max_map[:, None, None, None] - min_map[:, None, None, None])) / 255
# processing
#if self.crop_resize:
# tmp0 = tools.valid_crop_resize(data_numpy[:,:,:,:,0], valid_frame_num_source, self.p_interval, self.window_size)
# if self.data_path_target != 'None':
# tmp1 = tools.valid_crop_resize(data_numpy[:,:,:,:,1], valid_frame_num_target, self.p_interval, self.window_size)
# data_numpy = np.concatenate((tmp0[:,:,:,:,None] , tmp1[:,:,:,:,None]),axis = 4)
# else:
# data_numpy = np.concatenate((tmp0[:,:,:,:,None] , np.zeros((tmp0.shape),dtype = np.float32)[:,:,:,:,None]),axis = 4)
if self.crop_resize:
tmp0 = tools.valid_crop_resize(data_numpy[:, :, :, :, 0],
valid_frame_num_source,
self.p_interval, self.window_size)
if self.data_path_target != 'None':
#tmp1 = tools.random_get_frame(data_numpy[:,:,:,:,1], valid_frame_num_target, self.p_interval, self.window_size)
tmp1 = tools.valid_crop_resize(data_numpy[:, :, :, :, 1],
valid_frame_num_target,
self.p_interval,
self.window_size)
data_numpy = np.concatenate(
(tmp0[:, :, :, :, None], tmp1[:, :, :, :, None]),
axis=4).astype(np.float32)
else:
data_numpy = np.concatenate(
(tmp0[:, :, :, :, None],
np.zeros(
(tmp0.shape), dtype=np.float32)[:, :, :, :, None]),
axis=4).astype(np.float32)
if self.rand_rotate > 0:
data_numpy = tools.rand_rotate(data_numpy, self.rand_rotate)
if self.random_choose:
data_numpy = tools.random_choose(data_numpy,
self.window_size,
auto_pad=True)
if self.random_valid_choose:
data_numpy = tools.valid_choose(data_numpy,
self.window_size,
random_pad=True)
elif self.window_size > 0 and (not self.crop_resize) and (
not self.random_choose):
data_numpy = tools.valid_choose(data_numpy,
self.window_size,
random_pad=False)
if self.random_shift:
data_numpy = tools.random_shift(data_numpy)
if self.random_move:
data_numpy = tools.random_move(data_numpy)
return data_numpy, label, rl_label