def __getitem__(self, idx):
batch_idxs = self.shuffled_idx[
self.batch_size*idx:self.batch_size*(idx+1)]
if self.sample_method == 'central':
batch_gt = self.ground_truth.loc[batch_idxs]
batch_x, batch_y = get_data(batch_gt, pose_style=self.pose_style,
num_classes=self.num_classes, **self.data_kwargs)
elif self.sample_method == 'all':
batch_x, batch_y = [], []
for mapping_idx in batch_idxs:
video_idx, seq_idx = self.seqs_mapping[mapping_idx]
video_x, video_y = get_data(self.ground_truth.loc[[video_idx]],
pose_style=self.pose_style, num_classes=self.num_classes,
sample_method=self.sample_method, **self.data_kwargs)
seq_data = [ j[seq_idx] for j in video_x]
batch_x.append(seq_data)
batch_y.append(video_y[seq_idx])
batch_x = np.array(batch_x).transpose((1,0,2))
batch_y = np.array(batch_y)
if self.shuffle_indiv_order:
### Always swap half of the batch
num_joints = len(batch_x)//2
p1_joints = list(range(num_joints))
p2_joints = list(range(num_joints, num_joints*2))
swap_index = sorted(np.random.choice(list(range(batch_y.shape[0])),
batch_y.shape[0]//2, replace=False))
batch_x = np.array(batch_x)
batch_x[:,swap_index] = np.squeeze(
batch_x[p2_joints + p1_joints][:,[swap_index]], axis=1)
batch_x = [ np.array(input) for input in batch_x]
return batch_x, batch_y
def get_val(fold_num, **kwargs):
from misc import data_io
gt_split = get_val_gt(fold_num)
X, Y = data_io.get_data(gt_split, pose_style='NTU-V2', **kwargs)
return X, Y
def get_val(fold_num, **kwargs):
if fold_num < 0 or fold_num > NUM_FOLDS:
raise ValueError("fold_num must be within 0 and " + NUM_FOLDS +
", value entered: " + str(fold_num))
ground_truth = get_ground_truth()
gt_split = ground_truth[ground_truth.fold == fold_num]
X, Y = data_io.get_data(gt_split, pose_style='YMJA', **kwargs)
return X, Y
def get_train(fold_num, **kwargs):
if fold_num < 0 or fold_num > 5:
raise ValueError("fold_num must be within 0 and 5, value entered: " +
str(fold_num))
ground_truth = get_ground_truth()
gt_split = ground_truth[ground_truth.fold != fold_num]
X, Y = data_io.get_data(gt_split, pose_style='SBU', **kwargs)
return X, Y
def get_seqs(setid, selected_sequences, **kwargs):
if setid < 1 or setid > 2:
raise ValueError("setid must be 1 or 2, value entered: "+str(setid))
ground_truth = get_ground_truth()
gt_split = ground_truth[ground_truth.setid == setid]
gt_split = gt_split[gt_split.seq.isin(selected_sequences)]
X, Y = data_io.get_data(gt_split, pose_style='OpenPose', **kwargs)
return X, Y
def __getitem__(self, idx):
batch_idxs = self.shuffled_idx[self.batch_size * idx:self.batch_size *
(idx + 1)]
if not self.buffer_data:
batch_gt = self.ground_truth.loc[batch_idxs]
batch_x, batch_y = get_data(batch_gt,
pose_style=self.pose_style,
num_classes=self.num_classes,
**self.data_kwargs)
else:
batch_x = [self.buffer[0][idx] for idx in batch_idxs]
batch_y = self.buffer[1][batch_idxs]
if self.shuffle_indiv_order:
### Always swap half of the batch
num_joints = len(batch_x[0][0]) // 2
p1_joints = list(range(num_joints))
p2_joints = list(range(num_joints, num_joints * 2))
swap_index = sorted(
np.random.choice(list(range(batch_y.shape[0])),
batch_y.shape[0] // 2,
replace=False))
for video_swap_id in swap_index:
video_seqs = np.array(batch_x[video_swap_id])
batch_x[video_swap_id] = video_seqs[:,
p2_joints + p1_joints, :]
# Padding sequences
if self.pad_sequences:
maxlen = self.maxlen
for idx, video_seqs in enumerate(batch_x):
video_seqs = np.array(video_seqs)
padded_video_seqs = np.zeros((maxlen, ) +
np.array(batch_x[0][0]).shape)
if self.padding == 'pre':
padded_video_seqs[-len(video_seqs[:maxlen]):,
...] = video_seqs[:maxlen, ...]
elif self.padding == 'post':
padded_video_seqs[:len(video_seqs[:maxlen]),
...] = video_seqs[:maxlen, ...]
batch_x[idx] = padded_video_seqs
batch_x = np.array(batch_x)
return batch_x, batch_y
def __init__(self,
dataset_name,
dataset_fold,
subset,
batch_size=32,
reshuffle=False,
shuffle_indiv_order=False,
pad_sequences=False,
maxlen=None,
padding='pre',
buffer_data=False,
**data_kwargs):
if dataset_name == 'UT':
dataset = UT
max_framenum = 183
self.pose_style = 'OpenPose'
elif dataset_name == 'SBU':
dataset = SBU
max_framenum = 46
self.pose_style = 'SBU'
elif dataset_name == 'YMJA':
dataset = YMJA
max_framenum = 64
self.pose_style = 'YMJA'
elif dataset_name == 'NTU':
dataset = NTU
max_framenum = 300 # for all videos
# max_framenum = 205 # for mutual videos
self.pose_style = 'NTU'
elif dataset_name == 'NTU-V2':
dataset = NTU_V2
# max_framenum = 300 # 214 for mutual videos
max_framenum = 214 # 300 for all videos
self.pose_style = 'NTU-V2'
self.batch_size = batch_size
self.reshuffle = reshuffle
self.data_kwargs = data_kwargs
self.shuffle_indiv_order = shuffle_indiv_order
self.subset = subset
self.pad_sequences = pad_sequences
self.padding = padding
self.buffer_data = buffer_data
self.data_kwargs['flat_seqs'] = False
self.data_kwargs['sample_method'] = 'all'
if self.data_kwargs.get('seq_step') is None:
self.data_kwargs['seq_step'] = self.data_kwargs['timesteps'] // 2
if maxlen is not None:
self.maxlen = maxlen
else: # Estimate maxlen
self.maxlen = max_framenum // data_kwargs['seq_step']
if data_kwargs['skip_timesteps'] is not None:
self.maxlen = self.maxlen // data_kwargs['skip_timesteps']
if subset == 'train':
self.ground_truth = dataset.get_train_gt(dataset_fold)
elif subset == 'validation':
self.ground_truth = dataset.get_val_gt(dataset_fold)
self.num_classes = self.ground_truth.action.max() + 1
num_videos = self.ground_truth.shape[0]
self.num_batches = int(np.ceil(num_videos / batch_size))
if not self.buffer_data:
if subset == 'train':
self.shuffled_idx = np.random.choice(
self.ground_truth.index.values, num_videos, replace=False)
elif subset == 'validation':
self.shuffled_idx = self.ground_truth.index.values
# Validating data_kwargs
_ = get_data(self.ground_truth.sample(),
pose_style=self.pose_style,
num_classes=self.num_classes,
**self.data_kwargs)
else:
if subset == 'train':
self.shuffled_idx = np.random.choice(range(num_videos),
num_videos,
replace=False)
elif subset == 'validation':
self.shuffled_idx = list(range(num_videos))
self.buffer = get_data(self.ground_truth,
pose_style=self.pose_style,
num_classes=self.num_classes,
**self.data_kwargs)
def getSampleData(self, idx, new_arch_sample):
batch_idxs = self.shuffled_idx[self.batch_size * idx:self.batch_size *
(idx + 1)]
if self.sample_method == 'central':
batch_gt = self.ground_truth.loc[batch_idxs]
batch_x, batch_y = get_data(batch_gt,
pose_style=self.pose_style,
num_classes=self.num_classes,
**self.data_kwargs)
elif self.sample_method == 'all':
batch_x, batch_y = [], []
for mapping_idx in batch_idxs:
video_idx, seq_idx = self.seqs_mapping[mapping_idx]
video_x, video_y = get_data(self.ground_truth.loc[[video_idx]],
pose_style=self.pose_style,
num_classes=self.num_classes,
sample_method=self.sample_method,
**self.data_kwargs)
seq_data = [j[seq_idx] for j in video_x]
batch_x.append(seq_data)
batch_y.append(video_y[seq_idx])
batch_x = np.array(batch_x).transpose((1, 0, 2))
batch_y = np.array(batch_y)
if self.shuffle_indiv_order:
NUM_PEOPLE = 2
NUM_DIM = 3
if self.pose_style == 'YMJA':
NUM_DIM = 2
if 'arch' in self.data_kwargs and (
self.data_kwargs['arch'] == 'joint'
or self.data_kwargs['arch'] == 'temp'):
# Convert to form (num joints, num samples, timesteps, num people, num dimensions) for joint stream
# Convert to form (timesteps, num_samples, num_joints, num_people, num_dimensions) for temporal stream
batch_x = batch_x.reshape(
(batch_x.shape[0], batch_x.shape[1],
batch_x.shape[2] // (NUM_PEOPLE * NUM_DIM), NUM_PEOPLE,
NUM_DIM))
# For half of the samples in the batch, we want to flip the people axis
swap_index = np.random.choice(batch_x.shape[1],
batch_x.shape[1] // 2,
replace=False)
batch_x[:, swap_index] = np.flip(batch_x[:, swap_index],
axis=3)
# Reshape it to old form
batch_x = batch_x.reshape(
(batch_x.shape[0], batch_x.shape[1],
batch_x.shape[2] * batch_x.shape[3] * batch_x.shape[4]))
elif 'arch' in self.data_kwargs and self.data_kwargs[
'arch'] == 'joint_temp_fused':
new_batch_x = []
# Iterate over both streams
for batch_x_comp in batch_x:
# Convert to form (num joints, num samples, timesteps, num people, num dimensions) for joint stream
# Convert to form (timesteps, num_samples, num_joints, num_people, num_dimensions) for temporal stream
batch_x_comp = batch_x_comp.reshape(
(batch_x_comp.shape[0], batch_x_comp.shape[1],
batch_x_comp.shape[2] // (NUM_PEOPLE * NUM_DIM),
NUM_PEOPLE, NUM_DIM))
# For half of the samples in the batch, we want to flip the people axis
swap_index = np.random.choice(batch_x_comp.shape[1],
batch_x_comp.shape[1] // 2,
replace=False)
batch_x_comp[:, swap_index] = np.flip(
batch_x_comp[:, swap_index], axis=3)
# Reshape it to old form
batch_x_comp = batch_x_comp.reshape(
(batch_x_comp.shape[0], batch_x_comp.shape[1],
batch_x_comp.shape[2] * batch_x_comp.shape[3] *
batch_x_comp.shape[4]))
batch_x_comp = [np.array(input) for input in batch_x_comp]
# For fused model during training, must append to single list of 25 instead of multiple lists
if new_arch_sample:
new_batch_x.append(batch_x_comp)
else:
new_batch_x += batch_x_comp
return new_batch_x, batch_y
else:
### Always swap half of the batch
### Swaps between the P1 joints and the P2 joints
#
# In half of samples in batch, switch between who is P1 and P2
#
# For instance, if 4 joints, dimension batch_x = (4,8)
# Dimension batch_y = (8) (num samples)
#
# >>> batch_x original
# array([[ 0, 1, 2, 3, 4, 5, 6, 7],
# [ 8, 9, 10, 11, 12, 13, 14, 15],
# [16, 17, 18, 19, 20, 21, 22, 23],
# [24, 25, 26, 27, 28, 29, 30, 31]])
# >>> batch_x new
# array([[16, 1, 18, 3, 20, 21, 6, 7],
# [24, 9, 26, 11, 28, 29, 14, 15],
# [ 0, 17, 2, 19, 4, 5, 22, 23],
# [ 8, 25, 10, 27, 12, 13, 30, 31]])
num_joints = len(batch_x) // 2
p1_joints = list(range(num_joints))
p2_joints = list(range(num_joints, num_joints * 2))
swap_index = sorted(
np.random.choice(list(range(batch_y.shape[0])),
batch_y.shape[0] // 2,
replace=False))
batch_x = np.array(batch_x)
batch_x[:, swap_index] = np.squeeze(
batch_x[p2_joints + p1_joints][:, [swap_index]], axis=1)
batch_x = [np.array(input) for input in batch_x]
# Must append to single list of 25 for training and validation
if 'arch' in self.data_kwargs and self.data_kwargs[
'arch'] == 'joint_temp_fused' and not new_arch_sample:
batch_x[0] = [np.array(input) for input in batch_x[0]]
batch_x[1] = [np.array(input) for input in batch_x[1]]
batch_x = batch_x[0] + batch_x[1]
return batch_x, batch_y
def __init__(self,
dataset_name,
dataset_fold,
subset,
batch_size=32,
reshuffle=False,
shuffle_indiv_order=False,
sample_method='central',
**data_kwargs):
if dataset_name == 'UT':
dataset = UT
self.pose_style = 'OpenPose'
elif dataset_name == 'SBU':
dataset = SBU
self.pose_style = 'SBU'
elif dataset_name == 'YMJA':
dataset = YMJA
self.pose_style = 'YMJA'
elif dataset_name == 'NTU':
dataset = NTU
self.pose_style = 'NTU'
elif dataset_name == 'NTU-V2':
dataset = NTU_V2
self.pose_style = 'NTU-V2'
self.batch_size = batch_size
self.reshuffle = reshuffle
self.data_kwargs = data_kwargs
self.shuffle_indiv_order = shuffle_indiv_order
self.subset = subset
self.sample_method = sample_method
if subset == 'train':
self.ground_truth = dataset.get_train_gt(dataset_fold)
elif subset == 'validation':
self.ground_truth = dataset.get_val_gt(dataset_fold)
self.num_classes = self.ground_truth.action.max() + 1
if sample_method == 'central':
num_videos = self.ground_truth.shape[0]
if subset == 'train':
self.shuffled_idx = np.random.choice(
self.ground_truth.index.values, num_videos, replace=False)
elif subset == 'validation':
self.shuffled_idx = self.ground_truth.index.values
self.num_batches = int(np.ceil(num_videos / batch_size))
elif sample_method == 'all':
self.data_kwargs['flat_seqs'] = True
if self.data_kwargs.get('seq_step') is None:
self.data_kwargs[
'seq_step'] = self.data_kwargs['timesteps'] // 2
seqs_mapping_file = 'seqs_mapping-subset_{}-fold_{}-timesteps_{}-{}-seq_step_{}.csv'.format(
subset, dataset_fold, data_kwargs['timesteps'],
data_kwargs['skip_timesteps'], data_kwargs['seq_step'])
seqs_mapping_filepath = dataset.DATA_DIR + '/seqs_mapping/' + seqs_mapping_file
if os.path.exists(seqs_mapping_filepath):
type_index = type(self.ground_truth.index[0])
seqs_mapping = np.genfromtxt(seqs_mapping_filepath,
dtype='str')
seqs_mapping = [[type_index(r[0]), int(r[1])]
for r in seqs_mapping]
else:
seqs_mapping = [
] # Array with mapping for each sequence: (video_idx, seq_idx)
for video_idx in self.ground_truth.index.values:
_, Y = get_data(self.ground_truth.loc[[video_idx]],
pose_style=self.pose_style,
sample_method=sample_method,
**self.data_kwargs)
num_seqs = len(Y)
seqs_mapping += [[video_idx, seq_idx]
for seq_idx in range(num_seqs)]
np.savetxt(seqs_mapping_filepath, seqs_mapping, fmt='%s')
self.seqs_mapping = seqs_mapping
if subset == 'train':
self.shuffled_idx = np.random.choice(range(len(seqs_mapping)),
len(seqs_mapping),
replace=False)
elif subset == 'validation':
self.shuffled_idx = list(range(len(seqs_mapping)))
self.num_batches = int(np.ceil(len(seqs_mapping) / batch_size))
# Validating data_kwargs
_ = get_data(self.ground_truth.sample(),
pose_style=self.pose_style,
sample_method=sample_method,
**self.data_kwargs)
