def load_cmu_data(_path,
V,
C,
joints_to_model=None,
frame_drop=1,
add_mirrored=False):
data_path = os.path.join(_path, 'data_cmu_cleaned')
cmu_data_dict_file = os.path.join(
_path, 'cmu_data_dict_drop_' + str(frame_drop) + '.npz')
try:
data_dict = np.load(cmu_data_dict_file,
allow_pickle=True)['data_dict'].item()
min_time_steps = np.load(cmu_data_dict_file,
allow_pickle=True)['min_time_steps'].item()
print('Data file found. Returning data.')
except FileNotFoundError:
cmu_data_files = glob.glob(os.path.join(data_path, '*.bvh'))
channel_map = {'Xrotation': 'x', 'Yrotation': 'y', 'Zrotation': 'z'}
mocap = MocapDataset(V, C)
data_dict = dict()
labels_file = os.path.join(_path, 'labels_cmu/cmu_labels.csv')
data_name = ['' for _ in range(len(cmu_data_files))]
labels = []
with open(os.path.join(labels_file)) as csv_file:
read_lines = csv.reader(csv_file, delimiter=',')
row_count = -1
for row in read_lines:
row_count += 1
if row_count == 0:
labels_order = [x.lower() for x in row[1:]]
continue
data_name[row_count - 1] = row[0]
labels.append(list(map(float, row[1:])))
labels = np.stack(labels)
labels /= np.linalg.norm(labels, ord=1, axis=-1)[..., None]
emo_idx = [
labels_order.index(x)
for x in ['happy', 'sad', 'angry', 'neutral']
]
labels = labels[:, emo_idx]
num_files = len(cmu_data_files)
min_time_steps = np.inf
for data_counter, file in enumerate(cmu_data_files):
offsets, positions, orientations, rot_in, rotations =\
mocap.load_bvh(file, channel_map, joints_to_model=joints_to_model)
if len(positions) - 1 < min_time_steps:
min_time_steps = len(positions) - 1
data_dict[str(data_counter)] = mocap.get_features_from_data(
'cmu',
offsets=offsets,
positions=positions[1::frame_drop],
orientations=orientations[1::frame_drop],
rotations=rotations[1::frame_drop])
file_name = file.split('/')[-1].split('.')[0]
# mocap.save_as_bvh(np.expand_dims(positions[:, 0], axis=0),
# np.expand_dims(np.expand_dims(orientations, axis=-1), axis=0),
# np.expand_dims(rotations, axis=0),
# np.expand_dims(rot_in, axis=0),
# dataset_name='cmu', subset_name='test', save_file_names=[file_name])
data_dict[str(data_counter)]['labels'] = labels[data_name.index(
file_name)]
print('\rData file not found. Processing file {}/{}: {:3.2f}%'.
format(data_counter + 1, num_files,
data_counter * 100. / num_files),
end='')
min_time_steps = int(min_time_steps / frame_drop)
print('\rData file not found. Processing files: done. Saving...',
end='')
np.savez_compressed(cmu_data_dict_file,
data_dict=data_dict,
min_time_steps=min_time_steps)
print('done. Returning data.')
return data_dict, min_time_steps
def load_edin_data(_path,
num_labels,
frame_drop=1,
add_mirrored=False,
randomized=True):
if add_mirrored:
edin_data_dict_file = os.path.join(
_path,
'edin_data_dict_with_mirrored_drop_' + str(frame_drop) + '.npz')
else:
edin_data_dict_file = os.path.join(
_path, 'edin_data_dict_drop_' + str(frame_drop) + '.npz')
try:
data_dict = np.load(edin_data_dict_file,
allow_pickle=True)['data_dict'].item()
print('Data file found. Returning data.')
except FileNotFoundError:
data_dict = dict()
data_counter = 0
bvh_files = glob.glob(
os.path.join(_path, 'data_edin_original') + '/*.bvh')
num_files = len(bvh_files)
discard_idx = []
for fidx, bvh_file in enumerate(bvh_files):
names, parents, offsets, \
positions, rotations = MocapDataset.load_bvh([f for f in bvh_files if str(fidx).zfill(6) in f][0])
if len(positions) < 241:
discard_idx.append(fidx)
continue
positions_down_sampled = positions[1::frame_drop]
rotations_down_sampled = rotations[1::frame_drop]
joints_dict = dict()
joints_dict['joints_to_model'] = np.arange(len(parents))
joints_dict['joint_parents_all'] = parents
joints_dict['joint_parents'] = parents
joints_dict['joint_names_all'] = names
joints_dict['joint_names'] = names
joints_dict['joint_offsets_all'] = offsets
joints_dict['joints_left'] = [
idx for idx, name in enumerate(names)
if 'left' in name.lower()
]
joints_dict['joints_right'] = [
idx for idx, name in enumerate(names)
if 'right' in name.lower()
]
dict_key = str(data_counter)
data_counter += 1
data_dict[dict_key] = dict()
data_dict[dict_key]['joints_dict'] = joints_dict
data_dict[dict_key]['positions'] = positions_down_sampled
data_dict[dict_key]['rotations'] = rotations_down_sampled
data_dict[dict_key][
'affective_features'] = MocapDataset.get_affective_features(
positions_down_sampled)
data_dict[dict_key]['trans_and_controls'] =\
MocapDataset.compute_translations_and_controls(data_dict[dict_key])
data_dict[dict_key]['spline'] = MocapDataset.compute_splines(
data_dict[dict_key])
print('\rData file not found. Processing file: {:3.2f}%'.format(
fidx * 100. / num_files),
end='')
print('\rData file not found. Processing file: done. Saving...',
end='')
labels, num_annotators = load_edin_labels(
_path, np.array([num_files], dtype='int'))
if add_mirrored:
labels = np.repeat(labels, 2, axis=0)
label_partitions = np.append([0], np.cumsum(num_labels))
for lpidx in range(len(num_labels)):
labels[:, label_partitions[lpidx]:label_partitions[lpidx + 1]] = \
labels[:, label_partitions[lpidx]:label_partitions[lpidx + 1]] / \
np.linalg.norm(labels[:, label_partitions[lpidx]:label_partitions[lpidx + 1]], ord=1, axis=1)[:, None]
for data_counter, idx in enumerate(
np.setdiff1d(range(num_files), discard_idx)):
data_dict[str(data_counter)]['labels'] = labels[idx]
np.savez_compressed(edin_data_dict_file, data_dict=data_dict)
print('done. Returning data.')
return data_dict, split_data_dict(data_dict, randomized=randomized)
def load_data_with_glove(_path,
dataset,
embedding_src,
frame_drop=1,
add_mirrored=False):
data_path = os.path.join(_path, dataset)
data_dict_file = os.path.join(
data_path, 'data_dict_glove_drop_' + str(frame_drop) + '.npz')
try:
data_dict = np.load(data_dict_file,
allow_pickle=True)['data_dict'].item()
word2idx = np.load(data_dict_file,
allow_pickle=True)['word2idx'].item()
embedding_table = np.load(data_dict_file,
allow_pickle=True)['embedding_table']
tag_categories = list(
np.load(data_dict_file, allow_pickle=True)['tag_categories'])
max_time_steps = np.load(data_dict_file,
allow_pickle=True)['max_time_steps'].item()
print('Data file found. Returning data.')
except FileNotFoundError:
data_dict = []
word2idx = []
embedding_table = []
tag_categories = []
max_time_steps = 0.
if dataset == 'mpi':
channel_map = {
'Xrotation': 'x',
'Yrotation': 'y',
'Zrotation': 'z'
}
data_dict = dict()
tag_names = []
with open(os.path.join(data_path, 'tag_names.txt')) as names_file:
for line in names_file.readlines():
line = line[:-1]
tag_names.append(line)
id = tag_names.index('ID')
relevant_tags = [
'Intended emotion', 'Intended polarity', 'Perceived category',
'Perceived polarity', 'Acting task', 'Gender', 'Age',
'Handedness', 'Native tongue', 'Text'
]
tag_categories = [[] for _ in range(len(relevant_tags) - 1)]
tag_files = glob.glob(os.path.join(data_path, 'tags/*.txt'))
num_files = len(tag_files)
for tag_file in tag_files:
tag_data = []
with open(tag_file) as f:
for line in f.readlines():
line = line[:-1]
tag_data.append(line)
for category in range(len(tag_categories)):
tag_to_append = relevant_tags[category]
if tag_data[tag_names.index(
tag_to_append)] not in tag_categories[category]:
tag_categories[category].append(
tag_data[tag_names.index(tag_to_append)])
all_texts = [[] for _ in range(len(tag_files))]
for data_counter, tag_file in enumerate(tag_files):
tag_data = []
with open(tag_file) as f:
for line in f.readlines():
line = line[:-1]
tag_data.append(line)
bvh_file = os.path.join(data_path,
'bvh/' + tag_data[id] + '.bvh')
names, parents, offsets,\
positions, rotations = MocapDataset.load_bvh(bvh_file, channel_map)
positions_down_sampled = positions[1::frame_drop]
rotations_down_sampled = rotations[1::frame_drop]
if len(positions_down_sampled) > max_time_steps:
max_time_steps = len(positions_down_sampled)
joints_dict = dict()
joints_dict['joints_to_model'] = np.arange(len(parents))
joints_dict['joints_parents_all'] = parents
joints_dict['joints_parents'] = parents
joints_dict['joints_names_all'] = names
joints_dict['joints_names'] = names
joints_dict['joints_offsets_all'] = offsets
joints_dict['joints_left'] = [
idx for idx, name in enumerate(names)
if 'left' in name.lower()
]
joints_dict['joints_right'] = [
idx for idx, name in enumerate(names)
if 'right' in name.lower()
]
data_dict[tag_data[id]] = dict()
data_dict[tag_data[id]]['joints_dict'] = joints_dict
data_dict[tag_data[id]]['positions'] = positions_down_sampled
data_dict[tag_data[id]]['rotations'] = rotations_down_sampled
data_dict[tag_data[id]]['affective_features'] =\
MocapDataset.get_mpi_affective_features(positions_down_sampled)
for tag_index, tag_name in enumerate(relevant_tags):
if tag_name.lower() == 'text':
all_texts[data_counter] = [
e for e in str.split(tag_data[tag_names.index(
tag_name)]) if e.isalnum()
]
data_dict[tag_data[id]][tag_name] = tag_data[
tag_names.index(tag_name)]
text_length = len(data_dict[tag_data[id]][tag_name])
continue
if tag_name.lower() == 'age':
data_dict[tag_data[id]][tag_name] = float(
tag_data[tag_names.index(tag_name)]) / 100.
continue
if tag_name is 'Perceived category':
categories = tag_categories[0]
elif tag_name is 'Perceived polarity':
categories = tag_categories[1]
else:
categories = tag_categories[tag_index]
data_dict[tag_data[id]][tag_name] = to_one_hot(
tag_data[tag_names.index(tag_name)], categories)
print(
'\rData file not found. Reading data files {}/{}: {:3.2f}%'
.format(data_counter + 1, num_files,
data_counter * 100. / num_files),
end='')
print('\rData file not found. Reading files: done.')
print('Preparing embedding table:')
word2idx = build_vocab_idx(all_texts, min_word_count=0)
embedding_table = build_embedding_table(embedding_src, word2idx)
np.savez_compressed(data_dict_file,
data_dict=data_dict,
word2idx=word2idx,
embedding_table=embedding_table,
tag_categories=tag_categories,
max_time_steps=max_time_steps)
print('done. Returning data.')
elif dataset == 'creative_it':
mocap_data_dirs = os.listdir(os.path.join(data_path, 'mocap'))
for mocap_dir in mocap_data_dirs:
mocap_data_files = glob.glob(
os.path.join(data_path, 'mocap/' + mocap_dir + '/*.txt'))
else:
raise FileNotFoundError('Dataset not found.')
return data_dict, word2idx, embedding_table, tag_categories, max_time_steps
def load_data(_path, dataset, frame_drop=1, add_mirrored=False):
data_path = os.path.join(_path, dataset)
data_dict_file = os.path.join(data_path,
'data_dict_drop_' + str(frame_drop) + '.npz')
try:
data_dict = np.load(data_dict_file,
allow_pickle=True)['data_dict'].item()
tag_categories = list(
np.load(data_dict_file, allow_pickle=True)['tag_categories'])
max_text_length = np.load(data_dict_file,
allow_pickle=True)['max_text_length'].item()
max_time_steps = np.load(data_dict_file,
allow_pickle=True)['max_time_steps'].item()
print('Data file found. Returning data.')
except FileNotFoundError:
data_dict = []
tag_categories = []
max_text_length = 0.
max_time_steps = 0.
if dataset == 'mpi':
channel_map = {
'Xrotation': 'x',
'Yrotation': 'y',
'Zrotation': 'z'
}
data_dict = dict()
tag_names = []
with open(os.path.join(data_path, 'tag_names.txt')) as names_file:
for line in names_file.readlines():
line = line[:-1]
tag_names.append(line)
id = tag_names.index('ID')
relevant_tags = [
'Intended emotion', 'Intended polarity', 'Perceived category',
'Perceived polarity', 'Acting task', 'Gender', 'Age',
'Handedness', 'Native tongue', 'Text'
]
tag_categories = [[] for _ in range(len(relevant_tags) - 1)]
tag_files = glob.glob(os.path.join(data_path, 'tags/*.txt'))
num_files = len(tag_files)
for tag_file in tag_files:
tag_data = []
with open(tag_file) as f:
for line in f.readlines():
line = line[:-1]
tag_data.append(line)
for category in range(len(tag_categories)):
tag_to_append = relevant_tags[category]
if tag_data[tag_names.index(
tag_to_append)] not in tag_categories[category]:
tag_categories[category].append(
tag_data[tag_names.index(tag_to_append)])
for data_counter, tag_file in enumerate(tag_files):
tag_data = []
with open(tag_file) as f:
for line in f.readlines():
line = line[:-1]
tag_data.append(line)
bvh_file = os.path.join(data_path,
'bvh/' + tag_data[id] + '.bvh')
names, parents, offsets,\
positions, rotations, base_fps = MocapDataset.load_bvh(bvh_file, channel_map)
positions_down_sampled = positions[1::frame_drop]
rotations_down_sampled = rotations[1::frame_drop]
if len(positions_down_sampled) > max_time_steps:
max_time_steps = len(positions_down_sampled)
joints_dict = dict()
joints_dict['joints_to_model'] = np.arange(len(parents))
joints_dict['joints_parents_all'] = parents
joints_dict['joints_parents'] = parents
joints_dict['joints_names_all'] = names
joints_dict['joints_names'] = names
joints_dict['joints_offsets_all'] = offsets
joints_dict['joints_left'] = [
idx for idx, name in enumerate(names)
if 'left' in name.lower()
]
joints_dict['joints_right'] = [
idx for idx, name in enumerate(names)
if 'right' in name.lower()
]
data_dict[tag_data[id]] = dict()
data_dict[tag_data[id]]['joints_dict'] = joints_dict
data_dict[tag_data[id]]['positions'] = positions_down_sampled
data_dict[tag_data[id]]['rotations'] = rotations_down_sampled
data_dict[tag_data[id]]['affective_features'] =\
MocapDataset.get_mpi_affective_features(positions_down_sampled)
for tag_index, tag_name in enumerate(relevant_tags):
if tag_name.lower() == 'text':
data_dict[tag_data[id]][tag_name] =\
tag_data[tag_names.index(tag_name)].replace(' s ', '\'s ').replace(' t ', '\'t ')
text_vad = []
words = data_dict[tag_data[id]][tag_name].split(' ')
for lexeme in words:
if lexeme.isalpha():
if len(lexeme) == 1 and not (
lexeme.lower() is 'a'
or lexeme.lower() is 'i'):
continue
text_vad.append(get_vad(lexeme))
try:
data_dict[tag_data[id]][
tag_name + ' VAD'] = np.stack(text_vad)
data_dict[tag_data[id]]['best_tts_rate'],\
data_dict[tag_data[id]]['gesture_splits'] =\
get_gesture_splits(data_dict[tag_data[id]][tag_name], words,
len(data_dict[tag_data[id]]['positions']),
base_fps / frame_drop)
except ValueError:
data_dict[tag_data[id]][tag_name +
' VAD'] = np.zeros((0, 3))
text_length = len(data_dict[tag_data[id]][tag_name])
if text_length > max_text_length:
max_text_length = text_length
continue
if tag_name.lower() == 'age':
data_dict[tag_data[id]][tag_name] = float(
tag_data[tag_names.index(tag_name)]) / 100.
continue
if tag_name is 'Perceived category':
categories = tag_categories[0]
elif tag_name is 'Perceived polarity':
categories = tag_categories[1]
else:
categories = tag_categories[tag_index]
data_dict[tag_data[id]][tag_name] = to_one_hot(
tag_data[tag_names.index(tag_name)], categories)
if tag_name is 'Intended emotion' or tag_name is 'Perceived category':
data_dict[tag_data[id]][tag_name + ' VAD'] = get_vad(
tag_data[tag_names.index(tag_name)])
print('\rData file not found. Processing file {}/{}: {:3.2f}%'.
format(data_counter + 1, num_files,
data_counter * 100. / num_files),
end='')
print('\rData file not found. Processing files: done. Saving...',
end='')
np.savez_compressed(data_dict_file,
data_dict=data_dict,
tag_categories=tag_categories,
max_text_length=max_text_length,
max_time_steps=max_time_steps)
print('done. Returning data.')
elif dataset == 'creative_it':
mocap_data_dirs = os.listdir(os.path.join(data_path, 'mocap'))
for mocap_dir in mocap_data_dirs:
mocap_data_files = glob.glob(
os.path.join(data_path, 'mocap/' + mocap_dir + '/*.txt'))
else:
raise FileNotFoundError('Dataset not found.')
return data_dict, tag_categories, max_text_length, max_time_steps
def load_data_MPI(_path, _ftype, coords, joints, cycles=3):
# Counts: 'Hindi': 292, 'German': 955, 'English': 200
bvhDirectory = os.path.join(_path, "bvh")
tagDirectory = os.path.join(_path, "tags")
data_list = {}
num_samples = 0
time_steps = 0
labels_list = {}
fileIDs = []
for filenum in range(1, 1452):
filename = str(filenum).zfill(6)
# print(filenum)
if not os.path.exists(os.path.join(tagDirectory, filename + ".txt")):
print(os.path.join(tagDirectory, filename + ".txt"), " not found!")
continue
names, parents, offsets, positions, rotations = MocapDataset.load_bvh(
os.path.join(bvhDirectory, filename + ".bvh"))
tag, text = MocapDataset.read_tags(
os.path.join(tagDirectory, filename + ".txt"))
num_samples += 1
positions = np.reshape(
positions,
(positions.shape[0], positions.shape[1] * positions.shape[2]))
data_list[filenum] = list(positions)
time_steps_curr = len(positions)
if time_steps_curr > time_steps:
time_steps = time_steps_curr
if "Hindi" in tag:
labels_list[filenum] = 0
elif "German" in tag:
labels_list[filenum] = 1
elif "English" in tag:
labels_list[filenum] = 2
else:
print("ERROR: ", tag)
fileIDs.append(filenum)
labels = np.empty(num_samples)
data = np.empty((num_samples, time_steps * cycles, joints * coords))
index = 0
for si in fileIDs:
data_list_curr = np.tile(
data_list[si], (int(np.ceil(time_steps / len(data_list[si]))), 1))
if (data_list_curr.shape[1] != 69):
continue
for ci in range(cycles):
data[index, time_steps * ci:time_steps *
(ci + 1), :] = data_list_curr[0:time_steps]
labels[index] = labels_list[si]
index += 1
data = data[:index]
labels = labels[:index]
print(index, num_samples)
# data = common.get_affective_features(np.reshape(data, (data.shape[0], data.shape[1], joints, coords)))[:, :, :48]
data_train, data_test, labels_train, labels_test = train_test_split(
data, labels, test_size=0.1)
data_train, labels_train = balance_classes(data_train, labels_train)
return data, labels, data_train, labels_train, data_test, labels_test
import torch
from utils.mocap_dataset import MocapDataset as MD
anim = MD.load_bvh(
'/media/uttaran/repo0/Gamma/MotionSim/src/quater_long_term_emonet_2/render/bvh/edin/test/000001.bvh'
)
animation_pred = {
'joint_names': anim[0],
'joint_offsets': torch.from_numpy(anim[2][1:]).unsqueeze(0),
'joint_parents': anim[1],
'positions': torch.from_numpy(anim[3]).unsqueeze(0),
'rotations': torch.from_numpy(anim[4][1:]).unsqueeze(0)
}
MD.save_as_bvh(animation_pred, dataset_name='edin', subset_name='fixed')