def load_seq_datasets(data_path,
filelist,
timesteps,
feature_dimension,
ext,
mask_value=0): #This pads zeros according to max seq len
startfile = 1
num_records = 0
filenames = file(filelist).readlines()
merlin_io = binary_io.BinaryIOCollection()
for filename in filenames:
load_mat = merlin_io.load_binary_file(
data_path + filename.strip() + ext, feature_dimension)
remaining_frames = load_mat.shape[0] % timesteps
num_mask_frames = 0 if remaining_frames == 0 else timesteps - remaining_frames
mask_mat = np.ones((num_mask_frames, load_mat.shape[1]),
dtype=float) * float(mask_value)
if startfile:
startfile = 0
data_mat = np.r_[load_mat, mask_mat]
num_records += np.ceil(load_mat.shape[0] / float(timesteps))
else:
data_mat = np.r_[data_mat, load_mat, mask_mat]
num_records += np.ceil(load_mat.shape[0] / float(timesteps))
#num_records=num_of expected time chunks for BPTT
#The input to LSTM needs to be a 3-D tensor of shape: num_of_utterances x num_timesteps x feature_dimension
reshaped_data_mat = np.reshape(data_mat,
(-1, timesteps, feature_dimension))
assert (
reshaped_data_mat.shape[0] == num_records
), "Num of records do not match reshpaed matrix. num of records = %d, reshaped size = %d" % (
num_records, reshaped_data_mat.shape[0])
return reshaped_data_mat
def predict_tris(data_path, test_filelist, model_dir, chkpt, save_dir,ext, n_input_dim=1488):
#sgd = SGD(lr=0.01, decay=1e-6,momentum=0.9, nesterov=False, clipnorm=0.01)
# load weights into new model
model_name= model_dir + 'chkpt-' + str(chkpt)
json_file = open(model_name+'.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json)
model.load_weights(model_name+'.h5')
print("Loaded model from disk")
sgd = SGD(lr=0.04, momentum =0.5, clipvalue=0.01, decay=1e-8)
model.compile(loss={'ymean': 'mean_sum_error','ystd': 'mean_sum_error', 'nmean': 'mean_sum_error','nstd': 'mean_sum_error', 'ling_out': 'mean_sum_error','QA_out': 'mean_sum_error'}, loss_weights={'ymean':1., 'ystd':0.6, 'nmean':1., 'nstd':0.6, 'ling_out':0.1, 'QA_out':0.1}, optimizer=sgd)
merlin_io = binary_io.BinaryIOCollection()
with open(test_filelist, 'r') as fl:
for filename in fl:
print(filename)
ling_feat, qa, mean_feat, std_feat = load_tris_data_file(data_path, save_dir, filename, int(n_input_dim), ext=ext[0])
prediction = model.predict({'ling_in':ling_feat, 'QA_in':qa, 'AD_mean':mean_feat, 'AD_std':std_feat}, batch_size=1,
verbose=2)
print(len(prediction))
if not os.path.exists( save_dir):
os.makedirs(save_dir)
fname= save_dir + filename.strip()
merlin_io.array_to_binary_file(prediction[0], fname +'Y.mean')
merlin_io.array_to_binary_file(prediction[1], fname + 'Y.std')
merlin_io.array_to_binary_file(prediction[2], fname + 'N.mean')
merlin_io.array_to_binary_file(prediction[3], fname + 'N.std')
print("predicted features saved in ", save_dir)
return
def load_stats(data_path, filelist, feature_dimension, ext):
startfile = 1
filenames = file(filelist).readlines()
merlin_io = binary_io.BinaryIOCollection()
for filename in filenames:
ym = merlin_io.load_binary_file(
data_path + filename.strip() + '.ymean', feature_dimension)
ys = merlin_io.load_binary_file(data_path + filename.strip() + '.ystd',
feature_dimension)
nm = merlin_io.load_binary_file(
data_path + filename.strip() + '.nmean', feature_dimension)
ns = merlin_io.load_binary_file(data_path + filename.strip() + '.nstd',
feature_dimension)
if startfile:
startfile = 0
ymean = ym
ystd = ys
nmean = nm
nstd = ns
else:
ymean = np.r_[ymean, ym]
ystd = np.r_[ystd, ys]
nmean = np.r_[nmean, nm]
nstd = np.r_[nstd, ns]
return ymean, ystd, nmean, nstd
def load_tris_data(data_path, filelist, feature_dimension, ext):
startfile=1
filenames=file(filelist).readlines()
merlin_io=binary_io.BinaryIOCollection()
for filename in filenames:
load_mat=merlin_io.load_binary_file(data_path + filename.strip() + ext, feature_dimension)
mean_feat=merlin_io.load_binary_file(data_path + filename.strip() + '.mean', 52)
std_feat=merlin_io.load_binary_file(data_path + filename.strip() + '.std', 52)
qa = load_mat[:,:777]
assert qa.shape[1] == 777
ling_feat=load_mat[:,777:]
assert ling_feat.shape[1] == 711
if startfile:
startfile=0
ling_mat=ling_feat
qa_mat = qa
mean_mat = mean_feat
std_mat = std_feat
else:
ling_mat=np.r_[ling_mat, ling_feat]
qa_mat=np.r_[qa_mat, qa]
mean_mat=np.r_[mean_mat, mean_feat]
std_mat = np.r_[std_mat, std_feat]
return ling_mat, qa_mat, mean_mat, std_mat
def make_binary_tris_feats(tris_file, indir, outdir):
io_funcs = binary_io.BinaryIOCollection()
main_path = '/home/pbaljeka/TRIS_Exps3/'
source_name = 'cmu_us_slt'
source_path = main_path + source_name + '/festival/' + indir + '/' #Change the input folder to tris if you want it nodewise else coeffs
full_path = source_path + tris_file.strip()
node_stats_path = main_path + source_name + '/festival/node_stats/'
tris_utils = main_path + '/utils/'
outdir = main_path + source_name + '/festival/' + outdir + '/' #Change the output folder to nn_tris_trees to save the nodewise feats there. else nn_tris
if not os.path.exists(outdir):
os.makedirs(outdir)
outfile = outdir + tris_file.strip()
nodenames = tris_utils + '/nodenames'
q_list = question_list(tris_utils + '/question_list')
statenames = tris_utils + '/statenames'
phonenames = tris_utils + '/phonenames'
node_dict_mean, node_dict_std = make_node_dict_stat(
nodenames, node_stats_path)
tris_mat = []
mean_mat = []
std_mat = []
with open(full_path + '.tris', 'r') as f:
for line in f:
if line.strip().split()[0][-1] <> "L":
binary_tris_frame, node_mean, node_std = get_binary_tris(
line.strip(), node_dict_mean, node_dict_std, q_list,
statenames, phonenames)
tris_mat.append(binary_tris_frame)
mean_mat.append(node_mean)
std_mat.append(node_std)
io_funcs.array_to_binary_file(tris_mat, outfile + '.tris')
io_funcs.array_to_binary_file(mean_mat, outfile + '.mean')
io_funcs.array_to_binary_file(std_mat, outfile + '.std')
return
def do_acoustic_denormalisation_trees(data_dir,
utils_dir,
model_name,
n_input_dim=52):
logger = logging.getLogger("acoustic_denormalization")
logger.info('normalising acoustic (output) features using MVN')
all_list = utils_dir + '/senones'
#in_data_dir=data_dir + '/norm_nn_tris_trees/'
#out_data_dir=data_dir + '/check_denorm_trees/'
in_data_dir = data_dir + '/predicted_norm_nn_tris_trees/' + model_name + '/'
out_data_dir = data_dir + '/predicted_norm_nn_tris_trees/un_norm_' + model_name + '/'
if not os.path.exists(out_data_dir):
os.makedirs(out_data_dir)
merlin_io = binary_io.BinaryIOCollection()
for stat in ['mean', 'std']:
mean = merlin_io.load_binary_file(utils_dir + stat + '.mean', int(52))
std = merlin_io.load_binary_file(utils_dir + stat + '.std', int(52))
with open(all_list, 'r') as f:
for filename in f:
denorm_data = denormalize_data(in_data_dir,
filename.strip(),
mean,
std,
'.' + stat,
feature_dimension=52)
outfile = out_data_dir + filename.strip() + '.' + stat
merlin_io.array_to_binary_file(denorm_data, outfile)
def do_acoustic_normalisation_coeffs(data_dir, utils_dir, n_input_dim=52):
logger = logging.getLogger("acoustic_normalization")
logger.info('normalising acoustic (output) features using MVN')
all_list = utils_dir + '/all_list'
in_data_dir = data_dir + '/nn_tris/'
out_data_dir = data_dir + '/norm_nn_tris/'
if not os.path.exists(out_data_dir):
os.makedirs(out_data_dir)
merlin_io = binary_io.BinaryIOCollection()
for ext in ['.mean', '.ymean', '.nmean']:
mean = merlin_io.load_binary_file(utils_dir + 'mean.mean', int(52))
std = merlin_io.load_binary_file(utils_dir + 'mean.std', int(52))
with open(all_list, 'r') as f:
for filename in f:
norm_data = normalize_data(in_data_dir,
filename.strip(),
mean,
std,
ext,
feature_dimension=52)
outfile = out_data_dir + filename.strip() + ext
merlin_io.array_to_binary_file(norm_data, outfile)
for ext in ['.std', '.ystd', '.nstd']:
mean = merlin_io.load_binary_file(utils_dir + 'std.mean', int(52))
std = merlin_io.load_binary_file(utils_dir + 'std.std', int(52))
with open(all_list, 'r') as f:
for filename in f:
norm_data = normalize_data(in_data_dir,
filename.strip(),
mean,
std,
ext,
feature_dimension=52)
outfile = out_data_dir + filename.strip() + ext
merlin_io.array_to_binary_file(norm_data, outfile)
def make_acoustic_tris_feats(tris_file, node, stat):
io_funcs = binary_io.BinaryIOCollection()
main_path='/home/pbaljeka/TRIS_Exps3/'
source_name= 'cmu_us_slt'
source_path = main_path + source_name + '/festival/coeffs/'
stats_path = main_path + source_name + '/festival/node_stats/'
full_path= source_path + tris_file.strip()
tris_utils= main_path + '/utils/'
outdir= main_path + source_name + '/festival/nn_tris/'
if not os.path.exists(outdir):
os.makedirs(outdir)
ext=node[0] + stat
outfile= outdir + tris_file.strip() + '.' + ext
final_mat=[]
if node == "yes":
column_num=1
elif node== "no":
column_num=2
else:
print("Wrong option")
collapse_nodes=['pau_3_NNYNNYYYYNYNNYYNNYN','pau_3_NNNYNNYYNYNYYN','pau_3_NNYNNYYYYNYNNYYYYNYN', 's_2_YYYYN']
with open(full_path + '.tris', 'r') as f:
for line in f:
columns=line.strip().split()
if columns[0][-1] <> "L":
if columns[int(column_num)] in collapse_nodes:
column = columns[int(column_num - 1)]
else:
column = columns[int(column_num)]
#print(columns[int(column_num)])
final_mat.append(get_stat(column, stats_path,int(52), '.' + stat))
#print(final_mat[-1])
io_funcs.array_to_binary_file(final_mat, outfile)
return
def normalize_data(data_path, filename, mean, std, ext, feature_dimension=52):
merlin_io = binary_io.BinaryIOCollection()
data_mat = merlin_io.load_binary_file(data_path + filename.strip() + ext,
feature_dimension)
mean_mat = np.tile(mean, (data_mat.shape[0], 1))
std_mat = np.tile(std, (data_mat.shape[0], 1)) + 0.000000001
norm_data = (data_mat - mean_mat) / std_mat
return norm_data
def load_tris_data_file(data_path, predict_data_path, filename, feature_dimension, ext):
merlin_io=binary_io.BinaryIOCollection()
load_mat=merlin_io.load_binary_file(data_path + filename.strip() + ext, feature_dimension)
mean_feat=merlin_io.load_binary_file(predict_data_path + filename.strip() + '.mean', 52)
std_feat=merlin_io.load_binary_file(predict_data_path + filename.strip() + '.std', 52)
qa = load_mat[:,:777]
assert qa.shape[1] == 777
ling_feat=load_mat[:,777:]
assert ling_feat.shape[1] == 711
return ling_feat, qa, mean_feat, std_feat
def load_data(
data_path,
filename,
):
merlin_io = binary_io.BinaryIOCollection()
mean_mat = merlin_io.load_binary_file(data_path + filename + '.mean',
int(52))
assert mean_mat.shape[1] == 52, "Data loading improper"
std_mat = merlin_io.load_binary_file(data_path + filename + '.std',
int(52))
assert std_mat.shape[1] == 52, "Data loading improper"
return mean_mat, std_mat
def load_batch_datasets(data_path, filenames, feature_dimension, ext):
startfile = 1
merlin_io = binary_io.BinaryIOCollection()
for filename in filenames:
load_mat = merlin_io.load_binary_file(
data_path + filename.strip() + ext, feature_dimension)
if startfile:
startfile = 0
data_mat = load_mat
else:
data_mat = np.r_[data_mat, load_mat]
return data_mat, data_mat.shape[0]
def make_node_dict_stat(nodelist, node_stats_path):
"""This takes the list of nodes and creates a node dict, for the mean and std"""
node_dict_mean={}
node_dict_std={}
merlin_io=binary_io.BinaryIOCollection()
with open(nodelist, 'r') as f:
for node_num, line in enumerate(f):
filename=line.strip()
node_mean=merlin_io.load_binary_file(node_stats_path + filename + '.mean' ,52)
node_std=merlin_io.load_binary_file(node_stats_path + filename + '.std' ,52)
node_dict_mean[filename]=node_mean
node_dict_std[filename]=node_std
return node_dict_mean, node_dict_std
def load_datasets(data_path, filelist, feature_dimension, ext):
startfile = 1
filenames = file(filelist).readlines()
merlin_io = binary_io.BinaryIOCollection()
for filename in filenames:
load_mat = merlin_io.load_binary_file(
data_path + filename.strip() + ext, feature_dimension)
if startfile:
startfile = 0
data_mat = load_mat
else:
data_mat = np.r_[data_mat, load_mat]
return data_mat
def calc_acoustic_stats(data_path,
utils_dir,
chunksize=100,
feature_dimension=52):
filelist = utils_dir + '/all_list'
merlin_io = binary_io.BinaryIOCollection()
for ext in ['mean', 'std']:
print('Calculating stats for : ', ext)
mean_vec = calc_mean(data_path, filelist, '.' + ext, chunksize,
feature_dimension)
std_vec = calc_std(data_path, filelist, '.' + ext, mean_vec, chunksize,
feature_dimension)
merlin_io.array_to_binary_file(mean_vec, utils_dir + ext + '.mean')
merlin_io.array_to_binary_file(std_vec, utils_dir + ext + '.std')
def load_test_seq(data_path,
filename,
timesteps,
feature_dimension,
ext,
mask_value=0): #This pads zeros according to max seq len
merlin_io = binary_io.BinaryIOCollection()
load_mat = merlin_io.load_binary_file(data_path + filename.strip() + ext,
feature_dimension)
true_length = load_mat.shape[0]
remaining_frames = load_mat.shape[0] % timesteps
num_mask_frames = 0 if remaining_frames == 0 else timesteps - remaining_frames
mask_mat = np.ones(
(num_mask_frames, load_mat.shape[1]), dtype=float) * float(mask_value)
data_mat = np.r_[load_mat, mask_mat]
reshaped_data_mat = np.reshape(data_mat,
(-1, timesteps, feature_dimension))
return reshaped_data_mat, true_length
def predict(main_path,
test_filelist,
model_name,
chkpt,
timesteps=300,
n_input_dim=711,
n_output_dim=199):
data_path = main_path + '/data/'
save_dir = main_path + '/predicted_feats/' + model_name + '/'
model_path = main_path + '/models/' + model_name + '/chkpt-' + str(chkpt)
#sgd = SGD(lr=0.01, decay=1e-6,momentum=0.9, nesterov=False, clipnorm=0.01)
json_file = open(model_path + '.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights(model_path + '.h5')
print("Loaded model from disk")
loaded_model.compile(loss='mean_sum_error', optimizer='adam')
merlin_io = binary_io.BinaryIOCollection()
with open(test_filelist, 'r') as fl:
for filename in fl:
print(filename)
test_x, test_seq_length = load_test_seq(
data_path,
filename,
timesteps=timesteps,
feature_dimension=n_input_dim,
ext='.lab',
mask_value=0)
prediction = loaded_model.predict(test_x, batch_size=1, verbose=2)
actual_prediction = np.reshape(prediction, (-1, n_output_dim))
final_prediction = actual_prediction[:test_seq_length]
if not os.path.exists(save_dir):
os.makedirs(save_dir)
fname = save_dir + filename.strip() + '.cmp'
merlin_io.array_to_binary_file(final_prediction, fname)
print("predicted features saved in ", save_dir)
return
def binary_file_length(filename, feature_dimension):
merlin_io = binary_io.BinaryIOCollection()
load_mat = merlin_io.load_bianry_file(filename, feature_dimension)
return int(load_mat.shape[0])
def get_stat(nodename, main_path, feature_dimension, ext):
io_funcs = binary_io.BinaryIOCollection()
stat=io_funcs.load_binary_file(main_path +nodename.strip() + ext, feature_dimension)
return stat
import os
import sys
import numpy as np
import binary_io
io_funcs = binary_io.BinaryIOCollection()
#feature_file for non-terminal is:
#root_node, left_node, right_node,-3
#question, operator, operand-3
#root node name -1
#f0, acoustic feature, voicing - 1+ 50 + 1
#frame number, linguistic feature- 1+65
#TOTAL=123
def interleave(mean_vec, std_vec, outfile):
interleave_vec = np.vstack((mean_vec, std_vec)).reshape((-1, ), order='F')
#print(interleave_vec.shape)
with open(outfile, 'a+') as f:
np.savetxt(f,
interleave_vec.reshape(1, interleave_vec.shape[0]),
fmt='%.7f')
#np.savetxt(f, interleave_vec[None,:], delimiter='ii', newline='\n', fmt='%10.5f')
def load_data(
data_path,
filename,
):
merlin_io = binary_io.BinaryIOCollection()
mean_mat = merlin_io.load_binary_file(data_path + filename + '.mean',