def get_data_files(egs_dir, egs_type):
if egs_type == 'egs':
cmd = 'find ' + egs_dir + ' -iname "data.*.egs"'
elif egs_type == 'post':
cmd = 'find ' + egs_dir + ' -iname "data.*.post"'
else:
print_log(
'The egs type {egs_type} is not supported, use "egs" or "post", exiting script!'
.format(egs_type=egs_type))
sys.exit()
proc = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE)
x = proc.stdout.decode('utf-8')
data_files = []
for line in x.splitlines():
line = line.strip()
data_files.append(line)
return data_files
def get_egs(data_train, data_test, egs_dir, split_num, nnet_type):
os.mkdir(join(egs_dir, 'test'))
os.mkdir(join(egs_dir, 'train'))
print_log('Getting data dimension')
dim = int(get_dim(join(args.data_test, 'feats.scp')))
dim_2 = int(get_dim(join(args.data_train, 'feats.scp')))
if dim_2 != dim:
sys.exit(
'%s: Data dimensions of training and test data do not match, something is wrong, exiting script!'
% sys.argv[0])
with open(join(egs_dir, 'dim'), 'w') as fid:
fid.write('%d' % dim)
# Get egs for test data
split_dir = join(data_test, 'split' + str(split_num))
for batch in range(1, split_num + 1):
test_data, keys = fetch_feats(join(split_dir, str(batch)))
test_labels = fetch_labels(data_test, keys)
if nnet_type == 'cnn':
pic_dim = int(np.sqrt(dim))
test_data = np.reshape(test_data, (-1, pic_dim, pic_dim))
test_data = test_data[:, np.newaxis, :, :]
with open(join(egs_dir, 'test', 'data.' + str(batch) + '.egs'),
'wb') as fid:
pickle.dump(test_data, fid)
with open(join(egs_dir, 'test', 'labels.' + str(batch) + '.egs'),
'wb') as fid:
pickle.dump(test_labels, fid)
print_log('Finished generating test examples')
# Get egs for train data
split_dir = join(data_train, 'split' + str(split_num))
for batch in range(1, split_num + 1):
train_data, keys = fetch_feats(join(split_dir, str(batch)))
train_labels = fetch_labels(data_train, keys)
if nnet_type == 'cnn':
pic_dim = int(np.sqrt(dim))
train_data = np.reshape(train_data, (-1, pic_dim, pic_dim))
train_data = train_data[:, np.newaxis, :, :]
with open(join(egs_dir, 'train', 'data.' + str(batch) + '.egs'),
'wb') as fid:
pickle.dump(train_data, fid)
with open(join(egs_dir, 'train', 'labels.' + str(batch) + '.egs'),
'wb') as fid:
pickle.dump(train_labels, fid)
print_log('Finished generating nnet training examples')
proc = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE)
x = proc.stdout.decode('utf-8')
data_files = []
for line in x.splitlines():
line = line.strip()
data_files.append(line)
return data_files
if __name__ == '__main__':
args = get_args()
print_log('Combining data files of type {egs_type} from {dir1} and {dir2}'.
format(egs_type=args.egs_type,
dir1=args.egs_dir_1,
dir2=args.egs_dir_2))
files_1 = get_data_files(args.egs_dir_1, args.egs_type)
files_2 = get_data_files(args.egs_dir_2, args.egs_type)
if len(files_1) != len(files_2):
print_log(
'Number of data files of type {egs_type} in {dir1} and {dir2} are not equal, exiting script!'
.format(egs_type=args.egs_type,
dir1=args.egs_dir_1,
dir2=args.egs_dir_2))
sys.exit()
for i in range(len(files_1)):
comb_dat = combine(files_1[i], files_2[i])
print('%s: The final test performance is: %.2f %%' %
(sys.argv[0], re_er * 100))
# Save result
res_file = join(dirname(outmodel), 'result')
with open(res_file, 'w') as f:
f.write('Test set Frame Error Rate: %.2f %%' % (re_er * 100))
# Save model
with open(outmodel, 'wb') as fid:
pickle.dump(model, fid)
if __name__ == '__main__':
print_log('# BEGIN CNN TRAINING')
args = get_args()
gpu_id = get_device_id()
if gpu_id != -1:
print('%s: Using GPU device %d for nnet' % (sys.argv[0], gpu_id))
else:
print_log('Training nnet on single CPU, this will take some time!')
print_log('Defining nnet model')
with open(join(args.egs_dir, 'dim'), 'r') as fid:
insize = int(np.sqrt(int(fid.readline())))
model = cnn_model(args.nlayers, args.ndepth, args.ksize, args.ntargets,
soft_out=np.zeros((frame_num,feat_dim))
for i in range(frame_num):
soft_out[i,:]=np.exp(x[i,:])/(np.sum(np.exp(x[i,:])))
return soft_out
def get_post(model,egs):
nnet=pickle.load(open(model,'rb'))
data=pickle.load(open(egs,'rb'))
x=Variable(torch.from_numpy(data).float())
x=nnet(x)
post=softmax(x.data.numpy())
return post
if __name__=='__main__':
args=get_args()
print_log('Obtaining the data files from {egs_dir}'.format(egs_dir=args.egs_dir))
files=get_data_files(args.egs_dir)
print_log('Obtaining posteriors')
count=1
for i in files:
post=get_post(args.model,i)
pickle.dump(post,open(join(args.post_dir,'data.'+str(count)+'.post'),'wb'))
count+=1
print_log('Finished obtaining posteriors from {egs_dir}'.format(egs_dir=args.egs_dir))
train_labels = fetch_labels(data_train, keys)
if nnet_type == 'cnn':
pic_dim = int(np.sqrt(dim))
train_data = np.reshape(train_data, (-1, pic_dim, pic_dim))
train_data = train_data[:, np.newaxis, :, :]
with open(join(egs_dir, 'train', 'data.' + str(batch) + '.egs'),
'wb') as fid:
pickle.dump(train_data, fid)
with open(join(egs_dir, 'train', 'labels.' + str(batch) + '.egs'),
'wb') as fid:
pickle.dump(train_labels, fid)
print_log('Finished generating nnet training examples')
if __name__ == '__main__':
args = get_args()
if args.nnet_type != 'vanilla' and args.nnet_type != 'cnn':
sys.exit(
'%s: Nnet type %s is not supported by pyspeech, exiting script!' %
(sys.argv[0], args.nnet_type))
print_log('Generating examples for nnet training')
get_egs(args.data_train, args.data_test, args.egs_dir, args.split_num,
args.nnet_type)
with open(res_file, 'w') as f:
f.write('Test set Frame Error Rate: %.2f %%' % (cv_er * 100))
with open(outmodel, 'wb') as fid:
pickle.dump(model, fid)
if __name__ == '__main__':
args = get_args()
gpu_id = get_device_id()
if gpu_id != -1:
print('%s: Using GPU device %d for nnet' % (sys.argv[0], gpu_id))
else:
print_log('Training nnet on sinlge CPU, this will take some time!')
# Activation Function
if args.activation == 'sigmoid':
activ = nn.Sigmoid()
elif args.activation == 'tanh':
activ = nn.Tanh()
elif args.activation == 'relu':
activ = nn.ReLU()
else:
sys.exit('%s: The activation function %s is invalid, exiting script!' %
(sys.argv[0], args.activation))
print_log('Defining nnet model')
with open(join(args.egs_dir, 'dim'), 'r') as fid: