def run(self):
# with open(self.name + '/outtest', 'w') as o:
# print >> o, self.configuration
work_dir = self.name
xforms_dir = work_dir + '/xforms'
classes_dir = work_dir + '/classes'
files_dir = work_dir + '/files'
model = self.model.configuration['model_dir'] + '/' + self.configuration['model_name']
recog_scp = work_dir +"/recog.scp"
rscp = self.model.configuration['recognize_scp']
if not self.configuration['recognize_scp'].startswith('_'):
rscp = self.configuration['recognize_scp']
data_manipulation.copy_scp_file(rscp, recog_scp)
#shutil.copyfile(self.model.configuration['recognize_scp'], recog_scp)
#dict_hvite = self.model.configuration['dict_hvite']
dict_hdecode = self.model.configuration['dict_hdecode']
tiedlist = self.model.configuration['tiedlist']
lm = self.model.configuration['lm']
lm_rescore = self.model.configuration['lm_rescore']
lm_scale = self.configuration['lm_scale']
beam = self.configuration['beam']
end_beam = self.configuration['end_beam']
max_pruning = self.configuration['max_pruning']
num_tokens = self.configuration['num_tokens']
hdecode_mlf = work_dir + '/hdecode.mlf'
rescore_mlf = work_dir + '/rescore.mlf'
recog_trn = work_dir + '/recog.trn'
configs = [self.model.configuration['config']]
speaker_name_width = self.model.configuration['speaker_name_width']
# adaptation_dir=work_dir + '/xforms'
try:
#if os.path.exists(work_dir): shutil.rmtree(work_dir)
htk_lat_dir = os.path.join(work_dir, 'lattices.htk')
rescore_lat_dir = os.path.join(work_dir, 'lattices.rescore')
log_dir = os.path.join(work_dir, 'log')
for dir in [work_dir, htk_lat_dir, rescore_lat_dir, log_dir, xforms_dir, classes_dir, files_dir]:
if not os.path.exists(dir):
os.mkdir(dir)
current_parent_transform = None
for i, adaptation in enumerate(self.adaptations):
current_parent_transform = adaptation.make_adaptation(i,current_parent_transform,self)
adap_dirs = None
speaker_name_width = self.model.configuration['speaker_name_width']
if current_parent_transform is not None:
configs.append(current_parent_transform[1])
extension = current_parent_transform[0]
adap_dirs = [(xforms_dir, extension),(classes_dir, None)]
speaker_name_width = current_parent_transform[2]
print "Start step: %d (%s)" % (0, 'Generating lattices with HDecode')
htk.HDecode(log_dir, recog_scp, model, dict_hdecode, tiedlist, lm, htk_lat_dir, num_tokens,
hdecode_mlf, configs, lm_scale, beam, end_beam, max_pruning, adap_dirs, speaker_name_width)
if lm_rescore is not None:
print "Start step: %d (%s)" % (0, 'Rescoring lattices with lattice-tool')
htk.lattice_rescore(log_dir, htk_lat_dir, rescore_lat_dir, lm_rescore + '.gz', lm_scale)
print "Start step: %d (%s)" % (0, 'Decoding lattices with lattice-tool')
htk.lattice_decode(log_dir, rescore_lat_dir, rescore_mlf, lm_scale)
data_manipulation.mlf_to_trn(rescore_mlf, recog_trn, self.model.configuration['speaker_name_width'])
else:
with open(rescore_mlf, 'w') as out_file:
prev = ""
for line in open(hdecode_mlf):
if line.rstrip() == '.' and prev != '</s>':
print >> out_file, "</s>"
print >> out_file, line.rstrip().replace(htk_lat_dir, '*')
prev = line.rstrip()
#shutil.copyfile(hdecode_mlf, rescore_mlf)
data_manipulation.mlf_to_trn(hdecode_mlf, recog_trn, self.model.configuration['speaker_name_width'])
self.done = True
return
except Experiment:
self.done = False
pass
def run(self):
# with open(self.name + '/outtest', 'w') as o:
# print >> o, self.configuration
work_dir = self.name
xforms_dir = work_dir + '/xforms'
classes_dir = work_dir + '/classes'
files_dir = work_dir + '/files'
model = self.model.configuration[
'model_dir'] + '/' + self.configuration['model_name']
recog_scp = work_dir + "/recog.scp"
rscp = self.model.configuration['recognize_scp']
if not self.configuration['recognize_scp'].startswith('_'):
rscp = self.configuration['recognize_scp']
data_manipulation.copy_scp_file(rscp, recog_scp)
#shutil.copyfile(self.model.configuration['recognize_scp'], recog_scp)
#dict_hvite = self.model.configuration['dict_hvite']
dict_hdecode = self.model.configuration['dict_hdecode']
tiedlist = self.model.configuration['tiedlist']
lm = self.model.configuration['lm']
lm_rescore = self.model.configuration['lm_rescore']
lm_scale = self.configuration['lm_scale']
beam = self.configuration['beam']
end_beam = self.configuration['end_beam']
max_pruning = self.configuration['max_pruning']
num_tokens = self.configuration['num_tokens']
hdecode_mlf = work_dir + '/hdecode.mlf'
rescore_mlf = work_dir + '/rescore.mlf'
recog_trn = work_dir + '/recog.trn'
configs = [self.model.configuration['config']]
speaker_name_width = self.model.configuration['speaker_name_width']
# adaptation_dir=work_dir + '/xforms'
try:
#if os.path.exists(work_dir): shutil.rmtree(work_dir)
htk_lat_dir = os.path.join(work_dir, 'lattices.htk')
rescore_lat_dir = os.path.join(work_dir, 'lattices.rescore')
log_dir = os.path.join(work_dir, 'log')
for dir in [
work_dir, htk_lat_dir, rescore_lat_dir, log_dir,
xforms_dir, classes_dir, files_dir
]:
if not os.path.exists(dir):
os.mkdir(dir)
current_parent_transform = None
for i, adaptation in enumerate(self.adaptations):
current_parent_transform = adaptation.make_adaptation(
i, current_parent_transform, self)
adap_dirs = None
speaker_name_width = self.model.configuration['speaker_name_width']
if current_parent_transform is not None:
configs.append(current_parent_transform[1])
extension = current_parent_transform[0]
adap_dirs = [(xforms_dir, extension), (classes_dir, None)]
speaker_name_width = current_parent_transform[2]
print "Start step: %d (%s)" % (0,
'Generating lattices with HDecode')
htk.HDecode(log_dir, recog_scp, model, dict_hdecode, tiedlist, lm,
htk_lat_dir, num_tokens, hdecode_mlf, configs,
lm_scale, beam, end_beam, max_pruning, adap_dirs,
speaker_name_width)
if lm_rescore is not None:
print "Start step: %d (%s)" % (
0, 'Rescoring lattices with lattice-tool')
htk.lattice_rescore(log_dir, htk_lat_dir, rescore_lat_dir,
lm_rescore + '.gz', lm_scale)
print "Start step: %d (%s)" % (
0, 'Decoding lattices with lattice-tool')
htk.lattice_decode(log_dir, rescore_lat_dir, rescore_mlf,
lm_scale)
data_manipulation.mlf_to_trn(
rescore_mlf, recog_trn,
self.model.configuration['speaker_name_width'])
else:
with open(rescore_mlf, 'w') as out_file:
prev = ""
for line in open(hdecode_mlf):
if line.rstrip() == '.' and prev != '</s>':
print >> out_file, "</s>"
print >> out_file, line.rstrip().replace(
htk_lat_dir, '*')
prev = line.rstrip()
#shutil.copyfile(hdecode_mlf, rescore_mlf)
data_manipulation.mlf_to_trn(
hdecode_mlf, recog_trn,
self.model.configuration['speaker_name_width'])
self.done = True
return
except Experiment:
self.done = False
pass
def make_adaptation(self, id, parent_transform, experiment):
work_dir = experiment.name
classes_dir = work_dir + '/classes'
xforms_dir = work_dir + '/xforms'
files_dir = work_dir + '/files'
log_dir = os.path.join(work_dir, 'log')
target_extension = 'mllr%d' % id
source_extension = None
if parent_transform is not None:
source_extension = parent_transform[0]
scp_file = self.configuration['scp']
if scp_file is None or len(scp_file) == 0:
scp_file = work_dir + '/recog.scp'
adap_scp = work_dir +"/adap%d.scp" %id
data_manipulation.copy_scp_file(scp_file, adap_scp)
# shutil.copyfile(scp_file, adap_scp)
model_hvite = experiment.model.configuration['model_dir'] + '/' + self.configuration['model_hvite']
model_adapt = experiment.model.configuration['model_dir'] + '/' + self.configuration['model_adapt']
phones_list = experiment.model.configuration['model_dir'] + '/files/tiedlist'
dict_hvite = experiment.model.configuration['dict_hvite']
source_mlf = self.configuration['mlf']
source_mlf_ext = self.configuration['mlf_ext']
adapt_mlf = work_dir + '/adap%d.mlf'%id
num_nodes = self.configuration['nodes']
hvite_config = experiment.model.configuration['model_dir'] + '/config/config'
standard_config = experiment.model.configuration['config']
adapt_config = files_dir + '/adapt%d.config' % id
speaker_name_width = self.configuration['speaker_name_width']
num_adaptation_samples = None
if self.configuration['num_adaptation_samples'] >= 0:
num_adaptation_samples = self.configuration['num_adaptation_samples']
# align transcription
htk.HVite(log_dir, adap_scp, model_hvite, dict_hvite, phones_list, source_mlf, adapt_mlf, source_mlf_ext, hvite_config)
if self.configuration['type'] == 'tree':
regtree_hed = files_dir + '/regtree%d.hed'%id
regtree = classes_dir+'/regtree%d.tree'%id
data_manipulation.write_regtree_hed_file(regtree_hed, model_adapt, num_nodes, 'regtree%d'%id)
data_manipulation.write_tree_cmlllr_config(adapt_config, regtree, '"IntVec 3 13 13 13"')
htk.HHEd(log_dir, model_adapt, classes_dir, regtree_hed, phones_list, '/dev/null')
htk.HERest_estimate_transform(log_dir, adap_scp, model_adapt, xforms_dir, phones_list, adapt_mlf,
num_adaptation_samples,
[hvite_config, adapt_config], speaker_name_width, target_extension,
[(xforms_dir, source_extension), (classes_dir, None)], False,
[(xforms_dir, source_extension), (classes_dir, None)])
elif self.configuration['type'] == 'base':
global_f = classes_dir + '/global%d' % id
mask = None
if parent_transform is not None and parent_transform[2] != speaker_name_width:
mask = '%%%'
if parent_transform[2] > 0:
mask = ('%' * parent_transform[2]) + "*.*"
data_manipulation.write_base_cmllr_config(adapt_config, global_f, mask)
data_manipulation.write_global(global_f, 'global%d'%id)
input_transforms = [(classes_dir, None)]
parent_transforms = []
if parent_transform is not None:
input_transforms.append((xforms_dir, source_extension))
parent_transforms.append((xforms_dir, source_extension))
htk.HERest_estimate_transform(log_dir, adap_scp, model_adapt, xforms_dir, phones_list, adapt_mlf,
num_adaptation_samples,
[hvite_config, adapt_config], speaker_name_width, target_extension,
input_transforms, False, parent_transforms)
else:
raise Exception("Unknown adaptation type!")
return (target_extension, adapt_config, speaker_name_width)
def make_adaptation(self, id, parent_transform, experiment):
work_dir = experiment.name
classes_dir = work_dir + '/classes'
xforms_dir = work_dir + '/xforms'
files_dir = work_dir + '/files'
log_dir = os.path.join(work_dir, 'log')
target_extension = 'mllr%d' % id
source_extension = None
if parent_transform is not None:
source_extension = parent_transform[0]
scp_file = self.configuration['scp']
if scp_file is None or len(scp_file) == 0:
scp_file = work_dir + '/recog.scp'
adap_scp = work_dir + "/adap%d.scp" % id
data_manipulation.copy_scp_file(scp_file, adap_scp)
# shutil.copyfile(scp_file, adap_scp)
model_hvite = experiment.model.configuration[
'model_dir'] + '/' + self.configuration['model_hvite']
model_adapt = experiment.model.configuration[
'model_dir'] + '/' + self.configuration['model_adapt']
phones_list = experiment.model.configuration[
'model_dir'] + '/files/tiedlist'
dict_hvite = experiment.model.configuration['dict_hvite']
source_mlf = self.configuration['mlf']
source_mlf_ext = self.configuration['mlf_ext']
adapt_mlf = work_dir + '/adap%d.mlf' % id
num_nodes = self.configuration['nodes']
hvite_config = experiment.model.configuration[
'model_dir'] + '/config/config'
standard_config = experiment.model.configuration['config']
adapt_config = files_dir + '/adapt%d.config' % id
speaker_name_width = self.configuration['speaker_name_width']
num_adaptation_samples = None
if self.configuration['num_adaptation_samples'] >= 0:
num_adaptation_samples = self.configuration[
'num_adaptation_samples']
# align transcription
htk.HVite(log_dir, adap_scp, model_hvite, dict_hvite, phones_list,
source_mlf, adapt_mlf, source_mlf_ext, hvite_config)
if self.configuration['type'] == 'tree':
regtree_hed = files_dir + '/regtree%d.hed' % id
regtree = classes_dir + '/regtree%d.tree' % id
data_manipulation.write_regtree_hed_file(regtree_hed, model_adapt,
num_nodes,
'regtree%d' % id)
data_manipulation.write_tree_cmlllr_config(adapt_config, regtree,
'"IntVec 3 13 13 13"')
htk.HHEd(log_dir, model_adapt, classes_dir, regtree_hed,
phones_list, '/dev/null')
htk.HERest_estimate_transform(log_dir, adap_scp, model_adapt,
xforms_dir, phones_list, adapt_mlf,
num_adaptation_samples,
[hvite_config, adapt_config],
speaker_name_width, target_extension,
[(xforms_dir, source_extension),
(classes_dir, None)], False,
[(xforms_dir, source_extension),
(classes_dir, None)])
elif self.configuration['type'] == 'base':
global_f = classes_dir + '/global%d' % id
mask = None
if parent_transform is not None and parent_transform[
2] != speaker_name_width:
mask = '%%%'
if parent_transform[2] > 0:
mask = ('%' * parent_transform[2]) + "*.*"
data_manipulation.write_base_cmllr_config(adapt_config, global_f,
mask)
data_manipulation.write_global(global_f, 'global%d' % id)
input_transforms = [(classes_dir, None)]
parent_transforms = []
if parent_transform is not None:
input_transforms.append((xforms_dir, source_extension))
parent_transforms.append((xforms_dir, source_extension))
htk.HERest_estimate_transform(
log_dir, adap_scp, model_adapt, xforms_dir, phones_list,
adapt_mlf, num_adaptation_samples,
[hvite_config, adapt_config], speaker_name_width,
target_extension, input_transforms, False, parent_transforms)
else:
raise Exception("Unknown adaptation type!")
return (target_extension, adapt_config, speaker_name_width)