def test_tedlium(self):
transitions = durmodel_utils.read_transitions(os.path.dirname(__file__) + "/test_data/tedlium/transitions.txt")
nonsilence_phonemes = set()
for l in open(os.path.dirname(__file__) + "/test_data/tedlium/nonsilence.txt"):
nonsilence_phonemes.add(l.partition("_")[0])
word_list = []
for l in codecs.open(os.path.dirname(__file__) + "/test_data/tedlium/words.txt", encoding="UTF-8"):
word_list.append(l.split()[0])
stress_dict = durmodel_utils.load_stress_dict(os.path.dirname(__file__) + "/data/en/cmudict.0.7a.lc")
# about
word_id = 437
frames = [int(s) for s in "1794_2062_2061_2061_2300_5602_5601_5650_5662_5661_5661_4808_4807_4807_4807_4807_4832_4831_4831_4831_4860_4859_4859_22924_22923_22923_23018_23118".split("_")]
features_and_dur_seq = durmodel_utils.make_local(0, word_id, frames, transitions, word_list, nonsilence_phonemes, language="ENGLISH", stress_dict=stress_dict)
print features_and_dur_seq
self.assert_(('syllable', 1) in features_and_dur_seq[0][0])
self.assert_(('AH', 1) in features_and_dur_seq[0][0])
self.assert_(('vowel', 1) in features_and_dur_seq[0][0])
self.assert_(5, features_and_dur_seq[0][1])
self.assert_(('stress1', 1) in features_and_dur_seq[2][0])
self.assert_(('syllable', 2) in features_and_dur_seq[1][0])
self.assert_(('stop', 1) in features_and_dur_seq[-1][0])
self.assert_(5, features_and_dur_seq[0][-1])
def get_features_and_durs(train_lattice, sequences):
num_sentences_read = 0
sentence_lines = []
log.debug('Opening train lattice: %s', train_lattice)
for l in gzip.open(train_lattice):
if len(l.strip()) > 0:
sentence_lines.append(l)
elif len(sentence_lines) > 0:
try:
features_and_durs = []
lat = lattice.parse_aligned_lattice(sentence_lines)
log.info('Processing lattice %s', lat.name)
for arc in lat.arcs:
features_and_dur_seq = durmodel_utils.make_local(
arc.start_frame,
arc.word_id,
arc.phone_ids,
transitions,
word_list,
nonsilence_phonemes,
language=args.language,
stress_dict=stress_dict)
if sequences:
features_and_durs.extend(features_and_dur_seq)
else:
features_and_durs.append(features_and_dur_seq)
if sequences:
yield features_and_durs
else:
utt_full_features_and_durs = durmodel_utils.make_linear(
features_and_durs,
nonsilence_phonemes,
utt2spkid[lat.name] if utt2spkid else 0)
yield utt_full_features_and_durs
except IOError as e:
log.error(
'I/O error({0}): {1} -- {2} when processing lattice {3}'.format(
e.errno, e.strerror, e.message, lat.name))
except ValueError as e:
log.error(
'ValueError({0}): {1} -- {2} when processing lattice {3}'.format(
0, '', e.message, lat.name))
except Exception as e:
log.error('Exception({0}): {1} -- {2}'.format(
e.errno, e.strerror, e.message))
num_sentences_read += 1
sentence_lines = []
log.info('Read alignments for %d utterances', num_sentences_read)
def test_tedlium(self):
transitions = durmodel_utils.read_transitions(
os.path.dirname(__file__) + "/test_data/tedlium/transitions.txt")
nonsilence_phonemes = set()
for l in open(
os.path.dirname(__file__) +
"/test_data/tedlium/nonsilence.txt"):
nonsilence_phonemes.add(l.partition("_")[0])
word_list = []
for l in codecs.open(os.path.dirname(__file__) +
"/test_data/tedlium/words.txt",
encoding="UTF-8"):
word_list.append(l.split()[0])
stress_dict = durmodel_utils.load_stress_dict(
os.path.dirname(__file__) + "/data/en/cmudict.0.7a.lc")
# about
word_id = 437
frames = [
int(s) for s in
"1794_2062_2061_2061_2300_5602_5601_5650_5662_5661_5661_4808_4807_4807_4807_4807_4832_4831_4831_4831_4860_4859_4859_22924_22923_22923_23018_23118"
.split("_")
]
features_and_dur_seq = durmodel_utils.make_local(
0,
word_id,
frames,
transitions,
word_list,
nonsilence_phonemes,
language="ENGLISH",
stress_dict=stress_dict)
print features_and_dur_seq
self.assert_(('syllable', 1) in features_and_dur_seq[0][0])
self.assert_(('AH', 1) in features_and_dur_seq[0][0])
self.assert_(('vowel', 1) in features_and_dur_seq[0][0])
self.assert_(5, features_and_dur_seq[0][1])
self.assert_(('stress1', 1) in features_and_dur_seq[2][0])
self.assert_(('syllable', 2) in features_and_dur_seq[1][0])
self.assert_(('stop', 1) in features_and_dur_seq[-1][0])
self.assert_(5, features_and_dur_seq[0][-1])
def _test_linear_lattice(self):
seqs = []
words = ['<eps>', 'komm', 'Tanel']
# komm
seqs.append((
1,
'59_59_59_59_59_93_93_93_93_93_93_93_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80'
.split('_')))
# SIL
seqs.append((0, '6_6_6_6_6_6_6'.split('_')))
# Tanel
seqs.append((
2,
'135_135_135_135_135_135_135_135_13_13_13_13_13_13_13_13_13_13_85_85_85_85_85_85_85_25_25_25_25_25_25_25_25_25_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68'
.split('_')))
phone_map = {
6: 'SIL',
13: 'a',
25: 'e',
59: 'k',
68: 'l',
93: 'o',
80: 'mm',
85: 'n',
135: 't'
}
features_and_durs = []
for (word, seq) in seqs:
features_and_durs.append(
durmodel_utils.make_local(word, seq, phone_map, words))
# print features_and_durs
full_features_and_durs = durmodel_utils.make_linear(features_and_durs)
self.assertEqual(8, len(full_features_and_durs))
print(full_features_and_durs[1][0])
self.assert_(
'pos-1:dur=%f' %
durmodel_utils.dur_function(5) in full_features_and_durs[1][0])
for (f, d) in full_features_and_durs:
print('%d %s' % (d, ' '.join(f)))
def _test_linear_lattice(self):
seqs = []
words = ["<eps>", "komm", "Tanel"]
#komm
seqs.append((1, "59_59_59_59_59_93_93_93_93_93_93_93_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80".split("_")))
#SIL
seqs.append((0, "6_6_6_6_6_6_6".split("_")))
#Tanel
seqs.append((2, "135_135_135_135_135_135_135_135_13_13_13_13_13_13_13_13_13_13_85_85_85_85_85_85_85_25_25_25_25_25_25_25_25_25_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68".split("_")))
phone_map = {6: "SIL", 13: "a", 25: "e", 59 : "k", 68: "l", 93 : "o", 80: "mm", 85 : "n", 135 : "t"}
features_and_durs = []
for (word, seq) in seqs:
features_and_durs.append(durmodel_utils.make_local(word, seq, phone_map, words))
#print features_and_durs
full_features_and_durs = durmodel_utils.make_linear(features_and_durs)
self.assertEqual(8, len(full_features_and_durs))
print full_features_and_durs[1][0]
self.assert_("pos-1:dur=%f" % durmodel_utils.dur_function(5) in full_features_and_durs[1][0])
for (f, d) in full_features_and_durs:
print "%d %s" % (d, " ".join(f))
def _test_linear_lattice(self):
seqs = []
words = ["<eps>", "komm", "Tanel"]
#komm
seqs.append((
1,
"59_59_59_59_59_93_93_93_93_93_93_93_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80"
.split("_")))
#SIL
seqs.append((0, "6_6_6_6_6_6_6".split("_")))
#Tanel
seqs.append((
2,
"135_135_135_135_135_135_135_135_13_13_13_13_13_13_13_13_13_13_85_85_85_85_85_85_85_25_25_25_25_25_25_25_25_25_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68"
.split("_")))
phone_map = {
6: "SIL",
13: "a",
25: "e",
59: "k",
68: "l",
93: "o",
80: "mm",
85: "n",
135: "t"
}
features_and_durs = []
for (word, seq) in seqs:
features_and_durs.append(
durmodel_utils.make_local(word, seq, phone_map, words))
#print features_and_durs
full_features_and_durs = durmodel_utils.make_linear(features_and_durs)
self.assertEqual(8, len(full_features_and_durs))
print full_features_and_durs[1][0]
self.assert_(
"pos-1:dur=%f" %
durmodel_utils.dur_function(5) in full_features_and_durs[1][0])
for (f, d) in full_features_and_durs:
print "%d %s" % (d, " ".join(f))
num_sentences_read = 0
full_features_and_durs = []
sentence_lines = []
print >> sys.stderr, "Processing alignements..."
for l in gzip.open(args.train_lattice):
if len(l.strip()) > 0:
sentence_lines.append(l)
elif len(sentence_lines) > 0:
try:
lat = lattice.parse_aligned_lattice(sentence_lines)
#print >> sys.stderr, "Processing lattice %s" % lat.name
features_and_durs = []
for arc in lat.arcs:
features_and_dur_seq = durmodel_utils.make_local(arc.start_frame, arc.word_id, arc.phone_ids, transitions, word_list, nonsilence_phonemes,
language=args.language, stress_dict=stress_dict)
#print features_and_dur_seq
features_and_durs.append(features_and_dur_seq)
full_features_and_durs.extend(durmodel_utils.make_linear(features_and_durs, nonsilence_phonemes))
except IOError as e:
print >> sys.stderr, "I/O error({0}): {1} -- {2} when processing lattice {3}".format(e.errno, e.strerror, e.message, lat.name)
num_sentences_read += 1
sentence_lines = []
print >> sys.stderr, "Read alignments for %d utterances" % num_sentences_read
feature_dict = OrderedDict()
if args.read_features_filename:
print ".. Reading features from %s" % args.read_features_filename
feature_dict = OrderedDict()
for (i, feature) in enumerate(codecs.open(args.read_features_filename, encoding="UTF-8")):
feature_dict[feature.strip()] = i
model.layers[-1].logprob(
sym_y, sym_pdf_params))
lattice_lines = []
for l in sys.stdin:
l = l.strip()
if len(l) == 0:
lat = lattice.parse_aligned_lattice(lattice_lines)
print >> sys.stderr, "Processing lattice %s" % lat.name
features_and_durs = []
for arc in lat.arcs:
features_and_dur_seq = durmodel_utils.make_local(
arc.start_frame,
arc.word_id,
arc.phone_ids,
transitions,
word_list,
nonsilence_phonemes,
language=args.language,
stress_dict=stress_dict)
features_and_durs.append(features_and_dur_seq)
contexts = durmodel_utils.get_context_features_and_durs(
lat, features_and_durs)
assert len(contexts) == len(features_and_durs)
assert len(contexts) == len(lat.arcs)
i = 0
for (context,
local_feature_and_dur_seq) in zip(contexts,
features_and_durs):
#print >> sys.stderr, word_list[lat.arcs[i].word_id]
if args.skip_fillers:
