def get_features_and_durs(train_lattice, sequences):
num_sentences_read = 0
sentence_lines = []
log.debug('Opening train lattice: %s', train_lattice)
for l in gzip.open(train_lattice):
if len(l.strip()) > 0:
sentence_lines.append(l)
elif len(sentence_lines) > 0:
try:
features_and_durs = []
lat = lattice.parse_aligned_lattice(sentence_lines)
log.info('Processing lattice %s', lat.name)
for arc in lat.arcs:
features_and_dur_seq = durmodel_utils.make_local(
arc.start_frame,
arc.word_id,
arc.phone_ids,
transitions,
word_list,
nonsilence_phonemes,
language=args.language,
stress_dict=stress_dict)
if sequences:
features_and_durs.extend(features_and_dur_seq)
else:
features_and_durs.append(features_and_dur_seq)
if sequences:
yield features_and_durs
else:
utt_full_features_and_durs = durmodel_utils.make_linear(
features_and_durs,
nonsilence_phonemes,
utt2spkid[lat.name] if utt2spkid else 0)
yield utt_full_features_and_durs
except IOError as e:
log.error(
'I/O error({0}): {1} -- {2} when processing lattice {3}'.format(
e.errno, e.strerror, e.message, lat.name))
except ValueError as e:
log.error(
'ValueError({0}): {1} -- {2} when processing lattice {3}'.format(
0, '', e.message, lat.name))
except Exception as e:
log.error('Exception({0}): {1} -- {2}'.format(
e.errno, e.strerror, e.message))
num_sentences_read += 1
sentence_lines = []
log.info('Read alignments for %d utterances', num_sentences_read)
if args.stress_dict_filename:
print >> sys.stderr, "Reading stress dictionary"
stress_dict = durmodel_utils.load_stress_dict(args.stress_dict_filename)
num_sentences_read = 0
full_features_and_durs = []
sentence_lines = []
print >> sys.stderr, "Processing alignements..."
for l in gzip.open(args.train_lattice):
if len(l.strip()) > 0:
sentence_lines.append(l)
elif len(sentence_lines) > 0:
try:
lat = lattice.parse_aligned_lattice(sentence_lines)
#print >> sys.stderr, "Processing lattice %s" % lat.name
features_and_durs = []
for arc in lat.arcs:
features_and_dur_seq = durmodel_utils.make_local(arc.start_frame, arc.word_id, arc.phone_ids, transitions, word_list, nonsilence_phonemes,
language=args.language, stress_dict=stress_dict)
#print features_and_dur_seq
features_and_durs.append(features_and_dur_seq)
full_features_and_durs.extend(durmodel_utils.make_linear(features_and_durs, nonsilence_phonemes))
except IOError as e:
print >> sys.stderr, "I/O error({0}): {1} -- {2} when processing lattice {3}".format(e.errno, e.strerror, e.message, lat.name)
num_sentences_read += 1
sentence_lines = []
print >> sys.stderr, "Read alignments for %d utterances" % num_sentences_read
feature_dict = OrderedDict()
use_speaker_id = False
sym_input = theano.tensor.matrix(dtype=theano.config.floatX)
model_fprop_function = theano.function([sym_input],
model.fprop(sym_input))
sym_y = theano.tensor.matrix(dtype=theano.config.floatX)
sym_pdf_params = theano.tensor.matrix(dtype=theano.config.floatX)
model_logprob_function = theano.function([sym_y, sym_pdf_params],
model.layers[-1].logprob(
sym_y, sym_pdf_params))
lattice_lines = []
for l in sys.stdin:
l = l.strip()
if len(l) == 0:
lat = lattice.parse_aligned_lattice(lattice_lines)
print >> sys.stderr, "Processing lattice %s" % lat.name
features_and_durs = []
for arc in lat.arcs:
features_and_dur_seq = durmodel_utils.make_local(
arc.start_frame,
arc.word_id,
arc.phone_ids,
transitions,
word_list,
nonsilence_phonemes,
language=args.language,
stress_dict=stress_dict)
features_and_durs.append(features_and_dur_seq)
contexts = durmodel_utils.get_context_features_and_durs(
lat, features_and_durs)
model_fprop_function = theano.function(sym_input, sym_Y, on_unused_input='warn')
except:
use_speaker_id=False
sym_input = theano.tensor.matrix(dtype=theano.config.floatX)
model_fprop_function = theano.function([sym_input], model.fprop(sym_input))
sym_y = theano.tensor.matrix(dtype=theano.config.floatX)
sym_pdf_params = theano.tensor.matrix(dtype=theano.config.floatX)
model_logprob_function = theano.function([sym_y, sym_pdf_params], model.layers[-1].logprob(sym_y, sym_pdf_params))
lattice_lines = []
for l in sys.stdin:
l = l.strip()
if len(l) == 0:
lat = lattice.parse_aligned_lattice(lattice_lines)
print >> sys.stderr, "Processing lattice %s" % lat.name
features_and_durs = []
for arc in lat.arcs:
features_and_dur_seq = durmodel_utils.make_local(arc.start_frame, arc.word_id, arc.phone_ids, transitions, word_list,
nonsilence_phonemes, language=args.language, stress_dict=stress_dict)
features_and_durs.append(features_and_dur_seq)
contexts = durmodel_utils.get_context_features_and_durs(lat, features_and_durs)
assert len(contexts) == len(features_and_durs)
assert len(contexts) == len(lat.arcs)
i = 0
for (context, local_feature_and_dur_seq) in zip(contexts, features_and_durs):
#print >> sys.stderr, word_list[lat.arcs[i].word_id]
if args.skip_fillers:
if word_list[lat.arcs[i].word_id] in filler_words:
i += 1
utt2spkid[ss[0]] = speaker_ids.setdefault(ss[1], len(speaker_ids))
num_sentences_read = 0
sentence_lines = []
print >> sys.stderr, "Processing alignements..."
if args.sequences:
all_features_and_durs = []
for l in gzip.open(args.train_lattice):
if len(l.strip()) > 0:
sentence_lines.append(l)
elif len(sentence_lines) > 0:
try:
lat = lattice.parse_aligned_lattice(sentence_lines)
#print >> sys.stderr, "Processing lattice %s" % lat.name
features_and_durs = []
for arc in lat.arcs:
features_and_dur_seq = durmodel_utils.make_local(
arc.start_frame,
arc.word_id,
arc.phone_ids,
transitions,
word_list,
nonsilence_phonemes,
language=args.language,
stress_dict=stress_dict)
#print features_and_dur_seq
features_and_durs.extend(features_and_dur_seq)
all_features_and_durs.append(features_and_durs)