def get_features_and_durs(train_lattice, sequences): num_sentences_read = 0 sentence_lines = [] log.debug('Opening train lattice: %s', train_lattice) for l in gzip.open(train_lattice): if len(l.strip()) > 0: sentence_lines.append(l) elif len(sentence_lines) > 0: try: features_and_durs = [] lat = lattice.parse_aligned_lattice(sentence_lines) log.info('Processing lattice %s', lat.name) for arc in lat.arcs: features_and_dur_seq = durmodel_utils.make_local( arc.start_frame, arc.word_id, arc.phone_ids, transitions, word_list, nonsilence_phonemes, language=args.language, stress_dict=stress_dict) if sequences: features_and_durs.extend(features_and_dur_seq) else: features_and_durs.append(features_and_dur_seq) if sequences: yield features_and_durs else: utt_full_features_and_durs = durmodel_utils.make_linear( features_and_durs, nonsilence_phonemes, utt2spkid[lat.name] if utt2spkid else 0) yield utt_full_features_and_durs except IOError as e: log.error( 'I/O error({0}): {1} -- {2} when processing lattice {3}'.format( e.errno, e.strerror, e.message, lat.name)) except ValueError as e: log.error( 'ValueError({0}): {1} -- {2} when processing lattice {3}'.format( 0, '', e.message, lat.name)) except Exception as e: log.error('Exception({0}): {1} -- {2}'.format( e.errno, e.strerror, e.message)) num_sentences_read += 1 sentence_lines = [] log.info('Read alignments for %d utterances', num_sentences_read)
def _test_linear_lattice(self): seqs = [] words = ['<eps>', 'komm', 'Tanel'] # komm seqs.append(( 1, '59_59_59_59_59_93_93_93_93_93_93_93_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80' .split('_'))) # SIL seqs.append((0, '6_6_6_6_6_6_6'.split('_'))) # Tanel seqs.append(( 2, '135_135_135_135_135_135_135_135_13_13_13_13_13_13_13_13_13_13_85_85_85_85_85_85_85_25_25_25_25_25_25_25_25_25_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68' .split('_'))) phone_map = { 6: 'SIL', 13: 'a', 25: 'e', 59: 'k', 68: 'l', 93: 'o', 80: 'mm', 85: 'n', 135: 't' } features_and_durs = [] for (word, seq) in seqs: features_and_durs.append( durmodel_utils.make_local(word, seq, phone_map, words)) # print features_and_durs full_features_and_durs = durmodel_utils.make_linear(features_and_durs) self.assertEqual(8, len(full_features_and_durs)) print(full_features_and_durs[1][0]) self.assert_( 'pos-1:dur=%f' % durmodel_utils.dur_function(5) in full_features_and_durs[1][0]) for (f, d) in full_features_and_durs: print('%d %s' % (d, ' '.join(f)))
def _test_linear_lattice(self): seqs = [] words = ["<eps>", "komm", "Tanel"] #komm seqs.append((1, "59_59_59_59_59_93_93_93_93_93_93_93_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80".split("_"))) #SIL seqs.append((0, "6_6_6_6_6_6_6".split("_"))) #Tanel seqs.append((2, "135_135_135_135_135_135_135_135_13_13_13_13_13_13_13_13_13_13_85_85_85_85_85_85_85_25_25_25_25_25_25_25_25_25_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68".split("_"))) phone_map = {6: "SIL", 13: "a", 25: "e", 59 : "k", 68: "l", 93 : "o", 80: "mm", 85 : "n", 135 : "t"} features_and_durs = [] for (word, seq) in seqs: features_and_durs.append(durmodel_utils.make_local(word, seq, phone_map, words)) #print features_and_durs full_features_and_durs = durmodel_utils.make_linear(features_and_durs) self.assertEqual(8, len(full_features_and_durs)) print full_features_and_durs[1][0] self.assert_("pos-1:dur=%f" % durmodel_utils.dur_function(5) in full_features_and_durs[1][0]) for (f, d) in full_features_and_durs: print "%d %s" % (d, " ".join(f))
def _test_linear_lattice(self): seqs = [] words = ["<eps>", "komm", "Tanel"] #komm seqs.append(( 1, "59_59_59_59_59_93_93_93_93_93_93_93_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80_80" .split("_"))) #SIL seqs.append((0, "6_6_6_6_6_6_6".split("_"))) #Tanel seqs.append(( 2, "135_135_135_135_135_135_135_135_13_13_13_13_13_13_13_13_13_13_85_85_85_85_85_85_85_25_25_25_25_25_25_25_25_25_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68_68" .split("_"))) phone_map = { 6: "SIL", 13: "a", 25: "e", 59: "k", 68: "l", 93: "o", 80: "mm", 85: "n", 135: "t" } features_and_durs = [] for (word, seq) in seqs: features_and_durs.append( durmodel_utils.make_local(word, seq, phone_map, words)) #print features_and_durs full_features_and_durs = durmodel_utils.make_linear(features_and_durs) self.assertEqual(8, len(full_features_and_durs)) print full_features_and_durs[1][0] self.assert_( "pos-1:dur=%f" % durmodel_utils.dur_function(5) in full_features_and_durs[1][0]) for (f, d) in full_features_and_durs: print "%d %s" % (d, " ".join(f))
print >> sys.stderr, "Processing alignements..." for l in gzip.open(args.train_lattice): if len(l.strip()) > 0: sentence_lines.append(l) elif len(sentence_lines) > 0: try: lat = lattice.parse_aligned_lattice(sentence_lines) #print >> sys.stderr, "Processing lattice %s" % lat.name features_and_durs = [] for arc in lat.arcs: features_and_dur_seq = durmodel_utils.make_local(arc.start_frame, arc.word_id, arc.phone_ids, transitions, word_list, nonsilence_phonemes, language=args.language, stress_dict=stress_dict) #print features_and_dur_seq features_and_durs.append(features_and_dur_seq) full_features_and_durs.extend(durmodel_utils.make_linear(features_and_durs, nonsilence_phonemes)) except IOError as e: print >> sys.stderr, "I/O error({0}): {1} -- {2} when processing lattice {3}".format(e.errno, e.strerror, e.message, lat.name) num_sentences_read += 1 sentence_lines = [] print >> sys.stderr, "Read alignments for %d utterances" % num_sentences_read feature_dict = OrderedDict() if args.read_features_filename: print ".. Reading features from %s" % args.read_features_filename feature_dict = OrderedDict() for (i, feature) in enumerate(codecs.open(args.read_features_filename, encoding="UTF-8")): feature_dict[feature.strip()] = i else: for (_features, d) in full_features_and_durs:
features_and_durs = [] for arc in lat.arcs: features_and_dur_seq = durmodel_utils.make_local( arc.start_frame, arc.word_id, arc.phone_ids, transitions, word_list, nonsilence_phonemes, language=args.language, stress_dict=stress_dict) #print features_and_dur_seq features_and_durs.append(features_and_dur_seq) full_features_and_durs.extend( durmodel_utils.make_linear( features_and_durs, nonsilence_phonemes, utt2spkid[lat.name] if utt2spkid else 0)) except IOError as e: print >> sys.stderr, "I/O error({0}): {1} -- {2} when processing lattice {3}".format( e.errno, e.strerror, e.message, lat.name) except ValueError as e: print >> sys.stderr, "ValueError({0}): {1} -- {2} when processing lattice {3}".format( 0, "", e.message, lat.name) num_sentences_read += 1 sentence_lines = [] print >> sys.stderr, "Read alignments for %d utterances" % num_sentences_read feature_dict = OrderedDict() if args.read_features_filename: print ".. Reading features from %s" % args.read_features_filename