def read_vocab(self):
# don't try reading vocabulary for encoders that take pre-computed features
self.vocabs = [
None if binary else utils.initialize_vocabulary(vocab_path)
for vocab_path, binary in zip(self.filenames.vocab, self.binary)
]
self.src_vocab, self.trg_vocab = self.vocabs[:len(self.src_ext)], self.vocabs[len(self.src_ext):]
def read_vocab(self):
# don't try reading vocabulary for encoders that take pre-computed features
# [({key:val, key:val,....}, [word, word, ...., word]), (), (), () ,....., ()]
self.vocabs = [
None if binary else utils.initialize_vocabulary(vocab_path)
for vocab_path, binary in zip(self.filenames.vocab, self.binary)
]
# trg_vocab: [(name1: {key:val, key:val,....}, name2:[word, word, ...., word])]
self.src_vocab, self.trg_vocab = self.vocabs[:len(self.src_ext)], self.vocabs[len(self.src_ext):]
def _read_vocab(self):
# don't try reading vocabulary for encoders that take pre-computed features
self.vocabs = [
utils.initialize_vocabulary(vocab_path) if not binary else None
for ext, vocab_path, binary in zip(
self.extensions, self.filenames.vocab, self.binary_input)
]
self.src_vocab = self.vocabs[:-1]
self.trg_vocab = self.vocabs[-1]
self.ngrams = self.filenames.lm_path and utils.read_ngrams(
self.filenames.lm_path, self.trg_vocab.vocab)