def build_dictionary(cls,
filenames,
workers=1,
threshold=-1,
nwords=-1,
padding_factor=8):
"""Build the dictionary
Args:
filenames (list): list of filenames
workers (int): number of concurrent workers
threshold (int): defines the minimum word count
nwords (int): defines the total number of words in the final dictionary,
including special symbols
padding_factor (int): can be used to pad the dictionary size to be a
multiple of 8, which is important on some hardware (e.g., Nvidia
Tensor Cores).
"""
d = AsrDictionary()
for filename in filenames:
AsrDictionary.add_file_to_dictionary(filename, d,
tokenizer.tokenize_line,
workers)
d.finalize(threshold=threshold,
nwords=nwords,
padding_factor=padding_factor)
return d
def make_dictionary():
"""construct dictionary."""
d = AsrDictionary()
alphabet = string.ascii_lowercase
for token in alphabet:
d.add_symbol(token)
d.add_symbol("<space>")
d.finalize(padding_factor=1) # don't add extra padding symbols
d.space_index = d.indices.get("<space>", -1)
return d
def make_dictionary(vocab, non_lang_syms=[]):
"""construct dictionary."""
assert isinstance(vocab, list) and isinstance(non_lang_syms, list)
d = AsrDictionary()
for token in vocab:
d.add_symbol(token)
d.add_symbol('<space>')
for token in non_lang_syms:
d.add_symbol(token)
d.finalize(padding_factor=1) # don't add extra padding symbols
d.space_index = d.indices.get('<space>', -1)
return d
def make_dictionary(vocab, non_lang_syms=[]):
"""construct dictionary."""
assert isinstance(vocab, list) and isinstance(non_lang_syms, list)
d = AsrDictionary()
d.non_lang_syms = non_lang_syms
args = Namespace(bpe="characters_asr")
d.build_bpe(args)
for token in vocab:
d.add_symbol(token)
d.add_symbol("<space>")
for token in non_lang_syms:
d.add_symbol(token)
d.finalize(padding_factor=1) # don't add extra padding symbols
d.space_index = d.indices.get("<space>", -1)
return d
