def build_dictionary(cls, filenames, tokenize_func=None, workers=1, threshold=-1, nwords=-1, padding_factor=8): """Build the dictionary Args: filenames (list): list of filenames workers (int): number of concurrent workers threshold (int): defines the minimum word count nwords (int): defines the total number of words in the final dictionary, including special symbols padding_factor (int): can be used to pad the dictionary size to be a multiple of 8, which is important on some hardware (e.g., Nvidia Tensor Cores). """ d = Dictionary() for filename in filenames: Dictionary.add_file_to_dictionary(filename, d, tokenize_func, num_workers=workers) d.finalize(threshold=threshold, nwords=nwords, padding_factor=padding_factor) return d
def build_dictionary( cls, filenames, workers=1, threshold=-1, nwords=-1, padding_factor=8, tokenize_func=SPACE_SPLITTER, **kwargs, ): """Build the dictionary Args: filenames (list): list of filenames workers (int): number of concurrent workers threshold (int): defines the minimum word count nwords (int): defines the total number of words in the final dictionary, including special symbols padding_factor (int): can be used to pad the dictionary size to be a multiple of 8, which is important on some hardware (e.g., Nvidia Tensor Cores). """ d = Dictionary( pad=kwargs.get('pad', constants.PAD), bos=kwargs.get('bos', constants.BOS), eos=kwargs.get('eos', constants.EOS), unk=kwargs.get('unk', constants.UNK), extra_special_symbols=kwargs.get('extra_special_symbols', None), ) for filename in filenames: Dictionary.add_file_to_dictionary( filename, d, tokenize_func, d.eos_word, workers ) d.finalize(threshold=threshold, nwords=nwords, padding_factor=padding_factor) return d