def test_get_topn(self, instances): single_instance = instances["single_instance"] MAX_NUM_WORDS = 100 vocab_builder = Vocab(instances=single_instance, max_num_tokens=MAX_NUM_WORDS) vocab_builder.build_vocab() words_freqs = vocab_builder.get_topn_frequent_words(n=1) assert words_freqs[0][0] == "i" assert words_freqs[0][1] == 3
def test_get_topn(self, instances, include_special_vocab): single_instance = instances["single_instance"] MAX_NUM_WORDS = None vocab_builder = Vocab( instances=single_instance, max_num_tokens=MAX_NUM_WORDS, include_special_vocab=include_special_vocab, ) vocab_builder.build_vocab() words_freqs = vocab_builder.get_topn_frequent_words(n=1) assert words_freqs[0][0] == "i" assert words_freqs[0][1] == 3