def test_build_vocab_single_instance_has_words(self, instances): single_instance = instances["single_instance"] vocab_builder = Vocab(instances=single_instance, max_num_tokens=1000) vocab = vocab_builder.map_tokens_to_freq_idx() assert "i" in vocab.keys() assert "like" in vocab.keys() assert "nlp" in vocab.keys()
def test_vocab_always_has_special_tokens(self, instances): single_instance = instances["single_instance"] vocab_builder = Vocab(instances=single_instance, max_num_tokens=1000, min_count=1) vocab = vocab_builder.map_tokens_to_freq_idx() assert vocab_builder.unk_token in vocab.keys() assert vocab_builder.pad_token in vocab.keys() assert vocab_builder.start_token in vocab.keys() assert vocab_builder.end_token in vocab.keys()
def test_single_instance_clip_on_max_num(self, instances): single_instance = instances["single_instance"] MAX_NUM_WORDS = 1 vocab_builder = Vocab(instances=single_instance, max_num_tokens=MAX_NUM_WORDS) vocab_builder.build_vocab() vocab = vocab_builder.map_tokens_to_freq_idx() vocab = vocab_builder.clip_on_max_num(vocab) vocab_len = len(set(idx for freq, idx in vocab.values())) assert vocab_len == MAX_NUM_WORDS + len(vocab_builder.special_vocab)
def test_single_instance_min_count(self, instances): single_instance = instances["single_instance"] vocab_builder = Vocab(instances=single_instance, max_num_tokens=1000, min_count=2) vocab_builder.build_vocab() vocab = vocab_builder.map_tokens_to_freq_idx() vocab = vocab_builder.clip_on_mincount(vocab) # check that is mapped to unk nlp_freq, nlp_idx = vocab["nlp"] assert nlp_idx == vocab_builder.token2idx["<UNK>"]
def test_build_vocab_single_instance_descending_order(self, instances): single_instance = instances["single_instance"] vocab_builder = Vocab(instances=single_instance, max_num_tokens=1000, min_count=1) vocab = vocab_builder.map_tokens_to_freq_idx() i_freq, i_idx = vocab["i"] like_freq, like_idx = vocab["like"] nlp_freq, nlp_idx = vocab["nlp"] # since 'i' appears more number of times than 'like' appears more # number of times than nlp assert i_idx < like_idx < nlp_idx assert i_freq > like_freq > nlp_freq