def test_idx2token_out_of_bounds(self, instances): single_instance = instances["single_instance"] MAX_NUM_WORDS = 100 vocab_builder = Vocab(instances=single_instance, max_num_tokens=MAX_NUM_WORDS) vocab_builder.build_vocab() print(vocab_builder.get_idx2token_mapping()) with pytest.raises(ValueError): vocab_builder.get_token_from_idx(100)
def test_idx2token_for_unk(self, instances): """" Many words map to UNK in the vocab. For example say the index for UNK is 3. Then mapping 3 to the token should always map to UNK and not any other word """ single_instance = instances["single_instance"] MAX_NUM_WORDS = 100 vocab_builder = Vocab( instances=single_instance, max_num_tokens=MAX_NUM_WORDS, start_token="<SOS>", end_token="<EOS>", pad_token="<PAD>", unk_token="<UNK>", ) vocab_builder.build_vocab() UNK_IDX = vocab_builder.special_vocab[vocab_builder.unk_token][1] assert vocab_builder.get_token_from_idx(UNK_IDX) == "<UNK>"
def test_add_tokens(self, instances, tmpdir): instance_dict = instances single_instance = instance_dict["single_instance"] MAX_NUM_WORDS = 100 vocab_file = tmpdir.mkdir("tempdir").join("vocab.json") vocab = Vocab( instances=single_instance, max_num_tokens=MAX_NUM_WORDS, store_location=vocab_file, ) vocab.build_vocab() vocab.add_tokens(["very", "much"]) assert "very" in vocab.vocab.keys() assert "much" in vocab.vocab.keys() assert vocab.vocab["very"] == (1, 7) assert vocab.vocab["much"] == (1, 8) assert vocab.get_token_from_idx(7) == "very" assert vocab.get_token_from_idx(8) == "much" assert vocab.get_idx_from_token("very") == 7 assert vocab.get_idx_from_token("much") == 8