def test_idx2token_cries_for_vocab(self, instances): single_instance = instances["single_instance"] MAX_NUM_WORDS = 100 vocab_builder = Vocab(instances=single_instance, max_num_tokens=MAX_NUM_WORDS) with pytest.raises(ValueError): vocab_builder.get_idx_from_token(1)
def test_add_tokens(self, instances, tmpdir): instance_dict = instances single_instance = instance_dict["single_instance"] MAX_NUM_WORDS = 100 vocab_file = tmpdir.mkdir("tempdir").join("vocab.json") vocab = Vocab( instances=single_instance, max_num_tokens=MAX_NUM_WORDS, store_location=vocab_file, ) vocab.build_vocab() vocab.add_tokens(["very", "much"]) assert "very" in vocab.vocab.keys() assert "much" in vocab.vocab.keys() assert vocab.vocab["very"] == (1, 7) assert vocab.vocab["much"] == (1, 8) assert vocab.get_token_from_idx(7) == "very" assert vocab.get_token_from_idx(8) == "much" assert vocab.get_idx_from_token("very") == 7 assert vocab.get_idx_from_token("much") == 8