def test_count_vocab_items_correctly_indexes_tags(self): tags = ["B", "I", "O", "O", "O"] tag_field = TagField(tags, self.text, tag_namespace="tags") counter = defaultdict(lambda: defaultdict(int)) tag_field.count_vocab_items(counter) assert counter["tags"]["B"] == 1 assert counter["tags"]["I"] == 1 assert counter["tags"]["O"] == 3 assert set(counter.keys()) == {"tags"}
def test_count_vocab_items_correctly_indexes_tags(self): text = TextField(["here", "are", "some", "words", "."], [token_indexers["single id"]("words")]) tags = ["B", "I", "O", "O", "O"] tag_field = TagField(tags, text, tag_namespace="tags") counter = defaultdict(lambda: defaultdict(int)) tag_field.count_vocab_items(counter) assert counter["tags"]["B"] == 1 assert counter["tags"]["I"] == 1 assert counter["tags"]["O"] == 3 assert set(counter.keys()) == {"tags"}