def test_count_vocab_items_correctly_indexes_tags(self):
tags = ["B", "I", "O", "O", "O"]
tag_field = TagField(tags, self.text, tag_namespace="tags")
counter = defaultdict(lambda: defaultdict(int))
tag_field.count_vocab_items(counter)
assert counter["tags"]["B"] == 1
assert counter["tags"]["I"] == 1
assert counter["tags"]["O"] == 3
assert set(counter.keys()) == {"tags"}
def test_count_vocab_items_correctly_indexes_tags(self):
text = TextField(["here", "are", "some", "words", "."],
[token_indexers["single id"]("words")])
tags = ["B", "I", "O", "O", "O"]
tag_field = TagField(tags, text, tag_namespace="tags")
counter = defaultdict(lambda: defaultdict(int))
tag_field.count_vocab_items(counter)
assert counter["tags"]["B"] == 1
assert counter["tags"]["I"] == 1
assert counter["tags"]["O"] == 3
assert set(counter.keys()) == {"tags"}
