def test_unknown_token(self):
# pylint: disable=protected-access
# We're putting this behavior in a test so that the behavior is documented. There is
# solver code that depends in a small way on how we treat the unknown token, so any
# breaking change to this behavior should break a test, so you know you've done something
# that needs more consideration.
vocab = Vocabulary()
oov_token = vocab._oov_token
oov_index = vocab.get_token_index(oov_token)
assert oov_index == 1
assert vocab.get_token_index("unseen word") == oov_index
def test_set_from_file_reads_padded_files(self):
# pylint: disable=protected-access
vocab_filename = self.TEST_DIR / 'vocab_file'
with codecs.open(vocab_filename, 'w', 'utf-8') as vocab_file:
vocab_file.write('<S>\n')
vocab_file.write('</S>\n')
vocab_file.write('<UNK>\n')
vocab_file.write('a\n')
vocab_file.write('tricky\x0bchar\n')
vocab_file.write('word\n')
vocab_file.write('another\n')
vocab = Vocabulary()
vocab.set_from_file(vocab_filename, is_padded=True, oov_token="<UNK>")
assert vocab._oov_token == DEFAULT_OOV_TOKEN
assert vocab.get_token_index("random string") == 3
assert vocab.get_token_index("<S>") == 1
assert vocab.get_token_index("</S>") == 2
assert vocab.get_token_index(DEFAULT_OOV_TOKEN) == 3
assert vocab.get_token_index("a") == 4
assert vocab.get_token_index("tricky\x0bchar") == 5
assert vocab.get_token_index("word") == 6
assert vocab.get_token_index("another") == 7
assert vocab.get_token_from_index(0) == vocab._padding_token
assert vocab.get_token_from_index(1) == "<S>"
assert vocab.get_token_from_index(2) == "</S>"
assert vocab.get_token_from_index(3) == DEFAULT_OOV_TOKEN
assert vocab.get_token_from_index(4) == "a"
assert vocab.get_token_from_index(5) == "tricky\x0bchar"
assert vocab.get_token_from_index(6) == "word"
assert vocab.get_token_from_index(7) == "another"
def test_add_word_to_index_gives_consistent_results(self):
vocab = Vocabulary()
initial_vocab_size = vocab.get_vocab_size()
word_index = vocab.add_token_to_namespace("word")
assert "word" in vocab.get_index_to_token_vocabulary().values()
assert vocab.get_token_index("word") == word_index
assert vocab.get_token_from_index(word_index) == "word"
assert vocab.get_vocab_size() == initial_vocab_size + 1
# Now add it again, and make sure nothing changes.
vocab.add_token_to_namespace("word")
assert "word" in vocab.get_index_to_token_vocabulary().values()
assert vocab.get_token_index("word") == word_index
assert vocab.get_token_from_index(word_index) == "word"
assert vocab.get_vocab_size() == initial_vocab_size + 1
def index(self, vocab: Vocabulary):
if self._indexed_labels is None:
self._indexed_labels = [
vocab.get_token_index(label,
self._label_namespace) # type: ignore
for label in self.labels
]
def tokens_to_indices(self, tokens: List[Token], vocabulary: Vocabulary,
index_name: str) -> Dict[str, List[int]]:
dep_labels = [token.dep_ or 'NONE' for token in tokens]
return {
index_name: [
vocabulary.get_token_index(dep_label, self.namespace)
for dep_label in dep_labels
]
}
def test_namespaces(self):
vocab = Vocabulary()
initial_vocab_size = vocab.get_vocab_size()
word_index = vocab.add_token_to_namespace("word", namespace='1')
assert "word" in vocab.get_index_to_token_vocabulary(namespace='1').values()
assert vocab.get_token_index("word", namespace='1') == word_index
assert vocab.get_token_from_index(word_index, namespace='1') == "word"
assert vocab.get_vocab_size(namespace='1') == initial_vocab_size + 1
# Now add it again, in a different namespace and a different word, and make sure it's like
# new.
word2_index = vocab.add_token_to_namespace("word2", namespace='2')
word_index = vocab.add_token_to_namespace("word", namespace='2')
assert "word" in vocab.get_index_to_token_vocabulary(namespace='2').values()
assert "word2" in vocab.get_index_to_token_vocabulary(namespace='2').values()
assert vocab.get_token_index("word", namespace='2') == word_index
assert vocab.get_token_index("word2", namespace='2') == word2_index
assert vocab.get_token_from_index(word_index, namespace='2') == "word"
assert vocab.get_token_from_index(word2_index, namespace='2') == "word2"
assert vocab.get_vocab_size(namespace='2') == initial_vocab_size + 2
def tokens_to_indices(self, tokens: List[Token], vocabulary: Vocabulary,
index_name: str) -> Dict[str, List[int]]:
tags = [
'NONE' if token.ent_type_ is None else token.ent_type_
for token in tokens
]
return {
index_name:
[vocabulary.get_token_index(tag, self._namespace) for tag in tags]
}
def _get_vocab_index_mapping(self, archived_vocab: Vocabulary) -> List[Tuple[int, int]]:
vocab_index_mapping: List[Tuple[int, int]] = []
for index in range(self.vocab.get_vocab_size(namespace='tokens')):
token = self.vocab.get_token_from_index(index=index, namespace='tokens')
archived_token_index = archived_vocab.get_token_index(token, namespace='tokens')
# Checking if we got the UNK token index, because we don't want all new token
# representations initialized to UNK token's representation. We do that by checking if
# the two tokens are the same. They will not be if the token at the archived index is
# UNK.
if archived_vocab.get_token_from_index(archived_token_index, namespace="tokens") == token:
vocab_index_mapping.append((index, archived_token_index))
return vocab_index_mapping
def test_set_from_file_reads_non_padded_files(self):
# pylint: disable=protected-access
vocab_filename = self.TEST_DIR / 'vocab_file'
with codecs.open(vocab_filename, 'w', 'utf-8') as vocab_file:
vocab_file.write('B-PERS\n')
vocab_file.write('I-PERS\n')
vocab_file.write('O\n')
vocab_file.write('B-ORG\n')
vocab_file.write('I-ORG\n')
vocab = Vocabulary()
vocab.set_from_file(vocab_filename, is_padded=False, namespace='tags')
assert vocab.get_token_index("B-PERS", namespace='tags') == 0
assert vocab.get_token_index("I-PERS", namespace='tags') == 1
assert vocab.get_token_index("O", namespace='tags') == 2
assert vocab.get_token_index("B-ORG", namespace='tags') == 3
assert vocab.get_token_index("I-ORG", namespace='tags') == 4
assert vocab.get_token_from_index(0, namespace='tags') == "B-PERS"
assert vocab.get_token_from_index(1, namespace='tags') == "I-PERS"
assert vocab.get_token_from_index(2, namespace='tags') == "O"
assert vocab.get_token_from_index(3, namespace='tags') == "B-ORG"
assert vocab.get_token_from_index(4, namespace='tags') == "I-ORG"
def tokens_to_indices(self,
tokens: List[Token],
vocabulary: Vocabulary,
index_name: str) -> Dict[str, List[int]]:
indices: List[int] = []
for token in tokens:
if getattr(token, 'text_id', None) is not None:
# `text_id` being set on the token means that we aren't using the vocab, we just use
# this id instead.
indices.append(token.text_id)
else:
text = token.text
if self.lowercase_tokens:
text = text.lower()
indices.append(vocabulary.get_token_index(text, self.namespace))
return {index_name: indices}
def tokens_to_indices(self, tokens: List[Token], vocabulary: Vocabulary,
index_name: str) -> Dict[str, List[List[int]]]:
indices: List[List[int]] = []
for token in tokens:
token_indices: List[int] = []
if token.text is None:
raise ConfigurationError(
'TokenCharactersIndexer needs a tokenizer that retains text'
)
for character in self._character_tokenizer.tokenize(token.text):
if getattr(character, 'text_id', None) is not None:
# `text_id` being set on the token means that we aren't using the vocab, we just
# use this id instead.
index = character.text_id
else:
index = vocabulary.get_token_index(character.text,
self._namespace)
token_indices.append(index)
indices.append(token_indices)
return {index_name: indices}
def index(self, vocab: Vocabulary):
if self.is_global_rule and self._rule_id is None:
self._rule_id = vocab.get_token_index(self.rule,
self._vocab_namespace)
def index(self, vocab: Vocabulary):
if self._label_id is None:
self._label_id = vocab.get_token_index(self.label, self._label_namespace) # type: ignore
def test_from_params_valid_vocab_extension_thoroughly(self):
'''
Tests for Valid Vocab Extension thoroughly: Vocab extension is valid
when overlapping namespaces have same padding behaviour (padded/non-padded)
Summary of namespace paddings in this test:
original_vocab namespaces
tokens0 padded
tokens1 non-padded
tokens2 padded
tokens3 non-padded
instances namespaces
tokens0 padded
tokens1 non-padded
tokens4 padded
tokens5 non-padded
TypicalExtention example: (of tokens1 namespace)
-> original_vocab index2token
apple #0->apple
bat #1->bat
cat #2->cat
-> Token to be extended with: cat, an, apple, banana, atom, bat
-> extended_vocab: index2token
apple #0->apple
bat #1->bat
cat #2->cat
an #3->an
atom #4->atom
banana #5->banana
'''
vocab_dir = self.TEST_DIR / 'vocab_save'
original_vocab = Vocabulary(non_padded_namespaces=["tokens1", "tokens3"])
original_vocab.add_token_to_namespace("apple", namespace="tokens0") # index:2
original_vocab.add_token_to_namespace("bat", namespace="tokens0") # index:3
original_vocab.add_token_to_namespace("cat", namespace="tokens0") # index:4
original_vocab.add_token_to_namespace("apple", namespace="tokens1") # index:0
original_vocab.add_token_to_namespace("bat", namespace="tokens1") # index:1
original_vocab.add_token_to_namespace("cat", namespace="tokens1") # index:2
original_vocab.add_token_to_namespace("a", namespace="tokens2") # index:0
original_vocab.add_token_to_namespace("b", namespace="tokens2") # index:1
original_vocab.add_token_to_namespace("c", namespace="tokens2") # index:2
original_vocab.add_token_to_namespace("p", namespace="tokens3") # index:0
original_vocab.add_token_to_namespace("q", namespace="tokens3") # index:1
original_vocab.save_to_files(vocab_dir)
text_field0 = TextField([Token(t) for t in ["cat", "an", "apple", "banana", "atom", "bat"]],
{"tokens0": SingleIdTokenIndexer("tokens0")})
text_field1 = TextField([Token(t) for t in ["cat", "an", "apple", "banana", "atom", "bat"]],
{"tokens1": SingleIdTokenIndexer("tokens1")})
text_field4 = TextField([Token(t) for t in ["l", "m", "n", "o"]],
{"tokens4": SingleIdTokenIndexer("tokens4")})
text_field5 = TextField([Token(t) for t in ["x", "y", "z"]],
{"tokens5": SingleIdTokenIndexer("tokens5")})
instances = Batch([Instance({"text0": text_field0, "text1": text_field1,
"text4": text_field4, "text5": text_field5})])
params = Params({"directory_path": vocab_dir,
"extend": True,
"non_padded_namespaces": ["tokens1", "tokens5"]})
extended_vocab = Vocabulary.from_params(params, instances)
# namespaces: tokens0, tokens1 is common.
# tokens2, tokens3 only vocab has. tokens4, tokens5 only instances
extended_namespaces = {*extended_vocab._token_to_index}
assert extended_namespaces == {"tokens{}".format(i) for i in range(6)}
# # Check that _non_padded_namespaces list is consistent after extension
assert extended_vocab._non_padded_namespaces == {"tokens1", "tokens3", "tokens5"}
# # original_vocab["tokens1"] has 3 tokens, instances of "tokens1" ns has 5 tokens. 2 overlapping
assert extended_vocab.get_vocab_size("tokens1") == 6
assert extended_vocab.get_vocab_size("tokens0") == 8 # 2 extra overlapping because padded
# namespace tokens3, tokens4 was only in original_vocab,
# and its token count should be same in extended_vocab
assert extended_vocab.get_vocab_size("tokens2") == original_vocab.get_vocab_size("tokens2")
assert extended_vocab.get_vocab_size("tokens3") == original_vocab.get_vocab_size("tokens3")
# namespace tokens2 was only in instances,
# and its token count should be same in extended_vocab
assert extended_vocab.get_vocab_size("tokens4") == 6 # l,m,n,o + oov + padding
assert extended_vocab.get_vocab_size("tokens5") == 3 # x,y,z
# Word2index mapping of all words in all namespaces of original_vocab
# should be maintained in extended_vocab
for namespace, token2index in original_vocab._token_to_index.items():
for token, _ in token2index.items():
vocab_index = original_vocab.get_token_index(token, namespace)
extended_vocab_index = extended_vocab.get_token_index(token, namespace)
assert vocab_index == extended_vocab_index
# And same for Index2Word mapping
for namespace, index2token in original_vocab._index_to_token.items():
for index, _ in index2token.items():
vocab_token = original_vocab.get_token_from_index(index, namespace)
extended_vocab_token = extended_vocab.get_token_from_index(index, namespace)
assert vocab_token == extended_vocab_token
def index(self, vocab: Vocabulary):
if self._label_ids is None:
self._label_ids = [vocab.get_token_index(label, self._label_namespace) # type: ignore
for label in self.labels]
if not self._num_labels:
self._num_labels = vocab.get_vocab_size(self._label_namespace)