Python DepLabelIndexer.tokens_to_indices Examples

Example #1

File: dep_label_indexer_test.py Project: apmoore1/allennlp

 def test_tokens_to_indices_uses_pos_tags(self):
     tokens = self.tokenizer.split_words("This is a sentence.")
     tokens = [t for t in tokens] + [Token("</S>")]
     vocab = Vocabulary()
     root_index = vocab.add_token_to_namespace('ROOT', namespace='dep_labels')
     none_index = vocab.add_token_to_namespace('NONE', namespace='dep_labels')
     indexer = DepLabelIndexer()
     assert indexer.tokens_to_indices([tokens[1]], vocab, "tokens1") == {"tokens1": [root_index]}
     assert indexer.tokens_to_indices([tokens[-1]], vocab, "tokens-1") == {"tokens-1": [none_index]}

Example #2

 def test_tokens_to_indices_uses_pos_tags(self):
     tokens = self.tokenizer.split_words("This is a sentence.")
     tokens = [t for t in tokens] + [Token("</S>")]
     vocab = Vocabulary()
     root_index = vocab.add_token_to_namespace('ROOT', namespace='dep_labels')
     none_index = vocab.add_token_to_namespace('NONE', namespace='dep_labels')
     indexer = DepLabelIndexer()
     assert indexer.tokens_to_indices([tokens[1]], vocab, "tokens1") == {"tokens1": [root_index]}
     assert indexer.tokens_to_indices([tokens[-1]], vocab, "tokens-1") == {"tokens-1": [none_index]}

Example #3

File: dep_label_indexer_test.py Project: zulushakaka/allennlp

 def test_tokens_to_indices_uses_pos_tags(self):
     tokens = self.tokenizer.tokenize("This is a sentence.")
     tokens = [t for t in tokens] + [Token("</S>")]
     vocab = Vocabulary()
     root_index = vocab.add_token_to_namespace("ROOT", namespace="dep_labels")
     none_index = vocab.add_token_to_namespace("NONE", namespace="dep_labels")
     indexer = DepLabelIndexer()
     assert indexer.tokens_to_indices([tokens[1]], vocab) == {"tokens": [root_index]}
     assert indexer.tokens_to_indices([tokens[-1]], vocab) == {"tokens": [none_index]}