def test_raw_tokens_on_sentence_wit_mixed_element(self):
expected = [
Multiword(first_index=1, last_index=2),
Word(index=3),
Multiword(first_index=4, last_index=6),
Word(index=7)
]
sentence = Sentence([
expected[0], # 1-2
Word(index=1),
Word(index=2),
expected[1], # 3
EmptyNode(main_index=3, sub_index=1),
EmptyNode(main_index=3, sub_index=2),
expected[2], # 4-6
Word(index=4),
EmptyNode(main_index=4, sub_index=1),
EmptyNode(main_index=4, sub_index=2),
Word(index=5),
Word(index=6),
expected[3] # 7
])
result = sentence.raw_tokens()
self.assertIsInstance(result, Generator)
self.assertEqual(expected, list(result))
def test_raw_tokens_on_sentence_without_word_and_multiwords(self):
sentence = Sentence([
EmptyNode(main_index=0, sub_index=1),
])
result = sentence.raw_tokens()
self.assertIsInstance(result, Generator)
self.assertEqual([], list(result))
def test_raw_tokens_on_empty_sentence(self):
sentence = Sentence()
result = sentence.raw_tokens()
self.assertIsInstance(result, Generator)
self.assertEqual([], list(result))