def test_raw_tokens_on_sentence_wit_mixed_element(self): expected = [ Multiword(first_index=1, last_index=2), Word(index=3), Multiword(first_index=4, last_index=6), Word(index=7) ] sentence = Sentence([ expected[0], # 1-2 Word(index=1), Word(index=2), expected[1], # 3 EmptyNode(main_index=3, sub_index=1), EmptyNode(main_index=3, sub_index=2), expected[2], # 4-6 Word(index=4), EmptyNode(main_index=4, sub_index=1), EmptyNode(main_index=4, sub_index=2), Word(index=5), Word(index=6), expected[3] # 7 ]) result = sentence.raw_tokens() self.assertIsInstance(result, Generator) self.assertEqual(expected, list(result))
def test_raw_tokens_on_sentence_without_word_and_multiwords(self): sentence = Sentence([ EmptyNode(main_index=0, sub_index=1), ]) result = sentence.raw_tokens() self.assertIsInstance(result, Generator) self.assertEqual([], list(result))
def test_raw_tokens_on_empty_sentence(self): sentence = Sentence() result = sentence.raw_tokens() self.assertIsInstance(result, Generator) self.assertEqual([], list(result))