def test_tokenizer_words_from_current_pos(self):
tokenizer = Tokenizer()
self.assertIsNotNone(tokenizer)
self.assertEqual("", tokenizer.words_from_current_pos(None, 0))
words = ["aaa", "bbb", "ccc"]
self.assertEqual("aaa bbb ccc",
tokenizer.words_from_current_pos(words, 0))
self.assertEqual("bbb ccc", tokenizer.words_from_current_pos(words, 1))
self.assertEqual("ccc", tokenizer.words_from_current_pos(words, 2))
self.assertEqual("", tokenizer.words_from_current_pos(words, 3))
self.assertEqual("ccc", tokenizer.words_from_current_pos(words, -1))
self.assertEqual("bbb ccc",
tokenizer.words_from_current_pos(words, -2))
self.assertEqual("aaa bbb ccc",
tokenizer.words_from_current_pos(words, -3))
self.assertEqual("aaa bbb ccc",
tokenizer.words_from_current_pos(words, -4))
def test_words_from_current_pos_no_words(self):
tokenizer = Tokenizer()
self.assertIsNotNone(tokenizer)
self.assertEquals("", tokenizer.words_from_current_pos([], 5))
self.assertEquals("", tokenizer.words_from_current_pos(None, 5))
def test_words_from_current_pos_past_end(self):
tokenizer = Tokenizer()
self.assertIsNotNone(tokenizer)
self.assertEquals("",
tokenizer.words_from_current_pos(["A", "B", "C"], 5))