def test_nextword(self):
t = Tokenizer()
s = "word1 word2\n word3\n\tword4 "
t.init(StringIO(s))
words = ["word1", "word2", "\n", "word3", "\n", "word4", "\n"]
g = t.nextword()
for word in words:
self.assertEquals(word, g.next())
try:
g.next()
except StopIteration:
pass
