def test_nexttoken(self):
t = Tokenizer()
s = "[item1] word1 [item2] word2"
t.init(StringIO(s))
tokens = [
Token(Kinds.ITEM, "item1"),
Token(Kinds.WORD, "word1"),
Token(Kinds.ITEM, "item2"),
Token(Kinds.WORD, "word2")
]
g = t.nexttoken()
for token in tokens:
self.assertEquals(token, g.next())
try:
g.next()
except StopIteration:
pass
