def test_nexttoken(self): t = Tokenizer() s = u"[item1] word1 [item2] word2(ícaro)" t.init(StringIO(s)) tokens = [ Token(Kinds.ITEM, u"item1"), Token(Kinds.PUNCTUATION, u" "), Token(Kinds.WORD, u"word1"), Token(Kinds.PUNCTUATION, u" "), Token(Kinds.ITEM, u"item2"), Token(Kinds.PUNCTUATION, u" "), Token(Kinds.WORD, u"word2"), Token(Kinds.PUNCTUATION, u"("), Token(Kinds.WORD, u"ícaro"), Token(Kinds.PUNCTUATION, u")") ] g = t.tokens() for expected in tokens: actual = g.next() self.assertEquals(expected, actual) try: g.next() except StopIteration: pass
def test_readline(self): t = Tokenizer() l1 = "linea1\n" l2 = "linea2\n" l3 = "linea3" s = "{}{}{}".format(l1, l2, l3) f = StringIO(s) t.init(f) g = t.lines() self.assertEquals(l1, g.next()) self.assertEquals(l2, g.next()) self.assertEquals(l3 + "\n", g.next())
def test_nextword(self): t = Tokenizer() s = "word1 word2\n word3\n\tword4 " t.init(StringIO(s)) words = ["word1", "word2", "\n", "word3", "\n", "word4", "\n"] g = t.nextword() for word in words: self.assertEquals(word, g.next()) try: g.next() except StopIteration: pass
def test_nexttoken(self): t = Tokenizer() s = "[item1] word1 [item2] word2" t.init(StringIO(s)) tokens = [ Token(Kinds.ITEM, "item1"), Token(Kinds.WORD, "word1"), Token(Kinds.ITEM, "item2"), Token(Kinds.WORD, "word2") ] g = t.nexttoken() for token in tokens: self.assertEquals(token, g.next()) try: g.next() except StopIteration: pass