def testMarkRewind(self):
"""CommonTokenStream.mark()/rewind()"""
self.source.tokens.append(
antlr3.CommonToken(type=12)
)
self.source.tokens.append(
antlr3.CommonToken(type=13)
)
self.source.tokens.append(
antlr3.CommonToken(type=antlr3.EOF)
)
stream = antlr3.CommonTokenStream(self.source)
stream.fillBuffer()
stream.consume()
marker = stream.mark()
stream.consume()
stream.rewind(marker)
self.failUnlessEqual(stream.LA(1), 13)
def testConsume(self):
"""CommonTokenStream.consume()"""
self.source.tokens.append(
antlr3.CommonToken(type=12)
)
self.source.tokens.append(
antlr3.CommonToken(type=13)
)
self.source.tokens.append(
antlr3.CommonToken(type=antlr3.EOF)
)
stream = antlr3.CommonTokenStream(self.source)
self.failUnlessEqual(stream.LA(1), 12)
stream.consume()
self.failUnlessEqual(stream.LA(1), 13)
stream.consume()
self.failUnlessEqual(stream.LA(1), antlr3.EOF)
stream.consume()
self.failUnlessEqual(stream.LA(1), antlr3.EOF)
def testFillBuffer(self):
"""CommonTokenStream.fillBuffer()"""
self.source.tokens.append(
antlr3.CommonToken(type=12)
)
self.source.tokens.append(
antlr3.CommonToken(type=13)
)
self.source.tokens.append(
antlr3.CommonToken(type=14)
)
self.source.tokens.append(
antlr3.CommonToken(type=antlr3.EOF)
)
stream = antlr3.CommonTokenStream(self.source)
stream.fillBuffer()
self.failUnlessEqual(len(stream.tokens), 3)
self.failUnlessEqual(stream.tokens[0].type, 12)
self.failUnlessEqual(stream.tokens[1].type, 13)
self.failUnlessEqual(stream.tokens[2].type, 14)
def testLBBeyondBegin(self):
"""CommonTokenStream.LB(-1): beyond begin"""
self.source.tokens.append(
antlr3.CommonToken(type=12)
)
self.source.tokens.append(
antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
)
self.source.tokens.append(
antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
)
self.source.tokens.append(
antlr3.CommonToken(type=13)
)
stream = antlr3.CommonTokenStream(self.source)
self.failUnless(stream.LB(1) is None)
stream.consume()
stream.consume()
self.failUnless(stream.LB(3) is None)
def testLTZero(self):
"""CommonTokenStream.LT(0)"""
self.source.tokens.append(antlr3.CommonToken(type=12))
self.source.tokens.append(antlr3.CommonToken(type=13))
stream = antlr3.CommonTokenStream(self.source)
lt1 = stream.LT(0)
self.failUnless(lt1 is None)
def testLTZero(self):
"""CommonTokenStream.LT(0)"""
self.source.tokens.append(antlr3.CommonToken(type=12))
self.source.tokens.append(antlr3.CommonToken(type=13))
stream = antlr3.CommonTokenStream(self.source)
lt1 = stream.LT(0)
self.assertIsNone(lt1)
def testLB1(self):
"""CommonTokenStream.LB(1)"""
self.source.tokens.append(antlr3.CommonToken(type=12))
self.source.tokens.append(antlr3.CommonToken(type=13))
stream = antlr3.CommonTokenStream(self.source)
stream.fillBuffer()
stream.consume()
self.failUnlessEqual(stream.LB(1).type, 12)
def testLT2BeyondEnd(self):
"""CommonTokenStream.LT(2): beyond end"""
self.source.tokens.append(antlr3.CommonToken(type=12))
self.source.tokens.append(
antlr3.CommonToken(type=13, channel=antlr3.HIDDEN_CHANNEL))
stream = antlr3.CommonTokenStream(self.source)
lt1 = stream.LT(2)
self.failUnlessEqual(lt1.type, antlr3.EOF)
def testLT1WithHidden(self):
"""CommonTokenStream.LT(1): with hidden tokens"""
self.source.tokens.append(
antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL))
self.source.tokens.append(antlr3.CommonToken(type=13))
stream = antlr3.CommonTokenStream(self.source)
lt1 = stream.LT(1)
self.failUnlessEqual(lt1.type, 13)
def testLTNegative(self):
"""CommonTokenStream.LT(-1): look back"""
self.source.tokens.append(antlr3.CommonToken(type=12))
self.source.tokens.append(antlr3.CommonToken(type=13))
stream = antlr3.CommonTokenStream(self.source)
stream.fillBuffer()
stream.consume()
lt1 = stream.LT(-1)
self.failUnlessEqual(lt1.type, 12)
def testToString(self):
"""CommonTokenStream.toString()"""
self.source.tokens.append(antlr3.CommonToken(type=12, text="foo"))
self.source.tokens.append(antlr3.CommonToken(type=13, text="bar"))
self.source.tokens.append(antlr3.CommonToken(type=14, text="gnurz"))
self.source.tokens.append(antlr3.CommonToken(type=15, text="blarz"))
stream = antlr3.CommonTokenStream(self.source)
assert stream.toString() == "foobargnurzblarz"
assert stream.toString(1, 2) == "bargnurz"
assert stream.toString(stream.tokens[1],
stream.tokens[-2]) == "bargnurz"
def testSeek(self):
"""CommonTokenStream.seek()"""
self.source.tokens.append(antlr3.CommonToken(type=12))
self.source.tokens.append(antlr3.CommonToken(type=13))
self.source.tokens.append(antlr3.CommonToken(type=antlr3.EOF))
stream = antlr3.CommonTokenStream(self.source)
self.assertEqual(stream.LA(1), 12)
stream.seek(2)
self.assertEqual(stream.LA(1), antlr3.EOF)
stream.seek(0)
self.assertEqual(stream.LA(1), 12)
def testLT1(self):
"""CommonTokenStream.LT(1)"""
self.source.tokens.append(antlr3.CommonToken(type=12))
stream = antlr3.CommonTokenStream(self.source)
lt1 = stream.LT(1)
self.failUnlessEqual(lt1.type, 12)
def makeEOFToken(self):
return antlr3.CommonToken(type=antlr3.EOF)
