def test_lexer_offset(self):
"""Test the offset argument of Lexer.__init__()."""
lexer = Lexer('a b', 2)
self.assertEqual(lexer.start, 2)
self.assertEqual(lexer.end, 3)
self.assertEqual(lexer.token, Token.IDENT)
self.assertEqual(lexer.literal, 'b')
lexer.consume(Token.IDENT)
self.assertEqual(lexer.start, 3)
self.assertEqual(lexer.end, 3)
self.assertEqual(lexer.token, Token.EOF)
self.assertEqual(lexer.literal, None)
lexer.consume(Token.EOF)
def test_lexer_offset(self):
"""Test the offset argument of Lexer.__init__()."""
lexer = Lexer('a b', 2)
self.assertEqual(lexer.start, 2)
self.assertEqual(lexer.end, 3)
self.assertEqual(lexer.token, Token.IDENT)
self.assertEqual(lexer.literal, 'b')
lexer.consume(Token.IDENT)
self.assertEqual(lexer.start, 3)
self.assertEqual(lexer.end, 3)
self.assertEqual(lexer.token, Token.EOF)
self.assertEqual(lexer.literal, None)
lexer.consume(Token.EOF)
def test_lexer(self):
"""Test token, start, end, literal, and consume()."""
lexer = Lexer('a b //a\n "c"', 0)
self.assertEqual(lexer.start, 0)
self.assertEqual(lexer.end, 1)
self.assertEqual(lexer.token, Token.IDENT)
self.assertEqual(lexer.literal, 'a')
lexer.consume(Token.IDENT)
self.assertEqual(lexer.start, 2)
self.assertEqual(lexer.end, 3)
self.assertEqual(lexer.token, Token.IDENT)
self.assertEqual(lexer.literal, 'b')
lexer.consume(Token.IDENT)
self.assertEqual(lexer.start, 9)
self.assertEqual(lexer.end, 12)
self.assertEqual(lexer.token, Token.STRING)
self.assertEqual(lexer.literal, 'c')
lexer.consume(Token.STRING)
self.assertEqual(lexer.start, 12)
self.assertEqual(lexer.end, 12)
self.assertEqual(lexer.token, Token.EOF)
self.assertEqual(lexer.literal, None)
def test_lexer(self):
"""Test token, start, end, literal, and consume()."""
lexer = Lexer('a b //a\n "c"', 0)
self.assertEqual(lexer.start, 0)
self.assertEqual(lexer.end, 1)
self.assertEqual(lexer.token, Token.IDENT)
self.assertEqual(lexer.literal, 'a')
lexer.consume(Token.IDENT)
self.assertEqual(lexer.start, 2)
self.assertEqual(lexer.end, 3)
self.assertEqual(lexer.token, Token.IDENT)
self.assertEqual(lexer.literal, 'b')
lexer.consume(Token.IDENT)
self.assertEqual(lexer.start, 9)
self.assertEqual(lexer.end, 12)
self.assertEqual(lexer.token, Token.STRING)
self.assertEqual(lexer.literal, 'c')
lexer.consume(Token.STRING)
self.assertEqual(lexer.start, 12)
self.assertEqual(lexer.end, 12)
self.assertEqual(lexer.token, Token.EOF)
self.assertEqual(lexer.literal, None)
