def test_peek(self):
lexer = Lexer(drgn_test_lexer_func, "12345")
for i in range(5):
self.assertEqual(lexer.peek().kind, ord("1") + i)
self.assertEqual(lexer.pop().kind, ord("1") + i)
self.assertEqual(lexer.peek().kind, 0)
self.assertEqual(lexer.pop().kind, 0)
def test_push(self):
lexer = Lexer(drgn_test_lexer_func, "12345")
tokens = []
for i in range(4):
tokens.append(lexer.pop())
while tokens:
lexer.push(tokens.pop())
for i in range(5):
self.assertEqual(lexer.pop().kind, ord("1") + i)
self.assertEqual(lexer.pop().kind, 0)
def lex(self, s):
lexer = Lexer(drgn_lexer_c, s)
while True:
token = lexer.pop()
if token.kind == C_TOKEN.EOF:
break
yield token
def test_empty(self):
lexer = Lexer(drgn_lexer_c, "")
for i in range(64):
self.assertEqual(lexer.pop().kind, C_TOKEN.EOF)
def test_pop(self):
lexer = Lexer(drgn_test_lexer_func, '12345')
for i in range(5):
self.assertEqual(lexer.pop().kind, ord('1') + i)
self.assertEqual(lexer.pop().kind, 0)