def test_add_token_without_value(self):
lexer = Lexer('123')
lexer.token_buffer = '12345'
lexer.add_token(TokenType.IDENTIFIER, with_value=False)
self.assertEqual(TokenType.IDENTIFIER, lexer.tokens[0].type)
self.assertEqual('', lexer.tokens[0].value)
self.assertEqual('', lexer.token_buffer)
def test_add_token_no_rollback(self):
lexer = Lexer('123')
lexer.token_buffer = '12345'
lexer.state = LexingState.LIT_STR
lexer.offset = 4
lexer.add_token(TokenType.OP_PLUS)
self.assertEqual(1, len(lexer.tokens))
self.assertEqual('', lexer.token_buffer)
self.assertEqual(LexingState.START, lexer.state)
self.assertEqual(4, lexer.offset)