def test_add_token_rollback(self):
lexer = Lexer('123')
lexer.offset = 4
lexer.add_token(TokenType.OP_PLUS, rollback=True)
self.assertEqual(lexer.offset, 3)
def test_add_token_no_rollback(self):
lexer = Lexer('123')
lexer.token_buffer = '12345'
lexer.state = LexingState.LIT_STR
lexer.offset = 4
lexer.add_token(TokenType.OP_PLUS)
self.assertEqual(1, len(lexer.tokens))
self.assertEqual('', lexer.token_buffer)
self.assertEqual(LexingState.START, lexer.state)
self.assertEqual(4, lexer.offset)
