Python Lexer.get_tokens Exemples

Exemple #1

Fichier : test_lexer.py Projet : lcary/tranq

 def test_oneline_computation(self):
     text = "print: 4 * var + 1"
     lexer = Lexer()
     tokens = lexer.get_tokens(text)
     expect = [
         Token(token_type=TokenType.identifier, value='print'),
         Token(token_type=TokenType.operator, value=':'),
         Token(token_type=TokenType.number, value='4'),
         Token(token_type=TokenType.operator, value='*'),
         Token(token_type=TokenType.identifier, value='var'),
         Token(token_type=TokenType.operator, value='+'),
         Token(token_type=TokenType.number, value='1')]
     self.assertEqual(tokens, expect)

Exemple #2

def main():
    lexer = Lexer()
    parser = Parser()
    tokens = lexer.get_tokens(*sys.argv[1:])
    ast = parser.parse_ast(tokens)
    print(ast)