def test_select_simple_lexer_if_ply_not_present(self, rules): def _import(x): raise ImportError with patch('ox.lexer._import', _import): lexer = ox.make_lexer(rules) assert lexer.which == 'simple'
def make_lexer(): return ox.make_lexer([ ('NUMBER', r'[+-]?\d+(\.\d+)?'), ('SYMBOL', r'[a-zA-Z][_a-zA-Z0-9]*'), ('OP_SUM', r'[+-]'), ('OP_MUL', r'[*/]'), ('CONTROL', r'[(),=;%]'), ])
def lexer(string): lexer = ox.make_lexer([ ('ANSWER', r'ANSWER:\s*[a-zA-Z]'), ('OPTION', r'[a-zA-Z][.)]\s'), ('ANY', r'.*\n'), ]) return lexer(string)
def create_lexer(): lexer = ox.make_lexer([ ('SECTION_TITLE', r'\[[^\]]+\]\n*'), ('SUBSECTION_TITLE', r'\[\[[^\]]+\]\]\n*'), ('DATA', r'\{[^\}]+\}\n*'), ('ignore_COMMENT', r'\#[^\n]+\n*'), ('STRING', r'[^=^#^\n^\[^\]^\}^\{]+\n*'), ('EQUAL', r'='), ]) return lexer
def make_lexer(): return ox.make_lexer([ ('CONTROL', r'[()]'), ('NUMBER', r'[-+]?\d+(\.\d+)?'), ('SYMBOL', r'[-+*/!~@#$%&=?<>.,^]+'), ('NAME', r'[a-z]+'), ('STRING', r'"..."'), ('_COMMENT', r';...'), # Palavras reservadas ('r_IF', 'if'), ('r_LET', 'let'), ])
def lexer(self, string): lexer = ox.make_lexer([ ('ANSWER', r'=.+?(?=[~\#])'), ('OPTION', r'~.+?(?=[~}\#])'), ('FEEDBACK', r'\#.+?(?=[~}])'), ('QUESTION', r'.+?(?={)'), ('TRUE_VALUE', r'T|TRUE'), ('FALSE_VALUE', r'F|FALSE'), ('LBRACKET', r'{'), ('RBRACKET', r'}'), ]) return lexer(string)
def make_lexer(): lexer_rules = [ ('STRING', r'"[^"\n]*(\\"[^"\n]*)*"'), ('NUMBER', r'[+-]?\d+(\.\d+)?'), ('KEYWORD', r'true|false|null'), ('LBRACE', r'\{'), ('RBRACE', r'\}'), ('LBRACK', r'\['), ('RBRACK', r'\]'), ('COLON', r'\:'), ('COMMA', r'\,'), ] lexer = ox.make_lexer(lexer_rules, which='simple') tokens = [tk for tk, _ in lexer_rules] return lexer, tokens
from operator import add, mul, sub, truediv as div import ox lexer = ox.make_lexer({ ('MUL', r'\*'), ('ADD', r'\+'), ('SUB', r'\-'), ('DIV', r'\*'), ('NUMBER', r'\d+'), }) def make_ast(src): """ convert soruce code to a list of numbers and operations >>>makeast('1 2 +') [1.0, 2.0, add] """ tokens = lexer(src) return ... def evaluate (src): """ Avalia o resultado de uma expressao na notacao sufixa >>>evaluate('40 2 1 *+') 42,0 """ ast = make_ast(src) ... return 0.0
def ply_lexer(self, rules): return ox.make_lexer(rules, which='ply')
import ox import click import pprint lexer = ox.make_lexer([ ('LOOP', r'loop'), ('DEC', r'dec'), ('INC', r'inc'), ('LEFT_PAR', r'\('), ('RIGHT_PAR', r'\)'), ('RIGHT', r'right'), ('LEFT', r'left'), ('PRINT', r'print'), ('READ', r'read'), ('DO', r'do'), ('DO_AFTER', r'do-after'), ('DO_BEFORE', r'do-before'), ('ADD', r'add'), ('SUB', r'sub'), ('NUMBER', r'\d+'), ('DEF', r'def'), ('FUCTION', r'(?!loop)(?!dec)(?!inc)(?!right)(?!left)(?!print)(?!read)(?!do)(?!do)(?!do)(?!add)(?!sub)(?!def)(?!\d)[a-zA-Z0-9-\+_]+' ), ('ignore_COMMENT', r';[ \S]*'), ('ignore_SPACE', r'\s+') ]) tokens_list = [ 'DEC', 'INC', 'LOOP', 'LEFT_PAR', 'RIGHT_PAR', 'RIGHT', 'LEFT', 'PRINT', 'READ', 'DO', 'DO_AFTER', 'DO_BEFORE', 'ADD', 'SUB', 'NUMBER', 'DEF', 'FUCTION' ] parser = ox.make_parser([('expr : LEFT_PAR RIGHT_PAR', lambda x, y: '()'), ('expr : LEFT_PAR term RIGHT_PAR', lambda x, y, z: y), ('term : atom term', lambda x, y: (x, ) + y), ('term : atom', lambda x: (x, )), ('atom : expr', lambda x: x), ('atom : DEC', lambda x: x), ('atom : INC', lambda x: x), ('atom : LOOP', lambda x: x), ('atom : RIGHT', lambda x: x), ('atom : LEFT', lambda x: x),
import ox import click import pprint lexer = ox.make_lexer([ ('PARANTHESIS_OPENED', r'\('), ('PARANTHESIS_CLOSED', r'\)'), ('PLAIN_TEXT', r'[-a-zA-Z]+'), ('NUMBERS', r'\d+'), ('ignore_NEWLINE', r'\s+'), ('ignore_COMMENT', r';[^\n]*'), ]) tokens = [ 'PARANTHESIS_OPENED', 'PARANTHESIS_CLOSED', 'PLAIN_TEXT', 'NUMBERS', 'ignore_NEWLINE', 'ignore_COMMENT', ] parser = ox.make_parser([ ('atom : PLAIN_TEXT', lambda x: x), ('atom : NUMBERS', lambda x: x), ('expr : atom expr', lambda x, y: (x, ) + y), ('expr : atom', lambda x: (x, )), ('block : PARANTHESIS_OPENED PARANTHESIS_CLOSED', lambda x, y: '()'), ('block : PARANTHESIS_OPENED expr PARANTHESIS_CLOSED', lambda x, y, z: y), ('atom : block', lambda x: x), ], tokens)
from operator import add, mul, sub, truediv as div import ox lexer = ox.make_lexer([ ('MUL', r'\*'), ('ADD', r'\*'), ('SUB', r'\*'), ('DIV', r'\*'), ('NUMBER', r'\d+'), ]) def make_ast(src): """ Converte código fonte para uma lista de números e operações >>> make_ast('1 2 +') [1.0, 2.0, add] """ tokens = lexer(src) return ... def evaluate(src): """ Avalia o resultado de uma expressão na notação sufixa. >>> evaluate('40 2 1 * +') 42.0 """ ast = make_ast(src) ... return 0.0
import ox import operator as op OPERATORS = {'+': op.add, '-': op.sub, '*': op.mul, '/': op.truediv} lexer = ox.make_lexer([ ('NUMBER', r'[0-9]+(\.[0-9]+)?'), ('OP_SUM', r'[+-]'), ('OP_MUL', r'[*/]'), ('LPAR', r'\('), ('RPAR', r'\)'), ]) identity = lambda x: x def compute_operation(x, op, y): return OPERATORS[op](x, y) parser = ox.make_parser([ ('expr : term OP_SUM expr', compute_operation), ('expr : term', identity), ('term : atom OP_MUL term', compute_operation), ('term : atom', identity), ('atom : NUMBER', float), ('atom : LPAR expr RPAR', lambda x, y, z: y), ], ['NUMBER', 'OP_SUM', 'OP_MUL', 'LPAR', 'RPAR']) while True: from pprint import pprint
import ox import click from interpreter import Interpreter lexer = ox.make_lexer([('NAME', r'[-a-zA-Z]+'), ('NUMBER', r'\d+'), ('PARENTESE_A', r'\('), ('PARENTESE_F', r'\)'), ('COMMENT', r';.*'), ('NEWLINE', r'\n'), ('SPACE', r'\s+')]) tokens = ['PARENTESE_F', 'PARENTESE_A', 'NUMBER', 'NAME'] name = lambda name: (name) number = lambda number: (int(number)) op = lambda op: (op) parser = ox.make_parser([ ('program : PARENTESE_A expr PARENTESE_F', lambda x, y, z: y), ('program : PARENTESE_A PARENTESE_F', lambda x, y: '()'), ('expr : operator expr', lambda x, y: (x, ) + y), ('expr : operator', lambda x: (x, )), ('operator : program', op), ('operator : NAME', name), ('operator : NUMBER', number), ], tokens) @click.command() @click.argument('source', type=click.File('r')) def make_tree(source): program = source.read()
def test_issues_an_error_with_invalid_lexer_name(self, rules): with pytest.raises(ValueError): ox.make_lexer(rules, which='foobar')
import ox import click import re from getch import getche lexer = ox.make_lexer([ ('NAME',r'[a-zA-Z]+'), ('NUMBER', r'\d+'), ('OPENING_PARENTHESES', r'\('), ('CLOSING_PARENTHESES', r'\)'), ]) tokens_list = ['NUMBER', 'NAME','OPENING_PARENTHESES','CLOSING_PARENTHESES','COMMA'] atom_number = lambda value: ('atom_number', float(value)) parser = ox.make_parser([ ('simple_block : simple_block simple_term', lambda first, second: (first, second)), ('simple_block : simple_term', lambda simple_block: simple_block), ('simple_term : OPENING_PARENTHESES simple_term CLOSING_PARENTHESES', lambda opening_paretheses, term, closing_parentheses: (opening_paretheses, term, closing_parentheses)), ('simple_term : atom simple_term',lambda first_term, second_term : (first_term, second_term)), ('simple_term : atom COMMA simple_term',lambda atom, comma, simple_term : (atom, comma, simple_term)), ('simple_term : atom', lambda term: term), ('atom : OPENING_PARENTHESES atom CLOSING_PARENTHESES', lambda opening_paretheses, term, closing_parentheses: (opening_paretheses, term, closing_parentheses)), ('atom : NUMBER', atom_number), ('atom : NAME',lambda name : name), ], tokens_list) def pretty_print(code_p): indent = 0
('DIGITS', r'\d+'), ('ignore_NEWLINE', r'\s+'), ('ignore_COMMENT', r';[^\n]*'), ] parser_rules = [ ('block : OPEN_PARANTHESIS CLOSE_PARANTHESIS', lambda x, y: '()'), ('block : OPEN_PARANTHESIS expr CLOSE_PARANTHESIS', lambda x, y, z: y), ('atom : TEXT', lambda x: x), ('atom : DIGITS', lambda x: x), ('expr : atom expr', lambda x, y: (x, ) + y), ('expr : atom', lambda x: (x, )), ('atom : block', lambda x: x), ] lexer = ox.make_lexer(lexer_rules) parser = ox.make_parser(parser_rules, tokens) @click.command() @click.argument('lispfcktree', type=click.File('r')) def ast(lispfcktree): """Create ast.""" lispcode = lispfcktree.read() tokens = lexer(lispcode) tree = parser(tokens) pprint.pprint(tree) if __name__ == '__main__': ast()
import ox import click from getch import getche """ Create tokens list and its lexer and parser """ lexer = ox.make_lexer([ ('NUMBER', r'\d+'), ('NAME', r'[-a-zA-Z]+'), ('LPARAN', r'[(]'), ('RPARAN', r'[)]'), ('COMMENT', r';.*'), ('NEWLINE', r'\s+'), ]) tokens_list = ['NAME', 'NUMBER', 'LPARAN', 'RPARAN'] parser = ox.make_parser([ ('stmt : LPARAN RPARAN', lambda x,y: '()'), ('stmt : LPARAN expr RPARAN', lambda x,y,z: y), ('expr : term expr', lambda x,y: [x] + y), ('expr : term', lambda x: [x]), ('term : stmt', lambda x: x), ('term : NUMBER', lambda x: int(x)), ('term : NAME', lambda x: x), ], tokens_list) collection = [0] p = 0
import ox import operator as operator # \d representa digitos de 0 a 9 lexer = ox.make_lexer([ ('NUMBER', r'\d+(\.\d*)?'), ('OP_S', r'[-+]'), ('OP_M', r'[*/]'), ('OP_P', r'[\^]'), ('PAR_O', r'\('), ('PAR_C', r'\)'), ]) tokens_list = ['NUMBER', 'OP_S', 'OP_M', 'OP_P', 'PAR_O', 'PAR_C'] infix = lambda x, op, y: (op, x, y) atom = lambda x: ('atom', float(x)) parser = ox.make_parser([ ('expr : expr OP_S term', infix), ('expr : term', lambda x: x), ('term : term OP_M res', infix), ('term : res', lambda x: x), ('res : res OP_P atom', infix), ('res : atom', lambda x: x), ('atom : NUMBER', atom), ], tokens_list) OP_TO_FUNC = { '+': lambda x, y: x + y, '-': lambda x, y: x - y, '*': lambda x, y: x * y,
import ox import operator as op OP_TO_FUNC = {'+': op.add, '-': op.sub, '*': op.mul, '/': op.truediv} lexer = ox.make_lexer([ ('NUMBER', r'\d+(\.\d*)?'), ('TERM_OP', r'[*/]'), ('EXP_OP', r'[-+]'), ('LPAREN', r'\('), ('RPAREN', r'\)') ]) parser = ox.make_parser([ ('expr : expr EXP_OP term', lambda x, op, y: (op, x, y)), ('term : term TERM_OP value', lambda x, op, y: (op, x, y)), ('expr : term', lambda x: x), ('term : value', lambda x: x), ('value : NUMBER', lambda x: ('atom', x)), ], ['NUMBER', 'TERM_OP', 'EXP_OP']) def eval(ast): head, *tail = ast if head == 'atom': return float(tail[0]) else: x, y = tail
def simple_lexer(self, rules): return ox.make_lexer(rules, which='simple')
import ox import click import pprint lexer = ox.make_lexer([ ('RIGHT', r'right'), # > in brainfuck ('LEFT', r'left'), # < in brainfuck ('INC', r'inc'), # + in brainfuck ('DEC', r'dec'), # - in brainfuck ('PRINT', r'print'), # . in brainfuck ('READ', r'read'), # , in brainfuck ('DO', r'do'), ('ADD', r'add'), ('SUB', r'sub'), ('LOOP', r'loop'), # [] in brainfuck ('DEF', r'def'), ('NUMBER', r'\d+'), ('PARENTESE_A', r'\('), ('PARENTESE_F', r'\)'), ('NAME', r'[-a-zA-Z]+'), ('COMMENT', r';.*'), ('NEWLINE', r'\n'), ('SPACE', r'\s+') ]) tokens = [ 'RIGHT', 'LEFT', 'INC', 'DEC', 'SUB', 'ADD', 'NUMBER', 'PRINT', 'LOOP', 'READ', 'DEF', 'PARENTESE_F', 'PARENTESE_A', 'DO', 'NAME' ] operator = lambda type_op: (type_op)
FCall = sk.opt(name=str, fargs=list) Number = Expr.Number BinOp = Expr.BinOp FCall = Expr.FCall # # Calc lexer # lexer = ox.make_lexer([ ('NUMBER', r'\d+(\.\d+)?'), ('STRING', r'"[^"]*"'), ('OP', r'[-+*/<>=?@&$^~%]+'), ('COMMENT', r'\#[^\n]*'), ('LPAR', r'\('), ('RPAR', r'\)'), ('FNAME', r'[a-z]+'), ('COMMA', r'\,'), ]) tokens = ['NUMBER', 'OP', 'LPAR', 'RPAR', 'FNAME', 'COMMA'] # # Calc parser # identity = (lambda x: x) op_call = (lambda x, op, y: BinOp(op, x, y)) fcall = (lambda x, y, z, w: FCall(x, z))
import ox import click import pprint lexer = ox.make_lexer([ ('OPEN_P', r'\('), ('CLOSE_P', r'\)'), ('OP', r'[-a-zA-Z]+'), ('NUMBER', r'[0-9]+'), ('ignore_COMMENT', r';[^\n]*'), ('ignore_NEWLINE', r'\s+'), ]) tokens = ['OP', 'NUMBER', 'OPEN_P', 'CLOSE_P'] atom = lambda x: x term = lambda x: (x, ) comp = lambda x, y: (x, ) + y pare = lambda x, y: '()' expr = lambda x, y, z: y parser = ox.make_parser([ ('expr : OPEN_P term CLOSE_P', expr), ('expr : OPEN_P CLOSE_P', pare), ('term : atom term', comp), ('term : atom', term), ('atom : expr', atom), ('atom : OP', atom), ('atom : NUMBER', atom), ], tokens)
""" Implementa uma calculadora simples utilizando a biblioteca ox. """ import ox import operator as op OP_TO_FUNCTION = {'+': op.add, '-': op.sub, '*': op.mul, '/': op.truediv} # # Lexer # lexer = ox.make_lexer([ ('NUMBER', r'[0-9]+(\.[0-9]+)?'), ('OP', r'[-+*/]'), ]) # # Parser # binop = (lambda a, op, b: OP_TO_FUNCTION[op](a, b)) parser = ox.make_parser([ ('term : term OP atom', binop), ('term : atom', lambda x: x), ('atom : NUMBER', float), ]) # # # if __name__ == '__main__':
from getch import getche import click import pprint import ox lexer = ox.make_lexer([ ('COMMENT', r';(.)*'), ('NEW_LINE', r'\n+'), ('OPEN_BRACKET', r'\('), ('CLOSE_BRACKET', r'\)'), ('NAME', r'[a-zA-Z_][a-zA-Z_0-9-]*'), ('NUMBER', r'\d+(\.\d*)?'), ]) token_list = [ 'NAME', 'NUMBER', 'OPEN_BRACKET', 'CLOSE_BRACKET', ] identity = lambda x: x parser = ox.make_parser([ ('tuple : OPEN_BRACKET elements CLOSE_BRACKET', lambda a, x, b: x), ('tuple : OPEN_BRACKET CLOSE_BRACKET', lambda a, b: '[]'), ('elements : term elements', lambda x, xs: [x] + xs), ('elements : term', lambda x: [x]), ('term : atom', identity), ('term : tuple', identity), ('atom : NAME', identity),
import ox import click import pprint from lisp_interpreter import LispInterpreter #Define lexer rules lisp_lexer = ox.make_lexer([('COMMENT', r';.*'), ('NEWLINE', r'\n'), ('SPACE', r'\s+'), ('NAME', r'[-a-zA-Z]+'), ('NUMBER', r'\d+'), ('PARENT_OPEN', r'\('), ('PARENT_CLOSE', r'\)')]) #Define tokens tokens = ['PARENT_CLOSE', 'PARENT_OPEN', 'NUMBER', 'NAME'] #Methods for parser def name(name): return name def number(number): return int(number) def single_operator(operator): return operator def parent_expr(parent_open, expr, parent_close): return expr
import ox import click import pprint lexer = ox.make_lexer([ ('LOOP', r'loop'), ('DEC', r'dec'), ('INC', r'inc'), ('LPAR', r'\('), ('RPAR', r'\)'), ('RIGHT', r'right'), ('LEFT', r'left'), ('PRINT', r'print'), ('READ', r'read'), ('DO', r'do'), ('DO_AFTER', r'do-after'), ('DO_BEFORE', r'do-before'), ('ADD', r'add'), ('SUB', r'sub'), ('NUMBER', r'[0-9]+'), ('ignore_COMMENT', r';[^\n]*'), ('ignore_BREAK_LINE', r'\n'), ('ignore_SPACE', r'\s+') ]) tokens_list = ['LOOP', 'DEC', 'INC', 'LPAR', 'RPAR', 'RIGHT',
import ox #biblioteca do parse -- Só funciona em python3 import operator as op #Criar lexer : Função que recebe string de código e retorna tokens (Recebe lista de tuplas - nome do token e expressão regular) lexer = ox.make_lexer([ ('NUMBER', r'\d+(\.\d*)?'), # /d = [0-9] ('OP_S', r'[-+]'), ('OP_M', r'[*/]'), ]) # Função recebe o token jogado na funcao func, retorna objeto que é associado à "átomo" #Converte string func = float em número par uso posterior tokens_list = ['NUMBER','OP'] infix = lambda x, op, y : (op, x, y) #Só organiza a ordem dos operadores #Lambda define uma funcao simples (lambda argumentos e :retorno) atom = lambda x: ('atom',float(x)) # Expressão "SEXY ;p" -- função átomo recebe um token e retorna a árvore sintática parser = ox.make_parser([ ('expr : expr OP_S term', infix), ('expr : term', lambda x : x), ('term : term OP_M atom', infix), ('term : atom', lambda x : x), ('atom : NUMBER', atom) # Reduz numéro em um átomo # ('expr : atom OP atom', --> (OP x y)) # lisp (lista em python) - Operadores infixo, prefixo, sufixo ], tokens_list) # ------------------------FUNÇÕES ---------------------------------------------------------------------------- OP_TO_FUNC = { '+': lambda x , y : x + y,
import ox lexer = ox.make_lexer([('NUMBER', r'[0-9]+(\.[0-9]+?)'), ('OP_PRI', r'[*/]'), ('OP_SEC', r'[-+]')]) OPERATORS = { '+': lambda x, y: x + y, '-': lambda x, y: x - y, '*': lambda x, y: x * y, '/': lambda x, y: x / y, } tokens = ['NUMBER', 'OP'] parser = ox.make_parser([('term : term OP atom', lambda a, op, b: OPERATORS[op] (a, b)), ('term : atom', lambda x: x), ('atom : NUMBER', float)], tokens) st = input('expr: ') tokens = lexer(st) print(lexer(st)) res = parser(tokens) print(res)