tok_string = token("string") dot = token(".") colon = token(":") single_quote = token('"') double_quote = token("'") quote = (single_quote | double_quote) open_sq_brace = token("[") close_sq_brace = token("]") open_rnd_brace = token("(") close_rnd_brace = token(")") tok_constant = p.some(lambda t: t.value in {'nil', 'true', 'false'}) iden_start = p.skip(p.some(lambda t: t.type not in ".:")) tok_splash = (p.a(Token("iden", "splash")) + iden_start) >> token_value iden = token("iden") opt_iden = iden | p.pure("") # =========== Expressions parser # FIXME: it should be rewritten using full Lua 5.2 grammar. BINARY_OPS = set("+-*/^%><") | {"..", "==", "~=", ">=", "<=", "and", "or"} UNARY_OPS = {"not", "-", "#"} binary_op = p.some(lambda t: t.value in BINARY_OPS) >> token_value unary_op = p.some(lambda t: t.value in UNARY_OPS) >> token_value # expressions with binary and unary ops + parenthesis @p.with_forward_decls def value(): single_value = table | tok_number | tok_string | tok_constant | iden return single_value | (close_rnd_brace + expr + open_rnd_brace)
lparen = some(lambda tok: tok == "(") rparen = some(lambda tok: tok == ")") op = some(lambda tok: tok in "+-*^&|") eof = some(lambda tok: tok == EOF) number = some(lambda tok: tok.isdigit()) >> make_number paren_expr = with_forward_decls( lambda: lparen + expr + rparen ) # *Mark here are not really required, but if you are going to do # anything complex that requires that you discern between different # parsing paths, marks are often give you least hassle. expr = with_forward_decls( lambda: (number + pure(NumberMark) + expr_rest | paren_expr + pure(ParenMark) + expr_rest) >> make_expr) # This one allows us to add more complex expressions like function # application and ternary operators to the above definition with ease. # Otherwise terms such as `apply = expr lparen many(expr) rpanen` # would be impossible to add, always leading to infinite left recursion. expr_rest = maybe(op + expr) toplev = expr + skip(eof) @py.test.mark.parametrize("given, parser, expected", [ ("1", number, Number("1")), ("+", op, "+"), ("-", op, "-"),
tok_string = token("string") dot = token(".") colon = token(":") single_quote = token('"') double_quote = token("'") quote = (single_quote | double_quote) open_sq_brace = token("[") close_sq_brace = token("]") open_rnd_brace = token("(") close_rnd_brace = token(")") tok_constant = p.some(lambda t: t.value in {'nil', 'true', 'false'}) iden_start = p.skip(p.some(lambda t: t.type not in ".:")) tok_splash = (p.a(Token("iden", "splash")) + iden_start) >> token_value iden = token("iden") opt_iden = iden | p.pure("") # =========== Expressions parser # FIXME: it should be rewritten using full Lua 5.2 grammar. BINARY_OPS = set("+-*/^%><") | {"..", "==", "~=", ">=", "<=", "and", "or"} UNARY_OPS = {"not", "-", "#"} binary_op = p.some(lambda t: t.value in BINARY_OPS) >> token_value unary_op = p.some(lambda t: t.value in UNARY_OPS) >> token_value # expressions with binary and unary ops + parenthesis @p.with_forward_decls def value(): single_value = table | tok_number | tok_string | tok_constant | iden
# -*- coding: utf-8 -*- 'Tests for issue #8: prevent definitions of non-halting parsers.' from funcparserlib.parser import (a, many, fwd, maybe, pure, oneplus, GrammarError, makes_progress, non_halting) from funcparserlib.contrib.common import const from nose.tools import ok_, assert_raises x = a('x') p1 = maybe(x) p3 = maybe(x) + x p4 = many(p3) p5 = x | many(x) p8 = x >> const(True) p9 = pure(True) def test_makes_progress(): ok_(not makes_progress(p1)) ok_(not makes_progress(p4)) ok_(not makes_progress(p5)) ok_(not makes_progress(p9)) ok_(makes_progress(p3)) ok_(makes_progress(p8)) def test_non_halting_many(): assert_raises(GrammarError, lambda: many(many(x)).parse('')) assert_raises(GrammarError, lambda: oneplus(many(x)).parse('')) assert_raises(GrammarError, lambda: many(p1).parse(''))
'Tests for issue #8: prevent definitions of non-halting parsers.' from funcparserlib.parser import ( a, many, fwd, maybe, pure, oneplus, GrammarError, makes_progress, non_halting) from funcparserlib.contrib.common import const from nose.tools import ok_, assert_raises x = a('x') p1 = maybe(x) p3 = maybe(x) + x p4 = many(p3) p5 = x | many(x) p8 = x >> const(True) p9 = pure(True) def test_makes_progress(): ok_(not makes_progress(p1)) ok_(not makes_progress(p4)) ok_(not makes_progress(p5)) ok_(not makes_progress(p9)) ok_(makes_progress(p3)) ok_(makes_progress(p8)) def test_non_halting_many(): assert_raises(GrammarError, lambda: many(many(x)).parse('')) assert_raises(GrammarError, lambda: oneplus(many(x)).parse('')) assert_raises(GrammarError, lambda: many(p1).parse('')) assert_raises(GrammarError, lambda: many(p5).parse('')) assert_raises(GrammarError, lambda: (x + many(p4)).parse(''))