def setUp(self): signed = uncurry(lambda ops, x: x * product(-1 for s in ops if s == '-')) g = GrammarBuilder() g.number = Regexp(r'\d+') >= float g.atom = g.number | flatten('(' + g.expr + ')' >= itemgetter(0)) g.signed = repeat(keep('+') | keep('-')) + g.atom >= signed g.product_expr = +repeat(g.signed, separator='*') >= product g.expr = +repeat(g.product_expr, separator='+') >= sum g.whitespace = builtins.horizontal_whitespace self.grammar = g(start=g.expr, tokenize=[ignore(g.whitespace)], drop_terminals=True)
def setUp(self): super().setUp() g = GrammarBuilder() g.number_literal = Regexp(r'-?(?:[1-9]\d*|0)(?:\.\d*)?(?:[eE][+-]?\d+)?') g.string_literal = Regexp(r'"(?:[^"]|\\(?:["\\nbfrt]|u[0-9a-fA-F]{4}))*"') g.array = '[' + flatten(repeat(g.expr, separator=drop(','))) + ']' g.object_ = '{' + flatten(repeat(flatten(g.string_literal + ':' + g.expr), separator=',')) + '}' g.expr = flatten(g.number_literal | g.string_literal | g.array | g.object_) g.whitespace = Regexp('\\s+') self.grammar = g(start=g.expr, tokenize=[ignore(g.whitespace)], drop_terminals=True)
def setUp(self): signed = uncurry( lambda ops, x: x * product(-1 for s in ops if s == '-')) g = GrammarBuilder() g.number = Regexp(r'\d+') >= float g.atom = g.number | flatten('(' + g.expr + ')' >= itemgetter(0)) g.signed = repeat(keep('+') | keep('-')) + g.atom >= signed g.product_expr = +repeat(g.signed, separator='*') >= product g.expr = +repeat(g.product_expr, separator='+') >= sum g.whitespace = builtins.horizontal_whitespace self.grammar = g(start=g.expr, tokenize=[ignore(g.whitespace)], drop_terminals=True)
def setUp(self): super().setUp() g = GrammarBuilder() g.number_literal = Regexp(r'-?(?:[1-9]\d*|0)(?:\.\d*)?(?:[eE][+-]?\d+)?') >= float g.string_literal = Regexp(r'"(?:[^"]|\\(?:["\\nbfrt]|u[0-9a-fA-F]{4}))*"') >= (lambda s: s[1:-1]) g.array = '[' + flatten(repeat(g.expr, separator=',')) + ']' >= list g.object_item = g.string_literal + ':' + g.expr >= tuple g.object_ = '{' + flatten(repeat(g.object_item, separator=',')) + '}' >= dict g.boolean = keep('true') | keep('false') >= (lambda s: s == 'true') g.null = keep('null') >= const(None) g.expr = flatten(g.number_literal | g.string_literal | g.array | g.object_ | g.boolean | g.null) g.whitespace = Regexp('\\s+') self.grammar = g(start=g.expr, tokenize=[ignore(g.whitespace)], drop_terminals=True)
def test_indention_tokenizer(self): g = GrammarBuilder() g.whitespace = Regexp(r'\s+') g.block = INDENT + repeat(g.expr) + DEDENT g.label = Regexp(r'\w+') g.expr = g.label + Optional(g.block) | g.label + '(' + g.expr + ')' grammar = g( start=g.expr, tokenize=[ indent('(', ')'), ignore(g.whitespace), ] ) format = GrammarFormatter() source = textwrap.dedent(""" foo bar baz boo x y """).strip() self.assertEqual( [t.lexeme or t.symbol.name for t in grammar.tokenize(source)], ['foo', 'INDENT', 'bar', 'baz', 'INDENT', 'boo', 'DEDENT', 'x', 'y', 'DEDENT'] ) source = textwrap.dedent(""" foo foo baz(arg) boo baz( arg ) boo baz(aaa bbb) xxx """).strip() self.assertEqual( [t.lexeme or t.symbol.name for t in grammar.tokenize(source)], [ 'foo', 'foo', 'INDENT', 'baz', '(', 'arg', ')', 'boo', 'INDENT', 'baz', '(', 'arg', ')', 'DEDENT', 'boo', 'INDENT', 'baz', '(', 'aaa', 'bbb', ')', 'xxx', 'DEDENT', 'DEDENT' ] )
def setUp(self): super().setUp() g = GrammarBuilder() g.number_literal = Regexp( r'-?(?:[1-9]\d*|0)(?:\.\d*)?(?:[eE][+-]?\d+)?') g.string_literal = Regexp( r'"(?:[^"]|\\(?:["\\nbfrt]|u[0-9a-fA-F]{4}))*"') g.array = '[' + flatten(repeat(g.expr, separator=drop(','))) + ']' g.object_ = '{' + flatten( repeat(flatten(g.string_literal + ':' + g.expr), separator=',')) + '}' g.expr = flatten(g.number_literal | g.string_literal | g.array | g.object_) g.whitespace = Regexp('\\s+') self.grammar = g(start=g.expr, tokenize=[ignore(g.whitespace)], drop_terminals=True)
def setUp(self): super().setUp() g = GrammarBuilder() g.number_literal = Regexp( r'-?(?:[1-9]\d*|0)(?:\.\d*)?(?:[eE][+-]?\d+)?') >= float g.string_literal = Regexp( r'"(?:[^"]|\\(?:["\\nbfrt]|u[0-9a-fA-F]{4}))*"') >= ( lambda s: s[1:-1]) g.array = '[' + flatten(repeat(g.expr, separator=',')) + ']' >= list g.object_item = g.string_literal + ':' + g.expr >= tuple g.object_ = '{' + flatten(repeat(g.object_item, separator=',')) + '}' >= dict g.boolean = keep('true') | keep('false') >= (lambda s: s == 'true') g.null = keep('null') >= const(None) g.expr = flatten(g.number_literal | g.string_literal | g.array | g.object_ | g.boolean | g.null) g.whitespace = Regexp('\\s+') self.grammar = g(start=g.expr, tokenize=[ignore(g.whitespace)], drop_terminals=True)
def setUp(self): super().setUp() g = GrammarBuilder() g.ab = Terminal('A') | Terminal('B') g.seq = repeat(g.ab, separator=',', trailing=True) self.grammar = g(start=g.seq)
def setUp(self): super().setUp() g = GrammarBuilder() g.ab = flatten(Terminal('A') | Terminal('B')) g.seq = repeat(g.ab) self.grammar = g(start=g.seq)
def setUp(self): super().setUp() g = GrammarBuilder() g.seq = repeat('A', separator=',', leading=True) self.grammar = g(start=g.seq)
def setUp(self): super().setUp() g = GrammarBuilder() g.ab = Terminal('A') | Terminal('B') g.seq = repeat(g.ab, separator=',', trailing=True) self.grammar = g(start=g.seq)
def setUp(self): super().setUp() g = GrammarBuilder() g.ab = flatten(Terminal('A') | Terminal('B')) g.seq = repeat(g.ab) self.grammar = g(start=g.seq)
def setUp(self): super().setUp() g = GrammarBuilder() g.seq = repeat('A', separator=',', leading=True) self.grammar = g(start=g.seq)