def setUp(self):
signed = uncurry(lambda ops, x: x * product(-1 for s in ops if s == '-'))
g = GrammarBuilder()
g.number = Regexp(r'\d+') >= float
g.atom = g.number | flatten('(' + g.expr + ')' >= itemgetter(0))
g.signed = repeat(keep('+') | keep('-')) + g.atom >= signed
g.product_expr = +repeat(g.signed, separator='*') >= product
g.expr = +repeat(g.product_expr, separator='+') >= sum
g.whitespace = builtins.horizontal_whitespace
self.grammar = g(start=g.expr, tokenize=[ignore(g.whitespace)], drop_terminals=True)
def setUp(self):
super().setUp()
g = GrammarBuilder()
g.number_literal = Regexp(r'-?(?:[1-9]\d*|0)(?:\.\d*)?(?:[eE][+-]?\d+)?')
g.string_literal = Regexp(r'"(?:[^"]|\\(?:["\\nbfrt]|u[0-9a-fA-F]{4}))*"')
g.array = '[' + flatten(repeat(g.expr, separator=drop(','))) + ']'
g.object_ = '{' + flatten(repeat(flatten(g.string_literal + ':' + g.expr), separator=',')) + '}'
g.expr = flatten(g.number_literal | g.string_literal | g.array | g.object_)
g.whitespace = Regexp('\\s+')
self.grammar = g(start=g.expr, tokenize=[ignore(g.whitespace)], drop_terminals=True)
def setUp(self):
signed = uncurry(
lambda ops, x: x * product(-1 for s in ops if s == '-'))
g = GrammarBuilder()
g.number = Regexp(r'\d+') >= float
g.atom = g.number | flatten('(' + g.expr + ')' >= itemgetter(0))
g.signed = repeat(keep('+') | keep('-')) + g.atom >= signed
g.product_expr = +repeat(g.signed, separator='*') >= product
g.expr = +repeat(g.product_expr, separator='+') >= sum
g.whitespace = builtins.horizontal_whitespace
self.grammar = g(start=g.expr,
tokenize=[ignore(g.whitespace)],
drop_terminals=True)
def setUp(self):
super().setUp()
g = GrammarBuilder()
g.number_literal = Regexp(r'-?(?:[1-9]\d*|0)(?:\.\d*)?(?:[eE][+-]?\d+)?') >= float
g.string_literal = Regexp(r'"(?:[^"]|\\(?:["\\nbfrt]|u[0-9a-fA-F]{4}))*"') >= (lambda s: s[1:-1])
g.array = '[' + flatten(repeat(g.expr, separator=',')) + ']' >= list
g.object_item = g.string_literal + ':' + g.expr >= tuple
g.object_ = '{' + flatten(repeat(g.object_item, separator=',')) + '}' >= dict
g.boolean = keep('true') | keep('false') >= (lambda s: s == 'true')
g.null = keep('null') >= const(None)
g.expr = flatten(g.number_literal | g.string_literal | g.array | g.object_ | g.boolean | g.null)
g.whitespace = Regexp('\\s+')
self.grammar = g(start=g.expr, tokenize=[ignore(g.whitespace)], drop_terminals=True)
def test_indention_tokenizer(self):
g = GrammarBuilder()
g.whitespace = Regexp(r'\s+')
g.block = INDENT + repeat(g.expr) + DEDENT
g.label = Regexp(r'\w+')
g.expr = g.label + Optional(g.block) | g.label + '(' + g.expr + ')'
grammar = g(
start=g.expr,
tokenize=[
indent('(', ')'),
ignore(g.whitespace),
]
)
format = GrammarFormatter()
source = textwrap.dedent("""
foo
bar
baz
boo
x
y
""").strip()
self.assertEqual(
[t.lexeme or t.symbol.name for t in grammar.tokenize(source)],
['foo', 'INDENT', 'bar', 'baz', 'INDENT', 'boo', 'DEDENT', 'x', 'y', 'DEDENT']
)
source = textwrap.dedent("""
foo
foo
baz(arg)
boo
baz(
arg
)
boo
baz(aaa
bbb)
xxx
""").strip()
self.assertEqual(
[t.lexeme or t.symbol.name for t in grammar.tokenize(source)],
[
'foo', 'foo', 'INDENT',
'baz', '(', 'arg', ')',
'boo', 'INDENT',
'baz', '(', 'arg', ')', 'DEDENT',
'boo', 'INDENT',
'baz', '(', 'aaa', 'bbb', ')', 'xxx', 'DEDENT',
'DEDENT'
]
)
def setUp(self):
super().setUp()
g = GrammarBuilder()
g.number_literal = Regexp(
r'-?(?:[1-9]\d*|0)(?:\.\d*)?(?:[eE][+-]?\d+)?')
g.string_literal = Regexp(
r'"(?:[^"]|\\(?:["\\nbfrt]|u[0-9a-fA-F]{4}))*"')
g.array = '[' + flatten(repeat(g.expr, separator=drop(','))) + ']'
g.object_ = '{' + flatten(
repeat(flatten(g.string_literal + ':' + g.expr),
separator=',')) + '}'
g.expr = flatten(g.number_literal | g.string_literal | g.array
| g.object_)
g.whitespace = Regexp('\\s+')
self.grammar = g(start=g.expr,
tokenize=[ignore(g.whitespace)],
drop_terminals=True)
def setUp(self):
super().setUp()
g = GrammarBuilder()
g.number_literal = Regexp(
r'-?(?:[1-9]\d*|0)(?:\.\d*)?(?:[eE][+-]?\d+)?') >= float
g.string_literal = Regexp(
r'"(?:[^"]|\\(?:["\\nbfrt]|u[0-9a-fA-F]{4}))*"') >= (
lambda s: s[1:-1])
g.array = '[' + flatten(repeat(g.expr, separator=',')) + ']' >= list
g.object_item = g.string_literal + ':' + g.expr >= tuple
g.object_ = '{' + flatten(repeat(g.object_item,
separator=',')) + '}' >= dict
g.boolean = keep('true') | keep('false') >= (lambda s: s == 'true')
g.null = keep('null') >= const(None)
g.expr = flatten(g.number_literal | g.string_literal | g.array
| g.object_ | g.boolean | g.null)
g.whitespace = Regexp('\\s+')
self.grammar = g(start=g.expr,
tokenize=[ignore(g.whitespace)],
drop_terminals=True)
def setUp(self):
super().setUp()
g = GrammarBuilder()
g.ab = Terminal('A') | Terminal('B')
g.seq = repeat(g.ab, separator=',', trailing=True)
self.grammar = g(start=g.seq)
def setUp(self):
super().setUp()
g = GrammarBuilder()
g.ab = flatten(Terminal('A') | Terminal('B'))
g.seq = repeat(g.ab)
self.grammar = g(start=g.seq)
def setUp(self):
super().setUp()
g = GrammarBuilder()
g.seq = repeat('A', separator=',', leading=True)
self.grammar = g(start=g.seq)
def setUp(self):
super().setUp()
g = GrammarBuilder()
g.ab = Terminal('A') | Terminal('B')
g.seq = repeat(g.ab, separator=',', trailing=True)
self.grammar = g(start=g.seq)
def setUp(self):
super().setUp()
g = GrammarBuilder()
g.ab = flatten(Terminal('A') | Terminal('B'))
g.seq = repeat(g.ab)
self.grammar = g(start=g.seq)
def setUp(self):
super().setUp()
g = GrammarBuilder()
g.seq = repeat('A', separator=',', leading=True)
self.grammar = g(start=g.seq)