def do_test(self, pattern, target, dfa_result, nfa_result, parser_factory):
alphabet = LineAwareAlphabet(UnicodeAlphabet.instance(), parser_factory)
compiler = Compiler.single(alphabet, pattern)
str(compiler.expression)
# text = str(compiler.expression)
# assert text == format('(?P<label>{0!s})', pattern), text
factory = LineAwareStreamFactory(alphabet)
target = factory.from_string(target)
dfa = compiler.dfa()
result = dfa.match(target)
if result:
(a, b, c) = result
(p, q, r) = dfa_result
assert a == p, result
assert b == q, result
assert_str(repr(c), r)
else:
assert dfa_result == None, dfa_result
nfa = compiler.nfa()
result = list(nfa.match(target))
assert len(result) == len(nfa_result), result
for ((a,b,c), (p,q,r)) in zip(result, nfa_result):
assert a == p, result
assert b == q, result
assert_str(repr(c), r)
def do_test(self, pattern, target, dfa_result, nfa_result, parser_factory):
alphabet = LineAwareAlphabet(UnicodeAlphabet.instance(),
parser_factory)
compiler = Compiler.single(alphabet, pattern)
str(compiler.expression)
# text = str(compiler.expression)
# assert text == format('(?P<label>{0!s})', pattern), text
factory = LineAwareStreamFactory(alphabet)
target = factory.from_string(target)
dfa = compiler.dfa()
result = dfa.match(target)
if result:
(a, b, c) = result
(p, q, r) = dfa_result
assert a == p, result
assert b == q, result
assert_str(repr(c), r)
else:
assert dfa_result == None, dfa_result
nfa = compiler.nfa()
result = list(nfa.match(target))
assert len(result) == len(nfa_result), result
for ((a, b, c), (p, q, r)) in zip(result, nfa_result):
assert a == p, result
assert b == q, result
assert_str(repr(c), r)
def test_node(self):
#basicConfig(level=DEBUG)
class Term(Node):
pass
class Factor(Node):
pass
class Expression(Node):
pass
expression = Delayed()
number = Digit()[1:, ...] > 'number'
term = (number | '(' / expression / ')') > Term
muldiv = Any('*/') > 'operator'
factor = (term / (muldiv / term)[0::]) > Factor
addsub = Any('+-') > 'operator'
expression += (factor / (addsub / factor)[0::]) > Expression
p = expression.get_parse_string()
ast = p('1 + 2 * (3 + 4 - 5)')
assert_str(
ast[0], """Expression
+- Factor
| +- Term
| | `- number '1'
| `- ' '
+- operator '+'
+- ' '
`- Factor
+- Term
| `- number '2'
+- ' '
+- operator '*'
+- ' '
`- Term
+- '('
+- Expression
| +- Factor
| | +- Term
| | | `- number '3'
| | `- ' '
| +- operator '+'
| +- ' '
| +- Factor
| | +- Term
| | | `- number '4'
| | `- ' '
| +- operator '-'
| +- ' '
| `- Factor
| `- Term
| `- number '5'
`- ')'""")
def examples(self, examples):
'''
Run each example and check expected against actual output.
'''
for (example, target) in examples:
try:
result = str(example())
except Exception as e:
getLogger('lepl._example.support.Example').debug(format_exc())
result = ''.join(format_exception_only(type(e), e))
assert_str(target, result)
def examples(self, examples):
'''
Run each example and check expected against actual output.
'''
for (example, target) in examples:
try:
result = str(example())
except Exception as e:
getLogger('lepl._example.support.Example').debug(format_exc())
result = ''.join(format_exception_only(type(e), e))
assert_str(target, result)
def test_right(self):
#CLine = ContinuedBLineFactory(Token(r'\\'))
expr0 = Token("[A-Za-z_][A-Za-z0-9_]*")
expr1 = Delayed()
call = expr1 & expr0 > List # Deliberately not expr0 & expr1
expr1 += call | Empty() | expr0
program = expr1 & Eos()
parsed = program.parse("a b c")
assert_str(parsed[0], """List
+- List
| +- 'a'
| `- 'b'
`- 'c'""")
def test_left(self):
CLine = ContinuedBLineFactory(Token(r'\\'))
expr0 = Token("[A-Za-z_][A-Za-z0-9_]*")
expr1 = Delayed()
call = expr1 & expr0 > List # Deliberately not expr0 & expr1
expr1 += call | Empty() | expr0
program = (CLine(expr1) & Eos())
program.config.default_line_aware(block_policy=rightmost)
parsed = program.parse("a b c")
assert_str(parsed[0], """List
+- List
| +- 'a'
| `- 'b'
`- 'c'""")
def test_context(self):
#basicConfig(level=DEBUG)
output = StringIO()
with TraceVariables(out=output):
bar = Any()
bar.config.no_full_first_match()
repr(bar)
list(bar.match('abc'))
text = output.getvalue()
assert_str(text, ''' bar = ['a'] stream = 'bc'
bar failed stream = 'abc'
''')
def test_right(self):
#CLine = ContinuedBLineFactory(Token(r'\\'))
expr0 = Token("[A-Za-z_][A-Za-z0-9_]*")
expr1 = Delayed()
call = expr1 & expr0 > List # Deliberately not expr0 & expr1
expr1 += call | Empty () | expr0
program = expr1 & Eos()
parsed = program.parse("a b c")
assert_str(parsed[0],
"""List
+- List
| +- 'a'
| `- 'b'
`- 'c'""")
def test_nested(self):
class Term(Node):
pass
class Factor(Node):
pass
class Expression(Node):
pass
expr = Delayed()
number = Digit()[1:, ...] > "number"
with Separator(Drop(Regexp(r"\s*"))):
term = number | "(" & expr & ")" > Term
muldiv = Any("*/") > "operator"
factor = (term & (muldiv & term)[:] > Node) > "factor"
addsub = Any("+-") > "operator"
expr += factor & (addsub & factor)[:] > Expression
line = expr & Eos()
ast = line.parse_string("1 + 2 * (3 + 4 - 5)")[0]
text = str(ast)
assert_str(
text,
"""Expression
+- factor
| `- Term
| `- number '1'
+- operator '+'
`- factor
+- Term
| `- number '2'
+- operator '*'
`- Term
+- '('
+- Expression
| +- factor
| | `- Term
| | `- number '3'
| +- operator '+'
| +- factor
| | `- Term
| | `- number '4'
| +- operator '-'
| `- factor
| `- Term
| `- number '5'
`- ')'""",
)
def run_test(self, name, text, parse, match2, match3, error,
config=lambda x: None, **kargs):
matcher = Any()[:, ...]
config(matcher)
parser = getattr(matcher, 'parse' + name)
result = str(parser(text, **kargs))
assert_str(result, parse)
matcher = Any()[2, ...]
matcher.config.no_full_first_match()
config(matcher)
parser = getattr(matcher, 'match' + name)
result = str(list(parser(text, **kargs)))
assert_str(result, match2)
matcher = Any()[3, ...]
matcher.config.no_full_first_match()
config(matcher)
parser = getattr(matcher, 'match' + name)
result = str(list(parser(text, **kargs)))
assert_str(result, match3)
matcher = Any()
config(matcher)
parser = getattr(matcher, 'parse' + name)
try:
parser(text)
except FullFirstMatchException as e:
assert_str(e, error)
def test_context(self):
#basicConfig(level=DEBUG)
output = StringIO()
with TraceVariables(out=output):
bar = Any()
bar.config.no_full_first_match()
repr(bar)
list(bar.match('abc'))
text = output.getvalue()
assert_str(
text,
''' bar = ['a'] stream = 'bc'
bar failed stream = 'abc'
''')
def test_node(self):
#basicConfig(level=DEBUG)
class Term(Node): pass
class Factor(Node): pass
class Expression(Node): pass
expression = Delayed()
number = Digit()[1:,...] > 'number'
term = (number | '(' / expression / ')') > Term
muldiv = Any('*/') > 'operator'
factor = (term / (muldiv / term)[0::]) > Factor
addsub = Any('+-') > 'operator'
expression += (factor / (addsub / factor)[0::]) > Expression
p = expression.get_parse_string()
ast = p('1 + 2 * (3 + 4 - 5)')
assert_str(ast[0], """Expression
+- Factor
| +- Term
| | `- number '1'
| `- ' '
+- operator '+'
+- ' '
`- Factor
+- Term
| `- number '2'
+- ' '
+- operator '*'
+- ' '
`- Term
+- '('
+- Expression
| +- Factor
| | +- Term
| | | `- number '3'
| | `- ' '
| +- operator '+'
| +- ' '
| +- Factor
| | +- Term
| | | `- number '4'
| | `- ' '
| +- operator '-'
| +- ' '
| `- Factor
| `- Term
| `- number '5'
`- ')'""")
def test_ast(self):
class Term(List):
pass
class Factor(List):
pass
class Expression(List):
pass
expr = Delayed()
number = Digit()[1:, ...] >> int
with Separator(Drop(Regexp(r'\s*'))):
term = number | '(' & expr & ')' > Term
muldiv = Any('*/')
factor = term & (muldiv & term)[:] > Factor
addsub = Any('+-')
expr += factor & (addsub & factor)[:] > Expression
line = expr & Eos()
ast = line.parse_string('1 + 2 * (3 + 4 - 5)')[0]
text = str(ast)
assert_str(
text, """Expression
+- Factor
| `- Term
| `- 1
+- '+'
`- Factor
+- Term
| `- 2
+- '*'
`- Term
+- '('
+- Expression
| +- Factor
| | `- Term
| | `- 3
| +- '+'
| +- Factor
| | `- Term
| | `- 4
| +- '-'
| `- Factor
| `- Term
| `- 5
`- ')'""")
def test_nested(self):
class Term(Node):
pass
class Factor(Node):
pass
class Expression(Node):
pass
expr = Delayed()
number = Digit()[1:, ...] > 'number'
with Separator(Drop(Regexp(r'\s*'))):
term = number | '(' & expr & ')' > Term
muldiv = Any('*/') > 'operator'
factor = (term & (muldiv & term)[:] > Node) > 'factor'
addsub = Any('+-') > 'operator'
expr += factor & (addsub & factor)[:] > Expression
line = expr & Eos()
ast = line.parse_string('1 + 2 * (3 + 4 - 5)')[0]
text = str(ast)
assert_str(
text, """Expression
+- factor
| `- Term
| `- number '1'
+- operator '+'
`- factor
+- Term
| `- number '2'
+- operator '*'
`- Term
+- '('
+- Expression
| +- factor
| | `- Term
| | `- number '3'
| +- operator '+'
| +- factor
| | `- Term
| | `- number '4'
| +- operator '-'
| `- factor
| `- Term
| `- number '5'
`- ')'""")
def test_left(self):
CLine = ContinuedBLineFactory(Token(r'\\'))
expr0 = Token("[A-Za-z_][A-Za-z0-9_]*")
expr1 = Delayed()
call = expr1 & expr0 > List # Deliberately not expr0 & expr1
expr1 += call | Empty () | expr0
program = (CLine(expr1) & Eos())
program.config.default_line_aware(block_policy=rightmost)
parsed = program.parse("a b c")
assert_str(parsed[0],
"""List
+- List
| +- 'a'
| `- 'b'
`- 'c'""")
def test_left(self):
CLine = ContinuedLineFactory(r'\\')
expr0 = Token("[A-Za-z_][A-Za-z0-9_]*")
expr1 = Delayed()
call = (expr1 & expr0) > List # Deliberately not expr0 & expr1
expr1 += (call | Empty() | expr0)
program = (CLine(expr1) & Eos())
program.config.lines(block_policy=explicit).auto_memoize()
parsed = program.parse("a b c")
assert_str(
parsed[0], """List
+- List
| +- List
| | `- 'a'
| `- 'b'
`- 'c'""")
def test_left(self):
CLine = ContinuedLineFactory(r'\\')
expr0 = Token("[A-Za-z_][A-Za-z0-9_]*")
expr1 = Delayed()
call = (expr1 & expr0) > List # Deliberately not expr0 & expr1
expr1 += (call | Empty () | expr0)
program = (CLine(expr1) & Eos())
program.config.lines(block_policy=explicit).auto_memoize()
parsed = program.parse("a b c")
assert_str(parsed[0],
"""List
+- List
| +- List
| | `- 'a'
| `- 'b'
`- 'c'""")
def test_ast(self):
class Term(List): pass
class Factor(List): pass
class Expression(List): pass
expr = Delayed()
number = Digit()[1:,...] >> int
with Separator(Drop(Regexp(r'\s*'))):
term = number | '(' & expr & ')' > Term
muldiv = Any('*/')
factor = term & (muldiv & term)[:] > Factor
addsub = Any('+-')
expr += factor & (addsub & factor)[:] > Expression
line = expr & Eos()
ast = line.parse_string('1 + 2 * (3 + 4 - 5)')[0]
text = str(ast)
assert_str(text, """Expression
+- Factor
| `- Term
| `- 1
+- '+'
`- Factor
+- Term
| `- 2
+- '*'
`- Term
+- '('
+- Expression
| +- Factor
| | `- Term
| | `- 3
| +- '+'
| +- Factor
| | `- Term
| | `- 4
| +- '-'
| `- Factor
| `- Term
| `- 5
`- ')'""")
def test_right_no_lexer(self):
#basicConfig(level=DEBUG)
word = Any()
expr1 = Delayed()
call = (expr1 & word) > List
expr1 += (call | Empty() | word)
program = expr1 & Eos()
program.config.trace_stack().auto_memoize()
parser = program.get_parse()
print(parser.matcher.tree())
parsed = parser("abc")
assert_str(parsed[0],
"""List
+- List
| +- List
| | `- 'a'
| `- 'b'
`- 'c'""")
def test_right_no_lexer(self):
#basicConfig(level=DEBUG)
word = Any()
expr1 = Delayed()
call = (expr1 & word) > List
expr1 += (call | Empty() | word)
program = expr1 & Eos()
program.config.trace_stack().auto_memoize()
parser = program.get_parse()
print(parser.matcher.tree())
parsed = parser("abc")
assert_str(
parsed[0], """List
+- List
| +- List
| | `- 'a'
| `- 'b'
`- 'c'""")
def test_wrapper(self):
output = StringIO()
matcher = NamedResult('foo', Any()[:], out=output)
repr(matcher)
matcher.config.clear()
parser = matcher.get_match_string()
list(parser('abc'))
text = output.getvalue()
assert_str(text, '''foo = ['a', 'b', 'c']
'abc' -> <EOS>
foo (2) = ['a', 'b']
'abc' -> 'c'
foo (3) = ['a']
'abc' -> 'bc'
foo (4) = []
'abc' -> 'abc'
! foo (after 4 matches)
'abc'
''')
def test_wrapper(self):
output = StringIO()
matcher = NamedResult('foo', Any()[:], out=output)
repr(matcher)
matcher.config.clear()
parser = matcher.get_match_string()
list(parser('abc'))
text = output.getvalue()
assert_str(text, '''foo = ['a', 'b', 'c']
"abc" -> ""
foo (2) = ['a', 'b']
"abc" -> "c"
foo (3) = ['a']
"abc" -> "bc"
foo (4) = []
"abc" -> "abc"
! foo (after 4 matches)
"abc"
''')
def test_nested(self):
class Term(Node): pass
class Factor(Node): pass
class Expression(Node): pass
expr = Delayed()
number = Digit()[1:,...] > 'number'
with Separator(Drop(Regexp(r'\s*'))):
term = number | '(' & expr & ')' > Term
muldiv = Any('*/') > 'operator'
factor = (term & (muldiv & term)[:] > Node) > 'factor'
addsub = Any('+-') > 'operator'
expr += factor & (addsub & factor)[:] > Expression
line = expr & Eos()
ast = line.parse_string('1 + 2 * (3 + 4 - 5)')[0]
text = str(ast)
assert_str(text, """Expression
+- factor
| `- Term
| `- number '1'
+- operator '+'
`- factor
+- Term
| `- number '2'
+- operator '*'
`- Term
+- '('
+- Expression
| +- factor
| | `- Term
| | `- number '3'
| +- operator '+'
| +- factor
| | `- Term
| | `- number '4'
| +- operator '-'
| `- factor
| `- Term
| `- number '5'
`- ')'""")
def test_right(self):
#basicConfig(level=DEBUG)
#CLine = ContinuedBLineFactory(Token(r'\\'))
word = Token("[A-Za-z_][A-Za-z0-9_]*")
expr1 = Delayed()
call = (expr1 & word) > List
expr1 += (call | Empty() | word)
program = Trace(expr1 & Eos())
program.config.trace_stack().auto_memoize()
parser = program.get_parse()
#print(parser.matcher.tree())
parsed = parser("a b c")
assert_str(parsed[0],
"""List
+- List
| +- List
| | `- 'a'
| `- 'b'
`- 'c'""")
def test_right(self):
#basicConfig(level=DEBUG)
#CLine = ContinuedBLineFactory(Token(r'\\'))
word = Token("[A-Za-z_][A-Za-z0-9_]*")
expr1 = Delayed()
call = (expr1 & word) > List
expr1 += (call | Empty() | word)
program = Trace(expr1 & Eos())
program.config.trace_stack().auto_memoize()
parser = program.get_parse()
#print(parser.matcher.tree())
parsed = parser("a b c")
assert_str(
parsed[0], """List
+- List
| +- List
| | `- 'a'
| `- 'b'
`- 'c'""")
def test_wrapper(self):
output = StringIO()
matcher = NamedResult("foo", Any()[:], out=output)
repr(matcher)
matcher.config.clear()
parser = matcher.get_match_string()
list(parser("abc"))
text = output.getvalue()
assert_str(
text,
"""foo = ['a', 'b', 'c']
'abc' -> <EOS>
foo (2) = ['a', 'b']
'abc' -> 'c'
foo (3) = ['a']
'abc' -> 'bc'
foo (4) = []
'abc' -> 'abc'
! foo (after 4 matches)
'abc'
""",
)
def assert_same(self, text1, text2):
assert_str(text1, text2)