Example #1
0
    def do_test(self, pattern, target, dfa_result, nfa_result, parser_factory):
        alphabet = LineAwareAlphabet(UnicodeAlphabet.instance(), parser_factory)
        compiler = Compiler.single(alphabet, pattern)
        str(compiler.expression)
#        text = str(compiler.expression)
#        assert text == format('(?P<label>{0!s})', pattern), text
        
        factory = LineAwareStreamFactory(alphabet)
        target = factory.from_string(target)
        
        dfa = compiler.dfa()
        result = dfa.match(target)
        if result:
            (a, b, c) = result
            (p, q, r) = dfa_result
            assert a == p, result
            assert b == q, result
            assert_str(repr(c), r)
        else:
            assert dfa_result == None, dfa_result

        nfa = compiler.nfa()
        result = list(nfa.match(target))
        assert len(result) == len(nfa_result), result
        for ((a,b,c), (p,q,r)) in zip(result, nfa_result):
            assert a == p, result
            assert b == q, result
            assert_str(repr(c), r)
Example #2
0
    def do_test(self, pattern, target, dfa_result, nfa_result, parser_factory):
        alphabet = LineAwareAlphabet(UnicodeAlphabet.instance(),
                                     parser_factory)
        compiler = Compiler.single(alphabet, pattern)
        str(compiler.expression)
        #        text = str(compiler.expression)
        #        assert text == format('(?P<label>{0!s})', pattern), text

        factory = LineAwareStreamFactory(alphabet)
        target = factory.from_string(target)

        dfa = compiler.dfa()
        result = dfa.match(target)
        if result:
            (a, b, c) = result
            (p, q, r) = dfa_result
            assert a == p, result
            assert b == q, result
            assert_str(repr(c), r)
        else:
            assert dfa_result == None, dfa_result

        nfa = compiler.nfa()
        result = list(nfa.match(target))
        assert len(result) == len(nfa_result), result
        for ((a, b, c), (p, q, r)) in zip(result, nfa_result):
            assert a == p, result
            assert b == q, result
            assert_str(repr(c), r)
Example #3
0
    def test_node(self):
        #basicConfig(level=DEBUG)

        class Term(Node):
            pass

        class Factor(Node):
            pass

        class Expression(Node):
            pass

        expression = Delayed()
        number = Digit()[1:, ...] > 'number'
        term = (number | '(' / expression / ')') > Term
        muldiv = Any('*/') > 'operator'
        factor = (term / (muldiv / term)[0::]) > Factor
        addsub = Any('+-') > 'operator'
        expression += (factor / (addsub / factor)[0::]) > Expression

        p = expression.get_parse_string()
        ast = p('1 + 2 * (3 + 4 - 5)')
        assert_str(
            ast[0], """Expression
 +- Factor
 |   +- Term
 |   |   `- number '1'
 |   `- ' '
 +- operator '+'
 +- ' '
 `- Factor
     +- Term
     |   `- number '2'
     +- ' '
     +- operator '*'
     +- ' '
     `- Term
         +- '('
         +- Expression
         |   +- Factor
         |   |   +- Term
         |   |   |   `- number '3'
         |   |   `- ' '
         |   +- operator '+'
         |   +- ' '
         |   +- Factor
         |   |   +- Term
         |   |   |   `- number '4'
         |   |   `- ' '
         |   +- operator '-'
         |   +- ' '
         |   `- Factor
         |       `- Term
         |           `- number '5'
         `- ')'""")
Example #4
0
 def examples(self, examples):
     '''
     Run each example and check expected against actual output. 
     '''
     for (example, target) in examples:
         try:
             result = str(example())
         except Exception as e:
             getLogger('lepl._example.support.Example').debug(format_exc())
             result = ''.join(format_exception_only(type(e), e))
         assert_str(target, result)
Example #5
0
 def examples(self, examples):
     '''
     Run each example and check expected against actual output. 
     '''
     for (example, target) in examples:
         try:
             result = str(example())
         except Exception as e:
             getLogger('lepl._example.support.Example').debug(format_exc())
             result = ''.join(format_exception_only(type(e), e))
         assert_str(target, result)
Example #6
0
    def test_right(self):
        #CLine = ContinuedBLineFactory(Token(r'\\'))
        expr0 = Token("[A-Za-z_][A-Za-z0-9_]*")
        expr1 = Delayed()
        call = expr1 & expr0 > List  # Deliberately not expr0 & expr1
        expr1 += call | Empty() | expr0
        program = expr1 & Eos()
        parsed = program.parse("a b c")
        assert_str(parsed[0], """List
 +- List
 |   +- 'a'
 |   `- 'b'
 `- 'c'""")
Example #7
0
    def test_left(self):
        CLine = ContinuedBLineFactory(Token(r'\\'))
        expr0 = Token("[A-Za-z_][A-Za-z0-9_]*")
        expr1 = Delayed()
        call = expr1 & expr0 > List  # Deliberately not expr0 & expr1
        expr1 += call | Empty() | expr0
        program = (CLine(expr1) & Eos())
        program.config.default_line_aware(block_policy=rightmost)
        parsed = program.parse("a b c")
        assert_str(parsed[0], """List
 +- List
 |   +- 'a'
 |   `- 'b'
 `- 'c'""")
Example #8
0
    def test_context(self):
        #basicConfig(level=DEBUG)
        output = StringIO()
        with TraceVariables(out=output):
            bar = Any()
        bar.config.no_full_first_match()
        repr(bar)
        list(bar.match('abc'))
        text = output.getvalue()
        assert_str(text, '''         bar = ['a']                            stream = 'bc'
         bar failed                             stream = 'abc'
''')

        
Example #9
0
    def test_right(self):
        #CLine = ContinuedBLineFactory(Token(r'\\'))
        expr0 = Token("[A-Za-z_][A-Za-z0-9_]*")
        expr1 = Delayed()
        call = expr1 & expr0 > List # Deliberately not expr0 & expr1
        expr1 += call | Empty () | expr0
        program = expr1 & Eos()
        parsed = program.parse("a b c")
        assert_str(parsed[0],
"""List
 +- List
 |   +- 'a'
 |   `- 'b'
 `- 'c'""")
Example #10
0
    def test_nested(self):
        class Term(Node):
            pass

        class Factor(Node):
            pass

        class Expression(Node):
            pass

        expr = Delayed()
        number = Digit()[1:, ...] > "number"

        with Separator(Drop(Regexp(r"\s*"))):
            term = number | "(" & expr & ")" > Term
            muldiv = Any("*/") > "operator"
            factor = (term & (muldiv & term)[:] > Node) > "factor"
            addsub = Any("+-") > "operator"
            expr += factor & (addsub & factor)[:] > Expression
            line = expr & Eos()

        ast = line.parse_string("1 + 2 * (3 + 4 - 5)")[0]
        text = str(ast)
        assert_str(
            text,
            """Expression
 +- factor
 |   `- Term
 |       `- number '1'
 +- operator '+'
 `- factor
     +- Term
     |   `- number '2'
     +- operator '*'
     `- Term
         +- '('
         +- Expression
         |   +- factor
         |   |   `- Term
         |   |       `- number '3'
         |   +- operator '+'
         |   +- factor
         |   |   `- Term
         |   |       `- number '4'
         |   +- operator '-'
         |   `- factor
         |       `- Term
         |           `- number '5'
         `- ')'""",
        )
Example #11
0
 def run_test(self, name, text, parse, match2, match3, error, 
              config=lambda x: None, **kargs):
     matcher = Any()[:, ...]
     config(matcher)
     parser = getattr(matcher, 'parse' + name)
     result = str(parser(text, **kargs))
     assert_str(result, parse)
     matcher = Any()[2, ...]
     matcher.config.no_full_first_match()
     config(matcher)
     parser = getattr(matcher, 'match' + name)
     result = str(list(parser(text, **kargs)))
     assert_str(result, match2)
     matcher = Any()[3, ...]
     matcher.config.no_full_first_match()
     config(matcher)
     parser = getattr(matcher, 'match' + name)
     result = str(list(parser(text, **kargs)))
     assert_str(result, match3)
     matcher = Any()
     config(matcher)
     parser = getattr(matcher, 'parse' + name)
     try:
         parser(text)
     except FullFirstMatchException as e:
         assert_str(e, error)
Example #12
0
    def test_context(self):
        #basicConfig(level=DEBUG)
        output = StringIO()
        with TraceVariables(out=output):
            bar = Any()
        bar.config.no_full_first_match()
        repr(bar)
        list(bar.match('abc'))
        text = output.getvalue()
        assert_str(
            text,
            '''         bar = ['a']                            stream = 'bc'
         bar failed                             stream = 'abc'
''')
Example #13
0
    def test_node(self):
        #basicConfig(level=DEBUG)
        
        class Term(Node): pass
        class Factor(Node): pass
        class Expression(Node): pass

        expression  = Delayed()
        number      = Digit()[1:,...]                      > 'number'
        term        = (number | '(' / expression / ')')    > Term
        muldiv      = Any('*/')                            > 'operator'
        factor      = (term / (muldiv / term)[0::])        > Factor
        addsub      = Any('+-')                            > 'operator'
        expression += (factor / (addsub / factor)[0::])    > Expression
        
        p = expression.get_parse_string()
        ast = p('1 + 2 * (3 + 4 - 5)')
        assert_str(ast[0], """Expression
 +- Factor
 |   +- Term
 |   |   `- number '1'
 |   `- ' '
 +- operator '+'
 +- ' '
 `- Factor
     +- Term
     |   `- number '2'
     +- ' '
     +- operator '*'
     +- ' '
     `- Term
         +- '('
         +- Expression
         |   +- Factor
         |   |   +- Term
         |   |   |   `- number '3'
         |   |   `- ' '
         |   +- operator '+'
         |   +- ' '
         |   +- Factor
         |   |   +- Term
         |   |   |   `- number '4'
         |   |   `- ' '
         |   +- operator '-'
         |   +- ' '
         |   `- Factor
         |       `- Term
         |           `- number '5'
         `- ')'""")
Example #14
0
    def test_ast(self):
        class Term(List):
            pass

        class Factor(List):
            pass

        class Expression(List):
            pass

        expr = Delayed()
        number = Digit()[1:, ...] >> int

        with Separator(Drop(Regexp(r'\s*'))):
            term = number | '(' & expr & ')' > Term
            muldiv = Any('*/')
            factor = term & (muldiv & term)[:] > Factor
            addsub = Any('+-')
            expr += factor & (addsub & factor)[:] > Expression
            line = expr & Eos()

        ast = line.parse_string('1 + 2 * (3 + 4 - 5)')[0]
        text = str(ast)
        assert_str(
            text, """Expression
 +- Factor
 |   `- Term
 |       `- 1
 +- '+'
 `- Factor
     +- Term
     |   `- 2
     +- '*'
     `- Term
         +- '('
         +- Expression
         |   +- Factor
         |   |   `- Term
         |   |       `- 3
         |   +- '+'
         |   +- Factor
         |   |   `- Term
         |   |       `- 4
         |   +- '-'
         |   `- Factor
         |       `- Term
         |           `- 5
         `- ')'""")
Example #15
0
    def test_nested(self):
        class Term(Node):
            pass

        class Factor(Node):
            pass

        class Expression(Node):
            pass

        expr = Delayed()
        number = Digit()[1:, ...] > 'number'

        with Separator(Drop(Regexp(r'\s*'))):
            term = number | '(' & expr & ')' > Term
            muldiv = Any('*/') > 'operator'
            factor = (term & (muldiv & term)[:] > Node) > 'factor'
            addsub = Any('+-') > 'operator'
            expr += factor & (addsub & factor)[:] > Expression
            line = expr & Eos()

        ast = line.parse_string('1 + 2 * (3 + 4 - 5)')[0]
        text = str(ast)
        assert_str(
            text, """Expression
 +- factor
 |   `- Term
 |       `- number '1'
 +- operator '+'
 `- factor
     +- Term
     |   `- number '2'
     +- operator '*'
     `- Term
         +- '('
         +- Expression
         |   +- factor
         |   |   `- Term
         |   |       `- number '3'
         |   +- operator '+'
         |   +- factor
         |   |   `- Term
         |   |       `- number '4'
         |   +- operator '-'
         |   `- factor
         |       `- Term
         |           `- number '5'
         `- ')'""")
Example #16
0
    def test_left(self):
        CLine = ContinuedBLineFactory(Token(r'\\'))
        expr0 = Token("[A-Za-z_][A-Za-z0-9_]*")
        expr1 = Delayed()
        call = expr1 & expr0 > List # Deliberately not expr0 & expr1
        expr1 += call | Empty () | expr0
        program = (CLine(expr1) & Eos())
        program.config.default_line_aware(block_policy=rightmost)
        parsed = program.parse("a b c")
        assert_str(parsed[0],
"""List
 +- List
 |   +- 'a'
 |   `- 'b'
 `- 'c'""")
        
Example #17
0
    def test_left(self):
        CLine = ContinuedLineFactory(r'\\')
        expr0 = Token("[A-Za-z_][A-Za-z0-9_]*")
        expr1 = Delayed()
        call = (expr1 & expr0) > List  # Deliberately not expr0 & expr1
        expr1 += (call | Empty() | expr0)
        program = (CLine(expr1) & Eos())
        program.config.lines(block_policy=explicit).auto_memoize()
        parsed = program.parse("a b c")
        assert_str(
            parsed[0], """List
 +- List
 |   +- List
 |   |   `- 'a'
 |   `- 'b'
 `- 'c'""")
Example #18
0
    def test_left(self):
        CLine = ContinuedLineFactory(r'\\')
        expr0 = Token("[A-Za-z_][A-Za-z0-9_]*")
        expr1 = Delayed()
        call = (expr1 & expr0) > List # Deliberately not expr0 & expr1
        expr1 += (call | Empty () | expr0)
        program = (CLine(expr1) & Eos())
        program.config.lines(block_policy=explicit).auto_memoize()
        parsed = program.parse("a b c")
        assert_str(parsed[0],
"""List
 +- List
 |   +- List
 |   |   `- 'a'
 |   `- 'b'
 `- 'c'""")
        
Example #19
0
    def test_ast(self):
        
        class Term(List): pass
        class Factor(List): pass
        class Expression(List): pass
            
        expr   = Delayed()
        number = Digit()[1:,...]                         >> int
        
        with Separator(Drop(Regexp(r'\s*'))):
            term    = number | '(' & expr & ')'          > Term
            muldiv  = Any('*/')
            factor  = term & (muldiv & term)[:]          > Factor
            addsub  = Any('+-')
            expr   += factor & (addsub & factor)[:]      > Expression
            line    = expr & Eos()
            
        ast = line.parse_string('1 + 2 * (3 + 4 - 5)')[0]
        text = str(ast)
        assert_str(text, """Expression
 +- Factor
 |   `- Term
 |       `- 1
 +- '+'
 `- Factor
     +- Term
     |   `- 2
     +- '*'
     `- Term
         +- '('
         +- Expression
         |   +- Factor
         |   |   `- Term
         |   |       `- 3
         |   +- '+'
         |   +- Factor
         |   |   `- Term
         |   |       `- 4
         |   +- '-'
         |   `- Factor
         |       `- Term
         |           `- 5
         `- ')'""")

        
Example #20
0
    def test_right_no_lexer(self):
        #basicConfig(level=DEBUG)
        word = Any()
        expr1 = Delayed()
        call = (expr1 & word) > List
        expr1 += (call | Empty() | word)
        program = expr1 & Eos()
        program.config.trace_stack().auto_memoize()
        parser = program.get_parse()
        print(parser.matcher.tree())
        parsed = parser("abc")
        assert_str(parsed[0],
"""List
 +- List
 |   +- List
 |   |   `- 'a'
 |   `- 'b'
 `- 'c'""")
Example #21
0
    def test_right_no_lexer(self):
        #basicConfig(level=DEBUG)
        word = Any()
        expr1 = Delayed()
        call = (expr1 & word) > List
        expr1 += (call | Empty() | word)
        program = expr1 & Eos()
        program.config.trace_stack().auto_memoize()
        parser = program.get_parse()
        print(parser.matcher.tree())
        parsed = parser("abc")
        assert_str(
            parsed[0], """List
 +- List
 |   +- List
 |   |   `- 'a'
 |   `- 'b'
 `- 'c'""")
Example #22
0
    def test_wrapper(self):
        output = StringIO()
        matcher = NamedResult('foo', Any()[:], out=output)
        repr(matcher)
        matcher.config.clear()
        parser = matcher.get_match_string()
        list(parser('abc'))
        text = output.getvalue()
        assert_str(text, '''foo = ['a', 'b', 'c']
    'abc' -> <EOS>
foo (2) = ['a', 'b']
    'abc' -> 'c'
foo (3) = ['a']
    'abc' -> 'bc'
foo (4) = []
    'abc' -> 'abc'
! foo (after 4 matches)
    'abc'
''')
Example #23
0
    def test_wrapper(self):
        output = StringIO()
        matcher = NamedResult('foo', Any()[:], out=output)
        repr(matcher)
        matcher.config.clear()
        parser = matcher.get_match_string()
        list(parser('abc'))
        text = output.getvalue()
        assert_str(text, '''foo = ['a', 'b', 'c']
    "abc" -> ""
foo (2) = ['a', 'b']
    "abc" -> "c"
foo (3) = ['a']
    "abc" -> "bc"
foo (4) = []
    "abc" -> "abc"
! foo (after 4 matches)
    "abc"
''')
Example #24
0
    def test_nested(self):
        
        class Term(Node): pass
        class Factor(Node): pass
        class Expression(Node): pass
            
        expr   = Delayed()
        number = Digit()[1:,...]                         > 'number'
        
        with Separator(Drop(Regexp(r'\s*'))):
            term    = number | '(' & expr & ')'          > Term
            muldiv  = Any('*/')                          > 'operator'
            factor  = (term & (muldiv & term)[:] > Node) > 'factor'
            addsub  = Any('+-')                          > 'operator'
            expr   += factor & (addsub & factor)[:]      > Expression
            line    = expr & Eos()
            
        ast = line.parse_string('1 + 2 * (3 + 4 - 5)')[0]
        text = str(ast)
        assert_str(text, """Expression
 +- factor
 |   `- Term
 |       `- number '1'
 +- operator '+'
 `- factor
     +- Term
     |   `- number '2'
     +- operator '*'
     `- Term
         +- '('
         +- Expression
         |   +- factor
         |   |   `- Term
         |   |       `- number '3'
         |   +- operator '+'
         |   +- factor
         |   |   `- Term
         |   |       `- number '4'
         |   +- operator '-'
         |   `- factor
         |       `- Term
         |           `- number '5'
         `- ')'""")
Example #25
0
    def test_right(self):
        #basicConfig(level=DEBUG)
        #CLine = ContinuedBLineFactory(Token(r'\\'))
        word = Token("[A-Za-z_][A-Za-z0-9_]*")
        expr1 = Delayed()
        call = (expr1 & word) > List
        expr1 += (call | Empty() | word)
        program = Trace(expr1 & Eos())
        program.config.trace_stack().auto_memoize()
        parser = program.get_parse()
        #print(parser.matcher.tree())
        parsed = parser("a b c")
        assert_str(parsed[0],
"""List
 +- List
 |   +- List
 |   |   `- 'a'
 |   `- 'b'
 `- 'c'""")
Example #26
0
    def test_right(self):
        #basicConfig(level=DEBUG)
        #CLine = ContinuedBLineFactory(Token(r'\\'))
        word = Token("[A-Za-z_][A-Za-z0-9_]*")
        expr1 = Delayed()
        call = (expr1 & word) > List
        expr1 += (call | Empty() | word)
        program = Trace(expr1 & Eos())
        program.config.trace_stack().auto_memoize()
        parser = program.get_parse()
        #print(parser.matcher.tree())
        parsed = parser("a b c")
        assert_str(
            parsed[0], """List
 +- List
 |   +- List
 |   |   `- 'a'
 |   `- 'b'
 `- 'c'""")
Example #27
0
    def test_wrapper(self):
        output = StringIO()
        matcher = NamedResult("foo", Any()[:], out=output)
        repr(matcher)
        matcher.config.clear()
        parser = matcher.get_match_string()
        list(parser("abc"))
        text = output.getvalue()
        assert_str(
            text,
            """foo = ['a', 'b', 'c']
    'abc' -> <EOS>
foo (2) = ['a', 'b']
    'abc' -> 'c'
foo (3) = ['a']
    'abc' -> 'bc'
foo (4) = []
    'abc' -> 'abc'
! foo (after 4 matches)
    'abc'
""",
        )
Example #28
0
 def assert_same(self, text1, text2):
     assert_str(text1, text2)