Пример #1
0
    def test_36_unichars(self):
        grammar = '''
            start = { rule_positional | rule_keywords | rule_all }* $ ;

            rule_positional("ÄÖÜäöüß") = 'a' ;

            rule_keywords(k1='äöüÄÖÜß') = 'b' ;

            rule_all('ßÄÖÜäöü', k1="ßäöüÄÖÜ") = 'c' ;
        '''

        def _trydelete(pymodule):
            import os
            try:
                os.unlink(pymodule + ".py")
            except EnvironmentError:
                pass
            try:
                os.unlink(pymodule + ".pyc")
            except EnvironmentError:
                pass
            try:
                os.unlink(pymodule + ".pyo")
            except EnvironmentError:
                pass

        def assert_equal(target, value):
            self.assertEqual(target, value)

        class UnicharsSemantics(object):
            """Check all rule parameters for expected types and values"""
            def rule_positional(self, ast, p1):
                assert_equal("ÄÖÜäöüß", p1)
                return ast

            def rule_keyword(self, ast, k1):
                assert_equal("äöüÄÖÜß", k1)
                return ast

            def rule_all(self, ast, p1, k1):
                assert_equal("ßÄÖÜäöü", p1)
                assert_equal("ßäöüÄÖÜ", k1)
                return ast

        m = genmodel("UnicodeRuleArguments", grammar)
        ast = m.parse("a b c")
        self.assertEqual(['a', 'b', 'c'], ast)

        semantics = UnicharsSemantics()
        ast = m.parse("a b c", semantics=semantics)
        self.assertEqual(['a', 'b', 'c'], ast)

        code = codegen(m)
        import codecs
        with codecs.open("tc36unicharstest.py", "w", "utf-8") as f:
            f.write(code)
        import tc36unicharstest
        tc36unicharstest
        _trydelete("tc36unicharstest")
Пример #2
0
    def test_eol_comments_re_directive(self):
        grammar = '''
            @@eol_comments :: /#.*?$/

            test = "test" $;
        '''
        model = genmodel("test", grammar)
        code = codegen(model)
        compile(code, 'test.py', 'exec')
Пример #3
0
    def test_whitespace_directive(self):
        grammar = '''
            @@whitespace :: /[\t ]+/

            test = "test" $;
        '''
        model = genmodel("test", grammar)
        code = codegen(model)
        compile(code, 'test.py', 'exec')
Пример #4
0
    def test_failed_ref(self):
        grammar = """
            final = object;
            type = /[^\s=()]+/;
            object = '('type')' '{' @:{pair} {',' @:{pair}}* [','] '}';
            pair = key '=' value;
            list = '('type')' '[' @:{object} {',' @:{object}}* [','] ']';
            key = /[^\s=]+/;
            value = @:(string|list|object|unset|boolean|number|null) [','];
            null = '('type')' @:{ 'null' };
            boolean = /(true|false)/;
            unset = '<unset>';
            string = '"' @:/[^"]*/ '"';
            number = /-?[0-9]+/;
        """

        model = genmodel("final", grammar)
        codegen(model)
        model.parse('(sometype){boolean = true}')
Пример #5
0
    def test_left_recursion_directive(self):
        grammar = '''
            @@left_recursion :: False

            test = "test" $;
        '''
        model = genmodel("test", grammar)
        self.assertFalse(model.directives.get('left_recursion'))
        self.assertFalse(model.left_recursion)

        code = codegen(model)
        compile(code, 'test.py', 'exec')
Пример #6
0
 def test_keyword_params(self):
     grammar = '''
         start(k1=1, k2=2)
             =
             {'a'} $
             ;
     '''
     g = GrakoGrammarGenerator('Keywords')
     model = g.parse(grammar, trace=False)
     code = codegen(model)
     self.assertEquals('#!/usr/bin/env python', code.splitlines()[0])
     pass
Пример #7
0
    def test_bootstrap(self):
        print()

        if os.path.isfile('tmp/00.ast'):
            shutil.rmtree('tmp')
        if not os.path.isdir('tmp'):
            os.mkdir('tmp')
        print('-' * 20, 'phase 00 - parse using the bootstrap grammar')
        with open('etc/grako.ebnf') as f:
            text = str(f.read())
        g = GrakoParser('GrakoBootstrap')
        grammar0 = g.parse(text)
        ast0 = json.dumps(grammar0, indent=2)
        with open('tmp/00.ast', 'w') as f:
            f.write(ast0)

        print('-' * 20, 'phase 01 - parse with parser generator')
        with open('etc/grako.ebnf') as f:
            text = str(f.read())
        g = GrakoGrammarGenerator('GrakoBootstrap')
        g.parse(text, trace=False)

        generated_grammar1 = str(g.ast['grammar'])
        with open('tmp/01.ebnf', 'w') as f:
            f.write(generated_grammar1)

        print('-' * 20, 'phase 02 - parse previous output with the parser generator')
        with open('tmp/01.ebnf', 'r') as f:
            text = str(f.read())
        g = GrakoGrammarGenerator('GrakoBootstrap')
        g.parse(text, trace=False)
        generated_grammar2 = str(g.ast['grammar'])
        with open('tmp/02.ebnf', 'w') as f:
            f.write(generated_grammar2)
        self.assertEqual(generated_grammar2, generated_grammar1)

        print('-' * 20, 'phase 03 - repeat')
        with open('tmp/02.ebnf') as f:
            text = f.read()
        g = GrakoParser('GrakoBootstrap')
        ast3 = g.parse(text)
        with open('tmp/03.ast', 'w') as f:
            f.write(json.dumps(ast3, indent=2))

        print('-' * 20, 'phase 04 - repeat')
        with open('tmp/02.ebnf') as f:
            text = f.read()
        g = GrakoGrammarGenerator('GrakoBootstrap')
        g.parse(text)
        parser = g.ast['grammar']
    #    pprint(parser.first_sets, indent=2, depth=3)
        generated_grammar4 = str(parser)
        with open('tmp/04.ebnf', 'w') as f:
            f.write(generated_grammar4)
        self.assertEqual(generated_grammar4, generated_grammar2)

        print('-' * 20, 'phase 05 - parse using the grammar model')
        with open('tmp/04.ebnf') as f:
            text = f.read()
        ast5 = parser.parse(text)
        with open('tmp/05.ast', 'w') as f:
            f.write(json.dumps(ast5, indent=2))

        print('-' * 20, 'phase 06 - generate parser code')
        gencode6 = codegen(parser)
        with open('tmp/g06.py', 'w') as f:
            f.write(gencode6)

        print('-' * 20, 'phase 07 - import generated code')
        from g06 import GrakoBootstrapParser as GenParser  # @UnresolvedImport

        print('-' * 20, 'phase 08 - compile using generated code')
        parser = GenParser(trace=False)
        result = parser.parse(
            text,
            'grammar',
            comments_re=COMMENTS_RE,
            eol_comments_re=EOL_COMMENTS_RE
        )
        self.assertEqual(result, parser.ast['grammar'])
        ast8 = parser.ast['grammar']
        json8 = json.dumps(ast8, indent=2)
        open('tmp/08.ast', 'w').write(json8)
        self.assertEqual(ast5, ast8)

        print('-' * 20, 'phase 09 - Generate parser with semantics')
        with open('etc/grako.ebnf') as f:
            text = f.read()
        parser = GrakoGrammarGenerator('GrakoBootstrap')
        g9 = parser.parse(text)
        generated_grammar9 = str(g9)
        with open('tmp/09.ebnf', 'w') as f:
            f.write(generated_grammar9)
        self.assertEqual(generated_grammar9, generated_grammar1)

        print('-' * 20, 'phase 10 - Parse with a model using a semantics')
        g10 = g9.parse(
            text,
            start_rule='grammar',
            semantics=GrakoSemantics('GrakoBootstrap')
        )
        generated_grammar10 = str(g10)
        with open('tmp/10.ebnf', 'w') as f:
            f.write(generated_grammar10)
        gencode10 = codegen(g10)
        with open('tmp/g10.py', 'w') as f:
            f.write(gencode10)

        print('-' * 20, 'phase 11 - Pickle the model and try again.')
        with open('tmp/11.grako', 'wb') as f:
            pickle.dump(g10, f, protocol=2)
        with open('tmp/11.grako', 'rb') as f:
            g11 = pickle.load(f)
        r11 = g11.parse(
            text,
            start_rule='grammar',
            semantics=GrakoSemantics('GrakoBootstrap')
        )
        with open('tmp/11.ebnf', 'w') as f:
            f.write(str(g11))
        gencode11 = codegen(r11)
        with open('tmp/g11.py', 'w') as f:
            f.write(gencode11)

        print('-' * 20, 'phase 12 - Walker')

        class PrintNameWalker(DepthFirstWalker):
            def __init__(self):
                self.walked = []

            def walk_default(self, o, children):
                self.walked.append(o.__class__.__name__)

        v = PrintNameWalker()
        v.walk(g11)
        with open('tmp/12.txt', 'w') as f:
            f.write('\n'.join(v.walked))

        print('-' * 20, 'phase 13 - Graphics')
        try:
            from grako.diagrams import draw
        except ImportError:
            print('PyGraphViz not found!')
        else:
            draw('tmp/13.png', g11)
Пример #8
0
    def test_36_param_combinations(self):
        def assert_equal(target, value):
            self.assertEqual(target, value)

        class TC36Semantics(object):
            """Check all rule parameters for expected types and values"""
            def rule_positional(self, ast, p1, p2, p3, p4):
                assert_equal("ABC", p1)
                assert_equal(123, p2)
                assert_equal('=', p3)
                assert_equal("+", p4)
                return ast

            def rule_keyword(self, ast, k1, k2, k3, k4):
                assert_equal("ABC", k1)
                assert_equal(123, k2)
                assert_equal('=', k3)
                assert_equal('+', k4)
                return ast

            def rule_all(self, ast, p1, p2, p3, p4, k1, k2, k3, k4):
                assert_equal("DEF", p1)
                assert_equal(456, p2)
                assert_equal('=', p3)
                assert_equal("+", p4)
                assert_equal("HIJ", k1)
                assert_equal(789, k2)
                assert_equal('=', k3)
                assert_equal('+', k4)
                return ast

        grammar = '''
            start
                =
                {rule_positional | rule_keywords | rule_all} $
                ;


            rule_positional('ABC', 123, '=', '+')
                =
                'a'
                ;


            rule_keywords(k1=ABC, k3='=', k4='+', k2=123)
                =
                'b'
                ;


            rule_all('DEF', 456, '=', '+', k1=HIJ, k3='=', k4='+', k2=789)
                =
                'c'
                ;
        '''

        pretty = '''
            start
                =
                {rule_positional | rule_keywords | rule_all} $
                ;


            rule_positional(ABC, 123, '=', '+')
                =
                'a'
                ;


            rule_keywords(k1=ABC, k3='=', k4='+', k2=123)
                =
                'b'
                ;


            rule_all(DEF, 456, '=', '+', k1=HIJ, k3='=', k4='+', k2=789)
                =
                'c'
                ;
        '''

        model = genmodel('RuleArguments', grammar)
        self.assertEqual(trim(pretty), ustr(model))
        model = genmodel('RuleArguments', pretty)

        ast = model.parse("a b c")
        self.assertEqual(['a', 'b', 'c'], ast)
        semantics = TC36Semantics()
        ast = model.parse("a b c", semantics=semantics)
        self.assertEqual(['a', 'b', 'c'], ast)
        codegen(model)