コード例 #1
0
ファイル: grammars.py プロジェクト: brigittebigi/sppas
    def __str__(self):
        comments = self.comments_str()
        params = ', '.join(
            self.param_repr(p) for p in self.params
        ) if self.params else ''

        kwparams = ''
        if self.kwparams:
            kwparams = ', '.join(
                '%s=%s' % (k, self.param_repr(v)) for (k, v)
                in self.kwparams.items()
            )

        if params and kwparams:
            params = '(%s, %s)' % (params, kwparams)
        elif kwparams:
                params = '(%s)' % (kwparams)
        elif params:
            params = '(%s)' % params

        base = ' < %s' % ustr(self.base.name) if self.base else ''

        return trim(self.str_template).format(
            name=self.name,
            base=base,
            params=params,
            exp=indent(str(self.exp)),
            comments=comments
        )
コード例 #2
0
    def test_numbers_and_unicode(self):
        grammar = '''
            rúle(1, -23, 4.56, 7.89e-11, 0xABCDEF, Añez)
                =
                'a'
                ;
        '''
        rule2 = '''

            rulé(Añez)
                =
                '\\xf1'
                ;
        '''
        rule3 = '''

            rúlé(Añez)
                =
                'ñ'
                ;
        '''
        if PY3:
            grammar += rule3
        else:
            grammar += rule2

        model = genmodel("test", grammar)
        self.assertEquals(trim(grammar), ustr(model))
コード例 #3
0
ファイル: grammar_test.py プロジェクト: brigittebigi/sppas
    def test_numbers_and_unicode(self):
        grammar = '''
            rúle(1, -23, 4.56, 7.89e-11, 0xABCDEF, Añez)
                =
                'a'
                ;
        '''
        rule2 = '''

            rulé(Añez)
                =
                '\\xf1'
                ;
        '''
        rule3 = '''

            rúlé(Añez)
                =
                'ñ'
                ;
        '''
        if PY3:
            grammar += rule3
        else:
            grammar += rule2

        model = genmodel("test", grammar)
        self.assertEquals(trim(grammar), ustr(model))
コード例 #4
0
ファイル: parsing_test.py プロジェクト: lym0302/sppas
 def test_include(self):
     text = '''\
         first
             #include :: "something"
         last\
     '''
     buf = MockIncludeBuffer(trim(text))
     self.assertEqual('first\n\nINCLUDED "something"\nlast', buf.text)
コード例 #5
0
ファイル: parsing_test.py プロジェクト: brigittebigi/sppas
 def test_include(self):
     text = '''\
         first
             #include :: "something"
         last\
     '''
     buf = MockIncludeBuffer(trim(text))
     self.assertEqual('first\n\nINCLUDED "something"\nlast', buf.text)
コード例 #6
0
ファイル: grammars.py プロジェクト: brigittebigi/sppas
 def __str__(self):
     exp = ustr(self.exp)
     template = '[%s]'
     if isinstance(self.exp, Choice):
         template = trim(self.str_template)
     elif isinstance(self.exp, Group):
         exp = self.exp.exp
     return template % exp
コード例 #7
0
ファイル: grammar_test.py プロジェクト: brigittebigi/sppas
 def test_36_params_and_keyword_params(self):
     grammar = '''
         rule(A, kwdB=B)
             =
             'a'
             ;
     '''
     model = genmodel("test", grammar)
     self.assertEquals(trim(grammar), ustr(model))
コード例 #8
0
 def test_36_params_and_keyword_params(self):
     grammar = '''
         rule(A, kwdB=B)
             =
             'a'
             ;
     '''
     model = genmodel("test", grammar)
     self.assertEquals(trim(grammar), ustr(model))
コード例 #9
0
ファイル: python.py プロジェクト: brigittebigi/sppas
 def render_fields(self, fields):
     template = trim(self.option_template)
     options = [template.format(option=indent(self.rend(o))) for o in self.node.options]
     options = "\n".join(o for o in options)
     firstset = " ".join(f[0] for f in sorted(self.node.firstset) if f)
     if firstset:
         error = "expecting one of: " + firstset
     else:
         error = "no available options"
     fields.update(n=self.counter(), options=indent(options), error=urepr(error))
コード例 #10
0
ファイル: grammar_test.py プロジェクト: brigittebigi/sppas
    def test_whitespace_no_newlines(self):
        grammar = """
            @@whitespace :: /[\t ]+/
            # this is just a token with any character but space and newline
            # it should finish before it capture space or newline character
            token = /[^ \n]+/;
            # expect whitespace to capture spaces between tokens, but newline should be captured afterwards
            token2 = {token}* /\n/;
            # document is just list of this strings of tokens
            document = {@+:token2}* $;
        """
        text = trim("""\
            a b
            c d
            e f
        """)

        expected = [
            [
                [
                    "a",
                    "b"
                ],
                "\n"
            ],
            [
                [
                    "c",
                    "d"
                ],
                "\n"
            ],
            [
                [
                    "e",
                    "f"
                ],
                "\n"
            ]
        ]

        model = genmodel("document", grammar)
        ast = model.parse(text, start='document')
        self.assertEqual(expected, ast)
コード例 #11
0
ファイル: rendering.py プロジェクト: brigittebigi/sppas
    def render(self, template=None, **fields):
        fields.update(__class__=self.__class__.__name__)
        fields.update({k: v for k, v in vars(self).items() if not k.startswith('_')})

        override = self.render_fields(fields)
        if override is not None:
            template = override
        elif template is None:
            template = self.template

        try:
            return self._formatter.format(trim(template), **fields)
        except KeyError:
            # find the missing key
            keys = (p[1] for p in self._formatter.parse(template))
            for key in keys:
                if key and key not in fields:
                    raise KeyError(key, type(self))
            raise
コード例 #12
0
ファイル: python.py プロジェクト: brigittebigi/sppas
    def render_fields(self, fields):
        self.reset_counter()

        params = kwparams = ""
        if self.node.params:
            params = ", ".join(self.param_repr(self.rend(p)) for p in self.node.params)
        if self.node.kwparams:
            kwparams = ", ".join("%s=%s" % (k, self.param_repr(self.rend(v))) for k, v in self.kwparams.items())

        if params and kwparams:
            params = params + ", " + kwparams
        elif kwparams:
            params = kwparams

        fields.update(params=params)

        defines = compress_seq(self.defines())
        sdefs = [d for d, l in defines if not l]
        ldefs = [d for d, l in defines if l]
        if not (sdefs or ldefs):
            sdefines = ""
        else:
            sdefs = "[%s]" % ", ".join(urepr(d) for d in sdefs)
            ldefs = "[%s]" % ", ".join(urepr(d) for d in ldefs)
            if not ldefs:
                sdefines = "\n\n    self.ast._define(%s, %s)" % (sdefs, ldefs)
            else:
                sdefines = indent(
                    "\n\n"
                    + trim(
                        """\
                                                self.ast._define(
                                                    %s,
                                                    %s
                                                )"""
                        % (sdefs, ldefs)
                    )
                )

        fields.update(defines=sdefines)
コード例 #13
0
    def test_whitespace_no_newlines(self):
        grammar = """
            @@whitespace :: /[\t ]+/
            # this is just a token with any character but space and newline
            # it should finish before it capture space or newline character
            token = /[^ \n]+/;
            # expect whitespace to capture spaces between tokens, but newline should be captured afterwards
            token2 = {token}* /\n/;
            # document is just list of this strings of tokens
            document = {@+:token2}* $;
        """
        text = trim("""\
            a b
            c d
            e f
        """)

        expected = [[["a", "b"], "\n"], [["c", "d"], "\n"], [["e", "f"], "\n"]]

        model = genmodel("document", grammar)
        ast = model.parse(text, start='document')
        self.assertEqual(expected, ast)
コード例 #14
0
ファイル: python.py プロジェクト: brigittebigi/sppas
    def render_fields(self, fields):
        abstract_template = trim(self.abstract_rule_template)
        abstract_rules = [abstract_template.format(name=safe_name(rule.name)) for rule in self.node.rules]
        abstract_rules = indent("\n".join(abstract_rules))

        if self.node.whitespace is not None:
            whitespace = urepr(self.node.whitespace)
        elif self.node.directives.get("whitespace") is not None:
            whitespace = "re.compile({0}, RE_FLAGS | re.DOTALL)".format(urepr(self.node.directives.get("whitespace")))
        else:
            whitespace = "None"

        if self.node.nameguard is not None:
            nameguard = urepr(self.node.nameguard)
        elif self.node.directives.get("nameguard") is not None:
            nameguard = self.node.directives.get("nameguard")
        else:
            nameguard = "None"

        comments_re = urepr(self.node.directives.get("comments"))
        eol_comments_re = urepr(self.node.directives.get("eol_comments"))
        ignorecase = self.node.directives.get("ignorecase", "None")
        left_recursion = self.node.directives.get("left_recursion", True)

        rules = "\n".join([self.get_renderer(rule).render() for rule in self.node.rules])

        version = str(tuple(int(n) for n in str(timestamp()).split(".")))

        fields.update(
            rules=indent(rules),
            abstract_rules=abstract_rules,
            version=version,
            whitespace=whitespace,
            nameguard=nameguard,
            comments_re=comments_re,
            eol_comments_re=eol_comments_re,
            ignorecase=ignorecase,
            left_recursion=left_recursion,
        )
コード例 #15
0
ファイル: grammar_test.py プロジェクト: brigittebigi/sppas
    def test_based_rule(self):
        grammar = '''\
            start
                =
                b $
                ;


            a
                =
                @:'a'
                ;


            b < a
                =
                {@:'b'}
                ;
            '''
        model = genmodel("test", grammar)
        ast = model.parse("abb", nameguard=False)
        self.assertEquals(['a', 'b', 'b'], ast)
        self.assertEqual(trim(grammar), ustr(model))
コード例 #16
0
    def test_based_rule(self):
        grammar = '''\
            start
                =
                b $
                ;


            a
                =
                @:'a'
                ;


            b < a
                =
                {@:'b'}
                ;
            '''
        model = genmodel("test", grammar)
        ast = model.parse("abb", nameguard=False)
        self.assertEquals(['a', 'b', 'b'], ast)
        self.assertEqual(trim(grammar), ustr(model))
コード例 #17
0
ファイル: grammar_test.py プロジェクト: brigittebigi/sppas
    def test_36_param_combinations(self):
        def assert_equal(target, value):
            self.assertEqual(target, value)

        class TC36Semantics(object):

            """Check all rule parameters for expected types and values"""

            def rule_positional(self, ast, p1, p2, p3, p4):
                assert_equal("ABC", p1)
                assert_equal(123, p2)
                assert_equal('=', p3)
                assert_equal("+", p4)
                return ast

            def rule_keyword(self, ast, k1, k2, k3, k4):
                assert_equal("ABC", k1)
                assert_equal(123, k2)
                assert_equal('=', k3)
                assert_equal('+', k4)
                return ast

            def rule_all(self, ast, p1, p2, p3, p4, k1, k2, k3, k4):
                assert_equal("DEF", p1)
                assert_equal(456, p2)
                assert_equal('=', p3)
                assert_equal("+", p4)
                assert_equal("HIJ", k1)
                assert_equal(789, k2)
                assert_equal('=', k3)
                assert_equal('+', k4)
                return ast

        grammar = '''
            start
                =
                {rule_positional | rule_keywords | rule_all} $
                ;


            rule_positional('ABC', 123, '=', '+')
                =
                'a'
                ;


            rule_keywords(k1=ABC, k3='=', k4='+', k2=123)
                =
                'b'
                ;


            rule_all('DEF', 456, '=', '+', k1=HIJ, k3='=', k4='+', k2=789)
                =
                'c'
                ;
        '''

        pretty = '''
            start
                =
                {rule_positional | rule_keywords | rule_all} $
                ;


            rule_positional(ABC, 123, '=', '+')
                =
                'a'
                ;


            rule_keywords(k1=ABC, k3='=', k4='+', k2=123)
                =
                'b'
                ;


            rule_all(DEF, 456, '=', '+', k1=HIJ, k3='=', k4='+', k2=789)
                =
                'c'
                ;
        '''

        model = genmodel('RuleArguments', grammar)
        self.assertEqual(trim(pretty), ustr(model))
        model = genmodel('RuleArguments', pretty)

        ast = model.parse("a b c")
        self.assertEqual(['a', 'b', 'c'], ast)
        semantics = TC36Semantics()
        ast = model.parse("a b c", semantics=semantics)
        self.assertEqual(['a', 'b', 'c'], ast)
        codegen(model)
コード例 #18
0
ファイル: grammars.py プロジェクト: brigittebigi/sppas
 def __str__(self):
     exp = ustr(self.exp)
     if len(exp.splitlines()) > 1:
         return '(\n%s\n)' % indent(exp)
     else:
         return '(%s)' % trim(exp)
コード例 #19
0
ファイル: rendering.py プロジェクト: brigittebigi/sppas
 def trim(self, item, tabwidth=4):
     return trim(self.rend(item), tabwidth=tabwidth)
コード例 #20
0
    def test_36_param_combinations(self):
        def assert_equal(target, value):
            self.assertEqual(target, value)

        class TC36Semantics(object):
            """Check all rule parameters for expected types and values"""
            def rule_positional(self, ast, p1, p2, p3, p4):
                assert_equal("ABC", p1)
                assert_equal(123, p2)
                assert_equal('=', p3)
                assert_equal("+", p4)
                return ast

            def rule_keyword(self, ast, k1, k2, k3, k4):
                assert_equal("ABC", k1)
                assert_equal(123, k2)
                assert_equal('=', k3)
                assert_equal('+', k4)
                return ast

            def rule_all(self, ast, p1, p2, p3, p4, k1, k2, k3, k4):
                assert_equal("DEF", p1)
                assert_equal(456, p2)
                assert_equal('=', p3)
                assert_equal("+", p4)
                assert_equal("HIJ", k1)
                assert_equal(789, k2)
                assert_equal('=', k3)
                assert_equal('+', k4)
                return ast

        grammar = '''
            start
                =
                {rule_positional | rule_keywords | rule_all} $
                ;


            rule_positional('ABC', 123, '=', '+')
                =
                'a'
                ;


            rule_keywords(k1=ABC, k3='=', k4='+', k2=123)
                =
                'b'
                ;


            rule_all('DEF', 456, '=', '+', k1=HIJ, k3='=', k4='+', k2=789)
                =
                'c'
                ;
        '''

        pretty = '''
            start
                =
                {rule_positional | rule_keywords | rule_all} $
                ;


            rule_positional(ABC, 123, '=', '+')
                =
                'a'
                ;


            rule_keywords(k1=ABC, k3='=', k4='+', k2=123)
                =
                'b'
                ;


            rule_all(DEF, 456, '=', '+', k1=HIJ, k3='=', k4='+', k2=789)
                =
                'c'
                ;
        '''

        model = genmodel('RuleArguments', grammar)
        self.assertEqual(trim(pretty), ustr(model))
        model = genmodel('RuleArguments', pretty)

        ast = model.parse("a b c")
        self.assertEqual(['a', 'b', 'c'], ast)
        semantics = TC36Semantics()
        ast = model.parse("a b c", semantics=semantics)
        self.assertEqual(['a', 'b', 'c'], ast)
        codegen(model)