def __str__(self): comments = self.comments_str() params = ', '.join( self.param_repr(p) for p in self.params ) if self.params else '' kwparams = '' if self.kwparams: kwparams = ', '.join( '%s=%s' % (k, self.param_repr(v)) for (k, v) in self.kwparams.items() ) if params and kwparams: params = '(%s, %s)' % (params, kwparams) elif kwparams: params = '(%s)' % (kwparams) elif params: params = '(%s)' % params base = ' < %s' % ustr(self.base.name) if self.base else '' return trim(self.str_template).format( name=self.name, base=base, params=params, exp=indent(str(self.exp)), comments=comments )
def test_numbers_and_unicode(self): grammar = ''' rúle(1, -23, 4.56, 7.89e-11, 0xABCDEF, Añez) = 'a' ; ''' rule2 = ''' rulé(Añez) = '\\xf1' ; ''' rule3 = ''' rúlé(Añez) = 'ñ' ; ''' if PY3: grammar += rule3 else: grammar += rule2 model = genmodel("test", grammar) self.assertEquals(trim(grammar), ustr(model))
def test_include(self): text = '''\ first #include :: "something" last\ ''' buf = MockIncludeBuffer(trim(text)) self.assertEqual('first\n\nINCLUDED "something"\nlast', buf.text)
def __str__(self): exp = ustr(self.exp) template = '[%s]' if isinstance(self.exp, Choice): template = trim(self.str_template) elif isinstance(self.exp, Group): exp = self.exp.exp return template % exp
def test_36_params_and_keyword_params(self): grammar = ''' rule(A, kwdB=B) = 'a' ; ''' model = genmodel("test", grammar) self.assertEquals(trim(grammar), ustr(model))
def render_fields(self, fields): template = trim(self.option_template) options = [template.format(option=indent(self.rend(o))) for o in self.node.options] options = "\n".join(o for o in options) firstset = " ".join(f[0] for f in sorted(self.node.firstset) if f) if firstset: error = "expecting one of: " + firstset else: error = "no available options" fields.update(n=self.counter(), options=indent(options), error=urepr(error))
def test_whitespace_no_newlines(self): grammar = """ @@whitespace :: /[\t ]+/ # this is just a token with any character but space and newline # it should finish before it capture space or newline character token = /[^ \n]+/; # expect whitespace to capture spaces between tokens, but newline should be captured afterwards token2 = {token}* /\n/; # document is just list of this strings of tokens document = {@+:token2}* $; """ text = trim("""\ a b c d e f """) expected = [ [ [ "a", "b" ], "\n" ], [ [ "c", "d" ], "\n" ], [ [ "e", "f" ], "\n" ] ] model = genmodel("document", grammar) ast = model.parse(text, start='document') self.assertEqual(expected, ast)
def render(self, template=None, **fields): fields.update(__class__=self.__class__.__name__) fields.update({k: v for k, v in vars(self).items() if not k.startswith('_')}) override = self.render_fields(fields) if override is not None: template = override elif template is None: template = self.template try: return self._formatter.format(trim(template), **fields) except KeyError: # find the missing key keys = (p[1] for p in self._formatter.parse(template)) for key in keys: if key and key not in fields: raise KeyError(key, type(self)) raise
def render_fields(self, fields): self.reset_counter() params = kwparams = "" if self.node.params: params = ", ".join(self.param_repr(self.rend(p)) for p in self.node.params) if self.node.kwparams: kwparams = ", ".join("%s=%s" % (k, self.param_repr(self.rend(v))) for k, v in self.kwparams.items()) if params and kwparams: params = params + ", " + kwparams elif kwparams: params = kwparams fields.update(params=params) defines = compress_seq(self.defines()) sdefs = [d for d, l in defines if not l] ldefs = [d for d, l in defines if l] if not (sdefs or ldefs): sdefines = "" else: sdefs = "[%s]" % ", ".join(urepr(d) for d in sdefs) ldefs = "[%s]" % ", ".join(urepr(d) for d in ldefs) if not ldefs: sdefines = "\n\n self.ast._define(%s, %s)" % (sdefs, ldefs) else: sdefines = indent( "\n\n" + trim( """\ self.ast._define( %s, %s )""" % (sdefs, ldefs) ) ) fields.update(defines=sdefines)
def test_whitespace_no_newlines(self): grammar = """ @@whitespace :: /[\t ]+/ # this is just a token with any character but space and newline # it should finish before it capture space or newline character token = /[^ \n]+/; # expect whitespace to capture spaces between tokens, but newline should be captured afterwards token2 = {token}* /\n/; # document is just list of this strings of tokens document = {@+:token2}* $; """ text = trim("""\ a b c d e f """) expected = [[["a", "b"], "\n"], [["c", "d"], "\n"], [["e", "f"], "\n"]] model = genmodel("document", grammar) ast = model.parse(text, start='document') self.assertEqual(expected, ast)
def render_fields(self, fields): abstract_template = trim(self.abstract_rule_template) abstract_rules = [abstract_template.format(name=safe_name(rule.name)) for rule in self.node.rules] abstract_rules = indent("\n".join(abstract_rules)) if self.node.whitespace is not None: whitespace = urepr(self.node.whitespace) elif self.node.directives.get("whitespace") is not None: whitespace = "re.compile({0}, RE_FLAGS | re.DOTALL)".format(urepr(self.node.directives.get("whitespace"))) else: whitespace = "None" if self.node.nameguard is not None: nameguard = urepr(self.node.nameguard) elif self.node.directives.get("nameguard") is not None: nameguard = self.node.directives.get("nameguard") else: nameguard = "None" comments_re = urepr(self.node.directives.get("comments")) eol_comments_re = urepr(self.node.directives.get("eol_comments")) ignorecase = self.node.directives.get("ignorecase", "None") left_recursion = self.node.directives.get("left_recursion", True) rules = "\n".join([self.get_renderer(rule).render() for rule in self.node.rules]) version = str(tuple(int(n) for n in str(timestamp()).split("."))) fields.update( rules=indent(rules), abstract_rules=abstract_rules, version=version, whitespace=whitespace, nameguard=nameguard, comments_re=comments_re, eol_comments_re=eol_comments_re, ignorecase=ignorecase, left_recursion=left_recursion, )
def test_based_rule(self): grammar = '''\ start = b $ ; a = @:'a' ; b < a = {@:'b'} ; ''' model = genmodel("test", grammar) ast = model.parse("abb", nameguard=False) self.assertEquals(['a', 'b', 'b'], ast) self.assertEqual(trim(grammar), ustr(model))
def test_36_param_combinations(self): def assert_equal(target, value): self.assertEqual(target, value) class TC36Semantics(object): """Check all rule parameters for expected types and values""" def rule_positional(self, ast, p1, p2, p3, p4): assert_equal("ABC", p1) assert_equal(123, p2) assert_equal('=', p3) assert_equal("+", p4) return ast def rule_keyword(self, ast, k1, k2, k3, k4): assert_equal("ABC", k1) assert_equal(123, k2) assert_equal('=', k3) assert_equal('+', k4) return ast def rule_all(self, ast, p1, p2, p3, p4, k1, k2, k3, k4): assert_equal("DEF", p1) assert_equal(456, p2) assert_equal('=', p3) assert_equal("+", p4) assert_equal("HIJ", k1) assert_equal(789, k2) assert_equal('=', k3) assert_equal('+', k4) return ast grammar = ''' start = {rule_positional | rule_keywords | rule_all} $ ; rule_positional('ABC', 123, '=', '+') = 'a' ; rule_keywords(k1=ABC, k3='=', k4='+', k2=123) = 'b' ; rule_all('DEF', 456, '=', '+', k1=HIJ, k3='=', k4='+', k2=789) = 'c' ; ''' pretty = ''' start = {rule_positional | rule_keywords | rule_all} $ ; rule_positional(ABC, 123, '=', '+') = 'a' ; rule_keywords(k1=ABC, k3='=', k4='+', k2=123) = 'b' ; rule_all(DEF, 456, '=', '+', k1=HIJ, k3='=', k4='+', k2=789) = 'c' ; ''' model = genmodel('RuleArguments', grammar) self.assertEqual(trim(pretty), ustr(model)) model = genmodel('RuleArguments', pretty) ast = model.parse("a b c") self.assertEqual(['a', 'b', 'c'], ast) semantics = TC36Semantics() ast = model.parse("a b c", semantics=semantics) self.assertEqual(['a', 'b', 'c'], ast) codegen(model)
def __str__(self): exp = ustr(self.exp) if len(exp.splitlines()) > 1: return '(\n%s\n)' % indent(exp) else: return '(%s)' % trim(exp)
def trim(self, item, tabwidth=4): return trim(self.rend(item), tabwidth=tabwidth)