def test_postlex(self): from lark.indenter import Indenter class MyIndenter(Indenter): NL_type = '_NEWLINE' OPEN_PAREN_types = ['LPAR', 'LSQB', 'LBRACE'] CLOSE_PAREN_types = ['RPAR', 'RSQB', 'RBRACE'] INDENT_type = '_INDENT' DEDENT_type = '_DEDENT' tab_len = 8 grammar = r""" start: "(" ")" _NEWLINE _NEWLINE: /\n/ """ context = self._create_standalone(grammar) _Lark = context['Lark_StandAlone'] l = _Lark(postlex=MyIndenter()) x = l.parse('()\n') self.assertEqual(x, Tree('start', [])) l = _Lark(postlex=MyIndenter()) x = l.parse('(\n)\n') self.assertEqual(x, Tree('start', []))
def test_parse_hello_world(): p = 'print("hello world")' assert parse(p) == Tree('start', [ Token('IDENTIFIER', 'print'), Tree('arguments', [Token('STRING', '"hello world"')]) ])
def test_fruitflies_ambig(self): grammar = """ start: noun verb noun -> simple | noun verb "like" noun -> comparative noun: adj? NOUN verb: VERB adj: ADJ NOUN: "flies" | "bananas" | "fruit" VERB: "like" | "flies" ADJ: "fruit" %import common.WS %ignore WS """ parser = Lark(grammar, ambiguity='explicit', lexer=LEXER) tree = parser.parse('fruit flies like bananas') expected = Tree('_ambig', [ Tree('comparative', [ Tree('noun', ['fruit']), Tree('verb', ['flies']), Tree('noun', ['bananas']) ]), Tree('simple', [ Tree('noun', [Tree('adj', ['fruit']), 'flies']), Tree('verb', ['like']), Tree('noun', ['bananas']) ]) ]) # self.assertEqual(tree, expected) self.assertEqual(tree.data, expected.data) self.assertEqual(set(tree.children), set(expected.children))
def test_transformer_variants(self): tree = Tree('start', [ Tree( 'add', [Token('N', '1'), Token('N', '2'), Token('IGNORE_TOKEN', '4')]), Tree('add', [Token('N', '3'), Token('N', '4')]), Tree('ignore_tree', [Token('DO', 'NOT PANIC')]), ]) for base in (Transformer, Transformer_InPlace, Transformer_NonRecursive, Transformer_InPlaceRecursive): class T(base): def add(self, children): return sum(children) def N(self, token): return int(token) def ignore_tree(self, children): raise Discard def IGNORE_TOKEN(self, token): raise Discard copied = copy.deepcopy(tree) result = T().transform(copied) self.assertEqual(result, Tree('start', [3, 7]))
def test_interp(self): t = Tree('a', [Tree('b', []), Tree('c', []), 'd']) class Interp1(Interpreter): def a(self, tree): return self.visit_children(tree) + ['e'] def b(self, tree): return 'B' def c(self, tree): return 'C' self.assertEqual(Interp1().visit(t), list('BCde')) class Interp2(Interpreter): @visit_children_decor def a(self, values): return values + ['e'] def b(self, tree): return 'B' def c(self, tree): return 'C' self.assertEqual(Interp2().visit(t), list('BCde'))
def create_variable(name, time): return Tree( "variable", [ Tree("name", [Token("NAME", name)]), Tree("date", [Token("NUMBER", str(time))]), ], )
def translate_tree(self, t: Tree) -> Tree: # print(t) if t.data == "mll": return Tree(t.data, self.transform(t.children)) if t.data == "pyt": m = apply(t, lambda x: x, clean_tok) m.children = m.children + [Token("WS", "\n")] return m if t.data == "model": return self.translate_model(t) if t.data == "comp": return Tree(t.data, escape(t.children)) if t.data == "comment": return None if t.data == "parmac": # cprint("entro in parmac","yellow") self.insert_parmac(t) if t.data == "summa": # cprint("entro in parmac","yellow") from mll.simple_model import SimpleModel from mll.dispatcher import Dispatcher rest = Dispatcher(self).translate_e( Tree("e", [ Token("ID", clean_tok(t.children[2]).value), Tree("e", [Token("ID", clean_tok(t.children[0]).value)]) ])) print(rest) m = apply([ Token("ID", clean_tok(t.children[0]).value), Token("EQ", "=") ] + rest.children, lambda x: x, self.substitute_model) print(m) m = m + [Token("WS", "\n\n")] return m # if t.data == "macro_mod": # create_macro_mod(self,t) # # if t.data == "macro_exp": # create_macro_exp(self,t) # # if t.data == "macro_pip": # create_macro_pip(self,t) if t.data == "macro": create_macro_pip(self, t)
def test_utf8(self): g = u"""start: a a: "±a" """ l = _Lark(g) self.assertEqual(l.parse(u'±a'), Tree('start', [Tree('a', [])])) g = u"""start: A A: "±a" """ l = _Lark(g) self.assertEqual(l.parse(u'±a'), Tree('start', [u'\xb1a']))
def symbol(self, *args): tok = args[0][0] val = tok.value if val in self.__variables__: return Tree( "variable", [ Tree("name", [Token("NAME", val)]), Tree("date", [Token("NUMBER", "0")]), ], ) else: return Tree("symbol", *args)
def test_earley_repeating_empty(self): # This was a sneaky bug! grammar = """ !start: "a" empty empty "b" empty: empty2 empty2: """ parser = Lark(grammar, parser='earley', lexer=LEXER) res = parser.parse('ab') empty_tree = Tree('empty', [Tree('empty2', [])]) self.assertSequenceEqual(res.children, ['a', empty_tree, empty_tree, 'b'])
def test_partial(self): tree = Tree("start", [Tree("a", ["test1"]), Tree("b", ["test2"])]) def test(prefix, s, postfix): return prefix + s.upper() + postfix @v_args(inline=True) class T(Transformer): a = functools.partial(test, "@", postfix="!") b = functools.partial(lambda s: s + "!") res = T().transform(tree) assert res.children == ["@TEST1!", "test2!"]
def test_vargs_override(self): t = Tree('add', [Tree('sub', [Tree('i', ['3']), Tree('f', ['1.1'])]), Tree('i', ['1'])]) @v_args(inline=True) class T(Transformer): i = int f = float sub = lambda self, a, b: a-b @v_args(inline=False) def add(self, values): return sum(values) res = T().transform(t) self.assertEqual(res, 2.9)
def test_ambiguity2(self): grammar = """ ANY: /[a-zA-Z0-9 ]+/ a.2: "A" b+ b.2: "B" c: ANY start: (a|c)* """ l = Lark(grammar, parser='earley', lexer=LEXER) res = l.parse('ABX') expected = Tree('start', [Tree('a', [Tree('b', [])]), Tree('c', ['X'])]) self.assertEqual(res, expected)
def assignVar(self, name, vals): """ Assign a value to a variable """ if not isinstance(vals, list): vals = [vals] if len(vals) == 1: val = vals[0] # print("setting "+name+" to "+val) if isinstance(val, Token): if val.type == 'VARIABLE': if val.value in self.vars: ret = self.vars.get(val.value) else: raise SyntaxError('Undefined variable %s' % val.value) elif val.type in ('ATOM', 'FILENAME', 'CNAME'): ret = self.__getFileName(val) # elif isinstance(val, Tree): # process the tree and store the result in the var # ret = self.run_instruction(val) else: ret = val else: # process the tree and store the result in the var tree = Tree('command', vals) ret = self.run_instruction(tree) self.vars[name] = ret
def null_value(self, tree: Tree): token = find_token_in_ast(tree.children, ['NULL']) newtoken = Token( token.type, None, token.pos_in_stream, token.line, token.column ) tree.children = [newtoken] return tree
def true_value(self, tree: Tree): token = find_token_in_ast(tree.children, ['TRUE']) newtoken = Token( token.type, True, token.pos_in_stream, token.line, token.column ) tree.children = [newtoken] return tree
def false_value(self, tree: Tree): token = find_token_in_ast(tree.children, ['FALSE']) newtoken = Token( token.type, False, token.pos_in_stream, token.line, token.column ) tree.children = [newtoken] return tree
def float_value(self, tree: Tree): token = find_token_in_ast(tree.children, ['SIGNED_FLOAT']) newtoken = Token( token.type, float(token.value), token.pos_in_stream, token.line, token.column ) tree.children = [newtoken] return tree
def string_value(self, tree: Tree): token = find_token_in_ast(tree.children, ['STRING']) tmp = bytes(token.value[1:-1], "utf-8").decode('unicode-escape') newtoken = Token( token.type, tmp, token.pos_in_stream, token.line, token.column ) tree.children = [newtoken] return tree
def symbol(self, children): name = children[0].value if name == "inf": s = name else: s = stringify_parameter(name) return Tree("symbol", [Token("NAME", s)])
def variable(self, children): name = children[0].children[0].value try: date = int(children[1].children[0].value) except: date = 0 if self.shift == "S": new_date = "0" else: new_date = str(date + self.shift) return Tree( "variable", [ Tree("name", [Token("NAME", name)]), Tree("date", [Token("NUMBER", new_date)]), ], )
def variable(self, children): name = children[0].children[0].value if len(children) == 1: date = 0 else: date = int(children[1].children[0].value) s = stringify_variable((name, date)) return Tree("symbol", [Token("NAME", s)])
def test_inline_static(self): @v_args(inline=True) class T(Transformer): @staticmethod def test(a, b): return a + b x = T().transform(Tree('test', ['a', 'b'])) self.assertEqual(x, 'ab')
def test_transformer_variants(self): tree = Tree('start', [ Tree('add', [Token('N', '1'), Token('N', '2')]), Tree('add', [Token('N', '3'), Token('N', '4')]) ]) for base in (Transformer, Transformer_InPlace, Transformer_NonRecursive, Transformer_InPlaceRecursive): class T(base): def add(self, children): return sum(children) def N(self, token): return int(token) copied = copy.deepcopy(tree) result = T().transform(copied) self.assertEqual(result, Tree('start', [3, 7]))
def test_vargs(self): @v_args() class MyTransformer(Transformer): @staticmethod def integer(args): return 1 # some code here @classmethod def integer2(cls, args): return 2 # some code here hello = staticmethod(lambda args: 'hello') x = MyTransformer().transform( Tree('integer', [2])) self.assertEqual(x, 1) x = MyTransformer().transform( Tree('integer2', [2])) self.assertEqual(x, 2) x = MyTransformer().transform( Tree('hello', [2])) self.assertEqual(x, 'hello')
def test_ranged_repeat_terms(self): g = u"""!start: AAA AAA: "A"~3 """ l = _Lark(g) self.assertEqual(l.parse(u'AAA'), Tree('start', ["AAA"])) self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AA') self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAA') g = u"""!start: AABB CC AABB: "A"~0..2 "B"~2 CC: "C"~1..2 """ l = _Lark(g) self.assertEqual(l.parse(u'AABBCC'), Tree('start', ['AABB', 'CC'])) self.assertEqual(l.parse(u'BBC'), Tree('start', ['BB', 'C'])) self.assertEqual(l.parse(u'ABBCC'), Tree('start', ['ABB', 'CC'])) self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAB') self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAABBB') self.assertRaises((ParseError, UnexpectedInput), l.parse, u'ABB') self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAABB')
def test_vargs_set_name(self): # Test with cached_property if available. That actually uses __set_name__ prop = getattr(functools, "cached_property", property) class T(Transformer): @v_args(inline=True) @prop # Not sure why you would ever want to use a property here, but we support it def test(self): return lambda a, b: (self, a, b) t = T() self.assertEqual(t.transform(Tree("test", [1, 2])), (t, 1, 2))
def test_ranged_repeat_rules(self): g = u"""!start: "A"~3 """ l = _Lark(g) self.assertEqual(l.parse(u'AAA'), Tree('start', ["A", "A", "A"])) self.assertRaises(ParseError, l.parse, u'AA') self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAA') g = u"""!start: "A"~0..2 """ if PARSER != 'cyk': # XXX CYK currently doesn't support empty grammars l = _Lark(g) self.assertEqual(l.parse(u''), Tree('start', [])) self.assertEqual(l.parse(u'A'), Tree('start', ['A'])) self.assertEqual(l.parse(u'AA'), Tree('start', ['A', 'A'])) self.assertRaises((UnexpectedToken, UnexpectedInput), l.parse, u'AAA') g = u"""!start: "A"~3..2 """ self.assertRaises(GrammarError, _Lark, g) g = u"""!start: "A"~2..3 "B"~2 """ l = _Lark(g) self.assertEqual(l.parse(u'AABB'), Tree('start', ['A', 'A', 'B', 'B'])) self.assertEqual(l.parse(u'AAABB'), Tree('start', ['A', 'A', 'A', 'B', 'B'])) self.assertRaises(ParseError, l.parse, u'AAAB') self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAABBB') self.assertRaises((ParseError, UnexpectedInput), l.parse, u'ABB') self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAABB')
def factor(self, f): if len(f) == 2: op, operand = f op = { '+': UAdd(), '-': USub(), }.get(op.value, None) if op is not None: return UnaryOp(op=op, operand=operand) # print(f) return Tree(data='factor', children=f)
def _load_ptr(self, t, var): # if the pointer is located on stack, copy the address to a temporary # pointer located at zero-page in order to perform indexed indirect # addressing if isinstance(var.loc, StackOffset): tmp_ptr_lo = f'${hex(self.tmp_ptr)[2:]}' tmp_ptr_hi = f'${hex(self.tmp_ptr + 1)[2:]}' if var.is_array: # load into A, X the *stack location address*, at which the array begins self._getref(t, var) self._instr(t, Op.STA, tmp_ptr_lo) self._instr(t, Op.TXA) self._instr(t, Op.STA, tmp_ptr_hi) else: # create two fake byte-sized "anchors" to the stack-located address # LO and HI parts addr_lo, addr_hi = Tree(None, []), Tree(None, []) self._setloc(addr_lo, var.loc) self._setsize(addr_lo, 1) self._setloc(addr_hi, type(var.loc)(var.loc + 1)) self._setsize(addr_hi, 1) # pull the LO part and store it into temporary pointer LO location self._pull(t, addr_lo) self._instr(t, Op.STA, tmp_ptr_lo) # pull the HI part and store it in related part of the temporary # pointer self._pull(t, addr_hi) self._instr(t, Op.STA, tmp_ptr_hi) # now the location of the tree is the temporary pointer of the zero # page memory self._setloc(t, Pointer(self.tmp_ptr)) # the pointer is in a global variable and is already in zero-page memory else: self._setloc(t, Pointer(var.loc))