def test_postlex(self):
from lark.indenter import Indenter
class MyIndenter(Indenter):
NL_type = '_NEWLINE'
OPEN_PAREN_types = ['LPAR', 'LSQB', 'LBRACE']
CLOSE_PAREN_types = ['RPAR', 'RSQB', 'RBRACE']
INDENT_type = '_INDENT'
DEDENT_type = '_DEDENT'
tab_len = 8
grammar = r"""
start: "(" ")" _NEWLINE
_NEWLINE: /\n/
"""
context = self._create_standalone(grammar)
_Lark = context['Lark_StandAlone']
l = _Lark(postlex=MyIndenter())
x = l.parse('()\n')
self.assertEqual(x, Tree('start', []))
l = _Lark(postlex=MyIndenter())
x = l.parse('(\n)\n')
self.assertEqual(x, Tree('start', []))
def test_parse_hello_world():
p = 'print("hello world")'
assert parse(p) == Tree('start', [
Token('IDENTIFIER', 'print'),
Tree('arguments', [Token('STRING', '"hello world"')])
])
def test_fruitflies_ambig(self):
grammar = """
start: noun verb noun -> simple
| noun verb "like" noun -> comparative
noun: adj? NOUN
verb: VERB
adj: ADJ
NOUN: "flies" | "bananas" | "fruit"
VERB: "like" | "flies"
ADJ: "fruit"
%import common.WS
%ignore WS
"""
parser = Lark(grammar, ambiguity='explicit', lexer=LEXER)
tree = parser.parse('fruit flies like bananas')
expected = Tree('_ambig', [
Tree('comparative', [
Tree('noun', ['fruit']),
Tree('verb', ['flies']),
Tree('noun', ['bananas'])
]),
Tree('simple', [
Tree('noun', [Tree('adj', ['fruit']), 'flies']),
Tree('verb', ['like']),
Tree('noun', ['bananas'])
])
])
# self.assertEqual(tree, expected)
self.assertEqual(tree.data, expected.data)
self.assertEqual(set(tree.children), set(expected.children))
def test_transformer_variants(self):
tree = Tree('start', [
Tree(
'add',
[Token('N', '1'),
Token('N', '2'),
Token('IGNORE_TOKEN', '4')]),
Tree('add', [Token('N', '3'), Token('N', '4')]),
Tree('ignore_tree', [Token('DO', 'NOT PANIC')]),
])
for base in (Transformer, Transformer_InPlace,
Transformer_NonRecursive, Transformer_InPlaceRecursive):
class T(base):
def add(self, children):
return sum(children)
def N(self, token):
return int(token)
def ignore_tree(self, children):
raise Discard
def IGNORE_TOKEN(self, token):
raise Discard
copied = copy.deepcopy(tree)
result = T().transform(copied)
self.assertEqual(result, Tree('start', [3, 7]))
def test_interp(self):
t = Tree('a', [Tree('b', []), Tree('c', []), 'd'])
class Interp1(Interpreter):
def a(self, tree):
return self.visit_children(tree) + ['e']
def b(self, tree):
return 'B'
def c(self, tree):
return 'C'
self.assertEqual(Interp1().visit(t), list('BCde'))
class Interp2(Interpreter):
@visit_children_decor
def a(self, values):
return values + ['e']
def b(self, tree):
return 'B'
def c(self, tree):
return 'C'
self.assertEqual(Interp2().visit(t), list('BCde'))
def create_variable(name, time):
return Tree(
"variable",
[
Tree("name", [Token("NAME", name)]),
Tree("date", [Token("NUMBER", str(time))]),
],
)
def translate_tree(self, t: Tree) -> Tree:
# print(t)
if t.data == "mll":
return Tree(t.data, self.transform(t.children))
if t.data == "pyt":
m = apply(t, lambda x: x, clean_tok)
m.children = m.children + [Token("WS", "\n")]
return m
if t.data == "model":
return self.translate_model(t)
if t.data == "comp":
return Tree(t.data, escape(t.children))
if t.data == "comment":
return None
if t.data == "parmac":
# cprint("entro in parmac","yellow")
self.insert_parmac(t)
if t.data == "summa":
# cprint("entro in parmac","yellow")
from mll.simple_model import SimpleModel
from mll.dispatcher import Dispatcher
rest = Dispatcher(self).translate_e(
Tree("e", [
Token("ID",
clean_tok(t.children[2]).value),
Tree("e", [Token("ID",
clean_tok(t.children[0]).value)])
]))
print(rest)
m = apply([
Token("ID",
clean_tok(t.children[0]).value),
Token("EQ", "=")
] + rest.children, lambda x: x, self.substitute_model)
print(m)
m = m + [Token("WS", "\n\n")]
return m
# if t.data == "macro_mod":
# create_macro_mod(self,t)
#
# if t.data == "macro_exp":
# create_macro_exp(self,t)
#
# if t.data == "macro_pip":
# create_macro_pip(self,t)
if t.data == "macro":
create_macro_pip(self, t)
def test_utf8(self):
g = u"""start: a
a: "±a"
"""
l = _Lark(g)
self.assertEqual(l.parse(u'±a'), Tree('start', [Tree('a', [])]))
g = u"""start: A
A: "±a"
"""
l = _Lark(g)
self.assertEqual(l.parse(u'±a'), Tree('start', [u'\xb1a']))
def symbol(self, *args):
tok = args[0][0]
val = tok.value
if val in self.__variables__:
return Tree(
"variable",
[
Tree("name", [Token("NAME", val)]),
Tree("date", [Token("NUMBER", "0")]),
],
)
else:
return Tree("symbol", *args)
def test_earley_repeating_empty(self):
# This was a sneaky bug!
grammar = """
!start: "a" empty empty "b"
empty: empty2
empty2:
"""
parser = Lark(grammar, parser='earley', lexer=LEXER)
res = parser.parse('ab')
empty_tree = Tree('empty', [Tree('empty2', [])])
self.assertSequenceEqual(res.children, ['a', empty_tree, empty_tree, 'b'])
def test_partial(self):
tree = Tree("start", [Tree("a", ["test1"]), Tree("b", ["test2"])])
def test(prefix, s, postfix):
return prefix + s.upper() + postfix
@v_args(inline=True)
class T(Transformer):
a = functools.partial(test, "@", postfix="!")
b = functools.partial(lambda s: s + "!")
res = T().transform(tree)
assert res.children == ["@TEST1!", "test2!"]
def test_vargs_override(self):
t = Tree('add', [Tree('sub', [Tree('i', ['3']), Tree('f', ['1.1'])]), Tree('i', ['1'])])
@v_args(inline=True)
class T(Transformer):
i = int
f = float
sub = lambda self, a, b: a-b
@v_args(inline=False)
def add(self, values):
return sum(values)
res = T().transform(t)
self.assertEqual(res, 2.9)
def test_ambiguity2(self):
grammar = """
ANY: /[a-zA-Z0-9 ]+/
a.2: "A" b+
b.2: "B"
c: ANY
start: (a|c)*
"""
l = Lark(grammar, parser='earley', lexer=LEXER)
res = l.parse('ABX')
expected = Tree('start',
[Tree('a', [Tree('b', [])]),
Tree('c', ['X'])])
self.assertEqual(res, expected)
def assignVar(self, name, vals):
"""
Assign a value to a variable
"""
if not isinstance(vals, list):
vals = [vals]
if len(vals) == 1:
val = vals[0]
# print("setting "+name+" to "+val)
if isinstance(val, Token):
if val.type == 'VARIABLE':
if val.value in self.vars:
ret = self.vars.get(val.value)
else:
raise SyntaxError('Undefined variable %s' % val.value)
elif val.type in ('ATOM', 'FILENAME', 'CNAME'):
ret = self.__getFileName(val)
# elif isinstance(val, Tree):
# process the tree and store the result in the var
# ret = self.run_instruction(val)
else:
ret = val
else:
# process the tree and store the result in the var
tree = Tree('command', vals)
ret = self.run_instruction(tree)
self.vars[name] = ret
def null_value(self, tree: Tree):
token = find_token_in_ast(tree.children, ['NULL'])
newtoken = Token(
token.type, None, token.pos_in_stream, token.line, token.column
)
tree.children = [newtoken]
return tree
def true_value(self, tree: Tree):
token = find_token_in_ast(tree.children, ['TRUE'])
newtoken = Token(
token.type, True, token.pos_in_stream, token.line, token.column
)
tree.children = [newtoken]
return tree
def false_value(self, tree: Tree):
token = find_token_in_ast(tree.children, ['FALSE'])
newtoken = Token(
token.type, False, token.pos_in_stream, token.line, token.column
)
tree.children = [newtoken]
return tree
def float_value(self, tree: Tree):
token = find_token_in_ast(tree.children, ['SIGNED_FLOAT'])
newtoken = Token(
token.type, float(token.value), token.pos_in_stream, token.line,
token.column
)
tree.children = [newtoken]
return tree
def string_value(self, tree: Tree):
token = find_token_in_ast(tree.children, ['STRING'])
tmp = bytes(token.value[1:-1], "utf-8").decode('unicode-escape')
newtoken = Token(
token.type, tmp, token.pos_in_stream, token.line, token.column
)
tree.children = [newtoken]
return tree
def symbol(self, children):
name = children[0].value
if name == "inf":
s = name
else:
s = stringify_parameter(name)
return Tree("symbol", [Token("NAME", s)])
def variable(self, children):
name = children[0].children[0].value
try:
date = int(children[1].children[0].value)
except:
date = 0
if self.shift == "S":
new_date = "0"
else:
new_date = str(date + self.shift)
return Tree(
"variable",
[
Tree("name", [Token("NAME", name)]),
Tree("date", [Token("NUMBER", new_date)]),
],
)
def variable(self, children):
name = children[0].children[0].value
if len(children) == 1:
date = 0
else:
date = int(children[1].children[0].value)
s = stringify_variable((name, date))
return Tree("symbol", [Token("NAME", s)])
def test_inline_static(self):
@v_args(inline=True)
class T(Transformer):
@staticmethod
def test(a, b):
return a + b
x = T().transform(Tree('test', ['a', 'b']))
self.assertEqual(x, 'ab')
def test_transformer_variants(self):
tree = Tree('start', [
Tree('add', [Token('N', '1'), Token('N', '2')]),
Tree('add', [Token('N', '3'), Token('N', '4')])
])
for base in (Transformer, Transformer_InPlace,
Transformer_NonRecursive, Transformer_InPlaceRecursive):
class T(base):
def add(self, children):
return sum(children)
def N(self, token):
return int(token)
copied = copy.deepcopy(tree)
result = T().transform(copied)
self.assertEqual(result, Tree('start', [3, 7]))
def test_vargs(self):
@v_args()
class MyTransformer(Transformer):
@staticmethod
def integer(args):
return 1 # some code here
@classmethod
def integer2(cls, args):
return 2 # some code here
hello = staticmethod(lambda args: 'hello')
x = MyTransformer().transform( Tree('integer', [2]))
self.assertEqual(x, 1)
x = MyTransformer().transform( Tree('integer2', [2]))
self.assertEqual(x, 2)
x = MyTransformer().transform( Tree('hello', [2]))
self.assertEqual(x, 'hello')
def test_ranged_repeat_terms(self):
g = u"""!start: AAA
AAA: "A"~3
"""
l = _Lark(g)
self.assertEqual(l.parse(u'AAA'), Tree('start', ["AAA"]))
self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AA')
self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAA')
g = u"""!start: AABB CC
AABB: "A"~0..2 "B"~2
CC: "C"~1..2
"""
l = _Lark(g)
self.assertEqual(l.parse(u'AABBCC'), Tree('start', ['AABB', 'CC']))
self.assertEqual(l.parse(u'BBC'), Tree('start', ['BB', 'C']))
self.assertEqual(l.parse(u'ABBCC'), Tree('start', ['ABB', 'CC']))
self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAB')
self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAABBB')
self.assertRaises((ParseError, UnexpectedInput), l.parse, u'ABB')
self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAABB')
def test_vargs_set_name(self):
# Test with cached_property if available. That actually uses __set_name__
prop = getattr(functools, "cached_property", property)
class T(Transformer):
@v_args(inline=True)
@prop # Not sure why you would ever want to use a property here, but we support it
def test(self):
return lambda a, b: (self, a, b)
t = T()
self.assertEqual(t.transform(Tree("test", [1, 2])), (t, 1, 2))
def test_ranged_repeat_rules(self):
g = u"""!start: "A"~3
"""
l = _Lark(g)
self.assertEqual(l.parse(u'AAA'), Tree('start', ["A", "A", "A"]))
self.assertRaises(ParseError, l.parse, u'AA')
self.assertRaises((ParseError, UnexpectedInput), l.parse, u'AAAA')
g = u"""!start: "A"~0..2
"""
if PARSER != 'cyk': # XXX CYK currently doesn't support empty grammars
l = _Lark(g)
self.assertEqual(l.parse(u''), Tree('start', []))
self.assertEqual(l.parse(u'A'), Tree('start', ['A']))
self.assertEqual(l.parse(u'AA'), Tree('start', ['A', 'A']))
self.assertRaises((UnexpectedToken, UnexpectedInput), l.parse,
u'AAA')
g = u"""!start: "A"~3..2
"""
self.assertRaises(GrammarError, _Lark, g)
g = u"""!start: "A"~2..3 "B"~2
"""
l = _Lark(g)
self.assertEqual(l.parse(u'AABB'),
Tree('start', ['A', 'A', 'B', 'B']))
self.assertEqual(l.parse(u'AAABB'),
Tree('start', ['A', 'A', 'A', 'B', 'B']))
self.assertRaises(ParseError, l.parse, u'AAAB')
self.assertRaises((ParseError, UnexpectedInput), l.parse,
u'AAABBB')
self.assertRaises((ParseError, UnexpectedInput), l.parse, u'ABB')
self.assertRaises((ParseError, UnexpectedInput), l.parse,
u'AAAABB')
def factor(self, f):
if len(f) == 2:
op, operand = f
op = {
'+': UAdd(),
'-': USub(),
}.get(op.value, None)
if op is not None:
return UnaryOp(op=op, operand=operand)
# print(f)
return Tree(data='factor', children=f)
def _load_ptr(self, t, var):
# if the pointer is located on stack, copy the address to a temporary
# pointer located at zero-page in order to perform indexed indirect
# addressing
if isinstance(var.loc, StackOffset):
tmp_ptr_lo = f'${hex(self.tmp_ptr)[2:]}'
tmp_ptr_hi = f'${hex(self.tmp_ptr + 1)[2:]}'
if var.is_array:
# load into A, X the *stack location address*, at which the array begins
self._getref(t, var)
self._instr(t, Op.STA, tmp_ptr_lo)
self._instr(t, Op.TXA)
self._instr(t, Op.STA, tmp_ptr_hi)
else:
# create two fake byte-sized "anchors" to the stack-located address
# LO and HI parts
addr_lo, addr_hi = Tree(None, []), Tree(None, [])
self._setloc(addr_lo, var.loc)
self._setsize(addr_lo, 1)
self._setloc(addr_hi, type(var.loc)(var.loc + 1))
self._setsize(addr_hi, 1)
# pull the LO part and store it into temporary pointer LO location
self._pull(t, addr_lo)
self._instr(t, Op.STA, tmp_ptr_lo)
# pull the HI part and store it in related part of the temporary
# pointer
self._pull(t, addr_hi)
self._instr(t, Op.STA, tmp_ptr_hi)
# now the location of the tree is the temporary pointer of the zero
# page memory
self._setloc(t, Pointer(self.tmp_ptr))
# the pointer is in a global variable and is already in zero-page memory
else:
self._setloc(t, Pointer(var.loc))
