def test_function_def_anonymous(self):
tree = ast.parse(r'f = function() local a end')
exp = Chunk(
Block([
Assign(targets=[Name('f')],
values=[
AnonymousFunction(args=[],
body=Block([
LocalAssign(
targets=[Name('a')],
values=[])
]))
])
]))
self.assertEqual(exp, tree)
# test tokens
nodes = list(ast.walk(tree))
self.assertIsInstance(nodes[3], AnonymousFunction)
self.assertEqual(nodes[3].tokens[0].value.text, ' ')
self.assertEqual(nodes[3].tokens[1].value.text, 'function')
self.assertEqual(nodes[3].tokens[2].value.text, '(')
self.assertEqual(nodes[3].tokens[3].value.text, ')')
self.assertEqual(nodes[3].tokens[4].value.text, ' ')
self.assertEqual(nodes[3].tokens[5].value.text, 'local')
self.assertEqual(nodes[3].tokens[6].value.text, ' ')
self.assertEqual(nodes[3].tokens[7].value.text, 'a')
self.assertEqual(nodes[3].tokens[8].value.text, ' ')
self.assertEqual(nodes[3].tokens[9].value.text, 'end')
def process(self) -> None:
total_processed: int = 0
for node in ast.walk(self.luatree):
if isinstance(node, astnodes.Field) and hasattr(field.key, 'n'):
self.process_field(node)
total_processed += 1
print(f'Processed total {total_processed} nodes')
def test_function_nested_invoke(self):
tree = ast.parse(r'foo:bar():print("hello")')
exp = Chunk(
Block([
Invoke(source=Invoke(source=Name('foo'),
func=Name('bar'),
args=[]),
func=Name('print'),
args=[String('hello')])
]))
self.assertEqual(exp, tree)
# test tokens
nodes = list(ast.walk(tree))
self.assertIsInstance(nodes[2], Invoke)
self.assertEqual(nodes[2].tokens[0].value.text, 'foo')
self.assertEqual(nodes[2].tokens[1].value.text, ':')
self.assertEqual(nodes[2].tokens[2].value.text, 'bar')
self.assertEqual(nodes[2].tokens[3].value.text, '(')
self.assertEqual(nodes[2].tokens[4].value.text, ')')
self.assertEqual(nodes[2].tokens[5].value.text, ':')
self.assertEqual(nodes[2].tokens[6].value.text, 'print')
self.assertEqual(nodes[2].tokens[7].value.text, '(')
self.assertEqual(nodes[2].tokens[8].value.text, '"hello"')
self.assertEqual(nodes[2].tokens[9].value.text, ')')
self.assertIsInstance(nodes[5], Invoke)
self.assertEqual(nodes[5].tokens[0].value.text, 'foo')
self.assertEqual(nodes[5].tokens[1].value.text, ':')
self.assertEqual(nodes[5].tokens[2].value.text, 'bar')
self.assertEqual(nodes[5].tokens[3].value.text, '(')
self.assertEqual(nodes[5].tokens[4].value.text, ')')
def test_dict(self):
tree = ast.parse(r'a = {foo = "bar", bar = "foo"}')
exp = Chunk(
Block([
Assign(targets=[Name('a')],
values=[
Table(keys=[Name('foo'), Name('bar')],
values=[String('bar'),
String('foo')])
])
]))
self.assertEqual(exp, tree)
# test tokens
nodes = list(ast.walk(tree))
self.assertIsInstance(nodes[3], Table)
self.assertEqual(nodes[3].tokens[0].value.text, ' ')
self.assertEqual(nodes[3].tokens[1].value.text, '{')
self.assertEqual(nodes[3].tokens[2].value.text, 'foo')
self.assertEqual(nodes[3].tokens[3].value.text, ' ')
self.assertEqual(nodes[3].tokens[4].value.text, '=')
self.assertEqual(nodes[3].tokens[5].value.text, ' ')
self.assertEqual(nodes[3].tokens[6].value.text, '"bar"')
self.assertEqual(nodes[3].tokens[7].value.text, ',')
self.assertEqual(nodes[3].tokens[8].value.text, ' ')
self.assertEqual(nodes[3].tokens[9].value.text, 'bar')
self.assertEqual(nodes[3].tokens[10].value.text, ' ')
self.assertEqual(nodes[3].tokens[11].value.text, '=')
self.assertEqual(nodes[3].tokens[12].value.text, ' ')
self.assertEqual(nodes[3].tokens[13].value.text, '"foo"')
self.assertEqual(nodes[3].tokens[14].value.text, '}')
def test_addition(self):
tree = ast.parse(r'a = 1 + 0.2')
exp = Chunk(
Block([
Assign(targets=[Name('a')],
values=[AddOp(left=Number(1), right=Number(0.2))])
]))
self.assertEqual(exp, tree)
# test tokens
nodes = list(ast.walk(tree))
self.assertIsInstance(nodes[0], Chunk)
self.assertEqual(nodes[0].tokens[0].value.text, 'a')
self.assertEqual(nodes[0].tokens[1].value.text, ' ')
self.assertEqual(nodes[0].tokens[2].value.text, '=')
self.assertEqual(nodes[0].tokens[3].value.text, ' ')
self.assertEqual(nodes[0].tokens[4].value.text, '1')
self.assertEqual(nodes[0].tokens[5].value.text, ' ')
self.assertEqual(nodes[0].tokens[6].value.text, '+')
self.assertEqual(nodes[0].tokens[7].value.text, ' ')
self.assertEqual(nodes[0].tokens[8].value.text, '0.2')
self.assertIsInstance(nodes[1], Block)
self.assertEqual(nodes[1].tokens[0].value.text, 'a')
self.assertEqual(nodes[1].tokens[1].value.text, ' ')
self.assertEqual(nodes[1].tokens[2].value.text, '=')
self.assertEqual(nodes[1].tokens[3].value.text, ' ')
self.assertEqual(nodes[1].tokens[4].value.text, '1')
self.assertEqual(nodes[1].tokens[5].value.text, ' ')
self.assertEqual(nodes[1].tokens[6].value.text, '+')
self.assertEqual(nodes[1].tokens[7].value.text, ' ')
self.assertEqual(nodes[1].tokens[8].value.text, '0.2')
self.assertIsInstance(nodes[2], Assign)
self.assertEqual(nodes[2].tokens[0].value.text, 'a')
self.assertEqual(nodes[2].tokens[1].value.text, ' ')
self.assertEqual(nodes[2].tokens[2].value.text, '=')
self.assertEqual(nodes[2].tokens[3].value.text, ' ')
self.assertEqual(nodes[2].tokens[4].value.text, '1')
self.assertEqual(nodes[2].tokens[5].value.text, ' ')
self.assertEqual(nodes[2].tokens[6].value.text, '+')
self.assertEqual(nodes[2].tokens[7].value.text, ' ')
self.assertEqual(nodes[2].tokens[8].value.text, '0.2')
self.assertIsInstance(nodes[3], AddOp)
self.assertEqual(nodes[3].tokens[0].value.text, ' ')
self.assertEqual(nodes[3].tokens[1].value.text, '1')
self.assertEqual(nodes[3].tokens[2].value.text, ' ')
self.assertEqual(nodes[3].tokens[3].value.text, '+')
self.assertEqual(nodes[3].tokens[4].value.text, ' ')
self.assertEqual(nodes[3].tokens[5].value.text, '0.2')
self.assertIsInstance(nodes[4], Number)
self.assertEqual(nodes[4].tokens[0].value.text, ' ')
self.assertEqual(nodes[4].tokens[1].value.text, '0.2')
self.assertIsInstance(nodes[5], Number)
self.assertEqual(nodes[5].tokens[0].value.text, ' ')
self.assertEqual(nodes[5].tokens[1].value.text, '1')
def test_function_call_simple(self):
tree = ast.parse(r'print("hello")')
exp = Chunk(Block([Call(func=Name('print'), args=[String('hello')])]))
self.assertEqual(exp, tree)
# test tokens
nodes = list(ast.walk(tree))
self.assertIsInstance(nodes[2], Call)
self.assertEqual(nodes[2].tokens[0].value.text, 'print')
self.assertEqual(nodes[2].tokens[1].value.text, '(')
self.assertEqual(nodes[2].tokens[2].value.text, '"hello"')
self.assertEqual(nodes[2].tokens[3].value.text, ')')
def test_length_op(self):
tree = ast.parse(r'len = #t')
exp = Chunk(
Block([
Assign(targets=[Name('len')],
values=[ULengthOP(operand=Name('t'))])
]))
self.assertEqual(exp, tree)
# test tokens
nodes = list(ast.walk(tree))
self.assertIsInstance(nodes[3], ULengthOP)
self.assertEqual(nodes[3].tokens[0].value.text, ' ')
self.assertEqual(nodes[3].tokens[1].value.text, '#')
self.assertEqual(nodes[3].tokens[2].value.text, 't')
def test_label(self):
tree = ast.parse(
textwrap.dedent("""
::foo::
"""))
exp = Chunk(Block([Label(Name('foo'))]))
self.assertEqual(exp, tree)
# test tokens
nodes = list(ast.walk(tree))
self.assertIsInstance(nodes[2], Label)
self.assertEqual(nodes[2].tokens[0].value.text, '\n')
self.assertEqual(nodes[2].tokens[1].value.text, '::')
self.assertEqual(nodes[2].tokens[2].value.text, 'foo')
self.assertEqual(nodes[2].tokens[3].value.text, '::')
def test_ast_tokens_1(self):
src = textwrap.dedent("""\
-- exemple
local a = 42""")
tree = ast.parse(src)
nodes = list(ast.walk(tree))
self.assertIsInstance(nodes[0], Chunk)
self.assertEqual(nodes[0].tokens[0].value.text, '-- exemple')
self.assertEqual(nodes[0].tokens[1].value.text, '\n')
self.assertEqual(nodes[0].tokens[2].value.text, 'local')
self.assertEqual(nodes[0].tokens[3].value.text, ' ')
self.assertEqual(nodes[0].tokens[4].value.text, 'a')
self.assertEqual(nodes[0].tokens[5].value.text, ' ')
self.assertEqual(nodes[0].tokens[6].value.text, '=')
self.assertEqual(nodes[0].tokens[7].value.text, ' ')
self.assertEqual(nodes[0].tokens[8].value.text, '42')
self.assertIsInstance(nodes[1], Block)
self.assertEqual(nodes[1].tokens[0].value.text, '-- exemple')
self.assertEqual(nodes[1].tokens[1].value.text, '\n')
self.assertEqual(nodes[1].tokens[2].value.text, 'local')
self.assertEqual(nodes[1].tokens[3].value.text, ' ')
self.assertEqual(nodes[1].tokens[4].value.text, 'a')
self.assertEqual(nodes[1].tokens[5].value.text, ' ')
self.assertEqual(nodes[1].tokens[6].value.text, '=')
self.assertEqual(nodes[1].tokens[7].value.text, ' ')
self.assertEqual(nodes[1].tokens[8].value.text, '42')
self.assertIsInstance(nodes[2], LocalAssign)
self.assertEqual(nodes[1].tokens[0].value.text, '-- exemple')
self.assertEqual(nodes[1].tokens[1].value.text, '\n')
self.assertEqual(nodes[2].tokens[2].value.text, 'local')
self.assertEqual(nodes[2].tokens[3].value.text, ' ')
self.assertEqual(nodes[2].tokens[4].value.text, 'a')
self.assertEqual(nodes[2].tokens[5].value.text, ' ')
self.assertEqual(nodes[2].tokens[6].value.text, '=')
self.assertEqual(nodes[2].tokens[7].value.text, ' ')
self.assertEqual(nodes[2].tokens[8].value.text, '42')
self.assertIsInstance(nodes[3], Number)
self.assertEqual(nodes[3].tokens[0].value.text, ' ')
self.assertEqual(nodes[3].tokens[1].value.text, '42')
self.assertIsInstance(nodes[4], Name)
self.assertEqual(nodes[4].tokens[0].value.text, ' ')
self.assertEqual(nodes[4].tokens[1].value.text, 'a')
def test_walk_1(self):
src = textwrap.dedent("""
local a = 1
""")
tree = ast.parse(src)
chunk, block, local, name, number = False, False, False, False, False
for node in ast.walk(tree):
if isinstance(node, Chunk): chunk = True
if isinstance(node, Block): block = True
if isinstance(node, LocalAssign): local = True
if isinstance(node, Name): name = True
if isinstance(node, Number): number = True
self.assertTrue(chunk)
self.assertTrue(block)
self.assertTrue(local)
self.assertTrue(name)
self.assertTrue(number)
def test_cont_int_5(self):
tree = ast.parse(
textwrap.dedent("""
function table.pack(...)
repeat
print("value of a:", a)
a = a + 1;
until( a > 15 )
end
"""))
nodes = ast.walk(tree)
expected_cls = [
Chunk,
Block,
Function,
Index,
Name,
Name,
Varargs,
Block,
Repeat,
Block,
Call,
Name,
String,
Name,
Assign,
Name,
AddOp,
Name,
Number,
SemiColon,
GreaterThanOp,
Name,
Number,
]
for node, exp in zip(nodes, expected_cls):
self.assertIsInstance(node, exp)
from luaparser import ast
from luaparser import astnodes
a = ""
with open("test_kernels.lua", "r") as f:
a = f.read()
tree, builder = ast.parse(a)
#print(ast.to_lua_str(tree))
output = "import \"regent\"\n"
output += "require(\"src/particles/core_part\")\n"
output += "require(\"src/neighbour_search/cell_pair_tradequeues/import_cell_pair\")\n\n"
for node in ast.walk(tree):
if type(node) == astnodes.Particle_Type:
output += ast.to_lua_str(node)
output += "variables = {}\n"
output += "variables.config = regentlib.newsymbol(\"config\")\n"
output += "variables.particle_array = regentlib.newsymbol(\"particle_region\")\n"
output += "require(\"src/config/space\")\n"
output += "require(\"src/config/default_config\")\n"
output += "neighbour_init = require(\"src/neighbour_search/cell_pair_tradequeues/neighbour_init\")\n"
output += "require(\"src/neighbour_search/cell_pair_tradequeues/neighbour_search\")\n"
output += "require(\"src/neighbour_search/cell_pair_tradequeues/cell\")\n"
output += "require(\"src/io_modules/empty_io/initialisation\")\n"
output += ast.to_lua_str(tree)
print(output)
def test_for_in_1(self):
tree = ast.parse(
textwrap.dedent("""
for k, v in pairs({}) do
print(k, v)
end
"""))
exp = Chunk(
Block([
Forin(Block(
[Call(func=Name('print'), args=[Name('k'),
Name('v')])]),
iter=Call(func=Name('pairs'),
args=[Table(keys=[], values=[])]),
targets=[Name('k'), Name('v')])
]))
self.assertEqual(exp, tree)
# test tokens
nodes = list(ast.walk(tree))
self.assertIsInstance(nodes[2], Forin)
self.assertEqual(nodes[2].tokens[0].value.text, '\n')
self.assertEqual(nodes[2].tokens[1].value.text, 'for')
self.assertEqual(nodes[2].tokens[2].value.text, ' ')
self.assertEqual(nodes[2].tokens[3].value.text, 'k')
self.assertEqual(nodes[2].tokens[4].value.text, ',')
self.assertEqual(nodes[2].tokens[5].value.text, ' ')
self.assertEqual(nodes[2].tokens[6].value.text, 'v')
self.assertEqual(nodes[2].tokens[7].value.text, ' ')
self.assertEqual(nodes[2].tokens[8].value.text, 'in')
self.assertEqual(nodes[2].tokens[9].value.text, ' ')
self.assertEqual(nodes[2].tokens[10].value.text, 'pairs')
self.assertEqual(nodes[2].tokens[11].value.text, '(')
self.assertEqual(nodes[2].tokens[12].value.text, '{')
self.assertEqual(nodes[2].tokens[13].value.text, '}')
self.assertEqual(nodes[2].tokens[14].value.text, ')')
self.assertEqual(nodes[2].tokens[15].value.text, ' ')
self.assertEqual(nodes[2].tokens[16].value.text, 'do')
self.assertEqual(nodes[2].tokens[17].value.text, '\n')
self.assertEqual(nodes[2].tokens[18].value.text, ' ')
self.assertEqual(nodes[2].tokens[19].value.text, 'print')
self.assertEqual(nodes[2].tokens[20].value.text, '(')
self.assertEqual(nodes[2].tokens[21].value.text, 'k')
self.assertEqual(nodes[2].tokens[22].value.text, ',')
self.assertEqual(nodes[2].tokens[23].value.text, ' ')
self.assertEqual(nodes[2].tokens[24].value.text, 'v')
self.assertEqual(nodes[2].tokens[25].value.text, ')')
self.assertEqual(nodes[2].tokens[26].value.text, '\n')
self.assertEqual(nodes[2].tokens[27].value.text, 'end')
self.assertIsInstance(nodes[5], Call)
self.assertEqual(nodes[5].tokens[0].value.text, ' ')
self.assertEqual(nodes[5].tokens[1].value.text, 'pairs')
self.assertEqual(nodes[5].tokens[2].value.text, '(')
self.assertEqual(nodes[5].tokens[3].value.text, '{')
self.assertEqual(nodes[5].tokens[4].value.text, '}')
self.assertEqual(nodes[5].tokens[5].value.text, ')')
self.assertIsInstance(nodes[6], Table)
self.assertEqual(nodes[6].tokens[0].value.text, '{')
self.assertEqual(nodes[6].tokens[1].value.text, '}')
self.assertIsInstance(nodes[8], Block)
self.assertEqual(nodes[8].tokens[0].value.text, '\n')
self.assertEqual(nodes[8].tokens[1].value.text, ' ')
self.assertEqual(nodes[8].tokens[2].value.text, 'print')
self.assertEqual(nodes[8].tokens[3].value.text, '(')
self.assertEqual(nodes[8].tokens[4].value.text, 'k')
self.assertEqual(nodes[8].tokens[5].value.text, ',')
self.assertEqual(nodes[8].tokens[6].value.text, ' ')
self.assertEqual(nodes[8].tokens[7].value.text, 'v')
self.assertEqual(nodes[8].tokens[8].value.text, ')')
def test_if_elseif_elseif_else(self):
tree = ast.parse(
textwrap.dedent("""
if true then
elseif false then
elseif 42 then
else
return true
end
"""))
exp = Chunk(
Block([
If(test=TrueExpr(),
body=Block([]),
orelse=ElseIf(test=FalseExpr(),
body=Block([]),
orelse=ElseIf(test=Number(42),
body=Block([]),
orelse=Block(
[Return([TrueExpr()])]))))
]))
self.assertEqual(exp, tree)
# test tokens
nodes = list(ast.walk(tree))
self.assertIsInstance(nodes[2], If)
self.assertEqual(nodes[2].tokens[0].value.text, '\n')
self.assertEqual(nodes[2].tokens[1].value.text, 'if')
self.assertEqual(nodes[2].tokens[2].value.text, ' ')
self.assertEqual(nodes[2].tokens[3].value.text, 'true')
self.assertEqual(nodes[2].tokens[4].value.text, ' ')
self.assertEqual(nodes[2].tokens[5].value.text, 'then')
self.assertEqual(nodes[2].tokens[6].value.text, '\n')
self.assertEqual(nodes[2].tokens[7].value.text, 'elseif')
self.assertEqual(nodes[2].tokens[8].value.text, ' ')
self.assertEqual(nodes[2].tokens[9].value.text, 'false')
self.assertEqual(nodes[2].tokens[10].value.text, ' ')
self.assertEqual(nodes[2].tokens[11].value.text, 'then')
self.assertEqual(nodes[2].tokens[12].value.text, '\n')
self.assertEqual(nodes[2].tokens[13].value.text, 'elseif')
self.assertEqual(nodes[2].tokens[14].value.text, ' ')
self.assertEqual(nodes[2].tokens[15].value.text, '42')
self.assertEqual(nodes[2].tokens[16].value.text, ' ')
self.assertEqual(nodes[2].tokens[17].value.text, 'then')
self.assertEqual(nodes[2].tokens[18].value.text, '\n')
self.assertEqual(nodes[2].tokens[19].value.text, 'else')
self.assertEqual(nodes[2].tokens[20].value.text, '\n')
self.assertEqual(nodes[2].tokens[21].value.text, ' ')
self.assertEqual(nodes[2].tokens[22].value.text, 'return')
self.assertEqual(nodes[2].tokens[23].value.text, ' ')
self.assertEqual(nodes[2].tokens[24].value.text, 'true')
self.assertEqual(nodes[2].tokens[25].value.text, '\n')
self.assertEqual(nodes[2].tokens[26].value.text, 'end')
self.assertIsInstance(nodes[3], ElseIf)
self.assertEqual(nodes[3].tokens[0].value.text, '\n')
self.assertEqual(nodes[3].tokens[1].value.text, 'elseif')
self.assertEqual(nodes[3].tokens[2].value.text, ' ')
self.assertEqual(nodes[3].tokens[3].value.text, 'false')
self.assertEqual(nodes[3].tokens[4].value.text, ' ')
self.assertEqual(nodes[3].tokens[5].value.text, 'then')
self.assertIsInstance(nodes[4], ElseIf)
self.assertEqual(nodes[4].tokens[0].value.text, '\n')
self.assertEqual(nodes[4].tokens[1].value.text, 'elseif')
self.assertEqual(nodes[4].tokens[2].value.text, ' ')
self.assertEqual(nodes[4].tokens[3].value.text, '42')
self.assertEqual(nodes[4].tokens[4].value.text, ' ')
self.assertEqual(nodes[4].tokens[5].value.text, 'then')
def parse_lua(lua_file):
print('Decoding lua input file {0}...'.format(lua_file.name))
tree = ast.parse(lua_file.read())
block = {}
for node in ast.walk(tree):
if isinstance(node, astnodes.Block):
print('Contains {0} blocks'.format(len(node.body)))
for body_block in node.body:
key_name = body_block.targets[0].id
fields = body_block.values[0].fields
print('Decoding text block \'{0}\''.format(key_name))
print('Text block contains {0} entries'.format(len(fields)))
sub_blocks = {}
for index in range(len(fields)):
field = fields[index]
field_name = field.key.id
if isinstance(field.value, luaparser.astnodes.String):
field_value = decode_hex_utf8_string(field.value.s)
pass
elif isinstance(field.value, luaparser.astnodes.Concat):
values = []
tmp = field.value
while isinstance(tmp, astnodes.Concat):
if isinstance(tmp.left, astnodes.String):
values.append(tmp.left.s)
pass
values.append(tmp.right.s)
tmp = tmp.left
continue
value = []
if len(values) == 2:
value.append(decode_hex_utf8_string(values[0]))
value.append(decode_hex_utf8_string(values[1]))
pass
elif len(values) == 3:
value.append(decode_hex_utf8_string(values[1]))
value.append(decode_hex_utf8_string(values[2]))
value.append(decode_hex_utf8_string(values[0]))
pass
else:
print(
'More lines than expected in lua script file!')
pass
field_value = value
pass
sub_blocks[index] = {
field_name: field_value
} # Add our mapped value
continue
block.update({key_name: sub_blocks})
continue
continue
continue
return block