def test_function_def_anonymous(self): tree = ast.parse(r'f = function() local a end') exp = Chunk( Block([ Assign(targets=[Name('f')], values=[ AnonymousFunction(args=[], body=Block([ LocalAssign( targets=[Name('a')], values=[]) ])) ]) ])) self.assertEqual(exp, tree) # test tokens nodes = list(ast.walk(tree)) self.assertIsInstance(nodes[3], AnonymousFunction) self.assertEqual(nodes[3].tokens[0].value.text, ' ') self.assertEqual(nodes[3].tokens[1].value.text, 'function') self.assertEqual(nodes[3].tokens[2].value.text, '(') self.assertEqual(nodes[3].tokens[3].value.text, ')') self.assertEqual(nodes[3].tokens[4].value.text, ' ') self.assertEqual(nodes[3].tokens[5].value.text, 'local') self.assertEqual(nodes[3].tokens[6].value.text, ' ') self.assertEqual(nodes[3].tokens[7].value.text, 'a') self.assertEqual(nodes[3].tokens[8].value.text, ' ') self.assertEqual(nodes[3].tokens[9].value.text, 'end')
def process(self) -> None: total_processed: int = 0 for node in ast.walk(self.luatree): if isinstance(node, astnodes.Field) and hasattr(field.key, 'n'): self.process_field(node) total_processed += 1 print(f'Processed total {total_processed} nodes')
def test_function_nested_invoke(self): tree = ast.parse(r'foo:bar():print("hello")') exp = Chunk( Block([ Invoke(source=Invoke(source=Name('foo'), func=Name('bar'), args=[]), func=Name('print'), args=[String('hello')]) ])) self.assertEqual(exp, tree) # test tokens nodes = list(ast.walk(tree)) self.assertIsInstance(nodes[2], Invoke) self.assertEqual(nodes[2].tokens[0].value.text, 'foo') self.assertEqual(nodes[2].tokens[1].value.text, ':') self.assertEqual(nodes[2].tokens[2].value.text, 'bar') self.assertEqual(nodes[2].tokens[3].value.text, '(') self.assertEqual(nodes[2].tokens[4].value.text, ')') self.assertEqual(nodes[2].tokens[5].value.text, ':') self.assertEqual(nodes[2].tokens[6].value.text, 'print') self.assertEqual(nodes[2].tokens[7].value.text, '(') self.assertEqual(nodes[2].tokens[8].value.text, '"hello"') self.assertEqual(nodes[2].tokens[9].value.text, ')') self.assertIsInstance(nodes[5], Invoke) self.assertEqual(nodes[5].tokens[0].value.text, 'foo') self.assertEqual(nodes[5].tokens[1].value.text, ':') self.assertEqual(nodes[5].tokens[2].value.text, 'bar') self.assertEqual(nodes[5].tokens[3].value.text, '(') self.assertEqual(nodes[5].tokens[4].value.text, ')')
def test_dict(self): tree = ast.parse(r'a = {foo = "bar", bar = "foo"}') exp = Chunk( Block([ Assign(targets=[Name('a')], values=[ Table(keys=[Name('foo'), Name('bar')], values=[String('bar'), String('foo')]) ]) ])) self.assertEqual(exp, tree) # test tokens nodes = list(ast.walk(tree)) self.assertIsInstance(nodes[3], Table) self.assertEqual(nodes[3].tokens[0].value.text, ' ') self.assertEqual(nodes[3].tokens[1].value.text, '{') self.assertEqual(nodes[3].tokens[2].value.text, 'foo') self.assertEqual(nodes[3].tokens[3].value.text, ' ') self.assertEqual(nodes[3].tokens[4].value.text, '=') self.assertEqual(nodes[3].tokens[5].value.text, ' ') self.assertEqual(nodes[3].tokens[6].value.text, '"bar"') self.assertEqual(nodes[3].tokens[7].value.text, ',') self.assertEqual(nodes[3].tokens[8].value.text, ' ') self.assertEqual(nodes[3].tokens[9].value.text, 'bar') self.assertEqual(nodes[3].tokens[10].value.text, ' ') self.assertEqual(nodes[3].tokens[11].value.text, '=') self.assertEqual(nodes[3].tokens[12].value.text, ' ') self.assertEqual(nodes[3].tokens[13].value.text, '"foo"') self.assertEqual(nodes[3].tokens[14].value.text, '}')
def test_addition(self): tree = ast.parse(r'a = 1 + 0.2') exp = Chunk( Block([ Assign(targets=[Name('a')], values=[AddOp(left=Number(1), right=Number(0.2))]) ])) self.assertEqual(exp, tree) # test tokens nodes = list(ast.walk(tree)) self.assertIsInstance(nodes[0], Chunk) self.assertEqual(nodes[0].tokens[0].value.text, 'a') self.assertEqual(nodes[0].tokens[1].value.text, ' ') self.assertEqual(nodes[0].tokens[2].value.text, '=') self.assertEqual(nodes[0].tokens[3].value.text, ' ') self.assertEqual(nodes[0].tokens[4].value.text, '1') self.assertEqual(nodes[0].tokens[5].value.text, ' ') self.assertEqual(nodes[0].tokens[6].value.text, '+') self.assertEqual(nodes[0].tokens[7].value.text, ' ') self.assertEqual(nodes[0].tokens[8].value.text, '0.2') self.assertIsInstance(nodes[1], Block) self.assertEqual(nodes[1].tokens[0].value.text, 'a') self.assertEqual(nodes[1].tokens[1].value.text, ' ') self.assertEqual(nodes[1].tokens[2].value.text, '=') self.assertEqual(nodes[1].tokens[3].value.text, ' ') self.assertEqual(nodes[1].tokens[4].value.text, '1') self.assertEqual(nodes[1].tokens[5].value.text, ' ') self.assertEqual(nodes[1].tokens[6].value.text, '+') self.assertEqual(nodes[1].tokens[7].value.text, ' ') self.assertEqual(nodes[1].tokens[8].value.text, '0.2') self.assertIsInstance(nodes[2], Assign) self.assertEqual(nodes[2].tokens[0].value.text, 'a') self.assertEqual(nodes[2].tokens[1].value.text, ' ') self.assertEqual(nodes[2].tokens[2].value.text, '=') self.assertEqual(nodes[2].tokens[3].value.text, ' ') self.assertEqual(nodes[2].tokens[4].value.text, '1') self.assertEqual(nodes[2].tokens[5].value.text, ' ') self.assertEqual(nodes[2].tokens[6].value.text, '+') self.assertEqual(nodes[2].tokens[7].value.text, ' ') self.assertEqual(nodes[2].tokens[8].value.text, '0.2') self.assertIsInstance(nodes[3], AddOp) self.assertEqual(nodes[3].tokens[0].value.text, ' ') self.assertEqual(nodes[3].tokens[1].value.text, '1') self.assertEqual(nodes[3].tokens[2].value.text, ' ') self.assertEqual(nodes[3].tokens[3].value.text, '+') self.assertEqual(nodes[3].tokens[4].value.text, ' ') self.assertEqual(nodes[3].tokens[5].value.text, '0.2') self.assertIsInstance(nodes[4], Number) self.assertEqual(nodes[4].tokens[0].value.text, ' ') self.assertEqual(nodes[4].tokens[1].value.text, '0.2') self.assertIsInstance(nodes[5], Number) self.assertEqual(nodes[5].tokens[0].value.text, ' ') self.assertEqual(nodes[5].tokens[1].value.text, '1')
def test_function_call_simple(self): tree = ast.parse(r'print("hello")') exp = Chunk(Block([Call(func=Name('print'), args=[String('hello')])])) self.assertEqual(exp, tree) # test tokens nodes = list(ast.walk(tree)) self.assertIsInstance(nodes[2], Call) self.assertEqual(nodes[2].tokens[0].value.text, 'print') self.assertEqual(nodes[2].tokens[1].value.text, '(') self.assertEqual(nodes[2].tokens[2].value.text, '"hello"') self.assertEqual(nodes[2].tokens[3].value.text, ')')
def test_length_op(self): tree = ast.parse(r'len = #t') exp = Chunk( Block([ Assign(targets=[Name('len')], values=[ULengthOP(operand=Name('t'))]) ])) self.assertEqual(exp, tree) # test tokens nodes = list(ast.walk(tree)) self.assertIsInstance(nodes[3], ULengthOP) self.assertEqual(nodes[3].tokens[0].value.text, ' ') self.assertEqual(nodes[3].tokens[1].value.text, '#') self.assertEqual(nodes[3].tokens[2].value.text, 't')
def test_label(self): tree = ast.parse( textwrap.dedent(""" ::foo:: """)) exp = Chunk(Block([Label(Name('foo'))])) self.assertEqual(exp, tree) # test tokens nodes = list(ast.walk(tree)) self.assertIsInstance(nodes[2], Label) self.assertEqual(nodes[2].tokens[0].value.text, '\n') self.assertEqual(nodes[2].tokens[1].value.text, '::') self.assertEqual(nodes[2].tokens[2].value.text, 'foo') self.assertEqual(nodes[2].tokens[3].value.text, '::')
def test_ast_tokens_1(self): src = textwrap.dedent("""\ -- exemple local a = 42""") tree = ast.parse(src) nodes = list(ast.walk(tree)) self.assertIsInstance(nodes[0], Chunk) self.assertEqual(nodes[0].tokens[0].value.text, '-- exemple') self.assertEqual(nodes[0].tokens[1].value.text, '\n') self.assertEqual(nodes[0].tokens[2].value.text, 'local') self.assertEqual(nodes[0].tokens[3].value.text, ' ') self.assertEqual(nodes[0].tokens[4].value.text, 'a') self.assertEqual(nodes[0].tokens[5].value.text, ' ') self.assertEqual(nodes[0].tokens[6].value.text, '=') self.assertEqual(nodes[0].tokens[7].value.text, ' ') self.assertEqual(nodes[0].tokens[8].value.text, '42') self.assertIsInstance(nodes[1], Block) self.assertEqual(nodes[1].tokens[0].value.text, '-- exemple') self.assertEqual(nodes[1].tokens[1].value.text, '\n') self.assertEqual(nodes[1].tokens[2].value.text, 'local') self.assertEqual(nodes[1].tokens[3].value.text, ' ') self.assertEqual(nodes[1].tokens[4].value.text, 'a') self.assertEqual(nodes[1].tokens[5].value.text, ' ') self.assertEqual(nodes[1].tokens[6].value.text, '=') self.assertEqual(nodes[1].tokens[7].value.text, ' ') self.assertEqual(nodes[1].tokens[8].value.text, '42') self.assertIsInstance(nodes[2], LocalAssign) self.assertEqual(nodes[1].tokens[0].value.text, '-- exemple') self.assertEqual(nodes[1].tokens[1].value.text, '\n') self.assertEqual(nodes[2].tokens[2].value.text, 'local') self.assertEqual(nodes[2].tokens[3].value.text, ' ') self.assertEqual(nodes[2].tokens[4].value.text, 'a') self.assertEqual(nodes[2].tokens[5].value.text, ' ') self.assertEqual(nodes[2].tokens[6].value.text, '=') self.assertEqual(nodes[2].tokens[7].value.text, ' ') self.assertEqual(nodes[2].tokens[8].value.text, '42') self.assertIsInstance(nodes[3], Number) self.assertEqual(nodes[3].tokens[0].value.text, ' ') self.assertEqual(nodes[3].tokens[1].value.text, '42') self.assertIsInstance(nodes[4], Name) self.assertEqual(nodes[4].tokens[0].value.text, ' ') self.assertEqual(nodes[4].tokens[1].value.text, 'a')
def test_walk_1(self): src = textwrap.dedent(""" local a = 1 """) tree = ast.parse(src) chunk, block, local, name, number = False, False, False, False, False for node in ast.walk(tree): if isinstance(node, Chunk): chunk = True if isinstance(node, Block): block = True if isinstance(node, LocalAssign): local = True if isinstance(node, Name): name = True if isinstance(node, Number): number = True self.assertTrue(chunk) self.assertTrue(block) self.assertTrue(local) self.assertTrue(name) self.assertTrue(number)
def test_cont_int_5(self): tree = ast.parse( textwrap.dedent(""" function table.pack(...) repeat print("value of a:", a) a = a + 1; until( a > 15 ) end """)) nodes = ast.walk(tree) expected_cls = [ Chunk, Block, Function, Index, Name, Name, Varargs, Block, Repeat, Block, Call, Name, String, Name, Assign, Name, AddOp, Name, Number, SemiColon, GreaterThanOp, Name, Number, ] for node, exp in zip(nodes, expected_cls): self.assertIsInstance(node, exp)
from luaparser import ast from luaparser import astnodes a = "" with open("test_kernels.lua", "r") as f: a = f.read() tree, builder = ast.parse(a) #print(ast.to_lua_str(tree)) output = "import \"regent\"\n" output += "require(\"src/particles/core_part\")\n" output += "require(\"src/neighbour_search/cell_pair_tradequeues/import_cell_pair\")\n\n" for node in ast.walk(tree): if type(node) == astnodes.Particle_Type: output += ast.to_lua_str(node) output += "variables = {}\n" output += "variables.config = regentlib.newsymbol(\"config\")\n" output += "variables.particle_array = regentlib.newsymbol(\"particle_region\")\n" output += "require(\"src/config/space\")\n" output += "require(\"src/config/default_config\")\n" output += "neighbour_init = require(\"src/neighbour_search/cell_pair_tradequeues/neighbour_init\")\n" output += "require(\"src/neighbour_search/cell_pair_tradequeues/neighbour_search\")\n" output += "require(\"src/neighbour_search/cell_pair_tradequeues/cell\")\n" output += "require(\"src/io_modules/empty_io/initialisation\")\n" output += ast.to_lua_str(tree) print(output)
def test_for_in_1(self): tree = ast.parse( textwrap.dedent(""" for k, v in pairs({}) do print(k, v) end """)) exp = Chunk( Block([ Forin(Block( [Call(func=Name('print'), args=[Name('k'), Name('v')])]), iter=Call(func=Name('pairs'), args=[Table(keys=[], values=[])]), targets=[Name('k'), Name('v')]) ])) self.assertEqual(exp, tree) # test tokens nodes = list(ast.walk(tree)) self.assertIsInstance(nodes[2], Forin) self.assertEqual(nodes[2].tokens[0].value.text, '\n') self.assertEqual(nodes[2].tokens[1].value.text, 'for') self.assertEqual(nodes[2].tokens[2].value.text, ' ') self.assertEqual(nodes[2].tokens[3].value.text, 'k') self.assertEqual(nodes[2].tokens[4].value.text, ',') self.assertEqual(nodes[2].tokens[5].value.text, ' ') self.assertEqual(nodes[2].tokens[6].value.text, 'v') self.assertEqual(nodes[2].tokens[7].value.text, ' ') self.assertEqual(nodes[2].tokens[8].value.text, 'in') self.assertEqual(nodes[2].tokens[9].value.text, ' ') self.assertEqual(nodes[2].tokens[10].value.text, 'pairs') self.assertEqual(nodes[2].tokens[11].value.text, '(') self.assertEqual(nodes[2].tokens[12].value.text, '{') self.assertEqual(nodes[2].tokens[13].value.text, '}') self.assertEqual(nodes[2].tokens[14].value.text, ')') self.assertEqual(nodes[2].tokens[15].value.text, ' ') self.assertEqual(nodes[2].tokens[16].value.text, 'do') self.assertEqual(nodes[2].tokens[17].value.text, '\n') self.assertEqual(nodes[2].tokens[18].value.text, ' ') self.assertEqual(nodes[2].tokens[19].value.text, 'print') self.assertEqual(nodes[2].tokens[20].value.text, '(') self.assertEqual(nodes[2].tokens[21].value.text, 'k') self.assertEqual(nodes[2].tokens[22].value.text, ',') self.assertEqual(nodes[2].tokens[23].value.text, ' ') self.assertEqual(nodes[2].tokens[24].value.text, 'v') self.assertEqual(nodes[2].tokens[25].value.text, ')') self.assertEqual(nodes[2].tokens[26].value.text, '\n') self.assertEqual(nodes[2].tokens[27].value.text, 'end') self.assertIsInstance(nodes[5], Call) self.assertEqual(nodes[5].tokens[0].value.text, ' ') self.assertEqual(nodes[5].tokens[1].value.text, 'pairs') self.assertEqual(nodes[5].tokens[2].value.text, '(') self.assertEqual(nodes[5].tokens[3].value.text, '{') self.assertEqual(nodes[5].tokens[4].value.text, '}') self.assertEqual(nodes[5].tokens[5].value.text, ')') self.assertIsInstance(nodes[6], Table) self.assertEqual(nodes[6].tokens[0].value.text, '{') self.assertEqual(nodes[6].tokens[1].value.text, '}') self.assertIsInstance(nodes[8], Block) self.assertEqual(nodes[8].tokens[0].value.text, '\n') self.assertEqual(nodes[8].tokens[1].value.text, ' ') self.assertEqual(nodes[8].tokens[2].value.text, 'print') self.assertEqual(nodes[8].tokens[3].value.text, '(') self.assertEqual(nodes[8].tokens[4].value.text, 'k') self.assertEqual(nodes[8].tokens[5].value.text, ',') self.assertEqual(nodes[8].tokens[6].value.text, ' ') self.assertEqual(nodes[8].tokens[7].value.text, 'v') self.assertEqual(nodes[8].tokens[8].value.text, ')')
def test_if_elseif_elseif_else(self): tree = ast.parse( textwrap.dedent(""" if true then elseif false then elseif 42 then else return true end """)) exp = Chunk( Block([ If(test=TrueExpr(), body=Block([]), orelse=ElseIf(test=FalseExpr(), body=Block([]), orelse=ElseIf(test=Number(42), body=Block([]), orelse=Block( [Return([TrueExpr()])])))) ])) self.assertEqual(exp, tree) # test tokens nodes = list(ast.walk(tree)) self.assertIsInstance(nodes[2], If) self.assertEqual(nodes[2].tokens[0].value.text, '\n') self.assertEqual(nodes[2].tokens[1].value.text, 'if') self.assertEqual(nodes[2].tokens[2].value.text, ' ') self.assertEqual(nodes[2].tokens[3].value.text, 'true') self.assertEqual(nodes[2].tokens[4].value.text, ' ') self.assertEqual(nodes[2].tokens[5].value.text, 'then') self.assertEqual(nodes[2].tokens[6].value.text, '\n') self.assertEqual(nodes[2].tokens[7].value.text, 'elseif') self.assertEqual(nodes[2].tokens[8].value.text, ' ') self.assertEqual(nodes[2].tokens[9].value.text, 'false') self.assertEqual(nodes[2].tokens[10].value.text, ' ') self.assertEqual(nodes[2].tokens[11].value.text, 'then') self.assertEqual(nodes[2].tokens[12].value.text, '\n') self.assertEqual(nodes[2].tokens[13].value.text, 'elseif') self.assertEqual(nodes[2].tokens[14].value.text, ' ') self.assertEqual(nodes[2].tokens[15].value.text, '42') self.assertEqual(nodes[2].tokens[16].value.text, ' ') self.assertEqual(nodes[2].tokens[17].value.text, 'then') self.assertEqual(nodes[2].tokens[18].value.text, '\n') self.assertEqual(nodes[2].tokens[19].value.text, 'else') self.assertEqual(nodes[2].tokens[20].value.text, '\n') self.assertEqual(nodes[2].tokens[21].value.text, ' ') self.assertEqual(nodes[2].tokens[22].value.text, 'return') self.assertEqual(nodes[2].tokens[23].value.text, ' ') self.assertEqual(nodes[2].tokens[24].value.text, 'true') self.assertEqual(nodes[2].tokens[25].value.text, '\n') self.assertEqual(nodes[2].tokens[26].value.text, 'end') self.assertIsInstance(nodes[3], ElseIf) self.assertEqual(nodes[3].tokens[0].value.text, '\n') self.assertEqual(nodes[3].tokens[1].value.text, 'elseif') self.assertEqual(nodes[3].tokens[2].value.text, ' ') self.assertEqual(nodes[3].tokens[3].value.text, 'false') self.assertEqual(nodes[3].tokens[4].value.text, ' ') self.assertEqual(nodes[3].tokens[5].value.text, 'then') self.assertIsInstance(nodes[4], ElseIf) self.assertEqual(nodes[4].tokens[0].value.text, '\n') self.assertEqual(nodes[4].tokens[1].value.text, 'elseif') self.assertEqual(nodes[4].tokens[2].value.text, ' ') self.assertEqual(nodes[4].tokens[3].value.text, '42') self.assertEqual(nodes[4].tokens[4].value.text, ' ') self.assertEqual(nodes[4].tokens[5].value.text, 'then')
def parse_lua(lua_file): print('Decoding lua input file {0}...'.format(lua_file.name)) tree = ast.parse(lua_file.read()) block = {} for node in ast.walk(tree): if isinstance(node, astnodes.Block): print('Contains {0} blocks'.format(len(node.body))) for body_block in node.body: key_name = body_block.targets[0].id fields = body_block.values[0].fields print('Decoding text block \'{0}\''.format(key_name)) print('Text block contains {0} entries'.format(len(fields))) sub_blocks = {} for index in range(len(fields)): field = fields[index] field_name = field.key.id if isinstance(field.value, luaparser.astnodes.String): field_value = decode_hex_utf8_string(field.value.s) pass elif isinstance(field.value, luaparser.astnodes.Concat): values = [] tmp = field.value while isinstance(tmp, astnodes.Concat): if isinstance(tmp.left, astnodes.String): values.append(tmp.left.s) pass values.append(tmp.right.s) tmp = tmp.left continue value = [] if len(values) == 2: value.append(decode_hex_utf8_string(values[0])) value.append(decode_hex_utf8_string(values[1])) pass elif len(values) == 3: value.append(decode_hex_utf8_string(values[1])) value.append(decode_hex_utf8_string(values[2])) value.append(decode_hex_utf8_string(values[0])) pass else: print( 'More lines than expected in lua script file!') pass field_value = value pass sub_blocks[index] = { field_name: field_value } # Add our mapped value continue block.update({key_name: sub_blocks}) continue continue continue return block