def main():
import argparse
import dis
try:
from astpretty import pprint
except ImportError:
pprint = ast.dump
parser = argparse.ArgumentParser()
parser.add_argument('--dis',
'-d',
default=False,
action="store_true",
help="disassemble bytecode")
parser.add_argument('--ast',
'-a',
default=False,
action="store_true",
help="dump AST")
parser.add_argument('files', nargs='+')
args = parser.parse_args()
for fn in args.files:
with open(fn, 'rb') as fp:
buf = fp.read()
tree = parse(buf, fn)
if args.ast:
pprint(tree)
if args.dis:
co = compile(tree, fn, 'exec')
dis.dis(co)
def get_td(self, expr, debug=False):
tree = ast.parse(expr)
if debug:
astpretty.pprint(tree)
tds = TypeDeducerState({})
td = TypeDeducer(tds)
td.visit(tree)
return td
def myprint(x, *args, **kwargs):
if isinstance(x, ast.AST):
try:
astpretty.pprint(x, *args, **kwargs)
except:
print("Failed pretty printing AST")
_print(x, *args, **kwargs)
else:
_print(x, *args, **kwargs)
def mod_ast(func):
src, line_num = inspect.getsourcelines(func)
src_file = inspect.getabsfile(func)
# print(src_file)
# remove global indent
indent = len(mod_ast_re_indent.match(src[0]).group(1))
src = [l[indent:] for l in src]
src = reduce(lambda x, y: x+y, src)
# print('\nsources of {}'.format(func.__name__))
# for l in src: print(l, end='')
# pad with empty lines to match line numbering
src = '\n'*(line_num-1) + src
# get ast
a = ast.parse(src)
# remove the first decorator applied, presumably this one
a.body[0].decorator_list = a.body[0].decorator_list[:-1]
# # inherit caller locals callback
# mod_ast_inherit_locals(a.body[0])
print('='*10+'ast before')
print(astpretty.pprint(a))
for mod in mod_ast_modifiers: a = mod.visit(a)
print('='*10+'ast after')
print(astpretty.pprint(a))
# compile and exec code in previous global env
c = compile(a, src_file, mode='exec')
caller_globals = inspect.stack()[1][0].f_globals
caller_locals = inspect.stack()[1][0].f_locals
global mod_ast_last_caller_locals
mod_ast_last_caller_locals = caller_locals
for (v, k) in mod_ast_global_modifiers: caller_globals[k] = v
# pour locals in global
# allow sub-funcs to refer to funcs / class declared in the parent func
caller_globals = dict(caller_globals, **caller_locals)
# print('in mod, locals', caller_locals)
# print('in mod, global', caller_globals)
exec(c, caller_globals, caller_locals)
modfunc = caller_locals[func.__name__]
return modfunc
def func(*args, **kw):
code_str = inspect.getsource(method)
print(code_str)
code_ast = ast.parse(code_str)
func_def = code_ast.body[0]
func_name = func_def.name
func_args = func_def.args.args
func_body = func_def.body
cuda = OPPU_REGISTRY.get(func_name)
if cuda:
cuda.exec_cython()
cuda = Cuda(func_name)
astpretty.pprint(code_ast)
for func_arg in func_args:
arg_name = func_arg.arg
arg_type = func_arg.annotation
parse_ast_arg(arg_name, arg_type, cuda)
kernel_depth = 0
for statement in func_body:
if type(statement) == ast.AnnAssign:
arg_name = statement.target.id
arg_type = statement.annotation
parse_ast_arg(arg_name, arg_type, cuda)
elif type(statement) == ast.For:
if kernel_depth == 0:
kernel_depth = cuda.get_dim_min()
print("kernel_depth: {}".format(kernel_depth))
targets = []
iters = []
st = statement
for _ in range(kernel_depth):
targets.append(st.target.id)
iters.append(st.iter.args[-1].id)
st = st.body[0]
targets.append(st.target.id)
iters.append(st.iter.args[-1].id)
# for s in st.body:
print("inputs: {}".format(len(cuda.inputs)))
for arg in cuda.inputs:
print(arg.name, arg.get_full_type())
print("outputs: {}".format(len(cuda.outputs)))
for arg in cuda.outputs:
print(arg.name, arg.get_full_type())
print("params: {}".format(len(cuda.params)))
for arg in cuda.params:
print(arg.name, arg.get_full_type())
print("dims: {}".format(len(cuda.dims)))
for arg in cuda.dims:
print(arg.name, arg.get_full_type())
render("cuda_h.j2", cuda)
def pp(src: str):
""" Pretty print of AST """
try:
import astpretty
astpretty.pprint(ast.parse(src), show_offsets=False)
except ImportError:
print(
"Warning: couldn't find module 'astpretty'; falling back to ast.dump()"
)
ast.dump(ast.parse(src))
def get_tree(source_data, ext):
is_script = has_main(source_data)
if ext in [".dart", ".kt", ".rs"] and not is_script:
source_data = f"if __name__ == '__main__':\n {source_data}"
print(f">{source_data}<")
tree = ast.parse(source_data)
astpretty.pprint(tree)
proc = run([sys.executable, "-c", source_data], capture_output=True)
if proc.returncode:
raise RuntimeError(f"Invalid case {source_data}:\n{proc.stdout}{proc.stderr}")
return tree
def main():
# input_stream = FileStream('./sql_stmt.sql')
# lexer = sqlLexer(input_stream)
# stream = CommonTokenStream(lexer)
# parser = sqlParser(stream)
# tree = parser.root()
#
# t = Trees.Trees.toStringTree(tree, parser.ruleNames)
# print(t)
ast_tree = parse(read_file('./sql_stmt.sql'))
astpretty.pprint(ast_tree)
def visitProgram(self, ir):
python_nodes = [element.accept(self) for element in ir.body]
body = [self._ensureStmt(node) for node in python_nodes if node]
module = ast.Module()
module.body = [
self.import_('torch'),
self.import_('pyro'),
self.import_('pyro.distributions', 'dist')
]
module.body.append(self.buildModel([x for x in body]))
ast.fix_missing_locations(module)
astpretty.pprint(module)
print(astor.to_source(module))
return module
def eval_attribute (a):
if isinstance(a.value, ast.Name):
receiver = a.value.id
ret = receiver + '.' + a.attr
elif isinstance(a.value, ast.Attribute):
prefix = eval_attribute(a.value)
ret = prefix + '.' + a.attr
else:
import astpretty
astpretty.pprint(a)
print (f'{type(a.value)}')
raise NotImplementedError
return ret
def get_expression_part_info(node: ast.AST) -> Mapping[str, Any]:
node_type_sid = None
for types, node_type_name in TYPES_MAP:
if isinstance(node, types): # type: ignore
node_type_sid = node_type_name
break
else:
pprint(node) # noqa
raise AssertionError('should always get node type')
return {
'type': node_type_sid,
'subnodes': _get_sub_nodes(node, node_type_sid),
}
def optimize_comprehensions(func):
source = inspect.getsource(func)
in_node = parse(source)
astpretty.pprint(in_node)
out_node = OptimizeComprehensions().visit(in_node)
print("Modified AST")
astpretty.pprint(out_node)
new_func_name = out_node.body[0].name
func_scope = func.__globals__
# Compile the new method in the old methods scope
exec(compile(out_node, '<string>', 'exec'), func_scope)
return func_scope[new_func_name]
def main(argv):
with open(argv[1], 'r') as f:
start = time.time()
tree = ast.parse(f.read())
end = time.time()
print(end - start)
if (argv[0] == "pprint"):
astpretty.pprint(tree)
elif (argv[0] == "json"):
json_str = serialize(tree)
with open(argv[2], 'w') as output_f:
output_f.write(json_str)
def exitProgram(self, ctx):
ctx.ast = Module()
ctx.ast.body = [
Import(names=[alias(name='torch', asname=None)]),
ImportFrom(module='torch.distributions',
names=[alias(name='*', asname=None)],
level=0)
]
ctx.ast.body += gatherChildrenASTList(ctx)
ast.fix_missing_locations(ctx.ast)
astpretty.pprint(ctx.ast)
print('\n-----------------\n')
print(astor.to_source(ctx.ast))
print('\n-----------------\n')
exec(compile(ctx.ast, filename="<ast>", mode="exec"))
def easy_debug(code: str, should_exec=False, ctx=None):
res = to_tagged_ast(parse(code).result)
c = py_compile(res)
print("-----------code")
print(code)
print("-----------Python")
print(dis.dis(code))
print("-----------YaPyPy")
print(dis.dis(c))
print("-----------astpretty")
astpretty.pprint(ast.parse(code))
print("----------- Python exec result")
exec(code, ctx or {})
print("-----------YaPyPy exec result")
if should_exec:
exec(c, ctx or {})
else:
print("\t(skip)")
def test_fstring(self):
xml = Parser(self.get_xml("dict.xml"))
module = xml.parse()
code = compile(module, "<ast>", "exec")
mymodule = ast.Module([
ast.Assign([ast.Name('name', ast.Store())], ast.Str('Batuhan')),
ast.Assign([ast.Name('age', ast.Store())], ast.Num(15)),
ast.Expr(
ast.Call(ast.Name('print', ast.Load()), [
ast.JoinedStr(values=[
ast.Str(s='Benim adım'),
ast.FormattedValue(
value=ast.Name(id='name', ctx=ast.Load()),
conversion=-1,
format_spec=None,
),
ast.Str(s=', yaşım'),
ast.FormattedValue(
value=ast.Name(id='age', ctx=ast.Load()),
conversion=-1,
format_spec=None,
),
ast.Str(s='. Hakkımda geri kalan bilgi:'),
ast.FormattedValue(
value=ast.Call(
func=ast.Name(id='str', ctx=ast.Load()),
args=[ast.Num(n=235)],
keywords=[],
),
conversion=-1,
format_spec=None,
),
ast.Str(s=''),
])
], []))
])
ast.fix_missing_locations(mymodule)
pprint(module)
mycode = compile(mymodule, "<ast>", "exec")
self.assertEqual(code, mycode)
def pprint_ast_expr(exp: Union[str, ast3.AST, ast.AST],
only_expr: bool = True,
fix_missing_locations: bool = False) -> None:
if isinstance(exp, str):
exp = ast3.parse(exp)
if only_expr:
exp = exp.body[0] # type: ignore
if isinstance(exp, ast3.AST):
exp = typed_ast3_to_ast(exp)
if not isinstance(exp, ast.AST):
raise Exception(
"I cannot print the type {}, it isn't a type of an AST I know".
format(exp))
if fix_missing_locations:
exp = ast.fix_missing_locations(exp)
import astpretty
astpretty.pprint(exp)
def test_simple_types(self):
# string, integer, float, force casting
xml = Parser(self.get_xml("basic_types.xml"))
module = xml.parse()
code = compile(module, "<ast>", "exec")
mymodule = ast.Module([
ast.Expr(ast.Str("batuhan")),
ast.Expr(ast.Num(15)),
ast.Expr(ast.Num(15.5)),
ast.Expr(ast.Str("13")),
ast.Expr(ast.Ellipsis()),
ast.Expr(ast.Bytes(bytes("a", "ASCII"))),
ast.Expr(ast.Bytes(bytes("ş", "utf-8"))),
])
ast.fix_missing_locations(mymodule)
mycode = compile(mymodule, "<ast>", "exec")
pprint(mymodule)
self.assertEqual(code, mycode)
def print_tree(tree,message,verbose,buildKeyStep=False):
'''
Helper function to print trees mid-optimization.
We can only do buildKeyStep is verbose is true as well.
'''
# print('print_tree')
# print(f'{verbose} is verbose')
if verbose:
print('*'*30)
print(message)
astpretty.pprint(tree)
print('*'*30)
else:
buildKeyStep=False
build_key_step(message,buildKeyStep)
def print_obj(obj,message,verbose):
'''
Pretty-prints the object.
'''
if verbose:
print(message)
# We print out individual trees.
if is_list(obj):
for t in obj:
print('='*30)
if isinstance(t,AST):
print('printing fArg tree')
try:
astpretty.pprint(t)
except Exception as e:
print(ast.dump(t))
else:
print('printing fArg')
pp.pprint(t)
elif is_string(obj) or is_int(obj) or is_float(obj):
print(obj)
else:
pp.pprint(obj)
class GlobalVariable:
__name__ = "GlobalVariable"
class Arg:
__name__ = "Arg"
class Kwarg:
__name__ = "Kwarg"
class Base:
__name__ = "Base"
if __name__ == '__main__':
import astpretty
import ast
s = """
async def run():
conn = await asyncpg.connect(user='******', password='******',
database='database', host='127.0.0.1')
values = await conn.fetch('''SELECT * FROM mytable''')
await conn.close()
"""
print(astpretty.pprint(ast.parse(s).body[0]))
import ast import astor import codegen import astpretty import sys import pprint # parse the target program to ast target_path = "./ast2.py" source = open(target_path).read() tree = ast.parse(source, target_path) astpretty.pprint(tree) #print(ast.dump(tree)) print (astor.to_source(tree)) #astpretty.pprint(tree) #pprint.pprint(ast.dump(tree))
# moshmosh?
# +template-python
@quote
def f(x):
x + 1
x = y + 1
from moshmosh.ast_compat import ast
from astpretty import pprint
stmts = f(ast.Name("a"))
pprint(ast.fix_missing_locations(stmts[0]))
pprint(ast.fix_missing_locations(stmts[1]))
import ast
import astpretty
source="""
def functionsample(a,b):
c = a
d = b
return c,d
"""
node_structure = ast.parse(source)
print(node_structure)
astpretty.pprint(node_structure)
import ast
content = open("test.py").read()
#print ast.dump( ast.parse(content) )
import astpretty
astpretty.pprint(ast.parse(content))
#!/usr/bin/python3
# This script outputs the AST of its first argument
import sys
import astpretty
import ast
if __name__ == '__main__':
tree = ast.parse(sys.argv[1])
astpretty.pprint(tree, show_offsets=False, indent=' ')
def optimized(*args):
'''
If we've already compiled this function, then just grab it from the
function cache and evaluate it.
'''
if originalFuncName in FUNCTION_CACHE:
return FUNCTION_CACHE[originalFuncName](*args)
'''
First, we get a tree from the source code.
'''
tree=parse(src)
if verbose:
print('*'*30)
print('parse tree before')
astpretty.pprint(tree)
print('*'*30)
'''
Now, we check if the function has 1 argument and no return, in which
case we build a return around the argument.
'''
'''
tree,recompiled_pype_func=apply_tree_transformation(tree,
NoReturnReplacer(),
originalFuncName,
glbls)
if verbose:
print('*'*30)
print('after no return replacer tree is')
astpretty.pprint(tree)
'''
'''
Now, we want to replace any name, either in the global variables or the
function body, that appears in the function body with NameBookmark.
We recompile the function into the recompiledReplacedNamespace, and
extract the fArgs.
'''
callNameReplacer=CallNameReplacer(aliases)
tree,recompiled_pype_func=apply_tree_transformation(tree,
callNameReplacer,
originalFuncName,
glbls)
accumNode=callNameReplacer.accumNode
if verbose:
print('*'*30)
print('after call name replacer tree is')
astpretty.pprint(tree)
tree,recompiled_pype_func=apply_tree_transformation(tree,
PypeValReplacer(),
originalFuncName,
glbls,
)
if verbose:
print('*'*30)
print('after operator replacer tree is')
astpretty.pprint(tree)
fArgs=recompiled_pype_func(*args)
if verbose:
print('*'*30)
print('printing fArgs')
pp.pprint(fArgs)
'''
Now, we run the optimizations and convert the fArgs into a list of trees.
'''
fArgTrees=optimize_f_args(fArgs,accumNode)
#if verbose:
# print('*'*30)
# print('printing replacedTree')
# for fArgTree in fArgTrees:
# print(f'{str(ast.dump(fArgTree))[:100]}')
recompiledReplacerNamespace={}
tree=parse(src)
fArgReplacer=FArgReplacer(fArgTrees,aliases)
fArgReplacer.visit(tree)
if verbose:
print('*'*30)
print('parse tree after')
astpretty.pprint(tree)
print('*'*30)
#pp.pprint(astunparse.dump(tree))
exec(compile(tree,
filename='<ast>',
mode='exec'),
glbls,
recompiledReplacerNamespace)
recompiled_pype_func=recompiledReplacerNamespace[originalFuncName]
'''
This is extremely dangerous, but the alternative is to add a flag to the
function, checking if it's been compiled. What we are doing here is
taking the global namespace of pype_func, and replacing that pype_func with
recompiled_pype_func, so that any calls to pype_func will automatically call
recompiled_pype_func.
'''
FUNCTION_CACHE[originalFuncName]=recompiled_pype_func
#glbls[originalFuncName]=recompiled_pype_func
#setattr(mod,originalFuncName,recompiled_pype_func)
#print(f'successfully recompiled {recompiled_pype_func}')
#print(f'recomiled {originalFuncName} in globals: {glbls[originalFuncName]}')
#print(f'{pype_func.__module__} is module')
#print('*'*30)
return FUNCTION_CACHE[originalFuncName](*args)
def test_pprint(capsys):
astpretty.pprint(_to_expr_value('x'))
out, _ = capsys.readouterr()
assert out == "Name(lineno=1, col_offset=0, id='x', ctx=Load())\n"
class CodeGeneratorTests(unittest.TestCase):
LANGS = list(_get_all_settings(Mock(indent=4)).keys())
maxDiff = None
def setUp(self):
os.chdir(TESTS_DIR)
@foreach(sorted(LANGS))
@foreach(sorted(TEST_CASES.keys()))
def test_snippet(self, case, lang):
env = os.environ.copy()
if ENV.get(lang):
env.update(ENV.get(lang))
settings = _get_all_settings(Mock(indent=4), env=env)[lang]
ext = settings.ext
source_data = TEST_CASES[case]
is_script = has_main(source_data)
if ext in [".dart", ".kt", ".rs"] and not is_script:
source_data = f"def main():\n {source_data}"
print(f">{source_data}<")
tree = ast.parse(source_data)
astpretty.pprint(tree)
proc = run([sys.executable, "-c", source_data], capture_output=True)
if proc.returncode:
raise RuntimeError(
f"Invalid case {case}:\n{proc.stdout}{proc.stderr}")
try:
result = transpile(
"stdin",
tree,
settings.transpiler,
settings.rewriters,
settings.transformers,
settings.post_rewriters,
)
except NotImplementedError as e:
raise unittest.SkipTest(str(e))
if settings.formatter:
if not spawn.find_executable(settings.formatter[0]):
raise unittest.SkipTest(
f"{settings.formatter[0]} not available")
if ext == ".kt":
class_name = str(case.title()) + "Kt"
exe = TESTS_DIR / (class_name + ".class")
elif ext == ".cpp":
exe = TESTS_DIR / "a.out"
elif ext == ".dart" or (ext == ".nim" and sys.platform == "win32"):
exe = TESTS_DIR / "cases" / f"{case}.exe"
else:
exe = TESTS_DIR / "cases" / f"{case}"
exe.unlink(missing_ok=True)
case_output = TESTS_DIR / "cases" / f"{case}{ext}"
expect_success = f"{case}{ext}" in EXPECTED_SUCCESSES
with open(case_output, "w") as f:
f.write(result)
if settings.formatter:
if settings.ext == ".kt" and case_output.is_absolute():
# KtLint does not support absolute path in globs
case_output = case_output.relative_to(Path.cwd())
proc = run([*settings.formatter, case_output],
env=env,
capture_output=True)
if proc.returncode:
raise unittest.SkipTest(
f"Error: Could not reformat using {settings.formatter}:\n{proc.stdout}{proc.stderr}"
)
try:
compiler = COMPILERS[lang]
if compiler:
if not spawn.find_executable(compiler[0]):
raise unittest.SkipTest(f"{compiler[0]} not available")
proc = run([*compiler, case_output],
env=env,
capture_output=True)
if proc.returncode and not expect_success:
raise unittest.SkipTest(
f"{case}{ext} doesnt compile:\n{proc.stdout}{proc.stderr}"
)
if exe.exists() and os.access(exe, os.X_OK):
if not expect_success:
raise AssertionError(f"{case}{ext} compiled")
elif INVOKER.get(lang):
invoker = INVOKER.get(lang)
if not spawn.find_executable(invoker[0]):
raise unittest.SkipTest(f"{invoker[0]} not available")
proc = run([*invoker, case_output],
env=env,
capture_output=True)
if proc.returncode and not expect_success:
raise unittest.SkipTest(
f"Execution of {case}{ext} failed:\n{proc.stdout}{proc.stderr}"
)
if not expect_success:
raise AssertionError(f"{case}{ext} invoked")
else:
return
finally:
if not KEEP_GENERATED:
case_output.unlink(missing_ok=True)
exe.unlink(missing_ok=True)
if not expect_success:
assert False
def get_expression_part_info(node: ast.AST) -> Mapping[str, Any]:
types_map = [
(ast.UnaryOp, 'unary_op'),
(
(
ast.Expr,
ast.Return,
ast.Starred,
ast.Index,
ast.Yield,
ast.YieldFrom,
ast.FormattedValue,
),
'item_with_value',
),
(ast.Assert, 'assert'),
(ast.Delete, 'delete'),
(ast.Assign, 'assign'),
((ast.AugAssign, ast.AnnAssign), 'featured_assign'),
(ast.Call, 'call'),
(ast.Await, 'await'),
((ast.List, ast.Set, ast.Tuple), 'sized'),
(ast.Dict, 'dict'),
(ast.DictComp, 'dict_comprehension'),
((ast.ListComp, ast.GeneratorExp, ast.SetComp),
'simple_comprehensions'),
(ast.comprehension, 'base_comprehension'),
(ast.Compare, 'compare'),
(ast.Subscript, 'subscript'),
(ast.Slice, 'slice'),
(ast.ExtSlice, 'ext_slice'),
(ast.BinOp, 'binary_op'),
(ast.Lambda, 'lambda'),
(ast.IfExp, 'if_expr'),
(ast.BoolOp, 'bool_op'),
(ast.Attribute, 'attribute'),
(ast.JoinedStr, 'fstring'),
(ast.ClassDef, 'classdef'),
(
(
ast.Name,
ast.Import,
ast.Str,
ast.Num,
ast.NameConstant,
ast.Bytes,
ast.Nonlocal,
ast.ImportFrom,
ast.Pass,
ast.Raise,
ast.Break,
ast.Continue,
type(None),
ast.Ellipsis,
),
'simple_type',
),
]
node_type_sid = None
for types, node_type_name in types_map:
if isinstance(node, types): # type: ignore
node_type_sid = node_type_name
break
else:
pprint(node) # noqa
raise AssertionError('should always get node type')
return {
'type': node_type_sid,
'subnodes': _get_sub_nodes(node, node_type_sid),
}
