def flatten_ast(node: Any, prefix="", path="") -> str:
if isinstance(node, ast.AST):
acc = [f"{prefix}/_type={type(node).__name__}\n"]
if isinstance(node, ast.expr):
node_repr = remove_context("", ast.dump(node))
expr_hash = hex(hash(node_repr) & 0xFFFFFFFF)
acc.append(f"{prefix}/_hash={expr_hash}\n")
if "lineno" in node._attributes:
acc.append(f"{prefix}/_pos={node.lineno}:{path[2:]}\n")
fields = ast.iter_fields(node)
if isinstance(node, ast.FunctionDef):
# reject `body` behind `decorator_list` and `returns`, whose line number is those of `def` clause
fields = iter(sorted(fields, key=lambda c: c[0] == "body"))
for (i, (name, x)) in enumerate(fields):
if name == "orelse" and isinstance(node, (ast.For, ast.While, ast.AsyncFor)):
name = "loopelse" # this makes the "orelse" clause specific to conditionals
elif name == "targets" and isinstance(node, ast.Assign):
name = "assigntargets"
elif name == "target" and isinstance(node, ast.AugAssign):
name = "assigntarget"
elif name == "value" and isinstance(node, (ast.Assign, ast.AugAssign)):
name = "assignvalue"
acc.append(flatten_ast(x, f"{prefix}/{name}", f"{path}{i}-"))
return "".join(acc)
elif isinstance(node, list):
acc = [f"{prefix}/length={len(node)}\n"]
for (i, x) in enumerate(node, 1):
acc.append(flatten_ast(x, f"{prefix}/{i}", f"{path}{i}-"))
return "".join(acc)
elif prefix.endswith(("/s", "/module")):
return f"""{prefix}={repr(node)}\n"""
else:
return f"""{prefix}={repr(node).strip("'")}\n"""
def _get_bases_from_info(
self, info: typeshed_client.resolver.ResolvedName, mod: str
) -> Optional[List[Value]]:
if info is None:
return None
elif isinstance(info, typeshed_client.ImportedInfo):
return self._get_bases_from_info(info.info, ".".join(info.source_module))
elif isinstance(info, typeshed_client.NameInfo):
if isinstance(info.ast, ast3.ClassDef):
bases = info.ast.bases
return [self._parse_type(base, mod) for base in bases]
elif isinstance(info.ast, ast3.Assign):
val = self._parse_type(info.ast.value, mod)
if isinstance(val, KnownValue) and isinstance(val.val, type):
return self.get_bases(val.val)
else:
return [val]
elif isinstance(
info.ast,
(
# overloads are not supported yet
typeshed_client.OverloadedName,
typeshed_client.ImportedName,
# typeshed pretends the class is a function
ast3.FunctionDef,
),
):
return None
else:
raise NotImplementedError(ast3.dump(info.ast))
return None
def _parse_type(self, node: ast3.AST, module: str) -> Value:
val = self._parse_expr(node, module)
ctx = _AnnotationContext(finder=self, module=module)
typ = type_from_value(val, ctx=ctx)
if self.verbose and typ is UNRESOLVED_VALUE:
self.log("Got UNRESOLVED_VALUE", (ast3.dump(node), module))
return typ
def use(fn: Callable[..., Any]):
t = ast.parse(inspect.getsource(fn))
assert len(t.body) == 1
body = t.body[0].body
for node in body:
print(ast.dump(node))
def _unsupported_syntax(self, tree):
unparsed = 'invalid'
try:
unparsed = '"""{}"""'.format(
typed_astunparse.unparse(tree).strip())
except AttributeError:
pass
self.fill('unsupported_syntax')
raise SyntaxError(
'unparsing {} like """{}""" ({} in Python) is unsupported for {}'.
format(tree.__class__.__name__, typed_ast3.dump(tree), unparsed,
self.lang_name))
def flatten_node(
node: Any,
prefix: str = "",
path: str = "",
remove_context: Callable = regex.compile(r", ctx=.+?\(\)").sub,
) -> str:
r"""Traverse recursively (in pre-order) the given AST node and flatten its subtree.
Args:
node (Any): The node to traverse. Initially, the whole AST.
prefix (str, optional): The prefix of the current line to dump. Defaults to `""`.
path (str, optional): The path of the current node. Defaults to `""`.
remove_context (Callable, optional): A function removing the node context encoded in the
result of `ast3.dump()`.
[Not to be explicitly provided.](developer_manual/index.html#default-argument-trick)
Defaults to `regex.compile(r", ctx=.+?\(\)").sub`.
Returns:
str: A flat representation of the given node.
"""
if isinstance(node, ast.AST):
acc = [f"{prefix}/_type={type(node).__name__}\n"]
if isinstance(node, ast.expr):
node_repr = remove_context("", ast.dump(node))
acc.append(f"{prefix}/_hash={pseudo_hash(node_repr)}\n")
if "lineno" in node._attributes:
acc.append(f"{prefix}/_pos={node.lineno}:{path[2:]}\n")
fields = ast.iter_fields(node)
if isinstance(node, ast.FunctionDef):
# make `body` the last item of a `def` clause
fields = iter(sorted(fields, key=lambda c: c[0] == "body"))
for (i, (name, x)) in enumerate(fields):
if name == "orelse" and isinstance(
node, (ast.For, ast.While, ast.AsyncFor)):
name = "loopelse" # make the `orelse` clause specific to conditionals
elif name == "targets" and isinstance(node, ast.Assign):
name = "assigntargets" # `targets` is also used for `Delete`, etc.
elif name == "target" and isinstance(node, ast.AugAssign):
name = "assigntarget" # `target` is also used for comprehension, etc.
elif name == "value" and isinstance(node,
(ast.Assign, ast.AugAssign)):
name = "assignvalue" # `value` is also used for comprehension, etc.
acc.append(flatten_node(x, f"{prefix}/{name}", f"{path}{i}-"))
return "".join(acc)
elif isinstance(node, list):
acc = [f"{prefix}/_length={len(node)}\n"]
for (i, x) in enumerate(node, 1): # number the children from 1
acc.append(flatten_node(x, f"{prefix}/{i}", f"{path}{i}-"))
return "".join(acc)
else:
# If the node is a terminal value, dump it unquoted
return f"""{prefix}={repr(node).strip("'")}\n"""
def resolve_types(self, node):
resolved_types = {}
for type_ in ['FundamentalType', 'PointerType']:
type_nodes = self.get_all(node, './{}'.format(type_))
resolved_types.update(
dict(self.transform_all(type_nodes, parent=node)))
fix_resolved_types(resolved_types)
_LOG.debug(
'Detected types:\n%s',
pprint.pformat(
{k: typed_ast3.dump(v)
for k, v in resolved_types.items()}))
return resolved_types
def visit_node(self, node):
if not isinstance(node, typed_ast3.Str):
return node
try:
resolved_type = self.resolved_types[node.s]
except KeyError:
# raise NotImplementedError('cannot completely resolve') from err
_LOG.debug('cannot currently resolve %s', node.s)
return node
_LOG.debug('resolved %s into %s', node.s,
typed_ast3.dump(resolved_type))
self.modified = True
return resolved_type
def visit_node(self, node):
if not isinstance(node, typed_ast3.Str):
return node
try:
resolved_type = self.resolved_types[node.s]
except KeyError:
self.unresolved_types.add(node.s)
_LOG.debug('cannot currently resolve %s', node.s)
return node
if node.s in self.unresolved_types:
self.unresolved_types.remove(node.s)
_LOG.debug('resolved %s into %s', node.s,
typed_ast3.dump(resolved_type))
self.modified = True
return resolved_type
def make_call_from_slice(slice_: typed_ast3.Slice) -> typed_ast3.Call:
"""Transform code like '0:n:2' into 'slice(0, n, 2)'."""
assert isinstance(slice_, typed_ast3.Slice), type(slice_)
lower, upper, step = slice_.lower, slice_.upper, slice_.step
if lower is None and upper is None and step is None:
args = [typed_ast3.NameConstant(None)]
elif lower is not None and upper is None and step is None:
args = [lower, typed_ast3.NameConstant(None)]
elif lower is None and upper is not None and step is None:
args = [typed_ast3.NameConstant(None), upper]
elif lower is not None and upper is not None and step is None:
args = [lower, upper]
elif lower is not None and upper is None and step is not None:
args = [lower, typed_ast3.NameConstant(None), step]
elif lower is not None and upper is not None and step is not None:
args = [lower, upper, step]
else:
raise NotImplementedError('unsupported slice form: "{}"'.format(typed_ast3.dump(slice_)))
return typed_ast3.Call(typed_ast3.Name('slice', typed_ast3.Load()), args, [])
from typed_ast import ast3
code = """
def hello():
name = "world"
print(f"hello {world}")
"""
t = ast3.parse(code)
print(t)
print("----------------------------------------")
print(ast3.dump(t))
print("----------------------------------------")
for node in ast3.walk(t):
print(node)
def syntax_matches(syntax, target):
_01 = typed_ast3.dump(syntax)
_02 = typed_ast3.dump(target)
return _01 == _02
SOURCE = """
#import math
def f(a: float) -> float:
b = 1
return a
def add(a: int, b: int, tt: Tensor[int, L[20,20]]) -> int:
uu = tt[0,0]
e = f(6.1) + 1
g = math.sqrt(5.1)
for i in range(0,10):
v = 1 + i
#c = a + 1 + 5
#if True:
# c = c + 5
#r = 5
#c = 1 + 1.1
return a
"""
if __name__ == "__main__":
root = ast3.parse(SOURCE)
options = config.Options.create(python_version=(3, 7))
ast_factory = lambda unused_options: ast3
module = annotate_ast.annotate_source(SOURCE, ast_factory, options)
print(ast3.dump(root))
visitor = Expr()
visitor.visit(module)
print(str(visitor.module))
visitor.module.compile()
print(visitor.module.get_ir())
def run_pytropos( # noqa: C901
file: str,
filename: str,
cursorline: 'Optional[int]' = None,
console: bool = False,
pt_globals: 'Optional[Dict[str, Any]]' = None
) -> 'Tuple[int, Optional[Store]]':
dprint("Starting pytropos", verb=1)
dprint(
"Parsing and un-parsing a python file (it should preserve all type comments)",
verb=2)
if debug_print.verbosity > 1:
try:
from typed_astunparse import unparse
except ModuleNotFoundError:
print(
"Sorry! You need to install `typed_astunparse` for bigger verbosity levels.\n"
"Note: If you are using python 3.7 we recommend you to install \n"
" `typed_astunparse` with `pip install -r git_requirements.txt` \n"
" (you can find `git_requirements.txt` in {})\n".format(
metadata.url),
file=sys.stderr)
exit(1)
# Parsing file
ast_: ast3.Module
try:
ast_ = ast3.parse(file, filename=filename) # type: ignore
except SyntaxError as msg:
derror(
f"{msg.filename}:{msg.lineno}:{msg.offset-1}: {type(msg).__name__}: {msg.msg}"
)
return (2, None)
except (OverflowError, ValueError) as msg:
derror(f"{filename}::: {type(msg).__name__}")
return (2, None)
if debug_print.verbosity > 1:
dprint("Original file:", verb=2)
dprint("AST dump of original file:", ast3.dump(ast_), verb=3)
dprint(unparse(ast_), verb=2)
# Converting AST (code) into Pytropos representation
newast: ast3.Module
try:
newast = PytroposTransformer( # type: ignore
filename, cursorline=cursorline, console=console).visit(ast_)
except AstTransformerError:
derror(
"Sorry it seems Pytropos cannot run the file. Pytropos doesn't support "
"some Python characteristic it uses right now. Sorry :(")
traceback.print_exc()
# derror(msg)
return (2, None)
if debug_print.verbosity > 1:
dprint("Modified file:", verb=2)
dprint("AST dump of modified file:", ast3.dump(newast), verb=3)
dprint(unparse(newast), verb=2)
newast_py = ast.fix_missing_locations(typed_ast3_to_ast(newast))
# TODO(helq): add these lines of code for optional debugging
# import astpretty
# astpretty.pprint(newast_py)
try:
newast_comp = compile(newast_py, '<generated type checking ast>',
'exec') # type: ignore
except (ValueError, TypeError):
derror(
"PYTROPOS INTERNAL ERROR. Sorry, it seems something unexpected happened. "
"Please report this issue: run pytropos again with the same input and "
"the flag -vvv")
traceback.print_exc()
# derror(msg)
return (2, None)
exitvalues = run_transformed_type_checking_code(newast_comp, pt_globals)
TypeCheckLogger.clean_sing()
dprint("Closing pytropos", verb=1)
return exitvalues
def ast(code):
return dump(parse(code))
