def flatten_ast(node: Any, prefix="", path="") -> str: if isinstance(node, ast.AST): acc = [f"{prefix}/_type={type(node).__name__}\n"] if isinstance(node, ast.expr): node_repr = remove_context("", ast.dump(node)) expr_hash = hex(hash(node_repr) & 0xFFFFFFFF) acc.append(f"{prefix}/_hash={expr_hash}\n") if "lineno" in node._attributes: acc.append(f"{prefix}/_pos={node.lineno}:{path[2:]}\n") fields = ast.iter_fields(node) if isinstance(node, ast.FunctionDef): # reject `body` behind `decorator_list` and `returns`, whose line number is those of `def` clause fields = iter(sorted(fields, key=lambda c: c[0] == "body")) for (i, (name, x)) in enumerate(fields): if name == "orelse" and isinstance(node, (ast.For, ast.While, ast.AsyncFor)): name = "loopelse" # this makes the "orelse" clause specific to conditionals elif name == "targets" and isinstance(node, ast.Assign): name = "assigntargets" elif name == "target" and isinstance(node, ast.AugAssign): name = "assigntarget" elif name == "value" and isinstance(node, (ast.Assign, ast.AugAssign)): name = "assignvalue" acc.append(flatten_ast(x, f"{prefix}/{name}", f"{path}{i}-")) return "".join(acc) elif isinstance(node, list): acc = [f"{prefix}/length={len(node)}\n"] for (i, x) in enumerate(node, 1): acc.append(flatten_ast(x, f"{prefix}/{i}", f"{path}{i}-")) return "".join(acc) elif prefix.endswith(("/s", "/module")): return f"""{prefix}={repr(node)}\n""" else: return f"""{prefix}={repr(node).strip("'")}\n"""
def _get_bases_from_info( self, info: typeshed_client.resolver.ResolvedName, mod: str ) -> Optional[List[Value]]: if info is None: return None elif isinstance(info, typeshed_client.ImportedInfo): return self._get_bases_from_info(info.info, ".".join(info.source_module)) elif isinstance(info, typeshed_client.NameInfo): if isinstance(info.ast, ast3.ClassDef): bases = info.ast.bases return [self._parse_type(base, mod) for base in bases] elif isinstance(info.ast, ast3.Assign): val = self._parse_type(info.ast.value, mod) if isinstance(val, KnownValue) and isinstance(val.val, type): return self.get_bases(val.val) else: return [val] elif isinstance( info.ast, ( # overloads are not supported yet typeshed_client.OverloadedName, typeshed_client.ImportedName, # typeshed pretends the class is a function ast3.FunctionDef, ), ): return None else: raise NotImplementedError(ast3.dump(info.ast)) return None
def _parse_type(self, node: ast3.AST, module: str) -> Value: val = self._parse_expr(node, module) ctx = _AnnotationContext(finder=self, module=module) typ = type_from_value(val, ctx=ctx) if self.verbose and typ is UNRESOLVED_VALUE: self.log("Got UNRESOLVED_VALUE", (ast3.dump(node), module)) return typ
def use(fn: Callable[..., Any]): t = ast.parse(inspect.getsource(fn)) assert len(t.body) == 1 body = t.body[0].body for node in body: print(ast.dump(node))
def _unsupported_syntax(self, tree): unparsed = 'invalid' try: unparsed = '"""{}"""'.format( typed_astunparse.unparse(tree).strip()) except AttributeError: pass self.fill('unsupported_syntax') raise SyntaxError( 'unparsing {} like """{}""" ({} in Python) is unsupported for {}'. format(tree.__class__.__name__, typed_ast3.dump(tree), unparsed, self.lang_name))
def flatten_node( node: Any, prefix: str = "", path: str = "", remove_context: Callable = regex.compile(r", ctx=.+?\(\)").sub, ) -> str: r"""Traverse recursively (in pre-order) the given AST node and flatten its subtree. Args: node (Any): The node to traverse. Initially, the whole AST. prefix (str, optional): The prefix of the current line to dump. Defaults to `""`. path (str, optional): The path of the current node. Defaults to `""`. remove_context (Callable, optional): A function removing the node context encoded in the result of `ast3.dump()`. [Not to be explicitly provided.](developer_manual/index.html#default-argument-trick) Defaults to `regex.compile(r", ctx=.+?\(\)").sub`. Returns: str: A flat representation of the given node. """ if isinstance(node, ast.AST): acc = [f"{prefix}/_type={type(node).__name__}\n"] if isinstance(node, ast.expr): node_repr = remove_context("", ast.dump(node)) acc.append(f"{prefix}/_hash={pseudo_hash(node_repr)}\n") if "lineno" in node._attributes: acc.append(f"{prefix}/_pos={node.lineno}:{path[2:]}\n") fields = ast.iter_fields(node) if isinstance(node, ast.FunctionDef): # make `body` the last item of a `def` clause fields = iter(sorted(fields, key=lambda c: c[0] == "body")) for (i, (name, x)) in enumerate(fields): if name == "orelse" and isinstance( node, (ast.For, ast.While, ast.AsyncFor)): name = "loopelse" # make the `orelse` clause specific to conditionals elif name == "targets" and isinstance(node, ast.Assign): name = "assigntargets" # `targets` is also used for `Delete`, etc. elif name == "target" and isinstance(node, ast.AugAssign): name = "assigntarget" # `target` is also used for comprehension, etc. elif name == "value" and isinstance(node, (ast.Assign, ast.AugAssign)): name = "assignvalue" # `value` is also used for comprehension, etc. acc.append(flatten_node(x, f"{prefix}/{name}", f"{path}{i}-")) return "".join(acc) elif isinstance(node, list): acc = [f"{prefix}/_length={len(node)}\n"] for (i, x) in enumerate(node, 1): # number the children from 1 acc.append(flatten_node(x, f"{prefix}/{i}", f"{path}{i}-")) return "".join(acc) else: # If the node is a terminal value, dump it unquoted return f"""{prefix}={repr(node).strip("'")}\n"""
def resolve_types(self, node): resolved_types = {} for type_ in ['FundamentalType', 'PointerType']: type_nodes = self.get_all(node, './{}'.format(type_)) resolved_types.update( dict(self.transform_all(type_nodes, parent=node))) fix_resolved_types(resolved_types) _LOG.debug( 'Detected types:\n%s', pprint.pformat( {k: typed_ast3.dump(v) for k, v in resolved_types.items()})) return resolved_types
def visit_node(self, node): if not isinstance(node, typed_ast3.Str): return node try: resolved_type = self.resolved_types[node.s] except KeyError: # raise NotImplementedError('cannot completely resolve') from err _LOG.debug('cannot currently resolve %s', node.s) return node _LOG.debug('resolved %s into %s', node.s, typed_ast3.dump(resolved_type)) self.modified = True return resolved_type
def visit_node(self, node): if not isinstance(node, typed_ast3.Str): return node try: resolved_type = self.resolved_types[node.s] except KeyError: self.unresolved_types.add(node.s) _LOG.debug('cannot currently resolve %s', node.s) return node if node.s in self.unresolved_types: self.unresolved_types.remove(node.s) _LOG.debug('resolved %s into %s', node.s, typed_ast3.dump(resolved_type)) self.modified = True return resolved_type
def make_call_from_slice(slice_: typed_ast3.Slice) -> typed_ast3.Call: """Transform code like '0:n:2' into 'slice(0, n, 2)'.""" assert isinstance(slice_, typed_ast3.Slice), type(slice_) lower, upper, step = slice_.lower, slice_.upper, slice_.step if lower is None and upper is None and step is None: args = [typed_ast3.NameConstant(None)] elif lower is not None and upper is None and step is None: args = [lower, typed_ast3.NameConstant(None)] elif lower is None and upper is not None and step is None: args = [typed_ast3.NameConstant(None), upper] elif lower is not None and upper is not None and step is None: args = [lower, upper] elif lower is not None and upper is None and step is not None: args = [lower, typed_ast3.NameConstant(None), step] elif lower is not None and upper is not None and step is not None: args = [lower, upper, step] else: raise NotImplementedError('unsupported slice form: "{}"'.format(typed_ast3.dump(slice_))) return typed_ast3.Call(typed_ast3.Name('slice', typed_ast3.Load()), args, [])
from typed_ast import ast3 code = """ def hello(): name = "world" print(f"hello {world}") """ t = ast3.parse(code) print(t) print("----------------------------------------") print(ast3.dump(t)) print("----------------------------------------") for node in ast3.walk(t): print(node)
def syntax_matches(syntax, target): _01 = typed_ast3.dump(syntax) _02 = typed_ast3.dump(target) return _01 == _02
SOURCE = """ #import math def f(a: float) -> float: b = 1 return a def add(a: int, b: int, tt: Tensor[int, L[20,20]]) -> int: uu = tt[0,0] e = f(6.1) + 1 g = math.sqrt(5.1) for i in range(0,10): v = 1 + i #c = a + 1 + 5 #if True: # c = c + 5 #r = 5 #c = 1 + 1.1 return a """ if __name__ == "__main__": root = ast3.parse(SOURCE) options = config.Options.create(python_version=(3, 7)) ast_factory = lambda unused_options: ast3 module = annotate_ast.annotate_source(SOURCE, ast_factory, options) print(ast3.dump(root)) visitor = Expr() visitor.visit(module) print(str(visitor.module)) visitor.module.compile() print(visitor.module.get_ir())
def run_pytropos( # noqa: C901 file: str, filename: str, cursorline: 'Optional[int]' = None, console: bool = False, pt_globals: 'Optional[Dict[str, Any]]' = None ) -> 'Tuple[int, Optional[Store]]': dprint("Starting pytropos", verb=1) dprint( "Parsing and un-parsing a python file (it should preserve all type comments)", verb=2) if debug_print.verbosity > 1: try: from typed_astunparse import unparse except ModuleNotFoundError: print( "Sorry! You need to install `typed_astunparse` for bigger verbosity levels.\n" "Note: If you are using python 3.7 we recommend you to install \n" " `typed_astunparse` with `pip install -r git_requirements.txt` \n" " (you can find `git_requirements.txt` in {})\n".format( metadata.url), file=sys.stderr) exit(1) # Parsing file ast_: ast3.Module try: ast_ = ast3.parse(file, filename=filename) # type: ignore except SyntaxError as msg: derror( f"{msg.filename}:{msg.lineno}:{msg.offset-1}: {type(msg).__name__}: {msg.msg}" ) return (2, None) except (OverflowError, ValueError) as msg: derror(f"{filename}::: {type(msg).__name__}") return (2, None) if debug_print.verbosity > 1: dprint("Original file:", verb=2) dprint("AST dump of original file:", ast3.dump(ast_), verb=3) dprint(unparse(ast_), verb=2) # Converting AST (code) into Pytropos representation newast: ast3.Module try: newast = PytroposTransformer( # type: ignore filename, cursorline=cursorline, console=console).visit(ast_) except AstTransformerError: derror( "Sorry it seems Pytropos cannot run the file. Pytropos doesn't support " "some Python characteristic it uses right now. Sorry :(") traceback.print_exc() # derror(msg) return (2, None) if debug_print.verbosity > 1: dprint("Modified file:", verb=2) dprint("AST dump of modified file:", ast3.dump(newast), verb=3) dprint(unparse(newast), verb=2) newast_py = ast.fix_missing_locations(typed_ast3_to_ast(newast)) # TODO(helq): add these lines of code for optional debugging # import astpretty # astpretty.pprint(newast_py) try: newast_comp = compile(newast_py, '<generated type checking ast>', 'exec') # type: ignore except (ValueError, TypeError): derror( "PYTROPOS INTERNAL ERROR. Sorry, it seems something unexpected happened. " "Please report this issue: run pytropos again with the same input and " "the flag -vvv") traceback.print_exc() # derror(msg) return (2, None) exitvalues = run_transformed_type_checking_code(newast_comp, pt_globals) TypeCheckLogger.clean_sing() dprint("Closing pytropos", verb=1) return exitvalues
def ast(code): return dump(parse(code))