def record_asname( self, original_node: Union[cst.Import, cst.ImportFrom]) -> None: # Record the import's `as` name if it has one, and set the attribute mapping. names = original_node.names if not isinstance(names, Sequence): return for import_alias in names: alias_name = get_full_name_for_node(import_alias.name) if isinstance(original_node, cst.ImportFrom): module = original_node.module if module is None: return module_name = get_full_name_for_node(module) if module_name is None: return qual_name = f"{module_name}.{alias_name}" else: qual_name = alias_name if qual_name is not None and alias_name is not None: if qual_name == self.old_name or self.old_name.startswith( qual_name + "."): as_name_optional = import_alias.asname as_name_node = (as_name_optional.name if as_name_optional is not None else None) if as_name_node is not None and isinstance( as_name_node, (cst.Name, cst.Attribute)): full_as_name = get_full_name_for_node(as_name_node) if full_as_name is not None: self.as_name = (full_as_name, alias_name)
def _annotate_single_target( self, node: cst.Assign, updated_node: cst.Assign ) -> Union[cst.Assign, cst.AnnAssign]: only_target = node.targets[0].target if isinstance(only_target, (cst.Tuple, cst.List)): for element in only_target.elements: value = element.value name = get_full_name_for_node(value) if name: self._add_to_toplevel_annotations(name) elif isinstance(only_target, (cst.Subscript)): pass else: name = get_full_name_for_node(only_target) if name is not None: self.qualifier.append(name) if self._qualifier_name() in self.annotations.attribute_annotations and not isinstance( only_target, cst.Subscript ): annotation = self.annotations.attribute_annotations[ self._qualifier_name() ] self.qualifier.pop() return cst.AnnAssign(cst.Name(name), annotation, node.value) else: self.qualifier.pop() return updated_node
def find_qualified_name_for_import_alike( assignment_node: Union[cst.Import, cst.ImportFrom], full_name: str) -> Set[QualifiedName]: module = "" results = set() if isinstance(assignment_node, cst.ImportFrom): module_attr = assignment_node.module if module_attr: # TODO: for relative import, keep the relative Dot in the qualified name module = get_full_name_for_node(module_attr) import_names = assignment_node.names if not isinstance(import_names, cst.ImportStar): for name in import_names: real_name = get_full_name_for_node(name.name) as_name = real_name if name and name.asname: name_asname = name.asname if name_asname: as_name = cst.ensure_type(name_asname.name, cst.Name).value if as_name and full_name.startswith(as_name): if module: real_name = f"{module}.{real_name}" if real_name: remaining_name = full_name.split(as_name)[1].lstrip( ".") results.add( QualifiedName( f"{real_name}.{remaining_name}" if remaining_name else real_name, QualifiedNameSource.IMPORT, )) return results
def _get_import_alias_names(import_aliases: Sequence[cst.ImportAlias]) -> Set[str]: import_names = set() for imported_name in import_aliases: asname = imported_name.asname if asname: import_names.add(get_full_name_for_node(asname.name)) else: import_names.add(get_full_name_for_node(imported_name.name)) return import_names
def leave_ImportFrom(self, original_node: cst.ImportFrom, updated_node: cst.ImportFrom) -> cst.ImportFrom: module = updated_node.module if module is None: return updated_node imported_module_name = get_full_name_for_node(module) names = original_node.names if imported_module_name is None or not isinstance(names, Sequence): return updated_node else: new_names = [] for import_alias in names: alias_name = get_full_name_for_node(import_alias.name) if alias_name is not None: qual_name = f"{imported_module_name}.{alias_name}" if self.old_name == qual_name: replacement_module = self.gen_replacement_module( imported_module_name) replacement_obj = self.gen_replacement(alias_name) if not replacement_obj: # The user has requested an `import` statement rather than an `from ... import`. # This will be taken care of in `leave_Module`, in the meantime, schedule for potential removal. new_names.append(import_alias) self.scheduled_removals.add(original_node) continue new_import_alias_name: Union[ cst.Attribute, cst.Name] = self.gen_name_or_attr_node( replacement_obj) # Rename on the spot only if this is the only imported name under the module. if len(names) == 1: self.bypass_import = True return updated_node.with_changes( module=cst.parse_expression( replacement_module), names=(cst.ImportAlias( name=new_import_alias_name), ), ) # Or if the module name is to stay the same. elif replacement_module == imported_module_name: self.bypass_import = True new_names.append( cst.ImportAlias(name=new_import_alias_name)) else: if self.old_name.startswith(qual_name + "."): # This import might be in use elsewhere in the code, so schedule a potential removal. self.scheduled_removals.add(original_node) new_names.append(import_alias) return updated_node.with_changes(names=new_names) return updated_node
def _add_annotation_to_imports( self, annotation: cst.Attribute ) -> Union[cst.Name, cst.Attribute]: key = get_full_name_for_node(annotation.value) if key is not None: # Don't attempt to re-import existing imports. if key in self.existing_imports: return annotation import_name = get_full_name_for_node(annotation.attr) if import_name is not None: AddImportsVisitor.add_needed_import(self.context, key, import_name) return annotation.attr
def leave_Attribute( self, original_node: cst.Attribute, updated_node: cst.Attribute) -> Union[cst.Name, cst.Attribute]: full_name_for_node = get_full_name_for_node(original_node) if full_name_for_node is None: raise Exception("Could not parse full name for Attribute node.") full_replacement_name = self.gen_replacement(full_name_for_node) # If a node has no associated QualifiedName, we are still inside an import statement. inside_import_statement: bool = not self.get_metadata( QualifiedNameProvider, original_node, set()) if (QualifiedNameProvider.has_name( self, original_node, self.old_name, ) or (inside_import_statement and full_replacement_name == self.new_name)): new_value, new_attr = self.new_module, self.new_mod_or_obj if not inside_import_statement: self.scheduled_removals.add(original_node.value) if full_replacement_name == self.new_name: return updated_node.with_changes( value=cst.parse_expression(new_value), attr=cst.Name(value=new_attr.rstrip(".")), ) return self.gen_name_or_attr_node(new_attr) return updated_node
def visit_ClassDef(self, node: cst.ClassDef) -> Optional[bool]: self.scope.record_assignment(node.name.value, node) for decorator in node.decorators: decorator.visit(self) for base in node.bases: base.visit(self) for keyword in node.keywords: keyword.visit(self) with self._new_scope(ClassScope, node, get_full_name_for_node(node.name)): for statement in node.body.body: statement.visit(self) # visit remaining attributes for attr in [ node.lpar, node.rpar, node.leading_lines, node.lines_after_decorators, node.whitespace_after_class, node.whitespace_after_name, node.whitespace_before_colon, ]: if isinstance(attr, cst.CSTNode): attr.visit(self) return False
def visit_FunctionDef(self, node: cst.FunctionDef) -> Optional[bool]: self.scope.record_assignment(node.name.value, node) self.provider.set_metadata(node.name, self.scope) with self._new_scope(FunctionScope, node, get_full_name_for_node(node.name)): node.params.visit(self) node.body.visit(self) # visit remaining attributes for attr in [ node.asynchronous, node.leading_lines, node.lines_after_decorators, node.whitespace_after_def, node.whitespace_after_name, node.whitespace_before_params, node.whitespace_before_colon, ]: if isinstance(attr, cst.CSTNode): attr.visit(self) for decorator in node.decorators: decorator.visit(self) returns = node.returns if returns: returns.visit(self) return False
def visit_Assign(self, node: libcst.Assign) -> bool: for target_node in node.targets: target = get_full_name_for_node(target_node.target) if target == "__all__": self._in_assignment += 1 return True return False
def _visit_name_attr_alike(self, node: Union[cst.Name, cst.Attribute]) -> None: # Look up the local name of this node. local_name = get_full_name_for_node(node) if local_name is None: return # Look up the scope for this node, remove the import that caused it to exist. metadata_wrapper = self.context.wrapper if metadata_wrapper is None: raise Exception("Cannot look up import, metadata is not computed for node!") scope_provider = metadata_wrapper.resolve(ScopeProvider) try: scope = scope_provider[node] if scope is None: # This object has no scope, so we can't remove it. return except KeyError: # This object has no scope, so we can't remove it. return while True: for assignment in scope.assignments[node] or set(): # We only care about non-builtins. if isinstance(assignment, Assignment): import_node = assignment.node if isinstance(import_node, cst.Import): self._remove_imports_from_import_stmt(local_name, import_node) elif isinstance(import_node, cst.ImportFrom): self._remove_imports_from_importfrom_stmt( local_name, import_node ) if scope is scope.parent: break scope = scope.parent
def get_qualified_names_for( self, node: Union[str, cst.CSTNode]) -> Collection[QualifiedName]: """Get all :class:`~libcst.metadata.QualifiedName` in current scope given a :class:`~libcst.CSTNode`. The source of a qualified name can be either :attr:`QualifiedNameSource.IMPORT`, :attr:`QualifiedNameSource.BUILTIN` or :attr:`QualifiedNameSource.LOCAL`. Given the following example, ``c`` has qualified name ``a.b.c`` with source ``IMPORT``, ``f`` has qualified name ``Cls.f`` with source ``LOCAL``, ``a`` has qualified name ``Cls.f.<locals>.a``, ``i`` has qualified name ``Cls.f.<locals>.<comprehension>.i``, and the builtin ``int`` has qualified name ``builtins.int`` with source ``BUILTIN``:: from a.b import c class Cls: def f(self) -> "c": c() a = int("1") [i for i in c()] We extends `PEP-3155 <https://www.python.org/dev/peps/pep-3155/>`_ (defines ``__qualname__`` for class and function only; function namespace is followed by a ``<locals>``) to provide qualified name for all :class:`~libcst.CSTNode` recorded by :class:`~libcst.metadata.Assignment` and :class:`~libcst.metadata.Access`. The namespace of a comprehension (:class:`~libcst.ListComp`, :class:`~libcst.SetComp`, :class:`~libcst.DictComp`) is represented with ``<comprehension>``. An imported name may be used for type annotation with :class:`~libcst.SimpleString` and currently resolving the qualified given :class:`~libcst.SimpleString` is not supported considering it could be a complex type annotation in the string which is hard to resolve, e.g. ``List[Union[int, str]]``. """ results = set() full_name = get_full_name_for_node(node) if full_name is None: return results assignments = set() parts = full_name.split(".") for i in range(len(parts), 0, -1): prefix = ".".join(parts[:i]) if prefix in self: assignments = self[prefix] break for assignment in assignments: if isinstance(assignment, Assignment): assignment_node = assignment.node if isinstance(assignment_node, (cst.Import, cst.ImportFrom)): names = _NameUtil.find_qualified_name_for_import_alike( assignment_node, full_name) else: names = _NameUtil.find_qualified_name_for_non_import( assignment, full_name) if not isinstance(node, str) and _is_assignment( node, assignment_node): return names else: results |= names elif isinstance(assignment, BuiltinAssignment): results.add( QualifiedName(f"builtins.{assignment.name}", QualifiedNameSource.BUILTIN)) return results
def visit_AnnAssign(self, node: cst.AnnAssign) -> bool: name = get_full_name_for_node(node.target) if name is not None: self.qualifier.append(name) annotation_value = self._create_import_from_annotation(node.annotation) self.attribute_annotations[".".join(self.qualifier)] = annotation_value return True
def visit_Import(self, node: cst.Import) -> None: for import_alias in node.names: alias_name = get_full_name_for_node(import_alias.name) if alias_name is not None: if alias_name == self.old_name or alias_name.startswith( self.old_name + "."): # If the import statement is exactly equivalent to the old name, or we are renaming a top-level module of the import, # it will be taken care of in `leave_Name` or `leave_Attribute` when visiting the Name and Attribute children of this Import. self.bypass_import = True
def get_module_name_for_import(self) -> str: module = "" if isinstance(self.node, cst.ImportFrom): module_attr = self.node.module relative = self.node.relative if module_attr: module = get_full_name_for_node(module_attr) or "" if relative: module = "." * len(relative) + module return module
def get_module_name_for_import_alike( assignment_node: Union[cst.Import, cst.ImportFrom]) -> str: module = "" if isinstance(assignment_node, cst.ImportFrom): module_attr = assignment_node.module relative = assignment_node.relative if module_attr: module = get_full_name_for_node(module_attr) or "" if relative: module = "." * len(relative) + module return module
def test_get_full_name_for_expression( self, input: Union[str, cst.CSTNode], output: Optional[str], ) -> None: self.assertEqual(get_full_name_for_node(input), output) if output is None: with self.assertRaises(Exception): get_full_name_for_node_or_raise(input) else: self.assertEqual(get_full_name_for_node_or_raise(input), output)
def record_typevar( self, node: cst.Call, ) -> None: # pyre-ignore current_assign is never None here name = get_full_name_for_node(self.current_assign.targets[0].target) if name is not None: # pyre-ignore current_assign is never None here self.annotations.typevars[name] = self.current_assign self._handle_qualification_and_should_qualify("typing.TypeVar") self.current_assign = None
def visit_AnnAssign( self, node: cst.AnnAssign, ) -> bool: name = get_full_name_for_node(node.target) if name is not None: self.qualifier.append(name) annotation_value = self._handle_Annotation(annotation=node.annotation) self.annotations.attributes[".".join( self.qualifier)] = annotation_value return True
def find_qualified_name_for_import_alike( assignment_node: Union[cst.Import, cst.ImportFrom], full_name: str) -> Set[QualifiedName]: module = "" results = set() if isinstance(assignment_node, cst.ImportFrom): module_attr = assignment_node.module if module_attr: # TODO: for relative import, keep the relative Dot in the qualified name module = get_full_name_for_node(module_attr) import_names = assignment_node.names if not isinstance(import_names, cst.ImportStar): for name in import_names: real_name = get_full_name_for_node(name.name) if not real_name: continue # real_name can contain `.` for dotted imports # for these we want to find the longest prefix that matches full_name parts = real_name.split(".") real_names = [ ".".join(parts[:i]) for i in range(len(parts), 0, -1) ] for real_name in real_names: as_name = real_name if module: real_name = f"{module}.{real_name}" if name and name.asname: eval_alias = name.evaluated_alias if eval_alias is not None: as_name = eval_alias if full_name.startswith(as_name): remaining_name = full_name.split(as_name, 1)[1].lstrip(".") results.add( QualifiedName( f"{real_name}.{remaining_name}" if remaining_name else real_name, QualifiedNameSource.IMPORT, )) break return results
def visit_ImportFrom(self, node: cst.ImportFrom) -> None: module = node.module if module is None: return imported_module_name = get_full_name_for_node(module) if imported_module_name is None: return if imported_module_name == self.old_name or imported_module_name.startswith( self.old_name + "."): # If the imported module is exactly equivalent to the old name or we are renaming a parent module of the current module, # it will be taken care of in `leave_Name` or `leave_Attribute` when visiting the children of this ImportFrom. self.bypass_import = True
def _handle_assign_target(self, target: cst.BaseExpression, value: cst.BaseExpression) -> bool: target_name = get_full_name_for_node(target) if target_name == "__all__": # Assignments such as `__all__ = ["os"]` # or `__all__ = exports = ["os"]` if isinstance(value, (cst.List, cst.Tuple, cst.Set)): self._is_assigned_export.add(value) return True elif isinstance(target, cst.Tuple) and isinstance(value, cst.Tuple): # Assignments such as `__all__, x = ["os"], []` for element_idx, element_node in enumerate(target.elements): element_name = get_full_name_for_node(element_node.value) if element_name == "__all__": element_value = value.elements[element_idx].value if isinstance(element_value, (cst.List, cst.Tuple, cst.Set)): self._is_assigned_export.add(value) self._is_assigned_export.add(element_value) return True return False
def visit_ClassDef(self, node: cst.ClassDef) -> Optional[bool]: self.scope.record_assignment(node.name.value, node) for decorator in node.decorators: decorator.visit(self) for base in node.bases: base.visit(self) for keyword in node.keywords: keyword.visit(self) with self._new_scope(ClassScope, node, get_full_name_for_node(node.name)): for statement in node.body.body: statement.visit(self) return False
def visit_ImportFrom(self, node: cst.ImportFrom) -> None: module = node.module names = node.names # module is None for relative imports like `from .. import foo`. # We ignore these for now. if module is None or isinstance(names, cst.ImportStar): return module_name = get_full_name_for_node(module) if module_name is not None: for import_name in _get_import_alias_names(names): AddImportsVisitor.add_needed_import(self.context, module_name, import_name)
def record_typevar( self, node: cst.Call, ) -> None: # pyre-ignore current_assign is never None here name = get_full_name_for_node(self.current_assign.targets[0].target) if name is not None: # Preserve the whole node, even though we currently just use the # name, so that we can match bounds and variance at some point and # determine if two typevars with the same name are indeed the same. # pyre-ignore current_assign is never None here self.typevars[name] = self.current_assign self.current_assign = None
def visit_FunctionDef(self, node: cst.FunctionDef) -> Optional[bool]: self.scope.record_assignment(node.name.value, node) self.provider.set_metadata(node.name, self.scope) with self._new_scope(FunctionScope, node, get_full_name_for_node(node.name)): node.params.visit(self) node.body.visit(self) for decorator in node.decorators: decorator.visit(self) returns = node.returns if returns: returns.visit(self) return False
def leave_Assign( self, original_node: cst.Assign, updated_node: cst.Assign ) -> Union[cst.Assign, cst.AnnAssign]: if len(original_node.targets) > 1: for assign in original_node.targets: target = assign.target if isinstance(target, (cst.Name, cst.Attribute)): name = get_full_name_for_node(target) if name is not None: # Add separate top-level annotations for `a = b = 1` # as `a: int` and `b: int`. self._add_to_toplevel_annotations(name) return updated_node else: return self._annotate_single_target(original_node, updated_node)
def _get_unique_qualified_name(self, node: cst.CSTNode) -> str: name = None names = [ q.name for q in self.get_metadata(QualifiedNameProvider, node) ] if len(names) == 0: # we hit this branch if the stub is directly using a fully # qualified name, which is not technically valid python but is # convenient to allow. name = get_full_name_for_node(node) elif len(names) == 1 and isinstance(names[0], str): name = names[0] if name is None: start = self.get_metadata(PositionProvider, node).start raise ValueError( "Could not resolve a unique qualified name for type " + f"{get_full_name_for_node(node)} at {start.line}:{start.column}. " + f"Candidate names were: {names!r}") return name
def _is_awaitable_callable(annotation: str) -> bool: if not (annotation.startswith("typing.Callable") or annotation.startswith("typing.ClassMethod") or annotation.startswith("StaticMethod")): # Exit early if this is not even a `typing.Callable` annotation. return False try: # Wrap this in a try-except since the type annotation may not be parse-able as a module. # If it is not parse-able, we know it's not what we are looking for anyway, so return `False`. parsed_ann = cst.parse_module(annotation) except Exception: return False # If passed annotation does not match the expected annotation structure for a `typing.Callable` with # typing.Coroutine as the return type, matched_callable_ann will simply be `None`. # The expected structure of an awaitable callable annotation from Pyre is: typing.Callable()[[...], typing.Coroutine[...]] matched_callable_ann: Optional[Dict[str, Union[ Sequence[cst.CSTNode], cst.CSTNode]]] = m.extract( parsed_ann, m.Module(body=[ m.SimpleStatementLine(body=[ m.Expr(value=m.Subscript(slice=[ m.SubscriptElement(), m.SubscriptElement(slice=m.Index(value=m.Subscript( value=m.SaveMatchedNode( m.Attribute(), "base_return_type", )))), ], )) ]), ]), ) if (matched_callable_ann is not None and "base_return_type" in matched_callable_ann): base_return_type = get_full_name_for_node( cst.ensure_type(matched_callable_ann["base_return_type"], cst.CSTNode)) return (base_return_type is not None and base_return_type == "typing.Coroutine") return False
def visit_Lambda(self, node: cst.Lambda) -> None: if m.matches( node, m.Lambda( params=m.MatchIfTrue(self._is_simple_parameter_spec), body=m.Call(args=[ m.Arg(value=m.Name(value=param.name.value), star="", keyword=None) for param in node.params.params ]), ), ): call = cst.ensure_type(node.body, cst.Call) full_name = get_full_name_for_node(call) if full_name is None: full_name = "function" self.report( node, UNNECESSARY_LAMBDA.format(function=full_name), replacement=call.func, )