def record_asname(
self, original_node: Union[cst.Import, cst.ImportFrom]) -> None:
# Record the import's `as` name if it has one, and set the attribute mapping.
names = original_node.names
if not isinstance(names, Sequence):
return
for import_alias in names:
alias_name = get_full_name_for_node(import_alias.name)
if isinstance(original_node, cst.ImportFrom):
module = original_node.module
if module is None:
return
module_name = get_full_name_for_node(module)
if module_name is None:
return
qual_name = f"{module_name}.{alias_name}"
else:
qual_name = alias_name
if qual_name is not None and alias_name is not None:
if qual_name == self.old_name or self.old_name.startswith(
qual_name + "."):
as_name_optional = import_alias.asname
as_name_node = (as_name_optional.name
if as_name_optional is not None else None)
if as_name_node is not None and isinstance(
as_name_node, (cst.Name, cst.Attribute)):
full_as_name = get_full_name_for_node(as_name_node)
if full_as_name is not None:
self.as_name = (full_as_name, alias_name)
def _annotate_single_target(
self, node: cst.Assign, updated_node: cst.Assign
) -> Union[cst.Assign, cst.AnnAssign]:
only_target = node.targets[0].target
if isinstance(only_target, (cst.Tuple, cst.List)):
for element in only_target.elements:
value = element.value
name = get_full_name_for_node(value)
if name:
self._add_to_toplevel_annotations(name)
elif isinstance(only_target, (cst.Subscript)):
pass
else:
name = get_full_name_for_node(only_target)
if name is not None:
self.qualifier.append(name)
if self._qualifier_name() in self.annotations.attribute_annotations and not isinstance(
only_target, cst.Subscript
):
annotation = self.annotations.attribute_annotations[
self._qualifier_name()
]
self.qualifier.pop()
return cst.AnnAssign(cst.Name(name), annotation, node.value)
else:
self.qualifier.pop()
return updated_node
def find_qualified_name_for_import_alike(
assignment_node: Union[cst.Import, cst.ImportFrom],
full_name: str) -> Set[QualifiedName]:
module = ""
results = set()
if isinstance(assignment_node, cst.ImportFrom):
module_attr = assignment_node.module
if module_attr:
# TODO: for relative import, keep the relative Dot in the qualified name
module = get_full_name_for_node(module_attr)
import_names = assignment_node.names
if not isinstance(import_names, cst.ImportStar):
for name in import_names:
real_name = get_full_name_for_node(name.name)
as_name = real_name
if name and name.asname:
name_asname = name.asname
if name_asname:
as_name = cst.ensure_type(name_asname.name,
cst.Name).value
if as_name and full_name.startswith(as_name):
if module:
real_name = f"{module}.{real_name}"
if real_name:
remaining_name = full_name.split(as_name)[1].lstrip(
".")
results.add(
QualifiedName(
f"{real_name}.{remaining_name}"
if remaining_name else real_name,
QualifiedNameSource.IMPORT,
))
return results
def _get_import_alias_names(import_aliases: Sequence[cst.ImportAlias]) -> Set[str]:
import_names = set()
for imported_name in import_aliases:
asname = imported_name.asname
if asname:
import_names.add(get_full_name_for_node(asname.name))
else:
import_names.add(get_full_name_for_node(imported_name.name))
return import_names
def leave_ImportFrom(self, original_node: cst.ImportFrom,
updated_node: cst.ImportFrom) -> cst.ImportFrom:
module = updated_node.module
if module is None:
return updated_node
imported_module_name = get_full_name_for_node(module)
names = original_node.names
if imported_module_name is None or not isinstance(names, Sequence):
return updated_node
else:
new_names = []
for import_alias in names:
alias_name = get_full_name_for_node(import_alias.name)
if alias_name is not None:
qual_name = f"{imported_module_name}.{alias_name}"
if self.old_name == qual_name:
replacement_module = self.gen_replacement_module(
imported_module_name)
replacement_obj = self.gen_replacement(alias_name)
if not replacement_obj:
# The user has requested an `import` statement rather than an `from ... import`.
# This will be taken care of in `leave_Module`, in the meantime, schedule for potential removal.
new_names.append(import_alias)
self.scheduled_removals.add(original_node)
continue
new_import_alias_name: Union[
cst.Attribute,
cst.Name] = self.gen_name_or_attr_node(
replacement_obj)
# Rename on the spot only if this is the only imported name under the module.
if len(names) == 1:
self.bypass_import = True
return updated_node.with_changes(
module=cst.parse_expression(
replacement_module),
names=(cst.ImportAlias(
name=new_import_alias_name), ),
)
# Or if the module name is to stay the same.
elif replacement_module == imported_module_name:
self.bypass_import = True
new_names.append(
cst.ImportAlias(name=new_import_alias_name))
else:
if self.old_name.startswith(qual_name + "."):
# This import might be in use elsewhere in the code, so schedule a potential removal.
self.scheduled_removals.add(original_node)
new_names.append(import_alias)
return updated_node.with_changes(names=new_names)
return updated_node
def _add_annotation_to_imports(
self, annotation: cst.Attribute
) -> Union[cst.Name, cst.Attribute]:
key = get_full_name_for_node(annotation.value)
if key is not None:
# Don't attempt to re-import existing imports.
if key in self.existing_imports:
return annotation
import_name = get_full_name_for_node(annotation.attr)
if import_name is not None:
AddImportsVisitor.add_needed_import(self.context, key, import_name)
return annotation.attr
def leave_Attribute(
self, original_node: cst.Attribute,
updated_node: cst.Attribute) -> Union[cst.Name, cst.Attribute]:
full_name_for_node = get_full_name_for_node(original_node)
if full_name_for_node is None:
raise Exception("Could not parse full name for Attribute node.")
full_replacement_name = self.gen_replacement(full_name_for_node)
# If a node has no associated QualifiedName, we are still inside an import statement.
inside_import_statement: bool = not self.get_metadata(
QualifiedNameProvider, original_node, set())
if (QualifiedNameProvider.has_name(
self,
original_node,
self.old_name,
) or (inside_import_statement
and full_replacement_name == self.new_name)):
new_value, new_attr = self.new_module, self.new_mod_or_obj
if not inside_import_statement:
self.scheduled_removals.add(original_node.value)
if full_replacement_name == self.new_name:
return updated_node.with_changes(
value=cst.parse_expression(new_value),
attr=cst.Name(value=new_attr.rstrip(".")),
)
return self.gen_name_or_attr_node(new_attr)
return updated_node
def visit_ClassDef(self, node: cst.ClassDef) -> Optional[bool]:
self.scope.record_assignment(node.name.value, node)
for decorator in node.decorators:
decorator.visit(self)
for base in node.bases:
base.visit(self)
for keyword in node.keywords:
keyword.visit(self)
with self._new_scope(ClassScope, node,
get_full_name_for_node(node.name)):
for statement in node.body.body:
statement.visit(self)
# visit remaining attributes
for attr in [
node.lpar,
node.rpar,
node.leading_lines,
node.lines_after_decorators,
node.whitespace_after_class,
node.whitespace_after_name,
node.whitespace_before_colon,
]:
if isinstance(attr, cst.CSTNode):
attr.visit(self)
return False
def visit_FunctionDef(self, node: cst.FunctionDef) -> Optional[bool]:
self.scope.record_assignment(node.name.value, node)
self.provider.set_metadata(node.name, self.scope)
with self._new_scope(FunctionScope, node,
get_full_name_for_node(node.name)):
node.params.visit(self)
node.body.visit(self)
# visit remaining attributes
for attr in [
node.asynchronous,
node.leading_lines,
node.lines_after_decorators,
node.whitespace_after_def,
node.whitespace_after_name,
node.whitespace_before_params,
node.whitespace_before_colon,
]:
if isinstance(attr, cst.CSTNode):
attr.visit(self)
for decorator in node.decorators:
decorator.visit(self)
returns = node.returns
if returns:
returns.visit(self)
return False
def visit_Assign(self, node: libcst.Assign) -> bool:
for target_node in node.targets:
target = get_full_name_for_node(target_node.target)
if target == "__all__":
self._in_assignment += 1
return True
return False
def _visit_name_attr_alike(self, node: Union[cst.Name, cst.Attribute]) -> None:
# Look up the local name of this node.
local_name = get_full_name_for_node(node)
if local_name is None:
return
# Look up the scope for this node, remove the import that caused it to exist.
metadata_wrapper = self.context.wrapper
if metadata_wrapper is None:
raise Exception("Cannot look up import, metadata is not computed for node!")
scope_provider = metadata_wrapper.resolve(ScopeProvider)
try:
scope = scope_provider[node]
if scope is None:
# This object has no scope, so we can't remove it.
return
except KeyError:
# This object has no scope, so we can't remove it.
return
while True:
for assignment in scope.assignments[node] or set():
# We only care about non-builtins.
if isinstance(assignment, Assignment):
import_node = assignment.node
if isinstance(import_node, cst.Import):
self._remove_imports_from_import_stmt(local_name, import_node)
elif isinstance(import_node, cst.ImportFrom):
self._remove_imports_from_importfrom_stmt(
local_name, import_node
)
if scope is scope.parent:
break
scope = scope.parent
def get_qualified_names_for(
self, node: Union[str, cst.CSTNode]) -> Collection[QualifiedName]:
"""Get all :class:`~libcst.metadata.QualifiedName` in current scope given a
:class:`~libcst.CSTNode`.
The source of a qualified name can be either :attr:`QualifiedNameSource.IMPORT`,
:attr:`QualifiedNameSource.BUILTIN` or :attr:`QualifiedNameSource.LOCAL`.
Given the following example, ``c`` has qualified name ``a.b.c`` with source ``IMPORT``,
``f`` has qualified name ``Cls.f`` with source ``LOCAL``, ``a`` has qualified name
``Cls.f.<locals>.a``, ``i`` has qualified name ``Cls.f.<locals>.<comprehension>.i``,
and the builtin ``int`` has qualified name ``builtins.int`` with source ``BUILTIN``::
from a.b import c
class Cls:
def f(self) -> "c":
c()
a = int("1")
[i for i in c()]
We extends `PEP-3155 <https://www.python.org/dev/peps/pep-3155/>`_
(defines ``__qualname__`` for class and function only; function namespace is followed
by a ``<locals>``) to provide qualified name for all :class:`~libcst.CSTNode`
recorded by :class:`~libcst.metadata.Assignment` and :class:`~libcst.metadata.Access`.
The namespace of a comprehension (:class:`~libcst.ListComp`, :class:`~libcst.SetComp`,
:class:`~libcst.DictComp`) is represented with ``<comprehension>``.
An imported name may be used for type annotation with :class:`~libcst.SimpleString` and
currently resolving the qualified given :class:`~libcst.SimpleString` is not supported
considering it could be a complex type annotation in the string which is hard to
resolve, e.g. ``List[Union[int, str]]``.
"""
results = set()
full_name = get_full_name_for_node(node)
if full_name is None:
return results
assignments = set()
parts = full_name.split(".")
for i in range(len(parts), 0, -1):
prefix = ".".join(parts[:i])
if prefix in self:
assignments = self[prefix]
break
for assignment in assignments:
if isinstance(assignment, Assignment):
assignment_node = assignment.node
if isinstance(assignment_node, (cst.Import, cst.ImportFrom)):
names = _NameUtil.find_qualified_name_for_import_alike(
assignment_node, full_name)
else:
names = _NameUtil.find_qualified_name_for_non_import(
assignment, full_name)
if not isinstance(node, str) and _is_assignment(
node, assignment_node):
return names
else:
results |= names
elif isinstance(assignment, BuiltinAssignment):
results.add(
QualifiedName(f"builtins.{assignment.name}",
QualifiedNameSource.BUILTIN))
return results
def visit_AnnAssign(self, node: cst.AnnAssign) -> bool:
name = get_full_name_for_node(node.target)
if name is not None:
self.qualifier.append(name)
annotation_value = self._create_import_from_annotation(node.annotation)
self.attribute_annotations[".".join(self.qualifier)] = annotation_value
return True
def visit_Import(self, node: cst.Import) -> None:
for import_alias in node.names:
alias_name = get_full_name_for_node(import_alias.name)
if alias_name is not None:
if alias_name == self.old_name or alias_name.startswith(
self.old_name + "."):
# If the import statement is exactly equivalent to the old name, or we are renaming a top-level module of the import,
# it will be taken care of in `leave_Name` or `leave_Attribute` when visiting the Name and Attribute children of this Import.
self.bypass_import = True
def get_module_name_for_import(self) -> str:
module = ""
if isinstance(self.node, cst.ImportFrom):
module_attr = self.node.module
relative = self.node.relative
if module_attr:
module = get_full_name_for_node(module_attr) or ""
if relative:
module = "." * len(relative) + module
return module
def get_module_name_for_import_alike(
assignment_node: Union[cst.Import, cst.ImportFrom]) -> str:
module = ""
if isinstance(assignment_node, cst.ImportFrom):
module_attr = assignment_node.module
relative = assignment_node.relative
if module_attr:
module = get_full_name_for_node(module_attr) or ""
if relative:
module = "." * len(relative) + module
return module
def test_get_full_name_for_expression(
self,
input: Union[str, cst.CSTNode],
output: Optional[str],
) -> None:
self.assertEqual(get_full_name_for_node(input), output)
if output is None:
with self.assertRaises(Exception):
get_full_name_for_node_or_raise(input)
else:
self.assertEqual(get_full_name_for_node_or_raise(input), output)
def record_typevar(
self,
node: cst.Call,
) -> None:
# pyre-ignore current_assign is never None here
name = get_full_name_for_node(self.current_assign.targets[0].target)
if name is not None:
# pyre-ignore current_assign is never None here
self.annotations.typevars[name] = self.current_assign
self._handle_qualification_and_should_qualify("typing.TypeVar")
self.current_assign = None
def visit_AnnAssign(
self,
node: cst.AnnAssign,
) -> bool:
name = get_full_name_for_node(node.target)
if name is not None:
self.qualifier.append(name)
annotation_value = self._handle_Annotation(annotation=node.annotation)
self.annotations.attributes[".".join(
self.qualifier)] = annotation_value
return True
def find_qualified_name_for_import_alike(
assignment_node: Union[cst.Import, cst.ImportFrom],
full_name: str) -> Set[QualifiedName]:
module = ""
results = set()
if isinstance(assignment_node, cst.ImportFrom):
module_attr = assignment_node.module
if module_attr:
# TODO: for relative import, keep the relative Dot in the qualified name
module = get_full_name_for_node(module_attr)
import_names = assignment_node.names
if not isinstance(import_names, cst.ImportStar):
for name in import_names:
real_name = get_full_name_for_node(name.name)
if not real_name:
continue
# real_name can contain `.` for dotted imports
# for these we want to find the longest prefix that matches full_name
parts = real_name.split(".")
real_names = [
".".join(parts[:i]) for i in range(len(parts), 0, -1)
]
for real_name in real_names:
as_name = real_name
if module:
real_name = f"{module}.{real_name}"
if name and name.asname:
eval_alias = name.evaluated_alias
if eval_alias is not None:
as_name = eval_alias
if full_name.startswith(as_name):
remaining_name = full_name.split(as_name,
1)[1].lstrip(".")
results.add(
QualifiedName(
f"{real_name}.{remaining_name}"
if remaining_name else real_name,
QualifiedNameSource.IMPORT,
))
break
return results
def visit_ImportFrom(self, node: cst.ImportFrom) -> None:
module = node.module
if module is None:
return
imported_module_name = get_full_name_for_node(module)
if imported_module_name is None:
return
if imported_module_name == self.old_name or imported_module_name.startswith(
self.old_name + "."):
# If the imported module is exactly equivalent to the old name or we are renaming a parent module of the current module,
# it will be taken care of in `leave_Name` or `leave_Attribute` when visiting the children of this ImportFrom.
self.bypass_import = True
def _handle_assign_target(self, target: cst.BaseExpression,
value: cst.BaseExpression) -> bool:
target_name = get_full_name_for_node(target)
if target_name == "__all__":
# Assignments such as `__all__ = ["os"]`
# or `__all__ = exports = ["os"]`
if isinstance(value, (cst.List, cst.Tuple, cst.Set)):
self._is_assigned_export.add(value)
return True
elif isinstance(target, cst.Tuple) and isinstance(value, cst.Tuple):
# Assignments such as `__all__, x = ["os"], []`
for element_idx, element_node in enumerate(target.elements):
element_name = get_full_name_for_node(element_node.value)
if element_name == "__all__":
element_value = value.elements[element_idx].value
if isinstance(element_value,
(cst.List, cst.Tuple, cst.Set)):
self._is_assigned_export.add(value)
self._is_assigned_export.add(element_value)
return True
return False
def visit_ClassDef(self, node: cst.ClassDef) -> Optional[bool]:
self.scope.record_assignment(node.name.value, node)
for decorator in node.decorators:
decorator.visit(self)
for base in node.bases:
base.visit(self)
for keyword in node.keywords:
keyword.visit(self)
with self._new_scope(ClassScope, node, get_full_name_for_node(node.name)):
for statement in node.body.body:
statement.visit(self)
return False
def visit_ImportFrom(self, node: cst.ImportFrom) -> None:
module = node.module
names = node.names
# module is None for relative imports like `from .. import foo`.
# We ignore these for now.
if module is None or isinstance(names, cst.ImportStar):
return
module_name = get_full_name_for_node(module)
if module_name is not None:
for import_name in _get_import_alias_names(names):
AddImportsVisitor.add_needed_import(self.context, module_name,
import_name)
def record_typevar(
self,
node: cst.Call,
) -> None:
# pyre-ignore current_assign is never None here
name = get_full_name_for_node(self.current_assign.targets[0].target)
if name is not None:
# Preserve the whole node, even though we currently just use the
# name, so that we can match bounds and variance at some point and
# determine if two typevars with the same name are indeed the same.
# pyre-ignore current_assign is never None here
self.typevars[name] = self.current_assign
self.current_assign = None
def visit_FunctionDef(self, node: cst.FunctionDef) -> Optional[bool]:
self.scope.record_assignment(node.name.value, node)
self.provider.set_metadata(node.name, self.scope)
with self._new_scope(FunctionScope, node, get_full_name_for_node(node.name)):
node.params.visit(self)
node.body.visit(self)
for decorator in node.decorators:
decorator.visit(self)
returns = node.returns
if returns:
returns.visit(self)
return False
def leave_Assign(
self, original_node: cst.Assign, updated_node: cst.Assign
) -> Union[cst.Assign, cst.AnnAssign]:
if len(original_node.targets) > 1:
for assign in original_node.targets:
target = assign.target
if isinstance(target, (cst.Name, cst.Attribute)):
name = get_full_name_for_node(target)
if name is not None:
# Add separate top-level annotations for `a = b = 1`
# as `a: int` and `b: int`.
self._add_to_toplevel_annotations(name)
return updated_node
else:
return self._annotate_single_target(original_node, updated_node)
def _get_unique_qualified_name(self, node: cst.CSTNode) -> str:
name = None
names = [
q.name for q in self.get_metadata(QualifiedNameProvider, node)
]
if len(names) == 0:
# we hit this branch if the stub is directly using a fully
# qualified name, which is not technically valid python but is
# convenient to allow.
name = get_full_name_for_node(node)
elif len(names) == 1 and isinstance(names[0], str):
name = names[0]
if name is None:
start = self.get_metadata(PositionProvider, node).start
raise ValueError(
"Could not resolve a unique qualified name for type " +
f"{get_full_name_for_node(node)} at {start.line}:{start.column}. "
+ f"Candidate names were: {names!r}")
return name
def _is_awaitable_callable(annotation: str) -> bool:
if not (annotation.startswith("typing.Callable")
or annotation.startswith("typing.ClassMethod")
or annotation.startswith("StaticMethod")):
# Exit early if this is not even a `typing.Callable` annotation.
return False
try:
# Wrap this in a try-except since the type annotation may not be parse-able as a module.
# If it is not parse-able, we know it's not what we are looking for anyway, so return `False`.
parsed_ann = cst.parse_module(annotation)
except Exception:
return False
# If passed annotation does not match the expected annotation structure for a `typing.Callable` with
# typing.Coroutine as the return type, matched_callable_ann will simply be `None`.
# The expected structure of an awaitable callable annotation from Pyre is: typing.Callable()[[...], typing.Coroutine[...]]
matched_callable_ann: Optional[Dict[str, Union[
Sequence[cst.CSTNode], cst.CSTNode]]] = m.extract(
parsed_ann,
m.Module(body=[
m.SimpleStatementLine(body=[
m.Expr(value=m.Subscript(slice=[
m.SubscriptElement(),
m.SubscriptElement(slice=m.Index(value=m.Subscript(
value=m.SaveMatchedNode(
m.Attribute(),
"base_return_type",
)))),
], ))
]),
]),
)
if (matched_callable_ann is not None
and "base_return_type" in matched_callable_ann):
base_return_type = get_full_name_for_node(
cst.ensure_type(matched_callable_ann["base_return_type"],
cst.CSTNode))
return (base_return_type is not None
and base_return_type == "typing.Coroutine")
return False
def visit_Lambda(self, node: cst.Lambda) -> None:
if m.matches(
node,
m.Lambda(
params=m.MatchIfTrue(self._is_simple_parameter_spec),
body=m.Call(args=[
m.Arg(value=m.Name(value=param.name.value),
star="",
keyword=None) for param in node.params.params
]),
),
):
call = cst.ensure_type(node.body, cst.Call)
full_name = get_full_name_for_node(call)
if full_name is None:
full_name = "function"
self.report(
node,
UNNECESSARY_LAMBDA.format(function=full_name),
replacement=call.func,
)
