def leave_SimpleStatementLine(
self, original_node: SimpleStatementLine, updated_node: SimpleStatementLine
) -> Union[BaseStatement, RemovalSentinel]:
for n in updated_node.body:
if m.matches(n, m.ImportFrom(module=m.Name("__future__"))):
self.python_future_updated_node = updated_node
elif m.matches(n, m.ImportFrom(module=m.Name("builtins"))):
self.builtins_updated_node = updated_node
elif m.matches(
n,
m.ImportFrom(
module=m.Attribute(value=m.Name("future"), attr=m.Name("utils"))
),
):
self.future_utils_updated_node = updated_node
return updated_node
def _test_import_from(self, node: ImportFrom) -> bool:
return m.matches(
node,
m.ImportFrom(module=m.Attribute(
value=m.Attribute(value=m.Name("django"),
attr=m.Name(value="utils")),
attr=m.Name("translation"),
), ),
)
def leave_ImportFrom(
self, original_node: ImportFrom, updated_node: ImportFrom
) -> Union[BaseSmallStatement, RemovalSentinel]:
base_cls_matcher = []
if m.matches(
updated_node,
m.ImportFrom(module=module_matcher(["django", "contrib", "admin"])),
):
for imported_name in updated_node.names:
if m.matches(
imported_name, m.ImportAlias(name=m.Name("TabularInline"))
):
base_cls_matcher.append(m.Arg(m.Name("TabularInline")))
if m.matches(
imported_name, m.ImportAlias(name=m.Name("StackedInline"))
):
base_cls_matcher.append(m.Arg(m.Name("StackedInline")))
if m.matches(
updated_node,
m.ImportFrom(module=module_matcher(["django", "contrib"])),
):
for imported_name in updated_node.names:
if m.matches(imported_name, m.ImportAlias(name=m.Name("admin"))):
base_cls_matcher.extend(
[
m.Arg(
m.Attribute(
value=m.Name("admin"), attr=m.Name("TabularInline")
)
),
m.Arg(
m.Attribute(
value=m.Name("admin"), attr=m.Name("StackedInline")
)
),
]
)
# Save valid matchers in the context
if base_cls_matcher:
self.context.scratch[self.ctx_key_base_cls_matcher] = m.OneOf(
*base_cls_matcher
)
return super().leave_ImportFrom(original_node, updated_node)
def leave_ImportFrom(
self, original_node: ImportFrom, updated_node: ImportFrom
) -> Union[BaseSmallStatement, RemovalSentinel]:
if isinstance(updated_node.names, ImportStar):
return super().leave_ImportFrom(original_node, updated_node)
if m.matches(
updated_node,
m.ImportFrom(module=module_matcher(["django", "db"])),
):
for imported_name in updated_node.names:
if m.matches(imported_name, m.ImportAlias(name=m.Name("models"))):
self.add_decorator_matcher(
m.Decorator(
decorator=m.Attribute(
value=m.Name("models"), attr=m.Name("permalink")
)
)
)
if m.matches(
updated_node,
m.ImportFrom(module=module_matcher(["django", "db", "models"])),
):
updated_names = []
for imported_name in updated_node.names:
if m.matches(imported_name, m.ImportAlias(name=m.Name("permalink"))):
decorator_name_str = (
imported_name.evaluated_alias or imported_name.evaluated_name
)
self.add_decorator_matcher(
m.Decorator(decorator=m.Name(decorator_name_str))
)
else:
updated_names.append(imported_name)
if not updated_names:
return RemoveFromParent()
# sort imports
new_names = sorted(updated_names, key=lambda n: n.evaluated_name)
# remove any trailing commas
last_name = new_names[-1]
if last_name.comma != MaybeSentinel.DEFAULT:
new_names[-1] = last_name.with_changes(comma=MaybeSentinel.DEFAULT)
return updated_node.with_changes(names=new_names)
return super().leave_ImportFrom(original_node, updated_node)
def visit_ImportFrom(self, node: cst.ImportFrom) -> None:
if m.matches(
node,
m.ImportFrom(
module=m.Name("__future__"),
names=[
m.ZeroOrMore(),
m.ImportAlias(name=m.Name("annotations")),
m.ZeroOrMore(),
],
),
):
self.has_future_annotations_import = True
def _is_import_line(
line: Union[cst.SimpleStatementLine,
cst.BaseCompoundStatement]) -> bool:
return m.matches(line,
m.SimpleStatementLine(body=[m.Import() | m.ImportFrom()]))
class Modernizer(m.MatcherDecoratableTransformer):
METADATA_DEPENDENCIES = (PositionProvider,)
# FIXME use a stack of e.g. SimpleStatementLine then proper visit_Import/ImportFrom to store the ssl node
def __init__(
self, path: Path, verbose: bool = False, ignored: Optional[List[str]] = None
):
super().__init__()
self.path = path
self.verbose = verbose
self.ignored = set(ignored or [])
self.errors = False
self.stack: List[Tuple[str, ...]] = []
self.annotations: Dict[
Tuple[str, ...], Comment # key: tuple of canonical variable name
] = {}
self.python_future_updated_node: Optional[SimpleStatementLine] = None
self.python_future_imports: Dict[str, str] = {}
self.python_future_new_imports: Set[str] = set()
self.builtins_imports: Dict[str, str] = {}
self.builtins_new_imports: Set[str] = set()
self.builtins_updated_node: Optional[SimpleStatementLine] = None
self.future_utils_imports: Dict[str, str] = {}
self.future_utils_new_imports: Set[str] = set()
self.future_utils_updated_node: Optional[SimpleStatementLine] = None
# self.last_import_node: Optional[CSTNode] = None
self.last_import_node_stmt: Optional[CSTNode] = None
# @m.call_if_inside(m.ImportFrom(module=m.Name("__future__")))
# @m.visit(m.ImportAlias() | m.ImportStar())
# def import_python_future_check(self, node: Union[ImportAlias, ImportStar]) -> None:
# self.add_import(self.python_future_imports, node)
# @m.leave(m.ImportFrom(module=m.Name("__future__")))
# def import_python_future_modify(
# self, original_node: ImportFrom, updated_node: ImportFrom
# ) -> Union[BaseSmallStatement, RemovalSentinel]:
# return updated_node
@m.call_if_inside(m.ImportFrom(module=m.Name("builtins")))
@m.visit(m.ImportAlias() | m.ImportStar())
def import_builtins_check(self, node: Union[ImportAlias, ImportStar]) -> None:
self.add_import(self.builtins_imports, node)
# @m.leave(m.ImportFrom(module=m.Name("builtins")))
# def builtins_modify(
# self, original_node: ImportFrom, updated_node: ImportFrom
# ) -> Union[BaseSmallStatement, RemovalSentinel]:
# return updated_node
@m.call_if_inside(
m.ImportFrom(module=m.Attribute(value=m.Name("future"), attr=m.Name("utils")))
)
@m.visit(m.ImportAlias() | m.ImportStar())
def import_future_utils_check(self, node: Union[ImportAlias, ImportStar]) -> None:
self.add_import(self.future_utils_imports, node)
# @m.leave(
# m.ImportFrom(module=m.Attribute(value=m.Name("future"), attr=m.Name("utils")))
# )
# def future_utils_modify(
# self, original_node: ImportFrom, updated_node: ImportFrom
# ) -> Union[BaseSmallStatement, RemovalSentinel]:
# return updated_node
@staticmethod
def add_import(
imports: Dict[str, str], node: Union[ImportAlias, ImportStar]
) -> None:
if isinstance(node, ImportAlias):
imports[node.name.value] = (
node.asname.name.value if node.asname else node.name.value
)
else:
imports["*"] = "*"
# @m.call_if_not_inside(m.BaseCompoundStatement())
# def visit_Import(self, node: Import) -> Optional[bool]:
# self.last_import_node = node
# return None
# @m.call_if_not_inside(m.BaseCompoundStatement())
# def visit_ImportFrom(self, node: ImportFrom) -> Optional[bool]:
# self.last_import_node = node
# return None
@m.call_if_not_inside(m.ClassDef() | m.FunctionDef() | m.If())
def visit_SimpleStatementLine(self, node: SimpleStatementLine) -> Optional[bool]:
for n in node.body:
if m.matches(n, m.Import() | m.ImportFrom()):
self.last_import_node_stmt = node
return None
@m.call_if_not_inside(m.ClassDef() | m.FunctionDef() | m.If())
def leave_SimpleStatementLine(
self, original_node: SimpleStatementLine, updated_node: SimpleStatementLine
) -> Union[BaseStatement, RemovalSentinel]:
for n in updated_node.body:
if m.matches(n, m.ImportFrom(module=m.Name("__future__"))):
self.python_future_updated_node = updated_node
elif m.matches(n, m.ImportFrom(module=m.Name("builtins"))):
self.builtins_updated_node = updated_node
elif m.matches(
n,
m.ImportFrom(
module=m.Attribute(value=m.Name("future"), attr=m.Name("utils"))
),
):
self.future_utils_updated_node = updated_node
return updated_node
# @m.visit(
# m.AllOf(
# m.SimpleStatementLine(),
# m.MatchIfTrue(
# lambda node: any(m.matches(c, m.Assign()) for c in node.children)
# ),
# m.MatchIfTrue(
# lambda node: "# type:" in node.trailing_whitespace.comment.value
# ),
# )
# )
# def visit_assign(self, node: SimpleStatementSuite) -> None:
# return None
def visit_Param(self, node: Param) -> Optional[bool]:
class Visitor(m.MatcherDecoratableVisitor):
def __init__(self):
super().__init__()
self.ptype: Optional[str] = None
def visit_TrailingWhitespace_comment(
self, node: "TrailingWhitespace"
) -> None:
if node.comment and "type:" in node.comment.value:
mo = re.match(r"#\s*type:\s*(\S*)", node.comment.value)
self.ptype = mo.group(1) if mo else None
return None
v = Visitor()
node.visit(v)
if self.verbose:
pos = self.get_metadata(PositionProvider, node).start
print(
f"{self.path}:{pos.line}:{pos.column}: parameter {node.name.value}: {v.ptype or 'unknown type'}"
)
return None
@m.visit(m.SimpleStatementLine())
def visit_simple_stmt(self, node: SimpleStatementLine) -> None:
assign = None
for c in node.children:
if m.matches(c, m.Assign()):
assign = ensure_type(c, Assign)
if assign:
if m.MatchIfTrue(
lambda n: n.trailing_whitespace.comment
and "type:" in n.trailing_whitespace.comment.value
):
class TypingVisitor(m.MatcherDecoratableVisitor):
def __init__(self):
super().__init__()
self.vtype = None
def visit_TrailingWhitespace_comment(
self, node: "TrailingWhitespace"
) -> None:
if node.comment:
mo = re.match(r"#\s*type:\s*(\S*)", node.comment.value)
if mo:
vtype = mo.group(1)
return None
tv = TypingVisitor()
node.visit(tv)
vtype = tv.vtype
else:
vtype = None
class NameVisitor(m.MatcherDecoratableVisitor):
def __init__(self):
super().__init__()
self.names: List[str] = []
def visit_Name(self, node: Name) -> Optional[bool]:
self.names.append(node.value)
return None
if self.verbose:
pos = self.get_metadata(PositionProvider, node).start
for target in assign.targets:
v = NameVisitor()
target.visit(v)
for name in v.names:
print(
f"{self.path}:{pos.line}:{pos.column}: variable {name}: {vtype or 'unknown type'}"
)
def visit_FunctionDef_body(self, node: FunctionDef) -> None:
class Visitor(m.MatcherDecoratableVisitor):
def __init__(self):
super().__init__()
def visit_EmptyLine_comment(self, node: "EmptyLine") -> None:
# FIXME too many matches on test_param_02
if not node.comment:
return
# TODO: use comment.value
return None
v = Visitor()
node.visit(v)
return None
map_matcher = m.Call(
func=m.Name("filter") | m.Name("map") | m.Name("zip") | m.Name("range")
)
@m.visit(map_matcher)
def visit_map(self, node: Call) -> None:
func_name = ensure_type(node.func, Name).value
if func_name not in self.builtins_imports:
self.builtins_new_imports.add(func_name)
@m.call_if_not_inside(
m.Call(
func=m.Name("list")
| m.Name("set")
| m.Name("tuple")
| m.Attribute(attr=m.Name("join"))
)
| m.CompFor()
| m.For()
)
@m.leave(map_matcher)
def fix_map(self, original_node: Call, updated_node: Call) -> BaseExpression:
# TODO test with CompFor etc.
# TODO improve join test
func_name = ensure_type(updated_node.func, Name).value
if func_name not in self.builtins_imports:
updated_node = Call(func=Name("list"), args=[Arg(updated_node)])
return updated_node
@m.visit(m.Call(func=m.Name("xrange") | m.Name("raw_input")))
def visit_xrange(self, node: Call) -> None:
orig_func_name = ensure_type(node.func, Name).value
func_name = "range" if orig_func_name == "xrange" else "input"
if func_name not in self.builtins_imports:
self.builtins_new_imports.add(func_name)
@m.leave(m.Call(func=m.Name("xrange") | m.Name("raw_input")))
def fix_xrange(self, original_node: Call, updated_node: Call) -> BaseExpression:
orig_func_name = ensure_type(updated_node.func, Name).value
func_name = "range" if orig_func_name == "xrange" else "input"
return updated_node.with_changes(func=Name(func_name))
iter_matcher = m.Call(
func=m.Attribute(
attr=m.Name("iterkeys") | m.Name("itervalues") | m.Name("iteritems")
)
)
@m.visit(iter_matcher)
def visit_iter(self, node: Call) -> None:
func_name = ensure_type(node.func, Attribute).attr.value
if func_name not in self.future_utils_imports:
self.future_utils_new_imports.add(func_name)
@m.leave(iter_matcher)
def fix_iter(self, original_node: Call, updated_node: Call) -> BaseExpression:
attribute = ensure_type(updated_node.func, Attribute)
func_name = attribute.attr
dict_name = attribute.value
return updated_node.with_changes(func=func_name, args=[Arg(dict_name)])
not_iter_matcher = m.Call(
func=m.Attribute(attr=m.Name("keys") | m.Name("values") | m.Name("items"))
)
@m.call_if_not_inside(
m.Call(
func=m.Name("list")
| m.Name("set")
| m.Name("tuple")
| m.Attribute(attr=m.Name("join"))
)
| m.CompFor()
| m.For()
)
@m.leave(not_iter_matcher)
def fix_not_iter(self, original_node: Call, updated_node: Call) -> BaseExpression:
updated_node = Call(func=Name("list"), args=[Arg(updated_node)])
return updated_node
@m.call_if_not_inside(m.Import() | m.ImportFrom())
@m.leave(m.Name(value="unicode"))
def fix_unicode(self, original_node: Name, updated_node: Name) -> BaseExpression:
value = "text_type"
if value not in self.future_utils_imports:
self.future_utils_new_imports.add(value)
return updated_node.with_changes(value=value)
def leave_Module(self, original_node: Module, updated_node: Module) -> Module:
updated_node = self.update_imports(
original_node,
updated_node,
"builtins",
self.builtins_updated_node,
self.builtins_imports,
self.builtins_new_imports,
True,
)
updated_node = self.update_imports(
original_node,
updated_node,
"future.utils",
self.future_utils_updated_node,
self.future_utils_imports,
self.future_utils_new_imports,
False,
)
return updated_node
def update_imports(
self,
original_module: Module,
updated_module: Module,
import_name: str,
updated_import_node: SimpleStatementLine,
current_imports: Dict[str, str],
new_imports: Set[str],
noqa: bool,
) -> Module:
if not new_imports:
return updated_module
noqa_comment = " # noqa" if noqa else ""
if not updated_import_node:
i = -1
blank_lines = "\n\n"
if self.last_import_node_stmt:
blank_lines = ""
for i, (original, updated) in enumerate(
zip(original_module.body, updated_module.body)
):
if original is self.last_import_node_stmt:
break
stmt = parse_module(
f"from {import_name} import {', '.join(sorted(new_imports))}{noqa_comment}\n{blank_lines}",
config=updated_module.config_for_parsing,
)
body = list(updated_module.body)
self.last_import_node_stmt = stmt
return updated_module.with_changes(
body=body[: i + 1] + stmt.children + body[i + 1 :]
)
else:
if "*" not in current_imports:
current_imports_set = {
f"{k}" if k == v else f"{k} as {v}"
for k, v in current_imports.items()
}
stmt = parse_statement(
f"from {import_name} import {', '.join(sorted(new_imports | current_imports_set))}{noqa_comment}"
)
return updated_module.deep_replace(updated_import_node, stmt)
# for i, (original, updated) in enumerate(
# zip(original_module.body, updated_module.body)
# ):
# if original is original_import_node:
# body = list(updated_module.body)
# return updated_module.with_changes(
# body=body[:i] + [stmt] + body[i + 1 :]
# )
return updated_module
def visit_SimpleStatementLine(self, node: SimpleStatementLine) -> Optional[bool]:
for n in node.body:
if m.matches(n, m.Import() | m.ImportFrom()):
self.last_import_node_stmt = node
return None
class Checker(m.MatcherDecoratableVisitor):
METADATA_DEPENDENCIES = (PositionProvider,)
def __init__(
self, path: Path, verbose: bool = False, ignored: Optional[List[str]] = None
):
super().__init__()
self.path = path
self.verbose = verbose
self.ignored = set(ignored or [])
self.future_division = False
self.errors = False
self.stack: List[str] = []
@m.call_if_inside(m.ImportFrom(module=m.Name("__future__")))
@m.visit(m.ImportAlias(name=m.Name("division")))
def import_div(self, node: ImportAlias) -> None:
self.future_division = True
@m.visit(m.BinaryOperation(operator=m.Divide()))
def check_div(self, node: BinaryOperation) -> None:
if "division" in self.ignored:
return
if not self.future_division:
pos = self.get_metadata(PositionProvider, node).start
print(
f"{self.path}:{pos.line}:{pos.column}: division without `from __future__ import division`"
)
self.errors = True
@m.visit(m.Attribute(attr=m.Name("maxint"), value=m.Name("sys")))
def check_maxint(self, node: Attribute) -> None:
if "sys.maxint" in self.ignored:
return
pos = self.get_metadata(PositionProvider, node).start
print(f"{self.path}:{pos.line}:{pos.column}: use of sys.maxint")
self.errors = True
def visit_ClassDef(self, node: ClassDef) -> None:
self.stack.append(node.name.value)
def leave_ClassDef(self, node: ClassDef) -> None:
self.stack.pop()
def visit_FunctionDef(self, node: FunctionDef) -> None:
self.stack.append(node.name.value)
def leave_FunctionDef(self, node: FunctionDef) -> None:
self.stack.pop()
def visit_ClassDef_bases(self, node: "ClassDef") -> None:
return
@m.visit(
m.Call(
func=m.Attribute(attr=m.Name("assertEquals") | m.Name("assertItemsEqual"))
)
)
def visit_old_assert(self, node: Call) -> None:
name = ensure_type(node.func, Attribute).attr.value
if name in self.ignored:
return
pos = self.get_metadata(PositionProvider, node).start
print(f"{self.path}:{pos.line}:{pos.column}: use of {name}")
self.errors = True
def import_from_matches(node: ImportFrom, module_parts: Sequence[str]) -> bool:
"""Check if an `ImportFrom` node matches sequence of module parts."""
return m.matches(node, m.ImportFrom(module=module_matcher(module_parts)))
class GatherUnusedImportsVisitor(ContextAwareVisitor):
"""
Collects all imports from a module not directly used in the same module.
Intended to be instantiated and passed to a :class:`libcst.Module`
:meth:`~libcst.CSTNode.visit` method to process the full module.
Note that imports that are only used indirectly (from other modules) are
still collected.
After visiting a module the attribute ``unused_imports`` will contain a
set of unused :class:`~libcst.ImportAlias` objects, paired with their
parent import node.
"""
METADATA_DEPENDENCIES: Tuple[ProviderT] = (
*GatherNamesFromStringAnnotationsVisitor.METADATA_DEPENDENCIES,
ScopeProvider,
)
def __init__(self, context: CodemodContext) -> None:
super().__init__(context)
self._string_annotation_names: Set[str] = set()
self._exported_names: Set[str] = set()
#: Contains a set of (alias, parent_import) pairs that are not used
#: in the module after visiting.
self.unused_imports: Set[Tuple[cst.ImportAlias,
Union[cst.Import,
cst.ImportFrom]]] = set()
def visit_Module(self, node: cst.Module) -> bool:
export_collector = GatherExportsVisitor(self.context)
node.visit(export_collector)
self._exported_names = export_collector.explicit_exported_objects
annotation_visitor = GatherNamesFromStringAnnotationsVisitor(
self.context)
node.visit(annotation_visitor)
self._string_annotation_names = annotation_visitor.names
return True
@m.visit(m.Import()
| m.ImportFrom(
module=m.DoesNotMatch(m.Name("__future__")),
names=m.DoesNotMatch(m.ImportStar()),
))
def handle_import(self, node: Union[cst.Import, cst.ImportFrom]) -> None:
names = node.names
assert not isinstance(names, cst.ImportStar) # hello, type checker
for alias in names:
self.unused_imports.add((alias, node))
def leave_Module(self, original_node: cst.Module) -> None:
self.unused_imports = self.filter_unused_imports(self.unused_imports)
def filter_unused_imports(
self,
candidates: Iterable[Tuple[cst.ImportAlias, Union[cst.Import,
cst.ImportFrom]]],
) -> Set[Tuple[cst.ImportAlias, Union[cst.Import, cst.ImportFrom]]]:
"""
Return the imports in ``candidates`` which are not used.
This function implements the main logic of this visitor, and is called after traversal. It calls :meth:`~is_in_use` on each import.
Override this in a subclass for additional filtering.
"""
unused_imports = set()
for (alias, parent) in candidates:
scope = self.get_metadata(ScopeProvider, parent)
if scope is None:
continue
if not self.is_in_use(scope, alias):
unused_imports.add((alias, parent))
return unused_imports
def is_in_use(self, scope: cst.metadata.Scope,
alias: cst.ImportAlias) -> bool:
"""
Check if ``alias`` is in use in the given ``scope``.
An alias is in use if it's directly referenced, exported, or appears in
a string type annotation. Override this in a subclass for additional
filtering.
"""
asname = alias.asname
names = _gen_dotted_names(
cst.ensure_type(asname.name, cst.Name
) if asname is not None else alias.name)
for name_or_alias, _ in names:
if (name_or_alias in self._exported_names
or name_or_alias in self._string_annotation_names):
return True
for assignment in scope[name_or_alias]:
if (isinstance(assignment, cst.metadata.Assignment)
and isinstance(assignment.node,
(cst.ImportFrom, cst.Import))
and len(assignment.references) > 0):
return True
return False
class TransformerWithUnionReturnAnnotation(
m.MatcherDecoratableTransformer):
@m.leave(m.ImportFrom(module=m.Name(value="typing")))
def test(self, original_node: cst.ImportFrom,
updated_node: cst.ImportFrom) -> FakeUnion:
pass
def _test_import_from(self, node: ImportFrom) -> bool:
"""Check if 'import from' should be updated."""
return m.matches(
node, m.ImportFrom(module=module_matcher(self.old_module_parts))
)