def leave_SimpleStatementLine( self, original_node: SimpleStatementLine, updated_node: SimpleStatementLine ) -> Union[BaseStatement, RemovalSentinel]: for n in updated_node.body: if m.matches(n, m.ImportFrom(module=m.Name("__future__"))): self.python_future_updated_node = updated_node elif m.matches(n, m.ImportFrom(module=m.Name("builtins"))): self.builtins_updated_node = updated_node elif m.matches( n, m.ImportFrom( module=m.Attribute(value=m.Name("future"), attr=m.Name("utils")) ), ): self.future_utils_updated_node = updated_node return updated_node
def _test_import_from(self, node: ImportFrom) -> bool: return m.matches( node, m.ImportFrom(module=m.Attribute( value=m.Attribute(value=m.Name("django"), attr=m.Name(value="utils")), attr=m.Name("translation"), ), ), )
def leave_ImportFrom( self, original_node: ImportFrom, updated_node: ImportFrom ) -> Union[BaseSmallStatement, RemovalSentinel]: base_cls_matcher = [] if m.matches( updated_node, m.ImportFrom(module=module_matcher(["django", "contrib", "admin"])), ): for imported_name in updated_node.names: if m.matches( imported_name, m.ImportAlias(name=m.Name("TabularInline")) ): base_cls_matcher.append(m.Arg(m.Name("TabularInline"))) if m.matches( imported_name, m.ImportAlias(name=m.Name("StackedInline")) ): base_cls_matcher.append(m.Arg(m.Name("StackedInline"))) if m.matches( updated_node, m.ImportFrom(module=module_matcher(["django", "contrib"])), ): for imported_name in updated_node.names: if m.matches(imported_name, m.ImportAlias(name=m.Name("admin"))): base_cls_matcher.extend( [ m.Arg( m.Attribute( value=m.Name("admin"), attr=m.Name("TabularInline") ) ), m.Arg( m.Attribute( value=m.Name("admin"), attr=m.Name("StackedInline") ) ), ] ) # Save valid matchers in the context if base_cls_matcher: self.context.scratch[self.ctx_key_base_cls_matcher] = m.OneOf( *base_cls_matcher ) return super().leave_ImportFrom(original_node, updated_node)
def leave_ImportFrom( self, original_node: ImportFrom, updated_node: ImportFrom ) -> Union[BaseSmallStatement, RemovalSentinel]: if isinstance(updated_node.names, ImportStar): return super().leave_ImportFrom(original_node, updated_node) if m.matches( updated_node, m.ImportFrom(module=module_matcher(["django", "db"])), ): for imported_name in updated_node.names: if m.matches(imported_name, m.ImportAlias(name=m.Name("models"))): self.add_decorator_matcher( m.Decorator( decorator=m.Attribute( value=m.Name("models"), attr=m.Name("permalink") ) ) ) if m.matches( updated_node, m.ImportFrom(module=module_matcher(["django", "db", "models"])), ): updated_names = [] for imported_name in updated_node.names: if m.matches(imported_name, m.ImportAlias(name=m.Name("permalink"))): decorator_name_str = ( imported_name.evaluated_alias or imported_name.evaluated_name ) self.add_decorator_matcher( m.Decorator(decorator=m.Name(decorator_name_str)) ) else: updated_names.append(imported_name) if not updated_names: return RemoveFromParent() # sort imports new_names = sorted(updated_names, key=lambda n: n.evaluated_name) # remove any trailing commas last_name = new_names[-1] if last_name.comma != MaybeSentinel.DEFAULT: new_names[-1] = last_name.with_changes(comma=MaybeSentinel.DEFAULT) return updated_node.with_changes(names=new_names) return super().leave_ImportFrom(original_node, updated_node)
def visit_ImportFrom(self, node: cst.ImportFrom) -> None: if m.matches( node, m.ImportFrom( module=m.Name("__future__"), names=[ m.ZeroOrMore(), m.ImportAlias(name=m.Name("annotations")), m.ZeroOrMore(), ], ), ): self.has_future_annotations_import = True
def _is_import_line( line: Union[cst.SimpleStatementLine, cst.BaseCompoundStatement]) -> bool: return m.matches(line, m.SimpleStatementLine(body=[m.Import() | m.ImportFrom()]))
class Modernizer(m.MatcherDecoratableTransformer): METADATA_DEPENDENCIES = (PositionProvider,) # FIXME use a stack of e.g. SimpleStatementLine then proper visit_Import/ImportFrom to store the ssl node def __init__( self, path: Path, verbose: bool = False, ignored: Optional[List[str]] = None ): super().__init__() self.path = path self.verbose = verbose self.ignored = set(ignored or []) self.errors = False self.stack: List[Tuple[str, ...]] = [] self.annotations: Dict[ Tuple[str, ...], Comment # key: tuple of canonical variable name ] = {} self.python_future_updated_node: Optional[SimpleStatementLine] = None self.python_future_imports: Dict[str, str] = {} self.python_future_new_imports: Set[str] = set() self.builtins_imports: Dict[str, str] = {} self.builtins_new_imports: Set[str] = set() self.builtins_updated_node: Optional[SimpleStatementLine] = None self.future_utils_imports: Dict[str, str] = {} self.future_utils_new_imports: Set[str] = set() self.future_utils_updated_node: Optional[SimpleStatementLine] = None # self.last_import_node: Optional[CSTNode] = None self.last_import_node_stmt: Optional[CSTNode] = None # @m.call_if_inside(m.ImportFrom(module=m.Name("__future__"))) # @m.visit(m.ImportAlias() | m.ImportStar()) # def import_python_future_check(self, node: Union[ImportAlias, ImportStar]) -> None: # self.add_import(self.python_future_imports, node) # @m.leave(m.ImportFrom(module=m.Name("__future__"))) # def import_python_future_modify( # self, original_node: ImportFrom, updated_node: ImportFrom # ) -> Union[BaseSmallStatement, RemovalSentinel]: # return updated_node @m.call_if_inside(m.ImportFrom(module=m.Name("builtins"))) @m.visit(m.ImportAlias() | m.ImportStar()) def import_builtins_check(self, node: Union[ImportAlias, ImportStar]) -> None: self.add_import(self.builtins_imports, node) # @m.leave(m.ImportFrom(module=m.Name("builtins"))) # def builtins_modify( # self, original_node: ImportFrom, updated_node: ImportFrom # ) -> Union[BaseSmallStatement, RemovalSentinel]: # return updated_node @m.call_if_inside( m.ImportFrom(module=m.Attribute(value=m.Name("future"), attr=m.Name("utils"))) ) @m.visit(m.ImportAlias() | m.ImportStar()) def import_future_utils_check(self, node: Union[ImportAlias, ImportStar]) -> None: self.add_import(self.future_utils_imports, node) # @m.leave( # m.ImportFrom(module=m.Attribute(value=m.Name("future"), attr=m.Name("utils"))) # ) # def future_utils_modify( # self, original_node: ImportFrom, updated_node: ImportFrom # ) -> Union[BaseSmallStatement, RemovalSentinel]: # return updated_node @staticmethod def add_import( imports: Dict[str, str], node: Union[ImportAlias, ImportStar] ) -> None: if isinstance(node, ImportAlias): imports[node.name.value] = ( node.asname.name.value if node.asname else node.name.value ) else: imports["*"] = "*" # @m.call_if_not_inside(m.BaseCompoundStatement()) # def visit_Import(self, node: Import) -> Optional[bool]: # self.last_import_node = node # return None # @m.call_if_not_inside(m.BaseCompoundStatement()) # def visit_ImportFrom(self, node: ImportFrom) -> Optional[bool]: # self.last_import_node = node # return None @m.call_if_not_inside(m.ClassDef() | m.FunctionDef() | m.If()) def visit_SimpleStatementLine(self, node: SimpleStatementLine) -> Optional[bool]: for n in node.body: if m.matches(n, m.Import() | m.ImportFrom()): self.last_import_node_stmt = node return None @m.call_if_not_inside(m.ClassDef() | m.FunctionDef() | m.If()) def leave_SimpleStatementLine( self, original_node: SimpleStatementLine, updated_node: SimpleStatementLine ) -> Union[BaseStatement, RemovalSentinel]: for n in updated_node.body: if m.matches(n, m.ImportFrom(module=m.Name("__future__"))): self.python_future_updated_node = updated_node elif m.matches(n, m.ImportFrom(module=m.Name("builtins"))): self.builtins_updated_node = updated_node elif m.matches( n, m.ImportFrom( module=m.Attribute(value=m.Name("future"), attr=m.Name("utils")) ), ): self.future_utils_updated_node = updated_node return updated_node # @m.visit( # m.AllOf( # m.SimpleStatementLine(), # m.MatchIfTrue( # lambda node: any(m.matches(c, m.Assign()) for c in node.children) # ), # m.MatchIfTrue( # lambda node: "# type:" in node.trailing_whitespace.comment.value # ), # ) # ) # def visit_assign(self, node: SimpleStatementSuite) -> None: # return None def visit_Param(self, node: Param) -> Optional[bool]: class Visitor(m.MatcherDecoratableVisitor): def __init__(self): super().__init__() self.ptype: Optional[str] = None def visit_TrailingWhitespace_comment( self, node: "TrailingWhitespace" ) -> None: if node.comment and "type:" in node.comment.value: mo = re.match(r"#\s*type:\s*(\S*)", node.comment.value) self.ptype = mo.group(1) if mo else None return None v = Visitor() node.visit(v) if self.verbose: pos = self.get_metadata(PositionProvider, node).start print( f"{self.path}:{pos.line}:{pos.column}: parameter {node.name.value}: {v.ptype or 'unknown type'}" ) return None @m.visit(m.SimpleStatementLine()) def visit_simple_stmt(self, node: SimpleStatementLine) -> None: assign = None for c in node.children: if m.matches(c, m.Assign()): assign = ensure_type(c, Assign) if assign: if m.MatchIfTrue( lambda n: n.trailing_whitespace.comment and "type:" in n.trailing_whitespace.comment.value ): class TypingVisitor(m.MatcherDecoratableVisitor): def __init__(self): super().__init__() self.vtype = None def visit_TrailingWhitespace_comment( self, node: "TrailingWhitespace" ) -> None: if node.comment: mo = re.match(r"#\s*type:\s*(\S*)", node.comment.value) if mo: vtype = mo.group(1) return None tv = TypingVisitor() node.visit(tv) vtype = tv.vtype else: vtype = None class NameVisitor(m.MatcherDecoratableVisitor): def __init__(self): super().__init__() self.names: List[str] = [] def visit_Name(self, node: Name) -> Optional[bool]: self.names.append(node.value) return None if self.verbose: pos = self.get_metadata(PositionProvider, node).start for target in assign.targets: v = NameVisitor() target.visit(v) for name in v.names: print( f"{self.path}:{pos.line}:{pos.column}: variable {name}: {vtype or 'unknown type'}" ) def visit_FunctionDef_body(self, node: FunctionDef) -> None: class Visitor(m.MatcherDecoratableVisitor): def __init__(self): super().__init__() def visit_EmptyLine_comment(self, node: "EmptyLine") -> None: # FIXME too many matches on test_param_02 if not node.comment: return # TODO: use comment.value return None v = Visitor() node.visit(v) return None map_matcher = m.Call( func=m.Name("filter") | m.Name("map") | m.Name("zip") | m.Name("range") ) @m.visit(map_matcher) def visit_map(self, node: Call) -> None: func_name = ensure_type(node.func, Name).value if func_name not in self.builtins_imports: self.builtins_new_imports.add(func_name) @m.call_if_not_inside( m.Call( func=m.Name("list") | m.Name("set") | m.Name("tuple") | m.Attribute(attr=m.Name("join")) ) | m.CompFor() | m.For() ) @m.leave(map_matcher) def fix_map(self, original_node: Call, updated_node: Call) -> BaseExpression: # TODO test with CompFor etc. # TODO improve join test func_name = ensure_type(updated_node.func, Name).value if func_name not in self.builtins_imports: updated_node = Call(func=Name("list"), args=[Arg(updated_node)]) return updated_node @m.visit(m.Call(func=m.Name("xrange") | m.Name("raw_input"))) def visit_xrange(self, node: Call) -> None: orig_func_name = ensure_type(node.func, Name).value func_name = "range" if orig_func_name == "xrange" else "input" if func_name not in self.builtins_imports: self.builtins_new_imports.add(func_name) @m.leave(m.Call(func=m.Name("xrange") | m.Name("raw_input"))) def fix_xrange(self, original_node: Call, updated_node: Call) -> BaseExpression: orig_func_name = ensure_type(updated_node.func, Name).value func_name = "range" if orig_func_name == "xrange" else "input" return updated_node.with_changes(func=Name(func_name)) iter_matcher = m.Call( func=m.Attribute( attr=m.Name("iterkeys") | m.Name("itervalues") | m.Name("iteritems") ) ) @m.visit(iter_matcher) def visit_iter(self, node: Call) -> None: func_name = ensure_type(node.func, Attribute).attr.value if func_name not in self.future_utils_imports: self.future_utils_new_imports.add(func_name) @m.leave(iter_matcher) def fix_iter(self, original_node: Call, updated_node: Call) -> BaseExpression: attribute = ensure_type(updated_node.func, Attribute) func_name = attribute.attr dict_name = attribute.value return updated_node.with_changes(func=func_name, args=[Arg(dict_name)]) not_iter_matcher = m.Call( func=m.Attribute(attr=m.Name("keys") | m.Name("values") | m.Name("items")) ) @m.call_if_not_inside( m.Call( func=m.Name("list") | m.Name("set") | m.Name("tuple") | m.Attribute(attr=m.Name("join")) ) | m.CompFor() | m.For() ) @m.leave(not_iter_matcher) def fix_not_iter(self, original_node: Call, updated_node: Call) -> BaseExpression: updated_node = Call(func=Name("list"), args=[Arg(updated_node)]) return updated_node @m.call_if_not_inside(m.Import() | m.ImportFrom()) @m.leave(m.Name(value="unicode")) def fix_unicode(self, original_node: Name, updated_node: Name) -> BaseExpression: value = "text_type" if value not in self.future_utils_imports: self.future_utils_new_imports.add(value) return updated_node.with_changes(value=value) def leave_Module(self, original_node: Module, updated_node: Module) -> Module: updated_node = self.update_imports( original_node, updated_node, "builtins", self.builtins_updated_node, self.builtins_imports, self.builtins_new_imports, True, ) updated_node = self.update_imports( original_node, updated_node, "future.utils", self.future_utils_updated_node, self.future_utils_imports, self.future_utils_new_imports, False, ) return updated_node def update_imports( self, original_module: Module, updated_module: Module, import_name: str, updated_import_node: SimpleStatementLine, current_imports: Dict[str, str], new_imports: Set[str], noqa: bool, ) -> Module: if not new_imports: return updated_module noqa_comment = " # noqa" if noqa else "" if not updated_import_node: i = -1 blank_lines = "\n\n" if self.last_import_node_stmt: blank_lines = "" for i, (original, updated) in enumerate( zip(original_module.body, updated_module.body) ): if original is self.last_import_node_stmt: break stmt = parse_module( f"from {import_name} import {', '.join(sorted(new_imports))}{noqa_comment}\n{blank_lines}", config=updated_module.config_for_parsing, ) body = list(updated_module.body) self.last_import_node_stmt = stmt return updated_module.with_changes( body=body[: i + 1] + stmt.children + body[i + 1 :] ) else: if "*" not in current_imports: current_imports_set = { f"{k}" if k == v else f"{k} as {v}" for k, v in current_imports.items() } stmt = parse_statement( f"from {import_name} import {', '.join(sorted(new_imports | current_imports_set))}{noqa_comment}" ) return updated_module.deep_replace(updated_import_node, stmt) # for i, (original, updated) in enumerate( # zip(original_module.body, updated_module.body) # ): # if original is original_import_node: # body = list(updated_module.body) # return updated_module.with_changes( # body=body[:i] + [stmt] + body[i + 1 :] # ) return updated_module
def visit_SimpleStatementLine(self, node: SimpleStatementLine) -> Optional[bool]: for n in node.body: if m.matches(n, m.Import() | m.ImportFrom()): self.last_import_node_stmt = node return None
class Checker(m.MatcherDecoratableVisitor): METADATA_DEPENDENCIES = (PositionProvider,) def __init__( self, path: Path, verbose: bool = False, ignored: Optional[List[str]] = None ): super().__init__() self.path = path self.verbose = verbose self.ignored = set(ignored or []) self.future_division = False self.errors = False self.stack: List[str] = [] @m.call_if_inside(m.ImportFrom(module=m.Name("__future__"))) @m.visit(m.ImportAlias(name=m.Name("division"))) def import_div(self, node: ImportAlias) -> None: self.future_division = True @m.visit(m.BinaryOperation(operator=m.Divide())) def check_div(self, node: BinaryOperation) -> None: if "division" in self.ignored: return if not self.future_division: pos = self.get_metadata(PositionProvider, node).start print( f"{self.path}:{pos.line}:{pos.column}: division without `from __future__ import division`" ) self.errors = True @m.visit(m.Attribute(attr=m.Name("maxint"), value=m.Name("sys"))) def check_maxint(self, node: Attribute) -> None: if "sys.maxint" in self.ignored: return pos = self.get_metadata(PositionProvider, node).start print(f"{self.path}:{pos.line}:{pos.column}: use of sys.maxint") self.errors = True def visit_ClassDef(self, node: ClassDef) -> None: self.stack.append(node.name.value) def leave_ClassDef(self, node: ClassDef) -> None: self.stack.pop() def visit_FunctionDef(self, node: FunctionDef) -> None: self.stack.append(node.name.value) def leave_FunctionDef(self, node: FunctionDef) -> None: self.stack.pop() def visit_ClassDef_bases(self, node: "ClassDef") -> None: return @m.visit( m.Call( func=m.Attribute(attr=m.Name("assertEquals") | m.Name("assertItemsEqual")) ) ) def visit_old_assert(self, node: Call) -> None: name = ensure_type(node.func, Attribute).attr.value if name in self.ignored: return pos = self.get_metadata(PositionProvider, node).start print(f"{self.path}:{pos.line}:{pos.column}: use of {name}") self.errors = True
def import_from_matches(node: ImportFrom, module_parts: Sequence[str]) -> bool: """Check if an `ImportFrom` node matches sequence of module parts.""" return m.matches(node, m.ImportFrom(module=module_matcher(module_parts)))
class GatherUnusedImportsVisitor(ContextAwareVisitor): """ Collects all imports from a module not directly used in the same module. Intended to be instantiated and passed to a :class:`libcst.Module` :meth:`~libcst.CSTNode.visit` method to process the full module. Note that imports that are only used indirectly (from other modules) are still collected. After visiting a module the attribute ``unused_imports`` will contain a set of unused :class:`~libcst.ImportAlias` objects, paired with their parent import node. """ METADATA_DEPENDENCIES: Tuple[ProviderT] = ( *GatherNamesFromStringAnnotationsVisitor.METADATA_DEPENDENCIES, ScopeProvider, ) def __init__(self, context: CodemodContext) -> None: super().__init__(context) self._string_annotation_names: Set[str] = set() self._exported_names: Set[str] = set() #: Contains a set of (alias, parent_import) pairs that are not used #: in the module after visiting. self.unused_imports: Set[Tuple[cst.ImportAlias, Union[cst.Import, cst.ImportFrom]]] = set() def visit_Module(self, node: cst.Module) -> bool: export_collector = GatherExportsVisitor(self.context) node.visit(export_collector) self._exported_names = export_collector.explicit_exported_objects annotation_visitor = GatherNamesFromStringAnnotationsVisitor( self.context) node.visit(annotation_visitor) self._string_annotation_names = annotation_visitor.names return True @m.visit(m.Import() | m.ImportFrom( module=m.DoesNotMatch(m.Name("__future__")), names=m.DoesNotMatch(m.ImportStar()), )) def handle_import(self, node: Union[cst.Import, cst.ImportFrom]) -> None: names = node.names assert not isinstance(names, cst.ImportStar) # hello, type checker for alias in names: self.unused_imports.add((alias, node)) def leave_Module(self, original_node: cst.Module) -> None: self.unused_imports = self.filter_unused_imports(self.unused_imports) def filter_unused_imports( self, candidates: Iterable[Tuple[cst.ImportAlias, Union[cst.Import, cst.ImportFrom]]], ) -> Set[Tuple[cst.ImportAlias, Union[cst.Import, cst.ImportFrom]]]: """ Return the imports in ``candidates`` which are not used. This function implements the main logic of this visitor, and is called after traversal. It calls :meth:`~is_in_use` on each import. Override this in a subclass for additional filtering. """ unused_imports = set() for (alias, parent) in candidates: scope = self.get_metadata(ScopeProvider, parent) if scope is None: continue if not self.is_in_use(scope, alias): unused_imports.add((alias, parent)) return unused_imports def is_in_use(self, scope: cst.metadata.Scope, alias: cst.ImportAlias) -> bool: """ Check if ``alias`` is in use in the given ``scope``. An alias is in use if it's directly referenced, exported, or appears in a string type annotation. Override this in a subclass for additional filtering. """ asname = alias.asname names = _gen_dotted_names( cst.ensure_type(asname.name, cst.Name ) if asname is not None else alias.name) for name_or_alias, _ in names: if (name_or_alias in self._exported_names or name_or_alias in self._string_annotation_names): return True for assignment in scope[name_or_alias]: if (isinstance(assignment, cst.metadata.Assignment) and isinstance(assignment.node, (cst.ImportFrom, cst.Import)) and len(assignment.references) > 0): return True return False
class TransformerWithUnionReturnAnnotation( m.MatcherDecoratableTransformer): @m.leave(m.ImportFrom(module=m.Name(value="typing"))) def test(self, original_node: cst.ImportFrom, updated_node: cst.ImportFrom) -> FakeUnion: pass
def _test_import_from(self, node: ImportFrom) -> bool: """Check if 'import from' should be updated.""" return m.matches( node, m.ImportFrom(module=module_matcher(self.old_module_parts)) )