class DependencyVisitor(TraverserVisitor): def __init__(self, type_map: Dict[Expression, Type], python_version: Tuple[int, int], alias_deps: 'DefaultDict[str, Set[str]]') -> None: self.scope = Scope() self.type_map = type_map self.python2 = python_version[0] == 2 # This attribute holds a mapping from target to names of type aliases # it depends on. These need to be processed specially, since they are # only present in expanded form in symbol tables. For example, after: # A = List[int] # x: A # The module symbol table will just have a Var `x` with type `List[int]`, # and the dependency of `x` on `A` is lost. Therefore the alias dependencies # are preserved at alias expansion points in `semanal.py`, stored as an attribute # on MypyFile, and then passed here. self.alias_deps = alias_deps self.map = {} # type: Dict[str, Set[str]] self.is_class = False self.is_package_init_file = False # TODO (incomplete): # await # protocols def visit_mypy_file(self, o: MypyFile) -> None: self.scope.enter_file(o.fullname()) self.is_package_init_file = o.is_package_init_file() self.add_type_alias_deps(self.scope.current_target()) super().visit_mypy_file(o) self.scope.leave() def visit_func_def(self, o: FuncDef) -> None: self.scope.enter_function(o) target = self.scope.current_target() if o.type: if self.is_class and isinstance(o.type, FunctionLike): signature = bind_self(o.type) # type: Type else: signature = o.type for trigger in get_type_triggers(signature): self.add_dependency(trigger) self.add_dependency(trigger, target=make_trigger(target)) if o.info: for base in non_trivial_bases(o.info): self.add_dependency(make_trigger(base.fullname() + '.' + o.name())) self.add_type_alias_deps(self.scope.current_target()) super().visit_func_def(o) variants = set(o.expanded) - {o} for ex in variants: if isinstance(ex, FuncDef): super().visit_func_def(ex) self.scope.leave() def visit_decorator(self, o: Decorator) -> None: # We don't need to recheck outer scope for an overload, only overload itself. # Also if any decorator is nested, it is not externally visible, so we don't need to # generate dependency. if not o.func.is_overload and self.scope.current_function_name() is None: self.add_dependency(make_trigger(o.func.fullname())) super().visit_decorator(o) def visit_class_def(self, o: ClassDef) -> None: self.scope.enter_class(o.info) target = self.scope.current_full_target() self.add_dependency(make_trigger(target), target) old_is_class = self.is_class self.is_class = True # Add dependencies to type variables of a generic class. for tv in o.type_vars: self.add_dependency(make_trigger(tv.fullname), target) self.process_type_info(o.info) super().visit_class_def(o) self.is_class = old_is_class self.scope.leave() def visit_newtype_expr(self, o: NewTypeExpr) -> None: if o.info: self.scope.enter_class(o.info) self.process_type_info(o.info) self.scope.leave() def process_type_info(self, info: TypeInfo) -> None: target = self.scope.current_full_target() for base in info.bases: self.add_type_dependencies(base, target=target) if info.tuple_type: self.add_type_dependencies(info.tuple_type, target=make_trigger(target)) if info.typeddict_type: self.add_type_dependencies(info.typeddict_type, target=make_trigger(target)) if info.declared_metaclass: self.add_type_dependencies(info.declared_metaclass, target=make_trigger(target)) self.add_type_alias_deps(self.scope.current_target()) for name, node in info.names.items(): if isinstance(node.node, Var): for base_info in non_trivial_bases(info): # If the type of an attribute changes in a base class, we make references # to the attribute in the subclass stale. self.add_dependency(make_trigger(base_info.fullname() + '.' + name), target=make_trigger(info.fullname() + '.' + name)) for base_info in non_trivial_bases(info): for name, node in base_info.names.items(): self.add_dependency(make_trigger(base_info.fullname() + '.' + name), target=make_trigger(info.fullname() + '.' + name)) self.add_dependency(make_trigger(base_info.fullname() + '.__init__'), target=make_trigger(info.fullname() + '.__init__')) self.add_dependency(make_trigger(base_info.fullname() + '.__new__'), target=make_trigger(info.fullname() + '.__new__')) def visit_import(self, o: Import) -> None: for id, as_id in o.ids: self.add_dependency(make_trigger(id), self.scope.current_target()) def visit_import_from(self, o: ImportFrom) -> None: module_id, _ = correct_relative_import(self.scope.current_module_id(), o.relative, o.id, self.is_package_init_file) for name, as_name in o.names: self.add_dependency(make_trigger(module_id + '.' + name)) def visit_import_all(self, o: ImportAll) -> None: module_id, _ = correct_relative_import(self.scope.current_module_id(), o.relative, o.id, self.is_package_init_file) # The current target needs to be rechecked if anything "significant" changes in the # target module namespace (as the imported definitions will need to be updated). self.add_dependency(make_wildcard_trigger(module_id)) def visit_block(self, o: Block) -> None: if not o.is_unreachable: super().visit_block(o) def visit_assignment_stmt(self, o: AssignmentStmt) -> None: rvalue = o.rvalue if isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, TypeVarExpr): analyzed = rvalue.analyzed self.add_type_dependencies(analyzed.upper_bound, target=make_trigger(analyzed.fullname())) for val in analyzed.values: self.add_type_dependencies(val, target=make_trigger(analyzed.fullname())) # We need to re-analyze the definition if bound or value is deleted. super().visit_call_expr(rvalue) elif isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, NamedTupleExpr): # Depend on types of named tuple items. info = rvalue.analyzed.info prefix = '%s.%s' % (self.scope.current_full_target(), info.name()) for name, symnode in info.names.items(): if not name.startswith('_') and isinstance(symnode.node, Var): typ = symnode.node.type if typ: self.add_type_dependencies(typ) self.add_type_dependencies(typ, target=make_trigger(prefix)) attr_target = make_trigger('%s.%s' % (prefix, name)) self.add_type_dependencies(typ, target=attr_target) elif isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, TypedDictExpr): # Depend on the underlying typeddict type info = rvalue.analyzed.info assert info.typeddict_type is not None prefix = '%s.%s' % (self.scope.current_full_target(), info.name()) self.add_type_dependencies(info.typeddict_type, target=make_trigger(prefix)) elif isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, EnumCallExpr): # Enum values are currently not checked, but for future we add the deps on them for name, symnode in rvalue.analyzed.info.names.items(): if isinstance(symnode.node, Var) and symnode.node.type: self.add_type_dependencies(symnode.node.type) elif o.is_alias_def: assert len(o.lvalues) == 1 lvalue = o.lvalues[0] assert isinstance(lvalue, NameExpr) # TODO: get rid of this extra dependency from __init__ to alias definition scope typ = self.type_map.get(lvalue) if isinstance(typ, FunctionLike) and typ.is_type_obj(): class_name = typ.type_object().fullname() self.add_dependency(make_trigger(class_name + '.__init__')) self.add_dependency(make_trigger(class_name + '.__new__')) if isinstance(rvalue, IndexExpr) and isinstance(rvalue.analyzed, TypeAliasExpr): self.add_type_dependencies(rvalue.analyzed.type) else: # Normal assignment super().visit_assignment_stmt(o) for lvalue in o.lvalues: self.process_lvalue(lvalue) items = o.lvalues + [rvalue] for i in range(len(items) - 1): lvalue = items[i] rvalue = items[i + 1] if isinstance(lvalue, TupleExpr): self.add_attribute_dependency_for_expr(rvalue, '__iter__') if o.type: for trigger in get_type_triggers(o.type): self.add_dependency(trigger) def process_lvalue(self, lvalue: Expression) -> None: """Generate additional dependencies for an lvalue.""" if isinstance(lvalue, IndexExpr): self.add_operator_method_dependency(lvalue.base, '__setitem__') elif isinstance(lvalue, NameExpr): if lvalue.kind in (MDEF, GDEF): # Assignment to an attribute in the class body, or direct assignment to a # global variable. lvalue_type = self.get_non_partial_lvalue_type(lvalue) type_triggers = get_type_triggers(lvalue_type) attr_trigger = make_trigger('%s.%s' % (self.scope.current_full_target(), lvalue.name)) for type_trigger in type_triggers: self.add_dependency(type_trigger, attr_trigger) elif isinstance(lvalue, MemberExpr): if lvalue.kind is None: # Reference to a non-module attribute if lvalue.expr not in self.type_map: # Unreachable assignment -> not checked so no dependencies to generate. return object_type = self.type_map[lvalue.expr] lvalue_type = self.get_non_partial_lvalue_type(lvalue) type_triggers = get_type_triggers(lvalue_type) for attr_trigger in self.attribute_triggers(object_type, lvalue.name): for type_trigger in type_triggers: self.add_dependency(type_trigger, attr_trigger) elif isinstance(lvalue, TupleExpr): for item in lvalue.items: self.process_lvalue(item) # TODO: star lvalue def get_non_partial_lvalue_type(self, lvalue: RefExpr) -> Type: if lvalue not in self.type_map: # Likely a block considered unreachable during type checking. return UninhabitedType() lvalue_type = self.type_map[lvalue] if isinstance(lvalue_type, PartialType): if isinstance(lvalue.node, Var) and lvalue.node.type: lvalue_type = lvalue.node.type else: # Probably a secondary, non-definition assignment that doesn't # result in a non-partial type. We won't be able to infer any # dependencies from this so just return something. (The first, # definition assignment with a partial type is handled # differently, in the semantic analyzer.) assert not lvalue.is_new_def return UninhabitedType() return lvalue_type def visit_operator_assignment_stmt(self, o: OperatorAssignmentStmt) -> None: super().visit_operator_assignment_stmt(o) self.process_lvalue(o.lvalue) method = op_methods[o.op] self.add_attribute_dependency_for_expr(o.lvalue, method) if o.op in ops_with_inplace_method: inplace_method = '__i' + method[2:] self.add_attribute_dependency_for_expr(o.lvalue, inplace_method) def visit_for_stmt(self, o: ForStmt) -> None: super().visit_for_stmt(o) # __getitem__ is only used if __iter__ is missing but for simplicity we # just always depend on both. self.add_attribute_dependency_for_expr(o.expr, '__iter__') self.add_attribute_dependency_for_expr(o.expr, '__getitem__') self.process_lvalue(o.index) if isinstance(o.index, TupleExpr): # Process multiple assignment to index variables. item_type = o.inferred_item_type if item_type: # This is similar to above. self.add_attribute_dependency(item_type, '__iter__') self.add_attribute_dependency(item_type, '__getitem__') if o.index_type: self.add_type_dependencies(o.index_type) def visit_with_stmt(self, o: WithStmt) -> None: super().visit_with_stmt(o) for e in o.expr: self.add_attribute_dependency_for_expr(e, '__enter__') self.add_attribute_dependency_for_expr(e, '__exit__') if o.target_type: self.add_type_dependencies(o.target_type) def visit_print_stmt(self, o: PrintStmt) -> None: super().visit_print_stmt(o) if o.target: self.add_attribute_dependency_for_expr(o.target, 'write') def visit_del_stmt(self, o: DelStmt) -> None: super().visit_del_stmt(o) if isinstance(o.expr, IndexExpr): self.add_attribute_dependency_for_expr(o.expr.base, '__delitem__') # Expressions def process_global_ref_expr(self, o: RefExpr) -> None: if o.fullname is not None: self.add_dependency(make_trigger(o.fullname)) # If this is a reference to a type, generate a dependency to its # constructor. # TODO: avoid generating spurious dependencies for isinstancce checks, # except statements, class attribute reference, etc, if perf problem. typ = self.type_map.get(o) if isinstance(typ, FunctionLike) and typ.is_type_obj(): class_name = typ.type_object().fullname() self.add_dependency(make_trigger(class_name + '.__init__')) self.add_dependency(make_trigger(class_name + '.__new__')) def visit_name_expr(self, o: NameExpr) -> None: if o.kind == LDEF: # We don't track depdendencies to local variables, since they # aren't externally visible. return if o.kind == MDEF: # Direct reference to member is only possible in the scope that # defined the name, so no dependency is required. return self.process_global_ref_expr(o) def visit_member_expr(self, e: MemberExpr) -> None: super().visit_member_expr(e) if e.kind is not None: # Reference to a module attribute self.process_global_ref_expr(e) else: # Reference to a non-module attribute if e.expr not in self.type_map: # No type available -- this happens for unreachable code. Since it's unreachable, # it wasn't type checked and we don't need to generate dependencies. return typ = self.type_map[e.expr] self.add_attribute_dependency(typ, e.name) def visit_super_expr(self, e: SuperExpr) -> None: super().visit_super_expr(e) if e.info is not None: self.add_dependency(make_trigger(e.info.fullname() + '.' + e.name)) def visit_call_expr(self, e: CallExpr) -> None: super().visit_call_expr(e) def visit_cast_expr(self, e: CastExpr) -> None: super().visit_cast_expr(e) self.add_type_dependencies(e.type) def visit_type_application(self, e: TypeApplication) -> None: super().visit_type_application(e) for typ in e.types: self.add_type_dependencies(typ) def visit_index_expr(self, e: IndexExpr) -> None: super().visit_index_expr(e) self.add_operator_method_dependency(e.base, '__getitem__') def visit_unary_expr(self, e: UnaryExpr) -> None: super().visit_unary_expr(e) if e.op not in unary_op_methods: return method = unary_op_methods[e.op] self.add_operator_method_dependency(e.expr, method) def visit_op_expr(self, e: OpExpr) -> None: super().visit_op_expr(e) self.process_binary_op(e.op, e.left, e.right) def visit_comparison_expr(self, e: ComparisonExpr) -> None: super().visit_comparison_expr(e) for i, op in enumerate(e.operators): left = e.operands[i] right = e.operands[i + 1] self.process_binary_op(op, left, right) if self.python2 and op in ('==', '!=', '<', '<=', '>', '>='): self.add_operator_method_dependency(left, '__cmp__') self.add_operator_method_dependency(right, '__cmp__') def process_binary_op(self, op: str, left: Expression, right: Expression) -> None: method = op_methods.get(op) if method: if op == 'in': self.add_operator_method_dependency(right, method) else: self.add_operator_method_dependency(left, method) rev_method = reverse_op_methods.get(method) if rev_method: self.add_operator_method_dependency(right, rev_method) def add_operator_method_dependency(self, e: Expression, method: str) -> None: typ = self.type_map.get(e) if typ is not None: self.add_operator_method_dependency_for_type(typ, method) def add_operator_method_dependency_for_type(self, typ: Type, method: str) -> None: # Note that operator methods can't be (non-metaclass) methods of type objects # (that is, TypeType objects or Callables representing a type). if isinstance(typ, TypeVarType): typ = typ.upper_bound if isinstance(typ, TupleType): typ = typ.fallback if isinstance(typ, Instance): trigger = make_trigger(typ.type.fullname() + '.' + method) self.add_dependency(trigger) elif isinstance(typ, UnionType): for item in typ.items: self.add_operator_method_dependency_for_type(item, method) elif isinstance(typ, FunctionLike) and typ.is_type_obj(): self.add_operator_method_dependency_for_type(typ.fallback, method) elif isinstance(typ, TypeType): if isinstance(typ.item, Instance) and typ.item.type.metaclass_type is not None: self.add_operator_method_dependency_for_type(typ.item.type.metaclass_type, method) def visit_generator_expr(self, e: GeneratorExpr) -> None: super().visit_generator_expr(e) for seq in e.sequences: self.add_iter_dependency(seq) def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> None: super().visit_dictionary_comprehension(e) for seq in e.sequences: self.add_iter_dependency(seq) def visit_star_expr(self, e: StarExpr) -> None: super().visit_star_expr(e) self.add_iter_dependency(e.expr) def visit_yield_from_expr(self, e: YieldFromExpr) -> None: super().visit_yield_from_expr(e) self.add_iter_dependency(e.expr) # Helpers def add_type_alias_deps(self, target: str) -> None: # Type aliases are special, because some of the dependencies are calculated # in semanal.py, before they are expanded. if target in self.alias_deps: for alias in self.alias_deps[target]: self.add_dependency(make_trigger(alias)) def add_dependency(self, trigger: str, target: Optional[str] = None) -> None: """Add dependency from trigger to a target. If the target is not given explicitly, use the current target. """ if trigger.startswith(('<builtins.', '<typing.')): # Don't track dependencies to certain builtins to keep the size of # the dependencies manageable. These dependencies should only # change on mypy version updates, which will require a full rebuild # anyway. return if target is None: target = self.scope.current_target() self.map.setdefault(trigger, set()).add(target) def add_type_dependencies(self, typ: Type, target: Optional[str] = None) -> None: """Add dependencies to all components of a type. Args: target: If not None, override the default (current) target of the generated dependency. """ # TODO: Use this method in more places where get_type_triggers() + add_dependency() # are called together. for trigger in get_type_triggers(typ): self.add_dependency(trigger, target) def add_attribute_dependency(self, typ: Type, name: str) -> None: """Add dependencies for accessing a named attribute of a type.""" targets = self.attribute_triggers(typ, name) for target in targets: self.add_dependency(target) def attribute_triggers(self, typ: Type, name: str) -> List[str]: """Return all triggers associated with the attribute of a type.""" if isinstance(typ, TypeVarType): typ = typ.upper_bound if isinstance(typ, TupleType): typ = typ.fallback if isinstance(typ, Instance): member = '%s.%s' % (typ.type.fullname(), name) return [make_trigger(member)] elif isinstance(typ, FunctionLike) and typ.is_type_obj(): member = '%s.%s' % (typ.type_object().fullname(), name) triggers = [make_trigger(member)] triggers.extend(self.attribute_triggers(typ.fallback, name)) return triggers elif isinstance(typ, UnionType): targets = [] for item in typ.items: targets.extend(self.attribute_triggers(item, name)) return targets elif isinstance(typ, TypeType): triggers = self.attribute_triggers(typ.item, name) if isinstance(typ.item, Instance) and typ.item.type.metaclass_type is not None: triggers.append(make_trigger('%s.%s' % (typ.item.type.metaclass_type.type.fullname(), name))) return triggers else: return [] def add_attribute_dependency_for_expr(self, e: Expression, name: str) -> None: typ = self.type_map.get(e) if typ is not None: self.add_attribute_dependency(typ, name) def add_iter_dependency(self, node: Expression) -> None: typ = self.type_map.get(node) if typ: self.add_attribute_dependency(typ, '__iter__')
class DependencyVisitor(TraverserVisitor): def __init__(self, type_map: Dict[Expression, Type], python_version: Tuple[int, int], alias_deps: 'DefaultDict[str, Set[str]]') -> None: self.scope = Scope() self.type_map = type_map self.python2 = python_version[0] == 2 # This attribute holds a mapping from target to names of type aliases # it depends on. These need to be processed specially, since they are # only present in expanded form in symbol tables. For example, after: # A = List[int] # x: A # The module symbol table will just have a Var `x` with type `List[int]`, # and the dependency of `x` on `A` is lost. Therefore the alias dependencies # are preserved at alias expansion points in `semanal.py`, stored as an attribute # on MypyFile, and then passed here. self.alias_deps = alias_deps self.map = {} # type: Dict[str, Set[str]] self.is_class = False self.is_package_init_file = False # TODO (incomplete): # from m import * # await # protocols # metaclasses # functional enum # type variable with value restriction def visit_mypy_file(self, o: MypyFile) -> None: self.scope.enter_file(o.fullname()) self.is_package_init_file = o.is_package_init_file() self.add_type_alias_deps(self.scope.current_target()) super().visit_mypy_file(o) self.scope.leave() def visit_func_def(self, o: FuncDef) -> None: self.scope.enter_function(o) target = self.scope.current_target() if o.type: if self.is_class and isinstance(o.type, FunctionLike): signature = bind_self(o.type) # type: Type else: signature = o.type for trigger in get_type_triggers(signature): self.add_dependency(trigger) self.add_dependency(trigger, target=make_trigger(target)) if o.info: for base in non_trivial_bases(o.info): self.add_dependency( make_trigger(base.fullname() + '.' + o.name())) self.add_type_alias_deps(self.scope.current_target()) super().visit_func_def(o) self.scope.leave() def visit_decorator(self, o: Decorator) -> None: self.add_dependency(make_trigger(o.func.fullname())) super().visit_decorator(o) def visit_class_def(self, o: ClassDef) -> None: self.scope.enter_class(o.info) target = self.scope.current_full_target() self.add_dependency(make_trigger(target), target) old_is_class = self.is_class self.is_class = True # Add dependencies to type variables of a generic class. for tv in o.type_vars: self.add_dependency(make_trigger(tv.fullname), target) # Add dependencies to base types. for base in o.info.bases: self.add_type_dependencies(base, target=target) if o.info.tuple_type: self.add_type_dependencies(o.info.tuple_type, target=make_trigger(target)) if o.info.typeddict_type: self.add_type_dependencies(o.info.typeddict_type, target=make_trigger(target)) # TODO: Add dependencies based on remaining TypeInfo attributes. super().visit_class_def(o) self.add_type_alias_deps(self.scope.current_target()) self.is_class = old_is_class info = o.info for name, node in info.names.items(): if isinstance(node.node, Var): for base_info in non_trivial_bases(info): # If the type of an attribute changes in a base class, we make references # to the attribute in the subclass stale. self.add_dependency( make_trigger(base_info.fullname() + '.' + name), target=make_trigger(info.fullname() + '.' + name)) for base_info in non_trivial_bases(info): for name, node in base_info.names.items(): self.add_dependency( make_trigger(base_info.fullname() + '.' + name), target=make_trigger(info.fullname() + '.' + name)) self.add_dependency( make_trigger(base_info.fullname() + '.__init__'), target=make_trigger(info.fullname() + '.__init__')) self.scope.leave() def visit_import(self, o: Import) -> None: for id, as_id in o.ids: # TODO: as_id self.add_dependency(make_trigger(id), self.scope.current_target()) def visit_import_from(self, o: ImportFrom) -> None: module_id, _ = correct_relative_import(self.scope.current_module_id(), o.relative, o.id, self.is_package_init_file) for name, as_name in o.names: self.add_dependency(make_trigger(module_id + '.' + name)) def visit_block(self, o: Block) -> None: if not o.is_unreachable: super().visit_block(o) def visit_assignment_stmt(self, o: AssignmentStmt) -> None: # TODO: Implement all assignment special forms, including these: # Enum rvalue = o.rvalue if isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, TypeVarExpr): # TODO: Support type variable value restriction analyzed = rvalue.analyzed self.add_type_dependencies(analyzed.upper_bound, target=make_trigger( analyzed.fullname())) elif isinstance(rvalue, CallExpr) and isinstance( rvalue.analyzed, NamedTupleExpr): # Depend on types of named tuple items. info = rvalue.analyzed.info prefix = '%s.%s' % (self.scope.current_full_target(), info.name()) for name, symnode in info.names.items(): if not name.startswith('_') and isinstance(symnode.node, Var): typ = symnode.node.type if typ: self.add_type_dependencies(typ) self.add_type_dependencies(typ, target=make_trigger(prefix)) attr_target = make_trigger('%s.%s' % (prefix, name)) self.add_type_dependencies(typ, target=attr_target) elif isinstance(rvalue, CallExpr) and isinstance( rvalue.analyzed, TypedDictExpr): # Depend on the underlying typeddict type info = rvalue.analyzed.info assert info.typeddict_type is not None prefix = '%s.%s' % (self.scope.current_full_target(), info.name()) self.add_type_dependencies(info.typeddict_type, target=make_trigger(prefix)) elif o.is_alias_def: assert len(o.lvalues) == 1 lvalue = o.lvalues[0] assert isinstance(lvalue, NameExpr) # TODO: get rid of this extra dependency from __init__ to alias definition scope typ = self.type_map.get(lvalue) if isinstance(typ, FunctionLike) and typ.is_type_obj(): class_name = typ.type_object().fullname() self.add_dependency(make_trigger(class_name + '.__init__')) if isinstance(rvalue, IndexExpr) and isinstance( rvalue.analyzed, TypeAliasExpr): self.add_type_dependencies(rvalue.analyzed.type) else: # Normal assignment super().visit_assignment_stmt(o) for lvalue in o.lvalues: self.process_lvalue(lvalue) items = o.lvalues + [rvalue] for i in range(len(items) - 1): lvalue = items[i] rvalue = items[i + 1] if isinstance(lvalue, TupleExpr): self.add_attribute_dependency_for_expr(rvalue, '__iter__') if o.type: for trigger in get_type_triggers(o.type): self.add_dependency(trigger) def process_lvalue(self, lvalue: Expression) -> None: """Generate additional dependencies for an lvalue.""" if isinstance(lvalue, IndexExpr): self.add_operator_method_dependency(lvalue.base, '__setitem__') elif isinstance(lvalue, NameExpr): if lvalue.kind in (MDEF, GDEF): # Assignment to an attribute in the class body, or direct assignment to a # global variable. lvalue_type = self.get_non_partial_lvalue_type(lvalue) type_triggers = get_type_triggers(lvalue_type) attr_trigger = make_trigger( '%s.%s' % (self.scope.current_full_target(), lvalue.name)) for type_trigger in type_triggers: self.add_dependency(type_trigger, attr_trigger) elif isinstance(lvalue, MemberExpr): if lvalue.kind is None: # Reference to a non-module attribute if lvalue.expr not in self.type_map: # Unreachable assignment -> not checked so no dependencies to generate. return object_type = self.type_map[lvalue.expr] lvalue_type = self.get_non_partial_lvalue_type(lvalue) type_triggers = get_type_triggers(lvalue_type) for attr_trigger in self.attribute_triggers( object_type, lvalue.name): for type_trigger in type_triggers: self.add_dependency(type_trigger, attr_trigger) elif isinstance(lvalue, TupleExpr): for item in lvalue.items: self.process_lvalue(item) # TODO: star lvalue def get_non_partial_lvalue_type(self, lvalue: RefExpr) -> Type: if lvalue not in self.type_map: # Likely a block considered unreachable during type checking. return UninhabitedType() lvalue_type = self.type_map[lvalue] if isinstance(lvalue_type, PartialType): if isinstance(lvalue.node, Var) and lvalue.node.type: lvalue_type = lvalue.node.type else: # Probably a secondary, non-definition assignment that doesn't # result in a non-partial type. We won't be able to infer any # dependencies from this so just return something. (The first, # definition assignment with a partial type is handled # differently, in the semantic analyzer.) assert not lvalue.is_new_def return UninhabitedType() return lvalue_type def visit_operator_assignment_stmt(self, o: OperatorAssignmentStmt) -> None: super().visit_operator_assignment_stmt(o) self.process_lvalue(o.lvalue) method = op_methods[o.op] self.add_attribute_dependency_for_expr(o.lvalue, method) if o.op in ops_with_inplace_method: inplace_method = '__i' + method[2:] self.add_attribute_dependency_for_expr(o.lvalue, inplace_method) def visit_for_stmt(self, o: ForStmt) -> None: super().visit_for_stmt(o) # __getitem__ is only used if __iter__ is missing but for simplicity we # just always depend on both. self.add_attribute_dependency_for_expr(o.expr, '__iter__') self.add_attribute_dependency_for_expr(o.expr, '__getitem__') self.process_lvalue(o.index) if isinstance(o.index, TupleExpr): # Process multiple assignment to index variables. item_type = o.inferred_item_type if item_type: # This is similar to above. self.add_attribute_dependency(item_type, '__iter__') self.add_attribute_dependency(item_type, '__getitem__') if o.index_type: self.add_type_dependencies(o.index_type) def visit_with_stmt(self, o: WithStmt) -> None: super().visit_with_stmt(o) for e in o.expr: self.add_attribute_dependency_for_expr(e, '__enter__') self.add_attribute_dependency_for_expr(e, '__exit__') if o.target_type: self.add_type_dependencies(o.target_type) def visit_print_stmt(self, o: PrintStmt) -> None: super().visit_print_stmt(o) if o.target: self.add_attribute_dependency_for_expr(o.target, 'write') def visit_del_stmt(self, o: DelStmt) -> None: super().visit_del_stmt(o) if isinstance(o.expr, IndexExpr): self.add_attribute_dependency_for_expr(o.expr.base, '__delitem__') # Expressions def process_global_ref_expr(self, o: RefExpr) -> None: if o.fullname is not None: self.add_dependency(make_trigger(o.fullname)) # If this is a reference to a type, generate a dependency to its # constructor. # TODO: avoid generating spurious dependencies for isinstancce checks, # except statements, class attribute reference, etc, if perf problem. typ = self.type_map.get(o) if isinstance(typ, FunctionLike) and typ.is_type_obj(): class_name = typ.type_object().fullname() self.add_dependency(make_trigger(class_name + '.__init__')) def visit_name_expr(self, o: NameExpr) -> None: if o.kind == LDEF: # We don't track depdendencies to local variables, since they # aren't externally visible. return if o.kind == MDEF: # Direct reference to member is only possible in the scope that # defined the name, so no dependency is required. return self.process_global_ref_expr(o) def visit_member_expr(self, e: MemberExpr) -> None: super().visit_member_expr(e) if e.kind is not None: # Reference to a module attribute self.process_global_ref_expr(e) else: # Reference to a non-module attribute if e.expr not in self.type_map: # No type available -- this happens for unreachable code. Since it's unreachable, # it wasn't type checked and we don't need to generate dependencies. return typ = self.type_map[e.expr] self.add_attribute_dependency(typ, e.name) def visit_super_expr(self, e: SuperExpr) -> None: super().visit_super_expr(e) if e.info is not None: self.add_dependency(make_trigger(e.info.fullname() + '.' + e.name)) def visit_call_expr(self, e: CallExpr) -> None: super().visit_call_expr(e) def visit_cast_expr(self, e: CastExpr) -> None: super().visit_cast_expr(e) self.add_type_dependencies(e.type) def visit_type_application(self, e: TypeApplication) -> None: super().visit_type_application(e) for typ in e.types: self.add_type_dependencies(typ) def visit_index_expr(self, e: IndexExpr) -> None: super().visit_index_expr(e) self.add_operator_method_dependency(e.base, '__getitem__') def visit_unary_expr(self, e: UnaryExpr) -> None: super().visit_unary_expr(e) if e.op not in unary_op_methods: return method = unary_op_methods[e.op] self.add_operator_method_dependency(e.expr, method) def visit_op_expr(self, e: OpExpr) -> None: super().visit_op_expr(e) self.process_binary_op(e.op, e.left, e.right) def visit_comparison_expr(self, e: ComparisonExpr) -> None: super().visit_comparison_expr(e) for i, op in enumerate(e.operators): left = e.operands[i] right = e.operands[i + 1] self.process_binary_op(op, left, right) if self.python2 and op in ('==', '!=', '<', '<=', '>', '>='): self.add_operator_method_dependency(left, '__cmp__') self.add_operator_method_dependency(right, '__cmp__') def process_binary_op(self, op: str, left: Expression, right: Expression) -> None: method = op_methods.get(op) if method: if op == 'in': self.add_operator_method_dependency(right, method) else: self.add_operator_method_dependency(left, method) rev_method = reverse_op_methods.get(method) if rev_method: self.add_operator_method_dependency(right, rev_method) def add_operator_method_dependency(self, e: Expression, method: str) -> None: typ = self.type_map.get(e) if typ is not None: self.add_operator_method_dependency_for_type(typ, method) def add_operator_method_dependency_for_type(self, typ: Type, method: str) -> None: # Note that operator methods can't be (non-metaclass) methods of type objects # (that is, TypeType objects or Callables representing a type). # TODO: TypedDict # TODO: metaclasses if isinstance(typ, TypeVarType): typ = typ.upper_bound if isinstance(typ, TupleType): typ = typ.fallback if isinstance(typ, Instance): trigger = make_trigger(typ.type.fullname() + '.' + method) self.add_dependency(trigger) elif isinstance(typ, UnionType): for item in typ.items: self.add_operator_method_dependency_for_type(item, method) def visit_generator_expr(self, e: GeneratorExpr) -> None: super().visit_generator_expr(e) for seq in e.sequences: self.add_iter_dependency(seq) def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> None: super().visit_dictionary_comprehension(e) for seq in e.sequences: self.add_iter_dependency(seq) def visit_star_expr(self, e: StarExpr) -> None: super().visit_star_expr(e) self.add_iter_dependency(e.expr) def visit_yield_from_expr(self, e: YieldFromExpr) -> None: super().visit_yield_from_expr(e) self.add_iter_dependency(e.expr) # Helpers def add_type_alias_deps(self, target: str) -> None: # Type aliases are special, because some of the dependencies are calculated # in semanal.py, before they are expanded. if target in self.alias_deps: for alias in self.alias_deps[target]: self.add_dependency(make_trigger(alias)) def add_dependency(self, trigger: str, target: Optional[str] = None) -> None: """Add dependency from trigger to a target. If the target is not given explicitly, use the current target. """ if trigger.startswith(('<builtins.', '<typing.')): # Don't track dependencies to certain builtins to keep the size of # the dependencies manageable. These dependencies should only # change on mypy version updates, which will require a full rebuild # anyway. return if target is None: target = self.scope.current_target() self.map.setdefault(trigger, set()).add(target) def add_type_dependencies(self, typ: Type, target: Optional[str] = None) -> None: """Add dependencies to all components of a type. Args: target: If not None, override the default (current) target of the generated dependency. """ # TODO: Use this method in more places where get_type_triggers() + add_dependency() # are called together. for trigger in get_type_triggers(typ): self.add_dependency(trigger, target) def add_attribute_dependency(self, typ: Type, name: str) -> None: """Add dependencies for accessing a named attribute of a type.""" targets = self.attribute_triggers(typ, name) for target in targets: self.add_dependency(target) def attribute_triggers(self, typ: Type, name: str) -> List[str]: """Return all triggers associated with the attribute of a type.""" if isinstance(typ, TypeVarType): typ = typ.upper_bound if isinstance(typ, TupleType): typ = typ.fallback if isinstance(typ, Instance): member = '%s.%s' % (typ.type.fullname(), name) return [make_trigger(member)] elif isinstance(typ, FunctionLike) and typ.is_type_obj(): member = '%s.%s' % (typ.type_object().fullname(), name) return [make_trigger(member)] elif isinstance(typ, UnionType): targets = [] for item in typ.items: targets.extend(self.attribute_triggers(item, name)) return targets elif isinstance(typ, TypeType): # TODO: Metaclass attribute lookup return self.attribute_triggers(typ.item, name) else: return [] def add_attribute_dependency_for_expr(self, e: Expression, name: str) -> None: typ = self.type_map.get(e) if typ is not None: self.add_attribute_dependency(typ, name) def add_iter_dependency(self, node: Expression) -> None: typ = self.type_map.get(node) if typ: self.add_attribute_dependency(typ, '__iter__')
class DependencyVisitor(TraverserVisitor): def __init__(self, type_map: Dict[Expression, Type], python_version: Tuple[int, int], alias_deps: 'DefaultDict[str, Set[str]]', options: Optional[Options] = None) -> None: self.scope = Scope() self.type_map = type_map self.python2 = python_version[0] == 2 # This attribute holds a mapping from target to names of type aliases # it depends on. These need to be processed specially, since they are # only present in expanded form in symbol tables. For example, after: # A = List[int] # x: A # The module symbol table will just have a Var `x` with type `List[int]`, # and the dependency of `x` on `A` is lost. Therefore the alias dependencies # are preserved at alias expansion points in `semanal.py`, stored as an attribute # on MypyFile, and then passed here. self.alias_deps = alias_deps self.map = {} # type: Dict[str, Set[str]] self.is_class = False self.is_package_init_file = False self.options = options def visit_mypy_file(self, o: MypyFile) -> None: self.scope.enter_file(o.fullname()) self.is_package_init_file = o.is_package_init_file() self.add_type_alias_deps(self.scope.current_target()) super().visit_mypy_file(o) self.scope.leave() def visit_func_def(self, o: FuncDef) -> None: self.scope.enter_function(o) target = self.scope.current_target() if o.type: if self.is_class and isinstance(o.type, FunctionLike): signature = bind_self(o.type) # type: Type else: signature = o.type for trigger in get_type_triggers(signature): self.add_dependency(trigger) self.add_dependency(trigger, target=make_trigger(target)) if o.info: for base in non_trivial_bases(o.info): self.add_dependency( make_trigger(base.fullname() + '.' + o.name())) self.add_type_alias_deps(self.scope.current_target()) super().visit_func_def(o) variants = set(o.expanded) - {o} for ex in variants: if isinstance(ex, FuncDef): super().visit_func_def(ex) self.scope.leave() def visit_decorator(self, o: Decorator) -> None: # We don't need to recheck outer scope for an overload, only overload itself. # Also if any decorator is nested, it is not externally visible, so we don't need to # generate dependency. if not o.func.is_overload and self.scope.current_function_name( ) is None: self.add_dependency(make_trigger(o.func.fullname())) if self.options is not None and self.options.logical_deps: # Add logical dependencies from decorators to the function. For example, # if we have # @dec # def func(): ... # then if `dec` is unannotated, then it will "spoil" `func` and consequently # all call sites, making them all `Any`. for d in o.decorators: tname = None # type: Optional[str] if isinstance(d, RefExpr) and d.fullname is not None: tname = d.fullname if (isinstance(d, CallExpr) and isinstance(d.callee, RefExpr) and d.callee.fullname is not None): tname = d.callee.fullname if tname is not None: self.add_dependency(make_trigger(tname), make_trigger(o.func.fullname())) super().visit_decorator(o) def visit_class_def(self, o: ClassDef) -> None: self.scope.enter_class(o.info) target = self.scope.current_full_target() self.add_dependency(make_trigger(target), target) old_is_class = self.is_class self.is_class = True # Add dependencies to type variables of a generic class. for tv in o.type_vars: self.add_dependency(make_trigger(tv.fullname), target) self.process_type_info(o.info) super().visit_class_def(o) self.is_class = old_is_class self.scope.leave() def visit_newtype_expr(self, o: NewTypeExpr) -> None: if o.info: self.scope.enter_class(o.info) self.process_type_info(o.info) self.scope.leave() def process_type_info(self, info: TypeInfo) -> None: target = self.scope.current_full_target() for base in info.bases: self.add_type_dependencies(base, target=target) if info.tuple_type: self.add_type_dependencies(info.tuple_type, target=make_trigger(target)) if info.typeddict_type: self.add_type_dependencies(info.typeddict_type, target=make_trigger(target)) if info.declared_metaclass: self.add_type_dependencies(info.declared_metaclass, target=make_trigger(target)) if info.is_protocol: for base_info in info.mro[:-1]: # We add dependencies from whole MRO to cover explicit subprotocols. # For example: # # class Super(Protocol): # x: int # class Sub(Super, Protocol): # y: int # # In this example we add <Super[wildcard]> -> <Sub>, to invalidate Sub if # a new member is added to Super. self.add_dependency(make_wildcard_trigger( base_info.fullname()), target=make_trigger(target)) # More protocol dependencies are collected in TypeState._snapshot_protocol_deps # after a full run or update is finished. self.add_type_alias_deps(self.scope.current_target()) for name, node in info.names.items(): if isinstance(node.node, Var): # Recheck Liskov if needed, self definitions are checked in the defining method if node.node.is_initialized_in_class and has_user_bases(info): self.add_dependency( make_trigger(info.fullname() + '.' + name)) for base_info in non_trivial_bases(info): # If the type of an attribute changes in a base class, we make references # to the attribute in the subclass stale. self.add_dependency( make_trigger(base_info.fullname() + '.' + name), target=make_trigger(info.fullname() + '.' + name)) for base_info in non_trivial_bases(info): for name, node in base_info.names.items(): if self.options and self.options.logical_deps: # Skip logical dependency if an attribute is not overridden. For example, # in case of: # class Base: # x = 1 # y = 2 # class Sub(Base): # x = 3 # we skip <Base.y> -> <Child.y>, because even if `y` is unannotated it # doesn't affect precision of Liskov checking. if name not in info.names: continue self.add_dependency( make_trigger(base_info.fullname() + '.' + name), target=make_trigger(info.fullname() + '.' + name)) self.add_dependency( make_trigger(base_info.fullname() + '.__init__'), target=make_trigger(info.fullname() + '.__init__')) self.add_dependency( make_trigger(base_info.fullname() + '.__new__'), target=make_trigger(info.fullname() + '.__new__')) # If the set of abstract attributes change, this may invalidate class # instantiation, or change the generated error message, since Python checks # class abstract status when creating an instance. # # TODO: We should probably add this dependency only from the __init__ of the # current class, and independent of bases (to trigger changes in message # wording, as errors may enumerate all abstract attributes). self.add_dependency( make_trigger(base_info.fullname() + '.(abstract)'), target=make_trigger(info.fullname() + '.__init__')) # If the base class abstract attributes change, subclass abstract # attributes need to be recalculated. self.add_dependency( make_trigger(base_info.fullname() + '.(abstract)')) def visit_import(self, o: Import) -> None: for id, as_id in o.ids: self.add_dependency(make_trigger(id), self.scope.current_target()) def visit_import_from(self, o: ImportFrom) -> None: module_id, _ = correct_relative_import(self.scope.current_module_id(), o.relative, o.id, self.is_package_init_file) for name, as_name in o.names: self.add_dependency(make_trigger(module_id + '.' + name)) def visit_import_all(self, o: ImportAll) -> None: module_id, _ = correct_relative_import(self.scope.current_module_id(), o.relative, o.id, self.is_package_init_file) # The current target needs to be rechecked if anything "significant" changes in the # target module namespace (as the imported definitions will need to be updated). self.add_dependency(make_wildcard_trigger(module_id)) def visit_block(self, o: Block) -> None: if not o.is_unreachable: super().visit_block(o) def visit_assignment_stmt(self, o: AssignmentStmt) -> None: rvalue = o.rvalue if isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, TypeVarExpr): analyzed = rvalue.analyzed self.add_type_dependencies(analyzed.upper_bound, target=make_trigger( analyzed.fullname())) for val in analyzed.values: self.add_type_dependencies(val, target=make_trigger( analyzed.fullname())) # We need to re-analyze the definition if bound or value is deleted. super().visit_call_expr(rvalue) elif isinstance(rvalue, CallExpr) and isinstance( rvalue.analyzed, NamedTupleExpr): # Depend on types of named tuple items. info = rvalue.analyzed.info prefix = '%s.%s' % (self.scope.current_full_target(), info.name()) for name, symnode in info.names.items(): if not name.startswith('_') and isinstance(symnode.node, Var): typ = symnode.node.type if typ: self.add_type_dependencies(typ) self.add_type_dependencies(typ, target=make_trigger(prefix)) attr_target = make_trigger('%s.%s' % (prefix, name)) self.add_type_dependencies(typ, target=attr_target) elif isinstance(rvalue, CallExpr) and isinstance( rvalue.analyzed, TypedDictExpr): # Depend on the underlying typeddict type info = rvalue.analyzed.info assert info.typeddict_type is not None prefix = '%s.%s' % (self.scope.current_full_target(), info.name()) self.add_type_dependencies(info.typeddict_type, target=make_trigger(prefix)) elif isinstance(rvalue, CallExpr) and isinstance( rvalue.analyzed, EnumCallExpr): # Enum values are currently not checked, but for future we add the deps on them for name, symnode in rvalue.analyzed.info.names.items(): if isinstance(symnode.node, Var) and symnode.node.type: self.add_type_dependencies(symnode.node.type) elif o.is_alias_def: assert len(o.lvalues) == 1 lvalue = o.lvalues[0] assert isinstance(lvalue, NameExpr) # TODO: get rid of this extra dependency from __init__ to alias definition scope typ = self.type_map.get(lvalue) if isinstance(typ, FunctionLike) and typ.is_type_obj(): class_name = typ.type_object().fullname() self.add_dependency(make_trigger(class_name + '.__init__')) self.add_dependency(make_trigger(class_name + '.__new__')) if isinstance(rvalue, IndexExpr) and isinstance( rvalue.analyzed, TypeAliasExpr): self.add_type_dependencies(rvalue.analyzed.type) elif typ: self.add_type_dependencies(typ) else: # Normal assignment super().visit_assignment_stmt(o) for lvalue in o.lvalues: self.process_lvalue(lvalue) items = o.lvalues + [rvalue] for i in range(len(items) - 1): lvalue = items[i] rvalue = items[i + 1] if isinstance(lvalue, TupleExpr): self.add_attribute_dependency_for_expr(rvalue, '__iter__') if o.type: for trigger in get_type_triggers(o.type): self.add_dependency(trigger) if self.options and self.options.logical_deps and o.unanalyzed_type is None: # Special case: for definitions without an explicit type like this: # x = func(...) # we add a logical dependency <func> -> <x>, because if `func` is not annotated, # then it will make all points of use of `x` unchecked. if (isinstance(rvalue, CallExpr) and isinstance(rvalue.callee, RefExpr) and rvalue.callee.fullname is not None): fname = None # type: Optional[str] if isinstance(rvalue.callee.node, TypeInfo): # use actual __init__ as a dependency source init = rvalue.callee.node.get('__init__') if init and isinstance(init.node, FuncBase): fname = init.node.fullname() else: fname = rvalue.callee.fullname if fname is None: return for lv in o.lvalues: if isinstance(lv, RefExpr) and lv.fullname and lv.is_new_def: if lv.kind == LDEF: return # local definitions don't generate logical deps self.add_dependency(make_trigger(fname), make_trigger(lv.fullname)) def process_lvalue(self, lvalue: Expression) -> None: """Generate additional dependencies for an lvalue.""" if isinstance(lvalue, IndexExpr): self.add_operator_method_dependency(lvalue.base, '__setitem__') elif isinstance(lvalue, NameExpr): if lvalue.kind in (MDEF, GDEF): # Assignment to an attribute in the class body, or direct assignment to a # global variable. lvalue_type = self.get_non_partial_lvalue_type(lvalue) type_triggers = get_type_triggers(lvalue_type) attr_trigger = make_trigger( '%s.%s' % (self.scope.current_full_target(), lvalue.name)) for type_trigger in type_triggers: self.add_dependency(type_trigger, attr_trigger) elif isinstance(lvalue, MemberExpr): if self.is_self_member_ref(lvalue) and lvalue.is_new_def: node = lvalue.node if isinstance(node, Var): info = node.info if info and has_user_bases(info): # Recheck Liskov for self definitions self.add_dependency( make_trigger(info.fullname() + '.' + lvalue.name)) if lvalue.kind is None: # Reference to a non-module attribute if lvalue.expr not in self.type_map: # Unreachable assignment -> not checked so no dependencies to generate. return object_type = self.type_map[lvalue.expr] lvalue_type = self.get_non_partial_lvalue_type(lvalue) type_triggers = get_type_triggers(lvalue_type) for attr_trigger in self.attribute_triggers( object_type, lvalue.name): for type_trigger in type_triggers: self.add_dependency(type_trigger, attr_trigger) elif isinstance(lvalue, TupleExpr): for item in lvalue.items: self.process_lvalue(item) elif isinstance(lvalue, StarExpr): self.process_lvalue(lvalue.expr) def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: """Does memberexpr to refer to an attribute of self?""" if not isinstance(memberexpr.expr, NameExpr): return False node = memberexpr.expr.node return isinstance(node, Var) and node.is_self def get_non_partial_lvalue_type(self, lvalue: RefExpr) -> Type: if lvalue not in self.type_map: # Likely a block considered unreachable during type checking. return UninhabitedType() lvalue_type = self.type_map[lvalue] if isinstance(lvalue_type, PartialType): if isinstance(lvalue.node, Var) and lvalue.node.type: lvalue_type = lvalue.node.type else: # Probably a secondary, non-definition assignment that doesn't # result in a non-partial type. We won't be able to infer any # dependencies from this so just return something. (The first, # definition assignment with a partial type is handled # differently, in the semantic analyzer.) assert not lvalue.is_new_def return UninhabitedType() return lvalue_type def visit_operator_assignment_stmt(self, o: OperatorAssignmentStmt) -> None: super().visit_operator_assignment_stmt(o) self.process_lvalue(o.lvalue) method = op_methods[o.op] self.add_attribute_dependency_for_expr(o.lvalue, method) if o.op in ops_with_inplace_method: inplace_method = '__i' + method[2:] self.add_attribute_dependency_for_expr(o.lvalue, inplace_method) def visit_for_stmt(self, o: ForStmt) -> None: super().visit_for_stmt(o) if not o.is_async: # __getitem__ is only used if __iter__ is missing but for simplicity we # just always depend on both. self.add_attribute_dependency_for_expr(o.expr, '__iter__') self.add_attribute_dependency_for_expr(o.expr, '__getitem__') if o.inferred_iterator_type: if self.python2: method = 'next' else: method = '__next__' self.add_attribute_dependency(o.inferred_iterator_type, method) else: self.add_attribute_dependency_for_expr(o.expr, '__aiter__') if o.inferred_iterator_type: self.add_attribute_dependency(o.inferred_iterator_type, '__anext__') self.process_lvalue(o.index) if isinstance(o.index, TupleExpr): # Process multiple assignment to index variables. item_type = o.inferred_item_type if item_type: # This is similar to above. self.add_attribute_dependency(item_type, '__iter__') self.add_attribute_dependency(item_type, '__getitem__') if o.index_type: self.add_type_dependencies(o.index_type) def visit_with_stmt(self, o: WithStmt) -> None: super().visit_with_stmt(o) for e in o.expr: if not o.is_async: self.add_attribute_dependency_for_expr(e, '__enter__') self.add_attribute_dependency_for_expr(e, '__exit__') else: self.add_attribute_dependency_for_expr(e, '__aenter__') self.add_attribute_dependency_for_expr(e, '__aexit__') if o.target_type: self.add_type_dependencies(o.target_type) def visit_print_stmt(self, o: PrintStmt) -> None: super().visit_print_stmt(o) if o.target: self.add_attribute_dependency_for_expr(o.target, 'write') def visit_del_stmt(self, o: DelStmt) -> None: super().visit_del_stmt(o) if isinstance(o.expr, IndexExpr): self.add_attribute_dependency_for_expr(o.expr.base, '__delitem__') # Expressions def process_global_ref_expr(self, o: RefExpr) -> None: if o.fullname is not None: self.add_dependency(make_trigger(o.fullname)) # If this is a reference to a type, generate a dependency to its # constructor. # TODO: avoid generating spurious dependencies for isinstance checks, # except statements, class attribute reference, etc, if perf problem. typ = self.type_map.get(o) if isinstance(typ, FunctionLike) and typ.is_type_obj(): class_name = typ.type_object().fullname() self.add_dependency(make_trigger(class_name + '.__init__')) self.add_dependency(make_trigger(class_name + '.__new__')) def visit_name_expr(self, o: NameExpr) -> None: if o.kind == LDEF: # We don't track dependencies to local variables, since they # aren't externally visible. return if o.kind == MDEF: # Direct reference to member is only possible in the scope that # defined the name, so no dependency is required. return self.process_global_ref_expr(o) def visit_member_expr(self, e: MemberExpr) -> None: super().visit_member_expr(e) if e.kind is not None: # Reference to a module attribute self.process_global_ref_expr(e) else: # Reference to a non-module attribute if e.expr not in self.type_map: # No type available -- this happens for unreachable code. Since it's unreachable, # it wasn't type checked and we don't need to generate dependencies. return typ = self.type_map[e.expr] self.add_attribute_dependency(typ, e.name) def visit_super_expr(self, e: SuperExpr) -> None: super().visit_super_expr(e) if e.info is not None: self.add_dependency(make_trigger(e.info.fullname() + '.' + e.name)) def visit_call_expr(self, e: CallExpr) -> None: super().visit_call_expr(e) def visit_cast_expr(self, e: CastExpr) -> None: super().visit_cast_expr(e) self.add_type_dependencies(e.type) def visit_type_application(self, e: TypeApplication) -> None: super().visit_type_application(e) for typ in e.types: self.add_type_dependencies(typ) def visit_index_expr(self, e: IndexExpr) -> None: super().visit_index_expr(e) self.add_operator_method_dependency(e.base, '__getitem__') def visit_unary_expr(self, e: UnaryExpr) -> None: super().visit_unary_expr(e) if e.op not in unary_op_methods: return method = unary_op_methods[e.op] self.add_operator_method_dependency(e.expr, method) def visit_op_expr(self, e: OpExpr) -> None: super().visit_op_expr(e) self.process_binary_op(e.op, e.left, e.right) def visit_comparison_expr(self, e: ComparisonExpr) -> None: super().visit_comparison_expr(e) for i, op in enumerate(e.operators): left = e.operands[i] right = e.operands[i + 1] self.process_binary_op(op, left, right) if self.python2 and op in ('==', '!=', '<', '<=', '>', '>='): self.add_operator_method_dependency(left, '__cmp__') self.add_operator_method_dependency(right, '__cmp__') def process_binary_op(self, op: str, left: Expression, right: Expression) -> None: method = op_methods.get(op) if method: if op == 'in': self.add_operator_method_dependency(right, method) else: self.add_operator_method_dependency(left, method) rev_method = reverse_op_methods.get(method) if rev_method: self.add_operator_method_dependency(right, rev_method) def add_operator_method_dependency(self, e: Expression, method: str) -> None: typ = self.type_map.get(e) if typ is not None: self.add_operator_method_dependency_for_type(typ, method) def add_operator_method_dependency_for_type(self, typ: Type, method: str) -> None: # Note that operator methods can't be (non-metaclass) methods of type objects # (that is, TypeType objects or Callables representing a type). if isinstance(typ, TypeVarType): typ = typ.upper_bound if isinstance(typ, TupleType): typ = typ.fallback if isinstance(typ, Instance): trigger = make_trigger(typ.type.fullname() + '.' + method) self.add_dependency(trigger) elif isinstance(typ, UnionType): for item in typ.items: self.add_operator_method_dependency_for_type(item, method) elif isinstance(typ, FunctionLike) and typ.is_type_obj(): self.add_operator_method_dependency_for_type(typ.fallback, method) elif isinstance(typ, TypeType): if isinstance( typ.item, Instance) and typ.item.type.metaclass_type is not None: self.add_operator_method_dependency_for_type( typ.item.type.metaclass_type, method) def visit_generator_expr(self, e: GeneratorExpr) -> None: super().visit_generator_expr(e) for seq in e.sequences: self.add_iter_dependency(seq) def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> None: super().visit_dictionary_comprehension(e) for seq in e.sequences: self.add_iter_dependency(seq) def visit_star_expr(self, e: StarExpr) -> None: super().visit_star_expr(e) self.add_iter_dependency(e.expr) def visit_yield_from_expr(self, e: YieldFromExpr) -> None: super().visit_yield_from_expr(e) self.add_iter_dependency(e.expr) def visit_await_expr(self, e: AwaitExpr) -> None: super().visit_await_expr(e) self.add_attribute_dependency_for_expr(e.expr, '__await__') # Helpers def add_type_alias_deps(self, target: str) -> None: # Type aliases are special, because some of the dependencies are calculated # in semanal.py, before they are expanded. if target in self.alias_deps: for alias in self.alias_deps[target]: self.add_dependency(make_trigger(alias)) def add_dependency(self, trigger: str, target: Optional[str] = None) -> None: """Add dependency from trigger to a target. If the target is not given explicitly, use the current target. """ if trigger.startswith(('<builtins.', '<typing.')): # Don't track dependencies to certain builtins to keep the size of # the dependencies manageable. These dependencies should only # change on mypy version updates, which will require a full rebuild # anyway. return if target is None: target = self.scope.current_target() self.map.setdefault(trigger, set()).add(target) def add_type_dependencies(self, typ: Type, target: Optional[str] = None) -> None: """Add dependencies to all components of a type. Args: target: If not None, override the default (current) target of the generated dependency. """ # TODO: Use this method in more places where get_type_triggers() + add_dependency() # are called together. for trigger in get_type_triggers(typ): self.add_dependency(trigger, target) def add_attribute_dependency(self, typ: Type, name: str) -> None: """Add dependencies for accessing a named attribute of a type.""" targets = self.attribute_triggers(typ, name) for target in targets: self.add_dependency(target) def attribute_triggers(self, typ: Type, name: str) -> List[str]: """Return all triggers associated with the attribute of a type.""" if isinstance(typ, TypeVarType): typ = typ.upper_bound if isinstance(typ, TupleType): typ = typ.fallback if isinstance(typ, Instance): member = '%s.%s' % (typ.type.fullname(), name) return [make_trigger(member)] elif isinstance(typ, FunctionLike) and typ.is_type_obj(): member = '%s.%s' % (typ.type_object().fullname(), name) triggers = [make_trigger(member)] triggers.extend(self.attribute_triggers(typ.fallback, name)) return triggers elif isinstance(typ, UnionType): targets = [] for item in typ.items: targets.extend(self.attribute_triggers(item, name)) return targets elif isinstance(typ, TypeType): triggers = self.attribute_triggers(typ.item, name) if isinstance( typ.item, Instance) and typ.item.type.metaclass_type is not None: triggers.append( make_trigger( '%s.%s' % (typ.item.type.metaclass_type.type.fullname(), name))) return triggers else: return [] def add_attribute_dependency_for_expr(self, e: Expression, name: str) -> None: typ = self.type_map.get(e) if typ is not None: self.add_attribute_dependency(typ, name) def add_iter_dependency(self, node: Expression) -> None: typ = self.type_map.get(node) if typ: self.add_attribute_dependency(typ, '__iter__')
class DependencyVisitor(TraverserVisitor): def __init__(self, type_map: Dict[Expression, Type], python_version: Tuple[int, int], alias_deps: 'DefaultDict[str, Set[str]]', options: Optional[Options] = None) -> None: self.scope = Scope() self.type_map = type_map self.python2 = python_version[0] == 2 # This attribute holds a mapping from target to names of type aliases # it depends on. These need to be processed specially, since they are # only present in expanded form in symbol tables. For example, after: # A = List[int] # x: A # The module symbol table will just have a Var `x` with type `List[int]`, # and the dependency of `x` on `A` is lost. Therefore the alias dependencies # are preserved at alias expansion points in `semanal.py`, stored as an attribute # on MypyFile, and then passed here. self.alias_deps = alias_deps self.map = {} # type: Dict[str, Set[str]] self.is_class = False self.is_package_init_file = False self.options = options def visit_mypy_file(self, o: MypyFile) -> None: self.scope.enter_file(o.fullname()) self.is_package_init_file = o.is_package_init_file() self.add_type_alias_deps(self.scope.current_target()) for trigger, targets in o.plugin_deps.items(): self.map.setdefault(trigger, set()).update(targets) super().visit_mypy_file(o) self.scope.leave() def visit_func_def(self, o: FuncDef) -> None: self.scope.enter_function(o) target = self.scope.current_target() if o.type: if self.is_class and isinstance(o.type, FunctionLike): signature = bind_self(o.type) # type: Type else: signature = o.type for trigger in self.get_type_triggers(signature): self.add_dependency(trigger) self.add_dependency(trigger, target=make_trigger(target)) if o.info: for base in non_trivial_bases(o.info): # Base class __init__/__new__ doesn't generate a logical # dependency since the override can be incompatible. if not self.use_logical_deps() or o.name() not in ('__init__', '__new__'): self.add_dependency(make_trigger(base.fullname() + '.' + o.name())) self.add_type_alias_deps(self.scope.current_target()) super().visit_func_def(o) variants = set(o.expanded) - {o} for ex in variants: if isinstance(ex, FuncDef): super().visit_func_def(ex) self.scope.leave() def visit_decorator(self, o: Decorator) -> None: if not self.use_logical_deps(): # We don't need to recheck outer scope for an overload, only overload itself. # Also if any decorator is nested, it is not externally visible, so we don't need to # generate dependency. if not o.func.is_overload and self.scope.current_function_name() is None: self.add_dependency(make_trigger(o.func.fullname())) else: # Add logical dependencies from decorators to the function. For example, # if we have # @dec # def func(): ... # then if `dec` is unannotated, then it will "spoil" `func` and consequently # all call sites, making them all `Any`. for d in o.decorators: tname = None # type: Optional[str] if isinstance(d, RefExpr) and d.fullname is not None: tname = d.fullname if (isinstance(d, CallExpr) and isinstance(d.callee, RefExpr) and d.callee.fullname is not None): tname = d.callee.fullname if tname is not None: self.add_dependency(make_trigger(tname), make_trigger(o.func.fullname())) super().visit_decorator(o) def visit_class_def(self, o: ClassDef) -> None: self.scope.enter_class(o.info) target = self.scope.current_full_target() self.add_dependency(make_trigger(target), target) old_is_class = self.is_class self.is_class = True # Add dependencies to type variables of a generic class. for tv in o.type_vars: self.add_dependency(make_trigger(tv.fullname), target) self.process_type_info(o.info) super().visit_class_def(o) self.is_class = old_is_class self.scope.leave() def visit_newtype_expr(self, o: NewTypeExpr) -> None: if o.info: self.scope.enter_class(o.info) self.process_type_info(o.info) self.scope.leave() def process_type_info(self, info: TypeInfo) -> None: target = self.scope.current_full_target() for base in info.bases: self.add_type_dependencies(base, target=target) if info.tuple_type: self.add_type_dependencies(info.tuple_type, target=make_trigger(target)) if info.typeddict_type: self.add_type_dependencies(info.typeddict_type, target=make_trigger(target)) if info.declared_metaclass: self.add_type_dependencies(info.declared_metaclass, target=make_trigger(target)) if info.is_protocol: for base_info in info.mro[:-1]: # We add dependencies from whole MRO to cover explicit subprotocols. # For example: # # class Super(Protocol): # x: int # class Sub(Super, Protocol): # y: int # # In this example we add <Super[wildcard]> -> <Sub>, to invalidate Sub if # a new member is added to Super. self.add_dependency(make_wildcard_trigger(base_info.fullname()), target=make_trigger(target)) # More protocol dependencies are collected in TypeState._snapshot_protocol_deps # after a full run or update is finished. self.add_type_alias_deps(self.scope.current_target()) for name, node in info.names.items(): if isinstance(node.node, Var): # Recheck Liskov if needed, self definitions are checked in the defining method if node.node.is_initialized_in_class and has_user_bases(info): self.add_dependency(make_trigger(info.fullname() + '.' + name)) for base_info in non_trivial_bases(info): # If the type of an attribute changes in a base class, we make references # to the attribute in the subclass stale. self.add_dependency(make_trigger(base_info.fullname() + '.' + name), target=make_trigger(info.fullname() + '.' + name)) for base_info in non_trivial_bases(info): for name, node in base_info.names.items(): if self.use_logical_deps(): # Skip logical dependency if an attribute is not overridden. For example, # in case of: # class Base: # x = 1 # y = 2 # class Sub(Base): # x = 3 # we skip <Base.y> -> <Child.y>, because even if `y` is unannotated it # doesn't affect precision of Liskov checking. if name not in info.names: continue # __init__ and __new__ can be overridden with different signatures, so no # logical depedency. if name in ('__init__', '__new__'): continue self.add_dependency(make_trigger(base_info.fullname() + '.' + name), target=make_trigger(info.fullname() + '.' + name)) if not self.use_logical_deps(): # These dependencies are only useful for propagating changes -- # they aren't logical dependencies since __init__ and __new__ can be # overridden with a different signature. self.add_dependency(make_trigger(base_info.fullname() + '.__init__'), target=make_trigger(info.fullname() + '.__init__')) self.add_dependency(make_trigger(base_info.fullname() + '.__new__'), target=make_trigger(info.fullname() + '.__new__')) # If the set of abstract attributes change, this may invalidate class # instantiation, or change the generated error message, since Python checks # class abstract status when creating an instance. # # TODO: We should probably add this dependency only from the __init__ of the # current class, and independent of bases (to trigger changes in message # wording, as errors may enumerate all abstract attributes). self.add_dependency(make_trigger(base_info.fullname() + '.(abstract)'), target=make_trigger(info.fullname() + '.__init__')) # If the base class abstract attributes change, subclass abstract # attributes need to be recalculated. self.add_dependency(make_trigger(base_info.fullname() + '.(abstract)')) def visit_import(self, o: Import) -> None: for id, as_id in o.ids: self.add_dependency(make_trigger(id), self.scope.current_target()) def visit_import_from(self, o: ImportFrom) -> None: if self.use_logical_deps(): # Just importing a name doesn't create a logical dependency. return module_id, _ = correct_relative_import(self.scope.current_module_id(), o.relative, o.id, self.is_package_init_file) for name, as_name in o.names: self.add_dependency(make_trigger(module_id + '.' + name)) def visit_import_all(self, o: ImportAll) -> None: module_id, _ = correct_relative_import(self.scope.current_module_id(), o.relative, o.id, self.is_package_init_file) # The current target needs to be rechecked if anything "significant" changes in the # target module namespace (as the imported definitions will need to be updated). self.add_dependency(make_wildcard_trigger(module_id)) def visit_block(self, o: Block) -> None: if not o.is_unreachable: super().visit_block(o) def visit_assignment_stmt(self, o: AssignmentStmt) -> None: rvalue = o.rvalue if isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, TypeVarExpr): analyzed = rvalue.analyzed self.add_type_dependencies(analyzed.upper_bound, target=make_trigger(analyzed.fullname())) for val in analyzed.values: self.add_type_dependencies(val, target=make_trigger(analyzed.fullname())) # We need to re-analyze the definition if bound or value is deleted. super().visit_call_expr(rvalue) elif isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, NamedTupleExpr): # Depend on types of named tuple items. info = rvalue.analyzed.info prefix = '%s.%s' % (self.scope.current_full_target(), info.name()) for name, symnode in info.names.items(): if not name.startswith('_') and isinstance(symnode.node, Var): typ = symnode.node.type if typ: self.add_type_dependencies(typ) self.add_type_dependencies(typ, target=make_trigger(prefix)) attr_target = make_trigger('%s.%s' % (prefix, name)) self.add_type_dependencies(typ, target=attr_target) elif isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, TypedDictExpr): # Depend on the underlying typeddict type info = rvalue.analyzed.info assert info.typeddict_type is not None prefix = '%s.%s' % (self.scope.current_full_target(), info.name()) self.add_type_dependencies(info.typeddict_type, target=make_trigger(prefix)) elif isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, EnumCallExpr): # Enum values are currently not checked, but for future we add the deps on them for name, symnode in rvalue.analyzed.info.names.items(): if isinstance(symnode.node, Var) and symnode.node.type: self.add_type_dependencies(symnode.node.type) elif o.is_alias_def: assert len(o.lvalues) == 1 lvalue = o.lvalues[0] assert isinstance(lvalue, NameExpr) # TODO: get rid of this extra dependency from __init__ to alias definition scope typ = self.type_map.get(lvalue) if isinstance(typ, FunctionLike) and typ.is_type_obj(): class_name = typ.type_object().fullname() self.add_dependency(make_trigger(class_name + '.__init__')) self.add_dependency(make_trigger(class_name + '.__new__')) if isinstance(rvalue, IndexExpr) and isinstance(rvalue.analyzed, TypeAliasExpr): self.add_type_dependencies(rvalue.analyzed.type) elif typ: self.add_type_dependencies(typ) else: # Normal assignment super().visit_assignment_stmt(o) for lvalue in o.lvalues: self.process_lvalue(lvalue) items = o.lvalues + [rvalue] for i in range(len(items) - 1): lvalue = items[i] rvalue = items[i + 1] if isinstance(lvalue, TupleExpr): self.add_attribute_dependency_for_expr(rvalue, '__iter__') if o.type: for trigger in self.get_type_triggers(o.type): self.add_dependency(trigger) if self.use_logical_deps() and o.unanalyzed_type is None: # Special case: for definitions without an explicit type like this: # x = func(...) # we add a logical dependency <func> -> <x>, because if `func` is not annotated, # then it will make all points of use of `x` unchecked. if (isinstance(rvalue, CallExpr) and isinstance(rvalue.callee, RefExpr) and rvalue.callee.fullname is not None): fname = None # type: Optional[str] if isinstance(rvalue.callee.node, TypeInfo): # use actual __init__ as a dependency source init = rvalue.callee.node.get('__init__') if init and isinstance(init.node, FuncBase): fname = init.node.fullname() else: fname = rvalue.callee.fullname if fname is None: return for lv in o.lvalues: if isinstance(lv, RefExpr) and lv.fullname and lv.is_new_def: if lv.kind == LDEF: return # local definitions don't generate logical deps self.add_dependency(make_trigger(fname), make_trigger(lv.fullname)) def process_lvalue(self, lvalue: Expression) -> None: """Generate additional dependencies for an lvalue.""" if isinstance(lvalue, IndexExpr): self.add_operator_method_dependency(lvalue.base, '__setitem__') elif isinstance(lvalue, NameExpr): if lvalue.kind in (MDEF, GDEF): # Assignment to an attribute in the class body, or direct assignment to a # global variable. lvalue_type = self.get_non_partial_lvalue_type(lvalue) type_triggers = self.get_type_triggers(lvalue_type) attr_trigger = make_trigger('%s.%s' % (self.scope.current_full_target(), lvalue.name)) for type_trigger in type_triggers: self.add_dependency(type_trigger, attr_trigger) elif isinstance(lvalue, MemberExpr): if self.is_self_member_ref(lvalue) and lvalue.is_new_def: node = lvalue.node if isinstance(node, Var): info = node.info if info and has_user_bases(info): # Recheck Liskov for self definitions self.add_dependency(make_trigger(info.fullname() + '.' + lvalue.name)) if lvalue.kind is None: # Reference to a non-module attribute if lvalue.expr not in self.type_map: # Unreachable assignment -> not checked so no dependencies to generate. return object_type = self.type_map[lvalue.expr] lvalue_type = self.get_non_partial_lvalue_type(lvalue) type_triggers = self.get_type_triggers(lvalue_type) for attr_trigger in self.attribute_triggers(object_type, lvalue.name): for type_trigger in type_triggers: self.add_dependency(type_trigger, attr_trigger) elif isinstance(lvalue, TupleExpr): for item in lvalue.items: self.process_lvalue(item) elif isinstance(lvalue, StarExpr): self.process_lvalue(lvalue.expr) def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: """Does memberexpr to refer to an attribute of self?""" if not isinstance(memberexpr.expr, NameExpr): return False node = memberexpr.expr.node return isinstance(node, Var) and node.is_self def get_non_partial_lvalue_type(self, lvalue: RefExpr) -> Type: if lvalue not in self.type_map: # Likely a block considered unreachable during type checking. return UninhabitedType() lvalue_type = self.type_map[lvalue] if isinstance(lvalue_type, PartialType): if isinstance(lvalue.node, Var) and lvalue.node.type: lvalue_type = lvalue.node.type else: # Probably a secondary, non-definition assignment that doesn't # result in a non-partial type. We won't be able to infer any # dependencies from this so just return something. (The first, # definition assignment with a partial type is handled # differently, in the semantic analyzer.) assert not lvalue.is_new_def return UninhabitedType() return lvalue_type def visit_operator_assignment_stmt(self, o: OperatorAssignmentStmt) -> None: super().visit_operator_assignment_stmt(o) self.process_lvalue(o.lvalue) method = op_methods[o.op] self.add_attribute_dependency_for_expr(o.lvalue, method) if o.op in ops_with_inplace_method: inplace_method = '__i' + method[2:] self.add_attribute_dependency_for_expr(o.lvalue, inplace_method) def visit_for_stmt(self, o: ForStmt) -> None: super().visit_for_stmt(o) if not o.is_async: # __getitem__ is only used if __iter__ is missing but for simplicity we # just always depend on both. self.add_attribute_dependency_for_expr(o.expr, '__iter__') self.add_attribute_dependency_for_expr(o.expr, '__getitem__') if o.inferred_iterator_type: if self.python2: method = 'next' else: method = '__next__' self.add_attribute_dependency(o.inferred_iterator_type, method) else: self.add_attribute_dependency_for_expr(o.expr, '__aiter__') if o.inferred_iterator_type: self.add_attribute_dependency(o.inferred_iterator_type, '__anext__') self.process_lvalue(o.index) if isinstance(o.index, TupleExpr): # Process multiple assignment to index variables. item_type = o.inferred_item_type if item_type: # This is similar to above. self.add_attribute_dependency(item_type, '__iter__') self.add_attribute_dependency(item_type, '__getitem__') if o.index_type: self.add_type_dependencies(o.index_type) def visit_with_stmt(self, o: WithStmt) -> None: super().visit_with_stmt(o) for e in o.expr: if not o.is_async: self.add_attribute_dependency_for_expr(e, '__enter__') self.add_attribute_dependency_for_expr(e, '__exit__') else: self.add_attribute_dependency_for_expr(e, '__aenter__') self.add_attribute_dependency_for_expr(e, '__aexit__') for typ in o.analyzed_types: self.add_type_dependencies(typ) def visit_print_stmt(self, o: PrintStmt) -> None: super().visit_print_stmt(o) if o.target: self.add_attribute_dependency_for_expr(o.target, 'write') def visit_del_stmt(self, o: DelStmt) -> None: super().visit_del_stmt(o) if isinstance(o.expr, IndexExpr): self.add_attribute_dependency_for_expr(o.expr.base, '__delitem__') # Expressions def process_global_ref_expr(self, o: RefExpr) -> None: if o.fullname is not None: self.add_dependency(make_trigger(o.fullname)) # If this is a reference to a type, generate a dependency to its # constructor. # IDEA: Avoid generating spurious dependencies for except statements, # class attribute references, etc., if performance is a problem. typ = self.type_map.get(o) if isinstance(typ, FunctionLike) and typ.is_type_obj(): class_name = typ.type_object().fullname() self.add_dependency(make_trigger(class_name + '.__init__')) self.add_dependency(make_trigger(class_name + '.__new__')) def visit_name_expr(self, o: NameExpr) -> None: if o.kind == LDEF: # We don't track dependencies to local variables, since they # aren't externally visible. return if o.kind == MDEF: # Direct reference to member is only possible in the scope that # defined the name, so no dependency is required. return self.process_global_ref_expr(o) def visit_member_expr(self, e: MemberExpr) -> None: if isinstance(e.expr, RefExpr) and isinstance(e.expr.node, TypeInfo): # Special case class attribute so that we don't depend on "__init__". self.add_dependency(make_trigger(e.expr.node.fullname())) else: super().visit_member_expr(e) if e.kind is not None: # Reference to a module attribute self.process_global_ref_expr(e) else: # Reference to a non-module (or missing) attribute if e.expr not in self.type_map: # No type available -- this happens for unreachable code. Since it's unreachable, # it wasn't type checked and we don't need to generate dependencies. return if isinstance(e.expr, RefExpr) and isinstance(e.expr.node, MypyFile): # Special case: reference to a missing module attribute. self.add_dependency(make_trigger(e.expr.node.fullname() + '.' + e.name)) return typ = self.type_map[e.expr] self.add_attribute_dependency(typ, e.name) if self.use_logical_deps() and isinstance(typ, AnyType): name = self.get_unimported_fullname(e, typ) if name is not None: # Generate a logical dependency from an unimported # definition (which comes from a missing module). # Example: # import missing # "missing" not in build # # def g() -> None: # missing.f() # Generate dependency from "missing.f" self.add_dependency(make_trigger(name)) def get_unimported_fullname(self, e: MemberExpr, typ: AnyType) -> Optional[str]: """If e refers to an unimported definition, infer the fullname of this. Return None if e doesn't refer to an unimported definition or if we can't determine the name. """ suffix = '' # Unwrap nested member expression to handle cases like "a.b.c.d" where # "a.b" is a known reference to an unimported module. Find the base # reference to an unimported module (such as "a.b") and the name suffix # (such as "c.d") needed to build a full name. while typ.type_of_any == TypeOfAny.from_another_any and isinstance(e.expr, MemberExpr): suffix = '.' + e.name + suffix e = e.expr if e.expr not in self.type_map: return None obj_type = self.type_map[e.expr] if not isinstance(obj_type, AnyType): # Can't find the base reference to the unimported module. return None typ = obj_type if typ.type_of_any == TypeOfAny.from_unimported_type and typ.missing_import_name: # Infer the full name of the unimported definition. return typ.missing_import_name + '.' + e.name + suffix return None def visit_super_expr(self, e: SuperExpr) -> None: # Arguments in "super(C, self)" won't generate useful logical deps. if not self.use_logical_deps(): super().visit_super_expr(e) if e.info is not None: name = e.name for base in non_trivial_bases(e.info): self.add_dependency(make_trigger(base.fullname() + '.' + name)) if name in base.names: # No need to depend on further base classes, since we found # the target. This is safe since if the target gets # deleted or modified, we'll trigger it. break def visit_call_expr(self, e: CallExpr) -> None: if isinstance(e.callee, RefExpr) and e.callee.fullname == 'builtins.isinstance': self.process_isinstance_call(e) else: super().visit_call_expr(e) def process_isinstance_call(self, e: CallExpr) -> None: """Process "isinstance(...)" in a way to avoid some extra dependencies.""" if len(e.args) == 2: arg = e.args[1] if (isinstance(arg, RefExpr) and arg.kind == GDEF and isinstance(arg.node, TypeInfo) and arg.fullname): # Special case to avoid redundant dependencies from "__init__". self.add_dependency(make_trigger(arg.fullname)) return # In uncommon cases generate normal dependencies. These will include # spurious dependencies, but the performance impact is small. super().visit_call_expr(e) def visit_cast_expr(self, e: CastExpr) -> None: super().visit_cast_expr(e) self.add_type_dependencies(e.type) def visit_type_application(self, e: TypeApplication) -> None: super().visit_type_application(e) for typ in e.types: self.add_type_dependencies(typ) def visit_index_expr(self, e: IndexExpr) -> None: super().visit_index_expr(e) self.add_operator_method_dependency(e.base, '__getitem__') def visit_unary_expr(self, e: UnaryExpr) -> None: super().visit_unary_expr(e) if e.op not in unary_op_methods: return method = unary_op_methods[e.op] self.add_operator_method_dependency(e.expr, method) def visit_op_expr(self, e: OpExpr) -> None: super().visit_op_expr(e) self.process_binary_op(e.op, e.left, e.right) def visit_comparison_expr(self, e: ComparisonExpr) -> None: super().visit_comparison_expr(e) for i, op in enumerate(e.operators): left = e.operands[i] right = e.operands[i + 1] self.process_binary_op(op, left, right) if self.python2 and op in ('==', '!=', '<', '<=', '>', '>='): self.add_operator_method_dependency(left, '__cmp__') self.add_operator_method_dependency(right, '__cmp__') def process_binary_op(self, op: str, left: Expression, right: Expression) -> None: method = op_methods.get(op) if method: if op == 'in': self.add_operator_method_dependency(right, method) else: self.add_operator_method_dependency(left, method) rev_method = reverse_op_methods.get(method) if rev_method: self.add_operator_method_dependency(right, rev_method) def add_operator_method_dependency(self, e: Expression, method: str) -> None: typ = self.type_map.get(e) if typ is not None: self.add_operator_method_dependency_for_type(typ, method) def add_operator_method_dependency_for_type(self, typ: Type, method: str) -> None: # Note that operator methods can't be (non-metaclass) methods of type objects # (that is, TypeType objects or Callables representing a type). if isinstance(typ, TypeVarType): typ = typ.upper_bound if isinstance(typ, TupleType): typ = typ.partial_fallback if isinstance(typ, Instance): trigger = make_trigger(typ.type.fullname() + '.' + method) self.add_dependency(trigger) elif isinstance(typ, UnionType): for item in typ.items: self.add_operator_method_dependency_for_type(item, method) elif isinstance(typ, FunctionLike) and typ.is_type_obj(): self.add_operator_method_dependency_for_type(typ.fallback, method) elif isinstance(typ, TypeType): if isinstance(typ.item, Instance) and typ.item.type.metaclass_type is not None: self.add_operator_method_dependency_for_type(typ.item.type.metaclass_type, method) def visit_generator_expr(self, e: GeneratorExpr) -> None: super().visit_generator_expr(e) for seq in e.sequences: self.add_iter_dependency(seq) def visit_dictionary_comprehension(self, e: DictionaryComprehension) -> None: super().visit_dictionary_comprehension(e) for seq in e.sequences: self.add_iter_dependency(seq) def visit_star_expr(self, e: StarExpr) -> None: super().visit_star_expr(e) self.add_iter_dependency(e.expr) def visit_yield_from_expr(self, e: YieldFromExpr) -> None: super().visit_yield_from_expr(e) self.add_iter_dependency(e.expr) def visit_await_expr(self, e: AwaitExpr) -> None: super().visit_await_expr(e) self.add_attribute_dependency_for_expr(e.expr, '__await__') # Helpers def add_type_alias_deps(self, target: str) -> None: # Type aliases are special, because some of the dependencies are calculated # in semanal.py, before they are expanded. if target in self.alias_deps: for alias in self.alias_deps[target]: self.add_dependency(make_trigger(alias)) def add_dependency(self, trigger: str, target: Optional[str] = None) -> None: """Add dependency from trigger to a target. If the target is not given explicitly, use the current target. """ if trigger.startswith(('<builtins.', '<typing.', '<mypy_extensions.')): # Don't track dependencies to certain library modules to keep the size of # the dependencies manageable. These dependencies should only # change on mypy version updates, which will require a full rebuild # anyway. return if target is None: target = self.scope.current_target() self.map.setdefault(trigger, set()).add(target) def add_type_dependencies(self, typ: Type, target: Optional[str] = None) -> None: """Add dependencies to all components of a type. Args: target: If not None, override the default (current) target of the generated dependency. """ # TODO: Use this method in more places where get_type_triggers() + add_dependency() # are called together. for trigger in self.get_type_triggers(typ): self.add_dependency(trigger, target) def add_attribute_dependency(self, typ: Type, name: str) -> None: """Add dependencies for accessing a named attribute of a type.""" targets = self.attribute_triggers(typ, name) for target in targets: self.add_dependency(target) def attribute_triggers(self, typ: Type, name: str) -> List[str]: """Return all triggers associated with the attribute of a type.""" if isinstance(typ, TypeVarType): typ = typ.upper_bound if isinstance(typ, TupleType): typ = typ.partial_fallback if isinstance(typ, Instance): member = '%s.%s' % (typ.type.fullname(), name) return [make_trigger(member)] elif isinstance(typ, FunctionLike) and typ.is_type_obj(): member = '%s.%s' % (typ.type_object().fullname(), name) triggers = [make_trigger(member)] triggers.extend(self.attribute_triggers(typ.fallback, name)) return triggers elif isinstance(typ, UnionType): targets = [] for item in typ.items: targets.extend(self.attribute_triggers(item, name)) return targets elif isinstance(typ, TypeType): triggers = self.attribute_triggers(typ.item, name) if isinstance(typ.item, Instance) and typ.item.type.metaclass_type is not None: triggers.append(make_trigger('%s.%s' % (typ.item.type.metaclass_type.type.fullname(), name))) return triggers else: return [] def add_attribute_dependency_for_expr(self, e: Expression, name: str) -> None: typ = self.type_map.get(e) if typ is not None: self.add_attribute_dependency(typ, name) def add_iter_dependency(self, node: Expression) -> None: typ = self.type_map.get(node) if typ: self.add_attribute_dependency(typ, '__iter__') def use_logical_deps(self) -> bool: return self.options is not None and self.options.logical_deps def get_type_triggers(self, typ: Type) -> List[str]: return get_type_triggers(typ, self.use_logical_deps())