def make_simplified_union(items: Sequence[Type], line: int = -1, column: int = -1, *, keep_erased: bool = False) -> ProperType: """Build union type with redundant union items removed. If only a single item remains, this may return a non-union type. Examples: * [int, str] -> Union[int, str] * [int, object] -> object * [int, int] -> int * [int, Any] -> Union[int, Any] (Any types are not simplified away!) * [Any, Any] -> Any Note: This must NOT be used during semantic analysis, since TypeInfos may not be fully initialized. The keep_erased flag is used for type inference against union types containing type variables. If set to True, keep all ErasedType items. """ items = get_proper_types(items) while any(isinstance(typ, UnionType) for typ in items): all_items = [] # type: List[ProperType] for typ in items: if isinstance(typ, UnionType): all_items.extend(get_proper_types(typ.items)) else: all_items.append(typ) items = all_items from mypy.subtypes import is_proper_subtype removed = set() # type: Set[int] for i, ti in enumerate(items): if i in removed: continue # Keep track of the truishness info for deleted subtypes which can be relevant cbt = cbf = False for j, tj in enumerate(items): if i != j and is_proper_subtype( tj, ti, keep_erased_types=keep_erased): # We found a redundant item in the union. removed.add(j) cbt = cbt or tj.can_be_true cbf = cbf or tj.can_be_false # if deleted subtypes had more general truthiness, use that if not ti.can_be_true and cbt: items[i] = true_or_false(ti) elif not ti.can_be_false and cbf: items[i] = true_or_false(ti) simplified_set = [items[i] for i in range(len(items)) if i not in removed] return UnionType.make_union(simplified_set, line, column)
def make_simplified_union(items: Sequence[Type], line: int = -1, column: int = -1, *, keep_erased: bool = False, contract_literals: bool = True) -> ProperType: """Build union type with redundant union items removed. If only a single item remains, this may return a non-union type. Examples: * [int, str] -> Union[int, str] * [int, object] -> object * [int, int] -> int * [int, Any] -> Union[int, Any] (Any types are not simplified away!) * [Any, Any] -> Any Note: This must NOT be used during semantic analysis, since TypeInfos may not be fully initialized. The keep_erased flag is used for type inference against union types containing type variables. If set to True, keep all ErasedType items. The contract_literals flag indicates whether we need to contract literal types back into a sum type. Set it to False when called by try_expanding_sum_type_ to_union(). """ items = get_proper_types(items) # Step 1: expand all nested unions while any(isinstance(typ, UnionType) for typ in items): all_items: List[ProperType] = [] for typ in items: if isinstance(typ, UnionType): all_items.extend(get_proper_types(typ.items)) else: all_items.append(typ) items = all_items # Step 2: remove redundant unions simplified_set = _remove_redundant_union_items(items, keep_erased) # Step 3: If more than one literal exists in the union, try to simplify if contract_literals and sum( isinstance(item, LiteralType) for item in simplified_set) > 1: simplified_set = try_contracting_literals_in_union(simplified_set) return UnionType.make_union(simplified_set, line, column)
def is_special_target(right: ProperType) -> bool: """Whitelist some special cases for use in isinstance() with improper types.""" if isinstance(right, CallableType) and right.is_type_obj(): if right.type_object().fullname() == 'builtins.tuple': # Used with Union[Type, Tuple[Type, ...]]. return True if right.type_object().fullname() in ('mypy.types.Type', 'mypy.types.ProperType', 'mypy.types.TypeAliasType'): # Special case: things like assert isinstance(typ, ProperType) are always OK. return True if right.type_object().fullname() in ('mypy.types.UnboundType', 'mypy.types.TypeVarType', 'mypy.types.RawExpressionType', 'mypy.types.EllipsisType', 'mypy.types.StarType', 'mypy.types.TypeList', 'mypy.types.CallableArgument', 'mypy.types.PartialType', 'mypy.types.ErasedType'): # Special case: these are not valid targets for a type alias and thus safe. # TODO: introduce a SyntheticType base to simplify this? return True elif isinstance(right, TupleType): return all(is_special_target(t) for t in get_proper_types(right.items)) return False
def try_getting_str_literals(expr: Expression, typ: Type) -> Optional[List[str]]: """If the given expression or type corresponds to a string literal or a union of string literals, returns a list of the underlying strings. Otherwise, returns None. Specifically, this function is guaranteed to return a list with one or more strings if one one the following is true: 1. 'expr' is a StrExpr 2. 'typ' is a LiteralType containing a string 3. 'typ' is a UnionType containing only LiteralType of strings """ typ = get_proper_type(typ) if isinstance(expr, StrExpr): return [expr.value] if isinstance(typ, Instance) and typ.last_known_value is not None: possible_literals = [typ.last_known_value] # type: List[Type] elif isinstance(typ, UnionType): possible_literals = list(typ.items) else: possible_literals = [typ] strings = [] for lit in get_proper_types(possible_literals): if isinstance(lit, LiteralType) and lit.fallback.type.fullname( ) == 'builtins.str': val = lit.value assert isinstance(val, str) strings.append(val) else: return None return strings
def visit_callable_type(self, t: CallableType) -> ProperType: if isinstance(self.s, CallableType) and is_similar_callables( t, self.s): if is_equivalent(t, self.s): return combine_similar_callables(t, self.s) result = join_similar_callables(t, self.s) # We set the from_type_type flag to suppress error when a collection of # concrete class objects gets inferred as their common abstract superclass. if not ( (t.is_type_obj() and t.type_object().is_abstract) or (self.s.is_type_obj() and self.s.type_object().is_abstract)): result.from_type_type = True if any( isinstance(tp, (NoneType, UninhabitedType)) for tp in get_proper_types(result.arg_types)): # We don't want to return unusable Callable, attempt fallback instead. return join_types(t.fallback, self.s) return result elif isinstance(self.s, Overloaded): # Switch the order of arguments to that we'll get to visit_overloaded. return join_types(t, self.s) elif isinstance(self.s, Instance) and self.s.type.is_protocol: call = unpack_callback_protocol(self.s) if call: return join_types(t, call) return join_types(t.fallback, self.s)
def visit_union_type(self, t: UnionType) -> Type: new = cast(UnionType, super().visit_union_type(t)) # Erasure can result in many duplicate items; merge them. # Call make_simplified_union only on lists of instance types # that all have the same fullname, to avoid simplifying too # much. instances = [item for item in new.items if isinstance(get_proper_type(item), Instance)] # Avoid merge in simple cases such as optional types. if len(instances) > 1: instances_by_name: Dict[str, List[Instance]] = {} new_items = get_proper_types(new.items) for item in new_items: if isinstance(item, Instance) and not item.args: instances_by_name.setdefault(item.type.fullname, []).append(item) merged: List[Type] = [] for item in new_items: if isinstance(item, Instance) and not item.args: types = instances_by_name.get(item.type.fullname) if types is not None: if len(types) == 1: merged.append(item) else: from mypy.typeops import make_simplified_union merged.append(make_simplified_union(types)) del instances_by_name[item.type.fullname] else: merged.append(item) return UnionType.make_union(merged) return new
def narrow_declared_type(declared: Type, narrowed: Type) -> Type: """Return the declared type narrowed down to another type.""" # TODO: check infinite recursion for aliases here. declared = get_proper_type(declared) narrowed = get_proper_type(narrowed) if declared == narrowed: return declared if isinstance(declared, UnionType): return make_simplified_union([narrow_declared_type(x, narrowed) for x in declared.relevant_items()]) elif not is_overlapping_types(declared, narrowed, prohibit_none_typevar_overlap=True): if state.strict_optional: return UninhabitedType() else: return NoneType() elif isinstance(narrowed, UnionType): return make_simplified_union([narrow_declared_type(declared, x) for x in narrowed.relevant_items()]) elif isinstance(narrowed, AnyType): return narrowed elif isinstance(declared, TypeType) and isinstance(narrowed, TypeType): return TypeType.make_normalized(narrow_declared_type(declared.item, narrowed.item)) elif isinstance(declared, (Instance, TupleType, TypeType, LiteralType)): return meet_types(declared, narrowed) elif isinstance(declared, TypedDictType) and isinstance(narrowed, Instance): # Special case useful for selecting TypedDicts from unions using isinstance(x, dict). if (narrowed.type.fullname == 'builtins.dict' and all(isinstance(t, AnyType) for t in get_proper_types(narrowed.args))): return declared return meet_types(declared, narrowed) return narrowed
def try_getting_literals_from_type(typ: Type, target_literal_type: TypingType[T], target_fullname: str) -> Optional[List[T]]: """If the given expression or type corresponds to a Literal or union of Literals where the underlying values corresponds to the given target type, returns a list of those underlying values. Otherwise, returns None. """ typ = get_proper_type(typ) if isinstance(typ, Instance) and typ.last_known_value is not None: possible_literals: List[Type] = [typ.last_known_value] elif isinstance(typ, UnionType): possible_literals = list(typ.items) else: possible_literals = [typ] literals: List[T] = [] for lit in get_proper_types(possible_literals): if isinstance( lit, LiteralType) and lit.fallback.type.fullname == target_fullname: val = lit.value if isinstance(val, target_literal_type): literals.append(val) else: return None else: return None return literals
def check_specs_in_format_call(self, call: CallExpr, specs: List[ConversionSpecifier], format_value: str) -> None: """Perform pairwise checks for conversion specifiers vs their replacements. The core logic for format checking is implemented in this method. """ assert all(s.key for s in specs), "Keys must be auto-generated first!" replacements = self.find_replacements_in_call(call, [cast(str, s.key) for s in specs]) assert len(replacements) == len(specs) for spec, repl in zip(specs, replacements): repl = self.apply_field_accessors(spec, repl, ctx=call) actual_type = repl.type if isinstance(repl, TempNode) else self.chk.type_map.get(repl) assert actual_type is not None # Special case custom formatting. if (spec.format_spec and spec.non_standard_format_spec and # Exclude "dynamic" specifiers (i.e. containing nested formatting). not ('{' in spec.format_spec or '}' in spec.format_spec)): if (not custom_special_method(actual_type, '__format__', check_all=True) or spec.conversion): # TODO: add support for some custom specs like datetime? self.msg.fail('Unrecognized format' ' specification "{}"'.format(spec.format_spec[1:]), call, code=codes.STRING_FORMATTING) continue # Adjust expected and actual types. if not spec.type: expected_type = AnyType(TypeOfAny.special_form) # type: Optional[Type] else: assert isinstance(call.callee, MemberExpr) if isinstance(call.callee.expr, (StrExpr, UnicodeExpr)): format_str = call.callee.expr else: format_str = StrExpr(format_value) expected_type = self.conversion_type(spec.type, call, format_str, format_call=True) if spec.conversion is not None: # If the explicit conversion is given, then explicit conversion is called _first_. if spec.conversion[1] not in 'rsa': self.msg.fail('Invalid conversion type "{}",' ' must be one of "r", "s" or "a"'.format(spec.conversion[1]), call, code=codes.STRING_FORMATTING) actual_type = self.named_type('builtins.str') # Perform the checks for given types. if expected_type is None: continue a_type = get_proper_type(actual_type) actual_items = (get_proper_types(a_type.items) if isinstance(a_type, UnionType) else [a_type]) for a_type in actual_items: if custom_special_method(a_type, '__format__'): continue self.check_placeholder_type(a_type, expected_type, call) self.perform_special_format_checks(spec, call, repl, a_type, expected_type)
def type(self, t: Optional[Type]) -> None: t = get_proper_type(t) if not t: # If an expression does not have a type, it is often due to dead code. # Don't count these because there can be an unanalyzed value on a line with other # analyzed expressions, which overwrite the TYPE_UNANALYZED. self.record_line(self.line, TYPE_UNANALYZED) return if isinstance(t, AnyType) and is_special_form_any(t): # TODO: What if there is an error in special form definition? self.record_line(self.line, TYPE_PRECISE) return if isinstance(t, AnyType): self.log(' !! Any type around line %d' % self.line) self.num_any_exprs += 1 self.record_line(self.line, TYPE_ANY) elif ((not self.all_nodes and is_imprecise(t)) or (self.all_nodes and is_imprecise2(t))): self.log(' !! Imprecise type around line %d' % self.line) self.num_imprecise_exprs += 1 self.record_line(self.line, TYPE_IMPRECISE) else: self.num_precise_exprs += 1 self.record_line(self.line, TYPE_PRECISE) for typ in get_proper_types(collect_all_inner_types(t)) + [t]: if isinstance(typ, AnyType): typ = get_original_any(typ) if is_special_form_any(typ): continue self.type_of_any_counter[typ.type_of_any] += 1 self.num_any_types += 1 if self.line in self.any_line_map: self.any_line_map[self.line].append(typ) else: self.any_line_map[self.line] = [typ] elif isinstance(typ, Instance): if typ.args: if any(is_complex(arg) for arg in typ.args): self.num_complex_types += 1 else: self.num_generic_types += 1 else: self.num_simple_types += 1 elif isinstance(typ, FunctionLike): self.num_function_types += 1 elif isinstance(typ, TupleType): if any(is_complex(item) for item in typ.items): self.num_complex_types += 1 else: self.num_tuple_types += 1 elif isinstance(typ, TypeVarType): self.num_typevar_types += 1
def apply_generic_arguments( callable: CallableType, orig_types: Sequence[Optional[Type]], report_incompatible_typevar_value: Callable[[CallableType, Type, str, Context], None], context: Context, skip_unsatisfied: bool = False) -> CallableType: """Apply generic type arguments to a callable type. For example, applying [int] to 'def [T] (T) -> T' results in 'def (int) -> int'. Note that each type can be None; in this case, it will not be applied. If `skip_unsatisfied` is True, then just skip the types that don't satisfy type variable bound or constraints, instead of giving an error. """ tvars = callable.variables assert len(tvars) == len(orig_types) # Check that inferred type variable values are compatible with allowed # values and bounds. Also, promote subtype values to allowed values. types = get_proper_types(orig_types) # Create a map from type variable id to target type. id_to_type: Dict[TypeVarId, Type] = {} for tvar, type in zip(tvars, types): assert not isinstance(type, PartialType), "Internal error: must never apply partial type" if type is None: continue target_type = get_target_type( tvar, type, callable, report_incompatible_typevar_value, context, skip_unsatisfied ) if target_type is not None: id_to_type[tvar.id] = target_type param_spec = callable.param_spec() if param_spec is not None: nt = id_to_type.get(param_spec.id) if nt is not None: nt = get_proper_type(nt) if isinstance(nt, CallableType): callable = callable.expand_param_spec(nt) # Apply arguments to argument types. arg_types = [expand_type(at, id_to_type) for at in callable.arg_types] # The callable may retain some type vars if only some were applied. remaining_tvars = [tv for tv in tvars if tv.id not in id_to_type] return callable.copy_modified( arg_types=arg_types, ret_type=expand_type(callable.ret_type, id_to_type), variables=remaining_tvars, )
def narrow_declared_type(declared: Type, narrowed: Type) -> Type: """Return the declared type narrowed down to another type.""" # TODO: check infinite recursion for aliases here. if isinstance(narrowed, TypeGuardedType): # type: ignore[misc] # A type guard forces the new type even if it doesn't overlap the old. return narrowed.type_guard declared = get_proper_type(declared) narrowed = get_proper_type(narrowed) if declared == narrowed: return declared if isinstance(declared, UnionType): return make_simplified_union([ narrow_declared_type(x, narrowed) for x in declared.relevant_items() ]) if is_enum_overlapping_union(declared, narrowed): return narrowed elif not is_overlapping_types( declared, narrowed, prohibit_none_typevar_overlap=True): if state.strict_optional: return UninhabitedType() else: return NoneType() elif isinstance(narrowed, UnionType): return make_simplified_union([ narrow_declared_type(declared, x) for x in narrowed.relevant_items() ]) elif isinstance(narrowed, AnyType): return narrowed elif isinstance(narrowed, TypeVarType) and is_subtype( narrowed.upper_bound, declared): return narrowed elif isinstance(declared, TypeType) and isinstance(narrowed, TypeType): return TypeType.make_normalized( narrow_declared_type(declared.item, narrowed.item)) elif (isinstance(declared, TypeType) and isinstance(narrowed, Instance) and narrowed.type.is_metaclass()): # We'd need intersection types, so give up. return declared elif isinstance(declared, (Instance, TupleType, TypeType, LiteralType)): return meet_types(declared, narrowed) elif isinstance(declared, TypedDictType) and isinstance( narrowed, Instance): # Special case useful for selecting TypedDicts from unions using isinstance(x, dict). if (narrowed.type.fullname == 'builtins.dict' and all( isinstance(t, AnyType) for t in get_proper_types(narrowed.args))): return declared return meet_types(declared, narrowed) return narrowed
def are_tuples_overlapping(left: Type, right: Type, *, ignore_promotions: bool = False, prohibit_none_typevar_overlap: bool = False) -> bool: """Returns true if left and right are overlapping tuples.""" left, right = get_proper_types((left, right)) left = adjust_tuple(left, right) or left right = adjust_tuple(right, left) or right assert isinstance(left, TupleType), 'Type {} is not a tuple'.format(left) assert isinstance(right, TupleType), 'Type {} is not a tuple'.format(right) if len(left.items) != len(right.items): return False return all(is_overlapping_types(l, r, ignore_promotions=ignore_promotions, prohibit_none_typevar_overlap=prohibit_none_typevar_overlap) for l, r in zip(left.items, right.items))
def visit_callable_type(self, t: CallableType) -> ProperType: if isinstance(self.s, CallableType) and is_similar_callables(t, self.s): if is_equivalent(t, self.s): return combine_similar_callables(t, self.s) result = join_similar_callables(t, self.s) if any(isinstance(tp, (NoneType, UninhabitedType)) for tp in get_proper_types(result.arg_types)): # We don't want to return unusable Callable, attempt fallback instead. return join_types(t.fallback, self.s) return result elif isinstance(self.s, Overloaded): # Switch the order of arguments to that we'll get to visit_overloaded. return join_types(t, self.s) elif isinstance(self.s, Instance) and self.s.type.is_protocol: call = unpack_callback_protocol(self.s) if call: return join_types(t, call) return join_types(t.fallback, self.s)
def typed_dict_mapping_pair(left: Type, right: Type) -> bool: """Is this a pair where one type is a TypedDict and another one is an instance of Mapping? This case requires a precise/principled consideration because there are two use cases that push the boundary the opposite ways: we need to avoid spurious overlaps to avoid false positives for overloads, but we also need to avoid spuriously non-overlapping types to avoid false positives with --strict-equality. """ left, right = get_proper_types((left, right)) assert not isinstance(left, TypedDictType) or not isinstance(right, TypedDictType) if isinstance(left, TypedDictType): _, other = left, right elif isinstance(right, TypedDictType): _, other = right, left else: return False return isinstance(other, Instance) and other.type.has_base('typing.Mapping')
def get_target_type(tvar: TypeVarLikeType, type: ProperType, callable: CallableType, report_incompatible_typevar_value: Callable[ [CallableType, Type, str, Context], None], context: Context, skip_unsatisfied: bool) -> Optional[Type]: if isinstance(tvar, ParamSpecType): return type if isinstance(tvar, TypeVarTupleType): return type assert isinstance(tvar, TypeVarType) values = get_proper_types(tvar.values) if values: if isinstance(type, AnyType): return type if isinstance(type, TypeVarType) and type.values: # Allow substituting T1 for T if every allowed value of T1 # is also a legal value of T. if all( any(mypy.sametypes.is_same_type(v, v1) for v in values) for v1 in type.values): return type matching = [] for value in values: if mypy.subtypes.is_subtype(type, value): matching.append(value) if matching: best = matching[0] # If there are more than one matching value, we select the narrowest for match in matching[1:]: if mypy.subtypes.is_subtype(match, best): best = match return best if skip_unsatisfied: return None report_incompatible_typevar_value(callable, type, tvar.name, context) else: upper_bound = tvar.upper_bound if not mypy.subtypes.is_subtype(type, upper_bound): if skip_unsatisfied: return None report_incompatible_typevar_value(callable, type, tvar.name, context) return type
def _type_object_overlap(left: Type, right: Type) -> bool: """Special cases for type object types overlaps.""" # TODO: these checks are a bit in gray area, adjust if they cause problems. left, right = get_proper_types((left, right)) # 1. Type[C] vs Callable[..., C], where the latter is class object. if isinstance(left, TypeType) and isinstance(right, CallableType) and right.is_type_obj(): return _is_overlapping_types(left.item, right.ret_type) # 2. Type[C] vs Meta, where Meta is a metaclass for C. if isinstance(left, TypeType) and isinstance(right, Instance): if isinstance(left.item, Instance): left_meta = left.item.type.metaclass_type if left_meta is not None: return _is_overlapping_types(left_meta, right) # builtins.type (default metaclass) overlaps with all metaclasses return right.type.has_base('builtins.type') elif isinstance(left.item, AnyType): return right.type.has_base('builtins.type') # 3. Callable[..., C] vs Meta is considered below, when we switch to fallbacks. return False
def check_type_var_values(self, type: TypeInfo, actuals: List[Type], arg_name: str, valids: List[Type], arg_number: int, context: Context) -> None: for actual in get_proper_types(actuals): if (not isinstance(actual, AnyType) and not any(is_same_type(actual, value) for value in valids)): if len(actuals) > 1 or not isinstance(actual, Instance): self.fail('Invalid type argument value for "{}"'.format( type.name), context, code=codes.TYPE_VAR) else: class_name = '"{}"'.format(type.name) actual_type_name = '"{}"'.format(actual.type.name) self.fail( message_registry.INCOMPATIBLE_TYPEVAR_VALUE.format( arg_name, class_name, actual_type_name), context, code=codes.TYPE_VAR)
def make_simplified_union(items: Sequence[Type], line: int = -1, column: int = -1, *, keep_erased: bool = False, contract_literals: bool = True) -> ProperType: """Build union type with redundant union items removed. If only a single item remains, this may return a non-union type. Examples: * [int, str] -> Union[int, str] * [int, object] -> object * [int, int] -> int * [int, Any] -> Union[int, Any] (Any types are not simplified away!) * [Any, Any] -> Any Note: This must NOT be used during semantic analysis, since TypeInfos may not be fully initialized. The keep_erased flag is used for type inference against union types containing type variables. If set to True, keep all ErasedType items. """ items = get_proper_types(items) while any(isinstance(typ, UnionType) for typ in items): all_items: List[ProperType] = [] for typ in items: if isinstance(typ, UnionType): all_items.extend(get_proper_types(typ.items)) else: all_items.append(typ) items = all_items from mypy.subtypes import is_proper_subtype removed: Set[int] = set() seen: Set[Tuple[str, str]] = set() # NB: having a separate fast path for Union of Literal and slow path for other things # would arguably be cleaner, however it breaks down when simplifying the Union of two # different enum types as try_expanding_enum_to_union works recursively and will # trigger intermediate simplifications that would render the fast path useless for i, item in enumerate(items): if i in removed: continue # Avoid slow nested for loop for Union of Literal of strings/enums (issue #9169) if is_simple_literal(item): assert isinstance(item, LiteralType) assert isinstance(item.value, str) k = (item.value, item.fallback.type.fullname) if k in seen: removed.add(i) continue # NB: one would naively expect that it would be safe to skip the slow path # always for literals. One would be sorely mistaken. Indeed, some simplifications # such as that of None/Optional when strict optional is false, do require that we # proceed with the slow path. Thankfully, all literals will have the same subtype # relationship to non-literal types, so we only need to do that walk for the first # literal, which keeps the fast path fast even in the presence of a mixture of # literals and other types. safe_skip = len(seen) > 0 seen.add(k) if safe_skip: continue # Keep track of the truishness info for deleted subtypes which can be relevant cbt = cbf = False for j, tj in enumerate(items): # NB: we don't need to check literals as the fast path above takes care of that if (i != j and not is_simple_literal(tj) and is_proper_subtype( tj, item, keep_erased_types=keep_erased) and is_redundant_literal_instance(item, tj) # XXX? ): # We found a redundant item in the union. removed.add(j) cbt = cbt or tj.can_be_true cbf = cbf or tj.can_be_false # if deleted subtypes had more general truthiness, use that if not item.can_be_true and cbt: items[i] = true_or_false(item) elif not item.can_be_false and cbf: items[i] = true_or_false(item) simplified_set = [items[i] for i in range(len(items)) if i not in removed] # If more than one literal exists in the union, try to simplify if (contract_literals and sum(isinstance(item, LiteralType) for item in simplified_set) > 1): simplified_set = try_contracting_literals_in_union(simplified_set) return UnionType.make_union(simplified_set, line, column)
def typed_dict_mapping_overlap( left: ProperType, right: ProperType, overlapping: Callable[[Type, Type], bool]) -> bool: """Check if a TypedDict type is overlapping with a Mapping. The basic logic here consists of two rules: * A TypedDict with some required keys is overlapping with Mapping[str, <some type>] if and only if every key type is overlapping with <some type>. For example: - TypedDict(x=int, y=str) overlaps with Dict[str, Union[str, int]] - TypedDict(x=int, y=str) doesn't overlap with Dict[str, int] Note that any additional non-required keys can't change the above result. * A TypedDict with no required keys overlaps with Mapping[str, <some type>] if and only if at least one of key types overlaps with <some type>. For example: - TypedDict(x=str, y=str, total=False) overlaps with Dict[str, str] - TypedDict(x=str, y=str, total=False) doesn't overlap with Dict[str, int] - TypedDict(x=int, y=str, total=False) overlaps with Dict[str, str] As usual empty, dictionaries lie in a gray area. In general, List[str] and List[str] are considered non-overlapping despite empty list belongs to both. However, List[int] and List[<nothing>] are considered overlapping. So here we follow the same logic: a TypedDict with no required keys is considered non-overlapping with Mapping[str, <some type>], but is considered overlapping with Mapping[<nothing>, <nothing>]. This way we avoid false positives for overloads, and also avoid false positives for comparisons like SomeTypedDict == {} under --strict-equality. """ assert not isinstance(left, TypedDictType) or not isinstance( right, TypedDictType) if isinstance(left, TypedDictType): assert isinstance(right, Instance) typed, other = left, right else: assert isinstance(left, Instance) assert isinstance(right, TypedDictType) typed, other = right, left mapping = next(base for base in other.type.mro if base.fullname() == 'typing.Mapping') other = map_instance_to_supertype(other, mapping) key_type, value_type = get_proper_types(other.args) # TODO: is there a cleaner way to get str_type here? fallback = typed.as_anonymous().fallback str_type = fallback.type.bases[0].args[ 0] # typing._TypedDict inherits Mapping[str, object] # Special case: a TypedDict with no required keys overlaps with an empty dict. if isinstance(key_type, UninhabitedType) and isinstance( value_type, UninhabitedType): return not typed.required_keys if typed.required_keys: if not overlapping(key_type, str_type): return False return all( overlapping(typed.items[k], value_type) for k in typed.required_keys) else: if not overlapping(key_type, str_type): return False non_required = set(typed.items.keys()) - typed.required_keys return any( overlapping(typed.items[k], value_type) for k in non_required)
def is_none_typevar_overlap(t1: Type, t2: Type) -> bool: t1, t2 = get_proper_types((t1, t2)) return isinstance(t1, NoneType) and isinstance(t2, TypeVarType)
def is_overlapping_types(left: Type, right: Type, ignore_promotions: bool = False, prohibit_none_typevar_overlap: bool = False) -> bool: """Can a value of type 'left' also be of type 'right' or vice-versa? If 'ignore_promotions' is True, we ignore promotions while checking for overlaps. If 'prohibit_none_typevar_overlap' is True, we disallow None from overlapping with TypeVars (in both strict-optional and non-strict-optional mode). """ left, right = get_proper_types((left, right)) def _is_overlapping_types(left: Type, right: Type) -> bool: '''Encode the kind of overlapping check to perform. This function mostly exists so we don't have to repeat keyword arguments everywhere.''' return is_overlapping_types( left, right, ignore_promotions=ignore_promotions, prohibit_none_typevar_overlap=prohibit_none_typevar_overlap) # We should never encounter this type. if isinstance(left, PartialType) or isinstance(right, PartialType): assert False, "Unexpectedly encountered partial type" # We should also never encounter these types, but it's possible a few # have snuck through due to unrelated bugs. For now, we handle these # in the same way we handle 'Any'. # # TODO: Replace these with an 'assert False' once we are more confident. illegal_types = (UnboundType, ErasedType, DeletedType) if isinstance(left, illegal_types) or isinstance(right, illegal_types): return True # When running under non-strict optional mode, simplify away types of # the form 'Union[A, B, C, None]' into just 'Union[A, B, C]'. if not state.strict_optional: if isinstance(left, UnionType): left = UnionType.make_union(left.relevant_items()) if isinstance(right, UnionType): right = UnionType.make_union(right.relevant_items()) left, right = get_proper_types((left, right)) # 'Any' may or may not be overlapping with the other type if isinstance(left, AnyType) or isinstance(right, AnyType): return True # We check for complete overlaps next as a general-purpose failsafe. # If this check fails, we start checking to see if there exists a # *partial* overlap between types. # # These checks will also handle the NoneType and UninhabitedType cases for us. if (is_proper_subtype(left, right, ignore_promotions=ignore_promotions) or is_proper_subtype( right, left, ignore_promotions=ignore_promotions)): return True # See the docstring for 'get_possible_variants' for more info on what the # following lines are doing. left_possible = get_possible_variants(left) right_possible = get_possible_variants(right) # We start by checking multi-variant types like Unions first. We also perform # the same logic if either type happens to be a TypeVar. # # Handling the TypeVars now lets us simulate having them bind to the corresponding # type -- if we deferred these checks, the "return-early" logic of the other # checks will prevent us from detecting certain overlaps. # # If both types are singleton variants (and are not TypeVars), we've hit the base case: # we skip these checks to avoid infinitely recursing. def is_none_typevar_overlap(t1: Type, t2: Type) -> bool: t1, t2 = get_proper_types((t1, t2)) return isinstance(t1, NoneType) and isinstance(t2, TypeVarType) if prohibit_none_typevar_overlap: if is_none_typevar_overlap(left, right) or is_none_typevar_overlap( right, left): return False if (len(left_possible) > 1 or len(right_possible) > 1 or isinstance(left, TypeVarType) or isinstance(right, TypeVarType)): for l in left_possible: for r in right_possible: if _is_overlapping_types(l, r): return True return False # Now that we've finished handling TypeVars, we're free to end early # if one one of the types is None and we're running in strict-optional mode. # (None only overlaps with None in strict-optional mode). # # We must perform this check after the TypeVar checks because # a TypeVar could be bound to None, for example. if state.strict_optional and isinstance(left, NoneType) != isinstance( right, NoneType): return False # Next, we handle single-variant types that may be inherently partially overlapping: # # - TypedDicts # - Tuples # # If we cannot identify a partial overlap and end early, we degrade these two types # into their 'Instance' fallbacks. if isinstance(left, TypedDictType) and isinstance(right, TypedDictType): return are_typed_dicts_overlapping(left, right, ignore_promotions=ignore_promotions) elif typed_dict_mapping_pair(left, right): # Overlaps between TypedDicts and Mappings require dedicated logic. return typed_dict_mapping_overlap(left, right, overlapping=_is_overlapping_types) elif isinstance(left, TypedDictType): left = left.fallback elif isinstance(right, TypedDictType): right = right.fallback if is_tuple(left) and is_tuple(right): return are_tuples_overlapping(left, right, ignore_promotions=ignore_promotions) elif isinstance(left, TupleType): left = tuple_fallback(left) elif isinstance(right, TupleType): right = tuple_fallback(right) # Next, we handle single-variant types that cannot be inherently partially overlapping, # but do require custom logic to inspect. # # As before, we degrade into 'Instance' whenever possible. if isinstance(left, TypeType) and isinstance(right, TypeType): return _is_overlapping_types(left.item, right.item) def _type_object_overlap(left: Type, right: Type) -> bool: """Special cases for type object types overlaps.""" # TODO: these checks are a bit in gray area, adjust if they cause problems. left, right = get_proper_types((left, right)) # 1. Type[C] vs Callable[..., C], where the latter is class object. if isinstance(left, TypeType) and isinstance( right, CallableType) and right.is_type_obj(): return _is_overlapping_types(left.item, right.ret_type) # 2. Type[C] vs Meta, where Meta is a metaclass for C. if isinstance(left, TypeType) and isinstance(right, Instance): if isinstance(left.item, Instance): left_meta = left.item.type.metaclass_type if left_meta is not None: return _is_overlapping_types(left_meta, right) # builtins.type (default metaclass) overlaps with all metaclasses return right.type.has_base('builtins.type') elif isinstance(left.item, AnyType): return right.type.has_base('builtins.type') # 3. Callable[..., C] vs Meta is considered below, when we switch to fallbacks. return False if isinstance(left, TypeType) or isinstance(right, TypeType): return _type_object_overlap(left, right) or _type_object_overlap( right, left) if isinstance(left, CallableType) and isinstance(right, CallableType): return is_callable_compatible(left, right, is_compat=_is_overlapping_types, ignore_pos_arg_names=True, allow_partial_overlap=True) elif isinstance(left, CallableType): left = left.fallback elif isinstance(right, CallableType): right = right.fallback if isinstance(left, LiteralType) and isinstance(right, LiteralType): if left.value == right.value: # If values are the same, we still need to check if fallbacks are overlapping, # this is done below. left = left.fallback right = right.fallback else: return False elif isinstance(left, LiteralType): left = left.fallback elif isinstance(right, LiteralType): right = right.fallback # Finally, we handle the case where left and right are instances. if isinstance(left, Instance) and isinstance(right, Instance): # First we need to handle promotions and structural compatibility for instances # that came as fallbacks, so simply call is_subtype() to avoid code duplication. if (is_subtype(left, right, ignore_promotions=ignore_promotions) or is_subtype(right, left, ignore_promotions=ignore_promotions)): return True # Two unrelated types cannot be partially overlapping: they're disjoint. if left.type.has_base(right.type.fullname): left = map_instance_to_supertype(left, right.type) elif right.type.has_base(left.type.fullname): right = map_instance_to_supertype(right, left.type) else: return False if len(left.args) == len(right.args): # Note: we don't really care about variance here, since the overlapping check # is symmetric and since we want to return 'True' even for partial overlaps. # # For example, suppose we have two types Wrapper[Parent] and Wrapper[Child]. # It doesn't matter whether Wrapper is covariant or contravariant since # either way, one of the two types will overlap with the other. # # Similarly, if Wrapper was invariant, the two types could still be partially # overlapping -- what if Wrapper[Parent] happened to contain only instances of # specifically Child? # # Or, to use a more concrete example, List[Union[A, B]] and List[Union[B, C]] # would be considered partially overlapping since it's possible for both lists # to contain only instances of B at runtime. if all( _is_overlapping_types(left_arg, right_arg) for left_arg, right_arg in zip(left.args, right.args)): return True return False # We ought to have handled every case by now: we conclude the # two types are not overlapping, either completely or partially. # # Note: it's unclear however, whether returning False is the right thing # to do when inferring reachability -- see https://github.com/python/mypy/issues/5529 assert type(left) != type(right) return False
def callable_has_any(t: CallableType) -> int: # We count a bare None in argument position as Any, since # pyannotate turns it into Optional[Any] return (any( isinstance(at, NoneType) for at in get_proper_types(t.arg_types)) or has_any_type(t))
def make_simplified_union(items: Sequence[Type], line: int = -1, column: int = -1, *, keep_erased: bool = False, contract_literals: bool = True) -> ProperType: """Build union type with redundant union items removed. If only a single item remains, this may return a non-union type. Examples: * [int, str] -> Union[int, str] * [int, object] -> object * [int, int] -> int * [int, Any] -> Union[int, Any] (Any types are not simplified away!) * [Any, Any] -> Any Note: This must NOT be used during semantic analysis, since TypeInfos may not be fully initialized. The keep_erased flag is used for type inference against union types containing type variables. If set to True, keep all ErasedType items. """ items = get_proper_types(items) while any(isinstance(typ, UnionType) for typ in items): all_items = [] # type: List[ProperType] for typ in items: if isinstance(typ, UnionType): all_items.extend(get_proper_types(typ.items)) else: all_items.append(typ) items = all_items from mypy.subtypes import is_proper_subtype removed = set() # type: Set[int] # Avoid slow nested for loop for Union of Literal of strings (issue #9169) if all((isinstance(item, LiteralType) and item.fallback.type.fullname == 'builtins.str') for item in items): seen = set() # type: Set[str] for index, item in enumerate(items): assert isinstance(item, LiteralType) assert isinstance(item.value, str) if item.value in seen: removed.add(index) seen.add(item.value) else: for i, ti in enumerate(items): if i in removed: continue # Keep track of the truishness info for deleted subtypes which can be relevant cbt = cbf = False for j, tj in enumerate(items): if i != j and is_proper_subtype(tj, ti, keep_erased_types=keep_erased) and \ is_redundant_literal_instance(ti, tj): # We found a redundant item in the union. removed.add(j) cbt = cbt or tj.can_be_true cbf = cbf or tj.can_be_false # if deleted subtypes had more general truthiness, use that if not ti.can_be_true and cbt: items[i] = true_or_false(ti) elif not ti.can_be_false and cbf: items[i] = true_or_false(ti) simplified_set = [items[i] for i in range(len(items)) if i not in removed] # If more than one literal exists in the union, try to simplify if (contract_literals and sum(isinstance(item, LiteralType) for item in simplified_set) > 1): simplified_set = try_contracting_literals_in_union(simplified_set) return UnionType.make_union(simplified_set, line, column)
def apply_generic_arguments(callable: CallableType, orig_types: Sequence[Optional[Type]], report_incompatible_typevar_value: Callable[ [CallableType, Type, str, Context], None], context: Context, skip_unsatisfied: bool = False) -> CallableType: """Apply generic type arguments to a callable type. For example, applying [int] to 'def [T] (T) -> T' results in 'def (int) -> int'. Note that each type can be None; in this case, it will not be applied. If `skip_unsatisfied` is True, then just skip the types that don't satisfy type variable bound or constraints, instead of giving an error. """ tvars = callable.variables assert len(tvars) == len(orig_types) # Check that inferred type variable values are compatible with allowed # values and bounds. Also, promote subtype values to allowed values. types = get_proper_types(orig_types) for i, type in enumerate(types): assert not isinstance( type, PartialType), "Internal error: must never apply partial type" values = get_proper_types(callable.variables[i].values) if type is None: continue if values: if isinstance(type, AnyType): continue if isinstance(type, TypeVarType) and type.values: # Allow substituting T1 for T if every allowed value of T1 # is also a legal value of T. if all( any( mypy.sametypes.is_same_type(v, v1) for v in values) for v1 in type.values): continue matching = [] for value in values: if mypy.subtypes.is_subtype(type, value): matching.append(value) if matching: best = matching[0] # If there are more than one matching value, we select the narrowest for match in matching[1:]: if mypy.subtypes.is_subtype(match, best): best = match types[i] = best else: if skip_unsatisfied: types[i] = None else: report_incompatible_typevar_value( callable, type, callable.variables[i].name, context) else: upper_bound = callable.variables[i].upper_bound if not mypy.subtypes.is_subtype(type, upper_bound): if skip_unsatisfied: types[i] = None else: report_incompatible_typevar_value( callable, type, callable.variables[i].name, context) # Create a map from type variable id to target type. id_to_type = {} # type: Dict[TypeVarId, Type] for i, tv in enumerate(tvars): typ = types[i] if typ: id_to_type[tv.id] = typ # Apply arguments to argument types. arg_types = [expand_type(at, id_to_type) for at in callable.arg_types] # The callable may retain some type vars if only some were applied. remaining_tvars = [tv for tv in tvars if tv.id not in id_to_type] return callable.copy_modified( arg_types=arg_types, ret_type=expand_type(callable.ret_type, id_to_type), variables=remaining_tvars, )
def takes_callable_and_args_callback(ctx: FunctionContext) -> Type: """Automate the boilerplate for writing functions that accept arbitrary positional arguments of the same type accepted by a callable. For example, this supports writing:: @trio_typing.takes_callable_and_args def start_soon( self, async_fn: Callable[[VarArg()], Any], *args: Any, ) -> None: ... instead of:: T1 = TypeVar("T1") T2 = TypeVar("T2") T3 = TypeVar("T3") T4 = TypeVar("T4") @overload def start_soon( self, async_fn: Callable[[], Any], ) -> None: ... @overload def start_soon( self, async_fn: Callable[[T1], Any], __arg1: T1, ) -> None: ... @overload def start_soon( self, async_fn: Callable[[T1, T2], Any], __arg1: T1, __arg2: T2 ) -> None: ... # etc """ try: if not ctx.arg_types or len(ctx.arg_types[0]) != 1: raise ValueError("must be used as a decorator") fn_type = get_proper_type(ctx.arg_types[0][0]) if not isinstance(fn_type, CallableType) or not isinstance( get_proper_type(ctx.default_return_type), CallableType): raise ValueError("must be used as a decorator") callable_idx = -1 # index in function arguments of the callable callable_args_idx = -1 # index in callable arguments of the StarArgs callable_ty = None # type: Optional[CallableType] args_idx = -1 # index in function arguments of the StarArgs for idx, (kind, ty) in enumerate(zip(fn_type.arg_kinds, fn_type.arg_types)): ty = get_proper_type(ty) if isinstance(ty, AnyType) and kind == ARG_STAR: assert args_idx == -1 args_idx = idx elif isinstance(ty, (UnionType, CallableType)) and kind == ARG_POS: # turn Union[Callable[..., T], Callable[[VarArg()], T]] # into Callable[[VarArg()], T] # (the union makes it not fail when the plugin is not being used) if isinstance(ty, UnionType): for arm in get_proper_types(ty.items): if (isinstance(arm, CallableType) and not arm.is_ellipsis_args and any(kind_ == ARG_STAR for kind_ in arm.arg_kinds)): ty = arm break else: continue for idx_, (kind_, ty_) in enumerate(zip(ty.arg_kinds, ty.arg_types)): ty_ = get_proper_type(ty_) if isinstance(ty_, AnyType) and kind_ == ARG_STAR: if callable_idx != -1: raise ValueError( "decorated function may only take one callable " "that has an argument of type VarArg()") callable_idx = idx callable_args_idx = idx_ callable_ty = ty if args_idx == -1: raise ValueError("decorated function must take *args: Any") if callable_idx == -1 or callable_ty is None: raise ValueError( "decorated function must take a callable that has a " "argument of type mypy_extensions.VarArg()") expanded_fns = [] # type: List[CallableType] type_var_defs = [] # type: List[TypeVarDef] type_var_types = [] # type: List[Type] for arg_idx in range( 1, 7): # provides overloads for 0 through 5 arguments arg_types = list(fn_type.arg_types) arg_types[callable_idx] = callable_ty.copy_modified( arg_types=(callable_ty.arg_types[:callable_args_idx] + type_var_types + callable_ty.arg_types[callable_args_idx + 1:]), arg_kinds=(callable_ty.arg_kinds[:callable_args_idx] + ([ARG_POS] * len(type_var_types)) + callable_ty.arg_kinds[callable_args_idx + 1:]), arg_names=(callable_ty.arg_names[:callable_args_idx] + ([None] * len(type_var_types)) + callable_ty.arg_names[callable_args_idx + 1:]), variables=(callable_ty.variables + type_var_defs), ) expanded_fns.append( fn_type.copy_modified( arg_types=(arg_types[:args_idx] + type_var_types + arg_types[args_idx + 1:]), arg_kinds=(fn_type.arg_kinds[:args_idx] + ([ARG_POS] * len(type_var_types)) + fn_type.arg_kinds[args_idx + 1:]), arg_names=(fn_type.arg_names[:args_idx] + ([None] * len(type_var_types)) + fn_type.arg_names[args_idx + 1:]), variables=(fn_type.variables + type_var_defs), )) type_var_defs.append( TypeVarDef( "__T{}".format(arg_idx), "__T{}".format(arg_idx), -len(fn_type.variables) - arg_idx - 1, [], ctx.api.named_generic_type("builtins.object", []), )) type_var_types.append( TypeVarType(type_var_defs[-1], ctx.context.line, ctx.context.column)) return Overloaded(expanded_fns) except ValueError as ex: ctx.api.fail("invalid use of @takes_callable_and_args: {}".format(ex), ctx.context) return ctx.default_return_type