def make_simplified_union(items: Sequence[Type],
line: int = -1,
column: int = -1,
*,
keep_erased: bool = False) -> ProperType:
"""Build union type with redundant union items removed.
If only a single item remains, this may return a non-union type.
Examples:
* [int, str] -> Union[int, str]
* [int, object] -> object
* [int, int] -> int
* [int, Any] -> Union[int, Any] (Any types are not simplified away!)
* [Any, Any] -> Any
Note: This must NOT be used during semantic analysis, since TypeInfos may not
be fully initialized.
The keep_erased flag is used for type inference against union types
containing type variables. If set to True, keep all ErasedType items.
"""
items = get_proper_types(items)
while any(isinstance(typ, UnionType) for typ in items):
all_items = [] # type: List[ProperType]
for typ in items:
if isinstance(typ, UnionType):
all_items.extend(get_proper_types(typ.items))
else:
all_items.append(typ)
items = all_items
from mypy.subtypes import is_proper_subtype
removed = set() # type: Set[int]
for i, ti in enumerate(items):
if i in removed: continue
# Keep track of the truishness info for deleted subtypes which can be relevant
cbt = cbf = False
for j, tj in enumerate(items):
if i != j and is_proper_subtype(
tj, ti, keep_erased_types=keep_erased):
# We found a redundant item in the union.
removed.add(j)
cbt = cbt or tj.can_be_true
cbf = cbf or tj.can_be_false
# if deleted subtypes had more general truthiness, use that
if not ti.can_be_true and cbt:
items[i] = true_or_false(ti)
elif not ti.can_be_false and cbf:
items[i] = true_or_false(ti)
simplified_set = [items[i] for i in range(len(items)) if i not in removed]
return UnionType.make_union(simplified_set, line, column)
def make_simplified_union(items: Sequence[Type],
line: int = -1,
column: int = -1,
*,
keep_erased: bool = False,
contract_literals: bool = True) -> ProperType:
"""Build union type with redundant union items removed.
If only a single item remains, this may return a non-union type.
Examples:
* [int, str] -> Union[int, str]
* [int, object] -> object
* [int, int] -> int
* [int, Any] -> Union[int, Any] (Any types are not simplified away!)
* [Any, Any] -> Any
Note: This must NOT be used during semantic analysis, since TypeInfos may not
be fully initialized.
The keep_erased flag is used for type inference against union types
containing type variables. If set to True, keep all ErasedType items.
The contract_literals flag indicates whether we need to contract literal types
back into a sum type. Set it to False when called by try_expanding_sum_type_
to_union().
"""
items = get_proper_types(items)
# Step 1: expand all nested unions
while any(isinstance(typ, UnionType) for typ in items):
all_items: List[ProperType] = []
for typ in items:
if isinstance(typ, UnionType):
all_items.extend(get_proper_types(typ.items))
else:
all_items.append(typ)
items = all_items
# Step 2: remove redundant unions
simplified_set = _remove_redundant_union_items(items, keep_erased)
# Step 3: If more than one literal exists in the union, try to simplify
if contract_literals and sum(
isinstance(item, LiteralType) for item in simplified_set) > 1:
simplified_set = try_contracting_literals_in_union(simplified_set)
return UnionType.make_union(simplified_set, line, column)
def is_special_target(right: ProperType) -> bool:
"""Whitelist some special cases for use in isinstance() with improper types."""
if isinstance(right, CallableType) and right.is_type_obj():
if right.type_object().fullname() == 'builtins.tuple':
# Used with Union[Type, Tuple[Type, ...]].
return True
if right.type_object().fullname() in ('mypy.types.Type',
'mypy.types.ProperType',
'mypy.types.TypeAliasType'):
# Special case: things like assert isinstance(typ, ProperType) are always OK.
return True
if right.type_object().fullname() in ('mypy.types.UnboundType',
'mypy.types.TypeVarType',
'mypy.types.RawExpressionType',
'mypy.types.EllipsisType',
'mypy.types.StarType',
'mypy.types.TypeList',
'mypy.types.CallableArgument',
'mypy.types.PartialType',
'mypy.types.ErasedType'):
# Special case: these are not valid targets for a type alias and thus safe.
# TODO: introduce a SyntheticType base to simplify this?
return True
elif isinstance(right, TupleType):
return all(is_special_target(t) for t in get_proper_types(right.items))
return False
def try_getting_str_literals(expr: Expression,
typ: Type) -> Optional[List[str]]:
"""If the given expression or type corresponds to a string literal
or a union of string literals, returns a list of the underlying strings.
Otherwise, returns None.
Specifically, this function is guaranteed to return a list with
one or more strings if one one the following is true:
1. 'expr' is a StrExpr
2. 'typ' is a LiteralType containing a string
3. 'typ' is a UnionType containing only LiteralType of strings
"""
typ = get_proper_type(typ)
if isinstance(expr, StrExpr):
return [expr.value]
if isinstance(typ, Instance) and typ.last_known_value is not None:
possible_literals = [typ.last_known_value] # type: List[Type]
elif isinstance(typ, UnionType):
possible_literals = list(typ.items)
else:
possible_literals = [typ]
strings = []
for lit in get_proper_types(possible_literals):
if isinstance(lit, LiteralType) and lit.fallback.type.fullname(
) == 'builtins.str':
val = lit.value
assert isinstance(val, str)
strings.append(val)
else:
return None
return strings
def visit_callable_type(self, t: CallableType) -> ProperType:
if isinstance(self.s, CallableType) and is_similar_callables(
t, self.s):
if is_equivalent(t, self.s):
return combine_similar_callables(t, self.s)
result = join_similar_callables(t, self.s)
# We set the from_type_type flag to suppress error when a collection of
# concrete class objects gets inferred as their common abstract superclass.
if not (
(t.is_type_obj() and t.type_object().is_abstract) or
(self.s.is_type_obj() and self.s.type_object().is_abstract)):
result.from_type_type = True
if any(
isinstance(tp, (NoneType, UninhabitedType))
for tp in get_proper_types(result.arg_types)):
# We don't want to return unusable Callable, attempt fallback instead.
return join_types(t.fallback, self.s)
return result
elif isinstance(self.s, Overloaded):
# Switch the order of arguments to that we'll get to visit_overloaded.
return join_types(t, self.s)
elif isinstance(self.s, Instance) and self.s.type.is_protocol:
call = unpack_callback_protocol(self.s)
if call:
return join_types(t, call)
return join_types(t.fallback, self.s)
def visit_union_type(self, t: UnionType) -> Type:
new = cast(UnionType, super().visit_union_type(t))
# Erasure can result in many duplicate items; merge them.
# Call make_simplified_union only on lists of instance types
# that all have the same fullname, to avoid simplifying too
# much.
instances = [item for item in new.items
if isinstance(get_proper_type(item), Instance)]
# Avoid merge in simple cases such as optional types.
if len(instances) > 1:
instances_by_name: Dict[str, List[Instance]] = {}
new_items = get_proper_types(new.items)
for item in new_items:
if isinstance(item, Instance) and not item.args:
instances_by_name.setdefault(item.type.fullname, []).append(item)
merged: List[Type] = []
for item in new_items:
if isinstance(item, Instance) and not item.args:
types = instances_by_name.get(item.type.fullname)
if types is not None:
if len(types) == 1:
merged.append(item)
else:
from mypy.typeops import make_simplified_union
merged.append(make_simplified_union(types))
del instances_by_name[item.type.fullname]
else:
merged.append(item)
return UnionType.make_union(merged)
return new
def narrow_declared_type(declared: Type, narrowed: Type) -> Type:
"""Return the declared type narrowed down to another type."""
# TODO: check infinite recursion for aliases here.
declared = get_proper_type(declared)
narrowed = get_proper_type(narrowed)
if declared == narrowed:
return declared
if isinstance(declared, UnionType):
return make_simplified_union([narrow_declared_type(x, narrowed)
for x in declared.relevant_items()])
elif not is_overlapping_types(declared, narrowed,
prohibit_none_typevar_overlap=True):
if state.strict_optional:
return UninhabitedType()
else:
return NoneType()
elif isinstance(narrowed, UnionType):
return make_simplified_union([narrow_declared_type(declared, x)
for x in narrowed.relevant_items()])
elif isinstance(narrowed, AnyType):
return narrowed
elif isinstance(declared, TypeType) and isinstance(narrowed, TypeType):
return TypeType.make_normalized(narrow_declared_type(declared.item, narrowed.item))
elif isinstance(declared, (Instance, TupleType, TypeType, LiteralType)):
return meet_types(declared, narrowed)
elif isinstance(declared, TypedDictType) and isinstance(narrowed, Instance):
# Special case useful for selecting TypedDicts from unions using isinstance(x, dict).
if (narrowed.type.fullname == 'builtins.dict' and
all(isinstance(t, AnyType) for t in get_proper_types(narrowed.args))):
return declared
return meet_types(declared, narrowed)
return narrowed
def try_getting_literals_from_type(typ: Type,
target_literal_type: TypingType[T],
target_fullname: str) -> Optional[List[T]]:
"""If the given expression or type corresponds to a Literal or
union of Literals where the underlying values corresponds to the given
target type, returns a list of those underlying values. Otherwise,
returns None.
"""
typ = get_proper_type(typ)
if isinstance(typ, Instance) and typ.last_known_value is not None:
possible_literals: List[Type] = [typ.last_known_value]
elif isinstance(typ, UnionType):
possible_literals = list(typ.items)
else:
possible_literals = [typ]
literals: List[T] = []
for lit in get_proper_types(possible_literals):
if isinstance(
lit,
LiteralType) and lit.fallback.type.fullname == target_fullname:
val = lit.value
if isinstance(val, target_literal_type):
literals.append(val)
else:
return None
else:
return None
return literals
def check_specs_in_format_call(self, call: CallExpr,
specs: List[ConversionSpecifier], format_value: str) -> None:
"""Perform pairwise checks for conversion specifiers vs their replacements.
The core logic for format checking is implemented in this method.
"""
assert all(s.key for s in specs), "Keys must be auto-generated first!"
replacements = self.find_replacements_in_call(call, [cast(str, s.key) for s in specs])
assert len(replacements) == len(specs)
for spec, repl in zip(specs, replacements):
repl = self.apply_field_accessors(spec, repl, ctx=call)
actual_type = repl.type if isinstance(repl, TempNode) else self.chk.type_map.get(repl)
assert actual_type is not None
# Special case custom formatting.
if (spec.format_spec and spec.non_standard_format_spec and
# Exclude "dynamic" specifiers (i.e. containing nested formatting).
not ('{' in spec.format_spec or '}' in spec.format_spec)):
if (not custom_special_method(actual_type, '__format__', check_all=True) or
spec.conversion):
# TODO: add support for some custom specs like datetime?
self.msg.fail('Unrecognized format'
' specification "{}"'.format(spec.format_spec[1:]),
call, code=codes.STRING_FORMATTING)
continue
# Adjust expected and actual types.
if not spec.type:
expected_type = AnyType(TypeOfAny.special_form) # type: Optional[Type]
else:
assert isinstance(call.callee, MemberExpr)
if isinstance(call.callee.expr, (StrExpr, UnicodeExpr)):
format_str = call.callee.expr
else:
format_str = StrExpr(format_value)
expected_type = self.conversion_type(spec.type, call, format_str,
format_call=True)
if spec.conversion is not None:
# If the explicit conversion is given, then explicit conversion is called _first_.
if spec.conversion[1] not in 'rsa':
self.msg.fail('Invalid conversion type "{}",'
' must be one of "r", "s" or "a"'.format(spec.conversion[1]),
call, code=codes.STRING_FORMATTING)
actual_type = self.named_type('builtins.str')
# Perform the checks for given types.
if expected_type is None:
continue
a_type = get_proper_type(actual_type)
actual_items = (get_proper_types(a_type.items) if isinstance(a_type, UnionType)
else [a_type])
for a_type in actual_items:
if custom_special_method(a_type, '__format__'):
continue
self.check_placeholder_type(a_type, expected_type, call)
self.perform_special_format_checks(spec, call, repl, a_type, expected_type)
def type(self, t: Optional[Type]) -> None:
t = get_proper_type(t)
if not t:
# If an expression does not have a type, it is often due to dead code.
# Don't count these because there can be an unanalyzed value on a line with other
# analyzed expressions, which overwrite the TYPE_UNANALYZED.
self.record_line(self.line, TYPE_UNANALYZED)
return
if isinstance(t, AnyType) and is_special_form_any(t):
# TODO: What if there is an error in special form definition?
self.record_line(self.line, TYPE_PRECISE)
return
if isinstance(t, AnyType):
self.log(' !! Any type around line %d' % self.line)
self.num_any_exprs += 1
self.record_line(self.line, TYPE_ANY)
elif ((not self.all_nodes and is_imprecise(t))
or (self.all_nodes and is_imprecise2(t))):
self.log(' !! Imprecise type around line %d' % self.line)
self.num_imprecise_exprs += 1
self.record_line(self.line, TYPE_IMPRECISE)
else:
self.num_precise_exprs += 1
self.record_line(self.line, TYPE_PRECISE)
for typ in get_proper_types(collect_all_inner_types(t)) + [t]:
if isinstance(typ, AnyType):
typ = get_original_any(typ)
if is_special_form_any(typ):
continue
self.type_of_any_counter[typ.type_of_any] += 1
self.num_any_types += 1
if self.line in self.any_line_map:
self.any_line_map[self.line].append(typ)
else:
self.any_line_map[self.line] = [typ]
elif isinstance(typ, Instance):
if typ.args:
if any(is_complex(arg) for arg in typ.args):
self.num_complex_types += 1
else:
self.num_generic_types += 1
else:
self.num_simple_types += 1
elif isinstance(typ, FunctionLike):
self.num_function_types += 1
elif isinstance(typ, TupleType):
if any(is_complex(item) for item in typ.items):
self.num_complex_types += 1
else:
self.num_tuple_types += 1
elif isinstance(typ, TypeVarType):
self.num_typevar_types += 1
def apply_generic_arguments(
callable: CallableType, orig_types: Sequence[Optional[Type]],
report_incompatible_typevar_value: Callable[[CallableType, Type, str, Context], None],
context: Context,
skip_unsatisfied: bool = False) -> CallableType:
"""Apply generic type arguments to a callable type.
For example, applying [int] to 'def [T] (T) -> T' results in
'def (int) -> int'.
Note that each type can be None; in this case, it will not be applied.
If `skip_unsatisfied` is True, then just skip the types that don't satisfy type variable
bound or constraints, instead of giving an error.
"""
tvars = callable.variables
assert len(tvars) == len(orig_types)
# Check that inferred type variable values are compatible with allowed
# values and bounds. Also, promote subtype values to allowed values.
types = get_proper_types(orig_types)
# Create a map from type variable id to target type.
id_to_type: Dict[TypeVarId, Type] = {}
for tvar, type in zip(tvars, types):
assert not isinstance(type, PartialType), "Internal error: must never apply partial type"
if type is None:
continue
target_type = get_target_type(
tvar, type, callable, report_incompatible_typevar_value, context, skip_unsatisfied
)
if target_type is not None:
id_to_type[tvar.id] = target_type
param_spec = callable.param_spec()
if param_spec is not None:
nt = id_to_type.get(param_spec.id)
if nt is not None:
nt = get_proper_type(nt)
if isinstance(nt, CallableType):
callable = callable.expand_param_spec(nt)
# Apply arguments to argument types.
arg_types = [expand_type(at, id_to_type) for at in callable.arg_types]
# The callable may retain some type vars if only some were applied.
remaining_tvars = [tv for tv in tvars if tv.id not in id_to_type]
return callable.copy_modified(
arg_types=arg_types,
ret_type=expand_type(callable.ret_type, id_to_type),
variables=remaining_tvars,
)
def narrow_declared_type(declared: Type, narrowed: Type) -> Type:
"""Return the declared type narrowed down to another type."""
# TODO: check infinite recursion for aliases here.
if isinstance(narrowed, TypeGuardedType): # type: ignore[misc]
# A type guard forces the new type even if it doesn't overlap the old.
return narrowed.type_guard
declared = get_proper_type(declared)
narrowed = get_proper_type(narrowed)
if declared == narrowed:
return declared
if isinstance(declared, UnionType):
return make_simplified_union([
narrow_declared_type(x, narrowed)
for x in declared.relevant_items()
])
if is_enum_overlapping_union(declared, narrowed):
return narrowed
elif not is_overlapping_types(
declared, narrowed, prohibit_none_typevar_overlap=True):
if state.strict_optional:
return UninhabitedType()
else:
return NoneType()
elif isinstance(narrowed, UnionType):
return make_simplified_union([
narrow_declared_type(declared, x)
for x in narrowed.relevant_items()
])
elif isinstance(narrowed, AnyType):
return narrowed
elif isinstance(narrowed, TypeVarType) and is_subtype(
narrowed.upper_bound, declared):
return narrowed
elif isinstance(declared, TypeType) and isinstance(narrowed, TypeType):
return TypeType.make_normalized(
narrow_declared_type(declared.item, narrowed.item))
elif (isinstance(declared, TypeType) and isinstance(narrowed, Instance)
and narrowed.type.is_metaclass()):
# We'd need intersection types, so give up.
return declared
elif isinstance(declared, (Instance, TupleType, TypeType, LiteralType)):
return meet_types(declared, narrowed)
elif isinstance(declared, TypedDictType) and isinstance(
narrowed, Instance):
# Special case useful for selecting TypedDicts from unions using isinstance(x, dict).
if (narrowed.type.fullname == 'builtins.dict' and all(
isinstance(t, AnyType)
for t in get_proper_types(narrowed.args))):
return declared
return meet_types(declared, narrowed)
return narrowed
def are_tuples_overlapping(left: Type, right: Type, *,
ignore_promotions: bool = False,
prohibit_none_typevar_overlap: bool = False) -> bool:
"""Returns true if left and right are overlapping tuples."""
left, right = get_proper_types((left, right))
left = adjust_tuple(left, right) or left
right = adjust_tuple(right, left) or right
assert isinstance(left, TupleType), 'Type {} is not a tuple'.format(left)
assert isinstance(right, TupleType), 'Type {} is not a tuple'.format(right)
if len(left.items) != len(right.items):
return False
return all(is_overlapping_types(l, r,
ignore_promotions=ignore_promotions,
prohibit_none_typevar_overlap=prohibit_none_typevar_overlap)
for l, r in zip(left.items, right.items))
def visit_callable_type(self, t: CallableType) -> ProperType:
if isinstance(self.s, CallableType) and is_similar_callables(t, self.s):
if is_equivalent(t, self.s):
return combine_similar_callables(t, self.s)
result = join_similar_callables(t, self.s)
if any(isinstance(tp, (NoneType, UninhabitedType))
for tp in get_proper_types(result.arg_types)):
# We don't want to return unusable Callable, attempt fallback instead.
return join_types(t.fallback, self.s)
return result
elif isinstance(self.s, Overloaded):
# Switch the order of arguments to that we'll get to visit_overloaded.
return join_types(t, self.s)
elif isinstance(self.s, Instance) and self.s.type.is_protocol:
call = unpack_callback_protocol(self.s)
if call:
return join_types(t, call)
return join_types(t.fallback, self.s)
def typed_dict_mapping_pair(left: Type, right: Type) -> bool:
"""Is this a pair where one type is a TypedDict and another one is an instance of Mapping?
This case requires a precise/principled consideration because there are two use cases
that push the boundary the opposite ways: we need to avoid spurious overlaps to avoid
false positives for overloads, but we also need to avoid spuriously non-overlapping types
to avoid false positives with --strict-equality.
"""
left, right = get_proper_types((left, right))
assert not isinstance(left, TypedDictType) or not isinstance(right, TypedDictType)
if isinstance(left, TypedDictType):
_, other = left, right
elif isinstance(right, TypedDictType):
_, other = right, left
else:
return False
return isinstance(other, Instance) and other.type.has_base('typing.Mapping')
def get_target_type(tvar: TypeVarLikeType, type: ProperType,
callable: CallableType,
report_incompatible_typevar_value: Callable[
[CallableType, Type, str, Context],
None], context: Context,
skip_unsatisfied: bool) -> Optional[Type]:
if isinstance(tvar, ParamSpecType):
return type
if isinstance(tvar, TypeVarTupleType):
return type
assert isinstance(tvar, TypeVarType)
values = get_proper_types(tvar.values)
if values:
if isinstance(type, AnyType):
return type
if isinstance(type, TypeVarType) and type.values:
# Allow substituting T1 for T if every allowed value of T1
# is also a legal value of T.
if all(
any(mypy.sametypes.is_same_type(v, v1) for v in values)
for v1 in type.values):
return type
matching = []
for value in values:
if mypy.subtypes.is_subtype(type, value):
matching.append(value)
if matching:
best = matching[0]
# If there are more than one matching value, we select the narrowest
for match in matching[1:]:
if mypy.subtypes.is_subtype(match, best):
best = match
return best
if skip_unsatisfied:
return None
report_incompatible_typevar_value(callable, type, tvar.name, context)
else:
upper_bound = tvar.upper_bound
if not mypy.subtypes.is_subtype(type, upper_bound):
if skip_unsatisfied:
return None
report_incompatible_typevar_value(callable, type, tvar.name,
context)
return type
def _type_object_overlap(left: Type, right: Type) -> bool:
"""Special cases for type object types overlaps."""
# TODO: these checks are a bit in gray area, adjust if they cause problems.
left, right = get_proper_types((left, right))
# 1. Type[C] vs Callable[..., C], where the latter is class object.
if isinstance(left, TypeType) and isinstance(right, CallableType) and right.is_type_obj():
return _is_overlapping_types(left.item, right.ret_type)
# 2. Type[C] vs Meta, where Meta is a metaclass for C.
if isinstance(left, TypeType) and isinstance(right, Instance):
if isinstance(left.item, Instance):
left_meta = left.item.type.metaclass_type
if left_meta is not None:
return _is_overlapping_types(left_meta, right)
# builtins.type (default metaclass) overlaps with all metaclasses
return right.type.has_base('builtins.type')
elif isinstance(left.item, AnyType):
return right.type.has_base('builtins.type')
# 3. Callable[..., C] vs Meta is considered below, when we switch to fallbacks.
return False
def check_type_var_values(self, type: TypeInfo, actuals: List[Type],
arg_name: str, valids: List[Type],
arg_number: int, context: Context) -> None:
for actual in get_proper_types(actuals):
if (not isinstance(actual, AnyType) and
not any(is_same_type(actual, value) for value in valids)):
if len(actuals) > 1 or not isinstance(actual, Instance):
self.fail('Invalid type argument value for "{}"'.format(
type.name),
context,
code=codes.TYPE_VAR)
else:
class_name = '"{}"'.format(type.name)
actual_type_name = '"{}"'.format(actual.type.name)
self.fail(
message_registry.INCOMPATIBLE_TYPEVAR_VALUE.format(
arg_name, class_name, actual_type_name),
context,
code=codes.TYPE_VAR)
def make_simplified_union(items: Sequence[Type],
line: int = -1,
column: int = -1,
*,
keep_erased: bool = False,
contract_literals: bool = True) -> ProperType:
"""Build union type with redundant union items removed.
If only a single item remains, this may return a non-union type.
Examples:
* [int, str] -> Union[int, str]
* [int, object] -> object
* [int, int] -> int
* [int, Any] -> Union[int, Any] (Any types are not simplified away!)
* [Any, Any] -> Any
Note: This must NOT be used during semantic analysis, since TypeInfos may not
be fully initialized.
The keep_erased flag is used for type inference against union types
containing type variables. If set to True, keep all ErasedType items.
"""
items = get_proper_types(items)
while any(isinstance(typ, UnionType) for typ in items):
all_items: List[ProperType] = []
for typ in items:
if isinstance(typ, UnionType):
all_items.extend(get_proper_types(typ.items))
else:
all_items.append(typ)
items = all_items
from mypy.subtypes import is_proper_subtype
removed: Set[int] = set()
seen: Set[Tuple[str, str]] = set()
# NB: having a separate fast path for Union of Literal and slow path for other things
# would arguably be cleaner, however it breaks down when simplifying the Union of two
# different enum types as try_expanding_enum_to_union works recursively and will
# trigger intermediate simplifications that would render the fast path useless
for i, item in enumerate(items):
if i in removed:
continue
# Avoid slow nested for loop for Union of Literal of strings/enums (issue #9169)
if is_simple_literal(item):
assert isinstance(item, LiteralType)
assert isinstance(item.value, str)
k = (item.value, item.fallback.type.fullname)
if k in seen:
removed.add(i)
continue
# NB: one would naively expect that it would be safe to skip the slow path
# always for literals. One would be sorely mistaken. Indeed, some simplifications
# such as that of None/Optional when strict optional is false, do require that we
# proceed with the slow path. Thankfully, all literals will have the same subtype
# relationship to non-literal types, so we only need to do that walk for the first
# literal, which keeps the fast path fast even in the presence of a mixture of
# literals and other types.
safe_skip = len(seen) > 0
seen.add(k)
if safe_skip:
continue
# Keep track of the truishness info for deleted subtypes which can be relevant
cbt = cbf = False
for j, tj in enumerate(items):
# NB: we don't need to check literals as the fast path above takes care of that
if (i != j and not is_simple_literal(tj) and is_proper_subtype(
tj, item, keep_erased_types=keep_erased)
and is_redundant_literal_instance(item, tj) # XXX?
):
# We found a redundant item in the union.
removed.add(j)
cbt = cbt or tj.can_be_true
cbf = cbf or tj.can_be_false
# if deleted subtypes had more general truthiness, use that
if not item.can_be_true and cbt:
items[i] = true_or_false(item)
elif not item.can_be_false and cbf:
items[i] = true_or_false(item)
simplified_set = [items[i] for i in range(len(items)) if i not in removed]
# If more than one literal exists in the union, try to simplify
if (contract_literals
and sum(isinstance(item, LiteralType)
for item in simplified_set) > 1):
simplified_set = try_contracting_literals_in_union(simplified_set)
return UnionType.make_union(simplified_set, line, column)
def typed_dict_mapping_overlap(
left: ProperType, right: ProperType,
overlapping: Callable[[Type, Type], bool]) -> bool:
"""Check if a TypedDict type is overlapping with a Mapping.
The basic logic here consists of two rules:
* A TypedDict with some required keys is overlapping with Mapping[str, <some type>]
if and only if every key type is overlapping with <some type>. For example:
- TypedDict(x=int, y=str) overlaps with Dict[str, Union[str, int]]
- TypedDict(x=int, y=str) doesn't overlap with Dict[str, int]
Note that any additional non-required keys can't change the above result.
* A TypedDict with no required keys overlaps with Mapping[str, <some type>] if and
only if at least one of key types overlaps with <some type>. For example:
- TypedDict(x=str, y=str, total=False) overlaps with Dict[str, str]
- TypedDict(x=str, y=str, total=False) doesn't overlap with Dict[str, int]
- TypedDict(x=int, y=str, total=False) overlaps with Dict[str, str]
As usual empty, dictionaries lie in a gray area. In general, List[str] and List[str]
are considered non-overlapping despite empty list belongs to both. However, List[int]
and List[<nothing>] are considered overlapping.
So here we follow the same logic: a TypedDict with no required keys is considered
non-overlapping with Mapping[str, <some type>], but is considered overlapping with
Mapping[<nothing>, <nothing>]. This way we avoid false positives for overloads, and also
avoid false positives for comparisons like SomeTypedDict == {} under --strict-equality.
"""
assert not isinstance(left, TypedDictType) or not isinstance(
right, TypedDictType)
if isinstance(left, TypedDictType):
assert isinstance(right, Instance)
typed, other = left, right
else:
assert isinstance(left, Instance)
assert isinstance(right, TypedDictType)
typed, other = right, left
mapping = next(base for base in other.type.mro
if base.fullname() == 'typing.Mapping')
other = map_instance_to_supertype(other, mapping)
key_type, value_type = get_proper_types(other.args)
# TODO: is there a cleaner way to get str_type here?
fallback = typed.as_anonymous().fallback
str_type = fallback.type.bases[0].args[
0] # typing._TypedDict inherits Mapping[str, object]
# Special case: a TypedDict with no required keys overlaps with an empty dict.
if isinstance(key_type, UninhabitedType) and isinstance(
value_type, UninhabitedType):
return not typed.required_keys
if typed.required_keys:
if not overlapping(key_type, str_type):
return False
return all(
overlapping(typed.items[k], value_type)
for k in typed.required_keys)
else:
if not overlapping(key_type, str_type):
return False
non_required = set(typed.items.keys()) - typed.required_keys
return any(
overlapping(typed.items[k], value_type) for k in non_required)
def is_none_typevar_overlap(t1: Type, t2: Type) -> bool:
t1, t2 = get_proper_types((t1, t2))
return isinstance(t1, NoneType) and isinstance(t2, TypeVarType)
def is_overlapping_types(left: Type,
right: Type,
ignore_promotions: bool = False,
prohibit_none_typevar_overlap: bool = False) -> bool:
"""Can a value of type 'left' also be of type 'right' or vice-versa?
If 'ignore_promotions' is True, we ignore promotions while checking for overlaps.
If 'prohibit_none_typevar_overlap' is True, we disallow None from overlapping with
TypeVars (in both strict-optional and non-strict-optional mode).
"""
left, right = get_proper_types((left, right))
def _is_overlapping_types(left: Type, right: Type) -> bool:
'''Encode the kind of overlapping check to perform.
This function mostly exists so we don't have to repeat keyword arguments everywhere.'''
return is_overlapping_types(
left,
right,
ignore_promotions=ignore_promotions,
prohibit_none_typevar_overlap=prohibit_none_typevar_overlap)
# We should never encounter this type.
if isinstance(left, PartialType) or isinstance(right, PartialType):
assert False, "Unexpectedly encountered partial type"
# We should also never encounter these types, but it's possible a few
# have snuck through due to unrelated bugs. For now, we handle these
# in the same way we handle 'Any'.
#
# TODO: Replace these with an 'assert False' once we are more confident.
illegal_types = (UnboundType, ErasedType, DeletedType)
if isinstance(left, illegal_types) or isinstance(right, illegal_types):
return True
# When running under non-strict optional mode, simplify away types of
# the form 'Union[A, B, C, None]' into just 'Union[A, B, C]'.
if not state.strict_optional:
if isinstance(left, UnionType):
left = UnionType.make_union(left.relevant_items())
if isinstance(right, UnionType):
right = UnionType.make_union(right.relevant_items())
left, right = get_proper_types((left, right))
# 'Any' may or may not be overlapping with the other type
if isinstance(left, AnyType) or isinstance(right, AnyType):
return True
# We check for complete overlaps next as a general-purpose failsafe.
# If this check fails, we start checking to see if there exists a
# *partial* overlap between types.
#
# These checks will also handle the NoneType and UninhabitedType cases for us.
if (is_proper_subtype(left, right, ignore_promotions=ignore_promotions)
or is_proper_subtype(
right, left, ignore_promotions=ignore_promotions)):
return True
# See the docstring for 'get_possible_variants' for more info on what the
# following lines are doing.
left_possible = get_possible_variants(left)
right_possible = get_possible_variants(right)
# We start by checking multi-variant types like Unions first. We also perform
# the same logic if either type happens to be a TypeVar.
#
# Handling the TypeVars now lets us simulate having them bind to the corresponding
# type -- if we deferred these checks, the "return-early" logic of the other
# checks will prevent us from detecting certain overlaps.
#
# If both types are singleton variants (and are not TypeVars), we've hit the base case:
# we skip these checks to avoid infinitely recursing.
def is_none_typevar_overlap(t1: Type, t2: Type) -> bool:
t1, t2 = get_proper_types((t1, t2))
return isinstance(t1, NoneType) and isinstance(t2, TypeVarType)
if prohibit_none_typevar_overlap:
if is_none_typevar_overlap(left, right) or is_none_typevar_overlap(
right, left):
return False
if (len(left_possible) > 1 or len(right_possible) > 1
or isinstance(left, TypeVarType)
or isinstance(right, TypeVarType)):
for l in left_possible:
for r in right_possible:
if _is_overlapping_types(l, r):
return True
return False
# Now that we've finished handling TypeVars, we're free to end early
# if one one of the types is None and we're running in strict-optional mode.
# (None only overlaps with None in strict-optional mode).
#
# We must perform this check after the TypeVar checks because
# a TypeVar could be bound to None, for example.
if state.strict_optional and isinstance(left, NoneType) != isinstance(
right, NoneType):
return False
# Next, we handle single-variant types that may be inherently partially overlapping:
#
# - TypedDicts
# - Tuples
#
# If we cannot identify a partial overlap and end early, we degrade these two types
# into their 'Instance' fallbacks.
if isinstance(left, TypedDictType) and isinstance(right, TypedDictType):
return are_typed_dicts_overlapping(left,
right,
ignore_promotions=ignore_promotions)
elif typed_dict_mapping_pair(left, right):
# Overlaps between TypedDicts and Mappings require dedicated logic.
return typed_dict_mapping_overlap(left,
right,
overlapping=_is_overlapping_types)
elif isinstance(left, TypedDictType):
left = left.fallback
elif isinstance(right, TypedDictType):
right = right.fallback
if is_tuple(left) and is_tuple(right):
return are_tuples_overlapping(left,
right,
ignore_promotions=ignore_promotions)
elif isinstance(left, TupleType):
left = tuple_fallback(left)
elif isinstance(right, TupleType):
right = tuple_fallback(right)
# Next, we handle single-variant types that cannot be inherently partially overlapping,
# but do require custom logic to inspect.
#
# As before, we degrade into 'Instance' whenever possible.
if isinstance(left, TypeType) and isinstance(right, TypeType):
return _is_overlapping_types(left.item, right.item)
def _type_object_overlap(left: Type, right: Type) -> bool:
"""Special cases for type object types overlaps."""
# TODO: these checks are a bit in gray area, adjust if they cause problems.
left, right = get_proper_types((left, right))
# 1. Type[C] vs Callable[..., C], where the latter is class object.
if isinstance(left, TypeType) and isinstance(
right, CallableType) and right.is_type_obj():
return _is_overlapping_types(left.item, right.ret_type)
# 2. Type[C] vs Meta, where Meta is a metaclass for C.
if isinstance(left, TypeType) and isinstance(right, Instance):
if isinstance(left.item, Instance):
left_meta = left.item.type.metaclass_type
if left_meta is not None:
return _is_overlapping_types(left_meta, right)
# builtins.type (default metaclass) overlaps with all metaclasses
return right.type.has_base('builtins.type')
elif isinstance(left.item, AnyType):
return right.type.has_base('builtins.type')
# 3. Callable[..., C] vs Meta is considered below, when we switch to fallbacks.
return False
if isinstance(left, TypeType) or isinstance(right, TypeType):
return _type_object_overlap(left, right) or _type_object_overlap(
right, left)
if isinstance(left, CallableType) and isinstance(right, CallableType):
return is_callable_compatible(left,
right,
is_compat=_is_overlapping_types,
ignore_pos_arg_names=True,
allow_partial_overlap=True)
elif isinstance(left, CallableType):
left = left.fallback
elif isinstance(right, CallableType):
right = right.fallback
if isinstance(left, LiteralType) and isinstance(right, LiteralType):
if left.value == right.value:
# If values are the same, we still need to check if fallbacks are overlapping,
# this is done below.
left = left.fallback
right = right.fallback
else:
return False
elif isinstance(left, LiteralType):
left = left.fallback
elif isinstance(right, LiteralType):
right = right.fallback
# Finally, we handle the case where left and right are instances.
if isinstance(left, Instance) and isinstance(right, Instance):
# First we need to handle promotions and structural compatibility for instances
# that came as fallbacks, so simply call is_subtype() to avoid code duplication.
if (is_subtype(left, right, ignore_promotions=ignore_promotions) or
is_subtype(right, left, ignore_promotions=ignore_promotions)):
return True
# Two unrelated types cannot be partially overlapping: they're disjoint.
if left.type.has_base(right.type.fullname):
left = map_instance_to_supertype(left, right.type)
elif right.type.has_base(left.type.fullname):
right = map_instance_to_supertype(right, left.type)
else:
return False
if len(left.args) == len(right.args):
# Note: we don't really care about variance here, since the overlapping check
# is symmetric and since we want to return 'True' even for partial overlaps.
#
# For example, suppose we have two types Wrapper[Parent] and Wrapper[Child].
# It doesn't matter whether Wrapper is covariant or contravariant since
# either way, one of the two types will overlap with the other.
#
# Similarly, if Wrapper was invariant, the two types could still be partially
# overlapping -- what if Wrapper[Parent] happened to contain only instances of
# specifically Child?
#
# Or, to use a more concrete example, List[Union[A, B]] and List[Union[B, C]]
# would be considered partially overlapping since it's possible for both lists
# to contain only instances of B at runtime.
if all(
_is_overlapping_types(left_arg, right_arg)
for left_arg, right_arg in zip(left.args, right.args)):
return True
return False
# We ought to have handled every case by now: we conclude the
# two types are not overlapping, either completely or partially.
#
# Note: it's unclear however, whether returning False is the right thing
# to do when inferring reachability -- see https://github.com/python/mypy/issues/5529
assert type(left) != type(right)
return False
def callable_has_any(t: CallableType) -> int:
# We count a bare None in argument position as Any, since
# pyannotate turns it into Optional[Any]
return (any(
isinstance(at, NoneType) for at in get_proper_types(t.arg_types))
or has_any_type(t))
def make_simplified_union(items: Sequence[Type],
line: int = -1,
column: int = -1,
*,
keep_erased: bool = False,
contract_literals: bool = True) -> ProperType:
"""Build union type with redundant union items removed.
If only a single item remains, this may return a non-union type.
Examples:
* [int, str] -> Union[int, str]
* [int, object] -> object
* [int, int] -> int
* [int, Any] -> Union[int, Any] (Any types are not simplified away!)
* [Any, Any] -> Any
Note: This must NOT be used during semantic analysis, since TypeInfos may not
be fully initialized.
The keep_erased flag is used for type inference against union types
containing type variables. If set to True, keep all ErasedType items.
"""
items = get_proper_types(items)
while any(isinstance(typ, UnionType) for typ in items):
all_items = [] # type: List[ProperType]
for typ in items:
if isinstance(typ, UnionType):
all_items.extend(get_proper_types(typ.items))
else:
all_items.append(typ)
items = all_items
from mypy.subtypes import is_proper_subtype
removed = set() # type: Set[int]
# Avoid slow nested for loop for Union of Literal of strings (issue #9169)
if all((isinstance(item, LiteralType)
and item.fallback.type.fullname == 'builtins.str')
for item in items):
seen = set() # type: Set[str]
for index, item in enumerate(items):
assert isinstance(item, LiteralType)
assert isinstance(item.value, str)
if item.value in seen:
removed.add(index)
seen.add(item.value)
else:
for i, ti in enumerate(items):
if i in removed: continue
# Keep track of the truishness info for deleted subtypes which can be relevant
cbt = cbf = False
for j, tj in enumerate(items):
if i != j and is_proper_subtype(tj, ti, keep_erased_types=keep_erased) and \
is_redundant_literal_instance(ti, tj):
# We found a redundant item in the union.
removed.add(j)
cbt = cbt or tj.can_be_true
cbf = cbf or tj.can_be_false
# if deleted subtypes had more general truthiness, use that
if not ti.can_be_true and cbt:
items[i] = true_or_false(ti)
elif not ti.can_be_false and cbf:
items[i] = true_or_false(ti)
simplified_set = [items[i] for i in range(len(items)) if i not in removed]
# If more than one literal exists in the union, try to simplify
if (contract_literals
and sum(isinstance(item, LiteralType)
for item in simplified_set) > 1):
simplified_set = try_contracting_literals_in_union(simplified_set)
return UnionType.make_union(simplified_set, line, column)
def apply_generic_arguments(callable: CallableType,
orig_types: Sequence[Optional[Type]],
report_incompatible_typevar_value: Callable[
[CallableType, Type, str, Context], None],
context: Context,
skip_unsatisfied: bool = False) -> CallableType:
"""Apply generic type arguments to a callable type.
For example, applying [int] to 'def [T] (T) -> T' results in
'def (int) -> int'.
Note that each type can be None; in this case, it will not be applied.
If `skip_unsatisfied` is True, then just skip the types that don't satisfy type variable
bound or constraints, instead of giving an error.
"""
tvars = callable.variables
assert len(tvars) == len(orig_types)
# Check that inferred type variable values are compatible with allowed
# values and bounds. Also, promote subtype values to allowed values.
types = get_proper_types(orig_types)
for i, type in enumerate(types):
assert not isinstance(
type, PartialType), "Internal error: must never apply partial type"
values = get_proper_types(callable.variables[i].values)
if type is None:
continue
if values:
if isinstance(type, AnyType):
continue
if isinstance(type, TypeVarType) and type.values:
# Allow substituting T1 for T if every allowed value of T1
# is also a legal value of T.
if all(
any(
mypy.sametypes.is_same_type(v, v1) for v in values)
for v1 in type.values):
continue
matching = []
for value in values:
if mypy.subtypes.is_subtype(type, value):
matching.append(value)
if matching:
best = matching[0]
# If there are more than one matching value, we select the narrowest
for match in matching[1:]:
if mypy.subtypes.is_subtype(match, best):
best = match
types[i] = best
else:
if skip_unsatisfied:
types[i] = None
else:
report_incompatible_typevar_value(
callable, type, callable.variables[i].name, context)
else:
upper_bound = callable.variables[i].upper_bound
if not mypy.subtypes.is_subtype(type, upper_bound):
if skip_unsatisfied:
types[i] = None
else:
report_incompatible_typevar_value(
callable, type, callable.variables[i].name, context)
# Create a map from type variable id to target type.
id_to_type = {} # type: Dict[TypeVarId, Type]
for i, tv in enumerate(tvars):
typ = types[i]
if typ:
id_to_type[tv.id] = typ
# Apply arguments to argument types.
arg_types = [expand_type(at, id_to_type) for at in callable.arg_types]
# The callable may retain some type vars if only some were applied.
remaining_tvars = [tv for tv in tvars if tv.id not in id_to_type]
return callable.copy_modified(
arg_types=arg_types,
ret_type=expand_type(callable.ret_type, id_to_type),
variables=remaining_tvars,
)
def takes_callable_and_args_callback(ctx: FunctionContext) -> Type:
"""Automate the boilerplate for writing functions that accept
arbitrary positional arguments of the same type accepted by
a callable.
For example, this supports writing::
@trio_typing.takes_callable_and_args
def start_soon(
self,
async_fn: Callable[[VarArg()], Any],
*args: Any,
) -> None: ...
instead of::
T1 = TypeVar("T1")
T2 = TypeVar("T2")
T3 = TypeVar("T3")
T4 = TypeVar("T4")
@overload
def start_soon(
self,
async_fn: Callable[[], Any],
) -> None: ...
@overload
def start_soon(
self,
async_fn: Callable[[T1], Any],
__arg1: T1,
) -> None: ...
@overload
def start_soon(
self,
async_fn: Callable[[T1, T2], Any],
__arg1: T1,
__arg2: T2
) -> None: ...
# etc
"""
try:
if not ctx.arg_types or len(ctx.arg_types[0]) != 1:
raise ValueError("must be used as a decorator")
fn_type = get_proper_type(ctx.arg_types[0][0])
if not isinstance(fn_type, CallableType) or not isinstance(
get_proper_type(ctx.default_return_type), CallableType):
raise ValueError("must be used as a decorator")
callable_idx = -1 # index in function arguments of the callable
callable_args_idx = -1 # index in callable arguments of the StarArgs
callable_ty = None # type: Optional[CallableType]
args_idx = -1 # index in function arguments of the StarArgs
for idx, (kind,
ty) in enumerate(zip(fn_type.arg_kinds, fn_type.arg_types)):
ty = get_proper_type(ty)
if isinstance(ty, AnyType) and kind == ARG_STAR:
assert args_idx == -1
args_idx = idx
elif isinstance(ty, (UnionType, CallableType)) and kind == ARG_POS:
# turn Union[Callable[..., T], Callable[[VarArg()], T]]
# into Callable[[VarArg()], T]
# (the union makes it not fail when the plugin is not being used)
if isinstance(ty, UnionType):
for arm in get_proper_types(ty.items):
if (isinstance(arm, CallableType)
and not arm.is_ellipsis_args
and any(kind_ == ARG_STAR
for kind_ in arm.arg_kinds)):
ty = arm
break
else:
continue
for idx_, (kind_,
ty_) in enumerate(zip(ty.arg_kinds, ty.arg_types)):
ty_ = get_proper_type(ty_)
if isinstance(ty_, AnyType) and kind_ == ARG_STAR:
if callable_idx != -1:
raise ValueError(
"decorated function may only take one callable "
"that has an argument of type VarArg()")
callable_idx = idx
callable_args_idx = idx_
callable_ty = ty
if args_idx == -1:
raise ValueError("decorated function must take *args: Any")
if callable_idx == -1 or callable_ty is None:
raise ValueError(
"decorated function must take a callable that has a "
"argument of type mypy_extensions.VarArg()")
expanded_fns = [] # type: List[CallableType]
type_var_defs = [] # type: List[TypeVarDef]
type_var_types = [] # type: List[Type]
for arg_idx in range(
1, 7): # provides overloads for 0 through 5 arguments
arg_types = list(fn_type.arg_types)
arg_types[callable_idx] = callable_ty.copy_modified(
arg_types=(callable_ty.arg_types[:callable_args_idx] +
type_var_types +
callable_ty.arg_types[callable_args_idx + 1:]),
arg_kinds=(callable_ty.arg_kinds[:callable_args_idx] +
([ARG_POS] * len(type_var_types)) +
callable_ty.arg_kinds[callable_args_idx + 1:]),
arg_names=(callable_ty.arg_names[:callable_args_idx] +
([None] * len(type_var_types)) +
callable_ty.arg_names[callable_args_idx + 1:]),
variables=(callable_ty.variables + type_var_defs),
)
expanded_fns.append(
fn_type.copy_modified(
arg_types=(arg_types[:args_idx] + type_var_types +
arg_types[args_idx + 1:]),
arg_kinds=(fn_type.arg_kinds[:args_idx] +
([ARG_POS] * len(type_var_types)) +
fn_type.arg_kinds[args_idx + 1:]),
arg_names=(fn_type.arg_names[:args_idx] +
([None] * len(type_var_types)) +
fn_type.arg_names[args_idx + 1:]),
variables=(fn_type.variables + type_var_defs),
))
type_var_defs.append(
TypeVarDef(
"__T{}".format(arg_idx),
"__T{}".format(arg_idx),
-len(fn_type.variables) - arg_idx - 1,
[],
ctx.api.named_generic_type("builtins.object", []),
))
type_var_types.append(
TypeVarType(type_var_defs[-1], ctx.context.line,
ctx.context.column))
return Overloaded(expanded_fns)
except ValueError as ex:
ctx.api.fail("invalid use of @takes_callable_and_args: {}".format(ex),
ctx.context)
return ctx.default_return_type
