def test_heterogeneous_tuple(self):
ast = self._load_ast("a", """
from typing import Tuple
x = ... # type: Tuple[str, int]
""")
x = ast.Lookup("a.x").type
cls = self._ctx.convert.constant_to_value(x, {}, self._ctx.root_node)
instance = self._ctx.convert.constant_to_value(
abstract_utils.AsInstance(x), {}, self._ctx.root_node)
self.assertIsInstance(cls, abstract.TupleClass)
self.assertCountEqual(cls.formal_type_parameters.items(),
[(0, self._ctx.convert.str_type),
(1, self._ctx.convert.int_type),
(abstract_utils.T,
abstract.Union([
cls.formal_type_parameters[0],
cls.formal_type_parameters[1],
], self._ctx))])
self.assertIsInstance(instance, abstract.Tuple)
self.assertListEqual([v.data for v in instance.pyval],
[[self._ctx.convert.primitive_class_instances[str]],
[self._ctx.convert.primitive_class_instances[int]]])
# The order of option elements in Union is random
self.assertCountEqual(
instance.get_instance_type_parameter(abstract_utils.T).data, [
self._ctx.convert.primitive_class_instances[str],
self._ctx.convert.primitive_class_instances[int]
])
def test_callable_with_args(self):
ast = self._load_ast("a", """
from typing import Callable
x = ... # type: Callable[[int, bool], str]
""")
x = ast.Lookup("a.x").type
cls = self._ctx.convert.constant_to_value(x, {}, self._ctx.root_node)
instance = self._ctx.convert.constant_to_value(
abstract_utils.AsInstance(x), {}, self._ctx.root_node)
self.assertIsInstance(cls, abstract.CallableClass)
self.assertCountEqual(
cls.formal_type_parameters.items(),
[(0, self._ctx.convert.int_type),
(1, self._ctx.convert.primitive_classes[bool]),
(abstract_utils.ARGS,
abstract.Union(
[cls.formal_type_parameters[0], cls.formal_type_parameters[1]],
self._ctx)), (abstract_utils.RET, self._ctx.convert.str_type)])
self.assertIsInstance(instance, abstract.Instance)
self.assertEqual(instance.cls, cls)
self.assertCountEqual(
[(name, set(var.data))
for name, var in instance.instance_type_parameters.items()],
[(abstract_utils.full_type_name(instance, abstract_utils.ARGS), {
self._ctx.convert.primitive_class_instances[int],
self._ctx.convert.primitive_class_instances[bool]
}),
(abstract_utils.full_type_name(instance, abstract_utils.RET),
{self._ctx.convert.primitive_class_instances[str]})])
def test_classvar_instance(self):
ast = self._load_ast("a", """
from typing import ClassVar
class X:
v: ClassVar[int]
""")
pyval = ast.Lookup("a.X").Lookup("v").type
v = self._ctx.convert.constant_to_value(
abstract_utils.AsInstance(pyval), {}, self._ctx.root_node)
self.assertEqual(v, self._ctx.convert.primitive_class_instances[int])
def test_callable_no_args(self):
ast = self._load_ast("a", """
from typing import Callable
x = ... # type: Callable[[], ...]
""")
x = ast.Lookup("a.x").type
cls = self._ctx.convert.constant_to_value(x, {}, self._ctx.root_node)
instance = self._ctx.convert.constant_to_value(
abstract_utils.AsInstance(x), {}, self._ctx.root_node)
self.assertIsInstance(
cls.get_formal_type_parameter(abstract_utils.ARGS), abstract.Empty)
self.assertEqual(
abstract_utils.get_atomic_value(
instance.get_instance_type_parameter(abstract_utils.ARGS)),
self._ctx.convert.empty)
def _convert_member(self, member, subst=None):
"""Convert a member as a variable. For lazy lookup."""
subst = subst or datatypes.AliasingDict()
node = self.ctx.root_node
if isinstance(member, pytd.Constant):
return self.ctx.convert.constant_to_var(
abstract_utils.AsInstance(member.type), subst, node)
elif isinstance(member, pytd.Function):
c = self.ctx.convert.constant_to_value(member,
subst=subst,
node=node)
c.parent = self
return c.to_variable(node)
elif isinstance(member, pytd.Class):
return self.ctx.convert.constant_to_var(member,
subst=subst,
node=node)
else:
raise AssertionError("Invalid class member %s" %
pytd_utils.Print(member))
def test_plain_callable(self):
ast = self._load_ast("a", """
from typing import Callable
x = ... # type: Callable[..., int]
""")
x = ast.Lookup("a.x").type
cls = self._ctx.convert.constant_to_value(x, {}, self._ctx.root_node)
instance = self._ctx.convert.constant_to_value(
abstract_utils.AsInstance(x), {}, self._ctx.root_node)
self.assertIsInstance(cls, abstract.ParameterizedClass)
self.assertCountEqual(cls.formal_type_parameters.items(),
[(abstract_utils.ARGS, self._ctx.convert.unsolvable),
(abstract_utils.RET, self._ctx.convert.int_type)])
self.assertIsInstance(instance, abstract.Instance)
self.assertEqual(instance.cls, cls.base_cls)
self.assertCountEqual(
[(name, var.data)
for name, var in instance.instance_type_parameters.items()],
[(abstract_utils.full_type_name(
instance, abstract_utils.ARGS), [self._ctx.convert.unsolvable]),
(abstract_utils.full_type_name(instance, abstract_utils.RET),
[self._ctx.convert.primitive_class_instances[int]])])
def _constant_to_value(self, pyval, subst, get_node):
"""Create a BaseValue that represents a python constant.
This supports both constant from code constant pools and PyTD constants such
as classes. This also supports builtin python objects such as int and float.
Args:
pyval: The python or PyTD value to convert.
subst: The current type parameters.
get_node: A getter function for the current node.
Returns:
A Value that represents the constant, or None if we couldn't convert.
Raises:
NotImplementedError: If we don't know how to convert a value.
TypeParameterError: If we can't find a substitution for a type parameter.
"""
if isinstance(pyval, str):
return abstract.ConcreteValue(pyval, self.str_type, self.ctx)
elif isinstance(pyval, bytes):
return abstract.ConcreteValue(pyval, self.bytes_type, self.ctx)
elif isinstance(pyval, bool):
return self.true if pyval else self.false
elif isinstance(pyval, int) and -1 <= pyval <= _MAX_IMPORT_DEPTH:
# For small integers, preserve the actual value (for things like the
# level in IMPORT_NAME).
return abstract.ConcreteValue(pyval, self.int_type, self.ctx)
elif pyval.__class__ in self.primitive_classes:
return self.primitive_class_instances[pyval.__class__]
elif pyval.__class__ is frozenset:
instance = abstract.Instance(self.frozenset_type, self.ctx)
for element in pyval:
instance.merge_instance_type_parameter(
self.ctx.root_node, abstract_utils.T,
self.constant_to_var(element, subst, self.ctx.root_node))
return instance
elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)):
return abstract.ConcreteValue(pyval,
self.primitive_classes[types.CodeType],
self.ctx)
elif pyval is super:
return special_builtins.Super(self.ctx)
elif pyval is object:
return special_builtins.Object(self.ctx)
elif pyval.__class__ is type:
try:
return self.name_to_value(self._type_to_name(pyval), subst)
except (KeyError, AttributeError):
log.debug("Failed to find pytd", exc_info=True)
raise
elif isinstance(pyval, pytd.LateType):
actual = self._load_late_type(pyval)
return self._constant_to_value(actual, subst, get_node)
elif isinstance(pyval, pytd.TypeDeclUnit):
return self._create_module(pyval)
elif isinstance(pyval, pytd.Module):
mod = self.ctx.loader.import_name(pyval.module_name)
return self._create_module(mod)
elif isinstance(pyval, pytd.Class):
if pyval.name == "builtins.super":
return self.ctx.special_builtins["super"]
elif pyval.name == "builtins.object":
return self.object_type
elif pyval.name == "types.ModuleType":
return self.module_type
elif pyval.name == "_importlib_modulespec.ModuleType":
# Python 3's typeshed uses a stub file indirection to define ModuleType
# even though it is exported via types.pyi.
return self.module_type
elif pyval.name == "types.FunctionType":
return self.function_type
else:
module, dot, base_name = pyval.name.rpartition(".")
# typing.TypingContainer intentionally loads the underlying pytd types.
if (module not in ("typing", "typing_extensions") and
module in overlay_dict.overlays):
overlay = self.ctx.vm.import_module(module, module, 0)
if overlay.get_module(base_name) is overlay:
overlay.load_lazy_attribute(base_name)
return abstract_utils.get_atomic_value(overlay.members[base_name])
try:
cls = abstract.PyTDClass.make(base_name, pyval, self.ctx)
except mro.MROError as e:
self.ctx.errorlog.mro_error(self.ctx.vm.frames, base_name, e.mro_seqs)
cls = self.unsolvable
else:
if dot:
cls.module = module
cls.call_metaclass_init(get_node())
return cls
elif isinstance(pyval, pytd.Function):
signatures = [
abstract.PyTDSignature(pyval.name, sig, self.ctx)
for sig in pyval.signatures
]
type_new = self.ctx.loader.lookup_builtin("builtins.type").Lookup(
"__new__")
if pyval is type_new:
f_cls = special_builtins.TypeNew
else:
f_cls = abstract.PyTDFunction
f = f_cls(pyval.name, signatures, pyval.kind, self.ctx)
f.is_abstract = pyval.is_abstract
return f
elif isinstance(pyval, pytd.ClassType):
if pyval.cls:
cls = pyval.cls
else:
# If pyval is a reference to a class in builtins or typing, we can fill
# in the class ourselves. lookup_builtin raises a KeyError if the name
# is not found.
cls = self.ctx.loader.lookup_builtin(pyval.name)
assert isinstance(cls, pytd.Class)
return self.constant_to_value(cls, subst, self.ctx.root_node)
elif isinstance(pyval, pytd.NothingType):
return self.empty
elif isinstance(pyval, pytd.AnythingType):
return self.unsolvable
elif (isinstance(pyval, pytd.Constant) and
isinstance(pyval.type, pytd.AnythingType)):
# We allow "X = ... # type: Any" to declare X as a type.
return self.unsolvable
elif (isinstance(pyval, pytd.Constant) and
isinstance(pyval.type, pytd.GenericType) and
pyval.type.name == "builtins.type"):
# `X: Type[other_mod.X]` is equivalent to `X = other_mod.X`.
param, = pyval.type.parameters
return self.constant_to_value(param, subst, self.ctx.root_node)
elif isinstance(pyval, pytd.UnionType):
options = [
self.constant_to_value(t, subst, self.ctx.root_node)
for t in pyval.type_list
]
if len(options) > 1:
return abstract.Union(options, self.ctx)
else:
return options[0]
elif isinstance(pyval, pytd.TypeParameter):
constraints = tuple(
self.constant_to_value(c, {}, self.ctx.root_node)
for c in pyval.constraints)
bound = (
pyval.bound and
self.constant_to_value(pyval.bound, {}, self.ctx.root_node))
return abstract.TypeParameter(
pyval.name,
self.ctx,
constraints=constraints,
bound=bound,
module=pyval.scope)
elif isinstance(pyval, abstract_utils.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.LateType):
actual = self._load_late_type(cls)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
cls = actual.cls
if isinstance(cls, pytd.ClassType):
cls = cls.cls
if isinstance(cls, pytd.GenericType) and cls.name == "typing.ClassVar":
param, = cls.parameters
return self.constant_to_value(
abstract_utils.AsInstance(param), subst, self.ctx.root_node)
elif isinstance(cls, pytd.GenericType) or (isinstance(cls, pytd.Class) and
cls.template):
# If we're converting a generic Class, need to create a new instance of
# it. See test_classes.testGenericReinstantiated.
if isinstance(cls, pytd.Class):
params = tuple(t.type_param.upper_value for t in cls.template)
cls = pytd.GenericType(base_type=pytd.ClassType(cls.name, cls),
parameters=params)
if isinstance(cls.base_type, pytd.LateType):
actual = self._load_late_type(cls.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base_cls = actual.cls
else:
base_type = cls.base_type
assert isinstance(base_type, pytd.ClassType)
base_cls = base_type.cls
assert isinstance(base_cls, pytd.Class), base_cls
if base_cls.name == "builtins.type":
c, = cls.parameters
if isinstance(c, pytd.TypeParameter):
if not subst or c.full_name not in subst:
raise self.TypeParameterError(c.full_name)
# deformalize gets rid of any unexpected TypeVars, which can appear
# if something is annotated as Type[T].
return self.ctx.annotation_utils.deformalize(
self.merge_classes(subst[c.full_name].data))
else:
return self.constant_to_value(c, subst, self.ctx.root_node)
elif isinstance(cls, pytd.TupleType):
content = tuple(self.constant_to_var(abstract_utils.AsInstance(p),
subst, get_node())
for p in cls.parameters)
return self.tuple_to_value(content)
elif isinstance(cls, pytd.CallableType):
clsval = self.constant_to_value(cls, subst, self.ctx.root_node)
return abstract.Instance(clsval, self.ctx)
else:
clsval = self.constant_to_value(base_cls, subst, self.ctx.root_node)
instance = abstract.Instance(clsval, self.ctx)
num_params = len(cls.parameters)
assert num_params <= len(base_cls.template)
for i, formal in enumerate(base_cls.template):
if i < num_params:
node = get_node()
p = self.constant_to_var(
abstract_utils.AsInstance(cls.parameters[i]), subst, node)
else:
# An omitted type parameter implies `Any`.
node = self.ctx.root_node
p = self.unsolvable.to_variable(node)
instance.merge_instance_type_parameter(node, formal.name, p)
return instance
elif isinstance(cls, pytd.Class):
assert not cls.template
# This key is also used in __init__
key = (abstract.Instance, cls)
if key not in self._convert_cache:
if cls.name in ["builtins.type", "builtins.property"]:
# An instance of "type" or of an anonymous property can be anything.
instance = self._create_new_unknown_value("type")
else:
mycls = self.constant_to_value(cls, subst, self.ctx.root_node)
instance = abstract.Instance(mycls, self.ctx)
log.info("New pytd instance for %s: %r", cls.name, instance)
self._convert_cache[key] = instance
return self._convert_cache[key]
elif isinstance(cls, pytd.Literal):
return self.constant_to_value(
self._get_literal_value(cls.value), subst, self.ctx.root_node)
else:
return self.constant_to_value(cls, subst, self.ctx.root_node)
elif (isinstance(pyval, pytd.GenericType) and
pyval.name == "typing.ClassVar"):
param, = pyval.parameters
return self.constant_to_value(param, subst, self.ctx.root_node)
elif isinstance(pyval, pytd.GenericType):
if isinstance(pyval.base_type, pytd.LateType):
actual = self._load_late_type(pyval.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base = actual.cls
else:
assert isinstance(pyval.base_type, pytd.ClassType), pyval
base = pyval.base_type.cls
assert isinstance(base, pytd.Class), base
base_cls = self.constant_to_value(base, subst, self.ctx.root_node)
if not isinstance(base_cls, abstract.Class):
# base_cls can be, e.g., an unsolvable due to an mro error.
return self.unsolvable
if isinstance(pyval, pytd.TupleType):
abstract_class = abstract.TupleClass
template = list(range(len(pyval.parameters))) + [abstract_utils.T]
combined_parameter = pytd_utils.JoinTypes(pyval.parameters)
parameters = pyval.parameters + (combined_parameter,)
elif isinstance(pyval, pytd.CallableType):
abstract_class = abstract.CallableClass
template = list(range(len(pyval.args))) + [abstract_utils.ARGS,
abstract_utils.RET]
parameters = pyval.args + (pytd_utils.JoinTypes(pyval.args), pyval.ret)
else:
abstract_class = abstract.ParameterizedClass
if pyval.name == "typing.Generic":
pyval_template = pyval.parameters
else:
pyval_template = base.template
template = tuple(t.name for t in pyval_template)
parameters = pyval.parameters
assert (pyval.name in ("typing.Generic", "typing.Protocol") or
len(parameters) <= len(template))
# Delay type parameter loading to handle recursive types.
# See the ParameterizedClass.formal_type_parameters() property.
type_parameters = abstract_utils.LazyFormalTypeParameters(
template, parameters, subst)
return abstract_class(base_cls, type_parameters, self.ctx)
elif isinstance(pyval, pytd.Literal):
value = self.constant_to_value(
self._get_literal_value(pyval.value), subst, self.ctx.root_node)
return abstract.LiteralClass(value, self.ctx)
elif isinstance(pyval, pytd.Annotated):
typ = self.constant_to_value(pyval.base_type, subst, self.ctx.root_node)
if pyval.annotations[0] == "'pytype_metadata'":
try:
md = metadata.from_string(pyval.annotations[1])
if md["tag"] == "attr.ib":
ret = attr_overlay.AttribInstance.from_metadata(
self.ctx, self.ctx.root_node, typ, md)
return ret
elif md["tag"] == "attr.s":
ret = attr_overlay.Attrs.from_metadata(self.ctx, md)
return ret
except (IndexError, ValueError, TypeError, KeyError):
details = "Wrong format for pytype_metadata."
self.ctx.errorlog.invalid_annotation(self.ctx.vm.frames,
pyval.annotations[1], details)
return typ
else:
return typ
elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node
return self.tuple_to_value([
self.constant_to_var(item, subst, self.ctx.root_node)
for i, item in enumerate(pyval)
])
else:
raise NotImplementedError("Can't convert constant %s %r" %
(type(pyval), pyval))
def constant_to_var(self, pyval, subst=None, node=None, source_sets=None,
discard_concrete_values=False):
"""Convert a constant to a Variable.
This converts a constant to a cfg.Variable. Unlike constant_to_value, it
can handle things that need to be represented as a Variable with multiple
possible values (i.e., a union type), like pytd.Function.
Args:
pyval: The Python constant to convert. Can be a PyTD definition or a
builtin constant.
subst: The current type parameters.
node: The current CFG node. (For instances)
source_sets: An iterator over instances of SourceSet (or just tuples).
discard_concrete_values: Whether concrete values should be discarded from
type parameters.
Returns:
A cfg.Variable.
Raises:
TypeParameterError: if conversion is attempted on a type parameter without
a substitution.
ValueError: if pytype is not of a known type.
"""
source_sets = source_sets or [[]]
node = node or self.ctx.root_node
if isinstance(pyval, pytd.NothingType):
return self.ctx.program.NewVariable([], [], self.ctx.root_node)
elif isinstance(pyval, pytd.Alias):
return self.constant_to_var(pyval.type, subst, node, source_sets,
discard_concrete_values)
elif isinstance(pyval, abstract_utils.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.AnythingType):
return self.unsolvable.to_variable(node)
elif (isinstance(pyval, abstract_utils.AsReturnValue) and
isinstance(cls, pytd.NothingType)):
return self.no_return.to_variable(node)
elif isinstance(cls, pytd.GenericType) and cls.name == "typing.ClassVar":
param, = cls.parameters
return self.constant_to_var(abstract_utils.AsInstance(param), subst,
node, source_sets, discard_concrete_values)
var = self.ctx.program.NewVariable()
for t in pytd_utils.UnpackUnion(cls):
if isinstance(t, pytd.TypeParameter):
if not subst or t.full_name not in subst:
raise self.TypeParameterError(t.full_name)
else:
for v in subst[t.full_name].bindings:
for source_set in source_sets:
var.AddBinding(self.get_maybe_abstract_instance(v.data)
if discard_concrete_values else v.data,
source_set + [v], node)
elif isinstance(t, pytd.NothingType):
pass
else:
value = self.constant_to_value(
abstract_utils.AsInstance(t), subst, node)
for source_set in source_sets:
var.AddBinding(value, source_set, node)
return var
elif isinstance(pyval, pytd.Constant):
return self.constant_to_var(abstract_utils.AsInstance(pyval.type), subst,
node, source_sets, discard_concrete_values)
result = self.constant_to_value(pyval, subst, node)
if result is not None:
return result.to_variable(node)
# There might still be bugs on the abstract interpreter when it returns,
# e.g. a list of values instead of a list of types:
assert pyval.__class__ != cfg.Variable, pyval
if pyval.__class__ == tuple:
# This case needs to go at the end because many things are actually also
# tuples.
return self.build_tuple(self.ctx.root_node,
(self.constant_to_var(v, subst, node, source_sets,
discard_concrete_values)
for i, v in enumerate(pyval)))
raise ValueError(
"Cannot convert {} to an abstract value".format(pyval.__class__))
def _get_mutation(self, node, arg_dict, subst, retvar):
"""Mutation for changing the type parameters of mutable arguments.
This will adjust the type parameters as needed for pytd functions like:
def append_float(x: list[int]):
x = list[int or float]
This is called after all the signature matching has succeeded, and we
know we're actually calling this function.
Args:
node: The current CFG node.
arg_dict: A map of strings to pytd.Bindings instances.
subst: Current type parameters.
retvar: A variable of the return value.
Returns:
A list of Mutation instances.
Raises:
ValueError: If the pytd contains invalid information for mutated params.
"""
# Handle mutable parameters using the information type parameters
mutations = []
# It's possible that the signature contains type parameters that are used
# in mutations but are not filled in by the arguments, e.g. when starargs
# and starstarargs have type parameters but are not in the args. Check that
# subst has an entry for every type parameter, adding any that are missing.
if any(f.mutated_type for f in self.pytd_sig.params):
subst = subst.copy()
for t in pytd_utils.GetTypeParameters(self.pytd_sig):
if t.full_name not in subst:
subst[t.full_name] = self.ctx.convert.empty.to_variable(node)
for formal in self.pytd_sig.params:
actual = arg_dict[formal.name]
arg = actual.data
if (formal.mutated_type is not None and
isinstance(arg, _instance_base.SimpleValue)):
try:
all_names_actuals = self._collect_mutated_parameters(
formal.type, formal.mutated_type)
except ValueError as e:
log.error("Old: %s", pytd_utils.Print(formal.type))
log.error("New: %s", pytd_utils.Print(formal.mutated_type))
log.error("Actual: %r", actual)
raise ValueError("Mutable parameters setting a type to a "
"different base type is not allowed.") from e
for names_actuals in all_names_actuals:
for tparam, type_actual in names_actuals:
log.info("Mutating %s to %s",
tparam.name,
pytd_utils.Print(type_actual))
type_actual_val = self.ctx.convert.constant_to_var(
abstract_utils.AsInstance(type_actual),
subst,
node,
discard_concrete_values=True)
mutations.append(
function.Mutation(arg, tparam.full_name, type_actual_val))
if self.name == "__new__":
# This is a constructor, so check whether the constructed instance needs
# to be mutated.
for ret in retvar.data:
if ret.cls.full_name != "builtins.type":
for t in ret.cls.template:
if t.full_name in subst:
mutations.append(
function.Mutation(ret, t.full_name, subst[t.full_name]))
return mutations
def instantiate(self, node, container=None):
return self.ctx.convert.constant_to_var(
abstract_utils.AsInstance(self.pytd_cls), {}, node)
