def _to_pytd(datum, loader, ast):
if not datum:
return pytd.AnythingType()
t = pytd_utils.JoinTypes(v.to_type() for v in datum).Visit(
visitors.RemoveUnknownClasses())
return loader.resolve_type(t, ast)
def VisitUnionType(self, union):
return pytd_utils.JoinTypes(union.type_list)
def value_to_pytd_type(self, node, v, seen, view):
"""Get a PyTD type representing this object, as seen at a node.
Args:
node: The node from which we want to observe this object.
v: The object.
seen: The set of values seen before while computing the type.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, (abstract.Empty, typing_overlay.NoReturn)):
return pytd.NothingType()
elif isinstance(v, abstract.TypeParameterInstance):
if (v.module in self._scopes
or v.instance is abstract_utils.DUMMY_CONTAINER):
return self._typeparam_to_def(node, v.param, v.param.name)
elif v.instance.get_instance_type_parameter(v.full_name).bindings:
# The type parameter was initialized. Set the view to None, since we
# don't include v.instance in the view.
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, p, seen, None) for p in
v.instance.get_instance_type_parameter(v.full_name).data)
elif v.param.constraints:
return pytd_utils.JoinTypes(
self.value_instance_to_pytd_type(node, p, None, seen, view)
for p in v.param.constraints)
elif v.param.bound:
return self.value_instance_to_pytd_type(
node, v.param.bound, None, seen, view)
else:
return pytd.AnythingType()
elif isinstance(v, typing_overlay.TypeVar):
return pytd.NamedType("builtins.type")
elif isinstance(v, dataclass_overlay.FieldInstance):
if not v.default:
return pytd.AnythingType()
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, d, seen, view)
for d in v.default.data)
elif isinstance(v, attr_overlay.AttribInstance):
ret = self.value_to_pytd_type(node, v.typ, seen, view)
md = metadata.to_pytd(v.to_metadata())
return pytd.Annotated(ret, ("'pytype_metadata'", md))
elif isinstance(v, special_builtins.PropertyInstance):
return pytd.NamedType("builtins.property")
elif isinstance(v, typed_dict.TypedDict):
return pytd.NamedType(v.props.name)
elif isinstance(v, abstract.FUNCTION_TYPES):
try:
signatures = function.get_signatures(v)
except NotImplementedError:
return pytd.NamedType("typing.Callable")
if len(signatures) == 1:
val = self.signature_to_callable(signatures[0])
if not isinstance(
v, abstract.PYTD_FUNCTION_TYPES) or not val.formal:
# This is a workaround to make sure we don't put unexpected type
# parameters in call traces.
return self.value_instance_to_pytd_type(
node, val, None, seen, view)
return pytd.NamedType("typing.Callable")
elif isinstance(v, (abstract.ClassMethod, abstract.StaticMethod)):
return self.value_to_pytd_type(node, v.method, seen, view)
elif isinstance(v, (special_builtins.IsInstance,
special_builtins.ClassMethodCallable)):
return pytd.NamedType("typing.Callable")
elif isinstance(v, abstract.Class):
param = self.value_instance_to_pytd_type(node, v, None, seen, view)
return pytd.GenericType(base_type=pytd.NamedType("builtins.type"),
parameters=(param, ))
elif isinstance(v, abstract.Module):
return pytd.Alias(v.name,
pytd.Module(v.name, module_name=v.full_name))
elif (self._output_mode >= Converter.OutputMode.LITERAL
and isinstance(v, abstract.ConcreteValue)
and isinstance(v.pyval, (int, str, bytes))):
# LITERAL mode is used only for pretty-printing, so we just stringify the
# inner value rather than properly converting it.
return pytd.Literal(repr(v.pyval))
elif isinstance(v, abstract.SimpleValue):
ret = self.value_instance_to_pytd_type(node,
v.cls,
v,
seen=seen,
view=view)
ret.Visit(
visitors.FillInLocalPointers(
{"builtins": self.ctx.loader.builtins}))
return ret
elif isinstance(v, abstract.Union):
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, o, seen, view)
for o in v.options)
elif isinstance(v, special_builtins.SuperInstance):
return pytd.NamedType("builtins.super")
elif isinstance(v, abstract.TypeParameter):
# Arguably, the type of a type parameter is NamedType("typing.TypeVar"),
# but pytype doesn't know how to handle that, so let's just go with Any
# unless self._detailed is set.
if self._detailed:
return pytd.NamedType("typing.TypeVar")
else:
return pytd.AnythingType()
elif isinstance(v, abstract.Unsolvable):
return pytd.AnythingType()
elif isinstance(v, abstract.Unknown):
return pytd.NamedType(v.class_name)
elif isinstance(v, abstract.BuildClass):
return pytd.NamedType("typing.Callable")
elif isinstance(v, abstract.FinalAnnotation):
param = self.value_to_pytd_type(node, v.annotation, seen, view)
return pytd.GenericType(base_type=pytd.NamedType("typing.Final"),
parameters=(param, ))
else:
raise NotImplementedError(v.__class__.__name__)
def _constant_to_value(self, pyval, subst, get_node):
"""Create a AtomicAbstractValue that represents a python constant.
This supports both constant from code constant pools and PyTD constants such
as classes. This also supports builtin python objects such as int and float.
Args:
pyval: The python or PyTD value to convert.
subst: The current type parameters.
get_node: A getter function for the current node.
Returns:
A Value that represents the constant, or None if we couldn't convert.
Raises:
NotImplementedError: If we don't know how to convert a value.
TypeParameterError: If we can't find a substitution for a type parameter.
"""
if pyval.__class__ is str:
# We use a subclass of str, compat.BytesPy3, to mark Python 3
# bytestrings, which are converted to abstract bytes instances.
# compat.BytesType dispatches to this when appropriate.
return abstract.AbstractOrConcreteValue(pyval, self.str_type,
self.vm)
elif isinstance(pyval, compat.UnicodeType):
return abstract.AbstractOrConcreteValue(pyval, self.unicode_type,
self.vm)
elif isinstance(pyval, compat.BytesType):
return abstract.AbstractOrConcreteValue(pyval, self.bytes_type,
self.vm)
elif isinstance(pyval, bool):
return self.true if pyval else self.false
elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH:
# For small integers, preserve the actual value (for things like the
# level in IMPORT_NAME).
return abstract.AbstractOrConcreteValue(pyval, self.int_type,
self.vm)
elif isinstance(pyval, compat.LongType):
# long is aliased to int
return self.primitive_class_instances[int]
elif pyval.__class__ in self.primitive_classes:
return self.primitive_class_instances[pyval.__class__]
elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)):
return abstract.AbstractOrConcreteValue(
pyval, self.primitive_classes[types.CodeType], self.vm)
elif pyval is super:
return special_builtins.Super(self.vm)
elif pyval is object:
return special_builtins.Object(self.vm)
elif pyval.__class__ is type:
try:
return self.name_to_value(self._type_to_name(pyval), subst)
except (KeyError, AttributeError):
log.debug("Failed to find pytd", exc_info=True)
raise
elif isinstance(pyval, pytd.LateType):
actual = self._load_late_type(pyval)
return self._constant_to_value(actual, subst, get_node)
elif isinstance(pyval, pytd.TypeDeclUnit):
return self._create_module(pyval)
elif isinstance(pyval, pytd.Module):
mod = self.vm.loader.import_name(pyval.module_name)
return self._create_module(mod)
elif isinstance(pyval, pytd.Class):
if pyval.name == "__builtin__.super":
return self.vm.special_builtins["super"]
elif pyval.name == "__builtin__.object":
return self.object_type
elif pyval.name == "types.ModuleType":
return self.module_type
elif pyval.name == "_importlib_modulespec.ModuleType":
# Python 3's typeshed uses a stub file indirection to define ModuleType
# even though it is exported via types.pyi.
return self.module_type
else:
module, dot, base_name = pyval.name.rpartition(".")
# typing.TypingContainer intentionally loads the underlying pytd types.
if module != "typing" and module in self.vm.loaded_overlays:
overlay = self.vm.loaded_overlays[module]
if overlay.get_module(base_name) is overlay:
overlay.load_lazy_attribute(base_name)
return abstract_utils.get_atomic_value(
overlay.members[base_name])
try:
cls = abstract.PyTDClass(base_name, pyval, self.vm)
except mro.MROError as e:
self.vm.errorlog.mro_error(self.vm.frames, base_name,
e.mro_seqs)
cls = self.unsolvable
else:
if dot:
cls.module = module
return cls
elif isinstance(pyval, pytd.Function):
signatures = [
function.PyTDSignature(pyval.name, sig, self.vm)
for sig in pyval.signatures
]
type_new = self.vm.lookup_builtin("__builtin__.type").Lookup(
"__new__")
if pyval is type_new:
f_cls = special_builtins.TypeNew
else:
f_cls = abstract.PyTDFunction
f = f_cls(pyval.name, signatures, pyval.kind, self.vm)
f.is_abstract = pyval.is_abstract
return f
elif isinstance(pyval, pytd.ClassType):
assert pyval.cls
return self.constant_to_value(pyval.cls, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.NothingType):
return self.empty
elif isinstance(pyval, pytd.AnythingType):
return self.unsolvable
elif (isinstance(pyval, pytd.Constant)
and isinstance(pyval.type, pytd.AnythingType)):
# We allow "X = ... # type: Any" to declare X as a type.
return self.unsolvable
elif isinstance(pyval, pytd.FunctionType):
return self.constant_to_value(pyval.function, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.UnionType):
options = [
self.constant_to_value(t, subst, self.vm.root_cfg_node)
for t in pyval.type_list
]
if len(options) > 1:
return abstract.Union(options, self.vm)
else:
return options[0]
elif isinstance(pyval, pytd.TypeParameter):
constraints = tuple(
self.constant_to_value(c, {}, self.vm.root_cfg_node)
for c in pyval.constraints)
bound = (pyval.bound and self.constant_to_value(
pyval.bound, {}, self.vm.root_cfg_node))
return abstract.TypeParameter(pyval.name,
self.vm,
constraints=constraints,
bound=bound,
module=pyval.scope)
elif isinstance(pyval, abstract_utils.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.LateType):
actual = self._load_late_type(cls)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
cls = actual.cls
if isinstance(cls, pytd.ClassType):
cls = cls.cls
if (isinstance(cls, pytd.GenericType)
and cls.base_type.name == "typing.ClassVar"):
param, = cls.parameters
return self.constant_to_value(abstract_utils.AsInstance(param),
subst, self.vm.root_cfg_node)
elif isinstance(cls,
pytd.GenericType) or (isinstance(cls, pytd.Class)
and cls.template):
# If we're converting a generic Class, need to create a new instance of
# it. See test_classes.testGenericReinstantiated.
if isinstance(cls, pytd.Class):
params = tuple(t.type_param.upper_value
for t in cls.template)
cls = pytd.GenericType(base_type=pytd.ClassType(
cls.name, cls),
parameters=params)
if isinstance(cls.base_type, pytd.LateType):
actual = self._load_late_type(cls.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base_cls = actual.cls
else:
assert isinstance(cls.base_type, pytd.ClassType)
base_cls = cls.base_type.cls
assert isinstance(base_cls, pytd.Class), base_cls
if base_cls.name == "__builtin__.type":
c, = cls.parameters
if isinstance(c, pytd.TypeParameter):
if not subst or c.full_name not in subst:
raise self.TypeParameterError(c.full_name)
return self.merge_classes(get_node(),
subst[c.full_name].data)
else:
return self.constant_to_value(c, subst,
self.vm.root_cfg_node)
elif isinstance(cls, pytd.TupleType):
content = tuple(
self.constant_to_var(abstract_utils.AsInstance(
p), subst, get_node()) for p in cls.parameters)
return abstract.Tuple(content, self.vm)
elif isinstance(cls, pytd.CallableType):
clsval = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
return abstract.Instance(clsval, self.vm)
else:
clsval = self.constant_to_value(base_cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(clsval, self.vm)
num_params = len(cls.parameters)
assert num_params <= len(base_cls.template)
for i, formal in enumerate(base_cls.template):
if i < num_params:
node = get_node()
p = self.constant_to_var(
abstract_utils.AsInstance(cls.parameters[i]),
subst, node)
else:
# An omitted type parameter implies `Any`.
node = self.vm.root_cfg_node
p = self.unsolvable.to_variable(node)
instance.merge_instance_type_parameter(
node, formal.name, p)
return instance
elif isinstance(cls, pytd.Class):
assert not cls.template
# This key is also used in __init__
key = (abstract.Instance, cls)
if key not in self._convert_cache:
if cls.name in [
"__builtin__.type", "__builtin__.property"
]:
# An instance of "type" or of an anonymous property can be anything.
instance = self._create_new_unknown_value("type")
else:
mycls = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(mycls, self.vm)
log.info("New pytd instance for %s: %r", cls.name,
instance)
self._convert_cache[key] = instance
return self._convert_cache[key]
elif isinstance(cls, pytd.Literal):
return self.constant_to_value(
self._get_literal_value(cls.value), subst,
self.vm.root_cfg_node)
else:
return self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
elif (isinstance(pyval, pytd.GenericType)
and pyval.base_type.name == "typing.ClassVar"):
param, = pyval.parameters
return self.constant_to_value(param, subst, self.vm.root_cfg_node)
elif isinstance(pyval, pytd.GenericType):
if isinstance(pyval.base_type, pytd.LateType):
actual = self._load_late_type(pyval.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base = actual.cls
else:
assert isinstance(pyval.base_type, pytd.ClassType)
base = pyval.base_type.cls
assert isinstance(base, pytd.Class), base
base_cls = self.constant_to_value(base, subst,
self.vm.root_cfg_node)
if not isinstance(base_cls, mixin.Class):
# base_cls can be, e.g., an unsolvable due to an mro error.
return self.unsolvable
if isinstance(pyval, pytd.TupleType):
abstract_class = abstract.TupleClass
template = list(range(len(
pyval.parameters))) + [abstract_utils.T]
parameters = pyval.parameters + (pytd.UnionType(
pyval.parameters), )
elif isinstance(pyval, pytd.CallableType):
abstract_class = abstract.CallableClass
template = list(range(len(
pyval.args))) + [abstract_utils.ARGS, abstract_utils.RET]
parameters = pyval.args + (pytd_utils.JoinTypes(
pyval.args), pyval.ret)
else:
abstract_class = abstract.ParameterizedClass
if pyval.base_type.name == "typing.Generic":
pyval_template = pyval.parameters
else:
pyval_template = base.template
template = tuple(t.name for t in pyval_template)
parameters = pyval.parameters
assert (pyval.base_type.name == "typing.Generic"
or len(parameters) <= len(template))
# Delay type parameter loading to handle recursive types.
# See the ParameterizedClass.formal_type_parameters() property.
type_parameters = abstract_utils.LazyFormalTypeParameters(
template, parameters, subst)
return abstract_class(base_cls, type_parameters, self.vm)
elif isinstance(pyval, pytd.Literal):
value = self.constant_to_value(
self._get_literal_value(pyval.value), subst,
self.vm.root_cfg_node)
return abstract.LiteralClass(self.name_to_value("typing.Literal"),
value, self.vm)
elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node
return self.tuple_to_value([
self.constant_to_var(item, subst, self.vm.root_cfg_node)
for i, item in enumerate(pyval)
])
else:
raise NotImplementedError("Can't convert constant %s %r" %
(type(pyval), pyval))
def get_pytd(self, datum):
if not datum:
return pytd.AnythingType()
t = pytd_utils.JoinTypes(v.to_type() for v in datum).Visit(
visitors.RemoveUnknownClasses())
return self.loader.resolve_type(t, self.pytd_module)
def _class_to_def(self, node, v, class_name):
"""Convert an InterpreterClass to a PyTD definition."""
methods = {}
constants = collections.defaultdict(pytd_utils.TypeBuilder)
annots = abstract_utils.get_annotations_dict(v.members)
for name, t in self.annotations_to_instance_types(node, annots):
constants[name].add_type(t)
# class-level attributes
for name, member in v.members.items():
if name in CLASS_LEVEL_IGNORE or name in constants:
continue
for value in member.FilteredData(self.vm.exitpoint, strict=False):
if isinstance(value, special_builtins.PropertyInstance):
# For simplicity, output properties as constants, since our parser
# turns them into constants anyway.
if value.fget:
for typ in self._function_to_return_types(
node, value.fget):
constants[name].add_type(typ)
else:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, special_builtins.StaticMethodInstance):
try:
methods[name] = self._static_method_to_def(
node, value, name, pytd.STATICMETHOD)
except abstract_utils.ConversionError:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, special_builtins.ClassMethodInstance):
try:
methods[name] = self._class_method_to_def(
node, value, name, pytd.CLASSMETHOD)
except abstract_utils.ConversionError:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, abstract.Function):
# TODO(rechen): Removing mutations altogether won't work for generic
# classes. To support those, we'll need to change the mutated type's
# base to the current class, rename aliased type parameters, and
# replace any parameter not in the class or function template with
# its upper value.
methods[name] = self.value_to_pytd_def(
node, value,
name).Visit(visitors.DropMutableParameters())
else:
cls = self.vm.convert.merge_classes([value])
node, attr = self.vm.attribute_handler.get_attribute(
node, cls, "__get__")
if attr:
# This attribute is a descriptor. Its type is the return value of
# its __get__ method.
for typ in self._function_to_return_types(node, attr):
constants[name].add_type(typ)
else:
constants[name].add_type(value.to_type(node))
# instance-level attributes
for instance in set(v.instances):
for name, member in instance.members.items():
if name not in CLASS_LEVEL_IGNORE:
for value in member.FilteredData(self.vm.exitpoint,
strict=False):
constants[name].add_type(value.to_type(node))
for name in list(methods):
if name in constants:
# If something is both a constant and a method, it means that the class
# is, at some point, overwriting its own methods with an attribute.
del methods[name]
constants[name].add_type(pytd.AnythingType())
constants = [
pytd.Constant(name, builder.build())
for name, builder in constants.items() if builder
]
metaclass = v.metaclass(node)
if metaclass is not None:
metaclass = metaclass.get_instance_type(node)
# Some of the class's bases may not be in global scope, so they won't show
# up in the output. In that case, fold the base class's type information
# into this class's pytd.
bases = []
missing_bases = []
for basevar in v.bases():
if basevar.data == [self.vm.convert.oldstyleclass_type]:
continue
elif len(basevar.bindings) == 1:
b, = basevar.data
if b.official_name is None and isinstance(
b, abstract.InterpreterClass):
missing_bases.append(b)
else:
bases.append(b.get_instance_type(node))
else:
bases.append(
pytd_utils.JoinTypes(
b.get_instance_type(node) for b in basevar.data))
# Collect nested classes
# TODO(mdemello): We cannot put these in the output yet; they fail in
# load_dependencies because of the dotted class name (google/pytype#150)
classes = [
self._class_to_def(node, x, x.name) for x in v.get_inner_classes()
]
classes = [x.Replace(name=class_name + "." + x.name) for x in classes]
cls = pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(bases),
methods=tuple(methods.values()),
constants=tuple(constants),
classes=(),
slots=v.slots,
template=())
for base in missing_bases:
base_cls = self.value_to_pytd_def(node, base, base.name)
cls = pytd_utils.MergeBaseClass(cls, base_cls)
return cls
def pytd_for_types(self, defs):
# If a variable is annotated, we'll always output that type.
annotated_names = set()
data = []
annots = abstract_utils.get_annotations_dict(defs)
for name, t in self.ctx.pytd_convert.annotations_to_instance_types(
self.ctx.exitpoint, annots):
annotated_names.add(name)
data.append(pytd.Constant(name, t))
for name, var in defs.items():
if (name in abstract_utils.TOP_LEVEL_IGNORE
or name in annotated_names
or self._is_typing_member(name, var)):
continue
options = var.FilteredData(self.ctx.exitpoint, strict=False)
if (len(options) > 1 and not all(
isinstance(o, abstract.FUNCTION_TYPES) for o in options)):
if all(isinstance(o, abstract.TypeParameter) for o in options):
pytd_def = pytd_utils.JoinTypes(
t.to_pytd_def(self.ctx.exitpoint, name)
for t in options)
if isinstance(pytd_def, pytd.TypeParameter):
data.append(pytd_def)
else:
# We have multiple definitions for the same TypeVar name. There's no
# good way to handle this.
data.append(pytd.Constant(name, pytd.AnythingType()))
elif all(
isinstance(o, (abstract.ParameterizedClass,
abstract.Union))
for o in options): # type alias
pytd_def = pytd_utils.JoinTypes(
t.to_pytd_def(self.ctx.exitpoint, name).type
for t in options)
data.append(pytd.Alias(name, pytd_def))
else:
# It's ambiguous whether this is a type, a function or something
# else, so encode it as a constant.
combined_types = pytd_utils.JoinTypes(
t.to_type(self.ctx.exitpoint) for t in options)
data.append(pytd.Constant(name, combined_types))
elif options:
for option in options:
try:
d = option.to_pytd_def(self.ctx.exitpoint,
name) # Deep definition
except NotImplementedError:
d = option.to_type(self.ctx.exitpoint) # Type only
if isinstance(d, pytd.NothingType):
if isinstance(option, abstract.Empty):
d = pytd.AnythingType()
else:
assert isinstance(option,
typing_overlay.NoReturn)
if isinstance(d, pytd.Type) and not isinstance(
d, pytd.TypeParameter):
data.append(pytd.Constant(name, d))
else:
data.append(d)
else:
log.error("No visible options for %s", name)
data.append(pytd.Constant(name, pytd.AnythingType()))
return pytd_utils.WrapTypeDeclUnit("inferred", data)
def testJoinEmptyTypesToNothing(self):
"""Test that JoinTypes() simplifies empty unions to 'nothing'."""
self.assertIsInstance(pytd_utils.JoinTypes([]), pytd.NothingType)
def VisitParameter(self, p):
if p.mutated_type is None:
return p
else:
return p.Replace(type=pytd_utils.JoinTypes([p.type, p.mutated_type]),
mutated_type=None)
def _constant_to_value(self, pyval, subst, get_node):
"""Create a AtomicAbstractValue that represents a python constant.
This supports both constant from code constant pools and PyTD constants such
as classes. This also supports builtin python objects such as int and float.
Args:
pyval: The python or PyTD value to convert.
subst: The current type parameters.
get_node: A getter function for the current node.
Returns:
A Value that represents the constant, or None if we couldn't convert.
Raises:
NotImplementedError: If we don't know how to convert a value.
TypeParameterError: If we can't find a substitution for a type parameter.
"""
if isinstance(pyval, str):
return abstract.AbstractOrConcreteValue(pyval, self.str_type,
self.vm)
elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH:
# For small integers, preserve the actual value (for things like the
# level in IMPORT_NAME).
return abstract.AbstractOrConcreteValue(pyval, self.int_type,
self.vm)
elif isinstance(pyval, long):
# long is aliased to int
return self.primitive_class_instances[int]
elif pyval.__class__ in self.primitive_classes:
return self.primitive_class_instances[pyval.__class__]
elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)):
return abstract.AbstractOrConcreteValue(
pyval, self.primitive_classes[types.CodeType], self.vm)
elif pyval is super:
return special_builtins.Super(self.vm)
elif pyval is object:
return special_builtins.Object(self.vm)
elif pyval.__class__ is type:
if pyval is types.FunctionType:
classname = "typing.Callable"
else:
classname = "__builtin__." + pyval.__name__
try:
return self.name_to_value(classname, subst)
except (KeyError, AttributeError):
log.debug("Failed to find pytd", exc_info=True)
raise
elif isinstance(pyval, pytd.LateType):
actual = self._load_late_type(pyval)
return self._constant_to_value(actual, subst, get_node)
elif isinstance(pyval, pytd.TypeDeclUnit):
data = (pyval.constants + pyval.type_params + pyval.classes +
pyval.functions + pyval.aliases)
members = {val.name.rsplit(".")[-1]: val for val in data}
return abstract.Module(self.vm, pyval.name, members, pyval)
elif isinstance(pyval,
pytd.Class) and pyval.name == "__builtin__.super":
return self.vm.special_builtins["super"]
elif isinstance(pyval,
pytd.Class) and pyval.name == "__builtin__.object":
return self.object_type
elif isinstance(pyval,
pytd.Class) and pyval.name == "types.ModuleType":
return self.module_type
elif isinstance(pyval, pytd.Class):
module, dot, base_name = pyval.name.rpartition(".")
try:
cls = abstract.PyTDClass(base_name, pyval, self.vm)
except mro.MROError as e:
self.vm.errorlog.mro_error(self.vm.frames, base_name,
e.mro_seqs)
cls = self.unsolvable
else:
if dot:
cls.module = module
return cls
elif isinstance(pyval, pytd.Function):
signatures = [
abstract.PyTDSignature(pyval.name, sig, self.vm)
for sig in pyval.signatures
]
type_new = self.vm.lookup_builtin("__builtin__.type").Lookup(
"__new__")
if pyval is type_new:
f_cls = special_builtins.TypeNew
else:
f_cls = abstract.PyTDFunction
f = f_cls(pyval.name, signatures, pyval.kind, self.vm)
f.is_abstract = pyval.is_abstract
return f
elif isinstance(pyval, pytd.ClassType):
assert pyval.cls
return self.constant_to_value(pyval.cls, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.NothingType):
return self.nothing
elif isinstance(pyval, pytd.AnythingType):
return self.unsolvable
elif isinstance(pyval, pytd.FunctionType):
return self.constant_to_value(pyval.function, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.UnionType):
options = [
self.constant_to_value(t, subst, self.vm.root_cfg_node)
for t in pyval.type_list
]
if len(options) > 1:
return abstract.Union(options, self.vm)
else:
return options[0]
elif isinstance(pyval, pytd.TypeParameter):
constraints = tuple(
self.constant_to_value(c, {}, self.vm.root_cfg_node)
for c in pyval.constraints)
bound = (pyval.bound and self.constant_to_value(
pyval.bound, {}, self.vm.root_cfg_node))
return abstract.TypeParameter(pyval.name,
self.vm,
constraints=constraints,
bound=bound)
elif isinstance(pyval, abstract.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.LateType):
actual = self._load_late_type(cls)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
cls = actual.cls
if isinstance(cls, pytd.ClassType):
cls = cls.cls
if isinstance(cls,
pytd.GenericType) or (isinstance(cls, pytd.Class)
and cls.template):
# If we're converting a generic Class, need to create a new instance of
# it. See test_classes.testGenericReinstantiated.
if isinstance(cls, pytd.Class):
params = tuple(pytd.AnythingType() for t in cls.template)
cls = pytd.GenericType(base_type=pytd.ClassType(
cls.name, cls),
parameters=params)
if isinstance(cls.base_type, pytd.LateType):
actual = self._load_late_type(cls.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base_cls = actual.cls
else:
assert isinstance(cls.base_type, pytd.ClassType)
base_cls = cls.base_type.cls
assert isinstance(base_cls, pytd.Class), base_cls
if base_cls.name == "__builtin__.type":
c, = cls.parameters
if isinstance(c, pytd.TypeParameter):
if not subst or c.name not in subst:
raise self.TypeParameterError(c.name)
return self.merge_classes(get_node(),
subst[c.name].data)
else:
return self.constant_to_value(c, subst,
self.vm.root_cfg_node)
elif isinstance(cls, pytd.TupleType):
content = tuple(
self.constant_to_var(abstract.AsInstance(p), subst,
get_node())
for p in cls.parameters)
return abstract.Tuple(content, self.vm)
elif isinstance(cls, pytd.CallableType):
clsval = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
return abstract.Instance(clsval, self.vm)
else:
clsval = self.constant_to_value(base_cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(clsval, self.vm)
assert len(cls.parameters) <= len(base_cls.template)
for formal, actual in zip(base_cls.template,
cls.parameters):
p = self.constant_to_var(abstract.AsInstance(actual),
subst, get_node())
instance.initialize_type_parameter(
get_node(), formal.name, p)
return instance
elif isinstance(cls, pytd.Class):
# This key is also used in __init__
key = (abstract.Instance, cls)
if key not in self._convert_cache:
if cls.name in [
"__builtin__.type", "__builtin__.property"
]:
# An instance of "type" or of an anonymous property can be anything.
instance = self._create_new_unknown_value("type")
else:
mycls = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(mycls, self.vm)
instance.make_template_unsolvable(
cls.template, self.vm.root_cfg_node)
log.info("New pytd instance for %s: %r", cls.name,
instance)
self._convert_cache[key] = instance
return self._convert_cache[key]
else:
return self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.GenericType):
if isinstance(pyval.base_type, pytd.LateType):
actual = self._load_late_type(pyval.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base = actual.cls
else:
assert isinstance(pyval.base_type, pytd.ClassType)
base = pyval.base_type.cls
assert isinstance(base, pytd.Class), base
base_cls = self.constant_to_value(base, subst,
self.vm.root_cfg_node)
if not isinstance(base_cls, abstract.Class):
# base_cls can be, e.g., an unsolvable due to an mro error.
return self.unsolvable
if isinstance(pyval, pytd.TupleType):
abstract_class = abstract.TupleClass
template = range(len(pyval.parameters)) + [abstract.T]
parameters = pyval.parameters + (pytd.UnionType(
pyval.parameters), )
elif isinstance(pyval, pytd.CallableType):
abstract_class = abstract.Callable
template = range(len(
pyval.args)) + [abstract.ARGS, abstract.RET]
parameters = pyval.args + (pytd_utils.JoinTypes(
pyval.args), pyval.ret)
else:
abstract_class = abstract.ParameterizedClass
template = tuple(t.name for t in base.template)
parameters = pyval.parameters
assert (pyval.base_type.name == "typing.Generic"
or len(parameters) <= len(template))
type_parameters = utils.LazyDict()
for i, name in enumerate(template):
if i < len(parameters):
type_parameters.add_lazy_item(name, self.constant_to_value,
parameters[i], subst,
self.vm.root_cfg_node)
else:
type_parameters[name] = self.unsolvable
return abstract_class(base_cls, type_parameters, self.vm)
elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node
return self.tuple_to_value([
self.constant_to_var(item, subst, self.vm.root_cfg_node)
for i, item in enumerate(pyval)
])
else:
raise NotImplementedError("Can't convert constant %s %r" %
(type(pyval), pyval))
def _join_types(vals):
return pytd_utils.JoinTypes(v.to_type() for v in vals if v).Visit(
visitors.RemoveUnknownClasses())
def _class_to_def(self, node, v, class_name):
"""Convert an InterpreterClass to a PyTD definition."""
self._scopes.append(class_name)
methods = {}
constants = collections.defaultdict(pytd_utils.TypeBuilder)
annots = abstract_utils.get_annotations_dict(v.members)
annotated_names = set()
def add_constants(iterator):
for name, t in iterator:
if t is None:
# Remove the entry from constants
annotated_names.add(name)
elif name not in annotated_names:
constants[name].add_type(t)
annotated_names.add(name)
add_constants(
self._ordered_attrs_to_instance_types(node, v.metadata, annots))
add_constants(self.annotations_to_instance_types(node, annots))
def get_decorated_method(name, value, func_slot):
fvar = getattr(value, func_slot)
func = abstract_utils.get_atomic_value(fvar, abstract.Function)
defn = self.value_to_pytd_def(node, func, name)
defn = defn.Visit(visitors.DropMutableParameters())
return defn
def add_decorated_method(name, value, kind):
try:
defn = get_decorated_method(name, value, "func")
except (AttributeError, abstract_utils.ConversionError):
constants[name].add_type(pytd.AnythingType())
return
defn = defn.Replace(kind=kind)
methods[name] = defn
# If decorators are output as aliases to NamedTypes, they will be converted
# to Functions and fail a verification step if those functions have type
# parameters. Since we just want the function name, and since we have a
# fully resolved name at this stage, we just output a minimal pytd.Function
sig = pytd.Signature((), None, None, pytd.AnythingType(), (), ())
decorators = [
pytd.Alias(x, pytd.Function(x, (sig, ), pytd.MethodTypes.METHOD,
0)) for x in v.decorators
]
# class-level attributes
for name, member in v.members.items():
if name in CLASS_LEVEL_IGNORE or name in annotated_names:
continue
for value in member.FilteredData(self.vm.exitpoint, strict=False):
if isinstance(value, special_builtins.PropertyInstance):
# For simplicity, output properties as constants, since our parser
# turns them into constants anyway.
if value.fget:
for typ in self._function_to_return_types(
node, value.fget):
constants[name].add_type(
pytd.Annotated(typ, ("'property'", )))
else:
constants[name].add_type(
pytd.Annotated(pytd.AnythingType(),
("'property'", )))
elif isinstance(value, special_builtins.StaticMethodInstance):
add_decorated_method(name, value,
pytd.MethodTypes.STATICMETHOD)
elif isinstance(value, special_builtins.ClassMethodInstance):
add_decorated_method(name, value,
pytd.MethodTypes.CLASSMETHOD)
elif isinstance(value, abstract.Function):
# TODO(rechen): Removing mutations altogether won't work for generic
# classes. To support those, we'll need to change the mutated type's
# base to the current class, rename aliased type parameters, and
# replace any parameter not in the class or function template with
# its upper value.
methods[name] = self.value_to_pytd_def(
node, value,
name).Visit(visitors.DropMutableParameters())
else:
cls = self.vm.convert.merge_classes([value])
node, attr = self.vm.attribute_handler.get_attribute(
node, cls, "__get__")
if attr:
# This attribute is a descriptor. Its type is the return value of
# its __get__ method.
for typ in self._function_to_return_types(node, attr):
constants[name].add_type(typ)
else:
constants[name].add_type(value.to_type(node))
# Instance-level attributes: all attributes from 'canonical' instances (that
# is, ones created by analyze.py:analyze_class()) are added. Attributes from
# non-canonical instances are added if their canonical values do not contain
# type parameters.
ignore = set(annotated_names)
canonical_attributes = set()
def add_attributes_from(instance):
for name, member in instance.members.items():
if name in CLASS_LEVEL_IGNORE or name in ignore:
continue
for value in member.FilteredData(self.vm.exitpoint,
strict=False):
typ = value.to_type(node)
if pytd_utils.GetTypeParameters(typ):
# This attribute's type comes from an annotation that contains a
# type parameter; we do not want to merge in substituted values of
# the type parameter.
canonical_attributes.add(name)
constants[name].add_type(typ)
for instance in v.canonical_instances:
add_attributes_from(instance)
ignore |= canonical_attributes
for instance in v.instances - v.canonical_instances:
add_attributes_from(instance)
for name in list(methods):
if name in constants:
# If something is both a constant and a method, it means that the class
# is, at some point, overwriting its own methods with an attribute.
del methods[name]
constants[name].add_type(pytd.AnythingType())
constants = [
pytd.Constant(name, builder.build())
for name, builder in constants.items() if builder
]
metaclass = v.metaclass(node)
if metaclass is not None:
metaclass = metaclass.get_instance_type(node)
# Some of the class's bases may not be in global scope, so they won't show
# up in the output. In that case, fold the base class's type information
# into this class's pytd.
bases = []
missing_bases = []
for basevar in v.bases():
if basevar.data == [self.vm.convert.oldstyleclass_type]:
continue
elif len(basevar.bindings) == 1:
b, = basevar.data
if b.official_name is None and isinstance(
b, abstract.InterpreterClass):
missing_bases.append(b)
else:
bases.append(b.get_instance_type(node))
else:
bases.append(
pytd_utils.JoinTypes(
b.get_instance_type(node) for b in basevar.data))
# Collect nested classes
# TODO(mdemello): We cannot put these in the output yet; they fail in
# load_dependencies because of the dotted class name (google/pytype#150)
classes = [
self._class_to_def(node, x, x.name) for x in v.get_inner_classes()
]
classes = [x.Replace(name=class_name + "." + x.name) for x in classes]
cls = pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(bases),
methods=tuple(methods.values()),
constants=tuple(constants),
classes=(),
decorators=tuple(decorators),
slots=v.slots,
template=())
for base in missing_bases:
base_cls = self.value_to_pytd_def(node, base, base.name)
cls = pytd_utils.MergeBaseClass(cls, base_cls)
self._scopes.pop()
return cls
def _class_to_def(self, node, v, class_name):
"""Convert an InterpreterClass to a PyTD definition."""
self._scopes.append(class_name)
methods = {}
constants = collections.defaultdict(pytd_utils.TypeBuilder)
annots = abstract_utils.get_annotations_dict(v.members)
annotated_names = set()
def add_constants(iterator):
for name, t in iterator:
if t is None:
# Remove the entry from constants
annotated_names.add(name)
elif name not in annotated_names:
constants[name].add_type(t)
annotated_names.add(name)
add_constants(
self._ordered_attrs_to_instance_types(node, v.metadata, annots))
add_constants(self.annotations_to_instance_types(node, annots))
def add_final(defn, value):
if value.final:
return defn.Replace(flags=defn.flags | pytd.MethodFlag.FINAL)
else:
return defn
def get_decorated_method(name, value, func_slot):
fvar = getattr(value, func_slot)
func = abstract_utils.get_atomic_value(fvar, abstract.Function)
defn = self.value_to_pytd_def(node, func, name)
defn = defn.Visit(visitors.DropMutableParameters())
defn = add_final(defn, value)
return defn
def add_decorated_method(name, value, kind):
try:
defn = get_decorated_method(name, value, "func")
except (AttributeError, abstract_utils.ConversionError):
constants[name].add_type(pytd.AnythingType())
return
defn = defn.Replace(kind=kind)
methods[name] = defn
# If decorators are output as aliases to NamedTypes, they will be converted
# to Functions and fail a verification step if those functions have type
# parameters. Since we just want the function name, and since we have a
# fully resolved name at this stage, we just output a minimal pytd.Function
sig = pytd.Signature((), None, None, pytd.AnythingType(), (), ())
decorators = [
pytd.Alias(
x,
pytd.Function(x, (sig, ), pytd.MethodKind.METHOD,
pytd.MethodFlag.NONE)) for x in v.decorators
]
if v.final:
fn = pytd.Function("typing.final", (sig, ), pytd.MethodKind.METHOD,
pytd.MethodFlag.NONE)
decorators.append(pytd.Alias("final", fn))
# class-level attributes
for name, member in v.members.items():
if (name in abstract_utils.CLASS_LEVEL_IGNORE
or name in annotated_names
or (v.is_enum and name in ("__new__", "__eq__"))):
continue
for value in member.FilteredData(self.ctx.exitpoint, strict=False):
if isinstance(value, special_builtins.PropertyInstance):
# For simplicity, output properties as constants, since our parser
# turns them into constants anyway.
if value.fget:
for typ in self._function_to_return_types(
node, value.fget):
constants[name].add_type(
pytd.Annotated(typ, ("'property'", )))
else:
constants[name].add_type(
pytd.Annotated(pytd.AnythingType(),
("'property'", )))
elif isinstance(value, special_builtins.StaticMethodInstance):
add_decorated_method(name, value,
pytd.MethodKind.STATICMETHOD)
elif isinstance(value, special_builtins.ClassMethodInstance):
add_decorated_method(name, value,
pytd.MethodKind.CLASSMETHOD)
elif isinstance(value, abstract.Function):
# value_to_pytd_def returns different pytd node types depending on the
# input type, which pytype struggles to reason about.
method = cast(pytd.Function,
self.value_to_pytd_def(node, value, name))
keep = lambda name: not name or name.startswith(v.name)
signatures = tuple(
s for s in method.signatures
if not s.params or keep(s.params[0].type.name))
if signatures and signatures != method.signatures:
# Filter out calls made from subclasses unless they are the only
# ones recorded; when inferring types for ParentClass.__init__, we
# do not want `self: Union[ParentClass, Subclass]`.
method = method.Replace(signatures=signatures)
method = add_final(method, value)
# TODO(rechen): Removing mutations altogether won't work for generic
# classes. To support those, we'll need to change the mutated type's
# base to the current class, rename aliased type parameters, and
# replace any parameter not in the class or function template with
# its upper value.
methods[name] = method.Visit(
visitors.DropMutableParameters())
elif v.is_enum and self.ctx.options.use_enum_overlay:
if (any(
isinstance(enum_member, abstract.Instance)
and enum_member.cls == v
for enum_member in member.data)):
# i.e. if this is an enum that has any enum members, and the current
# member is an enum member.
# In this case, we would normally output:
# class M(enum.Enum):
# A: M
# However, this loses the type of A.value. Instead, annotate members
# with the type of their value. (This is what typeshed does.)
# class M(enum.Enum):
# A: int
enum_member = abstract_utils.get_atomic_value(member)
node, attr_var = self.ctx.attribute_handler.get_attribute(
node, enum_member, "value")
attr = abstract_utils.get_atomic_value(attr_var)
constants[name].add_type(attr.to_type(node))
else:
# i.e. this is an enum, and the current member is NOT an enum
# member. Which means it's a ClassVar.
cls_member = abstract_utils.get_atomic_value(member)
constants[name].add_type(
pytd.GenericType(
base_type=pytd.NamedType("typing.ClassVar"),
parameters=((cls_member.to_type(node), ))))
else:
cls = self.ctx.convert.merge_classes([value])
node, attr = self.ctx.attribute_handler.get_attribute(
node, cls, "__get__")
if attr:
# This attribute is a descriptor. Its type is the return value of
# its __get__ method.
for typ in self._function_to_return_types(node, attr):
constants[name].add_type(typ)
else:
constants[name].add_type(value.to_type(node))
# Instance-level attributes: all attributes from 'canonical' instances (that
# is, ones created by analyze.py:analyze_class()) are added. Attributes from
# non-canonical instances are added if their canonical values do not contain
# type parameters.
ignore = set(annotated_names)
# enums should not print "name" and "value" for instances.
if v.is_enum:
ignore.update(("name", "_name_", "value", "_value_"))
canonical_attributes = set()
def add_attributes_from(instance):
for name, member in instance.members.items():
if name in abstract_utils.CLASS_LEVEL_IGNORE or name in ignore:
continue
for value in member.FilteredData(self.ctx.exitpoint,
strict=False):
typ = value.to_type(node)
if pytd_utils.GetTypeParameters(typ):
# This attribute's type comes from an annotation that contains a
# type parameter; we do not want to merge in substituted values of
# the type parameter.
canonical_attributes.add(name)
if v.is_enum:
# If the containing class (v) is an enum, then output the instance
# attributes as properties.
# https://typing.readthedocs.io/en/latest/stubs.html#enums
typ = pytd.Annotated(typ, ("'property'", ))
constants[name].add_type(typ)
for instance in v.canonical_instances:
add_attributes_from(instance)
ignore |= canonical_attributes
for instance in v.instances - v.canonical_instances:
add_attributes_from(instance)
for name in list(methods):
if name in constants:
# If something is both a constant and a method, it means that the class
# is, at some point, overwriting its own methods with an attribute.
del methods[name]
constants[name].add_type(pytd.AnythingType())
constants = [
pytd.Constant(name, builder.build())
for name, builder in constants.items() if builder
]
metaclass = v.metaclass(node)
if metaclass is not None:
metaclass = metaclass.get_instance_type(node)
# Some of the class's bases may not be in global scope, so they won't show
# up in the output. In that case, fold the base class's type information
# into this class's pytd.
bases = []
missing_bases = []
for basevar in v.bases():
if len(basevar.bindings) == 1:
b, = basevar.data
if b.official_name is None and isinstance(
b, abstract.InterpreterClass):
missing_bases.append(b)
else:
bases.append(b.get_instance_type(node))
else:
bases.append(
pytd_utils.JoinTypes(
b.get_instance_type(node) for b in basevar.data))
# Collect nested classes
# TODO(mdemello): We cannot put these in the output yet; they fail in
# load_dependencies because of the dotted class name (google/pytype#150)
classes = [
self._class_to_def(node, x, x.name) for x in v.get_inner_classes()
]
classes = [x.Replace(name=class_name + "." + x.name) for x in classes]
cls = pytd.Class(name=class_name,
metaclass=metaclass,
bases=tuple(bases),
methods=tuple(methods.values()),
constants=tuple(constants),
classes=(),
decorators=tuple(decorators),
slots=v.slots,
template=())
for base in missing_bases:
base_cls = self.value_to_pytd_def(node, base, base.name)
cls = pytd_utils.MergeBaseClass(cls, base_cls)
self._scopes.pop()
return cls
def VisitUnionType(self, union):
"""Push unions down into containers.
This collects similar container types in unions and merges them into
single instances with the union type pushed down to the element_type level.
Arguments:
union: A pytd.Union instance. Might appear in a parameter, a return type,
a constant type, etc.
Returns:
A simplified pytd.Union.
"""
if not any(isinstance(t, pytd.GenericType) for t in union.type_list):
# Optimization: If we're not going to change anything, return original.
return union
union = pytd_utils.JoinTypes(union.type_list) # flatten
if not isinstance(union, pytd.UnionType):
union = pytd.UnionType((union, ))
merge_tuples = self._should_merge(pytd.TupleType, union)
merge_callables = self._should_merge(pytd.CallableType, union)
if merge_tuples or merge_callables:
type_list = []
for t in union.type_list:
if merge_tuples and isinstance(t, pytd.TupleType):
t = pytd.GenericType(base_type=t.base_type,
parameters=(pytd.UnionType(
t.parameters), ))
elif merge_callables and isinstance(t, pytd.CallableType):
t = pytd.GenericType(base_type=t.base_type,
parameters=(pytd.AnythingType(),
t.ret))
type_list.append(t)
union = union.Replace(type_list=tuple(type_list))
collect = {}
has_redundant_base_types = False
for t in union.type_list:
if isinstance(t, pytd.GenericType):
key = self._key(t)
if key in collect:
has_redundant_base_types = True
collect[key] = tuple(
pytd_utils.JoinTypes([p1, p2])
for p1, p2 in zip(collect[key], t.parameters))
else:
collect[key] = t.parameters
if not has_redundant_base_types:
return union
result = pytd.NothingType()
done = set()
for t in union.type_list:
if isinstance(t, pytd.GenericType):
key = self._key(t)
if key in done:
continue # already added
parameters = collect[key]
add = t.Replace(parameters=tuple(
p.Visit(CombineContainers()) for p in parameters))
done.add(key)
else:
add = t
result = pytd_utils.JoinTypes([result, add])
return result
def _join_type_defs(type_defs):
return pytd_utils.JoinTypes(v.to_type() for v in type_defs)
def testJoinSingleType(self):
"""Test that JoinTypes() returns single types as-is."""
a = pytd.NamedType("a")
self.assertEqual(pytd_utils.JoinTypes([a]), a)
self.assertEqual(pytd_utils.JoinTypes([a, a]), a)
def _class_to_def(self, node, v, class_name):
"""Convert an InterpreterClass to a PyTD definition."""
methods = {}
constants = collections.defaultdict(pytd_utils.TypeBuilder)
# class-level attributes
for name, member in v.members.items():
if name in CLASS_LEVEL_IGNORE:
continue
for value in member.FilteredData(self.vm.exitpoint):
if isinstance(value, special_builtins.PropertyInstance):
# For simplicity, output properties as constants, since our parser
# turns them into constants anyway.
if value.fget:
for typ in self._function_to_return_types(
node, value.fget):
constants[name].add_type(typ)
else:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, special_builtins.StaticMethodInstance):
try:
methods[name] = self._static_method_to_def(
node, value, name, pytd.STATICMETHOD)
except abstract.ConversionError:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, special_builtins.ClassMethodInstance):
try:
methods[name] = self._class_method_to_def(
node, value, name, pytd.CLASSMETHOD)
except abstract.ConversionError:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, abstract.Function):
# TODO(rechen): Removing mutations altogether won't work for generic
# classes. To support those, we'll need to change the mutated type's
# base to the current class, rename aliased type parameters, and
# replace any parameter not in the class or function template with
# its upper value.
methods[name] = self.value_to_pytd_def(
node, value,
name).Visit(visitors.DropMutableParameters())
else:
cls = self.vm.convert.merge_classes(node, [value])
node, attr = self.vm.attribute_handler.get_attribute(
node, cls, "__get__")
if attr:
# This attribute is a descriptor. Its type is the return value of
# its __get__ method.
for typ in self._function_to_return_types(node, attr):
constants[name].add_type(typ)
else:
constants[name].add_type(value.to_type(node))
# instance-level attributes
for instance in v.instances:
for name, member in instance.members.items():
if name not in CLASS_LEVEL_IGNORE:
for value in member.FilteredData(self.vm.exitpoint):
constants[name].add_type(value.to_type(node))
for name in list(methods):
if name in constants:
# If something is both a constant and a method, it means that the class
# is, at some point, overwriting its own methods with an attribute.
del methods[name]
constants[name].add_type(pytd.AnythingType())
bases = [
pytd_utils.JoinTypes(
b.get_instance_type(node) for b in basevar.data)
for basevar in v.bases()
if basevar.data != [self.vm.convert.oldstyleclass_type]
]
constants = [
pytd.Constant(name, builder.build())
for name, builder in constants.items() if builder
]
metaclass = v.metaclass(node)
if metaclass is not None:
metaclass = metaclass.get_instance_type(node)
return pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(bases),
methods=tuple(methods.values()),
constants=tuple(constants),
slots=v.slots,
template=())
def testJoinAnythingTypes(self):
"""Test that JoinTypes() simplifies unions containing '?'."""
types = [pytd.AnythingType(), pytd.NamedType("a")]
self.assertIsInstance(pytd_utils.JoinTypes(types), pytd.AnythingType)
def _constant_to_value(self, pyval, subst, get_node):
"""Create a BaseValue that represents a python constant.
This supports both constant from code constant pools and PyTD constants such
as classes. This also supports builtin python objects such as int and float.
Args:
pyval: The python or PyTD value to convert.
subst: The current type parameters.
get_node: A getter function for the current node.
Returns:
A Value that represents the constant, or None if we couldn't convert.
Raises:
NotImplementedError: If we don't know how to convert a value.
TypeParameterError: If we can't find a substitution for a type parameter.
"""
if isinstance(pyval, str):
return abstract.ConcreteValue(pyval, self.str_type, self.ctx)
elif isinstance(pyval, bytes):
return abstract.ConcreteValue(pyval, self.bytes_type, self.ctx)
elif isinstance(pyval, bool):
return self.true if pyval else self.false
elif isinstance(pyval, int) and -1 <= pyval <= _MAX_IMPORT_DEPTH:
# For small integers, preserve the actual value (for things like the
# level in IMPORT_NAME).
return abstract.ConcreteValue(pyval, self.int_type, self.ctx)
elif pyval.__class__ in self.primitive_classes:
return self.primitive_class_instances[pyval.__class__]
elif pyval.__class__ is frozenset:
instance = abstract.Instance(self.frozenset_type, self.ctx)
for element in pyval:
instance.merge_instance_type_parameter(
self.ctx.root_node, abstract_utils.T,
self.constant_to_var(element, subst, self.ctx.root_node))
return instance
elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)):
return abstract.ConcreteValue(pyval,
self.primitive_classes[types.CodeType],
self.ctx)
elif pyval is super:
return special_builtins.Super(self.ctx)
elif pyval is object:
return special_builtins.Object(self.ctx)
elif pyval.__class__ is type:
try:
return self.name_to_value(self._type_to_name(pyval), subst)
except (KeyError, AttributeError):
log.debug("Failed to find pytd", exc_info=True)
raise
elif isinstance(pyval, pytd.LateType):
actual = self._load_late_type(pyval)
return self._constant_to_value(actual, subst, get_node)
elif isinstance(pyval, pytd.TypeDeclUnit):
return self._create_module(pyval)
elif isinstance(pyval, pytd.Module):
mod = self.ctx.loader.import_name(pyval.module_name)
return self._create_module(mod)
elif isinstance(pyval, pytd.Class):
if pyval.name == "builtins.super":
return self.ctx.special_builtins["super"]
elif pyval.name == "builtins.object":
return self.object_type
elif pyval.name == "types.ModuleType":
return self.module_type
elif pyval.name == "_importlib_modulespec.ModuleType":
# Python 3's typeshed uses a stub file indirection to define ModuleType
# even though it is exported via types.pyi.
return self.module_type
elif pyval.name == "types.FunctionType":
return self.function_type
else:
module, dot, base_name = pyval.name.rpartition(".")
# typing.TypingContainer intentionally loads the underlying pytd types.
if (module not in ("typing", "typing_extensions") and
module in overlay_dict.overlays):
overlay = self.ctx.vm.import_module(module, module, 0)
if overlay.get_module(base_name) is overlay:
overlay.load_lazy_attribute(base_name)
return abstract_utils.get_atomic_value(overlay.members[base_name])
try:
cls = abstract.PyTDClass.make(base_name, pyval, self.ctx)
except mro.MROError as e:
self.ctx.errorlog.mro_error(self.ctx.vm.frames, base_name, e.mro_seqs)
cls = self.unsolvable
else:
if dot:
cls.module = module
cls.call_metaclass_init(get_node())
return cls
elif isinstance(pyval, pytd.Function):
signatures = [
abstract.PyTDSignature(pyval.name, sig, self.ctx)
for sig in pyval.signatures
]
type_new = self.ctx.loader.lookup_builtin("builtins.type").Lookup(
"__new__")
if pyval is type_new:
f_cls = special_builtins.TypeNew
else:
f_cls = abstract.PyTDFunction
f = f_cls(pyval.name, signatures, pyval.kind, self.ctx)
f.is_abstract = pyval.is_abstract
return f
elif isinstance(pyval, pytd.ClassType):
if pyval.cls:
cls = pyval.cls
else:
# If pyval is a reference to a class in builtins or typing, we can fill
# in the class ourselves. lookup_builtin raises a KeyError if the name
# is not found.
cls = self.ctx.loader.lookup_builtin(pyval.name)
assert isinstance(cls, pytd.Class)
return self.constant_to_value(cls, subst, self.ctx.root_node)
elif isinstance(pyval, pytd.NothingType):
return self.empty
elif isinstance(pyval, pytd.AnythingType):
return self.unsolvable
elif (isinstance(pyval, pytd.Constant) and
isinstance(pyval.type, pytd.AnythingType)):
# We allow "X = ... # type: Any" to declare X as a type.
return self.unsolvable
elif (isinstance(pyval, pytd.Constant) and
isinstance(pyval.type, pytd.GenericType) and
pyval.type.name == "builtins.type"):
# `X: Type[other_mod.X]` is equivalent to `X = other_mod.X`.
param, = pyval.type.parameters
return self.constant_to_value(param, subst, self.ctx.root_node)
elif isinstance(pyval, pytd.UnionType):
options = [
self.constant_to_value(t, subst, self.ctx.root_node)
for t in pyval.type_list
]
if len(options) > 1:
return abstract.Union(options, self.ctx)
else:
return options[0]
elif isinstance(pyval, pytd.TypeParameter):
constraints = tuple(
self.constant_to_value(c, {}, self.ctx.root_node)
for c in pyval.constraints)
bound = (
pyval.bound and
self.constant_to_value(pyval.bound, {}, self.ctx.root_node))
return abstract.TypeParameter(
pyval.name,
self.ctx,
constraints=constraints,
bound=bound,
module=pyval.scope)
elif isinstance(pyval, abstract_utils.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.LateType):
actual = self._load_late_type(cls)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
cls = actual.cls
if isinstance(cls, pytd.ClassType):
cls = cls.cls
if isinstance(cls, pytd.GenericType) and cls.name == "typing.ClassVar":
param, = cls.parameters
return self.constant_to_value(
abstract_utils.AsInstance(param), subst, self.ctx.root_node)
elif isinstance(cls, pytd.GenericType) or (isinstance(cls, pytd.Class) and
cls.template):
# If we're converting a generic Class, need to create a new instance of
# it. See test_classes.testGenericReinstantiated.
if isinstance(cls, pytd.Class):
params = tuple(t.type_param.upper_value for t in cls.template)
cls = pytd.GenericType(base_type=pytd.ClassType(cls.name, cls),
parameters=params)
if isinstance(cls.base_type, pytd.LateType):
actual = self._load_late_type(cls.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base_cls = actual.cls
else:
base_type = cls.base_type
assert isinstance(base_type, pytd.ClassType)
base_cls = base_type.cls
assert isinstance(base_cls, pytd.Class), base_cls
if base_cls.name == "builtins.type":
c, = cls.parameters
if isinstance(c, pytd.TypeParameter):
if not subst or c.full_name not in subst:
raise self.TypeParameterError(c.full_name)
# deformalize gets rid of any unexpected TypeVars, which can appear
# if something is annotated as Type[T].
return self.ctx.annotation_utils.deformalize(
self.merge_classes(subst[c.full_name].data))
else:
return self.constant_to_value(c, subst, self.ctx.root_node)
elif isinstance(cls, pytd.TupleType):
content = tuple(self.constant_to_var(abstract_utils.AsInstance(p),
subst, get_node())
for p in cls.parameters)
return self.tuple_to_value(content)
elif isinstance(cls, pytd.CallableType):
clsval = self.constant_to_value(cls, subst, self.ctx.root_node)
return abstract.Instance(clsval, self.ctx)
else:
clsval = self.constant_to_value(base_cls, subst, self.ctx.root_node)
instance = abstract.Instance(clsval, self.ctx)
num_params = len(cls.parameters)
assert num_params <= len(base_cls.template)
for i, formal in enumerate(base_cls.template):
if i < num_params:
node = get_node()
p = self.constant_to_var(
abstract_utils.AsInstance(cls.parameters[i]), subst, node)
else:
# An omitted type parameter implies `Any`.
node = self.ctx.root_node
p = self.unsolvable.to_variable(node)
instance.merge_instance_type_parameter(node, formal.name, p)
return instance
elif isinstance(cls, pytd.Class):
assert not cls.template
# This key is also used in __init__
key = (abstract.Instance, cls)
if key not in self._convert_cache:
if cls.name in ["builtins.type", "builtins.property"]:
# An instance of "type" or of an anonymous property can be anything.
instance = self._create_new_unknown_value("type")
else:
mycls = self.constant_to_value(cls, subst, self.ctx.root_node)
instance = abstract.Instance(mycls, self.ctx)
log.info("New pytd instance for %s: %r", cls.name, instance)
self._convert_cache[key] = instance
return self._convert_cache[key]
elif isinstance(cls, pytd.Literal):
return self.constant_to_value(
self._get_literal_value(cls.value), subst, self.ctx.root_node)
else:
return self.constant_to_value(cls, subst, self.ctx.root_node)
elif (isinstance(pyval, pytd.GenericType) and
pyval.name == "typing.ClassVar"):
param, = pyval.parameters
return self.constant_to_value(param, subst, self.ctx.root_node)
elif isinstance(pyval, pytd.GenericType):
if isinstance(pyval.base_type, pytd.LateType):
actual = self._load_late_type(pyval.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base = actual.cls
else:
assert isinstance(pyval.base_type, pytd.ClassType), pyval
base = pyval.base_type.cls
assert isinstance(base, pytd.Class), base
base_cls = self.constant_to_value(base, subst, self.ctx.root_node)
if not isinstance(base_cls, abstract.Class):
# base_cls can be, e.g., an unsolvable due to an mro error.
return self.unsolvable
if isinstance(pyval, pytd.TupleType):
abstract_class = abstract.TupleClass
template = list(range(len(pyval.parameters))) + [abstract_utils.T]
combined_parameter = pytd_utils.JoinTypes(pyval.parameters)
parameters = pyval.parameters + (combined_parameter,)
elif isinstance(pyval, pytd.CallableType):
abstract_class = abstract.CallableClass
template = list(range(len(pyval.args))) + [abstract_utils.ARGS,
abstract_utils.RET]
parameters = pyval.args + (pytd_utils.JoinTypes(pyval.args), pyval.ret)
else:
abstract_class = abstract.ParameterizedClass
if pyval.name == "typing.Generic":
pyval_template = pyval.parameters
else:
pyval_template = base.template
template = tuple(t.name for t in pyval_template)
parameters = pyval.parameters
assert (pyval.name in ("typing.Generic", "typing.Protocol") or
len(parameters) <= len(template))
# Delay type parameter loading to handle recursive types.
# See the ParameterizedClass.formal_type_parameters() property.
type_parameters = abstract_utils.LazyFormalTypeParameters(
template, parameters, subst)
return abstract_class(base_cls, type_parameters, self.ctx)
elif isinstance(pyval, pytd.Literal):
value = self.constant_to_value(
self._get_literal_value(pyval.value), subst, self.ctx.root_node)
return abstract.LiteralClass(value, self.ctx)
elif isinstance(pyval, pytd.Annotated):
typ = self.constant_to_value(pyval.base_type, subst, self.ctx.root_node)
if pyval.annotations[0] == "'pytype_metadata'":
try:
md = metadata.from_string(pyval.annotations[1])
if md["tag"] == "attr.ib":
ret = attr_overlay.AttribInstance.from_metadata(
self.ctx, self.ctx.root_node, typ, md)
return ret
elif md["tag"] == "attr.s":
ret = attr_overlay.Attrs.from_metadata(self.ctx, md)
return ret
except (IndexError, ValueError, TypeError, KeyError):
details = "Wrong format for pytype_metadata."
self.ctx.errorlog.invalid_annotation(self.ctx.vm.frames,
pyval.annotations[1], details)
return typ
else:
return typ
elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node
return self.tuple_to_value([
self.constant_to_var(item, subst, self.ctx.root_node)
for i, item in enumerate(pyval)
])
else:
raise NotImplementedError("Can't convert constant %s %r" %
(type(pyval), pyval))
def value_to_pytd_type(self, node, v, seen, view):
"""Get a PyTD type representing this object, as seen at a node.
Args:
node: The node from which we want to observe this object.
v: The object.
seen: The set of values seen before while computing the type.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, (abstract.Empty, typing_overlay.NoReturn)):
return pytd.NothingType()
elif isinstance(v, abstract.TypeParameterInstance):
if v.instance.get_instance_type_parameter(v.full_name).bindings:
# The type parameter was initialized. Set the view to None, since we
# don't include v.instance in the view.
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, p, seen, None) for p in
v.instance.get_instance_type_parameter(v.full_name).data)
elif v.param.constraints:
return pytd_utils.JoinTypes(
self.value_instance_to_pytd_type(node, p, None, seen, view)
for p in v.param.constraints)
elif v.param.bound:
return self.value_instance_to_pytd_type(
node, v.param.bound, None, seen, view)
else:
return pytd.AnythingType()
elif isinstance(v, typing_overlay.TypeVar):
return pytd.NamedType("__builtin__.type")
elif isinstance(v, abstract.FUNCTION_TYPES):
try:
signatures = abstract_utils.get_signatures(v)
except NotImplementedError:
return pytd.NamedType("typing.Callable")
if len(signatures) == 1:
val = self.signature_to_callable(signatures[0])
if not isinstance(
v, abstract.PYTD_FUNCTION_TYPES) or not val.formal:
# This is a workaround to make sure we don't put unexpected type
# parameters in call traces.
return self.value_instance_to_pytd_type(
node, val, None, seen, view)
return pytd.NamedType("typing.Callable")
elif isinstance(v, (abstract.ClassMethod, abstract.StaticMethod)):
return self.value_to_pytd_type(node, v.method, seen, view)
elif isinstance(v, (special_builtins.IsInstance,
special_builtins.ClassMethodCallable)):
return pytd.NamedType("typing.Callable")
elif isinstance(v, mixin.Class):
param = self.value_instance_to_pytd_type(node, v, None, seen, view)
return pytd.GenericType(
base_type=pytd.NamedType("__builtin__.type"),
parameters=(param, ))
elif isinstance(v, abstract.Module):
return pytd.NamedType("__builtin__.module")
elif isinstance(v, abstract.SimpleAbstractValue):
if v.cls:
ret = self.value_instance_to_pytd_type(node,
v.cls,
v,
seen=seen,
view=view)
ret.Visit(
visitors.FillInLocalPointers(
{"__builtin__": self.vm.loader.builtins}))
return ret
else:
# We don't know this type's __class__, so return AnythingType to
# indicate that we don't know anything about what this is.
# This happens e.g. for locals / globals, which are returned from the
# code in class declarations.
log.info("Using ? for %s", v.name)
return pytd.AnythingType()
elif isinstance(v, abstract.Union):
return pytd.UnionType(
tuple(
self.value_to_pytd_type(node, o, seen, view)
for o in v.options))
elif isinstance(v, special_builtins.SuperInstance):
return pytd.NamedType("__builtin__.super")
elif isinstance(v, abstract.TypeParameter):
# Arguably, the type of a type parameter is NamedType("typing.TypeVar"),
# but pytype doesn't know how to handle that, so let's just go with Any
# unless self._detailed is set.
if self._detailed:
return pytd.NamedType("typing.TypeVar")
else:
return pytd.AnythingType()
elif isinstance(v, abstract.Unsolvable):
return pytd.AnythingType()
elif isinstance(v, abstract.Unknown):
return pytd.NamedType(v.class_name)
elif isinstance(v, abstract.BuildClass):
return pytd.NamedType("typing.Callable")
else:
raise NotImplementedError(v.__class__.__name__)
def uninitialized_annotations_to_instance_types(self, node, annots, members):
"""Get instance types for annotations not present in the members map."""
for name in annots:
if name not in members:
yield name, pytd_utils.JoinTypes(
value.get_instance_type(node) for value in annots[name].data)
def assertOnlyHasReturnType(self, func, t):
"""Test that a given return type is the only one."""
ret = pytd_utils.JoinTypes(sig.return_type for sig in func.signatures)
self.assertEqual(
t, ret, "Return type %r != %r" %
(pytd_utils.Print(t), pytd_utils.Print(ret)))
def value_to_pytd_type(self, node, v, seen, view):
"""Get a PyTD type representing this object, as seen at a node.
Args:
node: The node from which we want to observe this object.
v: The object.
seen: The set of values seen before while computing the type.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, (abstract.Empty, typing.NoReturn)):
return pytd.NothingType()
elif isinstance(v, abstract.TypeParameterInstance):
if v.instance.type_parameters[v.name].bindings:
# The type parameter was initialized.
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, p, seen, view)
for p in v.instance.type_parameters[v.name].data)
elif v.param.constraints:
return pytd_utils.JoinTypes(
self.value_instance_to_pytd_type(node, p, None, seen, view)
for p in v.param.constraints)
elif v.param.bound:
return self.value_instance_to_pytd_type(
node, v.param.bound, None, seen, view)
else:
return pytd.AnythingType()
elif isinstance(v, typing.TypeVar):
return pytd.NamedType("__builtin__.type")
elif isinstance(
v,
(abstract.InterpreterFunction, abstract.BoundInterpreterFunction)):
sig, = abstract.get_signatures(v)
return self.value_instance_to_pytd_type(
node, self.signature_to_callable(sig, v.vm), None, seen, view)
elif isinstance(v,
(abstract.PyTDFunction, abstract.BoundPyTDFunction)):
signatures = abstract.get_signatures(v)
if len(signatures) == 1:
val = self.signature_to_callable(signatures[0], v.vm)
if not v.vm.annotations_util.get_type_parameters(val):
# This is a workaround to make sure we don't put unexpected type
# parameters in call traces.
return self.value_instance_to_pytd_type(
node, val, None, seen, view)
return pytd.NamedType("typing.Callable")
elif isinstance(
v,
(special_builtins.IsInstance, abstract.ClassMethod,
abstract.StaticMethod, special_builtins.ClassMethodCallable)):
return pytd.NamedType("typing.Callable")
elif isinstance(v, abstract.Class):
param = self.value_instance_to_pytd_type(node, v, None, seen, view)
return pytd.GenericType(
base_type=pytd.NamedType("__builtin__.type"),
parameters=(param, ))
elif isinstance(v, abstract.Module):
return pytd.NamedType("__builtin__.module")
elif isinstance(v, abstract.SimpleAbstractValue):
if v.cls:
classvalues = self._get_values(node, v.cls, view)
cls_types = []
for cls in classvalues:
cls_types.append(
self.value_instance_to_pytd_type(node,
cls,
v,
seen=seen,
view=view))
ret = pytd_utils.JoinTypes(cls_types)
ret.Visit(
visitors.FillInLocalPointers(
{"__builtin__": v.vm.loader.builtins}))
return ret
else:
# We don't know this type's __class__, so return AnythingType to
# indicate that we don't know anything about what this is.
# This happens e.g. for locals / globals, which are returned from the
# code in class declarations.
log.info("Using ? for %s", v.name)
return pytd.AnythingType()
elif isinstance(v, abstract.Union):
return pytd.UnionType(
tuple(
self.value_to_pytd_type(node, o, seen, view)
for o in v.options))
elif isinstance(v, special_builtins.SuperInstance):
return pytd.NamedType("__builtin__.super")
elif isinstance(v, (abstract.Unsolvable, abstract.TypeParameter)):
# Arguably, the type of a type parameter is NamedType("typing.TypeVar"),
# but pytype doesn't know how to handle that, so let's just go with Any.
return pytd.AnythingType()
elif isinstance(v, abstract.Unknown):
return pytd.NamedType(v.class_name)
else:
raise NotImplementedError(v.__class__.__name__)