def add_import(self, from_package, import_list):
"""Add an import.
Args:
from_package: A dotted package name if this is a "from" statement, or None
if it is an "import" statement.
import_list: A list of imported items, which are either strings or pairs
of strings. Pairs are used when items are renamed during import
using "as".
"""
if not self._current_condition.active:
return
if from_package:
# from a.b.c import d, ...
for item in import_list:
if isinstance(item, tuple):
name, new_name = item
else:
name = new_name = item
qualified_name = "%s.%s" % (from_package, name)
if from_package == "__PACKAGE__" and isinstance(item, str):
# This will always be a simple module import (from . cannot import a
# NamedType, and without 'as' the name will not be reexported).
t = pytd.Module(name=new_name, module_name=qualified_name)
else:
# We should ideally not need this check, but we have typing
# special-cased in some places.
if not qualified_name.startswith("typing.") and name != "*":
# Mark this as an externally imported type, so that AddNamePrefix
# does not prefix it with the current package name.
qualified_name = (parser_constants.EXTERNAL_NAME_PREFIX +
qualified_name)
t = pytd.NamedType(qualified_name)
if name == "*":
# A star import is stored as
# 'imported_mod.* = imported_mod.*'. The imported module needs to be
# in the alias name so that multiple star imports are handled
# properly. LookupExternalTypes() replaces the alias with the
# contents of the imported module.
assert new_name == name
new_name = t.name
self._type_map[new_name] = t
if from_package != "typing" or self._ast_name == "protocols":
self._aliases.append(pytd.Alias(new_name, t))
self._module_path_map[name] = qualified_name
else:
# import a, b as c, ...
for item in import_list:
if isinstance(item, tuple):
name, new_name = item
t = pytd.Module(name=new_name, module_name=name)
self._aliases.append(pytd.Alias(new_name, t))
else:
# We don't care about imports that are not aliased.
pass
def _add_module(self, import_item, from_package):
if isinstance(import_item, tuple):
assert len(import_item) == 2
name, alias = import_item
self._modules.append(pytd.Module(name=alias, module_name=name,
from_package=from_package))
# aliases with from_package are added to module_path_map in add_import.
if not from_package:
self._module_path_map[alias] = name
else:
self._modules.append(
pytd.Module(name=import_item, module_name=import_item,
from_package=from_package))
def process_from_import(self, from_package, item):
"""Process 'from a.b.c import d, ...'."""
if isinstance(item, tuple):
name, new_name = item
else:
name = new_name = item
qualified_name = self.qualify_name("%s.%s" % (from_package, name))
if (from_package in ["__PACKAGE__", "__PARENT__"]
and isinstance(item, str)):
# This will always be a simple module import (from . cannot import a
# NamedType, and without 'as' the name will not be reexported).
t = pytd.Module(name=new_name, module_name=qualified_name)
else:
# We should ideally not need this check, but we have typing
# special-cased in some places.
if not qualified_name.startswith("typing.") and name != "*":
# Mark this as an externally imported type, so that AddNamePrefix
# does not prefix it with the current package name.
qualified_name = (parser_constants.EXTERNAL_NAME_PREFIX +
qualified_name)
t = pytd.NamedType(qualified_name)
if name == "*":
# A star import is stored as
# 'imported_mod.* = imported_mod.*'. The imported module needs to be
# in the alias name so that multiple star imports are handled
# properly. LookupExternalTypes() replaces the alias with the
# contents of the imported module.
assert new_name == name
new_name = t.name
return Import(pytd_node=t,
name=name,
new_name=new_name,
qualified_name=qualified_name)
def value_to_pytd_def(self, node, v, name):
"""Get a PyTD definition for this object.
Args:
node: The node.
v: The object.
name: The object name.
Returns:
A PyTD definition.
"""
if isinstance(v, abstract.Module):
return pytd.Alias(name, pytd.Module(name, module_name=v.full_name))
elif isinstance(v, abstract.BoundFunction):
d = self.value_to_pytd_def(node, v.underlying, name)
assert isinstance(d, pytd.Function)
sigs = tuple(
sig.Replace(params=sig.params[1:]) for sig in d.signatures)
return d.Replace(signatures=sigs)
elif isinstance(v, attr_overlay.Attrs):
ret = pytd.NamedType("typing.Callable")
md = metadata.to_pytd(v.to_metadata())
return pytd.Annotated(ret, ("'pytype_metadata'", md))
elif (isinstance(v, abstract.PyTDFunction)
and not isinstance(v, typing_overlay.TypeVar)):
return pytd.Function(
name=name,
signatures=tuple(sig.pytd_sig for sig in v.signatures),
kind=v.kind,
flags=pytd.MethodFlag.abstract_flag(v.is_abstract))
elif isinstance(v, abstract.InterpreterFunction):
return self._function_to_def(node, v, name)
elif isinstance(v, abstract.SimpleFunction):
return self._simple_func_to_def(node, v, name)
elif isinstance(v, (abstract.ParameterizedClass, abstract.Union)):
return pytd.Alias(name, v.get_instance_type(node))
elif isinstance(v, abstract.PyTDClass) and v.module:
# This happens if a module does e.g. "from x import y as z", i.e., copies
# something from another module to the local namespace. We *could*
# reproduce the entire class, but we choose a more dense representation.
return v.to_type(node)
elif isinstance(v, typed_dict.TypedDictClass):
return self._typed_dict_to_def(node, v, name)
elif isinstance(v, abstract.PyTDClass): # a namedtuple instance
assert name != v.name
return pytd.Alias(name, pytd.NamedType(v.name))
elif isinstance(v, abstract.InterpreterClass):
if v.official_name is None or name == v.official_name:
return self._class_to_def(node, v, name)
else:
return pytd.Alias(name, pytd.NamedType(v.official_name))
elif isinstance(v, abstract.TypeParameter):
return self._typeparam_to_def(node, v, name)
elif isinstance(v, abstract.Unsolvable):
return pytd.Constant(name, v.to_type(node))
else:
raise NotImplementedError(v.__class__.__name__)
def process_import(self, item):
"""Process 'import a, b as c, ...'."""
if not isinstance(item, tuple):
# We don't care about imports that are not aliased.
return None
name, new_name = item
module_name = self.qualify_name(name)
as_name = self.qualify_name(new_name)
t = pytd.Module(name=as_name, module_name=module_name)
return Import(pytd_node=t, name=name, new_name=new_name)
def process_import(self, item):
"""Process 'import a, b as c, ...'."""
if isinstance(item, tuple):
name, new_name = item
else:
name = new_name = item
if name == new_name == "__builtin__":
# 'import __builtin__' should be completely ignored; this is the PY2 name
# of the builtins module.
return None
module_name = self.qualify_name(name)
as_name = self.qualify_name(new_name)
t = pytd.Module(name=as_name, module_name=module_name)
return Import(pytd_node=t, name=name, new_name=new_name)
def _load_late_type(self, late_type):
"""Resolve a late type, possibly by loading a module."""
if late_type.name not in self._resolved_late_types:
ast = self.ctx.loader.import_name(late_type.name)
if ast:
t = pytd.Module(name=late_type.name, module_name=late_type.name)
else:
ast, attr_name = self._load_late_type_module(late_type)
if ast is None:
log.error(
"During dependency resolution, couldn't resolve late type %r",
late_type.name)
t = pytd.AnythingType()
else:
try:
cls = pytd.LookupItemRecursive(ast, attr_name)
except KeyError:
if "__getattr__" not in ast:
log.warning("Couldn't resolve %s", late_type.name)
t = pytd.AnythingType()
else:
t = pytd.ToType(cls, allow_functions=True)
self._resolved_late_types[late_type.name] = t
return self._resolved_late_types[late_type.name]
def value_to_pytd_type(self, node, v, seen, view):
"""Get a PyTD type representing this object, as seen at a node.
Args:
node: The node from which we want to observe this object.
v: The object.
seen: The set of values seen before while computing the type.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, (abstract.Empty, typing_overlay.NoReturn)):
return pytd.NothingType()
elif isinstance(v, abstract.TypeParameterInstance):
if (v.module in self._scopes
or v.instance is abstract_utils.DUMMY_CONTAINER):
return self._typeparam_to_def(node, v.param, v.param.name)
elif v.instance.get_instance_type_parameter(v.full_name).bindings:
# The type parameter was initialized. Set the view to None, since we
# don't include v.instance in the view.
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, p, seen, None) for p in
v.instance.get_instance_type_parameter(v.full_name).data)
elif v.param.constraints:
return pytd_utils.JoinTypes(
self.value_instance_to_pytd_type(node, p, None, seen, view)
for p in v.param.constraints)
elif v.param.bound:
return self.value_instance_to_pytd_type(
node, v.param.bound, None, seen, view)
else:
return pytd.AnythingType()
elif isinstance(v, typing_overlay.TypeVar):
return pytd.NamedType("builtins.type")
elif isinstance(v, dataclass_overlay.FieldInstance):
if not v.default:
return pytd.AnythingType()
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, d, seen, view)
for d in v.default.data)
elif isinstance(v, attr_overlay.AttribInstance):
ret = self.value_to_pytd_type(node, v.typ, seen, view)
md = metadata.to_pytd(v.to_metadata())
return pytd.Annotated(ret, ("'pytype_metadata'", md))
elif isinstance(v, special_builtins.PropertyInstance):
return pytd.NamedType("builtins.property")
elif isinstance(v, typed_dict.TypedDict):
return pytd.NamedType(v.props.name)
elif isinstance(v, abstract.FUNCTION_TYPES):
try:
signatures = function.get_signatures(v)
except NotImplementedError:
return pytd.NamedType("typing.Callable")
if len(signatures) == 1:
val = self.signature_to_callable(signatures[0])
if not isinstance(
v, abstract.PYTD_FUNCTION_TYPES) or not val.formal:
# This is a workaround to make sure we don't put unexpected type
# parameters in call traces.
return self.value_instance_to_pytd_type(
node, val, None, seen, view)
return pytd.NamedType("typing.Callable")
elif isinstance(v, (abstract.ClassMethod, abstract.StaticMethod)):
return self.value_to_pytd_type(node, v.method, seen, view)
elif isinstance(v, (special_builtins.IsInstance,
special_builtins.ClassMethodCallable)):
return pytd.NamedType("typing.Callable")
elif isinstance(v, abstract.Class):
param = self.value_instance_to_pytd_type(node, v, None, seen, view)
return pytd.GenericType(base_type=pytd.NamedType("builtins.type"),
parameters=(param, ))
elif isinstance(v, abstract.Module):
return pytd.Alias(v.name,
pytd.Module(v.name, module_name=v.full_name))
elif (self._output_mode >= Converter.OutputMode.LITERAL
and isinstance(v, abstract.ConcreteValue)
and isinstance(v.pyval, (int, str, bytes))):
# LITERAL mode is used only for pretty-printing, so we just stringify the
# inner value rather than properly converting it.
return pytd.Literal(repr(v.pyval))
elif isinstance(v, abstract.SimpleValue):
ret = self.value_instance_to_pytd_type(node,
v.cls,
v,
seen=seen,
view=view)
ret.Visit(
visitors.FillInLocalPointers(
{"builtins": self.ctx.loader.builtins}))
return ret
elif isinstance(v, abstract.Union):
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, o, seen, view)
for o in v.options)
elif isinstance(v, special_builtins.SuperInstance):
return pytd.NamedType("builtins.super")
elif isinstance(v, abstract.TypeParameter):
# Arguably, the type of a type parameter is NamedType("typing.TypeVar"),
# but pytype doesn't know how to handle that, so let's just go with Any
# unless self._detailed is set.
if self._detailed:
return pytd.NamedType("typing.TypeVar")
else:
return pytd.AnythingType()
elif isinstance(v, abstract.Unsolvable):
return pytd.AnythingType()
elif isinstance(v, abstract.Unknown):
return pytd.NamedType(v.class_name)
elif isinstance(v, abstract.BuildClass):
return pytd.NamedType("typing.Callable")
elif isinstance(v, abstract.FinalAnnotation):
param = self.value_to_pytd_type(node, v.annotation, seen, view)
return pytd.GenericType(base_type=pytd.NamedType("typing.Final"),
parameters=(param, ))
else:
raise NotImplementedError(v.__class__.__name__)