def testMaybeFillInLocalPointers(self):
src = textwrap.dedent("""
class A(object):
def a(self, a: A, b: B) -> A or B:
raise A()
raise B()
""")
tree = self.Parse(src)
ty_a = pytd.ClassType("A")
ty_a.Visit(visitors.FillInLocalPointers({"": tree}))
self.assertIsNotNone(ty_a.cls)
ty_b = pytd.ClassType("B")
ty_b.Visit(visitors.FillInLocalPointers({"": tree}))
self.assertIsNone(ty_b.cls)
def _process_module(self, module_name, filename, ast):
"""Create a module from a loaded ast and save it to the loader cache.
Args:
module_name: The fully qualified name of the module being imported.
filename: The file the ast was generated from.
ast: The pytd.TypeDeclUnit representing the module.
Returns:
The ast (pytd.TypeDeclUnit) as represented in this loader.
"""
module = Module(module_name, filename, ast)
self._modules[module_name] = module
try:
module.ast = self._postprocess_pyi(module.ast)
# Now that any imported TypeVar instances have been resolved, adjust type
# parameters in classes and functions.
module.ast = module.ast.Visit(visitors.AdjustTypeParameters())
# Now we can fill in internal cls pointers to ClassType nodes in the
# module. This code executes when the module is first loaded, which
# happens before any others use it to resolve dependencies, so there are
# no external pointers into the module at this point.
module.ast.Visit(
visitors.FillInLocalPointers({"": module.ast,
module_name: module.ast}))
except:
# don't leave half-resolved modules around
del self._modules[module_name]
raise
return module.ast
def _FillLocalReferences(serializable_ast, module_map):
local_filler = visitors.FillInLocalPointers(module_map)
if serializable_ast.class_type_nodes:
for node in serializable_ast.class_type_nodes:
local_filler.EnterClassType(node)
if node.cls is None:
raise AssertionError("This should not happen: %s" % str(node))
else:
serializable_ast.ast.Visit(local_filler)
def GetBuiltinsAndTyping(python_version):
"""Get __builtin__.pytd and typing.pytd."""
assert python_version
global _cached_builtins_pytd
if _cached_builtins_pytd.cache:
assert _cached_builtins_pytd.version == python_version
else:
t = parser.parse_string(_FindBuiltinFile("typing", python_version),
name="typing",
python_version=python_version)
b = parser.parse_string(_FindBuiltinFile("__builtin__",
python_version),
name="__builtin__",
python_version=python_version)
b = b.Visit(
visitors.LookupExternalTypes({"typing": t},
full_names=True,
self_name="__builtin__"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"__builtin__": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"__builtin__": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
_cached_builtins_pytd = Cache(python_version, (b, t))
return _cached_builtins_pytd.cache
def ParseWithBuiltins(self, src):
ast = parser.parse_string(textwrap.dedent(src))
b, t = builtins.GetBuiltinsAndTyping()
ast = ast.Visit(
visitors.LookupExternalTypes({
"__builtin__": b,
"typing": t
},
full_names=True))
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.AdjustTypeParameters())
ast.Visit(visitors.FillInLocalPointers({"": ast, "__builtin__": b}))
ast.Visit(visitors.VerifyVisitor())
return ast
def GetBuiltinsAndTyping():
"""Get __builtin__.pytd and typing.pytd."""
global _cached_builtins_pytd
if not _cached_builtins_pytd:
t = parser.parse_string(_FindBuiltinFile("typing"), name="typing")
b = parser.parse_string(_FindBuiltinFile("__builtin__"),
name="__builtin__")
b = b.Visit(
visitors.LookupExternalTypes({"typing": t},
full_names=True,
self_name="__builtin__"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"__builtin__": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"__builtin__": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
_cached_builtins_pytd = b, t
return _cached_builtins_pytd
def ParseWithBuiltins(self, src):
ast = parser.parse_string(textwrap.dedent(src),
python_version=self.PYTHON_VERSION)
ast = ast.Visit(
visitors.LookupExternalTypes(
{
"__builtin__": self.loader.builtins,
"typing": self.loader.typing
},
full_names=True))
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.AdjustTypeParameters())
ast.Visit(
visitors.FillInLocalPointers({
"": ast,
"__builtin__": self.loader.builtins
}))
ast.Visit(visitors.VerifyVisitor())
return ast
def PrepareForExport(module_name, python_version, ast, loader):
"""Prepare an ast as if it was parsed and loaded.
External dependencies will not be resolved, as the ast generated by this
method is supposed to be exported.
Args:
module_name: The module_name as a string for the returned ast.
python_version: A tuple of (major, minor) python version as string
(see config.python_version).
ast: pytd.TypeDeclUnit, is only used if src is None.
loader: A load_pytd.Loader instance.
Returns:
A pytd.TypeDeclUnit representing the supplied AST as it would look after
being written to a file and parsed.
"""
# This is a workaround for functionality which crept into places it doesn't
# belong. Ideally this would call some transformation Visitors on ast to
# transform it into the same ast we get after parsing and loading (compare
# load_pytd.Loader.load_file). Unfortunately parsing has some special cases,
# e.g. '__init__' return type and '__new__' being a 'staticmethod', which
# need to be moved to visitors before we can do this. Printing an ast also
# applies transformations,
# e.g. visitors.PrintVisitor._FormatContainerContents, which need to move to
# their own visitors so they can be applied without printing.
src = pytd_utils.Print(ast)
ast = pytd_builtins.ParsePyTD(src=src,
module=module_name,
python_version=python_version)
ast = ast.Visit(visitors.LookupBuiltins(loader.builtins, full_names=False))
ast = ast.Visit(visitors.ExpandCompatibleBuiltins(loader.builtins))
ast = ast.Visit(visitors.LookupLocalTypes())
ast = ast.Visit(visitors.AdjustTypeParameters())
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.FillInLocalPointers({"": ast, module_name: ast}))
ast = ast.Visit(visitors.CanonicalOrderingVisitor())
ast = ast.Visit(
visitors.ClassTypeToLateType(
ignore=[module_name + ".", "__builtin__.", "typing."]))
return ast
def value_to_pytd_type(self, node, v, seen, view):
"""Get a PyTD type representing this object, as seen at a node.
Args:
node: The node from which we want to observe this object.
v: The object.
seen: The set of values seen before while computing the type.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, (abstract.Empty, abstract.Nothing)):
return pytd.NothingType()
elif isinstance(v, abstract.TypeParameterInstance):
if v.instance.type_parameters[v.name].bindings:
# The type parameter was initialized.
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, p, seen, view)
for p in v.instance.type_parameters[v.name].data)
elif v.param.constraints:
return pytd_utils.JoinTypes(
self.value_instance_to_pytd_type(node, p, None, seen, view)
for p in v.param.constraints)
else:
return pytd.AnythingType()
elif isinstance(v, typing.TypeVar):
return pytd.NamedType("__builtin__.type")
elif isinstance(
v,
(abstract.InterpreterFunction, abstract.BoundInterpreterFunction)):
sig, = abstract.get_signatures(v)
return self.value_instance_to_pytd_type(
node, self.signature_to_callable(sig, v.vm), None, seen, view)
elif isinstance(v,
(abstract.PyTDFunction, abstract.BoundPyTDFunction)):
signatures = abstract.get_signatures(v)
if len(signatures) == 1:
val = self.signature_to_callable(signatures[0], v.vm)
if not v.vm.annotations_util.get_type_parameters(val):
# This is a workaround to make sure we don't put unexpected type
# parameters in call traces.
return self.value_instance_to_pytd_type(
node, val, None, seen, view)
return pytd.NamedType("typing.Callable")
elif isinstance(
v,
(special_builtins.IsInstance, abstract.ClassMethod,
abstract.StaticMethod, special_builtins.ClassMethodCallable)):
return pytd.NamedType("typing.Callable")
elif isinstance(v, abstract.Class):
param = self.value_instance_to_pytd_type(node, v, None, seen, view)
return pytd.GenericType(
base_type=pytd.NamedType("__builtin__.type"),
parameters=(param, ))
elif isinstance(v, abstract.Module):
return pytd.NamedType("__builtin__.module")
elif isinstance(v, abstract.SimpleAbstractValue):
if v.cls:
classvalues = self._get_values(node, v.cls, view)
cls_types = []
for cls in classvalues:
cls_types.append(
self.value_instance_to_pytd_type(node,
cls,
v,
seen=seen,
view=view))
ret = pytd_utils.JoinTypes(cls_types)
ret.Visit(
visitors.FillInLocalPointers(
{"__builtin__": v.vm.loader.builtins}))
return ret
else:
# We don't know this type's __class__, so return AnythingType to
# indicate that we don't know anything about what this is.
# This happens e.g. for locals / globals, which are returned from the
# code in class declarations.
log.info("Using ? for %s", v.name)
return pytd.AnythingType()
elif isinstance(v, abstract.Union):
return pytd.UnionType(
tuple(
self.value_to_pytd_type(node, o, seen, view)
for o in v.options))
elif isinstance(v, special_builtins.SuperInstance):
return pytd.NamedType("__builtin__.super")
elif isinstance(v, (abstract.Unsolvable, abstract.TypeParameter)):
# Arguably, the type of a type parameter is NamedType("typing.TypeVar"),
# but pytype doesn't know how to handle that, so let's just go with Any.
return pytd.AnythingType()
elif isinstance(v, abstract.Unknown):
return pytd.NamedType(v.class_name)
else:
raise NotImplementedError(v.__class__.__name__)
def _finish_ast(self, ast): module_map = self._get_module_map() module_map[""] = ast # The module itself (local lookup) ast.Visit(visitors.FillInLocalPointers(module_map))
def testFillInFunctionTypePointers(self):
src = textwrap.dedent("def f(): ...")
tree = self.Parse(src)
ty = pytd.FunctionType("f", None)
ty.Visit(visitors.FillInLocalPointers({"": tree}))
self.assertEqual(ty.function, tree.Lookup("f"))
