def load(self, options):
"""Read builtins.pytd and typing.pytd, and return the parsed modules."""
t = self._parse_predefined("typing", options)
b = self._parse_predefined("builtins", options)
b = b.Visit(
visitors.LookupExternalTypes({"typing": t}, self_name="builtins"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"builtins": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"builtins": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
return b, t
def verify(self, mod_ast, *, mod_name=None):
try:
mod_ast.Visit(visitors.VerifyLookup(ignore_late_types=True))
except ValueError as e:
name = mod_name or mod_ast.name
raise BadDependencyError(utils.message(e), name) from e
mod_ast.Visit(visitors.VerifyContainers())
def _verify_pyi(self, pyval, ast_name=None):
try:
pyval.Visit(visitors.VerifyLookup(ignore_late_types=True))
except ValueError as e:
raise BadDependencyError(utils.message(e), ast_name
or pyval.name) from e
pyval.Visit(visitors.VerifyContainers())
def GetBuiltinsAndTyping(python_version): # Deprecated. Use load_pytd instead.
"""Get __builtin__.pytd and typing.pytd."""
assert python_version
global _cached_builtins_pytd
if _cached_builtins_pytd.cache:
assert _cached_builtins_pytd.version == python_version
else:
t = parser.parse_string(_FindBuiltinFile("typing", python_version),
name="typing",
python_version=python_version)
b = parser.parse_string(_FindBuiltinFile("__builtin__",
python_version),
name="__builtin__",
python_version=python_version)
b = b.Visit(
visitors.LookupExternalTypes({"typing": t},
full_names=True,
self_name="__builtin__"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"__builtin__": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"__builtin__": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
_cached_builtins_pytd = Cache(python_version, (b, t))
return _cached_builtins_pytd.cache
def test_star_import(self):
with file_utils.Tempdir() as d:
d.create_file("foo.pyi", "class A: ...")
d.create_file("bar.pyi", "from foo import *")
foo = _Module(module_name="foo", file_name="foo.pyi")
bar = _Module(module_name="bar", file_name="bar.pyi")
loader, _ = self._load_ast(d, module=bar)
self._pickle_modules(loader, d, foo, bar)
loaded_ast = self._load_pickled_module(d, bar)
loaded_ast.Visit(visitors.VerifyLookup())
self.assertEqual(pytd_utils.Print(loaded_ast), "from foo import A")
def testPackageRelativeImport(self):
with utils.Tempdir() as d:
d.create_file("pkg/foo.pyi", "class X: ...")
d.create_file("pkg/bar.pyi", """
from .foo import X
y = ... # type: X""")
foo = _Module(module_name="pkg.foo", file_name="pkg/foo.pyi")
bar = _Module(module_name="pkg.bar", file_name="pkg/bar.pyi")
loader, _ = self._LoadAst(d, module=bar)
self._PickleModules(loader, d, foo, bar)
loaded_ast = self._LoadPickledModule(d, bar)
loaded_ast.Visit(visitors.VerifyLookup())
def testStarImport(self):
with utils.Tempdir() as d:
d.create_file("foo.pyi", "class A(object): ...")
d.create_file("bar.pyi", "from foo import *")
foo = _Module(module_name="foo", file_name="foo.pyi")
bar = _Module(module_name="bar", file_name="bar.pyi")
loader, _ = self._LoadAst(d, module=bar)
self._PickleModules(loader, d, foo, bar)
loaded_ast = self._LoadPickledModule(d, bar)
loaded_ast.Visit(visitors.VerifyLookup())
self.assertMultiLineEqual(pytd.Print(loaded_ast), textwrap.dedent("""\
import foo
bar.A = foo.A"""))
def GetBuiltinsAndTyping(python_version): # Deprecated. Use load_pytd instead.
"""Get builtins.pytd and typing.pytd."""
assert python_version
if python_version not in _cached_builtins_pytd:
t = parser.parse_string(_FindBuiltinFile("typing", python_version),
name="typing",
python_version=python_version)
b = parser.parse_string(_FindBuiltinFile("builtins", python_version),
name="builtins",
python_version=python_version)
b = b.Visit(
visitors.LookupExternalTypes({"typing": t}, self_name="builtins"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"builtins": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"builtins": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
_cached_builtins_pytd[python_version] = (b, t)
return _cached_builtins_pytd[python_version]
def testLoadWithSameModuleName(self):
with utils.Tempdir() as d:
self._CreateFiles(tempdir=d)
module1 = _Module(module_name="foo.bar.module1", file_name="module1.pyi")
module2 = _Module(module_name="module2", file_name="module2.pyi")
loader, ast = self._LoadAst(tempdir=d, module=module1)
self._PickleModules(loader, d, module1, module2)
pickled_ast_filename = self._GetPath(d, module1.file_name + ".pickled")
result = serialize_ast.StoreAst(ast, pickled_ast_filename)
self.assertIsNone(result)
loaded_ast = self._LoadPickledModule(d, module1)
self.assertTrue(loaded_ast)
self.assertIsNot(loaded_ast, ast)
self.assertTrue(ast.ASTeq(loaded_ast))
loaded_ast.Visit(visitors.VerifyLookup())
def test_load_top_level(self):
"""Tests that a pickled file can be read."""
with file_utils.Tempdir() as d:
module_name = "module1"
pickled_ast_filename = os.path.join(d.path, "module1.pyi.pickled")
module_map = self._store_ast(d, module_name, pickled_ast_filename)
original_ast = module_map[module_name]
del module_map[module_name]
loaded_ast = serialize_ast.ProcessAst(
pytd_utils.LoadPickle(pickled_ast_filename), module_map)
self.assertTrue(loaded_ast)
self.assertIsNot(loaded_ast, original_ast)
self.assertEqual(loaded_ast.name, module_name)
self.assertTrue(pytd_utils.ASTeq(original_ast, loaded_ast))
loaded_ast.Visit(visitors.VerifyLookup())
def test_load_with_same_module_name(self):
with file_utils.Tempdir() as d:
self._create_files(tempdir=d)
module1 = _Module(module_name="foo.bar.module1", file_name="module1.pyi")
module2 = _Module(module_name="module2", file_name="module2.pyi")
loader, ast = self._load_ast(tempdir=d, module=module1)
self._pickle_modules(loader, d, module1, module2)
pickled_ast_filename = self._get_path(d, module1.file_name + ".pickled")
result = serialize_ast.StoreAst(ast, pickled_ast_filename)
self.assertIsNone(result)
loaded_ast = self._load_pickled_module(d, module1)
self.assertTrue(loaded_ast)
self.assertIsNot(loaded_ast, ast)
self.assertTrue(pytd_utils.ASTeq(ast, loaded_ast))
loaded_ast.Visit(visitors.VerifyLookup())
def testLookupClasses(self):
src = textwrap.dedent("""
class object(object):
pass
class A(object):
def a(self, a: A, b: B) -> A or B:
raise A()
raise B()
class B(object):
def b(self, a: A, b: B) -> A or B:
raise A()
raise B()
""")
tree = self.Parse(src)
new_tree = visitors.LookupClasses(tree)
self.AssertSourceEquals(new_tree, src)
new_tree.Visit(visitors.VerifyLookup())
def test_lookup_classes(self):
src = textwrap.dedent("""
from typing import Union
class object:
pass
class A:
def a(self, a: A, b: B) -> Union[A, B]:
raise A()
raise B()
class B:
def b(self, a: A, b: B) -> Union[A, B]:
raise A()
raise B()
""")
tree = self.Parse(src)
new_tree = visitors.LookupClasses(tree)
self.AssertSourceEquals(new_tree, src)
new_tree.Visit(visitors.VerifyLookup())
def test_node_index_visitor_usage(self):
"""Confirms that the node index is used.
This removes the first node from the class_type_nodes list and checks that
that node is not updated by ProcessAst.
"""
with file_utils.Tempdir() as d:
module_name = "module1"
pickled_ast_filename = os.path.join(d.path, "module1.pyi.pickled")
module_map = self._store_ast(d, module_name, pickled_ast_filename)
del module_map[module_name]
serialized_ast = pytd_utils.LoadPickle(pickled_ast_filename)
# The sorted makes the testcase more deterministic.
serialized_ast = serialized_ast.Replace(class_type_nodes=sorted(
serialized_ast.class_type_nodes)[1:])
loaded_ast = serialize_ast.ProcessAst(serialized_ast, module_map)
with six.assertRaisesRegex(
self, ValueError, "Unresolved class: '__builtin__.NoneType'"):
loaded_ast.Visit(visitors.VerifyLookup())
def test_load_with_same_module_name(self):
"""Explicitly set the module name and reload with the same name.
The difference to testLoadTopLevel is that the module name does not match
the filelocation.
"""
with file_utils.Tempdir() as d:
module_name = "foo.bar.module1"
pickled_ast_filename = os.path.join(d.path, "module1.pyi.pickled")
module_map = self._store_ast(d, module_name, pickled_ast_filename)
original_ast = module_map[module_name]
del module_map[module_name]
loaded_ast = serialize_ast.ProcessAst(
pytd_utils.LoadPickle(pickled_ast_filename), module_map)
self.assertTrue(loaded_ast)
self.assertIsNot(loaded_ast, original_ast)
self.assertEqual(loaded_ast.name, "foo.bar.module1")
self.assertTrue(pytd_utils.ASTeq(original_ast, loaded_ast))
loaded_ast.Visit(visitors.VerifyLookup())
def test_load_with_different_module_name(self):
with file_utils.Tempdir() as d:
original_module_name = "module1"
pickled_ast_filename = os.path.join(d.path, "module1.pyi.pickled")
module_map = self._store_ast(
d, original_module_name, pickled_ast_filename)
original_ast = module_map[original_module_name]
del module_map[original_module_name]
new_module_name = "wurstbrot.module2"
serializable_ast = pytd_utils.LoadPickle(pickled_ast_filename)
serializable_ast = serialize_ast.EnsureAstName(
serializable_ast, new_module_name, fix=True)
loaded_ast = serialize_ast.ProcessAst(serializable_ast, module_map)
self.assertTrue(loaded_ast)
self.assertIsNot(loaded_ast, original_ast)
self.assertEqual(loaded_ast.name, new_module_name)
loaded_ast.Visit(visitors.VerifyLookup())
self.assertFalse(pytd_utils.ASTeq(original_ast, loaded_ast))
ast_new_module, _ = self._get_ast(temp_dir=d, module_name=new_module_name)
self.assertTrue(pytd_utils.ASTeq(ast_new_module, loaded_ast))
def call(self, node, _, args):
"""Creates a namedtuple class definition.
Performs the same argument checking as collections.namedtuple, e.g. making
sure field names don't start with _ or digits, making sure no keywords are
used for the typename or field names, and so on. Because the methods of the
class have to be changed to match the number and names of the fields, we
construct pytd.Function and pytd.Constant instances for each member of the
class. Finally, the pytd.Class is wrapped in an abstract.PyTDClass.
If incorrect arguments are passed, a subclass of function.FailedFunctionCall
will be raised. Other cases may raise abstract_utils.ConversionError
exceptions, such as when the arguments are in unicode or any of the
arguments have multiple bindings, but these are caught and return Any. This
also occurs if an argument to namedtuple is invalid, e.g. a keyword is used
as a field name and rename is False.
Arguments:
node: the current CFG node
_: the func binding, ignored.
args: a function.Args instance
Returns:
a tuple of the given CFG node and an abstract.PyTDClass instance (wrapped
in a Variable) representing the constructed namedtuple class.
If a abstract_utils.ConversionError occurs or if field names are invalid,
this function returns Unsolvable (in a Variable) instead of a PyTDClass.
Raises:
function.FailedFunctionCall: Raised by _getargs if any of the arguments
do not match the function signature.
"""
# If we can't extract the arguments, we take the easy way out and return Any
try:
name_var, field_names, rename = self._getargs(node, args)
except abstract_utils.ConversionError:
return node, self.vm.new_unsolvable(node)
# We need the bare name for a few things, so pull that out now.
# The same unicode issue can strike here, so again return Any.
try:
name = abstract_utils.get_atomic_python_constant(name_var)
except abstract_utils.ConversionError:
return node, self.vm.new_unsolvable(node)
# namedtuple does some checking and optionally renaming of field names,
# so we do too.
try:
field_names = self._validate_and_rename_args(
name, field_names, rename)
except ValueError as e:
self.vm.errorlog.invalid_namedtuple_arg(self.vm.frames,
utils.message(e))
return node, self.vm.new_unsolvable(node)
name = namedtuple_name(name, field_names)
ast = namedtuple_ast(name,
field_names,
python_version=self.vm.python_version)
mapping = self._get_known_types_mapping()
# A truly well-formed pyi for the namedtuple will have references to the new
# namedtuple class itself for all `self` params and as the return type for
# methods like __new__, _replace and _make. In order to do that, we need a
# ClassType.
cls_type = pytd.ClassType(name)
mapping[name] = cls_type
cls = ast.Lookup(name).Visit(visitors.ReplaceTypes(mapping))
cls_type.cls = cls
# Make sure all NamedType nodes have been replaced by ClassType nodes with
# filled cls pointers.
cls.Visit(visitors.VerifyLookup())
# We can't build the PyTDClass directly, and instead must run it through
# convert.constant_to_value first, for caching.
instance = self.vm.convert.constant_to_value(cls, {},
self.vm.root_cfg_node)
self.vm.trace_namedtuple(instance)
return node, instance.to_variable(node)
def _verify_ast(self, ast):
try:
ast.Visit(visitors.VerifyLookup(ignore_late_types=True))
except ValueError as e:
raise BadDependencyError(utils.message(e), ast.name)
ast.Visit(visitors.VerifyContainers())
def test_get_builtins_pytd(self): self.assertIsNotNone(self.builtins) # Will throw an error for unresolved identifiers: self.builtins.Visit(visitors.VerifyLookup())
def test_unrestorable_child(self):
# Assume .cls in a ClassType X in module1 was referencing something for
# which, Visitors.LookupExternalTypes returned AnythingType.
# Now if something in module1 is referencing X external types need to be
# resolved before local types, so that we can resolve local types to the
# correct ClassType, as the ClassType instance changes, if .cls can not be
# filled and instead AnythingType is used.
class RenameVisitor(visitors.Visitor):
def __init__(self, *args, **kwargs):
super(RenameVisitor, self).__init__(*args, **kwargs)
self._init = False
def EnterFunction(self, func):
if func.name == "__init__":
self._init = True
return None
return False
def LeaveFunction(self, func):
self._init = False
def VisitClassType(self, cls_type):
if self._init:
cls_type = cls_type.Replace(
name="other_module.unknown_Reference")
# Needs to be copied manually as it is not part of the NamedTuple.
cls_type.cls = None
return cls_type
with file_utils.Tempdir() as d:
src = ("""
import other_module
x = other_module.UnusedReferenceNeededToKeepTheImport
class SomeClass(object):
def __init__(will_be_replaced_with_visitor) -> None:
pass
def func(a:SomeClass) -> None:
pass
""")
d.create_file(
"other_module.pyi", """
from typing import Any
def __getattr__(self, name) -> Any: ...""")
ast, loader = self._get_ast(temp_dir=d,
module_name="module1",
src=src)
ast = ast.Visit(RenameVisitor())
pickled_ast_filename = os.path.join(d.path, "module1.pyi.pickled")
module_map = self._store_ast(d,
"module1",
pickled_ast_filename,
ast=ast,
loader=loader)
del module_map["module1"]
serialized_ast = pytd_utils.LoadPickle(pickled_ast_filename)
loaded_ast = serialize_ast.ProcessAst(serialized_ast, module_map)
# Look up the "SomeClass" in "def func(a: SomeClass), then run
# VerifyLookup on it. We can't run VerifyLookup on the root node, since
# visitors don't descend into the "cls" attribute of ClassType.
cls = loaded_ast.functions[0].signatures[0].params[0].type.cls
cls.Visit(visitors.VerifyLookup())
