def extract_local(ast):
"""Extract all classes that are not unknowns of call records of builtins."""
return pytd.TypeDeclUnit(
name=ast.name,
classes=tuple(cls for cls in ast.classes if is_complete(cls)),
functions=tuple(f for f in ast.functions if is_complete(f)),
constants=tuple(c for c in ast.constants if is_complete(c)))
def EmptyModule(name="<empty>"):
return pytd.TypeDeclUnit(name,
type_params=(),
constants=(),
classes=(),
functions=(),
aliases=())
def VisitTypeDeclUnit(self, node):
return pytd.TypeDeclUnit(name=node.name,
constants=tuple(sorted(node.constants)),
type_params=tuple(sorted(node.type_params)),
functions=tuple(sorted(node.functions)),
classes=tuple(sorted(node.classes)),
aliases=tuple(sorted(node.aliases)))
def WrapTypeDeclUnit(name, items):
"""Given a list (classes, functions, etc.), wrap a pytd around them.
Args:
name: The name attribute of the resulting TypeDeclUnit.
items: A list of items. Can contain pytd.Class, pytd.Function and
pytd.Constant.
Returns:
A pytd.TypeDeclUnit.
Raises:
ValueError: In case of an invalid item in the list.
NameError: For name conflicts.
"""
functions = collections.OrderedDict()
classes = collections.OrderedDict()
constants = collections.defaultdict(TypeBuilder)
aliases = collections.OrderedDict()
for item in items:
if isinstance(item, pytd.Function):
if item.name in functions:
if item.kind != functions[item.name].kind:
raise ValueError("Can't combine %s and %s", item.kind,
functions[item.name].kind)
functions[item.name] = pytd.Function(
item.name,
functions[item.name].signatures + item.signatures,
item.kind)
else:
functions[item.name] = item
elif isinstance(item, pytd.Class):
if item.name in classes:
raise NameError("Duplicate top level class: %r", item.name)
classes[item.name] = item
elif isinstance(item, pytd.Constant):
constants[item.name].add_type(item.type)
elif isinstance(item, pytd.Alias):
if item.name in aliases:
raise NameError("Duplicate top level alias or import: %r",
item.name)
aliases[item.name] = item
else:
raise ValueError("Invalid top level pytd item: %r" % type(item))
_check_intersection(functions, classes, "function", "class")
_check_intersection(functions, constants, "functions", "constant")
_check_intersection(functions, aliases, "functions", "aliases")
_check_intersection(classes, constants, "class", "constant")
_check_intersection(classes, aliases, "class", "alias")
_check_intersection(constants, aliases, "constant", "alias")
return pytd.TypeDeclUnit(name=name,
constants=tuple(
pytd.Constant(name, t.build())
for name, t in sorted(constants.items())),
type_params=tuple(),
classes=tuple(classes.values()),
functions=tuple(functions.values()),
aliases=tuple(aliases.values()))
def build_type_decl_unit(self, defs) -> pytd.TypeDeclUnit:
"""Return a pytd.TypeDeclUnit for the given defs (plus parser state)."""
# defs contains both constant and function definitions.
constants, functions, aliases, slots, classes = _split_definitions(
defs)
assert not slots # slots aren't allowed on the module level
# TODO(mdemello): alias/constant handling is broken in some weird manner.
# assert not aliases # We handle top-level aliases in add_alias_or_constant
# constants.extend(self.constants)
if self.module_info.module_name == "builtins":
constants.extend(types.builtin_keyword_constants())
generated_classes = sum(self.generated_classes.values(), [])
classes = generated_classes + classes
functions = function.merge_method_signatures(functions)
name_to_class = {c.name: c for c in classes}
name_to_constant = {c.name: c for c in constants}
aliases = []
for a in self.aliases.values():
t = _maybe_resolve_alias(a, name_to_class, name_to_constant)
if t is None:
continue
elif isinstance(t, pytd.Function):
functions.append(t)
elif isinstance(t, pytd.Constant):
constants.append(t)
else:
assert isinstance(t, pytd.Alias)
aliases.append(t)
all_names = ([f.name for f in functions] + [c.name
for c in constants] +
[c.name for c in self.type_params] +
[c.name for c in classes] + [c.name for c in aliases])
duplicates = [
name for name, count in collections.Counter(all_names).items()
if count >= 2
]
if duplicates:
raise ParseError("Duplicate top-level identifier(s): " +
", ".join(duplicates))
properties = [x for x in functions if x.kind == pytd.PROPERTY]
if properties:
prop_names = ", ".join(p.name for p in properties)
raise ParseError(
"Module-level functions with property decorators: " +
prop_names)
return pytd.TypeDeclUnit(name=None,
constants=tuple(constants),
type_params=tuple(self.type_params),
functions=tuple(functions),
classes=tuple(classes),
aliases=tuple(aliases))
def CreateModule(name="<empty>", **kwargs):
module = pytd.TypeDeclUnit(name,
type_params=(),
constants=(),
classes=(),
functions=(),
aliases=())
return module.Replace(**kwargs)
def testDiffSamePickle(self):
ast = pytd.TypeDeclUnit("foo", (), (), (), (), ())
with file_utils.Tempdir() as d:
filename = os.path.join(d.path, "foo.pickled")
serialize_ast.StoreAst(ast, filename)
with open(filename, "rb") as fi:
data = fi.read()
named_pickles = [("foo", data)]
self.assertFalse(pytd_utils.DiffNamedPickles(named_pickles, named_pickles))
def Concat(*args, **kwargs):
"""Concatenate two or more pytd ASTs."""
assert all(isinstance(arg, pytd.TypeDeclUnit) for arg in args)
name = kwargs.get("name")
return pytd.TypeDeclUnit(name=name or " + ".join(arg.name for arg in args),
constants=sum((arg.constants for arg in args),
()),
classes=sum((arg.classes for arg in args), ()),
functions=sum((arg.functions for arg in args),
()))
def test_diff_pickle_length(self):
ast = pytd.TypeDeclUnit("foo", (), (), (), (), ())
with file_utils.Tempdir() as d:
filename = os.path.join(d.path, "foo.pickled")
serialize_ast.StoreAst(ast, filename)
with open(filename, "rb") as fi:
data = fi.read()
named_pickles1 = []
named_pickles2 = [("foo", data)]
self.assertTrue(pytd_utils.DiffNamedPickles(named_pickles1, named_pickles2))
def testPrintImportsNamedType(self):
# Can't get tree by parsing so build explicitly
node = pytd.Constant("x", pytd.NamedType("typing.List"))
tree = pytd.TypeDeclUnit(constants=(node,), type_params=(),
functions=(), classes=(), aliases=(), name=None)
expected_src = textwrap.dedent("""
import typing
x = ... # type: typing.List
""").strip()
res = pytd.Print(tree)
self.assertMultiLineEqual(res, expected_src)
def compute_types(self, defs, ignore):
self.program.Freeze()
ty = pytd_utils.Concat(
self.pytd_for_types(defs, ignore),
pytd.TypeDeclUnit(
"unknowns", (),
tuple(self.pytd_classes_for_unknowns()) +
tuple(self.pytd_classes_for_call_traces()),
tuple(self.pytd_functions_for_call_traces())))
ty = ty.Visit(optimize.PullInMethodClasses())
ty = ty.Visit(visitors.DefaceUnresolved(
[ty, self.loader.concat_all()], "~unknown"))
return ty
def Concat(*args, **kwargs):
"""Concatenate two or more pytd ASTs."""
assert all(isinstance(arg, pytd.TypeDeclUnit) for arg in args)
name = kwargs.get("name")
is_package = bool(kwargs.get("is_package"))
return pytd.TypeDeclUnit(
name=name or " + ".join(arg.name for arg in args),
is_package=is_package,
constants=sum((arg.constants for arg in args), ()),
type_params=sum((arg.type_params for arg in args), ()),
classes=sum((arg.classes for arg in args), ()),
functions=sum((arg.functions for arg in args), ()),
aliases=sum((arg.aliases for arg in args), ()))
def testDiffPickleAst(self):
ast1 = pytd.TypeDeclUnit("foo", (), (), (), (), ())
ast2 = ast1.Replace(type_params=(pytd.TypeParameter("T", (), None, None),))
with file_utils.Tempdir() as d:
data = []
for ast in (ast1, ast2):
filename = os.path.join(d.path, "foo.pickled")
serialize_ast.StoreAst(ast, filename)
with open(filename, "rb") as fi:
data.append(fi.read())
named_pickles1 = [("foo", data[0])]
named_pickles2 = [("foo", data[1])]
self.assertTrue(pytd_utils.DiffNamedPickles(named_pickles1, named_pickles2))
def _build_type_decl_unit(self, defs):
"""Return a pytd.TypeDeclUnit for the given defs (plus parser state)."""
# defs contains both constant and function definitions.
constants, functions, aliases, slots = _split_definitions(defs)
assert not slots # slots aren't allowed on the module level
assert not aliases # We handle top-level aliases in add_alias_or_constant.
constants.extend(self._constants)
generated_classes = [x for class_list in self._generated_classes.values()
for x in class_list]
classes = generated_classes + self._classes
all_names = (list(set(f.name for f in functions)) +
[c.name for c in constants] +
[c.name for c in self._type_params] +
[c.name for c in classes] +
[c.name for c in self._aliases])
duplicates = [name
for name, count in collections.Counter(all_names).items()
if count >= 2]
if duplicates:
raise ParseError(
"Duplicate top-level identifier(s): " + ", ".join(duplicates))
functions = _merge_method_signatures(functions)
properties = [x for x in functions if x.kind == pytd.PROPERTY]
if properties:
prop_names = ", ".join(p.name for p in properties)
raise ParseError(
"Module-level functions with property decorators: " + prop_names)
module_aliases = {}
for item in self._modules:
if item.alias and item.alias in module_aliases:
existing = module_aliases[item.alias]
if existing != item:
raise ParseError(
"Duplicate import aliases: %s as %s, %s as %s" % (
existing.module_name, existing.alias,
item.module_name, item.alias))
module_aliases[item.alias] = item
return pytd.TypeDeclUnit(name=None,
is_package=False,
constants=tuple(constants),
type_params=tuple(self._type_params),
functions=tuple(functions),
classes=tuple(classes),
modules=tuple(self._modules),
aliases=tuple(self._aliases))
def testSetModuleOnModule(self):
# A module's 'module' attribute should always remain None, and no one
# should attempt to set it to something besides the module's name or None.
ast = pytd.TypeDeclUnit("some_mod", False, (), (), (), (), ())
mod = abstract.Module(self._vm, ast.name, {}, ast)
mod.module = ast.name
self.assertIsNone(mod.module)
self.assertEqual(ast.name, mod.full_name)
mod.module = None
self.assertIsNone(mod.module)
self.assertEqual(ast.name, mod.full_name)
def set_module():
mod.module = "other_mod"
self.assertRaises(AssertionError, set_module)
def testAliasPrinting(self):
a = pytd.Alias("MyList", pytd.GenericType(
pytd.NamedType("typing.List"), (pytd.AnythingType(),)))
ty = pytd.TypeDeclUnit(
name="test",
constants=(),
type_params=(),
classes=(),
functions=(),
aliases=(a,))
expected = textwrap.dedent("""
from typing import Any, List
MyList = List[Any]""")
self.assertMultiLineEqual(expected.strip(), pytd.Print(ty).strip())
def compute_types(self, defs):
ty = pytd_utils.Concat(
self.pytd_for_types(defs),
pytd.TypeDeclUnit(
"unknowns",
constants=tuple(),
type_params=tuple(self.pytd_typevars()),
classes=tuple(self.pytd_classes_for_unknowns()) +
tuple(self.pytd_classes_for_call_traces()),
functions=tuple(self.pytd_functions_for_call_traces()),
aliases=tuple(self.pytd_aliases())))
ty = ty.Visit(optimize.PullInMethodClasses())
ty = ty.Visit(visitors.DefaceUnresolved(
[ty, self.loader.concat_all()], "~unknown"))
return ty.Visit(visitors.AdjustTypeParameters())
def p_unit(self, p):
"""unit : alldefs"""
funcdefs = [x for x in p[1] if isinstance(x, NameAndSig)]
constants = [x for x in p[1] if isinstance(x, pytd.Constant)]
classes = [x for x in p[1] if isinstance(x, pytd.Class)]
all_names = (list(set(f.name for f in funcdefs)) +
[c.name for c in constants] + [c.name for c in classes])
duplicates = [
name for name, count in collections.Counter(all_names).items()
if count >= 2
]
if duplicates:
make_syntax_error(
self,
'Duplicate top-level identifier(s):' + ', '.join(duplicates),
p)
p[0] = pytd.TypeDeclUnit(
name=None, # replaced later, in Parse
constants=tuple(constants),
functions=tuple(self.MergeSignatures(funcdefs)),
classes=tuple(classes))
def _build_type_decl_unit(self, defs):
"""Return a pytd.TypeDeclUnit for the given defs (plus parser state)."""
# defs contains both constant and function definitions.
constants, functions = _split_definitions(defs)
constants.extend(self._constants)
generated_classes = [
x for class_list in self._generated_classes.values()
for x in class_list
]
classes = generated_classes + self._classes
all_names = (list(set(f.name for f in functions)) +
[c.name for c in constants] +
[c.name
for c in self._type_params] + [c.name for c in classes] +
[c.name for c in self._aliases])
duplicates = [
name for name, count in collections.Counter(all_names).items()
if count >= 2
]
if duplicates:
raise ParseError("Duplicate top-level identifier(s): " +
", ".join(duplicates))
functions, properties = _merge_signatures(functions)
if properties:
prop_names = ", ".join(p.name for p in properties)
raise ParseError(
"Module-level functions with property decorators: " +
prop_names)
return pytd.TypeDeclUnit(name=None,
constants=tuple(constants),
type_params=tuple(self._type_params),
functions=tuple(functions),
classes=tuple(classes),
aliases=tuple(self._aliases))
def WrapTypeDeclUnit(name, items, is_package=False):
"""Given a list (classes, functions, etc.), wrap a pytd around them.
Args:
name: The name attribute of the resulting TypeDeclUnit.
items: A list of items. Can contain pytd.Class, pytd.Function and
pytd.Constant.
is_package: Whether the module is a package (e.g. foo/__init__.pyi)
Returns:
A pytd.TypeDeclUnit.
Raises:
ValueError: In case of an invalid item in the list.
NameError: For name conflicts.
"""
functions = collections.OrderedDict()
classes = collections.OrderedDict()
constants = collections.defaultdict(TypeBuilder)
aliases = collections.OrderedDict()
typevars = collections.OrderedDict()
for item in items:
if isinstance(item, pytd.Function):
if item.name in functions:
if item.kind != functions[item.name].kind:
raise ValueError("Can't combine %s and %s", item.kind,
functions[item.name].kind)
functions[item.name] = pytd.Function(
item.name, functions[item.name].signatures + item.signatures,
item.kind)
else:
functions[item.name] = item
elif isinstance(item, pytd.Class):
if item.name in classes:
raise NameError("Duplicate top level class: %r" % item.name)
classes[item.name] = item
elif isinstance(item, pytd.Constant):
constants[item.name].add_type(item.type)
elif isinstance(item, pytd.Alias):
if item.name in aliases:
raise NameError("Duplicate top level alias or import: %r" % item.name)
aliases[item.name] = item
elif isinstance(item, pytd.TypeParameter):
if item.name in typevars:
raise NameError("Duplicate top level type parameter: %r" % item.name)
typevars[item.name] = item
else:
raise ValueError("Invalid top level pytd item: %r" % type(item))
categories = {"function": functions, "class": classes, "constant": constants,
"alias": aliases, "typevar": typevars}
for c1, c2 in itertools.combinations(categories, 2):
_check_intersection(categories[c1], categories[c2], c1, c2)
return pytd.TypeDeclUnit(
name=name,
is_package=is_package,
constants=tuple(
pytd.Constant(name, t.build())
for name, t in sorted(constants.items())),
type_params=tuple(typevars.values()),
classes=tuple(classes.values()),
functions=tuple(functions.values()),
aliases=tuple(aliases.values()))
def test_write_pickle(self): ast = pytd.TypeDeclUnit(None, (), (), (), (), ()) options = config.Options.create(output="/dev/null") io.write_pickle(ast, options) # just make sure we don't crash
