def extract_local(ast): """Extract all classes that are not unknowns of call records of builtins.""" return pytd.TypeDeclUnit( name=ast.name, classes=tuple(cls for cls in ast.classes if is_complete(cls)), functions=tuple(f for f in ast.functions if is_complete(f)), constants=tuple(c for c in ast.constants if is_complete(c)))
def EmptyModule(name="<empty>"): return pytd.TypeDeclUnit(name, type_params=(), constants=(), classes=(), functions=(), aliases=())
def VisitTypeDeclUnit(self, node): return pytd.TypeDeclUnit(name=node.name, constants=tuple(sorted(node.constants)), type_params=tuple(sorted(node.type_params)), functions=tuple(sorted(node.functions)), classes=tuple(sorted(node.classes)), aliases=tuple(sorted(node.aliases)))
def WrapTypeDeclUnit(name, items): """Given a list (classes, functions, etc.), wrap a pytd around them. Args: name: The name attribute of the resulting TypeDeclUnit. items: A list of items. Can contain pytd.Class, pytd.Function and pytd.Constant. Returns: A pytd.TypeDeclUnit. Raises: ValueError: In case of an invalid item in the list. NameError: For name conflicts. """ functions = collections.OrderedDict() classes = collections.OrderedDict() constants = collections.defaultdict(TypeBuilder) aliases = collections.OrderedDict() for item in items: if isinstance(item, pytd.Function): if item.name in functions: if item.kind != functions[item.name].kind: raise ValueError("Can't combine %s and %s", item.kind, functions[item.name].kind) functions[item.name] = pytd.Function( item.name, functions[item.name].signatures + item.signatures, item.kind) else: functions[item.name] = item elif isinstance(item, pytd.Class): if item.name in classes: raise NameError("Duplicate top level class: %r", item.name) classes[item.name] = item elif isinstance(item, pytd.Constant): constants[item.name].add_type(item.type) elif isinstance(item, pytd.Alias): if item.name in aliases: raise NameError("Duplicate top level alias or import: %r", item.name) aliases[item.name] = item else: raise ValueError("Invalid top level pytd item: %r" % type(item)) _check_intersection(functions, classes, "function", "class") _check_intersection(functions, constants, "functions", "constant") _check_intersection(functions, aliases, "functions", "aliases") _check_intersection(classes, constants, "class", "constant") _check_intersection(classes, aliases, "class", "alias") _check_intersection(constants, aliases, "constant", "alias") return pytd.TypeDeclUnit(name=name, constants=tuple( pytd.Constant(name, t.build()) for name, t in sorted(constants.items())), type_params=tuple(), classes=tuple(classes.values()), functions=tuple(functions.values()), aliases=tuple(aliases.values()))
def build_type_decl_unit(self, defs) -> pytd.TypeDeclUnit: """Return a pytd.TypeDeclUnit for the given defs (plus parser state).""" # defs contains both constant and function definitions. constants, functions, aliases, slots, classes = _split_definitions( defs) assert not slots # slots aren't allowed on the module level # TODO(mdemello): alias/constant handling is broken in some weird manner. # assert not aliases # We handle top-level aliases in add_alias_or_constant # constants.extend(self.constants) if self.module_info.module_name == "builtins": constants.extend(types.builtin_keyword_constants()) generated_classes = sum(self.generated_classes.values(), []) classes = generated_classes + classes functions = function.merge_method_signatures(functions) name_to_class = {c.name: c for c in classes} name_to_constant = {c.name: c for c in constants} aliases = [] for a in self.aliases.values(): t = _maybe_resolve_alias(a, name_to_class, name_to_constant) if t is None: continue elif isinstance(t, pytd.Function): functions.append(t) elif isinstance(t, pytd.Constant): constants.append(t) else: assert isinstance(t, pytd.Alias) aliases.append(t) all_names = ([f.name for f in functions] + [c.name for c in constants] + [c.name for c in self.type_params] + [c.name for c in classes] + [c.name for c in aliases]) duplicates = [ name for name, count in collections.Counter(all_names).items() if count >= 2 ] if duplicates: raise ParseError("Duplicate top-level identifier(s): " + ", ".join(duplicates)) properties = [x for x in functions if x.kind == pytd.PROPERTY] if properties: prop_names = ", ".join(p.name for p in properties) raise ParseError( "Module-level functions with property decorators: " + prop_names) return pytd.TypeDeclUnit(name=None, constants=tuple(constants), type_params=tuple(self.type_params), functions=tuple(functions), classes=tuple(classes), aliases=tuple(aliases))
def CreateModule(name="<empty>", **kwargs): module = pytd.TypeDeclUnit(name, type_params=(), constants=(), classes=(), functions=(), aliases=()) return module.Replace(**kwargs)
def testDiffSamePickle(self): ast = pytd.TypeDeclUnit("foo", (), (), (), (), ()) with file_utils.Tempdir() as d: filename = os.path.join(d.path, "foo.pickled") serialize_ast.StoreAst(ast, filename) with open(filename, "rb") as fi: data = fi.read() named_pickles = [("foo", data)] self.assertFalse(pytd_utils.DiffNamedPickles(named_pickles, named_pickles))
def Concat(*args, **kwargs): """Concatenate two or more pytd ASTs.""" assert all(isinstance(arg, pytd.TypeDeclUnit) for arg in args) name = kwargs.get("name") return pytd.TypeDeclUnit(name=name or " + ".join(arg.name for arg in args), constants=sum((arg.constants for arg in args), ()), classes=sum((arg.classes for arg in args), ()), functions=sum((arg.functions for arg in args), ()))
def test_diff_pickle_length(self): ast = pytd.TypeDeclUnit("foo", (), (), (), (), ()) with file_utils.Tempdir() as d: filename = os.path.join(d.path, "foo.pickled") serialize_ast.StoreAst(ast, filename) with open(filename, "rb") as fi: data = fi.read() named_pickles1 = [] named_pickles2 = [("foo", data)] self.assertTrue(pytd_utils.DiffNamedPickles(named_pickles1, named_pickles2))
def testPrintImportsNamedType(self): # Can't get tree by parsing so build explicitly node = pytd.Constant("x", pytd.NamedType("typing.List")) tree = pytd.TypeDeclUnit(constants=(node,), type_params=(), functions=(), classes=(), aliases=(), name=None) expected_src = textwrap.dedent(""" import typing x = ... # type: typing.List """).strip() res = pytd.Print(tree) self.assertMultiLineEqual(res, expected_src)
def compute_types(self, defs, ignore): self.program.Freeze() ty = pytd_utils.Concat( self.pytd_for_types(defs, ignore), pytd.TypeDeclUnit( "unknowns", (), tuple(self.pytd_classes_for_unknowns()) + tuple(self.pytd_classes_for_call_traces()), tuple(self.pytd_functions_for_call_traces()))) ty = ty.Visit(optimize.PullInMethodClasses()) ty = ty.Visit(visitors.DefaceUnresolved( [ty, self.loader.concat_all()], "~unknown")) return ty
def Concat(*args, **kwargs): """Concatenate two or more pytd ASTs.""" assert all(isinstance(arg, pytd.TypeDeclUnit) for arg in args) name = kwargs.get("name") is_package = bool(kwargs.get("is_package")) return pytd.TypeDeclUnit( name=name or " + ".join(arg.name for arg in args), is_package=is_package, constants=sum((arg.constants for arg in args), ()), type_params=sum((arg.type_params for arg in args), ()), classes=sum((arg.classes for arg in args), ()), functions=sum((arg.functions for arg in args), ()), aliases=sum((arg.aliases for arg in args), ()))
def testDiffPickleAst(self): ast1 = pytd.TypeDeclUnit("foo", (), (), (), (), ()) ast2 = ast1.Replace(type_params=(pytd.TypeParameter("T", (), None, None),)) with file_utils.Tempdir() as d: data = [] for ast in (ast1, ast2): filename = os.path.join(d.path, "foo.pickled") serialize_ast.StoreAst(ast, filename) with open(filename, "rb") as fi: data.append(fi.read()) named_pickles1 = [("foo", data[0])] named_pickles2 = [("foo", data[1])] self.assertTrue(pytd_utils.DiffNamedPickles(named_pickles1, named_pickles2))
def _build_type_decl_unit(self, defs): """Return a pytd.TypeDeclUnit for the given defs (plus parser state).""" # defs contains both constant and function definitions. constants, functions, aliases, slots = _split_definitions(defs) assert not slots # slots aren't allowed on the module level assert not aliases # We handle top-level aliases in add_alias_or_constant. constants.extend(self._constants) generated_classes = [x for class_list in self._generated_classes.values() for x in class_list] classes = generated_classes + self._classes all_names = (list(set(f.name for f in functions)) + [c.name for c in constants] + [c.name for c in self._type_params] + [c.name for c in classes] + [c.name for c in self._aliases]) duplicates = [name for name, count in collections.Counter(all_names).items() if count >= 2] if duplicates: raise ParseError( "Duplicate top-level identifier(s): " + ", ".join(duplicates)) functions = _merge_method_signatures(functions) properties = [x for x in functions if x.kind == pytd.PROPERTY] if properties: prop_names = ", ".join(p.name for p in properties) raise ParseError( "Module-level functions with property decorators: " + prop_names) module_aliases = {} for item in self._modules: if item.alias and item.alias in module_aliases: existing = module_aliases[item.alias] if existing != item: raise ParseError( "Duplicate import aliases: %s as %s, %s as %s" % ( existing.module_name, existing.alias, item.module_name, item.alias)) module_aliases[item.alias] = item return pytd.TypeDeclUnit(name=None, is_package=False, constants=tuple(constants), type_params=tuple(self._type_params), functions=tuple(functions), classes=tuple(classes), modules=tuple(self._modules), aliases=tuple(self._aliases))
def testSetModuleOnModule(self): # A module's 'module' attribute should always remain None, and no one # should attempt to set it to something besides the module's name or None. ast = pytd.TypeDeclUnit("some_mod", False, (), (), (), (), ()) mod = abstract.Module(self._vm, ast.name, {}, ast) mod.module = ast.name self.assertIsNone(mod.module) self.assertEqual(ast.name, mod.full_name) mod.module = None self.assertIsNone(mod.module) self.assertEqual(ast.name, mod.full_name) def set_module(): mod.module = "other_mod" self.assertRaises(AssertionError, set_module)
def testAliasPrinting(self): a = pytd.Alias("MyList", pytd.GenericType( pytd.NamedType("typing.List"), (pytd.AnythingType(),))) ty = pytd.TypeDeclUnit( name="test", constants=(), type_params=(), classes=(), functions=(), aliases=(a,)) expected = textwrap.dedent(""" from typing import Any, List MyList = List[Any]""") self.assertMultiLineEqual(expected.strip(), pytd.Print(ty).strip())
def compute_types(self, defs): ty = pytd_utils.Concat( self.pytd_for_types(defs), pytd.TypeDeclUnit( "unknowns", constants=tuple(), type_params=tuple(self.pytd_typevars()), classes=tuple(self.pytd_classes_for_unknowns()) + tuple(self.pytd_classes_for_call_traces()), functions=tuple(self.pytd_functions_for_call_traces()), aliases=tuple(self.pytd_aliases()))) ty = ty.Visit(optimize.PullInMethodClasses()) ty = ty.Visit(visitors.DefaceUnresolved( [ty, self.loader.concat_all()], "~unknown")) return ty.Visit(visitors.AdjustTypeParameters())
def p_unit(self, p): """unit : alldefs""" funcdefs = [x for x in p[1] if isinstance(x, NameAndSig)] constants = [x for x in p[1] if isinstance(x, pytd.Constant)] classes = [x for x in p[1] if isinstance(x, pytd.Class)] all_names = (list(set(f.name for f in funcdefs)) + [c.name for c in constants] + [c.name for c in classes]) duplicates = [ name for name, count in collections.Counter(all_names).items() if count >= 2 ] if duplicates: make_syntax_error( self, 'Duplicate top-level identifier(s):' + ', '.join(duplicates), p) p[0] = pytd.TypeDeclUnit( name=None, # replaced later, in Parse constants=tuple(constants), functions=tuple(self.MergeSignatures(funcdefs)), classes=tuple(classes))
def _build_type_decl_unit(self, defs): """Return a pytd.TypeDeclUnit for the given defs (plus parser state).""" # defs contains both constant and function definitions. constants, functions = _split_definitions(defs) constants.extend(self._constants) generated_classes = [ x for class_list in self._generated_classes.values() for x in class_list ] classes = generated_classes + self._classes all_names = (list(set(f.name for f in functions)) + [c.name for c in constants] + [c.name for c in self._type_params] + [c.name for c in classes] + [c.name for c in self._aliases]) duplicates = [ name for name, count in collections.Counter(all_names).items() if count >= 2 ] if duplicates: raise ParseError("Duplicate top-level identifier(s): " + ", ".join(duplicates)) functions, properties = _merge_signatures(functions) if properties: prop_names = ", ".join(p.name for p in properties) raise ParseError( "Module-level functions with property decorators: " + prop_names) return pytd.TypeDeclUnit(name=None, constants=tuple(constants), type_params=tuple(self._type_params), functions=tuple(functions), classes=tuple(classes), aliases=tuple(self._aliases))
def WrapTypeDeclUnit(name, items, is_package=False): """Given a list (classes, functions, etc.), wrap a pytd around them. Args: name: The name attribute of the resulting TypeDeclUnit. items: A list of items. Can contain pytd.Class, pytd.Function and pytd.Constant. is_package: Whether the module is a package (e.g. foo/__init__.pyi) Returns: A pytd.TypeDeclUnit. Raises: ValueError: In case of an invalid item in the list. NameError: For name conflicts. """ functions = collections.OrderedDict() classes = collections.OrderedDict() constants = collections.defaultdict(TypeBuilder) aliases = collections.OrderedDict() typevars = collections.OrderedDict() for item in items: if isinstance(item, pytd.Function): if item.name in functions: if item.kind != functions[item.name].kind: raise ValueError("Can't combine %s and %s", item.kind, functions[item.name].kind) functions[item.name] = pytd.Function( item.name, functions[item.name].signatures + item.signatures, item.kind) else: functions[item.name] = item elif isinstance(item, pytd.Class): if item.name in classes: raise NameError("Duplicate top level class: %r" % item.name) classes[item.name] = item elif isinstance(item, pytd.Constant): constants[item.name].add_type(item.type) elif isinstance(item, pytd.Alias): if item.name in aliases: raise NameError("Duplicate top level alias or import: %r" % item.name) aliases[item.name] = item elif isinstance(item, pytd.TypeParameter): if item.name in typevars: raise NameError("Duplicate top level type parameter: %r" % item.name) typevars[item.name] = item else: raise ValueError("Invalid top level pytd item: %r" % type(item)) categories = {"function": functions, "class": classes, "constant": constants, "alias": aliases, "typevar": typevars} for c1, c2 in itertools.combinations(categories, 2): _check_intersection(categories[c1], categories[c2], c1, c2) return pytd.TypeDeclUnit( name=name, is_package=is_package, constants=tuple( pytd.Constant(name, t.build()) for name, t in sorted(constants.items())), type_params=tuple(typevars.values()), classes=tuple(classes.values()), functions=tuple(functions.values()), aliases=tuple(aliases.values()))
def test_write_pickle(self): ast = pytd.TypeDeclUnit(None, (), (), (), (), ()) options = config.Options.create(output="/dev/null") io.write_pickle(ast, options) # just make sure we don't crash