def check_types(py_src, pytd_src, py_filename, pytd_filename, errorlog, options, run_builtins=True, deep=True, cache_unknowns=False, init_maximum_depth=INIT_MAXIMUM_DEPTH): """Verify a PyTD against the Python code.""" tracer = CallTracer(errorlog=errorlog, options=options, module_name=_get_module_name(py_filename, options), cache_unknowns=cache_unknowns, analyze_annotated=True, generate_unknowns=False) loc, defs = tracer.run_program( py_src, py_filename, init_maximum_depth, run_builtins) if pytd_src is not None: del deep # ignored ast = builtins.ParsePyTD(pytd_src, pytd_filename, options.python_version, lookup_classes=True) ast = tracer.loader.resolve_ast(ast) tracer.check_types(loc, defs, ast, os.path.basename(py_filename), os.path.basename(pytd_filename)) elif deep: tracer.analyze(loc, defs, maximum_depth=(2 if options.quick else None)) _maybe_output_debug(options, tracer.program)
def parse_type_definition(pyi_subdir, module, python_version, typeshed_location, use_pickled): """Load and parse a *.pyi from typeshed. Args: pyi_subdir: the directory where the module should be found. module: the module name (without any file extension) python_version: sys.version_info[:2] typeshed_location: Location of the typeshed interface definitions. use_pickled: A boolean, iff True typeshed will try to load pickled files. Returns: The AST of the module; None if the module doesn't have a definition. """ typeshed = _get_typeshed(typeshed_location, use_pickled) try: filename, src = typeshed.get_module_file(pyi_subdir, module, python_version) except IOError: return None ast = builtins.ParsePyTD( src, filename=filename, module=module, python_version=python_version).Replace(name=module) return ast
def testParsePyTD(self): """Test ParsePyTD().""" ast = builtins.ParsePyTD("a = ... # type: int", "<inline>", python_version=self.PYTHON_VERSION, lookup_classes=True) a = ast.Lookup("a").type self.assertItemsEqual(a, pytd.ClassType("int")) self.assertIsNotNone(a.cls) # verify that the lookup succeeded
def load_file(self, module_name, filename, ast=None): """Load (or retrieve from cache) a module and resolve its dependencies.""" self._concatenated = None # invalidate existing = self._get_existing_ast(module_name, filename) if existing: return existing if not ast: ast = builtins.ParsePyTD(filename=filename, module=module_name, python_version=self.options.python_version) return self._process_module(module_name, filename, ast)
def _test_parse(pyi_file): python_version = (2, 7) module = os.path.splitext(os.path.basename(pyi_file))[0] if "__init__" == module: module = os.path.basename(os.path.dirname(pyi_file)) with open(pyi_file) as f: src = f.read() # Call ParsePyTD directly to avoid typeshed.get_typeshed_file logic builtins.ParsePyTD(src, filename=pyi_file, module=module, python_version=python_version)
def parse_type_definition(pyi_subdir, module, python_version): """Load and parse a *.pyi from typeshed. Args: pyi_subdir: the directory where the module should be found module: the module name (without any file extension) python_version: sys.version_info[:2] Returns: The AST of the module; None if the module doesn't have a definition. """ try: filename, src = get_typeshed_file(pyi_subdir, module, python_version) except IOError: return None return builtins.ParsePyTD( src, filename=filename, module=module, python_version=python_version).Replace(name=module)
def _load_file(self, module_name, filename, ast=None): """Load (or retrieve from cache) a module and resolve its dependencies.""" self._concatenated = None # invalidate existing = self._modules.get(module_name) if existing: if existing.filename != filename: raise AssertionError( "%s exists as both %s and %s" % (module_name, filename, existing.filename)) return existing.ast if not ast: ast = builtins.ParsePyTD( filename=filename, module=module_name, python_version=self.options.python_version) ast = self._postprocess_pyi(ast) module = Module(module_name, filename, ast) self._modules[module_name] = module try: module.ast = self._load_and_resolve_ast_dependencies( module.ast, module_name) # Now that any imported TypeVar instances have been resolved, adjust type # parameters in classes and functions. module.ast = visitors.AdjustTypeParameters(module.ast) # Now we can fill in internal cls pointers to ClassType nodes in the # module. This code executes when the module is first loaded, which # happens before any others use it to resolve dependencies, so there are # no external pointers into the module at this point. module.ast.Visit( visitors.FillInModuleClasses({ "": module.ast, module_name: module.ast })) # TODO(rechen): Once generics are supported in inline type annotations, a # VerifyContainers check should also be done on the final ast. module.ast.Visit(visitors.VerifyContainers()) except: del self._modules[ module_name] # don't leave half-resolved modules around raise return module.ast
def PrepareForExport(module_name, python_version, ast, loader): """Prepare an ast as if it was parsed and loaded. External dependencies will not be resolved, as the ast generated by this method is supposed to be exported. Args: module_name: The module_name as a string for the returned ast. python_version: A tuple of (major, minor) python version as string (see config.python_version). ast: pytd.TypeDeclUnit, is only used if src is None. loader: A load_pytd.Loader instance. Returns: A pytd.TypeDeclUnit representing the supplied AST as it would look after being written to a file and parsed. """ # This is a workaround for functionality which crept into places it doesn't # belong. Ideally this would call some transformation Visitors on ast to # transform it into the same ast we get after parsing and loading (compare # load_pytd.Loader.load_file). Unfortunately parsing has some special cases, # e.g. '__init__' return type and '__new__' being a 'staticmethod', which # need to be moved to visitors before we can do this. Printing an ast also # applies transformations, # e.g. visitors.PrintVisitor._FormatContainerContents, which need to move to # their own visitors so they can be applied without printing. src = pytd_utils.Print(ast) ast = pytd_builtins.ParsePyTD(src=src, module=module_name, python_version=python_version) ast = ast.Visit(visitors.LookupBuiltins(loader.builtins, full_names=False)) ast = ast.Visit(visitors.ExpandCompatibleBuiltins(loader.builtins)) ast = ast.Visit(visitors.LookupLocalTypes()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.FillInLocalPointers({"": ast, module_name: ast})) ast = ast.Visit(visitors.CanonicalOrderingVisitor()) ast = ast.Visit( visitors.ClassTypeToLateType( ignore=[module_name + ".", "__builtin__.", "typing."])) return ast
def check_types(py_src, pytd_src, py_filename, pytd_filename, errorlog, options, run_builtins=True, reverse_operators=False, cache_unknowns=False, init_maximum_depth=INIT_MAXIMUM_DEPTH): """Verify a PyTD against the Python code.""" tracer = CallTracer(errorlog=errorlog, options=options, module_name=_get_module_name(py_filename, options), reverse_operators=reverse_operators, cache_unknowns=cache_unknowns, generate_unknowns=False) loc, defs, builtin_names = tracer.run_program( py_src, py_filename, init_maximum_depth, run_builtins) if pytd_src is not None: ast = builtins.ParsePyTD(pytd_src, pytd_filename, options.python_version, lookup_classes=True) ast = tracer.loader.resolve_ast(ast) tracer.check_types(loc, defs, ast, os.path.basename(py_filename), os.path.basename(pytd_filename)) else: tracer.analyze(loc, defs, builtin_names, maximum_depth=(1 if options.quick else None))