def process_one_file(options): """Check a .py file or generate a .pyi for it, according to options. Args: options: config.Options object. Returns: An error code (0 means no error). """ log.info("Process %s => %s", options.input, options.output) loader = load_pytd.create_loader(options) try: errorlog, result, ast = check_or_generate_pyi(options, loader) except utils.UsageError as e: logging.error("Usage error: %s\n", utils.message(e)) return 1 if not options.check: if options.pickle_output: pyi_output = options.verify_pickle else: pyi_output = options.output # Write out the pyi file. if pyi_output: _write_pyi_output(options, result, pyi_output) # Write out the pickle file. if options.pickle_output: log.info("write pickle %r => %r", options.input, options.output) write_pickle(ast, options, loader) exit_status = handle_errors(errorlog, options) # If we have set return_success, set exit_status to 0 after the regular error # handler has been called. if options.return_success: exit_status = 0 # Touch output file upon success. if options.touch and not exit_status: with open(options.touch, "a"): os.utime(options.touch, None) return exit_status
def run_pytype(*, filename: str, python_version: str, typeshed_location: str) -> str | None: """Runs pytype, returning the stderr if any.""" if python_version not in _LOADERS: options = pytype_config.Options.create("", parse_pyi=True, python_version=python_version) loader = load_pytd.create_loader(options) _LOADERS[python_version] = (options, loader) options, loader = _LOADERS[python_version] stderr: str | None try: with pytype_config.verbosity_from(options): ast = loader.load_file(_get_module_name(filename), filename) loader.finish_and_verify_ast(ast) except Exception: stderr = traceback.format_exc() else: stderr = None return stderr
def process_file(options): """Process a single file and return cross references.""" # We bind the global ast variable in this function. global ast errorlog = errors.ErrorLog() loader = load_pytd.create_loader(options) src = io.read_source_file(options.input) vm = analyze.CallTracer(errorlog=errorlog, options=options, generate_unknowns=options.protocols, store_all_calls=False, loader=loader) try: analyze.infer_types(src=src, filename=options.input, errorlog=errorlog, options=options, loader=loader, tracer_vm=vm) except utils.UsageError as e: logging.error("Usage error: %s\n", utils.message(e)) return 1 major, minor = options.python_version if major == 2: # python2.7 is the only supported py2 version. a = ast27.parse(src, options.input) ast = ast27 else: a = ast3.parse(src, options.input, feature_version=minor) ast = ast3 # TODO(mdemello): Get from args module_name = "module" source = SourceFile(src, vm.opcode_traces, filename=options.input) ix = Indexer(source, module_name) ix.index(a) ix.finalize() return ix
def process_one_file(options): """Check a .py file or generate a .pyi for it, according to options. Args: options: config.Options object. Returns: An error code (0 means no error). """ log.info("Process %s => %s", options.input, options.output) errorlog = errors.ErrorLog() loader = load_pytd.create_loader(options) try: generated_values = check_or_generate_pyi(options, errorlog, loader) except utils.UsageError as e: logging.error("Usage error: %s\n", utils.message(e)) return 1 if not options.check: result, ast = generated_values if options.output == "-" or not options.output: sys.stdout.write(result) else: log.info("write pyi %r => %r", options.input, options.output) with open(options.output, "w") as fi: fi.write(result) if options.output_pickled: write_pickle(ast, loader, options) exit_status = handle_errors(errorlog, options) # If we have set return_success, set exit_status to 0 after the regular error # handler has been called. if options.return_success: exit_status = 0 # Touch output file upon success. if options.touch and not exit_status: with open(options.touch, "a"): os.utime(options.touch, None) return exit_status
def write_pickle(ast, options, loader=None): """Dump a pickle of the ast to a file.""" loader = loader or load_pytd.create_loader(options) try: ast = serialize_ast.PrepareForExport(options.module_name, ast, loader) except parser.ParseError as e: if options.nofail: ast = serialize_ast.PrepareForExport( options.module_name, pytd_builtins.GetDefaultAst(options.python_version), loader) log.warning("***Caught exception: %s", str(e), exc_info=True) else: raise if options.verify_pickle: ast1 = ast.Visit(visitors.LateTypeToClassType()) ast1 = ast1.Visit(visitors.ClearClassPointers()) ast2 = loader.load_file(options.module_name, options.verify_pickle) ast2 = ast2.Visit(visitors.ClearClassPointers()) if not pytd_utils.ASTeq(ast1, ast2): raise AssertionError() serialize_ast.StoreAst(ast, options.output, options.open_function)
def process_file(options): """Process a single file and return cross references.""" errorlog = errors.ErrorLog() loader = load_pytd.create_loader(options) src = io.read_source_file(options.input) vm = analyze.CallTracer(errorlog=errorlog, options=options, generate_unknowns=options.protocols, store_all_calls=False, loader=loader) try: analyze.infer_types(src=src, filename=options.input, errorlog=errorlog, options=options, loader=loader, tracer_vm=vm) except utils.UsageError as e: logging.error('Usage error: %s\n', utils.message(e)) return 1 display_traces(src, vm.opcode_traces)
def loader(self): if not _MatchLoaderConfig(self.options, self._loader): # Create a new loader only if the configuration in the current options # does not match the configuration in the current loader. self._loader = load_pytd.create_loader(self.options) return self._loader
def process_file(options, source_text=None, generate_callgraphs=False, preserve_pytype_vm=False): """Process a single file and return cross references. Args: options: A dictionary of pytype options. source_text: Optional text of the file; will be read from the file pointed to by options.input if not supplied. generate_callgraphs: Collect call graph information preserve_pytype_vm: Preserve the pytype vm in the indexer Returns: The Indexer object used for indexing. Raises: PytypeError if pytype fails. """ errorlog = errors.ErrorLog() loader = load_pytd.create_loader(options) src = source_text or io.read_source_file(options.input) vm = analyze.CallTracer(errorlog=errorlog, options=options, generate_unknowns=options.protocols, store_all_calls=True, loader=loader) with io.wrap_pytype_exceptions(PytypeError, filename=options.input): pytd_module, _ = analyze.infer_types(src=src, filename=options.input, errorlog=errorlog, options=options, loader=loader, tracer_vm=vm) major, minor = options.python_version if major == 2: # python2.7 is the only supported py2 version. ast_root_node = ast27.parse(src, options.input) ast = ast27 else: ast_root_node = ast3.parse(src, options.input, feature_version=minor) ast = ast3 # TODO(mdemello): Get from args module_name = "module" src_code = source.Code(src, vm.opcode_traces, VmTrace, filename=options.input) ix = Indexer(ast=ast, src=src_code, loader=vm.loader, module_name=module_name, pytd_module=pytd_module) ix.index(ast_root_node) ix.finalize() # Make the vm available via indexer.vm for post-finalize() functions. ix.vm = vm # Use the indexer as a single object to hold data for calling processes. if generate_callgraphs: ix.function_map = callgraph.collect_function_map(ix) # Release the vm before returning if not preserve_pytype_vm: ix.vm = None return ix
def parse_pyi(options): """Tries parsing a PYI file.""" loader = load_pytd.create_loader(options) loader.load_file(options.module_name, options.input)
def process_one_file(options): """Check a .py file or generate a .pyi for it, according to options. Args: options: config.Options object. Returns: An error code (0 means no error). """ log.info("Process %s => %s", options.input, options.output) errorlog = errors.ErrorLog() result = pytd_builtins.DEFAULT_SRC ast = pytd_builtins.GetDefaultAst(options.python_version) loader = load_pytd.create_loader(options) try: if options.check: check_py(input_filename=options.input, errorlog=errorlog, options=options, loader=loader) else: result, ast = generate_pyi(input_filename=options.input, errorlog=errorlog, options=options, loader=loader) except utils.UsageError as e: logging.error("Usage error: %s\n", utils.message(e)) return 1 except pyc.CompileError as e: errorlog.python_compiler_error(options.input, e.lineno, e.error) except IndentationError as e: errorlog.python_compiler_error(options.input, e.lineno, e.msg) except tokenize.TokenError as e: msg, (lineno, unused_column) = e.args # pylint: disable=unbalanced-tuple-unpacking errorlog.python_compiler_error(options.input, lineno, msg) except directors.SkipFile: result += "# skip-file found, file not analyzed" except Exception as e: # pylint: disable=broad-except if options.nofail: log.warn("***Caught exception: %s", str(e), exc_info=True) if not options.check: result += ( # pytype: disable=name-error "# Caught error in pytype: " + str(e).replace("\n", "\n#") + "\n# " + "\n# ".join(traceback.format_exc().splitlines())) else: e.args = (str(utils.message(e)) + "\nFile: %s" % options.input, ) + e.args[1:] raise if not options.check: if options.output == "-" or not options.output: sys.stdout.write(result) else: log.info("write pyi %r => %r", options.input, options.output) with open(options.output, "w") as fi: fi.write(result) if options.output_pickled: write_pickle(ast, loader, options) exit_status = handle_errors(errorlog, options) # If we have set return_success, set exit_status to 0 after the regular error # handler has been called. if options.return_success: exit_status = 0 # Touch output file upon success. if options.touch and not exit_status: with open(options.touch, "a"): os.utime(options.touch, None) return exit_status
def process_file(options, source_text=None, kythe_args=None, keep_pytype_data=False, ast_factory=None, annotate_ast=False): """Process a single file and return cross references. Args: options: A dictionary of pytype options. source_text: Optional text of the file; will be read from the file pointed to by options.input if not supplied. kythe_args: Extra args for generating the kythe index keep_pytype_data: Whether to preserve the Reference.data field. If true, the field will hold the type of the reference as a str or Tuple[str, str] (for attributes). Otherwise, it will be inaccessible. ast_factory: Callable to return an ast-module-compatible object to parse the source text into an ast-compatible object. It is passed the pytype Options object. If not specified, typed_ast will be used. annotate_ast: Whether to annotate the ast with type information. Nodes with type information will have these attributes added: * `.resolved_type`: the pytd information about the type * `.resolved_annotation`: A string representation of the type, as would be written in an annotation. Returns: The Indexer object used for indexing, and the created AST object. The AST object may have been modified if `annotate_ast=True`. Raises: PytypeError if pytype fails. """ # We bind the global ast variable in this function. global ast errorlog = errors.ErrorLog() loader = load_pytd.create_loader(options) src = source_text or io.read_source_file(options.input) vm = analyze.CallTracer( errorlog=errorlog, options=options, generate_unknowns=options.protocols, store_all_calls=False, loader=loader) with io.wrap_pytype_exceptions(PytypeError, filename=options.input): pytype_ast, _ = analyze.infer_types( src=src, filename=options.input, errorlog=errorlog, options=options, loader=loader, tracer_vm=vm) if ast_factory: ast = ast_factory(options) ast_root_node = ast.parse(src, options.input) else: major, minor = options.python_version if major == 2: # python2.7 is the only supported py2 version. ast_root_node = ast27.parse(src, options.input) ast = ast27 else: ast_root_node = ast3.parse(src, options.input, feature_version=minor) ast = ast3 # TODO(mdemello): Get from args module_name = "module" source = SourceFile(src, vm.opcode_traces, filename=options.input) ix = Indexer( source, vm.loader, module_name, kythe_args, annotate_ast=annotate_ast) ix.index(ast_root_node) ix.finalize(keep_pytype_data, pytype_ast) return ix, ast_root_node