def infer_types(src, errorlog, options, filename=None, run_builtins=True, deep=True, solve_unknowns=True, reverse_operators=True, cache_unknowns=False, maximum_depth=None): """Given Python source return its types. Args: src: A string containing Python source code. errorlog: Where error messages go. Instance of errors.ErrorLog. options: config.Options object filename: Filename of the program we're parsing. run_builtins: Whether to preload the native Python builtins when running the program. deep: If True, analyze all functions, even the ones not called by the main execution flow. solve_unknowns: If yes, try to replace structural types ("~unknowns") with nominal types. reverse_operators: If True, emulate operations like __radd__. cache_unknowns: If True, do a faster approximation of unknown types. maximum_depth: Depth of the analysis. Default: unlimited. Returns: A TypeDeclUnit Raises: AssertionError: In case of a bad parameter combination. """ tracer = CallTracer(errorlog=errorlog, options=options, module_name=_get_module_name(filename, options.pythonpath), reverse_operators=reverse_operators, cache_unknowns=cache_unknowns, maximum_depth=maximum_depth) loc, defs, builtin_names = tracer.run_program(src, filename, run_builtins) log.info("===Done run_program===") # TODO(pludemann): make test_inference.InferDedent and this code the same: if deep: tracer.exitpoint = tracer.analyze(loc, defs, builtin_names) else: tracer.exitpoint = loc ast = tracer.compute_types(defs, builtin_names) ast = tracer.loader.resolve_ast(ast) if solve_unknowns: log.info("=========== PyTD to solve =============\n%s", pytd.Print(ast)) ast = convert_structural.convert_pytd(ast, tracer.loader.concat_all()) if options.output_cfg or options.output_typegraph: if options.output_cfg and options.output_typegraph: raise AssertionError("Can output CFG or typegraph, but not both") dot = program_to_dot(tracer.program, set([]), bool(options.output_cfg)) proc = subprocess.Popen(["/usr/bin/dot", "-T", "svg", "-o", options.output_cfg or options.output_typegraph], stdin=subprocess.PIPE) proc.stdin.write(dot) proc.stdin.close() if options.output_pseudocode: src = program_to_pseudocode(tracer.program) with open(options.output_pseudocode, "w") as fi: fi.write(src) return ast
def test_isinstance(self): ast = self.parse(""" x = ... # type: `~unknown1` def `~__builtin__~isinstance`(object: int, class_or_type_or_tuple: tuple[nothing, ...]) -> `~unknown1` class `~unknown1`(object): pass """) expected = textwrap.dedent(""" x = ... # type: bool """).strip() ast = convert_structural.convert_pytd(ast, self.builtins_pytd) self.assertMultiLineEqual(pytd.Print(ast), expected)
def test_isinstance(self): sourcecode = textwrap.dedent(""" x = ... # type: `~unknown1` def `~isinstance`(object: int, class_or_type_or_tuple: tuple[nothing]) -> `~unknown1` class `~unknown1`(object): pass """) expected = textwrap.dedent(""" x = ... # type: bool """).strip() ast = parser.parse_string(sourcecode) ast = convert_structural.convert_pytd(ast, self.builtins_pytd) self.assertMultiLineEqual(pytd.Print(ast), expected)
def test_convert_with_type_params(self): ast = self.parse(""" from typing import Dict class A(object): def foo(self, x: `~unknown1`) -> bool class `~unknown1`(): def __setitem__(self, _1: str, _2: `~unknown2`) -> ? def update(self, _1: NoneType or Dict[nothing, nothing]) -> ? class `~unknown2`(): def append(self, _1:NoneType) -> NoneType """) ast = convert_structural.convert_pytd(ast, self.builtins_pytd) x = ast.Lookup("A").Lookup("foo").signatures[0].params[1].type self.assertIn("MutableSequence", pytd.Print(x))
def test_convert(self): sourcecode = textwrap.dedent(""" class A(object): def foo(self, x: `~unknown1`) -> ? def foobaz(self, x: int) -> int class `~unknown1`(object): def foobaz(self, x: int) -> int """) expected = textwrap.dedent(""" class A(object): def foo(self, x: A) -> Any: ... def foobaz(self, x: int) -> int: ... """).lstrip() ast = parser.parse_string(sourcecode) ast = convert_structural.convert_pytd(ast, self.builtins_pytd) self.assertMultiLineEqual(pytd.Print(ast), expected)
def test_convert(self): ast = self.parse(""" class A(object): def foo(self, x: `~unknown1`) -> ? def foobaz(self, x: int) -> int class `~unknown1`(object): def foobaz(self, x: int) -> int """) expected = textwrap.dedent(""" from typing import Any class A(object): def foo(self, x: A) -> Any: ... def foobaz(self, x: int) -> int: ... """).lstrip() ast = convert_structural.convert_pytd(ast, self.builtins_pytd) self.assertMultiLineEqual(pytd.Print(ast), expected)
def test_convert_with_type_params(self): sourcecode = textwrap.dedent(""" class A(object): def foo(self, x: `~unknown1`) -> bool class `~unknown1`(): def __setitem__(self, _1: str, _2: `~unknown2`) -> ? def update(self, _1: NoneType or Dict[nothing, nothing]) -> ? class `~unknown2`(): def append(self, v:NoneType) -> NoneType """) expected = textwrap.dedent(""" class A(object): def foo(self, x: Dict[str, List[Any]]) -> bool: ... """).lstrip() ast = parser.parse_string(sourcecode) ast = convert_structural.convert_pytd(ast, self.builtins_pytd) self.assertMultiLineEqual(pytd.Print(ast), expected)
def test_convert_with_type_params(self): ast = self.parse(""" class A(object): def foo(self, x: `~unknown1`) -> bool class `~unknown1`(): def __setitem__(self, _1: str, _2: `~unknown2`) -> ? def update(self, _1: NoneType or Dict[nothing, nothing]) -> ? class `~unknown2`(): def append(self, v:NoneType) -> NoneType """) expected = textwrap.dedent(""" from typing import Any, Dict, List class A(object): def foo(self, x: Dict[str, List[Any]]) -> bool: ... """).lstrip() ast = convert_structural.convert_pytd(ast, self.builtins_pytd) self.assertMultiLineEqual(pytd.Print(ast), expected)
def infer_types(src, errorlog, options, loader, filename=None, deep=True, init_maximum_depth=INIT_MAXIMUM_DEPTH, show_library_calls=False, maximum_depth=None, tracer_vm=None, **kwargs): """Given Python source return its types. Args: src: A string containing Python source code. errorlog: Where error messages go. Instance of errors.ErrorLog. options: config.Options object loader: A load_pytd.Loader instance to load PYI information. filename: Filename of the program we're parsing. deep: If True, analyze all functions, even the ones not called by the main execution flow. init_maximum_depth: Depth of analysis during module loading. show_library_calls: If True, call traces are kept in the output. maximum_depth: Depth of the analysis. Default: unlimited. tracer_vm: An instance of CallTracer, in case the caller wants to instantiate and retain the vm used for type inference. **kwargs: Additional parameters to pass to vm.VirtualMachine Returns: A tuple of (ast: TypeDeclUnit, builtins: TypeDeclUnit) Raises: AssertionError: In case of a bad parameter combination. """ # If the caller has passed in a vm, use that. if tracer_vm: assert isinstance(tracer_vm, CallTracer) tracer = tracer_vm else: tracer = CallTracer(errorlog=errorlog, options=options, generate_unknowns=options.protocols, store_all_calls=not deep, loader=loader, **kwargs) loc, defs = tracer.run_program(src, filename, init_maximum_depth) log.info("===Done running definitions and module-level code===") snapshotter = metrics.get_metric("memory", metrics.Snapshot) snapshotter.take_snapshot("analyze:infer_types:tracer") if deep: if maximum_depth is None: if not options.quick: maximum_depth = MAXIMUM_DEPTH elif options.analyze_annotated: # Since there's no point in analyzing annotated functions for inference, # the presence of this option means that the user wants checking, too. maximum_depth = QUICK_CHECK_MAXIMUM_DEPTH else: maximum_depth = QUICK_INFER_MAXIMUM_DEPTH tracer.exitpoint = tracer.analyze(loc, defs, maximum_depth) else: tracer.exitpoint = loc snapshotter.take_snapshot("analyze:infer_types:post") ast = tracer.compute_types(defs) ast = tracer.loader.resolve_ast(ast) if tracer.has_unknown_wildcard_imports or any( a in defs for a in abstract_utils.DYNAMIC_ATTRIBUTE_MARKERS): try: ast.Lookup("__getattr__") except KeyError: ast = pytd_utils.Concat( ast, builtins.GetDefaultAst(options.python_version)) # If merged with other if statement, triggers a ValueError: Unresolved class # when attempts to load from the protocols file if options.protocols: protocols_pytd = tracer.loader.import_name("protocols") else: protocols_pytd = None builtins_pytd = tracer.loader.concat_all() # Insert type parameters, where appropriate ast = ast.Visit(visitors.CreateTypeParametersForSignatures()) if options.protocols: log.info("=========== PyTD to solve =============\n%s", pytd_utils.Print(ast)) ast = convert_structural.convert_pytd(ast, builtins_pytd, protocols_pytd) elif not show_library_calls: log.info("Solving is turned off. Discarding call traces.") # Rename remaining "~unknown" to "?" ast = ast.Visit(visitors.RemoveUnknownClasses()) # Remove "~list" etc.: ast = convert_structural.extract_local(ast) _maybe_output_debug(options, tracer.program) return ast, builtins_pytd
def infer_types(src, errorlog, options, filename=None, run_builtins=True, deep=True, solve_unknowns=True, reverse_operators=False, cache_unknowns=False, extract_locals=True, init_maximum_depth=INIT_MAXIMUM_DEPTH, maximum_depth=None): """Given Python source return its types. Args: src: A string containing Python source code. errorlog: Where error messages go. Instance of errors.ErrorLog. options: config.Options object filename: Filename of the program we're parsing. run_builtins: Whether to preload the native Python builtins when running the program. deep: If True, analyze all functions, even the ones not called by the main execution flow. solve_unknowns: If yes, try to replace structural types ("~unknowns") with nominal types. reverse_operators: Experimental. Allow overloading __radd__ etc. For user-defined types only - our builtins don't need reverse operators. cache_unknowns: If True, do a faster approximation of unknown types. extract_locals: If not optimizing, should we at least remove the call traces? init_maximum_depth: Depth of analysis during module loading. maximum_depth: Depth of the analysis. Default: unlimited. Returns: A TypeDeclUnit Raises: AssertionError: In case of a bad parameter combination. """ tracer = CallTracer(errorlog=errorlog, options=options, module_name=_get_module_name(filename, options), reverse_operators=reverse_operators, cache_unknowns=cache_unknowns, generate_unknowns=not options.quick) loc, defs, builtin_names = tracer.run_program( src, filename, init_maximum_depth, run_builtins) log.info("===Done run_program===") if deep: tracer.exitpoint = tracer.analyze(loc, defs, builtin_names, maximum_depth) else: tracer.exitpoint = loc ast = tracer.compute_types(defs, builtin_names) ast = tracer.loader.resolve_ast(ast) if solve_unknowns: log.info("=========== PyTD to solve =============\n%s", pytd.Print(ast)) ast = convert_structural.convert_pytd(ast, tracer.loader.concat_all()) elif extract_locals: log.info("Solving is turned off. Discarding call traces.") # Rename "~unknown" to "?" ast = ast.Visit(visitors.RemoveUnknownClasses()) # Remove "~list" etc.: ast = convert_structural.extract_local(ast) if options.output_cfg or options.output_typegraph: if options.output_cfg and options.output_typegraph: raise AssertionError("Can output CFG or typegraph, but not both") dot = program_to_dot(tracer.program, set([]), bool(options.output_cfg)) proc = subprocess.Popen(["/usr/bin/dot", "-T", "svg", "-o", options.output_cfg or options.output_typegraph], stdin=subprocess.PIPE) proc.stdin.write(dot) proc.stdin.close() if options.output_debug: text = program_to_text(tracer.program) if options.output_debug == "-": log.info("=========== Program Dump =============\n%s", text) else: with open(options.output_debug, "w") as fi: fi.write(text) return ast
def infer_types(src, errorlog, options, loader, filename=None, run_builtins=True, deep=True, cache_unknowns=False, show_library_calls=False, analyze_annotated=False, init_maximum_depth=INIT_MAXIMUM_DEPTH, maximum_depth=None): """Given Python source return its types. Args: src: A string containing Python source code. errorlog: Where error messages go. Instance of errors.ErrorLog. options: config.Options object loader: A load_pytd.Loader instance to load PYI information. filename: Filename of the program we're parsing. run_builtins: Whether to preload the native Python builtins when running the program. deep: If True, analyze all functions, even the ones not called by the main execution flow. cache_unknowns: If True, do a faster approximation of unknown types. show_library_calls: If True, call traces are kept in the output. analyze_annotated: If True, analyze methods with type annotations, too. init_maximum_depth: Depth of analysis during module loading. maximum_depth: Depth of the analysis. Default: unlimited. Returns: A TypeDeclUnit Raises: AssertionError: In case of a bad parameter combination. """ tracer = CallTracer(errorlog=errorlog, options=options, module_name=get_module_name(filename, options), cache_unknowns=cache_unknowns, analyze_annotated=analyze_annotated, generate_unknowns=options.protocols, store_all_calls=not deep, loader=loader) loc, defs = tracer.run_program(src, filename, init_maximum_depth, run_builtins) log.info("===Done running definitions and module-level code===") snapshotter = metrics.get_metric("memory", metrics.Snapshot) snapshotter.take_snapshot("infer:infer_types:tracer") if deep: tracer.exitpoint = tracer.analyze(loc, defs, maximum_depth) else: tracer.exitpoint = loc snapshotter.take_snapshot("infer:infer_types:post") ast = tracer.compute_types(defs) ast = tracer.loader.resolve_ast(ast) if tracer.has_unknown_wildcard_imports or ("HAS_DYNAMIC_ATTRIBUTES" in defs or "has_dynamic_attributes" in defs): try: ast.Lookup("__getattr__") except KeyError: ast = pytd_utils.Concat( ast, builtins.GetDefaultAst(options.python_version)) # If merged with other if statement, triggers a ValueError: Unresolved class # when attempts to load from the protocols file if options.protocols: protocols_pytd = tracer.loader.import_name("protocols") else: protocols_pytd = None builtins_pytd = tracer.loader.concat_all() # Insert type parameters, where appropriate ast = ast.Visit(visitors.CreateTypeParametersForSignatures()) if options.protocols: log.info("=========== PyTD to solve =============\n%s", pytd.Print(ast)) ast = convert_structural.convert_pytd(ast, builtins_pytd, protocols_pytd) elif not show_library_calls: log.info("Solving is turned off. Discarding call traces.") # Rename remaining "~unknown" to "?" ast = ast.Visit(visitors.RemoveUnknownClasses()) # Remove "~list" etc.: ast = convert_structural.extract_local(ast) if options.output_cfg or options.output_typegraph: if options.output_cfg and options.output_typegraph: raise AssertionError("Can output CFG or typegraph, but not both") dot = debug.program_to_dot(tracer.program, set([]), bool(options.output_cfg)) proc = subprocess.Popen([ "/usr/bin/dot", "-T", "svg", "-o", options.output_cfg or options.output_typegraph ], stdin=subprocess.PIPE) proc.stdin.write(dot) proc.stdin.close() _maybe_output_debug(options, tracer.program) return ast, builtins_pytd
def infer_types( src, python_version, filename=None, run_builtins=True, pybuiltins_filename=None, pythonpath=(), find_pytd_import_ext=".pytd", import_drop_prefixes=(), output_cfg=None, output_typegraph=None, output_pseudocode=None, deep=True, solve_unknowns=True, reverse_operators=False, cache_unknowns=False, skip_repeat_calls=True, ): """Given Python source return its types. Args: src: A string containing Python source code. python_version: The python version to emulate (major, minor). filename: Filename of the program we're parsing. run_builtins: Whether to preload the native Python builtins when running the program. pybuiltins_filename: Path to Python builtins, or None for default. pythonpath: List of directories to search for .pytd-gen files. find_pytd_import_ext: Extension pattern to use when looking up import PyTD in pythonpath. import_drop_prefixes: List of prefixes to drop when resolving module names. output_cfg: A filename into which to save an SVG of the control flow graph. output_typegraph: A filename into which to save an SVG of the typegraph. output_pseudocode: A filename to write pseudo code to. deep: If True, analyze all functions, even the ones not called by the main execution flow. solve_unknowns: If yes, try to replace structural types ("~unknowns") with nominal types. reverse_operators: If True, emulate operations like __radd__. cache_unknowns: If True, do a faster approximation of unknown types. skip_repeat_calls: If True, don't rerun functions that have been called before with the same arguments and environment. Returns: A TypeDeclUnit Raises: AssertionError: In case of a bad parameter combination. """ tracer = CallTracer( python_version=python_version, module_name=_get_module_name(filename, pythonpath), reverse_operators=reverse_operators, cache_unknowns=cache_unknowns, pythonpath=pythonpath, find_pytd_import_ext=find_pytd_import_ext, import_drop_prefixes=import_drop_prefixes, pybuiltins_filename=pybuiltins_filename, skip_repeat_calls=skip_repeat_calls, ) loc, defs, builtin_names = tracer.run_program(src, filename, run_builtins) log.info("===Done run_program===") # TODO(pludemann): make test_inference.InferDedent and this code the same: if deep: tracer.exitpoint = tracer.analyze(loc, defs, builtin_names) else: tracer.exitpoint = loc ast = tracer.compute_types(defs, builtin_names) if solve_unknowns: log.info("=========== PyTD to solve =============\n%s", pytd.Print(ast)) ast = convert_structural.convert_pytd(ast, tracer.loader.concat_all()) if output_cfg or output_typegraph: if output_cfg and output_typegraph: raise AssertionError("Can output CFG or typegraph, but not both") dot = program_to_dot(tracer.program, set([]), bool(output_cfg)) proc = subprocess.Popen( ["/usr/bin/dot", "-T", "svg", "-o", output_cfg or output_typegraph], stdin=subprocess.PIPE ) proc.stdin.write(dot) proc.stdin.close() if output_pseudocode: src = program_to_pseudocode(tracer.program) with open(output_pseudocode, "w") as fi: fi.write(src) return ast
def infer_types(src, errorlog, options, filename=None, run_builtins=True, deep=True, solve_unknowns=True, cache_unknowns=False, show_library_calls=False, analyze_annotated=False, init_maximum_depth=INIT_MAXIMUM_DEPTH, maximum_depth=None): """Given Python source return its types. Args: src: A string containing Python source code. errorlog: Where error messages go. Instance of errors.ErrorLog. options: config.Options object filename: Filename of the program we're parsing. run_builtins: Whether to preload the native Python builtins when running the program. deep: If True, analyze all functions, even the ones not called by the main execution flow. solve_unknowns: If yes, try to replace structural types ("~unknowns") with nominal types. cache_unknowns: If True, do a faster approximation of unknown types. show_library_calls: If True, call traces are kept in the output. analyze_annotated: If True, analyze methods with type annotations, too. init_maximum_depth: Depth of analysis during module loading. maximum_depth: Depth of the analysis. Default: unlimited. Returns: A TypeDeclUnit Raises: AssertionError: In case of a bad parameter combination. """ tracer = CallTracer(errorlog=errorlog, options=options, module_name=_get_module_name(filename, options), cache_unknowns=cache_unknowns, analyze_annotated=analyze_annotated, generate_unknowns=not options.quick, store_all_calls=not deep) loc, defs = tracer.run_program( src, filename, init_maximum_depth, run_builtins) log.info("===Done running definitions and module-level code===") if deep: tracer.exitpoint = tracer.analyze(loc, defs, maximum_depth) else: tracer.exitpoint = loc ast = tracer.compute_types(defs) ast = tracer.loader.resolve_ast(ast) if tracer.has_unknown_wildcard_imports: try: ast.Lookup("__getattr__") except KeyError: ast = pytd_utils.Concat( ast, builtins.GetDefaultAst(options.python_version)) builtins_pytd = tracer.loader.concat_all() if solve_unknowns: log.info("=========== PyTD to solve =============\n%s", pytd.Print(ast)) ast = convert_structural.convert_pytd(ast, builtins_pytd) elif not show_library_calls: log.info("Solving is turned off. Discarding call traces.") # Rename "~unknown" to "?" ast = ast.Visit(visitors.RemoveUnknownClasses()) # Remove "~list" etc.: ast = convert_structural.extract_local(ast) if options.output_cfg or options.output_typegraph: if options.output_cfg and options.output_typegraph: raise AssertionError("Can output CFG or typegraph, but not both") dot = program_to_dot(tracer.program, set([]), bool(options.output_cfg)) proc = subprocess.Popen(["/usr/bin/dot", "-T", "svg", "-o", options.output_cfg or options.output_typegraph], stdin=subprocess.PIPE) proc.stdin.write(dot) proc.stdin.close() _maybe_output_debug(options, tracer.program) return ast, builtins_pytd
def infer_types(src, python_version, errorlog, filename=None, run_builtins=True, pybuiltins_filename=None, imports_map=None, pythonpath=(), find_pytd_import_ext=".pytd", import_drop_prefixes=(), output_cfg=None, output_typegraph=None, output_pseudocode=None, deep=True, solve_unknowns=True, reverse_operators=True, cache_unknowns=False, skip_repeat_calls=True, maximum_depth=None): """Given Python source return its types. Args: src: A string containing Python source code. python_version: The python version to emulate (major, minor). errorlog: Where error messages go. Instance of ErrorLog. filename: Filename of the program we're parsing. run_builtins: Whether to preload the native Python builtins when running the program. pybuiltins_filename: Path to Python builtins, or None for default. imports_map: map of .py file name to corresponding pytd file (generated by a separate invocation of pytype). pythonpath: List of directories to search for *.pytd (or *.${find_pytd_import_ext}) files. find_pytd_import_ext: Extension pattern to use when looking up import PyTD in pythonpath. import_drop_prefixes: List of prefixes to drop when resolving module names. output_cfg: A filename into which to save an SVG of the control flow graph. output_typegraph: A filename into which to save an SVG of the typegraph. output_pseudocode: A filename to write pseudo code to. deep: If True, analyze all functions, even the ones not called by the main execution flow. solve_unknowns: If yes, try to replace structural types ("~unknowns") with nominal types. reverse_operators: If True, emulate operations like __radd__. cache_unknowns: If True, do a faster approximation of unknown types. skip_repeat_calls: If True, don't rerun functions that have been called before with the same arguments and environment. maximum_depth: Depth of the analysis. Default: unlimited. Returns: A TypeDeclUnit Raises: AssertionError: In case of a bad parameter combination. """ tracer = CallTracer(python_version=python_version, imports_map=imports_map, errorlog=errorlog, module_name=_get_module_name(filename, pythonpath), reverse_operators=reverse_operators, cache_unknowns=cache_unknowns, pythonpath=pythonpath, find_pytd_import_ext=find_pytd_import_ext, import_drop_prefixes=import_drop_prefixes, pybuiltins_filename=pybuiltins_filename, skip_repeat_calls=skip_repeat_calls, maximum_depth=maximum_depth) loc, defs, builtin_names = tracer.run_program(src, filename, run_builtins) log.info("===Done run_program===") # TODO(pludemann): make test_inference.InferDedent and this code the same: if deep: tracer.exitpoint = tracer.analyze(loc, defs, builtin_names) else: tracer.exitpoint = loc ast = tracer.compute_types(defs, builtin_names) if solve_unknowns: log.info("=========== PyTD to solve =============\n%s", pytd.Print(ast)) ast = convert_structural.convert_pytd(ast, tracer.loader.concat_all()) if output_cfg or output_typegraph: if output_cfg and output_typegraph: raise AssertionError("Can output CFG or typegraph, but not both") dot = program_to_dot(tracer.program, set([]), bool(output_cfg)) proc = subprocess.Popen(["/usr/bin/dot", "-T", "svg", "-o", output_cfg or output_typegraph], stdin=subprocess.PIPE) proc.stdin.write(dot) proc.stdin.close() if output_pseudocode: src = program_to_pseudocode(tracer.program) with open(output_pseudocode, "w") as fi: fi.write(src) return ast