def __init__(self, test, srccode, deep=False,
solve_unknowns=False, extract_locals=False,
reverse_operators=True, extra_verbose=False,
report_errors=True, pythonpath=(),
find_pytd_import_ext=".pytd"):
"""Constructor for Infer.
Args:
test: the Testcase (see inferenceTest.Infer)
srccode: the Python source code to do type inferencing on
deep: see class comments (assume --api - analyize all methods, even those
that don't have a caller)
solve_unknowns: try to solve for all ~unknown types
extract_locals: strip ~unknown types from the output pytd
reverse_operators: Whether to try e.g. both __add__ and __radd__
extra_verbose: extra intermeidate output (for debugging)
report_errors: Whether to fail if the type inferencer reports any erros.
pythonpath: list of directories for imports
find_pytd_import_ext: the file extension pattern for imports
"""
# TODO(pludemann): There are eight possible combinations of these three
# boolean flags. Do all of these combinations make sense? Or would it be
# possible to simplify this into something like a "mode" parameter:
# mode="solve" => deep=True, solve_unknowns=True
# mode="structural" => deep=True, solve_unknowns=False, extract_locals=False
# mode="deep" => deep=True, solve_unknowns=False, extract_locals=True
# mode="main" => deep=False, solve_unknowns=False, extract_locals=True
self.srccode = textwrap.dedent(srccode)
self.inferred = None
self.optimized_types = None
self.extract_locals = None # gets set if extract_locals is set (below)
self.extra_verbose = extra_verbose
self.canonical_types = None
# We need to catch any exceptions here and preserve them for __exit__.
# Exceptions raised in the body of 'with' will be presented to __exit__.
try:
self.types = test._InferAndVerify(
self.srccode, deep=deep, solve_unknowns=solve_unknowns,
reverse_operators=reverse_operators, cache_unknowns=True,
report_errors=report_errors,
pythonpath=pythonpath, find_pytd_import_ext=find_pytd_import_ext)
self.inferred = self.types
if extract_locals:
# Rename "~unknown" to "?"
self.types = self.types.Visit(visitors.RemoveUnknownClasses())
# Remove "~list" etc.:
self.types = convert_structural.extract_local(self.types)
self.extract_locals = self.types
# TODO(pludemann): These flags are the same as those in main.py; there
# should be a way of ensuring that they're the same.
self.types = self.optimized_types = optimize.Optimize(
self.types, lossy=False, use_abcs=False,
max_union=7, remove_mutable=False)
self.types = self.canonical_types = pytd_utils.CanonicalOrdering(
self.types)
except Exception: # pylint: disable=broad-except
self.types = None
if not self.__exit__(*sys.exc_info()):
raise
def __init__(self, test, srccode, deep=False,
solve_unknowns=False, extract_locals=False,
extra_verbose=False, report_errors=True, **kwargs):
"""Constructor for Infer.
Args:
test: the Testcase (see inferenceTest.Infer)
srccode: the Python source code to do type inferencing on
deep: see class comments (assume --api - analyize all methods, even those
that don't have a caller)
solve_unknowns: try to solve for all ~unknown types
extract_locals: strip ~unknown types from the output pytd
extra_verbose: extra intermeidate output (for debugging)
report_errors: Whether to fail if the type inferencer reports any erros.
**kwargs: Additional options to pass through to infer_types().
"""
# TODO(pludemann): There are eight possible combinations of these three
# boolean flags. Do all of these combinations make sense? Or would it be
# possible to simplify this into something like a "mode" parameter:
# mode="solve" => deep=True, solve_unknowns=True
# mode="structural" => deep=True, solve_unknowns=False, extract_locals=False
# mode="deep" => deep=True, solve_unknowns=False, extract_locals=True
# mode="main" => deep=False, solve_unknowns=False, extract_locals=True
self.srccode = textwrap.dedent(srccode)
self.inferred = None
self.optimized_types = None
self.extract_locals = None # gets set if extract_locals is set (below)
self.extra_verbose = extra_verbose
self.canonical_types = None
# We need to catch any exceptions here and preserve them for __exit__.
# Exceptions raised in the body of 'with' will be presented to __exit__.
try:
self.types = test._InferAndVerify(
self.srccode, deep=deep, cache_unknowns=True,
solve_unknowns=solve_unknowns,
report_errors=report_errors, **kwargs)
self.inferred = self.types
if extract_locals:
# Rename "~unknown" to "?"
self.types = self.types.Visit(visitors.RemoveUnknownClasses())
# Remove "~list" etc.:
self.types = convert_structural.extract_local(self.types)
self.extract_locals = self.types
# TODO(pludemann): These flags are the same as those in main.py; there
# should be a way of ensuring that they're the same.
self.types = self.optimized_types = optimize.Optimize(
self.types, lossy=False, use_abcs=False,
max_union=7, remove_mutable=False)
self.types = self.canonical_types = pytd_utils.CanonicalOrdering(
self.types)
except Exception: # pylint: disable=broad-except
self.types = None
if not self.__exit__(*sys.exc_info()):
raise
def Infer(self, srccode, pythonpath=(), deep=False, solve_unknowns=False,
extract_locals=False, report_errors=True, **kwargs):
types = self._InferAndVerify(
textwrap.dedent(srccode), pythonpath=pythonpath, deep=deep,
cache_unknowns=True, solve_unknowns=solve_unknowns,
report_errors=report_errors, **kwargs)
if extract_locals:
# Rename "~unknown" to "?"
types = types.Visit(visitors.RemoveUnknownClasses())
# Remove "~list" etc.:
types = convert_structural.extract_local(types)
types = optimize.Optimize(types, lossy=False, use_abcs=False,
max_union=7, remove_mutable=False)
types = pytd_utils.CanonicalOrdering(types)
return types
def infer_types(src,
errorlog,
options,
loader,
filename=None,
deep=True,
init_maximum_depth=INIT_MAXIMUM_DEPTH,
show_library_calls=False,
maximum_depth=None,
tracer_vm=None,
**kwargs):
"""Given Python source return its types.
Args:
src: A string containing Python source code.
errorlog: Where error messages go. Instance of errors.ErrorLog.
options: config.Options object
loader: A load_pytd.Loader instance to load PYI information.
filename: Filename of the program we're parsing.
deep: If True, analyze all functions, even the ones not called by the main
execution flow.
init_maximum_depth: Depth of analysis during module loading.
show_library_calls: If True, call traces are kept in the output.
maximum_depth: Depth of the analysis. Default: unlimited.
tracer_vm: An instance of CallTracer, in case the caller wants to
instantiate and retain the vm used for type inference.
**kwargs: Additional parameters to pass to vm.VirtualMachine
Returns:
A tuple of (ast: TypeDeclUnit, builtins: TypeDeclUnit)
Raises:
AssertionError: In case of a bad parameter combination.
"""
# If the caller has passed in a vm, use that.
if tracer_vm:
assert isinstance(tracer_vm, CallTracer)
tracer = tracer_vm
else:
tracer = CallTracer(errorlog=errorlog,
options=options,
generate_unknowns=options.protocols,
store_all_calls=not deep,
loader=loader,
**kwargs)
loc, defs = tracer.run_program(src, filename, init_maximum_depth)
log.info("===Done running definitions and module-level code===")
snapshotter = metrics.get_metric("memory", metrics.Snapshot)
snapshotter.take_snapshot("analyze:infer_types:tracer")
if deep:
if maximum_depth is None:
if not options.quick:
maximum_depth = MAXIMUM_DEPTH
elif options.analyze_annotated:
# Since there's no point in analyzing annotated functions for inference,
# the presence of this option means that the user wants checking, too.
maximum_depth = QUICK_CHECK_MAXIMUM_DEPTH
else:
maximum_depth = QUICK_INFER_MAXIMUM_DEPTH
tracer.exitpoint = tracer.analyze(loc, defs, maximum_depth)
else:
tracer.exitpoint = loc
snapshotter.take_snapshot("analyze:infer_types:post")
ast = tracer.compute_types(defs)
ast = tracer.loader.resolve_ast(ast)
if tracer.has_unknown_wildcard_imports or any(
a in defs for a in abstract_utils.DYNAMIC_ATTRIBUTE_MARKERS):
try:
ast.Lookup("__getattr__")
except KeyError:
ast = pytd_utils.Concat(
ast, builtins.GetDefaultAst(options.python_version))
# If merged with other if statement, triggers a ValueError: Unresolved class
# when attempts to load from the protocols file
if options.protocols:
protocols_pytd = tracer.loader.import_name("protocols")
else:
protocols_pytd = None
builtins_pytd = tracer.loader.concat_all()
# Insert type parameters, where appropriate
ast = ast.Visit(visitors.CreateTypeParametersForSignatures())
if options.protocols:
log.info("=========== PyTD to solve =============\n%s",
pytd_utils.Print(ast))
ast = convert_structural.convert_pytd(ast, builtins_pytd,
protocols_pytd)
elif not show_library_calls:
log.info("Solving is turned off. Discarding call traces.")
# Rename remaining "~unknown" to "?"
ast = ast.Visit(visitors.RemoveUnknownClasses())
# Remove "~list" etc.:
ast = convert_structural.extract_local(ast)
_maybe_output_debug(options, tracer.program)
return ast, builtins_pytd
def infer_types(src,
errorlog, options,
filename=None, run_builtins=True,
deep=True, solve_unknowns=True,
reverse_operators=False, cache_unknowns=False,
extract_locals=True, init_maximum_depth=INIT_MAXIMUM_DEPTH,
maximum_depth=None):
"""Given Python source return its types.
Args:
src: A string containing Python source code.
errorlog: Where error messages go. Instance of errors.ErrorLog.
options: config.Options object
filename: Filename of the program we're parsing.
run_builtins: Whether to preload the native Python builtins when running
the program.
deep: If True, analyze all functions, even the ones not called by the main
execution flow.
solve_unknowns: If yes, try to replace structural types ("~unknowns") with
nominal types.
reverse_operators: Experimental. Allow overloading __radd__ etc.
For user-defined types only - our builtins don't need reverse operators.
cache_unknowns: If True, do a faster approximation of unknown types.
extract_locals: If not optimizing, should we at least remove the call
traces?
init_maximum_depth: Depth of analysis during module loading.
maximum_depth: Depth of the analysis. Default: unlimited.
Returns:
A TypeDeclUnit
Raises:
AssertionError: In case of a bad parameter combination.
"""
tracer = CallTracer(errorlog=errorlog, options=options,
module_name=_get_module_name(filename, options),
reverse_operators=reverse_operators,
cache_unknowns=cache_unknowns,
generate_unknowns=not options.quick)
loc, defs, builtin_names = tracer.run_program(
src, filename, init_maximum_depth, run_builtins)
log.info("===Done run_program===")
if deep:
tracer.exitpoint = tracer.analyze(loc, defs, builtin_names, maximum_depth)
else:
tracer.exitpoint = loc
ast = tracer.compute_types(defs, builtin_names)
ast = tracer.loader.resolve_ast(ast)
if solve_unknowns:
log.info("=========== PyTD to solve =============\n%s", pytd.Print(ast))
ast = convert_structural.convert_pytd(ast, tracer.loader.concat_all())
elif extract_locals:
log.info("Solving is turned off. Discarding call traces.")
# Rename "~unknown" to "?"
ast = ast.Visit(visitors.RemoveUnknownClasses())
# Remove "~list" etc.:
ast = convert_structural.extract_local(ast)
if options.output_cfg or options.output_typegraph:
if options.output_cfg and options.output_typegraph:
raise AssertionError("Can output CFG or typegraph, but not both")
dot = program_to_dot(tracer.program, set([]), bool(options.output_cfg))
proc = subprocess.Popen(["/usr/bin/dot", "-T", "svg", "-o",
options.output_cfg or options.output_typegraph],
stdin=subprocess.PIPE)
proc.stdin.write(dot)
proc.stdin.close()
if options.output_debug:
text = program_to_text(tracer.program)
if options.output_debug == "-":
log.info("=========== Program Dump =============\n%s", text)
else:
with open(options.output_debug, "w") as fi:
fi.write(text)
return ast
def infer_types(src,
errorlog,
options,
loader,
filename=None,
run_builtins=True,
deep=True,
cache_unknowns=False,
show_library_calls=False,
analyze_annotated=False,
init_maximum_depth=INIT_MAXIMUM_DEPTH,
maximum_depth=None):
"""Given Python source return its types.
Args:
src: A string containing Python source code.
errorlog: Where error messages go. Instance of errors.ErrorLog.
options: config.Options object
loader: A load_pytd.Loader instance to load PYI information.
filename: Filename of the program we're parsing.
run_builtins: Whether to preload the native Python builtins when running
the program.
deep: If True, analyze all functions, even the ones not called by the main
execution flow.
cache_unknowns: If True, do a faster approximation of unknown types.
show_library_calls: If True, call traces are kept in the output.
analyze_annotated: If True, analyze methods with type annotations, too.
init_maximum_depth: Depth of analysis during module loading.
maximum_depth: Depth of the analysis. Default: unlimited.
Returns:
A TypeDeclUnit
Raises:
AssertionError: In case of a bad parameter combination.
"""
tracer = CallTracer(errorlog=errorlog,
options=options,
module_name=get_module_name(filename, options),
cache_unknowns=cache_unknowns,
analyze_annotated=analyze_annotated,
generate_unknowns=options.protocols,
store_all_calls=not deep,
loader=loader)
loc, defs = tracer.run_program(src, filename, init_maximum_depth,
run_builtins)
log.info("===Done running definitions and module-level code===")
snapshotter = metrics.get_metric("memory", metrics.Snapshot)
snapshotter.take_snapshot("infer:infer_types:tracer")
if deep:
tracer.exitpoint = tracer.analyze(loc, defs, maximum_depth)
else:
tracer.exitpoint = loc
snapshotter.take_snapshot("infer:infer_types:post")
ast = tracer.compute_types(defs)
ast = tracer.loader.resolve_ast(ast)
if tracer.has_unknown_wildcard_imports or ("HAS_DYNAMIC_ATTRIBUTES" in defs
or "has_dynamic_attributes"
in defs):
try:
ast.Lookup("__getattr__")
except KeyError:
ast = pytd_utils.Concat(
ast, builtins.GetDefaultAst(options.python_version))
# If merged with other if statement, triggers a ValueError: Unresolved class
# when attempts to load from the protocols file
if options.protocols:
protocols_pytd = tracer.loader.import_name("protocols")
else:
protocols_pytd = None
builtins_pytd = tracer.loader.concat_all()
# Insert type parameters, where appropriate
ast = ast.Visit(visitors.CreateTypeParametersForSignatures())
if options.protocols:
log.info("=========== PyTD to solve =============\n%s",
pytd.Print(ast))
ast = convert_structural.convert_pytd(ast, builtins_pytd,
protocols_pytd)
elif not show_library_calls:
log.info("Solving is turned off. Discarding call traces.")
# Rename remaining "~unknown" to "?"
ast = ast.Visit(visitors.RemoveUnknownClasses())
# Remove "~list" etc.:
ast = convert_structural.extract_local(ast)
if options.output_cfg or options.output_typegraph:
if options.output_cfg and options.output_typegraph:
raise AssertionError("Can output CFG or typegraph, but not both")
dot = debug.program_to_dot(tracer.program, set([]),
bool(options.output_cfg))
proc = subprocess.Popen([
"/usr/bin/dot", "-T", "svg", "-o", options.output_cfg
or options.output_typegraph
],
stdin=subprocess.PIPE)
proc.stdin.write(dot)
proc.stdin.close()
_maybe_output_debug(options, tracer.program)
return ast, builtins_pytd
def infer_types(src,
errorlog, options,
filename=None, run_builtins=True,
deep=True, solve_unknowns=True,
cache_unknowns=False, show_library_calls=False,
analyze_annotated=False,
init_maximum_depth=INIT_MAXIMUM_DEPTH, maximum_depth=None):
"""Given Python source return its types.
Args:
src: A string containing Python source code.
errorlog: Where error messages go. Instance of errors.ErrorLog.
options: config.Options object
filename: Filename of the program we're parsing.
run_builtins: Whether to preload the native Python builtins when running
the program.
deep: If True, analyze all functions, even the ones not called by the main
execution flow.
solve_unknowns: If yes, try to replace structural types ("~unknowns") with
nominal types.
cache_unknowns: If True, do a faster approximation of unknown types.
show_library_calls: If True, call traces are kept in the output.
analyze_annotated: If True, analyze methods with type annotations, too.
init_maximum_depth: Depth of analysis during module loading.
maximum_depth: Depth of the analysis. Default: unlimited.
Returns:
A TypeDeclUnit
Raises:
AssertionError: In case of a bad parameter combination.
"""
tracer = CallTracer(errorlog=errorlog, options=options,
module_name=_get_module_name(filename, options),
cache_unknowns=cache_unknowns,
analyze_annotated=analyze_annotated,
generate_unknowns=not options.quick,
store_all_calls=not deep)
loc, defs = tracer.run_program(
src, filename, init_maximum_depth, run_builtins)
log.info("===Done running definitions and module-level code===")
if deep:
tracer.exitpoint = tracer.analyze(loc, defs, maximum_depth)
else:
tracer.exitpoint = loc
ast = tracer.compute_types(defs)
ast = tracer.loader.resolve_ast(ast)
if tracer.has_unknown_wildcard_imports:
try:
ast.Lookup("__getattr__")
except KeyError:
ast = pytd_utils.Concat(
ast, builtins.GetDefaultAst(options.python_version))
builtins_pytd = tracer.loader.concat_all()
if solve_unknowns:
log.info("=========== PyTD to solve =============\n%s", pytd.Print(ast))
ast = convert_structural.convert_pytd(ast, builtins_pytd)
elif not show_library_calls:
log.info("Solving is turned off. Discarding call traces.")
# Rename "~unknown" to "?"
ast = ast.Visit(visitors.RemoveUnknownClasses())
# Remove "~list" etc.:
ast = convert_structural.extract_local(ast)
if options.output_cfg or options.output_typegraph:
if options.output_cfg and options.output_typegraph:
raise AssertionError("Can output CFG or typegraph, but not both")
dot = program_to_dot(tracer.program, set([]), bool(options.output_cfg))
proc = subprocess.Popen(["/usr/bin/dot", "-T", "svg", "-o",
options.output_cfg or options.output_typegraph],
stdin=subprocess.PIPE)
proc.stdin.write(dot)
proc.stdin.close()
_maybe_output_debug(options, tracer.program)
return ast, builtins_pytd
