def testConcatTypeParameters(self):
"""Test for concatenating ASTs with type parameters."""
ast1 = self.Parse("""T = TypeVar("T")""", name="__builtin__")
ast2 = self.Parse("""T = TypeVar("T")""")
combined = utils.Concat(ast1, ast2)
self.assertEquals(combined.Lookup("__builtin__.T"),
pytd.TypeParameter("T", scope="__builtin__"))
self.assertEquals(combined.Lookup("T"),
pytd.TypeParameter("T", scope=None))
def GetBuiltinsPyTD():
"""Get the "default" AST used to lookup built in types.
Get an AST for all Python builtins as well as the most commonly used standard
libraries.
Returns:
A pytd.TypeDeclUnit instance. It'll directly contain the builtin classes
and functions, and submodules for each of the standard library modules.
"""
return utils.Concat(*GetBuiltinsAndTyping())
def convert_pytd(ast, builtins_pytd, protocols_pytd):
"""Convert pytd with unknowns (structural types) to one with nominal types."""
builtins_pytd = builtins_pytd.Visit(visitors.ClassTypeToNamedType())
mapping, result = solve(ast, builtins_pytd, protocols_pytd)
log_info_mapping(mapping)
lookup = pytd_utils.Concat(builtins_pytd, result)
result = insert_solution(result, mapping, lookup)
if log.isEnabledFor(logging.INFO):
log.info("=========== solve result =============\n%s", pytd.Print(result))
log.info("=========== solve result (end) =============")
return result
def testConcat3(self):
"""Test for concatenating three pytd ASTs."""
ast1 = self.Parse("""c1 = ... # type: int""")
ast2 = self.Parse("""c2 = ... # type: float""")
ast3 = self.Parse("""c3 = ... # type: bool""")
combined = utils.Concat(ast1, ast2, ast3)
expected = textwrap.dedent("""
c1 = ... # type: int
c2 = ... # type: float
c3 = ... # type: bool
""")
self.AssertSourceEquals(combined, expected)
def compute_types(self, defs, ignore):
self.program.Freeze()
ty = pytd_utils.Concat(
self.pytd_for_types(defs, ignore),
pytd.TypeDeclUnit(
"unknowns", (),
tuple(self.pytd_classes_for_unknowns()) +
tuple(self.pytd_classes_for_call_traces()),
tuple(self.pytd_functions_for_call_traces())))
ty = ty.Visit(optimize.PullInMethodClasses())
ty = ty.Visit(visitors.DefaceUnresolved(
[ty, self.loader.concat_all()], "~unknown"))
return ty
def compute_types(self, defs):
ty = pytd_utils.Concat(
self.pytd_for_types(defs),
pytd.TypeDeclUnit(
"unknowns",
constants=tuple(),
type_params=tuple(self.pytd_typevars()),
classes=tuple(self.pytd_classes_for_unknowns()) +
tuple(self.pytd_classes_for_call_traces()),
functions=tuple(self.pytd_functions_for_call_traces()),
aliases=tuple(self.pytd_aliases())))
ty = ty.Visit(optimize.PullInMethodClasses())
ty = ty.Visit(visitors.DefaceUnresolved(
[ty, self.loader.concat_all()], "~unknown"))
return ty.Visit(visitors.AdjustTypeParameters())
def test_name_conflict(self):
ty = self.Infer("""
import collections
X = collections.namedtuple("_", [])
Y = collections.namedtuple("_", [])
Z = collections.namedtuple("_", "a")
""")
name_x = collections_overlay.namedtuple_name("_", [])
name_z = collections_overlay.namedtuple_name("_", ["a"])
ast_x = collections_overlay.namedtuple_ast(name_x, [])
ast_z = collections_overlay.namedtuple_ast(name_z, ["a"])
ast = pytd_utils.Concat(ast_x, ast_z)
expected = pytd.Print(ast) + textwrap.dedent("""\
collections = ... # type: module
X = {name_x}
Y = {name_x}
Z = {name_z}""").format(name_x=name_x, name_z=name_z)
self.assertTypesMatchPytd(ty, expected)
def concat_all(self):
if not self._concatenated:
self._concatenated = pytd_utils.Concat(
*(module.ast for module in self._modules.values()),
name="<all>")
return self._concatenated
def setUp(self):
builtins = parser.parse_string(textwrap.dedent(_BUILTINS)).Replace(
name="__builtin__").Visit(visitors.AddNamePrefix())
typing = parser.parse_string("class Generic: ...").Replace(
name="typing").Visit(visitors.AddNamePrefix())
self.mini_builtins = pytd_utils.Concat(builtins, typing)
def infer_types(src,
errorlog,
options,
loader,
filename=None,
run_builtins=True,
deep=True,
cache_unknowns=False,
show_library_calls=False,
analyze_annotated=False,
init_maximum_depth=INIT_MAXIMUM_DEPTH,
maximum_depth=None):
"""Given Python source return its types.
Args:
src: A string containing Python source code.
errorlog: Where error messages go. Instance of errors.ErrorLog.
options: config.Options object
loader: A load_pytd.Loader instance to load PYI information.
filename: Filename of the program we're parsing.
run_builtins: Whether to preload the native Python builtins when running
the program.
deep: If True, analyze all functions, even the ones not called by the main
execution flow.
cache_unknowns: If True, do a faster approximation of unknown types.
show_library_calls: If True, call traces are kept in the output.
analyze_annotated: If True, analyze methods with type annotations, too.
init_maximum_depth: Depth of analysis during module loading.
maximum_depth: Depth of the analysis. Default: unlimited.
Returns:
A TypeDeclUnit
Raises:
AssertionError: In case of a bad parameter combination.
"""
tracer = CallTracer(errorlog=errorlog,
options=options,
module_name=get_module_name(filename, options),
cache_unknowns=cache_unknowns,
analyze_annotated=analyze_annotated,
generate_unknowns=options.protocols,
store_all_calls=not deep,
loader=loader)
loc, defs = tracer.run_program(src, filename, init_maximum_depth,
run_builtins)
log.info("===Done running definitions and module-level code===")
snapshotter = metrics.get_metric("memory", metrics.Snapshot)
snapshotter.take_snapshot("infer:infer_types:tracer")
if deep:
tracer.exitpoint = tracer.analyze(loc, defs, maximum_depth)
else:
tracer.exitpoint = loc
snapshotter.take_snapshot("infer:infer_types:post")
ast = tracer.compute_types(defs)
ast = tracer.loader.resolve_ast(ast)
if tracer.has_unknown_wildcard_imports or ("HAS_DYNAMIC_ATTRIBUTES" in defs
or "has_dynamic_attributes"
in defs):
try:
ast.Lookup("__getattr__")
except KeyError:
ast = pytd_utils.Concat(
ast, builtins.GetDefaultAst(options.python_version))
# If merged with other if statement, triggers a ValueError: Unresolved class
# when attempts to load from the protocols file
if options.protocols:
protocols_pytd = tracer.loader.import_name("protocols")
else:
protocols_pytd = None
builtins_pytd = tracer.loader.concat_all()
# Insert type parameters, where appropriate
ast = ast.Visit(visitors.CreateTypeParametersForSignatures())
if options.protocols:
log.info("=========== PyTD to solve =============\n%s",
pytd.Print(ast))
ast = convert_structural.convert_pytd(ast, builtins_pytd,
protocols_pytd)
elif not show_library_calls:
log.info("Solving is turned off. Discarding call traces.")
# Rename remaining "~unknown" to "?"
ast = ast.Visit(visitors.RemoveUnknownClasses())
# Remove "~list" etc.:
ast = convert_structural.extract_local(ast)
if options.output_cfg or options.output_typegraph:
if options.output_cfg and options.output_typegraph:
raise AssertionError("Can output CFG or typegraph, but not both")
dot = debug.program_to_dot(tracer.program, set([]),
bool(options.output_cfg))
proc = subprocess.Popen([
"/usr/bin/dot", "-T", "svg", "-o", options.output_cfg
or options.output_typegraph
],
stdin=subprocess.PIPE)
proc.stdin.write(dot)
proc.stdin.close()
_maybe_output_debug(options, tracer.program)
return ast, builtins_pytd
def setUp(self):
builtins = parser.parse_string(textwrap.dedent(_BUILTINS),
name="__builtin__")
typing = parser.parse_string("class Generic: ...", name="typing")
self.mini_builtins = pytd_utils.Concat(builtins, typing)
def infer_types(src,
errorlog, options,
filename=None, run_builtins=True,
deep=True, solve_unknowns=True,
cache_unknowns=False, show_library_calls=False,
analyze_annotated=False,
init_maximum_depth=INIT_MAXIMUM_DEPTH, maximum_depth=None):
"""Given Python source return its types.
Args:
src: A string containing Python source code.
errorlog: Where error messages go. Instance of errors.ErrorLog.
options: config.Options object
filename: Filename of the program we're parsing.
run_builtins: Whether to preload the native Python builtins when running
the program.
deep: If True, analyze all functions, even the ones not called by the main
execution flow.
solve_unknowns: If yes, try to replace structural types ("~unknowns") with
nominal types.
cache_unknowns: If True, do a faster approximation of unknown types.
show_library_calls: If True, call traces are kept in the output.
analyze_annotated: If True, analyze methods with type annotations, too.
init_maximum_depth: Depth of analysis during module loading.
maximum_depth: Depth of the analysis. Default: unlimited.
Returns:
A TypeDeclUnit
Raises:
AssertionError: In case of a bad parameter combination.
"""
tracer = CallTracer(errorlog=errorlog, options=options,
module_name=_get_module_name(filename, options),
cache_unknowns=cache_unknowns,
analyze_annotated=analyze_annotated,
generate_unknowns=not options.quick,
store_all_calls=not deep)
loc, defs = tracer.run_program(
src, filename, init_maximum_depth, run_builtins)
log.info("===Done running definitions and module-level code===")
if deep:
tracer.exitpoint = tracer.analyze(loc, defs, maximum_depth)
else:
tracer.exitpoint = loc
ast = tracer.compute_types(defs)
ast = tracer.loader.resolve_ast(ast)
if tracer.has_unknown_wildcard_imports:
try:
ast.Lookup("__getattr__")
except KeyError:
ast = pytd_utils.Concat(
ast, builtins.GetDefaultAst(options.python_version))
builtins_pytd = tracer.loader.concat_all()
if solve_unknowns:
log.info("=========== PyTD to solve =============\n%s", pytd.Print(ast))
ast = convert_structural.convert_pytd(ast, builtins_pytd)
elif not show_library_calls:
log.info("Solving is turned off. Discarding call traces.")
# Rename "~unknown" to "?"
ast = ast.Visit(visitors.RemoveUnknownClasses())
# Remove "~list" etc.:
ast = convert_structural.extract_local(ast)
if options.output_cfg or options.output_typegraph:
if options.output_cfg and options.output_typegraph:
raise AssertionError("Can output CFG or typegraph, but not both")
dot = program_to_dot(tracer.program, set([]), bool(options.output_cfg))
proc = subprocess.Popen(["/usr/bin/dot", "-T", "svg", "-o",
options.output_cfg or options.output_typegraph],
stdin=subprocess.PIPE)
proc.stdin.write(dot)
proc.stdin.close()
_maybe_output_debug(options, tracer.program)
return ast, builtins_pytd
