def Infer(self,
srccode,
pythonpath=(),
deep=True,
report_errors=True,
analyze_annotated=True,
pickle=False,
module_name=None,
**kwargs):
"""Runs inference on srccode."""
types, builtins_pytd = self._InferAndVerify(
_Format(srccode),
pythonpath=pythonpath,
deep=deep,
analyze_annotated=analyze_annotated,
module_name=module_name,
report_errors=report_errors,
**kwargs)
types = optimize.Optimize(types,
builtins_pytd,
lossy=False,
use_abcs=False,
max_union=7,
remove_mutable=False)
types = pytd_utils.CanonicalOrdering(types)
if pickle:
return self._Pickle(types, module_name)
else:
return types
def main():
argument_parser = make_parser()
opts = argument_parser.parse_args()
python_version = utils.version_from_string(opts.python_version)
try:
utils.validate_version(python_version)
except utils.UsageError as e:
sys.stderr.write("Usage error: %s\n" % utils.message(e))
sys.exit(1)
with open(opts.input) as fi:
sourcecode = fi.read()
try:
parsed = parser.parse_string(sourcecode, filename=opts.input,
python_version=python_version)
except parser.ParseError as e:
sys.stderr.write(str(e))
sys.exit(1)
if opts.optimize:
parsed = optimize.Optimize(parsed,
builtins.GetBuiltinsPyTD(python_version),
lossy=opts.lossy,
use_abcs=opts.use_abcs,
max_union=opts.max_union,
remove_mutable=opts.remove_mutable,
can_do_lookup=False)
if opts.output is not None:
out_text = pytd_utils.Print(parsed, opts.multiline_args)
if opts.output == "-":
sys.stdout.write(out_text)
else:
with open(opts.output, "w") as out:
out.write(out_text)
def Infer(self,
srccode,
pythonpath=(),
deep=False,
solve_unknowns=False,
report_errors=True,
analyze_annotated=True,
**kwargs):
types, builtins_pytd = self._InferAndVerify(
textwrap.dedent(srccode),
pythonpath=pythonpath,
deep=deep,
cache_unknowns=True,
analyze_annotated=analyze_annotated,
solve_unknowns=solve_unknowns,
report_errors=report_errors,
**kwargs)
types = optimize.Optimize(types,
builtins_pytd,
lossy=False,
use_abcs=False,
max_union=7,
remove_mutable=False)
types = pytd_utils.CanonicalOrdering(types)
return types
def _combine_multiple_returns(self, signatures):
"""Combines multiple return types.
Args:
signatures: The candidate signatures.
Returns:
The combined return type.
"""
options = []
for sig, _, _ in signatures:
t = sig.pytd_sig.return_type
params = pytd_utils.GetTypeParameters(t)
if params:
replacement = {}
for param_type in params:
replacement[param_type] = pytd.AnythingType()
replace_visitor = visitors.ReplaceTypeParameters(replacement)
t = t.Visit(replace_visitor)
options.append(t)
if len(set(options)) == 1:
return options[0]
# Optimizing and then removing unions allows us to preserve as much
# precision as possible while avoiding false positives.
ret_type = optimize.Optimize(pytd_utils.JoinTypes(options))
return ret_type.Visit(visitors.ReplaceUnionsWithAny())
def InferWithErrors(self,
code,
deep=True,
pythonpath=(),
module_name=None,
analyze_annotated=True,
quick=False,
imports_map=None,
**kwargs):
"""Runs inference on code expected to have type errors."""
kwargs.update(
self._SetUpErrorHandling(code, pythonpath, analyze_annotated,
quick, imports_map))
self.ConfigureOptions(module_name=module_name)
unit, builtins_pytd = analyze.infer_types(deep=deep, **kwargs)
unit.Visit(visitors.VerifyVisitor())
unit = optimize.Optimize(unit,
builtins_pytd,
lossy=False,
use_abcs=False,
max_union=7,
remove_mutable=False)
errorlog = kwargs["errorlog"]
errorlog.assert_errors_match_expected()
return pytd_utils.CanonicalOrdering(unit), errorlog
def _pytd_print(self, pytd_type):
name = pytd.Print(
pytd_utils.CanonicalOrdering(optimize.Optimize(pytd_type)))
# Clean up autogenerated namedtuple names, e.g. "namedtuple-X-a-_0-c"
# becomes just "X", by extracting out just the type name.
if "namedtuple-" in name:
return re.sub(r"\bnamedtuple-([^-]+)-[-_\w]*", r"\1", name)
return name
def InferWithErrors(self, code, deep=True, pythonpath=(),
analyze_annotated=True, quick=False, **kwargs):
kwargs.update(
self._SetUpErrorHandling(code, pythonpath, analyze_annotated, quick))
unit, builtins_pytd = analyze.infer_types(deep=deep, **kwargs)
unit.Visit(visitors.VerifyVisitor())
unit = optimize.Optimize(unit, builtins_pytd, lossy=False, use_abcs=False,
max_union=7, remove_mutable=False)
return pytd_utils.CanonicalOrdering(unit), kwargs["errorlog"]
def __init__(self, test, srccode, deep=False,
solve_unknowns=False, extract_locals=False,
extra_verbose=False, report_errors=True, **kwargs):
"""Constructor for Infer.
Args:
test: the Testcase (see inferenceTest.Infer)
srccode: the Python source code to do type inferencing on
deep: see class comments (assume --api - analyize all methods, even those
that don't have a caller)
solve_unknowns: try to solve for all ~unknown types
extract_locals: strip ~unknown types from the output pytd
extra_verbose: extra intermeidate output (for debugging)
report_errors: Whether to fail if the type inferencer reports any erros.
**kwargs: Additional options to pass through to infer_types().
"""
# TODO(pludemann): There are eight possible combinations of these three
# boolean flags. Do all of these combinations make sense? Or would it be
# possible to simplify this into something like a "mode" parameter:
# mode="solve" => deep=True, solve_unknowns=True
# mode="structural" => deep=True, solve_unknowns=False, extract_locals=False
# mode="deep" => deep=True, solve_unknowns=False, extract_locals=True
# mode="main" => deep=False, solve_unknowns=False, extract_locals=True
self.srccode = textwrap.dedent(srccode)
self.inferred = None
self.optimized_types = None
self.extract_locals = None # gets set if extract_locals is set (below)
self.extra_verbose = extra_verbose
self.canonical_types = None
# We need to catch any exceptions here and preserve them for __exit__.
# Exceptions raised in the body of 'with' will be presented to __exit__.
try:
self.types = test._InferAndVerify(
self.srccode, deep=deep, cache_unknowns=True,
solve_unknowns=solve_unknowns,
report_errors=report_errors, **kwargs)
self.inferred = self.types
if extract_locals:
# Rename "~unknown" to "?"
self.types = self.types.Visit(visitors.RemoveUnknownClasses())
# Remove "~list" etc.:
self.types = convert_structural.extract_local(self.types)
self.extract_locals = self.types
# TODO(pludemann): These flags are the same as those in main.py; there
# should be a way of ensuring that they're the same.
self.types = self.optimized_types = optimize.Optimize(
self.types, lossy=False, use_abcs=False,
max_union=7, remove_mutable=False)
self.types = self.canonical_types = pytd_utils.CanonicalOrdering(
self.types)
except Exception: # pylint: disable=broad-except
self.types = None
if not self.__exit__(*sys.exc_info()):
raise
def testLossy(self):
# Lossy compression is hard to test, since we don't know to which degree
# "compressible" items will be compressed. This test only checks that
# non-compressible things stay the same.
src = textwrap.dedent("""
def foo(a: int) -> float raises IndexError
def foo(a: str) -> complex raises AssertionError
""")
optimized = optimize.Optimize(self.Parse(src),
lossy=True, use_abcs=False)
self.AssertSourceEquals(optimized, src)
def testABCs(self):
src = textwrap.dedent("""
def foo(a: int or float) -> NoneType
def foo(a: int or complex or float) -> NoneType
""")
new_src = textwrap.dedent("""
def foo(a: Real) -> NoneType
def foo(a: Complex) -> NoneType
""")
optimized = optimize.Optimize(self.Parse(src),
lossy=True, use_abcs=True)
self.AssertSourceEquals(optimized, new_src)
def InferAndCheck(self, code, deep=True, pythonpath=(), **kwargs):
self.options.tweak(pythonpath=pythonpath)
code = textwrap.dedent(code)
errorlog = errors.ErrorLog()
unit, builtins_pytd = infer.infer_types(code,
errorlog,
self.options,
deep=deep,
analyze_annotated=True,
cache_unknowns=True,
**kwargs)
unit.Visit(visitors.VerifyVisitor())
unit = optimize.Optimize(unit,
builtins_pytd,
lossy=False,
use_abcs=False,
max_union=7,
remove_mutable=False)
return pytd_utils.CanonicalOrdering(unit), errorlog
def generate_pyi(input_filename, errorlog, options, loader):
"""Run the inferencer on one file, producing output.
Args:
input_filename: name of the file to process
errorlog: Where error messages go. Instance of errors.ErrorLog.
options: config.Options object.
loader: A load_pytd.Loader instance.
Returns:
A tuple, (PYI Ast as string, TypeDeclUnit).
Raises:
CompileError: If we couldn't parse the input file.
UsageError: If the input filepath is invalid.
"""
mod, builtins = _call(analyze.infer_types, input_filename, errorlog,
options, loader)
mod.Visit(visitors.VerifyVisitor())
mod = optimize.Optimize(
mod,
builtins,
# TODO(kramm): Add FLAGs for these
lossy=False,
use_abcs=False,
max_union=7,
remove_mutable=False)
mod = pytd_utils.CanonicalOrdering(mod, sort_signatures=True)
result = pytd.Print(mod)
log.info("=========== pyi optimized =============")
log.info("\n%s", result)
log.info("========================================")
if not result.endswith("\n"):
result += "\n"
result_prefix = ""
if options.quick:
result_prefix += "# (generated with --quick)\n"
if result_prefix:
result = result_prefix + "\n" + result
return result, mod
def testDuplicatesInUnions(self):
src = textwrap.dedent("""
def a(x: int or float or complex) -> bool
def b(x: int or float) -> bool
def c(x: int or int or int) -> bool
def d(x: int or int) -> bool
def e(x: float or int or int or float) -> bool
def f(x: float or int) -> bool
""")
new_src = textwrap.dedent("""
def a(x) -> bool # max_union=2 makes this object
def b(x: int or float) -> bool
def c(x: int) -> bool
def d(x: int) -> bool
def e(x: float or int) -> bool
def f(x: float or int) -> bool
""")
optimized = optimize.Optimize(self.Parse(src),
lossy=False,
max_union=2)
self.AssertSourceEquals(optimized, new_src)
def testDuplicatesInUnions(self):
src = textwrap.dedent("""
def a(x: int or float or complex) -> bool
def b(x: int or float) -> bool
def c(x: int or int or int) -> bool
def d(x: int or int) -> bool
def e(x: float or int or int or float) -> bool
def f(x: float or int) -> bool
""")
new_src = textwrap.dedent("""
def a(x) -> bool # max_union=2 makes this object
def b(x: int or float) -> bool
def c(x: int) -> bool
def d(x: int) -> bool
def e(x: float or int) -> bool
def f(x: float or int) -> bool
""")
ast = self.ParseAndResolve(src)
optimized = optimize.Optimize(ast, lossy=False, max_union=2)
optimized = optimized.Visit(visitors.DropBuiltinPrefix())
self.AssertSourceEquals(optimized, new_src)
def _pytd_print(self, pytd_type):
"""Print the name of the pytd type."""
name = pytd_utils.Print(pytd_utils.CanonicalOrdering(optimize.Optimize(
pytd_type.Visit(visitors.RemoveUnknownClasses()))))
# Clean up autogenerated namedtuple names, e.g. "namedtuple-X-a-_0-c"
# becomes just "X", by extracting out just the type name.
if "namedtuple" in name:
return escape.unpack_namedtuple(name)
nested_class_match = re.search(r"_(?:\w+)_DOT_", name)
if nested_class_match:
# Pytype doesn't have true support for nested classes. Instead, for
# class Foo:
# class Bar: ...
# it outputs:
# class _Foo_DOT_Bar: ...
# class Foo:
# Bar = ... # type: Type[_Foo_DOT_Bar]
# Replace _Foo_DOT_Bar with Foo.Bar in error messages for readability.
# TODO(b/35138984): Get rid of this hack.
start = nested_class_match.start()
return name[:start] + name[start+1:].replace("_DOT_", ".")
return name
def generate_pyi(src, options=None, loader=None):
"""Run the inferencer on a string of source code, producing output.
Args:
src: The source code.
options: config.Options object.
loader: A load_pytd.Loader instance.
Returns:
A tuple, (errors.ErrorLog, PYI Ast as string, TypeDeclUnit).
Raises:
CompileError: If we couldn't parse the input file.
UsageError: If the input filepath is invalid.
"""
options = options or config.Options.create()
with config.verbosity_from(options):
errorlog, (mod, builtins) = _call(analyze.infer_types, src, options,
loader)
mod.Visit(visitors.VerifyVisitor())
mod = optimize.Optimize(
mod,
builtins,
# TODO(b/159038508): Add FLAGs for these
lossy=False,
use_abcs=False,
max_union=7,
remove_mutable=False)
mod = pytd_utils.CanonicalOrdering(mod, sort_signatures=True)
result = pytd_utils.Print(mod)
log.info("=========== pyi optimized =============")
log.info("\n%s", result)
log.info("========================================")
result += "\n"
if options.quick:
result = "# (generated with --quick)\n\n" + result
return errorlog, result, mod
def Optimize(self, ast, **kwargs):
return optimize.Optimize(ast, self.builtins, **kwargs)
def OptimizedString(self, data):
tree = self.Parse(data)
new_tree = optimize.Optimize(tree)
return pytd.Print(new_tree)
def OptimizedString(self, data):
tree = self.Parse(data) if isinstance(data, basestring) else data
new_tree = optimize.Optimize(tree)
return pytd.Print(new_tree)
def Optimize(self, ast, **kwargs):
return optimize.Optimize(ast, builtins.GetBuiltinsPyTD(), **kwargs)
def Optimize(self, ast, **kwargs):
return optimize.Optimize(ast,
builtins.GetBuiltinsPyTD(self.PYTHON_VERSION),
**kwargs)
