def infer_types(source, filename, options):
"""Infer types for the provided source.
Args:
source: Text, the source code to analyze.
filename: Text, the filename the source came from. The file won't be read.
options: pytype.config.Options, the options to pass onto Pytype.
Returns:
Traces object with information gathered by Pytype.
"""
errorlog = errors.ErrorLog()
loader = load_pytd.create_loader(options)
vm = analyze.CallTracer(errorlog=errorlog,
options=options,
generate_unknowns=options.protocols,
loader=loader)
with io.wrap_pytype_exceptions(PytypeError, filename=filename):
analyze.infer_types(src=source,
filename=filename,
errorlog=errorlog,
options=options,
loader=loader,
show_library_calls=True,
tracer_vm=vm)
return Traces(vm)
def process_file(options, source_text=None, kythe_args=None):
"""Process a single file and return cross references.
Args:
options: A dictionary of pytype options.
source_text: Optional text of the file; will be read from the file pointed
to by options.input if not supplied.
kythe_args: Extra args for generating the kythe index
Returns:
An Indexer object with the indexed code
Raises:
PytypeError if pytype fails.
"""
# We bind the global ast variable in this function.
global ast
errorlog = errors.ErrorLog()
loader = load_pytd.create_loader(options)
src = source_text or io.read_source_file(options.input)
vm = analyze.CallTracer(
errorlog=errorlog,
options=options,
generate_unknowns=options.protocols,
store_all_calls=False,
loader=loader)
with io.wrap_pytype_exceptions(PytypeError, filename=options.input):
analyze.infer_types(
src=src,
filename=options.input,
errorlog=errorlog,
options=options,
loader=loader,
tracer_vm=vm)
major, minor = options.python_version
if major == 2:
# python2.7 is the only supported py2 version.
a = ast27.parse(src, options.input)
ast = ast27
else:
a = ast3.parse(src, options.input, feature_version=minor)
ast = ast3
# TODO(mdemello): Get from args
module_name = "module"
source = SourceFile(src, vm.opcode_traces, filename=options.input)
ix = Indexer(source, vm.loader, module_name, kythe_args)
ix.index(a)
ix.finalize()
return ix
def process_file(options, source_text=None):
"""Process a single file and return cross references.
Args:
options: A dictionary of pytype options.
source_text: Optional text of the file; will be read from the file pointed
to by options.input if not supplied.
Returns:
An Indexer object with the indexed code.
"""
# We bind the global ast variable in this function.
global ast
errorlog = errors.ErrorLog()
loader = load_pytd.create_loader(options)
src = source_text or io.read_source_file(options.input)
vm = analyze.CallTracer(
errorlog=errorlog,
options=options,
generate_unknowns=options.protocols,
store_all_calls=False,
loader=loader)
try:
analyze.infer_types(
src=src,
filename=options.input,
errorlog=errorlog,
options=options,
loader=loader,
tracer_vm=vm)
except utils.UsageError as e:
logging.error("Usage error: %s\n", utils.message(e))
return 1
major, minor = options.python_version
if major == 2:
# python2.7 is the only supported py2 version.
a = ast27.parse(src, options.input)
ast = ast27
else:
a = ast3.parse(src, options.input, feature_version=minor)
ast = ast3
# TODO(mdemello): Get from args
module_name = "module"
source = SourceFile(src, vm.opcode_traces, filename=options.input)
ix = Indexer(source, vm.loader.get_resolved_modules(), module_name)
ix.index(a)
ix.finalize()
return ix
def InferWithErrors(self,
code,
deep=True,
pythonpath=(),
module_name=None,
analyze_annotated=True,
quick=False,
imports_map=None,
**kwargs):
"""Runs inference on code expected to have type errors."""
kwargs.update(
self._SetUpErrorHandling(code, pythonpath, analyze_annotated,
quick, imports_map))
self.ConfigureOptions(module_name=module_name)
unit, builtins_pytd = analyze.infer_types(deep=deep, **kwargs)
unit.Visit(visitors.VerifyVisitor())
unit = optimize.Optimize(unit,
builtins_pytd,
lossy=False,
use_abcs=False,
max_union=7,
remove_mutable=False)
errorlog = kwargs["errorlog"]
errorlog.assert_errors_match_expected()
return pytd_utils.CanonicalOrdering(unit), errorlog
def trace(src, options=None):
"""Generates type traces for the given source code.
Args:
src: The source text.
options: A pytype.config.Options object that can be used to specify options
such as the target Python version.
Returns:
A source.Code object.
"""
errorlog = errors.ErrorLog()
options = options or config.Options.create()
loader = load_pytd.create_loader(options)
vm = analyze.CallTracer(
errorlog=errorlog,
options=options,
generate_unknowns=options.protocols,
loader=loader)
pytd_module, _ = analyze.infer_types(
src=src,
filename=options.input,
errorlog=errorlog,
options=options,
loader=loader,
tracer_vm=vm)
raw_traces = []
for op, symbol, data in vm.opcode_traces:
raw_traces.append(
(op, symbol, tuple(_to_pytd(d, loader, pytd_module) for d in data)))
return source.Code(src, raw_traces, TypeTrace, options.input)
def _InferAndVerify(
self, src, pythonpath, module_name, report_errors, analyze_annotated,
imports_map=None, quick=False, **kwargs):
"""Infer types for the source code treating it as a module.
Used by Infer().
Args:
src: The source code of a module. Treat it as "__main__".
pythonpath: --pythonpath as list/tuple of string
module_name: Name of the module we're analyzing. E.g. "foo.bar.mymodule".
report_errors: Whether to fail if the type inferencer reports any errors
in the program.
analyze_annotated: Whether to analyze functions with return annotations.
imports_map: --imports_info data
quick: Try to run faster, by avoiding costly computations.
**kwargs: Keyword parameters to pass through to the type inferencer.
Raises:
AssertionError: If report_errors is True and we found errors.
Returns:
A pytd.TypeDeclUnit
"""
self.ConfigureOptions(
module_name=module_name, quick=quick, use_pickled_files=True,
pythonpath=[""] if (not pythonpath and imports_map) else pythonpath,
imports_map=imports_map, analyze_annotated=analyze_annotated)
errorlog = errors.ErrorLog()
unit, builtins_pytd = analyze.infer_types(
src, errorlog, self.options, loader=self.loader, **kwargs)
unit.Visit(visitors.VerifyVisitor())
if report_errors and errorlog:
errorlog.print_to_stderr()
self.fail("Inferencer found %d errors" % len(errorlog))
return unit, builtins_pytd
def InferWithErrors(self, code, deep=True, pythonpath=(),
analyze_annotated=True, quick=False, **kwargs):
kwargs.update(
self._SetUpErrorHandling(code, pythonpath, analyze_annotated, quick))
unit, builtins_pytd = analyze.infer_types(deep=deep, **kwargs)
unit.Visit(visitors.VerifyVisitor())
unit = optimize.Optimize(unit, builtins_pytd, lossy=False, use_abcs=False,
max_union=7, remove_mutable=False)
return pytd_utils.CanonicalOrdering(unit), kwargs["errorlog"]
def process_file(options):
"""Process a single file and return cross references."""
# We bind the global ast variable in this function.
global ast
errorlog = errors.ErrorLog()
loader = load_pytd.create_loader(options)
src = io.read_source_file(options.input)
vm = analyze.CallTracer(errorlog=errorlog,
options=options,
generate_unknowns=options.protocols,
store_all_calls=False,
loader=loader)
try:
analyze.infer_types(src=src,
filename=options.input,
errorlog=errorlog,
options=options,
loader=loader,
tracer_vm=vm)
except utils.UsageError as e:
logging.error("Usage error: %s\n", utils.message(e))
return 1
major, minor = options.python_version
if major == 2:
# python2.7 is the only supported py2 version.
a = ast27.parse(src, options.input)
ast = ast27
else:
a = ast3.parse(src, options.input, feature_version=minor)
ast = ast3
# TODO(mdemello): Get from args
module_name = "module"
source = SourceFile(src, vm.opcode_traces, filename=options.input)
ix = Indexer(source, module_name)
ix.index(a)
ix.lookup_refs()
ix.process_deflocs()
ix.process_links()
return ix
def InferFromFile(self, filename, pythonpath):
with open(filename, "r") as fi:
code = fi.read()
errorlog = errors.ErrorLog()
self.ConfigureOptions(
module_name=load_pytd.get_module_name(filename, pythonpath),
pythonpath=pythonpath)
unit, _ = analyze.infer_types(code, errorlog, self.options,
loader=self.loader, filename=filename)
unit.Visit(visitors.VerifyVisitor())
return pytd_utils.CanonicalOrdering(unit)
def InferFromFile(self, filename, pythonpath):
with open(filename, "r") as fi:
code = fi.read()
errorlog = test_utils.TestErrorLog(code)
if errorlog.expected:
self.fail(
"Cannot assert errors with InferFromFile(); use InferWithErrors()")
self.ConfigureOptions(
module_name=load_pytd.get_module_name(filename, pythonpath),
pythonpath=pythonpath)
unit, _ = analyze.infer_types(code, errorlog, self.options,
loader=self.loader, filename=filename)
unit.Visit(visitors.VerifyVisitor())
return pytd_utils.CanonicalOrdering(unit)
def process_file(options):
"""Process a single file and return cross references."""
errorlog = errors.ErrorLog()
loader = load_pytd.create_loader(options)
src = io.read_source_file(options.input)
vm = analyze.CallTracer(errorlog=errorlog,
options=options,
generate_unknowns=options.protocols,
store_all_calls=False,
loader=loader)
try:
analyze.infer_types(src=src,
filename=options.input,
errorlog=errorlog,
options=options,
loader=loader,
tracer_vm=vm)
except utils.UsageError as e:
logging.error('Usage error: %s\n', utils.message(e))
return 1
display_traces(src, vm.opcode_traces)
def InferFromFile(self, filename, pythonpath, python_version=None):
with open(filename, "rb") as fi:
code = fi.read()
errorlog = errors.ErrorLog()
self.options.tweak(module_name=load_pytd.get_module_name(
filename, pythonpath),
pythonpath=pythonpath,
python_version=python_version)
self._CreateLoader()
unit, _ = analyze.infer_types(code,
errorlog,
self.options,
loader=self.loader,
filename=filename)
unit.Visit(visitors.VerifyVisitor())
return pytd_utils.CanonicalOrdering(unit)
def _InferAndVerify(self,
src,
pythonpath=(),
module_name=None,
imports_map=None,
report_errors=False,
quick=False,
**kwargs):
"""Infer types for the source code treating it as a module.
Used by Infer().
Args:
src: The source code of a module. Treat it as "__main__".
pythonpath: --pythonpath as list/tuple of string
module_name: Name of the module we're analyzing. E.g. "foo.bar.mymodule".
imports_map: --imports_info data
report_errors: Whether to fail if the type inferencer reports any errors
in the program.
quick: Try to run faster, by avoiding costly computations.
**kwargs: Keyword parameters to pass through to the type inferencer.
Raises:
AssertionError: If report_errors is True and we found errors.
Returns:
A pytd.TypeDeclUnit
"""
self.options.tweak(module_name=module_name, quick=quick)
errorlog = errors.ErrorLog()
self.loader = load_pytd.PickledPyiLoader(
use_pickled_typeshed=False,
base_module=module_name,
python_version=self.PYTHON_VERSION,
pythonpath=[""] if
(not pythonpath and imports_map) else pythonpath,
imports_map=imports_map)
unit, builtins_pytd = analyze.infer_types(src,
errorlog,
self.options,
loader=self.loader,
**kwargs)
unit.Visit(visitors.VerifyVisitor())
unit = pytd_utils.CanonicalOrdering(unit)
if report_errors and len(errorlog):
errorlog.print_to_stderr()
self.fail("Inferencer found %d errors" % len(errorlog))
return unit, builtins_pytd
def process_file(options,
source_text=None,
generate_callgraphs=False,
preserve_pytype_vm=False):
"""Process a single file and return cross references.
Args:
options: A dictionary of pytype options.
source_text: Optional text of the file; will be read from the file pointed
to by options.input if not supplied.
generate_callgraphs: Collect call graph information
preserve_pytype_vm: Preserve the pytype vm in the indexer
Returns:
The Indexer object used for indexing.
Raises:
PytypeError if pytype fails.
"""
errorlog = errors.ErrorLog()
loader = load_pytd.create_loader(options)
src = source_text or io.read_source_file(options.input)
vm = analyze.CallTracer(errorlog=errorlog,
options=options,
generate_unknowns=options.protocols,
store_all_calls=True,
loader=loader)
with io.wrap_pytype_exceptions(PytypeError, filename=options.input):
pytd_module, _ = analyze.infer_types(src=src,
filename=options.input,
errorlog=errorlog,
options=options,
loader=loader,
tracer_vm=vm)
major, minor = options.python_version
if major == 2:
# python2.7 is the only supported py2 version.
ast_root_node = ast27.parse(src, options.input)
ast = ast27
else:
ast_root_node = ast3.parse(src, options.input, feature_version=minor)
ast = ast3
# TODO(mdemello): Get from args
module_name = "module"
src_code = source.Code(src,
vm.opcode_traces,
VmTrace,
filename=options.input)
ix = Indexer(ast=ast,
src=src_code,
loader=vm.loader,
module_name=module_name,
pytd_module=pytd_module)
ix.index(ast_root_node)
ix.finalize()
# Make the vm available via indexer.vm for post-finalize() functions.
ix.vm = vm
# Use the indexer as a single object to hold data for calling processes.
if generate_callgraphs:
ix.function_map = callgraph.collect_function_map(ix)
# Release the vm before returning
if not preserve_pytype_vm:
ix.vm = None
return ix
def process_file(options,
source_text=None,
kythe_args=None,
keep_pytype_data=False,
ast_factory=None,
annotate_ast=False):
"""Process a single file and return cross references.
Args:
options: A dictionary of pytype options.
source_text: Optional text of the file; will be read from the file pointed
to by options.input if not supplied.
kythe_args: Extra args for generating the kythe index
keep_pytype_data: Whether to preserve the Reference.data field. If true, the
field will hold the type of the reference as a str or Tuple[str, str] (for
attributes). Otherwise, it will be inaccessible.
ast_factory: Callable to return an ast-module-compatible object to parse the
source text into an ast-compatible object. It is passed the pytype Options
object. If not specified, typed_ast will be used.
annotate_ast: Whether to annotate the ast with type information. Nodes with
type information will have these attributes added:
* `.resolved_type`: the pytd information about the type
* `.resolved_annotation`: A string representation of the type, as would
be written in an annotation.
Returns:
The Indexer object used for indexing, and the created AST object. The
AST object may have been modified if `annotate_ast=True`.
Raises:
PytypeError if pytype fails.
"""
# We bind the global ast variable in this function.
global ast
errorlog = errors.ErrorLog()
loader = load_pytd.create_loader(options)
src = source_text or io.read_source_file(options.input)
vm = analyze.CallTracer(
errorlog=errorlog,
options=options,
generate_unknowns=options.protocols,
store_all_calls=False,
loader=loader)
with io.wrap_pytype_exceptions(PytypeError, filename=options.input):
pytype_ast, _ = analyze.infer_types(
src=src,
filename=options.input,
errorlog=errorlog,
options=options,
loader=loader,
tracer_vm=vm)
if ast_factory:
ast = ast_factory(options)
ast_root_node = ast.parse(src, options.input)
else:
major, minor = options.python_version
if major == 2:
# python2.7 is the only supported py2 version.
ast_root_node = ast27.parse(src, options.input)
ast = ast27
else:
ast_root_node = ast3.parse(src, options.input, feature_version=minor)
ast = ast3
# TODO(mdemello): Get from args
module_name = "module"
source = SourceFile(src, vm.opcode_traces, filename=options.input)
ix = Indexer(
source, vm.loader, module_name, kythe_args, annotate_ast=annotate_ast)
ix.index(ast_root_node)
ix.finalize(keep_pytype_data, pytype_ast)
return ix, ast_root_node
