def test_analyze_annotated_output(self):
argv = ["--output=out.pyi", "test.py"]
opts = config.Options(argv, command_line=True)
self.assertFalse(opts.analyze_annotated) # default
argv.append("--analyze-annotated")
opts = config.Options(argv, command_line=True)
self.assertTrue(opts.analyze_annotated)
def test_analyze_annotated_check(self):
argv = ["--check", "test.py"]
opts = config.Options(argv, command_line=True)
self.assertTrue(opts.analyze_annotated) # default
argv.append("--analyze-annotated")
opts = config.Options(argv, command_line=True)
self.assertTrue(opts.analyze_annotated)
def parse_args(argv):
"""Parse command line args.
Arguments:
argv: Raw command line args, typically sys.argv[1:]
Returns:
A tuple of (
parsed_args: argparse.Namespace,
kythe_args: kythe.Args,
pytype_options: pytype.config.Options)
"""
parser = make_parser()
args = parser.parse_args(argv)
cli_args = args.inputs.copy()
# If we are passed an imports map we should look for pickled files as well.
if args.imports_info:
cli_args += [
"--imports_info", args.imports_info, "--use-pickled-files"
]
# We need to set this when creating Options (b/128032570)
if args.python_version:
cli_args += ["-V", pytype_utils.format_version(args.python_version)]
pytype_options = pytype_config.Options(cli_args)
pytype_options.tweak(**parser.get_pytype_kwargs(args))
kythe_args = kythe.Args(corpus=args.kythe_corpus, root=args.kythe_root)
return (args, kythe_args, pytype_options)
def main():
try:
options = config.Options(sys.argv[1:], command_line=True)
except utils.UsageError as e:
print(str(e), file=sys.stderr)
sys.exit(1)
if options.show_config:
print(options)
sys.exit(0)
if options.version:
print(io.get_pytype_version())
sys.exit(0)
node.SetCheckPreconditions(options.check_preconditions)
if options.timeout is not None:
signal.alarm(options.timeout)
with _ProfileContext(options.profile):
with metrics.MetricsContext(options.metrics, options.open_function):
with metrics.StopWatch("total_time"):
with metrics.Snapshot("memory",
enabled=options.memory_snapshots):
return _run_pytype(options)
def testRunPytypeGenerateBuiltins(self):
"""Basic smoke test for --generate-builtins."""
filename = self._TmpPath("builtins")
argv = ["--generate-builtins", filename]
options = config.Options(argv)
main_module._run_pytype(options)
self.assertTrue(os.path.isfile(filename))
def test_resolved_imports(self):
# We need all imports to be valid for pytype
code = """\
import foo
import x.y
import a.b as c
from a import b
from p import q as r
"""
stub = "class X: pass"
with file_utils.Tempdir() as d:
d.create_file("t.py", code)
d.create_file("foo.pyi", stub)
d.create_file("x/y.pyi", stub)
d.create_file("a/b.pyi", stub)
d.create_file("p/q.pyi", stub)
options = config.Options([d["t.py"]])
options.tweak(pythonpath=[d.path], version=self.python_version)
ix = indexer.process_file(options)
self.assertDef(ix, "module::foo", "foo", "Import")
self.assertDef(ix, "module::x.y", "x.y", "Import")
self.assertDef(ix, "module::c", "c", "Import")
self.assertDef(ix, "module::b", "b", "ImportFrom")
self.assertDef(ix, "module::r", "r", "ImportFrom")
self.assertEqual(ix.modules["module::foo"], "foo")
self.assertEqual(ix.modules["module::x.y"], "x.y")
self.assertEqual(ix.modules["module::b"], "a.b")
self.assertEqual(ix.modules["module::c"], "a.b")
self.assertEqual(ix.modules["module::r"], "p.q")
def run_pytype(self, module, report_errors):
"""Run pytype over a single module."""
# Create the output subdirectory for this file.
target_dir = os.path.join(self.pyi_dir, os.path.dirname(module.target))
try:
file_utils.makedirs(target_dir)
except OSError:
logging.error('Could not create output directory: %s', target_dir)
return
if report_errors:
print('%s' % module.target)
else:
print('%s*' % module.target)
args = self.get_pytype_args(module, report_errors)
logging.info('Running: pytype %s %s', ' '.join(args),
self.custom_options)
# TODO(rechen): Do we want to get rid of the --nofail option and use a
# try/except here instead? We'd control the failure behavior (e.g. we could
# potentially bring back the .errors file, or implement an "abort on first
# error" flag for quick iterative typechecking).
with debug.save_logging_level(
): # pytype_config changes the logging level
options = pytype_config.Options(args)
# TODO(rechen): Do this tweaking in get_pytype_args so it can be tested.
if report_errors:
options.tweak(**self.custom_options)
io.process_one_file(options)
def get_options(self, args):
self.assertEqual(args[0], 'pytype-single')
args = args[1:]
start, end = args.index('--imports_info'), args.index('$imports')
self.assertEqual(end - start, 1)
args.pop(end)
args.pop(start)
return pytype_config.Options(args)
def get_options(self, args):
nargs = len(pytype_runner.PYTYPE_SINGLE)
self.assertEqual(args[:nargs], pytype_runner.PYTYPE_SINGLE)
args = args[nargs:]
start, end = args.index('--imports_info'), args.index('$imports')
self.assertEqual(end - start, 1)
args.pop(end)
args.pop(start)
return pytype_config.Options(args)
def index_code(self, code, **kwargs):
"""Generate references from a code string."""
args = {"version": self.python_version}
args.update(kwargs)
with file_utils.Tempdir() as d:
d.create_file("t.py", code)
options = config.Options([d["t.py"]])
options.tweak(**args)
return indexer.process_file(options)
def testRunPytype(self):
"""Basic smoke test for _run_pytype."""
infile = self._TmpPath("input")
outfile = self._TmpPath("output")
with open(infile, "w") as f:
f.write("def f(x): pass")
argv = ["", "-o", outfile, infile]
options = config.Options(argv)
main_module._run_pytype(options)
self.assertTrue(os.path.isfile(outfile))
def test_source_text(self):
# Don't try to read from the filesystem if we supply source_text
code = textwrap.dedent("""
def f(x):
return 42
""")
options = config.Options(["/path/to/nonexistent/file.py"])
options.tweak(version=self.python_version)
ix = indexer.process_file(options, source_text=code)
self.assertDef(ix, "module.f", "f", "FunctionDef")
def generate_kythe(self, code, **kwargs):
"""Generate a kythe index from a code string."""
with file_utils.Tempdir() as d:
d.create_file("t.py", code)
options = config.Options([d["t.py"]])
options.tweak(pythonpath=[d.path], version=self.python_version)
kythe_args = kythe.Args(corpus="corpus", root="root")
ix = indexer.process_file(options, kythe_args=kythe_args)
# Collect all the references from the kythe graph.
kythe_index = [json.loads(x) for x in output.json_kythe_graph(ix)]
return kythe_index
def test_basic(self):
argv = [
"-V", "3.6", "--use-pickled-files", "-o", "out.pyi",
"--pythonpath", "foo:bar", "test.py"
]
opts = config.Options(argv)
self.assertEqual(opts.python_version, (3, 6))
self.assertEqual(opts.use_pickled_files, True)
self.assertEqual(opts.pythonpath, ["foo", "bar"])
self.assertEqual(opts.output, "out.pyi")
self.assertEqual(opts.input, "test.py")
def annotate(self, source):
source = textwrap.dedent(source.lstrip('\n'))
filename = 'unused.py'
ast_factory = lambda unused_options: ast
pytype_options = config.Options([filename])
pytype_options.tweak(version=self.python_version)
module = annotate_ast.annotate_source(source,
ast_factory,
pytype_options,
filename=filename)
return module
def testRunPytype(self):
"""Basic unit test (smoke test) for _run_pytype."""
# TODO(kramm): This is a unit test, whereas all other tests in this file
# are integration tests. Move this somewhere else?
infile = self._TmpPath("input")
outfile = self._TmpPath("output")
with open(infile, "w") as f:
f.write("def f(x): pass")
argv = ["-o", outfile, infile]
options = config.Options(argv)
main_module._run_pytype(options)
self.assertTrue(os.path.isfile(outfile))
def main():
try:
options = config.Options(sys.argv[1:])
except utils.UsageError as e:
print(str(e), file=sys.stderr)
sys.exit(1)
node.SetCheckPreconditions(options.check_preconditions)
if options.timeout is not None:
signal.alarm(options.timeout)
return process_file(options)
def test_basic(self):
version = sys.version_info[:2]
argv = [
"-V",
utils.format_version(version), "--use-pickled-files", "-o",
"out.pyi", "--pythonpath", "foo:bar", "test.py"
]
opts = config.Options(argv, command_line=True)
self.assertEqual(opts.python_version, version)
self.assertEqual(opts.use_pickled_files, True)
self.assertEqual(opts.pythonpath, ["foo", "bar"])
self.assertEqual(opts.output, "out.pyi")
self.assertEqual(opts.input, "test.py")
def add_pytype_and_parse(parser, argv):
"""Add basic pytype options and parse args.
Useful to generate a quick CLI for a library.
Args:
parser: An argparse.ArgumentParser
argv: Raw command line args, typically sys.argv[1:]
Returns:
A tuple of (
parsed_args: argparse.Namespace,
pytype_options: pytype.config.Options)
"""
# Add default --debug and input arguments.
parser.add_argument("--debug",
action="store_true",
dest="debug",
default=None,
help="Display debug output.")
parser.add_argument("inputs",
metavar="input",
nargs=1,
help="A .py file to index")
# Add options from pytype-single.
wrapper = datatypes.ParserWrapper(parser)
pytype_config.add_basic_options(wrapper)
pytype_config.add_feature_flags(wrapper)
parser = Parser(parser, wrapper.actions)
# Parse argv
args = parser.parse_args(argv)
cli_args = args.inputs.copy()
# Make sure we have a valid set of CLI options to pytype
## If we are passed an imports map we should look for pickled files as well.
if getattr(args, "imports_info", None):
cli_args += [
"--imports_info", args.imports_info, "--use-pickled-files"
]
## We need to set this when creating Options (b/128032570)
if args.python_version:
cli_args += ["-V", pytype_utils.format_version(args.python_version)]
pytype_options = pytype_config.Options(cli_args, command_line=True)
pytype_options.tweak(**parser.get_pytype_kwargs(args))
return (args, pytype_options)
def test_kythe_args(self):
code = textwrap.dedent("""
def f(x):
return 42
""")
with file_utils.Tempdir() as d:
d.create_file("t.py", code)
options = config.Options([d["t.py"]])
options.tweak(pythonpath=[d.path], version=self.python_version)
kythe_args = kythe.Args(corpus="corpus", root="root")
ix = indexer.process_file(options, kythe_args=kythe_args)
# Collect all the references from the kythe graph.
kythe_index = [json.loads(x) for x in output.json_kythe_graph(ix)]
k = kythe_index[0]["source"]
self.assertEqual(k["corpus"], "corpus")
self.assertEqual(k["root"], "root")
def run_pytype(self, module, report_errors):
"""Run pytype over a single module."""
self.create_output_dir(module)
args = self.get_pytype_args(module, report_errors)
logging.info('Running: pytype %s %s', ' '.join(args),
self.custom_options)
# TODO(rechen): Do we want to get rid of the --nofail option and use a
# try/except here instead? We'd control the failure behavior (e.g. we could
# potentially bring back the .errors file, or implement an "abort on first
# error" flag for quick iterative typechecking).
with debug.save_logging_level(
): # pytype_config changes the logging level
options = pytype_config.Options(args)
# TODO(rechen): Do this tweaking in get_pytype_args so it can be tested.
if report_errors:
options.tweak(**self.custom_options)
return io.process_one_file(options)
def main():
try:
options = config.Options(sys.argv[1:])
except utils.UsageError as e:
print(str(e), file=sys.stderr)
sys.exit(1)
node.SetCheckPreconditions(options.check_preconditions)
if options.timeout is not None:
signal.alarm(options.timeout)
ix = indexer.process_file(options)
if options.debug:
debug.show_index(ix)
else:
output.output_kythe_graph(ix)
def run_pytype(args, dry_run, typeshed_location):
"""Runs pytype, returning the stderr if any."""
if dry_run:
return None
old_typeshed_home = os.environ.get(TYPESHED_HOME, UNSET)
os.environ[TYPESHED_HOME] = typeshed_location
try:
io.parse_pyi(config.Options(args))
except Exception:
stderr = traceback.format_exc()
else:
stderr = None
if old_typeshed_home is UNSET:
del os.environ[TYPESHED_HOME]
else:
os.environ[TYPESHED_HOME] = old_typeshed_home
return stderr
def main():
try:
options = config.Options(sys.argv[1:])
except utils.UsageError as e:
print(str(e), file=sys.stderr)
sys.exit(1)
node.SetCheckPreconditions(options.check_preconditions)
if options.timeout is not None:
signal.alarm(options.timeout)
v = indexer.process_file(options)
output.show_defs(v)
print()
print("--------------------")
print()
output.show_refs(v)
def parse_args(argv):
"""Parse command line args.
Arguments:
argv: Raw command line args, typically sys.argv[1:]
Returns:
A tuple of (
parsed_args: argparse.Namespace,
kythe_args: kythe.Args,
pytype_options: pytype.config.Options)
"""
parser = make_parser()
args = parser.parse_args(argv)
pytype_options = pytype_config.Options(args.inputs)
pytype_options.tweak(**parser.get_pytype_kwargs(args))
kythe_args = kythe.Args(corpus=args.kythe_corpus, root=args.kythe_root)
return (args, kythe_args, pytype_options)
def communicate(self):
if self.results:
return self.results
old_env = {}
if self.env:
for var, val in self.env.items():
old_env[var] = os.environ.get(var, UNSET)
os.environ[var] = val
try:
io.parse_pyi(config.Options(self.args[1:]))
except Exception:
self.results = (1, '', traceback.format_exc())
else:
self.results = (0, '', '')
for var, val in old_env.items():
if val is UNSET:
del os.environ[var]
else:
os.environ[var] = val
return self.results
def index_code(self, code, return_ast=False, **kwargs):
"""Generate references from a code string."""
ast_factory = kwargs.pop("ast_factory", None)
keep_pytype_data = kwargs.pop("keep_pytype_data", False)
annotate_ast = kwargs.pop("annotate_ast", False)
args = {"version": self.python_version}
args.update(kwargs)
with file_utils.Tempdir() as d:
d.create_file("t.py", code)
options = config.Options([d["t.py"]])
options.tweak(**args)
ix, ast_root = indexer.process_file(
options,
keep_pytype_data=keep_pytype_data,
ast_factory=ast_factory,
annotate_ast=annotate_ast)
if return_ast:
return ix, ast_root
else:
return ix
def test_import_ref(self):
# We need all imports to be valid for pytype
code = """\
import foo
x = foo.Bar
"""
stub = "class Bar: pass"
with file_utils.Tempdir() as d:
d.create_file("t.py", code)
d.create_file("foo.pyi", stub)
options = config.Options([d["t.py"]])
options.tweak(pythonpath=[d.path], version=self.python_version)
ix = indexer.process_file(options)
self.assertDef(ix, "module::foo", "foo", "Import")
self.assertDef(ix, "module::x", "x", "Store")
self.assertEqual(ix.modules["module::foo"], "foo")
expected = [({
"name": "foo",
"typ": "Name",
"scope": "module",
"location": (2, 4)
}, {
"name": "foo",
"scope": "module"
}),
({
"name": "foo.Bar",
"typ": "Attribute",
"scope": "module",
"location": (2, 4)
}, {
"name": "Bar",
"scope": "foo/module"
})]
for (ref, defn), (expected_ref,
expected_defn) in zip(ix.links, expected):
self.assertAttrs(ref, expected_ref)
self.assertAttrs(defn, expected_defn)
def run_pytype(*, filename: str, python_version: str, python_exe: str,
typeshed_location: str) -> Optional[str]:
"""Runs pytype, returning the stderr if any."""
options = pytype_config.Options([
"--module-name={}".format(_get_module_name(filename)),
"--parse-pyi",
"-V {}".format(python_version),
"--python_exe={}".format(python_exe),
filename,
])
old_typeshed_home = os.environ.get(TYPESHED_HOME, UNSET)
os.environ[TYPESHED_HOME] = typeshed_location
try:
pytype_io.parse_pyi(options)
except Exception:
stderr = traceback.format_exc()
else:
stderr = None
if old_typeshed_home is UNSET:
del os.environ[TYPESHED_HOME]
else:
os.environ[TYPESHED_HOME] = old_typeshed_home
return stderr
def parse_args(argv):
"""Parse command line args.
Arguments:
argv: Raw command line args, typically sys.argv[1:]
Returns:
A tuple of (
parsed_args: argparse.Namespace,
kythe_args: kythe.Args,
pytype_options: pytype.config.Options)
"""
parser = make_parser()
args = parser.parse_args(argv)
cli_args = args.inputs
# If we are passed an imports map we should look for pickled files as well.
if args.imports_info:
cli_args += ["--imports_info", args.imports_info,
"--use-pickled-files"]
pytype_options = pytype_config.Options(cli_args)
pytype_options.tweak(**parser.get_pytype_kwargs(args))
kythe_args = kythe.Args(corpus=args.kythe_corpus, root=args.kythe_root)
return (args, kythe_args, pytype_options)