def test_executable_inference(self) -> None:
"""Test the --python-executable flag with --python-version"""
sys_ver_str = '{ver.major}.{ver.minor}'.format(ver=sys.version_info)
base = ['file.py'] # dummy file
# test inference given one (infer the other)
matching_version = base + [f'--python-version={sys_ver_str}']
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
matching_version = base + [f'--python-executable={sys.executable}']
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
# test inference given both
matching_version = base + [
f'--python-version={sys_ver_str}',
f'--python-executable={sys.executable}'
]
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
# test that --no-site-packages will disable executable inference
matching_version = base + [
f'--python-version={sys_ver_str}', '--no-site-packages'
]
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable is None
# Test setting python_version/executable from config file
special_opts = argparse.Namespace()
special_opts.python_executable = None
special_opts.python_version = None
special_opts.no_executable = None
# first test inferring executable from version
options = Options()
options.python_executable = None
options.python_version = sys.version_info[:2]
infer_python_executable(options, special_opts)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
# then test inferring version from executable
options = Options()
options.python_executable = sys.executable
infer_python_executable(options, special_opts)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
def test_executable_inference(self) -> None:
"""Test the --python-executable flag with --python-version"""
sys_ver_str = '{ver.major}.{ver.minor}'.format(ver=sys.version_info)
base = ['file.py'] # dummy file
# test inference given one (infer the other)
matching_version = base + ['--python-version={}'.format(sys_ver_str)]
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
matching_version = base + ['--python-executable={}'.format(sys.executable)]
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
# test inference given both
matching_version = base + ['--python-version={}'.format(sys_ver_str),
'--python-executable={}'.format(sys.executable)]
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
# test that --no-site-packages will disable executable inference
matching_version = base + ['--python-version={}'.format(sys_ver_str),
'--no-site-packages']
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable is None
# Test setting python_version/executable from config file
special_opts = argparse.Namespace()
special_opts.python_executable = None
special_opts.python_version = None
special_opts.no_executable = None
# first test inferring executable from version
options = Options()
options.python_executable = None
options.python_version = sys.version_info[:2]
infer_python_executable(options, special_opts)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
# then test inferring version from executable
options = Options()
options.python_executable = sys.executable
infer_python_executable(options, special_opts)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
def test_executable_inference(self) -> None:
"""Test the --python-executable flag with --python-version"""
sys_ver_str = '{ver.major}.{ver.minor}'.format(ver=sys.version_info)
base = ['file.py'] # dummy file
# test inference given one (infer the other)
matching_version = base + ['--python-version={}'.format(sys_ver_str)]
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
matching_version = base + [
'--python-executable={}'.format(sys.executable)
]
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
# test inference given both
matching_version = base + [
'--python-version={}'.format(sys_ver_str),
'--python-executable={}'.format(sys.executable)
]
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
# test that we error if the version mismatch
# argparse sys.exits on a parser.error, we need to check the raw inference function
options = Options()
special_opts = argparse.Namespace()
special_opts.python_executable = sys.executable
special_opts.python_version = (2, 10) # obviously wrong
special_opts.no_executable = None
with pytest.raises(PythonExecutableInferenceError) as e:
infer_python_version_and_executable(options, special_opts)
assert str(e.value) == 'Python version (2, 10) did not match executable {}, got' \
' version {}.'.format(sys.executable, sys.version_info[:2])
# test that --no-site-packages will disable executable inference
matching_version = base + [
'--python-version={}'.format(sys_ver_str), '--no-site-packages'
]
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable is None
def _mypy_analyze(mypy_conf_path: str,
root_path: str,
stubs_path: Optional[str] = None) -> BuildResult:
"""
Parse and analyze the types of the code in root_path.
:param mypy_conf_path: path to a mypy.ini
:param root_path: path to the code directory where the type analysis is started
:param stubs_path: path to the directory of stubs for mypy to use
:returns: Mypy's analysis result
"""
# The call to `build.build` is inspired by `mypy/mypy/main.py::main`
# `build` is not a documented public API
args = [
"--config-file",
mypy_conf_path,
"--no-incremental",
"--cache-dir=" + ("nul" if os.name == "nt" else "/dev/null"),
root_path,
]
sources, options = process_options(args)
if stubs_path is not None:
options = options.apply_changes({"mypy_path": [stubs_path]})
return build(sources, options, None, None)
def parse_options(self, program_text: str, testcase: DataDrivenTestCase,
incremental_step: int) -> Options:
options = Options()
flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
if incremental_step > 1:
flags2 = re.search('# flags{}: (.*)$'.format(incremental_step), program_text,
flags=re.MULTILINE)
if flags2:
flags = flags2
flag_list = None
if flags:
flag_list = flags.group(1).split()
targets, options = process_options(flag_list, require_targets=False)
if targets:
raise RuntimeError('Specifying targets via the flags pragma is not supported.')
else:
options = Options()
# Allow custom python version to override testcase_pyversion
if (not flag_list or
all(flag not in flag_list for flag in ['--python-version', '-2', '--py2'])):
options.python_version = testcase_pyversion(testcase.file, testcase.name)
options.use_builtins_fixtures = True
options.show_traceback = True
options.incremental = True
return options
def get_mypy_config(paths: List[str],
mypy_options: Optional[List[str]]) -> Tuple[List[BuildSource], Options]:
"""Construct mypy BuildSources and Options from file and options lists"""
# It is kind of silly to do this but oh well
mypy_options = mypy_options or []
mypy_options.append('--')
mypy_options.extend(paths)
sources, options = process_options(mypy_options)
# OSH PATCH
#if options.python_version[0] == 2:
if 0:
fail('Python 2 not supported')
# OSH Patch!
#if not options.strict_optional:
if 0:
fail('Disabling strict optional checking not supported')
options.show_traceback = True
# Needed to get types for all AST nodes
options.export_types = True
# TODO: Support incremental checking
options.incremental = False
# 10/2019: FIX for MyPy 0.730. Not sure why I need this but I do.
options.preserve_asts = True
for source in sources:
options.per_module_options.setdefault(source.module, {})['mypyc'] = True
return sources, options
def parse_options(self, program_text: str,
testcase: DataDrivenTestCase) -> Options:
options = Options()
flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
flag_list = None
if flags:
flag_list = flags.group(1).split()
targets, options = process_options(flag_list,
require_targets=False)
if targets:
# TODO: support specifying targets via the flags pragma
raise RuntimeError(
'Specifying targets via the flags pragma is not supported.'
)
else:
options = Options()
# Allow custom python version to override testcase_pyversion
if (not flag_list
or all(flag not in flag_list
for flag in ['--python-version', '-2', '--py2'])):
options.python_version = testcase_pyversion(
testcase.file, testcase.name)
return options
def parse_options(program_text: str, testcase: DataDrivenTestCase,
incremental_step: int) -> Options:
"""Parse comments like '# flags: --foo' in a test case."""
options = Options()
flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
if incremental_step > 1:
flags2 = re.search('# flags{}: (.*)$'.format(incremental_step),
program_text,
flags=re.MULTILINE)
if flags2:
flags = flags2
flag_list = None
if flags:
flag_list = flags.group(1).split()
targets, options = process_options(flag_list, require_targets=False)
if targets:
# TODO: support specifying targets via the flags pragma
raise RuntimeError(
'Specifying targets via the flags pragma is not supported.')
else:
options = Options()
# Allow custom python version to override testcase_pyversion
if (not flag_list or all(flag not in flag_list
for flag in ['--python-version', '-2', '--py2'])):
options.python_version = testcase_pyversion(testcase.file,
testcase.name)
return options
def get_mypy_config(
paths: List[str], mypy_options: Optional[List[str]]
) -> Tuple[List[BuildSource], Options]:
"""Construct mypy BuildSources and Options from file and options lists"""
# It is kind of silly to do this but oh well
mypy_options = mypy_options or []
mypy_options.append('--')
mypy_options.extend(paths)
sources, options = process_options(mypy_options)
# Override whatever python_version is inferred from the .ini file,
# and set the python_version to be the currently used version.
options.python_version = sys.version_info[:2]
if options.python_version[0] == 2:
fail('Python 2 not supported')
if not options.strict_optional:
fail('Disabling strict optional checking not supported')
options.show_traceback = True
# Needed to get types for all AST nodes
options.export_types = True
# TODO: Support incremental checking
options.incremental = False
options.preserve_asts = True
for source in sources:
options.per_module_options.setdefault(source.module,
{})['mypyc'] = True
return sources, options
def parse_options(program_text: str, testcase: DataDrivenTestCase,
incremental_step: int) -> Options:
"""Parse comments like '# flags: --foo' in a test case."""
options = Options()
flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
if incremental_step > 1:
flags2 = re.search('# flags{}: (.*)$'.format(incremental_step), program_text,
flags=re.MULTILINE)
if flags2:
flags = flags2
flag_list = None
if flags:
flag_list = flags.group(1).split()
targets, options = process_options(flag_list, require_targets=False)
if targets:
# TODO: support specifying targets via the flags pragma
raise RuntimeError('Specifying targets via the flags pragma is not supported.')
else:
options = Options()
# Allow custom python version to override testcase_pyversion
if (not flag_list or
all(flag not in flag_list for flag in ['--python-version', '-2', '--py2'])):
options.python_version = testcase_pyversion(testcase.file, testcase.name)
return options
def run_mypy_typechecking(cmd_options: List[str]) -> int:
fscache = FileSystemCache()
sources, options = process_options(cmd_options, fscache=fscache)
error_messages = []
def flush_errors(new_messages: List[str], serious: bool) -> None:
error_messages.extend(new_messages)
f = sys.stderr if serious else sys.stdout
try:
for msg in new_messages:
f.write(msg + "\n")
f.flush()
except BrokenPipeError:
sys.exit(ReturnCodes.FATAL_ERROR)
try:
build.build(sources,
options,
flush_errors=flush_errors,
fscache=fscache)
except SystemExit as sysexit:
return sysexit.code
finally:
fscache.flush()
if error_messages:
return ReturnCodes.FAIL
return ReturnCodes.SUCCESS
def parse_options(self, program_text: str, testcase: DataDrivenTestCase,
incremental_step: int) -> Options:
options = Options()
flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
if incremental_step > 1:
flags2 = re.search('# flags{}: (.*)$'.format(incremental_step),
program_text,
flags=re.MULTILINE)
if flags2:
flags = flags2
flag_list = None
if flags:
flag_list = flags.group(1).split()
targets, options = process_options(flag_list,
require_targets=False)
if targets:
raise RuntimeError(
'Specifying targets via the flags pragma is not supported.'
)
else:
options = Options()
# Allow custom python version to override testcase_pyversion
if (not flag_list
or all(flag not in flag_list
for flag in ['--python-version', '-2', '--py2'])):
options.python_version = testcase_pyversion(
testcase.file, testcase.name)
options.use_builtins_fixtures = True
options.show_traceback = True
options.incremental = True
return options
def parse_options(program_text: str, testcase: DataDrivenTestCase,
incremental_step: int) -> Options:
"""Parse comments like '# flags: --foo' in a test case."""
options = Options()
flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
if incremental_step > 1:
flags2 = re.search('# flags{}: (.*)$'.format(incremental_step), program_text,
flags=re.MULTILINE)
if flags2:
flags = flags2
flag_list = None
if flags:
flag_list = flags.group(1).split()
flag_list.append('--no-site-packages') # the tests shouldn't need an installed Python
targets, options = process_options(flag_list, require_targets=False)
if targets:
# TODO: support specifying targets via the flags pragma
raise RuntimeError('Specifying targets via the flags pragma is not supported.')
else:
options = Options()
# TODO: Enable strict optional in test cases by default (requires *many* test case changes)
options.strict_optional = False
# Allow custom python version to override testcase_pyversion
if (not flag_list or
all(flag not in flag_list for flag in ['--python-version', '-2', '--py2'])):
options.python_version = testcase_pyversion(testcase.file, testcase.name)
if testcase.config.getoption('--mypy-verbose'):
options.verbosity = testcase.config.getoption('--mypy-verbose')
return options
def test_coherence(self):
# We have to special case Options.BuildType because we're required to
# set a target
options = Options()
options.build_type = BuildType.PROGRAM_TEXT
_, parsed_options = process_options(['-c', 'cmd'])
assert_equal(options, parsed_options)
def get_ast():
sources, options = process_options(sys.argv[1:])
res = build.build(sources, options)
for module_name, f in res.files.items():
if f.path == sys.argv[1]:
print(str(f))
return
print('Module not found after analysis.')
def check_with_mypy(abs_path: Path, config_file_path: Path) -> int:
error_happened = False
with cd(abs_path):
sources, options = process_options(['--cache-dir', str(config_file_path.parent / '.mypy_cache'),
'--config-file', str(config_file_path),
str(abs_path)])
res = build.build(sources, options)
for error_line in res.errors:
if not is_ignored(error_line, abs_path.name):
error_happened = True
print(replace_with_clickable_location(error_line, abs_test_folder=abs_path))
return int(error_happened)
def _parse_options(self):
"""Parse options using mypy
"""
self.settings.append(self.filename)
# check MyPy version
match = re.match(r'(\d).(\d).(\d)', mypy.__version__)
if match is not None:
mypy_version = tuple(int(i) for i in (
match.group(1), match.group(2), match.group(3)))
else:
mypy_version = (0, 0, 0)
if mypy_version >= (0, 4, 3):
return mypy.process_options(self.settings)
sys_argv = sys.argv
sys.argv = [''] + self.settings
sources, options = mypy.process_options()
sys.argv = sys_argv
return sources, options
def check_with_mypy(abs_path: Path) -> int:
error_happened = False
with cd(abs_path):
sources, options = process_options([
'--cache-dir',
str(MYPY_CONFIG_FILE.parent / '.mypy_cache'), '--config-file',
str(MYPY_CONFIG_FILE),
str(abs_path)
])
res = build.build(sources, options)
for error_line in res.errors:
if not error_line.startswith('tests/'):
# only leave errors for tests/ directory
continue
if not is_ignored(error_line):
print(
replace_with_clickable_location(error_line,
abs_test_folder=abs_path))
return int(error_happened)
def get_mypy_config(
mypy_options: List[str],
only_compile_paths: Optional[Iterable[str]],
compiler_options: CompilerOptions,
fscache: Optional[FileSystemCache],
) -> Tuple[List[BuildSource], List[BuildSource], Options]:
"""Construct mypy BuildSources and Options from file and options lists"""
all_sources, options = process_options(mypy_options, fscache=fscache)
if only_compile_paths is not None:
paths_set = set(only_compile_paths)
mypyc_sources = [s for s in all_sources if s.path in paths_set]
else:
mypyc_sources = all_sources
if compiler_options.separate:
mypyc_sources = [
src for src in mypyc_sources
if src.path and not src.path.endswith('__init__.py')
]
if not mypyc_sources:
return mypyc_sources, all_sources, options
# Override whatever python_version is inferred from the .ini file,
# and set the python_version to be the currently used version.
options.python_version = sys.version_info[:2]
if options.python_version[0] == 2:
fail('Python 2 not supported')
if not options.strict_optional:
fail('Disabling strict optional checking not supported')
options.show_traceback = True
# Needed to get types for all AST nodes
options.export_types = True
# We use mypy incremental mode when doing separate/incremental mypyc compilation
options.incremental = compiler_options.separate
options.preserve_asts = True
for source in mypyc_sources:
options.per_module_options.setdefault(source.module,
{})['mypyc'] = True
return mypyc_sources, all_sources, options
def parse_options(self, program_text: str, testcase: DataDrivenTestCase) -> Options:
options = Options()
flags = re.search("# flags: (.*)$", program_text, flags=re.MULTILINE)
flag_list = None
if flags:
flag_list = flags.group(1).split()
targets, options = process_options(flag_list, require_targets=False)
if targets:
# TODO: support specifying targets via the flags pragma
raise RuntimeError("Specifying targets via the flags pragma is not supported.")
else:
options = Options()
# Allow custom python version to override testcase_pyversion
if not flag_list or all(flag not in flag_list for flag in ["--python-version", "-2", "--py2"]):
options.python_version = testcase_pyversion(testcase.file, testcase.name)
return options
def console_entry() -> None:
try:
main(None, sys.stdout, sys.stderr)
sys.stdout.flush()
sys.stderr.flush()
except BrokenPipeError:
# Python flushes standard streams on exit; redirect remaining output
# to devnull to avoid another BrokenPipeError at shutdown
devnull = os.open(os.devnull, os.O_WRONLY)
os.dup2(devnull, sys.stdout.fileno())
sys.exit(2)
except KeyboardInterrupt:
_, options = process_options(args=sys.argv[1:])
if options.show_traceback:
sys.stdout.write(traceback.format_exc())
formatter = FancyFormatter(sys.stdout, sys.stderr, False)
msg = "Interrupted\n"
sys.stdout.write(formatter.style(msg, color="red", bold=True))
sys.stdout.flush()
sys.stderr.flush()
sys.exit(2)
def parse_options(program_text: str, testcase: DataDrivenTestCase,
incremental_step: int) -> Options:
"""Parse comments like '# flags: --foo' in a test case."""
options = Options()
flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
if incremental_step > 1:
flags2 = re.search('# flags{}: (.*)$'.format(incremental_step), program_text,
flags=re.MULTILINE)
if flags2:
flags = flags2
flag_list = None
if flags:
flag_list = flags.group(1).split()
flag_list.append('--no-site-packages') # the tests shouldn't need an installed Python
targets, options = process_options(flag_list, require_targets=False)
if targets:
# TODO: support specifying targets via the flags pragma
raise RuntimeError('Specifying targets via the flags pragma is not supported.')
else:
options = Options()
# TODO: Enable strict optional in test cases by default (requires *many* test case changes)
options.strict_optional = False
# Allow custom python version to override testcase_pyversion
if (not flag_list or
all(flag not in flag_list for flag in ['--python-version', '-2', '--py2'])):
options.python_version = testcase_pyversion(testcase.file, testcase.name)
if testcase.config.getoption('--mypy-verbose'):
options.verbosity = testcase.config.getoption('--mypy-verbose')
if os.getenv('NEWSEMANAL'):
if not flag_list or '--no-new-semantic-analyzer' not in flag_list:
options.new_semantic_analyzer = True
return options
def run(filepath):
with open(filepath) as rf:
source = rf.read()
sources, options = process_options([filepath])
return parse(source, filepath, None, options)
def test_coherence(self):
options = Options()
_, parsed_options = process_options([], require_targets=False)
assert_equal(options, parsed_options)
def test_coherence(self) -> None:
options = Options()
_, parsed_options = process_options([], require_targets=False)
# FIX: test this too. Requires changing working dir to avoid finding 'setup.cfg'
options.config_file = parsed_options.config_file
assert_equal(options.snapshot(), parsed_options.snapshot())
def test_coherence(self) -> None:
options = Options()
_, parsed_options = process_options([], require_targets=False)
assert_equal(options, parsed_options)
