def test_error_stream(testcase: DataDrivenTestCase) -> None:
"""Perform a single error streaming test case.
The argument contains the description of the test case.
"""
options = Options()
options.show_traceback = True
logged_messages = [] # type: List[str]
def flush_errors(msgs: List[str], serious: bool) -> None:
if msgs:
logged_messages.append('==== Errors flushed ====')
logged_messages.extend(msgs)
sources = [BuildSource('main', '__main__', '\n'.join(testcase.input))]
try:
build.build(sources=sources,
options=options,
alt_lib_path=test_temp_dir,
flush_errors=flush_errors)
except CompileError as e:
assert e.messages == []
assert_string_arrays_equal(testcase.output, logged_messages,
'Invalid output ({}, line {})'.format(
testcase.file, testcase.line))
def test_coherence(self):
# We have to special case Options.BuildType because we're required to
# set a target
options = Options()
options.build_type = BuildType.PROGRAM_TEXT
_, parsed_options = process_options(['-c', 'cmd'])
assert_equal(options, parsed_options)
def get_semanal_options() -> Options:
options = Options()
options.use_builtins_fixtures = True
options.semantic_analysis_only = True
options.show_traceback = True
options.python_version = PYTHON3_VERSION
return options
def generate_stub(path: str,
output_dir: str,
_all_: Optional[List[str]] = None,
target: Optional[str] = None,
add_header: bool = False,
module: Optional[str] = None,
pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION,
include_private: bool = False
) -> None:
with open(path, 'rb') as f:
data = f.read()
source = mypy.util.decode_python_encoding(data, pyversion)
options = MypyOptions()
options.python_version = pyversion
try:
ast = mypy.parse.parse(source, fnam=path, module=module, errors=None, options=options)
except mypy.errors.CompileError as e:
# Syntax error!
for m in e.messages:
sys.stderr.write('%s\n' % m)
sys.exit(1)
gen = StubGenerator(_all_, pyversion=pyversion, include_private=include_private)
ast.accept(gen)
if not target:
target = os.path.join(output_dir, os.path.basename(path))
subdir = os.path.dirname(target)
if subdir and not os.path.isdir(subdir):
os.makedirs(subdir)
with open(target, 'w') as file:
if add_header:
write_header(file, module, pyversion=pyversion)
file.write(''.join(gen.output()))
def parse_options(self, program_text: str, testcase: DataDrivenTestCase,
incremental_step: int) -> Options:
options = Options()
flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
if incremental_step > 1:
flags2 = re.search('# flags{}: (.*)$'.format(incremental_step), program_text,
flags=re.MULTILINE)
if flags2:
flags = flags2
flag_list = None
if flags:
flag_list = flags.group(1).split()
targets, options = process_options(flag_list, require_targets=False)
if targets:
# TODO: support specifying targets via the flags pragma
raise RuntimeError('Specifying targets via the flags pragma is not supported.')
else:
options = Options()
# Allow custom python version to override testcase_pyversion
if (not flag_list or
all(flag not in flag_list for flag in ['--python-version', '-2', '--py2'])):
options.python_version = testcase_pyversion(testcase.file, testcase.name)
return options
def build(self, source: str) -> Tuple[List[str], Optional[Dict[str, MypyFile]]]:
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.cache_dir = os.devnull
try:
result = build.build(sources=[BuildSource('main', None, source)],
options=options,
alt_lib_path=test_temp_dir)
except CompileError as e:
# TODO: Is it okay to return None?
return e.messages, None
return result.errors, result.files
def run_case(self, testcase: DataDrivenTestCase) -> None:
try:
line = testcase.input[0]
mask = ''
if line.startswith('##'):
mask = '(' + line[2:].strip() + ')$'
src = '\n'.join(testcase.input)
options = Options()
options.strict_optional = False # TODO: Enable strict optional checking
options.use_builtins_fixtures = True
options.show_traceback = True
options.export_types = True
result = build.build(sources=[BuildSource('main', None, src)],
options=options,
alt_lib_path=test_temp_dir)
a = result.errors
map = result.types
nodes = map.keys()
# Ignore NameExpr nodes of variables with explicit (trivial) types
# to simplify output.
searcher = SkippedNodeSearcher()
for file in result.files.values():
file.accept(searcher)
ignored = searcher.nodes
# Filter nodes that should be included in the output.
keys = []
for node in nodes:
if node.line is not None and node.line != -1 and map[node]:
if ignore_node(node) or node in ignored:
continue
if (re.match(mask, short_type(node))
or (isinstance(node, NameExpr)
and re.match(mask, node.name))):
# Include node in output.
keys.append(node)
for key in sorted(keys,
key=lambda n: (n.line, short_type(n),
str(n) + str(map[n]))):
ts = str(map[key]).replace('*', '') # Remove erased tags
ts = ts.replace('__main__.', '')
a.append('{}({}) : {}'.format(short_type(key), key.line, ts))
except CompileError as e:
a = e.messages
assert_string_arrays_equal(
testcase.output, a,
'Invalid type checker output ({}, line {})'.format(testcase.file,
testcase.line))
def test_executable_inference(self) -> None:
"""Test the --python-executable flag with --python-version"""
sys_ver_str = '{ver.major}.{ver.minor}'.format(ver=sys.version_info)
base = ['file.py'] # dummy file
# test inference given one (infer the other)
matching_version = base + ['--python-version={}'.format(sys_ver_str)]
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
matching_version = base + ['--python-executable={}'.format(sys.executable)]
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
# test inference given both
matching_version = base + ['--python-version={}'.format(sys_ver_str),
'--python-executable={}'.format(sys.executable)]
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
# test that --no-site-packages will disable executable inference
matching_version = base + ['--python-version={}'.format(sys_ver_str),
'--no-site-packages']
_, options = process_options(matching_version)
assert options.python_version == sys.version_info[:2]
assert options.python_executable is None
# Test setting python_version/executable from config file
special_opts = argparse.Namespace()
special_opts.python_executable = None
special_opts.python_version = None
special_opts.no_executable = None
# first test inferring executable from version
options = Options()
options.python_executable = None
options.python_version = sys.version_info[:2]
infer_python_executable(options, special_opts)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
# then test inferring version from executable
options = Options()
options.python_executable = sys.executable
infer_python_executable(options, special_opts)
assert options.python_version == sys.version_info[:2]
assert options.python_executable == sys.executable
def build_dir(target_dir: str) -> Tuple[List[str], BuildManager, Graph]:
sources = expand_dir(target_dir)
options = Options()
options.incremental = True
options.show_traceback = True
options.cache_dir = os.devnull
try:
result = build.build(sources=sources,
options=options)
except CompileError as e:
# TODO: We need a manager and a graph in this case as well
assert False, str('\n'.join(e.messages))
return e.messages, None, None
return result.errors, result.manager, result.graph
def daemonize(options: Options,
status_file: str,
timeout: Optional[int] = None,
log_file: Optional[str] = None) -> int:
"""Create the daemon process via "dmypy daemon" and pass options via command line
When creating the daemon grandchild, we create it in a new console, which is
started hidden. We cannot use DETACHED_PROCESS since it will cause console windows
to pop up when starting. See
https://github.com/python/cpython/pull/4150#issuecomment-340215696
for more on why we can't have nice things.
It also pickles the options to be unpickled by mypy.
"""
command = [sys.executable, '-m', 'mypy.dmypy', '--status-file', status_file, 'daemon']
pickeled_options = pickle.dumps((options.snapshot(), timeout, log_file))
command.append('--options-data="{}"'.format(base64.b64encode(pickeled_options).decode()))
info = STARTUPINFO()
info.dwFlags = 0x1 # STARTF_USESHOWWINDOW aka use wShowWindow's value
info.wShowWindow = 0 # SW_HIDE aka make the window invisible
try:
subprocess.Popen(command,
creationflags=0x10, # CREATE_NEW_CONSOLE
startupinfo=info)
return 0
except subprocess.CalledProcessError as e:
return e.returncode
def build(self, source: str) -> Tuple[List[str], Optional[BuildManager], Dict[str, State]]:
options = Options()
options.incremental = True
options.use_builtins_fixtures = True
options.show_traceback = True
main_path = os.path.join(test_temp_dir, 'main')
with open(main_path, 'w') as f:
f.write(source)
try:
result = build.build(sources=[BuildSource(main_path, None, None)],
options=options,
alt_lib_path=test_temp_dir)
except CompileError as e:
# TODO: Is it okay to return None?
return e.messages, None, {}
return result.errors, result.manager, result.graph
def process_package_roots(fscache: Optional[FileSystemCache],
parser: argparse.ArgumentParser,
options: Options) -> None:
"""Validate and normalize package_root."""
if fscache is None:
parser.error("--package-root does not work here (no fscache)")
assert fscache is not None # Since mypy doesn't know parser.error() raises.
# Do some stuff with drive letters to make Windows happy (esp. tests).
current_drive, _ = os.path.splitdrive(os.getcwd())
dot = os.curdir
dotslash = os.curdir + os.sep
dotdotslash = os.pardir + os.sep
trivial_paths = {dot, dotslash}
package_root = []
for root in options.package_root:
if os.path.isabs(root):
parser.error("Package root cannot be absolute: %r" % root)
drive, root = os.path.splitdrive(root)
if drive and drive != current_drive:
parser.error("Package root must be on current drive: %r" % (drive + root))
# Empty package root is always okay.
if root:
root = os.path.relpath(root) # Normalize the heck out of it.
if root.startswith(dotdotslash):
parser.error("Package root cannot be above current directory: %r" % root)
if root in trivial_paths:
root = ''
elif not root.endswith(os.sep):
root = root + os.sep
package_root.append(root)
options.package_root = package_root
# Pass the package root on the the filesystem cache.
fscache.set_package_root(package_root)
def __init__(self, options: Options,
status_file: str,
timeout: Optional[int] = None) -> None:
"""Initialize the server with the desired mypy flags."""
self.options = options
# Snapshot the options info before we muck with it, to detect changes
self.options_snapshot = options.snapshot()
self.timeout = timeout
self.fine_grained_manager = None # type: Optional[FineGrainedBuildManager]
if os.path.isfile(status_file):
os.unlink(status_file)
self.fscache = FileSystemCache()
options.incremental = True
options.fine_grained_incremental = True
options.show_traceback = True
if options.use_fine_grained_cache:
# Using fine_grained_cache implies generating and caring
# about the fine grained cache
options.cache_fine_grained = True
else:
options.cache_dir = os.devnull
# Fine-grained incremental doesn't support general partial types
# (details in https://github.com/python/mypy/issues/4492)
options.local_partial_types = True
self.status_file = status_file
def do_daemon(args: argparse.Namespace) -> None:
"""Serve requests in the foreground."""
# Lazy import so this import doesn't slow down other commands.
from mypy.dmypy_server import Server, process_start_options
if args.options_data:
from mypy.options import Options
options_dict, timeout, log_file = pickle.loads(base64.b64decode(args.options_data))
options_obj = Options()
options = options_obj.apply_changes(options_dict)
if log_file:
sys.stdout = sys.stderr = open(log_file, 'a', buffering=1)
fd = sys.stdout.fileno()
os.dup2(fd, 2)
os.dup2(fd, 1)
else:
options = process_start_options(args.flags, allow_sources=False)
timeout = args.timeout
Server(options, args.status_file, timeout=timeout).serve()
def build(self,
source: str,
python_version: Tuple[int, int]) -> Tuple[List[str],
Optional[Dict[str, MypyFile]],
Optional[Dict[Expression, Type]]]:
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.cache_dir = os.devnull
options.python_version = python_version
try:
result = build.build(sources=[BuildSource('main', None, source)],
options=options,
alt_lib_path=test_temp_dir)
except CompileError as e:
# TODO: Should perhaps not return None here.
return e.messages, None, None
return result.errors, result.files, result.types
def parse_options(self, program_text: str, testcase: DataDrivenTestCase) -> Options:
options = Options()
flags = re.search("# flags: (.*)$", program_text, flags=re.MULTILINE)
flag_list = None
if flags:
flag_list = flags.group(1).split()
targets, options = process_options(flag_list, require_targets=False)
if targets:
# TODO: support specifying targets via the flags pragma
raise RuntimeError("Specifying targets via the flags pragma is not supported.")
else:
options = Options()
# Allow custom python version to override testcase_pyversion
if not flag_list or all(flag not in flag_list for flag in ["--python-version", "-2", "--py2"]):
options.python_version = testcase_pyversion(testcase.file, testcase.name)
return options
def build_stubs(mod):
data_dir = default_data_dir(None)
options = Options()
options.python_version = (3, 6)
lib_path = default_lib_path(data_dir,
options.python_version,
custom_typeshed_dir=None)
sources = find_modules_recursive(mod, lib_path)
try:
res = build.build(sources=sources,
options=options)
messages = res.errors
except CompileError as error:
messages = error.messages
if messages:
for msg in messages:
print(msg)
sys.exit(1)
return res.files
def test_parser(testcase):
"""Perform a single parser test case.
The argument contains the description of the test case.
"""
options = Options()
if testcase.file.endswith('python2.test'):
options.python_version = defaults.PYTHON2_VERSION
else:
options.python_version = defaults.PYTHON3_VERSION
try:
n = parse(bytes('\n'.join(testcase.input), 'ascii'),
fnam='main',
errors=None,
options=options)
a = str(n).split('\n')
except CompileError as e:
a = e.messages
assert_string_arrays_equal(testcase.output, a,
'Invalid parser output ({}, line {})'.format(
testcase.file, testcase.line))
def mypy_options(stubgen_options: Options) -> MypyOptions:
"""Generate mypy options using the flag passed by user."""
options = MypyOptions()
options.follow_imports = 'skip'
options.incremental = False
options.ignore_errors = True
options.semantic_analysis_only = True
options.python_version = stubgen_options.pyversion
return options
def infer_python_executable(options: Options,
special_opts: argparse.Namespace) -> None:
"""Infer the Python executable from the given version.
This function mutates options based on special_opts to infer the correct Python executable
to use.
"""
# TODO: (ethanhs) Look at folding these checks and the site packages subprocess calls into
# one subprocess call for speed.
# Use the command line specified executable, or fall back to one set in the
# config file. If an executable is not specified, infer it from the version
# (unless no_executable is set)
python_executable = special_opts.python_executable or options.python_executable
if python_executable is None:
if not special_opts.no_executable:
python_executable = _python_executable_from_version(options.python_version)
options.python_executable = python_executable
def parse_config_file(options: Options, filename: str) -> None:
"""Parse a config file into an Options object.
Errors are written to stderr but are not fatal.
"""
parser = configparser.RawConfigParser()
try:
parser.read(filename)
except configparser.Error as err:
print("%s: %s" % (filename, err), file=sys.stderr)
return
if 'mypy' not in parser:
print("%s: No [mypy] section in config file" % filename, file=sys.stderr)
else:
section = parser['mypy']
prefix = '%s: [%s]' % (filename, 'mypy')
updates, report_dirs = parse_section(prefix, options, section)
for k, v in updates.items():
setattr(options, k, v)
options.report_dirs.update(report_dirs)
for name, section in parser.items():
if name.startswith('mypy-'):
prefix = '%s: [%s]' % (filename, name)
updates, report_dirs = parse_section(prefix, options, section)
if report_dirs:
print("%s: Per-module sections should not specify reports (%s)" %
(prefix, ', '.join(s + '_report' for s in sorted(report_dirs))),
file=sys.stderr)
if set(updates) - Options.PER_MODULE_OPTIONS:
print("%s: Per-module sections should only specify per-module flags (%s)" %
(prefix, ', '.join(sorted(set(updates) - Options.PER_MODULE_OPTIONS))),
file=sys.stderr)
updates = {k: v for k, v in updates.items() if k in Options.PER_MODULE_OPTIONS}
globs = name[5:]
for glob in globs.split(','):
# For backwards compatibility, replace (back)slashes with dots.
glob = glob.replace(os.sep, '.')
if os.altsep:
glob = glob.replace(os.altsep, '.')
pattern = re.compile(fnmatch.translate(glob))
options.per_module_options[pattern] = updates
def process_cache_map(parser: argparse.ArgumentParser,
special_opts: argparse.Namespace,
options: Options) -> None:
"""Validate cache_map and copy into options.cache_map."""
n = len(special_opts.cache_map)
if n % 3 != 0:
parser.error("--cache-map requires one or more triples (see source)")
for i in range(0, n, 3):
source, meta_file, data_file = special_opts.cache_map[i:i + 3]
if source in options.cache_map:
parser.error("Duplicate --cache-map source %s)" % source)
if not source.endswith('.py') and not source.endswith('.pyi'):
parser.error("Invalid --cache-map source %s (triple[0] must be *.py[i])" % source)
if not meta_file.endswith('.meta.json'):
parser.error("Invalid --cache-map meta_file %s (triple[1] must be *.meta.json)" %
meta_file)
if not data_file.endswith('.data.json'):
parser.error("Invalid --cache-map data_file %s (triple[2] must be *.data.json)" %
data_file)
options.cache_map[source] = (meta_file, data_file)
def build(self, source: str) -> Optional[BuildResult]:
options = Options()
options.incremental = True
options.fine_grained_incremental = True
options.use_builtins_fixtures = True
options.show_traceback = True
options.python_version = PYTHON3_VERSION
main_path = os.path.join(test_temp_dir, 'main')
with open(main_path, 'w', encoding='utf8') as f:
f.write(source)
try:
result = build.build(sources=[BuildSource(main_path, None, None)],
options=options,
alt_lib_path=test_temp_dir)
except CompileError:
# TODO: Is it okay to return None?
return None
return result
def parse(source: Union[str, bytes],
fnam: str,
module: Optional[str],
errors: Optional[Errors] = None,
options: Optional[Options] = None) -> MypyFile:
"""Parse a source file, without doing any semantic analysis.
Return the parse tree. If errors is not provided, raise ParseError
on failure. Otherwise, use the errors object to report parse errors.
"""
raise_on_error = False
if errors is None:
errors = Errors()
raise_on_error = True
if options is None:
options = Options()
errors.set_file(fnam, module)
is_stub_file = fnam.endswith('.pyi')
try:
assert options.python_version[0] < 3 and not is_stub_file
ast = ast27.parse(source, fnam, 'exec')
tree = ASTConverter(options=options,
is_stub=is_stub_file,
errors=errors,
).visit(ast)
assert isinstance(tree, MypyFile)
tree.path = fnam
tree.is_stub = is_stub_file
except SyntaxError as e:
errors.report(e.lineno, e.offset, e.msg, blocker=True)
tree = MypyFile([], [], False, set())
if raise_on_error and errors.is_errors():
errors.raise_error()
return tree
def process_cache_map(parser: argparse.ArgumentParser,
special_opts: argparse.Namespace,
options: Options) -> None:
"""Validate cache_map and copy into options.cache_map."""
n = len(special_opts.cache_map)
if n % 3 != 0:
parser.error("--cache-map requires one or more triples (see source)")
for i in range(0, n, 3):
source, meta_file, data_file = special_opts.cache_map[i:i + 3]
if source in options.cache_map:
parser.error("Duplicate --cache-map source %s)" % source)
if not source.endswith('.py') and not source.endswith('.pyi'):
parser.error(
"Invalid --cache-map source %s (triple[0] must be *.py[i])" %
source)
if not meta_file.endswith('.meta.json'):
parser.error(
"Invalid --cache-map meta_file %s (triple[1] must be *.meta.json)"
% meta_file)
if not data_file.endswith('.data.json'):
parser.error(
"Invalid --cache-map data_file %s (triple[2] must be *.data.json)"
% data_file)
options.cache_map[source] = (meta_file, data_file)
def process_package_roots(fscache: Optional[FileSystemCache],
parser: argparse.ArgumentParser,
options: Options) -> None:
"""Validate and normalize package_root."""
if fscache is None:
parser.error("--package-root does not work here (no fscache)")
assert fscache is not None # Since mypy doesn't know parser.error() raises.
# Do some stuff with drive letters to make Windows happy (esp. tests).
current_drive, _ = os.path.splitdrive(os.getcwd())
dot = os.curdir
dotslash = os.curdir + os.sep
dotdotslash = os.pardir + os.sep
trivial_paths = {dot, dotslash}
package_root = []
for root in options.package_root:
if os.path.isabs(root):
parser.error("Package root cannot be absolute: %r" % root)
drive, root = os.path.splitdrive(root)
if drive and drive != current_drive:
parser.error("Package root must be on current drive: %r" %
(drive + root))
# Empty package root is always okay.
if root:
root = os.path.relpath(root) # Normalize the heck out of it.
if root.startswith(dotdotslash):
parser.error(
"Package root cannot be above current directory: %r" %
root)
if root in trivial_paths:
root = ''
elif not root.endswith(os.sep):
root = root + os.sep
package_root.append(root)
options.package_root = package_root
# Pass the package root on the the filesystem cache.
fscache.set_package_root(package_root)
def parse(source: Union[str, bytes], fnam: str = None, errors: Errors = None,
options: Options = Options()) -> MypyFile:
"""Parse a source file, without doing any semantic analysis.
Return the parse tree. If errors is not provided, raise ParseError
on failure. Otherwise, use the errors object to report parse errors.
"""
raise_on_error = False
if errors is None:
errors = Errors()
raise_on_error = True
errors.set_file('<input>' if fnam is None else fnam, None)
is_stub_file = bool(fnam) and fnam.endswith('.pyi')
try:
if is_stub_file:
feature_version = defaults.PYTHON3_VERSION[1]
else:
assert options.python_version[0] >= 3
feature_version = options.python_version[1]
ast = ast3.parse(source, fnam, 'exec', feature_version=feature_version)
tree = ASTConverter(options=options,
is_stub=is_stub_file,
errors=errors,
).visit(ast)
tree.path = fnam
tree.is_stub = is_stub_file
except SyntaxError as e:
errors.report(e.lineno, e.offset, e.msg)
tree = MypyFile([], [], False, set())
if raise_on_error and errors.is_errors():
errors.raise_error()
return tree
def build(
self, source: str
) -> Tuple[List[str], Optional[BuildManager], Dict[str, State]]:
options = Options()
options.incremental = True
options.fine_grained_incremental = True
options.use_builtins_fixtures = True
options.show_traceback = True
main_path = os.path.join(test_temp_dir, 'main')
with open(main_path, 'w') as f:
f.write(source)
try:
result = build.build(sources=[BuildSource(main_path, None, None)],
options=options,
alt_lib_path=test_temp_dir)
except CompileError as e:
# TODO: Is it okay to return None?
return e.messages, None, {}
return result.errors, result.manager, result.graph
def parse_options(program_text: str, testcase: DataDrivenTestCase,
incremental_step: int) -> Options:
"""Parse comments like '# flags: --foo' in a test case."""
options = Options()
flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
if incremental_step > 1:
flags2 = re.search('# flags{}: (.*)$'.format(incremental_step),
program_text,
flags=re.MULTILINE)
if flags2:
flags = flags2
flag_list = None
if flags:
flag_list = flags.group(1).split()
flag_list.append('--no-site-packages'
) # the tests shouldn't need an installed Python
targets, options = process_options(flag_list, require_targets=False)
if targets:
# TODO: support specifying targets via the flags pragma
raise RuntimeError(
'Specifying targets via the flags pragma is not supported.')
else:
options = Options()
# TODO: Enable strict optional in test cases by default (requires *many* test case changes)
options.strict_optional = False
# Allow custom python version to override testcase_pyversion
if (not flag_list or all(flag not in flag_list
for flag in ['--python-version', '-2', '--py2'])):
options.python_version = testcase_pyversion(testcase.file,
testcase.name)
if testcase.config.getoption('--mypy-verbose'):
options.verbosity = testcase.config.getoption('--mypy-verbose')
return options
def test_type_check(mocker, extensions):
mocker.patch("sys.argv", ["", "clean"])
runpy.run_module("gino.ext", run_name="__main__")
result = build(
[BuildSource(None, None, "from gino.ext.demo3 import s3")], Options()
)
assert result.errors
result = build(
[BuildSource(None, None, "from gino.ext.demo1 import s1")], Options()
)
assert result.errors
mocker.patch("sys.argv", ["", "stub"])
runpy.run_module("gino.ext", run_name="__main__")
runpy.run_module("gino.ext", run_name="__main__")
try:
result = build(
[BuildSource(None, None, "from gino.ext.demo1 import s1")], Options()
)
assert not result.errors
result = build(
[BuildSource(None, None, "from gino.ext.demo1 import s2")], Options()
)
assert result.errors
result = build(
[BuildSource(None, None, "from gino.ext.demo2 import s2")], Options()
)
assert not result.errors
result = build(
[BuildSource(None, None, "from gino.ext.demo2 import s1")], Options()
)
assert result.errors
finally:
mocker.patch("sys.argv", ["", "clean"])
runpy.run_module("gino.ext", run_name="__main__")
def test_transform(testcase: DataDrivenTestCase) -> None:
"""Perform an identity transform test case."""
try:
src = '\n'.join(testcase.input)
options = Options()
options.use_builtins_fixtures = True
options.semantic_analysis_only = True
options.show_traceback = True
options.python_version = testfile_pyversion(testcase.file)
result = build.build(sources=[BuildSource('main', None, src)],
options=options,
alt_lib_path=test_temp_dir)
a = result.errors
if a:
raise CompileError(a)
# Include string representations of the source files in the actual
# output.
for fnam in sorted(result.files.keys()):
f = result.files[fnam]
# Omit the builtins module and files with a special marker in the
# path.
# TODO the test is not reliable
if (not f.path.endswith((os.sep + 'builtins.pyi',
'typing.pyi',
'abc.pyi'))
and not os.path.basename(f.path).startswith('_')
and not os.path.splitext(
os.path.basename(f.path))[0].endswith('_')):
t = TypeAssertTransformVisitor()
f = t.mypyfile(f)
a += str(f).split('\n')
except CompileError as e:
a = e.messages
if testcase.normalize_output:
a = normalize_error_messages(a)
assert_string_arrays_equal(
testcase.output, a,
'Invalid semantic analyzer output ({}, line {})'.format(testcase.file,
testcase.line))
def setUp(self) -> None:
self.package_dir = os.path.relpath(os.path.join(
package_path,
"modulefinder-site-packages",
))
egg_dirs, site_packages = expand_site_packages([self.package_dir])
self.search_paths = SearchPaths(
python_path=(),
mypy_path=(os.path.join(data_path, "pkg1"),),
package_path=tuple(egg_dirs + site_packages),
typeshed_path=(),
)
options = Options()
options.namespace_packages = True
self.fmc_ns = FindModuleCache(self.search_paths, fscache=None, options=options)
options = Options()
options.namespace_packages = False
self.fmc_nons = FindModuleCache(self.search_paths, fscache=None, options=options)
def __init__(self, options: Options,
timeout: Optional[int] = None,
alt_lib_path: Optional[str] = None) -> None:
"""Initialize the server with the desired mypy flags."""
self.options = options
self.timeout = timeout
self.alt_lib_path = alt_lib_path
self.fine_grained_manager = None # type: Optional[FineGrainedBuildManager]
if os.path.isfile(STATUS_FILE):
os.unlink(STATUS_FILE)
options.incremental = True
options.fine_grained_incremental = True
options.show_traceback = True
if options.use_fine_grained_cache:
options.cache_fine_grained = True # set this so that cache options match
else:
options.cache_dir = os.devnull
# Fine-grained incremental doesn't support general partial types
# (details in https://github.com/python/mypy/issues/4492)
options.local_partial_types = True
def setUp(self) -> None:
self.search_paths = SearchPaths(
python_path=(),
mypy_path=(
os.path.join(data_path, "nsx-pkg1"),
os.path.join(data_path, "nsx-pkg2"),
os.path.join(data_path, "nsx-pkg3"),
os.path.join(data_path, "nsy-pkg1"),
os.path.join(data_path, "nsy-pkg2"),
os.path.join(data_path, "pkg1"),
os.path.join(data_path, "pkg2"),
),
package_path=(),
typeshed_path=(),
)
options = Options()
options.namespace_packages = True
self.fmc_ns = FindModuleCache(self.search_paths, options=options)
options = Options()
options.namespace_packages = False
self.fmc_nons = FindModuleCache(self.search_paths, options=options)
def __init__(self, options: Options,
timeout: Optional[int] = None,
alt_lib_path: Optional[str] = None) -> None:
"""Initialize the server with the desired mypy flags."""
self.options = options
self.timeout = timeout
self.alt_lib_path = alt_lib_path
self.fine_grained_manager = None # type: Optional[FineGrainedBuildManager]
if os.path.isfile(STATUS_FILE):
os.unlink(STATUS_FILE)
options.incremental = True
options.fine_grained_incremental = True
options.show_traceback = True
if options.use_fine_grained_cache:
options.cache_fine_grained = True # set this so that cache options match
else:
options.cache_dir = os.devnull
# Fine-grained incremental doesn't support general partial types
# (details in https://github.com/python/mypy/issues/4492)
options.local_partial_types = True
def setUp(self) -> None:
self.package_dir = os.path.relpath(os.path.join(
package_path,
"modulefinder-site-packages",
))
self.search_paths = SearchPaths(
python_path=(),
mypy_path=(
os.path.join(data_path, "pkg1"),
),
package_path=(
self.package_dir,
),
typeshed_path=(),
)
options = Options()
options.namespace_packages = True
self.fmc_ns = FindModuleCache(self.search_paths, options=options)
options = Options()
options.namespace_packages = False
self.fmc_nons = FindModuleCache(self.search_paths, options=options)
def parse_options(program_text: str, testcase: DataDrivenTestCase,
incremental_step: int) -> Options:
"""Parse comments like '# flags: --foo' in a test case."""
options = Options()
flags = re.search('# flags: (.*)$', program_text, flags=re.MULTILINE)
if incremental_step > 1:
flags2 = re.search('# flags{}: (.*)$'.format(incremental_step), program_text,
flags=re.MULTILINE)
if flags2:
flags = flags2
flag_list = None
if flags:
flag_list = flags.group(1).split()
flag_list.append('--no-site-packages') # the tests shouldn't need an installed Python
targets, options = process_options(flag_list, require_targets=False)
if targets:
# TODO: support specifying targets via the flags pragma
raise RuntimeError('Specifying targets via the flags pragma is not supported.')
else:
options = Options()
# TODO: Enable strict optional in test cases by default (requires *many* test case changes)
options.strict_optional = False
# Allow custom python version to override testcase_pyversion
if (not flag_list or
all(flag not in flag_list for flag in ['--python-version', '-2', '--py2'])):
options.python_version = testcase_pyversion(testcase.file, testcase.name)
if testcase.config.getoption('--mypy-verbose'):
options.verbosity = testcase.config.getoption('--mypy-verbose')
if os.getenv('NEWSEMANAL'):
if not flag_list or '--no-new-semantic-analyzer' not in flag_list:
options.new_semantic_analyzer = True
return options
def get_semanal_options() -> Options:
options = Options()
options.use_builtins_fixtures = True
options.semantic_analysis_only = True
options.show_traceback = True
return options
def test_find_sources_exclude(self) -> None:
options = Options()
options.namespace_packages = True
# default
for excluded_dir in [
"site-packages", ".whatever", "node_modules", ".x/.z"
]:
fscache = FakeFSCache(
{"/dir/a.py", "/dir/venv/{}/b.py".format(excluded_dir)})
assert find_sources(["/"], options, fscache) == [("a", "/dir")]
with pytest.raises(InvalidSourceList):
find_sources(["/dir/venv/"], options, fscache)
assert find_sources(["/dir/venv/{}".format(excluded_dir)], options,
fscache) == [
("b", "/dir/venv/{}".format(excluded_dir))
]
assert find_sources(["/dir/venv/{}/b.py".format(excluded_dir)],
options, fscache) == [
("b", "/dir/venv/{}".format(excluded_dir))
]
files = {
"/pkg/a1/b/c/d/e.py",
"/pkg/a1/b/f.py",
"/pkg/a2/__init__.py",
"/pkg/a2/b/c/d/e.py",
"/pkg/a2/b/f.py",
}
# file name
options.exclude = r"/f\.py$"
fscache = FakeFSCache(files)
assert find_sources(["/"], options, fscache) == [
("a2", "/pkg"),
("a2.b.c.d.e", "/pkg"),
("e", "/pkg/a1/b/c/d"),
]
assert find_sources(["/pkg/a1/b/f.py"], options,
fscache) == [('f', '/pkg/a1/b')]
assert find_sources(["/pkg/a2/b/f.py"], options,
fscache) == [('a2.b.f', '/pkg')]
# directory name
options.exclude = "/a1/"
fscache = FakeFSCache(files)
assert find_sources(["/"], options, fscache) == [
("a2", "/pkg"),
("a2.b.c.d.e", "/pkg"),
("a2.b.f", "/pkg"),
]
with pytest.raises(InvalidSourceList):
find_sources(["/pkg/a1"], options, fscache)
with pytest.raises(InvalidSourceList):
find_sources(["/pkg/a1/"], options, fscache)
with pytest.raises(InvalidSourceList):
find_sources(["/pkg/a1/b"], options, fscache)
options.exclude = "/a1/$"
assert find_sources(["/pkg/a1"], options,
fscache) == [('e', '/pkg/a1/b/c/d'),
('f', '/pkg/a1/b')]
# paths
options.exclude = "/pkg/a1/"
fscache = FakeFSCache(files)
assert find_sources(["/"], options, fscache) == [
("a2", "/pkg"),
("a2.b.c.d.e", "/pkg"),
("a2.b.f", "/pkg"),
]
with pytest.raises(InvalidSourceList):
find_sources(["/pkg/a1"], options, fscache)
options.exclude = "/(a1|a3)/"
fscache = FakeFSCache(files)
assert find_sources(["/"], options, fscache) == [
("a2", "/pkg"),
("a2.b.c.d.e", "/pkg"),
("a2.b.f", "/pkg"),
]
options.exclude = "b/c/"
fscache = FakeFSCache(files)
assert find_sources(["/"], options, fscache) == [
("a2", "/pkg"),
("a2.b.f", "/pkg"),
("f", "/pkg/a1/b"),
]
# nothing should be ignored as a result of this
options.exclude = "|".join((
"/pkg/a/",
"/2",
"/1",
"/pk/",
"/kg",
"/g.py",
"/bc",
"/xxx/pkg/a2/b/f.py"
"xxx/pkg/a2/b/f.py",
))
fscache = FakeFSCache(files)
assert len(find_sources(["/"], options, fscache)) == len(files)
files = {
"pkg/a1/b/c/d/e.py",
"pkg/a1/b/f.py",
"pkg/a2/__init__.py",
"pkg/a2/b/c/d/e.py",
"pkg/a2/b/f.py",
}
fscache = FakeFSCache(files)
assert len(find_sources(["."], options, fscache)) == len(files)
def process_options(
args: List[str],
require_targets: bool = True) -> Tuple[List[BuildSource], Options]:
"""Parse command line arguments."""
# Make the help output a little less jarring.
help_factory = (lambda prog: argparse.RawDescriptionHelpFormatter(
prog=prog, max_help_position=28))
parser = argparse.ArgumentParser(prog='mypy',
epilog=FOOTER,
fromfile_prefix_chars='@',
formatter_class=help_factory)
# Unless otherwise specified, arguments will be parsed directly onto an
# Options object. Options that require further processing should have
# their `dest` prefixed with `special-opts:`, which will cause them to be
# parsed into the separate special_opts namespace object.
parser.add_argument('-v',
'--verbose',
action='count',
dest='verbosity',
help="more verbose messages")
parser.add_argument('-V',
'--version',
action='version',
version='%(prog)s ' + __version__)
parser.add_argument('--python-version',
type=parse_version,
metavar='x.y',
help='use Python x.y')
parser.add_argument(
'--platform',
action='store',
metavar='PLATFORM',
help="typecheck special-cased code for the given OS platform "
"(defaults to sys.platform).")
parser.add_argument('-2',
'--py2',
dest='python_version',
action='store_const',
const=defaults.PYTHON2_VERSION,
help="use Python 2 mode")
parser.add_argument('-s',
'--silent-imports',
action='store_true',
help="don't follow imports to .py files")
parser.add_argument(
'--almost-silent',
action='store_true',
help="like --silent-imports but reports the imports as errors")
parser.add_argument(
'--disallow-untyped-calls',
action='store_true',
help="disallow calling functions without type annotations"
" from functions with type annotations")
parser.add_argument(
'--disallow-untyped-defs',
action='store_true',
help="disallow defining functions without type annotations"
" or with incomplete type annotations")
parser.add_argument(
'--check-untyped-defs',
action='store_true',
help="type check the interior of functions without type annotations")
parser.add_argument(
'--disallow-subclassing-any',
action='store_true',
help="disallow subclassing values of type 'Any' when defining classes")
parser.add_argument(
'--warn-incomplete-stub',
action='store_true',
help="warn if missing type annotation in typeshed, only relevant with"
" --check-untyped-defs enabled")
parser.add_argument(
'--warn-redundant-casts',
action='store_true',
help="warn about casting an expression to its inferred type")
parser.add_argument('--warn-no-return',
action='store_true',
help="warn about functions that end without returning")
parser.add_argument('--warn-unused-ignores',
action='store_true',
help="warn about unneeded '# type: ignore' comments")
parser.add_argument('--hide-error-context',
action='store_true',
dest='hide_error_context',
help="Hide context notes before errors")
parser.add_argument('--fast-parser',
action='store_true',
help="enable experimental fast parser")
parser.add_argument('-i',
'--incremental',
action='store_true',
help="enable experimental module cache")
parser.add_argument(
'--cache-dir',
action='store',
metavar='DIR',
help="store module cache info in the given folder in incremental mode "
"(defaults to '{}')".format(defaults.CACHE_DIR))
parser.add_argument('--strict-optional',
action='store_true',
dest='strict_optional',
help="enable experimental strict Optional checks")
parser.add_argument(
'--strict-optional-whitelist',
metavar='GLOB',
nargs='*',
help="suppress strict Optional errors in all but the provided files "
"(experimental -- read documentation before using!). "
"Implies --strict-optional. Has the undesirable side-effect of "
"suppressing other errors in non-whitelisted files.")
parser.add_argument('--junit-xml',
help="write junit.xml to the given file")
parser.add_argument('--pdb',
action='store_true',
help="invoke pdb on fatal error")
parser.add_argument('--show-traceback',
'--tb',
action='store_true',
help="show traceback on fatal error")
parser.add_argument('--stats',
action='store_true',
dest='dump_type_stats',
help="dump stats")
parser.add_argument('--inferstats',
action='store_true',
dest='dump_inference_stats',
help="dump type inference stats")
parser.add_argument('--custom-typing',
metavar='MODULE',
dest='custom_typing_module',
help="use a custom typing module")
parser.add_argument('--custom-typeshed-dir',
metavar='DIR',
help="use the custom typeshed in DIR")
parser.add_argument('--scripts-are-modules',
action='store_true',
help="Script x becomes module x instead of __main__")
parser.add_argument('--config-file',
help="Configuration file, must have a [mypy] section "
"(defaults to {})".format(defaults.CONFIG_FILE))
parser.add_argument('--show-column-numbers',
action='store_true',
dest='show_column_numbers',
help="Show column numbers in error messages")
parser.add_argument(
'--find-occurrences',
metavar='CLASS.MEMBER',
dest='special-opts:find_occurrences',
help="print out all usages of a class member (experimental)")
# hidden options
# --shadow-file a.py tmp.py will typecheck tmp.py in place of a.py.
# Useful for tools to make transformations to a file to get more
# information from a mypy run without having to change the file in-place
# (e.g. by adding a call to reveal_type).
parser.add_argument('--shadow-file',
metavar='PATH',
nargs=2,
dest='shadow_file',
help=argparse.SUPPRESS)
# --debug-cache will disable any cache-related compressions/optimizations,
# which will make the cache writing process output pretty-printed JSON (which
# is easier to debug).
parser.add_argument('--debug-cache',
action='store_true',
help=argparse.SUPPRESS)
# deprecated options
parser.add_argument('--silent',
action='store_true',
dest='special-opts:silent',
help=argparse.SUPPRESS)
parser.add_argument('-f',
'--dirty-stubs',
action='store_true',
dest='special-opts:dirty_stubs',
help=argparse.SUPPRESS)
parser.add_argument('--use-python-path',
action='store_true',
dest='special-opts:use_python_path',
help=argparse.SUPPRESS)
report_group = parser.add_argument_group(
title='report generation',
description='Generate a report in the specified format.')
for report_type in sorted(reporter_classes):
report_group.add_argument('--%s-report' %
report_type.replace('_', '-'),
metavar='DIR',
dest='special-opts:%s_report' % report_type)
code_group = parser.add_argument_group(
title='How to specify the code to type check')
code_group.add_argument(
'-m',
'--module',
action='append',
metavar='MODULE',
dest='special-opts:modules',
help="type-check module; can repeat for more modules")
# TODO: `mypy -p A -p B` currently silently ignores ignores A
# (last option wins). Perhaps -c, -m and -p could just be
# command-line flags that modify how we interpret self.files?
code_group.add_argument('-c',
'--command',
action='append',
metavar='PROGRAM_TEXT',
dest='special-opts:command',
help="type-check program passed in as string")
code_group.add_argument('-p',
'--package',
metavar='PACKAGE',
dest='special-opts:package',
help="type-check all files in a directory")
code_group.add_argument(metavar='files',
nargs='*',
dest='special-opts:files',
help="type-check given files or directories")
# Parse arguments once into a dummy namespace so we can get the
# filename for the config file.
dummy = argparse.Namespace()
parser.parse_args(args, dummy)
config_file = dummy.config_file or defaults.CONFIG_FILE
# Parse config file first, so command line can override.
options = Options()
if config_file and os.path.exists(config_file):
parse_config_file(options, config_file)
# Parse command line for real, using a split namespace.
special_opts = argparse.Namespace()
parser.parse_args(args,
SplitNamespace(options, special_opts, 'special-opts:'))
# --use-python-path is no longer supported; explain why.
if special_opts.use_python_path:
parser.error(
"Sorry, --use-python-path is no longer supported.\n"
"If you are trying this because your code depends on a library module,\n"
"you should really investigate how to obtain stubs for that module.\n"
"See https://github.com/python/mypy/issues/1411 for more discussion."
)
# warn about deprecated options
if special_opts.silent:
print("Warning: --silent is deprecated; use --silent-imports",
file=sys.stderr)
options.silent_imports = True
if special_opts.dirty_stubs:
print(
"Warning: -f/--dirty-stubs is deprecated and no longer necessary. Mypy no longer "
"checks the git status of stubs.",
file=sys.stderr)
# Check for invalid argument combinations.
if require_targets:
code_methods = sum(
bool(c) for c in [
special_opts.modules, special_opts.command,
special_opts.package, special_opts.files
])
if code_methods == 0:
parser.error("Missing target module, package, files, or command.")
elif code_methods > 1:
parser.error(
"May only specify one of: module, package, files, or command.")
# Set build flags.
if options.strict_optional_whitelist is not None:
# TODO: Deprecate, then kill this flag
options.strict_optional = True
if options.strict_optional:
experiments.STRICT_OPTIONAL = True
if special_opts.find_occurrences:
experiments.find_occurrences = special_opts.find_occurrences.split('.')
if len(experiments.find_occurrences) < 2:
parser.error("Can only find occurrences of class members.")
if len(experiments.find_occurrences) != 2:
parser.error(
"Can only find occurrences of non-nested class members.")
# Set reports.
for flag, val in vars(special_opts).items():
if flag.endswith('_report') and val is not None:
report_type = flag[:-7].replace('_', '-')
report_dir = val
options.report_dirs[report_type] = report_dir
# Set target.
if special_opts.modules:
options.build_type = BuildType.MODULE
targets = [BuildSource(None, m, None) for m in special_opts.modules]
return targets, options
elif special_opts.package:
if os.sep in special_opts.package or os.altsep and os.altsep in special_opts.package:
fail("Package name '{}' cannot have a slash in it.".format(
special_opts.package))
options.build_type = BuildType.MODULE
lib_path = [os.getcwd()] + build.mypy_path()
targets = build.find_modules_recursive(special_opts.package, lib_path)
if not targets:
fail("Can't find package '{}'".format(special_opts.package))
return targets, options
elif special_opts.command:
options.build_type = BuildType.PROGRAM_TEXT
return [BuildSource(None, None,
'\n'.join(special_opts.command))], options
else:
targets = []
for f in special_opts.files:
if f.endswith(PY_EXTENSIONS):
targets.append(BuildSource(f, crawl_up(f)[1], None))
elif os.path.isdir(f):
sub_targets = expand_dir(f)
if not sub_targets:
fail("There are no .py[i] files in directory '{}'".format(
f))
targets.extend(sub_targets)
else:
mod = os.path.basename(
f) if options.scripts_are_modules else None
targets.append(BuildSource(f, mod, None))
return targets, options
def main(argv):
# TODO: Put these in the shell script
mypy_options = [
'--py2', '--strict', '--no-implicit-optional', '--no-strict-optional',
# for consistency?
'--follow-imports=silent',
#'--verbose',
]
o = Options()
opts, argv = o.parse_args(argv)
paths = argv[1:] # e.g. asdl/typed_arith_parse.py
if 0:
print(opts)
print(paths)
return
# e.g. asdl/typed_arith_parse.py -> 'typed_arith_parse'
mod_names = [os.path.basename(p) for p in paths]
mod_names = [os.path.splitext(name)[0] for name in mod_names]
# Ditto
to_header = opts.to_header
#if to_header:
if 0:
to_header = [os.path.basename(p) for p in to_header]
to_header = [os.path.splitext(name)[0] for name in to_header]
#log('to_header %s', to_header)
sources, options = get_mypy_config(paths, mypy_options)
if 0:
for source in sources:
log('source %s', source)
#log('options %s', options)
#result = emitmodule.parse_and_typecheck(sources, options)
import time
start_time = time.time()
result = mypy_build(sources=sources, options=options)
#log('elapsed 1: %f', time.time() - start_time)
if result.errors:
log('')
log('-'* 80)
for e in result.errors:
log(e)
log('-'* 80)
log('')
# Important functions in mypyc/build.py:
#
# generate_c (251 lines)
# parse_and_typecheck
# compile_modules_to_c
# mypyc/emitmodule.py (487 lines)
# def compile_modules_to_c(result: BuildResult, module_names: List[str],
# class ModuleGenerator:
# # This generates a whole bunch of textual code!
# literals, modules, errors = genops.build_ir(file_nodes, result.graph,
# result.types)
# TODO: Debug what comes out of here.
#build.dump_graph(result.graph)
#return
# no-op
for name in result.graph:
state = result.graph[name]
# GLOBAL Constant pass over all modules. We want to collect duplicate
# strings together. And have globally unique IDs str0, str1, ... strN.
const_lookup = {}
const_code = []
pass1 = const_pass.Collect(result.types, const_lookup, const_code)
to_compile = list(ModulesToCompile(result, mod_names))
# HACK: Why do I get oil.asdl.tdop in addition to asdl.tdop?
names = set(name for name, _ in to_compile)
filtered = []
seen = set()
for name, module in to_compile:
if name.startswith('oil.'):
name = name[4:]
if name not in seen: # remove dupe
filtered.append((name, module))
seen.add(name)
to_compile = filtered
import pickle
if 1:
for name, module in to_compile:
log('to_compile %s', name)
# can't pickle but now I see deserialize() nodes and stuff
#s = pickle.dumps(module)
#log('%d pickle', len(s))
# Print the tree for debugging
if 0:
for name, module in to_compile:
builder = debug_pass.Print(result.types)
builder.visit_mypy_file(module)
return
f = sys.stdout
gc = bool(os.getenv('GC'))
header_name = 'gc_heap' if gc else 'mylib'
#header_name = 'mylib'
# TODO: Add --cc-out? But there is a preamble and postamble.
f.write("""\
// BEGIN mycpp output
#include "%s.h"
using gc_heap::Alloc;
using gc_heap::kZeroMask;
using gc_heap::StackRoots;
""" % header_name)
if gc:
f.write("""\
#include "my_runtime.h"
#include "mylib2.h"
using gc_heap::NewStr;
using gc_heap::NewList;
using gc_heap::NewDict;
""")
if to_header:
f.write('#include "%s"\n' % os.path.basename(opts.header_out))
f.write('\n')
# Collect constants and then emit code.
for name, module in to_compile:
pass1.visit_mypy_file(module)
# Instead of top-level code, should we generate a function and call it from
# main?
for line in const_code:
f.write('%s\n' % line)
f.write('\n')
# Note: doesn't take into account module names!
virtual = pass_state.Virtual()
if opts.header_out:
header_f = open(opts.header_out, 'w') # Not closed
guard = 'RUNTIME_H'
header_f.write("""\
// %s: translated from Python by mycpp
#ifndef %s
#define %s
#include "%s.h"
""" % (os.path.basename(opts.header_out), guard, guard, header_name))
log('\tFORWARD DECL')
# Forward declarations first.
# class Foo; class Bar;
for name, module in to_compile:
log('forward decl name %s', name)
if name in to_header:
out_f = header_f
else:
out_f = f
p2 = cppgen_pass.Generate(result.types, const_lookup, out_f,
virtual=virtual, forward_decl=True)
p2.visit_mypy_file(module)
# After seeing class and method names in the first pass, figure out which
# ones are virtual. We use this info in the second pass.
virtual.Calculate()
#log('V %s', virtual.virtuals)
local_vars = {} # FuncDef node -> (name, c_type) list
# Node -> fmt_name, plus a hack for the counter
# TODO: This could be a class with 2 members
fmt_ids = {'_counter': 0}
log('\tDECL')
# First generate ALL C++ declarations / "headers".
# class Foo { void method(); }; class Bar { void method(); };
for name, module in to_compile:
log('decl name %s', name)
if name in to_header:
out_f = header_f
else:
out_f = f
p3 = cppgen_pass.Generate(result.types, const_lookup, out_f,
local_vars=local_vars, fmt_ids=fmt_ids,
virtual=virtual, decl=True)
p3.visit_mypy_file(module)
if opts.header_out:
header_f.write("""\
#endif // %s
""" % guard)
log('\tDEFINITION')
# Now the definitions / implementations.
# void Foo:method() { ... }
# void Bar:method() { ... }
for name, module in to_compile:
p4 = cppgen_pass.Generate(result.types, const_lookup, f,
local_vars=local_vars, fmt_ids=fmt_ids)
p4.visit_mypy_file(module)
def parse_config_file(options: Options, filename: Optional[str]) -> None:
"""Parse a config file into an Options object.
Errors are written to stderr but are not fatal.
If filename is None, fall back to default config files.
"""
if filename is not None:
config_files = (filename,) # type: Tuple[str, ...]
else:
config_files = tuple(map(os.path.expanduser, defaults.CONFIG_FILES))
parser = configparser.RawConfigParser()
for config_file in config_files:
if not os.path.exists(config_file):
continue
try:
parser.read(config_file)
except configparser.Error as err:
print("%s: %s" % (config_file, err), file=sys.stderr)
else:
file_read = config_file
options.config_file = file_read
break
else:
return
if 'mypy' not in parser:
if filename or file_read not in defaults.SHARED_CONFIG_FILES:
print("%s: No [mypy] section in config file" % file_read, file=sys.stderr)
else:
section = parser['mypy']
prefix = '%s: [%s]' % (file_read, 'mypy')
updates, report_dirs = parse_section(prefix, options, section)
for k, v in updates.items():
setattr(options, k, v)
options.report_dirs.update(report_dirs)
for name, section in parser.items():
if name.startswith('mypy-'):
prefix = '%s: [%s]' % (file_read, name)
updates, report_dirs = parse_section(prefix, options, section)
if report_dirs:
print("%s: Per-module sections should not specify reports (%s)" %
(prefix, ', '.join(s + '_report' for s in sorted(report_dirs))),
file=sys.stderr)
if set(updates) - PER_MODULE_OPTIONS:
print("%s: Per-module sections should only specify per-module flags (%s)" %
(prefix, ', '.join(sorted(set(updates) - PER_MODULE_OPTIONS))),
file=sys.stderr)
updates = {k: v for k, v in updates.items() if k in PER_MODULE_OPTIONS}
globs = name[5:]
for glob in globs.split(','):
# For backwards compatibility, replace (back)slashes with dots.
glob = glob.replace(os.sep, '.')
if os.altsep:
glob = glob.replace(os.altsep, '.')
if (any(c in glob for c in '?[]!') or
any('*' in x and x != '*' for x in glob.split('.'))):
print("%s: Patterns must be fully-qualified module names, optionally "
"with '*' in some components (e.g spam.*.eggs.*)"
% prefix,
file=sys.stderr)
else:
options.per_module_options[glob] = updates
def start_server_and_analyze(config, workspace, python_executable=None):
if settings is None:
log.error('Settings is None')
return
log.info(f'mypy version: {mypy_version}')
log.info(f'mypyls version: {mypyls_version}')
options = Options()
options.check_untyped_defs = True
if mypy_version < '0.780':
# Before mypy 0.780, follow_imports must be either 'error' or 'skip' in the daemon.
# Starting in 0.780, the daemon supports the default follow_imports=normal.
options.follow_imports = 'error'
options.use_fine_grained_cache = True
if python_executable is not None:
options.python_executable = python_executable
stderr_stream = StringIO()
config_file = settings.get('configFile')
if config_file == '':
# Use empty string rather than null in vscode settings, so that it's shown in settings editor GUI.
config_file = None
log.info(
f'Trying to read mypy config file from {config_file or "default locations"}'
)
with redirect_stderr(stderr_stream):
if mypy_version >= '0.770':
def set_strict_flags():
# The code to set all strict options using the 'strict' flag is in mypy.main.process_options,
# and we cannot access it from here, so we disable `strict = True` in the config file for now.
stderr_stream.write(
"Setting 'strict' in the configuration file is not supported by mypy-vscode for now. "
"The option will be ignored. You may set individual strict flags instead "
"(see 'mypy -h' for the list of flags enabled in strict mode)."
)
parse_config_file(options, set_strict_flags, config_file)
else:
parse_config_file(options, config_file)
stderr = stderr_stream.getvalue()
if stderr:
log.error(f'Error reading mypy config file:\n{stderr}')
workspace.show_message(f'Error reading mypy config file:\n{stderr}')
if options.config_file:
log.info(f'Read mypy config from: {options.config_file}')
else:
log.info(f'Mypy configuration not read, using defaults.')
if config_file:
workspace.show_message(
f'Mypy config file not found:\n{config_file}')
options.show_column_numbers = True
if mypy_version < '0.780' and options.follow_imports not in ('error',
'skip'):
workspace.show_message(
f"Cannot use follow_imports='{options.follow_imports}', using 'error' instead."
)
options.follow_imports = 'error'
if mypy_version > '0.720':
options.color_output = False
options.error_summary = False
options.pretty = False
log.info(
f'python_executable after applying config: {options.python_executable}'
)
workspace.mypy_server = Server(options, DEFAULT_STATUS_FILE)
mypy_check(workspace, config)
def run_case(self, testcase: DataDrivenTestCase) -> None:
bench = testcase.config.getoption('--bench', False) and 'Benchmark' in testcase.name
# setup.py wants to be run from the root directory of the package, which we accommodate
# by chdiring into tmp/
with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase), (
chdir_manager('tmp')):
text = '\n'.join(testcase.input)
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.strict_optional = True
options.python_version = (3, 6)
options.export_types = True
workdir = 'build'
os.mkdir(workdir)
source_path = 'native.py'
with open(source_path, 'w') as f:
f.write(text)
with open('interpreted.py', 'w') as f:
f.write(text)
source = build.BuildSource(source_path, 'native', text)
sources = [source]
module_names = ['native']
module_paths = [os.path.abspath('native.py')]
# Hard code another module name to compile in the same compilation unit.
to_delete = []
for fn, text in testcase.files:
fn = os.path.relpath(fn, test_temp_dir)
if os.path.basename(fn).startswith('other'):
name = os.path.basename(fn).split('.')[0]
module_names.append(name)
sources.append(build.BuildSource(fn, name, text))
to_delete.append(fn)
module_paths.append(os.path.abspath(fn))
for source in sources:
options.per_module_options.setdefault(source.module, {})['mypyc'] = True
try:
result = emitmodule.parse_and_typecheck(
sources=sources,
options=options,
alt_lib_path='.')
ctext = emitmodule.compile_modules_to_c(
result,
module_names=module_names,
use_shared_lib=len(module_names) > 1)
except CompileError as e:
for line in e.messages:
print(line)
assert False, 'Compile error'
cpath = os.path.abspath(os.path.join(workdir, '__native.c'))
with open(cpath, 'w') as f:
f.write(ctext)
setup_file = os.path.abspath(os.path.join(workdir, 'setup.py'))
with open(setup_file, 'w') as f:
f.write(setup_format.format(module_paths))
run_setup(setup_file, ['build_ext', '--inplace'])
# Oh argh run_setup doesn't propagate failure. For now we'll just assert
# that the file is there.
if not glob.glob('native.*.so'):
show_c(ctext)
assert False, "Compilation failed"
for p in to_delete:
os.remove(p)
driver_path = 'driver.py'
env = os.environ.copy()
env['MYPYC_RUN_BENCH'] = '1' if bench else '0'
# XXX: This is an ugly hack.
if 'MYPYC_RUN_GDB' in os.environ:
if platform.system() == 'Darwin':
subprocess.check_call(['lldb', '--', 'python', driver_path], env=env)
assert False, ("Test can't pass in lldb mode. (And remember to pass -s to "
"pytest)")
elif platform.system() == 'Linux':
subprocess.check_call(['gdb', '--args', 'python', driver_path], env=env)
assert False, ("Test can't pass in gdb mode. (And remember to pass -s to "
"pytest)")
else:
assert False, 'Unsupported OS'
proc = subprocess.Popen(['python', driver_path], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, env=env)
output, _ = proc.communicate()
output = output.decode('utf8')
outlines = output.splitlines()
show_c(ctext)
if proc.returncode != 0:
print()
print('*** Exit status: %d' % proc.returncode)
# Verify output.
if bench:
print('Test output:')
print(output)
else:
assert_test_output(testcase, outlines, 'Invalid output')
assert proc.returncode == 0
def test_coherence(self) -> None:
options = Options()
_, parsed_options = process_options([], require_targets=False)
# FIX: test this too. Requires changing working dir to avoid finding 'setup.cfg'
options.config_file = parsed_options.config_file
assert_equal(options.snapshot(), parsed_options.snapshot())
def process_options(args: List[str],
require_targets: bool = True
) -> Tuple[List[BuildSource], Options]:
"""Parse command line arguments."""
# Make the help output a little less jarring.
help_factory = (lambda prog:
argparse.RawDescriptionHelpFormatter(prog=prog,
max_help_position=28)) # type: Any
parser = argparse.ArgumentParser(prog='mypy', epilog=FOOTER,
fromfile_prefix_chars='@',
formatter_class=help_factory)
# Unless otherwise specified, arguments will be parsed directly onto an
# Options object. Options that require further processing should have
# their `dest` prefixed with `special-opts:`, which will cause them to be
# parsed into the separate special_opts namespace object.
parser.add_argument('-v', '--verbose', action='count', dest='verbosity',
help="more verbose messages")
parser.add_argument('-V', '--version', action='version',
version='%(prog)s ' + __version__)
parser.add_argument('--python-version', type=parse_version, metavar='x.y',
help='use Python x.y')
parser.add_argument('--platform', action='store', metavar='PLATFORM',
help="typecheck special-cased code for the given OS platform "
"(defaults to sys.platform).")
parser.add_argument('-2', '--py2', dest='python_version', action='store_const',
const=defaults.PYTHON2_VERSION, help="use Python 2 mode")
parser.add_argument('--ignore-missing-imports', action='store_true',
help="silently ignore imports of missing modules")
parser.add_argument('--follow-imports', choices=['normal', 'silent', 'skip', 'error'],
default='normal', help="how to treat imports (default normal)")
parser.add_argument('--disallow-untyped-calls', action='store_true',
help="disallow calling functions without type annotations"
" from functions with type annotations")
parser.add_argument('--disallow-untyped-defs', action='store_true',
help="disallow defining functions without type annotations"
" or with incomplete type annotations")
parser.add_argument('--check-untyped-defs', action='store_true',
help="type check the interior of functions without type annotations")
parser.add_argument('--disallow-subclassing-any', action='store_true',
help="disallow subclassing values of type 'Any' when defining classes")
parser.add_argument('--warn-incomplete-stub', action='store_true',
help="warn if missing type annotation in typeshed, only relevant with"
" --check-untyped-defs enabled")
parser.add_argument('--warn-redundant-casts', action='store_true',
help="warn about casting an expression to its inferred type")
parser.add_argument('--warn-no-return', action='store_true',
help="warn about functions that end without returning")
parser.add_argument('--warn-unused-ignores', action='store_true',
help="warn about unneeded '# type: ignore' comments")
parser.add_argument('--show-error-context', action='store_false',
dest='hide_error_context',
help='Precede errors with "note:" messages explaining context')
parser.add_argument('--fast-parser', action='store_true',
help="enable fast parser (recommended except on Windows)")
parser.add_argument('-i', '--incremental', action='store_true',
help="enable experimental module cache")
parser.add_argument('--cache-dir', action='store', metavar='DIR',
help="store module cache info in the given folder in incremental mode "
"(defaults to '{}')".format(defaults.CACHE_DIR))
parser.add_argument('--strict-optional', action='store_true',
dest='strict_optional',
help="enable experimental strict Optional checks")
parser.add_argument('--strict-optional-whitelist', metavar='GLOB', nargs='*',
help="suppress strict Optional errors in all but the provided files "
"(experimental -- read documentation before using!). "
"Implies --strict-optional. Has the undesirable side-effect of "
"suppressing other errors in non-whitelisted files.")
parser.add_argument('--junit-xml', help="write junit.xml to the given file")
parser.add_argument('--pdb', action='store_true', help="invoke pdb on fatal error")
parser.add_argument('--show-traceback', '--tb', action='store_true',
help="show traceback on fatal error")
parser.add_argument('--stats', action='store_true', dest='dump_type_stats', help="dump stats")
parser.add_argument('--inferstats', action='store_true', dest='dump_inference_stats',
help="dump type inference stats")
parser.add_argument('--custom-typing', metavar='MODULE', dest='custom_typing_module',
help="use a custom typing module")
parser.add_argument('--custom-typeshed-dir', metavar='DIR',
help="use the custom typeshed in DIR")
parser.add_argument('--scripts-are-modules', action='store_true',
help="Script x becomes module x instead of __main__")
parser.add_argument('--config-file',
help="Configuration file, must have a [mypy] section "
"(defaults to {})".format(defaults.CONFIG_FILE))
parser.add_argument('--show-column-numbers', action='store_true',
dest='show_column_numbers',
help="Show column numbers in error messages")
parser.add_argument('--find-occurrences', metavar='CLASS.MEMBER',
dest='special-opts:find_occurrences',
help="print out all usages of a class member (experimental)")
# hidden options
# --shadow-file a.py tmp.py will typecheck tmp.py in place of a.py.
# Useful for tools to make transformations to a file to get more
# information from a mypy run without having to change the file in-place
# (e.g. by adding a call to reveal_type).
parser.add_argument('--shadow-file', metavar='PATH', nargs=2, dest='shadow_file',
help=argparse.SUPPRESS)
# --debug-cache will disable any cache-related compressions/optimizations,
# which will make the cache writing process output pretty-printed JSON (which
# is easier to debug).
parser.add_argument('--debug-cache', action='store_true', help=argparse.SUPPRESS)
# --dump-graph will dump the contents of the graph of SCCs and exit.
parser.add_argument('--dump-graph', action='store_true', help=argparse.SUPPRESS)
parser.add_argument('--hide-error-context', action='store_true',
dest='hide_error_context',
help=argparse.SUPPRESS)
# deprecated options
parser.add_argument('-f', '--dirty-stubs', action='store_true',
dest='special-opts:dirty_stubs',
help=argparse.SUPPRESS)
parser.add_argument('--use-python-path', action='store_true',
dest='special-opts:use_python_path',
help=argparse.SUPPRESS)
parser.add_argument('-s', '--silent-imports', action='store_true',
dest='special-opts:silent_imports',
help=argparse.SUPPRESS)
parser.add_argument('--almost-silent', action='store_true',
dest='special-opts:almost_silent',
help=argparse.SUPPRESS)
report_group = parser.add_argument_group(
title='report generation',
description='Generate a report in the specified format.')
for report_type in sorted(reporter_classes):
report_group.add_argument('--%s-report' % report_type.replace('_', '-'),
metavar='DIR',
dest='special-opts:%s_report' % report_type)
code_group = parser.add_argument_group(title='How to specify the code to type check')
code_group.add_argument('-m', '--module', action='append', metavar='MODULE',
dest='special-opts:modules',
help="type-check module; can repeat for more modules")
# TODO: `mypy -p A -p B` currently silently ignores ignores A
# (last option wins). Perhaps -c, -m and -p could just be
# command-line flags that modify how we interpret self.files?
code_group.add_argument('-c', '--command', action='append', metavar='PROGRAM_TEXT',
dest='special-opts:command',
help="type-check program passed in as string")
code_group.add_argument('-p', '--package', metavar='PACKAGE', dest='special-opts:package',
help="type-check all files in a directory")
code_group.add_argument(metavar='files', nargs='*', dest='special-opts:files',
help="type-check given files or directories")
# Parse arguments once into a dummy namespace so we can get the
# filename for the config file.
dummy = argparse.Namespace()
parser.parse_args(args, dummy)
config_file = defaults.CONFIG_FILE
if dummy.config_file:
config_file = dummy.config_file
if not os.path.exists(config_file):
parser.error("Cannot file config file '%s'" % config_file)
# Parse config file first, so command line can override.
options = Options()
if config_file and os.path.exists(config_file):
parse_config_file(options, config_file)
# Parse command line for real, using a split namespace.
special_opts = argparse.Namespace()
parser.parse_args(args, SplitNamespace(options, special_opts, 'special-opts:'))
# --use-python-path is no longer supported; explain why.
if special_opts.use_python_path:
parser.error("Sorry, --use-python-path is no longer supported.\n"
"If you are trying this because your code depends on a library module,\n"
"you should really investigate how to obtain stubs for that module.\n"
"See https://github.com/python/mypy/issues/1411 for more discussion."
)
# Process deprecated options
if special_opts.almost_silent:
print("Warning: --almost-silent has been replaced by "
"--follow-imports=errors", file=sys.stderr)
if options.follow_imports == 'normal':
options.follow_imports = 'errors'
elif special_opts.silent_imports:
print("Warning: --silent-imports has been replaced by "
"--ignore-missing-imports --follow-imports=skip", file=sys.stderr)
options.ignore_missing_imports = True
if options.follow_imports == 'normal':
options.follow_imports = 'skip'
if special_opts.dirty_stubs:
print("Warning: -f/--dirty-stubs is deprecated and no longer necessary. Mypy no longer "
"checks the git status of stubs.",
file=sys.stderr)
# Check for invalid argument combinations.
if require_targets:
code_methods = sum(bool(c) for c in [special_opts.modules,
special_opts.command,
special_opts.package,
special_opts.files])
if code_methods == 0:
parser.error("Missing target module, package, files, or command.")
elif code_methods > 1:
parser.error("May only specify one of: module, package, files, or command.")
# Set build flags.
if options.strict_optional_whitelist is not None:
# TODO: Deprecate, then kill this flag
options.strict_optional = True
if options.strict_optional:
experiments.STRICT_OPTIONAL = True
if special_opts.find_occurrences:
experiments.find_occurrences = special_opts.find_occurrences.split('.')
if len(experiments.find_occurrences) < 2:
parser.error("Can only find occurrences of class members.")
if len(experiments.find_occurrences) != 2:
parser.error("Can only find occurrences of non-nested class members.")
# Set reports.
for flag, val in vars(special_opts).items():
if flag.endswith('_report') and val is not None:
report_type = flag[:-7].replace('_', '-')
report_dir = val
options.report_dirs[report_type] = report_dir
# Set target.
if special_opts.modules:
options.build_type = BuildType.MODULE
targets = [BuildSource(None, m, None) for m in special_opts.modules]
return targets, options
elif special_opts.package:
if os.sep in special_opts.package or os.altsep and os.altsep in special_opts.package:
fail("Package name '{}' cannot have a slash in it."
.format(special_opts.package))
options.build_type = BuildType.MODULE
lib_path = [os.getcwd()] + build.mypy_path()
targets = build.find_modules_recursive(special_opts.package, lib_path)
if not targets:
fail("Can't find package '{}'".format(special_opts.package))
return targets, options
elif special_opts.command:
options.build_type = BuildType.PROGRAM_TEXT
return [BuildSource(None, None, '\n'.join(special_opts.command))], options
else:
targets = []
for f in special_opts.files:
if f.endswith(PY_EXTENSIONS):
targets.append(BuildSource(f, crawl_up(f)[1], None))
elif os.path.isdir(f):
sub_targets = expand_dir(f)
if not sub_targets:
fail("There are no .py[i] files in directory '{}'"
.format(f))
targets.extend(sub_targets)
else:
mod = os.path.basename(f) if options.scripts_are_modules else None
targets.append(BuildSource(f, mod, None))
return targets, options
def test_stubs(args: argparse.Namespace,
use_builtins_fixtures: bool = False) -> int:
"""This is stubtest! It's time to test the stubs!"""
# Load the allowlist. This is a series of strings corresponding to Error.object_desc
# Values in the dict will store whether we used the allowlist entry or not.
allowlist = {
entry: False
for allowlist_file in args.allowlist
for entry in get_allowlist_entries(allowlist_file)
}
allowlist_regexes = {entry: re.compile(entry) for entry in allowlist}
# If we need to generate an allowlist, we store Error.object_desc for each error here.
generated_allowlist = set()
modules = args.modules
if args.check_typeshed:
assert not args.modules, "Cannot pass both --check-typeshed and a list of modules"
modules = get_typeshed_stdlib_modules(args.custom_typeshed_dir)
annoying_modules = {"antigravity", "this"}
modules = [m for m in modules if m not in annoying_modules]
assert modules, "No modules to check"
options = Options()
options.incremental = False
options.custom_typeshed_dir = args.custom_typeshed_dir
options.config_file = args.mypy_config_file
options.use_builtins_fixtures = use_builtins_fixtures
if options.config_file:
def set_strict_flags() -> None: # not needed yet
return
parse_config_file(options, set_strict_flags, options.config_file,
sys.stdout, sys.stderr)
try:
modules = build_stubs(modules,
options,
find_submodules=not args.check_typeshed)
except RuntimeError:
return 1
exit_code = 0
for module in modules:
for error in test_module(module):
# Filter errors
if args.ignore_missing_stub and error.is_missing_stub():
continue
if args.ignore_positional_only and error.is_positional_only_related(
):
continue
if error.object_desc in allowlist:
allowlist[error.object_desc] = True
continue
is_allowlisted = False
for w in allowlist:
if allowlist_regexes[w].fullmatch(error.object_desc):
allowlist[w] = True
is_allowlisted = True
break
if is_allowlisted:
continue
# We have errors, so change exit code, and output whatever necessary
exit_code = 1
if args.generate_allowlist:
generated_allowlist.add(error.object_desc)
continue
print(error.get_description(concise=args.concise))
# Print unused allowlist entries
if not args.ignore_unused_allowlist:
for w in allowlist:
# Don't consider an entry unused if it regex-matches the empty string
# This lets us allowlist errors that don't manifest at all on some systems
if not allowlist[w] and not allowlist_regexes[w].fullmatch(""):
exit_code = 1
print("note: unused allowlist entry {}".format(w))
# Print the generated allowlist
if args.generate_allowlist:
for e in sorted(generated_allowlist):
print(e)
exit_code = 0
return exit_code
def run_case_step(self, testcase: DataDrivenTestCase,
incremental_step: int) -> None:
bench = testcase.config.getoption(
'--bench', False) and 'Benchmark' in testcase.name
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.strict_optional = True
# N.B: We try to (and ought to!) run with the current
# version of python, since we are going to link and run
# against the current version of python.
# But a lot of the tests use type annotations so we can't say it is 3.5.
options.python_version = max(sys.version_info[:2], (3, 6))
options.export_types = True
options.preserve_asts = True
options.incremental = False
# Avoid checking modules/packages named 'unchecked', to provide a way
# to test interacting with code we don't have types for.
options.per_module_options['unchecked.*'] = {'follow_imports': 'error'}
source = build.BuildSource('native.py', 'native', None)
sources = [source]
module_names = ['native']
module_paths = ['native.py']
# Hard code another module name to compile in the same compilation unit.
to_delete = []
for fn, text in testcase.files:
fn = os.path.relpath(fn, test_temp_dir)
if os.path.basename(fn).startswith('other') and fn.endswith('.py'):
name = os.path.basename(fn).split('.')[0]
module_names.append(name)
sources.append(build.BuildSource(fn, name, None))
to_delete.append(fn)
module_paths.append(fn)
shutil.copyfile(
fn,
os.path.join(os.path.dirname(fn),
name + '_interpreted.py'))
for source in sources:
options.per_module_options.setdefault(source.module,
{})['mypyc'] = True
separate = (self.get_separate('\n'.join(
testcase.input), incremental_step) if self.separate else False)
groups = construct_groups(sources, separate, len(module_names) > 1)
try:
result = emitmodule.parse_and_typecheck(sources=sources,
options=options,
alt_lib_path='.')
errors = Errors()
compiler_options = CompilerOptions(multi_file=self.multi_file,
separate=self.separate)
ir, cfiles = emitmodule.compile_modules_to_c(
result,
compiler_options=compiler_options,
errors=errors,
groups=groups,
)
if errors.num_errors:
errors.flush_errors()
assert False, "Compile error"
except CompileError as e:
for line in e.messages:
print(line)
assert False, 'Compile error'
# Check that serialization works on this IR
check_serialization_roundtrip(ir)
setup_file = os.path.abspath(os.path.join(WORKDIR, 'setup.py'))
# We pass the C file information to the build script via setup.py unfortunately
with open(setup_file, 'w', encoding='utf-8') as f:
f.write(
setup_format.format(module_paths, separate, cfiles,
self.multi_file))
if not run_setup(setup_file, ['build_ext', '--inplace']):
if testcase.config.getoption('--mypyc-showc'):
show_c(cfiles)
assert False, "Compilation failed"
# Assert that an output file got created
suffix = 'pyd' if sys.platform == 'win32' else 'so'
assert glob.glob('native.*.{}'.format(suffix))
driver_path = 'driver.py'
env = os.environ.copy()
env['MYPYC_RUN_BENCH'] = '1' if bench else '0'
# XXX: This is an ugly hack.
if 'MYPYC_RUN_GDB' in os.environ:
if platform.system() == 'Darwin':
subprocess.check_call(
['lldb', '--', sys.executable, driver_path], env=env)
assert False, (
"Test can't pass in lldb mode. (And remember to pass -s to "
"pytest)")
elif platform.system() == 'Linux':
subprocess.check_call(
['gdb', '--args', sys.executable, driver_path], env=env)
assert False, (
"Test can't pass in gdb mode. (And remember to pass -s to "
"pytest)")
else:
assert False, 'Unsupported OS'
proc = subprocess.Popen([sys.executable, driver_path],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
output = proc.communicate()[0].decode('utf8')
outlines = output.splitlines()
if testcase.config.getoption('--mypyc-showc'):
show_c(cfiles)
if proc.returncode != 0:
print()
print('*** Exit status: %d' % proc.returncode)
# Verify output.
if bench:
print('Test output:')
print(output)
else:
if incremental_step == 1:
msg = 'Invalid output'
expected = testcase.output
else:
msg = 'Invalid output (step {})'.format(incremental_step)
expected = testcase.output2.get(incremental_step, [])
assert_test_output(testcase, outlines, msg, expected)
if incremental_step > 1 and options.incremental:
suffix = '' if incremental_step == 2 else str(incremental_step - 1)
expected_rechecked = testcase.expected_rechecked_modules.get(
incremental_step - 1)
if expected_rechecked is not None:
assert_module_equivalence('rechecked' + suffix,
expected_rechecked,
result.manager.rechecked_modules)
expected_stale = testcase.expected_stale_modules.get(
incremental_step - 1)
if expected_stale is not None:
assert_module_equivalence('stale' + suffix, expected_stale,
result.manager.stale_modules)
assert proc.returncode == 0
def process_options(args: List[str],
require_targets: bool = True,
server_options: bool = False,
fscache: Optional[FileSystemCache] = None,
) -> Tuple[List[BuildSource], Options]:
"""Parse command line arguments.
If a FileSystemCache is passed in, and package_root options are given,
call fscache.set_package_root() to set the cache's package root.
"""
parser = argparse.ArgumentParser(prog='mypy', epilog=FOOTER,
fromfile_prefix_chars='@',
formatter_class=AugmentedHelpFormatter)
strict_flag_names = [] # type: List[str]
strict_flag_assignments = [] # type: List[Tuple[str, bool]]
def add_invertible_flag(flag: str,
*,
inverse: Optional[str] = None,
default: bool,
dest: Optional[str] = None,
help: str,
strict_flag: bool = False,
group: Optional[argparse._ActionsContainer] = None
) -> None:
if inverse is None:
inverse = invert_flag_name(flag)
if group is None:
group = parser
if help is not argparse.SUPPRESS:
help += " (inverse: {})".format(inverse)
arg = group.add_argument(flag,
action='store_false' if default else 'store_true',
dest=dest,
help=help)
dest = arg.dest
arg = group.add_argument(inverse,
action='store_true' if default else 'store_false',
dest=dest,
help=argparse.SUPPRESS)
if strict_flag:
assert dest is not None
strict_flag_names.append(flag)
strict_flag_assignments.append((dest, not default))
# Unless otherwise specified, arguments will be parsed directly onto an
# Options object. Options that require further processing should have
# their `dest` prefixed with `special-opts:`, which will cause them to be
# parsed into the separate special_opts namespace object.
parser.add_argument('-v', '--verbose', action='count', dest='verbosity',
help="more verbose messages")
parser.add_argument('-V', '--version', action='version',
version='%(prog)s ' + __version__,
help="show program's version number and exit")
config_group = parser.add_argument_group(
title='config file',
description="Use a config file instead of command line arguments.")
config_group.add_argument(
'--config-file',
help="configuration file, must have a [mypy] section "
"(defaults to {})".format(', '.join(defaults.CONFIG_FILES)))
add_invertible_flag('--warn-unused-configs', default=False, strict_flag=True,
help="warn about unused '[mypy-<pattern>]' config sections",
group=config_group)
imports_group = parser.add_argument_group(
title='import discovery',
description="Configure how imports are discovered and followed.")
imports_group.add_argument(
'--ignore-missing-imports', action='store_true',
help="silently ignore imports of missing modules")
imports_group.add_argument(
'--follow-imports', choices=['normal', 'silent', 'skip', 'error'],
default='normal', help="how to treat imports (default normal)")
imports_group.add_argument(
'--python-executable', action='store', metavar='EXECUTABLE',
help="Python executable used for finding PEP 561 compliant installed"
" packages and stubs",
dest='special-opts:python_executable')
imports_group.add_argument(
'--no-site-packages', action='store_true',
dest='special-opts:no_executable',
help="do not search for installed PEP 561 compliant packages")
platform_group = parser.add_argument_group(
title='platform configuration',
description="Type check code assuming certain runtime conditions.")
platform_group.add_argument(
'--python-version', type=parse_version, metavar='x.y',
help='type check code assuming it will be running on Python x.y',
dest='special-opts:python_version')
platform_group.add_argument(
'-2', '--py2', dest='python_version', action='store_const',
const=defaults.PYTHON2_VERSION,
help="use Python 2 mode (same as --python-version 2.7)")
platform_group.add_argument(
'--platform', action='store', metavar='PLATFORM',
help="type check special-cased code for the given OS platform "
"(defaults to sys.platform)")
platform_group.add_argument(
'--always-true', metavar='NAME', action='append', default=[],
help="additional variable to be considered True (may be repeated)")
platform_group.add_argument(
'--always-false', metavar='NAME', action='append', default=[],
help="additional variable to be considered False (may be repeated)")
disallow_any_group = parser.add_argument_group(
title='Any type restrictions',
description="Disallow the use of the 'Any' type under certain conditions.")
disallow_any_group.add_argument(
'--disallow-any-unimported', default=False, action='store_true',
help="disallow Any types resulting from unfollowed imports")
add_invertible_flag('--disallow-subclassing-any', default=False, strict_flag=True,
help="disallow subclassing values of type 'Any' when defining classes",
group=disallow_any_group)
disallow_any_group.add_argument(
'--disallow-any-expr', default=False, action='store_true',
help='disallow all expressions that have type Any')
disallow_any_group.add_argument(
'--disallow-any-decorated', default=False, action='store_true',
help='disallow functions that have Any in their signature '
'after decorator transformation')
disallow_any_group.add_argument(
'--disallow-any-explicit', default=False, action='store_true',
help='disallow explicit Any in type positions')
disallow_any_group.add_argument(
'--disallow-any-generics', default=False, action='store_true',
help='disallow usage of generic types that do not specify explicit '
'type parameters')
untyped_group = parser.add_argument_group(
title='untyped definitions and calls',
description="Configure how untyped definitions and calls are handled.")
add_invertible_flag('--disallow-untyped-calls', default=False, strict_flag=True,
help="disallow calling functions without type annotations"
" from functions with type annotations",
group=untyped_group)
add_invertible_flag('--disallow-untyped-defs', default=False, strict_flag=True,
help="disallow defining functions without type annotations"
" or with incomplete type annotations",
group=untyped_group)
add_invertible_flag('--disallow-incomplete-defs', default=False, strict_flag=True,
help="disallow defining functions with incomplete type annotations",
group=untyped_group)
add_invertible_flag('--check-untyped-defs', default=False, strict_flag=True,
help="type check the interior of functions without type annotations",
group=untyped_group)
add_invertible_flag('--warn-incomplete-stub', default=False,
help="warn if missing type annotation in typeshed, only relevant with"
" --check-untyped-defs enabled",
group=untyped_group)
none_group = parser.add_argument_group(
title='None and Optional handling',
description="Adjust how values of type 'None' are handled.")
add_invertible_flag('--no-implicit-optional', default=False, strict_flag=True,
help="don't assume arguments with default values of None are Optional",
group=none_group)
none_group.add_argument(
'--strict-optional', action='store_true',
help=argparse.SUPPRESS)
none_group.add_argument(
'--no-strict-optional', action='store_false', dest='strict_optional',
help="disable strict Optional checks (inverse: --strict-optional)")
none_group.add_argument(
'--strict-optional-whitelist', metavar='GLOB', nargs='*',
help="suppress strict Optional errors in all but the provided files; "
"implies --strict-optional (may suppress certain other errors "
"in non-whitelisted files)")
lint_group = parser.add_argument_group(
title='warnings',
description="Detect code that is sound but redundant or problematic.")
add_invertible_flag('--warn-redundant-casts', default=False, strict_flag=True,
help="warn about casting an expression to its inferred type",
group=lint_group)
add_invertible_flag('--no-warn-no-return', dest='warn_no_return', default=True,
help="do not warn about functions that end without returning",
group=lint_group)
add_invertible_flag('--warn-return-any', default=False, strict_flag=True,
help="warn about returning values of type Any"
" from non-Any typed functions",
group=lint_group)
add_invertible_flag('--warn-unused-ignores', default=False, strict_flag=True,
help="warn about unneeded '# type: ignore' comments",
group=lint_group)
strictness_group = parser.add_argument_group(
title='other strictness checks',
description="Other miscellaneous strictness checks.")
add_invertible_flag('--disallow-untyped-decorators', default=False, strict_flag=True,
help="disallow decorating typed functions with untyped decorators",
group=strictness_group)
incremental_group = parser.add_argument_group(
title='incremental mode',
description="Adjust how mypy incrementally type checks and caches modules.")
incremental_group.add_argument(
'-i', '--incremental', action='store_true',
help=argparse.SUPPRESS)
incremental_group.add_argument(
'--no-incremental', action='store_false', dest='incremental',
help="disable module cache (inverse: --incremental)")
incremental_group.add_argument(
'--cache-dir', action='store', metavar='DIR',
help="store module cache info in the given folder in incremental mode "
"(defaults to '{}')".format(defaults.CACHE_DIR))
incremental_group.add_argument(
'--cache-fine-grained', action='store_true',
help="include fine-grained dependency information in the cache for the mypy daemon")
incremental_group.add_argument(
'--quick-and-dirty', action='store_true',
help="use cache even if dependencies out of date (implies --incremental)")
incremental_group.add_argument(
'--skip-version-check', action='store_true',
help="allow using cache written by older mypy version")
internals_group = parser.add_argument_group(
title='mypy internals',
description="Debug and customize mypy internals.")
internals_group.add_argument(
'--pdb', action='store_true', help="invoke pdb on fatal error")
internals_group.add_argument(
'--show-traceback', '--tb', action='store_true',
help="show traceback on fatal error")
internals_group.add_argument(
'--custom-typing', metavar='MODULE', dest='custom_typing_module',
help="use a custom typing module")
internals_group.add_argument(
'--custom-typeshed-dir', metavar='DIR',
help="use the custom typeshed in DIR")
internals_group.add_argument(
'--shadow-file', nargs=2, metavar=('SOURCE_FILE', 'SHADOW_FILE'),
dest='shadow_file', action='append',
help="when encountering SOURCE_FILE, read and type check "
"the contents of SHADOW_FILE instead.")
error_group = parser.add_argument_group(
title='error reporting',
description="Adjust the amount of detail shown in error messages.")
add_invertible_flag('--show-error-context', default=False,
dest='show_error_context',
help='precede errors with "note:" messages explaining context',
group=error_group)
add_invertible_flag('--show-column-numbers', default=False,
help="show column numbers in error messages",
group=error_group)
analysis_group = parser.add_argument_group(
title='extra analysis',
description="Extract additional information and analysis.")
analysis_group.add_argument(
'--stats', action='store_true', dest='dump_type_stats', help=argparse.SUPPRESS)
analysis_group.add_argument(
'--inferstats', action='store_true', dest='dump_inference_stats',
help=argparse.SUPPRESS)
analysis_group.add_argument(
'--find-occurrences', metavar='CLASS.MEMBER',
dest='special-opts:find_occurrences',
help="print out all usages of a class member (experimental)")
strict_help = "strict mode; enables the following flags: {}".format(
", ".join(strict_flag_names))
strictness_group.add_argument(
'--strict', action='store_true', dest='special-opts:strict',
help=strict_help)
report_group = parser.add_argument_group(
title='report generation',
description='Generate a report in the specified format.')
for report_type in sorted(reporter_classes):
report_group.add_argument('--%s-report' % report_type.replace('_', '-'),
metavar='DIR',
dest='special-opts:%s_report' % report_type)
other_group = parser.add_argument_group(
title='miscellaneous',
description="Other miscellaneous flags.")
other_group.add_argument(
'--junit-xml', help="write junit.xml to the given file")
other_group.add_argument(
'--scripts-are-modules', action='store_true',
help="script x becomes module x instead of __main__")
if server_options:
# TODO: This flag is superfluous; remove after a short transition (2018-03-16)
other_group.add_argument(
'--experimental', action='store_true', dest='fine_grained_incremental',
help="enable fine-grained incremental mode")
other_group.add_argument(
'--use-fine-grained-cache', action='store_true',
help="use the cache in fine-grained incremental mode")
# hidden options
# --debug-cache will disable any cache-related compressions/optimizations,
# which will make the cache writing process output pretty-printed JSON (which
# is easier to debug).
parser.add_argument('--debug-cache', action='store_true', help=argparse.SUPPRESS)
# --dump-deps will dump all fine-grained dependencies to stdout
parser.add_argument('--dump-deps', action='store_true', help=argparse.SUPPRESS)
# --dump-graph will dump the contents of the graph of SCCs and exit.
parser.add_argument('--dump-graph', action='store_true', help=argparse.SUPPRESS)
# --semantic-analysis-only does exactly that.
parser.add_argument('--semantic-analysis-only', action='store_true', help=argparse.SUPPRESS)
# --local-partial-types disallows partial types spanning module top level and a function
# (implicitly defined in fine-grained incremental mode)
parser.add_argument('--local-partial-types', action='store_true', help=argparse.SUPPRESS)
# --bazel changes some behaviors for use with Bazel (https://bazel.build).
parser.add_argument('--bazel', action='store_true', help=argparse.SUPPRESS)
# --package-root adds a directory below which directories are considered
# packages even without __init__.py. May be repeated.
parser.add_argument('--package-root', metavar='ROOT', action='append', default=[],
help=argparse.SUPPRESS)
# --cache-map FILE ... gives a mapping from source files to cache files.
# Each triple of arguments is a source file, a cache meta file, and a cache data file.
# Modules not mentioned in the file will go through cache_dir.
# Must be followed by another flag or by '--' (and then only file args may follow).
parser.add_argument('--cache-map', nargs='+', dest='special-opts:cache_map',
help=argparse.SUPPRESS)
# deprecated options
parser.add_argument('--disallow-any', dest='special-opts:disallow_any',
help=argparse.SUPPRESS)
add_invertible_flag('--strict-boolean', default=False,
help=argparse.SUPPRESS)
parser.add_argument('-f', '--dirty-stubs', action='store_true',
dest='special-opts:dirty_stubs',
help=argparse.SUPPRESS)
parser.add_argument('--use-python-path', action='store_true',
dest='special-opts:use_python_path',
help=argparse.SUPPRESS)
parser.add_argument('-s', '--silent-imports', action='store_true',
dest='special-opts:silent_imports',
help=argparse.SUPPRESS)
parser.add_argument('--almost-silent', action='store_true',
dest='special-opts:almost_silent',
help=argparse.SUPPRESS)
parser.add_argument('--fast-parser', action='store_true', dest='special-opts:fast_parser',
help=argparse.SUPPRESS)
parser.add_argument('--no-fast-parser', action='store_true',
dest='special-opts:no_fast_parser',
help=argparse.SUPPRESS)
code_group = parser.add_argument_group(title='specifying which code to type check')
code_group.add_argument('-m', '--module', action='append', metavar='MODULE',
default=[],
dest='special-opts:modules',
help="type-check module; can repeat for more modules")
code_group.add_argument('-p', '--package', action='append', metavar='PACKAGE',
default=[],
dest='special-opts:packages',
help="type-check package recursively; can be repeated")
code_group.add_argument('-c', '--command', action='append', metavar='PROGRAM_TEXT',
dest='special-opts:command',
help="type-check program passed in as string")
code_group.add_argument(metavar='files', nargs='*', dest='special-opts:files',
help="type-check given files or directories")
# Parse arguments once into a dummy namespace so we can get the
# filename for the config file and know if the user requested all strict options.
dummy = argparse.Namespace()
parser.parse_args(args, dummy)
config_file = dummy.config_file
if config_file is not None and not os.path.exists(config_file):
parser.error("Cannot find config file '%s'" % config_file)
# Parse config file first, so command line can override.
options = Options()
parse_config_file(options, config_file)
# Set strict flags before parsing (if strict mode enabled), so other command
# line options can override.
if getattr(dummy, 'special-opts:strict'):
for dest, value in strict_flag_assignments:
setattr(options, dest, value)
# Parse command line for real, using a split namespace.
special_opts = argparse.Namespace()
parser.parse_args(args, SplitNamespace(options, special_opts, 'special-opts:'))
# --use-python-path is no longer supported; explain why.
if special_opts.use_python_path:
parser.error("Sorry, --use-python-path is no longer supported.\n"
"If you are trying this because your code depends on a library module,\n"
"you should really investigate how to obtain stubs for that module.\n"
"See https://github.com/python/mypy/issues/1411 for more discussion."
)
# Process deprecated options
if special_opts.disallow_any:
print("--disallow-any option was split up into multiple flags. "
"See http://mypy.readthedocs.io/en/latest/command_line.html#disallow-any-flags")
if options.strict_boolean:
print("Warning: --strict-boolean is deprecated; "
"see https://github.com/python/mypy/issues/3195", file=sys.stderr)
if special_opts.almost_silent:
print("Warning: --almost-silent has been replaced by "
"--follow-imports=errors", file=sys.stderr)
if options.follow_imports == 'normal':
options.follow_imports = 'errors'
elif special_opts.silent_imports:
print("Warning: --silent-imports has been replaced by "
"--ignore-missing-imports --follow-imports=skip", file=sys.stderr)
options.ignore_missing_imports = True
if options.follow_imports == 'normal':
options.follow_imports = 'skip'
if special_opts.dirty_stubs:
print("Warning: -f/--dirty-stubs is deprecated and no longer necessary. Mypy no longer "
"checks the git status of stubs.",
file=sys.stderr)
if special_opts.fast_parser:
print("Warning: --fast-parser is now the default (and only) parser.")
if special_opts.no_fast_parser:
print("Warning: --no-fast-parser no longer has any effect. The fast parser "
"is now mypy's default and only parser.")
try:
infer_python_version_and_executable(options, special_opts)
except PythonExecutableInferenceError as e:
parser.error(str(e))
if special_opts.no_executable:
options.python_executable = None
# Check for invalid argument combinations.
if require_targets:
code_methods = sum(bool(c) for c in [special_opts.modules + special_opts.packages,
special_opts.command,
special_opts.files])
if code_methods == 0:
parser.error("Missing target module, package, files, or command.")
elif code_methods > 1:
parser.error("May only specify one of: module/package, files, or command.")
# Check for overlapping `--always-true` and `--always-false` flags.
overlap = set(options.always_true) & set(options.always_false)
if overlap:
parser.error("You can't make a variable always true and always false (%s)" %
', '.join(sorted(overlap)))
# Set build flags.
if options.strict_optional_whitelist is not None:
# TODO: Deprecate, then kill this flag
options.strict_optional = True
if special_opts.find_occurrences:
experiments.find_occurrences = special_opts.find_occurrences.split('.')
assert experiments.find_occurrences is not None
if len(experiments.find_occurrences) < 2:
parser.error("Can only find occurrences of class members.")
if len(experiments.find_occurrences) != 2:
parser.error("Can only find occurrences of non-nested class members.")
# Set reports.
for flag, val in vars(special_opts).items():
if flag.endswith('_report') and val is not None:
report_type = flag[:-7].replace('_', '-')
report_dir = val
options.report_dirs[report_type] = report_dir
# Process --package-root.
if options.package_root:
process_package_roots(fscache, parser, options)
# Process --cache-map.
if special_opts.cache_map:
process_cache_map(parser, special_opts, options)
# Let quick_and_dirty imply incremental.
if options.quick_and_dirty:
options.incremental = True
# Set target.
if special_opts.modules + special_opts.packages:
options.build_type = BuildType.MODULE
lib_path = [os.getcwd()] + build.mypy_path()
targets = []
# TODO: use the same cache that the BuildManager will
cache = build.FindModuleCache(fscache)
for p in special_opts.packages:
if os.sep in p or os.altsep and os.altsep in p:
fail("Package name '{}' cannot have a slash in it.".format(p))
p_targets = cache.find_modules_recursive(p, tuple(lib_path), options.python_executable)
if not p_targets:
fail("Can't find package '{}'".format(p))
targets.extend(p_targets)
for m in special_opts.modules:
targets.append(BuildSource(None, m, None))
return targets, options
elif special_opts.command:
options.build_type = BuildType.PROGRAM_TEXT
targets = [BuildSource(None, None, '\n'.join(special_opts.command))]
return targets, options
else:
try:
targets = create_source_list(special_opts.files, options, fscache)
except InvalidSourceList as e:
fail(str(e))
return targets, options
def process_options(args: List[str],
require_targets: bool = True,
server_options: bool = False,
fscache: Optional[FileSystemCache] = None,
program: str = 'mypy',
header: str = HEADER,
) -> Tuple[List[BuildSource], Options]:
"""Parse command line arguments.
If a FileSystemCache is passed in, and package_root options are given,
call fscache.set_package_root() to set the cache's package root.
"""
parser = argparse.ArgumentParser(prog=program,
usage=header,
description=DESCRIPTION,
epilog=FOOTER,
fromfile_prefix_chars='@',
formatter_class=AugmentedHelpFormatter,
add_help=False)
strict_flag_names = [] # type: List[str]
strict_flag_assignments = [] # type: List[Tuple[str, bool]]
def add_invertible_flag(flag: str,
*,
inverse: Optional[str] = None,
default: bool,
dest: Optional[str] = None,
help: str,
strict_flag: bool = False,
group: Optional[argparse._ActionsContainer] = None
) -> None:
if inverse is None:
inverse = invert_flag_name(flag)
if group is None:
group = parser
if help is not argparse.SUPPRESS:
help += " (inverse: {})".format(inverse)
arg = group.add_argument(flag,
action='store_false' if default else 'store_true',
dest=dest,
help=help)
dest = arg.dest
arg = group.add_argument(inverse,
action='store_true' if default else 'store_false',
dest=dest,
help=argparse.SUPPRESS)
if strict_flag:
assert dest is not None
strict_flag_names.append(flag)
strict_flag_assignments.append((dest, not default))
# Unless otherwise specified, arguments will be parsed directly onto an
# Options object. Options that require further processing should have
# their `dest` prefixed with `special-opts:`, which will cause them to be
# parsed into the separate special_opts namespace object.
# Note: we have a style guide for formatting the mypy --help text. See
# https://github.com/python/mypy/wiki/Documentation-Conventions
general_group = parser.add_argument_group(
title='Optional arguments')
general_group.add_argument(
'-h', '--help', action='help',
help="Show this help message and exit")
general_group.add_argument(
'-v', '--verbose', action='count', dest='verbosity',
help="More verbose messages")
general_group.add_argument(
'-V', '--version', action='version',
version='%(prog)s ' + __version__,
help="Show program's version number and exit")
config_group = parser.add_argument_group(
title='Config file',
description="Use a config file instead of command line arguments. "
"This is useful if you are using many flags or want "
"to set different options per each module.")
config_group.add_argument(
'--config-file',
help="Configuration file, must have a [mypy] section "
"(defaults to {})".format(', '.join(defaults.CONFIG_FILES)))
add_invertible_flag('--warn-unused-configs', default=False, strict_flag=True,
help="Warn about unused '[mypy-<pattern>]' config sections",
group=config_group)
imports_group = parser.add_argument_group(
title='Import discovery',
description="Configure how imports are discovered and followed.")
imports_group.add_argument(
'--ignore-missing-imports', action='store_true',
help="Silently ignore imports of missing modules")
imports_group.add_argument(
'--follow-imports', choices=['normal', 'silent', 'skip', 'error'],
default='normal', help="How to treat imports (default normal)")
imports_group.add_argument(
'--python-executable', action='store', metavar='EXECUTABLE',
help="Python executable used for finding PEP 561 compliant installed"
" packages and stubs",
dest='special-opts:python_executable')
imports_group.add_argument(
'--no-site-packages', action='store_true',
dest='special-opts:no_executable',
help="Do not search for installed PEP 561 compliant packages")
imports_group.add_argument(
'--no-silence-site-packages', action='store_true',
help="Do not silence errors in PEP 561 compliant installed packages")
add_invertible_flag(
'--namespace-packages', default=False,
help="Support namespace packages (PEP 420, __init__.py-less)",
group=imports_group)
platform_group = parser.add_argument_group(
title='Platform configuration',
description="Type check code assuming it will be run under certain "
"runtime conditions. By default, mypy assumes your code "
"will be run using the same operating system and Python "
"version you are using to run mypy itself.")
platform_group.add_argument(
'--python-version', type=parse_version, metavar='x.y',
help='Type check code assuming it will be running on Python x.y',
dest='special-opts:python_version')
platform_group.add_argument(
'-2', '--py2', dest='special-opts:python_version', action='store_const',
const=defaults.PYTHON2_VERSION,
help="Use Python 2 mode (same as --python-version 2.7)")
platform_group.add_argument(
'--platform', action='store', metavar='PLATFORM',
help="Type check special-cased code for the given OS platform "
"(defaults to sys.platform)")
platform_group.add_argument(
'--always-true', metavar='NAME', action='append', default=[],
help="Additional variable to be considered True (may be repeated)")
platform_group.add_argument(
'--always-false', metavar='NAME', action='append', default=[],
help="Additional variable to be considered False (may be repeated)")
disallow_any_group = parser.add_argument_group(
title='Dynamic typing',
description="Disallow the use of the dynamic 'Any' type under certain conditions.")
disallow_any_group.add_argument(
'--disallow-any-unimported', default=False, action='store_true',
help="Disallow Any types resulting from unfollowed imports")
add_invertible_flag('--disallow-subclassing-any', default=False, strict_flag=True,
help="Disallow subclassing values of type 'Any' when defining classes",
group=disallow_any_group)
disallow_any_group.add_argument(
'--disallow-any-expr', default=False, action='store_true',
help='Disallow all expressions that have type Any')
disallow_any_group.add_argument(
'--disallow-any-decorated', default=False, action='store_true',
help='Disallow functions that have Any in their signature '
'after decorator transformation')
disallow_any_group.add_argument(
'--disallow-any-explicit', default=False, action='store_true',
help='Disallow explicit Any in type positions')
add_invertible_flag('--disallow-any-generics', default=False, strict_flag=True,
help='Disallow usage of generic types that do not specify explicit type '
'parameters', group=disallow_any_group)
untyped_group = parser.add_argument_group(
title='Untyped definitions and calls',
description="Configure how untyped definitions and calls are handled. "
"Note: by default, mypy ignores any untyped function definitions "
"and assumes any calls to such functions have a return "
"type of 'Any'.")
add_invertible_flag('--disallow-untyped-calls', default=False, strict_flag=True,
help="Disallow calling functions without type annotations"
" from functions with type annotations",
group=untyped_group)
add_invertible_flag('--disallow-untyped-defs', default=False, strict_flag=True,
help="Disallow defining functions without type annotations"
" or with incomplete type annotations",
group=untyped_group)
add_invertible_flag('--disallow-incomplete-defs', default=False, strict_flag=True,
help="Disallow defining functions with incomplete type annotations",
group=untyped_group)
add_invertible_flag('--check-untyped-defs', default=False, strict_flag=True,
help="Type check the interior of functions without type annotations",
group=untyped_group)
add_invertible_flag('--disallow-untyped-decorators', default=False, strict_flag=True,
help="Disallow decorating typed functions with untyped decorators",
group=untyped_group)
none_group = parser.add_argument_group(
title='None and Optional handling',
description="Adjust how values of type 'None' are handled. For more context on "
"how mypy handles values of type 'None', see: "
"mypy.readthedocs.io/en/latest/kinds_of_types.html#no-strict-optional")
add_invertible_flag('--no-implicit-optional', default=False, strict_flag=True,
help="Don't assume arguments with default values of None are Optional",
group=none_group)
none_group.add_argument(
'--strict-optional', action='store_true',
help=argparse.SUPPRESS)
none_group.add_argument(
'--no-strict-optional', action='store_false', dest='strict_optional',
help="Disable strict Optional checks (inverse: --strict-optional)")
none_group.add_argument(
'--strict-optional-whitelist', metavar='GLOB', nargs='*',
help="Suppress strict Optional errors in all but the provided files; "
"implies --strict-optional (may suppress certain other errors "
"in non-whitelisted files)")
lint_group = parser.add_argument_group(
title='Warnings',
description="Detect code that is sound but redundant or problematic.")
add_invertible_flag('--warn-redundant-casts', default=False, strict_flag=True,
help="Warn about casting an expression to its inferred type",
group=lint_group)
add_invertible_flag('--warn-unused-ignores', default=False, strict_flag=True,
help="Warn about unneeded '# type: ignore' comments",
group=lint_group)
add_invertible_flag('--no-warn-no-return', dest='warn_no_return', default=True,
help="Do not warn about functions that end without returning",
group=lint_group)
add_invertible_flag('--warn-return-any', default=False, strict_flag=True,
help="Warn about returning values of type Any"
" from non-Any typed functions",
group=lint_group)
# Note: this group is intentionally added here even though we don't add
# --strict to this group near the end.
#
# That way, this group will appear after the various strictness groups
# but before the remaining flags.
# We add `--strict` near the end so we don't accidentally miss any strictness
# flags that are added after this group.
strictness_group = parser.add_argument_group(
title='Other strictness checks')
add_invertible_flag('--allow-untyped-globals', default=False, strict_flag=False,
help="Suppress toplevel errors caused by missing annotations",
group=strictness_group)
add_invertible_flag('--allow-redefinition', default=False, strict_flag=False,
help="Allow unconditional variable redefinition with a new type",
group=strictness_group)
add_invertible_flag('--strict-equality', default=False, strict_flag=False,
help="Prohibit equality, identity, and container checks for"
" non-overlapping types",
group=strictness_group)
incremental_group = parser.add_argument_group(
title='Incremental mode',
description="Adjust how mypy incrementally type checks and caches modules. "
"Mypy caches type information about modules into a cache to "
"let you speed up future invocations of mypy. Also see "
"mypy's daemon mode: "
"mypy.readthedocs.io/en/latest/mypy_daemon.html#mypy-daemon")
incremental_group.add_argument(
'-i', '--incremental', action='store_true',
help=argparse.SUPPRESS)
incremental_group.add_argument(
'--no-incremental', action='store_false', dest='incremental',
help="Disable module cache (inverse: --incremental)")
incremental_group.add_argument(
'--cache-dir', action='store', metavar='DIR',
help="Store module cache info in the given folder in incremental mode "
"(defaults to '{}')".format(defaults.CACHE_DIR))
add_invertible_flag('--sqlite-cache', default=False,
help="Use a sqlite database to store the cache",
group=incremental_group)
incremental_group.add_argument(
'--cache-fine-grained', action='store_true',
help="Include fine-grained dependency information in the cache for the mypy daemon")
incremental_group.add_argument(
'--skip-version-check', action='store_true',
help="Allow using cache written by older mypy version")
incremental_group.add_argument(
'--skip-cache-mtime-checks', action='store_true',
help="Skip cache internal consistency checks based on mtime")
internals_group = parser.add_argument_group(
title='Mypy internals',
description="Debug and customize mypy internals.")
internals_group.add_argument(
'--pdb', action='store_true', help="Invoke pdb on fatal error")
internals_group.add_argument(
'--show-traceback', '--tb', action='store_true',
help="Show traceback on fatal error")
internals_group.add_argument(
'--raise-exceptions', action='store_true', help="Raise exception on fatal error"
)
internals_group.add_argument(
'--custom-typing', metavar='MODULE', dest='custom_typing_module',
help="Use a custom typing module")
internals_group.add_argument(
'--custom-typeshed-dir', metavar='DIR',
help="Use the custom typeshed in DIR")
add_invertible_flag('--warn-incomplete-stub', default=False,
help="Warn if missing type annotation in typeshed, only relevant with"
" --disallow-untyped-defs or --disallow-incomplete-defs enabled",
group=internals_group)
internals_group.add_argument(
'--shadow-file', nargs=2, metavar=('SOURCE_FILE', 'SHADOW_FILE'),
dest='shadow_file', action='append',
help="When encountering SOURCE_FILE, read and type check "
"the contents of SHADOW_FILE instead.")
add_invertible_flag('--fast-exit', default=False, help=argparse.SUPPRESS,
group=internals_group)
add_invertible_flag('--new-semantic-analyzer', default=False, help=argparse.SUPPRESS,
group=internals_group)
error_group = parser.add_argument_group(
title='Error reporting',
description="Adjust the amount of detail shown in error messages.")
add_invertible_flag('--show-error-context', default=False,
dest='show_error_context',
help='Precede errors with "note:" messages explaining context',
group=error_group)
add_invertible_flag('--show-column-numbers', default=False,
help="Show column numbers in error messages",
group=error_group)
strict_help = "Strict mode; enables the following flags: {}".format(
", ".join(strict_flag_names))
strictness_group.add_argument(
'--strict', action='store_true', dest='special-opts:strict',
help=strict_help)
report_group = parser.add_argument_group(
title='Report generation',
description='Generate a report in the specified format.')
for report_type in sorted(defaults.REPORTER_NAMES):
report_group.add_argument('--%s-report' % report_type.replace('_', '-'),
metavar='DIR',
dest='special-opts:%s_report' % report_type)
other_group = parser.add_argument_group(
title='Miscellaneous')
other_group.add_argument(
'--quickstart-file', help=argparse.SUPPRESS)
other_group.add_argument(
'--junit-xml', help="Write junit.xml to the given file")
other_group.add_argument(
'--scripts-are-modules', action='store_true',
help="Script x becomes module x instead of __main__")
other_group.add_argument(
'--find-occurrences', metavar='CLASS.MEMBER',
dest='special-opts:find_occurrences',
help="Print out all usages of a class member (experimental)")
if server_options:
# TODO: This flag is superfluous; remove after a short transition (2018-03-16)
other_group.add_argument(
'--experimental', action='store_true', dest='fine_grained_incremental',
help="Enable fine-grained incremental mode")
other_group.add_argument(
'--use-fine-grained-cache', action='store_true',
help="Use the cache in fine-grained incremental mode")
# hidden options
parser.add_argument(
'--stats', action='store_true', dest='dump_type_stats', help=argparse.SUPPRESS)
parser.add_argument(
'--inferstats', action='store_true', dest='dump_inference_stats',
help=argparse.SUPPRESS)
# --debug-cache will disable any cache-related compressions/optimizations,
# which will make the cache writing process output pretty-printed JSON (which
# is easier to debug).
parser.add_argument('--debug-cache', action='store_true', help=argparse.SUPPRESS)
# --dump-deps will dump all fine-grained dependencies to stdout
parser.add_argument('--dump-deps', action='store_true', help=argparse.SUPPRESS)
# --dump-graph will dump the contents of the graph of SCCs and exit.
parser.add_argument('--dump-graph', action='store_true', help=argparse.SUPPRESS)
# --semantic-analysis-only does exactly that.
parser.add_argument('--semantic-analysis-only', action='store_true', help=argparse.SUPPRESS)
# --local-partial-types disallows partial types spanning module top level and a function
# (implicitly defined in fine-grained incremental mode)
parser.add_argument('--local-partial-types', action='store_true', help=argparse.SUPPRESS)
# --logical-deps adds some more dependencies that are not semantically needed, but
# may be helpful to determine relative importance of classes and functions for overall
# type precision in a code base. It also _removes_ some deps, so this flag should be never
# used except for generating code stats. This also automatically enables --cache-fine-grained.
# NOTE: This is an experimental option that may be modified or removed at any time.
parser.add_argument('--logical-deps', action='store_true', help=argparse.SUPPRESS)
# --bazel changes some behaviors for use with Bazel (https://bazel.build).
parser.add_argument('--bazel', action='store_true', help=argparse.SUPPRESS)
# --package-root adds a directory below which directories are considered
# packages even without __init__.py. May be repeated.
parser.add_argument('--package-root', metavar='ROOT', action='append', default=[],
help=argparse.SUPPRESS)
# --cache-map FILE ... gives a mapping from source files to cache files.
# Each triple of arguments is a source file, a cache meta file, and a cache data file.
# Modules not mentioned in the file will go through cache_dir.
# Must be followed by another flag or by '--' (and then only file args may follow).
parser.add_argument('--cache-map', nargs='+', dest='special-opts:cache_map',
help=argparse.SUPPRESS)
# options specifying code to check
code_group = parser.add_argument_group(
title="Running code",
description="Specify the code you want to type check. For more details, see "
"mypy.readthedocs.io/en/latest/running_mypy.html#running-mypy")
code_group.add_argument(
'-m', '--module', action='append', metavar='MODULE',
default=[],
dest='special-opts:modules',
help="Type-check module; can repeat for more modules")
code_group.add_argument(
'-p', '--package', action='append', metavar='PACKAGE',
default=[],
dest='special-opts:packages',
help="Type-check package recursively; can be repeated")
code_group.add_argument(
'-c', '--command', action='append', metavar='PROGRAM_TEXT',
dest='special-opts:command',
help="Type-check program passed in as string")
code_group.add_argument(
metavar='files', nargs='*', dest='special-opts:files',
help="Type-check given files or directories")
# Parse arguments once into a dummy namespace so we can get the
# filename for the config file and know if the user requested all strict options.
dummy = argparse.Namespace()
parser.parse_args(args, dummy)
config_file = dummy.config_file
# Don't explicitly test if "config_file is not None" for this check.
# This lets `--config-file=` (an empty string) be used to disable all config files.
if config_file and not os.path.exists(config_file):
parser.error("Cannot find config file '%s'" % config_file)
# Parse config file first, so command line can override.
options = Options()
parse_config_file(options, config_file)
# Set strict flags before parsing (if strict mode enabled), so other command
# line options can override.
if getattr(dummy, 'special-opts:strict'):
for dest, value in strict_flag_assignments:
setattr(options, dest, value)
# Parse command line for real, using a split namespace.
special_opts = argparse.Namespace()
parser.parse_args(args, SplitNamespace(options, special_opts, 'special-opts:'))
# The python_version is either the default, which can be overridden via a config file,
# or stored in special_opts and is passed via the command line.
options.python_version = special_opts.python_version or options.python_version
try:
infer_python_executable(options, special_opts)
except PythonExecutableInferenceError as e:
parser.error(str(e))
if special_opts.no_executable:
options.python_executable = None
# Paths listed in the config file will be ignored if any paths are passed on
# the command line.
if options.files and not special_opts.files:
special_opts.files = options.files
# Check for invalid argument combinations.
if require_targets:
code_methods = sum(bool(c) for c in [special_opts.modules + special_opts.packages,
special_opts.command,
special_opts.files])
if code_methods == 0:
parser.error("Missing target module, package, files, or command.")
elif code_methods > 1:
parser.error("May only specify one of: module/package, files, or command.")
# Check for overlapping `--always-true` and `--always-false` flags.
overlap = set(options.always_true) & set(options.always_false)
if overlap:
parser.error("You can't make a variable always true and always false (%s)" %
', '.join(sorted(overlap)))
# Set build flags.
if options.strict_optional_whitelist is not None:
# TODO: Deprecate, then kill this flag
options.strict_optional = True
if special_opts.find_occurrences:
state.find_occurrences = special_opts.find_occurrences.split('.')
assert state.find_occurrences is not None
if len(state.find_occurrences) < 2:
parser.error("Can only find occurrences of class members.")
if len(state.find_occurrences) != 2:
parser.error("Can only find occurrences of non-nested class members.")
# Set reports.
for flag, val in vars(special_opts).items():
if flag.endswith('_report') and val is not None:
report_type = flag[:-7].replace('_', '-')
report_dir = val
options.report_dirs[report_type] = report_dir
# Process --package-root.
if options.package_root:
process_package_roots(fscache, parser, options)
# Process --cache-map.
if special_opts.cache_map:
if options.sqlite_cache:
parser.error("--cache-map is incompatible with --sqlite-cache")
process_cache_map(parser, special_opts, options)
# Let logical_deps imply cache_fine_grained (otherwise the former is useless).
if options.logical_deps:
options.cache_fine_grained = True
# Set target.
if special_opts.modules + special_opts.packages:
options.build_type = BuildType.MODULE
search_paths = SearchPaths((os.getcwd(),), tuple(mypy_path()), (), ())
targets = []
# TODO: use the same cache that the BuildManager will
cache = FindModuleCache(search_paths, fscache)
for p in special_opts.packages:
if os.sep in p or os.altsep and os.altsep in p:
fail("Package name '{}' cannot have a slash in it.".format(p))
p_targets = cache.find_modules_recursive(p)
if not p_targets:
fail("Can't find package '{}'".format(p))
targets.extend(p_targets)
for m in special_opts.modules:
targets.append(BuildSource(None, m, None))
return targets, options
elif special_opts.command:
options.build_type = BuildType.PROGRAM_TEXT
targets = [BuildSource(None, None, '\n'.join(special_opts.command))]
return targets, options
else:
try:
targets = create_source_list(special_opts.files, options, fscache)
except InvalidSourceList as e:
fail(str(e))
return targets, options
def parse_config_file(options: Options, set_strict_flags: Callable[[], None],
filename: Optional[str],
stdout: Optional[TextIO] = None,
stderr: Optional[TextIO] = None) -> None:
"""Parse a config file into an Options object.
Errors are written to stderr but are not fatal.
If filename is None, fall back to default config files.
"""
stdout = stdout or sys.stdout
stderr = stderr or sys.stderr
if filename is not None:
config_files: Tuple[str, ...] = (filename,)
else:
config_files = tuple(map(os.path.expanduser, defaults.CONFIG_FILES))
config_parser = configparser.RawConfigParser()
for config_file in config_files:
if not os.path.exists(config_file):
continue
try:
if is_toml(config_file):
with open(config_file, encoding="utf-8") as f:
toml_data = tomli.loads(f.read())
# Filter down to just mypy relevant toml keys
toml_data = toml_data.get('tool', {})
if 'mypy' not in toml_data:
continue
toml_data = {'mypy': toml_data['mypy']}
parser: MutableMapping[str, Any] = destructure_overrides(toml_data)
config_types = toml_config_types
else:
config_parser.read(config_file)
parser = config_parser
config_types = ini_config_types
except (tomli.TOMLDecodeError, configparser.Error, ConfigTOMLValueError) as err:
print("%s: %s" % (config_file, err), file=stderr)
else:
if config_file in defaults.SHARED_CONFIG_FILES and 'mypy' not in parser:
continue
file_read = config_file
options.config_file = file_read
break
else:
return
os.environ['MYPY_CONFIG_FILE_DIR'] = os.path.dirname(
os.path.abspath(config_file))
if 'mypy' not in parser:
if filename or file_read not in defaults.SHARED_CONFIG_FILES:
print("%s: No [mypy] section in config file" % file_read, file=stderr)
else:
section = parser['mypy']
prefix = '%s: [%s]: ' % (file_read, 'mypy')
updates, report_dirs = parse_section(
prefix, options, set_strict_flags, section, config_types, stderr)
for k, v in updates.items():
setattr(options, k, v)
options.report_dirs.update(report_dirs)
for name, section in parser.items():
if name.startswith('mypy-'):
prefix = get_prefix(file_read, name)
updates, report_dirs = parse_section(
prefix, options, set_strict_flags, section, config_types, stderr)
if report_dirs:
print("%sPer-module sections should not specify reports (%s)" %
(prefix, ', '.join(s + '_report' for s in sorted(report_dirs))),
file=stderr)
if set(updates) - PER_MODULE_OPTIONS:
print("%sPer-module sections should only specify per-module flags (%s)" %
(prefix, ', '.join(sorted(set(updates) - PER_MODULE_OPTIONS))),
file=stderr)
updates = {k: v for k, v in updates.items() if k in PER_MODULE_OPTIONS}
globs = name[5:]
for glob in globs.split(','):
# For backwards compatibility, replace (back)slashes with dots.
glob = glob.replace(os.sep, '.')
if os.altsep:
glob = glob.replace(os.altsep, '.')
if (any(c in glob for c in '?[]!') or
any('*' in x and x != '*' for x in glob.split('.'))):
print("%sPatterns must be fully-qualified module names, optionally "
"with '*' in some components (e.g spam.*.eggs.*)"
% prefix,
file=stderr)
else:
options.per_module_options[glob] = updates
from mypy.options import Options
from mypy.dmypy_server import Server
from mypy.dmypy_util import DEFAULT_STATUS_FILE
from text import Position, Range, Diagnostic
import logging
import re
import lsp
line_pattern = r'([^:]+):(?:(\d+):)?(?:(\d+):)? (\w+): (.*)'
log = logging.getLogger(__name__)
settings = None
mypy_version = __version__
options = Options()
options.check_untyped_defs = True
options.follow_imports = 'error'
options.use_fine_grained_cache = True
options.show_column_numbers = True
def is_patched_mypy():
return 'langserver' in mypy_version
def parse_mypy_out(output: str):
lines = output.splitlines()
diags = []
for line in lines:
result = re.match(line_pattern, line)
def process_options(
args: List[str],
require_targets: bool = True,
server_options: bool = False,
) -> Tuple[List[BuildSource], Options]:
"""Parse command line arguments."""
parser = argparse.ArgumentParser(prog='mypy',
epilog=FOOTER,
fromfile_prefix_chars='@',
formatter_class=AugmentedHelpFormatter)
strict_flag_names = [] # type: List[str]
strict_flag_assignments = [] # type: List[Tuple[str, bool]]
def add_invertible_flag(flag: str,
*,
inverse: Optional[str] = None,
default: bool,
dest: Optional[str] = None,
help: str,
strict_flag: bool = False) -> None:
if inverse is None:
inverse = invert_flag_name(flag)
if help is not argparse.SUPPRESS:
help += " (inverse: {})".format(inverse)
arg = parser.add_argument(
flag, # type: ignore # incorrect stub for add_argument
action='store_false' if default else 'store_true',
dest=dest,
help=help)
dest = arg.dest
arg = parser.add_argument(
inverse, # type: ignore # incorrect stub for add_argument
action='store_true' if default else 'store_false',
dest=dest,
help=argparse.SUPPRESS)
if strict_flag:
assert dest is not None
strict_flag_names.append(flag)
strict_flag_assignments.append((dest, not default))
# Unless otherwise specified, arguments will be parsed directly onto an
# Options object. Options that require further processing should have
# their `dest` prefixed with `special-opts:`, which will cause them to be
# parsed into the separate special_opts namespace object.
parser.add_argument('-v',
'--verbose',
action='count',
dest='verbosity',
help="more verbose messages")
parser.add_argument('-V',
'--version',
action='version',
version='%(prog)s ' + __version__)
parser.add_argument('--python-version',
type=parse_version,
metavar='x.y',
help='use Python x.y')
parser.add_argument(
'--platform',
action='store',
metavar='PLATFORM',
help="typecheck special-cased code for the given OS platform "
"(defaults to sys.platform).")
parser.add_argument('-2',
'--py2',
dest='python_version',
action='store_const',
const=defaults.PYTHON2_VERSION,
help="use Python 2 mode")
parser.add_argument('--ignore-missing-imports',
action='store_true',
help="silently ignore imports of missing modules")
parser.add_argument('--follow-imports',
choices=['normal', 'silent', 'skip', 'error'],
default='normal',
help="how to treat imports (default normal)")
parser.add_argument(
'--disallow-any-unimported',
default=False,
action='store_true',
help="disallow Any types resulting from unfollowed imports")
parser.add_argument('--disallow-any-expr',
default=False,
action='store_true',
help='disallow all expressions that have type Any')
parser.add_argument(
'--disallow-any-decorated',
default=False,
action='store_true',
help='disallow functions that have Any in their signature '
'after decorator transformation')
parser.add_argument('--disallow-any-explicit',
default=False,
action='store_true',
help='disallow explicit Any in type positions')
parser.add_argument(
'--disallow-any-generics',
default=False,
action='store_true',
help='disallow usage of generic types that do not specify explicit '
'type parameters')
add_invertible_flag(
'--disallow-untyped-calls',
default=False,
strict_flag=True,
help="disallow calling functions without type annotations"
" from functions with type annotations")
add_invertible_flag(
'--disallow-untyped-defs',
default=False,
strict_flag=True,
help="disallow defining functions without type annotations"
" or with incomplete type annotations")
add_invertible_flag(
'--disallow-incomplete-defs',
default=False,
strict_flag=True,
help="disallow defining functions with incomplete type annotations")
add_invertible_flag(
'--check-untyped-defs',
default=False,
strict_flag=True,
help="type check the interior of functions without type annotations")
add_invertible_flag(
'--disallow-subclassing-any',
default=False,
strict_flag=True,
help="disallow subclassing values of type 'Any' when defining classes")
add_invertible_flag(
'--warn-incomplete-stub',
default=False,
help="warn if missing type annotation in typeshed, only relevant with"
" --check-untyped-defs enabled")
add_invertible_flag(
'--disallow-untyped-decorators',
default=False,
strict_flag=True,
help="disallow decorating typed functions with untyped decorators")
add_invertible_flag(
'--warn-redundant-casts',
default=False,
strict_flag=True,
help="warn about casting an expression to its inferred type")
add_invertible_flag(
'--no-warn-no-return',
dest='warn_no_return',
default=True,
help="do not warn about functions that end without returning")
add_invertible_flag('--warn-return-any',
default=False,
strict_flag=True,
help="warn about returning values of type Any"
" from non-Any typed functions")
add_invertible_flag('--warn-unused-ignores',
default=False,
strict_flag=True,
help="warn about unneeded '# type: ignore' comments")
add_invertible_flag(
'--warn-unused-configs',
default=False,
strict_flag=True,
help="warn about unused '[mypy-<pattern>]' config sections")
add_invertible_flag(
'--show-error-context',
default=False,
dest='show_error_context',
help='Precede errors with "note:" messages explaining context')
add_invertible_flag(
'--no-implicit-optional',
default=False,
strict_flag=True,
help="don't assume arguments with default values of None are Optional")
parser.add_argument(
'-i',
'--incremental',
action='store_true',
help="enable module cache, (inverse: --no-incremental)")
parser.add_argument('--no-incremental',
action='store_false',
dest='incremental',
help=argparse.SUPPRESS)
parser.add_argument('--quick-and-dirty',
action='store_true',
help="use cache even if dependencies out of date "
"(implies --incremental)")
parser.add_argument(
'--cache-dir',
action='store',
metavar='DIR',
help="store module cache info in the given folder in incremental mode "
"(defaults to '{}')".format(defaults.CACHE_DIR))
parser.add_argument(
'--cache-fine-grained',
action='store_true',
help="include fine-grained dependency information in the cache")
parser.add_argument('--skip-version-check',
action='store_true',
help="allow using cache written by older mypy version")
add_invertible_flag('--strict-optional',
default=False,
strict_flag=True,
help="enable experimental strict Optional checks")
parser.add_argument(
'--strict-optional-whitelist',
metavar='GLOB',
nargs='*',
help="suppress strict Optional errors in all but the provided files "
"(experimental -- read documentation before using!). "
"Implies --strict-optional. Has the undesirable side-effect of "
"suppressing other errors in non-whitelisted files.")
parser.add_argument('--junit-xml',
help="write junit.xml to the given file")
parser.add_argument('--pdb',
action='store_true',
help="invoke pdb on fatal error")
parser.add_argument('--show-traceback',
'--tb',
action='store_true',
help="show traceback on fatal error")
parser.add_argument('--stats',
action='store_true',
dest='dump_type_stats',
help="dump stats")
parser.add_argument('--inferstats',
action='store_true',
dest='dump_inference_stats',
help="dump type inference stats")
parser.add_argument('--custom-typing',
metavar='MODULE',
dest='custom_typing_module',
help="use a custom typing module")
parser.add_argument('--custom-typeshed-dir',
metavar='DIR',
help="use the custom typeshed in DIR")
parser.add_argument('--scripts-are-modules',
action='store_true',
help="Script x becomes module x instead of __main__")
parser.add_argument('--config-file',
help="Configuration file, must have a [mypy] section "
"(defaults to {})".format(defaults.CONFIG_FILE))
add_invertible_flag('--show-column-numbers',
default=False,
help="Show column numbers in error messages")
parser.add_argument(
'--find-occurrences',
metavar='CLASS.MEMBER',
dest='special-opts:find_occurrences',
help="print out all usages of a class member (experimental)")
strict_help = "Strict mode. Enables the following flags: {}".format(
", ".join(strict_flag_names))
parser.add_argument('--strict',
action='store_true',
dest='special-opts:strict',
help=strict_help)
parser.add_argument('--shadow-file',
nargs=2,
metavar=('SOURCE_FILE', 'SHADOW_FILE'),
dest='shadow_file',
help='Typecheck SHADOW_FILE in place of SOURCE_FILE.')
# hidden options
# --debug-cache will disable any cache-related compressions/optimizations,
# which will make the cache writing process output pretty-printed JSON (which
# is easier to debug).
parser.add_argument('--debug-cache',
action='store_true',
help=argparse.SUPPRESS)
# --dump-deps will dump all fine-grained dependencies to stdout
parser.add_argument('--dump-deps',
action='store_true',
help=argparse.SUPPRESS)
# --dump-graph will dump the contents of the graph of SCCs and exit.
parser.add_argument('--dump-graph',
action='store_true',
help=argparse.SUPPRESS)
# --semantic-analysis-only does exactly that.
parser.add_argument('--semantic-analysis-only',
action='store_true',
help=argparse.SUPPRESS)
# --local-partial-types disallows partial types spanning module top level and a function
# (implicitly defined in fine-grained incremental mode)
parser.add_argument('--local-partial-types',
action='store_true',
help=argparse.SUPPRESS)
# deprecated options
parser.add_argument('--disallow-any',
dest='special-opts:disallow_any',
help=argparse.SUPPRESS)
add_invertible_flag('--strict-boolean',
default=False,
help=argparse.SUPPRESS)
parser.add_argument('-f',
'--dirty-stubs',
action='store_true',
dest='special-opts:dirty_stubs',
help=argparse.SUPPRESS)
parser.add_argument('--use-python-path',
action='store_true',
dest='special-opts:use_python_path',
help=argparse.SUPPRESS)
parser.add_argument('-s',
'--silent-imports',
action='store_true',
dest='special-opts:silent_imports',
help=argparse.SUPPRESS)
parser.add_argument('--almost-silent',
action='store_true',
dest='special-opts:almost_silent',
help=argparse.SUPPRESS)
parser.add_argument('--fast-parser',
action='store_true',
dest='special-opts:fast_parser',
help=argparse.SUPPRESS)
parser.add_argument('--no-fast-parser',
action='store_true',
dest='special-opts:no_fast_parser',
help=argparse.SUPPRESS)
if server_options:
parser.add_argument('--experimental',
action='store_true',
dest='fine_grained_incremental',
help="enable fine-grained incremental mode")
parser.add_argument(
'--use-fine-grained-cache',
action='store_true',
help="use the cache in fine-grained incremental mode")
report_group = parser.add_argument_group(
title='report generation',
description='Generate a report in the specified format.')
for report_type in sorted(reporter_classes):
report_group.add_argument('--%s-report' %
report_type.replace('_', '-'),
metavar='DIR',
dest='special-opts:%s_report' % report_type)
code_group = parser.add_argument_group(
title='How to specify the code to type check')
code_group.add_argument(
'-m',
'--module',
action='append',
metavar='MODULE',
dest='special-opts:modules',
help="type-check module; can repeat for more modules")
# TODO: `mypy -p A -p B` currently silently ignores A
# (last option wins). Perhaps -c, -m and -p could just be
# command-line flags that modify how we interpret self.files?
code_group.add_argument('-c',
'--command',
action='append',
metavar='PROGRAM_TEXT',
dest='special-opts:command',
help="type-check program passed in as string")
code_group.add_argument('-p',
'--package',
metavar='PACKAGE',
dest='special-opts:package',
help="type-check all files in a directory")
code_group.add_argument(metavar='files',
nargs='*',
dest='special-opts:files',
help="type-check given files or directories")
# Parse arguments once into a dummy namespace so we can get the
# filename for the config file and know if the user requested all strict options.
dummy = argparse.Namespace()
parser.parse_args(args, dummy)
config_file = dummy.config_file
if config_file is not None and not os.path.exists(config_file):
parser.error("Cannot find config file '%s'" % config_file)
# Parse config file first, so command line can override.
options = Options()
parse_config_file(options, config_file)
# Set strict flags before parsing (if strict mode enabled), so other command
# line options can override.
if getattr(dummy, 'special-opts:strict'):
for dest, value in strict_flag_assignments:
setattr(options, dest, value)
# Parse command line for real, using a split namespace.
special_opts = argparse.Namespace()
parser.parse_args(args,
SplitNamespace(options, special_opts, 'special-opts:'))
# --use-python-path is no longer supported; explain why.
if special_opts.use_python_path:
parser.error(
"Sorry, --use-python-path is no longer supported.\n"
"If you are trying this because your code depends on a library module,\n"
"you should really investigate how to obtain stubs for that module.\n"
"See https://github.com/python/mypy/issues/1411 for more discussion."
)
# Process deprecated options
if special_opts.disallow_any:
print(
"--disallow-any option was split up into multiple flags. "
"See http://mypy.readthedocs.io/en/latest/command_line.html#disallow-any-flags"
)
if options.strict_boolean:
print(
"Warning: --strict-boolean is deprecated; "
"see https://github.com/python/mypy/issues/3195",
file=sys.stderr)
if special_opts.almost_silent:
print(
"Warning: --almost-silent has been replaced by "
"--follow-imports=errors",
file=sys.stderr)
if options.follow_imports == 'normal':
options.follow_imports = 'errors'
elif special_opts.silent_imports:
print(
"Warning: --silent-imports has been replaced by "
"--ignore-missing-imports --follow-imports=skip",
file=sys.stderr)
options.ignore_missing_imports = True
if options.follow_imports == 'normal':
options.follow_imports = 'skip'
if special_opts.dirty_stubs:
print(
"Warning: -f/--dirty-stubs is deprecated and no longer necessary. Mypy no longer "
"checks the git status of stubs.",
file=sys.stderr)
if special_opts.fast_parser:
print("Warning: --fast-parser is now the default (and only) parser.")
if special_opts.no_fast_parser:
print(
"Warning: --no-fast-parser no longer has any effect. The fast parser "
"is now mypy's default and only parser.")
# Check for invalid argument combinations.
if require_targets:
code_methods = sum(
bool(c) for c in [
special_opts.modules, special_opts.command,
special_opts.package, special_opts.files
])
if code_methods == 0:
parser.error("Missing target module, package, files, or command.")
elif code_methods > 1:
parser.error(
"May only specify one of: module, package, files, or command.")
# Set build flags.
if options.strict_optional_whitelist is not None:
# TODO: Deprecate, then kill this flag
options.strict_optional = True
if special_opts.find_occurrences:
experiments.find_occurrences = special_opts.find_occurrences.split('.')
assert experiments.find_occurrences is not None
if len(experiments.find_occurrences) < 2:
parser.error("Can only find occurrences of class members.")
if len(experiments.find_occurrences) != 2:
parser.error(
"Can only find occurrences of non-nested class members.")
# Set reports.
for flag, val in vars(special_opts).items():
if flag.endswith('_report') and val is not None:
report_type = flag[:-7].replace('_', '-')
report_dir = val
options.report_dirs[report_type] = report_dir
# Let quick_and_dirty imply incremental.
if options.quick_and_dirty:
options.incremental = True
# Set target.
if special_opts.modules:
options.build_type = BuildType.MODULE
targets = [BuildSource(None, m, None) for m in special_opts.modules]
return targets, options
elif special_opts.package:
if os.sep in special_opts.package or os.altsep and os.altsep in special_opts.package:
fail("Package name '{}' cannot have a slash in it.".format(
special_opts.package))
options.build_type = BuildType.MODULE
lib_path = [os.getcwd()] + build.mypy_path()
# TODO: use the same cache as the BuildManager will
targets = build.FindModuleCache().find_modules_recursive(
special_opts.package, lib_path)
if not targets:
fail("Can't find package '{}'".format(special_opts.package))
return targets, options
elif special_opts.command:
options.build_type = BuildType.PROGRAM_TEXT
targets = [BuildSource(None, None, '\n'.join(special_opts.command))]
return targets, options
else:
targets = create_source_list(special_opts.files, options)
return targets, options
def process_options(
args: List[str],
stdout: Optional[TextIO] = None,
stderr: Optional[TextIO] = None,
require_targets: bool = True,
server_options: bool = False,
fscache: Optional[FileSystemCache] = None,
program: str = 'mypy',
header: str = HEADER,
) -> Tuple[List[BuildSource], Options]:
"""Parse command line arguments.
If a FileSystemCache is passed in, and package_root options are given,
call fscache.set_package_root() to set the cache's package root.
"""
stdout = stdout or sys.stdout
stderr = stderr or sys.stderr
parser = argparse.ArgumentParser(prog=program,
usage=header,
description=DESCRIPTION,
epilog=FOOTER,
fromfile_prefix_chars='@',
formatter_class=AugmentedHelpFormatter,
add_help=False)
strict_flag_names = [] # type: List[str]
strict_flag_assignments = [] # type: List[Tuple[str, bool]]
def add_invertible_flag(
flag: str,
*,
inverse: Optional[str] = None,
default: bool,
dest: Optional[str] = None,
help: str,
strict_flag: bool = False,
group: Optional[argparse._ActionsContainer] = None) -> None:
if inverse is None:
inverse = invert_flag_name(flag)
if group is None:
group = parser
if help is not argparse.SUPPRESS:
help += " (inverse: {})".format(inverse)
arg = group.add_argument(
flag,
action='store_false' if default else 'store_true',
dest=dest,
help=help)
dest = arg.dest
arg = group.add_argument(
inverse,
action='store_true' if default else 'store_false',
dest=dest,
help=argparse.SUPPRESS)
if strict_flag:
assert dest is not None
strict_flag_names.append(flag)
strict_flag_assignments.append((dest, not default))
# Unless otherwise specified, arguments will be parsed directly onto an
# Options object. Options that require further processing should have
# their `dest` prefixed with `special-opts:`, which will cause them to be
# parsed into the separate special_opts namespace object.
# Note: we have a style guide for formatting the mypy --help text. See
# https://github.com/python/mypy/wiki/Documentation-Conventions
general_group = parser.add_argument_group(title='Optional arguments')
general_group.add_argument('-h',
'--help',
action='help',
help="Show this help message and exit")
general_group.add_argument('-v',
'--verbose',
action='count',
dest='verbosity',
help="More verbose messages")
general_group.add_argument('-V',
'--version',
action='version',
version='%(prog)s ' + __version__,
help="Show program's version number and exit")
config_group = parser.add_argument_group(
title='Config file',
description="Use a config file instead of command line arguments. "
"This is useful if you are using many flags or want "
"to set different options per each module.")
config_group.add_argument(
'--config-file',
help="Configuration file, must have a [mypy] section "
"(defaults to {})".format(', '.join(defaults.CONFIG_FILES)))
add_invertible_flag(
'--warn-unused-configs',
default=False,
strict_flag=True,
help="Warn about unused '[mypy-<pattern>]' config sections",
group=config_group)
imports_group = parser.add_argument_group(
title='Import discovery',
description="Configure how imports are discovered and followed.")
imports_group.add_argument(
'--ignore-missing-imports',
action='store_true',
help="Silently ignore imports of missing modules")
imports_group.add_argument('--follow-imports',
choices=['normal', 'silent', 'skip', 'error'],
default='normal',
help="How to treat imports (default normal)")
imports_group.add_argument(
'--python-executable',
action='store',
metavar='EXECUTABLE',
help="Python executable used for finding PEP 561 compliant installed"
" packages and stubs",
dest='special-opts:python_executable')
imports_group.add_argument(
'--no-site-packages',
action='store_true',
dest='special-opts:no_executable',
help="Do not search for installed PEP 561 compliant packages")
imports_group.add_argument(
'--no-silence-site-packages',
action='store_true',
help="Do not silence errors in PEP 561 compliant installed packages")
add_invertible_flag(
'--namespace-packages',
default=False,
help="Support namespace packages (PEP 420, __init__.py-less)",
group=imports_group)
platform_group = parser.add_argument_group(
title='Platform configuration',
description="Type check code assuming it will be run under certain "
"runtime conditions. By default, mypy assumes your code "
"will be run using the same operating system and Python "
"version you are using to run mypy itself.")
platform_group.add_argument(
'--python-version',
type=parse_version,
metavar='x.y',
help='Type check code assuming it will be running on Python x.y',
dest='special-opts:python_version')
platform_group.add_argument(
'-2',
'--py2',
dest='special-opts:python_version',
action='store_const',
const=defaults.PYTHON2_VERSION,
help="Use Python 2 mode (same as --python-version 2.7)")
platform_group.add_argument(
'--platform',
action='store',
metavar='PLATFORM',
help="Type check special-cased code for the given OS platform "
"(defaults to sys.platform)")
platform_group.add_argument(
'--always-true',
metavar='NAME',
action='append',
default=[],
help="Additional variable to be considered True (may be repeated)")
platform_group.add_argument(
'--always-false',
metavar='NAME',
action='append',
default=[],
help="Additional variable to be considered False (may be repeated)")
disallow_any_group = parser.add_argument_group(
title='Dynamic typing',
description=
"Disallow the use of the dynamic 'Any' type under certain conditions.")
disallow_any_group.add_argument(
'--disallow-any-unimported',
default=False,
action='store_true',
help="Disallow Any types resulting from unfollowed imports")
add_invertible_flag(
'--disallow-subclassing-any',
default=False,
strict_flag=True,
help="Disallow subclassing values of type 'Any' when defining classes",
group=disallow_any_group)
disallow_any_group.add_argument(
'--disallow-any-expr',
default=False,
action='store_true',
help='Disallow all expressions that have type Any')
disallow_any_group.add_argument(
'--disallow-any-decorated',
default=False,
action='store_true',
help='Disallow functions that have Any in their signature '
'after decorator transformation')
disallow_any_group.add_argument(
'--disallow-any-explicit',
default=False,
action='store_true',
help='Disallow explicit Any in type positions')
add_invertible_flag(
'--disallow-any-generics',
default=False,
strict_flag=True,
help='Disallow usage of generic types that do not specify explicit type '
'parameters',
group=disallow_any_group)
untyped_group = parser.add_argument_group(
title='Untyped definitions and calls',
description="Configure how untyped definitions and calls are handled. "
"Note: by default, mypy ignores any untyped function definitions "
"and assumes any calls to such functions have a return "
"type of 'Any'.")
add_invertible_flag(
'--disallow-untyped-calls',
default=False,
strict_flag=True,
help="Disallow calling functions without type annotations"
" from functions with type annotations",
group=untyped_group)
add_invertible_flag(
'--disallow-untyped-defs',
default=False,
strict_flag=True,
help="Disallow defining functions without type annotations"
" or with incomplete type annotations",
group=untyped_group)
add_invertible_flag(
'--disallow-incomplete-defs',
default=False,
strict_flag=True,
help="Disallow defining functions with incomplete type annotations",
group=untyped_group)
add_invertible_flag(
'--check-untyped-defs',
default=False,
strict_flag=True,
help="Type check the interior of functions without type annotations",
group=untyped_group)
add_invertible_flag(
'--disallow-untyped-decorators',
default=False,
strict_flag=True,
help="Disallow decorating typed functions with untyped decorators",
group=untyped_group)
none_group = parser.add_argument_group(
title='None and Optional handling',
description=
"Adjust how values of type 'None' are handled. For more context on "
"how mypy handles values of type 'None', see: "
"mypy.readthedocs.io/en/latest/kinds_of_types.html#no-strict-optional")
add_invertible_flag(
'--no-implicit-optional',
default=False,
strict_flag=True,
help="Don't assume arguments with default values of None are Optional",
group=none_group)
none_group.add_argument('--strict-optional',
action='store_true',
help=argparse.SUPPRESS)
none_group.add_argument(
'--no-strict-optional',
action='store_false',
dest='strict_optional',
help="Disable strict Optional checks (inverse: --strict-optional)")
none_group.add_argument(
'--strict-optional-whitelist',
metavar='GLOB',
nargs='*',
help="Suppress strict Optional errors in all but the provided files; "
"implies --strict-optional (may suppress certain other errors "
"in non-whitelisted files)")
lint_group = parser.add_argument_group(
title='Warnings',
description="Detect code that is sound but redundant or problematic.")
add_invertible_flag(
'--warn-redundant-casts',
default=False,
strict_flag=True,
help="Warn about casting an expression to its inferred type",
group=lint_group)
add_invertible_flag('--warn-unused-ignores',
default=False,
strict_flag=True,
help="Warn about unneeded '# type: ignore' comments",
group=lint_group)
add_invertible_flag(
'--no-warn-no-return',
dest='warn_no_return',
default=True,
help="Do not warn about functions that end without returning",
group=lint_group)
add_invertible_flag('--warn-return-any',
default=False,
strict_flag=True,
help="Warn about returning values of type Any"
" from non-Any typed functions",
group=lint_group)
# Note: this group is intentionally added here even though we don't add
# --strict to this group near the end.
#
# That way, this group will appear after the various strictness groups
# but before the remaining flags.
# We add `--strict` near the end so we don't accidentally miss any strictness
# flags that are added after this group.
strictness_group = parser.add_argument_group(
title='Other strictness checks')
add_invertible_flag(
'--allow-untyped-globals',
default=False,
strict_flag=False,
help="Suppress toplevel errors caused by missing annotations",
group=strictness_group)
add_invertible_flag(
'--allow-redefinition',
default=False,
strict_flag=False,
help="Allow unconditional variable redefinition with a new type",
group=strictness_group)
add_invertible_flag(
'--strict-equality',
default=False,
strict_flag=False,
help="Prohibit equality, identity, and container checks for"
" non-overlapping types",
group=strictness_group)
incremental_group = parser.add_argument_group(
title='Incremental mode',
description=
"Adjust how mypy incrementally type checks and caches modules. "
"Mypy caches type information about modules into a cache to "
"let you speed up future invocations of mypy. Also see "
"mypy's daemon mode: "
"mypy.readthedocs.io/en/latest/mypy_daemon.html#mypy-daemon")
incremental_group.add_argument('-i',
'--incremental',
action='store_true',
help=argparse.SUPPRESS)
incremental_group.add_argument(
'--no-incremental',
action='store_false',
dest='incremental',
help="Disable module cache (inverse: --incremental)")
incremental_group.add_argument(
'--cache-dir',
action='store',
metavar='DIR',
help="Store module cache info in the given folder in incremental mode "
"(defaults to '{}')".format(defaults.CACHE_DIR))
add_invertible_flag('--sqlite-cache',
default=False,
help="Use a sqlite database to store the cache",
group=incremental_group)
incremental_group.add_argument(
'--cache-fine-grained',
action='store_true',
help=
"Include fine-grained dependency information in the cache for the mypy daemon"
)
incremental_group.add_argument(
'--skip-version-check',
action='store_true',
help="Allow using cache written by older mypy version")
incremental_group.add_argument(
'--skip-cache-mtime-checks',
action='store_true',
help="Skip cache internal consistency checks based on mtime")
internals_group = parser.add_argument_group(
title='Mypy internals',
description="Debug and customize mypy internals.")
internals_group.add_argument('--pdb',
action='store_true',
help="Invoke pdb on fatal error")
internals_group.add_argument('--show-traceback',
'--tb',
action='store_true',
help="Show traceback on fatal error")
internals_group.add_argument('--raise-exceptions',
action='store_true',
help="Raise exception on fatal error")
internals_group.add_argument('--custom-typing',
metavar='MODULE',
dest='custom_typing_module',
help="Use a custom typing module")
internals_group.add_argument('--custom-typeshed-dir',
metavar='DIR',
help="Use the custom typeshed in DIR")
add_invertible_flag(
'--warn-incomplete-stub',
default=False,
help="Warn if missing type annotation in typeshed, only relevant with"
" --disallow-untyped-defs or --disallow-incomplete-defs enabled",
group=internals_group)
internals_group.add_argument(
'--shadow-file',
nargs=2,
metavar=('SOURCE_FILE', 'SHADOW_FILE'),
dest='shadow_file',
action='append',
help="When encountering SOURCE_FILE, read and type check "
"the contents of SHADOW_FILE instead.")
add_invertible_flag('--fast-exit',
default=False,
help=argparse.SUPPRESS,
group=internals_group)
add_invertible_flag('--new-semantic-analyzer',
default=False,
help=argparse.SUPPRESS,
group=internals_group)
error_group = parser.add_argument_group(
title='Error reporting',
description="Adjust the amount of detail shown in error messages.")
add_invertible_flag(
'--show-error-context',
default=False,
dest='show_error_context',
help='Precede errors with "note:" messages explaining context',
group=error_group)
add_invertible_flag('--show-column-numbers',
default=False,
help="Show column numbers in error messages",
group=error_group)
strict_help = "Strict mode; enables the following flags: {}".format(
", ".join(strict_flag_names))
strictness_group.add_argument('--strict',
action='store_true',
dest='special-opts:strict',
help=strict_help)
report_group = parser.add_argument_group(
title='Report generation',
description='Generate a report in the specified format.')
for report_type in sorted(defaults.REPORTER_NAMES):
report_group.add_argument('--%s-report' %
report_type.replace('_', '-'),
metavar='DIR',
dest='special-opts:%s_report' % report_type)
other_group = parser.add_argument_group(title='Miscellaneous')
other_group.add_argument('--quickstart-file', help=argparse.SUPPRESS)
other_group.add_argument('--junit-xml',
help="Write junit.xml to the given file")
other_group.add_argument(
'--scripts-are-modules',
action='store_true',
help="Script x becomes module x instead of __main__")
other_group.add_argument(
'--find-occurrences',
metavar='CLASS.MEMBER',
dest='special-opts:find_occurrences',
help="Print out all usages of a class member (experimental)")
if server_options:
# TODO: This flag is superfluous; remove after a short transition (2018-03-16)
other_group.add_argument('--experimental',
action='store_true',
dest='fine_grained_incremental',
help="Enable fine-grained incremental mode")
other_group.add_argument(
'--use-fine-grained-cache',
action='store_true',
help="Use the cache in fine-grained incremental mode")
# hidden options
parser.add_argument('--stats',
action='store_true',
dest='dump_type_stats',
help=argparse.SUPPRESS)
parser.add_argument('--inferstats',
action='store_true',
dest='dump_inference_stats',
help=argparse.SUPPRESS)
# --debug-cache will disable any cache-related compressions/optimizations,
# which will make the cache writing process output pretty-printed JSON (which
# is easier to debug).
parser.add_argument('--debug-cache',
action='store_true',
help=argparse.SUPPRESS)
# --dump-deps will dump all fine-grained dependencies to stdout
parser.add_argument('--dump-deps',
action='store_true',
help=argparse.SUPPRESS)
# --dump-graph will dump the contents of the graph of SCCs and exit.
parser.add_argument('--dump-graph',
action='store_true',
help=argparse.SUPPRESS)
# --semantic-analysis-only does exactly that.
parser.add_argument('--semantic-analysis-only',
action='store_true',
help=argparse.SUPPRESS)
# --local-partial-types disallows partial types spanning module top level and a function
# (implicitly defined in fine-grained incremental mode)
parser.add_argument('--local-partial-types',
action='store_true',
help=argparse.SUPPRESS)
# --logical-deps adds some more dependencies that are not semantically needed, but
# may be helpful to determine relative importance of classes and functions for overall
# type precision in a code base. It also _removes_ some deps, so this flag should be never
# used except for generating code stats. This also automatically enables --cache-fine-grained.
# NOTE: This is an experimental option that may be modified or removed at any time.
parser.add_argument('--logical-deps',
action='store_true',
help=argparse.SUPPRESS)
# --bazel changes some behaviors for use with Bazel (https://bazel.build).
parser.add_argument('--bazel', action='store_true', help=argparse.SUPPRESS)
# --package-root adds a directory below which directories are considered
# packages even without __init__.py. May be repeated.
parser.add_argument('--package-root',
metavar='ROOT',
action='append',
default=[],
help=argparse.SUPPRESS)
# --cache-map FILE ... gives a mapping from source files to cache files.
# Each triple of arguments is a source file, a cache meta file, and a cache data file.
# Modules not mentioned in the file will go through cache_dir.
# Must be followed by another flag or by '--' (and then only file args may follow).
parser.add_argument('--cache-map',
nargs='+',
dest='special-opts:cache_map',
help=argparse.SUPPRESS)
# options specifying code to check
code_group = parser.add_argument_group(
title="Running code",
description=
"Specify the code you want to type check. For more details, see "
"mypy.readthedocs.io/en/latest/running_mypy.html#running-mypy")
code_group.add_argument(
'-m',
'--module',
action='append',
metavar='MODULE',
default=[],
dest='special-opts:modules',
help="Type-check module; can repeat for more modules")
code_group.add_argument(
'-p',
'--package',
action='append',
metavar='PACKAGE',
default=[],
dest='special-opts:packages',
help="Type-check package recursively; can be repeated")
code_group.add_argument('-c',
'--command',
action='append',
metavar='PROGRAM_TEXT',
dest='special-opts:command',
help="Type-check program passed in as string")
code_group.add_argument(metavar='files',
nargs='*',
dest='special-opts:files',
help="Type-check given files or directories")
# Parse arguments once into a dummy namespace so we can get the
# filename for the config file and know if the user requested all strict options.
dummy = argparse.Namespace()
parser.parse_args(args, dummy)
config_file = dummy.config_file
# Don't explicitly test if "config_file is not None" for this check.
# This lets `--config-file=` (an empty string) be used to disable all config files.
if config_file and not os.path.exists(config_file):
parser.error("Cannot find config file '%s'" % config_file)
# Parse config file first, so command line can override.
options = Options()
parse_config_file(options, config_file, stdout, stderr)
# Set strict flags before parsing (if strict mode enabled), so other command
# line options can override.
if getattr(dummy, 'special-opts:strict'): # noqa
for dest, value in strict_flag_assignments:
setattr(options, dest, value)
# Parse command line for real, using a split namespace.
special_opts = argparse.Namespace()
parser.parse_args(args,
SplitNamespace(options, special_opts, 'special-opts:'))
# The python_version is either the default, which can be overridden via a config file,
# or stored in special_opts and is passed via the command line.
options.python_version = special_opts.python_version or options.python_version
try:
infer_python_executable(options, special_opts)
except PythonExecutableInferenceError as e:
parser.error(str(e))
if special_opts.no_executable:
options.python_executable = None
# Paths listed in the config file will be ignored if any paths are passed on
# the command line.
if options.files and not special_opts.files:
special_opts.files = options.files
# Check for invalid argument combinations.
if require_targets:
code_methods = sum(
bool(c) for c in [
special_opts.modules +
special_opts.packages, special_opts.command, special_opts.files
])
if code_methods == 0:
parser.error("Missing target module, package, files, or command.")
elif code_methods > 1:
parser.error(
"May only specify one of: module/package, files, or command.")
# Check for overlapping `--always-true` and `--always-false` flags.
overlap = set(options.always_true) & set(options.always_false)
if overlap:
parser.error(
"You can't make a variable always true and always false (%s)" %
', '.join(sorted(overlap)))
# Set build flags.
if options.strict_optional_whitelist is not None:
# TODO: Deprecate, then kill this flag
options.strict_optional = True
if special_opts.find_occurrences:
state.find_occurrences = special_opts.find_occurrences.split('.')
assert state.find_occurrences is not None
if len(state.find_occurrences) < 2:
parser.error("Can only find occurrences of class members.")
if len(state.find_occurrences) != 2:
parser.error(
"Can only find occurrences of non-nested class members.")
# Set reports.
for flag, val in vars(special_opts).items():
if flag.endswith('_report') and val is not None:
report_type = flag[:-7].replace('_', '-')
report_dir = val
options.report_dirs[report_type] = report_dir
# Process --package-root.
if options.package_root:
process_package_roots(fscache, parser, options)
# Process --cache-map.
if special_opts.cache_map:
if options.sqlite_cache:
parser.error("--cache-map is incompatible with --sqlite-cache")
process_cache_map(parser, special_opts, options)
# Let logical_deps imply cache_fine_grained (otherwise the former is useless).
if options.logical_deps:
options.cache_fine_grained = True
# Set target.
if special_opts.modules + special_opts.packages:
options.build_type = BuildType.MODULE
search_paths = SearchPaths((os.getcwd(), ), tuple(mypy_path()), (), ())
targets = []
# TODO: use the same cache that the BuildManager will
cache = FindModuleCache(search_paths, fscache)
for p in special_opts.packages:
if os.sep in p or os.altsep and os.altsep in p:
fail("Package name '{}' cannot have a slash in it.".format(p),
stderr)
p_targets = cache.find_modules_recursive(p)
if not p_targets:
fail("Can't find package '{}'".format(p), stderr)
targets.extend(p_targets)
for m in special_opts.modules:
targets.append(BuildSource(None, m, None))
return targets, options
elif special_opts.command:
options.build_type = BuildType.PROGRAM_TEXT
targets = [BuildSource(None, None, '\n'.join(special_opts.command))]
return targets, options
else:
try:
targets = create_source_list(special_opts.files, options, fscache)
# Variable named e2 instead of e to work around mypyc bug #620
# which causes issues when using the same variable to catch
# exceptions of different types.
except InvalidSourceList as e2:
fail(str(e2), stderr)
return targets, options
def run_case(self, testcase: DataDrivenTestCase) -> None:
bench = testcase.config.getoption('--bench', False) and 'Benchmark' in testcase.name
# setup.py wants to be run from the root directory of the package, which we accommodate
# by chdiring into tmp/
with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase), (
chdir_manager('tmp')):
text = '\n'.join(testcase.input)
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.strict_optional = True
# N.B: We try to (and ought to!) run with the current
# version of python, since we are going to link and run
# against the current version of python.
# But a lot of the tests use type annotations so we can't say it is 3.5.
options.python_version = max(sys.version_info[:2], (3, 6))
options.export_types = True
options.preserve_asts = True
# Avoid checking modules/packages named 'unchecked', to provide a way
# to test interacting with code we don't have types for.
options.per_module_options['unchecked.*'] = {'follow_imports': 'error'}
workdir = 'build'
os.mkdir(workdir)
source_path = 'native.py'
with open(source_path, 'w', encoding='utf-8') as f:
f.write(text)
with open('interpreted.py', 'w', encoding='utf-8') as f:
f.write(text)
shutil.copyfile(TESTUTIL_PATH, 'testutil.py')
source = build.BuildSource(source_path, 'native', text)
sources = [source]
module_names = ['native']
module_paths = [os.path.abspath('native.py')]
# Hard code another module name to compile in the same compilation unit.
to_delete = []
for fn, text in testcase.files:
fn = os.path.relpath(fn, test_temp_dir)
if os.path.basename(fn).startswith('other'):
name = os.path.basename(fn).split('.')[0]
module_names.append(name)
sources.append(build.BuildSource(fn, name, text))
to_delete.append(fn)
module_paths.append(os.path.abspath(fn))
shutil.copyfile(fn,
os.path.join(os.path.dirname(fn), name + '_interpreted.py'))
for source in sources:
options.per_module_options.setdefault(source.module, {})['mypyc'] = True
if len(module_names) == 1:
lib_name = None # type: Optional[str]
else:
lib_name = shared_lib_name([source.module for source in sources])
try:
result = emitmodule.parse_and_typecheck(
sources=sources,
options=options,
alt_lib_path='.')
errors = Errors()
compiler_options = CompilerOptions(multi_file=self.multi_file)
cfiles = emitmodule.compile_modules_to_c(
result,
module_names=module_names,
shared_lib_name=lib_name,
compiler_options=compiler_options,
errors=errors,
)
if errors.num_errors:
errors.flush_errors()
assert False, "Compile error"
except CompileError as e:
for line in e.messages:
print(line)
assert False, 'Compile error'
for cfile, ctext in cfiles:
with open(os.path.join(workdir, cfile), 'w', encoding='utf-8') as f:
f.write(ctext)
setup_file = os.path.abspath(os.path.join(workdir, 'setup.py'))
with open(setup_file, 'w') as f:
f.write(setup_format.format(module_paths))
if not run_setup(setup_file, ['build_ext', '--inplace']):
if testcase.config.getoption('--mypyc-showc'):
show_c(cfiles)
assert False, "Compilation failed"
# Assert that an output file got created
suffix = 'pyd' if sys.platform == 'win32' else 'so'
assert glob.glob('native.*.{}'.format(suffix))
for p in to_delete:
os.remove(p)
driver_path = 'driver.py'
env = os.environ.copy()
env['MYPYC_RUN_BENCH'] = '1' if bench else '0'
# XXX: This is an ugly hack.
if 'MYPYC_RUN_GDB' in os.environ:
if platform.system() == 'Darwin':
subprocess.check_call(['lldb', '--', sys.executable, driver_path], env=env)
assert False, ("Test can't pass in lldb mode. (And remember to pass -s to "
"pytest)")
elif platform.system() == 'Linux':
subprocess.check_call(['gdb', '--args', sys.executable, driver_path], env=env)
assert False, ("Test can't pass in gdb mode. (And remember to pass -s to "
"pytest)")
else:
assert False, 'Unsupported OS'
proc = subprocess.Popen([sys.executable, driver_path], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, env=env)
output = proc.communicate()[0].decode('utf8')
outlines = output.splitlines()
if testcase.config.getoption('--mypyc-showc'):
show_c(cfiles)
if proc.returncode != 0:
print()
print('*** Exit status: %d' % proc.returncode)
# Verify output.
if bench:
print('Test output:')
print(output)
else:
assert_test_output(testcase, outlines, 'Invalid output')
assert proc.returncode == 0
def parse_config_file(options: Options, filename: Optional[str]) -> None:
"""Parse a config file into an Options object.
Errors are written to stderr but are not fatal.
If filename is None, fall back to default config file and then
to setup.cfg.
"""
if filename is not None:
config_files = (filename, ) # type: Tuple[str, ...]
else:
config_files = (defaults.CONFIG_FILE, ) + SHARED_CONFIG_FILES
parser = configparser.RawConfigParser()
for config_file in config_files:
if not os.path.exists(config_file):
continue
try:
parser.read(config_file)
except configparser.Error as err:
print("%s: %s" % (config_file, err), file=sys.stderr)
else:
file_read = config_file
options.config_file = file_read
break
else:
return
if 'mypy' not in parser:
if filename or file_read not in SHARED_CONFIG_FILES:
print("%s: No [mypy] section in config file" % file_read,
file=sys.stderr)
else:
section = parser['mypy']
prefix = '%s: [%s]' % (file_read, 'mypy')
updates, report_dirs = parse_section(prefix, options, section)
for k, v in updates.items():
setattr(options, k, v)
options.report_dirs.update(report_dirs)
for name, section in parser.items():
if name.startswith('mypy-'):
prefix = '%s: [%s]' % (file_read, name)
updates, report_dirs = parse_section(prefix, options, section)
if report_dirs:
print(
"%s: Per-module sections should not specify reports (%s)" %
(prefix, ', '.join(s + '_report'
for s in sorted(report_dirs))),
file=sys.stderr)
if set(updates) - Options.PER_MODULE_OPTIONS:
print(
"%s: Per-module sections should only specify per-module flags (%s)"
% (prefix, ', '.join(
sorted(set(updates) - Options.PER_MODULE_OPTIONS))),
file=sys.stderr)
updates = {
k: v
for k, v in updates.items()
if k in Options.PER_MODULE_OPTIONS
}
globs = name[5:]
for glob in globs.split(','):
# For backwards compatibility, replace (back)slashes with dots.
glob = glob.replace(os.sep, '.')
if os.altsep:
glob = glob.replace(os.altsep, '.')
pattern = re.compile(fnmatch.translate(glob))
options.per_module_options[pattern] = updates
options.unused_configs[pattern] = glob
def test_coherence(self) -> None:
options = Options()
_, parsed_options = process_options([], require_targets=False)
assert_equal(options, parsed_options)
