def build(self,
source: str,
python_version: Tuple[int, int]) -> Tuple[List[str],
Optional[Dict[str, MypyFile]],
Optional[Dict[Expression, Type]]]:
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.cache_dir = os.devnull
options.python_version = python_version
try:
result = build.build(sources=[BuildSource('main', None, source)],
options=options,
alt_lib_path=test_temp_dir)
except CompileError as e:
# TODO: Should perhaps not return None here.
return e.messages, None, None
return result.errors, result.files, result.types
def build(self, source: str) -> Optional[BuildResult]:
options = Options()
options.incremental = True
options.fine_grained_incremental = True
options.use_builtins_fixtures = True
options.show_traceback = True
options.python_version = PYTHON3_VERSION
main_path = os.path.join(test_temp_dir, 'main')
with open(main_path, 'w') as f:
f.write(source)
try:
result = build.build(sources=[BuildSource(main_path, None, None)],
options=options,
alt_lib_path=test_temp_dir)
except CompileError as e:
# TODO: Is it okay to return None?
return None
return result
def _make_manager(self) -> BuildManager:
errors = Errors()
options = Options()
fscache = FileSystemCache()
manager = BuildManager(
data_dir='',
lib_path=[],
ignore_prefix='',
source_set=BuildSourceSet([]),
reports=Reports('', {}),
options=options,
version_id=__version__,
plugin=Plugin(options),
errors=errors,
flush_errors=lambda msgs, serious: None,
fscache=fscache,
)
return manager
def build_ir_for_single_file(input_lines: List[str]) -> List[FuncIR]:
program_text = '\n'.join(input_lines)
options = Options()
options.show_traceback = True
options.use_builtins_fixtures = True
options.strict_optional = True
source = build.BuildSource('main', '__main__', program_text)
# Construct input as a single single.
# Parse and type check the input program.
result = build.build(sources=[source],
options=options,
alt_lib_path=test_temp_dir)
if result.errors:
raise CompileError(result.errors)
module = genops.build_ir(result.files['__main__'], result.types)
return module.functions
def do_daemon(args: argparse.Namespace) -> None:
"""Serve requests in the foreground."""
# Lazy import so this import doesn't slow down other commands.
from mypy.dmypy_server import Server, process_start_options
if args.options_data:
from mypy.options import Options
options_dict, timeout, log_file = pickle.loads(base64.b64decode(args.options_data))
options_obj = Options()
options = options_obj.apply_changes(options_dict)
if log_file:
sys.stdout = sys.stderr = open(log_file, 'a', buffering=1)
fd = sys.stdout.fileno()
os.dup2(fd, 2)
os.dup2(fd, 1)
else:
options = process_start_options(args.flags, allow_sources=False)
timeout = args.timeout
Server(options, args.status_file, timeout=timeout).serve()
def build(
self, source: str
) -> Tuple[List[str], Optional[BuildManager], Dict[str, State]]:
options = Options()
options.incremental = True
options.use_builtins_fixtures = True
options.show_traceback = True
main_path = os.path.join(test_temp_dir, 'main')
with open(main_path, 'w') as f:
f.write(source)
try:
result = build.build(sources=[BuildSource(main_path, None, None)],
options=options,
alt_lib_path=test_temp_dir)
except CompileError as e:
# TODO: Is it okay to return None?
return e.messages, None, {}
return result.errors, result.manager, result.graph
def parse(source: Union[str, bytes],
fnam: str,
module: Optional[str],
errors: Optional[Errors] = None,
options: Optional[Options] = None) -> MypyFile:
"""Parse a source file, without doing any semantic analysis.
Return the parse tree. If errors is not provided, raise ParseError
on failure. Otherwise, use the errors object to report parse errors.
"""
raise_on_error = False
if errors is None:
errors = Errors()
raise_on_error = True
if options is None:
options = Options()
errors.set_file(fnam, module)
is_stub_file = fnam.endswith('.pyi')
try:
if is_stub_file:
feature_version = defaults.PYTHON3_VERSION[1]
else:
assert options.python_version[0] >= 3
feature_version = options.python_version[1]
ast = ast3.parse(source, fnam, 'exec', feature_version=feature_version)
tree = ASTConverter(
options=options,
is_stub=is_stub_file,
errors=errors,
).visit(ast)
tree.path = fnam
tree.is_stub = is_stub_file
except SyntaxError as e:
errors.reportErrorCode(errorcode.SYNTAX_ERROR(e.msg),
e.lineno,
e.offset,
blocker=True)
tree = MypyFile([], [], False, set())
if raise_on_error and errors.is_errors():
errors.raise_error()
return tree
def parse(source: Union[str, bytes],
fnam: str,
module: Optional[str],
errors: Optional[Errors] = None,
options: Optional[Options] = None) -> MypyFile:
"""Parse a source file, without doing any semantic analysis.
Return the parse tree. If errors is not provided, raise ParseError
on failure. Otherwise, use the errors object to report parse errors.
"""
raise_on_error = False
if errors is None:
errors = Errors()
raise_on_error = True
if options is None:
options = Options()
errors.set_file(fnam, module)
is_stub_file = fnam.endswith('.pyi')
try:
assert options.python_version[0] < 3 and not is_stub_file
# Disable deprecation warnings about <>.
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
ast = ast27.parse(source, fnam, 'exec')
tree = ASTConverter(
options=options,
errors=errors,
).visit(ast)
assert isinstance(tree, MypyFile)
tree.path = fnam
tree.is_stub = is_stub_file
except SyntaxError as e:
errors.report(e.lineno if e.lineno is not None else -1,
e.offset,
e.msg,
blocker=True,
code=codes.SYNTAX)
tree = MypyFile([], [], False, {})
if raise_on_error and errors.is_errors():
errors.raise_error()
return tree
def build_stubs(mod):
data_dir = default_data_dir(None)
options = Options()
options.python_version = (3, 6)
lib_path = default_lib_path(data_dir,
options.python_version,
custom_typeshed_dir=None)
sources = find_modules_recursive(mod, lib_path)
try:
res = build.build(sources=sources, options=options)
messages = res.errors
except CompileError as error:
messages = error.messages
if messages:
for msg in messages:
print(msg)
sys.exit(1)
return res.files
def test_find_sources_in_dir_namespace(self) -> None:
options = Options()
options.namespace_packages = True
files = {
"/pkg/a1/b/c/d/e.py",
"/pkg/a1/b/f.py",
"/pkg/a2/__init__.py",
"/pkg/a2/b/c/d/e.py",
"/pkg/a2/b/f.py",
}
finder = SourceFinder(FakeFSCache(files), options)
assert find_sources_in_dir(finder, "/") == [
("a2", "/pkg"),
("a2.b.c.d.e", "/pkg"),
("a2.b.f", "/pkg"),
("e", "/pkg/a1/b/c/d"),
("f", "/pkg/a1/b"),
]
def dump(fname: str,
python_version: Tuple[int, int],
quiet: bool = False) -> None:
"""
Parameters
----------
fname : str
DESCRIPTION.
python_version : Tuple[int, int]
DESCRIPTION.
quiet : bool, optional
DESCRIPTION. The default is False.
Returns
-------
None
DESCRIPTION.
"""
options = Options()
options.python_version = python_version
with open(fname, 'rb') as f_name:
string = f_name.read()
tree = parse(string, fname, None, errors=None, options=options)
from pudb import set_trace
seen = set()
if not quiet:
lst = tree.defs
for xs in lst:
if isinstance(xs, mypy.nodes.FuncDef):
seen.add((xs.line, xs.name))
if isinstance(xs, mypy.nodes.ExpressionStmt):
resp = xs.expr
for ix in resp.args:
seen.add((ix.line, ix))
if isinstance(xs, mypy.nodes.Decorator):
for node in xs.decorators:
seen.add((xs.line, xs.name))
for item in sorted(list(seen), key=lambda x: x[0]):
print(item[0], item[1])
def run_case(self, testcase: DataDrivenTestCase) -> None:
src = "\n".join(testcase.input)
options = Options()
options.show_traceback = True
options.python_version = sys.version_info[:2]
if testcase.name.endswith("_36"):
options.python_version = (3, 6)
else:
options.python_version = sys.version_info[:2]
options.plugins = ["trio_typing.plugin"]
# must specify something for config_file, else the plugins don't get loaded
options.config_file = "/dev/null"
result = build.build(
sources=[BuildSource("main", None, src)], options=options
)
assert_string_arrays_equal(
testcase.output,
result.errors,
"Unexpected output from {0.file} line {0.line}".format(testcase),
)
def test_transform(testcase: DataDrivenTestCase) -> None:
"""Perform an identity transform test case."""
try:
src = '\n'.join(testcase.input)
options = Options()
options.use_builtins_fixtures = True
options.semantic_analysis_only = True
options.show_traceback = True
options.python_version = testfile_pyversion(testcase.file)
result = build.build(sources=[BuildSource('main', None, src)],
options=options,
alt_lib_path=test_temp_dir)
a = result.errors
if a:
raise CompileError(a)
# Include string representations of the source files in the actual
# output.
for fnam in sorted(result.files.keys()):
f = result.files[fnam]
# Omit the builtins module and files with a special marker in the
# path.
# TODO the test is not reliable
if (not f.path.endswith((os.sep + 'builtins.pyi',
'typing.pyi',
'abc.pyi'))
and not os.path.basename(f.path).startswith('_')
and not os.path.splitext(
os.path.basename(f.path))[0].endswith('_')):
t = TypeAssertTransformVisitor()
f = t.mypyfile(f)
a += str(f).split('\n')
except CompileError as e:
a = e.messages
if testcase.normalize_output:
a = normalize_error_messages(a)
assert_string_arrays_equal(
testcase.output, a,
'Invalid semantic analyzer output ({}, line {})'.format(testcase.file,
testcase.line))
def run_case(self, testcase: DataDrivenTestCase) -> None:
"""Perform a runtime checking transformation test case."""
with use_custom_builtins(
os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase):
expected_output = remove_comment_lines(testcase.output)
program_text = '\n'.join(testcase.input)
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.strict_optional = True
options.python_version = (3, 6)
options.export_types = True
source = build.BuildSource('main', '__main__', program_text)
try:
# Construct input as a single single.
# Parse and type check the input program.
result = build.build(sources=[source],
options=options,
alt_lib_path=test_temp_dir)
except CompileError as e:
actual = e.messages
else:
if result.errors:
actual = result.errors
else:
modules = genops.build_ir([result.files['__main__']],
result.types)
module = modules[0][1]
actual = []
for fn in module.functions:
if is_empty_module_top_level(fn):
# Skip trivial module top levels that only return.
continue
actual.extend(format_func(fn))
assert_test_output(testcase, actual, 'Invalid source code output',
expected_output)
def build_stubs(id):
data_dir = default_data_dir(None)
options = Options()
options.python_version = (3, 6)
lib_path = default_lib_path(data_dir,
options.python_version,
custom_typeshed_dir=None)
sources = find_modules_recursive(id, lib_path)
if not sources:
sys.exit('Error: Cannot find module {}'.format(repr(id)))
msg = []
try:
res = build.build(sources=sources,
options=options)
msg = res.errors
except CompileError as e:
msg = e.messages
if msg:
for m in msg:
print(m)
sys.exit(1)
return res
def _make_manager(self) -> BuildManager:
errors = Errors()
options = Options()
fscache = FileSystemCache()
search_paths = SearchPaths((), (), (), ())
manager = BuildManager(
data_dir='',
search_paths=search_paths,
ignore_prefix='',
source_set=BuildSourceSet([]),
reports=Reports('', {}),
options=options,
version_id=__version__,
plugin=Plugin(options),
plugins_snapshot={},
errors=errors,
flush_errors=lambda msgs, serious: None,
fscache=fscache,
stdout=sys.stdout,
stderr=sys.stderr,
)
return manager
def build_ir_for_single_file2(input_lines: List[str],
compiler_options: Optional[CompilerOptions] = None
) -> ModuleIR:
program_text = '\n'.join(input_lines)
# By default generate IR compatible with the earliest supported Python C API.
# If a test needs more recent API features, this should be overridden.
compiler_options = compiler_options or CompilerOptions(capi_version=(3, 5))
options = Options()
options.show_traceback = True
options.use_builtins_fixtures = True
options.strict_optional = True
options.python_version = (3, 6)
options.export_types = True
options.preserve_asts = True
options.per_module_options['__main__'] = {'mypyc': True}
source = build.BuildSource('main', '__main__', program_text)
# Construct input as a single single.
# Parse and type check the input program.
result = build.build(sources=[source],
options=options,
alt_lib_path=test_temp_dir)
if result.errors:
raise CompileError(result.errors)
errors = Errors()
modules = build_ir(
[result.files['__main__']], result.graph, result.types,
Mapper({'__main__': None}),
compiler_options, errors)
if errors.num_errors:
raise CompileError(errors.new_messages())
module = list(modules.values())[0]
for fn in module.functions:
assert_func_ir_valid(fn)
return module
def test_parser(testcase):
"""Perform a single parser test case.
The argument contains the description of the test case.
"""
options = Options()
if testcase.file.endswith('python2.test'):
options.python_version = defaults.PYTHON2_VERSION
else:
options.python_version = defaults.PYTHON3_VERSION
try:
n = parse(bytes('\n'.join(testcase.input), 'ascii'),
fnam='main',
errors=None,
options=options)
a = str(n).split('\n')
except CompileError as e:
a = e.messages
assert_string_arrays_equal(testcase.output, a,
'Invalid parser output ({}, line {})'.format(
testcase.file, testcase.line))
def parse(
source: Union[str, bytes],
fnam: str,
module: Optional[str],
errors: Optional[Errors] = None,
options: Options = Options()) -> MypyFile:
"""Parse a source file, without doing any semantic analysis.
Return the parse tree. If errors is not provided, raise ParseError
on failure. Otherwise, use the errors object to report parse errors.
"""
raise_on_error = False
if errors is None:
errors = Errors()
raise_on_error = True
errors.set_file(fnam, module)
is_stub_file = fnam.endswith('.pyi')
try:
assert options.python_version[0] < 3 and not is_stub_file
ast = ast27.parse(source, fnam, 'exec')
tree = ASTConverter(
options=options,
is_stub=is_stub_file,
errors=errors,
).visit(ast)
assert isinstance(tree, MypyFile)
tree.path = fnam
tree.is_stub = is_stub_file
except SyntaxError as e:
errors.report(e.lineno, e.offset, e.msg, blocker=True)
tree = MypyFile([], [], False, set())
if raise_on_error and errors.is_errors():
errors.raise_error()
return tree
def build_ir_for_single_file(input_lines: List[str]) -> List[FuncIR]:
program_text = '\n'.join(input_lines)
options = Options()
options.show_traceback = True
options.use_builtins_fixtures = True
options.strict_optional = True
options.python_version = (3, 6)
options.export_types = True
options.per_module_options['__main__'] = {'mypyc': True}
source = build.BuildSource('main', '__main__', program_text)
# Construct input as a single single.
# Parse and type check the input program.
result = build.build(sources=[source],
options=options,
alt_lib_path=test_temp_dir)
if result.errors:
raise CompileError(result.errors)
_, modules, errors = genops.build_ir([result.files['__main__']], result.graph, result.types)
assert errors == 0
module = modules[0][1]
return module.functions
def test_type_check(mocker, extensions):
mocker.patch("sys.argv", ["", "clean"])
runpy.run_module("gino.ext", run_name="__main__")
result = build(
[BuildSource(None, None, "from gino.ext.demo3 import s3")], Options()
)
assert result.errors
result = build(
[BuildSource(None, None, "from gino.ext.demo1 import s1")], Options()
)
assert result.errors
mocker.patch("sys.argv", ["", "stub"])
runpy.run_module("gino.ext", run_name="__main__")
runpy.run_module("gino.ext", run_name="__main__")
try:
result = build(
[BuildSource(None, None, "from gino.ext.demo1 import s1")], Options()
)
assert not result.errors
result = build(
[BuildSource(None, None, "from gino.ext.demo1 import s2")], Options()
)
assert result.errors
result = build(
[BuildSource(None, None, "from gino.ext.demo2 import s2")], Options()
)
assert not result.errors
result = build(
[BuildSource(None, None, "from gino.ext.demo2 import s1")], Options()
)
assert result.errors
finally:
mocker.patch("sys.argv", ["", "clean"])
runpy.run_module("gino.ext", run_name="__main__")
def test_stubs(args: argparse.Namespace,
use_builtins_fixtures: bool = False) -> int:
"""This is stubtest! It's time to test the stubs!"""
# Load the allowlist. This is a series of strings corresponding to Error.object_desc
# Values in the dict will store whether we used the allowlist entry or not.
allowlist = {
entry: False
for allowlist_file in args.allowlist
for entry in get_allowlist_entries(allowlist_file)
}
allowlist_regexes = {entry: re.compile(entry) for entry in allowlist}
# If we need to generate an allowlist, we store Error.object_desc for each error here.
generated_allowlist = set()
modules = args.modules
if args.check_typeshed:
assert not args.modules, "Cannot pass both --check-typeshed and a list of modules"
modules = get_typeshed_stdlib_modules(args.custom_typeshed_dir)
annoying_modules = {"antigravity", "this"}
modules = [m for m in modules if m not in annoying_modules]
assert modules, "No modules to check"
options = Options()
options.incremental = False
options.custom_typeshed_dir = args.custom_typeshed_dir
options.config_file = args.mypy_config_file
options.use_builtins_fixtures = use_builtins_fixtures
if options.config_file:
def set_strict_flags() -> None: # not needed yet
return
parse_config_file(options, set_strict_flags, options.config_file,
sys.stdout, sys.stderr)
try:
modules = build_stubs(modules,
options,
find_submodules=not args.check_typeshed)
except RuntimeError:
return 1
exit_code = 0
for module in modules:
for error in test_module(module):
# Filter errors
if args.ignore_missing_stub and error.is_missing_stub():
continue
if args.ignore_positional_only and error.is_positional_only_related(
):
continue
if error.object_desc in allowlist:
allowlist[error.object_desc] = True
continue
is_allowlisted = False
for w in allowlist:
if allowlist_regexes[w].fullmatch(error.object_desc):
allowlist[w] = True
is_allowlisted = True
break
if is_allowlisted:
continue
# We have errors, so change exit code, and output whatever necessary
exit_code = 1
if args.generate_allowlist:
generated_allowlist.add(error.object_desc)
continue
print(error.get_description(concise=args.concise))
# Print unused allowlist entries
if not args.ignore_unused_allowlist:
for w in allowlist:
# Don't consider an entry unused if it regex-matches the empty string
# This lets us allowlist errors that don't manifest at all on some systems
if not allowlist[w] and not allowlist_regexes[w].fullmatch(""):
exit_code = 1
print("note: unused allowlist entry {}".format(w))
# Print the generated allowlist
if args.generate_allowlist:
for e in sorted(generated_allowlist):
print(e)
exit_code = 0
return exit_code
def process_options(
args: List[str],
require_targets: bool = True,
server_options: bool = False,
) -> Tuple[List[BuildSource], Options]:
"""Parse command line arguments."""
parser = argparse.ArgumentParser(prog='mypy',
epilog=FOOTER,
fromfile_prefix_chars='@',
formatter_class=AugmentedHelpFormatter)
strict_flag_names = [] # type: List[str]
strict_flag_assignments = [] # type: List[Tuple[str, bool]]
def add_invertible_flag(flag: str,
*,
inverse: Optional[str] = None,
default: bool,
dest: Optional[str] = None,
help: str,
strict_flag: bool = False) -> None:
if inverse is None:
inverse = invert_flag_name(flag)
if help is not argparse.SUPPRESS:
help += " (inverse: {})".format(inverse)
arg = parser.add_argument(
flag, # type: ignore # incorrect stub for add_argument
action='store_false' if default else 'store_true',
dest=dest,
help=help)
dest = arg.dest
arg = parser.add_argument(
inverse, # type: ignore # incorrect stub for add_argument
action='store_true' if default else 'store_false',
dest=dest,
help=argparse.SUPPRESS)
if strict_flag:
assert dest is not None
strict_flag_names.append(flag)
strict_flag_assignments.append((dest, not default))
# Unless otherwise specified, arguments will be parsed directly onto an
# Options object. Options that require further processing should have
# their `dest` prefixed with `special-opts:`, which will cause them to be
# parsed into the separate special_opts namespace object.
parser.add_argument('-v',
'--verbose',
action='count',
dest='verbosity',
help="more verbose messages")
parser.add_argument('-V',
'--version',
action='version',
version='%(prog)s ' + __version__)
parser.add_argument('--python-version',
type=parse_version,
metavar='x.y',
help='use Python x.y')
parser.add_argument(
'--platform',
action='store',
metavar='PLATFORM',
help="typecheck special-cased code for the given OS platform "
"(defaults to sys.platform).")
parser.add_argument('-2',
'--py2',
dest='python_version',
action='store_const',
const=defaults.PYTHON2_VERSION,
help="use Python 2 mode")
parser.add_argument('--ignore-missing-imports',
action='store_true',
help="silently ignore imports of missing modules")
parser.add_argument('--follow-imports',
choices=['normal', 'silent', 'skip', 'error'],
default='normal',
help="how to treat imports (default normal)")
parser.add_argument(
'--disallow-any-unimported',
default=False,
action='store_true',
help="disallow Any types resulting from unfollowed imports")
parser.add_argument('--disallow-any-expr',
default=False,
action='store_true',
help='disallow all expressions that have type Any')
parser.add_argument(
'--disallow-any-decorated',
default=False,
action='store_true',
help='disallow functions that have Any in their signature '
'after decorator transformation')
parser.add_argument('--disallow-any-explicit',
default=False,
action='store_true',
help='disallow explicit Any in type positions')
parser.add_argument(
'--disallow-any-generics',
default=False,
action='store_true',
help='disallow usage of generic types that do not specify explicit '
'type parameters')
add_invertible_flag(
'--disallow-untyped-calls',
default=False,
strict_flag=True,
help="disallow calling functions without type annotations"
" from functions with type annotations")
add_invertible_flag(
'--disallow-untyped-defs',
default=False,
strict_flag=True,
help="disallow defining functions without type annotations"
" or with incomplete type annotations")
add_invertible_flag(
'--disallow-incomplete-defs',
default=False,
strict_flag=True,
help="disallow defining functions with incomplete type annotations")
add_invertible_flag(
'--check-untyped-defs',
default=False,
strict_flag=True,
help="type check the interior of functions without type annotations")
add_invertible_flag(
'--disallow-subclassing-any',
default=False,
strict_flag=True,
help="disallow subclassing values of type 'Any' when defining classes")
add_invertible_flag(
'--warn-incomplete-stub',
default=False,
help="warn if missing type annotation in typeshed, only relevant with"
" --check-untyped-defs enabled")
add_invertible_flag(
'--disallow-untyped-decorators',
default=False,
strict_flag=True,
help="disallow decorating typed functions with untyped decorators")
add_invertible_flag(
'--warn-redundant-casts',
default=False,
strict_flag=True,
help="warn about casting an expression to its inferred type")
add_invertible_flag(
'--no-warn-no-return',
dest='warn_no_return',
default=True,
help="do not warn about functions that end without returning")
add_invertible_flag('--warn-return-any',
default=False,
strict_flag=True,
help="warn about returning values of type Any"
" from non-Any typed functions")
add_invertible_flag('--warn-unused-ignores',
default=False,
strict_flag=True,
help="warn about unneeded '# type: ignore' comments")
add_invertible_flag(
'--warn-unused-configs',
default=False,
strict_flag=True,
help="warn about unused '[mypy-<pattern>]' config sections")
add_invertible_flag(
'--show-error-context',
default=False,
dest='show_error_context',
help='Precede errors with "note:" messages explaining context')
add_invertible_flag(
'--no-implicit-optional',
default=False,
strict_flag=True,
help="don't assume arguments with default values of None are Optional")
parser.add_argument(
'-i',
'--incremental',
action='store_true',
help="enable module cache, (inverse: --no-incremental)")
parser.add_argument('--no-incremental',
action='store_false',
dest='incremental',
help=argparse.SUPPRESS)
parser.add_argument('--quick-and-dirty',
action='store_true',
help="use cache even if dependencies out of date "
"(implies --incremental)")
parser.add_argument(
'--cache-dir',
action='store',
metavar='DIR',
help="store module cache info in the given folder in incremental mode "
"(defaults to '{}')".format(defaults.CACHE_DIR))
parser.add_argument(
'--cache-fine-grained',
action='store_true',
help="include fine-grained dependency information in the cache")
parser.add_argument('--skip-version-check',
action='store_true',
help="allow using cache written by older mypy version")
add_invertible_flag('--strict-optional',
default=False,
strict_flag=True,
help="enable experimental strict Optional checks")
parser.add_argument(
'--strict-optional-whitelist',
metavar='GLOB',
nargs='*',
help="suppress strict Optional errors in all but the provided files "
"(experimental -- read documentation before using!). "
"Implies --strict-optional. Has the undesirable side-effect of "
"suppressing other errors in non-whitelisted files.")
parser.add_argument('--junit-xml',
help="write junit.xml to the given file")
parser.add_argument('--pdb',
action='store_true',
help="invoke pdb on fatal error")
parser.add_argument('--show-traceback',
'--tb',
action='store_true',
help="show traceback on fatal error")
parser.add_argument('--stats',
action='store_true',
dest='dump_type_stats',
help="dump stats")
parser.add_argument('--inferstats',
action='store_true',
dest='dump_inference_stats',
help="dump type inference stats")
parser.add_argument('--custom-typing',
metavar='MODULE',
dest='custom_typing_module',
help="use a custom typing module")
parser.add_argument('--custom-typeshed-dir',
metavar='DIR',
help="use the custom typeshed in DIR")
parser.add_argument('--scripts-are-modules',
action='store_true',
help="Script x becomes module x instead of __main__")
parser.add_argument('--config-file',
help="Configuration file, must have a [mypy] section "
"(defaults to {})".format(defaults.CONFIG_FILE))
add_invertible_flag('--show-column-numbers',
default=False,
help="Show column numbers in error messages")
parser.add_argument(
'--find-occurrences',
metavar='CLASS.MEMBER',
dest='special-opts:find_occurrences',
help="print out all usages of a class member (experimental)")
strict_help = "Strict mode. Enables the following flags: {}".format(
", ".join(strict_flag_names))
parser.add_argument('--strict',
action='store_true',
dest='special-opts:strict',
help=strict_help)
parser.add_argument('--shadow-file',
nargs=2,
metavar=('SOURCE_FILE', 'SHADOW_FILE'),
dest='shadow_file',
help='Typecheck SHADOW_FILE in place of SOURCE_FILE.')
# hidden options
# --debug-cache will disable any cache-related compressions/optimizations,
# which will make the cache writing process output pretty-printed JSON (which
# is easier to debug).
parser.add_argument('--debug-cache',
action='store_true',
help=argparse.SUPPRESS)
# --dump-deps will dump all fine-grained dependencies to stdout
parser.add_argument('--dump-deps',
action='store_true',
help=argparse.SUPPRESS)
# --dump-graph will dump the contents of the graph of SCCs and exit.
parser.add_argument('--dump-graph',
action='store_true',
help=argparse.SUPPRESS)
# --semantic-analysis-only does exactly that.
parser.add_argument('--semantic-analysis-only',
action='store_true',
help=argparse.SUPPRESS)
# --local-partial-types disallows partial types spanning module top level and a function
# (implicitly defined in fine-grained incremental mode)
parser.add_argument('--local-partial-types',
action='store_true',
help=argparse.SUPPRESS)
# deprecated options
parser.add_argument('--disallow-any',
dest='special-opts:disallow_any',
help=argparse.SUPPRESS)
add_invertible_flag('--strict-boolean',
default=False,
help=argparse.SUPPRESS)
parser.add_argument('-f',
'--dirty-stubs',
action='store_true',
dest='special-opts:dirty_stubs',
help=argparse.SUPPRESS)
parser.add_argument('--use-python-path',
action='store_true',
dest='special-opts:use_python_path',
help=argparse.SUPPRESS)
parser.add_argument('-s',
'--silent-imports',
action='store_true',
dest='special-opts:silent_imports',
help=argparse.SUPPRESS)
parser.add_argument('--almost-silent',
action='store_true',
dest='special-opts:almost_silent',
help=argparse.SUPPRESS)
parser.add_argument('--fast-parser',
action='store_true',
dest='special-opts:fast_parser',
help=argparse.SUPPRESS)
parser.add_argument('--no-fast-parser',
action='store_true',
dest='special-opts:no_fast_parser',
help=argparse.SUPPRESS)
if server_options:
parser.add_argument('--experimental',
action='store_true',
dest='fine_grained_incremental',
help="enable fine-grained incremental mode")
parser.add_argument(
'--use-fine-grained-cache',
action='store_true',
help="use the cache in fine-grained incremental mode")
report_group = parser.add_argument_group(
title='report generation',
description='Generate a report in the specified format.')
for report_type in sorted(reporter_classes):
report_group.add_argument('--%s-report' %
report_type.replace('_', '-'),
metavar='DIR',
dest='special-opts:%s_report' % report_type)
code_group = parser.add_argument_group(
title='How to specify the code to type check')
code_group.add_argument(
'-m',
'--module',
action='append',
metavar='MODULE',
dest='special-opts:modules',
help="type-check module; can repeat for more modules")
# TODO: `mypy -p A -p B` currently silently ignores A
# (last option wins). Perhaps -c, -m and -p could just be
# command-line flags that modify how we interpret self.files?
code_group.add_argument('-c',
'--command',
action='append',
metavar='PROGRAM_TEXT',
dest='special-opts:command',
help="type-check program passed in as string")
code_group.add_argument('-p',
'--package',
metavar='PACKAGE',
dest='special-opts:package',
help="type-check all files in a directory")
code_group.add_argument(metavar='files',
nargs='*',
dest='special-opts:files',
help="type-check given files or directories")
# Parse arguments once into a dummy namespace so we can get the
# filename for the config file and know if the user requested all strict options.
dummy = argparse.Namespace()
parser.parse_args(args, dummy)
config_file = dummy.config_file
if config_file is not None and not os.path.exists(config_file):
parser.error("Cannot find config file '%s'" % config_file)
# Parse config file first, so command line can override.
options = Options()
parse_config_file(options, config_file)
# Set strict flags before parsing (if strict mode enabled), so other command
# line options can override.
if getattr(dummy, 'special-opts:strict'):
for dest, value in strict_flag_assignments:
setattr(options, dest, value)
# Parse command line for real, using a split namespace.
special_opts = argparse.Namespace()
parser.parse_args(args,
SplitNamespace(options, special_opts, 'special-opts:'))
# --use-python-path is no longer supported; explain why.
if special_opts.use_python_path:
parser.error(
"Sorry, --use-python-path is no longer supported.\n"
"If you are trying this because your code depends on a library module,\n"
"you should really investigate how to obtain stubs for that module.\n"
"See https://github.com/python/mypy/issues/1411 for more discussion."
)
# Process deprecated options
if special_opts.disallow_any:
print(
"--disallow-any option was split up into multiple flags. "
"See http://mypy.readthedocs.io/en/latest/command_line.html#disallow-any-flags"
)
if options.strict_boolean:
print(
"Warning: --strict-boolean is deprecated; "
"see https://github.com/python/mypy/issues/3195",
file=sys.stderr)
if special_opts.almost_silent:
print(
"Warning: --almost-silent has been replaced by "
"--follow-imports=errors",
file=sys.stderr)
if options.follow_imports == 'normal':
options.follow_imports = 'errors'
elif special_opts.silent_imports:
print(
"Warning: --silent-imports has been replaced by "
"--ignore-missing-imports --follow-imports=skip",
file=sys.stderr)
options.ignore_missing_imports = True
if options.follow_imports == 'normal':
options.follow_imports = 'skip'
if special_opts.dirty_stubs:
print(
"Warning: -f/--dirty-stubs is deprecated and no longer necessary. Mypy no longer "
"checks the git status of stubs.",
file=sys.stderr)
if special_opts.fast_parser:
print("Warning: --fast-parser is now the default (and only) parser.")
if special_opts.no_fast_parser:
print(
"Warning: --no-fast-parser no longer has any effect. The fast parser "
"is now mypy's default and only parser.")
# Check for invalid argument combinations.
if require_targets:
code_methods = sum(
bool(c) for c in [
special_opts.modules, special_opts.command,
special_opts.package, special_opts.files
])
if code_methods == 0:
parser.error("Missing target module, package, files, or command.")
elif code_methods > 1:
parser.error(
"May only specify one of: module, package, files, or command.")
# Set build flags.
if options.strict_optional_whitelist is not None:
# TODO: Deprecate, then kill this flag
options.strict_optional = True
if special_opts.find_occurrences:
experiments.find_occurrences = special_opts.find_occurrences.split('.')
assert experiments.find_occurrences is not None
if len(experiments.find_occurrences) < 2:
parser.error("Can only find occurrences of class members.")
if len(experiments.find_occurrences) != 2:
parser.error(
"Can only find occurrences of non-nested class members.")
# Set reports.
for flag, val in vars(special_opts).items():
if flag.endswith('_report') and val is not None:
report_type = flag[:-7].replace('_', '-')
report_dir = val
options.report_dirs[report_type] = report_dir
# Let quick_and_dirty imply incremental.
if options.quick_and_dirty:
options.incremental = True
# Set target.
if special_opts.modules:
options.build_type = BuildType.MODULE
targets = [BuildSource(None, m, None) for m in special_opts.modules]
return targets, options
elif special_opts.package:
if os.sep in special_opts.package or os.altsep and os.altsep in special_opts.package:
fail("Package name '{}' cannot have a slash in it.".format(
special_opts.package))
options.build_type = BuildType.MODULE
lib_path = [os.getcwd()] + build.mypy_path()
# TODO: use the same cache as the BuildManager will
targets = build.FindModuleCache().find_modules_recursive(
special_opts.package, lib_path)
if not targets:
fail("Can't find package '{}'".format(special_opts.package))
return targets, options
elif special_opts.command:
options.build_type = BuildType.PROGRAM_TEXT
targets = [BuildSource(None, None, '\n'.join(special_opts.command))]
return targets, options
else:
targets = create_source_list(special_opts.files, options)
return targets, options
def process_options(
args: List[str],
stdout: Optional[TextIO] = None,
stderr: Optional[TextIO] = None,
require_targets: bool = True,
server_options: bool = False,
fscache: Optional[FileSystemCache] = None,
program: str = 'mypy',
header: str = HEADER,
) -> Tuple[List[BuildSource], Options]:
"""Parse command line arguments.
If a FileSystemCache is passed in, and package_root options are given,
call fscache.set_package_root() to set the cache's package root.
"""
stdout = stdout or sys.stdout
stderr = stderr or sys.stderr
parser = argparse.ArgumentParser(prog=program,
usage=header,
description=DESCRIPTION,
epilog=FOOTER,
fromfile_prefix_chars='@',
formatter_class=AugmentedHelpFormatter,
add_help=False)
strict_flag_names = [] # type: List[str]
strict_flag_assignments = [] # type: List[Tuple[str, bool]]
def add_invertible_flag(
flag: str,
*,
inverse: Optional[str] = None,
default: bool,
dest: Optional[str] = None,
help: str,
strict_flag: bool = False,
group: Optional[argparse._ActionsContainer] = None) -> None:
if inverse is None:
inverse = invert_flag_name(flag)
if group is None:
group = parser
if help is not argparse.SUPPRESS:
help += " (inverse: {})".format(inverse)
arg = group.add_argument(
flag,
action='store_false' if default else 'store_true',
dest=dest,
help=help)
dest = arg.dest
arg = group.add_argument(
inverse,
action='store_true' if default else 'store_false',
dest=dest,
help=argparse.SUPPRESS)
if strict_flag:
assert dest is not None
strict_flag_names.append(flag)
strict_flag_assignments.append((dest, not default))
# Unless otherwise specified, arguments will be parsed directly onto an
# Options object. Options that require further processing should have
# their `dest` prefixed with `special-opts:`, which will cause them to be
# parsed into the separate special_opts namespace object.
# Note: we have a style guide for formatting the mypy --help text. See
# https://github.com/python/mypy/wiki/Documentation-Conventions
general_group = parser.add_argument_group(title='Optional arguments')
general_group.add_argument('-h',
'--help',
action='help',
help="Show this help message and exit")
general_group.add_argument('-v',
'--verbose',
action='count',
dest='verbosity',
help="More verbose messages")
general_group.add_argument('-V',
'--version',
action='version',
version='%(prog)s ' + __version__,
help="Show program's version number and exit")
config_group = parser.add_argument_group(
title='Config file',
description="Use a config file instead of command line arguments. "
"This is useful if you are using many flags or want "
"to set different options per each module.")
config_group.add_argument(
'--config-file',
help="Configuration file, must have a [mypy] section "
"(defaults to {})".format(', '.join(defaults.CONFIG_FILES)))
add_invertible_flag(
'--warn-unused-configs',
default=False,
strict_flag=True,
help="Warn about unused '[mypy-<pattern>]' config sections",
group=config_group)
imports_group = parser.add_argument_group(
title='Import discovery',
description="Configure how imports are discovered and followed.")
imports_group.add_argument(
'--ignore-missing-imports',
action='store_true',
help="Silently ignore imports of missing modules")
imports_group.add_argument('--follow-imports',
choices=['normal', 'silent', 'skip', 'error'],
default='normal',
help="How to treat imports (default normal)")
imports_group.add_argument(
'--python-executable',
action='store',
metavar='EXECUTABLE',
help="Python executable used for finding PEP 561 compliant installed"
" packages and stubs",
dest='special-opts:python_executable')
imports_group.add_argument(
'--no-site-packages',
action='store_true',
dest='special-opts:no_executable',
help="Do not search for installed PEP 561 compliant packages")
imports_group.add_argument(
'--no-silence-site-packages',
action='store_true',
help="Do not silence errors in PEP 561 compliant installed packages")
add_invertible_flag(
'--namespace-packages',
default=False,
help="Support namespace packages (PEP 420, __init__.py-less)",
group=imports_group)
platform_group = parser.add_argument_group(
title='Platform configuration',
description="Type check code assuming it will be run under certain "
"runtime conditions. By default, mypy assumes your code "
"will be run using the same operating system and Python "
"version you are using to run mypy itself.")
platform_group.add_argument(
'--python-version',
type=parse_version,
metavar='x.y',
help='Type check code assuming it will be running on Python x.y',
dest='special-opts:python_version')
platform_group.add_argument(
'-2',
'--py2',
dest='special-opts:python_version',
action='store_const',
const=defaults.PYTHON2_VERSION,
help="Use Python 2 mode (same as --python-version 2.7)")
platform_group.add_argument(
'--platform',
action='store',
metavar='PLATFORM',
help="Type check special-cased code for the given OS platform "
"(defaults to sys.platform)")
platform_group.add_argument(
'--always-true',
metavar='NAME',
action='append',
default=[],
help="Additional variable to be considered True (may be repeated)")
platform_group.add_argument(
'--always-false',
metavar='NAME',
action='append',
default=[],
help="Additional variable to be considered False (may be repeated)")
disallow_any_group = parser.add_argument_group(
title='Dynamic typing',
description=
"Disallow the use of the dynamic 'Any' type under certain conditions.")
disallow_any_group.add_argument(
'--disallow-any-unimported',
default=False,
action='store_true',
help="Disallow Any types resulting from unfollowed imports")
add_invertible_flag(
'--disallow-subclassing-any',
default=False,
strict_flag=True,
help="Disallow subclassing values of type 'Any' when defining classes",
group=disallow_any_group)
disallow_any_group.add_argument(
'--disallow-any-expr',
default=False,
action='store_true',
help='Disallow all expressions that have type Any')
disallow_any_group.add_argument(
'--disallow-any-decorated',
default=False,
action='store_true',
help='Disallow functions that have Any in their signature '
'after decorator transformation')
disallow_any_group.add_argument(
'--disallow-any-explicit',
default=False,
action='store_true',
help='Disallow explicit Any in type positions')
add_invertible_flag(
'--disallow-any-generics',
default=False,
strict_flag=True,
help='Disallow usage of generic types that do not specify explicit type '
'parameters',
group=disallow_any_group)
untyped_group = parser.add_argument_group(
title='Untyped definitions and calls',
description="Configure how untyped definitions and calls are handled. "
"Note: by default, mypy ignores any untyped function definitions "
"and assumes any calls to such functions have a return "
"type of 'Any'.")
add_invertible_flag(
'--disallow-untyped-calls',
default=False,
strict_flag=True,
help="Disallow calling functions without type annotations"
" from functions with type annotations",
group=untyped_group)
add_invertible_flag(
'--disallow-untyped-defs',
default=False,
strict_flag=True,
help="Disallow defining functions without type annotations"
" or with incomplete type annotations",
group=untyped_group)
add_invertible_flag(
'--disallow-incomplete-defs',
default=False,
strict_flag=True,
help="Disallow defining functions with incomplete type annotations",
group=untyped_group)
add_invertible_flag(
'--check-untyped-defs',
default=False,
strict_flag=True,
help="Type check the interior of functions without type annotations",
group=untyped_group)
add_invertible_flag(
'--disallow-untyped-decorators',
default=False,
strict_flag=True,
help="Disallow decorating typed functions with untyped decorators",
group=untyped_group)
none_group = parser.add_argument_group(
title='None and Optional handling',
description=
"Adjust how values of type 'None' are handled. For more context on "
"how mypy handles values of type 'None', see: "
"mypy.readthedocs.io/en/latest/kinds_of_types.html#no-strict-optional")
add_invertible_flag(
'--no-implicit-optional',
default=False,
strict_flag=True,
help="Don't assume arguments with default values of None are Optional",
group=none_group)
none_group.add_argument('--strict-optional',
action='store_true',
help=argparse.SUPPRESS)
none_group.add_argument(
'--no-strict-optional',
action='store_false',
dest='strict_optional',
help="Disable strict Optional checks (inverse: --strict-optional)")
none_group.add_argument(
'--strict-optional-whitelist',
metavar='GLOB',
nargs='*',
help="Suppress strict Optional errors in all but the provided files; "
"implies --strict-optional (may suppress certain other errors "
"in non-whitelisted files)")
lint_group = parser.add_argument_group(
title='Warnings',
description="Detect code that is sound but redundant or problematic.")
add_invertible_flag(
'--warn-redundant-casts',
default=False,
strict_flag=True,
help="Warn about casting an expression to its inferred type",
group=lint_group)
add_invertible_flag('--warn-unused-ignores',
default=False,
strict_flag=True,
help="Warn about unneeded '# type: ignore' comments",
group=lint_group)
add_invertible_flag(
'--no-warn-no-return',
dest='warn_no_return',
default=True,
help="Do not warn about functions that end without returning",
group=lint_group)
add_invertible_flag('--warn-return-any',
default=False,
strict_flag=True,
help="Warn about returning values of type Any"
" from non-Any typed functions",
group=lint_group)
# Note: this group is intentionally added here even though we don't add
# --strict to this group near the end.
#
# That way, this group will appear after the various strictness groups
# but before the remaining flags.
# We add `--strict` near the end so we don't accidentally miss any strictness
# flags that are added after this group.
strictness_group = parser.add_argument_group(
title='Other strictness checks')
add_invertible_flag(
'--allow-untyped-globals',
default=False,
strict_flag=False,
help="Suppress toplevel errors caused by missing annotations",
group=strictness_group)
add_invertible_flag(
'--allow-redefinition',
default=False,
strict_flag=False,
help="Allow unconditional variable redefinition with a new type",
group=strictness_group)
add_invertible_flag(
'--strict-equality',
default=False,
strict_flag=False,
help="Prohibit equality, identity, and container checks for"
" non-overlapping types",
group=strictness_group)
incremental_group = parser.add_argument_group(
title='Incremental mode',
description=
"Adjust how mypy incrementally type checks and caches modules. "
"Mypy caches type information about modules into a cache to "
"let you speed up future invocations of mypy. Also see "
"mypy's daemon mode: "
"mypy.readthedocs.io/en/latest/mypy_daemon.html#mypy-daemon")
incremental_group.add_argument('-i',
'--incremental',
action='store_true',
help=argparse.SUPPRESS)
incremental_group.add_argument(
'--no-incremental',
action='store_false',
dest='incremental',
help="Disable module cache (inverse: --incremental)")
incremental_group.add_argument(
'--cache-dir',
action='store',
metavar='DIR',
help="Store module cache info in the given folder in incremental mode "
"(defaults to '{}')".format(defaults.CACHE_DIR))
add_invertible_flag('--sqlite-cache',
default=False,
help="Use a sqlite database to store the cache",
group=incremental_group)
incremental_group.add_argument(
'--cache-fine-grained',
action='store_true',
help=
"Include fine-grained dependency information in the cache for the mypy daemon"
)
incremental_group.add_argument(
'--skip-version-check',
action='store_true',
help="Allow using cache written by older mypy version")
incremental_group.add_argument(
'--skip-cache-mtime-checks',
action='store_true',
help="Skip cache internal consistency checks based on mtime")
internals_group = parser.add_argument_group(
title='Mypy internals',
description="Debug and customize mypy internals.")
internals_group.add_argument('--pdb',
action='store_true',
help="Invoke pdb on fatal error")
internals_group.add_argument('--show-traceback',
'--tb',
action='store_true',
help="Show traceback on fatal error")
internals_group.add_argument('--raise-exceptions',
action='store_true',
help="Raise exception on fatal error")
internals_group.add_argument('--custom-typing',
metavar='MODULE',
dest='custom_typing_module',
help="Use a custom typing module")
internals_group.add_argument('--custom-typeshed-dir',
metavar='DIR',
help="Use the custom typeshed in DIR")
add_invertible_flag(
'--warn-incomplete-stub',
default=False,
help="Warn if missing type annotation in typeshed, only relevant with"
" --disallow-untyped-defs or --disallow-incomplete-defs enabled",
group=internals_group)
internals_group.add_argument(
'--shadow-file',
nargs=2,
metavar=('SOURCE_FILE', 'SHADOW_FILE'),
dest='shadow_file',
action='append',
help="When encountering SOURCE_FILE, read and type check "
"the contents of SHADOW_FILE instead.")
add_invertible_flag('--fast-exit',
default=False,
help=argparse.SUPPRESS,
group=internals_group)
add_invertible_flag('--new-semantic-analyzer',
default=False,
help=argparse.SUPPRESS,
group=internals_group)
error_group = parser.add_argument_group(
title='Error reporting',
description="Adjust the amount of detail shown in error messages.")
add_invertible_flag(
'--show-error-context',
default=False,
dest='show_error_context',
help='Precede errors with "note:" messages explaining context',
group=error_group)
add_invertible_flag('--show-column-numbers',
default=False,
help="Show column numbers in error messages",
group=error_group)
strict_help = "Strict mode; enables the following flags: {}".format(
", ".join(strict_flag_names))
strictness_group.add_argument('--strict',
action='store_true',
dest='special-opts:strict',
help=strict_help)
report_group = parser.add_argument_group(
title='Report generation',
description='Generate a report in the specified format.')
for report_type in sorted(defaults.REPORTER_NAMES):
report_group.add_argument('--%s-report' %
report_type.replace('_', '-'),
metavar='DIR',
dest='special-opts:%s_report' % report_type)
other_group = parser.add_argument_group(title='Miscellaneous')
other_group.add_argument('--quickstart-file', help=argparse.SUPPRESS)
other_group.add_argument('--junit-xml',
help="Write junit.xml to the given file")
other_group.add_argument(
'--scripts-are-modules',
action='store_true',
help="Script x becomes module x instead of __main__")
other_group.add_argument(
'--find-occurrences',
metavar='CLASS.MEMBER',
dest='special-opts:find_occurrences',
help="Print out all usages of a class member (experimental)")
if server_options:
# TODO: This flag is superfluous; remove after a short transition (2018-03-16)
other_group.add_argument('--experimental',
action='store_true',
dest='fine_grained_incremental',
help="Enable fine-grained incremental mode")
other_group.add_argument(
'--use-fine-grained-cache',
action='store_true',
help="Use the cache in fine-grained incremental mode")
# hidden options
parser.add_argument('--stats',
action='store_true',
dest='dump_type_stats',
help=argparse.SUPPRESS)
parser.add_argument('--inferstats',
action='store_true',
dest='dump_inference_stats',
help=argparse.SUPPRESS)
# --debug-cache will disable any cache-related compressions/optimizations,
# which will make the cache writing process output pretty-printed JSON (which
# is easier to debug).
parser.add_argument('--debug-cache',
action='store_true',
help=argparse.SUPPRESS)
# --dump-deps will dump all fine-grained dependencies to stdout
parser.add_argument('--dump-deps',
action='store_true',
help=argparse.SUPPRESS)
# --dump-graph will dump the contents of the graph of SCCs and exit.
parser.add_argument('--dump-graph',
action='store_true',
help=argparse.SUPPRESS)
# --semantic-analysis-only does exactly that.
parser.add_argument('--semantic-analysis-only',
action='store_true',
help=argparse.SUPPRESS)
# --local-partial-types disallows partial types spanning module top level and a function
# (implicitly defined in fine-grained incremental mode)
parser.add_argument('--local-partial-types',
action='store_true',
help=argparse.SUPPRESS)
# --logical-deps adds some more dependencies that are not semantically needed, but
# may be helpful to determine relative importance of classes and functions for overall
# type precision in a code base. It also _removes_ some deps, so this flag should be never
# used except for generating code stats. This also automatically enables --cache-fine-grained.
# NOTE: This is an experimental option that may be modified or removed at any time.
parser.add_argument('--logical-deps',
action='store_true',
help=argparse.SUPPRESS)
# --bazel changes some behaviors for use with Bazel (https://bazel.build).
parser.add_argument('--bazel', action='store_true', help=argparse.SUPPRESS)
# --package-root adds a directory below which directories are considered
# packages even without __init__.py. May be repeated.
parser.add_argument('--package-root',
metavar='ROOT',
action='append',
default=[],
help=argparse.SUPPRESS)
# --cache-map FILE ... gives a mapping from source files to cache files.
# Each triple of arguments is a source file, a cache meta file, and a cache data file.
# Modules not mentioned in the file will go through cache_dir.
# Must be followed by another flag or by '--' (and then only file args may follow).
parser.add_argument('--cache-map',
nargs='+',
dest='special-opts:cache_map',
help=argparse.SUPPRESS)
# options specifying code to check
code_group = parser.add_argument_group(
title="Running code",
description=
"Specify the code you want to type check. For more details, see "
"mypy.readthedocs.io/en/latest/running_mypy.html#running-mypy")
code_group.add_argument(
'-m',
'--module',
action='append',
metavar='MODULE',
default=[],
dest='special-opts:modules',
help="Type-check module; can repeat for more modules")
code_group.add_argument(
'-p',
'--package',
action='append',
metavar='PACKAGE',
default=[],
dest='special-opts:packages',
help="Type-check package recursively; can be repeated")
code_group.add_argument('-c',
'--command',
action='append',
metavar='PROGRAM_TEXT',
dest='special-opts:command',
help="Type-check program passed in as string")
code_group.add_argument(metavar='files',
nargs='*',
dest='special-opts:files',
help="Type-check given files or directories")
# Parse arguments once into a dummy namespace so we can get the
# filename for the config file and know if the user requested all strict options.
dummy = argparse.Namespace()
parser.parse_args(args, dummy)
config_file = dummy.config_file
# Don't explicitly test if "config_file is not None" for this check.
# This lets `--config-file=` (an empty string) be used to disable all config files.
if config_file and not os.path.exists(config_file):
parser.error("Cannot find config file '%s'" % config_file)
# Parse config file first, so command line can override.
options = Options()
parse_config_file(options, config_file, stdout, stderr)
# Set strict flags before parsing (if strict mode enabled), so other command
# line options can override.
if getattr(dummy, 'special-opts:strict'): # noqa
for dest, value in strict_flag_assignments:
setattr(options, dest, value)
# Parse command line for real, using a split namespace.
special_opts = argparse.Namespace()
parser.parse_args(args,
SplitNamespace(options, special_opts, 'special-opts:'))
# The python_version is either the default, which can be overridden via a config file,
# or stored in special_opts and is passed via the command line.
options.python_version = special_opts.python_version or options.python_version
try:
infer_python_executable(options, special_opts)
except PythonExecutableInferenceError as e:
parser.error(str(e))
if special_opts.no_executable:
options.python_executable = None
# Paths listed in the config file will be ignored if any paths are passed on
# the command line.
if options.files and not special_opts.files:
special_opts.files = options.files
# Check for invalid argument combinations.
if require_targets:
code_methods = sum(
bool(c) for c in [
special_opts.modules +
special_opts.packages, special_opts.command, special_opts.files
])
if code_methods == 0:
parser.error("Missing target module, package, files, or command.")
elif code_methods > 1:
parser.error(
"May only specify one of: module/package, files, or command.")
# Check for overlapping `--always-true` and `--always-false` flags.
overlap = set(options.always_true) & set(options.always_false)
if overlap:
parser.error(
"You can't make a variable always true and always false (%s)" %
', '.join(sorted(overlap)))
# Set build flags.
if options.strict_optional_whitelist is not None:
# TODO: Deprecate, then kill this flag
options.strict_optional = True
if special_opts.find_occurrences:
state.find_occurrences = special_opts.find_occurrences.split('.')
assert state.find_occurrences is not None
if len(state.find_occurrences) < 2:
parser.error("Can only find occurrences of class members.")
if len(state.find_occurrences) != 2:
parser.error(
"Can only find occurrences of non-nested class members.")
# Set reports.
for flag, val in vars(special_opts).items():
if flag.endswith('_report') and val is not None:
report_type = flag[:-7].replace('_', '-')
report_dir = val
options.report_dirs[report_type] = report_dir
# Process --package-root.
if options.package_root:
process_package_roots(fscache, parser, options)
# Process --cache-map.
if special_opts.cache_map:
if options.sqlite_cache:
parser.error("--cache-map is incompatible with --sqlite-cache")
process_cache_map(parser, special_opts, options)
# Let logical_deps imply cache_fine_grained (otherwise the former is useless).
if options.logical_deps:
options.cache_fine_grained = True
# Set target.
if special_opts.modules + special_opts.packages:
options.build_type = BuildType.MODULE
search_paths = SearchPaths((os.getcwd(), ), tuple(mypy_path()), (), ())
targets = []
# TODO: use the same cache that the BuildManager will
cache = FindModuleCache(search_paths, fscache)
for p in special_opts.packages:
if os.sep in p or os.altsep and os.altsep in p:
fail("Package name '{}' cannot have a slash in it.".format(p),
stderr)
p_targets = cache.find_modules_recursive(p)
if not p_targets:
fail("Can't find package '{}'".format(p), stderr)
targets.extend(p_targets)
for m in special_opts.modules:
targets.append(BuildSource(None, m, None))
return targets, options
elif special_opts.command:
options.build_type = BuildType.PROGRAM_TEXT
targets = [BuildSource(None, None, '\n'.join(special_opts.command))]
return targets, options
else:
try:
targets = create_source_list(special_opts.files, options, fscache)
# Variable named e2 instead of e to work around mypyc bug #620
# which causes issues when using the same variable to catch
# exceptions of different types.
except InvalidSourceList as e2:
fail(str(e2), stderr)
return targets, options
def run_case_step(self, testcase: DataDrivenTestCase,
incremental_step: int) -> None:
bench = testcase.config.getoption(
'--bench', False) and 'Benchmark' in testcase.name
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.strict_optional = True
# N.B: We try to (and ought to!) run with the current
# version of python, since we are going to link and run
# against the current version of python.
# But a lot of the tests use type annotations so we can't say it is 3.5.
options.python_version = max(sys.version_info[:2], (3, 6))
options.export_types = True
options.preserve_asts = True
options.incremental = False
# Avoid checking modules/packages named 'unchecked', to provide a way
# to test interacting with code we don't have types for.
options.per_module_options['unchecked.*'] = {'follow_imports': 'error'}
source = build.BuildSource('native.py', 'native', None)
sources = [source]
module_names = ['native']
module_paths = ['native.py']
# Hard code another module name to compile in the same compilation unit.
to_delete = []
for fn, text in testcase.files:
fn = os.path.relpath(fn, test_temp_dir)
if os.path.basename(fn).startswith('other') and fn.endswith('.py'):
name = os.path.basename(fn).split('.')[0]
module_names.append(name)
sources.append(build.BuildSource(fn, name, None))
to_delete.append(fn)
module_paths.append(fn)
shutil.copyfile(
fn,
os.path.join(os.path.dirname(fn),
name + '_interpreted.py'))
for source in sources:
options.per_module_options.setdefault(source.module,
{})['mypyc'] = True
separate = (self.get_separate('\n'.join(
testcase.input), incremental_step) if self.separate else False)
groups = construct_groups(sources, separate, len(module_names) > 1)
try:
result = emitmodule.parse_and_typecheck(sources=sources,
options=options,
alt_lib_path='.')
errors = Errors()
compiler_options = CompilerOptions(multi_file=self.multi_file,
separate=self.separate)
ir, cfiles = emitmodule.compile_modules_to_c(
result,
compiler_options=compiler_options,
errors=errors,
groups=groups,
)
if errors.num_errors:
errors.flush_errors()
assert False, "Compile error"
except CompileError as e:
for line in e.messages:
print(line)
assert False, 'Compile error'
# Check that serialization works on this IR
check_serialization_roundtrip(ir)
setup_file = os.path.abspath(os.path.join(WORKDIR, 'setup.py'))
# We pass the C file information to the build script via setup.py unfortunately
with open(setup_file, 'w', encoding='utf-8') as f:
f.write(
setup_format.format(module_paths, separate, cfiles,
self.multi_file))
if not run_setup(setup_file, ['build_ext', '--inplace']):
if testcase.config.getoption('--mypyc-showc'):
show_c(cfiles)
assert False, "Compilation failed"
# Assert that an output file got created
suffix = 'pyd' if sys.platform == 'win32' else 'so'
assert glob.glob('native.*.{}'.format(suffix))
driver_path = 'driver.py'
env = os.environ.copy()
env['MYPYC_RUN_BENCH'] = '1' if bench else '0'
# XXX: This is an ugly hack.
if 'MYPYC_RUN_GDB' in os.environ:
if platform.system() == 'Darwin':
subprocess.check_call(
['lldb', '--', sys.executable, driver_path], env=env)
assert False, (
"Test can't pass in lldb mode. (And remember to pass -s to "
"pytest)")
elif platform.system() == 'Linux':
subprocess.check_call(
['gdb', '--args', sys.executable, driver_path], env=env)
assert False, (
"Test can't pass in gdb mode. (And remember to pass -s to "
"pytest)")
else:
assert False, 'Unsupported OS'
proc = subprocess.Popen([sys.executable, driver_path],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
output = proc.communicate()[0].decode('utf8')
outlines = output.splitlines()
if testcase.config.getoption('--mypyc-showc'):
show_c(cfiles)
if proc.returncode != 0:
print()
print('*** Exit status: %d' % proc.returncode)
# Verify output.
if bench:
print('Test output:')
print(output)
else:
if incremental_step == 1:
msg = 'Invalid output'
expected = testcase.output
else:
msg = 'Invalid output (step {})'.format(incremental_step)
expected = testcase.output2.get(incremental_step, [])
assert_test_output(testcase, outlines, msg, expected)
if incremental_step > 1 and options.incremental:
suffix = '' if incremental_step == 2 else str(incremental_step - 1)
expected_rechecked = testcase.expected_rechecked_modules.get(
incremental_step - 1)
if expected_rechecked is not None:
assert_module_equivalence('rechecked' + suffix,
expected_rechecked,
result.manager.rechecked_modules)
expected_stale = testcase.expected_stale_modules.get(
incremental_step - 1)
if expected_stale is not None:
assert_module_equivalence('stale' + suffix, expected_stale,
result.manager.stale_modules)
assert proc.returncode == 0
def test_coherence(self) -> None:
options = Options()
_, parsed_options = process_options([], require_targets=False)
# FIX: test this too. Requires changing working dir to avoid finding 'setup.cfg'
options.config_file = parsed_options.config_file
assert_equal(options.snapshot(), parsed_options.snapshot())
def get_semanal_options() -> Options:
options = Options()
options.use_builtins_fixtures = True
options.semantic_analysis_only = True
options.show_traceback = True
return options
def process_options(
args: List[str],
require_targets: bool = True) -> Tuple[List[BuildSource], Options]:
"""Parse command line arguments."""
# Make the help output a little less jarring.
help_factory = (lambda prog: argparse.RawDescriptionHelpFormatter(
prog=prog, max_help_position=28))
parser = argparse.ArgumentParser(prog='mypy',
epilog=FOOTER,
fromfile_prefix_chars='@',
formatter_class=help_factory)
# Unless otherwise specified, arguments will be parsed directly onto an
# Options object. Options that require further processing should have
# their `dest` prefixed with `special-opts:`, which will cause them to be
# parsed into the separate special_opts namespace object.
parser.add_argument('-v',
'--verbose',
action='count',
dest='verbosity',
help="more verbose messages")
parser.add_argument('-V',
'--version',
action='version',
version='%(prog)s ' + __version__)
parser.add_argument('--python-version',
type=parse_version,
metavar='x.y',
help='use Python x.y')
parser.add_argument(
'--platform',
action='store',
metavar='PLATFORM',
help="typecheck special-cased code for the given OS platform "
"(defaults to sys.platform).")
parser.add_argument('-2',
'--py2',
dest='python_version',
action='store_const',
const=defaults.PYTHON2_VERSION,
help="use Python 2 mode")
parser.add_argument('-s',
'--silent-imports',
action='store_true',
help="don't follow imports to .py files")
parser.add_argument(
'--almost-silent',
action='store_true',
help="like --silent-imports but reports the imports as errors")
parser.add_argument(
'--disallow-untyped-calls',
action='store_true',
help="disallow calling functions without type annotations"
" from functions with type annotations")
parser.add_argument(
'--disallow-untyped-defs',
action='store_true',
help="disallow defining functions without type annotations"
" or with incomplete type annotations")
parser.add_argument(
'--check-untyped-defs',
action='store_true',
help="type check the interior of functions without type annotations")
parser.add_argument(
'--disallow-subclassing-any',
action='store_true',
help="disallow subclassing values of type 'Any' when defining classes")
parser.add_argument(
'--warn-incomplete-stub',
action='store_true',
help="warn if missing type annotation in typeshed, only relevant with"
" --check-untyped-defs enabled")
parser.add_argument(
'--warn-redundant-casts',
action='store_true',
help="warn about casting an expression to its inferred type")
parser.add_argument('--warn-no-return',
action='store_true',
help="warn about functions that end without returning")
parser.add_argument('--warn-unused-ignores',
action='store_true',
help="warn about unneeded '# type: ignore' comments")
parser.add_argument('--hide-error-context',
action='store_true',
dest='hide_error_context',
help="Hide context notes before errors")
parser.add_argument('--fast-parser',
action='store_true',
help="enable experimental fast parser")
parser.add_argument('-i',
'--incremental',
action='store_true',
help="enable experimental module cache")
parser.add_argument(
'--cache-dir',
action='store',
metavar='DIR',
help="store module cache info in the given folder in incremental mode "
"(defaults to '{}')".format(defaults.CACHE_DIR))
parser.add_argument('--strict-optional',
action='store_true',
dest='strict_optional',
help="enable experimental strict Optional checks")
parser.add_argument(
'--strict-optional-whitelist',
metavar='GLOB',
nargs='*',
help="suppress strict Optional errors in all but the provided files "
"(experimental -- read documentation before using!). "
"Implies --strict-optional. Has the undesirable side-effect of "
"suppressing other errors in non-whitelisted files.")
parser.add_argument('--junit-xml',
help="write junit.xml to the given file")
parser.add_argument('--pdb',
action='store_true',
help="invoke pdb on fatal error")
parser.add_argument('--show-traceback',
'--tb',
action='store_true',
help="show traceback on fatal error")
parser.add_argument('--stats',
action='store_true',
dest='dump_type_stats',
help="dump stats")
parser.add_argument('--inferstats',
action='store_true',
dest='dump_inference_stats',
help="dump type inference stats")
parser.add_argument('--custom-typing',
metavar='MODULE',
dest='custom_typing_module',
help="use a custom typing module")
parser.add_argument('--custom-typeshed-dir',
metavar='DIR',
help="use the custom typeshed in DIR")
parser.add_argument('--scripts-are-modules',
action='store_true',
help="Script x becomes module x instead of __main__")
parser.add_argument('--config-file',
help="Configuration file, must have a [mypy] section "
"(defaults to {})".format(defaults.CONFIG_FILE))
parser.add_argument('--show-column-numbers',
action='store_true',
dest='show_column_numbers',
help="Show column numbers in error messages")
parser.add_argument(
'--find-occurrences',
metavar='CLASS.MEMBER',
dest='special-opts:find_occurrences',
help="print out all usages of a class member (experimental)")
# hidden options
# --shadow-file a.py tmp.py will typecheck tmp.py in place of a.py.
# Useful for tools to make transformations to a file to get more
# information from a mypy run without having to change the file in-place
# (e.g. by adding a call to reveal_type).
parser.add_argument('--shadow-file',
metavar='PATH',
nargs=2,
dest='shadow_file',
help=argparse.SUPPRESS)
# --debug-cache will disable any cache-related compressions/optimizations,
# which will make the cache writing process output pretty-printed JSON (which
# is easier to debug).
parser.add_argument('--debug-cache',
action='store_true',
help=argparse.SUPPRESS)
# deprecated options
parser.add_argument('--silent',
action='store_true',
dest='special-opts:silent',
help=argparse.SUPPRESS)
parser.add_argument('-f',
'--dirty-stubs',
action='store_true',
dest='special-opts:dirty_stubs',
help=argparse.SUPPRESS)
parser.add_argument('--use-python-path',
action='store_true',
dest='special-opts:use_python_path',
help=argparse.SUPPRESS)
report_group = parser.add_argument_group(
title='report generation',
description='Generate a report in the specified format.')
for report_type in sorted(reporter_classes):
report_group.add_argument('--%s-report' %
report_type.replace('_', '-'),
metavar='DIR',
dest='special-opts:%s_report' % report_type)
code_group = parser.add_argument_group(
title='How to specify the code to type check')
code_group.add_argument(
'-m',
'--module',
action='append',
metavar='MODULE',
dest='special-opts:modules',
help="type-check module; can repeat for more modules")
# TODO: `mypy -p A -p B` currently silently ignores ignores A
# (last option wins). Perhaps -c, -m and -p could just be
# command-line flags that modify how we interpret self.files?
code_group.add_argument('-c',
'--command',
action='append',
metavar='PROGRAM_TEXT',
dest='special-opts:command',
help="type-check program passed in as string")
code_group.add_argument('-p',
'--package',
metavar='PACKAGE',
dest='special-opts:package',
help="type-check all files in a directory")
code_group.add_argument(metavar='files',
nargs='*',
dest='special-opts:files',
help="type-check given files or directories")
# Parse arguments once into a dummy namespace so we can get the
# filename for the config file.
dummy = argparse.Namespace()
parser.parse_args(args, dummy)
config_file = dummy.config_file or defaults.CONFIG_FILE
# Parse config file first, so command line can override.
options = Options()
if config_file and os.path.exists(config_file):
parse_config_file(options, config_file)
# Parse command line for real, using a split namespace.
special_opts = argparse.Namespace()
parser.parse_args(args,
SplitNamespace(options, special_opts, 'special-opts:'))
# --use-python-path is no longer supported; explain why.
if special_opts.use_python_path:
parser.error(
"Sorry, --use-python-path is no longer supported.\n"
"If you are trying this because your code depends on a library module,\n"
"you should really investigate how to obtain stubs for that module.\n"
"See https://github.com/python/mypy/issues/1411 for more discussion."
)
# warn about deprecated options
if special_opts.silent:
print("Warning: --silent is deprecated; use --silent-imports",
file=sys.stderr)
options.silent_imports = True
if special_opts.dirty_stubs:
print(
"Warning: -f/--dirty-stubs is deprecated and no longer necessary. Mypy no longer "
"checks the git status of stubs.",
file=sys.stderr)
# Check for invalid argument combinations.
if require_targets:
code_methods = sum(
bool(c) for c in [
special_opts.modules, special_opts.command,
special_opts.package, special_opts.files
])
if code_methods == 0:
parser.error("Missing target module, package, files, or command.")
elif code_methods > 1:
parser.error(
"May only specify one of: module, package, files, or command.")
# Set build flags.
if options.strict_optional_whitelist is not None:
# TODO: Deprecate, then kill this flag
options.strict_optional = True
if options.strict_optional:
experiments.STRICT_OPTIONAL = True
if special_opts.find_occurrences:
experiments.find_occurrences = special_opts.find_occurrences.split('.')
if len(experiments.find_occurrences) < 2:
parser.error("Can only find occurrences of class members.")
if len(experiments.find_occurrences) != 2:
parser.error(
"Can only find occurrences of non-nested class members.")
# Set reports.
for flag, val in vars(special_opts).items():
if flag.endswith('_report') and val is not None:
report_type = flag[:-7].replace('_', '-')
report_dir = val
options.report_dirs[report_type] = report_dir
# Set target.
if special_opts.modules:
options.build_type = BuildType.MODULE
targets = [BuildSource(None, m, None) for m in special_opts.modules]
return targets, options
elif special_opts.package:
if os.sep in special_opts.package or os.altsep and os.altsep in special_opts.package:
fail("Package name '{}' cannot have a slash in it.".format(
special_opts.package))
options.build_type = BuildType.MODULE
lib_path = [os.getcwd()] + build.mypy_path()
targets = build.find_modules_recursive(special_opts.package, lib_path)
if not targets:
fail("Can't find package '{}'".format(special_opts.package))
return targets, options
elif special_opts.command:
options.build_type = BuildType.PROGRAM_TEXT
return [BuildSource(None, None,
'\n'.join(special_opts.command))], options
else:
targets = []
for f in special_opts.files:
if f.endswith(PY_EXTENSIONS):
targets.append(BuildSource(f, crawl_up(f)[1], None))
elif os.path.isdir(f):
sub_targets = expand_dir(f)
if not sub_targets:
fail("There are no .py[i] files in directory '{}'".format(
f))
targets.extend(sub_targets)
else:
mod = os.path.basename(
f) if options.scripts_are_modules else None
targets.append(BuildSource(f, mod, None))
return targets, options
def main(argv):
# TODO: Put these in the shell script
mypy_options = [
'--py2', '--strict', '--no-implicit-optional', '--no-strict-optional',
# for consistency?
'--follow-imports=silent',
#'--verbose',
]
o = Options()
opts, argv = o.parse_args(argv)
paths = argv[1:] # e.g. asdl/typed_arith_parse.py
if 0:
print(opts)
print(paths)
return
# e.g. asdl/typed_arith_parse.py -> 'typed_arith_parse'
mod_names = [os.path.basename(p) for p in paths]
mod_names = [os.path.splitext(name)[0] for name in mod_names]
# Ditto
to_header = opts.to_header
#if to_header:
if 0:
to_header = [os.path.basename(p) for p in to_header]
to_header = [os.path.splitext(name)[0] for name in to_header]
#log('to_header %s', to_header)
sources, options = get_mypy_config(paths, mypy_options)
if 0:
for source in sources:
log('source %s', source)
#log('options %s', options)
#result = emitmodule.parse_and_typecheck(sources, options)
import time
start_time = time.time()
result = mypy_build(sources=sources, options=options)
#log('elapsed 1: %f', time.time() - start_time)
if result.errors:
log('')
log('-'* 80)
for e in result.errors:
log(e)
log('-'* 80)
log('')
# Important functions in mypyc/build.py:
#
# generate_c (251 lines)
# parse_and_typecheck
# compile_modules_to_c
# mypyc/emitmodule.py (487 lines)
# def compile_modules_to_c(result: BuildResult, module_names: List[str],
# class ModuleGenerator:
# # This generates a whole bunch of textual code!
# literals, modules, errors = genops.build_ir(file_nodes, result.graph,
# result.types)
# TODO: Debug what comes out of here.
#build.dump_graph(result.graph)
#return
# no-op
for name in result.graph:
state = result.graph[name]
# GLOBAL Constant pass over all modules. We want to collect duplicate
# strings together. And have globally unique IDs str0, str1, ... strN.
const_lookup = {}
const_code = []
pass1 = const_pass.Collect(result.types, const_lookup, const_code)
to_compile = list(ModulesToCompile(result, mod_names))
# HACK: Why do I get oil.asdl.tdop in addition to asdl.tdop?
names = set(name for name, _ in to_compile)
filtered = []
seen = set()
for name, module in to_compile:
if name.startswith('oil.'):
name = name[4:]
if name not in seen: # remove dupe
filtered.append((name, module))
seen.add(name)
to_compile = filtered
import pickle
if 1:
for name, module in to_compile:
log('to_compile %s', name)
# can't pickle but now I see deserialize() nodes and stuff
#s = pickle.dumps(module)
#log('%d pickle', len(s))
# Print the tree for debugging
if 0:
for name, module in to_compile:
builder = debug_pass.Print(result.types)
builder.visit_mypy_file(module)
return
f = sys.stdout
gc = bool(os.getenv('GC'))
header_name = 'gc_heap' if gc else 'mylib'
#header_name = 'mylib'
# TODO: Add --cc-out? But there is a preamble and postamble.
f.write("""\
// BEGIN mycpp output
#include "%s.h"
using gc_heap::Alloc;
using gc_heap::kZeroMask;
using gc_heap::StackRoots;
""" % header_name)
if gc:
f.write("""\
#include "my_runtime.h"
#include "mylib2.h"
using gc_heap::NewStr;
using gc_heap::NewList;
using gc_heap::NewDict;
""")
if to_header:
f.write('#include "%s"\n' % os.path.basename(opts.header_out))
f.write('\n')
# Collect constants and then emit code.
for name, module in to_compile:
pass1.visit_mypy_file(module)
# Instead of top-level code, should we generate a function and call it from
# main?
for line in const_code:
f.write('%s\n' % line)
f.write('\n')
# Note: doesn't take into account module names!
virtual = pass_state.Virtual()
if opts.header_out:
header_f = open(opts.header_out, 'w') # Not closed
guard = 'RUNTIME_H'
header_f.write("""\
// %s: translated from Python by mycpp
#ifndef %s
#define %s
#include "%s.h"
""" % (os.path.basename(opts.header_out), guard, guard, header_name))
log('\tFORWARD DECL')
# Forward declarations first.
# class Foo; class Bar;
for name, module in to_compile:
log('forward decl name %s', name)
if name in to_header:
out_f = header_f
else:
out_f = f
p2 = cppgen_pass.Generate(result.types, const_lookup, out_f,
virtual=virtual, forward_decl=True)
p2.visit_mypy_file(module)
# After seeing class and method names in the first pass, figure out which
# ones are virtual. We use this info in the second pass.
virtual.Calculate()
#log('V %s', virtual.virtuals)
local_vars = {} # FuncDef node -> (name, c_type) list
# Node -> fmt_name, plus a hack for the counter
# TODO: This could be a class with 2 members
fmt_ids = {'_counter': 0}
log('\tDECL')
# First generate ALL C++ declarations / "headers".
# class Foo { void method(); }; class Bar { void method(); };
for name, module in to_compile:
log('decl name %s', name)
if name in to_header:
out_f = header_f
else:
out_f = f
p3 = cppgen_pass.Generate(result.types, const_lookup, out_f,
local_vars=local_vars, fmt_ids=fmt_ids,
virtual=virtual, decl=True)
p3.visit_mypy_file(module)
if opts.header_out:
header_f.write("""\
#endif // %s
""" % guard)
log('\tDEFINITION')
# Now the definitions / implementations.
# void Foo:method() { ... }
# void Bar:method() { ... }
for name, module in to_compile:
p4 = cppgen_pass.Generate(result.types, const_lookup, f,
local_vars=local_vars, fmt_ids=fmt_ids)
p4.visit_mypy_file(module)
def process_options(args: List[str],
require_targets: bool = True,
server_options: bool = False,
fscache: Optional[FileSystemCache] = None,
) -> Tuple[List[BuildSource], Options]:
"""Parse command line arguments.
If a FileSystemCache is passed in, and package_root options are given,
call fscache.set_package_root() to set the cache's package root.
"""
parser = argparse.ArgumentParser(prog='mypy', epilog=FOOTER,
fromfile_prefix_chars='@',
formatter_class=AugmentedHelpFormatter)
strict_flag_names = [] # type: List[str]
strict_flag_assignments = [] # type: List[Tuple[str, bool]]
def add_invertible_flag(flag: str,
*,
inverse: Optional[str] = None,
default: bool,
dest: Optional[str] = None,
help: str,
strict_flag: bool = False,
group: Optional[argparse._ActionsContainer] = None
) -> None:
if inverse is None:
inverse = invert_flag_name(flag)
if group is None:
group = parser
if help is not argparse.SUPPRESS:
help += " (inverse: {})".format(inverse)
arg = group.add_argument(flag,
action='store_false' if default else 'store_true',
dest=dest,
help=help)
dest = arg.dest
arg = group.add_argument(inverse,
action='store_true' if default else 'store_false',
dest=dest,
help=argparse.SUPPRESS)
if strict_flag:
assert dest is not None
strict_flag_names.append(flag)
strict_flag_assignments.append((dest, not default))
# Unless otherwise specified, arguments will be parsed directly onto an
# Options object. Options that require further processing should have
# their `dest` prefixed with `special-opts:`, which will cause them to be
# parsed into the separate special_opts namespace object.
parser.add_argument('-v', '--verbose', action='count', dest='verbosity',
help="more verbose messages")
parser.add_argument('-V', '--version', action='version',
version='%(prog)s ' + __version__,
help="show program's version number and exit")
config_group = parser.add_argument_group(
title='config file',
description="Use a config file instead of command line arguments.")
config_group.add_argument(
'--config-file',
help="configuration file, must have a [mypy] section "
"(defaults to {})".format(', '.join(defaults.CONFIG_FILES)))
add_invertible_flag('--warn-unused-configs', default=False, strict_flag=True,
help="warn about unused '[mypy-<pattern>]' config sections",
group=config_group)
imports_group = parser.add_argument_group(
title='import discovery',
description="Configure how imports are discovered and followed.")
imports_group.add_argument(
'--ignore-missing-imports', action='store_true',
help="silently ignore imports of missing modules")
imports_group.add_argument(
'--follow-imports', choices=['normal', 'silent', 'skip', 'error'],
default='normal', help="how to treat imports (default normal)")
imports_group.add_argument(
'--python-executable', action='store', metavar='EXECUTABLE',
help="Python executable used for finding PEP 561 compliant installed"
" packages and stubs",
dest='special-opts:python_executable')
imports_group.add_argument(
'--no-site-packages', action='store_true',
dest='special-opts:no_executable',
help="do not search for installed PEP 561 compliant packages")
platform_group = parser.add_argument_group(
title='platform configuration',
description="Type check code assuming certain runtime conditions.")
platform_group.add_argument(
'--python-version', type=parse_version, metavar='x.y',
help='type check code assuming it will be running on Python x.y',
dest='special-opts:python_version')
platform_group.add_argument(
'-2', '--py2', dest='python_version', action='store_const',
const=defaults.PYTHON2_VERSION,
help="use Python 2 mode (same as --python-version 2.7)")
platform_group.add_argument(
'--platform', action='store', metavar='PLATFORM',
help="type check special-cased code for the given OS platform "
"(defaults to sys.platform)")
platform_group.add_argument(
'--always-true', metavar='NAME', action='append', default=[],
help="additional variable to be considered True (may be repeated)")
platform_group.add_argument(
'--always-false', metavar='NAME', action='append', default=[],
help="additional variable to be considered False (may be repeated)")
disallow_any_group = parser.add_argument_group(
title='Any type restrictions',
description="Disallow the use of the 'Any' type under certain conditions.")
disallow_any_group.add_argument(
'--disallow-any-unimported', default=False, action='store_true',
help="disallow Any types resulting from unfollowed imports")
add_invertible_flag('--disallow-subclassing-any', default=False, strict_flag=True,
help="disallow subclassing values of type 'Any' when defining classes",
group=disallow_any_group)
disallow_any_group.add_argument(
'--disallow-any-expr', default=False, action='store_true',
help='disallow all expressions that have type Any')
disallow_any_group.add_argument(
'--disallow-any-decorated', default=False, action='store_true',
help='disallow functions that have Any in their signature '
'after decorator transformation')
disallow_any_group.add_argument(
'--disallow-any-explicit', default=False, action='store_true',
help='disallow explicit Any in type positions')
disallow_any_group.add_argument(
'--disallow-any-generics', default=False, action='store_true',
help='disallow usage of generic types that do not specify explicit '
'type parameters')
untyped_group = parser.add_argument_group(
title='untyped definitions and calls',
description="Configure how untyped definitions and calls are handled.")
add_invertible_flag('--disallow-untyped-calls', default=False, strict_flag=True,
help="disallow calling functions without type annotations"
" from functions with type annotations",
group=untyped_group)
add_invertible_flag('--disallow-untyped-defs', default=False, strict_flag=True,
help="disallow defining functions without type annotations"
" or with incomplete type annotations",
group=untyped_group)
add_invertible_flag('--disallow-incomplete-defs', default=False, strict_flag=True,
help="disallow defining functions with incomplete type annotations",
group=untyped_group)
add_invertible_flag('--check-untyped-defs', default=False, strict_flag=True,
help="type check the interior of functions without type annotations",
group=untyped_group)
add_invertible_flag('--warn-incomplete-stub', default=False,
help="warn if missing type annotation in typeshed, only relevant with"
" --check-untyped-defs enabled",
group=untyped_group)
none_group = parser.add_argument_group(
title='None and Optional handling',
description="Adjust how values of type 'None' are handled.")
add_invertible_flag('--no-implicit-optional', default=False, strict_flag=True,
help="don't assume arguments with default values of None are Optional",
group=none_group)
none_group.add_argument(
'--strict-optional', action='store_true',
help=argparse.SUPPRESS)
none_group.add_argument(
'--no-strict-optional', action='store_false', dest='strict_optional',
help="disable strict Optional checks (inverse: --strict-optional)")
none_group.add_argument(
'--strict-optional-whitelist', metavar='GLOB', nargs='*',
help="suppress strict Optional errors in all but the provided files; "
"implies --strict-optional (may suppress certain other errors "
"in non-whitelisted files)")
lint_group = parser.add_argument_group(
title='warnings',
description="Detect code that is sound but redundant or problematic.")
add_invertible_flag('--warn-redundant-casts', default=False, strict_flag=True,
help="warn about casting an expression to its inferred type",
group=lint_group)
add_invertible_flag('--no-warn-no-return', dest='warn_no_return', default=True,
help="do not warn about functions that end without returning",
group=lint_group)
add_invertible_flag('--warn-return-any', default=False, strict_flag=True,
help="warn about returning values of type Any"
" from non-Any typed functions",
group=lint_group)
add_invertible_flag('--warn-unused-ignores', default=False, strict_flag=True,
help="warn about unneeded '# type: ignore' comments",
group=lint_group)
strictness_group = parser.add_argument_group(
title='other strictness checks',
description="Other miscellaneous strictness checks.")
add_invertible_flag('--disallow-untyped-decorators', default=False, strict_flag=True,
help="disallow decorating typed functions with untyped decorators",
group=strictness_group)
incremental_group = parser.add_argument_group(
title='incremental mode',
description="Adjust how mypy incrementally type checks and caches modules.")
incremental_group.add_argument(
'-i', '--incremental', action='store_true',
help=argparse.SUPPRESS)
incremental_group.add_argument(
'--no-incremental', action='store_false', dest='incremental',
help="disable module cache (inverse: --incremental)")
incremental_group.add_argument(
'--cache-dir', action='store', metavar='DIR',
help="store module cache info in the given folder in incremental mode "
"(defaults to '{}')".format(defaults.CACHE_DIR))
incremental_group.add_argument(
'--cache-fine-grained', action='store_true',
help="include fine-grained dependency information in the cache for the mypy daemon")
incremental_group.add_argument(
'--quick-and-dirty', action='store_true',
help="use cache even if dependencies out of date (implies --incremental)")
incremental_group.add_argument(
'--skip-version-check', action='store_true',
help="allow using cache written by older mypy version")
internals_group = parser.add_argument_group(
title='mypy internals',
description="Debug and customize mypy internals.")
internals_group.add_argument(
'--pdb', action='store_true', help="invoke pdb on fatal error")
internals_group.add_argument(
'--show-traceback', '--tb', action='store_true',
help="show traceback on fatal error")
internals_group.add_argument(
'--custom-typing', metavar='MODULE', dest='custom_typing_module',
help="use a custom typing module")
internals_group.add_argument(
'--custom-typeshed-dir', metavar='DIR',
help="use the custom typeshed in DIR")
internals_group.add_argument(
'--shadow-file', nargs=2, metavar=('SOURCE_FILE', 'SHADOW_FILE'),
dest='shadow_file', action='append',
help="when encountering SOURCE_FILE, read and type check "
"the contents of SHADOW_FILE instead.")
error_group = parser.add_argument_group(
title='error reporting',
description="Adjust the amount of detail shown in error messages.")
add_invertible_flag('--show-error-context', default=False,
dest='show_error_context',
help='precede errors with "note:" messages explaining context',
group=error_group)
add_invertible_flag('--show-column-numbers', default=False,
help="show column numbers in error messages",
group=error_group)
analysis_group = parser.add_argument_group(
title='extra analysis',
description="Extract additional information and analysis.")
analysis_group.add_argument(
'--stats', action='store_true', dest='dump_type_stats', help=argparse.SUPPRESS)
analysis_group.add_argument(
'--inferstats', action='store_true', dest='dump_inference_stats',
help=argparse.SUPPRESS)
analysis_group.add_argument(
'--find-occurrences', metavar='CLASS.MEMBER',
dest='special-opts:find_occurrences',
help="print out all usages of a class member (experimental)")
strict_help = "strict mode; enables the following flags: {}".format(
", ".join(strict_flag_names))
strictness_group.add_argument(
'--strict', action='store_true', dest='special-opts:strict',
help=strict_help)
report_group = parser.add_argument_group(
title='report generation',
description='Generate a report in the specified format.')
for report_type in sorted(reporter_classes):
report_group.add_argument('--%s-report' % report_type.replace('_', '-'),
metavar='DIR',
dest='special-opts:%s_report' % report_type)
other_group = parser.add_argument_group(
title='miscellaneous',
description="Other miscellaneous flags.")
other_group.add_argument(
'--junit-xml', help="write junit.xml to the given file")
other_group.add_argument(
'--scripts-are-modules', action='store_true',
help="script x becomes module x instead of __main__")
if server_options:
# TODO: This flag is superfluous; remove after a short transition (2018-03-16)
other_group.add_argument(
'--experimental', action='store_true', dest='fine_grained_incremental',
help="enable fine-grained incremental mode")
other_group.add_argument(
'--use-fine-grained-cache', action='store_true',
help="use the cache in fine-grained incremental mode")
# hidden options
# --debug-cache will disable any cache-related compressions/optimizations,
# which will make the cache writing process output pretty-printed JSON (which
# is easier to debug).
parser.add_argument('--debug-cache', action='store_true', help=argparse.SUPPRESS)
# --dump-deps will dump all fine-grained dependencies to stdout
parser.add_argument('--dump-deps', action='store_true', help=argparse.SUPPRESS)
# --dump-graph will dump the contents of the graph of SCCs and exit.
parser.add_argument('--dump-graph', action='store_true', help=argparse.SUPPRESS)
# --semantic-analysis-only does exactly that.
parser.add_argument('--semantic-analysis-only', action='store_true', help=argparse.SUPPRESS)
# --local-partial-types disallows partial types spanning module top level and a function
# (implicitly defined in fine-grained incremental mode)
parser.add_argument('--local-partial-types', action='store_true', help=argparse.SUPPRESS)
# --bazel changes some behaviors for use with Bazel (https://bazel.build).
parser.add_argument('--bazel', action='store_true', help=argparse.SUPPRESS)
# --package-root adds a directory below which directories are considered
# packages even without __init__.py. May be repeated.
parser.add_argument('--package-root', metavar='ROOT', action='append', default=[],
help=argparse.SUPPRESS)
# --cache-map FILE ... gives a mapping from source files to cache files.
# Each triple of arguments is a source file, a cache meta file, and a cache data file.
# Modules not mentioned in the file will go through cache_dir.
# Must be followed by another flag or by '--' (and then only file args may follow).
parser.add_argument('--cache-map', nargs='+', dest='special-opts:cache_map',
help=argparse.SUPPRESS)
# deprecated options
parser.add_argument('--disallow-any', dest='special-opts:disallow_any',
help=argparse.SUPPRESS)
add_invertible_flag('--strict-boolean', default=False,
help=argparse.SUPPRESS)
parser.add_argument('-f', '--dirty-stubs', action='store_true',
dest='special-opts:dirty_stubs',
help=argparse.SUPPRESS)
parser.add_argument('--use-python-path', action='store_true',
dest='special-opts:use_python_path',
help=argparse.SUPPRESS)
parser.add_argument('-s', '--silent-imports', action='store_true',
dest='special-opts:silent_imports',
help=argparse.SUPPRESS)
parser.add_argument('--almost-silent', action='store_true',
dest='special-opts:almost_silent',
help=argparse.SUPPRESS)
parser.add_argument('--fast-parser', action='store_true', dest='special-opts:fast_parser',
help=argparse.SUPPRESS)
parser.add_argument('--no-fast-parser', action='store_true',
dest='special-opts:no_fast_parser',
help=argparse.SUPPRESS)
code_group = parser.add_argument_group(title='specifying which code to type check')
code_group.add_argument('-m', '--module', action='append', metavar='MODULE',
default=[],
dest='special-opts:modules',
help="type-check module; can repeat for more modules")
code_group.add_argument('-p', '--package', action='append', metavar='PACKAGE',
default=[],
dest='special-opts:packages',
help="type-check package recursively; can be repeated")
code_group.add_argument('-c', '--command', action='append', metavar='PROGRAM_TEXT',
dest='special-opts:command',
help="type-check program passed in as string")
code_group.add_argument(metavar='files', nargs='*', dest='special-opts:files',
help="type-check given files or directories")
# Parse arguments once into a dummy namespace so we can get the
# filename for the config file and know if the user requested all strict options.
dummy = argparse.Namespace()
parser.parse_args(args, dummy)
config_file = dummy.config_file
if config_file is not None and not os.path.exists(config_file):
parser.error("Cannot find config file '%s'" % config_file)
# Parse config file first, so command line can override.
options = Options()
parse_config_file(options, config_file)
# Set strict flags before parsing (if strict mode enabled), so other command
# line options can override.
if getattr(dummy, 'special-opts:strict'):
for dest, value in strict_flag_assignments:
setattr(options, dest, value)
# Parse command line for real, using a split namespace.
special_opts = argparse.Namespace()
parser.parse_args(args, SplitNamespace(options, special_opts, 'special-opts:'))
# --use-python-path is no longer supported; explain why.
if special_opts.use_python_path:
parser.error("Sorry, --use-python-path is no longer supported.\n"
"If you are trying this because your code depends on a library module,\n"
"you should really investigate how to obtain stubs for that module.\n"
"See https://github.com/python/mypy/issues/1411 for more discussion."
)
# Process deprecated options
if special_opts.disallow_any:
print("--disallow-any option was split up into multiple flags. "
"See http://mypy.readthedocs.io/en/latest/command_line.html#disallow-any-flags")
if options.strict_boolean:
print("Warning: --strict-boolean is deprecated; "
"see https://github.com/python/mypy/issues/3195", file=sys.stderr)
if special_opts.almost_silent:
print("Warning: --almost-silent has been replaced by "
"--follow-imports=errors", file=sys.stderr)
if options.follow_imports == 'normal':
options.follow_imports = 'errors'
elif special_opts.silent_imports:
print("Warning: --silent-imports has been replaced by "
"--ignore-missing-imports --follow-imports=skip", file=sys.stderr)
options.ignore_missing_imports = True
if options.follow_imports == 'normal':
options.follow_imports = 'skip'
if special_opts.dirty_stubs:
print("Warning: -f/--dirty-stubs is deprecated and no longer necessary. Mypy no longer "
"checks the git status of stubs.",
file=sys.stderr)
if special_opts.fast_parser:
print("Warning: --fast-parser is now the default (and only) parser.")
if special_opts.no_fast_parser:
print("Warning: --no-fast-parser no longer has any effect. The fast parser "
"is now mypy's default and only parser.")
try:
infer_python_version_and_executable(options, special_opts)
except PythonExecutableInferenceError as e:
parser.error(str(e))
if special_opts.no_executable:
options.python_executable = None
# Check for invalid argument combinations.
if require_targets:
code_methods = sum(bool(c) for c in [special_opts.modules + special_opts.packages,
special_opts.command,
special_opts.files])
if code_methods == 0:
parser.error("Missing target module, package, files, or command.")
elif code_methods > 1:
parser.error("May only specify one of: module/package, files, or command.")
# Check for overlapping `--always-true` and `--always-false` flags.
overlap = set(options.always_true) & set(options.always_false)
if overlap:
parser.error("You can't make a variable always true and always false (%s)" %
', '.join(sorted(overlap)))
# Set build flags.
if options.strict_optional_whitelist is not None:
# TODO: Deprecate, then kill this flag
options.strict_optional = True
if special_opts.find_occurrences:
experiments.find_occurrences = special_opts.find_occurrences.split('.')
assert experiments.find_occurrences is not None
if len(experiments.find_occurrences) < 2:
parser.error("Can only find occurrences of class members.")
if len(experiments.find_occurrences) != 2:
parser.error("Can only find occurrences of non-nested class members.")
# Set reports.
for flag, val in vars(special_opts).items():
if flag.endswith('_report') and val is not None:
report_type = flag[:-7].replace('_', '-')
report_dir = val
options.report_dirs[report_type] = report_dir
# Process --package-root.
if options.package_root:
process_package_roots(fscache, parser, options)
# Process --cache-map.
if special_opts.cache_map:
process_cache_map(parser, special_opts, options)
# Let quick_and_dirty imply incremental.
if options.quick_and_dirty:
options.incremental = True
# Set target.
if special_opts.modules + special_opts.packages:
options.build_type = BuildType.MODULE
lib_path = [os.getcwd()] + build.mypy_path()
targets = []
# TODO: use the same cache that the BuildManager will
cache = build.FindModuleCache(fscache)
for p in special_opts.packages:
if os.sep in p or os.altsep and os.altsep in p:
fail("Package name '{}' cannot have a slash in it.".format(p))
p_targets = cache.find_modules_recursive(p, tuple(lib_path), options.python_executable)
if not p_targets:
fail("Can't find package '{}'".format(p))
targets.extend(p_targets)
for m in special_opts.modules:
targets.append(BuildSource(None, m, None))
return targets, options
elif special_opts.command:
options.build_type = BuildType.PROGRAM_TEXT
targets = [BuildSource(None, None, '\n'.join(special_opts.command))]
return targets, options
else:
try:
targets = create_source_list(special_opts.files, options, fscache)
except InvalidSourceList as e:
fail(str(e))
return targets, options
