def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bool) -> Options: # This handles things like '# flags: --foo'. options = parse_options(source, testcase, incremental_step=1) options.incremental = True options.use_builtins_fixtures = True options.show_traceback = True options.error_summary = False options.fine_grained_incremental = not build_cache options.use_fine_grained_cache = self.use_cache and not build_cache options.cache_fine_grained = self.use_cache options.local_partial_types = True if options.follow_imports == 'normal': options.follow_imports = 'error' for name, _ in testcase.files: if 'mypy.ini' in name: parse_config_file(options, lambda: None, name) break return options
def process_options( args: List[str], stdout: Optional[TextIO] = None, stderr: Optional[TextIO] = None, require_targets: bool = True, server_options: bool = False, fscache: Optional[FileSystemCache] = None, program: str = 'mypy', header: str = HEADER, ) -> Tuple[List[BuildSource], Options]: """Parse command line arguments. If a FileSystemCache is passed in, and package_root options are given, call fscache.set_package_root() to set the cache's package root. """ stdout = stdout or sys.stdout stderr = stderr or sys.stderr parser = argparse.ArgumentParser(prog=program, usage=header, description=DESCRIPTION, epilog=FOOTER, fromfile_prefix_chars='@', formatter_class=AugmentedHelpFormatter, add_help=False) strict_flag_names = [] # type: List[str] strict_flag_assignments = [] # type: List[Tuple[str, bool]] def add_invertible_flag( flag: str, *, inverse: Optional[str] = None, default: bool, dest: Optional[str] = None, help: str, strict_flag: bool = False, group: Optional[argparse._ActionsContainer] = None) -> None: if inverse is None: inverse = invert_flag_name(flag) if group is None: group = parser if help is not argparse.SUPPRESS: help += " (inverse: {})".format(inverse) arg = group.add_argument( flag, action='store_false' if default else 'store_true', dest=dest, help=help) dest = arg.dest arg = group.add_argument( inverse, action='store_true' if default else 'store_false', dest=dest, help=argparse.SUPPRESS) if strict_flag: assert dest is not None strict_flag_names.append(flag) strict_flag_assignments.append((dest, not default)) # Unless otherwise specified, arguments will be parsed directly onto an # Options object. Options that require further processing should have # their `dest` prefixed with `special-opts:`, which will cause them to be # parsed into the separate special_opts namespace object. # Note: we have a style guide for formatting the mypy --help text. See # https://github.com/python/mypy/wiki/Documentation-Conventions general_group = parser.add_argument_group(title='Optional arguments') general_group.add_argument('-h', '--help', action='help', help="Show this help message and exit") general_group.add_argument('-v', '--verbose', action='count', dest='verbosity', help="More verbose messages") general_group.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__, help="Show program's version number and exit") config_group = parser.add_argument_group( title='Config file', description="Use a config file instead of command line arguments. " "This is useful if you are using many flags or want " "to set different options per each module.") config_group.add_argument( '--config-file', help="Configuration file, must have a [mypy] section " "(defaults to {})".format(', '.join(defaults.CONFIG_FILES))) add_invertible_flag( '--warn-unused-configs', default=False, strict_flag=True, help="Warn about unused '[mypy-<pattern>]' config sections", group=config_group) imports_group = parser.add_argument_group( title='Import discovery', description="Configure how imports are discovered and followed.") imports_group.add_argument( '--ignore-missing-imports', action='store_true', help="Silently ignore imports of missing modules") imports_group.add_argument('--follow-imports', choices=['normal', 'silent', 'skip', 'error'], default='normal', help="How to treat imports (default normal)") imports_group.add_argument( '--python-executable', action='store', metavar='EXECUTABLE', help="Python executable used for finding PEP 561 compliant installed" " packages and stubs", dest='special-opts:python_executable') imports_group.add_argument( '--no-site-packages', action='store_true', dest='special-opts:no_executable', help="Do not search for installed PEP 561 compliant packages") imports_group.add_argument( '--no-silence-site-packages', action='store_true', help="Do not silence errors in PEP 561 compliant installed packages") add_invertible_flag( '--namespace-packages', default=False, help="Support namespace packages (PEP 420, __init__.py-less)", group=imports_group) platform_group = parser.add_argument_group( title='Platform configuration', description="Type check code assuming it will be run under certain " "runtime conditions. By default, mypy assumes your code " "will be run using the same operating system and Python " "version you are using to run mypy itself.") platform_group.add_argument( '--python-version', type=parse_version, metavar='x.y', help='Type check code assuming it will be running on Python x.y', dest='special-opts:python_version') platform_group.add_argument( '-2', '--py2', dest='special-opts:python_version', action='store_const', const=defaults.PYTHON2_VERSION, help="Use Python 2 mode (same as --python-version 2.7)") platform_group.add_argument( '--platform', action='store', metavar='PLATFORM', help="Type check special-cased code for the given OS platform " "(defaults to sys.platform)") platform_group.add_argument( '--always-true', metavar='NAME', action='append', default=[], help="Additional variable to be considered True (may be repeated)") platform_group.add_argument( '--always-false', metavar='NAME', action='append', default=[], help="Additional variable to be considered False (may be repeated)") disallow_any_group = parser.add_argument_group( title='Dynamic typing', description= "Disallow the use of the dynamic 'Any' type under certain conditions.") disallow_any_group.add_argument( '--disallow-any-unimported', default=False, action='store_true', help="Disallow Any types resulting from unfollowed imports") add_invertible_flag( '--disallow-subclassing-any', default=False, strict_flag=True, help="Disallow subclassing values of type 'Any' when defining classes", group=disallow_any_group) disallow_any_group.add_argument( '--disallow-any-expr', default=False, action='store_true', help='Disallow all expressions that have type Any') disallow_any_group.add_argument( '--disallow-any-decorated', default=False, action='store_true', help='Disallow functions that have Any in their signature ' 'after decorator transformation') disallow_any_group.add_argument( '--disallow-any-explicit', default=False, action='store_true', help='Disallow explicit Any in type positions') add_invertible_flag( '--disallow-any-generics', default=False, strict_flag=True, help='Disallow usage of generic types that do not specify explicit type ' 'parameters', group=disallow_any_group) untyped_group = parser.add_argument_group( title='Untyped definitions and calls', description="Configure how untyped definitions and calls are handled. " "Note: by default, mypy ignores any untyped function definitions " "and assumes any calls to such functions have a return " "type of 'Any'.") add_invertible_flag( '--disallow-untyped-calls', default=False, strict_flag=True, help="Disallow calling functions without type annotations" " from functions with type annotations", group=untyped_group) add_invertible_flag( '--disallow-untyped-defs', default=False, strict_flag=True, help="Disallow defining functions without type annotations" " or with incomplete type annotations", group=untyped_group) add_invertible_flag( '--disallow-incomplete-defs', default=False, strict_flag=True, help="Disallow defining functions with incomplete type annotations", group=untyped_group) add_invertible_flag( '--check-untyped-defs', default=False, strict_flag=True, help="Type check the interior of functions without type annotations", group=untyped_group) add_invertible_flag( '--disallow-untyped-decorators', default=False, strict_flag=True, help="Disallow decorating typed functions with untyped decorators", group=untyped_group) none_group = parser.add_argument_group( title='None and Optional handling', description= "Adjust how values of type 'None' are handled. For more context on " "how mypy handles values of type 'None', see: " "mypy.readthedocs.io/en/latest/kinds_of_types.html#no-strict-optional") add_invertible_flag( '--no-implicit-optional', default=False, strict_flag=True, help="Don't assume arguments with default values of None are Optional", group=none_group) none_group.add_argument('--strict-optional', action='store_true', help=argparse.SUPPRESS) none_group.add_argument( '--no-strict-optional', action='store_false', dest='strict_optional', help="Disable strict Optional checks (inverse: --strict-optional)") none_group.add_argument( '--strict-optional-whitelist', metavar='GLOB', nargs='*', help="Suppress strict Optional errors in all but the provided files; " "implies --strict-optional (may suppress certain other errors " "in non-whitelisted files)") lint_group = parser.add_argument_group( title='Warnings', description="Detect code that is sound but redundant or problematic.") add_invertible_flag( '--warn-redundant-casts', default=False, strict_flag=True, help="Warn about casting an expression to its inferred type", group=lint_group) add_invertible_flag('--warn-unused-ignores', default=False, strict_flag=True, help="Warn about unneeded '# type: ignore' comments", group=lint_group) add_invertible_flag( '--no-warn-no-return', dest='warn_no_return', default=True, help="Do not warn about functions that end without returning", group=lint_group) add_invertible_flag('--warn-return-any', default=False, strict_flag=True, help="Warn about returning values of type Any" " from non-Any typed functions", group=lint_group) # Note: this group is intentionally added here even though we don't add # --strict to this group near the end. # # That way, this group will appear after the various strictness groups # but before the remaining flags. # We add `--strict` near the end so we don't accidentally miss any strictness # flags that are added after this group. strictness_group = parser.add_argument_group( title='Other strictness checks') add_invertible_flag( '--allow-untyped-globals', default=False, strict_flag=False, help="Suppress toplevel errors caused by missing annotations", group=strictness_group) add_invertible_flag( '--allow-redefinition', default=False, strict_flag=False, help="Allow unconditional variable redefinition with a new type", group=strictness_group) add_invertible_flag( '--strict-equality', default=False, strict_flag=False, help="Prohibit equality, identity, and container checks for" " non-overlapping types", group=strictness_group) incremental_group = parser.add_argument_group( title='Incremental mode', description= "Adjust how mypy incrementally type checks and caches modules. " "Mypy caches type information about modules into a cache to " "let you speed up future invocations of mypy. Also see " "mypy's daemon mode: " "mypy.readthedocs.io/en/latest/mypy_daemon.html#mypy-daemon") incremental_group.add_argument('-i', '--incremental', action='store_true', help=argparse.SUPPRESS) incremental_group.add_argument( '--no-incremental', action='store_false', dest='incremental', help="Disable module cache (inverse: --incremental)") incremental_group.add_argument( '--cache-dir', action='store', metavar='DIR', help="Store module cache info in the given folder in incremental mode " "(defaults to '{}')".format(defaults.CACHE_DIR)) add_invertible_flag('--sqlite-cache', default=False, help="Use a sqlite database to store the cache", group=incremental_group) incremental_group.add_argument( '--cache-fine-grained', action='store_true', help= "Include fine-grained dependency information in the cache for the mypy daemon" ) incremental_group.add_argument( '--skip-version-check', action='store_true', help="Allow using cache written by older mypy version") incremental_group.add_argument( '--skip-cache-mtime-checks', action='store_true', help="Skip cache internal consistency checks based on mtime") internals_group = parser.add_argument_group( title='Mypy internals', description="Debug and customize mypy internals.") internals_group.add_argument('--pdb', action='store_true', help="Invoke pdb on fatal error") internals_group.add_argument('--show-traceback', '--tb', action='store_true', help="Show traceback on fatal error") internals_group.add_argument('--raise-exceptions', action='store_true', help="Raise exception on fatal error") internals_group.add_argument('--custom-typing', metavar='MODULE', dest='custom_typing_module', help="Use a custom typing module") internals_group.add_argument('--custom-typeshed-dir', metavar='DIR', help="Use the custom typeshed in DIR") add_invertible_flag( '--warn-incomplete-stub', default=False, help="Warn if missing type annotation in typeshed, only relevant with" " --disallow-untyped-defs or --disallow-incomplete-defs enabled", group=internals_group) internals_group.add_argument( '--shadow-file', nargs=2, metavar=('SOURCE_FILE', 'SHADOW_FILE'), dest='shadow_file', action='append', help="When encountering SOURCE_FILE, read and type check " "the contents of SHADOW_FILE instead.") add_invertible_flag('--fast-exit', default=False, help=argparse.SUPPRESS, group=internals_group) add_invertible_flag('--new-semantic-analyzer', default=False, help=argparse.SUPPRESS, group=internals_group) error_group = parser.add_argument_group( title='Error reporting', description="Adjust the amount of detail shown in error messages.") add_invertible_flag( '--show-error-context', default=False, dest='show_error_context', help='Precede errors with "note:" messages explaining context', group=error_group) add_invertible_flag('--show-column-numbers', default=False, help="Show column numbers in error messages", group=error_group) strict_help = "Strict mode; enables the following flags: {}".format( ", ".join(strict_flag_names)) strictness_group.add_argument('--strict', action='store_true', dest='special-opts:strict', help=strict_help) report_group = parser.add_argument_group( title='Report generation', description='Generate a report in the specified format.') for report_type in sorted(defaults.REPORTER_NAMES): report_group.add_argument('--%s-report' % report_type.replace('_', '-'), metavar='DIR', dest='special-opts:%s_report' % report_type) other_group = parser.add_argument_group(title='Miscellaneous') other_group.add_argument('--quickstart-file', help=argparse.SUPPRESS) other_group.add_argument('--junit-xml', help="Write junit.xml to the given file") other_group.add_argument( '--scripts-are-modules', action='store_true', help="Script x becomes module x instead of __main__") other_group.add_argument( '--find-occurrences', metavar='CLASS.MEMBER', dest='special-opts:find_occurrences', help="Print out all usages of a class member (experimental)") if server_options: # TODO: This flag is superfluous; remove after a short transition (2018-03-16) other_group.add_argument('--experimental', action='store_true', dest='fine_grained_incremental', help="Enable fine-grained incremental mode") other_group.add_argument( '--use-fine-grained-cache', action='store_true', help="Use the cache in fine-grained incremental mode") # hidden options parser.add_argument('--stats', action='store_true', dest='dump_type_stats', help=argparse.SUPPRESS) parser.add_argument('--inferstats', action='store_true', dest='dump_inference_stats', help=argparse.SUPPRESS) # --debug-cache will disable any cache-related compressions/optimizations, # which will make the cache writing process output pretty-printed JSON (which # is easier to debug). parser.add_argument('--debug-cache', action='store_true', help=argparse.SUPPRESS) # --dump-deps will dump all fine-grained dependencies to stdout parser.add_argument('--dump-deps', action='store_true', help=argparse.SUPPRESS) # --dump-graph will dump the contents of the graph of SCCs and exit. parser.add_argument('--dump-graph', action='store_true', help=argparse.SUPPRESS) # --semantic-analysis-only does exactly that. parser.add_argument('--semantic-analysis-only', action='store_true', help=argparse.SUPPRESS) # --local-partial-types disallows partial types spanning module top level and a function # (implicitly defined in fine-grained incremental mode) parser.add_argument('--local-partial-types', action='store_true', help=argparse.SUPPRESS) # --logical-deps adds some more dependencies that are not semantically needed, but # may be helpful to determine relative importance of classes and functions for overall # type precision in a code base. It also _removes_ some deps, so this flag should be never # used except for generating code stats. This also automatically enables --cache-fine-grained. # NOTE: This is an experimental option that may be modified or removed at any time. parser.add_argument('--logical-deps', action='store_true', help=argparse.SUPPRESS) # --bazel changes some behaviors for use with Bazel (https://bazel.build). parser.add_argument('--bazel', action='store_true', help=argparse.SUPPRESS) # --package-root adds a directory below which directories are considered # packages even without __init__.py. May be repeated. parser.add_argument('--package-root', metavar='ROOT', action='append', default=[], help=argparse.SUPPRESS) # --cache-map FILE ... gives a mapping from source files to cache files. # Each triple of arguments is a source file, a cache meta file, and a cache data file. # Modules not mentioned in the file will go through cache_dir. # Must be followed by another flag or by '--' (and then only file args may follow). parser.add_argument('--cache-map', nargs='+', dest='special-opts:cache_map', help=argparse.SUPPRESS) # options specifying code to check code_group = parser.add_argument_group( title="Running code", description= "Specify the code you want to type check. For more details, see " "mypy.readthedocs.io/en/latest/running_mypy.html#running-mypy") code_group.add_argument( '-m', '--module', action='append', metavar='MODULE', default=[], dest='special-opts:modules', help="Type-check module; can repeat for more modules") code_group.add_argument( '-p', '--package', action='append', metavar='PACKAGE', default=[], dest='special-opts:packages', help="Type-check package recursively; can be repeated") code_group.add_argument('-c', '--command', action='append', metavar='PROGRAM_TEXT', dest='special-opts:command', help="Type-check program passed in as string") code_group.add_argument(metavar='files', nargs='*', dest='special-opts:files', help="Type-check given files or directories") # Parse arguments once into a dummy namespace so we can get the # filename for the config file and know if the user requested all strict options. dummy = argparse.Namespace() parser.parse_args(args, dummy) config_file = dummy.config_file # Don't explicitly test if "config_file is not None" for this check. # This lets `--config-file=` (an empty string) be used to disable all config files. if config_file and not os.path.exists(config_file): parser.error("Cannot find config file '%s'" % config_file) # Parse config file first, so command line can override. options = Options() parse_config_file(options, config_file, stdout, stderr) # Set strict flags before parsing (if strict mode enabled), so other command # line options can override. if getattr(dummy, 'special-opts:strict'): # noqa for dest, value in strict_flag_assignments: setattr(options, dest, value) # Parse command line for real, using a split namespace. special_opts = argparse.Namespace() parser.parse_args(args, SplitNamespace(options, special_opts, 'special-opts:')) # The python_version is either the default, which can be overridden via a config file, # or stored in special_opts and is passed via the command line. options.python_version = special_opts.python_version or options.python_version try: infer_python_executable(options, special_opts) except PythonExecutableInferenceError as e: parser.error(str(e)) if special_opts.no_executable: options.python_executable = None # Paths listed in the config file will be ignored if any paths are passed on # the command line. if options.files and not special_opts.files: special_opts.files = options.files # Check for invalid argument combinations. if require_targets: code_methods = sum( bool(c) for c in [ special_opts.modules + special_opts.packages, special_opts.command, special_opts.files ]) if code_methods == 0: parser.error("Missing target module, package, files, or command.") elif code_methods > 1: parser.error( "May only specify one of: module/package, files, or command.") # Check for overlapping `--always-true` and `--always-false` flags. overlap = set(options.always_true) & set(options.always_false) if overlap: parser.error( "You can't make a variable always true and always false (%s)" % ', '.join(sorted(overlap))) # Set build flags. if options.strict_optional_whitelist is not None: # TODO: Deprecate, then kill this flag options.strict_optional = True if special_opts.find_occurrences: state.find_occurrences = special_opts.find_occurrences.split('.') assert state.find_occurrences is not None if len(state.find_occurrences) < 2: parser.error("Can only find occurrences of class members.") if len(state.find_occurrences) != 2: parser.error( "Can only find occurrences of non-nested class members.") # Set reports. for flag, val in vars(special_opts).items(): if flag.endswith('_report') and val is not None: report_type = flag[:-7].replace('_', '-') report_dir = val options.report_dirs[report_type] = report_dir # Process --package-root. if options.package_root: process_package_roots(fscache, parser, options) # Process --cache-map. if special_opts.cache_map: if options.sqlite_cache: parser.error("--cache-map is incompatible with --sqlite-cache") process_cache_map(parser, special_opts, options) # Let logical_deps imply cache_fine_grained (otherwise the former is useless). if options.logical_deps: options.cache_fine_grained = True # Set target. if special_opts.modules + special_opts.packages: options.build_type = BuildType.MODULE search_paths = SearchPaths((os.getcwd(), ), tuple(mypy_path()), (), ()) targets = [] # TODO: use the same cache that the BuildManager will cache = FindModuleCache(search_paths, fscache) for p in special_opts.packages: if os.sep in p or os.altsep and os.altsep in p: fail("Package name '{}' cannot have a slash in it.".format(p), stderr) p_targets = cache.find_modules_recursive(p) if not p_targets: fail("Can't find package '{}'".format(p), stderr) targets.extend(p_targets) for m in special_opts.modules: targets.append(BuildSource(None, m, None)) return targets, options elif special_opts.command: options.build_type = BuildType.PROGRAM_TEXT targets = [BuildSource(None, None, '\n'.join(special_opts.command))] return targets, options else: try: targets = create_source_list(special_opts.files, options, fscache) # Variable named e2 instead of e to work around mypyc bug #620 # which causes issues when using the same variable to catch # exceptions of different types. except InvalidSourceList as e2: fail(str(e2), stderr) return targets, options
def test_stubs(args: argparse.Namespace, use_builtins_fixtures: bool = False) -> int: """This is stubtest! It's time to test the stubs!""" # Load the allowlist. This is a series of strings corresponding to Error.object_desc # Values in the dict will store whether we used the allowlist entry or not. allowlist = { entry: False for allowlist_file in args.allowlist for entry in get_allowlist_entries(allowlist_file) } allowlist_regexes = {entry: re.compile(entry) for entry in allowlist} # If we need to generate an allowlist, we store Error.object_desc for each error here. generated_allowlist = set() modules = args.modules if args.check_typeshed: assert not args.modules, "Cannot pass both --check-typeshed and a list of modules" modules = get_typeshed_stdlib_modules(args.custom_typeshed_dir) annoying_modules = {"antigravity", "this"} modules = [m for m in modules if m not in annoying_modules] assert modules, "No modules to check" options = Options() options.incremental = False options.custom_typeshed_dir = args.custom_typeshed_dir options.config_file = args.mypy_config_file options.use_builtins_fixtures = use_builtins_fixtures if options.config_file: def set_strict_flags() -> None: # not needed yet return parse_config_file(options, set_strict_flags, options.config_file, sys.stdout, sys.stderr) try: modules = build_stubs(modules, options, find_submodules=not args.check_typeshed) except RuntimeError: return 1 exit_code = 0 for module in modules: for error in test_module(module): # Filter errors if args.ignore_missing_stub and error.is_missing_stub(): continue if args.ignore_positional_only and error.is_positional_only_related( ): continue if error.object_desc in allowlist: allowlist[error.object_desc] = True continue is_allowlisted = False for w in allowlist: if allowlist_regexes[w].fullmatch(error.object_desc): allowlist[w] = True is_allowlisted = True break if is_allowlisted: continue # We have errors, so change exit code, and output whatever necessary exit_code = 1 if args.generate_allowlist: generated_allowlist.add(error.object_desc) continue print(error.get_description(concise=args.concise)) # Print unused allowlist entries if not args.ignore_unused_allowlist: for w in allowlist: # Don't consider an entry unused if it regex-matches the empty string # This lets us allowlist errors that don't manifest at all on some systems if not allowlist[w] and not allowlist_regexes[w].fullmatch(""): exit_code = 1 print("note: unused allowlist entry {}".format(w)) # Print the generated allowlist if args.generate_allowlist: for e in sorted(generated_allowlist): print(e) exit_code = 0 return exit_code