def test_stubs(args: argparse.Namespace) -> int:
"""This is stubtest! It's time to test the stubs!"""
# Load the whitelist. This is a series of strings corresponding to Error.object_desc
# Values in the dict will store whether we used the whitelist entry or not.
whitelist = {
entry: False
for whitelist_file in args.whitelist
for entry in get_whitelist_entries(whitelist_file)
}
# If we need to generate a whitelist, we store Error.object_desc for each error here.
generated_whitelist = set()
modules = args.modules
if args.check_typeshed:
assert not args.modules, "Cannot pass both --check-typeshed and a list of modules"
modules = get_typeshed_stdlib_modules(args.custom_typeshed_dir)
modules.remove("antigravity") # it's super annoying
assert modules, "No modules to check"
options = Options()
options.incremental = False
options.custom_typeshed_dir = args.custom_typeshed_dir
try:
modules = build_stubs(modules, options, find_submodules=not args.check_typeshed)
except RuntimeError:
return 1
exit_code = 0
for module in modules:
for error in test_module(module):
# Filter errors
if args.ignore_missing_stub and error.is_missing_stub():
continue
if args.ignore_positional_only and error.is_positional_only_related():
continue
if error.object_desc in whitelist:
whitelist[error.object_desc] = True
continue
# We have errors, so change exit code, and output whatever necessary
exit_code = 1
if args.generate_whitelist:
generated_whitelist.add(error.object_desc)
continue
print(error.get_description(concise=args.concise))
# Print unused whitelist entries
for w in whitelist:
if not whitelist[w]:
exit_code = 1
print("note: unused whitelist entry {}".format(w))
# Print the generated whitelist
if args.generate_whitelist:
for e in sorted(generated_whitelist):
print(e)
exit_code = 0
return exit_code
def test_stubs(args: argparse.Namespace,
use_builtins_fixtures: bool = False) -> int:
"""This is stubtest! It's time to test the stubs!"""
# Load the allowlist. This is a series of strings corresponding to Error.object_desc
# Values in the dict will store whether we used the allowlist entry or not.
allowlist = {
entry: False
for allowlist_file in args.allowlist
for entry in get_allowlist_entries(allowlist_file)
}
allowlist_regexes = {entry: re.compile(entry) for entry in allowlist}
# If we need to generate an allowlist, we store Error.object_desc for each error here.
generated_allowlist = set()
modules = args.modules
if args.check_typeshed:
assert not args.modules, "Cannot pass both --check-typeshed and a list of modules"
modules = get_typeshed_stdlib_modules(args.custom_typeshed_dir)
annoying_modules = {"antigravity", "this"}
modules = [m for m in modules if m not in annoying_modules]
assert modules, "No modules to check"
options = Options()
options.incremental = False
options.custom_typeshed_dir = args.custom_typeshed_dir
options.config_file = args.mypy_config_file
options.use_builtins_fixtures = use_builtins_fixtures
if options.config_file:
def set_strict_flags() -> None: # not needed yet
return
parse_config_file(options, set_strict_flags, options.config_file,
sys.stdout, sys.stderr)
try:
modules = build_stubs(modules,
options,
find_submodules=not args.check_typeshed)
except RuntimeError:
return 1
exit_code = 0
for module in modules:
for error in test_module(module):
# Filter errors
if args.ignore_missing_stub and error.is_missing_stub():
continue
if args.ignore_positional_only and error.is_positional_only_related(
):
continue
if error.object_desc in allowlist:
allowlist[error.object_desc] = True
continue
is_allowlisted = False
for w in allowlist:
if allowlist_regexes[w].fullmatch(error.object_desc):
allowlist[w] = True
is_allowlisted = True
break
if is_allowlisted:
continue
# We have errors, so change exit code, and output whatever necessary
exit_code = 1
if args.generate_allowlist:
generated_allowlist.add(error.object_desc)
continue
print(error.get_description(concise=args.concise))
# Print unused allowlist entries
if not args.ignore_unused_allowlist:
for w in allowlist:
# Don't consider an entry unused if it regex-matches the empty string
# This lets us allowlist errors that don't manifest at all on some systems
if not allowlist[w] and not allowlist_regexes[w].fullmatch(""):
exit_code = 1
print("note: unused allowlist entry {}".format(w))
# Print the generated allowlist
if args.generate_allowlist:
for e in sorted(generated_allowlist):
print(e)
exit_code = 0
return exit_code
def main() -> int:
assert sys.version_info >= (3, 5), "This script requires at least Python 3.5"
parser = argparse.ArgumentParser(
description="Compares stubs to objects introspected from the runtime."
)
parser.add_argument("modules", nargs="*", help="Modules to test")
parser.add_argument("--concise", action="store_true", help="Make output concise")
parser.add_argument(
"--ignore-missing-stub",
action="store_true",
help="Ignore errors for stub missing things that are present at runtime",
)
parser.add_argument(
"--ignore-positional-only",
action="store_true",
help="Ignore errors for whether an argument should or shouldn't be positional-only",
)
parser.add_argument(
"--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR"
)
parser.add_argument(
"--check-typeshed", action="store_true", help="Check all stdlib modules in typeshed"
)
parser.add_argument(
"--whitelist",
action="append",
metavar="FILE",
default=[],
help=(
"Use file as a whitelist. Can be passed multiple times to combine multiple "
"whitelists. Whitelist can be created with --generate-whitelist"
),
)
parser.add_argument(
"--generate-whitelist",
action="store_true",
help="Print a whitelist (to stdout) to be used with --whitelist",
)
args = parser.parse_args()
# Load the whitelist. This is a series of strings corresponding to Error.object_desc
# Values in the dict will store whether we used the whitelist entry or not.
whitelist = {
entry: False
for whitelist_file in args.whitelist
for entry in get_whitelist_entries(whitelist_file)
}
# If we need to generate a whitelist, we store Error.object_desc for each error here.
generated_whitelist = set()
modules = args.modules
if args.check_typeshed:
assert not args.modules, "Cannot pass both --check-typeshed and a list of modules"
modules = get_typeshed_stdlib_modules(args.custom_typeshed_dir)
modules.remove("antigravity") # it's super annoying
assert modules, "No modules to check"
options = Options()
options.incremental = False
options.custom_typeshed_dir = args.custom_typeshed_dir
modules = build_stubs(modules, options, find_submodules=not args.check_typeshed)
exit_code = 0
for module in modules:
for error in test_module(module):
# Filter errors
if args.ignore_missing_stub and error.is_missing_stub():
continue
if args.ignore_positional_only and error.is_positional_only_related():
continue
if error.object_desc in whitelist:
whitelist[error.object_desc] = True
continue
# We have errors, so change exit code, and output whatever necessary
exit_code = 1
if args.generate_whitelist:
generated_whitelist.add(error.object_desc)
continue
print(error.get_description(concise=args.concise))
# Print unused whitelist entries
for w in whitelist:
if not whitelist[w]:
exit_code = 1
print("note: unused whitelist entry {}".format(w))
# Print the generated whitelist
if args.generate_whitelist:
for e in sorted(generated_whitelist):
print(e)
exit_code = 0
return exit_code
