def find_module_paths_using_search(modules: List[str], packages: List[str],
search_path: List[str],
pyversion: Tuple[int, int]) -> List[StubSource]:
"""Find sources for modules and packages requested.
This function just looks for source files at the file system level.
This is used if user passes --no-import, and will not find C modules.
Exit if some of the modules or packages can't be found.
"""
result = [] # type: List[StubSource]
typeshed_path = default_lib_path(mypy.build.default_data_dir(), pyversion, None)
search_paths = SearchPaths(('.',) + tuple(search_path), (), (), tuple(typeshed_path))
cache = FindModuleCache(search_paths)
for module in modules:
module_path = cache.find_module(module)
if not module_path:
fail_missing(module)
result.append(StubSource(module, module_path))
for package in packages:
p_result = cache.find_modules_recursive(package)
if not p_result:
fail_missing(package)
sources = [StubSource(m.module, m.path) for m in p_result]
result.extend(sources)
return result
def find_added_suppressed(
self, graph: mypy.build.Graph, seen: Set[str],
search_paths: SearchPaths) -> List[Tuple[str, str]]:
"""Find suppressed modules that have been added (and not included in seen).
Args:
seen: reachable modules we've seen before (mutated here!!)
Return suppressed, added modules.
"""
all_suppressed = set()
for module, state in graph.items():
all_suppressed |= state.suppressed_set
# Filter out things that shouldn't actually be considered suppressed.
# TODO: Figure out why these are treated as suppressed
all_suppressed = {
module
for module in all_suppressed if module not in graph
}
# TODO: Namespace packages
finder = FindModuleCache(search_paths, self.fscache, self.options)
found = []
for module in all_suppressed:
result = finder.find_module(module)
if isinstance(result, str) and module not in seen:
found.append((module, result))
seen.add(module)
return found
def find_added_suppressed(
self, graph: mypy.build.Graph, seen: Set[str],
search_paths: SearchPaths) -> List[Tuple[str, str]]:
"""Find suppressed modules that have been added (and not included in seen).
Args:
seen: reachable modules we've seen before (mutated here!!)
Return suppressed, added modules.
"""
all_suppressed = set()
for state in graph.values():
all_suppressed |= state.suppressed_set
# Filter out things that shouldn't actually be considered suppressed.
#
# TODO: Figure out why these are treated as suppressed
all_suppressed = {
module
for module in all_suppressed
if module not in graph and not ignore_suppressed_imports(module)
}
# Optimization: skip top-level packages that are obviously not
# there, to avoid calling the relatively slow find_module()
# below too many times.
packages = {module.split('.', 1)[0] for module in all_suppressed}
packages = filter_out_missing_top_level_packages(
packages, search_paths, self.fscache)
# TODO: Namespace packages
finder = FindModuleCache(search_paths, self.fscache, self.options)
found = []
for module in all_suppressed:
top_level_pkg = module.split('.', 1)[0]
if top_level_pkg not in packages:
# Fast path: non-existent top-level package
continue
result = finder.find_module(module, fast_path=True)
if isinstance(result, str) and module not in seen:
# When not following imports, we only follow imports to .pyi files.
if not self.following_imports() and not result.endswith(
'.pyi'):
continue
found.append((module, result))
seen.add(module)
return found
def parse_module(self,
program_text: str,
incremental_step: int = 0) -> List[Tuple[str, str, str]]:
"""Return the module and program names for a test case.
Normally, the unit tests will parse the default ('__main__')
module and follow all the imports listed there. You can override
this behavior and instruct the tests to check multiple modules
by using a comment like this in the test case input:
# cmd: mypy -m foo.bar foo.baz
You can also use `# cmdN:` to have a different cmd for incremental
step N (2, 3, ...).
Return a list of tuples (module name, file name, program text).
"""
m = re.search('# cmd: mypy -m ([a-zA-Z0-9_. ]+)$',
program_text,
flags=re.MULTILINE)
if incremental_step > 1:
alt_regex = '# cmd{}: mypy -m ([a-zA-Z0-9_. ]+)$'.format(
incremental_step)
alt_m = re.search(alt_regex, program_text, flags=re.MULTILINE)
if alt_m is not None:
# Optionally return a different command if in a later step
# of incremental mode, otherwise default to reusing the
# original cmd.
m = alt_m
if m:
# The test case wants to use a non-default main
# module. Look up the module and give it as the thing to
# analyze.
module_names = m.group(1)
out = []
search_paths = SearchPaths((test_temp_dir, ), (), (), ())
cache = FindModuleCache(search_paths)
for module_name in module_names.split(' '):
path = cache.find_module(module_name)
assert isinstance(
path, str), "Can't find ad hoc case file: %s" % module_name
with open(path, encoding='utf8') as f:
program_text = f.read()
out.append((module_name, path, program_text))
return out
else:
return [('__main__', 'main', program_text)]
def parse_module(self,
program_text: str,
incremental_step: int = 0) -> List[Tuple[str, str, str]]:
"""Return the module and program names for a test case.
Normally, the unit tests will parse the default ('__main__')
module and follow all the imports listed there. You can override
this behavior and instruct the tests to check multiple modules
by using a comment like this in the test case input:
# cmd: mypy -m foo.bar foo.baz
You can also use `# cmdN:` to have a different cmd for incremental
step N (2, 3, ...).
Return a list of tuples (module name, file name, program text).
"""
m = re.search('# cmd: mypy -m ([a-zA-Z0-9_. ]+)$', program_text, flags=re.MULTILINE)
if incremental_step > 1:
alt_regex = '# cmd{}: mypy -m ([a-zA-Z0-9_. ]+)$'.format(incremental_step)
alt_m = re.search(alt_regex, program_text, flags=re.MULTILINE)
if alt_m is not None:
# Optionally return a different command if in a later step
# of incremental mode, otherwise default to reusing the
# original cmd.
m = alt_m
if m:
# The test case wants to use a non-default main
# module. Look up the module and give it as the thing to
# analyze.
module_names = m.group(1)
out = []
search_paths = SearchPaths((test_temp_dir,), (), (), ())
cache = FindModuleCache(search_paths)
for module_name in module_names.split(' '):
path = cache.find_module(module_name)
assert path is not None, "Can't find ad hoc case file"
with open(path, encoding='utf8') as f:
program_text = f.read()
out.append((module_name, path, program_text))
return out
else:
return [('__main__', 'main', program_text)]
class ModuleFinderSitePackagesSuite(Suite):
def setUp(self) -> None:
self.package_dir = os.path.relpath(
os.path.join(
package_path,
"modulefinder-site-packages",
))
egg_dirs, site_packages = expand_site_packages([self.package_dir])
self.search_paths = SearchPaths(
python_path=(),
mypy_path=(os.path.join(data_path, "pkg1"), ),
package_path=tuple(egg_dirs + site_packages),
typeshed_path=(),
)
options = Options()
options.namespace_packages = True
self.fmc_ns = FindModuleCache(self.search_paths,
fscache=None,
options=options)
options = Options()
options.namespace_packages = False
self.fmc_nons = FindModuleCache(self.search_paths,
fscache=None,
options=options)
def path(self, *parts: str) -> str:
return os.path.join(self.package_dir, *parts)
def test__packages_with_ns(self) -> None:
cases = [
# Namespace package with py.typed
("ns_pkg_typed", self.path("ns_pkg_typed")),
("ns_pkg_typed.a", self.path("ns_pkg_typed", "a.py")),
("ns_pkg_typed.b", self.path("ns_pkg_typed", "b")),
("ns_pkg_typed.b.c", self.path("ns_pkg_typed", "b", "c.py")),
("ns_pkg_typed.a.a_var", ModuleNotFoundReason.NOT_FOUND),
# Namespace package without py.typed
("ns_pkg_untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("ns_pkg_untyped.a",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("ns_pkg_untyped.b",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("ns_pkg_untyped.b.c",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("ns_pkg_untyped.a.a_var",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
# Namespace package without stub package
("ns_pkg_w_stubs", self.path("ns_pkg_w_stubs")),
("ns_pkg_w_stubs.typed",
self.path("ns_pkg_w_stubs-stubs", "typed", "__init__.pyi")),
("ns_pkg_w_stubs.typed_inline",
self.path("ns_pkg_w_stubs", "typed_inline", "__init__.py")),
("ns_pkg_w_stubs.untyped",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
# Regular package with py.typed
("pkg_typed", self.path("pkg_typed", "__init__.py")),
("pkg_typed.a", self.path("pkg_typed", "a.py")),
("pkg_typed.b", self.path("pkg_typed", "b", "__init__.py")),
("pkg_typed.b.c", self.path("pkg_typed", "b", "c.py")),
("pkg_typed.a.a_var", ModuleNotFoundReason.NOT_FOUND),
# Regular package without py.typed
("pkg_untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("pkg_untyped.a", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("pkg_untyped.b", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("pkg_untyped.b.c", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("pkg_untyped.a.a_var",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
# Top-level Python file in site-packages
("standalone", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("standalone.standalone_var",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
# Packages found by following .pth files
("baz_pkg", self.path("baz", "baz_pkg", "__init__.py")),
("ns_baz_pkg.a", self.path("baz", "ns_baz_pkg", "a.py")),
("neighbor_pkg",
self.path("..", "modulefinder-src", "neighbor_pkg",
"__init__.py")),
("ns_neighbor_pkg.a",
self.path("..", "modulefinder-src", "ns_neighbor_pkg", "a.py")),
# Something that doesn't exist
("does_not_exist", ModuleNotFoundReason.NOT_FOUND),
# A regular package with an installed set of stubs
("foo.bar", self.path("foo-stubs", "bar.pyi")),
# A regular, non-site-packages module
("a", os.path.join(data_path, "pkg1", "a.py")),
]
for module, expected in cases:
template = "Find(" + module + ") got {}; expected {}"
actual = self.fmc_ns.find_module(module)
assert_equal(actual, expected, template)
def test__packages_without_ns(self) -> None:
cases = [
# Namespace package with py.typed
("ns_pkg_typed", ModuleNotFoundReason.NOT_FOUND),
("ns_pkg_typed.a", ModuleNotFoundReason.NOT_FOUND),
("ns_pkg_typed.b", ModuleNotFoundReason.NOT_FOUND),
("ns_pkg_typed.b.c", ModuleNotFoundReason.NOT_FOUND),
("ns_pkg_typed.a.a_var", ModuleNotFoundReason.NOT_FOUND),
# Namespace package without py.typed
("ns_pkg_untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("ns_pkg_untyped.a",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("ns_pkg_untyped.b",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("ns_pkg_untyped.b.c",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("ns_pkg_untyped.a.a_var",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
# Namespace package without stub package
("ns_pkg_w_stubs", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("ns_pkg_w_stubs.typed",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("ns_pkg_w_stubs.typed_inline",
self.path("ns_pkg_w_stubs", "typed_inline", "__init__.py")),
("ns_pkg_w_stubs.untyped",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
# Regular package with py.typed
("pkg_typed", self.path("pkg_typed", "__init__.py")),
("pkg_typed.a", self.path("pkg_typed", "a.py")),
("pkg_typed.b", self.path("pkg_typed", "b", "__init__.py")),
("pkg_typed.b.c", self.path("pkg_typed", "b", "c.py")),
("pkg_typed.a.a_var", ModuleNotFoundReason.NOT_FOUND),
# Regular package without py.typed
("pkg_untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("pkg_untyped.a", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("pkg_untyped.b", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("pkg_untyped.b.c", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("pkg_untyped.a.a_var",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
# Top-level Python file in site-packages
("standalone", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
("standalone.standalone_var",
ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS),
# Packages found by following .pth files
("baz_pkg", self.path("baz", "baz_pkg", "__init__.py")),
("ns_baz_pkg.a", ModuleNotFoundReason.NOT_FOUND),
("neighbor_pkg",
self.path("..", "modulefinder-src", "neighbor_pkg",
"__init__.py")),
("ns_neighbor_pkg.a", ModuleNotFoundReason.NOT_FOUND),
# Something that doesn't exist
("does_not_exist", ModuleNotFoundReason.NOT_FOUND),
# A regular package with an installed set of stubs
("foo.bar", self.path("foo-stubs", "bar.pyi")),
# A regular, non-site-packages module
("a", os.path.join(data_path, "pkg1", "a.py")),
]
for module, expected in cases:
template = "Find(" + module + ") got {}; expected {}"
actual = self.fmc_nons.find_module(module)
assert_equal(actual, expected, template)
class ModuleFinderSuite(Suite):
def setUp(self) -> None:
self.search_paths = SearchPaths(
python_path=(),
mypy_path=(
os.path.join(data_path, "nsx-pkg1"),
os.path.join(data_path, "nsx-pkg2"),
os.path.join(data_path, "nsx-pkg3"),
os.path.join(data_path, "nsy-pkg1"),
os.path.join(data_path, "nsy-pkg2"),
os.path.join(data_path, "pkg1"),
os.path.join(data_path, "pkg2"),
),
package_path=(),
typeshed_path=(),
)
options = Options()
options.namespace_packages = True
self.fmc_ns = FindModuleCache(self.search_paths,
fscache=None,
options=options)
options = Options()
options.namespace_packages = False
self.fmc_nons = FindModuleCache(self.search_paths,
fscache=None,
options=options)
def test__no_namespace_packages__nsx(self) -> None:
"""
If namespace_packages is False, we shouldn't find nsx
"""
found_module = self.fmc_nons.find_module("nsx")
assert_equal(ModuleNotFoundReason.NOT_FOUND, found_module)
def test__no_namespace_packages__nsx_a(self) -> None:
"""
If namespace_packages is False, we shouldn't find nsx.a.
"""
found_module = self.fmc_nons.find_module("nsx.a")
assert_equal(ModuleNotFoundReason.NOT_FOUND, found_module)
def test__no_namespace_packages__find_a_in_pkg1(self) -> None:
"""
Find find pkg1/a.py for "a" with namespace_packages False.
"""
found_module = self.fmc_nons.find_module("a")
expected = os.path.join(data_path, "pkg1", "a.py")
assert_equal(expected, found_module)
def test__no_namespace_packages__find_b_in_pkg2(self) -> None:
found_module = self.fmc_ns.find_module("b")
expected = os.path.join(data_path, "pkg2", "b", "__init__.py")
assert_equal(expected, found_module)
def test__find_nsx_as_namespace_pkg_in_pkg1(self) -> None:
"""
There's no __init__.py in any of the nsx dirs, return
the path to the first one found in mypypath.
"""
found_module = self.fmc_ns.find_module("nsx")
expected = os.path.join(data_path, "nsx-pkg1", "nsx")
assert_equal(expected, found_module)
def test__find_nsx_a_init_in_pkg1(self) -> None:
"""
Find nsx-pkg1/nsx/a/__init__.py for "nsx.a" in namespace mode.
"""
found_module = self.fmc_ns.find_module("nsx.a")
expected = os.path.join(data_path, "nsx-pkg1", "nsx", "a",
"__init__.py")
assert_equal(expected, found_module)
def test__find_nsx_b_init_in_pkg2(self) -> None:
"""
Find nsx-pkg2/nsx/b/__init__.py for "nsx.b" in namespace mode.
"""
found_module = self.fmc_ns.find_module("nsx.b")
expected = os.path.join(data_path, "nsx-pkg2", "nsx", "b",
"__init__.py")
assert_equal(expected, found_module)
def test__find_nsx_c_c_in_pkg3(self) -> None:
"""
Find nsx-pkg3/nsx/c/c.py for "nsx.c.c" in namespace mode.
"""
found_module = self.fmc_ns.find_module("nsx.c.c")
expected = os.path.join(data_path, "nsx-pkg3", "nsx", "c", "c.py")
assert_equal(expected, found_module)
def test__find_nsy_a__init_pyi(self) -> None:
"""
Prefer nsy-pkg1/a/__init__.pyi file over __init__.py.
"""
found_module = self.fmc_ns.find_module("nsy.a")
expected = os.path.join(data_path, "nsy-pkg1", "nsy", "a",
"__init__.pyi")
assert_equal(expected, found_module)
def test__find_nsy_b__init_py(self) -> None:
"""
There is a nsy-pkg2/nsy/b.pyi, but also a nsy-pkg2/nsy/b/__init__.py.
We expect to find the latter when looking up "nsy.b" as
a package is preferred over a module.
"""
found_module = self.fmc_ns.find_module("nsy.b")
expected = os.path.join(data_path, "nsy-pkg2", "nsy", "b",
"__init__.py")
assert_equal(expected, found_module)
def test__find_nsy_c_pyi(self) -> None:
"""
There is a nsy-pkg2/nsy/c.pyi and nsy-pkg2/nsy/c.py
We expect to find the former when looking up "nsy.b" as
.pyi is preferred over .py.
"""
found_module = self.fmc_ns.find_module("nsy.c")
expected = os.path.join(data_path, "nsy-pkg2", "nsy", "c.pyi")
assert_equal(expected, found_module)
def test__find_a_in_pkg1(self) -> None:
found_module = self.fmc_ns.find_module("a")
expected = os.path.join(data_path, "pkg1", "a.py")
assert_equal(expected, found_module)
def test__find_b_init_in_pkg2(self) -> None:
found_module = self.fmc_ns.find_module("b")
expected = os.path.join(data_path, "pkg2", "b", "__init__.py")
assert_equal(expected, found_module)
def test__find_d_nowhere(self) -> None:
found_module = self.fmc_ns.find_module("d")
assert_equal(ModuleNotFoundReason.NOT_FOUND, found_module)