def find_module_paths_using_search(modules: List[str], packages: List[str], search_path: List[str], pyversion: Tuple[int, int]) -> List[StubSource]: """Find sources for modules and packages requested. This function just looks for source files at the file system level. This is used if user passes --no-import, and will not find C modules. Exit if some of the modules or packages can't be found. """ result = [] # type: List[StubSource] typeshed_path = default_lib_path(mypy.build.default_data_dir(), pyversion, None) search_paths = SearchPaths(('.',) + tuple(search_path), (), (), tuple(typeshed_path)) cache = FindModuleCache(search_paths) for module in modules: module_path = cache.find_module(module) if not module_path: fail_missing(module) result.append(StubSource(module, module_path)) for package in packages: p_result = cache.find_modules_recursive(package) if not p_result: fail_missing(package) sources = [StubSource(m.module, m.path) for m in p_result] result.extend(sources) return result
def find_added_suppressed( self, graph: mypy.build.Graph, seen: Set[str], search_paths: SearchPaths) -> List[Tuple[str, str]]: """Find suppressed modules that have been added (and not included in seen). Args: seen: reachable modules we've seen before (mutated here!!) Return suppressed, added modules. """ all_suppressed = set() for module, state in graph.items(): all_suppressed |= state.suppressed_set # Filter out things that shouldn't actually be considered suppressed. # TODO: Figure out why these are treated as suppressed all_suppressed = { module for module in all_suppressed if module not in graph } # TODO: Namespace packages finder = FindModuleCache(search_paths, self.fscache, self.options) found = [] for module in all_suppressed: result = finder.find_module(module) if isinstance(result, str) and module not in seen: found.append((module, result)) seen.add(module) return found
def find_added_suppressed( self, graph: mypy.build.Graph, seen: Set[str], search_paths: SearchPaths) -> List[Tuple[str, str]]: """Find suppressed modules that have been added (and not included in seen). Args: seen: reachable modules we've seen before (mutated here!!) Return suppressed, added modules. """ all_suppressed = set() for state in graph.values(): all_suppressed |= state.suppressed_set # Filter out things that shouldn't actually be considered suppressed. # # TODO: Figure out why these are treated as suppressed all_suppressed = { module for module in all_suppressed if module not in graph and not ignore_suppressed_imports(module) } # Optimization: skip top-level packages that are obviously not # there, to avoid calling the relatively slow find_module() # below too many times. packages = {module.split('.', 1)[0] for module in all_suppressed} packages = filter_out_missing_top_level_packages( packages, search_paths, self.fscache) # TODO: Namespace packages finder = FindModuleCache(search_paths, self.fscache, self.options) found = [] for module in all_suppressed: top_level_pkg = module.split('.', 1)[0] if top_level_pkg not in packages: # Fast path: non-existent top-level package continue result = finder.find_module(module, fast_path=True) if isinstance(result, str) and module not in seen: # When not following imports, we only follow imports to .pyi files. if not self.following_imports() and not result.endswith( '.pyi'): continue found.append((module, result)) seen.add(module) return found
def parse_module(self, program_text: str, incremental_step: int = 0) -> List[Tuple[str, str, str]]: """Return the module and program names for a test case. Normally, the unit tests will parse the default ('__main__') module and follow all the imports listed there. You can override this behavior and instruct the tests to check multiple modules by using a comment like this in the test case input: # cmd: mypy -m foo.bar foo.baz You can also use `# cmdN:` to have a different cmd for incremental step N (2, 3, ...). Return a list of tuples (module name, file name, program text). """ m = re.search('# cmd: mypy -m ([a-zA-Z0-9_. ]+)$', program_text, flags=re.MULTILINE) if incremental_step > 1: alt_regex = '# cmd{}: mypy -m ([a-zA-Z0-9_. ]+)$'.format( incremental_step) alt_m = re.search(alt_regex, program_text, flags=re.MULTILINE) if alt_m is not None: # Optionally return a different command if in a later step # of incremental mode, otherwise default to reusing the # original cmd. m = alt_m if m: # The test case wants to use a non-default main # module. Look up the module and give it as the thing to # analyze. module_names = m.group(1) out = [] search_paths = SearchPaths((test_temp_dir, ), (), (), ()) cache = FindModuleCache(search_paths) for module_name in module_names.split(' '): path = cache.find_module(module_name) assert isinstance( path, str), "Can't find ad hoc case file: %s" % module_name with open(path, encoding='utf8') as f: program_text = f.read() out.append((module_name, path, program_text)) return out else: return [('__main__', 'main', program_text)]
def parse_module(self, program_text: str, incremental_step: int = 0) -> List[Tuple[str, str, str]]: """Return the module and program names for a test case. Normally, the unit tests will parse the default ('__main__') module and follow all the imports listed there. You can override this behavior and instruct the tests to check multiple modules by using a comment like this in the test case input: # cmd: mypy -m foo.bar foo.baz You can also use `# cmdN:` to have a different cmd for incremental step N (2, 3, ...). Return a list of tuples (module name, file name, program text). """ m = re.search('# cmd: mypy -m ([a-zA-Z0-9_. ]+)$', program_text, flags=re.MULTILINE) if incremental_step > 1: alt_regex = '# cmd{}: mypy -m ([a-zA-Z0-9_. ]+)$'.format(incremental_step) alt_m = re.search(alt_regex, program_text, flags=re.MULTILINE) if alt_m is not None: # Optionally return a different command if in a later step # of incremental mode, otherwise default to reusing the # original cmd. m = alt_m if m: # The test case wants to use a non-default main # module. Look up the module and give it as the thing to # analyze. module_names = m.group(1) out = [] search_paths = SearchPaths((test_temp_dir,), (), (), ()) cache = FindModuleCache(search_paths) for module_name in module_names.split(' '): path = cache.find_module(module_name) assert path is not None, "Can't find ad hoc case file" with open(path, encoding='utf8') as f: program_text = f.read() out.append((module_name, path, program_text)) return out else: return [('__main__', 'main', program_text)]
class ModuleFinderSitePackagesSuite(Suite): def setUp(self) -> None: self.package_dir = os.path.relpath( os.path.join( package_path, "modulefinder-site-packages", )) egg_dirs, site_packages = expand_site_packages([self.package_dir]) self.search_paths = SearchPaths( python_path=(), mypy_path=(os.path.join(data_path, "pkg1"), ), package_path=tuple(egg_dirs + site_packages), typeshed_path=(), ) options = Options() options.namespace_packages = True self.fmc_ns = FindModuleCache(self.search_paths, fscache=None, options=options) options = Options() options.namespace_packages = False self.fmc_nons = FindModuleCache(self.search_paths, fscache=None, options=options) def path(self, *parts: str) -> str: return os.path.join(self.package_dir, *parts) def test__packages_with_ns(self) -> None: cases = [ # Namespace package with py.typed ("ns_pkg_typed", self.path("ns_pkg_typed")), ("ns_pkg_typed.a", self.path("ns_pkg_typed", "a.py")), ("ns_pkg_typed.b", self.path("ns_pkg_typed", "b")), ("ns_pkg_typed.b.c", self.path("ns_pkg_typed", "b", "c.py")), ("ns_pkg_typed.a.a_var", ModuleNotFoundReason.NOT_FOUND), # Namespace package without py.typed ("ns_pkg_untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("ns_pkg_untyped.a", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("ns_pkg_untyped.b", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("ns_pkg_untyped.b.c", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("ns_pkg_untyped.a.a_var", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), # Namespace package without stub package ("ns_pkg_w_stubs", self.path("ns_pkg_w_stubs")), ("ns_pkg_w_stubs.typed", self.path("ns_pkg_w_stubs-stubs", "typed", "__init__.pyi")), ("ns_pkg_w_stubs.typed_inline", self.path("ns_pkg_w_stubs", "typed_inline", "__init__.py")), ("ns_pkg_w_stubs.untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), # Regular package with py.typed ("pkg_typed", self.path("pkg_typed", "__init__.py")), ("pkg_typed.a", self.path("pkg_typed", "a.py")), ("pkg_typed.b", self.path("pkg_typed", "b", "__init__.py")), ("pkg_typed.b.c", self.path("pkg_typed", "b", "c.py")), ("pkg_typed.a.a_var", ModuleNotFoundReason.NOT_FOUND), # Regular package without py.typed ("pkg_untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("pkg_untyped.a", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("pkg_untyped.b", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("pkg_untyped.b.c", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("pkg_untyped.a.a_var", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), # Top-level Python file in site-packages ("standalone", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("standalone.standalone_var", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), # Packages found by following .pth files ("baz_pkg", self.path("baz", "baz_pkg", "__init__.py")), ("ns_baz_pkg.a", self.path("baz", "ns_baz_pkg", "a.py")), ("neighbor_pkg", self.path("..", "modulefinder-src", "neighbor_pkg", "__init__.py")), ("ns_neighbor_pkg.a", self.path("..", "modulefinder-src", "ns_neighbor_pkg", "a.py")), # Something that doesn't exist ("does_not_exist", ModuleNotFoundReason.NOT_FOUND), # A regular package with an installed set of stubs ("foo.bar", self.path("foo-stubs", "bar.pyi")), # A regular, non-site-packages module ("a", os.path.join(data_path, "pkg1", "a.py")), ] for module, expected in cases: template = "Find(" + module + ") got {}; expected {}" actual = self.fmc_ns.find_module(module) assert_equal(actual, expected, template) def test__packages_without_ns(self) -> None: cases = [ # Namespace package with py.typed ("ns_pkg_typed", ModuleNotFoundReason.NOT_FOUND), ("ns_pkg_typed.a", ModuleNotFoundReason.NOT_FOUND), ("ns_pkg_typed.b", ModuleNotFoundReason.NOT_FOUND), ("ns_pkg_typed.b.c", ModuleNotFoundReason.NOT_FOUND), ("ns_pkg_typed.a.a_var", ModuleNotFoundReason.NOT_FOUND), # Namespace package without py.typed ("ns_pkg_untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("ns_pkg_untyped.a", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("ns_pkg_untyped.b", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("ns_pkg_untyped.b.c", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("ns_pkg_untyped.a.a_var", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), # Namespace package without stub package ("ns_pkg_w_stubs", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("ns_pkg_w_stubs.typed", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("ns_pkg_w_stubs.typed_inline", self.path("ns_pkg_w_stubs", "typed_inline", "__init__.py")), ("ns_pkg_w_stubs.untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), # Regular package with py.typed ("pkg_typed", self.path("pkg_typed", "__init__.py")), ("pkg_typed.a", self.path("pkg_typed", "a.py")), ("pkg_typed.b", self.path("pkg_typed", "b", "__init__.py")), ("pkg_typed.b.c", self.path("pkg_typed", "b", "c.py")), ("pkg_typed.a.a_var", ModuleNotFoundReason.NOT_FOUND), # Regular package without py.typed ("pkg_untyped", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("pkg_untyped.a", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("pkg_untyped.b", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("pkg_untyped.b.c", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("pkg_untyped.a.a_var", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), # Top-level Python file in site-packages ("standalone", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), ("standalone.standalone_var", ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS), # Packages found by following .pth files ("baz_pkg", self.path("baz", "baz_pkg", "__init__.py")), ("ns_baz_pkg.a", ModuleNotFoundReason.NOT_FOUND), ("neighbor_pkg", self.path("..", "modulefinder-src", "neighbor_pkg", "__init__.py")), ("ns_neighbor_pkg.a", ModuleNotFoundReason.NOT_FOUND), # Something that doesn't exist ("does_not_exist", ModuleNotFoundReason.NOT_FOUND), # A regular package with an installed set of stubs ("foo.bar", self.path("foo-stubs", "bar.pyi")), # A regular, non-site-packages module ("a", os.path.join(data_path, "pkg1", "a.py")), ] for module, expected in cases: template = "Find(" + module + ") got {}; expected {}" actual = self.fmc_nons.find_module(module) assert_equal(actual, expected, template)
class ModuleFinderSuite(Suite): def setUp(self) -> None: self.search_paths = SearchPaths( python_path=(), mypy_path=( os.path.join(data_path, "nsx-pkg1"), os.path.join(data_path, "nsx-pkg2"), os.path.join(data_path, "nsx-pkg3"), os.path.join(data_path, "nsy-pkg1"), os.path.join(data_path, "nsy-pkg2"), os.path.join(data_path, "pkg1"), os.path.join(data_path, "pkg2"), ), package_path=(), typeshed_path=(), ) options = Options() options.namespace_packages = True self.fmc_ns = FindModuleCache(self.search_paths, fscache=None, options=options) options = Options() options.namespace_packages = False self.fmc_nons = FindModuleCache(self.search_paths, fscache=None, options=options) def test__no_namespace_packages__nsx(self) -> None: """ If namespace_packages is False, we shouldn't find nsx """ found_module = self.fmc_nons.find_module("nsx") assert_equal(ModuleNotFoundReason.NOT_FOUND, found_module) def test__no_namespace_packages__nsx_a(self) -> None: """ If namespace_packages is False, we shouldn't find nsx.a. """ found_module = self.fmc_nons.find_module("nsx.a") assert_equal(ModuleNotFoundReason.NOT_FOUND, found_module) def test__no_namespace_packages__find_a_in_pkg1(self) -> None: """ Find find pkg1/a.py for "a" with namespace_packages False. """ found_module = self.fmc_nons.find_module("a") expected = os.path.join(data_path, "pkg1", "a.py") assert_equal(expected, found_module) def test__no_namespace_packages__find_b_in_pkg2(self) -> None: found_module = self.fmc_ns.find_module("b") expected = os.path.join(data_path, "pkg2", "b", "__init__.py") assert_equal(expected, found_module) def test__find_nsx_as_namespace_pkg_in_pkg1(self) -> None: """ There's no __init__.py in any of the nsx dirs, return the path to the first one found in mypypath. """ found_module = self.fmc_ns.find_module("nsx") expected = os.path.join(data_path, "nsx-pkg1", "nsx") assert_equal(expected, found_module) def test__find_nsx_a_init_in_pkg1(self) -> None: """ Find nsx-pkg1/nsx/a/__init__.py for "nsx.a" in namespace mode. """ found_module = self.fmc_ns.find_module("nsx.a") expected = os.path.join(data_path, "nsx-pkg1", "nsx", "a", "__init__.py") assert_equal(expected, found_module) def test__find_nsx_b_init_in_pkg2(self) -> None: """ Find nsx-pkg2/nsx/b/__init__.py for "nsx.b" in namespace mode. """ found_module = self.fmc_ns.find_module("nsx.b") expected = os.path.join(data_path, "nsx-pkg2", "nsx", "b", "__init__.py") assert_equal(expected, found_module) def test__find_nsx_c_c_in_pkg3(self) -> None: """ Find nsx-pkg3/nsx/c/c.py for "nsx.c.c" in namespace mode. """ found_module = self.fmc_ns.find_module("nsx.c.c") expected = os.path.join(data_path, "nsx-pkg3", "nsx", "c", "c.py") assert_equal(expected, found_module) def test__find_nsy_a__init_pyi(self) -> None: """ Prefer nsy-pkg1/a/__init__.pyi file over __init__.py. """ found_module = self.fmc_ns.find_module("nsy.a") expected = os.path.join(data_path, "nsy-pkg1", "nsy", "a", "__init__.pyi") assert_equal(expected, found_module) def test__find_nsy_b__init_py(self) -> None: """ There is a nsy-pkg2/nsy/b.pyi, but also a nsy-pkg2/nsy/b/__init__.py. We expect to find the latter when looking up "nsy.b" as a package is preferred over a module. """ found_module = self.fmc_ns.find_module("nsy.b") expected = os.path.join(data_path, "nsy-pkg2", "nsy", "b", "__init__.py") assert_equal(expected, found_module) def test__find_nsy_c_pyi(self) -> None: """ There is a nsy-pkg2/nsy/c.pyi and nsy-pkg2/nsy/c.py We expect to find the former when looking up "nsy.b" as .pyi is preferred over .py. """ found_module = self.fmc_ns.find_module("nsy.c") expected = os.path.join(data_path, "nsy-pkg2", "nsy", "c.pyi") assert_equal(expected, found_module) def test__find_a_in_pkg1(self) -> None: found_module = self.fmc_ns.find_module("a") expected = os.path.join(data_path, "pkg1", "a.py") assert_equal(expected, found_module) def test__find_b_init_in_pkg2(self) -> None: found_module = self.fmc_ns.find_module("b") expected = os.path.join(data_path, "pkg2", "b", "__init__.py") assert_equal(expected, found_module) def test__find_d_nowhere(self) -> None: found_module = self.fmc_ns.find_module("d") assert_equal(ModuleNotFoundReason.NOT_FOUND, found_module)