def refresh_suppressed_submodules(module: str, path: Optional[str], deps: Dict[str, Set[str]], graph: Graph, fscache: FileSystemCache) -> None: """Look for submodules that are now suppressed in target package. If a submodule a.b gets added, we need to mark it as suppressed in modules that contain "from a import b". Previously we assumed that 'a.b' is not a module but a regular name. This is only relevant when following imports normally. Args: module: target package in which to look for submodules path: path of the module """ if path is None or not path.endswith(INIT_SUFFIXES): # Only packages have submodules. return # Find any submodules present in the directory. pkgdir = os.path.dirname(path) for fnam in fscache.listdir(pkgdir): if (not fnam.endswith(('.py', '.pyi')) or fnam.startswith("__init__.") or fnam.count('.') != 1): continue shortname = fnam.split('.')[0] submodule = module + '.' + shortname trigger = make_trigger(submodule) if trigger in deps: for dep in deps[trigger]: # TODO: <...> deps, etc. state = graph.get(dep) if not state: # Maybe it's a non-top-level target. We only care about the module. dep_module = module_prefix(graph, dep) if dep_module is not None: state = graph.get(dep_module) if state: tree = state.tree assert tree # TODO: What if doesn't exist? for imp in tree.imports: if isinstance(imp, ImportFrom): if (imp.id == module and any(name == shortname for name, _ in imp.names)): # TODO: Only if does not exist already state.suppressed.append(submodule) state.suppressed_set.add(submodule)
def filter_out_missing_top_level_packages(packages: Set[str], search_paths: SearchPaths, fscache: FileSystemCache) -> Set[str]: """Quickly filter out obviously missing top-level packages. Return packages with entries that can't be found removed. This is approximate: some packages that aren't actually valid may be included. However, all potentially valid packages must be returned. """ # Start with a empty set and add all potential top-level packages. found = set() paths = ( search_paths.python_path + search_paths.mypy_path + search_paths.package_path + search_paths.typeshed_path ) paths += tuple(os.path.join(p, '@python2') for p in search_paths.typeshed_path) for p in paths: try: entries = fscache.listdir(p) except Exception: entries = [] for entry in entries: # The code is hand-optimized for mypyc since this may be somewhat # performance-critical. if entry.endswith('.py'): entry = entry[:-3] elif entry.endswith('.pyi'): entry = entry[:-4] elif entry.endswith('-stubs'): # Possible PEP 561 stub package entry = entry[:-6] if entry.endswith('-python2'): entry = entry[:-8] if entry in packages: found.add(entry) return found
def refresh_suppressed_submodules( module: str, path: Optional[str], deps: Dict[str, Set[str]], graph: Graph, fscache: FileSystemCache, refresh_file: Callable[[str, str], List[str]]) -> Optional[List[str]]: """Look for submodules that are now suppressed in target package. If a submodule a.b gets added, we need to mark it as suppressed in modules that contain "from a import b". Previously we assumed that 'a.b' is not a module but a regular name. This is only relevant when following imports normally. Args: module: target package in which to look for submodules path: path of the module refresh_file: function that reads the AST of a module (returns error messages) Return a list of errors from refresh_file() if it was called. If the return value is None, we didn't call refresh_file(). """ messages = None if path is None or not path.endswith(INIT_SUFFIXES): # Only packages have submodules. return None # Find any submodules present in the directory. pkgdir = os.path.dirname(path) for fnam in fscache.listdir(pkgdir): if (not fnam.endswith(('.py', '.pyi')) or fnam.startswith("__init__.") or fnam.count('.') != 1): continue shortname = fnam.split('.')[0] submodule = module + '.' + shortname trigger = make_trigger(submodule) # We may be missing the required fine-grained deps. ensure_deps_loaded(module, deps, graph) if trigger in deps: for dep in deps[trigger]: # We can ignore <...> deps since a submodule can't trigger any. state = graph.get(dep) if not state: # Maybe it's a non-top-level target. We only care about the module. dep_module = module_prefix(graph, dep) if dep_module is not None: state = graph.get(dep_module) if state: # Is the file may missing an AST in case it's read from cache? if state.tree is None: # Create AST for the file. This may produce some new errors # that we need to propagate. assert state.path is not None messages = refresh_file(state.id, state.path) tree = state.tree assert tree # Will be fine, due to refresh_file() above for imp in tree.imports: if isinstance(imp, ImportFrom): if (imp.id == module and any(name == shortname for name, _ in imp.names) and submodule not in state.suppressed_set): state.suppressed.append(submodule) state.suppressed_set.add(submodule) return messages