def __init__(self): self.patcher_functions = {} if sys.version_info[0] >= 3: from importlib.machinery import PathFinder self.pathfinder = PathFinder()
def test_path_hook_installed(self): # PathFinder can only use it with sys.executable on sys.path with patch('sys.path', sys.path): sys.path = [p for p in sys.path if p != sys.executable] PathFinder.invalidate_caches() self.assertIsNone(PathFinder.find_spec("pwd")) sys.path.append(sys.executable) spec = PathFinder.find_spec("pwd") self.assert_spec(spec, "pwd", is_pkg=False, Loader=sys.__spec__.loader, origin="built-in")
def get_importable_spec(self, symbol_is_module=False): """Returns the specification of an actual importable module. This is a check based on the limitations that we do not actually perform an import, and assumes a directory structure with modules. :param symbol_is_module: if it's known that the symbol is also a module, include it in the search for an importable spec :type symbol_is_module: bool """ modules_paths = sys.path modules_paths.insert(0, self.get_relative_module_fs_path()) spec = None for component, previous in self._walk_importable_components( symbol_is_module): if previous: modules_paths = [ os.path.join(mod, previous) for mod in modules_paths[:] ] spec = PathFinder.find_spec(component, modules_paths) if spec is None: break return spec
def _find_import_match(parent_path, parent_module): """Attempts to find an importable module.""" modules_paths = [parent_path] + sys.path found_spec = PathFinder.find_spec(parent_module, modules_paths) if found_spec is None: raise ClassNotSuitable return found_spec
def name_path(mod, pkg): loader = None parent = None path = sys.path if pkg: pkg_path = name_path(pkg, None) if pkg_path: path = [dirname(pkg_path), ] + path parts = mod.split('.') prefix = '' for part in parts: if not part: prefix += '.' continue name = prefix + part loader = PathFinder.find_module(name, path) prefix = '' if loader: parent = dirname(loader.path) if loader.is_package(name): path = parent, else: path = tuple() if not loader: return None return loader.path
def find_spec(cls, fullname, path=None, target=None): parts = fullname.split('.') if len(parts) > 2 and parts[:2] == ['eddy', 'plugins']: from importlib.machinery import PathFinder return PathFinder.find_spec( fullname, [os.path.join(os.path.dirname(__file__), os.pardir)], target)
def find_spec( self, fullname: str, path: Optional[Sequence[Union[bytes, str]]], target: Optional[ModuleType] = None, ): if _managers: index = -1 module_spec = PathFinder.find_spec(fullname, path, target) if not module_spec: return module_origin = module_spec.origin if not module_origin: return module_path = Path(module_origin).resolve() while -index <= len(_managers): manager = _managers[index] if ( fullname in manager.plugins or module_path in manager.searched_plugins.values() ): module_spec.loader = PluginLoader(manager, fullname, module_origin) return module_spec index -= 1 return
def find_spec(cls, fullname, path, target=None): if any(fullname.startswith(rdir) for rdir in (RECIPE_DIR, CONFIG_DIR)): if path is None: path = [os.path.abspath('.')] elif isinstance(path, str): path = [os.path.abspath(os.path.join('.', path))] spec = PathFinder.find_spec(fullname, path, target) return spec
def find_spec(cls, fullname, path, target=None): for rprefix, rdir in ((gs.RECIPE_PREFIX, gs.RECIPE_PATH), (gs.CONFIG_PREFIX, gs.CONFIG_PATH)): if fullname.startswith(rprefix): if path is None: path = [os.path.abspath(rdir)] elif isinstance(path, str): path = [os.path.abspath(os.path.join(rdir, path))] spec = PathFinder.find_spec(fullname, path, target) return spec
def find_spec(cls, fullname, path=None, target=None): spec = PathFinder.find_spec(fullname, path=path, target=target) if spec is None: return if spec.loader is not None: if hasattr(spec.loader, 'exec_module'): spec.loader = CustomLoader(spec.loader) else: spec.loader = LegacyCustomLoader(spec.loader) return spec
def load(directory): curdir = os.path.join(directory, 'scripts', 'pbs', 'commands') commands = {} for dir, subdirs, files in os.walk(curdir): for filename in files: name, ext = os.path.splitext(filename) if ext == '.py' and name != '__init__': loader = PathFinder.find_module(name, [curdir]) commands[name] = loader.load_module() return commands
def find_spec( self, fullname: str, path: Optional[Sequence[Union[bytes, str]]] = None, target: Optional[types.ModuleType] = None, ) -> Optional[ModuleSpec]: if fullname != "pip": return None spec = PathFinder.find_spec(fullname, [PIP_SOURCES_ROOT], target) assert spec, (PIP_SOURCES_ROOT, fullname) return spec
def find_spec(cls, fullname: str, path=None, target=None): """Try to find a spec for 'fullname' on sys.path or 'path'. Search only for modules starting with 'labs.ext.' and use PathFinder to first search for submodule in the current build, then module starting with labs_. """ if not fullname.startswith(_prefix): return None # Not a labs extension _name = fullname[_offset:] if '.' in _name: return None # Extension already imported, this is a submodule. # Try importing non packaged version paths_1, paths_2 = _get_ctx_path() if paths_1: spec = PathFinder.find_spec(fullname, paths_1, target) if spec is not None: return _patch_spec(fullname, spec) # Correct the module's name. # Try importing a packaged version installed_name = f'labs_{_name}' if paths_2: spec = PathFinder.find_spec(installed_name, paths_2, target) if spec is not None: return _patch_spec(fullname, spec) # Correct the module's name. # Try importing an installed version spec = PathFinder.find_spec(installed_name, None, target) if spec is not None: return _patch_spec(fullname, spec) # Correct the module's name. return None
def _get_module(app, modname): # Find out the app's __path__ try: app_path = import_module(app).__path__ except AttributeError: # pragma: no cover return # Use importlib's PathFinder().find_spec() to find the app's modname.py if PathFinder().find_spec(modname, app_path) is None: return # Import the app's module file import_module('{}.{}'.format(app, modname))
def find_spec(self, fullname: str, path, target): if _manager_stack: index = -1 origin_spec = PathFinder.find_spec(fullname, path, target) while -index <= len(_manager_stack): manager = _manager_stack[index] rel_name = None if origin_spec and origin_spec.origin: rel_name = manager._check_absolute_import( origin_spec.origin) newname = manager._rewrite_module_name(rel_name or fullname) if newname: spec = PathFinder.find_spec( newname, path or [*manager.search_path, *sys.path], target) if spec: spec.loader = PluginLoader( # type: ignore manager, newname, spec.origin) return spec index -= 1 return None
def get_alternative(mod, pkg = None): if mod.startswith('.') and pkg: loader = PathFinder.find_module(pkg, path) if loader: modpath = ast.name_path(mod, pkg) mod = pkg + mod fname = mod.replace('.', '/') + '.js' # FIXME for p in path: if os.path.isdir(p): candidate = os.path.join(p, fname) if os.path.isfile(candidate): return candidate, mod return None, None
def get_package_directory(module_name): """Return Path of the package directory that the given module is in.""" base_module = module_name.split('.', 1)[0] # Find the import path of the package, excluding the `''` entry in `sys.path` so as # to avoid the false positive of returning the CWD if the user is in the development # directory (which may also be in the path for the case of an editable install, # which is why we skip `''` specifically, and not anything that expands to CWD). spec = PathFinder.find_spec(base_module, path=[p for p in sys.path if p]) if spec is None or spec.origin is None: raise ModuleNotFoundError(base_module) if not spec.parent: msg = f'{base_module} is not a package' raise ValueError(msg) return Path(spec.origin).parent
def find_spec(self, fullname: str, path, target): if _manager_stack: index = -1 while -index <= len(_manager_stack): manager = _manager_stack[index] newname = manager._rewrite_module_name(fullname) if newname: spec = PathFinder.find_spec(newname, list(manager.search_path), target) if spec: return spec index -= 1 return None
def _load_module(module_name, filename, *paths): name, ext = os.path.splitext(filename) if ext in ('.py', ''): loader = PathFinder.find_module(name, path=paths) if loader is None: loader = PathFinder.find_module(name) if loader is None: raise ImportError( 'Could not find module {} on path {} or sys.path'.format( name, paths)) else: loader = None for path in paths: p = os.path.normpath(os.path.join(path, filename)) if os.path.exists(p): loader = SourceFileLoader(module_name, p) if loader is None: raise ImportError( 'Could not find module {} on path {}'.format(name, paths)) loader.name = module_name return loader.load_module()
def find_spec(self, fullname, path, target=None): spec = None if fullname in self._soft_patches or fullname in self._hard_patches: spec = PathFinder.find_spec(fullname, path, target=target) if spec is None: raise Exception("Could not patch %s" % fullname) if fullname in self._soft_patches: self._wrap_soft_patches(spec.loader, self._soft_patches[fullname]) elif fullname in self._hard_patches: self._wrap_hard_patch(spec.loader, self._hard_patches[fullname]) return spec
def find_spec(self, fullname, path=None, target=None): if world.rank == 0: spec = PathFinder.find_spec(fullname, path, target) if spec is None: return None code = spec.loader.get_code(fullname) if code is None: # C extensions return None loader = BroadcastLoader(spec, self.module_cache) assert fullname == spec.name searchloc = spec.submodule_search_locations spec = ModuleSpec(fullname, loader, origin=spec.origin, is_package=searchloc is not None) if searchloc is not None: spec.submodule_search_locations += searchloc return spec else: if fullname not in self.module_cache: # Could this in principle interfere with builtin imports? return PathFinder.find_spec(fullname, path, target) searchloc, origin = self.module_cache[fullname][0] loader = BroadcastLoader(None, self.module_cache) spec = ModuleSpec(fullname, loader, origin=origin, is_package=searchloc is not None) if searchloc is not None: spec.submodule_search_locations += searchloc loader.spec = spec # XXX loader.loader is still None return spec
def find_spec(self, fullname: str, path, target): if _manager_stack: index = -1 while -index <= len(_manager_stack): manager = _manager_stack[index] newname = manager._rewrite_module_name(fullname) if newname: spec = PathFinder.find_spec( newname, [*manager.search_path, *(path or sys.path)], target) if spec: spec.loader = PluginLoader(manager, newname, spec.origin) return spec index -= 1 return None
def import_plugin(self, name, additional_paths=[]): """ Mimics Python's import mechanism to import plugins found in the specified plugin directories; returns the imported plugin module Args: name (str): The name of the plugin's module to be imported additional_paths (:obj:`list`, optional): Additional paths to search for the plugin in Returns: ModuleType: The imported plugin's module Raises: ImportError: If the name cannot be resolved without a package (e.g. a relative import) ModuleNotFoundError: If the name cannot be found in any of the searched directories. """ try: abs_name = importlib.util.resolve_name(name, None) except ImportError: raise ImportError(f'Invalid plugin name {name!r}') try: # if the plugin is already loaded, return it return self.plugins[abs_name] except KeyError: pass try: # if the plugin is already imported, return it return sys.modules[abs_name] except KeyError: pass search_paths = list(set(self.plugin_dirs + additional_paths)) spec = PathFinder.find_spec(abs_name, search_paths) if not spec: raise ModuleNotFoundError(f'No plugin named {abs_name!r}', name=abs_name) module = importlib.util.module_from_spec(spec) sys.modules[abs_name] = module spec.loader.exec_module(module) self.logger.debug("Imported plugin: %s", abs_name) return module
def find_spec(cls, fullname, path=None, target=None): """ From the official documentation: - path is the list of paths where to look for the module, if set to 'None', then search for sys.path. - target is set only in case this is a module reload request, otherwise it will always be 'None'. """ splitname = fullname.split('.') # CHECK IF NAME MATCHES THE PLUGIN PACKAGE PATH if splitname[:2] == ['eddy', 'plugins'] and len(splitname) >= 3: if splitname[2] in PluginManager.info: plugin_spec, plugin_path, plugin_class = PluginManager.info.get(splitname[2]) plugin_path = expandPath(plugin_path) return PathFinder.find_spec(fullname, [plugin_path]) return None
def findModules(module, submoduleName, pattern="*"): """Find the direct submodules from one or more DIRAC extension(s) that match a pattern :param list/str/module module: One or more Python modules or Python module names :param str submoduleName: The submodule under ``module`` in which to look :param str pattern: A ``fnmatch``-style pattern that the submodule must match :returns: list of tuples of the form (SystemName, ServiceName) """ for system in _findSystems(module): agentModule = PathFinder.find_spec( submoduleName, path=system.submodule_search_locations) if not agentModule: continue for _, name, _ in pkgutil.iter_modules( agentModule.submodule_search_locations): if fnmatch.fnmatch(name, pattern): yield system.name, name
def _get_source(self, module_name, package_path): if six.PY2: import imp (file, pathname, desc) = imp.find_module( module_name.split('.')[-1], package_path ) source_code = file.read() file.close() file_path = file.name else: from importlib.machinery import PathFinder loader = PathFinder.find_module(module_name, package_path) source_code = loader.get_source(module_name) file_path = loader.path return source_code, file_path
def find_spec( self, fullname: str, path: Optional[Sequence[Union[bytes, str]]], target: Optional[ModuleType] = None, ): if _managers: module_spec = PathFinder.find_spec(fullname, path, target) if not module_spec: return module_origin = module_spec.origin if not module_origin: return module_path = Path(module_origin).resolve() for manager in reversed(_managers): # use path instead of name in case of submodule name conflict if (fullname in manager.plugins or module_path in manager._searched_plugin_names.values()): module_spec.loader = PluginLoader(manager, fullname, module_origin) return module_spec return
def find_spec(cls, fullname, path, target=None): for load_path in gs.IMPORT_PATHS: if load_path.endswith(os.path.sep): module_dir = load_path[:-1] module_prefix = '' else: module_dir = os.path.dirname(load_path) if not module_dir: module_dir = '.' module_prefix = os.path.basename(load_path) if not module_prefix or fullname == module_prefix or fullname.startswith( module_prefix + '.'): if path is None: search_path = [os.path.abspath(module_dir)] elif isinstance(path, str): search_path = [ os.path.abspath(os.path.join(module_dir, path)) ] else: search_path = path spec = PathFinder.find_spec(fullname, search_path, target) if spec: return spec
def getPath(self, fullname): """ Return the proper path to the module specified in fullname """ actualName = self.getActualModuleName(fullname) return PathFinder.find_spec(actualName).origin
class CometModuleFinder(object): def __init__(self): self.patcher_functions = {} if sys.version_info[0] >= 3: from importlib.machinery import PathFinder self.pathfinder = PathFinder() def register(self, module_name, object_name, patcher_function): module_patchers = self.patcher_functions.setdefault(module_name, {}) module_patchers[object_name] = patcher_function def start(self): if self not in sys.meta_path and not in_notebook_environment(): sys.meta_path.insert(0, self) def find_module(self, fullname, path=None): """ Python 2 import hook """ if fullname not in self.patcher_functions: return return self def load_module(self, fullname): """ Python 2 import hook """ module = self._get_module(fullname) return self._patch(module, fullname) def find_spec(self, fullname, path=None, target=None): """ Python 3 import hook """ if fullname not in self.patcher_functions: return from importlib.machinery import ModuleSpec, SourceFileLoader spec = self.pathfinder.find_spec(fullname, path, target) loader = SourceFileLoader(fullname, spec.origin) return ModuleSpec(fullname, CustomFileLoader(loader, fullname, self)) def _get_module(self, fullname): splitted_name = fullname.split('.') parent = '.'.join(splitted_name[:-1]) if fullname in sys.modules: return sys.modules[fullname] elif parent in sys.modules: parent = sys.modules[parent] module_path = imp.find_module(splitted_name[-1], parent.__path__) return imp.load_module(fullname, *module_path) else: module_path = imp.find_module(fullname) return imp.load_module(fullname, *module_path) def _patch(self, module, fullname): objects_to_patch = self.patcher_functions.get(fullname, {}) for object_name, patcher_function in objects_to_patch.items(): object_path = object_name.split('.') original = self._get_object(module, object_path) if original is None: # TODO: Send back the error? continue new_object = patcher_function(original) self._set_object(module, object_path, new_object) return module def _get_object(self, module, object_path): current_object = module for part in object_path: try: current_object = getattr(current_object, part) except AttributeError: return None return current_object def _set_object(self, module, object_path, new_object): object_to_patch = self._get_object(module, object_path[:-1]) setattr(object_to_patch, object_path[-1], new_object)
def find_spec(self, name, path, target=None): ms = PathFinder.find_spec(name, path, target) if self.is_xhpy_module(name): ms.loader = XHPyLoader(ms.origin) return ms
def invalidate_caches(cls): PathFinder.invalidate_caches()
class CometModuleFinder(object): def __init__(self): self.patcher_functions = {} if sys.version_info[0] >= 3: from importlib.machinery import PathFinder self.pathfinder = PathFinder() def register_before(self, module_name, object_name, patcher_function, allow_empty_experiment=False): self._register("before", module_name, object_name, patcher_function, allow_empty_experiment) def register_after(self, module_name, object_name, patcher_function, allow_empty_experiment=False): self._register("after", module_name, object_name, patcher_function, allow_empty_experiment) def _register( self, lifecycle, module_name, object_name, patcher_function, allow_empty_experiment, ): module_patchers = self.patcher_functions.setdefault(module_name, {}) object_patchers = module_patchers.setdefault( object_name, { "before": [], "after": [], "allow_empty_experiment": allow_empty_experiment, }, ) object_patchers[lifecycle].append(patcher_function) def start(self): if self not in sys.meta_path: sys.meta_path.insert(0, self) def find_module(self, fullname, path=None): """ Python 2 import hook """ if fullname not in self.patcher_functions: return return self def load_module(self, fullname): """ Python 2 import hook """ module = self._get_module(fullname) return self._patch(module, fullname) def find_spec(self, fullname, path=None, target=None): """ Python 3 import hook """ if fullname not in self.patcher_functions: return spec = self.pathfinder.find_spec(fullname, path, target) if not spec: return new_loader = CustomFileLoader(spec.loader, fullname, self) spec.loader = new_loader return spec def _get_module(self, fullname): splitted_name = fullname.split(".") parent = ".".join(splitted_name[:-1]) if fullname in sys.modules: return sys.modules[fullname] elif parent in sys.modules: parent = sys.modules[parent] module_path = imp.find_module(splitted_name[-1], parent.__path__) return imp.load_module(fullname, *module_path) else: try: module_path = imp.find_module(fullname) return imp.load_module(fullname, *module_path) except ImportError: for p in sys.path: importer = get_pep_302_importer(p) # Ignore invalid paths if importer is None: continue module_path = importer.find_module(fullname) if module_path: return importer.load_module(fullname) def _patch(self, module, fullname): objects_to_patch = self.patcher_functions.get(fullname, {}) for object_name, patcher_callbacks in objects_to_patch.items(): object_path = object_name.split(".") original = self._get_object(module, object_path) if original is None: # TODO: Send back the error? continue new_object = Entrypoint(original, **patcher_callbacks) self._set_object(module, object_path, original, new_object) return module def _get_object(self, module, object_path): current_object = module for part in object_path: try: current_object = getattr(current_object, part) except AttributeError: return None return current_object def _set_object(self, module, object_path, original, new_object): object_to_patch = self._get_object(module, object_path[:-1]) original_self = getattr(original, "__self__", None) # Support classmethod if original_self and inspect.isclass(original_self): new_object = classmethod(new_object) # Support staticmethod elif (six.PY2 and inspect.isclass(object_to_patch) and isinstance(original, types.FunctionType)): new_object = staticmethod(new_object) setattr(object_to_patch, object_path[-1], new_object)
def find_module(self, fullname, path=None): if path is None and fullname in self._path_dct: p = self._path_dct[fullname] loader = PathFinder.find_module(fullname, path=[p]) return loader return None
def find_spec(self, fullname, path, target=None): module_spec = PathFinder.find_spec(fullname, path, target) if module_spec and module_spec.has_location and self._is_in_directory(module_spec.origin): loader = Loader(fullname, module_spec.origin, self._transformer) is_package = os.path.basename(module_spec.origin).lower() == "__init__.py" return spec_from_loader(fullname, loader, origin=module_spec.origin, is_package=is_package)
def find_spec(fullname, path=None, target=None): "Method for Python 3.4+." return PathFinder.find_spec(fullname, path or _get_paths(), target)
def find_module(cls, fullname, path=None): loader = PathFinder.find_module(fullname, path=path) if not loader: return loader = CustomLoader(loader) return loader