def __init__(self):
self.patcher_functions = {}
if sys.version_info[0] >= 3:
from importlib.machinery import PathFinder
self.pathfinder = PathFinder()
def test_path_hook_installed(self):
# PathFinder can only use it with sys.executable on sys.path
with patch('sys.path', sys.path):
sys.path = [p for p in sys.path if p != sys.executable]
PathFinder.invalidate_caches()
self.assertIsNone(PathFinder.find_spec("pwd"))
sys.path.append(sys.executable)
spec = PathFinder.find_spec("pwd")
self.assert_spec(spec,
"pwd",
is_pkg=False,
Loader=sys.__spec__.loader,
origin="built-in")
def get_importable_spec(self, symbol_is_module=False):
"""Returns the specification of an actual importable module.
This is a check based on the limitations that we do not
actually perform an import, and assumes a directory structure
with modules.
:param symbol_is_module: if it's known that the symbol is also
a module, include it in the search for
an importable spec
:type symbol_is_module: bool
"""
modules_paths = sys.path
modules_paths.insert(0, self.get_relative_module_fs_path())
spec = None
for component, previous in self._walk_importable_components(
symbol_is_module):
if previous:
modules_paths = [
os.path.join(mod, previous) for mod in modules_paths[:]
]
spec = PathFinder.find_spec(component, modules_paths)
if spec is None:
break
return spec
def _find_import_match(parent_path, parent_module):
"""Attempts to find an importable module."""
modules_paths = [parent_path] + sys.path
found_spec = PathFinder.find_spec(parent_module, modules_paths)
if found_spec is None:
raise ClassNotSuitable
return found_spec
def name_path(mod, pkg):
loader = None
parent = None
path = sys.path
if pkg:
pkg_path = name_path(pkg, None)
if pkg_path:
path = [dirname(pkg_path), ] + path
parts = mod.split('.')
prefix = ''
for part in parts:
if not part:
prefix += '.'
continue
name = prefix + part
loader = PathFinder.find_module(name, path)
prefix = ''
if loader:
parent = dirname(loader.path)
if loader.is_package(name):
path = parent,
else:
path = tuple()
if not loader:
return None
return loader.path
def find_spec(cls, fullname, path=None, target=None):
parts = fullname.split('.')
if len(parts) > 2 and parts[:2] == ['eddy', 'plugins']:
from importlib.machinery import PathFinder
return PathFinder.find_spec(
fullname, [os.path.join(os.path.dirname(__file__), os.pardir)],
target)
def find_spec(
self,
fullname: str,
path: Optional[Sequence[Union[bytes, str]]],
target: Optional[ModuleType] = None,
):
if _managers:
index = -1
module_spec = PathFinder.find_spec(fullname, path, target)
if not module_spec:
return
module_origin = module_spec.origin
if not module_origin:
return
module_path = Path(module_origin).resolve()
while -index <= len(_managers):
manager = _managers[index]
if (
fullname in manager.plugins
or module_path in manager.searched_plugins.values()
):
module_spec.loader = PluginLoader(manager, fullname, module_origin)
return module_spec
index -= 1
return
def find_spec(cls, fullname, path, target=None):
if any(fullname.startswith(rdir) for rdir in (RECIPE_DIR, CONFIG_DIR)):
if path is None:
path = [os.path.abspath('.')]
elif isinstance(path, str):
path = [os.path.abspath(os.path.join('.', path))]
spec = PathFinder.find_spec(fullname, path, target)
return spec
def find_spec(cls, fullname, path, target=None):
for rprefix, rdir in ((gs.RECIPE_PREFIX, gs.RECIPE_PATH),
(gs.CONFIG_PREFIX, gs.CONFIG_PATH)):
if fullname.startswith(rprefix):
if path is None:
path = [os.path.abspath(rdir)]
elif isinstance(path, str):
path = [os.path.abspath(os.path.join(rdir, path))]
spec = PathFinder.find_spec(fullname, path, target)
return spec
def find_spec(cls, fullname, path=None, target=None):
spec = PathFinder.find_spec(fullname, path=path, target=target)
if spec is None:
return
if spec.loader is not None:
if hasattr(spec.loader, 'exec_module'):
spec.loader = CustomLoader(spec.loader)
else:
spec.loader = LegacyCustomLoader(spec.loader)
return spec
def load(directory):
curdir = os.path.join(directory, 'scripts', 'pbs', 'commands')
commands = {}
for dir, subdirs, files in os.walk(curdir):
for filename in files:
name, ext = os.path.splitext(filename)
if ext == '.py' and name != '__init__':
loader = PathFinder.find_module(name, [curdir])
commands[name] = loader.load_module()
return commands
def find_spec(
self,
fullname: str,
path: Optional[Sequence[Union[bytes, str]]] = None,
target: Optional[types.ModuleType] = None,
) -> Optional[ModuleSpec]:
if fullname != "pip":
return None
spec = PathFinder.find_spec(fullname, [PIP_SOURCES_ROOT], target)
assert spec, (PIP_SOURCES_ROOT, fullname)
return spec
def find_spec(cls, fullname: str, path=None, target=None):
"""Try to find a spec for 'fullname' on sys.path or 'path'.
Search only for modules starting with 'labs.ext.' and use PathFinder to first search for
submodule in the current build, then module starting with labs_.
"""
if not fullname.startswith(_prefix):
return None # Not a labs extension
_name = fullname[_offset:]
if '.' in _name:
return None # Extension already imported, this is a submodule.
# Try importing non packaged version
paths_1, paths_2 = _get_ctx_path()
if paths_1:
spec = PathFinder.find_spec(fullname, paths_1, target)
if spec is not None:
return _patch_spec(fullname,
spec) # Correct the module's name.
# Try importing a packaged version
installed_name = f'labs_{_name}'
if paths_2:
spec = PathFinder.find_spec(installed_name, paths_2, target)
if spec is not None:
return _patch_spec(fullname,
spec) # Correct the module's name.
# Try importing an installed version
spec = PathFinder.find_spec(installed_name, None, target)
if spec is not None:
return _patch_spec(fullname, spec) # Correct the module's name.
return None
def _get_module(app, modname):
# Find out the app's __path__
try:
app_path = import_module(app).__path__
except AttributeError: # pragma: no cover
return
# Use importlib's PathFinder().find_spec() to find the app's modname.py
if PathFinder().find_spec(modname, app_path) is None:
return
# Import the app's module file
import_module('{}.{}'.format(app, modname))
def find_spec(self, fullname: str, path, target):
if _manager_stack:
index = -1
origin_spec = PathFinder.find_spec(fullname, path, target)
while -index <= len(_manager_stack):
manager = _manager_stack[index]
rel_name = None
if origin_spec and origin_spec.origin:
rel_name = manager._check_absolute_import(
origin_spec.origin)
newname = manager._rewrite_module_name(rel_name or fullname)
if newname:
spec = PathFinder.find_spec(
newname, path or [*manager.search_path, *sys.path],
target)
if spec:
spec.loader = PluginLoader( # type: ignore
manager, newname, spec.origin)
return spec
index -= 1
return None
def get_alternative(mod, pkg = None):
if mod.startswith('.') and pkg:
loader = PathFinder.find_module(pkg, path)
if loader:
modpath = ast.name_path(mod, pkg)
mod = pkg + mod
fname = mod.replace('.', '/') + '.js' # FIXME
for p in path:
if os.path.isdir(p):
candidate = os.path.join(p, fname)
if os.path.isfile(candidate):
return candidate, mod
return None, None
def get_package_directory(module_name):
"""Return Path of the package directory that the given module is in."""
base_module = module_name.split('.', 1)[0]
# Find the import path of the package, excluding the `''` entry in `sys.path` so as
# to avoid the false positive of returning the CWD if the user is in the development
# directory (which may also be in the path for the case of an editable install,
# which is why we skip `''` specifically, and not anything that expands to CWD).
spec = PathFinder.find_spec(base_module, path=[p for p in sys.path if p])
if spec is None or spec.origin is None:
raise ModuleNotFoundError(base_module)
if not spec.parent:
msg = f'{base_module} is not a package'
raise ValueError(msg)
return Path(spec.origin).parent
def find_spec(self, fullname: str, path, target):
if _manager_stack:
index = -1
while -index <= len(_manager_stack):
manager = _manager_stack[index]
newname = manager._rewrite_module_name(fullname)
if newname:
spec = PathFinder.find_spec(newname,
list(manager.search_path),
target)
if spec:
return spec
index -= 1
return None
def _load_module(module_name, filename, *paths):
name, ext = os.path.splitext(filename)
if ext in ('.py', ''):
loader = PathFinder.find_module(name, path=paths)
if loader is None:
loader = PathFinder.find_module(name)
if loader is None:
raise ImportError(
'Could not find module {} on path {} or sys.path'.format(
name, paths))
else:
loader = None
for path in paths:
p = os.path.normpath(os.path.join(path, filename))
if os.path.exists(p):
loader = SourceFileLoader(module_name, p)
if loader is None:
raise ImportError(
'Could not find module {} on path {}'.format(name, paths))
loader.name = module_name
return loader.load_module()
def find_spec(self, fullname, path, target=None):
spec = None
if fullname in self._soft_patches or fullname in self._hard_patches:
spec = PathFinder.find_spec(fullname, path, target=target)
if spec is None:
raise Exception("Could not patch %s" % fullname)
if fullname in self._soft_patches:
self._wrap_soft_patches(spec.loader,
self._soft_patches[fullname])
elif fullname in self._hard_patches:
self._wrap_hard_patch(spec.loader,
self._hard_patches[fullname])
return spec
def find_spec(self, fullname, path=None, target=None):
if world.rank == 0:
spec = PathFinder.find_spec(fullname, path, target)
if spec is None:
return None
code = spec.loader.get_code(fullname)
if code is None: # C extensions
return None
loader = BroadcastLoader(spec, self.module_cache)
assert fullname == spec.name
searchloc = spec.submodule_search_locations
spec = ModuleSpec(fullname,
loader,
origin=spec.origin,
is_package=searchloc is not None)
if searchloc is not None:
spec.submodule_search_locations += searchloc
return spec
else:
if fullname not in self.module_cache:
# Could this in principle interfere with builtin imports?
return PathFinder.find_spec(fullname, path, target)
searchloc, origin = self.module_cache[fullname][0]
loader = BroadcastLoader(None, self.module_cache)
spec = ModuleSpec(fullname,
loader,
origin=origin,
is_package=searchloc is not None)
if searchloc is not None:
spec.submodule_search_locations += searchloc
loader.spec = spec # XXX loader.loader is still None
return spec
def _load_module(module_name, filename, *paths):
name, ext = os.path.splitext(filename)
if ext in ('.py', ''):
loader = PathFinder.find_module(name, path=paths)
if loader is None:
loader = PathFinder.find_module(name)
if loader is None:
raise ImportError(
'Could not find module {} on path {} or sys.path'.format(
name, paths))
else:
loader = None
for path in paths:
p = os.path.normpath(os.path.join(path, filename))
if os.path.exists(p):
loader = SourceFileLoader(module_name, p)
if loader is None:
raise ImportError(
'Could not find module {} on path {}'.format(name, paths))
loader.name = module_name
return loader.load_module()
def find_spec(self, fullname: str, path, target):
if _manager_stack:
index = -1
while -index <= len(_manager_stack):
manager = _manager_stack[index]
newname = manager._rewrite_module_name(fullname)
if newname:
spec = PathFinder.find_spec(
newname, [*manager.search_path, *(path or sys.path)],
target)
if spec:
spec.loader = PluginLoader(manager, newname,
spec.origin)
return spec
index -= 1
return None
def import_plugin(self, name, additional_paths=[]):
""" Mimics Python's import mechanism to import plugins found in the
specified plugin directories; returns the imported plugin module
Args:
name (str): The name of the plugin's module to be imported
additional_paths (:obj:`list`, optional): Additional paths to
search for the plugin in
Returns:
ModuleType: The imported plugin's module
Raises:
ImportError: If the name cannot be resolved without a package
(e.g. a relative import)
ModuleNotFoundError: If the name cannot be found in any of the
searched directories.
"""
try:
abs_name = importlib.util.resolve_name(name, None)
except ImportError:
raise ImportError(f'Invalid plugin name {name!r}')
try:
# if the plugin is already loaded, return it
return self.plugins[abs_name]
except KeyError:
pass
try:
# if the plugin is already imported, return it
return sys.modules[abs_name]
except KeyError:
pass
search_paths = list(set(self.plugin_dirs + additional_paths))
spec = PathFinder.find_spec(abs_name, search_paths)
if not spec:
raise ModuleNotFoundError(f'No plugin named {abs_name!r}',
name=abs_name)
module = importlib.util.module_from_spec(spec)
sys.modules[abs_name] = module
spec.loader.exec_module(module)
self.logger.debug("Imported plugin: %s", abs_name)
return module
def find_spec(cls, fullname, path=None, target=None):
"""
From the official documentation:
- path is the list of paths where to look for the module,
if set to 'None', then search for sys.path.
- target is set only in case this is a module reload request,
otherwise it will always be 'None'.
"""
splitname = fullname.split('.')
# CHECK IF NAME MATCHES THE PLUGIN PACKAGE PATH
if splitname[:2] == ['eddy', 'plugins'] and len(splitname) >= 3:
if splitname[2] in PluginManager.info:
plugin_spec, plugin_path, plugin_class = PluginManager.info.get(splitname[2])
plugin_path = expandPath(plugin_path)
return PathFinder.find_spec(fullname, [plugin_path])
return None
def findModules(module, submoduleName, pattern="*"):
"""Find the direct submodules from one or more DIRAC extension(s) that match a pattern
:param list/str/module module: One or more Python modules or Python module names
:param str submoduleName: The submodule under ``module`` in which to look
:param str pattern: A ``fnmatch``-style pattern that the submodule must match
:returns: list of tuples of the form (SystemName, ServiceName)
"""
for system in _findSystems(module):
agentModule = PathFinder.find_spec(
submoduleName, path=system.submodule_search_locations)
if not agentModule:
continue
for _, name, _ in pkgutil.iter_modules(
agentModule.submodule_search_locations):
if fnmatch.fnmatch(name, pattern):
yield system.name, name
def _get_source(self, module_name, package_path):
if six.PY2:
import imp
(file, pathname, desc) = imp.find_module(
module_name.split('.')[-1],
package_path
)
source_code = file.read()
file.close()
file_path = file.name
else:
from importlib.machinery import PathFinder
loader = PathFinder.find_module(module_name, package_path)
source_code = loader.get_source(module_name)
file_path = loader.path
return source_code, file_path
def find_spec(
self,
fullname: str,
path: Optional[Sequence[Union[bytes, str]]],
target: Optional[ModuleType] = None,
):
if _managers:
module_spec = PathFinder.find_spec(fullname, path, target)
if not module_spec:
return
module_origin = module_spec.origin
if not module_origin:
return
module_path = Path(module_origin).resolve()
for manager in reversed(_managers):
# use path instead of name in case of submodule name conflict
if (fullname in manager.plugins or module_path
in manager._searched_plugin_names.values()):
module_spec.loader = PluginLoader(manager, fullname,
module_origin)
return module_spec
return
def find_spec(cls, fullname, path, target=None):
for load_path in gs.IMPORT_PATHS:
if load_path.endswith(os.path.sep):
module_dir = load_path[:-1]
module_prefix = ''
else:
module_dir = os.path.dirname(load_path)
if not module_dir:
module_dir = '.'
module_prefix = os.path.basename(load_path)
if not module_prefix or fullname == module_prefix or fullname.startswith(
module_prefix + '.'):
if path is None:
search_path = [os.path.abspath(module_dir)]
elif isinstance(path, str):
search_path = [
os.path.abspath(os.path.join(module_dir, path))
]
else:
search_path = path
spec = PathFinder.find_spec(fullname, search_path, target)
if spec:
return spec
def getPath(self, fullname):
""" Return the proper path to the module specified in fullname """
actualName = self.getActualModuleName(fullname)
return PathFinder.find_spec(actualName).origin
class CometModuleFinder(object):
def __init__(self):
self.patcher_functions = {}
if sys.version_info[0] >= 3:
from importlib.machinery import PathFinder
self.pathfinder = PathFinder()
def register(self, module_name, object_name, patcher_function):
module_patchers = self.patcher_functions.setdefault(module_name, {})
module_patchers[object_name] = patcher_function
def start(self):
if self not in sys.meta_path and not in_notebook_environment():
sys.meta_path.insert(0, self)
def find_module(self, fullname, path=None):
""" Python 2 import hook
"""
if fullname not in self.patcher_functions:
return
return self
def load_module(self, fullname):
""" Python 2 import hook
"""
module = self._get_module(fullname)
return self._patch(module, fullname)
def find_spec(self, fullname, path=None, target=None):
""" Python 3 import hook
"""
if fullname not in self.patcher_functions:
return
from importlib.machinery import ModuleSpec, SourceFileLoader
spec = self.pathfinder.find_spec(fullname, path, target)
loader = SourceFileLoader(fullname, spec.origin)
return ModuleSpec(fullname, CustomFileLoader(loader, fullname, self))
def _get_module(self, fullname):
splitted_name = fullname.split('.')
parent = '.'.join(splitted_name[:-1])
if fullname in sys.modules:
return sys.modules[fullname]
elif parent in sys.modules:
parent = sys.modules[parent]
module_path = imp.find_module(splitted_name[-1], parent.__path__)
return imp.load_module(fullname, *module_path)
else:
module_path = imp.find_module(fullname)
return imp.load_module(fullname, *module_path)
def _patch(self, module, fullname):
objects_to_patch = self.patcher_functions.get(fullname, {})
for object_name, patcher_function in objects_to_patch.items():
object_path = object_name.split('.')
original = self._get_object(module, object_path)
if original is None:
# TODO: Send back the error?
continue
new_object = patcher_function(original)
self._set_object(module, object_path, new_object)
return module
def _get_object(self, module, object_path):
current_object = module
for part in object_path:
try:
current_object = getattr(current_object, part)
except AttributeError:
return None
return current_object
def _set_object(self, module, object_path, new_object):
object_to_patch = self._get_object(module, object_path[:-1])
setattr(object_to_patch, object_path[-1], new_object)
def find_spec(self, name, path, target=None):
ms = PathFinder.find_spec(name, path, target)
if self.is_xhpy_module(name):
ms.loader = XHPyLoader(ms.origin)
return ms
def invalidate_caches(cls):
PathFinder.invalidate_caches()
class CometModuleFinder(object):
def __init__(self):
self.patcher_functions = {}
if sys.version_info[0] >= 3:
from importlib.machinery import PathFinder
self.pathfinder = PathFinder()
def register_before(self,
module_name,
object_name,
patcher_function,
allow_empty_experiment=False):
self._register("before", module_name, object_name, patcher_function,
allow_empty_experiment)
def register_after(self,
module_name,
object_name,
patcher_function,
allow_empty_experiment=False):
self._register("after", module_name, object_name, patcher_function,
allow_empty_experiment)
def _register(
self,
lifecycle,
module_name,
object_name,
patcher_function,
allow_empty_experiment,
):
module_patchers = self.patcher_functions.setdefault(module_name, {})
object_patchers = module_patchers.setdefault(
object_name,
{
"before": [],
"after": [],
"allow_empty_experiment": allow_empty_experiment,
},
)
object_patchers[lifecycle].append(patcher_function)
def start(self):
if self not in sys.meta_path:
sys.meta_path.insert(0, self)
def find_module(self, fullname, path=None):
""" Python 2 import hook
"""
if fullname not in self.patcher_functions:
return
return self
def load_module(self, fullname):
""" Python 2 import hook
"""
module = self._get_module(fullname)
return self._patch(module, fullname)
def find_spec(self, fullname, path=None, target=None):
""" Python 3 import hook
"""
if fullname not in self.patcher_functions:
return
spec = self.pathfinder.find_spec(fullname, path, target)
if not spec:
return
new_loader = CustomFileLoader(spec.loader, fullname, self)
spec.loader = new_loader
return spec
def _get_module(self, fullname):
splitted_name = fullname.split(".")
parent = ".".join(splitted_name[:-1])
if fullname in sys.modules:
return sys.modules[fullname]
elif parent in sys.modules:
parent = sys.modules[parent]
module_path = imp.find_module(splitted_name[-1], parent.__path__)
return imp.load_module(fullname, *module_path)
else:
try:
module_path = imp.find_module(fullname)
return imp.load_module(fullname, *module_path)
except ImportError:
for p in sys.path:
importer = get_pep_302_importer(p)
# Ignore invalid paths
if importer is None:
continue
module_path = importer.find_module(fullname)
if module_path:
return importer.load_module(fullname)
def _patch(self, module, fullname):
objects_to_patch = self.patcher_functions.get(fullname, {})
for object_name, patcher_callbacks in objects_to_patch.items():
object_path = object_name.split(".")
original = self._get_object(module, object_path)
if original is None:
# TODO: Send back the error?
continue
new_object = Entrypoint(original, **patcher_callbacks)
self._set_object(module, object_path, original, new_object)
return module
def _get_object(self, module, object_path):
current_object = module
for part in object_path:
try:
current_object = getattr(current_object, part)
except AttributeError:
return None
return current_object
def _set_object(self, module, object_path, original, new_object):
object_to_patch = self._get_object(module, object_path[:-1])
original_self = getattr(original, "__self__", None)
# Support classmethod
if original_self and inspect.isclass(original_self):
new_object = classmethod(new_object)
# Support staticmethod
elif (six.PY2 and inspect.isclass(object_to_patch)
and isinstance(original, types.FunctionType)):
new_object = staticmethod(new_object)
setattr(object_to_patch, object_path[-1], new_object)
def find_module(self, fullname, path=None):
if path is None and fullname in self._path_dct:
p = self._path_dct[fullname]
loader = PathFinder.find_module(fullname, path=[p])
return loader
return None
def find_spec(self, fullname, path, target=None):
module_spec = PathFinder.find_spec(fullname, path, target)
if module_spec and module_spec.has_location and self._is_in_directory(module_spec.origin):
loader = Loader(fullname, module_spec.origin, self._transformer)
is_package = os.path.basename(module_spec.origin).lower() == "__init__.py"
return spec_from_loader(fullname, loader, origin=module_spec.origin, is_package=is_package)
def find_spec(fullname, path=None, target=None):
"Method for Python 3.4+."
return PathFinder.find_spec(fullname, path or _get_paths(), target)
def find_spec(fullname, path=None, target=None):
"Method for Python 3.4+."
return PathFinder.find_spec(fullname, path or _get_paths(), target)
def find_module(cls, fullname, path=None):
loader = PathFinder.find_module(fullname, path=path)
if not loader:
return
loader = CustomLoader(loader)
return loader
