def find_module_path(module_name, search_path=None):
"""
Find the module path (pyd / so), while accounting for platform/arch naming
:param module_name: The name of the module
:param search_path: The path to search in. If None, searches system path.
:return: The full path to the library or None if not found.
"""
# Use importlib if python 3.4+, else imp
if sys.version_info[0] > 3 or (sys.version_info[0] == 3 and sys.version_info[1] >= 4):
from importlib.machinery import FileFinder, ExtensionFileLoader, EXTENSION_SUFFIXES
file_finder = FileFinder(search_path, (ExtensionFileLoader, EXTENSION_SUFFIXES))
# The search caches must be cleared to guaranteed find dynamically created modules
file_finder.invalidate_caches()
result = file_finder.find_spec(module_name)
return None if not result else result.origin
else:
from imp import find_module # Deprecated in 3.4
try:
result = find_module(module_name, [search_path])
except ImportError:
result = None
return None if not result else result[1]
def find_module_path(module_name, search_path=None):
"""
Find the module path (pyd / so), while accounting for platform/arch naming
:param module_name: The name of the module
:param search_path: The path to search in. If None, searches system path.
:return: The full path to the library or None if not found.
"""
# Use importlib if python 3.4+, else imp
if sys.version_info[0] > 3 or (sys.version_info[0] == 3
and sys.version_info[1] >= 4):
from importlib.machinery import FileFinder, ExtensionFileLoader, EXTENSION_SUFFIXES
file_finder = FileFinder(search_path,
(ExtensionFileLoader, EXTENSION_SUFFIXES))
# The search caches must be cleared to guaranteed find dynamically created modules
file_finder.invalidate_caches()
result = file_finder.find_spec(module_name)
return None if not result else result.origin
else:
from imp import find_module # Deprecated in 3.4
try:
result = find_module(module_name, [search_path])
except ImportError:
result = None
return None if not result else result[1]
def load_module(name, path=None):
finder = FileFinder(path, *_loader_details)
spec = finder.find_spec(name)
if not spec or not spec.loader:
raise ImportError(f"no module named {name}")
mod = module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def loadModule (path): folder = os.path.dirname(path) filename = os.path.basename(path) (moduleName, dot, extension) = filename.rpartition(filename) loaderArgs = (SourceFileLoader, [dot+extension]) finder = FileFinder(folder, loaderArgs) spec = finder.find_spec(moduleName) module = importlib.util.module_from_spec(spec) loader = spec.loader module.moduleSpec = spec module.exec = lambda: loader.exec_module(module) return module
def _path_importer_cache(cls, path):
try:
finder = cls.path_importer_cache[path]
except KeyError:
finder = FileFinder(path, (PTLFileLoader, [PTL_EXT]))
cls.path_importer_cache[path] = finder
return finder
def install():
loader_details = HookedLoader, [".py"]
# insert the path hook ahead of other path hooks
sys.path_hooks.insert(0, FileFinder.path_hook(loader_details))
# clear any loaders that might already be in use by the FileFinder
sys.path_importer_cache.clear()
importlib.invalidate_caches()
def import_module(name, fullname=None):
if fullname is None:
fullname = 'trytond.modules.' + name
try:
module = importlib.import_module(fullname)
except ImportError:
if name not in EGG_MODULES:
raise
ep = EGG_MODULES[name]
# Can not use ep.load because modules are declared in an importable
# path and it can not import submodule.
path = os.path.join(ep.dist.location, *ep.module_name.split('.')[:-1])
if not os.path.isdir(path):
# Find module in path
for path in sys.path:
path = os.path.join(path, *ep.module_name.split('.')[:-1])
if os.path.isdir(os.path.join(path, name)):
break
else:
# When testing modules from setuptools location is the
# module directory
path = os.path.dirname(ep.dist.location)
spec = FileFinder(
path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(fullname)
if spec.loader:
module = spec.loader.load_module()
else:
raise
return module
def available_players(
player_paths: T.Optional[T.Sequence[str]] = None
) -> T.Sequence[ModuleInfo]:
import othello
module_paths = set(player_paths) if player_paths else set()
othello_path: str = othello.__path__ # type: ignore # mypy issue #1422
package_paths = {
*(path.join(p, "models", "players") for p in othello_path),
*(dir_path for dir_path in module_paths if path.isdir(dir_path)),
}
module_paths -= package_paths
return (
*walk_packages(package_paths),
*(
ModuleInfo(
FileFinder(
path.dirname(module_path),
(SourceFileLoader,
(".py", )), # type: ignore # typeshed is incorrect
),
path.splitext(path.basename(module_path))[0],
False,
) for module_path in module_paths
if (path.isfile(module_path) and module_path.endswith(".py")
and not path.basename(module_path).startswith("_"))),
)
def install() -> None:
"""Installs a loader which is capable of loading and validating strict modules"""
supported_loaders = _get_supported_file_loaders()
for index, hook in enumerate(sys.path_hooks):
if not isinstance(hook, type):
# pyre-fixme[6]: For 1st param expected `Tuple[Type[Loader], List[str]]`
# but got `Tuple[Loader, List[str]]`.
sys.path_hooks.insert(index, FileFinder.path_hook(*supported_loaders))
break
else:
# pyre-fixme[6]: For 1st param expected `Tuple[Type[Loader], List[str]]` but
# got `Tuple[Loader, List[str]]`.
sys.path_hooks.insert(0, FileFinder.path_hook(*supported_loaders))
# We need to clear the path_importer_cache so that our new FileFinder will
# start being used for existing directories we've loaded modules from.
sys.path_importer_cache.clear()
def install(haxe_bin='haxe'):
HaxeLoader.haxe_bin = haxe_bin
loader_details = HaxeLoader, [".hx"]
# insert the path hook ahead of other path hooks
sys.path_hooks.insert(0, FileFinder.path_hook(loader_details))
# clear any loaders that might already be in use by the FileFinder
sys.path_importer_cache.clear()
invalidate_caches()
def _install_source_loader_helper(source_loader_type):
extensions = ExtensionFileLoader, _imp.extension_suffixes()
source = source_loader_type, SOURCE_SUFFIXES
bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
supported_loaders = [extensions, source, bytecode]
sys.path_hooks[:] = [
zipimport.zipimporter,
FileFinder.path_hook(*supported_loaders),
]
sys.path_importer_cache.clear()
def lib2to3importer(fixers, prefix=None):
"""Returns finder closure to be prepended to sys.path_hooks.
:param fixers: e.g. ["lib2to3.fixes.fix_exec", "lib2to3.fixes.fix_long"]
:param prefix: Prefix of module or package name to apply fixes.
Unless provided, fixes are applied to all of modules and packages.
"""
extensions = ExtensionFileLoader, EXTENSION_SUFFIXES
source = Lib2to3FileLoader.apply(fixers, prefix), SOURCE_SUFFIXES
bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
return FileFinder.path_hook(extensions, source, bytecode)
def file_loader(loader: object) -> Generator[None, None, None]:
orig_hooks = sys.path_hooks[:]
sys.path_hooks[:] = [
FileFinder.path_hook(
loader) # pyre-ignore[6]: Loader type isn't expose
]
sys.path_importer_cache.clear()
try:
yield
finally:
sys.path_hooks[:] = orig_hooks
sys.path_importer_cache.clear()
def install_hooks():
_ctx.update({"start_time": time.time(), "depth": 0})
ExtensionFileLoader.exec_module = profile(ExtensionFileLoader.exec_module)
SourceFileLoader.exec_module = profile(SourceFileLoader.exec_module)
SourcelessFileLoader.exec_module = profile(
SourcelessFileLoader.exec_module)
sys.path_hooks.insert(
0,
FileFinder.path_hook(
(ExtensionFileLoader, EXTENSION_SUFFIXES),
(SourceFileLoader, SOURCE_SUFFIXES),
(SourcelessFileLoader, BYTECODE_SUFFIXES),
),
)
def install_loader():
"""
Install the import hook
Once this has been run, the enhancements will be available in any imported code
This will run automatically when the package is imported; to disable::
builtins.__dict__["__perl__disable_automatic_import"] = True
"""
# Set up import hook
loader_details = PerlLoader, [".py"]
sys.path_hooks.insert(0, FileFinder.path_hook(loader_details))
sys.path_importer_cache.clear()
invalidate_caches()
# Inject dependencies for rewritten code
builtins.__dict__["re"] = re
builtins.__dict__["__perl__re_match"] = re_match
builtins.__dict__["__perl__reset_vars"] = reset_vars
def hotfix(module_names):
global OLD_MODULES
global BUILTINS_OBJ
OLD_MODULES.clear()
for name in module_names:
if name in sys.modules:
OLD_MODULES[name] = sys.modules[name]
gc.disable()
OLD_MODULES = sys.modules.copy()
BUILTINS_OBJ = object
sys.modules["builtins"].object = HotfixerObject
sys.path_hooks.insert(0, FileFinder.path_hook((MetaPathLoader, [".py"])))
# clear any loaders that might already be in use by the FileFinder
sys.path_importer_cache.clear()
importlib.invalidate_caches()
for name in module_names:
try:
old_module = sys.modules.pop(name, None)
new_module = importlib.import_module(name)
if old_module and not is_skip_hotfix(new_module):
for attr_name, new_attr in inspect.getmembers(new_module):
setattr(old_module, attr_name, new_attr)
except Exception as ex:
traceback.print_exc()
finally:
if name in OLD_MODULES:
sys.modules[name] = OLD_MODULES[name]
sys.path_hooks.pop(0)
sys.modules["builtins"].object = BUILTINS_OBJ
BUILTINS_OBJ = None
for obj in NEW_OBJECTS:
hotfix_obj(obj)
gc.enable()
gc.collect()
def available_players(player_paths: T.Optional[T.Sequence[str]] = None) -> T.Sequence[ModuleInfo]:
import othello
module_path = othello.__path__ # type: ignore # mypy issue #1422
player_paths = player_paths or tuple()
dir_player_paths = (
*(path.join(p, "models", "players") for p in module_path),
*(dir_path for dir_path in player_paths if path.isdir(dir_path)),
)
return (
*walk_packages(dir_player_paths, onerror=lambda _: None),
*(
ModuleInfo(
FileFinder(path.dirname(file_path), (SourceFileLoader, (".py", ".pyc"))),
path.splitext(path.basename(file_path))[0],
False,
)
for file_path in player_paths
if path.isfile(file_path)
),
)
def update_hooks(*loaders):
"""Append custom loaders to the `sys.path_hooks`, they must
have a tuple attribute EXTENSION_SUFFIXES to discover the correct path.
_NATIVE_HOOK resides in the global scope to reset the original sys.path_hooks
if necessary..
"""
global _NATIVE_HOOK
from importlib.machinery import FileFinder
if loaders:
for i, hook in enumerate(sys.path_hooks):
__closure__ = getattr(hook, '__closure__', None)
if __closure__ and issubclass(__closure__[0].cell_contents, FileFinder):
_NATIVE_HOOK = globals().get('_NATIVE_HOOK', (i, hook))
sys.path_hooks[i] = FileFinder.path_hook(
*_NATIVE_HOOK[1].__closure__[1].cell_contents,
*((loader, loader.EXTENSION_SUFFIXES) for loader in loaders
))
else:
sys.path_hooks[_NATIVE_HOOK[0]] = _NATIVE_HOOK[1]
"""https://docs.python.org/3/library/sys.html#sys.path_importer_cache"""
sys.path_importer_cache.clear()
def find_and_load_module(modulename, folder=""):
finder = FileFinder(folder, loader_details)
spec = finder.find_spec(modulename)
module = module_from_spec(spec)
spec.loader.exec_module(module)
return module
def find_and_load_module(modulename, folder=""):
finder = FileFinder(folder, loader_details)
spec = finder.find_spec(modulename)
return spec.loader.load_module()
def get_module_path(modulename, folder=""):
finder = FileFinder(folder, loader_details)
return finder.find_spec(modulename).origin
def install(loader_details):
# Insert the path hook ahead of other path hooks.
sys.path_hooks.insert(0, FileFinder.path_hook(loader_details))
# Clear any loaders that might already be in use by the FileFinder.
sys.path_importer_cache.clear()
invalidate_caches()
cmake_args.append(f"-DCUDA_TOOLKIT_ROOT_DIR:STRING={cuda_root}")
elif dp_variant == "rocm":
cmake_args.append("-DUSE_ROCM_TOOLKIT:BOOL=TRUE")
rocm_root = os.environ.get("ROCM_ROOT")
if rocm_root:
cmake_args.append(f"-DROCM_ROOT:STRING={rocm_root}")
else:
raise RuntimeError("Unsupported DP_VARIANT option: %s" % dp_variant)
# get tensorflow spec
tf_spec = find_spec("tensorflow")
if not tf_spec:
# purelib gets site-packages path
site_packages = get_path("purelib")
if site_packages:
tf_spec = FileFinder(site_packages).find_spec("tensorflow")
# get install dir from spec
try:
tf_install_dir = tf_spec.submodule_search_locations[0] # type: ignore
# AttributeError if ft_spec is None
# TypeError if submodule_search_locations are None
# IndexError if submodule_search_locations is an empty list
except (AttributeError, TypeError, IndexError):
setup_requires.extend(extras_require['cpu'])
# setuptools will re-find tensorflow after installing setup_requires
tf_install_dir = None
# add cmake as a build requirement if cmake>3.7 is not installed
try:
cmake_version = get_cmake_version()
def find_and_load_module(modulename, folder=""): finder = FileFinder(folder, loader_details) spec = finder.find_spec(modulename) return spec.loader.load_module()
def _getFinder(self, path):
try:
return self._finderCache[path]
except KeyError:
self._finderCache[path] = FileFinder(path, (EPSLoader, ['.eps']))
return self._finderCache[path]
def init_enforce() -> None:
loader_details = (Loader, SOURCE_SUFFIXES)
sys.path_hooks.insert(0,
FileFinder.path_hook(loader_details)) # type: ignore
def installImportOverride():
# insert the path hook ahead of other path hooks
sys.path_hooks.insert(0, FileFinder.path_hook(det))
# clear any loaders that might already be in use by the FileFinder
sys.path_importer_cache.clear()
invalidate_caches()
def get_module_path(modulename, folder=""): finder = FileFinder(folder, loader_details) return finder.find_spec(modulename).origin
def find_and_load_module(modulename, folder=""): finder = FileFinder(folder, loader_details) spec = finder.find_spec(modulename) module = module_from_spec(spec) spec.loader.exec_module(module) return module
