def find_module_path(module_name, search_path=None): """ Find the module path (pyd / so), while accounting for platform/arch naming :param module_name: The name of the module :param search_path: The path to search in. If None, searches system path. :return: The full path to the library or None if not found. """ # Use importlib if python 3.4+, else imp if sys.version_info[0] > 3 or (sys.version_info[0] == 3 and sys.version_info[1] >= 4): from importlib.machinery import FileFinder, ExtensionFileLoader, EXTENSION_SUFFIXES file_finder = FileFinder(search_path, (ExtensionFileLoader, EXTENSION_SUFFIXES)) # The search caches must be cleared to guaranteed find dynamically created modules file_finder.invalidate_caches() result = file_finder.find_spec(module_name) return None if not result else result.origin else: from imp import find_module # Deprecated in 3.4 try: result = find_module(module_name, [search_path]) except ImportError: result = None return None if not result else result[1]
def load_module(name, path=None): finder = FileFinder(path, *_loader_details) spec = finder.find_spec(name) if not spec or not spec.loader: raise ImportError(f"no module named {name}") mod = module_from_spec(spec) spec.loader.exec_module(mod) return mod
def loadModule (path): folder = os.path.dirname(path) filename = os.path.basename(path) (moduleName, dot, extension) = filename.rpartition(filename) loaderArgs = (SourceFileLoader, [dot+extension]) finder = FileFinder(folder, loaderArgs) spec = finder.find_spec(moduleName) module = importlib.util.module_from_spec(spec) loader = spec.loader module.moduleSpec = spec module.exec = lambda: loader.exec_module(module) return module
def _path_importer_cache(cls, path): try: finder = cls.path_importer_cache[path] except KeyError: finder = FileFinder(path, (PTLFileLoader, [PTL_EXT])) cls.path_importer_cache[path] = finder return finder
def install(): loader_details = HookedLoader, [".py"] # insert the path hook ahead of other path hooks sys.path_hooks.insert(0, FileFinder.path_hook(loader_details)) # clear any loaders that might already be in use by the FileFinder sys.path_importer_cache.clear() importlib.invalidate_caches()
def import_module(name, fullname=None): if fullname is None: fullname = 'trytond.modules.' + name try: module = importlib.import_module(fullname) except ImportError: if name not in EGG_MODULES: raise ep = EGG_MODULES[name] # Can not use ep.load because modules are declared in an importable # path and it can not import submodule. path = os.path.join(ep.dist.location, *ep.module_name.split('.')[:-1]) if not os.path.isdir(path): # Find module in path for path in sys.path: path = os.path.join(path, *ep.module_name.split('.')[:-1]) if os.path.isdir(os.path.join(path, name)): break else: # When testing modules from setuptools location is the # module directory path = os.path.dirname(ep.dist.location) spec = FileFinder( path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(fullname) if spec.loader: module = spec.loader.load_module() else: raise return module
def available_players( player_paths: T.Optional[T.Sequence[str]] = None ) -> T.Sequence[ModuleInfo]: import othello module_paths = set(player_paths) if player_paths else set() othello_path: str = othello.__path__ # type: ignore # mypy issue #1422 package_paths = { *(path.join(p, "models", "players") for p in othello_path), *(dir_path for dir_path in module_paths if path.isdir(dir_path)), } module_paths -= package_paths return ( *walk_packages(package_paths), *( ModuleInfo( FileFinder( path.dirname(module_path), (SourceFileLoader, (".py", )), # type: ignore # typeshed is incorrect ), path.splitext(path.basename(module_path))[0], False, ) for module_path in module_paths if (path.isfile(module_path) and module_path.endswith(".py") and not path.basename(module_path).startswith("_"))), )
def install() -> None: """Installs a loader which is capable of loading and validating strict modules""" supported_loaders = _get_supported_file_loaders() for index, hook in enumerate(sys.path_hooks): if not isinstance(hook, type): # pyre-fixme[6]: For 1st param expected `Tuple[Type[Loader], List[str]]` # but got `Tuple[Loader, List[str]]`. sys.path_hooks.insert(index, FileFinder.path_hook(*supported_loaders)) break else: # pyre-fixme[6]: For 1st param expected `Tuple[Type[Loader], List[str]]` but # got `Tuple[Loader, List[str]]`. sys.path_hooks.insert(0, FileFinder.path_hook(*supported_loaders)) # We need to clear the path_importer_cache so that our new FileFinder will # start being used for existing directories we've loaded modules from. sys.path_importer_cache.clear()
def install(haxe_bin='haxe'): HaxeLoader.haxe_bin = haxe_bin loader_details = HaxeLoader, [".hx"] # insert the path hook ahead of other path hooks sys.path_hooks.insert(0, FileFinder.path_hook(loader_details)) # clear any loaders that might already be in use by the FileFinder sys.path_importer_cache.clear() invalidate_caches()
def _install_source_loader_helper(source_loader_type): extensions = ExtensionFileLoader, _imp.extension_suffixes() source = source_loader_type, SOURCE_SUFFIXES bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES supported_loaders = [extensions, source, bytecode] sys.path_hooks[:] = [ zipimport.zipimporter, FileFinder.path_hook(*supported_loaders), ] sys.path_importer_cache.clear()
def lib2to3importer(fixers, prefix=None): """Returns finder closure to be prepended to sys.path_hooks. :param fixers: e.g. ["lib2to3.fixes.fix_exec", "lib2to3.fixes.fix_long"] :param prefix: Prefix of module or package name to apply fixes. Unless provided, fixes are applied to all of modules and packages. """ extensions = ExtensionFileLoader, EXTENSION_SUFFIXES source = Lib2to3FileLoader.apply(fixers, prefix), SOURCE_SUFFIXES bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES return FileFinder.path_hook(extensions, source, bytecode)
def file_loader(loader: object) -> Generator[None, None, None]: orig_hooks = sys.path_hooks[:] sys.path_hooks[:] = [ FileFinder.path_hook( loader) # pyre-ignore[6]: Loader type isn't expose ] sys.path_importer_cache.clear() try: yield finally: sys.path_hooks[:] = orig_hooks sys.path_importer_cache.clear()
def install_hooks(): _ctx.update({"start_time": time.time(), "depth": 0}) ExtensionFileLoader.exec_module = profile(ExtensionFileLoader.exec_module) SourceFileLoader.exec_module = profile(SourceFileLoader.exec_module) SourcelessFileLoader.exec_module = profile( SourcelessFileLoader.exec_module) sys.path_hooks.insert( 0, FileFinder.path_hook( (ExtensionFileLoader, EXTENSION_SUFFIXES), (SourceFileLoader, SOURCE_SUFFIXES), (SourcelessFileLoader, BYTECODE_SUFFIXES), ), )
def install_loader(): """ Install the import hook Once this has been run, the enhancements will be available in any imported code This will run automatically when the package is imported; to disable:: builtins.__dict__["__perl__disable_automatic_import"] = True """ # Set up import hook loader_details = PerlLoader, [".py"] sys.path_hooks.insert(0, FileFinder.path_hook(loader_details)) sys.path_importer_cache.clear() invalidate_caches() # Inject dependencies for rewritten code builtins.__dict__["re"] = re builtins.__dict__["__perl__re_match"] = re_match builtins.__dict__["__perl__reset_vars"] = reset_vars
def hotfix(module_names): global OLD_MODULES global BUILTINS_OBJ OLD_MODULES.clear() for name in module_names: if name in sys.modules: OLD_MODULES[name] = sys.modules[name] gc.disable() OLD_MODULES = sys.modules.copy() BUILTINS_OBJ = object sys.modules["builtins"].object = HotfixerObject sys.path_hooks.insert(0, FileFinder.path_hook((MetaPathLoader, [".py"]))) # clear any loaders that might already be in use by the FileFinder sys.path_importer_cache.clear() importlib.invalidate_caches() for name in module_names: try: old_module = sys.modules.pop(name, None) new_module = importlib.import_module(name) if old_module and not is_skip_hotfix(new_module): for attr_name, new_attr in inspect.getmembers(new_module): setattr(old_module, attr_name, new_attr) except Exception as ex: traceback.print_exc() finally: if name in OLD_MODULES: sys.modules[name] = OLD_MODULES[name] sys.path_hooks.pop(0) sys.modules["builtins"].object = BUILTINS_OBJ BUILTINS_OBJ = None for obj in NEW_OBJECTS: hotfix_obj(obj) gc.enable() gc.collect()
def available_players(player_paths: T.Optional[T.Sequence[str]] = None) -> T.Sequence[ModuleInfo]: import othello module_path = othello.__path__ # type: ignore # mypy issue #1422 player_paths = player_paths or tuple() dir_player_paths = ( *(path.join(p, "models", "players") for p in module_path), *(dir_path for dir_path in player_paths if path.isdir(dir_path)), ) return ( *walk_packages(dir_player_paths, onerror=lambda _: None), *( ModuleInfo( FileFinder(path.dirname(file_path), (SourceFileLoader, (".py", ".pyc"))), path.splitext(path.basename(file_path))[0], False, ) for file_path in player_paths if path.isfile(file_path) ), )
def update_hooks(*loaders): """Append custom loaders to the `sys.path_hooks`, they must have a tuple attribute EXTENSION_SUFFIXES to discover the correct path. _NATIVE_HOOK resides in the global scope to reset the original sys.path_hooks if necessary.. """ global _NATIVE_HOOK from importlib.machinery import FileFinder if loaders: for i, hook in enumerate(sys.path_hooks): __closure__ = getattr(hook, '__closure__', None) if __closure__ and issubclass(__closure__[0].cell_contents, FileFinder): _NATIVE_HOOK = globals().get('_NATIVE_HOOK', (i, hook)) sys.path_hooks[i] = FileFinder.path_hook( *_NATIVE_HOOK[1].__closure__[1].cell_contents, *((loader, loader.EXTENSION_SUFFIXES) for loader in loaders )) else: sys.path_hooks[_NATIVE_HOOK[0]] = _NATIVE_HOOK[1] """https://docs.python.org/3/library/sys.html#sys.path_importer_cache""" sys.path_importer_cache.clear()
def find_and_load_module(modulename, folder=""): finder = FileFinder(folder, loader_details) spec = finder.find_spec(modulename) module = module_from_spec(spec) spec.loader.exec_module(module) return module
def find_and_load_module(modulename, folder=""): finder = FileFinder(folder, loader_details) spec = finder.find_spec(modulename) return spec.loader.load_module()
def get_module_path(modulename, folder=""): finder = FileFinder(folder, loader_details) return finder.find_spec(modulename).origin
def install(loader_details): # Insert the path hook ahead of other path hooks. sys.path_hooks.insert(0, FileFinder.path_hook(loader_details)) # Clear any loaders that might already be in use by the FileFinder. sys.path_importer_cache.clear() invalidate_caches()
cmake_args.append(f"-DCUDA_TOOLKIT_ROOT_DIR:STRING={cuda_root}") elif dp_variant == "rocm": cmake_args.append("-DUSE_ROCM_TOOLKIT:BOOL=TRUE") rocm_root = os.environ.get("ROCM_ROOT") if rocm_root: cmake_args.append(f"-DROCM_ROOT:STRING={rocm_root}") else: raise RuntimeError("Unsupported DP_VARIANT option: %s" % dp_variant) # get tensorflow spec tf_spec = find_spec("tensorflow") if not tf_spec: # purelib gets site-packages path site_packages = get_path("purelib") if site_packages: tf_spec = FileFinder(site_packages).find_spec("tensorflow") # get install dir from spec try: tf_install_dir = tf_spec.submodule_search_locations[0] # type: ignore # AttributeError if ft_spec is None # TypeError if submodule_search_locations are None # IndexError if submodule_search_locations is an empty list except (AttributeError, TypeError, IndexError): setup_requires.extend(extras_require['cpu']) # setuptools will re-find tensorflow after installing setup_requires tf_install_dir = None # add cmake as a build requirement if cmake>3.7 is not installed try: cmake_version = get_cmake_version()
def _getFinder(self, path): try: return self._finderCache[path] except KeyError: self._finderCache[path] = FileFinder(path, (EPSLoader, ['.eps'])) return self._finderCache[path]
def init_enforce() -> None: loader_details = (Loader, SOURCE_SUFFIXES) sys.path_hooks.insert(0, FileFinder.path_hook(loader_details)) # type: ignore
def installImportOverride(): # insert the path hook ahead of other path hooks sys.path_hooks.insert(0, FileFinder.path_hook(det)) # clear any loaders that might already be in use by the FileFinder sys.path_importer_cache.clear() invalidate_caches()