def create_script_spec(script_name): # There are 3 places that the scripts can live # 1. In the dev environment # 2. In the docker image # 3. Installed through wheel SMRF_SCRIPT_HOME = [ os.path.abspath(__file__ + '../../../../../') + '/scripts/', '/usr/local/bin/' ] found = False for sch in SMRF_SCRIPT_HOME: filename = os.path.join(sch, script_name) if os.path.exists(filename): SMRF_SCRIPT_HOME = filename found = True break # if it wasn't found then it should be installed if not found: SMRF_SCRIPT_HOME = shutil.which(script_name) return spec_from_loader( script_name, SourceFileLoader( script_name, SMRF_SCRIPT_HOME ) )
def _load_module(self): loader = SourceFileLoader("<unnamed module>", path=str(self.load_from)) spec = spec_from_loader(loader.name, loader) mod = importlib.util.module_from_spec(spec) loader.exec_module(mod) self.source = inspect.getsource(mod) return mod
def import_extension_module(module_name: str, path_to_extension_module: str): # you don't want to know how long it took me to get this right loader = ExtensionFileLoader(module_name, path_to_extension_module) spec = spec_from_loader(module_name, loader) module = module_from_spec(spec) loader.exec_module(module) return module
def __initialize_plugins(self, configured_plugins, plugin_module_files): # Initialize the plugins for class_name in configured_plugins.keys(): if not plugin_module_files.get(class_name): raise TidenPluginException( 'Python module not found in plugins/* for configured plugin %s' % class_name) plugin_file = plugin_module_files[class_name] # Load module loader = machinery.SourceFileLoader( path.basename(plugin_file)[:-3], plugin_file) spec = util.spec_from_loader(loader.name, loader) plugin_module = util.module_from_spec(spec) loader.exec_module(plugin_module) preloaded_plugin_config = { 'file': plugin_file, 'class': class_name, } # Check mandatory constants in a plugin module for const in self.mandatory_constants: preloaded_plugin_config[const] = getattr(plugin_module, const) # Get plugin options from config plugin_opts = configured_plugins[class_name] # Check version if needed if not plugin_opts.get('version') or \ plugin_opts['version'] == preloaded_plugin_config['TIDEN_PLUGIN_VERSION']: # Get the instance preloaded_plugin_config['instance'] \ = getattr(plugin_module, class_name)(class_name, self.config) self.plugins[class_name] = preloaded_plugin_config configured_plugins[class_name]['module'] = plugin_file
def inspectable_compile(s, modname=None): """ Compile and executable the input string, returning what was constructed :param s: Input source :type s: ```str``` :param modname: Module name, generates a random one if None :type modname: ```Optional[str]``` :return: The compiled and executed input source module, such that `inspect.getsource` works :rtype: ```Any``` """ filename = NamedTemporaryFile(suffix=".py").name modname = modname or path.splitext(path.basename(filename))[0] assert modname not in modules # our loader is a dummy one which just spits out our source loader = ShowSourceLoader(modname, s) spec = spec_from_loader(modname, loader, origin=filename) module = module_from_spec(spec) # the code must be compiled so the function's code object has a filename code = compile(s, mode="exec", filename=filename) exec(code, module.__dict__) # inspect.getmodule(...) requires it to be in sys.modules setattr(module, "__file__", s) modules[modname] = module return module
def __init__(self, folders: Union[str, List[str]]) -> None: """Import all menu models into the models namespace. Instantiation of the namespace requires either a path to the folder that contains model files or a list of such folders. Parameters ---------- folders : Union[str, List[str]] a folder or a list of folders to import models from """ if isinstance(folders, str): folders = [folders] for folder in folders: menu_models = [( f.replace("_model.py", ""), os.path.abspath(os.path.join(folder, f)), ) for f in os.listdir(folder) if f.endswith("_model.py")] for model_name, model_file in menu_models: loader = machinery.SourceFileLoader(model_name, model_file) spec = util.spec_from_loader(model_name, loader) if spec is not None: setattr(self, model_name, util.module_from_spec(spec)) loader.exec_module(getattr(self, model_name)) else: pass
def import_script(self): # Imports the script and returns the module and the module's callable spec = spec_from_loader( self.name.split(".")[0], StringLoader(self.file.read())) module = module_from_spec(spec) spec.loader.exec_module(module) return module, module._CALLABLE
def find_spec(self, fullname, path, target=None): spec = self._find_spec_nomacro(fullname, path, target) if spec is None or not (hasattr(spec.loader, 'get_source') and callable(spec.loader.get_source)): # noqa: E128 if fullname != 'org': # stdlib pickle.py at line 94 contains a ``from # org.python.core for Jython which is always failing, # of course logging.debug('Failed finding spec for %s', fullname) return origin = spec.origin if origin == 'builtin': return # # try to find already exported module # # TODO: are these the right arguments? # # NOTE: This is a noop # module = macropy.core.exporters.NullExporter().find( # file_path, file_path, "", module_name, package_path) # if module: # return _MacroLoader(ast.mod) try: source = spec.loader.get_source(fullname) except ImportError: logging.debug('Loader for %s was unable to find the sources', fullname) return except Exception: logging.exception('Loader for %s raised an error', fullname) return code, tree = self.expand_macros(source, origin, spec) if not code: # no macros! return loader = MacroLoader(spec, code, tree) return spec_from_loader(fullname, loader)
def load_python_launch_file_as_module(python_launch_file_path): """Load a given Python launch file (by path) as a Python module.""" loader = SourceFileLoader('python_launch_file', python_launch_file_path) spec = spec_from_loader(loader.name, loader) mod = module_from_spec(spec) loader.exec_module(mod) return mod
def load_module(module_name, path): """Load arbitrary Python source file""" loader = SourceFileLoader(module_name, path) spec = spec_from_loader(loader.name, loader) module = module_from_spec(spec) loader.exec_module(module) return module
def find_spec(self, fullname, path, target=None): spec = self._find_spec_nomacro(fullname, path, target) if spec is None or not (hasattr(spec.loader, 'get_source') and callable( spec.loader.get_source)): # noqa: E128 if fullname != 'org': # stdlib pickle.py at line 94 contains a ``from # org.python.core for Jython which is always failing, # of course logging.debug('Failed finding spec for %s', fullname) return origin = spec.origin if origin == 'builtin': return # # try to find already exported module # # TODO: are these the right arguments? # # NOTE: This is a noop # module = macropy.core.exporters.NullExporter().find( # file_path, file_path, "", module_name, package_path) # if module: # return _MacroLoader(ast.mod) try: source = spec.loader.get_source(fullname) except ImportError: logging.debug('Loader for %s was unable to find the sources', fullname) return except Exception: logging.exception('Loader for %s raised an error', fullname) return code, tree = self.expand_macros(source, origin, spec) if not code: # no macros! return loader = MacroLoader(spec, code, tree) return spec_from_loader(fullname, loader)
def _find_spec_legacy(finder, name, path): # This would be a good place for a DeprecationWarning if # we ended up going that route. loader = finder.find_module(name, path) if loader is None: return None return spec_from_loader(name, loader)
def import_module_from_file(module_name, source_file, sys_path=None): ''' Import a specific file as a module instance, return the module instance. Parameters: * `module_name`: the name to assign to the module * `source_file`: the source file to load * `sys_path`: optional list of paths to set as `sys.path` for the duration of this import; the default is the current value of `sys.path` Note that this is a "bare" import; the module instance is not inserted into `sys.modules`. *Warning*: `sys.path` is modified for the duration of this function, which may affect multithreaded applications. ''' if sys_path is None: sys_path = sys.path with stackattrs(sys, path=sys_path): loader = SourceFileLoader(module_name, source_file) spec = spec_from_loader(loader.name, loader) M = module_from_spec(spec) loader.exec_module(M) return M
def test_name_main(self) -> None: """Test the `if __name__ == '__main___'` block""" argparse_mock = MagicMock() loader = SourceFileLoader( "__main__", os.path.join( os.path.dirname(os.path.dirname(__file__)), "__main__{extsep}py".format(extsep=extsep), ), ) with patch("argparse.ArgumentParser._print_message", argparse_mock), patch( "sys.argv", []), self.assertRaises(SystemExit) as e: loader.exec_module( module_from_spec(spec_from_loader(loader.name, loader))) self.assertEqual(e.exception.code, SystemExit(2).code) self.assertEqual( (lambda output: output[(output.rfind(" ") + 1):][:-1])( (argparse_mock.call_args.args if PY3_8 else argparse_mock.call_args[0])[0]), "command", )
def execute_dsl( self, patcher: dag_runner_patcher.DagRunnerPatcher) -> Dict[str, Any]: self._check_pipeline_dsl_path() dsl_path = self.flags_dict[labels.PIPELINE_DSL_PATH] with patcher.patch() as context: # Simluate python script execution. # - Need to add the script directory as a first entry of sys.path. # - Load the script as if we are in __main__ module. dir_path = os.path.dirname(os.path.realpath(dsl_path)) sys.path.insert(0, dir_path) loader = machinery.SourceFileLoader('__main__', dsl_path) try: loader.exec_module( import_util.module_from_spec( import_util.spec_from_loader(loader.name, loader))) except SystemExit as system_exit: # Swallow normal exit in absl.app.run() if system_exit.code != 0 and system_exit.code is not None: raise sys.path.pop(0) if not patcher.run_called: sys.exit('Cannot find ' + patcher.get_runner_class().__name__ + '.run() in ' + dsl_path) return context
def find_spec(self, fullname, path, target=None): spec = self._find_plain_spec(fullname, path, target) if spec is None or not (hasattr(spec.loader, 'get_source') and callable( spec.loader.get_source)): # noqa: E128 if fullname != 'org': # stdlib pickle.py at line 94 contains a ``from # org.python.core for Jython which is always failing, # of course logging.debug('Failed finding spec for %s', fullname) return try: source = spec.loader.get_source(fullname) except ImportError: logging.debug('Loader for %s was unable to find the sources', fullname) return except Exception: logging.exception('Loader for %s raised an error', fullname) return if not source or 'birdseye' not in source: return deep, trace_stmt = should_trace(source) if not trace_stmt: return loader = BirdsEyeLoader(spec, source, deep) return spec_from_loader(fullname, loader)
def find_spec(self, name, target=None): if name in self.available: origin = "{}/{}.py".format(self.url, name) loader = URLLoader() return spec_from_loader(name, loader, origin=origin) else: return None
def run_program(arguments="", raises=DummyException): """ Run program at given path with given arguments. If raises is specified, ensure the given exception is raised. """ arguments = arguments.replace('\\', '\\\\') path, *args = shlex.split(arguments) old_args = sys.argv assert all(isinstance(a, str) for a in args) try: sys.argv = [path] + args with redirect_stdout(StringIO()) as output: with redirect_stderr(output): try: if '__main__' in sys.modules: del sys.modules['__main__'] loader = SourceFileLoader('__main__', path) spec = spec_from_loader(loader.name, loader) module = module_from_spec(spec) sys.modules['__main__'] = module loader.exec_module(module) except raises: return output.getvalue() except SystemExit as e: if e.args != (0,): raise SystemExit(output.getvalue()) from e finally: sys.modules['__main__'].__dict__.clear() sys.modules.pop('__main__', None) # Closes any open files if raises is not DummyException: raise AssertionError("{} not raised".format(raises)) return output.getvalue() finally: sys.argv = old_args
def import_module_from_file(file_name, module_name='module'): spec = spec_from_loader(module_name, SourceFileLoader(module_name, file_name)) mod = module_from_spec(spec) spec.loader.exec_module(mod) return mod
def read_config(self): ''' Reads the configuration file. Return values: 0: Config file was read successfully 1: Config file does not exist 2: Config file was read successfully, but it is outdated ''' if not os.path.exists(self.filename): return 1 dummy_module_dir, module_file = os.path.split(self.filename) module_name, dummy_module_ext = os.path.splitext(module_file) # Here we call loader directly inorder to avoid extension analyses (tricky!) spec = spec_from_loader(module_name, SourceFileLoader(module_name, self.filename)) self.config = module_from_spec(spec) spec.loader.exec_module(self.config) try: t = int(self.config.default_wood) self.config.default_wood = t except ValueError: pass vnum = version_number(self.config.version) if vnum < self.create_version_number: return 2 return 0
def launch(): discover_paths = set([os.path.abspath(".")]) for discover_path in discover_paths: sys.path.append(discover_path) parser = argparse.ArgumentParser( description= 'Run a task multiple times, controlled by envrironment variables.') parser.add_argument("--adaptor", type=str, default="shell", choices=Adaptors.choices(), help='the adaptor with which to run the task.') parser.add_argument( "--data-dir", type=str, help= 'the directory which holds the dataset. Can also be set through the DATA_DIR environment variable.', required=False) parser.add_argument( "--output-dir", type=str, help= 'the directory in which to output results. Can also be set through the OUTPUT_DIR environment variable.', required=False) subparsers = parser.add_subparsers(title='tasks', description='valid tasks', dest="task") tasks: Dict[str, Any] = {} for discover_path in discover_paths: for task in glob.glob("{}/**/tasks.py".format(discover_path), recursive=True): loader = SourceFileLoader("task", task) spec = util.spec_from_loader("task", loader) task = util.module_from_spec(spec) # type: ignore spec.loader.exec_module(task) # type: ignore for c in task.tasks: # type: ignore c.arguments(subparsers) tasks[c.name] = c assert len(tasks) > 0, "No tasks found in {}".format(discover_paths) for adaptor_name in Adaptors.choices(): adaptor = Adaptors.get(adaptor_name) adaptor.arguments(parser) args = vars(parser.parse_args()) if args["data_dir"] is None and "DATA_DIR" in os.environ: args["data_dir"] = os.environ["DATA_DIR"] if args["output_dir"] is None and "OUTPUT_DIR" in os.environ: args["output_dir"] = os.environ["OUTPUT_DIR"] assert args["data_dir"] is not None, "No data directory selected." assert args["output_dir"] is not None, "No output directory selected." assert args["task"] is not None, "No task selected." adaptor = Adaptors.get(args["adaptor"]) task = tasks[args["task"]] adaptor.execute(task, args)
def test_01_loading_scripts(self): for script in SCRIPTS: with self.subTest(script=script): loader = SourceFileLoader(script, os.path.join('tools', script)) spec = spec_from_loader(loader.name, loader) mod = module_from_spec(spec) loader.exec_module(mod)
def load_module(p: Path) -> Any: module_name = p.name.rsplit('.')[0] with sys_path_starting(with_=p.parent): loader = SourceFileLoader(module_name, str(p)) spec = spec_from_loader(module_name, loader, is_package=False) module = module_from_spec(spec) loader.exec_module(module) return module
def find_spec(self, fullname, path, target=None): loader = self._get_loader(fullname, path) if loader: spec = spec_from_loader(fullname, loader) if spec is not None and hasattr(loader, '_subpackage_search_paths'): spec.submodule_search_locations = loader._subpackage_search_paths return spec else: return None
def _get_bcc_trace_tool(args): sys.path.append('/usr/lib/python3/dist-packages') loader = machinery.SourceFileLoader('bcc_trace', '/usr/share/bcc/tools/trace') spec = util.spec_from_loader(loader.name, loader) bcc_trace = util.module_from_spec(spec) loader.exec_module(bcc_trace) sys.argv = args return bcc_trace.Tool()
def load(module_name, filename): """ Helper function to load the specified filename into the specified module name """ loader = PerlLoader(module_name, filename) spec = spec_from_loader(loader.name, loader) module = module_from_spec(spec) loader.exec_module(module) return module
def import_module_from_file(module_name, path): """Load a modules from a raw file path.""" loader = SourceFileLoader(module_name, path) spec = spec_from_loader(module_name, loader) module = module_from_spec(spec) spec.loader.exec_module(module) return module
def load_libyear_module(): """ As the module has no extension, this workaround is needed to load """ libyear_path = str(Path(__file__).parent.parent / "libyear/libyear") spec = spec_from_loader("libyear", SourceFileLoader("libyear", libyear_path)) libyear = module_from_spec(spec) spec.loader.exec_module(libyear) sys.modules['libyear'] = libyear return libyear
def _load_generator(self, generator_file: Path): if not (generator_file.exists() and generator_file.is_file()): raise FileNotFoundError("`{generator_file}` not found.") loader = machinery.SourceFileLoader("testgen", str(generator_file)) spec = util.spec_from_loader(loader.name, loader) mod = util.module_from_spec(spec) loader.exec_module(mod) return mod
def find_spec(self, fullname, path, target=None): mod_path, is_package = get_path(fullname) module = (self.modules.get(mod_path) or get_code(open(mod_path).read())) return spec_from_loader(fullname, TestModuleLoader(module, mod_path), is_package=is_package)
def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec: """Get ModuleSpec for builtin or extension module""" if modinfo.state == ModuleState.SHARED: location = os.fspath(self.get_location(modinfo)) loader = ExtensionFileLoader(modinfo.name, location) return spec_from_file_location(modinfo.name, location, loader=loader) elif modinfo.state == ModuleState.BUILTIN: return spec_from_loader(modinfo.name, loader=BuiltinImporter) else: raise ValueError(modinfo)
def __init__(self): super(Module, self).__init__(__name__) self.__file__ = __file__ if sys.version_info < (3, 4): self.__loader__ = loader else: from importlib.util import spec_from_loader self.__spec__ = spec_from_loader(__name__, loader, origin=__spec__.origin)
def find_spec(cls, fullname, path=None, target=None): modules = [m for m in path or cls._cached_hs_modules() if m == fullname or m.startswith(fullname+'.')] if modules: spec = util.spec_from_loader(fullname, HaskellLoader, origin='haskell', is_package=len(modules) > 1) if len(modules) > 1: spec.submodule_search_locations = list(modules) return spec return None
def new_module(self, source_code=None): loader = TestingImporter() if source_code is not None: loader.source_code = source_code spec = util.spec_from_loader(TestingImporter.module_name, util.LazyLoader(loader)) module = spec.loader.create_module(spec) module.__spec__ = spec module.__loader__ = spec.loader spec.loader.exec_module(module) # Module is now lazy. self.assertIsNone(loader.loaded) return module
def _find_spec_nomacro(self, fullname, path, target=None): """Try to find the original, non macro-expanded module using all the remaining meta_path finders. This one is installed by ``macropy.activate`` at index 0.""" spec = None for finder in sys.meta_path: # when testing with pytest, it installs a finder that for # some yet unknown reasons makes macros expansion # fail. For now it will just avoid using it and pass to # the next one if finder is self or 'pytest' in finder.__module__: continue if hasattr(finder, 'find_spec'): spec = finder.find_spec(fullname, path, target=target) elif hasattr(finder, 'load_module'): spec = spec_from_loader(fullname, finder) if spec is not None: break return spec
# GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with nautilus-dropbox. If not, see <http://www.gnu.org/licenses/>. # import os import shutil import socket import sys import unittest from importlib.util import spec_from_loader, module_from_spec from importlib.machinery import SourceFileLoader from unittest.mock import MagicMock spec = spec_from_loader("dropbox", SourceFileLoader("dropbox", "./dropbox")) dropbox = module_from_spec(spec) if spec.loader: spec.loader.exec_module(dropbox) class TestDropbox(unittest.TestCase): def test_plat_fails_on_non_linux(self): sys.platform = 'darwin' #with self.assertRaises(dropbox.FatalVisibleError): dropbox.plat() def test_reroll_autostart_without_config_dir(self): os.listdir = MagicMock(return_value=[]) os.makedirs = MagicMock() os.remove = MagicMock()
def find_spec(self, name, path, target=None): if name != self.module_name: return None return util.spec_from_loader(name, util.LazyLoader(self))
def find_spec(self, fullname, path, target=None): module_spec = PathFinder.find_spec(fullname, path, target) if module_spec and module_spec.has_location and self._is_in_directory(module_spec.origin): loader = Loader(fullname, module_spec.origin, self._transformer) is_package = os.path.basename(module_spec.origin).lower() == "__init__.py" return spec_from_loader(fullname, loader, origin=module_spec.origin, is_package=is_package)