def create_script_spec(script_name):
# There are 3 places that the scripts can live
# 1. In the dev environment
# 2. In the docker image
# 3. Installed through wheel
SMRF_SCRIPT_HOME = [
os.path.abspath(__file__ + '../../../../../') + '/scripts/',
'/usr/local/bin/'
]
found = False
for sch in SMRF_SCRIPT_HOME:
filename = os.path.join(sch, script_name)
if os.path.exists(filename):
SMRF_SCRIPT_HOME = filename
found = True
break
# if it wasn't found then it should be installed
if not found:
SMRF_SCRIPT_HOME = shutil.which(script_name)
return spec_from_loader(
script_name,
SourceFileLoader(
script_name,
SMRF_SCRIPT_HOME
)
)
def _load_module(self):
loader = SourceFileLoader("<unnamed module>", path=str(self.load_from))
spec = spec_from_loader(loader.name, loader)
mod = importlib.util.module_from_spec(spec)
loader.exec_module(mod)
self.source = inspect.getsource(mod)
return mod
def import_extension_module(module_name: str, path_to_extension_module: str):
# you don't want to know how long it took me to get this right
loader = ExtensionFileLoader(module_name, path_to_extension_module)
spec = spec_from_loader(module_name, loader)
module = module_from_spec(spec)
loader.exec_module(module)
return module
def __initialize_plugins(self, configured_plugins, plugin_module_files):
# Initialize the plugins
for class_name in configured_plugins.keys():
if not plugin_module_files.get(class_name):
raise TidenPluginException(
'Python module not found in plugins/* for configured plugin %s'
% class_name)
plugin_file = plugin_module_files[class_name]
# Load module
loader = machinery.SourceFileLoader(
path.basename(plugin_file)[:-3], plugin_file)
spec = util.spec_from_loader(loader.name, loader)
plugin_module = util.module_from_spec(spec)
loader.exec_module(plugin_module)
preloaded_plugin_config = {
'file': plugin_file,
'class': class_name,
}
# Check mandatory constants in a plugin module
for const in self.mandatory_constants:
preloaded_plugin_config[const] = getattr(plugin_module, const)
# Get plugin options from config
plugin_opts = configured_plugins[class_name]
# Check version if needed
if not plugin_opts.get('version') or \
plugin_opts['version'] == preloaded_plugin_config['TIDEN_PLUGIN_VERSION']:
# Get the instance
preloaded_plugin_config['instance'] \
= getattr(plugin_module, class_name)(class_name, self.config)
self.plugins[class_name] = preloaded_plugin_config
configured_plugins[class_name]['module'] = plugin_file
def inspectable_compile(s, modname=None):
"""
Compile and executable the input string, returning what was constructed
:param s: Input source
:type s: ```str```
:param modname: Module name, generates a random one if None
:type modname: ```Optional[str]```
:return: The compiled and executed input source module, such that `inspect.getsource` works
:rtype: ```Any```
"""
filename = NamedTemporaryFile(suffix=".py").name
modname = modname or path.splitext(path.basename(filename))[0]
assert modname not in modules
# our loader is a dummy one which just spits out our source
loader = ShowSourceLoader(modname, s)
spec = spec_from_loader(modname, loader, origin=filename)
module = module_from_spec(spec)
# the code must be compiled so the function's code object has a filename
code = compile(s, mode="exec", filename=filename)
exec(code, module.__dict__)
# inspect.getmodule(...) requires it to be in sys.modules
setattr(module, "__file__", s)
modules[modname] = module
return module
def __init__(self, folders: Union[str, List[str]]) -> None:
"""Import all menu models into the models namespace.
Instantiation of the namespace requires either a path to the folder that
contains model files or a list of such folders.
Parameters
----------
folders : Union[str, List[str]]
a folder or a list of folders to import models from
"""
if isinstance(folders, str):
folders = [folders]
for folder in folders:
menu_models = [(
f.replace("_model.py", ""),
os.path.abspath(os.path.join(folder, f)),
) for f in os.listdir(folder) if f.endswith("_model.py")]
for model_name, model_file in menu_models:
loader = machinery.SourceFileLoader(model_name, model_file)
spec = util.spec_from_loader(model_name, loader)
if spec is not None:
setattr(self, model_name, util.module_from_spec(spec))
loader.exec_module(getattr(self, model_name))
else:
pass
def import_script(self):
# Imports the script and returns the module and the module's callable
spec = spec_from_loader(
self.name.split(".")[0], StringLoader(self.file.read()))
module = module_from_spec(spec)
spec.loader.exec_module(module)
return module, module._CALLABLE
def find_spec(self, fullname, path, target=None):
spec = self._find_spec_nomacro(fullname, path, target)
if spec is None or not (hasattr(spec.loader, 'get_source') and
callable(spec.loader.get_source)): # noqa: E128
if fullname != 'org':
# stdlib pickle.py at line 94 contains a ``from
# org.python.core for Jython which is always failing,
# of course
logging.debug('Failed finding spec for %s', fullname)
return
origin = spec.origin
if origin == 'builtin':
return
# # try to find already exported module
# # TODO: are these the right arguments?
# # NOTE: This is a noop
# module = macropy.core.exporters.NullExporter().find(
# file_path, file_path, "", module_name, package_path)
# if module:
# return _MacroLoader(ast.mod)
try:
source = spec.loader.get_source(fullname)
except ImportError:
logging.debug('Loader for %s was unable to find the sources',
fullname)
return
except Exception:
logging.exception('Loader for %s raised an error', fullname)
return
code, tree = self.expand_macros(source, origin, spec)
if not code: # no macros!
return
loader = MacroLoader(spec, code, tree)
return spec_from_loader(fullname, loader)
def load_python_launch_file_as_module(python_launch_file_path):
"""Load a given Python launch file (by path) as a Python module."""
loader = SourceFileLoader('python_launch_file', python_launch_file_path)
spec = spec_from_loader(loader.name, loader)
mod = module_from_spec(spec)
loader.exec_module(mod)
return mod
def load_module(module_name, path):
"""Load arbitrary Python source file"""
loader = SourceFileLoader(module_name, path)
spec = spec_from_loader(loader.name, loader)
module = module_from_spec(spec)
loader.exec_module(module)
return module
def find_spec(self, fullname, path, target=None):
spec = self._find_spec_nomacro(fullname, path, target)
if spec is None or not (hasattr(spec.loader, 'get_source')
and callable(
spec.loader.get_source)): # noqa: E128
if fullname != 'org':
# stdlib pickle.py at line 94 contains a ``from
# org.python.core for Jython which is always failing,
# of course
logging.debug('Failed finding spec for %s', fullname)
return
origin = spec.origin
if origin == 'builtin':
return
# # try to find already exported module
# # TODO: are these the right arguments?
# # NOTE: This is a noop
# module = macropy.core.exporters.NullExporter().find(
# file_path, file_path, "", module_name, package_path)
# if module:
# return _MacroLoader(ast.mod)
try:
source = spec.loader.get_source(fullname)
except ImportError:
logging.debug('Loader for %s was unable to find the sources',
fullname)
return
except Exception:
logging.exception('Loader for %s raised an error', fullname)
return
code, tree = self.expand_macros(source, origin, spec)
if not code: # no macros!
return
loader = MacroLoader(spec, code, tree)
return spec_from_loader(fullname, loader)
def _find_spec_legacy(finder, name, path):
# This would be a good place for a DeprecationWarning if
# we ended up going that route.
loader = finder.find_module(name, path)
if loader is None:
return None
return spec_from_loader(name, loader)
def import_module_from_file(module_name, source_file, sys_path=None):
''' Import a specific file as a module instance,
return the module instance.
Parameters:
* `module_name`: the name to assign to the module
* `source_file`: the source file to load
* `sys_path`: optional list of paths to set as `sys.path`
for the duration of this import;
the default is the current value of `sys.path`
Note that this is a "bare" import;
the module instance is not inserted into `sys.modules`.
*Warning*: `sys.path` is modified for the duration of this function,
which may affect multithreaded applications.
'''
if sys_path is None:
sys_path = sys.path
with stackattrs(sys, path=sys_path):
loader = SourceFileLoader(module_name, source_file)
spec = spec_from_loader(loader.name, loader)
M = module_from_spec(spec)
loader.exec_module(M)
return M
def test_name_main(self) -> None:
"""Test the `if __name__ == '__main___'` block"""
argparse_mock = MagicMock()
loader = SourceFileLoader(
"__main__",
os.path.join(
os.path.dirname(os.path.dirname(__file__)),
"__main__{extsep}py".format(extsep=extsep),
),
)
with patch("argparse.ArgumentParser._print_message",
argparse_mock), patch(
"sys.argv", []), self.assertRaises(SystemExit) as e:
loader.exec_module(
module_from_spec(spec_from_loader(loader.name, loader)))
self.assertEqual(e.exception.code, SystemExit(2).code)
self.assertEqual(
(lambda output: output[(output.rfind(" ") + 1):][:-1])(
(argparse_mock.call_args.args
if PY3_8 else argparse_mock.call_args[0])[0]),
"command",
)
def execute_dsl(
self,
patcher: dag_runner_patcher.DagRunnerPatcher) -> Dict[str, Any]:
self._check_pipeline_dsl_path()
dsl_path = self.flags_dict[labels.PIPELINE_DSL_PATH]
with patcher.patch() as context:
# Simluate python script execution.
# - Need to add the script directory as a first entry of sys.path.
# - Load the script as if we are in __main__ module.
dir_path = os.path.dirname(os.path.realpath(dsl_path))
sys.path.insert(0, dir_path)
loader = machinery.SourceFileLoader('__main__', dsl_path)
try:
loader.exec_module(
import_util.module_from_spec(
import_util.spec_from_loader(loader.name, loader)))
except SystemExit as system_exit: # Swallow normal exit in absl.app.run()
if system_exit.code != 0 and system_exit.code is not None:
raise
sys.path.pop(0)
if not patcher.run_called:
sys.exit('Cannot find ' + patcher.get_runner_class().__name__ +
'.run() in ' + dsl_path)
return context
def find_spec(self, fullname, path, target=None):
spec = self._find_plain_spec(fullname, path, target)
if spec is None or not (hasattr(spec.loader, 'get_source')
and callable(
spec.loader.get_source)): # noqa: E128
if fullname != 'org':
# stdlib pickle.py at line 94 contains a ``from
# org.python.core for Jython which is always failing,
# of course
logging.debug('Failed finding spec for %s', fullname)
return
try:
source = spec.loader.get_source(fullname)
except ImportError:
logging.debug('Loader for %s was unable to find the sources',
fullname)
return
except Exception:
logging.exception('Loader for %s raised an error', fullname)
return
if not source or 'birdseye' not in source:
return
deep, trace_stmt = should_trace(source)
if not trace_stmt:
return
loader = BirdsEyeLoader(spec, source, deep)
return spec_from_loader(fullname, loader)
def find_spec(self, name, target=None):
if name in self.available:
origin = "{}/{}.py".format(self.url, name)
loader = URLLoader()
return spec_from_loader(name, loader, origin=origin)
else:
return None
def run_program(arguments="", raises=DummyException):
"""
Run program at given path with given arguments.
If raises is specified, ensure the given exception is raised.
"""
arguments = arguments.replace('\\', '\\\\')
path, *args = shlex.split(arguments)
old_args = sys.argv
assert all(isinstance(a, str) for a in args)
try:
sys.argv = [path] + args
with redirect_stdout(StringIO()) as output:
with redirect_stderr(output):
try:
if '__main__' in sys.modules:
del sys.modules['__main__']
loader = SourceFileLoader('__main__', path)
spec = spec_from_loader(loader.name, loader)
module = module_from_spec(spec)
sys.modules['__main__'] = module
loader.exec_module(module)
except raises:
return output.getvalue()
except SystemExit as e:
if e.args != (0,):
raise SystemExit(output.getvalue()) from e
finally:
sys.modules['__main__'].__dict__.clear()
sys.modules.pop('__main__', None) # Closes any open files
if raises is not DummyException:
raise AssertionError("{} not raised".format(raises))
return output.getvalue()
finally:
sys.argv = old_args
def import_module_from_file(file_name, module_name='module'):
spec = spec_from_loader(module_name,
SourceFileLoader(module_name, file_name))
mod = module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def read_config(self):
'''
Reads the configuration file. Return values:
0: Config file was read successfully
1: Config file does not exist
2: Config file was read successfully, but it is outdated
'''
if not os.path.exists(self.filename):
return 1
dummy_module_dir, module_file = os.path.split(self.filename)
module_name, dummy_module_ext = os.path.splitext(module_file)
# Here we call loader directly inorder to avoid extension analyses (tricky!)
spec = spec_from_loader(module_name,
SourceFileLoader(module_name, self.filename))
self.config = module_from_spec(spec)
spec.loader.exec_module(self.config)
try:
t = int(self.config.default_wood)
self.config.default_wood = t
except ValueError:
pass
vnum = version_number(self.config.version)
if vnum < self.create_version_number:
return 2
return 0
def launch():
discover_paths = set([os.path.abspath(".")])
for discover_path in discover_paths:
sys.path.append(discover_path)
parser = argparse.ArgumentParser(
description=
'Run a task multiple times, controlled by envrironment variables.')
parser.add_argument("--adaptor",
type=str,
default="shell",
choices=Adaptors.choices(),
help='the adaptor with which to run the task.')
parser.add_argument(
"--data-dir",
type=str,
help=
'the directory which holds the dataset. Can also be set through the DATA_DIR environment variable.',
required=False)
parser.add_argument(
"--output-dir",
type=str,
help=
'the directory in which to output results. Can also be set through the OUTPUT_DIR environment variable.',
required=False)
subparsers = parser.add_subparsers(title='tasks',
description='valid tasks',
dest="task")
tasks: Dict[str, Any] = {}
for discover_path in discover_paths:
for task in glob.glob("{}/**/tasks.py".format(discover_path),
recursive=True):
loader = SourceFileLoader("task", task)
spec = util.spec_from_loader("task", loader)
task = util.module_from_spec(spec) # type: ignore
spec.loader.exec_module(task) # type: ignore
for c in task.tasks: # type: ignore
c.arguments(subparsers)
tasks[c.name] = c
assert len(tasks) > 0, "No tasks found in {}".format(discover_paths)
for adaptor_name in Adaptors.choices():
adaptor = Adaptors.get(adaptor_name)
adaptor.arguments(parser)
args = vars(parser.parse_args())
if args["data_dir"] is None and "DATA_DIR" in os.environ:
args["data_dir"] = os.environ["DATA_DIR"]
if args["output_dir"] is None and "OUTPUT_DIR" in os.environ:
args["output_dir"] = os.environ["OUTPUT_DIR"]
assert args["data_dir"] is not None, "No data directory selected."
assert args["output_dir"] is not None, "No output directory selected."
assert args["task"] is not None, "No task selected."
adaptor = Adaptors.get(args["adaptor"])
task = tasks[args["task"]]
adaptor.execute(task, args)
def test_01_loading_scripts(self):
for script in SCRIPTS:
with self.subTest(script=script):
loader = SourceFileLoader(script,
os.path.join('tools', script))
spec = spec_from_loader(loader.name, loader)
mod = module_from_spec(spec)
loader.exec_module(mod)
def load_module(p: Path) -> Any:
module_name = p.name.rsplit('.')[0]
with sys_path_starting(with_=p.parent):
loader = SourceFileLoader(module_name, str(p))
spec = spec_from_loader(module_name, loader, is_package=False)
module = module_from_spec(spec)
loader.exec_module(module)
return module
def find_spec(self, fullname, path, target=None):
loader = self._get_loader(fullname, path)
if loader:
spec = spec_from_loader(fullname, loader)
if spec is not None and hasattr(loader, '_subpackage_search_paths'):
spec.submodule_search_locations = loader._subpackage_search_paths
return spec
else:
return None
def _get_bcc_trace_tool(args):
sys.path.append('/usr/lib/python3/dist-packages')
loader = machinery.SourceFileLoader('bcc_trace',
'/usr/share/bcc/tools/trace')
spec = util.spec_from_loader(loader.name, loader)
bcc_trace = util.module_from_spec(spec)
loader.exec_module(bcc_trace)
sys.argv = args
return bcc_trace.Tool()
def load(module_name, filename):
"""
Helper function to load the specified filename into the specified module name
"""
loader = PerlLoader(module_name, filename)
spec = spec_from_loader(loader.name, loader)
module = module_from_spec(spec)
loader.exec_module(module)
return module
def import_module_from_file(module_name, path):
"""Load a modules from a raw file path."""
loader = SourceFileLoader(module_name, path)
spec = spec_from_loader(module_name, loader)
module = module_from_spec(spec)
spec.loader.exec_module(module)
return module
def load_libyear_module():
""" As the module has no extension, this workaround is needed to load """
libyear_path = str(Path(__file__).parent.parent / "libyear/libyear")
spec = spec_from_loader("libyear",
SourceFileLoader("libyear", libyear_path))
libyear = module_from_spec(spec)
spec.loader.exec_module(libyear)
sys.modules['libyear'] = libyear
return libyear
def _load_generator(self, generator_file: Path):
if not (generator_file.exists() and generator_file.is_file()):
raise FileNotFoundError("`{generator_file}` not found.")
loader = machinery.SourceFileLoader("testgen", str(generator_file))
spec = util.spec_from_loader(loader.name, loader)
mod = util.module_from_spec(spec)
loader.exec_module(mod)
return mod
def find_spec(self, fullname, path, target=None):
mod_path, is_package = get_path(fullname)
module = (self.modules.get(mod_path)
or get_code(open(mod_path).read()))
return spec_from_loader(fullname,
TestModuleLoader(module, mod_path),
is_package=is_package)
def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec:
"""Get ModuleSpec for builtin or extension module"""
if modinfo.state == ModuleState.SHARED:
location = os.fspath(self.get_location(modinfo))
loader = ExtensionFileLoader(modinfo.name, location)
return spec_from_file_location(modinfo.name, location, loader=loader)
elif modinfo.state == ModuleState.BUILTIN:
return spec_from_loader(modinfo.name, loader=BuiltinImporter)
else:
raise ValueError(modinfo)
def __init__(self):
super(Module, self).__init__(__name__)
self.__file__ = __file__
if sys.version_info < (3, 4):
self.__loader__ = loader
else:
from importlib.util import spec_from_loader
self.__spec__ = spec_from_loader(__name__, loader,
origin=__spec__.origin)
def find_spec(cls, fullname, path=None, target=None):
modules = [m for m in path or cls._cached_hs_modules()
if m == fullname or m.startswith(fullname+'.')]
if modules:
spec = util.spec_from_loader(fullname, HaskellLoader,
origin='haskell',
is_package=len(modules) > 1)
if len(modules) > 1:
spec.submodule_search_locations = list(modules)
return spec
return None
def new_module(self, source_code=None):
loader = TestingImporter()
if source_code is not None:
loader.source_code = source_code
spec = util.spec_from_loader(TestingImporter.module_name,
util.LazyLoader(loader))
module = spec.loader.create_module(spec)
module.__spec__ = spec
module.__loader__ = spec.loader
spec.loader.exec_module(module)
# Module is now lazy.
self.assertIsNone(loader.loaded)
return module
def _find_spec_nomacro(self, fullname, path, target=None):
"""Try to find the original, non macro-expanded module using all the
remaining meta_path finders. This one is installed by
``macropy.activate`` at index 0."""
spec = None
for finder in sys.meta_path:
# when testing with pytest, it installs a finder that for
# some yet unknown reasons makes macros expansion
# fail. For now it will just avoid using it and pass to
# the next one
if finder is self or 'pytest' in finder.__module__:
continue
if hasattr(finder, 'find_spec'):
spec = finder.find_spec(fullname, path, target=target)
elif hasattr(finder, 'load_module'):
spec = spec_from_loader(fullname, finder)
if spec is not None:
break
return spec
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with nautilus-dropbox. If not, see <http://www.gnu.org/licenses/>.
#
import os
import shutil
import socket
import sys
import unittest
from importlib.util import spec_from_loader, module_from_spec
from importlib.machinery import SourceFileLoader
from unittest.mock import MagicMock
spec = spec_from_loader("dropbox", SourceFileLoader("dropbox", "./dropbox"))
dropbox = module_from_spec(spec)
if spec.loader:
spec.loader.exec_module(dropbox)
class TestDropbox(unittest.TestCase):
def test_plat_fails_on_non_linux(self):
sys.platform = 'darwin'
#with self.assertRaises(dropbox.FatalVisibleError):
dropbox.plat()
def test_reroll_autostart_without_config_dir(self):
os.listdir = MagicMock(return_value=[])
os.makedirs = MagicMock()
os.remove = MagicMock()
def find_spec(self, name, path, target=None):
if name != self.module_name:
return None
return util.spec_from_loader(name, util.LazyLoader(self))
def find_spec(self, fullname, path, target=None):
module_spec = PathFinder.find_spec(fullname, path, target)
if module_spec and module_spec.has_location and self._is_in_directory(module_spec.origin):
loader = Loader(fullname, module_spec.origin, self._transformer)
is_package = os.path.basename(module_spec.origin).lower() == "__init__.py"
return spec_from_loader(fullname, loader, origin=module_spec.origin, is_package=is_package)
