def list_opts():
"""Returns a list of oslo.config options available in the library.
The returned list includes all oslo.config options which may be registered
at runtime by the library.
Each element of the list is a tuple. The first element is the name of the
group under which the list of elements in the second element will be
registered. A group name of None corresponds to the [DEFAULT] group in
config files.
The purpose of this is to allow tools like the Oslo sample config file
generator to discover the options exposed to users by this library.
:returns: a list of (group_name, opts) tuples
"""
key_manager_opts = []
key_manager_opts.extend(key_manager.key_manager_opts)
key_manager_opts.extend(utils.credential_opts)
opts = [('key_manager', key_manager_opts)]
ext_mgr = ExtensionManager("castellan.drivers",
invoke_on_load=True,
invoke_args=[cfg.CONF])
for driver in ext_mgr.names():
opts.extend(ext_mgr[driver].obj.list_options_for_discovery())
return opts
def entrypoint(args=None):
parser = argparse.ArgumentParser()
parser.add_argument('--debug', '-D', action='store_true')
subparsers = parser.add_subparsers(dest='command')
subparsers.required = True
commands = {}
def register_extension(ext, commands=commands):
try:
parser = subparsers.add_parser(ext.name)
commands[ext.name] = ext.plugin(parser)
except Exception:
logger.exception('Error while loading command {}.'.format(
ext.name))
from stevedore import ExtensionManager
mgr = ExtensionManager(namespace='bonobo.commands')
mgr.map(register_extension)
args = parser.parse_args(args).__dict__
if args.pop('debug', False):
settings.DEBUG = True
settings.LOGGING_LEVEL = logging.DEBUG
logging.set_level(settings.LOGGING_LEVEL)
logger.debug('Command: ' + args['command'] + ' Arguments: ' + repr(args))
commands[args.pop('command')](**args)
def test_zmq_client_configure_eslogger(self):
dummy_ep = importlib.metadata.EntryPoint(
'dummy', 'test.test_plugins:DummyEventTarget',
'es_logger.plugins.event_target')
dummy2_ep = importlib.metadata.EntryPoint(
'dummy2', 'test.test_plugins:DummyEventTarget',
'es_logger.plugins.event_target')
ExtensionManager.ENTRY_POINT_CACHE = {
'es_logger.plugins.event_target': [dummy_ep, dummy2_ep]
}
with unittest.mock.patch(
'configparser.ConfigParser') as mock_config_parser:
config = self.config_setup(mock_config_parser)
self.set_default_config(config)
self.set_eslogger_targets_config(config)
self.zmqd.configure()
# Reset stevedore
ExtensionManager.ENTRY_POINT_CACHE = {}
mgr = ExtensionManager(namespace='es_logger.plugins.event_target',
invoke_on_load=False)
mgr.names()
# Perform validation after resetting stevedore state
expected_targets = {
'dummy': {
'var1': 'var1'
},
'dummy2': {
'var2': 'var2',
'var3': 'var3'
}
}
nose.tools.ok_(
self.zmqd.targets == expected_targets,
"{} did not match {}".format(expected_targets, self.zmqd.targets))
def entrypoint(args=None):
"""
Main callable for "bonobo" entrypoint.
Will load commands from "bonobo.commands" entrypoints, using stevedore.
"""
mondrian.setup(excepthook=True)
logger = logging.getLogger()
logger.setLevel(settings.LOGGING_LEVEL.get())
parser = argparse.ArgumentParser()
parser.add_argument("--debug", "-D", action="store_true")
subparsers = parser.add_subparsers(dest="command")
subparsers.required = True
commands = {}
def register_extension(ext):
nonlocal commands
try:
parser = subparsers.add_parser(ext.name)
if isinstance(ext.plugin, type) and issubclass(
ext.plugin, BaseCommand):
# current way, class based.
cmd = ext.plugin()
cmd.add_arguments(parser)
cmd.__name__ = ext.name
commands[ext.name] = cmd.handle
else:
# old school, function based.
commands[ext.name] = ext.plugin(parser)
except Exception:
logger.exception("Error while loading command {}.".format(
ext.name))
from stevedore import ExtensionManager
mgr = ExtensionManager(namespace="bonobo.commands")
mgr.map(register_extension)
parsed_args = parser.parse_args(args).__dict__
if parsed_args.pop("debug", False):
settings.DEBUG.set(True)
settings.LOGGING_LEVEL.set(logging.DEBUG)
logger.setLevel(settings.LOGGING_LEVEL.get())
logger.debug("Command: " + parsed_args["command"] + " Arguments: " +
repr(parsed_args))
# Get command handler, execute, rince.
command = commands[parsed_args.pop("command")]
command(**parsed_args)
return 0
def load_feature_extensions():
if not _all_features:
def register_feature(ext, all_features=_all_features):
all_features[ext.name] = ext.plugin
mgr = ExtensionManager(namespace="medikit.feature")
mgr.map(register_feature)
return _all_features
def __init__(self):
self._configs = {}
self._features = {}
self._pipelines = {}
def register_feature(ext):
self._features[ext.name] = ext.plugin
mgr = ExtensionManager(namespace='medikit.feature')
mgr.map(register_feature)
def get_event_target_plugin_help():
ret = ''
mgr = ExtensionManager(namespace='es_logger.plugins.event_target',
invoke_on_load=False)
for target_plugin in mgr.names(): # All known target plugins
drv = driver.DriverManager(
namespace='es_logger.plugins.event_target',
invoke_on_load=False,
name=target_plugin)
ret = ret + '\n{}'.format(drv.driver.get_help_string())
return ret
def __init__(self):
self._configs = {}
self._features = {}
self._pipelines = {}
self._variables = OrderedDict()
def register_feature(ext):
self._features[ext.name] = ext.plugin
def on_load_feature_failure(mgr, entrypoint, err):
logger.exception("Exception caught while loading {}.".format(entrypoint), err)
mgr = ExtensionManager(namespace="medikit.feature", on_load_failure_callback=on_load_feature_failure)
mgr.map(register_feature)
def test_manager_should_call_all(self):
em = ExtensionManager.make_test_instance([test_extension2,
test_extension])
func = Mock()
em.map(func)
func.assert_any_call(test_extension2)
func.assert_any_call(test_extension)
def test_manager_should_eat_exceptions(self):
em = ExtensionManager.make_test_instance([test_extension])
func = Mock(side_effect=RuntimeError('hard coded error'))
results = em.map(func, 1, 2, a='A', b='B')
self.assertEqual(results, [])
def find_plugins(self) -> List[PluginSpec]:
from stevedore import ExtensionManager
from stevedore.exception import NoMatches
manager = ExtensionManager(
self.namespace,
invoke_on_load=False,
on_load_failure_callback=self._on_load_failure_callback,
)
# creates for each loadable stevedore extension a PluginSpec
try:
return manager.map(self.to_plugin_spec)
except NoMatches:
LOG.debug("no extensions found in namespace %s", self.namespace)
return []
def list_plugins(data_only=False,
namespaces=[
'gather_build_data', 'console_log_processor',
'event_generator',
'event_generator.console_log_events', 'event_target'
]):
def print_plugin(ext):
print("\t{}".format(ext.entry_point.name))
plugins = {}
for namespace in namespaces:
plugins[namespace] = ExtensionManager(
namespace='es_logger.plugins.' + namespace,
invoke_on_load=False)
if data_only:
ret_list = []
for namespace in namespaces:
ret_list += plugins[namespace].names()
return ret_list
for namespace in namespaces:
print("{}:".format('es_logger.plugins.' + namespace))
if len(plugins[namespace].names()) > 0:
plugins[namespace].map(print_plugin)
else:
print("\tNone found")
def load_var_adapters():
"""Loading a dictlet finder extension.
Returns:
ExtensionManager: the extension manager holding the extensions
"""
log2 = logging.getLogger("stevedore")
out_hdlr = logging.StreamHandler(sys.stdout)
out_hdlr.setFormatter(
logging.Formatter("load adapter plugin error -> %(message)s"))
out_hdlr.setLevel(logging.DEBUG)
log2.addHandler(out_hdlr)
log2.setLevel(logging.INFO)
log.debug("Loading var adapters...")
mgr = ExtensionManager(
namespace="freckles.var_adapters",
invoke_on_load=True,
propagate_map_exceptions=True,
)
result = {}
for plugin in mgr:
name = plugin.name
ep = plugin.entry_point
adapter = ep.load()
result[name] = adapter()
return result
def test_manager_should_call():
em = ExtensionManager.make_test_instance([test_extension])
func = Mock()
em.map(func)
func.assert_called_once_with(test_extension)
def test_manager_return_values():
def mapped(ext, *args, **kwds):
return ext.name
em = ExtensionManager.make_test_instance([test_extension2, test_extension])
results = em.map(mapped)
assert sorted(results) == ['another_one', 'test_extension']
def test_manager_should_call():
em = ExtensionManager.make_test_instance([test_extension])
func = Mock()
em.map(func)
func.assert_called_once_with(test_extension)
def test_manager_should_call_all(self):
em = ExtensionManager.make_test_instance(
[test_extension2, test_extension])
func = Mock()
em.map(func)
func.assert_any_call(test_extension2)
func.assert_any_call(test_extension)
def test_manager_should_eat_exceptions(self):
em = ExtensionManager.make_test_instance([test_extension])
func = Mock(side_effect=RuntimeError('hard coded error'))
results = em.map(func, 1, 2, a='A', b='B')
self.assertEqual(results, [])
def load_plugins():
"""Load PyFibre plugins via Stevedore. """
mgr = ExtensionManager(namespace='pyfibre.plugins', invoke_on_load=True)
plugins = [ext.obj for ext in mgr]
return plugins
def test_manager_return_values(self):
def mapped(ext, *args, **kwds):
return ext.name
em = ExtensionManager.make_test_instance([test_extension2,
test_extension])
results = em.map(mapped)
self.assertEqual(sorted(results), ['another_one', 'test_extension'])
def _load_extensions(namespace: str) -> None:
global MANAGERS
logger.debug("creating manager for {0}".format(namespace))
MANAGERS[namespace] = ExtensionManager(
namespace=namespace,
invoke_on_load=False,
verify_requirements=True,
propagate_map_exceptions=True,
on_load_failure_callback=stevedore_error_handler)
def test_register_plugin():
namespace = "roast.serial"
name = "dummy_serial"
register_plugin(name, "serial", "tests.test_plugin:DummySerial")
e = ExtensionManager(namespace)
serial_entries = [
entry_point.name for entry_point in e.ENTRY_POINT_CACHE.get(namespace)
]
assert name in serial_entries
namespace = "roast.board"
name = "dummy_board"
register_plugin(name, "board", "tests.test_plugin:DummyBoard")
e = ExtensionManager(namespace)
board_entries = [
entry_point.name for entry_point in e.ENTRY_POINT_CACHE.get(namespace)
]
assert name in board_entries
def get_interceptor(name):
"""Get an interceptor by name.
Attrs:
name: The name of the plugin containing the interceptor.
Returns: A callable object which must accept a single `WorkDB` argument.
"""
return ExtensionManager('cosmic_ray.interceptors')[name].plugin
def interceptor_names():
"""Get all interceptor plugin names.
Returns: A sequence of interceptor plugin names.
"""
return ExtensionManager(
'cosmic_ray.interceptors',
on_load_failure_callback=_log_extension_loading_failure,
).names()
def test_runner_names():
"""Get all test-runner plugin names.
Returns: A sequence of test-runner plugin names.
"""
return ExtensionManager(
'cosmic_ray.test_runners',
on_load_failure_callback=_log_extension_loading_failure,
).names()
def execution_engine_names():
"""Get all execution-engine plugin names.
Returns: A sequence of execution-engine names.
"""
return ExtensionManager(
'cosmic_ray.execution_engines',
on_load_failure_callback=_log_extension_loading_failure,
).names()
def distributor_names():
"""Get all distributor plugin names.
Returns:
A sequence of distributor names.
"""
return ExtensionManager(
"cosmic_ray.distributors",
on_load_failure_callback=_log_extension_loading_failure,
).names()
def get_operator(name):
"""Get an operator class from a plugin.
Attrs:
name: The name of the plugin containing the operator class.
Returns: The operator *class object* (i.e. not an instance) provided by the
plugin named `name`.
"""
return ExtensionManager('cosmic_ray.operators')[name].plugin
def entrypoint(args=None):
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest='command')
subparsers.required = True
commands = {}
def register_extension(ext, commands=commands):
try:
parser = subparsers.add_parser(ext.name)
commands[ext.name] = ext.plugin(parser)
except Exception:
logging.exception('Error while loading command {}.'.format(ext.name))
mgr = ExtensionManager(
namespace='bonobo.commands',
)
mgr.map(register_extension)
args = parser.parse_args(args).__dict__
commands[args.pop('command')](**args)
def __init__(self, dispatcher: EventDispatcher):
self._configs = {}
self._features = {}
self._pipelines = {}
self._variables = OrderedDict()
self.dispatcher = dispatcher
self.resources = OrderedDict()
def register_feature(ext):
self._features[ext.name] = ext.plugin
def on_load_feature_failure(mgr, entrypoint, err):
logger.exception(
"Exception caught while loading {}.".format(entrypoint), err)
mgr = ExtensionManager(
namespace="medikit.feature",
on_load_failure_callback=on_load_feature_failure)
mgr.map(register_feature)
dispatcher.add_listener(medikit.on_end, self.write_resources)
def run_hooks(hook, options, args, output=None):
log = logging.getLogger('virtualenvwrapper.hook_loader')
if output is None:
output = sys.stdout
namespace = 'virtualenvwrapper.%s' % hook
if options.names:
log.debug('looking for %s hooks %s' % (namespace, options.names))
hook_mgr = NamedExtensionManager(namespace, options.names)
else:
log.debug('looking for %s hooks' % namespace)
hook_mgr = ExtensionManager(namespace)
if options.listing:
def show(ext):
output.write(' %-10s -- %s\n' %
(ext.name, inspect.getdoc(ext.plugin) or ''))
try:
hook_mgr.map(show)
except RuntimeError: # no templates
output.write(' No templates installed.\n')
elif options.sourcing:
def get_source(ext, args):
# Show the shell commands so they can
# be run in the calling shell.
log.debug('getting source instructions for %s' % ext.name)
contents = (ext.plugin(args) or '').strip()
if contents:
output.write('# %s\n' % ext.name)
output.write(contents)
output.write("\n")
try:
hook_mgr.map(get_source, args[1:])
except RuntimeError:
pass
else:
# Just run the plugin ourselves
def invoke(ext, args):
log.debug('running %s' % ext.name)
ext.plugin(args)
try:
hook_mgr.map(invoke, args[1:])
except RuntimeError:
pass
def register_plugin(name, plugin_type, entry_point) -> None:
"""Registers a plugin dynamically without needing to install as a package.
Args:
name (str): Name of plugin to be referenced.
plugin_type (str): Type to determine plugin namespace.
entry_point (str): Entry point in the form: some.module:some.attr
Raises:
Exception: Raised when plugin_type is not supported.
"""
if plugin_type == "system":
namespace = "roast.component.system"
elif plugin_type == "testsuite":
namespace = "roast.component.testsuite"
elif plugin_type == "serial":
namespace = "roast.serial"
elif plugin_type == "board":
namespace = "roast.board"
elif plugin_type == "relay":
namespace = "roast.relay"
else:
err_msg = f"Plugin type {plugin_type} is not supported."
log.error(err_msg)
raise Exception(err_msg)
ep = importlib_metadata.EntryPoint(name, entry_point, namespace)
e = ExtensionManager(namespace)
if namespace in e.ENTRY_POINT_CACHE:
entry_points = e.ENTRY_POINT_CACHE.get(namespace)
if name not in [entry_point.name for entry_point in entry_points]:
entry_points.append(ep)
e.ENTRY_POINT_CACHE[namespace] = entry_points
else:
e.ENTRY_POINT_CACHE[namespace] = [ep]
ep.load()
def job_runner(self):
# Lazily import this since this module will be loaded on hadoop worker nodes however stevedore will not be
# available in that environment.
from stevedore import ExtensionManager
extension_manager = ExtensionManager('mapreduce.engine')
try:
engine_class = extension_manager[self.mapreduce_engine].plugin
except KeyError:
raise KeyError('A map reduce engine must be specified in order to run MapReduceJobTasks')
if issubclass(engine_class, MapReduceJobRunner):
engine_kwargs = self._get_engine_parameters_from_targets()
return engine_class(**engine_kwargs)
else:
return engine_class()
def run_hooks(hook, options, args, output=None):
if output is None:
output = sys.stdout
namespace = 'virtualenvwrapper.%s' % hook
if options.names:
hook_mgr = NamedExtensionManager(namespace, options.names)
else:
hook_mgr = ExtensionManager(namespace)
if options.listing:
def show(ext):
output.write(' %-10s -- %s\n' %
(ext.name, inspect.getdoc(ext.plugin) or ''))
hook_mgr.map(show)
elif options.sourcing:
def get_source(ext, args):
# Show the shell commands so they can
# be run in the calling shell.
contents = (ext.plugin(args) or '').strip()
if contents:
output.write('# %s\n' % ext.name)
output.write(contents)
output.write("\n")
try:
hook_mgr.map(get_source, args[1:])
except RuntimeError:
pass
else:
# Just run the plugin ourselves
def invoke(ext, args):
ext.plugin(args)
try:
hook_mgr.map(invoke, args[1:])
except RuntimeError:
pass
def load_from_pkg_resources(namespace):
"""Discover ExtensionPoints using pkg_resources.
This uses pkg_resources to scan the entry-points for `namespace`. Each dotted entrypoint name should have two parts.
The first will be `namespace` and the second will be assumed to be the name of the entry point to which it belongs.
Each discovered entry point should be a callable that takes a single argument of an ExtensionPoint. The extension
should populate the ExtensionPoint appropriately, e.g. by calling `ExtensionPoint.add()`.
Args:
namespace: The pkg_resources namespace to scan.
Returns: An iterable of ExtensionPoints.
"""
import pkg_resources
from stevedore import ExtensionManager
for entry_point_name in pkg_resources.get_entry_map(namespace):
toks = entry_point_name.split('.')
if len(toks) == 1:
# This happens for e.g. console_scripts
continue
# TODO: Do we need to extend for the case of an entry_point_name with more than two tokens? We could return a
# tree of ExtensionPoint instead of just a flat mapping.
assert toks[0] == namespace
extension_point = ExtensionPoint(toks[1])
manager = ExtensionManager(
'.'.join(toks),
on_load_failure_callback=_log_extension_loading_failure)
for extension in manager:
extension.plugin(extension_point)
yield extension_point
def test_instance_should_use_supplied_namespace():
namespace = 'testing.1.2.3'
em = ExtensionManager.make_test_instance([], namespace=namespace)
assert namespace == em.namespace
def test_instance_should_have_default_namespace(self):
em = ExtensionManager.make_test_instance([])
self.assertEqual(em.namespace, 'TESTING')
def test_instance_should_use_supplied_namespace(self):
namespace = 'testing.1.2.3'
em = ExtensionManager.make_test_instance([], namespace=namespace)
self.assertEqual(namespace, em.namespace)
def test_extension_name_should_be_listed(self):
em = ExtensionManager.make_test_instance([test_extension])
self.assertIn(test_extension.name, em.names())
def test_manager_should_allow_name_access(self):
em = ExtensionManager.make_test_instance([test_extension])
self.assertEqual(test_extension, em[test_extension.name])
def test_manager_should_allow_name_access():
em = ExtensionManager.make_test_instance([test_extension])
assert test_extension == em[test_extension.name]
def test_instance_should_use_supplied_extensions():
extensions = [test_extension, test_extension2]
em = ExtensionManager.make_test_instance(extensions)
assert extensions == em.extensions
def test_instance_should_have_default_namespace():
em = ExtensionManager.make_test_instance([])
assert em.namespace
def test_iterator_should_yield_extension():
em = ExtensionManager.make_test_instance([test_extension])
assert test_extension == next(iter(em))
def test_manager_should_propagate_exceptions(self):
em = ExtensionManager.make_test_instance([test_extension],
propagate_map_exceptions=True)
self.skipTest('Skipping temporarily')
func = Mock(side_effect=RuntimeError('hard coded error'))
em.map(func, 1, 2, a='A', b='B')
def test_instance_should_use_supplied_extensions(self):
extensions = [test_extension, test_extension2]
em = ExtensionManager.make_test_instance(extensions)
self.assertEqual(extensions, em.extensions)
def test_extension_name_should_be_listed():
em = ExtensionManager.make_test_instance([test_extension])
assert test_extension.name in em.names()
def test_iterator_should_yield_extension(self):
em = ExtensionManager.make_test_instance([test_extension])
self.assertEqual(test_extension, next(iter(em)))
def test_manager_should_propagate_exceptions():
em = ExtensionManager.make_test_instance([test_extension],
propagate_map_exceptions=True)
func = Mock(side_effect=RuntimeError('hard coded error'))
em.map(func, 1, 2, a='A', b='B')
