def list_opts():
    """Returns a list of oslo.config options available in the library.

    The returned list includes all oslo.config options which may be registered
    at runtime by the library.

    Each element of the list is a tuple. The first element is the name of the
    group under which the list of elements in the second element will be
    registered. A group name of None corresponds to the [DEFAULT] group in
    config files.

    The purpose of this is to allow tools like the Oslo sample config file
    generator to discover the options exposed to users by this library.

    :returns: a list of (group_name, opts) tuples
    """
    key_manager_opts = []
    key_manager_opts.extend(key_manager.key_manager_opts)
    key_manager_opts.extend(utils.credential_opts)
    opts = [('key_manager', key_manager_opts)]

    ext_mgr = ExtensionManager("castellan.drivers",
                               invoke_on_load=True,
                               invoke_args=[cfg.CONF])

    for driver in ext_mgr.names():
        opts.extend(ext_mgr[driver].obj.list_options_for_discovery())

    return opts
Beispiel #2
0
def entrypoint(args=None):
    parser = argparse.ArgumentParser()
    parser.add_argument('--debug', '-D', action='store_true')

    subparsers = parser.add_subparsers(dest='command')
    subparsers.required = True

    commands = {}

    def register_extension(ext, commands=commands):
        try:
            parser = subparsers.add_parser(ext.name)
            commands[ext.name] = ext.plugin(parser)
        except Exception:
            logger.exception('Error while loading command {}.'.format(
                ext.name))

    from stevedore import ExtensionManager
    mgr = ExtensionManager(namespace='bonobo.commands')
    mgr.map(register_extension)

    args = parser.parse_args(args).__dict__
    if args.pop('debug', False):
        settings.DEBUG = True
        settings.LOGGING_LEVEL = logging.DEBUG
        logging.set_level(settings.LOGGING_LEVEL)

    logger.debug('Command: ' + args['command'] + ' Arguments: ' + repr(args))
    commands[args.pop('command')](**args)
Beispiel #3
0
    def test_zmq_client_configure_eslogger(self):
        dummy_ep = importlib.metadata.EntryPoint(
            'dummy', 'test.test_plugins:DummyEventTarget',
            'es_logger.plugins.event_target')
        dummy2_ep = importlib.metadata.EntryPoint(
            'dummy2', 'test.test_plugins:DummyEventTarget',
            'es_logger.plugins.event_target')
        ExtensionManager.ENTRY_POINT_CACHE = {
            'es_logger.plugins.event_target': [dummy_ep, dummy2_ep]
        }
        with unittest.mock.patch(
                'configparser.ConfigParser') as mock_config_parser:
            config = self.config_setup(mock_config_parser)
            self.set_default_config(config)
            self.set_eslogger_targets_config(config)
            self.zmqd.configure()
        # Reset stevedore
        ExtensionManager.ENTRY_POINT_CACHE = {}
        mgr = ExtensionManager(namespace='es_logger.plugins.event_target',
                               invoke_on_load=False)
        mgr.names()

        # Perform validation after resetting stevedore state
        expected_targets = {
            'dummy': {
                'var1': 'var1'
            },
            'dummy2': {
                'var2': 'var2',
                'var3': 'var3'
            }
        }
        nose.tools.ok_(
            self.zmqd.targets == expected_targets,
            "{} did not match {}".format(expected_targets, self.zmqd.targets))
Beispiel #4
0
def entrypoint(args=None):
    """
    Main callable for "bonobo" entrypoint.

    Will load commands from "bonobo.commands" entrypoints, using stevedore.

    """

    mondrian.setup(excepthook=True)
    logger = logging.getLogger()
    logger.setLevel(settings.LOGGING_LEVEL.get())

    parser = argparse.ArgumentParser()
    parser.add_argument("--debug", "-D", action="store_true")

    subparsers = parser.add_subparsers(dest="command")
    subparsers.required = True

    commands = {}

    def register_extension(ext):
        nonlocal commands

        try:
            parser = subparsers.add_parser(ext.name)
            if isinstance(ext.plugin, type) and issubclass(
                    ext.plugin, BaseCommand):
                # current way, class based.
                cmd = ext.plugin()
                cmd.add_arguments(parser)
                cmd.__name__ = ext.name
                commands[ext.name] = cmd.handle
            else:
                # old school, function based.
                commands[ext.name] = ext.plugin(parser)
        except Exception:
            logger.exception("Error while loading command {}.".format(
                ext.name))

    from stevedore import ExtensionManager

    mgr = ExtensionManager(namespace="bonobo.commands")
    mgr.map(register_extension)

    parsed_args = parser.parse_args(args).__dict__

    if parsed_args.pop("debug", False):
        settings.DEBUG.set(True)
        settings.LOGGING_LEVEL.set(logging.DEBUG)
        logger.setLevel(settings.LOGGING_LEVEL.get())

    logger.debug("Command: " + parsed_args["command"] + " Arguments: " +
                 repr(parsed_args))

    # Get command handler, execute, rince.
    command = commands[parsed_args.pop("command")]
    command(**parsed_args)

    return 0
Beispiel #5
0
def load_feature_extensions():
    if not _all_features:

        def register_feature(ext, all_features=_all_features):
            all_features[ext.name] = ext.plugin

        mgr = ExtensionManager(namespace="medikit.feature")
        mgr.map(register_feature)

    return _all_features
Beispiel #6
0
    def __init__(self):
        self._configs = {}
        self._features = {}
        self._pipelines = {}

        def register_feature(ext):
            self._features[ext.name] = ext.plugin

        mgr = ExtensionManager(namespace='medikit.feature')
        mgr.map(register_feature)
Beispiel #7
0
 def get_event_target_plugin_help():
     ret = ''
     mgr = ExtensionManager(namespace='es_logger.plugins.event_target',
                            invoke_on_load=False)
     for target_plugin in mgr.names():  # All known target plugins
         drv = driver.DriverManager(
             namespace='es_logger.plugins.event_target',
             invoke_on_load=False,
             name=target_plugin)
         ret = ret + '\n{}'.format(drv.driver.get_help_string())
     return ret
Beispiel #8
0
    def __init__(self):
        self._configs = {}
        self._features = {}
        self._pipelines = {}
        self._variables = OrderedDict()

        def register_feature(ext):
            self._features[ext.name] = ext.plugin

        def on_load_feature_failure(mgr, entrypoint, err):
            logger.exception("Exception caught while loading {}.".format(entrypoint), err)

        mgr = ExtensionManager(namespace="medikit.feature", on_load_failure_callback=on_load_feature_failure)
        mgr.map(register_feature)
 def test_manager_should_call_all(self):
     em = ExtensionManager.make_test_instance([test_extension2,
                                               test_extension])
     func = Mock()
     em.map(func)
     func.assert_any_call(test_extension2)
     func.assert_any_call(test_extension)
    def test_manager_should_eat_exceptions(self):
        em = ExtensionManager.make_test_instance([test_extension])

        func = Mock(side_effect=RuntimeError('hard coded error'))

        results = em.map(func, 1, 2, a='A', b='B')
        self.assertEqual(results, [])
Beispiel #11
0
    def find_plugins(self) -> List[PluginSpec]:
        from stevedore import ExtensionManager
        from stevedore.exception import NoMatches

        manager = ExtensionManager(
            self.namespace,
            invoke_on_load=False,
            on_load_failure_callback=self._on_load_failure_callback,
        )

        # creates for each loadable stevedore extension a PluginSpec
        try:
            return manager.map(self.to_plugin_spec)
        except NoMatches:
            LOG.debug("no extensions found in namespace %s", self.namespace)
            return []
Beispiel #12
0
    def list_plugins(data_only=False,
                     namespaces=[
                         'gather_build_data', 'console_log_processor',
                         'event_generator',
                         'event_generator.console_log_events', 'event_target'
                     ]):
        def print_plugin(ext):
            print("\t{}".format(ext.entry_point.name))

        plugins = {}
        for namespace in namespaces:
            plugins[namespace] = ExtensionManager(
                namespace='es_logger.plugins.' + namespace,
                invoke_on_load=False)
        if data_only:
            ret_list = []
            for namespace in namespaces:
                ret_list += plugins[namespace].names()
            return ret_list

        for namespace in namespaces:
            print("{}:".format('es_logger.plugins.' + namespace))
            if len(plugins[namespace].names()) > 0:
                plugins[namespace].map(print_plugin)
            else:
                print("\tNone found")
Beispiel #13
0
def load_var_adapters():
    """Loading a dictlet finder extension.

    Returns:
      ExtensionManager: the extension manager holding the extensions
    """

    log2 = logging.getLogger("stevedore")
    out_hdlr = logging.StreamHandler(sys.stdout)
    out_hdlr.setFormatter(
        logging.Formatter("load adapter plugin error -> %(message)s"))
    out_hdlr.setLevel(logging.DEBUG)
    log2.addHandler(out_hdlr)
    log2.setLevel(logging.INFO)

    log.debug("Loading var adapters...")

    mgr = ExtensionManager(
        namespace="freckles.var_adapters",
        invoke_on_load=True,
        propagate_map_exceptions=True,
    )

    result = {}
    for plugin in mgr:
        name = plugin.name
        ep = plugin.entry_point
        adapter = ep.load()
        result[name] = adapter()

    return result
def test_manager_should_call():
    em = ExtensionManager.make_test_instance([test_extension])
    func = Mock()

    em.map(func)

    func.assert_called_once_with(test_extension)
def test_manager_return_values():
    def mapped(ext, *args, **kwds):
        return ext.name

    em = ExtensionManager.make_test_instance([test_extension2, test_extension])
    results = em.map(mapped)
    assert sorted(results) == ['another_one', 'test_extension']
def test_manager_should_call():
    em = ExtensionManager.make_test_instance([test_extension])
    func = Mock()

    em.map(func)

    func.assert_called_once_with(test_extension)
 def test_manager_should_call_all(self):
     em = ExtensionManager.make_test_instance(
         [test_extension2, test_extension])
     func = Mock()
     em.map(func)
     func.assert_any_call(test_extension2)
     func.assert_any_call(test_extension)
    def test_manager_should_eat_exceptions(self):
        em = ExtensionManager.make_test_instance([test_extension])

        func = Mock(side_effect=RuntimeError('hard coded error'))

        results = em.map(func, 1, 2, a='A', b='B')
        self.assertEqual(results, [])
Beispiel #19
0
def load_plugins():
    """Load PyFibre plugins via Stevedore. """

    mgr = ExtensionManager(namespace='pyfibre.plugins', invoke_on_load=True)

    plugins = [ext.obj for ext in mgr]

    return plugins
    def test_manager_return_values(self):
        def mapped(ext, *args, **kwds):
            return ext.name

        em = ExtensionManager.make_test_instance([test_extension2,
                                                  test_extension])
        results = em.map(mapped)
        self.assertEqual(sorted(results), ['another_one', 'test_extension'])
Beispiel #21
0
def _load_extensions(namespace: str) -> None:
    global MANAGERS
    logger.debug("creating manager for {0}".format(namespace))
    MANAGERS[namespace] = ExtensionManager(
        namespace=namespace,
        invoke_on_load=False,
        verify_requirements=True,
        propagate_map_exceptions=True,
        on_load_failure_callback=stevedore_error_handler)
Beispiel #22
0
def test_register_plugin():
    namespace = "roast.serial"
    name = "dummy_serial"
    register_plugin(name, "serial", "tests.test_plugin:DummySerial")
    e = ExtensionManager(namespace)
    serial_entries = [
        entry_point.name for entry_point in e.ENTRY_POINT_CACHE.get(namespace)
    ]
    assert name in serial_entries

    namespace = "roast.board"
    name = "dummy_board"
    register_plugin(name, "board", "tests.test_plugin:DummyBoard")
    e = ExtensionManager(namespace)
    board_entries = [
        entry_point.name for entry_point in e.ENTRY_POINT_CACHE.get(namespace)
    ]
    assert name in board_entries
Beispiel #23
0
def get_interceptor(name):
    """Get an interceptor by name.

    Attrs:
        name: The name of the plugin containing the interceptor.

    Returns: A callable object which must accept a single `WorkDB` argument.
    """
    return ExtensionManager('cosmic_ray.interceptors')[name].plugin
Beispiel #24
0
def interceptor_names():
    """Get all interceptor plugin names.

    Returns: A sequence of interceptor plugin names.
    """
    return ExtensionManager(
        'cosmic_ray.interceptors',
        on_load_failure_callback=_log_extension_loading_failure,
    ).names()
Beispiel #25
0
def test_runner_names():
    """Get all test-runner plugin names.

    Returns: A sequence of test-runner plugin names.
    """
    return ExtensionManager(
        'cosmic_ray.test_runners',
        on_load_failure_callback=_log_extension_loading_failure,
    ).names()
Beispiel #26
0
def execution_engine_names():
    """Get all execution-engine plugin names.

    Returns: A sequence of execution-engine names.
    """
    return ExtensionManager(
        'cosmic_ray.execution_engines',
        on_load_failure_callback=_log_extension_loading_failure,
    ).names()
Beispiel #27
0
def distributor_names():
    """Get all distributor plugin names.

    Returns:
        A sequence of distributor names.
    """
    return ExtensionManager(
        "cosmic_ray.distributors",
        on_load_failure_callback=_log_extension_loading_failure,
    ).names()
Beispiel #28
0
def get_operator(name):
    """Get an operator class from a plugin.

    Attrs:
        name: The name of the plugin containing the operator class.

    Returns: The operator *class object* (i.e. not an instance) provided by the
        plugin named `name`.
    """
    return ExtensionManager('cosmic_ray.operators')[name].plugin
Beispiel #29
0
def entrypoint(args=None):
    parser = argparse.ArgumentParser()

    subparsers = parser.add_subparsers(dest='command')
    subparsers.required = True

    commands = {}

    def register_extension(ext, commands=commands):
        try:
            parser = subparsers.add_parser(ext.name)
            commands[ext.name] = ext.plugin(parser)
        except Exception:
            logging.exception('Error while loading command {}.'.format(ext.name))

    mgr = ExtensionManager(
        namespace='bonobo.commands',
    )
    mgr.map(register_extension)

    args = parser.parse_args(args).__dict__
    commands[args.pop('command')](**args)
Beispiel #30
0
    def __init__(self, dispatcher: EventDispatcher):
        self._configs = {}
        self._features = {}
        self._pipelines = {}
        self._variables = OrderedDict()

        self.dispatcher = dispatcher
        self.resources = OrderedDict()

        def register_feature(ext):
            self._features[ext.name] = ext.plugin

        def on_load_feature_failure(mgr, entrypoint, err):
            logger.exception(
                "Exception caught while loading {}.".format(entrypoint), err)

        mgr = ExtensionManager(
            namespace="medikit.feature",
            on_load_failure_callback=on_load_feature_failure)
        mgr.map(register_feature)

        dispatcher.add_listener(medikit.on_end, self.write_resources)
Beispiel #31
0
def run_hooks(hook, options, args, output=None):
    log = logging.getLogger('virtualenvwrapper.hook_loader')
    if output is None:
        output = sys.stdout

    namespace = 'virtualenvwrapper.%s' % hook
    if options.names:
        log.debug('looking for %s hooks %s' % (namespace, options.names))
        hook_mgr = NamedExtensionManager(namespace, options.names)
    else:
        log.debug('looking for %s hooks' % namespace)
        hook_mgr = ExtensionManager(namespace)

    if options.listing:

        def show(ext):
            output.write('  %-10s -- %s\n' %
                         (ext.name, inspect.getdoc(ext.plugin) or ''))

        try:
            hook_mgr.map(show)
        except RuntimeError:  # no templates
            output.write('  No templates installed.\n')

    elif options.sourcing:

        def get_source(ext, args):
            # Show the shell commands so they can
            # be run in the calling shell.
            log.debug('getting source instructions for %s' % ext.name)
            contents = (ext.plugin(args) or '').strip()
            if contents:
                output.write('# %s\n' % ext.name)
                output.write(contents)
                output.write("\n")

        try:
            hook_mgr.map(get_source, args[1:])
        except RuntimeError:
            pass

    else:
        # Just run the plugin ourselves
        def invoke(ext, args):
            log.debug('running %s' % ext.name)
            ext.plugin(args)

        try:
            hook_mgr.map(invoke, args[1:])
        except RuntimeError:
            pass
Beispiel #32
0
def register_plugin(name, plugin_type, entry_point) -> None:
    """Registers a plugin dynamically without needing to install as a package.

    Args:
        name (str): Name of plugin to be referenced.
        plugin_type (str): Type to determine plugin namespace.
        entry_point (str): Entry point in the form: some.module:some.attr

    Raises:
        Exception: Raised when plugin_type is not supported.
    """
    if plugin_type == "system":
        namespace = "roast.component.system"
    elif plugin_type == "testsuite":
        namespace = "roast.component.testsuite"
    elif plugin_type == "serial":
        namespace = "roast.serial"
    elif plugin_type == "board":
        namespace = "roast.board"
    elif plugin_type == "relay":
        namespace = "roast.relay"
    else:
        err_msg = f"Plugin type {plugin_type} is not supported."
        log.error(err_msg)
        raise Exception(err_msg)

    ep = importlib_metadata.EntryPoint(name, entry_point, namespace)
    e = ExtensionManager(namespace)
    if namespace in e.ENTRY_POINT_CACHE:
        entry_points = e.ENTRY_POINT_CACHE.get(namespace)
        if name not in [entry_point.name for entry_point in entry_points]:
            entry_points.append(ep)
            e.ENTRY_POINT_CACHE[namespace] = entry_points
    else:
        e.ENTRY_POINT_CACHE[namespace] = [ep]
    ep.load()
    def job_runner(self):
        # Lazily import this since this module will be loaded on hadoop worker nodes however stevedore will not be
        # available in that environment.
        from stevedore import ExtensionManager

        extension_manager = ExtensionManager('mapreduce.engine')
        try:
            engine_class = extension_manager[self.mapreduce_engine].plugin
        except KeyError:
            raise KeyError('A map reduce engine must be specified in order to run MapReduceJobTasks')

        if issubclass(engine_class, MapReduceJobRunner):
            engine_kwargs = self._get_engine_parameters_from_targets()
            return engine_class(**engine_kwargs)
        else:
            return engine_class()
Beispiel #34
0
def run_hooks(hook, options, args, output=None):
    if output is None:
        output = sys.stdout

    namespace = 'virtualenvwrapper.%s' % hook
    if options.names:
        hook_mgr = NamedExtensionManager(namespace, options.names)
    else:
        hook_mgr = ExtensionManager(namespace)

    if options.listing:

        def show(ext):
            output.write('  %-10s -- %s\n' %
                         (ext.name, inspect.getdoc(ext.plugin) or ''))

        hook_mgr.map(show)

    elif options.sourcing:

        def get_source(ext, args):
            # Show the shell commands so they can
            # be run in the calling shell.
            contents = (ext.plugin(args) or '').strip()
            if contents:
                output.write('# %s\n' % ext.name)
                output.write(contents)
                output.write("\n")

        try:
            hook_mgr.map(get_source, args[1:])
        except RuntimeError:
            pass

    else:
        # Just run the plugin ourselves
        def invoke(ext, args):
            ext.plugin(args)

        try:
            hook_mgr.map(invoke, args[1:])
        except RuntimeError:
            pass
Beispiel #35
0
def load_from_pkg_resources(namespace):
    """Discover ExtensionPoints using pkg_resources.

    This uses pkg_resources to scan the entry-points for `namespace`. Each dotted entrypoint name should have two parts.
    The first will be `namespace` and the second will be assumed to be the name of the entry point to which it belongs.

    Each discovered entry point should be a callable that takes a single argument of an ExtensionPoint. The extension
    should populate the ExtensionPoint appropriately, e.g. by calling `ExtensionPoint.add()`.

    Args:
        namespace: The pkg_resources namespace to scan.

    Returns: An iterable of ExtensionPoints.
    """
    import pkg_resources
    from stevedore import ExtensionManager

    for entry_point_name in pkg_resources.get_entry_map(namespace):
        toks = entry_point_name.split('.')
        if len(toks) == 1:
            # This happens for e.g. console_scripts
            continue

        # TODO: Do we need to extend for the case of an entry_point_name with more than two tokens? We could return a
        # tree of ExtensionPoint instead of just a flat mapping.

        assert toks[0] == namespace

        extension_point = ExtensionPoint(toks[1])

        manager = ExtensionManager(
            '.'.join(toks),
            on_load_failure_callback=_log_extension_loading_failure)

        for extension in manager:
            extension.plugin(extension_point)

        yield extension_point
def test_instance_should_use_supplied_namespace():
    namespace = 'testing.1.2.3'
    em = ExtensionManager.make_test_instance([], namespace=namespace)

    assert namespace == em.namespace
 def test_instance_should_have_default_namespace(self):
     em = ExtensionManager.make_test_instance([])
     self.assertEqual(em.namespace, 'TESTING')
 def test_instance_should_use_supplied_namespace(self):
     namespace = 'testing.1.2.3'
     em = ExtensionManager.make_test_instance([], namespace=namespace)
     self.assertEqual(namespace, em.namespace)
 def test_extension_name_should_be_listed(self):
     em = ExtensionManager.make_test_instance([test_extension])
     self.assertIn(test_extension.name, em.names())
 def test_manager_should_allow_name_access(self):
     em = ExtensionManager.make_test_instance([test_extension])
     self.assertEqual(test_extension, em[test_extension.name])
def test_manager_should_allow_name_access():
    em = ExtensionManager.make_test_instance([test_extension])

    assert test_extension == em[test_extension.name]
def test_instance_should_use_supplied_extensions():
    extensions = [test_extension, test_extension2]
    em = ExtensionManager.make_test_instance(extensions)

    assert extensions == em.extensions
def test_instance_should_have_default_namespace():
    em = ExtensionManager.make_test_instance([])

    assert em.namespace
def test_iterator_should_yield_extension():
    em = ExtensionManager.make_test_instance([test_extension])

    assert test_extension == next(iter(em))
 def test_manager_should_propagate_exceptions(self):
     em = ExtensionManager.make_test_instance([test_extension],
                                              propagate_map_exceptions=True)
     self.skipTest('Skipping temporarily')
     func = Mock(side_effect=RuntimeError('hard coded error'))
     em.map(func, 1, 2, a='A', b='B')
 def test_instance_should_use_supplied_extensions(self):
     extensions = [test_extension, test_extension2]
     em = ExtensionManager.make_test_instance(extensions)
     self.assertEqual(extensions, em.extensions)
def test_extension_name_should_be_listed():
    em = ExtensionManager.make_test_instance([test_extension])

    assert test_extension.name in em.names()
 def test_iterator_should_yield_extension(self):
     em = ExtensionManager.make_test_instance([test_extension])
     self.assertEqual(test_extension, next(iter(em)))
def test_manager_should_propagate_exceptions():
    em = ExtensionManager.make_test_instance([test_extension],
                                             propagate_map_exceptions=True)
    func = Mock(side_effect=RuntimeError('hard coded error'))

    em.map(func, 1, 2, a='A', b='B')