def test_extensions_listed_in_name_order(self):
# Since we don't know the "natural" order of the extensions, run
# the test both ways: if the sorting is broken, one of them will
# fail
em = named.NamedExtensionManager('stevedore.test.extension',
names=['t1', 't2'],
name_order=True)
actual = em.names()
self.assertEqual(actual, ['t1', 't2'])
em = named.NamedExtensionManager('stevedore.test.extension',
names=['t2', 't1'],
name_order=True)
actual = em.names()
self.assertEqual(actual, ['t2', 't1'])
def run(self):
# Delay startup so workers are jittered
time.sleep(self.startup_delay)
super(NotificationService, self).run()
self.managers = [ext.obj for ext in named.NamedExtensionManager(
namespace='ceilometer.notification.pipeline',
names=self.conf.notification.pipelines, invoke_on_load=True,
on_missing_entrypoints_callback=self._log_missing_pipeline,
invoke_args=(self.conf,))]
# FIXME(sileht): endpoint uses the notification_topics option
# and it should not because this is an oslo_messaging option
# not a ceilometer. Until we have something to get the
# notification_topics in another way, we must create a transport
# to ensure the option has been registered by oslo_messaging.
messaging.get_notifier(messaging.get_transport(self.conf), '')
endpoints = []
for pipe_mgr in self.managers:
endpoints.extend(pipe_mgr.get_main_endpoints())
targets = self.get_targets()
urls = self.conf.notification.messaging_urls or [None]
for url in urls:
transport = messaging.get_transport(self.conf, url)
# NOTE(gordc): ignore batching as we want pull
# to maintain sequencing as much as possible.
listener = messaging.get_batch_notification_listener(
transport, targets, endpoints, allow_requeue=True)
listener.start(
override_pool_size=self.conf.max_parallel_requests
)
self.listeners.append(listener)
def _configure_extensions(self):
"""
Configure the Stevedore NamedExtensionManager.
Raises:
ConfigurationException
"""
# These are the names of all of our configured extensions
configured_extension_names = self.extensions.keys()
# Load Stevedore extensions that we are configured for (and only those)
self.mgr = named.NamedExtensionManager(
names=configured_extension_names,
namespace='annotation_finder.searchers',
invoke_on_load=True,
on_load_failure_callback=self._plugin_load_failed_handler,
invoke_args=(self, self.echo),
)
# Output extension names listed in configuration
self.echo.echo_vv("Configured extension names: {}".format(" ".join(configured_extension_names)))
# Output found extension entry points from setup.py|cfg (whether or not they were loaded)
self.echo.echo_vv("Stevedore entry points found: {}".format(str(self.mgr.list_entry_points())))
# Output extensions that were actually able to load
self.echo.echo_v("Loaded extensions: {}".format(" ".join([x.name for x in self.mgr.extensions])))
if len(self.mgr.extensions) != len(configured_extension_names):
raise ConfigurationException('Not all configured extensions could be loaded! Asked for {} got {}.'.format(
configured_extension_names, self.mgr.extensions
))
def api_app(conf):
cfg.CONF(args=[], project='monasca')
log_levels = (cfg.CONF.default_log_levels)
cfg.set_defaults(log.log_opts, default_log_levels=log_levels)
log.setup('monasca')
dispatcher_manager = named.NamedExtensionManager(
namespace=DISPATCHER_NAMESPACE,
names=cfg.CONF.dispatcher,
invoke_on_load=True,
invoke_args=[cfg.CONF])
if not list(dispatcher_manager):
LOG.error('Failed to load any dispatchers for %s' %
DISPATCHER_NAMESPACE)
return None
# Create the application
app = resource_api.ResourceAPI()
# add each dispatcher to the application to serve requests offered by
# each dispatcher
for driver in dispatcher_manager:
app.add_route(None, driver.obj)
LOG.debug('Dispatcher drivers have been added to the routes!')
return app
def __init__(self,
transport,
publisher_id=None,
driver=None,
serializer=None,
retry=None,
topics=None):
"""Construct a Notifier object.
:param transport: the transport to use for sending messages
:type transport: oslo_messaging.Transport
:param publisher_id: field in notifications sent, for example
'compute.host1'
:type publisher_id: str
:param driver: a driver to lookup from oslo_messaging.notify.drivers
:type driver: str
:param serializer: an optional entity serializer
:type serializer: Serializer
:param retry: connection retries configuration (used by the messaging
driver):
None or -1 means to retry forever.
0 means no retry is attempted.
N means attempt at most N retries.
:type retry: int
:param topics: the topics which to send messages on
:type topics: list of strings
"""
conf = transport.conf
conf.register_opts(_notifier_opts,
group='oslo_messaging_notifications')
if not isinstance(transport, msg_transport.NotificationTransport):
_LOG.warning("Using RPC transport for notifications. Please use "
"get_notification_transport to obtain a "
"notification transport instance.")
self.transport = transport
self.publisher_id = publisher_id
if retry is not None:
self.retry = retry
else:
self.retry = conf.oslo_messaging_notifications.retry
self._driver_names = ([driver] if driver is not None else
conf.oslo_messaging_notifications.driver)
if topics is not None:
self._topics = topics
else:
self._topics = conf.oslo_messaging_notifications.topics
self._serializer = serializer or msg_serializer.NoOpSerializer()
self._driver_mgr = named.NamedExtensionManager(
'oslo.messaging.notify.drivers',
names=self._driver_names,
invoke_on_load=True,
invoke_args=[conf],
invoke_kwds={
'topics': self._topics,
'transport': self.transport,
})
def ems():
names = ['validate_salary', 'validate_duplicates']
mgr = named.NamedExtensionManager(namespace=f'pangadfs.validate',
names=names,
invoke_on_load=True,
name_order=True)
return {'validate': mgr}
def __init__(self):
threshold_rules = ('gnocchi_resources_threshold',
'gnocchi_aggregation_by_metrics_threshold',
'gnocchi_aggregation_by_resources_threshold')
CompositeRule.threshold_plugins = named.NamedExtensionManager(
"aodh.alarm.rule", threshold_rules)
super(CompositeRule, self).__init__()
def factory(global_config, **local_conf):
if not cfg.CONF['service:api'].enable_api_v1:
def disabled_app(environ, start_response):
status = '404 Not Found'
start_response(status, [])
return []
return disabled_app
app = flask.Flask('designate.api.v1')
app.request_class = DesignateRequest
app.json_encoder = JSONEncoder
app.config.update(
PROPAGATE_EXCEPTIONS=True
)
# Install custom converters (URL param varidators)
app.url_map.converters['uuid'] = UUIDConverter
# disable strict slashes. This allows trailing slashes in the URLS.
app.url_map.strict_slashes = False
# Ensure all error responses are JSON
def _json_error(ex):
code = ex.code if isinstance(ex, wexceptions.HTTPException) else 500
response = {
'code': code
}
if code == 405:
response['type'] = 'invalid_method'
response = flask.jsonify(**response)
response.status_code = code
return response
for code in wexceptions.default_exceptions.iterkeys():
app.error_handler_spec[None][code] = _json_error
# TODO(kiall): Ideally, we want to make use of the Plugin class here.
# This works for the moment though.
def _register_blueprint(ext):
app.register_blueprint(ext.plugin)
# Add all in-built APIs
mgr = extension.ExtensionManager('designate.api.v1')
mgr.map(_register_blueprint)
# Add any (enabled) optional extensions
extensions = cfg.CONF['service:api'].enabled_extensions_v1
if len(extensions) > 0:
extmgr = named.NamedExtensionManager('designate.api.v1.extensions',
names=extensions)
extmgr.map(_register_blueprint)
return app
def __init__(self,
transport,
publisher_id=None,
driver=None,
topic=None,
serializer=None,
retry=None,
topics=None):
"""Construct a Notifier object.
:param transport: the transport to use for sending messages
:type transport: oslo_messaging.Transport
:param publisher_id: field in notifications sent, for example
'compute.host1'
:type publisher_id: str
:param driver: a driver to lookup from oslo_messaging.notify.drivers
:type driver: str
:param topic: the topic which to send messages on
:type topic: str
:param serializer: an optional entity serializer
:type serializer: Serializer
:param retry: an connection retries configuration
None or -1 means to retry forever
0 means no retry
N means N retries
:type retry: int
:param topics: the topics which to send messages on
:type topic: list of strings
"""
conf = transport.conf
conf.register_opts(_notifier_opts,
group='oslo_messaging_notifications')
self.transport = transport
self.publisher_id = publisher_id
self.retry = retry
self._driver_names = ([driver] if driver is not None else
conf.oslo_messaging_notifications.driver)
if topics is not None:
self._topics = topics
elif topic is not None:
self._topics = [topic]
else:
self._topics = conf.oslo_messaging_notifications.topics
self._serializer = serializer or msg_serializer.NoOpSerializer()
self._driver_mgr = named.NamedExtensionManager(
'oslo.messaging.notify.drivers',
names=self._driver_names,
invoke_on_load=True,
invoke_args=[conf],
invoke_kwds={
'topics': self._topics,
'transport': self.transport,
})
def _get_stats_handlers():
global _STATS_HANDLERS
if _STATS_HANDLERS is None:
_STATS_HANDLERS = stevedore_named.NamedExtensionManager(
namespace='octavia.statistics.drivers',
names=CONF.controller_worker.statistics_drivers,
invoke_on_load=True,
propagate_map_exceptions=False)
return _STATS_HANDLERS
def create_component(self, component_types):
system_mgr = named.NamedExtensionManager(
namespace="roast.component.system",
names=component_types,
invoke_on_load=True,
invoke_args=(self.config,),
name_order=True,
)
return system_mgr
def get_import_plugins(**kwargs):
task_list = CONF.image_import_opts.image_import_plugins
extensions = named.NamedExtensionManager('glance.image_import.plugins',
names=task_list,
name_order=True,
invoke_on_load=True,
invoke_kwds=kwargs)
for extension in extensions.extensions:
yield extension.obj
def test_named(self):
em = named.NamedExtensionManager(
'stevedore.test.extension',
names=['t1'],
invoke_on_load=True,
invoke_args=('a', ),
invoke_kwds={'b': 'B'},
)
actual = em.names()
self.assertEqual(actual, ['t1'])
def test_named():
em = named.NamedExtensionManager(
'stevedore.test.extension',
['t1'],
invoke_on_load=True,
invoke_args=('a', ),
invoke_kwds={'b': 'B'},
)
assert len(em.extensions) == 1
assert em.names() == ['t1']
def load_dispatcher_manager():
LOG.debug(_('loading dispatchers from %s'), DISPATCHER_NAMESPACE)
dispatcher_manager = named.NamedExtensionManager(
namespace=DISPATCHER_NAMESPACE,
names=cfg.CONF.dispatcher,
invoke_on_load=True,
invoke_args=[cfg.CONF])
if not list(dispatcher_manager):
LOG.warning(_('Failed to load any dispatchers for %s'),
DISPATCHER_NAMESPACE)
return dispatcher_manager
def start(self):
super(DispatchedService, self).start()
LOG.debug(_('loading dispatchers from %s'), self.DISPATCHER_NAMESPACE)
self.dispatcher_manager = named.NamedExtensionManager(
namespace=self.DISPATCHER_NAMESPACE,
names=cfg.CONF.dispatcher,
invoke_on_load=True,
invoke_args=[cfg.CONF])
if not list(self.dispatcher_manager):
LOG.warning(_('Failed to load any dispatchers for %s'),
self.DISPATCHER_NAMESPACE)
def __init__(self, *args, **kwargs):
super(CollectorBase, self).__init__(*args, **kwargs)
LOG.debug('loading dispatchers from %s', self.DISPATCHER_NAMESPACE)
self.dispatcher_manager = named.NamedExtensionManager(
namespace=self.DISPATCHER_NAMESPACE,
names=cfg.CONF.collector.dispatcher,
invoke_on_load=True,
invoke_args=[cfg.CONF])
if not list(self.dispatcher_manager):
LOG.warning('Failed to load any dispatchers for %s',
self.DISPATCHER_NAMESPACE)
def get_recovery_flow(task_list, **kwargs):
"""This is used create extension object from provided task_list.
This method returns the extension object of the each task provided
in a list using stevedore extension manager.
"""
extensions = named.NamedExtensionManager(
'masakari.task_flow.tasks', names=task_list,
name_order=True, invoke_on_load=True, invoke_kwds=kwargs)
for extension in extensions.extensions:
yield extension.obj
def test_missing_entrypoints_callback(self, load_fn):
errors = set()
def callback(names):
errors.update(names)
load_fn.return_value = [extension.Extension('foo', None, None, None)]
named.NamedExtensionManager('stevedore.test.extension',
names=['foo', 'bar'],
invoke_on_load=True,
on_missing_entrypoints_callback=callback)
self.assertEqual(errors, {'bar'})
def load_dispatcher():
"""Load dispatchers."""
global dispatchers
LOG.debug("Loading dispatchers")
dispatchers = named.NamedExtensionManager(namespace="senlin.dispatchers",
names=cfg.CONF.event_dispatchers,
invoke_on_load=True,
propagate_map_exceptions=True)
if not list(dispatchers):
LOG.warning("No dispatchers configured for 'senlin.dispatchers'")
else:
LOG.info("Loaded dispatchers: %s", dispatchers.names())
def test_load_fail_ignored_when_sorted(self):
em = named.NamedExtensionManager(
'stevedore.test.extension',
names=['e1', 't2', 'e2', 't1'],
name_order=True,
invoke_on_load=True,
invoke_args=('a', ),
invoke_kwds={'b': 'B'},
)
actual = em.names()
self.assertEqual(['t2', 't1'], actual)
em = named.NamedExtensionManager(
'stevedore.test.extension',
names=['e1', 't1'],
name_order=False,
invoke_on_load=True,
invoke_args=('a', ),
invoke_kwds={'b': 'B'},
)
actual = em.names()
self.assertEqual(['t1'], actual)
def __init__(self):
self.handlers = named.NamedExtensionManager(
namespace='dnh.handler',
names=cfg.CONF[CFG_GRP].handlers,
invoke_on_load=True)
# Some handlers may have required options. They're loaded by stevedore
# after the initial config parsing, so we reparse the config file here
# to make sure handlers get all their (required) options parsed.
cfg.CONF()
self.connection = kombu.Connection(
'amqp://%s:%d' % (cfg.CONF[CFG_GRP].host, cfg.CONF[CFG_GRP].port))
signal.signal(signal.SIGTERM, self.on_signal_stop)
signal.signal(signal.SIGINT, self.on_signal_stop)
def run(self):
# Delay startup so workers are jittered
time.sleep(self.startup_delay)
super(NotificationService, self).run()
self.coord_lock = threading.Lock()
self.managers = [
ext.obj for ext in named.NamedExtensionManager(
namespace='ceilometer.notification.pipeline',
names=self.conf.notification.pipelines,
invoke_on_load=True,
on_missing_entrypoints_callback=self._log_missing_pipeline,
invoke_args=(self.conf,
self.conf.notification.workload_partitioning))
]
self.transport = messaging.get_transport(self.conf)
if self.conf.notification.workload_partitioning:
self.partition_coordinator.start(start_heart=True)
else:
# FIXME(sileht): endpoint uses the notification_topics option
# and it should not because this is an oslo_messaging option
# not a ceilometer. Until we have something to get the
# notification_topics in another way, we must create a transport
# to ensure the option has been registered by oslo_messaging.
messaging.get_notifier(self.transport, '')
self._configure_main_queue_listeners()
if self.conf.notification.workload_partitioning:
# join group after all manager set up is configured
self.hashring = self.partition_coordinator.join_partitioned_group(
self.NOTIFICATION_NAMESPACE)
@periodics.periodic(spacing=self.conf.coordination.check_watchers,
run_immediately=True)
def run_watchers():
self.partition_coordinator.run_watchers()
if self.group_state != self.hashring.ring.nodes:
self.group_state = self.hashring.ring.nodes.copy()
self._refresh_agent()
self.periodic = periodics.PeriodicWorker.create(
[],
executor_factory=lambda: futures.ThreadPoolExecutor(max_workers
=10))
self.periodic.add(run_watchers)
utils.spawn_thread(self.periodic.start)
def load_dispatcher_manager():
LOG.debug(_('loading dispatchers from %s'), DISPATCHER_NAMESPACE)
# set propagate_map_exceptions to True to enable stevedore
# to propagate exceptions.
dispatcher_manager = named.NamedExtensionManager(
namespace=DISPATCHER_NAMESPACE,
names=cfg.CONF.dispatcher,
invoke_on_load=True,
invoke_args=[cfg.CONF],
propagate_map_exceptions=True)
if not list(dispatcher_manager):
LOG.warning(_('Failed to load any dispatchers for %s'),
DISPATCHER_NAMESPACE)
return dispatcher_manager
def __init__(self):
enabled_ext = cfg.CONF['service:api'].enabled_extensions_admin
if len(enabled_ext) > 0:
self._mgr = named.NamedExtensionManager(
namespace='designate.api.admin.extensions',
names=enabled_ext,
invoke_on_load=True)
for ext in self._mgr:
controller = self
path = ext.obj.get_path()
for p in path.split('.')[:-1]:
if p != '':
controller = getattr(controller, p)
setattr(controller, path.split('.')[-1], ext.obj)
def get_import_plugin(**kwargs):
method_list = CONF.enabled_import_methods
import_method = kwargs.get('import_req')['method']['name']
if import_method in method_list:
import_method = import_method.replace("-", "_")
task_list = [import_method]
# TODO(jokke): Implement error handling of non-listed methods.
extensions = named.NamedExtensionManager(
'glance.image_import.internal_plugins',
names=task_list,
name_order=True,
invoke_on_load=True,
invoke_kwds=kwargs)
for extension in extensions.extensions:
return extension.obj
def _load_dispatcher_manager(conf, dispatcher_type):
namespace = 'ceilometer.dispatcher.%s' % dispatcher_type
conf_name = '%s_dispatchers' % dispatcher_type
LOG.debug('loading dispatchers from %s', namespace)
# set propagate_map_exceptions to True to enable stevedore
# to propagate exceptions.
dispatcher_manager = named.NamedExtensionManager(
namespace=namespace,
names=getattr(conf, conf_name),
invoke_on_load=True,
invoke_args=[conf],
propagate_map_exceptions=True)
if not list(dispatcher_manager):
LOG.warning(_LW('Failed to load any dispatchers for %s'), namespace)
return dispatcher_manager
def _get_import_flows(**kwargs):
# NOTE(flaper87): Until we have a better infrastructure to enable
# and disable tasks plugins, hard-code the tasks we know exist,
# instead of loading everything from the namespace. This guarantees
# both, the load order of these plugins and the fact that no random
# plugins will be added/loaded until we feel comfortable with this.
# Future patches will keep using NamedExtensionManager but they'll
# rely on a config option to control this process.
extensions = named.NamedExtensionManager('glance.flows.import',
names=['convert', 'introspect'],
name_order=True,
invoke_on_load=True,
invoke_kwds=kwargs)
for ext in extensions.extensions:
yield ext.obj
def provider(self):
"""provider.
:returns mgr
"""
LOG.debug((u'Loading provider extension(s)'))
# create the driver manager to load the appropriate drivers
provider_type = 'poppy.provider'
args = [self.conf]
provider_names = self.driver_conf.providers
mgr = named.NamedExtensionManager(namespace=provider_type,
names=provider_names,
invoke_on_load=True,
invoke_args=args)
return mgr
def notification(self):
"""notification.
:returns mgr
"""
LOG.debug((u'Loading notification extension(s)'))
# create the driver manager to load the appropriate drivers
notification_type = 'poppy.notification'
args = [self.conf]
notification_type_names = self.driver_conf.notifications
mgr = named.NamedExtensionManager(namespace=notification_type,
names=notification_type_names,
invoke_on_load=True,
invoke_args=args)
return mgr