def subscribe():
try:
log.register_options(CONF)
except cfg.ArgsAlreadyParsedError:
pass
log.setup(CONF, 'storyboard')
CONF(project='storyboard')
CONF.register_opts(NOTIFICATION_OPTS, "notifications")
subscriber = Subscriber(CONF.notifications)
subscriber.start()
manager = enabled.EnabledExtensionManager(
namespace='storyboard.plugin.worker',
check_func=check_enabled,
invoke_on_load=True,
invoke_args=(CONF, ))
while subscriber.started:
(method, properties, body) = subscriber.get()
if not method or not properties:
LOG.debug(_("No messages available, sleeping for 5 seconds."))
time.sleep(5)
continue
manager.map(handle_event, body)
# Ack the message
subscriber.ack(method.delivery_tag)
def _load_job_binaries(self):
config_jb = CONF.job_binary_types
extension_manager = enabled.EnabledExtensionManager(
check_func=lambda ext: ext.name in config_jb,
namespace='sahara.job_binary.types',
invoke_on_load=True
)
for ext in extension_manager.extensions:
if ext.name in self.job_binaries:
raise ex.ConfigurationError(
_("Job binary with name '%s' already exists.") %
ext.name)
ext.obj.name = ext.name
self.job_binaries[ext.name] = ext.obj
LOG.info(_LI("Job binary name {jb_name} loaded {entry_point}")
.format(jb_name=ext.name,
entry_point=ext.entry_point_target))
if len(self.job_binaries) < len(config_jb):
loaded_jb = set(six.iterkeys(self.job_binaries))
requested_jb = set(config_jb)
raise ex.ConfigurationError(
_("Job binaries couldn't be loaded: %s") %
", ".join(requested_jb - loaded_jb))
def main():
args = parse_args()
mgr = enabled.EnabledExtensionManager(
namespace='sensormgr.drivers',
check_func=check_enable,
invoke_on_load=False,
)
print('Available drivers:', mgr.names())
with open(args.sensors) as fd:
config = yaml.load(fd)
sensors = []
for spec in config['sensors']:
if spec['method'] not in mgr.names():
print('skipping %(name)s: no driver available for method '
'%(method)s.' % spec)
continue
sensor = mgr[spec['method']].plugin(spec)
sensors.append(sensor)
while True:
for sensor in sensors:
print('%s: %s' % (sensor.cfg['name'], sensor.sense()))
time.sleep(5)
def _get_plugins(self):
"""Return dict of resource-plugin class pairs."""
config_plugins = CONF.manager.plugins
plugins = {}
extension_manager = enabled.EnabledExtensionManager(
check_func=lambda ext: ext.name in config_plugins,
namespace='climate.resource.plugins',
invoke_on_load=False
)
for ext in extension_manager.extensions:
try:
plugin_obj = ext.plugin()
except Exception as e:
LOG.warning("Could not load {0} plugin "
"for resource type {1} '{2}'".format(
ext.name, ext.plugin.resource_type, e))
else:
if plugin_obj.resource_type in plugins:
msg = ("You have provided several plugins for "
"one resource type in configuration file. "
"Please set one plugin per resource type.")
raise exceptions.PluginConfigurationError(error=msg)
plugins[plugin_obj.resource_type] = plugin_obj
return plugins
def __init__(self, migration_config):
self._manager = enabled.EnabledExtensionManager(
MIGRATION_NAMESPACE,
check_plugin_enabled,
invoke_kwds={'migration_config': migration_config},
invoke_on_load=True)
if not self._plugins:
raise ValueError('There must be at least one plugin active.')
def __init__(self, resource_tracker):
self.cpu_monitor_loaded = False
ns = 'nova.compute.monitors.cpu'
cpu_plugin_mgr = enabled.EnabledExtensionManager(
namespace=ns,
invoke_on_load=True,
check_func=self.check_enabled_cpu_monitor,
invoke_args=(resource_tracker, ))
self.monitors = [obj.obj for obj in cpu_plugin_mgr]
def _load_plugins(config):
def check_func(extension):
enabled_archives = config['enabled_plugins']['archives']
return extension.name in enabled_archives
enabled.EnabledExtensionManager(
namespace='wazo_purge_db.archives',
check_func=check_func,
invoke_args=(config,),
invoke_on_load=True,
)
def test_enabled():
def check_enabled(ep):
return ep.name == 't2'
em = enabled.EnabledExtensionManager(
'stevedore.test.extension',
check_enabled,
invoke_on_load=True,
invoke_args=('a',),
invoke_kwds={'b': 'B'},
)
assert len(em.extensions) == 1
assert em.names() == ['t2']
def _start_provider_agents(exit_event):
extensions = stevedore_enabled.EnabledExtensionManager(
namespace='octavia.driver_agent.provider_agents',
check_func=_check_if_provider_agent_enabled)
for ext in extensions:
ext_process = multiprocessing.Process(
name=ext.name, target=_process_wrapper,
args=(exit_event, 'provider_agent', ext.plugin),
kwargs={'agent_name': ext.name})
PROVIDER_AGENT_PROCESSES.append(ext_process)
ext_process.start()
LOG.info('Started enabled provider agent: "%s" with PID: %d.',
ext.name, ext_process.pid)
def __init__(self, resource_tracker):
# Dictionary keyed by the monitor type namespace. Value is the
# first loaded monitor of that namespace or False.
self.type_monitor_loaded = {ns: False for ns in self.NAMESPACES}
self.monitors = []
for ns in self.NAMESPACES:
plugin_mgr = enabled.EnabledExtensionManager(
namespace=ns,
invoke_on_load=True,
check_func=self.check_enabled_monitor,
invoke_args=(resource_tracker, ))
self.monitors += [ext.obj for ext in plugin_mgr]
def test_enabled(self):
def check_enabled(ep):
return ep.name == 't2'
em = enabled.EnabledExtensionManager(
'stevedore.test.extension',
check_enabled,
invoke_on_load=True,
invoke_args=('a', ),
invoke_kwds={'b': 'B'},
)
self.assertEqual(len(em.extensions), 1)
self.assertEqual(em.names(), ['t2'])
def reload(self):
"""Force a reload of the plugins for this instances namespace.
:returns: None.
"""
self._extensions = {}
self._extension_manager = enabled.EnabledExtensionManager(
self.namespace, lambda x: True, invoke_on_load=False)
if not self._extension_manager.names():
LOG.debug("No plugins found in namespace: ", self.namespace)
return
self._extension_manager.map(self._add_extension)
def initialise():
global _mgr
if _mgr:
return
def client_is_available(client_plugin):
if not hasattr(client_plugin.plugin, 'is_available'):
# if the client does not have a is_available() class method, then
# we assume it wants to be always available
return True
# let the client plugin decide if it wants to register or not
return client_plugin.plugin.is_available()
_mgr = enabled.EnabledExtensionManager(namespace='bilean.clients',
check_func=client_is_available,
invoke_on_load=False)
def __init__(self, namespace):
self.mgr = enabled.EnabledExtensionManager(
check_func=self._gen_check_func(),
namespace=namespace,
propagate_map_exceptions=True,
on_load_failure_callback=self._on_load_failure)
self.plugin_map = {'by_typename': {},
'by_endpoint': {}}
def _add_extension(ext):
"""
Plugins can be loaded as entry_point=single plugin and
entry_point=PLUGIN_LIST, where PLUGIN_LIST is a python variable
holding a list of plugins
"""
def _load_one(plugin):
if issubclass(plugin, definitions.ArtifactType):
# make sure that have correct plugin name
art_name = plugin.metadata.type_name
if art_name != ext.name:
raise exception.ArtifactNonMatchingTypeName(
name=art_name, plugin=ext.name)
# make sure that no plugin with the same name and version
# already exists
exists = self._get_plugins(ext.name)
new_tv = plugin.metadata.type_version
if any(e.metadata.type_version == new_tv for e in exists):
raise exception.ArtifactDuplicateNameTypeVersion()
self._add_plugin("by_endpoint", plugin.metadata.endpoint,
plugin)
self._add_plugin("by_typename", plugin.metadata.type_name,
plugin)
if isinstance(ext.plugin, list):
for p in ext.plugin:
_load_one(p)
else:
_load_one(ext.plugin)
# (ivasilevskaya) that looks pretty bad as RuntimeError is too general,
# but stevedore has awful exception wrapping with no specific class
# for this very case (no extensions for given namespace found)
try:
self.mgr.map(_add_extension)
except RuntimeError as re:
LOG.error(_LE("Unable to load artifacts: %s") % re.message)
def get_manager(namespace, names=None):
u"""
Get the stevedore extension manager for this namespace.
Args:
namespace (basestring): The entry point namespace to load plugins for.
names (list): A list of names to load. If this is ``None`` then all extension will be loaded from this
namespace.
Returns:
stevedore.enabled.EnabledExtensionManager: Extension manager with all extensions instantiated.
"""
return enabled.EnabledExtensionManager(
namespace=namespace,
check_func=partial(check_plugin, namespace=namespace, names=names),
invoke_on_load=True,
)
def __init__(self, migration_config, engine=None):
if engine is None:
if migration_config.get('db_url'):
engine = sqlalchemy.create_engine(
migration_config['db_url'],
poolclass=sqlalchemy.pool.NullPool,
)
else:
raise ValueError('Either database url or engine'
' must be provided.')
self._manager = enabled.EnabledExtensionManager(
MIGRATION_NAMESPACE,
check_plugin_enabled,
invoke_args=(engine, migration_config),
invoke_on_load=True)
if not self._plugins:
raise ValueError('There must be at least one plugin active.')
def get_extensions(cls, enabled_extensions=None):
"""Load a series of extensions"""
LOG.debug('Looking for extensions in %s' % cls.__plugin_ns__)
def _check_func(ext):
if enabled_extensions is None:
# All extensions are enabled by default, if no specific list
# is specified
return True
return ext.plugin.get_plugin_name() in enabled_extensions
mgr = enabled.EnabledExtensionManager(
cls.__plugin_ns__, check_func=_check_func,
propagate_map_exceptions=True)
return [e.plugin for e in mgr]
def initialise():
global _mgr
if _mgr:
#mgr已初始化，返回
return
def client_is_available(client_plugin):
if not hasattr(client_plugin.plugin, 'is_available'):
# if the client does not have a is_available() class method, then
# we assume it wants to be always available
return True
# let the client plugin decide if it wants to register or not
return client_plugin.plugin.is_available()
#加载heat.clients下的所有client
_mgr = enabled.EnabledExtensionManager(
namespace='heat.clients',
check_func=client_is_available,
invoke_on_load=False,
on_load_failure_callback=pluginutils.log_fail_msg)
def __init__(self):
extensions = []
self.extension_manager = enabled.EnabledExtensionManager(
check_func=lambda ext: ext.name in CONF.api.api_v2_controllers,
namespace='blazar.api.v2.controllers.extensions',
invoke_on_load=True)
self._log_missing_plugins(CONF.api.api_v2_controllers)
for ext in self.extension_manager.extensions:
try:
setattr(self, ext.obj.name, ext.obj)
except TypeError:
raise exceptions.BlazarException(
_("API name must be specified for "
"extension {0}").format(ext.name))
self._routes.update(ext.obj.extra_routes)
extensions.append(ext.obj.name)
LOG.debug("Loaded extensions: {0}".format(extensions))
def _purge_tables(config):
def check_func(extension):
enabled_purgers = config['enabled_plugins']['purgers']
return enabled_purgers.get(extension.name, False)
table_purgers = enabled.EnabledExtensionManager(
namespace='wazo_purge_db.purgers', check_func=check_func, invoke_on_load=True
)
with session_scope() as session:
for purger in table_purgers:
days_to_keep = config['days_to_keep_per_plugin'].get(
purger.name, config['days_to_keep']
)
logger.info(
'%s purger: deleting entries older than %s days',
purger.name,
days_to_keep,
)
purger.obj.purge(days_to_keep, session)
logger.debug('%s purger: finished', purger.name)
def _load_cluster_plugins(self):
config_plugins = CONF.plugins
extension_manager = enabled.EnabledExtensionManager(
check_func=lambda ext: ext.name in config_plugins,
namespace='sahara.cluster.plugins',
invoke_on_load=True)
for ext in extension_manager.extensions:
if ext.name in self.plugins:
raise ex.ConfigurationError(
"Plugin with name '%s' already exists." % ext.name)
ext.obj.name = ext.name
self.plugins[ext.name] = ext.obj
LOG.info("Plugin '%s' loaded (%s)" %
(ext.name, ext.entry_point_target))
if len(self.plugins) < len(config_plugins):
loaded_plugins = set(six.iterkeys(self.plugins))
requested_plugins = set(config_plugins)
raise ex.ConfigurationError(
"Plugins couldn't be loaded: %s" %
", ".join(requested_plugins - loaded_plugins))
def _load_plugins(self):
config_plugins = api.plugins
extension_manager = enabled.EnabledExtensionManager(
check_func=lambda ext: ext.name in config_plugins,
namespace='application_manager.execution.plugins',
invoke_on_load=True)
for ext in extension_manager.extensions:
if ext.name in self.plugins:
raise ex.ConfigurationError(
("Plugin with name '%s' already exists.") % ext.name)
ext.obj.name = ext.name
self.plugins[ext.name] = ext.obj
LOG.log("Plugin {plugin_name} loaded {entry_point}".format(
plugin_name=ext.name, entry_point=ext.entry_point_target))
if len(self.plugins) < len(config_plugins):
loaded_plugins = set(six.iterkeys(self.plugins))
requested_plugins = set(config_plugins)
raise ex.ConfigurationError(
("Plugins couldn't be loaded: %s") %
", ".join(requested_plugins - loaded_plugins))
def _load_data_sources(self):
config_ds = CONF.data_source_types
extension_manager = enabled.EnabledExtensionManager(
check_func=lambda ext: ext.name in config_ds,
namespace='sahara.data_source.types',
invoke_on_load=True)
for ext in extension_manager.extensions:
if ext.name in self.data_sources:
raise ex.ConfigurationError(
_("Data source with name '%s' already exists.") % ext.name)
ext.obj.name = ext.name
self.data_sources[ext.name] = ext.obj
LOG.info("Data source name {ds_name} loaded {entry_point}".format(
ds_name=ext.name, entry_point=ext.entry_point_target))
if len(self.data_sources) < len(config_ds):
loaded_ds = set(six.iterkeys(self.data_sources))
requested_ds = set(config_ds)
raise ex.ConfigurationError(
_("Data sources couldn't be loaded: %s") %
", ".join(requested_ds - loaded_ds))
def _load_cluster_plugins(self):
config_plugins = CONF.plugins
extension_manager = enabled.EnabledExtensionManager(
check_func=lambda ext: ext.name in config_plugins,
namespace='savanna.cluster.plugins',
invoke_on_load=True
)
for ext in extension_manager.extensions:
if ext.name in self.plugins:
# TODO(slukjanov): replace with specific exception
raise RuntimeError("Plugin with name '%s' already exists.")
ext.obj.name = ext.name
self.plugins[ext.name] = ext.obj
LOG.info("Plugin '%s' loaded (%s)"
% (ext.name, ext.entry_point_target))
if len(self.plugins) < len(config_plugins):
loaded_plugins = set(six.iterkeys(self.plugins))
requested_plugins = set(config_plugins)
# TODO(slukjanov): replace with specific exception
raise RuntimeError("Plugins couldn't be loaded: %s"
% ", ".join(requested_plugins - loaded_plugins))
def make_app():
"""App builder (wsgi).
Entry point for Blazar REST API server.
"""
app = flask.Flask('blazar.api')
app.route('/', methods=['GET'])(version_list)
app.route('/versions', methods=['GET'])(version_list)
LOG.debug("List of plugins: %s", cfg.CONF.manager.plugins)
plugins = cfg.CONF.manager.plugins + ['leases']
extension_manager = enabled.EnabledExtensionManager(
check_func=lambda ext: ext.name in plugins,
namespace='blazar.api.v1.extensions',
invoke_on_load=False)
for ext in extension_manager.extensions:
bp = ext.plugin()
app.register_blueprint(bp, url_prefix=bp.url_prefix)
for code in werkzeug_exceptions.default_exceptions:
app.register_error_handler(code, make_json_error)
if cfg.CONF.debug and not cfg.CONF.log_exchange:
LOG.debug('Logging of request/response exchange could be enabled '
'using flag --log_exchange')
if cfg.CONF.log_exchange:
app.wsgi_app = debug.Debug.factory(app.config)(app.wsgi_app)
app.wsgi_app = request_id.BlazarReqIdMiddleware(app.wsgi_app)
app.wsgi_app = request_log.RequestLog(app.wsgi_app)
app.wsgi_app = auth_token.filter_factory(app.config)(app.wsgi_app)
return app
def get_extensions():
def check_enabled(ext):
return ext.plugin.enabled()
return enabled.EnabledExtensionManager('subunit2sql.target',
check_func=check_enabled)
def run_playbook(self,
playbook_name,
user_name,
only_tags=None,
skip_tags=None,
extra_vars=None,
subset=None):
""" This function will launch a playbook. To handle logging, it will
use stevedore which will load all extensions registered under the
entrypoint cyclosible.plugins.storage. For example, it will let Cyclosible
save his log on a file, on S3, or something else.
:param playbook_name:
:param user_name:
:param only_tags:
:param skip_tags:
:param extra_vars:
:return:
"""
history = PlaybookRunHistory.objects.create(
playbook=Playbook.objects.get(name=playbook_name),
date_launched=timezone.now(),
status='RUNNING',
task_id=self.request.id,
launched_by=User.objects.get(username=user_name))
vault_password = None
if settings.VAULT_ENABLED:
try:
self.mgr_vault = driver.DriverManager(
namespace='cyclosible.plugins.vault',
name=settings.VAULT_ENABLED,
invoke_on_load=True,
)
vault_password = self.mgr_vault.driver.get_password()
except RuntimeError as e:
logger.error(e)
logger.debug('LOADED VAULT: {plugins} | Status: {status}'.format(
plugins=settings.VAULT_ENABLED,
status='OK' if vault_password else 'KO'))
inventory = Inventory(vault_password=vault_password)
# Here, we override the default ansible callbacks to pass our customs parameters
stats = callbacks.AggregateStats()
playbook_cb = PlaybookCallbacks(verbose=utils.VERBOSITY,
task_id=self.request.id)
runner_cb = PlaybookRunnerCallbacks(stats=stats,
verbose=utils.VERBOSITY,
task_id=self.request.id)
pb = ansible.playbook.PlayBook(playbook=''.join(
[settings.PLAYBOOK_PATH, playbook_name, '.yml']),
callbacks=playbook_cb,
runner_callbacks=runner_cb,
inventory=inventory,
stats=stats,
extra_vars=extra_vars,
only_tags=only_tags,
skip_tags=skip_tags,
vault_password=vault_password,
subset=subset)
try:
pb.run()
hosts = sorted(pb.stats.processed.keys())
logger.info(hosts)
playbook_cb.on_stats(pb.stats)
history.status = 'SUCCESS'
for h in hosts:
t = pb.stats.summarize(h)
display("%s : %s %s %s %s" %
(hostcolor(h, t), colorize('ok', t['ok'], 'green'),
colorize('changed', t['changed'], 'yellow'),
colorize('unreachable', t['unreachable'], 'red'),
colorize('failed', t['failures'], 'red')),
task_id=self.request.id)
except errors.AnsibleError as e:
history.status = 'FAILED'
display(u"ERROR: %s" % e, task_id=self.request.id)
logger.error(u"ERROR: %s" % e)
try:
self.mgr_storage = enabled.EnabledExtensionManager(
namespace='cyclosible.plugins.storage',
check_func=check_plugin_enabled,
invoke_on_load=True,
invoke_kwds={'task_id': self.request.id},
verify_requirements=True)
logger.debug('LOADED STORAGE: {plugins}'.format(
plugins=', '.join(self.mgr_storage.names())))
except RuntimeError as e:
logger.error(e)
try:
list_urls = []
self.mgr_storage.map(lambda ext: (ext.name, ext.obj.write_log()))
urls = self.mgr_storage.map(lambda ext:
(ext.name, ext.obj.get_url_log()))
for url in urls:
try:
if url[1]:
list_urls.append({url[0]: url[1]})
except IndexError:
logger.debug('Index does not exist in the url returned')
history.log_url = json.dumps(list_urls)
except RuntimeError:
logger.debug('No plugins available')
history.date_finished = timezone.now()
history.save()
if history.status == 'FAILED':
return 1
def run_playbook(self, playbook_name, user_name, only_tags=None, skip_tags=None, extra_vars=None):
""" This function will launch a playbook. To handle logging, it will
use stevedore which will load all extensions registered under the
entrypoint cyclosible.plugins.storage. For example, it will let Cyclosible
save his log on a file, on S3, or something else.
:param playbook_name:
:param user_name:
:param only_tags:
:param skip_tags:
:param extra_vars:
:return:
"""
history = PlaybookRunHistory.objects.create(
playbook=Playbook.objects.get(name=playbook_name),
date_launched=timezone.now(),
status='RUNNING',
task_id=self.request.id,
launched_by=User.objects.get(username=user_name)
)
# Here, we override the default ansible callbacks to pass our customs parameters
stats = callbacks.AggregateStats()
playbook_cb = PlaybookCallbacks(
verbose=utils.VERBOSITY,
task_id=self.request.id
)
runner_cb = PlaybookRunnerCallbacks(
stats=stats,
verbose=utils.VERBOSITY,
task_id=self.request.id
)
pb = ansible.playbook.PlayBook(
playbook=''.join([settings.PLAYBOOK_PATH, playbook_name, '.yml']),
callbacks=playbook_cb,
runner_callbacks=runner_cb,
stats=stats,
extra_vars=extra_vars,
only_tags=only_tags,
skip_tags=skip_tags,
)
try:
pb.run()
hosts = sorted(pb.stats.processed.keys())
logger.info(hosts)
playbook_cb.on_stats(pb.stats)
history.status = 'SUCCESS'
except errors.AnsibleError:
history.status = 'FAILED'
logger.error(u"ERROR: %s" % utils.unicode.to_unicode(errors.AnsibleError, nonstring='simplerepr'))
self.mgr_storage = enabled.EnabledExtensionManager(
namespace='cyclosible.plugins.storage',
check_func=check_plugin_enabled,
invoke_on_load=True,
invoke_kwds={'task_id': self.request.id},
verify_requirements=True
)
logger.debug('LOADED PLUGINS: {plugins}'.format(plugins=', '.join(self.mgr_storage.names())))
try:
list_urls = []
self.mgr_storage.map(lambda ext: (ext.name, ext.obj.write_log()))
urls = self.mgr_storage.map(lambda ext: (ext.name, ext.obj.get_url_log()))
for url in urls:
try:
if url[1]:
list_urls.append({url[0]: url[1]})
except IndexError:
logger.debug('Index does not exist in the url returned')
history.log_url = json.dumps(list_urls)
except RuntimeError:
logger.debug('No plugins available')
history.date_finished = timezone.now()
history.save()
if history.status == 'FAILED':
return 1
