def get_plugin_attributes(cls, cluster):
plugin_attributes = {}
for plugin_db in PluginCollection.all_newest():
plugin_adapter = wrap_plugin(plugin_db)
attributes = plugin_adapter.get_plugin_attributes(cluster)
plugin_attributes.update(attributes)
return plugin_attributes
def get_plugins_node_roles(cls, cluster):
result = {}
core_roles = set(cluster.release.roles_metadata)
for plugin_db in ClusterPlugins.get_enabled(cluster.id):
plugin_roles = wrap_plugin(plugin_db).normalized_roles_metadata
# we should check all possible cases of roles intersection
# with core ones and those from other plugins
# and afterwards show them in error message;
# thus role names for which following checks
# fails are accumulated in err_info variable
err_roles = set(r for r in plugin_roles
if r in core_roles or r in result)
if err_roles:
raise errors.AlreadyExists(
"Plugin (ID={0}) is unable to register the following "
"node roles: {1}".format(plugin_db.id,
", ".join(sorted(err_roles))))
# update info on processed roles in case of
# success of all intersection checks
result.update(plugin_roles)
return result
def get_cluster_plugins_with_tasks(cls, cluster):
cluster_plugins = []
for plugin_db in cluster.plugins:
plugin_adapter = wrap_plugin(plugin_db)
plugin_adapter.set_cluster_tasks()
cluster_plugins.append(plugin_adapter)
return cluster_plugins
def get_plugins_node_roles(cls, cluster):
result = {}
core_roles = set(cluster.release.roles_metadata)
for plugin_db in cluster.plugins:
plugin_roles = wrap_plugin(plugin_db).normalized_roles_metadata
# we should check all possible cases of roles intersection
# with core ones and those from other plugins
# and afterwards show them in error message;
# thus role names for which following checks
# fails are accumulated in err_info variable
err_roles = set()
if set(plugin_roles) & core_roles:
err_roles |= set(plugin_roles) & core_roles
if set(plugin_roles) & set(result):
err_roles |= set(plugin_roles) & set(result)
if err_roles:
raise errors.AlreadyExists(
"Plugin (ID={0}) is unable to register the following "
"node roles: {1}".format(plugin_db.id,
", ".join(err_roles))
)
# update info on processed roles in case of
# success of all intersection checks
result.update(plugin_roles)
return result
def get_cluster_plugins_with_tasks(cls, cluster):
cluster_plugins = []
for plugin_db in ClusterPlugins.get_enabled(cluster.id):
plugin_adapter = wrap_plugin(plugin_db)
plugin_adapter.set_cluster_tasks()
cluster_plugins.append(plugin_adapter)
return cluster_plugins
def test_get_particular_role_for_cluster_w_plugin(self):
plugin_data = self.env.get_default_plugin_metadata()
plugin_data['roles_metadata'] = self.ROLES
plugin_data['volumes_metadata'] = self.VOLUMES
plugin = objects.Plugin.create(plugin_data)
self.cluster.plugins.append(plugin)
objects.ClusterPlugin.set_attributes(self.cluster.id,
plugin.id,
enabled=True)
self.db.flush()
plugin_adapter = adapters.wrap_plugin(plugin)
role = self.app.get(
url=base.reverse(
'ClusterRolesHandler',
{'cluster_id': self.cluster.id, 'role_name': 'test_role'}
)
).json
self.assertEqual(role['name'], 'test_role')
self.assertDictEqual(
role['meta'],
plugin_adapter.normalized_roles_metadata['test_role']
)
self.assertItemsEqual(
role['volumes_roles_mapping'],
plugin_adapter.volumes_metadata[
'volumes_roles_mapping']['test_role']
)
def get_plugin_attributes(cls, cluster):
plugin_attributes = {}
for plugin_db in PluginCollection.all_newest():
plugin_adapter = wrap_plugin(plugin_db)
attributes = plugin_adapter.get_plugin_attributes(cluster)
plugin_attributes.update(attributes)
return plugin_attributes
def get_cluster_plugins_with_tasks(cls, cluster):
cluster_plugins = []
for plugin_db in cluster.plugins:
plugin_adapter = wrap_plugin(plugin_db)
plugin_adapter.set_cluster_tasks()
cluster_plugins.append(plugin_adapter)
return cluster_plugins
def test_get_particular_role_for_cluster_w_plugin(self):
plugin_data = self.env.get_default_plugin_metadata()
plugin_data['roles_metadata'] = self.ROLES
plugin_data['volumes_metadata'] = self.VOLUMES
plugin = objects.Plugin.create(plugin_data)
self.cluster.plugins.append(plugin)
objects.ClusterPlugin.set_attributes(self.cluster.id,
plugin.id,
enabled=True)
self.db.flush()
plugin_adapter = adapters.wrap_plugin(plugin)
role = self.app.get(
url=base.reverse('ClusterRolesHandler', {
'cluster_id': self.cluster.id,
'role_name': 'test_role'
})).json
self.assertEqual(role['name'], 'test_role')
self.assertDictEqual(
role['meta'],
plugin_adapter.normalized_roles_metadata['test_role'])
self.assertItemsEqual(
role['volumes_roles_mapping'],
plugin_adapter.volumes_metadata['volumes_roles_mapping']
['test_role'])
def get_cluster_plugins_with_tasks(cls, cluster):
cluster_plugins = []
for plugin_db in ClusterPlugins.get_enabled(cluster.id):
plugin_adapter = wrap_plugin(plugin_db)
plugin_adapter.set_cluster_tasks()
cluster_plugins.append(plugin_adapter)
return cluster_plugins
def sync_plugins_metadata(cls, plugin_ids=None):
"""Sync metadata for plugins by given ids. If there is not
ids all newest plugins will be synced
"""
if plugin_ids:
plugins = PluginCollection.get_by_uids(plugin_ids)
else:
plugins = PluginCollection.all()
for plugin in plugins:
plugin_adapter = wrap_plugin(plugin)
plugin_adapter.sync_metadata_to_db()
def sync_plugins_metadata(cls, plugin_ids=None):
"""Sync metadata for plugins by given ids. If there is not
ids all newest plugins will be synced
"""
if plugin_ids:
plugins = PluginCollection.get_by_uids(plugin_ids)
else:
plugins = PluginCollection.all()
for plugin in plugins:
plugin_adapter = wrap_plugin(plugin)
plugin_adapter.sync_metadata_to_db()
def is_compatible(cls, cluster, plugin):
"""Validates if plugin is compatible with cluster.
:param cluster: A cluster instance
:type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster
:param plugin: A plugin instance
:type plugin: nailgun.db.sqlalchemy.models.plugins.Plugin
:return: True if compatible, False if not
:rtype: bool
"""
plugin_adapter = wrap_plugin(plugin)
return plugin_adapter.validate_compatibility(cluster)
def is_compatible(cls, cluster, plugin):
"""Validates if plugin is compatible with cluster.
:param cluster: A cluster instance
:type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster
:param plugin: A plugin instance
:type plugin: nailgun.db.sqlalchemy.models.plugins.Plugin
:return: True if compatible, False if not
:rtype: bool
"""
plugin_adapter = wrap_plugin(plugin)
return plugin_adapter.validate_compatibility(cluster)
def get_plugins_attributes(cls,
cluster,
all_versions=False,
default=False):
"""Gets attributes of all plugins connected with given cluster.
:param cluster: A cluster instance
:type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster
:param all_versions: True to get attributes of all versions of plugins
:type all_versions: bool
:param default: True to return a default plugins attributes (for UI)
:type default: bool
:return: Plugins attributes
:rtype: dict
"""
plugins_attributes = {}
for plugin in ClusterPlugin.get_connected_plugins_data(cluster.id):
db_plugin = Plugin.get_by_uid(plugin.id)
plugin_adapter = wrap_plugin(db_plugin)
default_attrs = plugin_adapter.attributes_metadata
if all_versions:
container = plugins_attributes.setdefault(plugin.name, {})
enabled = plugin.enabled and not (all_versions and default)
cls.create_common_metadata(plugin, container, enabled)
container['metadata']['default'] = default
versions = container['metadata'].setdefault('versions', [])
if default:
actual_attrs = copy.deepcopy(default_attrs)
actual_attrs.setdefault('metadata', {})
else:
actual_attrs = copy.deepcopy(plugin.attributes)
actual_attrs['metadata'] = default_attrs.get(
'metadata', {})
cls.fill_plugin_metadata(plugin, actual_attrs['metadata'])
versions.append(actual_attrs)
container['metadata'].setdefault('chosen_id', plugin.id)
if enabled:
container['metadata']['chosen_id'] = plugin.id
elif plugin.enabled:
container = plugins_attributes.setdefault(plugin.name, {})
cls.create_common_metadata(plugin, container)
container['metadata'].update(default_attrs.get('metadata', {}))
cls.fill_plugin_metadata(plugin, container['metadata'])
container.update(plugin.attributes)
return plugins_attributes
def create(cls, data):
new_plugin = super(Plugin, cls).create(data)
# FIXME (vmygal): This is very ugly hack and it must be fixed ASAP.
# Need to remove the syncing of plugin metadata from here.
# All plugin metadata must be sent via 'data' argument of this
# function and it must be fixed in 'python-fuelclient' repository.
from nailgun.plugins.adapters import wrap_plugin
plugin_adapter = wrap_plugin(new_plugin)
plugin_adapter.sync_metadata_to_db()
ClusterPlugins.add_compatible_clusters(new_plugin)
return new_plugin
def create(cls, data):
new_plugin = super(Plugin, cls).create(data)
# FIXME (vmygal): This is very ugly hack and it must be fixed ASAP.
# Need to remove the syncing of plugin metadata from here.
# All plugin metadata must be sent via 'data' argument of this
# function and it must be fixed in 'python-fuelclient' repository.
from nailgun.plugins.adapters import wrap_plugin
plugin_adapter = wrap_plugin(new_plugin)
plugin_adapter.sync_metadata_to_db()
ClusterPlugins.add_compatible_clusters(new_plugin)
return new_plugin
def create(cls, data):
# accidental because i've seen this way of tasks creation only in tests
deployment_tasks = data.pop('deployment_tasks', [])
new_plugin = super(Plugin, cls).create(data)
# create default graph in any case
DeploymentGraph.create_for_model({'tasks': deployment_tasks},
new_plugin)
plugin_adapter = wrap_plugin(new_plugin)
cls.update(new_plugin, plugin_adapter.get_metadata())
ClusterPlugins.add_compatible_clusters(new_plugin)
return new_plugin
def setUp(self):
super(TestPrePostHooks, self).setUp()
self._requests_mock = mock.patch(
'nailgun.utils.debian.requests.get',
return_value=mock.Mock(text='Archive: test'))
self._requests_mock.start()
resp = self.create_plugin()
self.plugin = adapters.wrap_plugin(
objects.Plugin.get_by_uid(resp.json['id']))
self.cluster = self.create_cluster([
{'roles': ['controller'], 'pending_addition': True},
{'roles': ['compute'], 'pending_addition': True}])
self.enable_plugin(self.cluster, self.sample_plugin['name'])
def create(cls, data):
# accidental because i've seen this way of tasks creation only in tests
deployment_tasks = data.pop('deployment_tasks', [])
new_plugin = super(Plugin, cls).create(data)
# create default graph in any case
DeploymentGraph.create_for_model(
{'tasks': deployment_tasks}, new_plugin)
plugin_adapter = wrap_plugin(new_plugin)
cls.update(new_plugin, plugin_adapter.get_metadata())
ClusterPlugin.add_compatible_clusters(new_plugin)
return new_plugin
def setUp(self):
super(TestPrePostHooks, self).setUp()
self._requests_mock = mock.patch(
'nailgun.utils.debian.requests.get',
return_value=mock.Mock(text='Archive: test'))
self._requests_mock.start()
resp = self.create_plugin()
self.plugin = adapters.wrap_plugin(
objects.Plugin.get_by_uid(resp.json['id']))
self.cluster = self.create_cluster([
{'roles': ['controller'], 'pending_addition': True},
{'roles': ['compute'], 'pending_addition': True}])
self.enable_plugin(self.cluster, self.sample_plugin['name'])
def get_plugins_attributes(
cls, cluster, all_versions=False, default=False):
"""Gets attributes of all plugins connected with given cluster.
:param cluster: A cluster instance
:type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster
:param all_versions: True to get attributes of all versions of plugins
:type all_versions: bool
:param default: True to return a default plugins attributes (for UI)
:type default: bool
:return: Plugins attributes
:rtype: dict
"""
plugins_attributes = {}
for plugin in ClusterPlugins.get_connected_plugins_data(cluster.id):
db_plugin = Plugin.get_by_uid(plugin.id)
plugin_adapter = wrap_plugin(db_plugin)
default_attrs = plugin_adapter.attributes_metadata
if all_versions:
container = plugins_attributes.setdefault(plugin.name, {})
enabled = plugin.enabled and not (all_versions and default)
cls.create_common_metadata(plugin, container, enabled)
container['metadata']['default'] = default
versions = container['metadata'].setdefault('versions', [])
if default:
actual_attrs = copy.deepcopy(default_attrs)
actual_attrs.setdefault('metadata', {})
else:
actual_attrs = copy.deepcopy(plugin.attributes)
actual_attrs['metadata'] = default_attrs.get('metadata',
{})
cls.fill_plugin_metadata(plugin, actual_attrs['metadata'])
versions.append(actual_attrs)
container['metadata'].setdefault('chosen_id', plugin.id)
if enabled:
container['metadata']['chosen_id'] = plugin.id
elif plugin.enabled:
container = plugins_attributes.setdefault(plugin.name, {})
cls.create_common_metadata(plugin, container)
container['metadata'].update(default_attrs.get('metadata', {}))
cls.fill_plugin_metadata(plugin, container['metadata'])
container.update(plugin.attributes)
return plugins_attributes
def setUp(self):
super(TestPrePostHooks, self).setUp()
self._requests_mock = mock.patch(
"nailgun.utils.debian.requests.get", return_value=mock.Mock(text="Archive: test")
)
self._requests_mock.start()
resp = self.create_plugin()
self.plugin = adapters.wrap_plugin(objects.Plugin.get_by_uid(resp.json["id"]))
self.cluster = self.create_cluster(
[{"roles": ["controller"], "pending_addition": True}, {"roles": ["compute"], "pending_addition": True}]
)
objects.Cluster.prepare_for_deployment(self.cluster)
self.enable_plugin(self.cluster, self.sample_plugin["name"], resp.json["id"])
def sync_plugins_metadata(cls, plugin_ids=None):
"""Sync metadata for plugins by given IDs.
If there are no IDs, all newest plugins will be synced.
:param plugin_ids: list of plugin IDs
:type plugin_ids: list
"""
if plugin_ids:
plugins = PluginCollection.get_by_uids(plugin_ids)
else:
plugins = PluginCollection.all()
for plugin in plugins:
plugin_adapter = wrap_plugin(plugin)
plugin_adapter.sync_metadata_to_db()
def sync_plugins_metadata(cls, plugin_ids=None):
"""Sync metadata for plugins by given IDs.
If there are no IDs, all newest plugins will be synced.
:param plugin_ids: list of plugin IDs
:type plugin_ids: list
"""
if plugin_ids:
plugins = PluginCollection.get_by_uids(plugin_ids)
else:
plugins = PluginCollection.all()
for plugin in plugins:
plugin_adapter = wrap_plugin(plugin)
plugin_adapter.sync_metadata_to_db()
def setUp(self):
super(TestPluginBase, self).setUp()
self.plugin_metadata = self.env.get_default_plugin_metadata(
package_version=self.package_version)
self.plugin = Plugin.create(self.plugin_metadata)
self.env.create(
cluster_kwargs={'mode': 'multinode'},
release_kwargs={
'version': '2014.2-6.0',
'operating_system': 'Ubuntu'})
self.cluster = self.env.clusters[0]
self.plugin_adapter = adapters.wrap_plugin(self.plugin)
self.env_config = self.env.get_default_plugin_env_config()
self.get_config = lambda *args: mock.mock_open(
read_data=yaml.dump(self.env_config))()
db().flush()
def enable_plugins_by_components(cls, cluster):
"""Enable plugin by components.
:param cluster: A cluster instance
:type cluster: Cluster model
"""
cluster_components = set(cluster.components)
plugin_ids = [p.id for p in PluginCollection.all_newest()]
for plugin in ClusterPlugin.get_connected_plugins(cluster, plugin_ids):
plugin_adapter = wrap_plugin(plugin)
plugin_components = set(
component['name']
for component in plugin_adapter.components_metadata)
if cluster_components & plugin_components:
ClusterPlugin.set_attributes(cluster.id,
plugin.id,
enabled=True)
def enable_plugins_by_components(cls, cluster):
"""Enable plugin by components.
:param cluster: A cluster instance
:type cluster: Cluster model
"""
cluster_components = set(cluster.components)
plugin_ids = [p.id for p in PluginCollection.all_newest()]
for plugin in ClusterPlugins.get_connected_plugins(
cluster, plugin_ids):
plugin_adapter = wrap_plugin(plugin)
plugin_components = set(
component['name']
for component in plugin_adapter.components_metadata)
if cluster_components & plugin_components:
ClusterPlugins.set_attributes(
cluster.id, plugin.id, enabled=True)
def setUp(self):
super(TestPluginBase, self).setUp()
self.plugin_metadata = self.env.get_default_plugin_metadata(
package_version=self.package_version,
roles_metadata={
'role_x': {
'name': 'Role X',
'description': 'Role X is ...',
},
'role_y': {
'name': 'Role Y',
'description': 'Role Y is ...',
'restrictions': [],
'fault_tolerance': '5%'
},
'role_z': {
'name': 'Role Z',
'description': 'Role Z is ...',
'restrictions': ['settings:some.stuff.value == false'],
'fault_tolerance': '10%'
}
})
self.plugin = Plugin.create(self.plugin_metadata)
self.cluster = self.env.create(
cluster_kwargs={'mode': consts.CLUSTER_MODES.multinode},
release_kwargs={
'version':
'2015.1-8.0',
'operating_system':
'Ubuntu',
'modes': [
consts.CLUSTER_MODES.multinode,
consts.CLUSTER_MODES.ha_compact
]
})
self.plugin_adapter = adapters.wrap_plugin(self.plugin)
self.env_config = self.env.get_default_plugin_env_config()
self.get_config = lambda *args: mock.mock_open(read_data=yaml.dump(
self.env_config))()
db().flush()
def setUp(self):
super(TestPluginBase, self).setUp()
self.plugin_metadata = self.env.get_default_plugin_metadata(
package_version=self.package_version,
roles_metadata={
'role_x': {
'name': 'Role X',
'description': 'Role X is ...',
},
'role_y': {
'name': 'Role Y',
'description': 'Role Y is ...',
'restrictions': [],
'fault_tolerance': '5%'
},
'role_z': {
'name': 'Role Z',
'description': 'Role Z is ...',
'restrictions': [
'settings:some.stuff.value == false'
],
'fault_tolerance': '10%'
}
}
)
self.plugin = Plugin.create(self.plugin_metadata)
self.cluster = self.env.create(
cluster_kwargs={'mode': consts.CLUSTER_MODES.multinode},
release_kwargs={
'version': '2015.1-8.0',
'operating_system': 'Ubuntu',
'modes': [consts.CLUSTER_MODES.multinode,
consts.CLUSTER_MODES.ha_compact]})
self.plugin_adapter = adapters.wrap_plugin(self.plugin)
self.env_config = self.env.get_default_plugin_env_config()
self.get_config = lambda *args: mock.mock_open(
read_data=yaml.dump(self.env_config))()
db().flush()
def serialize_plugin(cls, cluster, plugin):
os_name = cluster.release.operating_system
adapter = adapters.wrap_plugin(plugin)
result = {
'name': plugin['name'],
'scripts': [
{
'remote_url': adapter.master_scripts_path(cluster),
'local_path': adapter.slaves_scripts_path
}
]
}
if not adapter.repo_files(cluster):
return result
# TODO(bgaifullin) move priority to plugin metadata
if os_name == consts.RELEASE_OS.centos:
repo = {
'type': 'rpm',
'name': adapter.full_name,
'uri': adapter.repo_url(cluster),
'priority': settings.REPO_PRIORITIES['plugins']['centos']
}
elif os_name == consts.RELEASE_OS.ubuntu:
repo = {
'type': 'deb',
'name': adapter.full_name,
'uri': adapter.repo_url(cluster),
'suite': '/',
'section': '',
'priority': settings.REPO_PRIORITIES['plugins']['ubuntu']
}
else:
logger.warning("Unsupported OS: %s.", os_name)
return result
result['repositories'] = [repo]
return result
def serialize_plugin(cls, cluster, plugin):
os_name = cluster.release.operating_system
adapter = adapters.wrap_plugin(plugin)
result = {
'name': plugin['name'],
'scripts': [
{
'remote_url': adapter.master_scripts_path(cluster),
'local_path': adapter.slaves_scripts_path
}
]
}
if not adapter.repo_files(cluster):
return result
# TODO(bgaifullin) move priority to plugin metadata
if os_name == consts.RELEASE_OS.centos:
repo = {
'type': 'rpm',
'name': adapter.full_name,
'uri': adapter.repo_url(cluster),
'priority': settings.REPO_PRIORITIES['plugins']['centos']
}
elif os_name == consts.RELEASE_OS.ubuntu:
repo = {
'type': 'deb',
'name': adapter.full_name,
'uri': adapter.repo_url(cluster),
'suite': '/',
'section': '',
'priority': settings.REPO_PRIORITIES['plugins']['ubuntu']
}
else:
logger.warning("Unsupported OS: %s.", os_name)
return result
result['repositories'] = [repo]
return result
def get_enabled_plugins(cls, cluster):
return [
wrap_plugin(plugin)
for plugin in ClusterPlugin.get_enabled(cluster.id)
]
def get_enabled_plugins(cls, cluster):
return [wrap_plugin(plugin)
for plugin in ClusterPlugins.get_enabled(cluster.id)]