def sync_metadata_to_db(self):
"""Sync metadata from all config yaml files to DB"""
super(PluginAdapterV3, self).sync_metadata_to_db()
data_to_update = {}
db_config_metadata_mapping = {
'attributes_metadata': self.environment_config_name,
'roles_metadata': self.node_roles_config_name,
'volumes_metadata': self.volumes_config_name,
'network_roles_metadata': self.network_roles_config_name,
'deployment_tasks': self.deployment_tasks_config_name,
'tasks': self.task_config_name
}
for attribute, config in six.iteritems(db_config_metadata_mapping):
config_file_path = os.path.join(self.plugin_path, config)
attribute_data = self._load_config(config_file_path)
# Plugin columns have constraints for nullable data, so
# we need to check it
if attribute_data:
if attribute == 'attributes_metadata':
attribute_data = attribute_data['attributes']
data_to_update[attribute] = attribute_data
Plugin.update(self.plugin, data_to_update)
def sync_metadata_to_db(self):
"""Sync metadata from config yaml files into DB"""
metadata_file_path = os.path.join(self.plugin_path,
self.plugin_metadata)
metadata = self._load_config(metadata_file_path) or {}
Plugin.update(self.plugin, metadata)
def sync_metadata_to_db(self):
"""Sync metadata from config yaml files into DB"""
metadata_file_path = os.path.join(
self.plugin_path, self.plugin_metadata)
metadata = self._load_config(metadata_file_path) or {}
Plugin.update(self.plugin, metadata)
def process_cluster_attributes(cls, cluster, attributes):
"""Generate Cluster-Plugins relation based on attributes.
Iterates through plugins attributes, creates
or deletes Cluster <-> Plugins relation if plugin
is enabled or disabled.
:param cluster: A cluster instance
:type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster
:param attributes: Cluster attributes
:type attributes: dict
"""
from nailgun.objects import Release
plugins = {}
# Detach plugins data
for k in list(attributes):
if cls.is_plugin_data(attributes[k]):
plugins[k] = attributes.pop(k)['metadata']
propagate_task_deploy = get_in(
attributes, 'common', 'propagate_task_deploy', 'value')
if propagate_task_deploy is not None:
legacy_tasks_are_ignored = not propagate_task_deploy
else:
legacy_tasks_are_ignored = not get_in(
cluster.attributes.editable,
'common', 'propagate_task_deploy', 'value')
for container in six.itervalues(plugins):
default = container.get('default', False)
for attrs in container.get('versions', []):
version_metadata = attrs.pop('metadata')
plugin_id = version_metadata['plugin_id']
plugin = Plugin.get_by_uid(plugin_id)
if not plugin:
logger.warning(
'Plugin with id "%s" is not found, skip it', plugin_id)
continue
enabled = container['enabled'] \
and plugin_id == container['chosen_id']
if (enabled and
Release.is_lcm_supported(cluster.release) and
legacy_tasks_are_ignored and
cls.contains_legacy_tasks(
wrap_plugin(Plugin.get_by_uid(plugin.id)))):
raise errors.InvalidData(
'Cannot enable plugin with legacy tasks unless '
'propagate_task_deploy attribute is set. '
'Ensure tasks.yaml is empty and all tasks '
'has version >= 2.0.0.')
ClusterPlugin.set_attributes(
cluster.id, plugin.id, enabled=enabled,
attrs=attrs if enabled or default else None
)
def process_cluster_attributes(cls, cluster, attributes):
"""Generate Cluster-Plugins relation based on attributes.
Iterates through plugins attributes, creates
or deletes Cluster <-> Plugins relation if plugin
is enabled or disabled.
:param cluster: A cluster instance
:type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster
:param attributes: Cluster attributes
:type attributes: dict
"""
from nailgun.objects import Release
plugins = {}
# Detach plugins data
for k in list(attributes):
if cls.is_plugin_data(attributes[k]):
plugins[k] = attributes.pop(k)['metadata']
propagate_task_deploy = get_in(attributes, 'common',
'propagate_task_deploy', 'value')
if propagate_task_deploy is not None:
legacy_tasks_are_ignored = not propagate_task_deploy
else:
legacy_tasks_are_ignored = not get_in(
cluster.attributes.editable, 'common', 'propagate_task_deploy',
'value')
for container in six.itervalues(plugins):
default = container.get('default', False)
for attrs in container.get('versions', []):
version_metadata = attrs.pop('metadata')
plugin_id = version_metadata['plugin_id']
plugin = Plugin.get_by_uid(plugin_id)
if not plugin:
logger.warning('Plugin with id "%s" is not found, skip it',
plugin_id)
continue
enabled = container['enabled'] \
and plugin_id == container['chosen_id']
if (enabled and Release.is_lcm_supported(cluster.release)
and legacy_tasks_are_ignored
and cls.contains_legacy_tasks(
wrap_plugin(Plugin.get_by_uid(plugin.id)))):
raise errors.InvalidData(
'Cannot enable plugin with legacy tasks unless '
'propagate_task_deploy attribute is set. '
'Ensure tasks.yaml is empty and all tasks '
'has version >= 2.0.0.')
ClusterPlugin.set_attributes(
cluster.id,
plugin.id,
enabled=enabled,
attrs=attrs if enabled or default else None)
def _plugin_update(cls, plugin):
"""Update plugin metadata.
:param plugin: A plugin instance
:type plugin: plugin model
"""
try:
plugin_adapter = wrap_plugin(plugin)
metadata = plugin_adapter.get_metadata()
Plugin.update(plugin, metadata)
except Exception as e:
logger.error("cannot update plugin {0} in DB. Reason: {1}".format(
plugin.name, str(e)))
def _update_plugin(self, mapping):
data_to_update = {}
for attribute, config in six.iteritems(mapping):
config_file_path = os.path.join(self.plugin_path, config)
attribute_data = self._load_config(config_file_path)
# Plugin columns have constraints for nullable data, so
# we need to check it
if attribute_data:
if attribute == 'attributes_metadata':
attribute_data = attribute_data['attributes']
data_to_update[attribute] = attribute_data
Plugin.update(self.plugin, data_to_update)
def _update_plugin(self, mapping):
data_to_update = {}
for attribute, config in six.iteritems(mapping):
config_file_path = os.path.join(self.plugin_path, config)
attribute_data = self._load_config(config_file_path)
# Plugin columns have constraints for nullable data, so
# we need to check it
if attribute_data:
if attribute == 'attributes_metadata':
attribute_data = attribute_data['attributes']
data_to_update[attribute] = attribute_data
Plugin.update(self.plugin, data_to_update)
def _plugin_create(cls, plugin_dir):
"""Create plugin using metadata.
:param plugin_dir: a plugin directory name on FS
:type plugin_dir: str
"""
plugin_path = os.path.join(settings.PLUGINS_PATH, plugin_dir,
'metadata.yaml')
try:
plugin_metadata = cls._parse_yaml_file(plugin_path)
Plugin.create(plugin_metadata)
except Exception as e:
logger.error("cannot create plugin {0} from FS. Reason: {1}"
.format(plugin_dir, str(e)))
def _plugin_create(cls, plugin_dir):
"""Create plugin using metadata.
:param plugin_dir: a plugin directory name on FS
:type plugin_dir: str
"""
plugin_path = os.path.join(settings.PLUGINS_PATH, plugin_dir,
'metadata.yaml')
try:
plugin_metadata = cls._parse_yaml_file(plugin_path)
Plugin.create(plugin_metadata)
except Exception as e:
logger.error(
"cannot create plugin {0} from FS. Reason: {1}".format(
plugin_dir, str(e)))
def _plugin_update(cls, plugin):
"""Update plugin metadata.
:param plugin: A plugin instance
:type plugin: plugin model
"""
try:
plugin_adapter = wrap_plugin(plugin)
metadata = plugin_adapter.get_metadata()
Plugin.update(plugin, metadata)
except errors.InvalidData:
raise
except Exception as e:
logger.error("cannot update plugin {0} in DB. Reason: {1}"
.format(plugin.name, str(e)))
def sync_plugins_metadata(cls, plugin_ids=None):
"""Sync metadata for plugins by given IDs.
If there are no IDs, all newest plugins will be synced.
:param plugin_ids: list of plugin IDs
:type plugin_ids: list
"""
if plugin_ids:
plugins = PluginCollection.get_by_uids(plugin_ids)
else:
plugins = PluginCollection.all()
for plugin in plugins:
plugin_adapter = wrap_plugin(plugin)
metadata = plugin_adapter.get_metadata()
Plugin.update(plugin, metadata)
def process_cluster_attributes(cls, cluster, attributes):
"""Generate Cluster-Plugins relation based on attributes.
Iterates through plugins attributes, creates
or deletes Cluster <-> Plugins relation if plugin
is enabled or disabled.
:param cluster: A cluster instance
:type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster
:param attributes: Cluster attributes
:type attributes: dict
"""
plugins = {}
# Detach plugins data
for k in list(attributes):
if cls.is_plugin_data(attributes[k]):
plugins[k] = attributes.pop(k)['metadata']
for container in six.itervalues(plugins):
default = container.get('default', False)
for attrs in container.get('versions', []):
version_metadata = attrs.pop('metadata')
plugin_id = version_metadata['plugin_id']
plugin = Plugin.get_by_uid(plugin_id)
if not plugin:
logger.warning(
'Plugin with id "%s" is not found, skip it', plugin_id)
continue
enabled = container['enabled']\
and plugin_id == container['chosen_id']
ClusterPlugin.set_attributes(
cluster.id, plugin.id, enabled=enabled,
attrs=attrs if enabled or default else None
)
def _process_attr(cls, cluster, attr):
if not isinstance(attr, dict):
return
metadata = attr.get('metadata', {})
plugin_id = metadata.get('plugin_id')
if not plugin_id:
return
plugin = Plugin.get_by_uid(plugin_id)
if not plugin:
logger.warning('Plugin with id "%s" is not found, skip it',
plugin_id)
return
enabled = metadata.get('enabled', False)
# Value is true and plugin is not enabled for this cluster
# that means plugin was enabled on this request
if enabled and cluster not in plugin.clusters:
plugin.clusters.append(cluster)
# Value is false and plugin is enabled for this cluster
# that means plugin was disabled on this request
elif not enabled and cluster in plugin.clusters:
plugin.clusters.remove(cluster)
def _process_attr(cls, cluster, attr):
if not isinstance(attr, dict):
return
metadata = attr.get('metadata', {})
plugin_id = metadata.get('plugin_id')
if not plugin_id:
return
plugin = Plugin.get_by_uid(plugin_id)
if not plugin:
logger.warning('Plugin with id "%s" is not found, skip it',
plugin_id)
return
enabled = metadata.get('enabled', False)
# Value is true and plugin is not enabled for this cluster
# that means plugin was enabled on this request
if enabled and cluster not in plugin.clusters:
plugin.clusters.append(cluster)
# Value is false and plugin is enabled for this cluster
# that means plugin was disabled on this request
elif not enabled and cluster in plugin.clusters:
plugin.clusters.remove(cluster)
def sync_plugins_metadata(cls, plugin_ids=None):
"""Sync metadata for plugins by given IDs.
If there are no IDs, all newest plugins will be synced.
:param plugin_ids: list of plugin IDs
:type plugin_ids: list
"""
if plugin_ids:
plugins = PluginCollection.get_by_uids(plugin_ids)
else:
plugins = PluginCollection.all()
for plugin in plugins:
plugin_adapter = wrap_plugin(plugin)
metadata = plugin_adapter.get_metadata()
Plugin.update(plugin, metadata)
def _plugin_delete(cls, plugin):
"""Delete plugin
:param plugin: A plugin instance
:type plugin: plugin model
"""
if cls._is_plugin_deletable(plugin):
try:
Plugin.delete(plugin)
except Exception as e:
logger.error("cannot delete plugin {0} from DB. Reason: {1}"
.format(plugin.name, str(e)))
else:
logger.error("cannot delete plugin {0} from DB. The one of "
"possible reasons: is still used at least in "
"one cluster, but it is already deleted on "
"filesystem. Please reinstall it using package "
"or disable it in every cluster".format(plugin.name))
def _plugin_delete(cls, plugin):
"""Delete plugin
:param plugin: A plugin instance
:type plugin: plugin model
"""
if cls._is_plugin_deletable(plugin):
try:
Plugin.delete(plugin)
except Exception as e:
logger.error(
"cannot delete plugin {0} from DB. Reason: {1}".format(
plugin.name, str(e)))
else:
logger.error("cannot delete plugin {0} from DB. The one of "
"possible reasons: is still used at least in "
"one cluster, but it is already deleted on "
"filesystem. Please reinstall it using package "
"or disable it in every cluster".format(plugin.name))
def process_cluster_attributes(cls, cluster, attributes):
"""Generate Cluster-Plugins relation based on attributes
Iterates through plugins attributes, creates
or deletes Cluster <-> Plugins relation if plugin
is enabled or disabled.
:param cluster: A cluster instance
:type cluster: nailgun.objects.cluster.Cluster
:param attributes: Cluster attributes
:type attributes: dict
"""
def _convert_attrs(plugin_id, attrs):
prefix = "#{0}_".format(plugin_id)
result = dict((title[len(prefix):], attrs[title])
for title in attrs if title.startswith(prefix))
for attr in six.itervalues(result):
if 'restrictions' not in attr:
continue
if len(attr['restrictions']) == 1:
attr.pop('restrictions')
else:
attr['restrictions'].pop()
return result
for attrs in six.itervalues(attributes):
if not isinstance(attrs, dict):
continue
plugin_versions = attrs.pop('plugin_versions', None)
if plugin_versions is None:
continue
metadata = attrs.pop('metadata', {})
plugin_enabled = metadata.get('enabled', False)
default = metadata.get('default', False)
for version in plugin_versions['values']:
pid = version.get('data')
plugin = Plugin.get_by_uid(pid)
if not plugin:
logger.warning('Plugin with id "%s" is not found, skip it',
pid)
continue
enabled = plugin_enabled and\
pid == plugin_versions['value']
ClusterPlugins.set_attributes(
cluster.id,
plugin.id,
enabled=enabled,
attrs=_convert_attrs(plugin.id, attrs)
if enabled or default else None)
def process_cluster_attributes(cls, cluster, attributes):
"""Generate Cluster-Plugins relation based on attributes
Iterates through plugins attributes, creates
or deletes Cluster <-> Plugins relation if plugin
is enabled or disabled.
:param cluster: A cluster instance
:type cluster: nailgun.objects.cluster.Cluster
:param attributes: Cluster attributes
:type attributes: dict
"""
def _convert_attrs(plugin_id, attrs):
prefix = "#{0}_".format(plugin_id)
result = dict((title[len(prefix):], attrs[title])
for title in attrs
if title.startswith(prefix))
for attr in six.itervalues(result):
if 'restrictions' not in attr:
continue
if len(attr['restrictions']) == 1:
attr.pop('restrictions')
else:
attr['restrictions'].pop()
return result
for attrs in six.itervalues(attributes):
if not isinstance(attrs, dict):
continue
plugin_versions = attrs.pop('plugin_versions', None)
if plugin_versions is None:
continue
metadata = attrs.pop('metadata', {})
plugin_enabled = metadata.get('enabled', False)
default = metadata.get('default', False)
for version in plugin_versions['values']:
pid = version.get('data')
plugin = Plugin.get_by_uid(pid)
if not plugin:
logger.warning(
'Plugin with id "%s" is not found, skip it', pid)
continue
enabled = plugin_enabled and\
pid == plugin_versions['value']
ClusterPlugins.set_attributes(
cluster.id, plugin.id, enabled=enabled,
attrs=_convert_attrs(plugin.id, attrs)
if enabled or default else None
)
def sync_metadata_to_db(self):
super(PluginAdapterV3, self).sync_metadata_to_db()
data_to_update = {}
db_config_metadata_mapping = {
'attributes_metadata': self.environment_config_name,
'roles_metadata': self.node_roles_config_name,
'volumes_metadata': self.volumes_config_name,
'deployment_tasks': self.deployment_tasks_config_name,
'tasks': self.task_config_name
}
for attribute, config in six.iteritems(db_config_metadata_mapping):
config_file_path = os.path.join(self.plugin_path, config)
attribute_data = self._load_config(config_file_path)
# Plugin columns have constraints for nullable data, so
# we need to check it
if attribute_data:
data_to_update[attribute] = attribute_data
Plugin.update(self.plugin, data_to_update)
def get_plugins_attributes(cls,
cluster,
all_versions=False,
default=False):
"""Gets attributes of all plugins connected with given cluster.
:param cluster: A cluster instance
:type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster
:param all_versions: True to get attributes of all versions of plugins
:type all_versions: bool
:param default: True to return a default plugins attributes (for UI)
:type default: bool
:return: Plugins attributes
:rtype: dict
"""
plugins_attributes = {}
for plugin in ClusterPlugin.get_connected_plugins_data(cluster.id):
db_plugin = Plugin.get_by_uid(plugin.id)
plugin_adapter = wrap_plugin(db_plugin)
default_attrs = plugin_adapter.attributes_metadata
if all_versions:
container = plugins_attributes.setdefault(plugin.name, {})
enabled = plugin.enabled and not (all_versions and default)
cls.create_common_metadata(plugin, container, enabled)
container['metadata']['default'] = default
versions = container['metadata'].setdefault('versions', [])
if default:
actual_attrs = copy.deepcopy(default_attrs)
actual_attrs.setdefault('metadata', {})
else:
actual_attrs = copy.deepcopy(plugin.attributes)
actual_attrs['metadata'] = default_attrs.get(
'metadata', {})
actual_attrs['metadata']['contains_legacy_tasks'] = \
cls.contains_legacy_tasks(plugin_adapter)
cls.fill_plugin_metadata(plugin, actual_attrs['metadata'])
versions.append(actual_attrs)
container['metadata'].setdefault('chosen_id', plugin.id)
if enabled:
container['metadata']['chosen_id'] = plugin.id
elif plugin.enabled:
container = plugins_attributes.setdefault(plugin.name, {})
cls.create_common_metadata(plugin, container)
container['metadata'].update(default_attrs.get('metadata', {}))
cls.fill_plugin_metadata(plugin, container['metadata'])
container.update(plugin.attributes)
return plugins_attributes
def get_plugins_attributes(
cls, cluster, all_versions=False, default=False):
"""Gets attributes of all plugins connected with given cluster.
:param cluster: A cluster instance
:type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster
:param all_versions: True to get attributes of all versions of plugins
:type all_versions: bool
:param default: True to return a default plugins attributes (for UI)
:type default: bool
:return: Plugins attributes
:rtype: dict
"""
plugins_attributes = {}
for plugin in ClusterPlugin.get_connected_plugins_data(cluster.id):
db_plugin = Plugin.get_by_uid(plugin.id)
plugin_adapter = wrap_plugin(db_plugin)
default_attrs = plugin_adapter.attributes_metadata
if all_versions:
container = plugins_attributes.setdefault(plugin.name, {})
enabled = plugin.enabled and not (all_versions and default)
cls.create_common_metadata(plugin, container, enabled)
container['metadata']['default'] = default
versions = container['metadata'].setdefault('versions', [])
if default:
actual_attrs = copy.deepcopy(default_attrs)
actual_attrs.setdefault('metadata', {})
else:
actual_attrs = copy.deepcopy(plugin.attributes)
actual_attrs['metadata'] = default_attrs.get('metadata',
{})
actual_attrs['metadata']['contains_legacy_tasks'] = \
cls.contains_legacy_tasks(plugin_adapter)
cls.fill_plugin_metadata(plugin, actual_attrs['metadata'])
versions.append(actual_attrs)
container['metadata'].setdefault('chosen_id', plugin.id)
if enabled:
container['metadata']['chosen_id'] = plugin.id
elif plugin.enabled:
container = plugins_attributes.setdefault(plugin.name, {})
cls.create_common_metadata(plugin, container)
container['metadata'].update(default_attrs.get('metadata', {}))
cls.fill_plugin_metadata(plugin, container['metadata'])
container.update(plugin.attributes)
return plugins_attributes
def process_cluster_attributes(cls, cluster, attributes):
"""Generate Cluster-Plugins relation based on attributes.
Iterates through plugins attributes, creates
or deletes Cluster <-> Plugins relation if plugin
is enabled or disabled.
:param cluster: A cluster instance
:type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster
:param attributes: Cluster attributes
:type attributes: dict
"""
plugins = {}
# Detach plugins data
for k in list(attributes):
if cls.is_plugin_data(attributes[k]):
plugins[k] = attributes.pop(k)['metadata']
cluster.attributes.editable.pop(k, None)
for container in six.itervalues(plugins):
default = container.get('default', False)
for attrs in container.get('versions', []):
version_metadata = attrs.pop('metadata')
plugin_id = version_metadata['plugin_id']
plugin = Plugin.get_by_uid(plugin_id)
if not plugin:
logger.warning('Plugin with id "%s" is not found, skip it',
plugin_id)
continue
enabled = container['enabled']\
and plugin_id == container['chosen_id']
ClusterPlugins.set_attributes(
cluster.id,
plugin.id,
enabled=enabled,
attrs=attrs if enabled or default else None)
