def sync_metadata_to_db(self): """Sync metadata from all config yaml files to DB""" super(PluginAdapterV3, self).sync_metadata_to_db() data_to_update = {} db_config_metadata_mapping = { 'attributes_metadata': self.environment_config_name, 'roles_metadata': self.node_roles_config_name, 'volumes_metadata': self.volumes_config_name, 'network_roles_metadata': self.network_roles_config_name, 'deployment_tasks': self.deployment_tasks_config_name, 'tasks': self.task_config_name } for attribute, config in six.iteritems(db_config_metadata_mapping): config_file_path = os.path.join(self.plugin_path, config) attribute_data = self._load_config(config_file_path) # Plugin columns have constraints for nullable data, so # we need to check it if attribute_data: if attribute == 'attributes_metadata': attribute_data = attribute_data['attributes'] data_to_update[attribute] = attribute_data Plugin.update(self.plugin, data_to_update)
def sync_metadata_to_db(self): """Sync metadata from config yaml files into DB""" metadata_file_path = os.path.join(self.plugin_path, self.plugin_metadata) metadata = self._load_config(metadata_file_path) or {} Plugin.update(self.plugin, metadata)
def sync_metadata_to_db(self): """Sync metadata from config yaml files into DB""" metadata_file_path = os.path.join( self.plugin_path, self.plugin_metadata) metadata = self._load_config(metadata_file_path) or {} Plugin.update(self.plugin, metadata)
def process_cluster_attributes(cls, cluster, attributes): """Generate Cluster-Plugins relation based on attributes. Iterates through plugins attributes, creates or deletes Cluster <-> Plugins relation if plugin is enabled or disabled. :param cluster: A cluster instance :type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster :param attributes: Cluster attributes :type attributes: dict """ from nailgun.objects import Release plugins = {} # Detach plugins data for k in list(attributes): if cls.is_plugin_data(attributes[k]): plugins[k] = attributes.pop(k)['metadata'] propagate_task_deploy = get_in( attributes, 'common', 'propagate_task_deploy', 'value') if propagate_task_deploy is not None: legacy_tasks_are_ignored = not propagate_task_deploy else: legacy_tasks_are_ignored = not get_in( cluster.attributes.editable, 'common', 'propagate_task_deploy', 'value') for container in six.itervalues(plugins): default = container.get('default', False) for attrs in container.get('versions', []): version_metadata = attrs.pop('metadata') plugin_id = version_metadata['plugin_id'] plugin = Plugin.get_by_uid(plugin_id) if not plugin: logger.warning( 'Plugin with id "%s" is not found, skip it', plugin_id) continue enabled = container['enabled'] \ and plugin_id == container['chosen_id'] if (enabled and Release.is_lcm_supported(cluster.release) and legacy_tasks_are_ignored and cls.contains_legacy_tasks( wrap_plugin(Plugin.get_by_uid(plugin.id)))): raise errors.InvalidData( 'Cannot enable plugin with legacy tasks unless ' 'propagate_task_deploy attribute is set. ' 'Ensure tasks.yaml is empty and all tasks ' 'has version >= 2.0.0.') ClusterPlugin.set_attributes( cluster.id, plugin.id, enabled=enabled, attrs=attrs if enabled or default else None )
def process_cluster_attributes(cls, cluster, attributes): """Generate Cluster-Plugins relation based on attributes. Iterates through plugins attributes, creates or deletes Cluster <-> Plugins relation if plugin is enabled or disabled. :param cluster: A cluster instance :type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster :param attributes: Cluster attributes :type attributes: dict """ from nailgun.objects import Release plugins = {} # Detach plugins data for k in list(attributes): if cls.is_plugin_data(attributes[k]): plugins[k] = attributes.pop(k)['metadata'] propagate_task_deploy = get_in(attributes, 'common', 'propagate_task_deploy', 'value') if propagate_task_deploy is not None: legacy_tasks_are_ignored = not propagate_task_deploy else: legacy_tasks_are_ignored = not get_in( cluster.attributes.editable, 'common', 'propagate_task_deploy', 'value') for container in six.itervalues(plugins): default = container.get('default', False) for attrs in container.get('versions', []): version_metadata = attrs.pop('metadata') plugin_id = version_metadata['plugin_id'] plugin = Plugin.get_by_uid(plugin_id) if not plugin: logger.warning('Plugin with id "%s" is not found, skip it', plugin_id) continue enabled = container['enabled'] \ and plugin_id == container['chosen_id'] if (enabled and Release.is_lcm_supported(cluster.release) and legacy_tasks_are_ignored and cls.contains_legacy_tasks( wrap_plugin(Plugin.get_by_uid(plugin.id)))): raise errors.InvalidData( 'Cannot enable plugin with legacy tasks unless ' 'propagate_task_deploy attribute is set. ' 'Ensure tasks.yaml is empty and all tasks ' 'has version >= 2.0.0.') ClusterPlugin.set_attributes( cluster.id, plugin.id, enabled=enabled, attrs=attrs if enabled or default else None)
def _plugin_update(cls, plugin): """Update plugin metadata. :param plugin: A plugin instance :type plugin: plugin model """ try: plugin_adapter = wrap_plugin(plugin) metadata = plugin_adapter.get_metadata() Plugin.update(plugin, metadata) except Exception as e: logger.error("cannot update plugin {0} in DB. Reason: {1}".format( plugin.name, str(e)))
def _update_plugin(self, mapping): data_to_update = {} for attribute, config in six.iteritems(mapping): config_file_path = os.path.join(self.plugin_path, config) attribute_data = self._load_config(config_file_path) # Plugin columns have constraints for nullable data, so # we need to check it if attribute_data: if attribute == 'attributes_metadata': attribute_data = attribute_data['attributes'] data_to_update[attribute] = attribute_data Plugin.update(self.plugin, data_to_update)
def _plugin_create(cls, plugin_dir): """Create plugin using metadata. :param plugin_dir: a plugin directory name on FS :type plugin_dir: str """ plugin_path = os.path.join(settings.PLUGINS_PATH, plugin_dir, 'metadata.yaml') try: plugin_metadata = cls._parse_yaml_file(plugin_path) Plugin.create(plugin_metadata) except Exception as e: logger.error("cannot create plugin {0} from FS. Reason: {1}" .format(plugin_dir, str(e)))
def _plugin_create(cls, plugin_dir): """Create plugin using metadata. :param plugin_dir: a plugin directory name on FS :type plugin_dir: str """ plugin_path = os.path.join(settings.PLUGINS_PATH, plugin_dir, 'metadata.yaml') try: plugin_metadata = cls._parse_yaml_file(plugin_path) Plugin.create(plugin_metadata) except Exception as e: logger.error( "cannot create plugin {0} from FS. Reason: {1}".format( plugin_dir, str(e)))
def _plugin_update(cls, plugin): """Update plugin metadata. :param plugin: A plugin instance :type plugin: plugin model """ try: plugin_adapter = wrap_plugin(plugin) metadata = plugin_adapter.get_metadata() Plugin.update(plugin, metadata) except errors.InvalidData: raise except Exception as e: logger.error("cannot update plugin {0} in DB. Reason: {1}" .format(plugin.name, str(e)))
def sync_plugins_metadata(cls, plugin_ids=None): """Sync metadata for plugins by given IDs. If there are no IDs, all newest plugins will be synced. :param plugin_ids: list of plugin IDs :type plugin_ids: list """ if plugin_ids: plugins = PluginCollection.get_by_uids(plugin_ids) else: plugins = PluginCollection.all() for plugin in plugins: plugin_adapter = wrap_plugin(plugin) metadata = plugin_adapter.get_metadata() Plugin.update(plugin, metadata)
def process_cluster_attributes(cls, cluster, attributes): """Generate Cluster-Plugins relation based on attributes. Iterates through plugins attributes, creates or deletes Cluster <-> Plugins relation if plugin is enabled or disabled. :param cluster: A cluster instance :type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster :param attributes: Cluster attributes :type attributes: dict """ plugins = {} # Detach plugins data for k in list(attributes): if cls.is_plugin_data(attributes[k]): plugins[k] = attributes.pop(k)['metadata'] for container in six.itervalues(plugins): default = container.get('default', False) for attrs in container.get('versions', []): version_metadata = attrs.pop('metadata') plugin_id = version_metadata['plugin_id'] plugin = Plugin.get_by_uid(plugin_id) if not plugin: logger.warning( 'Plugin with id "%s" is not found, skip it', plugin_id) continue enabled = container['enabled']\ and plugin_id == container['chosen_id'] ClusterPlugin.set_attributes( cluster.id, plugin.id, enabled=enabled, attrs=attrs if enabled or default else None )
def _process_attr(cls, cluster, attr): if not isinstance(attr, dict): return metadata = attr.get('metadata', {}) plugin_id = metadata.get('plugin_id') if not plugin_id: return plugin = Plugin.get_by_uid(plugin_id) if not plugin: logger.warning('Plugin with id "%s" is not found, skip it', plugin_id) return enabled = metadata.get('enabled', False) # Value is true and plugin is not enabled for this cluster # that means plugin was enabled on this request if enabled and cluster not in plugin.clusters: plugin.clusters.append(cluster) # Value is false and plugin is enabled for this cluster # that means plugin was disabled on this request elif not enabled and cluster in plugin.clusters: plugin.clusters.remove(cluster)
def _plugin_delete(cls, plugin): """Delete plugin :param plugin: A plugin instance :type plugin: plugin model """ if cls._is_plugin_deletable(plugin): try: Plugin.delete(plugin) except Exception as e: logger.error("cannot delete plugin {0} from DB. Reason: {1}" .format(plugin.name, str(e))) else: logger.error("cannot delete plugin {0} from DB. The one of " "possible reasons: is still used at least in " "one cluster, but it is already deleted on " "filesystem. Please reinstall it using package " "or disable it in every cluster".format(plugin.name))
def _plugin_delete(cls, plugin): """Delete plugin :param plugin: A plugin instance :type plugin: plugin model """ if cls._is_plugin_deletable(plugin): try: Plugin.delete(plugin) except Exception as e: logger.error( "cannot delete plugin {0} from DB. Reason: {1}".format( plugin.name, str(e))) else: logger.error("cannot delete plugin {0} from DB. The one of " "possible reasons: is still used at least in " "one cluster, but it is already deleted on " "filesystem. Please reinstall it using package " "or disable it in every cluster".format(plugin.name))
def process_cluster_attributes(cls, cluster, attributes): """Generate Cluster-Plugins relation based on attributes Iterates through plugins attributes, creates or deletes Cluster <-> Plugins relation if plugin is enabled or disabled. :param cluster: A cluster instance :type cluster: nailgun.objects.cluster.Cluster :param attributes: Cluster attributes :type attributes: dict """ def _convert_attrs(plugin_id, attrs): prefix = "#{0}_".format(plugin_id) result = dict((title[len(prefix):], attrs[title]) for title in attrs if title.startswith(prefix)) for attr in six.itervalues(result): if 'restrictions' not in attr: continue if len(attr['restrictions']) == 1: attr.pop('restrictions') else: attr['restrictions'].pop() return result for attrs in six.itervalues(attributes): if not isinstance(attrs, dict): continue plugin_versions = attrs.pop('plugin_versions', None) if plugin_versions is None: continue metadata = attrs.pop('metadata', {}) plugin_enabled = metadata.get('enabled', False) default = metadata.get('default', False) for version in plugin_versions['values']: pid = version.get('data') plugin = Plugin.get_by_uid(pid) if not plugin: logger.warning('Plugin with id "%s" is not found, skip it', pid) continue enabled = plugin_enabled and\ pid == plugin_versions['value'] ClusterPlugins.set_attributes( cluster.id, plugin.id, enabled=enabled, attrs=_convert_attrs(plugin.id, attrs) if enabled or default else None)
def process_cluster_attributes(cls, cluster, attributes): """Generate Cluster-Plugins relation based on attributes Iterates through plugins attributes, creates or deletes Cluster <-> Plugins relation if plugin is enabled or disabled. :param cluster: A cluster instance :type cluster: nailgun.objects.cluster.Cluster :param attributes: Cluster attributes :type attributes: dict """ def _convert_attrs(plugin_id, attrs): prefix = "#{0}_".format(plugin_id) result = dict((title[len(prefix):], attrs[title]) for title in attrs if title.startswith(prefix)) for attr in six.itervalues(result): if 'restrictions' not in attr: continue if len(attr['restrictions']) == 1: attr.pop('restrictions') else: attr['restrictions'].pop() return result for attrs in six.itervalues(attributes): if not isinstance(attrs, dict): continue plugin_versions = attrs.pop('plugin_versions', None) if plugin_versions is None: continue metadata = attrs.pop('metadata', {}) plugin_enabled = metadata.get('enabled', False) default = metadata.get('default', False) for version in plugin_versions['values']: pid = version.get('data') plugin = Plugin.get_by_uid(pid) if not plugin: logger.warning( 'Plugin with id "%s" is not found, skip it', pid) continue enabled = plugin_enabled and\ pid == plugin_versions['value'] ClusterPlugins.set_attributes( cluster.id, plugin.id, enabled=enabled, attrs=_convert_attrs(plugin.id, attrs) if enabled or default else None )
def sync_metadata_to_db(self): super(PluginAdapterV3, self).sync_metadata_to_db() data_to_update = {} db_config_metadata_mapping = { 'attributes_metadata': self.environment_config_name, 'roles_metadata': self.node_roles_config_name, 'volumes_metadata': self.volumes_config_name, 'deployment_tasks': self.deployment_tasks_config_name, 'tasks': self.task_config_name } for attribute, config in six.iteritems(db_config_metadata_mapping): config_file_path = os.path.join(self.plugin_path, config) attribute_data = self._load_config(config_file_path) # Plugin columns have constraints for nullable data, so # we need to check it if attribute_data: data_to_update[attribute] = attribute_data Plugin.update(self.plugin, data_to_update)
def get_plugins_attributes(cls, cluster, all_versions=False, default=False): """Gets attributes of all plugins connected with given cluster. :param cluster: A cluster instance :type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster :param all_versions: True to get attributes of all versions of plugins :type all_versions: bool :param default: True to return a default plugins attributes (for UI) :type default: bool :return: Plugins attributes :rtype: dict """ plugins_attributes = {} for plugin in ClusterPlugin.get_connected_plugins_data(cluster.id): db_plugin = Plugin.get_by_uid(plugin.id) plugin_adapter = wrap_plugin(db_plugin) default_attrs = plugin_adapter.attributes_metadata if all_versions: container = plugins_attributes.setdefault(plugin.name, {}) enabled = plugin.enabled and not (all_versions and default) cls.create_common_metadata(plugin, container, enabled) container['metadata']['default'] = default versions = container['metadata'].setdefault('versions', []) if default: actual_attrs = copy.deepcopy(default_attrs) actual_attrs.setdefault('metadata', {}) else: actual_attrs = copy.deepcopy(plugin.attributes) actual_attrs['metadata'] = default_attrs.get( 'metadata', {}) actual_attrs['metadata']['contains_legacy_tasks'] = \ cls.contains_legacy_tasks(plugin_adapter) cls.fill_plugin_metadata(plugin, actual_attrs['metadata']) versions.append(actual_attrs) container['metadata'].setdefault('chosen_id', plugin.id) if enabled: container['metadata']['chosen_id'] = plugin.id elif plugin.enabled: container = plugins_attributes.setdefault(plugin.name, {}) cls.create_common_metadata(plugin, container) container['metadata'].update(default_attrs.get('metadata', {})) cls.fill_plugin_metadata(plugin, container['metadata']) container.update(plugin.attributes) return plugins_attributes
def get_plugins_attributes( cls, cluster, all_versions=False, default=False): """Gets attributes of all plugins connected with given cluster. :param cluster: A cluster instance :type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster :param all_versions: True to get attributes of all versions of plugins :type all_versions: bool :param default: True to return a default plugins attributes (for UI) :type default: bool :return: Plugins attributes :rtype: dict """ plugins_attributes = {} for plugin in ClusterPlugin.get_connected_plugins_data(cluster.id): db_plugin = Plugin.get_by_uid(plugin.id) plugin_adapter = wrap_plugin(db_plugin) default_attrs = plugin_adapter.attributes_metadata if all_versions: container = plugins_attributes.setdefault(plugin.name, {}) enabled = plugin.enabled and not (all_versions and default) cls.create_common_metadata(plugin, container, enabled) container['metadata']['default'] = default versions = container['metadata'].setdefault('versions', []) if default: actual_attrs = copy.deepcopy(default_attrs) actual_attrs.setdefault('metadata', {}) else: actual_attrs = copy.deepcopy(plugin.attributes) actual_attrs['metadata'] = default_attrs.get('metadata', {}) actual_attrs['metadata']['contains_legacy_tasks'] = \ cls.contains_legacy_tasks(plugin_adapter) cls.fill_plugin_metadata(plugin, actual_attrs['metadata']) versions.append(actual_attrs) container['metadata'].setdefault('chosen_id', plugin.id) if enabled: container['metadata']['chosen_id'] = plugin.id elif plugin.enabled: container = plugins_attributes.setdefault(plugin.name, {}) cls.create_common_metadata(plugin, container) container['metadata'].update(default_attrs.get('metadata', {})) cls.fill_plugin_metadata(plugin, container['metadata']) container.update(plugin.attributes) return plugins_attributes
def process_cluster_attributes(cls, cluster, attributes): """Generate Cluster-Plugins relation based on attributes. Iterates through plugins attributes, creates or deletes Cluster <-> Plugins relation if plugin is enabled or disabled. :param cluster: A cluster instance :type cluster: nailgun.db.sqlalchemy.models.cluster.Cluster :param attributes: Cluster attributes :type attributes: dict """ plugins = {} # Detach plugins data for k in list(attributes): if cls.is_plugin_data(attributes[k]): plugins[k] = attributes.pop(k)['metadata'] cluster.attributes.editable.pop(k, None) for container in six.itervalues(plugins): default = container.get('default', False) for attrs in container.get('versions', []): version_metadata = attrs.pop('metadata') plugin_id = version_metadata['plugin_id'] plugin = Plugin.get_by_uid(plugin_id) if not plugin: logger.warning('Plugin with id "%s" is not found, skip it', plugin_id) continue enabled = container['enabled']\ and plugin_id == container['chosen_id'] ClusterPlugins.set_attributes( cluster.id, plugin.id, enabled=enabled, attrs=attrs if enabled or default else None)