def get_plugins_deployment_graph(cls, cluster, graph_type=None): deployment_tasks = [] processed_tasks = {} enabled_plugins = ClusterPlugin.get_enabled(cluster.id) graph_metadata = {} for plugin_adapter in map(wrap_plugin, enabled_plugins): depl_graph = plugin_adapter.get_deployment_graph(graph_type) depl_tasks = depl_graph.pop('tasks') dict_update(graph_metadata, depl_graph) for t in depl_tasks: t_id = t['id'] if t_id in processed_tasks: raise errors.AlreadyExists( 'Plugin {0} is overlapping with plugin {1} ' 'by introducing the same deployment task with ' 'id {2}'.format( plugin_adapter.full_name, processed_tasks[t_id], t_id ) ) processed_tasks[t_id] = plugin_adapter.full_name deployment_tasks.extend(depl_tasks) graph_metadata['tasks'] = deployment_tasks return graph_metadata
def get_common_attrs(self, cluster): """Cluster attributes.""" # tests call this method directly. # and we need this workaround to avoid refactoring a lot of tests. self._ensure_initialized_for(cluster) attrs = objects.Cluster.get_attributes(cluster) attrs = objects.Attributes.merged_attrs_values(attrs) attrs['deployment_mode'] = cluster.mode attrs['deployment_id'] = cluster.id attrs['openstack_version'] = cluster.release.version attrs['fuel_version'] = cluster.fuel_version attrs['nodes'] = self.node_list(self.all_nodes) # Adding params to workloads_collector if 'workloads_collector' not in attrs: attrs['workloads_collector'] = {} attrs['workloads_collector']['create_user'] = \ objects.MasterNodeSettings.must_send_stats() username = attrs['workloads_collector'].pop('user', None) attrs['workloads_collector']['username'] = username if self.role_resolver.resolve(['cinder']): attrs['use_cinder'] = True net_serializer = self.get_net_provider_serializer(cluster) net_common_attrs = net_serializer.get_common_attrs(cluster, attrs) utils.dict_update(attrs, net_common_attrs) self.inject_list_of_plugins(attrs, cluster) return attrs
def _get_current_state(cluster, nodes, tasks, force=False): # In case of force=True, the current state is {} which means: behave like # an intial deployment. if force: return {} nodes = {n.uid: n for n in nodes} nodes[consts.MASTER_NODE_UID] = None tasks_names = [ t['id'] for t in tasks if t['type'] not in consts.INTERNAL_TASKS ] txs = objects.TransactionCollection.get_successful_transactions_per_task( cluster.id, tasks_names, nodes ) state = {} for tx, data in itertools.groupby(txs, lambda x: x[0]): node_ids = [] deferred_state = {} for _, node_id, task_name in data: t_state = state.setdefault(task_name, {}) if _is_node_for_redeploy(nodes.get(node_id)): t_state[node_id] = {} else: t_state[node_id] = deferred_state.setdefault(node_id, {}) node_ids.append(node_id) dict_update( deferred_state, objects.Transaction.get_deployment_info(tx, node_uids=node_ids), level=2 ) return state
def _get_current_state(cluster, nodes, tasks, force=False): # In case of force=True, the current state is {} which means: behave like # an intial deployment. if force: return {} nodes = {n.uid: n for n in nodes} nodes[consts.MASTER_NODE_UID] = None tasks_names = [t["id"] for t in tasks if t["type"] not in consts.INTERNAL_TASKS] txs = objects.TransactionCollection.get_successful_transactions_per_task(cluster.id, tasks_names, nodes) state = {} for tx, data in itertools.groupby(txs, lambda x: x[0]): node_ids = [] deferred_state = {} for _, node_id, task_name in data: t_state = state.setdefault(task_name, {}) if _is_node_for_redeploy(nodes.get(node_id)): t_state[node_id] = {} else: t_state[node_id] = deferred_state.setdefault(node_id, {}) node_ids.append(node_id) dict_update(deferred_state, objects.Transaction.get_deployment_info(tx, node_uids=node_ids), level=2) return state
def get_common_attrs(self, cluster): """Cluster attributes.""" # tests call this method directly. # and we need this workaround to avoid refactoring a lot of tests. self._ensure_initialized_for(cluster) attrs = objects.Cluster.get_attributes(cluster) attrs = objects.Attributes.merged_attrs_values(attrs) attrs['deployment_mode'] = cluster.mode attrs['deployment_id'] = cluster.id attrs['openstack_version'] = cluster.release.version attrs['fuel_version'] = cluster.fuel_version attrs['nodes'] = self.node_list(self.all_nodes) # Adding params to workloads_collector if 'workloads_collector' not in attrs: attrs['workloads_collector'] = {} attrs['workloads_collector']['create_user'] = \ objects.MasterNodeSettings.must_send_stats() username = attrs['workloads_collector'].pop('user', None) attrs['workloads_collector']['username'] = username if self.resolver.resolve(['cinder']): attrs['use_cinder'] = True net_serializer = self.get_net_provider_serializer(cluster) net_common_attrs = net_serializer.get_common_attrs(cluster, attrs) utils.dict_update(attrs, net_common_attrs) self.inject_list_of_plugins(attrs, cluster) return attrs
def inject_provision_info(self, common_attrs, node, data): if node.replaced_provisioning_info: info = node.replaced_provision_info else: info = self._provision_serializer.serialize_node_info( common_attrs, node ) utils.dict_update(data.setdefault('provision', {}), info)
def inject_provision_info(self, node, data): # TODO(bgaifullin) serialize_node_info should be reworked if not self._cluster_info: self._cluster_info = self.get_common_attrs(node.cluster) if node.replaced_provisioning_info: info = node.replaced_provision_info else: info = self._provision_serializer.serialize_node_info( self._cluster_info, node) utils.dict_update(data.setdefault('provision', {}), info)
def get_common_attrs(self, cluster): attrs = super(DeploymentLCMSerializer, self).get_common_attrs( cluster ) attrs['cluster'] = objects.Cluster.to_dict(cluster) attrs['release'] = objects.Release.to_dict(cluster.release) provision = attrs.setdefault('provision', {}) utils.dict_update( provision, self._provision_serializer.serialize_cluster_info(cluster, attrs) ) return attrs
def test_dict_update_all_levels(self): target = { 'a': {'b': 1}, 'c': {'d': {'e': {'f': 2}}} } update1 = { 'a': {'b': 2}, 'c': {'d': {'g': 2}} } dict_update(target, update1) self.assertEqual({'b': 2}, target['a']) self.assertEqual({'d': {'g': 2, 'e': {'f': 2}}}, target['c'])
def test_dict_update_2nd_levels(self): target = { 'a': {'b': 1}, 'c': {'d': {'e': {'f': {'k': 2}}, 'l': 1}} } update1 = { 'a': {'b': 2}, 'c': {'d': {'e': {'g': 2}}} } dict_update(target, update1, level=2) self.assertEqual({'b': 2}, target['a']) self.assertEqual({'d': {'e': {'g': 2}, 'l': 1}}, target['c'])
def serialize_cluster(cls, cluster, data, **kwargs): if objects.Release.is_lcm_supported(cluster.release): return data else: serializer = get_serializer_for_cluster(cluster)() serializer.initialize(cluster) common_attrs = serializer.get_common_attrs(cluster) if cluster.replaced_deployment_info: # patch common attributes with custom deployment info utils.dict_update(common_attrs, cluster.replaced_deployment_info) return common_attrs
def inject_provision_info(self, node, data): # TODO(bgaifullin) serialize_node_info should be reworked if not self._cluster_info: self._cluster_info = self.get_common_attrs(node.cluster) if node.replaced_provisioning_info: info = node.replaced_provision_info else: info = self._provision_serializer.serialize_node_info( self._cluster_info, node ) utils.dict_update(data.setdefault('provision', {}), info)
def serialize(self, cluster, nodes, ignore_customized=False, skip_extensions=False): """Method generates facts which are passed to puppet.""" try: self.initialize(cluster) common_attrs = self.get_common_attrs(cluster) if not ignore_customized and cluster.replaced_deployment_info: # patch common attributes with custom deployment info utils.dict_update(common_attrs, cluster.replaced_deployment_info) if not skip_extensions: extensions.\ fire_callback_on_cluster_serialization_for_deployment( cluster, common_attrs ) serialized_nodes = [] origin_nodes = [] customized_nodes = [] if ignore_customized: origin_nodes = nodes else: for node in nodes: if node.replaced_deployment_info: customized_nodes.append(node) else: origin_nodes.append(node) serialized_nodes.extend( self.serialize_generated(origin_nodes, skip_extensions)) serialized_nodes.extend( self.serialize_customized(customized_nodes)) # NOTE(dshulyak) tasks should not be preserved from replaced # deployment info, there is different mechanism to control # changes in tasks introduced during granular deployment, # and that mech should be used self.set_tasks(serialized_nodes) deployment_info = { 'common': common_attrs, 'nodes': serialized_nodes } finally: self.finalize() return deployment_info
def serialize(self, cluster, nodes, ignore_customized=False, skip_extensions=False): """Method generates facts which are passed to puppet.""" try: self.initialize(cluster) common_attrs = self.get_common_attrs(cluster) if not ignore_customized and cluster.replaced_deployment_info: # patch common attributes with custom deployment info utils.dict_update( common_attrs, cluster.replaced_deployment_info ) if not skip_extensions: extensions.\ fire_callback_on_cluster_serialization_for_deployment( cluster, common_attrs ) serialized_nodes = [] origin_nodes = [] customized_nodes = [] if ignore_customized: origin_nodes = nodes else: for node in nodes: if node.replaced_deployment_info: customized_nodes.append(node) else: origin_nodes.append(node) serialized_nodes.extend( self.serialize_generated(origin_nodes, skip_extensions) ) serialized_nodes.extend( self.serialize_customized(customized_nodes) ) # NOTE(dshulyak) tasks should not be preserved from replaced # deployment info, there is different mechanism to control # changes in tasks introduced during granular deployment, # and that mech should be used self.set_tasks(serialized_nodes) deployment_info = {'common': common_attrs, 'nodes': serialized_nodes} finally: self.finalize() return deployment_info
def serialize_customized(self, nodes): for node in nodes: data = {} roles = [] for role_data in node.replaced_deployment_info: if 'role' in role_data: # if replaced_deployment_info consists # of old serialized info, the old info # have serialized data per role roles.append(role_data.pop('role')) utils.dict_update(data, role_data) if roles: data['roles'] = roles self.inject_provision_info(node, data) yield data
def inject_plugin_attribute_values(cls, attributes): """Inject given attributes with plugin attributes values. :param attributes: Cluster attributes :type attributes: dict """ for k, attrs in six.iteritems(attributes): if (not cls.is_plugin_data(attrs) or not attrs['metadata']['enabled']): continue metadata = attrs['metadata'] selected_plugin_attrs = cls._get_specific_version( metadata.get('versions', []), metadata.get('chosen_id')) selected_plugin_attrs.pop('metadata', None) dict_update(attrs, selected_plugin_attrs, 1)
def get_common_attrs(self, cluster): attrs = super(DeploymentLCMSerializer, self).get_common_attrs(cluster) attrs['cluster'] = objects.Cluster.to_dict(cluster, fields=("id", "name", "fuel_version", "status", "mode")) attrs['release'] = objects.Release.to_dict(cluster.release, fields=('name', 'version', 'operating_system')) # the ReleaseSerializer adds this attribute certainly attrs['release'].pop('is_deployable', None) provision = attrs.setdefault('provision', {}) utils.dict_update( provision, self._provision_serializer.serialize_cluster_info(cluster, attrs)) # TODO(bgaifullin) remove using cluster_info # in serialize_node_for_provision self._cluster_info = attrs return attrs
def serialize(self): configs = self.configs if configs is None: configs = objects.OpenstackConfigCollection.find_configs_for_nodes( self.cluster, self.nodes) node_configs = defaultdict(lambda: defaultdict(dict)) nodes_to_update = dict((node.id, node) for node in self.nodes) for config in configs: if config.config_type == consts.OPENSTACK_CONFIG_TYPES.cluster: for node_id in nodes_to_update: node_configs[node_id]['cluster'] = config.configuration elif config.config_type == consts.OPENSTACK_CONFIG_TYPES.role: for node in self.nodes: if config.node_role in node.all_roles: utils.dict_update(node_configs[node.id]['role'], config.configuration, level=1) elif config.config_type == consts.OPENSTACK_CONFIG_TYPES.node: if config.node_id in nodes_to_update: fqdn = objects.Node.get_node_fqdn( nodes_to_update[config.node_id]) node_configs[config.node_id][fqdn] = config.configuration for node_id in node_configs: for config_dest in node_configs[node_id]: path = os.path.join(consts.OVERRIDE_CONFIG_BASE_PATH, config_dest + '.yaml') # Converts config from MutableDict to dict # needs for further serialization to yaml data = { 'configuration': dict(node_configs[node_id][config_dest]) } node = nodes_to_update[node_id] yield templates.make_upload_task([node.uid], path=path, data=yaml.safe_dump(data))
def serialize(self): configs = self.configs if configs is None: configs = objects.OpenstackConfigCollection.find_configs_for_nodes( self.cluster, self.nodes) node_configs = defaultdict(lambda: defaultdict(dict)) nodes_to_update = dict((node.id, node) for node in self.nodes) for config in configs: if config.config_type == consts.OPENSTACK_CONFIG_TYPES.cluster: for node_id in nodes_to_update: node_configs[node_id]['cluster'] = config.configuration elif config.config_type == consts.OPENSTACK_CONFIG_TYPES.role: for node in self.nodes: if config.node_role in node.all_roles: utils.dict_update( node_configs[node.id]['role'], config.configuration, level=1 ) elif config.config_type == consts.OPENSTACK_CONFIG_TYPES.node: if config.node_id in nodes_to_update: fqdn = objects.Node.get_node_fqdn( nodes_to_update[config.node_id]) node_configs[config.node_id][fqdn] = config.configuration for node_id in node_configs: for config_dest in node_configs[node_id]: path = os.path.join(consts.OVERRIDE_CONFIG_BASE_PATH, config_dest + '.yaml') # Converts config from MutableDict to dict # needs for further serialization to yaml data = { 'configuration': dict(node_configs[node_id][config_dest])} node = nodes_to_update[node_id] yield templates.make_upload_task( [node.uid], path=path, data=yaml.safe_dump(data))
def get_common_attrs(self, cluster): attrs = super(DeploymentLCMSerializer, self).get_common_attrs( cluster ) attrs['cluster'] = objects.Cluster.to_dict( cluster, fields=("id", "name", "fuel_version", "status", "mode") ) attrs['release'] = objects.Release.to_dict( cluster.release, fields=('name', 'version', 'operating_system') ) # the ReleaseSerializer adds this attribute certainly attrs['release'].pop('is_deployable', None) provision = attrs.setdefault('provision', {}) utils.dict_update( provision, self._provision_serializer.serialize_cluster_info(cluster, attrs) ) # TODO(bgaifullin) remove using cluster_info # in serialize_node_for_provision self._cluster_info = attrs return attrs
def upgrade_tags_set(): connection = op.get_bind() q_get_role_tags_meta = sa.text( "SELECT id, version, roles_metadata, tags_metadata FROM releases") q_update_role_tags_meta = sa.text( "UPDATE releases " "SET roles_metadata = :roles_meta, tags_metadata = :tags_meta " "WHERE id = :obj_id") for obj_id, version, roles_meta, tags_meta in connection.execute( q_get_role_tags_meta): if not is_feature_supported(version, FUEL_TAGS_SUPPORT): continue roles_meta = jsonutils.loads(roles_meta or '{}') tags_meta = jsonutils.loads(tags_meta or '{}') dict_update(roles_meta, NEW_ROLES_META) dict_update(tags_meta, NEW_TAGS_META) connection.execute(q_update_role_tags_meta, roles_meta=jsonutils.dumps(roles_meta), tags_meta=jsonutils.dumps(tags_meta), obj_id=obj_id)
def downgrade_tags_set(): connection = op.get_bind() q_get_role_tags_meta = sa.text("SELECT id, roles_metadata, tags_metadata " "FROM releases") q_update_role_tags_meta = sa.text( "UPDATE releases " "SET roles_metadata = :roles_meta, tags_metadata = :tags_meta " "WHERE id = :obj_id") for obj_id, roles_meta, tags_meta in connection.execute( q_get_role_tags_meta): roles_meta = jsonutils.loads(roles_meta or '{}') tags_meta = jsonutils.loads(tags_meta or '{}') dict_update(roles_meta, OLD_ROLES_META) tags_to_remove = set(NEW_TAGS_META) & set(tags_meta) for tag_name in tags_to_remove: tags_meta.pop(tag_name) connection.execute(q_update_role_tags_meta, roles_meta=jsonutils.dumps(roles_meta), tags_meta=jsonutils.dumps(tags_meta), obj_id=obj_id)
def get_plugins_deployment_graph(cls, cluster, graph_type=None): deployment_tasks = [] processed_tasks = {} enabled_plugins = ClusterPlugin.get_enabled(cluster.id) graph_metadata = {} for plugin_adapter in map(wrap_plugin, enabled_plugins): depl_graph = plugin_adapter.get_deployment_graph(graph_type) depl_tasks = depl_graph.pop('tasks') dict_update(graph_metadata, depl_graph) for t in depl_tasks: t_id = t['id'] if t_id in processed_tasks: raise errors.AlreadyExists( 'Plugin {0} is overlapping with plugin {1} ' 'by introducing the same deployment task with ' 'id {2}'.format(plugin_adapter.full_name, processed_tasks[t_id], t_id)) processed_tasks[t_id] = plugin_adapter.full_name deployment_tasks.extend(depl_tasks) graph_metadata['tasks'] = deployment_tasks return graph_metadata
def inject_configs(self, node, roles, output): node_config = output.setdefault('configuration', {}) for config in self._configs: if config.config_type == consts.OPENSTACK_CONFIG_TYPES.cluster: utils.dict_update(node_config, config.configuration, 1) elif config.config_type == consts.OPENSTACK_CONFIG_TYPES.role: for role in roles: if NameMatchingPolicy.create(config.node_role).match(role): utils.dict_update(node_config, config.configuration, 1) elif config.config_type == consts.OPENSTACK_CONFIG_TYPES.node: if config.node_id == node.id: utils.dict_update(node_config, config.configuration, 1)
def inject_configs(self, node, output): node_config = output.setdefault('configuration', {}) for config in self._configs: if config.config_type == consts.OPENSTACK_CONFIG_TYPES.cluster: utils.dict_update(node_config, config.configuration, 1) elif config.config_type == consts.OPENSTACK_CONFIG_TYPES.role: # (asaprykin): objects.Node.all_roles() has a side effect, # it replaces "<rolename>" with "primary-<rolename>" # in case of primary role. for role in node.all_roles: if NameMatchingPolicy.create(config.node_role).match(role): utils.dict_update(node_config, config.configuration, 1) elif config.config_type == consts.OPENSTACK_CONFIG_TYPES.node: if config.node_id == node.id: utils.dict_update(node_config, config.configuration, 1)
def inject_configs(self, node, output): node_config = output.setdefault('configuration', {}) node_config_opts = output.setdefault('configuration_options', {}) for config in self._configs: # OpenstackConfig.configuration is MutableDict, so we copy # data for preventing changes in the DB config_data = config.configuration.copy() # TODO(akislitsky) refactor CLI and OpenstackConfig object # to allow serialize arbitrary data. Old configs data should be # modified to the structure {'configuration': old_configuration}. # Then new config data will have the structure: # {'configuration': old_configuration, # 'configuration_options': ..., # 'any_key': any_value # } # and new structure will be serialized to the node config. config_data_opts = config_data.pop('configuration_options', {}) if config.config_type == consts.OPENSTACK_CONFIG_TYPES.cluster: utils.dict_update(node_config, config_data, 1) utils.dict_update(node_config_opts, config_data_opts, 1) elif config.config_type == consts.OPENSTACK_CONFIG_TYPES.role: # (asaprykin): objects.Node.all_roles() has a side effect, # it replaces "<rolename>" with "primary-<rolename>" # in case of primary role. for role in node.all_roles: if NameMatchingPolicy.create(config.node_role).match(role): utils.dict_update(node_config, config_data, 1) utils.dict_update(node_config_opts, config_data_opts, 1) elif config.config_type == consts.OPENSTACK_CONFIG_TYPES.node: if config.node_id == node.id: utils.dict_update(node_config, config_data, 1) utils.dict_update(node_config_opts, config_data_opts, 1)