def _get_host_tmpl_vars_dict(self, host_id, global_vars_dict, **kwargs): """Generate template variables dictionary.""" vars_dict = {} if global_vars_dict: # Set cluster template vars_dict from cluster os_config. vars_dict = deepcopy(global_vars_dict) # Set hostname, MAC address and hostname, networks, dns and so on. host_baseinfo = self.config_manager.get_host_baseinfo(host_id) vars_dict[const.BASEINFO] = host_baseinfo # Set profile if self.PROFILE in kwargs: profile = kwargs[self.PROFILE] else: os_version = self.config_manager.get_os_version() profile = self._get_profile_from_server(os_version) vars_dict[const.BASEINFO][self.PROFILE] = profile metadata = self.config_manager.get_os_config_metadata() os_config = self.config_manager.get_host_os_config(host_id) # Get template variables values from host os_config host_vars_dict = self.get_tmpl_vars_from_metadata(metadata, os_config) util.merge_dict( vars_dict.setdefault(const.OS_CONFIG, {}), host_vars_dict ) return vars_dict
def _generate_system_config(self, host_id, host_vars_dict): """Generate updated system config from the template. :param host_vars_dict: dict of variables for the system template to generate system config dict for each host. """ os_version = self.config_manager.get_os_version() tmpl_path = os.path.join( os.path.join(self.tmpl_dir, os_version), self.SYS_TMPL_NAME ) if not os.path.exists(tmpl_path): err_msg = "Template '%s' does not exists!" % tmpl_path logging.error(err_msg) raise Exception(err_msg) system_config = self.get_config_from_template(tmpl_path, host_vars_dict) # update package config info to cobbler ksmeta if self.pk_installer_config and host_id in self.pk_installer_config: pk_config = self.pk_installer_config[host_id] ksmeta = system_config.setdefault("ksmeta", {}) util.merge_dict(ksmeta, pk_config) system_config["ksmeta"] = ksmeta return system_config
def test_update(self): """test update function.""" expected_config = copy.deepcopy(self.config_) config2 = {'9': 9, '10': {'10': 10}} util.merge_dict(expected_config, config2) self.ref_.update(config2) self.assertEqual(self.ref_.config, expected_config)
def _generate_system_config(self, host_id, host_vars_dict): """Generate updated system config from the template. :param host_vars_dict: dict of variables for the system template to generate system config dict for each host. """ os_version = self.config_manager.get_os_version() tmpl_path = os.path.join(os.path.join(self.tmpl_dir, os_version), self.SYS_TMPL_NAME) if not os.path.exists(tmpl_path): err_msg = "Template '%s' does not exists!" % tmpl_path logging.error(err_msg) raise Exception(err_msg) system_config = self.get_config_from_template(tmpl_path, host_vars_dict) # update package config info to cobbler ksmeta if self.pk_installer_config and host_id in self.pk_installer_config: pk_config = self.pk_installer_config[host_id] ksmeta = system_config.setdefault("ksmeta", {}) util.merge_dict(ksmeta, pk_config) system_config["ksmeta"] = ksmeta return system_config
def _generate_node_attributes(self, roles, host_vars_dict): """Generate node attributes. Generates from templates according to its roles. The templates are named by roles without '-'. Return the dictionary of attributes defined in the templates. :param list roles: The roles for this node, used to load the specific template. :param dict host_vars_dict: The dict used in cheetah searchList to render attributes from templates. """ if not roles: return {} node_tmpl_dir = os.path.join(self.tmpl_dir, self.NODE_TMPL_DIR) node_attr = {} for role in roles: role = role.replace('-', '_') tmpl_name = '.'.join((role, 'tmpl')) node_tmpl = os.path.join(node_tmpl_dir, tmpl_name) util.merge_dict( node_attr, self.get_config_from_template(node_tmpl, host_vars_dict) ) return node_attr
def merge(self, upper_ref, lower_refs): """merge upper config to lower configs.""" upper_sub_refs = self._get_upper_sub_refs(upper_ref) for ref_key, sub_ref in upper_sub_refs: sub_configs = self._get_mapping_from_upper_keys(ref_key, sub_ref) lower_sub_refs = {} for lower_key, lower_ref in lower_refs.items(): lower_sub_refs[lower_key] = lower_ref.setdefault(ref_key) lower_sub_configs = self._get_mapping_from_lower_keys( ref_key, lower_sub_refs) util.merge_dict(sub_configs, lower_sub_configs) values = self._get_values(ref_key, sub_ref, lower_sub_refs, sub_configs) logging.debug('%s set values %s to %s', ref_key, self.to_key_, values) for lower_key, lower_sub_ref in lower_sub_refs.items(): if lower_key not in values: logging.error('no key %s in %s', lower_key, values) continue value = values[lower_key] lower_to_ref = lower_sub_ref.setdefault(self.to_key_) override = self._get_override(ref_key, sub_ref, self.to_key_, lower_to_ref) lower_to_ref.update(value, override)
def test_inputs_not_dict(self): """test inputs not dict.""" lhs = [1, 2, 3] rhs = {1: 2} merged = util.merge_dict(lhs, rhs) expected = {1: 2} self.assertEqual(merged, expected) lhs = [1, 2, 3] rhs = {1: 2} merged = util.merge_dict(lhs, rhs, False) expected = [1, 2, 3] self.assertEqual(merged, expected) lhs = {1: 2} rhs = [1, 2, 3] merged = util.merge_dict(lhs, rhs) expected = [1, 2, 3] self.assertEqual(merged, expected) lhs = {1: 2} rhs = [1, 2, 3] merged = util.merge_dict(lhs, rhs, False) expected = {1: 2} self.assertEqual(merged, expected)
def deploy(self): """Deploy the cluster.""" deploy_config = {} pk_instl_confs = {} if self.pk_installer: # generate target system config which will be installed by OS # installer right after OS installation is completed. pk_instl_confs = self.package_installer.generate_installer_config() logging.debug('[DeployManager][deploy] pk_instl_confs is %s', pk_instl_confs) if self.os_installer: logging.info('[DeployManager][deploy]get OS installer %s', self.os_installer) # Send package installer config info to OS installer. if pk_instl_confs: self.os_installer.set_package_installer_config(pk_instl_confs) # start to deploy OS try: os_deploy_config = self.os_installer.deploy() deploy_config = os_deploy_config except Exception as ex: logging.error(ex.message) if self.pk_installer: logging.info('DeployManager][deploy]get package installer %s', self.pk_installer) pk_deploy_config = self.package_installer.deploy() util.merge_dict(deploy_config, pk_deploy_config) return deploy_config
def _get_host_tmpl_vars_dict(self, host_id, global_vars_dict, **kwargs): """Generate template variables dictionary.""" vars_dict = {} if global_vars_dict: # Set cluster template vars_dict from cluster os_config. vars_dict = deepcopy(global_vars_dict) # Set hostname, MAC address and hostname, networks, dns and so on. host_baseinfo = self.config_manager.get_host_baseinfo(host_id) vars_dict[const.BASEINFO] = host_baseinfo # Set profile if self.PROFILE in kwargs: profile = kwargs[self.PROFILE] else: os_version = self.config_manager.get_os_version() profile = self._get_profile_from_server(os_version) vars_dict[const.BASEINFO][self.PROFILE] = profile metadata = self.config_manager.get_os_config_metadata() os_config = self.config_manager.get_host_os_config(host_id) # Get template variables values from host os_config host_vars_dict = self.get_tmpl_vars_from_metadata(metadata, os_config) util.merge_dict(vars_dict.setdefault(const.OS_CONFIG, {}), host_vars_dict) return vars_dict
def update_cluster_and_host_configs(self, clusterid, hostids, update_hostids, os_version, target_system): """update cluster/host configs.""" logging.debug('update cluster %s with all hosts %s and update: %s', clusterid, hostids, update_hostids) global_config = self.get_global_config(os_version, target_system) self.update_global_config(global_config, os_version=os_version, target_system=target_system) cluster_config = self.get_cluster_config(clusterid, os_version=os_version, target_system=target_system) util.merge_dict(cluster_config, global_config, False) self.update_cluster_config(clusterid, cluster_config, os_version=os_version, target_system=target_system) host_configs = self.get_host_configs(hostids, os_version=os_version, target_system=target_system) CLUSTER_HOST_MERGER.merge(cluster_config, host_configs) update_host_configs = dict([ (hostid, host_config) for hostid, host_config in host_configs.items() if hostid in update_hostids ]) self.update_host_configs(update_host_configs, os_version=os_version, target_system=target_system)
def config(self): """config getter.""" config = {} if self.config_data: try: config.update(json.loads(self.config_data)) config.update({'hostid': self.id, 'hostname': self.hostname}) if self.cluster: config.update({'clusterid': self.cluster.id, 'clustername': self.cluster.name}) if self.machine: util.merge_dict( config, { 'networking': { 'interfaces': { 'management': { 'mac': self.machine.mac } } } }) except Exception as error: logging.error('failed to load config %s: %s', self.hostname, self.config_data) logging.exception(error) return config
def _get_flavors_metadata_from_configuration(): """Get flavor metadata from flavor metadata config dir.""" flavors_metadata = {} env_locals = {} env_locals.update(metadata_validator.VALIDATOR_LOCALS) env_locals.update(metadata_callback.CALLBACK_LOCALS) configs = util.load_configs( setting.FLAVOR_METADATA_DIR, env_locals=env_locals ) for config in configs: adapter_name = config['ADAPTER'] flavor_name = config['FLAVOR'] flavor_metadata = flavors_metadata.setdefault( adapter_name, {} ).setdefault(flavor_name, {}) for key, value in config['METADATA'].items(): flavor_metadata[key] = _get_metadata_from_configuration( key, key, value, FLAVOR_FIELDS ) packages_metadata = PACKAGES_METADATA adapters_flavors = adapter_api.ADAPTERS_FLAVORS for adapter_name, adapter_flavors in adapters_flavors.items(): package_metadata = packages_metadata.get(adapter_name, {}) for flavor_name, flavor in adapter_flavors.items(): flavor_metadata = flavors_metadata.setdefault( adapter_name, {} ).setdefault(flavor_name, {}) util.merge_dict(flavor_metadata, package_metadata, override=False) return flavors_metadata
def update_cluster_and_host_configs(self, clusterid, hostids, update_hostids, os_version, target_system): """update cluster/host configs.""" logging.debug('update cluster %s with all hosts %s and update: %s', clusterid, hostids, update_hostids) global_config = self.get_global_config(os_version, target_system) self.update_global_config(global_config, os_version=os_version, target_system=target_system) cluster_config = self.get_cluster_config( clusterid, os_version=os_version, target_system=target_system) util.merge_dict(cluster_config, global_config, False) self.update_cluster_config( clusterid, cluster_config, os_version=os_version, target_system=target_system) host_configs = self.get_host_configs( hostids, os_version=os_version, target_system=target_system) CLUSTER_HOST_MERGER.merge(cluster_config, host_configs) update_host_configs = dict( [(hostid, host_config) for hostid, host_config in host_configs.items() if hostid in update_hostids]) self.update_host_configs( update_host_configs, os_version=os_version, target_system=target_system)
def _get_host_tmpl_vars(self, host_id, global_vars_dict): """Get templates variables dictionary for cheetah searchList based on host package config. :param int host_id: The host ID. :param dict global_vars_dict: The vars_dict got from cluster level package_config. """ vars_dict = {} if global_vars_dict: temp = global_vars_dict[const.CLUSTER][const.DEPLOYED_PK_CONFIG] vars_dict[const.DEPLOYED_PK_CONFIG] = temp host_baseinfo = self.config_manager.get_host_baseinfo(host_id) util.merge_dict(vars_dict, host_baseinfo) pk_config = self.config_manager.get_host_package_config(host_id) if pk_config: # Get host template variables and merge to vars_dict metadata = self.config_manager.get_pk_config_meatadata() host_dict = self.get_tmpl_vars_from_metadata(metadata, pk_config) util.merge_dict(vars_dict[const.DEPLOYED_PK_CONFIG], host_dict) # Set role_mapping for host mapping = self.config_manager.get_host_roles_mapping(host_id) vars_dict[const.DEPLOYED_PK_CONFIG][const.ROLES_MAPPING] = mapping return {const.HOST: vars_dict}
def _get_flavors_metadata_from_configuration(): """Get flavor metadata from flavor metadata config dir.""" flavors_metadata = {} env_locals = {} env_locals.update(metadata_validator.VALIDATOR_LOCALS) env_locals.update(metadata_callback.CALLBACK_LOCALS) configs = util.load_configs(setting.FLAVOR_METADATA_DIR, env_locals=env_locals) for config in configs: adapter_name = config['ADAPTER'] flavor_name = config['FLAVOR'] flavor_metadata = flavors_metadata.setdefault(adapter_name, {}).setdefault( flavor_name, {}) for key, value in config['METADATA'].items(): flavor_metadata[key] = _get_metadata_from_configuration( key, key, value, FLAVOR_FIELDS) packages_metadata = PACKAGES_METADATA adapters_flavors = adapter_api.ADAPTERS_FLAVORS for adapter_name, adapter_flavors in adapters_flavors.items(): package_metadata = packages_metadata.get(adapter_name, {}) for flavor_name, flavor in adapter_flavors.items(): flavor_metadata = flavors_metadata.setdefault(adapter_name, {}).setdefault( flavor_name, {}) util.merge_dict(flavor_metadata, package_metadata, override=False) return flavors_metadata
def _get_host_tmpl_vars(self, host_id, global_vars_dict): """Generate templates variables dictionary for cheetah searchList based on host package config. :param int host_id: The host ID. :param dict global_vars_dict: The vars_dict got from cluster level package_config. The output format is the same as cluster_vars_dict. """ host_vars_dict = {} if global_vars_dict: host_vars_dict = deepcopy(global_vars_dict) # Update host basic info host_baseinfo = self.config_manager.get_host_baseinfo(host_id) util.merge_dict( host_vars_dict.setdefault(const.BASEINFO, {}), host_baseinfo ) pk_config = self.config_manager.get_host_package_config(host_id) if pk_config: # Get host template variables and merge to vars_dict metadata = self.config_manager.get_pk_config_meatadata() meta_dict = self.get_tmpl_vars_from_metadata(metadata, pk_config) util.merge_dict( host_vars_dict.setdefault(const.PK_CONFIG, {}), meta_dict ) # Override role_mapping for host if host role_mapping exists mapping = self.config_manager.get_host_roles_mapping(host_id) if mapping: host_vars_dict[const.ROLES_MAPPING] = mapping return host_vars_dict
def _get_host_tmpl_vars(self, host_id, global_vars_dict): """Generate templates variables dictionary. For cheetah searchList based on host package config. :param int host_id: The host ID. :param dict global_vars_dict: The vars_dict got from cluster level package_config. The output format is the same as cluster_vars_dict. """ host_vars_dict = {} if global_vars_dict: host_vars_dict = deepcopy(global_vars_dict) # Update host basic info host_baseinfo = self.config_manager.get_host_baseinfo(host_id) util.merge_dict(host_vars_dict.setdefault(const.BASEINFO, {}), host_baseinfo) pk_config = self.config_manager.get_host_package_config(host_id) if pk_config: # Get host template variables and merge to vars_dict metadata = self.config_manager.get_pk_config_meatadata() meta_dict = self.get_tmpl_vars_from_metadata(metadata, pk_config) util.merge_dict(host_vars_dict.setdefault(const.PK_CONFIG, {}), meta_dict) # Override role_mapping for host if host role_mapping exists mapping = self.config_manager.get_host_roles_mapping(host_id) if mapping: host_vars_dict[const.ROLES_MAPPING] = mapping return host_vars_dict
def merge(self, upper_ref, lower_refs): """merge upper config to lower configs.""" upper_sub_refs = self._get_upper_sub_refs(upper_ref) for ref_key, sub_ref in upper_sub_refs: sub_configs = self._get_mapping_from_upper_keys(ref_key, sub_ref) lower_sub_refs = {} for lower_key, lower_ref in lower_refs.items(): lower_sub_refs[lower_key] = lower_ref.setdefault(ref_key) lower_sub_configs = self._get_mapping_from_lower_keys( ref_key, lower_sub_refs) util.merge_dict(sub_configs, lower_sub_configs) values = self._get_values( ref_key, sub_ref, lower_sub_refs, sub_configs) logging.debug('%s set values %s to %s', ref_key, self.to_key_, values) for lower_key, lower_sub_ref in lower_sub_refs.items(): if lower_key not in values: logging.error('no key %s in %s', lower_key, values) continue value = values[lower_key] lower_to_ref = lower_sub_ref.setdefault(self.to_key_) override = self._get_override(self.to_key_, lower_to_ref) lower_to_ref.update(value, override)
def deploy(self): """Deploy the cluster.""" deployed_config = self.deploy_os() package_deployed_config = self.deploy_target_system() util.merge_dict(deployed_config, package_deployed_config) return deployed_config
def update_global_config(self, config, target_system, **kwargs): """update global config.""" global_bag_item = self._get_global_databag_item(target_system) translated_config = TO_GLOBAL_TRANSLATORS[target_system].translate( config) util.merge_dict(global_bag_item, translated_config, True) self._update_global_databag_item(target_system, global_bag_item)
def test_simple_merge(self): """simple test of merge merge_dict performs in-place merge of rhs to lhs """ lhs = {1: 1} rhs = {2: 2} util.merge_dict(lhs, rhs) self.assertEqual(lhs, {1: 1, 2: 2})
def get_cluster_config(self, clusterid, target_system, **kwargs): """get cluster config.""" bag = self._get_databag(target_system) global_bag_item = dict(self._get_global_databag_item(bag)) bag_item = dict(self._get_cluster_databag_item( bag, clusterid, target_system)) util.merge_dict(bag_item, global_bag_item, False) return FROM_CLUSTER_TRANSLATORS[target_system].translate(bag_item)
def test_change_after_merge(self): lhs = {1: {2: 3}} rhs = {1: {3: [4, 5, 6]}} util.merge_dict(lhs, rhs) self.assertEqual(lhs, {1: {2: 3, 3: [4, 5, 6]}}) self.assertEqual(rhs, {1: {3: [4, 5, 6]}}) rhs[1][3].append(7) self.assertEqual(lhs, {1: {2: 3, 3: [4, 5, 6]}}) self.assertEqual(rhs, {1: {3: [4, 5, 6, 7]}})
def get_cluster_config(self, clusterid, target_system, **kwargs): """get cluster config.""" bag = self._get_databag(target_system) global_bag_item = dict(self._get_global_databag_item(bag)) bag_item = dict( self._get_cluster_databag_item(bag, clusterid, target_system)) util.merge_dict(bag_item, global_bag_item, False) return FROM_CLUSTER_TRANSLATORS[target_system].translate(bag_item)
def test_merge_not_override(self): lhs = {1: 1} rhs = {1: 2} util.merge_dict(lhs, rhs, False) self.assertEqual(lhs, {1: 1}) lhs = {1: {2: 3, 3: 5}} rhs = {1: {2: 4, 4: 6}} util.merge_dict(lhs, rhs, False) self.assertEqual(lhs, {1: {2: 3, 3: 5, 4: 6}})
def test_init_with_parent_update(self): # create ConfigReference instance with parent param. # it will update the key value pair in parent config if it exists. config3 = {'5': {'7': 7}} ref3 = config_reference.ConfigReference(config3['5'], self.ref_, '5') expected_config = copy.deepcopy(self.config_) util.merge_dict(expected_config, config3) self.assertEqual(self.config_, expected_config) self.assertEqual(id(self.config_['5']), id(ref3.config)) self.assertEqual(id(ref3.config), id(config3['5']))
def test_init_with_parent(self): # create ConfigReference instance with parent param. # it will add a key value pair in parent config if not exists. config2 = {'5': {'6': 6}} ref2 = config_reference.ConfigReference(config2['5'], self.ref_, '5') expected_config = copy.deepcopy(self.config_) util.merge_dict(expected_config, config2) self.assertEqual(self.config_, expected_config) self.assertEqual(id(self.config_['5']), id(ref2.config)) self.assertEqual(id(ref2.config), id(config2['5']))
def get_host_completed_os_config(self, host_id): deployed_config = self.get_host_deployed_os_config(host_id) config = self.__get_deployed_config_item(deployed_config, const.COMPLETED_OS_CONFIG) if not config: config = self.get_cluster_os_config() util.merge_dict(config, self.get_host_os_config(host_id)) deployed_config[const.COMPLETED_OS_CONFIG] = config return config
def assign_roles_by_host_numbers(upper_ref, from_key, lower_refs, to_key, policy_by_host_numbers={}, default={}, **_kwargs): """Assign roles by role assign policy.""" host_numbers = str(len(lower_refs)) policy_kwargs = deepcopy(default) if host_numbers in policy_by_host_numbers: util.merge_dict(policy_kwargs, policy_by_host_numbers[host_numbers]) return assign_roles(upper_ref, from_key, lower_refs, to_key, **policy_kwargs)
def test_merge_override(self): """test merge override.""" lhs = {1: 1} rhs = {1: 2} util.merge_dict(lhs, rhs) self.assertEqual(lhs, {1: 2}) lhs = {1: {2: 3, 3: 5}} rhs = {1: {2: 4, 4: 6}} util.merge_dict(lhs, rhs) self.assertEqual(lhs, {1: {2: 4, 3: 5, 4: 6}})
def test_init(self): config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) config2 = {'5': {'6': 6}} ref2 = config_reference.ConfigReference(config2['5'], ref, '5') expected_config = deepcopy(config) util.merge_dict(expected_config, config2) self.assertEqual(ref.config, expected_config) self.assertEqual(id(ref.config['5']), id(ref2.config)) config3 = {'5': {'7': 7}} ref3 = config_reference.ConfigReference(config3['5'], ref, '5') self.assertEqual(id(ref.config['5']), id(ref3.config))
def _get_cluster_tmpl_vars(self): """Generate template variables dict Generates based on cluster level config. The vars_dict will be: { "baseinfo": { "id":1, "name": "cluster01", ... }, "package_config": { .... //mapped from original package config based on metadata }, "role_mapping": { .... } } """ cluster_vars_dict = {} # set cluster basic information to vars_dict cluster_baseinfo = self.config_manager.get_cluster_baseinfo() cluster_vars_dict[const.BASEINFO] = cluster_baseinfo # get and set template variables from cluster package config. pk_metadata = self.config_manager.get_pk_config_meatadata() pk_config = self.config_manager.get_cluster_package_config() # get os config as ansible needs them os_metadata = self.config_manager.get_os_config_metadata() os_config = self.config_manager.get_cluster_os_config() pk_meta_dict = self.get_tmpl_vars_from_metadata(pk_metadata, pk_config) os_meta_dict = self.get_tmpl_vars_from_metadata(os_metadata, os_config) util.merge_dict(pk_meta_dict, os_meta_dict) cluster_vars_dict[const.PK_CONFIG] = pk_meta_dict # get and set roles_mapping to vars_dict mapping = self.config_manager.get_cluster_roles_mapping() logging.info("cluster role mapping is %s", mapping) cluster_vars_dict[const.ROLES_MAPPING] = mapping # get ip settings to vars_dict hosts_ip_settings = self.config_manager.get_hosts_ip_settings( pk_meta_dict["network_cfg"]["ip_settings"], pk_meta_dict["network_cfg"]["sys_intf_mappings"] ) logging.info("hosts_ip_settings is %s", hosts_ip_settings) cluster_vars_dict["ip_settings"] = hosts_ip_settings return byteify(cluster_vars_dict)
def test_update(self): config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8} expected_config = deepcopy(config) ref = config_reference.ConfigReference(config) config2 = {'9': 9, '10': {'10': 10}} util.merge_dict(expected_config, config2) ref.update(config2) self.assertEqual(ref.config, expected_config) ref.update(10, False) self.assertEqual(ref.config, expected_config) ref.update(10) self.assertEqual(ref.config, 10)
def test_init(self): """test init function.""" config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) config2 = {'5': {'6': 6}} ref2 = config_reference.ConfigReference(config2['5'], ref, '5') expected_config = copy.deepcopy(config) util.merge_dict(expected_config, config2) self.assertEqual(ref.config, expected_config) self.assertEqual(id(ref.config['5']), id(ref2.config)) config3 = {'5': {'7': 7}} ref3 = config_reference.ConfigReference(config3['5'], ref, '5') self.assertEqual(id(ref.config['5']), id(ref3.config))
def update_cluster_config(self, clusterid, config, target_system, **kwargs): """update cluster config.""" self.clean_cluster_config(clusterid, config, target_system, **kwargs) global_bag_item = self._get_global_databag_item(target_system) cluster_bag_item = self._get_cluster_databag_item( target_system, clusterid) util.merge_dict(cluster_bag_item, global_bag_item, False) translated_config = TO_CLUSTER_TRANSLATORS[target_system].translate( config) util.merge_dict(cluster_bag_item, translated_config, True) self._update_cluster_databag_item(target_system, clusterid, cluster_bag_item)
def _add_flavors_from_configuration(): """Get flavors from flavor config dir and update to adapters.""" configs = util.load_configs(setting.ADAPTER_FLAVOR_DIR) for config in configs: logging.info('add config %s to flavor', config) adapter_name = config['ADAPTER_NAME'] adapter = ADAPTERS[adapter_name] adapter_flavors = ADAPTERS_FLAVORS.setdefault(adapter_name, {}) adapter_roles = ADAPTERS_ROLES[adapter_name] for flavor_dict in config['FLAVORS']: flavor_name = flavor_dict['flavor'] flavor_id = '%s:%s' % (adapter_name, flavor_name) flavor = { 'name': flavor_name, 'id': flavor_id, 'adapter_id': adapter_name, 'adapter_name': adapter_name, 'display_name': flavor_dict.get('display_name', flavor_name), 'template': flavor_dict.get('template', None) } flavor_roles = flavor_dict.get('roles', []) roles_in_flavor = [] for flavor_role in flavor_roles: if isinstance(flavor_role, basestring): role_name = flavor_role role_in_flavor = { 'name': role_name, 'flavor_id': flavor_id } else: role_in_flavor = flavor_role role_in_flavor['flavor_id'] = flavor_id if 'role' in role_in_flavor: role_in_flavor['name'] = role_in_flavor['role'] del role_in_flavor['role'] role_name = role_in_flavor['name'] role = adapter_roles[role_name] util.merge_dict(role_in_flavor, role, override=False) roles_in_flavor.append(role_in_flavor) flavor['roles'] = roles_in_flavor adapter_flavors[flavor_name] = flavor parents = {} for name, adapter in ADAPTERS.items(): parent = adapter.get('parent', None) parents[name] = parent for adapter_name, adapter_roles in ADAPTERS_FLAVORS.items(): util.recursive_merge_dict(adapter_name, ADAPTERS_FLAVORS, parents) for adapter_name, adapter_flavors in ADAPTERS_FLAVORS.items(): adapter = ADAPTERS[adapter_name] adapter['flavors'] = adapter_flavors.values()
def assign_roles_by_host_numbers(upper_ref, from_key, lower_refs, to_key, policy_by_host_numbers={}, default={}, **_kwargs): """Assign roles by role assign policy.""" host_numbers = str(len(lower_refs)) policy_kwargs = deepcopy(default) if host_numbers in policy_by_host_numbers: util.merge_dict(policy_kwargs, policy_by_host_numbers[host_numbers]) else: logging.debug('didnot find policy %s by host numbers %s', policy_by_host_numbers, host_numbers) return assign_roles(upper_ref, from_key, lower_refs, to_key, **policy_kwargs)
def test_update(self): """test update function.""" config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8} expected_config = copy.deepcopy(config) ref = config_reference.ConfigReference(config) config2 = {'9': 9, '10': {'10': 10}} util.merge_dict(expected_config, config2) ref.update(config2) self.assertEqual(ref.config, expected_config) ref.update(10, False) self.assertEqual(ref.config, expected_config) ref.update(10) self.assertEqual(ref.config, 10)
def update_cluster_config(self, clusterid, config, target_system, **kwargs): """update cluster config.""" self.clean_cluster_config(clusterid, config, target_system, **kwargs) global_bag_item = self._get_global_databag_item(target_system) cluster_bag_item = self._get_cluster_databag_item( target_system, clusterid) util.merge_dict(cluster_bag_item, global_bag_item, False) translated_config = TO_CLUSTER_TRANSLATORS[target_system].translate( config) util.merge_dict(cluster_bag_item, translated_config, True) self._update_cluster_databag_item( target_system, clusterid, cluster_bag_item)
def _get_cluster_tmpl_vars(self): vars_dict = {} cluster_baseinfo = self.config_manager.get_cluster_baseinfo() util.merge_dict(vars_dict, cluster_baseinfo) pk_metadata = self.config_manager.get_pk_config_meatadata() pk_config = self.config_manager.get_cluster_package_config() meta_dict = self.get_tmpl_vars_from_metadata(pk_metadata, pk_config) vars_dict[const.DEPLOYED_PK_CONFIG] = meta_dict mapping = self.config_manager.get_cluster_roles_mapping() vars_dict[const.DEPLOYED_PK_CONFIG][const.ROLES_MAPPING] = mapping return {const.CLUSTER: vars_dict}
def get_key_from_pattern( _ref, path, from_pattern='.*', to_pattern='', **kwargs): """Get translated key from pattern""" match = re.match(from_pattern, path) if not match: return None group = match.groupdict() util.merge_dict(group, kwargs) try: translated_key = to_pattern % group except Exception as error: logging.error('failed to get translated key from %s %% %s', to_pattern, group) raise error return translated_key
def get_config_from_template(self, tmpl_path, vars_dict): logging.debug("template path is %s", tmpl_path) logging.debug("vars_dict is %s", vars_dict) if not os.path.exists(tmpl_path) or not vars_dict: logging.info("Template dir or vars_dict is None!") return {} searchList = [] copy_vars_dict = deepcopy(vars_dict) for key, value in vars_dict.iteritems(): if isinstance(value, dict): temp = copy_vars_dict[key] del copy_vars_dict[key] searchList.append(temp) searchList.append(copy_vars_dict) # Load base template first if it exists base_config = {} base_tmpl_path = os.path.join(os.path.dirname(tmpl_path), 'base.tmpl') if os.path.isfile(base_tmpl_path) and base_tmpl_path != tmpl_path: base_tmpl = Template(file=base_tmpl_path, searchList=searchList) base_config = json.loads(base_tmpl.respond(), encoding='utf-8') base_config = json.loads(json.dumps(base_config), encoding='utf-8') # Load specific template for current adapter tmpl = Template(file=open(tmpl_path, "r"), searchList=searchList) config = json.loads(tmpl.respond(), encoding='utf-8') config = json.loads(json.dumps(config), encoding='utf-8') # Merge the two outputs config = util.merge_dict(base_config, config) logging.debug("get_config_from_template resulting %s", config) return config
def config(self, value): """config setter""" if not self.config_data: config = {} self.config_data = json.dumps(config) if value: try: config = json.loads(self.config_data) util.merge_dict(config, value) self.config_data = json.dumps(config) except Exception as error: logging.error('failed to dump config %s: %s', self.hostname, value) logging.exception(error) raise error
def update(self, config, override=True): """Update with config. :param config: config to update. :param override: if the instance config should be overrided :type override: bool """ if (self.config is not None and isinstance(self.config, dict) and isinstance(config, dict)): util.merge_dict(self.config, config, override) elif self.config is None or override: self.config = copy.deepcopy(config) else: return self.__init__(self.config, self.parent_, self.parent_key_)
def get_global_config(self, os_version, target_system): """Get global config.""" config = self.config_provider_.get_global_config() logging.debug('got global provider config from %s: %s', self.config_provider_, config) os_config = self.os_installer_.get_global_config( os_version=os_version, target_system=target_system) logging.debug('got global os config from %s: %s', self.os_installer_, os_config) package_config = self.package_installer_.get_global_config( os_version=os_version, target_system=target_system) logging.debug('got global package config from %s: %s', self.package_installer_, package_config) util.merge_dict(config, os_config) util.merge_dict(config, package_config) return config