def update(self, config, override=True): '''Update ConfigReference instance with config. Args: config: any type override: bool, if the instance config should be overrided. Returns: None Examples: config = {'networking': {'management': {'ip': '1.2.3.4'}}} ref = ConfigReference(config) new_config = {'networking': { 'management': {'mac': '00:00:00:00:00:00', 'ip': '2.3.4.5'} }} ref.update(new_config) ref.config == {'networking': { 'management': {'ip': '1.2.3.4', 'mac': '00:00:00:00:00:00'} }} ref.update(new_config, True) ref.config == {'networking': { 'management': {'ip': '2.3.4.5', 'mac': '00:00:00:00:00:00'} }} ''' if (self.config is not None and isinstance(self.config, dict) and isinstance(config, dict)): util.mergeDict(self.config, config, override) elif self.config is None or override: self.config = deepcopy(config) else: return self.__init__(self.config, self.parent, self.parent_key)
def merge(self, upper_ref, lower_refs): '''merge upper config to lower configs.''' upper_sub_refs = self._getUpperSubRefs(upper_ref) for ref_key, sub_ref in upper_sub_refs: sub_configs = self._getMappingFromUpperKeys(ref_key, sub_ref) lower_sub_refs = {} for lower_key, lower_ref in lower_refs.items(): lower_sub_refs[lower_key] = lower_ref.setdefault(ref_key) lower_sub_configs = self._getMappingFromLowerKeys( ref_key, lower_sub_refs) util.mergeDict(sub_configs, lower_sub_configs) values = self._getValues(ref_key, sub_ref, lower_sub_refs, sub_configs) logging.debug('%s set values %s to %s', ref_key, self.to_key, values) for lower_key, lower_sub_ref in lower_sub_refs.items(): if lower_key not in values: logging.error('no key %s in %s', lower_key, values) continue value = values[lower_key] lower_sub_ref.setdefault(self.to_key, value)
def merge(self, upper_ref, lower_refs): '''merge upper config to lower configs.''' upper_sub_refs = self._getUpperSubRefs(upper_ref) for ref_key, sub_ref in upper_sub_refs: sub_configs = self._getMappingFromUpperKeys(ref_key, sub_ref) lower_sub_refs = {} for lower_key, lower_ref in lower_refs.items(): lower_sub_refs[lower_key] = lower_ref.setdefault(ref_key) lower_sub_configs = self._getMappingFromLowerKeys( ref_key, lower_sub_refs) util.mergeDict(sub_configs, lower_sub_configs) values = self._getValues( ref_key, sub_ref, lower_sub_refs, sub_configs) logging.debug('%s set values %s to %s', ref_key, self.to_key, values) for lower_key, lower_sub_ref in lower_sub_refs.items(): if lower_key not in values: logging.error('no key %s in %s', lower_key, values) continue value = values[lower_key] lower_sub_ref.setdefault(self.to_key, value)
def config(self): '''config getter.''' config = {} if self.config_data: try: config.update(json.loads(self.config_data)) config.update({'hostid': self.id, 'hostname': self.hostname}) if self.cluster: config.update({ 'clusterid': self.cluster.id, 'clustername': self.cluster.name }) if self.machine: util.mergeDict( config, { 'networking': { 'interfaces': { 'management': { 'mac': self.machine.mac } } } }) except Exception as error: logging.error('failed to load config %s: %s', self.hostname, self.config_data) logging.exception(error) return config
def updateClusterAndHostConfigs(self, clusterid, hostids, update_hostids, os_version, target_system): '''update cluster/host configs.''' logging.debug('update cluster %s with all hosts %s and update: %s', clusterid, hostids, update_hostids) global_config = self.getGlobalConfig(os_version, target_system) self.updateGlobalConfig(global_config, os_version=os_version, target_system=target_system) cluster_config = self.getClusterConfig(clusterid, os_version=os_version, target_system=target_system) util.mergeDict(cluster_config, global_config, False) self.updateClusterConfig(clusterid, cluster_config, os_version=os_version, target_system=target_system) host_configs = self.getHostConfigs(hostids, os_version=os_version, target_system=target_system) CLUSTER_HOST_MERGER.merge(cluster_config, host_configs) update_host_configs = dict([ (hostid, host_config) for hostid, host_config in host_configs.items() if hostid in update_hostids ]) self.updateHostConfigs(update_host_configs, os_version=os_version, target_system=target_system)
def config(self): '''config getter.''' config = {} if self.config_data: try: config.update(json.loads(self.config_data)) config.update({'hostid': self.id, 'hostname': self.hostname}) if self.cluster: config.update({'clusterid': self.cluster.id, 'clustername': self.cluster.name}) if self.machine: util.mergeDict( config, { 'networking': { 'interfaces': { 'management': { 'mac': self.machine.mac } } } }) except Exception as error: logging.error('failed to load config %s: %s', self.hostname, self.config_data) logging.exception(error) return config
def updateClusterAndHostConfigs(self, clusterid, hostids, update_hostids, os_version, target_system): '''update cluster/host configs.''' logging.debug('update cluster %s with all hosts %s and update: %s', clusterid, hostids, update_hostids) global_config = self.getGlobalConfig(os_version, target_system) self.updateGlobalConfig(global_config, os_version=os_version, target_system=target_system) cluster_config = self.getClusterConfig( clusterid, os_version=os_version, target_system=target_system) util.mergeDict(cluster_config, global_config, False) self.updateClusterConfig( clusterid, cluster_config, os_version=os_version, target_system=target_system) host_configs = self.getHostConfigs( hostids, os_version=os_version, target_system=target_system) CLUSTER_HOST_MERGER.merge(cluster_config, host_configs) update_host_configs = dict( [(hostid, host_config) for hostid, host_config in host_configs.items() if hostid in update_hostids]) self.updateHostConfigs( update_host_configs, os_version=os_version, target_system=target_system)
def test_change_after_merge(self): lhs = {1: {2: 3}} rhs = {1: {3: [4, 5, 6]}} util.mergeDict(lhs, rhs) self.assertEqual(lhs, {1: {2: 3, 3: [4, 5, 6]}}) self.assertEqual(rhs, {1: {3: [4, 5, 6]}}) rhs[1][3].append(7) self.assertEqual(lhs, {1: {2: 3, 3: [4, 5, 6]}}) self.assertEqual(rhs, {1: {3: [4, 5, 6, 7]}})
def test_merge_not_override(self): lhs = {1: 1} rhs = {1: 2} util.mergeDict(lhs, rhs, False) self.assertEqual(lhs, {1: 1}) lhs = {1: {2: 3, 3: 5}} rhs = {1: {2: 4, 4: 6}} util.mergeDict(lhs, rhs, False) self.assertEqual(lhs, {1: {2: 3, 3: 5, 4: 6}})
def assignRolesByHostNumbers(upper_ref, from_key, lower_refs, to_key, policy_by_host_numbers={}, default={}, **_kwargs): '''assign roles by role assign policy.''' host_numbers = str(len(lower_refs)) policy_kwargs = deepcopy(default) if host_numbers in policy_by_host_numbers: util.mergeDict(policy_kwargs, policy_by_host_numbers[host_numbers]) return assignRoles(upper_ref, from_key, lower_refs, to_key, **policy_kwargs)
def test_update(self): config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8} expected_config = deepcopy(config) ref = config_reference.ConfigReference(config) config2 = {'9': 9, '10': {'10': 10}} util.mergeDict(expected_config, config2) ref.update(config2) self.assertEqual(ref.config, expected_config) ref.update(10, False) self.assertEqual(ref.config, expected_config) ref.update(10) self.assertEqual(ref.config, 10)
def getClusterConfig(self, clusterid, **kwargs): '''get cluster config.''' target_system = kwargs['target_system'] if target_system not in FROM_CLUSTER_TRANSLATORS: return {} bag = self.getDataBag(target_system) global_bag_item = dict(self.getGlobalDataBagItem(bag)) bag_item = dict( self.getClusterDataBagItem(bag, clusterid, target_system)) util.mergeDict(bag_item, global_bag_item, False) return FROM_CLUSTER_TRANSLATORS[target_system].translate(bag_item)
def getClusterConfig(self, clusterid, **kwargs): '''get cluster config.''' target_system = kwargs['target_system'] if target_system not in FROM_CLUSTER_TRANSLATORS: return {} bag = self.getDataBag(target_system) global_bag_item = dict(self.getGlobalDataBagItem(bag)) bag_item = dict(self.getClusterDataBagItem( bag, clusterid, target_system)) util.mergeDict(bag_item, global_bag_item, False) return FROM_CLUSTER_TRANSLATORS[target_system].translate(bag_item)
def getKeyFromPattern(_ref, path, from_pattern='.*', to_pattern='', **kwargs): '''Get translated key from pattern''' match = re.match(from_pattern, path) if not match: return None group = match.groupdict() util.mergeDict(group, kwargs) try: translated_key = to_pattern % group except Exception as error: logging.error('failed to get translated key from %s %% %s', to_pattern, group) logging.exception(error) return None return translated_key
def config(self, value): 'config setter' '' if not self.config_data: config = {} self.config_data = json.dumps(config) if value: try: config = json.loads(self.config_data) util.mergeDict(config, value) self.config_data = json.dumps(config) except Exception as error: logging.error('failed to dump config %s: %s', self.hostname, value) logging.exception(error)
def test_init(self): config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) self.assertIsNone(ref.parent) self.assertIsNone(ref.parent_key) config2 = {'5': {'6': 6}} ref2 = config_reference.ConfigReference(config2['5'], ref, '5') expected_config = deepcopy(config) util.mergeDict(expected_config, config2) self.assertEqual(ref.config, expected_config) self.assertEqual(id(ref.config['5']), id(ref2.config)) self.assertEqual(ref2.parent, ref) self.assertEqual(ref2.parent_key, '5') config3 = {'5': {'7': 7}} ref3 = config_reference.ConfigReference(config3['5'], ref, '5') self.assertEqual(id(ref.config['5']), id(ref3.config))
def getKeyFromPattern( _ref, path, from_pattern='.*', to_pattern='', **kwargs): '''Get translated key from pattern''' match = re.match(from_pattern, path) if not match: return None group = match.groupdict() util.mergeDict(group, kwargs) try: translated_key = to_pattern % group except Exception as error: logging.error('failed to get translated key from %s %% %s', to_pattern, group) logging.exception(error) return None return translated_key
def config(self, value): 'config setter''' if not self.config_data: config = { } self.config_data = json.dumps(config) if value: try: config = json.loads(self.config_data) util.mergeDict(config, value) self.config_data = json.dumps(config) except Exception as error: logging.error('failed to dump config %s: %s', self.hostname, value) logging.exception(error)
def getGlobalConfig(self, os_version, target_system): '''Get global config.''' config = self.config_provider.getGlobalConfig() logging.debug('got global provider config from %s: %s', self.config_provider, config) os_config = self.os_installer.getGlobalConfig( os_version=os_version, target_system=target_system) logging.debug('got global os config from %s: %s', self.os_installer, os_config) package_config = self.package_installer.getGlobalConfig( os_version=os_version, target_system=target_system) logging.debug('got global package config from %s: %s', self.package_installer, package_config) util.mergeDict(config, os_config) util.mergeDict(config, package_config) logging.debug('got global config: %s', config) return config
def getHostConfig(self, hostid, os_version, target_system): '''get host config.''' config = self.config_provider.getHostConfig(hostid) logging.debug('got host %s config from %s: %s', hostid, self.config_provider, config) os_config = self.os_installer.getHostConfig( hostid, os_version=os_version, target_system=target_system) logging.debug('got host %s config from %s: %s', hostid, self.os_installer, os_config) package_config = self.package_installer.getHostConfig( hostid, os_version=os_version, target_system=target_system) logging.debug('got host %s config from %s: %s', hostid, self.package_installer, package_config) util.mergeDict(config, os_config) util.mergeDict(config, package_config) logging.debug('got host %s config: %s', hostid, config) return config
def updateClusterConfig(self, clusterid, config, **kwargs): '''update cluster config.''' target_system = kwargs['target_system'] bag = self.getDataBag(target_system) global_bag_item = dict(self.getGlobalDataBagItem(bag)) bag_item = self.getClusterDataBagItem(bag, clusterid, target_system) bag_item_dict = dict(bag_item) util.mergeDict(bag_item_dict, global_bag_item, False) if target_system not in TO_CLUSTER_TRANSLATORS: return translated_config = TO_CLUSTER_TRANSLATORS[target_system].translate( config) util.mergeDict(bag_item_dict, translated_config) for key, value in bag_item_dict.items(): bag_item[key] = value bag_item.save()
def getClusterConfig(self, clusterid, os_version, target_system): '''get cluster config.''' config = self.config_provider.getClusterConfig(clusterid) logging.debug('got cluster %s config from %s: %s', clusterid, self.config_provider, config) os_config = self.os_installer.getClusterConfig( clusterid, os_version=os_version, target_system=target_system) logging.debug('got cluster %s config from %s: %s', clusterid, self.os_installer, os_config) package_config = self.package_installer.getClusterConfig( clusterid, os_version=os_version, target_system=target_system) logging.debug('got cluster %s config from %s: %s', clusterid, self.package_installer, package_config) util.mergeDict(config, os_config) util.mergeDict(config, package_config) logging.debug('got cluster %s config: %s', clusterid, config) return config
def _getModifySystemFromConfig(self, hostname, profile, config, **kwargs): '''get modified system config.''' logging.debug('%s[%s] get modify system from config: %s', self, hostname, config) system_config = { 'name': hostname, 'hostname': hostname, 'profile': profile, } translated_config = TO_HOST_TRANSLATOR.translate(config) logging.debug('%s[%s] get translated config: %s', self.NAME, hostname, translated_config) util.mergeDict(system_config, translated_config) ksmeta = system_config.setdefault('ksmeta', {}) package_config = {'tool': self.package_installer.NAME} util.mergeDict( package_config, self.package_installer.os_installer_config( config, **kwargs)) logging.debug('%s[%s] get package config: %s', self.NAME, hostname, package_config) util.mergeDict(ksmeta, package_config) return system_config
def _getModifySystemFromConfig(self, hostname, profile, config, **kwargs): '''get modified system config.''' logging.debug('%s[%s] get modify system from config: %s', self, hostname, config) system_config = { 'name': hostname, 'hostname': hostname, 'profile': profile, } translated_config = TO_HOST_TRANSLATOR.translate(config) logging.debug('%s[%s] get translated config: %s', self.NAME, hostname, translated_config) util.mergeDict(system_config, translated_config) ksmeta = system_config.setdefault('ksmeta', {}) package_config = {'tool': self.package_installer.NAME} util.mergeDict( package_config, self.package_installer.os_installer_config(config, **kwargs)) logging.debug('%s[%s] get package config: %s', self.NAME, hostname, package_config) util.mergeDict(ksmeta, package_config) return system_config
def config(self): '''get config from security, networking, partition''' config = {} if self.raw_config: try: config = json.loads(self.raw_config) except Exception as error: logging.error('failed to load raw config %s: %s', self.id, self.raw_config) logging.exception(error) util.mergeDict(config, {'security': self.security}) util.mergeDict(config, {'networking': self.networking}) util.mergeDict(config, {'partition': self.partition}) util.mergeDict(config, {'clusterid': self.id, 'clustername': self.name}) return config
def config(self): '''get config from security, networking, partition''' config = {} if self.raw_config: try: config = json.loads(self.raw_config) except Exception as error: logging.error('failed to load raw config %s: %s', self.id, self.raw_config) logging.exception(error) util.mergeDict(config, {'security': self.security}) util.mergeDict(config, {'networking': self.networking}) util.mergeDict(config, {'partition': self.partition}) util.mergeDict(config, { 'clusterid': self.id, 'clustername': self.name }) return config
def test_recursive_merge(self): lhs = {1: {2: 3}} rhs = {1: {3: 4}} util.mergeDict(lhs, rhs) self.assertEqual(lhs, {1: {2: 3, 3: 4}})
def test_simple_merge(self): lhs = {1: 1} rhs = {2: 2} util.mergeDict(lhs, rhs) self.assertEqual(lhs, {1: 1, 2: 2})