def test_merge_all_matching_keys_copied(self): # all the keys matching the pattern will be copied # from upper config to lower configs. upper_config = {'key': 'abc', 'key2': 'def'} upper_ref = config_reference.ConfigReference(upper_config) lower_configs = {1: {}, 2: {}, 3: {}} lower_refs = {} for lower_key, lower_config in lower_configs.items(): lower_refs[lower_key] = config_reference.ConfigReference( lower_config) merger = config_merger.ConfigMapping(path_list=['key*']) merger.merge(upper_ref, lower_refs) self.assertEqual( lower_configs, { 1: { 'key': 'abc', 'key2': 'def' }, 2: { 'key': 'abc', 'key2': 'def' }, 3: { 'key': 'abc', 'key2': 'def' } })
def test_translate_override_conditions(self): # override param can be set from some config fields. config = {'key': 'abc', 'key_suffix': 'A', 'key_prefix': 'B'} ref = config_reference.ConfigReference(config) translated_config = {'BmA': 'BmA', 'mnq': 'mnq'} translated_ref = config_reference.ConfigReference(translated_config) def _generate_override2( sub_ref, ref_key, translated_sub_ref, translated_key, prefix='', suffix='', ): return (translated_key.startswith(prefix) and translated_key.endswith(suffix)) translator = config_translator.KeyTranslator( translated_keys=['BmA', 'mnq'], override=_generate_override2, override_conditions={ 'prefix': '/key_prefix', 'suffix': '/key_suffix' }) translator.translate(ref, 'key', translated_ref) self.assertEqual(translated_config, {'BmA': 'abc', 'mnq': 'mnq'})
def test_merge_value_set_by_callback(self): # key in lower config is called a callback to expected value . upper_config = {'key': 'abc', 'key2': 'def'} upper_ref = config_reference.ConfigReference(upper_config) lower_configs = {1: {}, 2: {}, 3: {}} lower_refs = {} for lower_key, lower_config in lower_configs.items(): lower_refs[lower_key] = config_reference.ConfigReference( lower_config) def _merge_value(sub_ref, ref_key, lower_sub_refs, to_key): values = {} for lower_key, lower_sub_ref in lower_sub_refs.items(): values[lower_key] = '%s.%s' % (sub_ref.config, lower_key) return values merger = config_merger.ConfigMapping(path_list=['key'], value=_merge_value) merger.merge(upper_ref, lower_refs) self.assertEqual(lower_configs, { 1: { 'key': 'abc.1' }, 2: { 'key': 'abc.2' }, 3: { 'key': 'abc.3' } })
def merge(self, upper_config, lower_configs): '''merge upper config to lower configs. Args: upper_config: any type. lower_configs: dict of {str: any type} Returns: None ''' upper_ref = config_reference.ConfigReference(upper_config) lower_refs = {} for lower_key, lower_config in lower_configs.items(): lower_refs[lower_key] = config_reference.ConfigReference( lower_config) for mapping in self.mappings: logging.debug('apply merging from the rule %s', mapping) mapping.merge(upper_ref, lower_refs) for lower_key, lower_config in lower_configs.items(): lower_configs[lower_key] = config_reference.getCleanConfig( lower_config) logging.debug('merged upper config\n%s\nto lower configs:\n%s', upper_config, lower_configs)
def test_merge_multikey_to_sam_tokey_override(self): # if multi key in upper config writes the same dest key, # the later one will override the former one if override is True. upper_config = {'key1': 'abc', 'key2': 'bcd'} upper_ref = config_reference.ConfigReference(upper_config) lower_configs = {1: {}, 2: {}, 3: {}} lower_refs = {} for lower_key, lower_config in lower_configs.items(): lower_refs[lower_key] = config_reference.ConfigReference( lower_config) merger = config_merger.ConfigMapping(path_list=['key1', 'key2'], to_key='/key', override=True) merger.merge(upper_ref, lower_refs) self.assertEqual( lower_configs, { 1: { 'key': 'bcd', 'key1': None, 'key2': None }, 2: { 'key': 'bcd', 'key1': None, 'key2': None }, 3: { 'key': 'bcd', 'key1': None, 'key2': None } })
def test_translate_from_keys(self): # generate translated_keys from some keys from config. config = {'key': 'abc', 'key_suffix': 'A', 'key_prefix': 'B'} def _generate_key(sub_ref, ref_key, prefix='', suffix=''): return '%s%s%s' % (prefix, ref_key, suffix) def _generate_keys(sub_ref, ref_key, prefix='', suffix=''): return ['%s%s%s' % (prefix, ref_key, suffix)] ref = config_reference.ConfigReference(config) translated_config = {} translated_ref = config_reference.ConfigReference(translated_config) translator = config_translator.KeyTranslator( translated_keys=[_generate_key], from_keys={ 'prefix': '/key_prefix', 'suffix': '/key_suffix' }) translator.translate(ref, 'key', translated_ref) self.assertEqual(translated_config, {'BkeyA': 'abc'}) translated_config = {} translated_ref = config_reference.ConfigReference(translated_config) translator = config_translator.KeyTranslator( translated_keys=_generate_keys, from_keys={ 'prefix': '/key_prefix', 'suffix': '/key_suffix' }) translator.translate(ref, 'key', translated_ref) self.assertEqual(translated_config, {'BkeyA': 'abc'})
def test_add_value(self): config = 'hello' ref = config_reference.ConfigReference(config) translated_config = ['hi'] translated_ref = config_reference.ConfigReference(translated_config) new_value = config_translator_callbacks.add_value( ref, None, translated_ref, None) self.assertEqual(new_value, ['hi', 'hello'])
def test_init_config_keys(self): # config key should be string. config_reference.ConfigReference(1) config_reference.ConfigReference('1') config_reference.ConfigReference([1, 2]) config_reference.ConfigReference({'1': 2}) config_reference.ConfigReference({u'1': 2}) self.assertRaises(TypeError, config_reference.ConfigReference, {1: 2})
def test_merge_from_upper_and_lower_configs(self): # set lower configs from some keys of upper config and lower configs. upper_config = {'key': 'abc', 'key_prefix': 'A', 'key_suffix': 'B'} upper_ref = config_reference.ConfigReference(upper_config) lower_configs = { 1: { 'name': 'hello' }, 2: { 'name': 'hi' }, 3: { 'name': 'today' } } lower_refs = {} for lower_key, lower_config in lower_configs.items(): lower_refs[lower_key] = config_reference.ConfigReference( lower_config) def _merge_value2(sub_ref, ref_key, lower_sub_refs, to_key, prefix='', suffix='', names={}): values = {} for lower_key, lower_sub_ref in lower_sub_refs.items(): values[lower_key] = '%s%s%s' % (prefix, names.get( lower_key, ''), suffix) return values merger = config_merger.ConfigMapping( path_list=['key'], value=_merge_value2, from_upper_keys={ 'prefix': '/key_prefix', 'suffix': '/key_suffix' }, from_lower_keys={'names': '/name'}) merger.merge(upper_ref, lower_refs) self.assertEqual( lower_configs, { 1: { 'name': 'hello', 'key': 'AhelloB' }, 2: { 'name': 'hi', 'key': 'AhiB' }, 3: { 'name': 'today', 'key': 'AtodayB' } })
def filter(self, config): '''Filter config''' ref = config_reference.ConfigReference(config) filtered_ref = config_reference.ConfigReference({}) self._filterAllows(ref, filtered_ref) self._filterDenies(filtered_ref) filtered_config = config_reference.getCleanConfig(filtered_ref.config) logging.debug('filter config %s to %s', config, filtered_config) return filtered_config
def test_translate_translated_value(self): # translated_value can be set explictly. config = {'key': 'abc', 'key_suffix': 'A', 'key_prefix': 'B'} ref = config_reference.ConfigReference(config) translated_config = {} translated_ref = config_reference.ConfigReference(translated_config) translator = config_translator.KeyTranslator(translated_keys=['mnq'], translated_value='mnq') translator.translate(ref, 'key', translated_ref) self.assertEqual(translated_config, {'mnq': 'mnq'})
def test_merge_override_callback(self): # override param can be set from callback. upper_config = { 'key1': 'abc', 'key2': 'bcd', 'key3': 'def', 'key_prefix': 'b', 'key_suffix': 'd' } upper_ref = config_reference.ConfigReference(upper_config) lower_configs = {1: {}, 2: {}, 3: {}} lower_refs = {} for lower_key, lower_config in lower_configs.items(): lower_refs[lower_key] = config_reference.ConfigReference( lower_config) def _generate_override(sub_ref, ref_key, lower_ref, to_key, prefix='', suffix=''): return (sub_ref.config.startswith(prefix) and sub_ref.config.endswith(suffix)) merger = config_merger.ConfigMapping( path_list=['key1', 'key2', 'key3'], to_key='/key', override=_generate_override, override_conditions={ 'prefix': '/key_prefix', 'suffix': '/key_suffix' }) merger.merge(upper_ref, lower_refs) self.assertEqual( lower_configs, { 1: { 'key': 'bcd', 'key1': None, 'key2': None, 'key3': None }, 2: { 'key': 'bcd', 'key1': None, 'key2': None, 'key3': None }, 3: { 'key': 'bcd', 'key1': None, 'key2': None, 'key3': None } })
def test_translate(self): # test get translated keys. # only keys in translated_keys is set in translted config. config = {'key1': 'abc', 'key2': 'bcd', 'key3': 'mnq'} ref = config_reference.ConfigReference(config) translated_config = {} translated_ref = config_reference.ConfigReference(translated_config) translator = config_translator.KeyTranslator( translated_keys=['key2', 'key3']) translator.translate(ref, 'key1', translated_ref) self.assertEqual(translated_config, {'key2': 'abc', 'key3': 'abc'})
def test_translate_translated_keys_each_key_callback(self): # each translated key can be a callback to dynamically # get the translated key. config = {'key1': 'abc', 'key2': 'bcd', 'key3': 'mnq'} ref = config_reference.ConfigReference(config) translated_config = {} translated_ref = config_reference.ConfigReference(translated_config) translator = config_translator.KeyTranslator( translated_keys=['key1', (lambda sub_ref, ref_key: 'mkey2')]) translator.translate(ref, 'key1', translated_ref) self.assertEqual(translated_config, {'key1': 'abc', 'mkey2': 'abc'})
def test_init(self): config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) config2 = {'5': {'6': 6}} ref2 = config_reference.ConfigReference(config2['5'], ref, '5') expected_config = deepcopy(config) util.merge_dict(expected_config, config2) self.assertEqual(ref.config, expected_config) self.assertEqual(id(ref.config['5']), id(ref2.config)) config3 = {'5': {'7': 7}} ref3 = config_reference.ConfigReference(config3['5'], ref, '5') self.assertEqual(id(ref.config['5']), id(ref3.config))
def test_translate_override(self): # the translated config will be overrided if override param is True. config = { 'key': 'abc', } ref = config_reference.ConfigReference(config) translated_config = {'mnq': 'mnq'} translated_ref = config_reference.ConfigReference(translated_config) translator = config_translator.KeyTranslator(translated_keys=['mnq'], override=True) translator.translate(ref, 'key', translated_ref) self.assertEqual(translated_config, {'mnq': 'abc'})
def test_len(self): ref = config_reference.ConfigReference({}) self.assertEqual(len(ref), 0) ref = config_reference.ConfigReference({'1': '2', '2': '4'}) self.assertEqual(len(ref), 2) ref = config_reference.ConfigReference({ '1': { '2': '3', '4': '5' }, '2': '4' }) self.assertEqual(len(ref), 4)
def test_translate_nooverride(self): # the translated key will be ignored when the key has already existed # in translated config and override is False. config = { 'key': 'abc', } ref = config_reference.ConfigReference(config) translated_config = {'mnq': 'mnq'} translated_ref = config_reference.ConfigReference(translated_config) translator = config_translator.KeyTranslator(translated_keys=['mnq'], override=False) translator.translate(ref, 'key', translated_ref) self.assertEqual(translated_config, {'mnq': 'mnq'})
def test_setdefault(self): config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) self.assertEqual(ref.setdefault('1/2').config, config['1']['2']) self.assertIsNone(ref.setdefault('1/3').config) self.assertEqual(ref.setdefault('1/4', 4).config, 4) self.assertEqual(4, config['1']['4'])
def test_assign_roles_by_host_number_host_number_host_number_int(self): default = { 'roles': ['control', 'api', 'compute'], 'maxs': { 'control': 1, 'api': 1, 'compute': -1 }, 'default_min': 1, 'exclusives': ['control'] } policy_by_host_numbers = { 1: { 'bundles': [['control', 'api', 'compute']] }, 2: { 'bundles': [['control', 'api']] }, } lower_configs = {1: {}} lower_refs = {} for hostid, config in lower_configs.items(): lower_refs[hostid] = config_reference.ConfigReference(config) self.assertRaises(ValueError, config_merger_callbacks.assign_roles_by_host_numbers, None, None, lower_refs, 'roles', policy_by_host_numbers=policy_by_host_numbers, default=default)
def test_assign_ips_validate(self): lower_configs = {1: {}} lower_refs = {} for hostid, config in lower_configs.items(): lower_refs[hostid] = config_reference.ConfigReference(config) # ip_start and ip_end param should be the correct format. self.assertRaises(ValueError, config_merger_callbacks.assign_ips, None, None, lower_refs, 'ip', ip_start='') self.assertRaises(ValueError, config_merger_callbacks.assign_ips, None, None, lower_refs, 'ip', ip_start='100') self.assertRaises(ValueError, config_merger_callbacks.assign_ips, None, None, lower_refs, 'ip', ip_end='') self.assertRaises(ValueError, config_merger_callbacks.assign_ips, None, None, lower_refs, 'ip', ip_end='100')
def test_noproxy(self): lower_configs = {1: {}, 2: {}} lower_refs = {} for hostid, config in lower_configs.items(): lower_refs[hostid] = config_reference.ConfigReference(config) assigned = config_merger_callbacks.assign_noproxy( None, None, lower_refs, 'noproxy', default=['127.0.0.1', 'compass', '10.145.88.3'], clusterid=1, noproxy_pattern='%(hostname)s.%(clusterid)s,%(ip)s', hostnames={ 1: 'host1', 2: 'host2' }, ips={ 1: '10.145.88.1', 2: '10.145.88.2' }) self.assertEqual( assigned, { 1: ('127.0.0.1,compass,10.145.88.3,' 'host1.1,10.145.88.1,host2.1,10.145.88.2'), 2: ('127.0.0.1,compass,10.145.88.3,' 'host1.1,10.145.88.1,host2.1,10.145.88.2') })
def test_lower_config_empty(self): lower_config = '' lower_ref = config_reference.ConfigReference(lower_config) override = config_merger_callbacks.override_if_empty( None, None, lower_ref, 'override') self.assertTrue(override) lower_config = [] lower_ref = config_reference.ConfigReference(lower_config) override = config_merger_callbacks.override_if_empty( None, None, lower_ref, 'override') self.assertTrue(override) lower_config = {} lower_ref = config_reference.ConfigReference(lower_config) override = config_merger_callbacks.override_if_empty( None, None, lower_ref, 'override') self.assertTrue(override)
def filter(self, config): """Filter config :param config: configuration to filter. :type config: dict :returns: filtered configuration as dict """ ref = config_reference.ConfigReference(config) filtered_ref = config_reference.ConfigReference({}) self._filter_allows(ref, filtered_ref) self._filter_denies(filtered_ref) filtered_config = config_reference.get_clean_config( filtered_ref.config) logging.debug('filter config %s to %s', config, filtered_config) return filtered_config
def test_filter(self): config = {'1': {'2': 'abcdef', '4': 4}, '3': ['efg', 'hij', 'k']} ref = config_reference.ConfigReference(config) self.assertEqual(ref.filter(['1/2', '1/4', '5']), { '1/2': 'abcdef', '1/4': 4 })
def test_assign_roles_by_host_number_host_number_not_found(self): lower_configs = {1: {}, 2: {}, 3: {}} lower_refs = {} for hostid, config in lower_configs.items(): lower_refs[hostid] = config_reference.ConfigReference(config) default = { 'roles': ['control', 'api', 'compute'], 'maxs': { 'control': 1, 'api': 1, 'compute': -1 }, 'default_min': 1, 'exclusives': ['control'] } policy_by_host_numbers = { '1': { 'bundles': [['control', 'api', 'compute']] }, '2': { 'bundles': [['control', 'api']] }, } assigned = config_merger_callbacks.assign_roles_by_host_numbers( None, None, lower_refs, 'roles', policy_by_host_numbers=policy_by_host_numbers, default=default) self.assertEqual(assigned, { 1: ['control'], 2: ['api'], 3: ['compute'] })
def test_ref(self): config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) self.assertRaises(KeyError, ref.ref, '') self.assertRaises(KeyError, ref.ref, '/1/2/4') self.assertEqual(ref.ref('.').config, config) self.assertEqual(ref.ref('..').config, config) self.assertEqual(ref.ref('/').config, config) self.assertEqual(ref.ref('1').config, config['1']) self.assertEqual(ref.ref('1/2').config, config['1']['2']) self.assertEqual(ref.ref('1/2/.').config, config['1']['2']) self.assertEqual(ref.ref('1/2/..').config, config['1']) self.assertEqual(ref.ref('1/2//').config, config['1']['2']) self.assertEqual(ref.ref('/1').config, config['1']) self.assertEqual(ref.ref('/1/2').config, config['1']['2']) subref = ref.ref('1') self.assertEqual(subref.ref('2').config, config['1']['2']) self.assertEqual(subref.ref('2/..').config, config['1']) self.assertEqual(subref.ref('..').config, config) self.assertEqual(subref.ref('../..').config, config) self.assertEqual(subref.ref('/').config, config) self.assertEqual(subref.ref('/4').config, config['4']) self.assertRaises(KeyError, subref.ref, '/4/5') self.assertRaises(KeyError, subref.ref, '/9') subref2 = ref.ref('9', True) self.assertEqual(ref.ref('9'), subref2)
def test_get(self): config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8} ref = config_reference.ConfigReference(config) self.assertEqual(ref.get('1/2'), config['1']['2']) self.assertIsNone(ref.get('1/3')) self.assertEqual(ref.get('1/3', 3), 3) self.assertNotIn('3', config['1'])
def test_assign_roles_hosts_portion_by_default_roles(self): roles = ['control', 'api', 'compute', 'mysql'] maxs = {'control': 1, 'api': 2, 'compute': -1, 'mysql': -1} exclusives = ['control'] bundles = [['control', 'api']] lower_configs = {1: {}, 2: {}, 3: {}, 4: {}, 5: {}, 6: {}} lower_refs = {} for hostid, config in lower_configs.items(): lower_refs[hostid] = config_reference.ConfigReference(config) assigned = config_merger_callbacks.assign_roles( None, None, lower_refs, 'roles', roles=roles, maxs=maxs, default_min=self.default_min_, exclusives=exclusives, bundles=bundles) self.assertEqual( assigned, { 1: ['control', 'api'], 2: ['compute'], 3: ['mysql'], 4: ['compute'], 5: ['mysql'], 6: ['compute'] })
def test_translate_override_by_callback(self): # override param can be set from callback. config = { 'key': 'abc', } ref = config_reference.ConfigReference(config) translated_config = {'klm': 'klm', 'mnq': 'mnq'} translated_ref = config_reference.ConfigReference(translated_config) def _generate_override(sub_ref, ref_key, translated_sub_ref, translated_key): return translated_key == 'klm' translator = config_translator.KeyTranslator( translated_keys=['klm', 'mnq'], override=_generate_override) translator.translate(ref, 'key', translated_ref) self.assertEqual(translated_config, {'klm': 'abc', 'mnq': 'mnq'})