def test_merge_all_matching_keys_copied(self):
# all the keys matching the pattern will be copied
# from upper config to lower configs.
upper_config = {'key': 'abc', 'key2': 'def'}
upper_ref = config_reference.ConfigReference(upper_config)
lower_configs = {1: {}, 2: {}, 3: {}}
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
merger = config_merger.ConfigMapping(path_list=['key*'])
merger.merge(upper_ref, lower_refs)
self.assertEqual(
lower_configs, {
1: {
'key': 'abc',
'key2': 'def'
},
2: {
'key': 'abc',
'key2': 'def'
},
3: {
'key': 'abc',
'key2': 'def'
}
})
def test_translate_override_conditions(self):
# override param can be set from some config fields.
config = {'key': 'abc', 'key_suffix': 'A', 'key_prefix': 'B'}
ref = config_reference.ConfigReference(config)
translated_config = {'BmA': 'BmA', 'mnq': 'mnq'}
translated_ref = config_reference.ConfigReference(translated_config)
def _generate_override2(
sub_ref,
ref_key,
translated_sub_ref,
translated_key,
prefix='',
suffix='',
):
return (translated_key.startswith(prefix)
and translated_key.endswith(suffix))
translator = config_translator.KeyTranslator(
translated_keys=['BmA', 'mnq'],
override=_generate_override2,
override_conditions={
'prefix': '/key_prefix',
'suffix': '/key_suffix'
})
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'BmA': 'abc', 'mnq': 'mnq'})
def test_merge_value_set_by_callback(self):
# key in lower config is called a callback to expected value .
upper_config = {'key': 'abc', 'key2': 'def'}
upper_ref = config_reference.ConfigReference(upper_config)
lower_configs = {1: {}, 2: {}, 3: {}}
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
def _merge_value(sub_ref, ref_key, lower_sub_refs, to_key):
values = {}
for lower_key, lower_sub_ref in lower_sub_refs.items():
values[lower_key] = '%s.%s' % (sub_ref.config, lower_key)
return values
merger = config_merger.ConfigMapping(path_list=['key'],
value=_merge_value)
merger.merge(upper_ref, lower_refs)
self.assertEqual(lower_configs, {
1: {
'key': 'abc.1'
},
2: {
'key': 'abc.2'
},
3: {
'key': 'abc.3'
}
})
def merge(self, upper_config, lower_configs):
'''merge upper config to lower configs.
Args:
upper_config: any type.
lower_configs: dict of {str: any type}
Returns:
None
'''
upper_ref = config_reference.ConfigReference(upper_config)
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
for mapping in self.mappings:
logging.debug('apply merging from the rule %s', mapping)
mapping.merge(upper_ref, lower_refs)
for lower_key, lower_config in lower_configs.items():
lower_configs[lower_key] = config_reference.getCleanConfig(
lower_config)
logging.debug('merged upper config\n%s\nto lower configs:\n%s',
upper_config, lower_configs)
def test_merge_multikey_to_sam_tokey_override(self):
# if multi key in upper config writes the same dest key,
# the later one will override the former one if override is True.
upper_config = {'key1': 'abc', 'key2': 'bcd'}
upper_ref = config_reference.ConfigReference(upper_config)
lower_configs = {1: {}, 2: {}, 3: {}}
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
merger = config_merger.ConfigMapping(path_list=['key1', 'key2'],
to_key='/key',
override=True)
merger.merge(upper_ref, lower_refs)
self.assertEqual(
lower_configs, {
1: {
'key': 'bcd',
'key1': None,
'key2': None
},
2: {
'key': 'bcd',
'key1': None,
'key2': None
},
3: {
'key': 'bcd',
'key1': None,
'key2': None
}
})
def test_translate_from_keys(self):
# generate translated_keys from some keys from config.
config = {'key': 'abc', 'key_suffix': 'A', 'key_prefix': 'B'}
def _generate_key(sub_ref, ref_key, prefix='', suffix=''):
return '%s%s%s' % (prefix, ref_key, suffix)
def _generate_keys(sub_ref, ref_key, prefix='', suffix=''):
return ['%s%s%s' % (prefix, ref_key, suffix)]
ref = config_reference.ConfigReference(config)
translated_config = {}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(
translated_keys=[_generate_key],
from_keys={
'prefix': '/key_prefix',
'suffix': '/key_suffix'
})
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'BkeyA': 'abc'})
translated_config = {}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(
translated_keys=_generate_keys,
from_keys={
'prefix': '/key_prefix',
'suffix': '/key_suffix'
})
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'BkeyA': 'abc'})
def test_add_value(self):
config = 'hello'
ref = config_reference.ConfigReference(config)
translated_config = ['hi']
translated_ref = config_reference.ConfigReference(translated_config)
new_value = config_translator_callbacks.add_value(
ref, None, translated_ref, None)
self.assertEqual(new_value, ['hi', 'hello'])
def test_init_config_keys(self):
# config key should be string.
config_reference.ConfigReference(1)
config_reference.ConfigReference('1')
config_reference.ConfigReference([1, 2])
config_reference.ConfigReference({'1': 2})
config_reference.ConfigReference({u'1': 2})
self.assertRaises(TypeError, config_reference.ConfigReference, {1: 2})
def test_merge_from_upper_and_lower_configs(self):
# set lower configs from some keys of upper config and lower configs.
upper_config = {'key': 'abc', 'key_prefix': 'A', 'key_suffix': 'B'}
upper_ref = config_reference.ConfigReference(upper_config)
lower_configs = {
1: {
'name': 'hello'
},
2: {
'name': 'hi'
},
3: {
'name': 'today'
}
}
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
def _merge_value2(sub_ref,
ref_key,
lower_sub_refs,
to_key,
prefix='',
suffix='',
names={}):
values = {}
for lower_key, lower_sub_ref in lower_sub_refs.items():
values[lower_key] = '%s%s%s' % (prefix, names.get(
lower_key, ''), suffix)
return values
merger = config_merger.ConfigMapping(
path_list=['key'],
value=_merge_value2,
from_upper_keys={
'prefix': '/key_prefix',
'suffix': '/key_suffix'
},
from_lower_keys={'names': '/name'})
merger.merge(upper_ref, lower_refs)
self.assertEqual(
lower_configs, {
1: {
'name': 'hello',
'key': 'AhelloB'
},
2: {
'name': 'hi',
'key': 'AhiB'
},
3: {
'name': 'today',
'key': 'AtodayB'
}
})
def filter(self, config):
'''Filter config'''
ref = config_reference.ConfigReference(config)
filtered_ref = config_reference.ConfigReference({})
self._filterAllows(ref, filtered_ref)
self._filterDenies(filtered_ref)
filtered_config = config_reference.getCleanConfig(filtered_ref.config)
logging.debug('filter config %s to %s', config, filtered_config)
return filtered_config
def test_translate_translated_value(self):
# translated_value can be set explictly.
config = {'key': 'abc', 'key_suffix': 'A', 'key_prefix': 'B'}
ref = config_reference.ConfigReference(config)
translated_config = {}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(translated_keys=['mnq'],
translated_value='mnq')
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'mnq': 'mnq'})
def test_merge_override_callback(self):
# override param can be set from callback.
upper_config = {
'key1': 'abc',
'key2': 'bcd',
'key3': 'def',
'key_prefix': 'b',
'key_suffix': 'd'
}
upper_ref = config_reference.ConfigReference(upper_config)
lower_configs = {1: {}, 2: {}, 3: {}}
lower_refs = {}
for lower_key, lower_config in lower_configs.items():
lower_refs[lower_key] = config_reference.ConfigReference(
lower_config)
def _generate_override(sub_ref,
ref_key,
lower_ref,
to_key,
prefix='',
suffix=''):
return (sub_ref.config.startswith(prefix)
and sub_ref.config.endswith(suffix))
merger = config_merger.ConfigMapping(
path_list=['key1', 'key2', 'key3'],
to_key='/key',
override=_generate_override,
override_conditions={
'prefix': '/key_prefix',
'suffix': '/key_suffix'
})
merger.merge(upper_ref, lower_refs)
self.assertEqual(
lower_configs, {
1: {
'key': 'bcd',
'key1': None,
'key2': None,
'key3': None
},
2: {
'key': 'bcd',
'key1': None,
'key2': None,
'key3': None
},
3: {
'key': 'bcd',
'key1': None,
'key2': None,
'key3': None
}
})
def test_translate(self):
# test get translated keys.
# only keys in translated_keys is set in translted config.
config = {'key1': 'abc', 'key2': 'bcd', 'key3': 'mnq'}
ref = config_reference.ConfigReference(config)
translated_config = {}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(
translated_keys=['key2', 'key3'])
translator.translate(ref, 'key1', translated_ref)
self.assertEqual(translated_config, {'key2': 'abc', 'key3': 'abc'})
def test_translate_translated_keys_each_key_callback(self):
# each translated key can be a callback to dynamically
# get the translated key.
config = {'key1': 'abc', 'key2': 'bcd', 'key3': 'mnq'}
ref = config_reference.ConfigReference(config)
translated_config = {}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(
translated_keys=['key1', (lambda sub_ref, ref_key: 'mkey2')])
translator.translate(ref, 'key1', translated_ref)
self.assertEqual(translated_config, {'key1': 'abc', 'mkey2': 'abc'})
def test_init(self):
config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
config2 = {'5': {'6': 6}}
ref2 = config_reference.ConfigReference(config2['5'], ref, '5')
expected_config = deepcopy(config)
util.merge_dict(expected_config, config2)
self.assertEqual(ref.config, expected_config)
self.assertEqual(id(ref.config['5']), id(ref2.config))
config3 = {'5': {'7': 7}}
ref3 = config_reference.ConfigReference(config3['5'], ref, '5')
self.assertEqual(id(ref.config['5']), id(ref3.config))
def test_translate_override(self):
# the translated config will be overrided if override param is True.
config = {
'key': 'abc',
}
ref = config_reference.ConfigReference(config)
translated_config = {'mnq': 'mnq'}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(translated_keys=['mnq'],
override=True)
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'mnq': 'abc'})
def test_len(self):
ref = config_reference.ConfigReference({})
self.assertEqual(len(ref), 0)
ref = config_reference.ConfigReference({'1': '2', '2': '4'})
self.assertEqual(len(ref), 2)
ref = config_reference.ConfigReference({
'1': {
'2': '3',
'4': '5'
},
'2': '4'
})
self.assertEqual(len(ref), 4)
def test_translate_nooverride(self):
# the translated key will be ignored when the key has already existed
# in translated config and override is False.
config = {
'key': 'abc',
}
ref = config_reference.ConfigReference(config)
translated_config = {'mnq': 'mnq'}
translated_ref = config_reference.ConfigReference(translated_config)
translator = config_translator.KeyTranslator(translated_keys=['mnq'],
override=False)
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'mnq': 'mnq'})
def test_setdefault(self):
config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
self.assertEqual(ref.setdefault('1/2').config, config['1']['2'])
self.assertIsNone(ref.setdefault('1/3').config)
self.assertEqual(ref.setdefault('1/4', 4).config, 4)
self.assertEqual(4, config['1']['4'])
def test_assign_roles_by_host_number_host_number_host_number_int(self):
default = {
'roles': ['control', 'api', 'compute'],
'maxs': {
'control': 1,
'api': 1,
'compute': -1
},
'default_min': 1,
'exclusives': ['control']
}
policy_by_host_numbers = {
1: {
'bundles': [['control', 'api', 'compute']]
},
2: {
'bundles': [['control', 'api']]
},
}
lower_configs = {1: {}}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
self.assertRaises(ValueError,
config_merger_callbacks.assign_roles_by_host_numbers,
None,
None,
lower_refs,
'roles',
policy_by_host_numbers=policy_by_host_numbers,
default=default)
def test_assign_ips_validate(self):
lower_configs = {1: {}}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
# ip_start and ip_end param should be the correct format.
self.assertRaises(ValueError,
config_merger_callbacks.assign_ips,
None,
None,
lower_refs,
'ip',
ip_start='')
self.assertRaises(ValueError,
config_merger_callbacks.assign_ips,
None,
None,
lower_refs,
'ip',
ip_start='100')
self.assertRaises(ValueError,
config_merger_callbacks.assign_ips,
None,
None,
lower_refs,
'ip',
ip_end='')
self.assertRaises(ValueError,
config_merger_callbacks.assign_ips,
None,
None,
lower_refs,
'ip',
ip_end='100')
def test_noproxy(self):
lower_configs = {1: {}, 2: {}}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_noproxy(
None,
None,
lower_refs,
'noproxy',
default=['127.0.0.1', 'compass', '10.145.88.3'],
clusterid=1,
noproxy_pattern='%(hostname)s.%(clusterid)s,%(ip)s',
hostnames={
1: 'host1',
2: 'host2'
},
ips={
1: '10.145.88.1',
2: '10.145.88.2'
})
self.assertEqual(
assigned, {
1: ('127.0.0.1,compass,10.145.88.3,'
'host1.1,10.145.88.1,host2.1,10.145.88.2'),
2: ('127.0.0.1,compass,10.145.88.3,'
'host1.1,10.145.88.1,host2.1,10.145.88.2')
})
def test_lower_config_empty(self):
lower_config = ''
lower_ref = config_reference.ConfigReference(lower_config)
override = config_merger_callbacks.override_if_empty(
None, None, lower_ref, 'override')
self.assertTrue(override)
lower_config = []
lower_ref = config_reference.ConfigReference(lower_config)
override = config_merger_callbacks.override_if_empty(
None, None, lower_ref, 'override')
self.assertTrue(override)
lower_config = {}
lower_ref = config_reference.ConfigReference(lower_config)
override = config_merger_callbacks.override_if_empty(
None, None, lower_ref, 'override')
self.assertTrue(override)
def filter(self, config):
"""Filter config
:param config: configuration to filter.
:type config: dict
:returns: filtered configuration as dict
"""
ref = config_reference.ConfigReference(config)
filtered_ref = config_reference.ConfigReference({})
self._filter_allows(ref, filtered_ref)
self._filter_denies(filtered_ref)
filtered_config = config_reference.get_clean_config(
filtered_ref.config)
logging.debug('filter config %s to %s', config, filtered_config)
return filtered_config
def test_filter(self):
config = {'1': {'2': 'abcdef', '4': 4}, '3': ['efg', 'hij', 'k']}
ref = config_reference.ConfigReference(config)
self.assertEqual(ref.filter(['1/2', '1/4', '5']), {
'1/2': 'abcdef',
'1/4': 4
})
def test_assign_roles_by_host_number_host_number_not_found(self):
lower_configs = {1: {}, 2: {}, 3: {}}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
default = {
'roles': ['control', 'api', 'compute'],
'maxs': {
'control': 1,
'api': 1,
'compute': -1
},
'default_min': 1,
'exclusives': ['control']
}
policy_by_host_numbers = {
'1': {
'bundles': [['control', 'api', 'compute']]
},
'2': {
'bundles': [['control', 'api']]
},
}
assigned = config_merger_callbacks.assign_roles_by_host_numbers(
None,
None,
lower_refs,
'roles',
policy_by_host_numbers=policy_by_host_numbers,
default=default)
self.assertEqual(assigned, {
1: ['control'],
2: ['api'],
3: ['compute']
})
def test_ref(self):
config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
self.assertRaises(KeyError, ref.ref, '')
self.assertRaises(KeyError, ref.ref, '/1/2/4')
self.assertEqual(ref.ref('.').config, config)
self.assertEqual(ref.ref('..').config, config)
self.assertEqual(ref.ref('/').config, config)
self.assertEqual(ref.ref('1').config, config['1'])
self.assertEqual(ref.ref('1/2').config, config['1']['2'])
self.assertEqual(ref.ref('1/2/.').config, config['1']['2'])
self.assertEqual(ref.ref('1/2/..').config, config['1'])
self.assertEqual(ref.ref('1/2//').config, config['1']['2'])
self.assertEqual(ref.ref('/1').config, config['1'])
self.assertEqual(ref.ref('/1/2').config, config['1']['2'])
subref = ref.ref('1')
self.assertEqual(subref.ref('2').config, config['1']['2'])
self.assertEqual(subref.ref('2/..').config, config['1'])
self.assertEqual(subref.ref('..').config, config)
self.assertEqual(subref.ref('../..').config, config)
self.assertEqual(subref.ref('/').config, config)
self.assertEqual(subref.ref('/4').config, config['4'])
self.assertRaises(KeyError, subref.ref, '/4/5')
self.assertRaises(KeyError, subref.ref, '/9')
subref2 = ref.ref('9', True)
self.assertEqual(ref.ref('9'), subref2)
def test_get(self):
config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
self.assertEqual(ref.get('1/2'), config['1']['2'])
self.assertIsNone(ref.get('1/3'))
self.assertEqual(ref.get('1/3', 3), 3)
self.assertNotIn('3', config['1'])
def test_assign_roles_hosts_portion_by_default_roles(self):
roles = ['control', 'api', 'compute', 'mysql']
maxs = {'control': 1, 'api': 2, 'compute': -1, 'mysql': -1}
exclusives = ['control']
bundles = [['control', 'api']]
lower_configs = {1: {}, 2: {}, 3: {}, 4: {}, 5: {}, 6: {}}
lower_refs = {}
for hostid, config in lower_configs.items():
lower_refs[hostid] = config_reference.ConfigReference(config)
assigned = config_merger_callbacks.assign_roles(
None,
None,
lower_refs,
'roles',
roles=roles,
maxs=maxs,
default_min=self.default_min_,
exclusives=exclusives,
bundles=bundles)
self.assertEqual(
assigned, {
1: ['control', 'api'],
2: ['compute'],
3: ['mysql'],
4: ['compute'],
5: ['mysql'],
6: ['compute']
})
def test_translate_override_by_callback(self):
# override param can be set from callback.
config = {
'key': 'abc',
}
ref = config_reference.ConfigReference(config)
translated_config = {'klm': 'klm', 'mnq': 'mnq'}
translated_ref = config_reference.ConfigReference(translated_config)
def _generate_override(sub_ref, ref_key, translated_sub_ref,
translated_key):
return translated_key == 'klm'
translator = config_translator.KeyTranslator(
translated_keys=['klm', 'mnq'], override=_generate_override)
translator.translate(ref, 'key', translated_ref)
self.assertEqual(translated_config, {'klm': 'abc', 'mnq': 'mnq'})
