def update(self, config, override=True):
'''Update ConfigReference instance with config.
Args:
config: any type
override: bool, if the instance config should be overrided.
Returns:
None
Examples:
config = {'networking': {'management': {'ip': '1.2.3.4'}}}
ref = ConfigReference(config)
new_config = {'networking': {
'management': {'mac': '00:00:00:00:00:00', 'ip': '2.3.4.5'}
}}
ref.update(new_config)
ref.config == {'networking': {
'management': {'ip': '1.2.3.4', 'mac': '00:00:00:00:00:00'}
}}
ref.update(new_config, True)
ref.config == {'networking': {
'management': {'ip': '2.3.4.5', 'mac': '00:00:00:00:00:00'}
}}
'''
if (self.config is not None and isinstance(self.config, dict)
and isinstance(config, dict)):
util.mergeDict(self.config, config, override)
elif self.config is None or override:
self.config = deepcopy(config)
else:
return
self.__init__(self.config, self.parent, self.parent_key)
def merge(self, upper_ref, lower_refs):
'''merge upper config to lower configs.'''
upper_sub_refs = self._getUpperSubRefs(upper_ref)
for ref_key, sub_ref in upper_sub_refs:
sub_configs = self._getMappingFromUpperKeys(ref_key, sub_ref)
lower_sub_refs = {}
for lower_key, lower_ref in lower_refs.items():
lower_sub_refs[lower_key] = lower_ref.setdefault(ref_key)
lower_sub_configs = self._getMappingFromLowerKeys(
ref_key, lower_sub_refs)
util.mergeDict(sub_configs, lower_sub_configs)
values = self._getValues(ref_key, sub_ref, lower_sub_refs,
sub_configs)
logging.debug('%s set values %s to %s', ref_key, self.to_key,
values)
for lower_key, lower_sub_ref in lower_sub_refs.items():
if lower_key not in values:
logging.error('no key %s in %s', lower_key, values)
continue
value = values[lower_key]
lower_sub_ref.setdefault(self.to_key, value)
def merge(self, upper_ref, lower_refs):
'''merge upper config to lower configs.'''
upper_sub_refs = self._getUpperSubRefs(upper_ref)
for ref_key, sub_ref in upper_sub_refs:
sub_configs = self._getMappingFromUpperKeys(ref_key, sub_ref)
lower_sub_refs = {}
for lower_key, lower_ref in lower_refs.items():
lower_sub_refs[lower_key] = lower_ref.setdefault(ref_key)
lower_sub_configs = self._getMappingFromLowerKeys(
ref_key, lower_sub_refs)
util.mergeDict(sub_configs, lower_sub_configs)
values = self._getValues(
ref_key, sub_ref, lower_sub_refs, sub_configs)
logging.debug('%s set values %s to %s',
ref_key, self.to_key, values)
for lower_key, lower_sub_ref in lower_sub_refs.items():
if lower_key not in values:
logging.error('no key %s in %s', lower_key, values)
continue
value = values[lower_key]
lower_sub_ref.setdefault(self.to_key, value)
def config(self):
'''config getter.'''
config = {}
if self.config_data:
try:
config.update(json.loads(self.config_data))
config.update({'hostid': self.id, 'hostname': self.hostname})
if self.cluster:
config.update({
'clusterid': self.cluster.id,
'clustername': self.cluster.name
})
if self.machine:
util.mergeDict(
config, {
'networking': {
'interfaces': {
'management': {
'mac': self.machine.mac
}
}
}
})
except Exception as error:
logging.error('failed to load config %s: %s', self.hostname,
self.config_data)
logging.exception(error)
return config
def updateClusterAndHostConfigs(self, clusterid, hostids, update_hostids,
os_version, target_system):
'''update cluster/host configs.'''
logging.debug('update cluster %s with all hosts %s and update: %s',
clusterid, hostids, update_hostids)
global_config = self.getGlobalConfig(os_version, target_system)
self.updateGlobalConfig(global_config,
os_version=os_version,
target_system=target_system)
cluster_config = self.getClusterConfig(clusterid,
os_version=os_version,
target_system=target_system)
util.mergeDict(cluster_config, global_config, False)
self.updateClusterConfig(clusterid,
cluster_config,
os_version=os_version,
target_system=target_system)
host_configs = self.getHostConfigs(hostids,
os_version=os_version,
target_system=target_system)
CLUSTER_HOST_MERGER.merge(cluster_config, host_configs)
update_host_configs = dict([
(hostid, host_config)
for hostid, host_config in host_configs.items()
if hostid in update_hostids
])
self.updateHostConfigs(update_host_configs,
os_version=os_version,
target_system=target_system)
def config(self):
'''config getter.'''
config = {}
if self.config_data:
try:
config.update(json.loads(self.config_data))
config.update({'hostid': self.id, 'hostname': self.hostname})
if self.cluster:
config.update({'clusterid': self.cluster.id,
'clustername': self.cluster.name})
if self.machine:
util.mergeDict(
config, {
'networking': {
'interfaces': {
'management': {
'mac': self.machine.mac
}
}
}
})
except Exception as error:
logging.error('failed to load config %s: %s',
self.hostname, self.config_data)
logging.exception(error)
return config
def updateClusterAndHostConfigs(self,
clusterid,
hostids,
update_hostids,
os_version,
target_system):
'''update cluster/host configs.'''
logging.debug('update cluster %s with all hosts %s and update: %s',
clusterid, hostids, update_hostids)
global_config = self.getGlobalConfig(os_version, target_system)
self.updateGlobalConfig(global_config, os_version=os_version,
target_system=target_system)
cluster_config = self.getClusterConfig(
clusterid, os_version=os_version, target_system=target_system)
util.mergeDict(cluster_config, global_config, False)
self.updateClusterConfig(
clusterid, cluster_config, os_version=os_version,
target_system=target_system)
host_configs = self.getHostConfigs(
hostids, os_version=os_version,
target_system=target_system)
CLUSTER_HOST_MERGER.merge(cluster_config, host_configs)
update_host_configs = dict(
[(hostid, host_config)
for hostid, host_config in host_configs.items()
if hostid in update_hostids])
self.updateHostConfigs(
update_host_configs, os_version=os_version,
target_system=target_system)
def test_change_after_merge(self):
lhs = {1: {2: 3}}
rhs = {1: {3: [4, 5, 6]}}
util.mergeDict(lhs, rhs)
self.assertEqual(lhs, {1: {2: 3, 3: [4, 5, 6]}})
self.assertEqual(rhs, {1: {3: [4, 5, 6]}})
rhs[1][3].append(7)
self.assertEqual(lhs, {1: {2: 3, 3: [4, 5, 6]}})
self.assertEqual(rhs, {1: {3: [4, 5, 6, 7]}})
def test_change_after_merge(self):
lhs = {1: {2: 3}}
rhs = {1: {3: [4, 5, 6]}}
util.mergeDict(lhs, rhs)
self.assertEqual(lhs, {1: {2: 3, 3: [4, 5, 6]}})
self.assertEqual(rhs, {1: {3: [4, 5, 6]}})
rhs[1][3].append(7)
self.assertEqual(lhs, {1: {2: 3, 3: [4, 5, 6]}})
self.assertEqual(rhs, {1: {3: [4, 5, 6, 7]}})
def test_merge_not_override(self):
lhs = {1: 1}
rhs = {1: 2}
util.mergeDict(lhs, rhs, False)
self.assertEqual(lhs, {1: 1})
lhs = {1: {2: 3, 3: 5}}
rhs = {1: {2: 4, 4: 6}}
util.mergeDict(lhs, rhs, False)
self.assertEqual(lhs, {1: {2: 3, 3: 5, 4: 6}})
def test_merge_not_override(self):
lhs = {1: 1}
rhs = {1: 2}
util.mergeDict(lhs, rhs, False)
self.assertEqual(lhs, {1: 1})
lhs = {1: {2: 3, 3: 5}}
rhs = {1: {2: 4, 4: 6}}
util.mergeDict(lhs, rhs, False)
self.assertEqual(lhs, {1: {2: 3, 3: 5, 4: 6}})
def assignRolesByHostNumbers(upper_ref, from_key, lower_refs, to_key,
policy_by_host_numbers={}, default={},
**_kwargs):
'''assign roles by role assign policy.'''
host_numbers = str(len(lower_refs))
policy_kwargs = deepcopy(default)
if host_numbers in policy_by_host_numbers:
util.mergeDict(policy_kwargs, policy_by_host_numbers[host_numbers])
return assignRoles(upper_ref, from_key, lower_refs,
to_key, **policy_kwargs)
def test_update(self):
config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8}
expected_config = deepcopy(config)
ref = config_reference.ConfigReference(config)
config2 = {'9': 9, '10': {'10': 10}}
util.mergeDict(expected_config, config2)
ref.update(config2)
self.assertEqual(ref.config, expected_config)
ref.update(10, False)
self.assertEqual(ref.config, expected_config)
ref.update(10)
self.assertEqual(ref.config, 10)
def test_update(self):
config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8}
expected_config = deepcopy(config)
ref = config_reference.ConfigReference(config)
config2 = {'9': 9, '10': {'10': 10}}
util.mergeDict(expected_config, config2)
ref.update(config2)
self.assertEqual(ref.config, expected_config)
ref.update(10, False)
self.assertEqual(ref.config, expected_config)
ref.update(10)
self.assertEqual(ref.config, 10)
def getClusterConfig(self, clusterid, **kwargs):
'''get cluster config.'''
target_system = kwargs['target_system']
if target_system not in FROM_CLUSTER_TRANSLATORS:
return {}
bag = self.getDataBag(target_system)
global_bag_item = dict(self.getGlobalDataBagItem(bag))
bag_item = dict(
self.getClusterDataBagItem(bag, clusterid, target_system))
util.mergeDict(bag_item, global_bag_item, False)
return FROM_CLUSTER_TRANSLATORS[target_system].translate(bag_item)
def getClusterConfig(self, clusterid, **kwargs):
'''get cluster config.'''
target_system = kwargs['target_system']
if target_system not in FROM_CLUSTER_TRANSLATORS:
return {}
bag = self.getDataBag(target_system)
global_bag_item = dict(self.getGlobalDataBagItem(bag))
bag_item = dict(self.getClusterDataBagItem(
bag, clusterid, target_system))
util.mergeDict(bag_item, global_bag_item, False)
return FROM_CLUSTER_TRANSLATORS[target_system].translate(bag_item)
def assignRolesByHostNumbers(upper_ref,
from_key,
lower_refs,
to_key,
policy_by_host_numbers={},
default={},
**_kwargs):
'''assign roles by role assign policy.'''
host_numbers = str(len(lower_refs))
policy_kwargs = deepcopy(default)
if host_numbers in policy_by_host_numbers:
util.mergeDict(policy_kwargs, policy_by_host_numbers[host_numbers])
return assignRoles(upper_ref, from_key, lower_refs, to_key,
**policy_kwargs)
def getKeyFromPattern(_ref, path, from_pattern='.*', to_pattern='', **kwargs):
'''Get translated key from pattern'''
match = re.match(from_pattern, path)
if not match:
return None
group = match.groupdict()
util.mergeDict(group, kwargs)
try:
translated_key = to_pattern % group
except Exception as error:
logging.error('failed to get translated key from %s %% %s', to_pattern,
group)
logging.exception(error)
return None
return translated_key
def config(self, value):
'config setter' ''
if not self.config_data:
config = {}
self.config_data = json.dumps(config)
if value:
try:
config = json.loads(self.config_data)
util.mergeDict(config, value)
self.config_data = json.dumps(config)
except Exception as error:
logging.error('failed to dump config %s: %s', self.hostname,
value)
logging.exception(error)
def test_init(self):
config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
self.assertIsNone(ref.parent)
self.assertIsNone(ref.parent_key)
config2 = {'5': {'6': 6}}
ref2 = config_reference.ConfigReference(config2['5'], ref, '5')
expected_config = deepcopy(config)
util.mergeDict(expected_config, config2)
self.assertEqual(ref.config, expected_config)
self.assertEqual(id(ref.config['5']), id(ref2.config))
self.assertEqual(ref2.parent, ref)
self.assertEqual(ref2.parent_key, '5')
config3 = {'5': {'7': 7}}
ref3 = config_reference.ConfigReference(config3['5'], ref, '5')
self.assertEqual(id(ref.config['5']), id(ref3.config))
def test_init(self):
config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
self.assertIsNone(ref.parent)
self.assertIsNone(ref.parent_key)
config2 = {'5': {'6': 6}}
ref2 = config_reference.ConfigReference(config2['5'], ref, '5')
expected_config = deepcopy(config)
util.mergeDict(expected_config, config2)
self.assertEqual(ref.config, expected_config)
self.assertEqual(id(ref.config['5']), id(ref2.config))
self.assertEqual(ref2.parent, ref)
self.assertEqual(ref2.parent_key, '5')
config3 = {'5': {'7': 7}}
ref3 = config_reference.ConfigReference(config3['5'], ref, '5')
self.assertEqual(id(ref.config['5']), id(ref3.config))
def getKeyFromPattern(
_ref, path, from_pattern='.*',
to_pattern='', **kwargs):
'''Get translated key from pattern'''
match = re.match(from_pattern, path)
if not match:
return None
group = match.groupdict()
util.mergeDict(group, kwargs)
try:
translated_key = to_pattern % group
except Exception as error:
logging.error('failed to get translated key from %s %% %s',
to_pattern, group)
logging.exception(error)
return None
return translated_key
def config(self, value):
'config setter'''
if not self.config_data:
config = {
}
self.config_data = json.dumps(config)
if value:
try:
config = json.loads(self.config_data)
util.mergeDict(config, value)
self.config_data = json.dumps(config)
except Exception as error:
logging.error('failed to dump config %s: %s',
self.hostname, value)
logging.exception(error)
def getGlobalConfig(self, os_version, target_system):
'''Get global config.'''
config = self.config_provider.getGlobalConfig()
logging.debug('got global provider config from %s: %s',
self.config_provider, config)
os_config = self.os_installer.getGlobalConfig(
os_version=os_version, target_system=target_system)
logging.debug('got global os config from %s: %s', self.os_installer,
os_config)
package_config = self.package_installer.getGlobalConfig(
os_version=os_version, target_system=target_system)
logging.debug('got global package config from %s: %s',
self.package_installer, package_config)
util.mergeDict(config, os_config)
util.mergeDict(config, package_config)
logging.debug('got global config: %s', config)
return config
def getHostConfig(self, hostid, os_version, target_system):
'''get host config.'''
config = self.config_provider.getHostConfig(hostid)
logging.debug('got host %s config from %s: %s', hostid,
self.config_provider, config)
os_config = self.os_installer.getHostConfig(
hostid, os_version=os_version, target_system=target_system)
logging.debug('got host %s config from %s: %s', hostid,
self.os_installer, os_config)
package_config = self.package_installer.getHostConfig(
hostid, os_version=os_version, target_system=target_system)
logging.debug('got host %s config from %s: %s', hostid,
self.package_installer, package_config)
util.mergeDict(config, os_config)
util.mergeDict(config, package_config)
logging.debug('got host %s config: %s', hostid, config)
return config
def getGlobalConfig(self, os_version, target_system):
'''Get global config.'''
config = self.config_provider.getGlobalConfig()
logging.debug('got global provider config from %s: %s',
self.config_provider, config)
os_config = self.os_installer.getGlobalConfig(
os_version=os_version, target_system=target_system)
logging.debug('got global os config from %s: %s',
self.os_installer, os_config)
package_config = self.package_installer.getGlobalConfig(
os_version=os_version,
target_system=target_system)
logging.debug('got global package config from %s: %s',
self.package_installer, package_config)
util.mergeDict(config, os_config)
util.mergeDict(config, package_config)
logging.debug('got global config: %s', config)
return config
def updateClusterConfig(self, clusterid, config, **kwargs):
'''update cluster config.'''
target_system = kwargs['target_system']
bag = self.getDataBag(target_system)
global_bag_item = dict(self.getGlobalDataBagItem(bag))
bag_item = self.getClusterDataBagItem(bag, clusterid, target_system)
bag_item_dict = dict(bag_item)
util.mergeDict(bag_item_dict, global_bag_item, False)
if target_system not in TO_CLUSTER_TRANSLATORS:
return
translated_config = TO_CLUSTER_TRANSLATORS[target_system].translate(
config)
util.mergeDict(bag_item_dict, translated_config)
for key, value in bag_item_dict.items():
bag_item[key] = value
bag_item.save()
def getClusterConfig(self, clusterid, os_version, target_system):
'''get cluster config.'''
config = self.config_provider.getClusterConfig(clusterid)
logging.debug('got cluster %s config from %s: %s', clusterid,
self.config_provider, config)
os_config = self.os_installer.getClusterConfig(
clusterid, os_version=os_version, target_system=target_system)
logging.debug('got cluster %s config from %s: %s', clusterid,
self.os_installer, os_config)
package_config = self.package_installer.getClusterConfig(
clusterid, os_version=os_version, target_system=target_system)
logging.debug('got cluster %s config from %s: %s', clusterid,
self.package_installer, package_config)
util.mergeDict(config, os_config)
util.mergeDict(config, package_config)
logging.debug('got cluster %s config: %s', clusterid, config)
return config
def updateClusterConfig(self, clusterid, config, **kwargs):
'''update cluster config.'''
target_system = kwargs['target_system']
bag = self.getDataBag(target_system)
global_bag_item = dict(self.getGlobalDataBagItem(bag))
bag_item = self.getClusterDataBagItem(bag, clusterid, target_system)
bag_item_dict = dict(bag_item)
util.mergeDict(bag_item_dict, global_bag_item, False)
if target_system not in TO_CLUSTER_TRANSLATORS:
return
translated_config = TO_CLUSTER_TRANSLATORS[target_system].translate(
config)
util.mergeDict(bag_item_dict, translated_config)
for key, value in bag_item_dict.items():
bag_item[key] = value
bag_item.save()
def _getModifySystemFromConfig(self, hostname,
profile, config, **kwargs):
'''get modified system config.'''
logging.debug('%s[%s] get modify system from config: %s',
self, hostname, config)
system_config = {
'name': hostname,
'hostname': hostname,
'profile': profile,
}
translated_config = TO_HOST_TRANSLATOR.translate(config)
logging.debug('%s[%s] get translated config: %s',
self.NAME, hostname, translated_config)
util.mergeDict(system_config, translated_config)
ksmeta = system_config.setdefault('ksmeta', {})
package_config = {'tool': self.package_installer.NAME}
util.mergeDict(
package_config,
self.package_installer.os_installer_config(
config, **kwargs))
logging.debug('%s[%s] get package config: %s',
self.NAME, hostname, package_config)
util.mergeDict(ksmeta, package_config)
return system_config
def _getModifySystemFromConfig(self, hostname, profile, config, **kwargs):
'''get modified system config.'''
logging.debug('%s[%s] get modify system from config: %s', self,
hostname, config)
system_config = {
'name': hostname,
'hostname': hostname,
'profile': profile,
}
translated_config = TO_HOST_TRANSLATOR.translate(config)
logging.debug('%s[%s] get translated config: %s', self.NAME, hostname,
translated_config)
util.mergeDict(system_config, translated_config)
ksmeta = system_config.setdefault('ksmeta', {})
package_config = {'tool': self.package_installer.NAME}
util.mergeDict(
package_config,
self.package_installer.os_installer_config(config, **kwargs))
logging.debug('%s[%s] get package config: %s', self.NAME, hostname,
package_config)
util.mergeDict(ksmeta, package_config)
return system_config
def getClusterConfig(self, clusterid, os_version, target_system):
'''get cluster config.'''
config = self.config_provider.getClusterConfig(clusterid)
logging.debug('got cluster %s config from %s: %s',
clusterid, self.config_provider, config)
os_config = self.os_installer.getClusterConfig(
clusterid, os_version=os_version,
target_system=target_system)
logging.debug('got cluster %s config from %s: %s',
clusterid, self.os_installer, os_config)
package_config = self.package_installer.getClusterConfig(
clusterid, os_version=os_version,
target_system=target_system)
logging.debug('got cluster %s config from %s: %s',
clusterid, self.package_installer, package_config)
util.mergeDict(config, os_config)
util.mergeDict(config, package_config)
logging.debug('got cluster %s config: %s', clusterid, config)
return config
def getHostConfig(self, hostid, os_version, target_system):
'''get host config.'''
config = self.config_provider.getHostConfig(hostid)
logging.debug('got host %s config from %s: %s',
hostid, self.config_provider, config)
os_config = self.os_installer.getHostConfig(
hostid, os_version=os_version,
target_system=target_system)
logging.debug('got host %s config from %s: %s',
hostid, self.os_installer, os_config)
package_config = self.package_installer.getHostConfig(
hostid, os_version=os_version,
target_system=target_system)
logging.debug('got host %s config from %s: %s',
hostid, self.package_installer, package_config)
util.mergeDict(config, os_config)
util.mergeDict(config, package_config)
logging.debug('got host %s config: %s', hostid, config)
return config
def config(self):
'''get config from security, networking, partition'''
config = {}
if self.raw_config:
try:
config = json.loads(self.raw_config)
except Exception as error:
logging.error('failed to load raw config %s: %s',
self.id, self.raw_config)
logging.exception(error)
util.mergeDict(config, {'security': self.security})
util.mergeDict(config, {'networking': self.networking})
util.mergeDict(config, {'partition': self.partition})
util.mergeDict(config, {'clusterid': self.id,
'clustername': self.name})
return config
def update(self, config, override=True):
'''Update ConfigReference instance with config.
Args:
config: any type
override: bool, if the instance config should be overrided.
Returns:
None
Examples:
config = {'networking': {'management': {'ip': '1.2.3.4'}}}
ref = ConfigReference(config)
new_config = {'networking': {
'management': {'mac': '00:00:00:00:00:00', 'ip': '2.3.4.5'}
}}
ref.update(new_config)
ref.config == {'networking': {
'management': {'ip': '1.2.3.4', 'mac': '00:00:00:00:00:00'}
}}
ref.update(new_config, True)
ref.config == {'networking': {
'management': {'ip': '2.3.4.5', 'mac': '00:00:00:00:00:00'}
}}
'''
if (self.config is not None and
isinstance(self.config, dict) and
isinstance(config, dict)):
util.mergeDict(self.config, config, override)
elif self.config is None or override:
self.config = deepcopy(config)
else:
return
self.__init__(self.config, self.parent, self.parent_key)
def config(self):
'''get config from security, networking, partition'''
config = {}
if self.raw_config:
try:
config = json.loads(self.raw_config)
except Exception as error:
logging.error('failed to load raw config %s: %s', self.id,
self.raw_config)
logging.exception(error)
util.mergeDict(config, {'security': self.security})
util.mergeDict(config, {'networking': self.networking})
util.mergeDict(config, {'partition': self.partition})
util.mergeDict(config, {
'clusterid': self.id,
'clustername': self.name
})
return config
def test_recursive_merge(self):
lhs = {1: {2: 3}}
rhs = {1: {3: 4}}
util.mergeDict(lhs, rhs)
self.assertEqual(lhs, {1: {2: 3, 3: 4}})
def test_recursive_merge(self):
lhs = {1: {2: 3}}
rhs = {1: {3: 4}}
util.mergeDict(lhs, rhs)
self.assertEqual(lhs, {1: {2: 3, 3: 4}})
def test_simple_merge(self):
lhs = {1: 1}
rhs = {2: 2}
util.mergeDict(lhs, rhs)
self.assertEqual(lhs, {1: 1, 2: 2})
def test_simple_merge(self):
lhs = {1: 1}
rhs = {2: 2}
util.mergeDict(lhs, rhs)
self.assertEqual(lhs, {1: 1, 2: 2})
