def _get_host_tmpl_vars_dict(self, host_id, global_vars_dict, **kwargs):
"""Generate template variables dictionary."""
vars_dict = {}
if global_vars_dict:
# Set cluster template vars_dict from cluster os_config.
vars_dict = deepcopy(global_vars_dict)
# Set hostname, MAC address and hostname, networks, dns and so on.
host_baseinfo = self.config_manager.get_host_baseinfo(host_id)
vars_dict[const.BASEINFO] = host_baseinfo
# Set profile
if self.PROFILE in kwargs:
profile = kwargs[self.PROFILE]
else:
os_version = self.config_manager.get_os_version()
profile = self._get_profile_from_server(os_version)
vars_dict[const.BASEINFO][self.PROFILE] = profile
metadata = self.config_manager.get_os_config_metadata()
os_config = self.config_manager.get_host_os_config(host_id)
# Get template variables values from host os_config
host_vars_dict = self.get_tmpl_vars_from_metadata(metadata, os_config)
util.merge_dict(
vars_dict.setdefault(const.OS_CONFIG, {}), host_vars_dict
)
return vars_dict
def _generate_system_config(self, host_id, host_vars_dict):
"""Generate updated system config from the template.
:param host_vars_dict: dict of variables for the system template to
generate system config dict for each host.
"""
os_version = self.config_manager.get_os_version()
tmpl_path = os.path.join(
os.path.join(self.tmpl_dir, os_version), self.SYS_TMPL_NAME
)
if not os.path.exists(tmpl_path):
err_msg = "Template '%s' does not exists!" % tmpl_path
logging.error(err_msg)
raise Exception(err_msg)
system_config = self.get_config_from_template(tmpl_path,
host_vars_dict)
# update package config info to cobbler ksmeta
if self.pk_installer_config and host_id in self.pk_installer_config:
pk_config = self.pk_installer_config[host_id]
ksmeta = system_config.setdefault("ksmeta", {})
util.merge_dict(ksmeta, pk_config)
system_config["ksmeta"] = ksmeta
return system_config
def test_update(self):
"""test update function."""
expected_config = copy.deepcopy(self.config_)
config2 = {'9': 9, '10': {'10': 10}}
util.merge_dict(expected_config, config2)
self.ref_.update(config2)
self.assertEqual(self.ref_.config, expected_config)
def _generate_system_config(self, host_id, host_vars_dict):
"""Generate updated system config from the template.
:param host_vars_dict: dict of variables for the system template to
generate system config dict for each host.
"""
os_version = self.config_manager.get_os_version()
tmpl_path = os.path.join(os.path.join(self.tmpl_dir, os_version),
self.SYS_TMPL_NAME)
if not os.path.exists(tmpl_path):
err_msg = "Template '%s' does not exists!" % tmpl_path
logging.error(err_msg)
raise Exception(err_msg)
system_config = self.get_config_from_template(tmpl_path,
host_vars_dict)
# update package config info to cobbler ksmeta
if self.pk_installer_config and host_id in self.pk_installer_config:
pk_config = self.pk_installer_config[host_id]
ksmeta = system_config.setdefault("ksmeta", {})
util.merge_dict(ksmeta, pk_config)
system_config["ksmeta"] = ksmeta
return system_config
def _generate_node_attributes(self, roles, host_vars_dict):
"""Generate node attributes.
Generates from templates according to its roles. The
templates are named by roles without '-'. Return the dictionary
of attributes defined in the templates.
:param list roles: The roles for this node, used to load the
specific template.
:param dict host_vars_dict: The dict used in cheetah searchList to
render attributes from templates.
"""
if not roles:
return {}
node_tmpl_dir = os.path.join(self.tmpl_dir, self.NODE_TMPL_DIR)
node_attr = {}
for role in roles:
role = role.replace('-', '_')
tmpl_name = '.'.join((role, 'tmpl'))
node_tmpl = os.path.join(node_tmpl_dir, tmpl_name)
util.merge_dict(
node_attr,
self.get_config_from_template(node_tmpl, host_vars_dict)
)
return node_attr
def merge(self, upper_ref, lower_refs):
"""merge upper config to lower configs."""
upper_sub_refs = self._get_upper_sub_refs(upper_ref)
for ref_key, sub_ref in upper_sub_refs:
sub_configs = self._get_mapping_from_upper_keys(ref_key, sub_ref)
lower_sub_refs = {}
for lower_key, lower_ref in lower_refs.items():
lower_sub_refs[lower_key] = lower_ref.setdefault(ref_key)
lower_sub_configs = self._get_mapping_from_lower_keys(
ref_key, lower_sub_refs)
util.merge_dict(sub_configs, lower_sub_configs)
values = self._get_values(ref_key, sub_ref, lower_sub_refs,
sub_configs)
logging.debug('%s set values %s to %s', ref_key, self.to_key_,
values)
for lower_key, lower_sub_ref in lower_sub_refs.items():
if lower_key not in values:
logging.error('no key %s in %s', lower_key, values)
continue
value = values[lower_key]
lower_to_ref = lower_sub_ref.setdefault(self.to_key_)
override = self._get_override(ref_key, sub_ref, self.to_key_,
lower_to_ref)
lower_to_ref.update(value, override)
def test_inputs_not_dict(self):
"""test inputs not dict."""
lhs = [1, 2, 3]
rhs = {1: 2}
merged = util.merge_dict(lhs, rhs)
expected = {1: 2}
self.assertEqual(merged, expected)
lhs = [1, 2, 3]
rhs = {1: 2}
merged = util.merge_dict(lhs, rhs, False)
expected = [1, 2, 3]
self.assertEqual(merged, expected)
lhs = {1: 2}
rhs = [1, 2, 3]
merged = util.merge_dict(lhs, rhs)
expected = [1, 2, 3]
self.assertEqual(merged, expected)
lhs = {1: 2}
rhs = [1, 2, 3]
merged = util.merge_dict(lhs, rhs, False)
expected = {1: 2}
self.assertEqual(merged, expected)
def deploy(self):
"""Deploy the cluster."""
deploy_config = {}
pk_instl_confs = {}
if self.pk_installer:
# generate target system config which will be installed by OS
# installer right after OS installation is completed.
pk_instl_confs = self.package_installer.generate_installer_config()
logging.debug('[DeployManager][deploy] pk_instl_confs is %s',
pk_instl_confs)
if self.os_installer:
logging.info('[DeployManager][deploy]get OS installer %s',
self.os_installer)
# Send package installer config info to OS installer.
if pk_instl_confs:
self.os_installer.set_package_installer_config(pk_instl_confs)
# start to deploy OS
try:
os_deploy_config = self.os_installer.deploy()
deploy_config = os_deploy_config
except Exception as ex:
logging.error(ex.message)
if self.pk_installer:
logging.info('DeployManager][deploy]get package installer %s',
self.pk_installer)
pk_deploy_config = self.package_installer.deploy()
util.merge_dict(deploy_config, pk_deploy_config)
return deploy_config
def test_update(self):
"""test update function."""
expected_config = copy.deepcopy(self.config_)
config2 = {'9': 9, '10': {'10': 10}}
util.merge_dict(expected_config, config2)
self.ref_.update(config2)
self.assertEqual(self.ref_.config, expected_config)
def _get_host_tmpl_vars_dict(self, host_id, global_vars_dict, **kwargs):
"""Generate template variables dictionary."""
vars_dict = {}
if global_vars_dict:
# Set cluster template vars_dict from cluster os_config.
vars_dict = deepcopy(global_vars_dict)
# Set hostname, MAC address and hostname, networks, dns and so on.
host_baseinfo = self.config_manager.get_host_baseinfo(host_id)
vars_dict[const.BASEINFO] = host_baseinfo
# Set profile
if self.PROFILE in kwargs:
profile = kwargs[self.PROFILE]
else:
os_version = self.config_manager.get_os_version()
profile = self._get_profile_from_server(os_version)
vars_dict[const.BASEINFO][self.PROFILE] = profile
metadata = self.config_manager.get_os_config_metadata()
os_config = self.config_manager.get_host_os_config(host_id)
# Get template variables values from host os_config
host_vars_dict = self.get_tmpl_vars_from_metadata(metadata, os_config)
util.merge_dict(vars_dict.setdefault(const.OS_CONFIG, {}),
host_vars_dict)
return vars_dict
def update_cluster_and_host_configs(self, clusterid, hostids,
update_hostids, os_version,
target_system):
"""update cluster/host configs."""
logging.debug('update cluster %s with all hosts %s and update: %s',
clusterid, hostids, update_hostids)
global_config = self.get_global_config(os_version, target_system)
self.update_global_config(global_config,
os_version=os_version,
target_system=target_system)
cluster_config = self.get_cluster_config(clusterid,
os_version=os_version,
target_system=target_system)
util.merge_dict(cluster_config, global_config, False)
self.update_cluster_config(clusterid,
cluster_config,
os_version=os_version,
target_system=target_system)
host_configs = self.get_host_configs(hostids,
os_version=os_version,
target_system=target_system)
CLUSTER_HOST_MERGER.merge(cluster_config, host_configs)
update_host_configs = dict([
(hostid, host_config)
for hostid, host_config in host_configs.items()
if hostid in update_hostids
])
self.update_host_configs(update_host_configs,
os_version=os_version,
target_system=target_system)
def config(self):
"""config getter."""
config = {}
if self.config_data:
try:
config.update(json.loads(self.config_data))
config.update({'hostid': self.id, 'hostname': self.hostname})
if self.cluster:
config.update({'clusterid': self.cluster.id,
'clustername': self.cluster.name})
if self.machine:
util.merge_dict(
config, {
'networking': {
'interfaces': {
'management': {
'mac': self.machine.mac
}
}
}
})
except Exception as error:
logging.error('failed to load config %s: %s',
self.hostname, self.config_data)
logging.exception(error)
return config
def _get_flavors_metadata_from_configuration():
"""Get flavor metadata from flavor metadata config dir."""
flavors_metadata = {}
env_locals = {}
env_locals.update(metadata_validator.VALIDATOR_LOCALS)
env_locals.update(metadata_callback.CALLBACK_LOCALS)
configs = util.load_configs(
setting.FLAVOR_METADATA_DIR,
env_locals=env_locals
)
for config in configs:
adapter_name = config['ADAPTER']
flavor_name = config['FLAVOR']
flavor_metadata = flavors_metadata.setdefault(
adapter_name, {}
).setdefault(flavor_name, {})
for key, value in config['METADATA'].items():
flavor_metadata[key] = _get_metadata_from_configuration(
key, key, value, FLAVOR_FIELDS
)
packages_metadata = PACKAGES_METADATA
adapters_flavors = adapter_api.ADAPTERS_FLAVORS
for adapter_name, adapter_flavors in adapters_flavors.items():
package_metadata = packages_metadata.get(adapter_name, {})
for flavor_name, flavor in adapter_flavors.items():
flavor_metadata = flavors_metadata.setdefault(
adapter_name, {}
).setdefault(flavor_name, {})
util.merge_dict(flavor_metadata, package_metadata, override=False)
return flavors_metadata
def update_cluster_and_host_configs(self,
clusterid,
hostids,
update_hostids,
os_version,
target_system):
"""update cluster/host configs."""
logging.debug('update cluster %s with all hosts %s and update: %s',
clusterid, hostids, update_hostids)
global_config = self.get_global_config(os_version, target_system)
self.update_global_config(global_config, os_version=os_version,
target_system=target_system)
cluster_config = self.get_cluster_config(
clusterid, os_version=os_version, target_system=target_system)
util.merge_dict(cluster_config, global_config, False)
self.update_cluster_config(
clusterid, cluster_config, os_version=os_version,
target_system=target_system)
host_configs = self.get_host_configs(
hostids, os_version=os_version,
target_system=target_system)
CLUSTER_HOST_MERGER.merge(cluster_config, host_configs)
update_host_configs = dict(
[(hostid, host_config)
for hostid, host_config in host_configs.items()
if hostid in update_hostids])
self.update_host_configs(
update_host_configs, os_version=os_version,
target_system=target_system)
def config(self):
"""config getter."""
config = {}
if self.config_data:
try:
config.update(json.loads(self.config_data))
config.update({'hostid': self.id, 'hostname': self.hostname})
if self.cluster:
config.update({'clusterid': self.cluster.id,
'clustername': self.cluster.name})
if self.machine:
util.merge_dict(
config, {
'networking': {
'interfaces': {
'management': {
'mac': self.machine.mac
}
}
}
})
except Exception as error:
logging.error('failed to load config %s: %s',
self.hostname, self.config_data)
logging.exception(error)
return config
def test_inputs_not_dict(self):
"""test inputs not dict."""
lhs = [1, 2, 3]
rhs = {1: 2}
merged = util.merge_dict(lhs, rhs)
expected = {1: 2}
self.assertEqual(merged, expected)
lhs = [1, 2, 3]
rhs = {1: 2}
merged = util.merge_dict(lhs, rhs, False)
expected = [1, 2, 3]
self.assertEqual(merged, expected)
lhs = {1: 2}
rhs = [1, 2, 3]
merged = util.merge_dict(lhs, rhs)
expected = [1, 2, 3]
self.assertEqual(merged, expected)
lhs = {1: 2}
rhs = [1, 2, 3]
merged = util.merge_dict(lhs, rhs, False)
expected = {1: 2}
self.assertEqual(merged, expected)
def _get_host_tmpl_vars(self, host_id, global_vars_dict):
"""Get templates variables dictionary for cheetah searchList based
on host package config.
:param int host_id: The host ID.
:param dict global_vars_dict: The vars_dict got from cluster level
package_config.
"""
vars_dict = {}
if global_vars_dict:
temp = global_vars_dict[const.CLUSTER][const.DEPLOYED_PK_CONFIG]
vars_dict[const.DEPLOYED_PK_CONFIG] = temp
host_baseinfo = self.config_manager.get_host_baseinfo(host_id)
util.merge_dict(vars_dict, host_baseinfo)
pk_config = self.config_manager.get_host_package_config(host_id)
if pk_config:
# Get host template variables and merge to vars_dict
metadata = self.config_manager.get_pk_config_meatadata()
host_dict = self.get_tmpl_vars_from_metadata(metadata, pk_config)
util.merge_dict(vars_dict[const.DEPLOYED_PK_CONFIG], host_dict)
# Set role_mapping for host
mapping = self.config_manager.get_host_roles_mapping(host_id)
vars_dict[const.DEPLOYED_PK_CONFIG][const.ROLES_MAPPING] = mapping
return {const.HOST: vars_dict}
def _get_flavors_metadata_from_configuration():
"""Get flavor metadata from flavor metadata config dir."""
flavors_metadata = {}
env_locals = {}
env_locals.update(metadata_validator.VALIDATOR_LOCALS)
env_locals.update(metadata_callback.CALLBACK_LOCALS)
configs = util.load_configs(setting.FLAVOR_METADATA_DIR,
env_locals=env_locals)
for config in configs:
adapter_name = config['ADAPTER']
flavor_name = config['FLAVOR']
flavor_metadata = flavors_metadata.setdefault(adapter_name,
{}).setdefault(
flavor_name, {})
for key, value in config['METADATA'].items():
flavor_metadata[key] = _get_metadata_from_configuration(
key, key, value, FLAVOR_FIELDS)
packages_metadata = PACKAGES_METADATA
adapters_flavors = adapter_api.ADAPTERS_FLAVORS
for adapter_name, adapter_flavors in adapters_flavors.items():
package_metadata = packages_metadata.get(adapter_name, {})
for flavor_name, flavor in adapter_flavors.items():
flavor_metadata = flavors_metadata.setdefault(adapter_name,
{}).setdefault(
flavor_name, {})
util.merge_dict(flavor_metadata, package_metadata, override=False)
return flavors_metadata
def _get_host_tmpl_vars(self, host_id, global_vars_dict):
"""Generate templates variables dictionary for cheetah searchList based
on host package config.
:param int host_id: The host ID.
:param dict global_vars_dict: The vars_dict got from cluster level
package_config.
The output format is the same as cluster_vars_dict.
"""
host_vars_dict = {}
if global_vars_dict:
host_vars_dict = deepcopy(global_vars_dict)
# Update host basic info
host_baseinfo = self.config_manager.get_host_baseinfo(host_id)
util.merge_dict(
host_vars_dict.setdefault(const.BASEINFO, {}), host_baseinfo
)
pk_config = self.config_manager.get_host_package_config(host_id)
if pk_config:
# Get host template variables and merge to vars_dict
metadata = self.config_manager.get_pk_config_meatadata()
meta_dict = self.get_tmpl_vars_from_metadata(metadata, pk_config)
util.merge_dict(
host_vars_dict.setdefault(const.PK_CONFIG, {}), meta_dict
)
# Override role_mapping for host if host role_mapping exists
mapping = self.config_manager.get_host_roles_mapping(host_id)
if mapping:
host_vars_dict[const.ROLES_MAPPING] = mapping
return host_vars_dict
def _get_host_tmpl_vars(self, host_id, global_vars_dict):
"""Generate templates variables dictionary.
For cheetah searchList based on host package config.
:param int host_id: The host ID.
:param dict global_vars_dict: The vars_dict got from cluster level
package_config.
The output format is the same as cluster_vars_dict.
"""
host_vars_dict = {}
if global_vars_dict:
host_vars_dict = deepcopy(global_vars_dict)
# Update host basic info
host_baseinfo = self.config_manager.get_host_baseinfo(host_id)
util.merge_dict(host_vars_dict.setdefault(const.BASEINFO, {}),
host_baseinfo)
pk_config = self.config_manager.get_host_package_config(host_id)
if pk_config:
# Get host template variables and merge to vars_dict
metadata = self.config_manager.get_pk_config_meatadata()
meta_dict = self.get_tmpl_vars_from_metadata(metadata, pk_config)
util.merge_dict(host_vars_dict.setdefault(const.PK_CONFIG, {}),
meta_dict)
# Override role_mapping for host if host role_mapping exists
mapping = self.config_manager.get_host_roles_mapping(host_id)
if mapping:
host_vars_dict[const.ROLES_MAPPING] = mapping
return host_vars_dict
def merge(self, upper_ref, lower_refs):
"""merge upper config to lower configs."""
upper_sub_refs = self._get_upper_sub_refs(upper_ref)
for ref_key, sub_ref in upper_sub_refs:
sub_configs = self._get_mapping_from_upper_keys(ref_key, sub_ref)
lower_sub_refs = {}
for lower_key, lower_ref in lower_refs.items():
lower_sub_refs[lower_key] = lower_ref.setdefault(ref_key)
lower_sub_configs = self._get_mapping_from_lower_keys(
ref_key, lower_sub_refs)
util.merge_dict(sub_configs, lower_sub_configs)
values = self._get_values(
ref_key, sub_ref, lower_sub_refs, sub_configs)
logging.debug('%s set values %s to %s',
ref_key, self.to_key_, values)
for lower_key, lower_sub_ref in lower_sub_refs.items():
if lower_key not in values:
logging.error('no key %s in %s', lower_key, values)
continue
value = values[lower_key]
lower_to_ref = lower_sub_ref.setdefault(self.to_key_)
override = self._get_override(self.to_key_, lower_to_ref)
lower_to_ref.update(value, override)
def deploy(self):
"""Deploy the cluster."""
deployed_config = self.deploy_os()
package_deployed_config = self.deploy_target_system()
util.merge_dict(deployed_config, package_deployed_config)
return deployed_config
def update_global_config(self, config, target_system, **kwargs):
"""update global config."""
global_bag_item = self._get_global_databag_item(target_system)
translated_config = TO_GLOBAL_TRANSLATORS[target_system].translate(
config)
util.merge_dict(global_bag_item, translated_config, True)
self._update_global_databag_item(target_system, global_bag_item)
def update_global_config(self, config, target_system, **kwargs):
"""update global config."""
global_bag_item = self._get_global_databag_item(target_system)
translated_config = TO_GLOBAL_TRANSLATORS[target_system].translate(
config)
util.merge_dict(global_bag_item, translated_config, True)
self._update_global_databag_item(target_system, global_bag_item)
def test_simple_merge(self):
"""simple test of merge
merge_dict performs in-place merge of rhs to lhs
"""
lhs = {1: 1}
rhs = {2: 2}
util.merge_dict(lhs, rhs)
self.assertEqual(lhs, {1: 1, 2: 2})
def test_simple_merge(self):
"""simple test of merge
merge_dict performs in-place merge of rhs to lhs
"""
lhs = {1: 1}
rhs = {2: 2}
util.merge_dict(lhs, rhs)
self.assertEqual(lhs, {1: 1, 2: 2})
def deploy(self):
"""Deploy the cluster."""
deployed_config = self.deploy_os()
package_deployed_config = self.deploy_target_system()
util.merge_dict(deployed_config, package_deployed_config)
return deployed_config
def get_cluster_config(self, clusterid, target_system, **kwargs):
"""get cluster config."""
bag = self._get_databag(target_system)
global_bag_item = dict(self._get_global_databag_item(bag))
bag_item = dict(self._get_cluster_databag_item(
bag, clusterid, target_system))
util.merge_dict(bag_item, global_bag_item, False)
return FROM_CLUSTER_TRANSLATORS[target_system].translate(bag_item)
def test_change_after_merge(self):
lhs = {1: {2: 3}}
rhs = {1: {3: [4, 5, 6]}}
util.merge_dict(lhs, rhs)
self.assertEqual(lhs, {1: {2: 3, 3: [4, 5, 6]}})
self.assertEqual(rhs, {1: {3: [4, 5, 6]}})
rhs[1][3].append(7)
self.assertEqual(lhs, {1: {2: 3, 3: [4, 5, 6]}})
self.assertEqual(rhs, {1: {3: [4, 5, 6, 7]}})
def test_change_after_merge(self):
lhs = {1: {2: 3}}
rhs = {1: {3: [4, 5, 6]}}
util.merge_dict(lhs, rhs)
self.assertEqual(lhs, {1: {2: 3, 3: [4, 5, 6]}})
self.assertEqual(rhs, {1: {3: [4, 5, 6]}})
rhs[1][3].append(7)
self.assertEqual(lhs, {1: {2: 3, 3: [4, 5, 6]}})
self.assertEqual(rhs, {1: {3: [4, 5, 6, 7]}})
def get_cluster_config(self, clusterid, target_system, **kwargs):
"""get cluster config."""
bag = self._get_databag(target_system)
global_bag_item = dict(self._get_global_databag_item(bag))
bag_item = dict(
self._get_cluster_databag_item(bag, clusterid, target_system))
util.merge_dict(bag_item, global_bag_item, False)
return FROM_CLUSTER_TRANSLATORS[target_system].translate(bag_item)
def test_merge_not_override(self):
lhs = {1: 1}
rhs = {1: 2}
util.merge_dict(lhs, rhs, False)
self.assertEqual(lhs, {1: 1})
lhs = {1: {2: 3, 3: 5}}
rhs = {1: {2: 4, 4: 6}}
util.merge_dict(lhs, rhs, False)
self.assertEqual(lhs, {1: {2: 3, 3: 5, 4: 6}})
def test_init_with_parent_update(self):
# create ConfigReference instance with parent param.
# it will update the key value pair in parent config if it exists.
config3 = {'5': {'7': 7}}
ref3 = config_reference.ConfigReference(config3['5'], self.ref_, '5')
expected_config = copy.deepcopy(self.config_)
util.merge_dict(expected_config, config3)
self.assertEqual(self.config_, expected_config)
self.assertEqual(id(self.config_['5']), id(ref3.config))
self.assertEqual(id(ref3.config), id(config3['5']))
def test_init_with_parent(self):
# create ConfigReference instance with parent param.
# it will add a key value pair in parent config if not exists.
config2 = {'5': {'6': 6}}
ref2 = config_reference.ConfigReference(config2['5'], self.ref_, '5')
expected_config = copy.deepcopy(self.config_)
util.merge_dict(expected_config, config2)
self.assertEqual(self.config_, expected_config)
self.assertEqual(id(self.config_['5']), id(ref2.config))
self.assertEqual(id(ref2.config), id(config2['5']))
def test_init_with_parent_update(self):
# create ConfigReference instance with parent param.
# it will update the key value pair in parent config if it exists.
config3 = {'5': {'7': 7}}
ref3 = config_reference.ConfigReference(config3['5'], self.ref_, '5')
expected_config = copy.deepcopy(self.config_)
util.merge_dict(expected_config, config3)
self.assertEqual(self.config_, expected_config)
self.assertEqual(id(self.config_['5']), id(ref3.config))
self.assertEqual(id(ref3.config), id(config3['5']))
def test_init_with_parent(self):
# create ConfigReference instance with parent param.
# it will add a key value pair in parent config if not exists.
config2 = {'5': {'6': 6}}
ref2 = config_reference.ConfigReference(config2['5'], self.ref_, '5')
expected_config = copy.deepcopy(self.config_)
util.merge_dict(expected_config, config2)
self.assertEqual(self.config_, expected_config)
self.assertEqual(id(self.config_['5']), id(ref2.config))
self.assertEqual(id(ref2.config), id(config2['5']))
def get_host_completed_os_config(self, host_id):
deployed_config = self.get_host_deployed_os_config(host_id)
config = self.__get_deployed_config_item(deployed_config,
const.COMPLETED_OS_CONFIG)
if not config:
config = self.get_cluster_os_config()
util.merge_dict(config, self.get_host_os_config(host_id))
deployed_config[const.COMPLETED_OS_CONFIG] = config
return config
def test_merge_not_override(self):
lhs = {1: 1}
rhs = {1: 2}
util.merge_dict(lhs, rhs, False)
self.assertEqual(lhs, {1: 1})
lhs = {1: {2: 3, 3: 5}}
rhs = {1: {2: 4, 4: 6}}
util.merge_dict(lhs, rhs, False)
self.assertEqual(lhs, {1: {2: 3, 3: 5, 4: 6}})
def assign_roles_by_host_numbers(upper_ref, from_key, lower_refs, to_key,
policy_by_host_numbers={}, default={},
**_kwargs):
"""Assign roles by role assign policy."""
host_numbers = str(len(lower_refs))
policy_kwargs = deepcopy(default)
if host_numbers in policy_by_host_numbers:
util.merge_dict(policy_kwargs, policy_by_host_numbers[host_numbers])
return assign_roles(upper_ref, from_key, lower_refs,
to_key, **policy_kwargs)
def test_merge_override(self):
"""test merge override."""
lhs = {1: 1}
rhs = {1: 2}
util.merge_dict(lhs, rhs)
self.assertEqual(lhs, {1: 2})
lhs = {1: {2: 3, 3: 5}}
rhs = {1: {2: 4, 4: 6}}
util.merge_dict(lhs, rhs)
self.assertEqual(lhs, {1: {2: 4, 3: 5, 4: 6}})
def test_merge_override(self):
"""test merge override."""
lhs = {1: 1}
rhs = {1: 2}
util.merge_dict(lhs, rhs)
self.assertEqual(lhs, {1: 2})
lhs = {1: {2: 3, 3: 5}}
rhs = {1: {2: 4, 4: 6}}
util.merge_dict(lhs, rhs)
self.assertEqual(lhs, {1: {2: 4, 3: 5, 4: 6}})
def test_init(self):
config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
config2 = {'5': {'6': 6}}
ref2 = config_reference.ConfigReference(config2['5'], ref, '5')
expected_config = deepcopy(config)
util.merge_dict(expected_config, config2)
self.assertEqual(ref.config, expected_config)
self.assertEqual(id(ref.config['5']), id(ref2.config))
config3 = {'5': {'7': 7}}
ref3 = config_reference.ConfigReference(config3['5'], ref, '5')
self.assertEqual(id(ref.config['5']), id(ref3.config))
def _get_cluster_tmpl_vars(self):
"""Generate template variables dict
Generates based on cluster level config.
The vars_dict will be:
{
"baseinfo": {
"id":1,
"name": "cluster01",
...
},
"package_config": {
.... //mapped from original package config based on metadata
},
"role_mapping": {
....
}
}
"""
cluster_vars_dict = {}
# set cluster basic information to vars_dict
cluster_baseinfo = self.config_manager.get_cluster_baseinfo()
cluster_vars_dict[const.BASEINFO] = cluster_baseinfo
# get and set template variables from cluster package config.
pk_metadata = self.config_manager.get_pk_config_meatadata()
pk_config = self.config_manager.get_cluster_package_config()
# get os config as ansible needs them
os_metadata = self.config_manager.get_os_config_metadata()
os_config = self.config_manager.get_cluster_os_config()
pk_meta_dict = self.get_tmpl_vars_from_metadata(pk_metadata, pk_config)
os_meta_dict = self.get_tmpl_vars_from_metadata(os_metadata, os_config)
util.merge_dict(pk_meta_dict, os_meta_dict)
cluster_vars_dict[const.PK_CONFIG] = pk_meta_dict
# get and set roles_mapping to vars_dict
mapping = self.config_manager.get_cluster_roles_mapping()
logging.info("cluster role mapping is %s", mapping)
cluster_vars_dict[const.ROLES_MAPPING] = mapping
# get ip settings to vars_dict
hosts_ip_settings = self.config_manager.get_hosts_ip_settings(
pk_meta_dict["network_cfg"]["ip_settings"],
pk_meta_dict["network_cfg"]["sys_intf_mappings"]
)
logging.info("hosts_ip_settings is %s", hosts_ip_settings)
cluster_vars_dict["ip_settings"] = hosts_ip_settings
return byteify(cluster_vars_dict)
def _get_cluster_tmpl_vars(self):
"""Generate template variables dict
Generates based on cluster level config.
The vars_dict will be:
{
"baseinfo": {
"id":1,
"name": "cluster01",
...
},
"package_config": {
.... //mapped from original package config based on metadata
},
"role_mapping": {
....
}
}
"""
cluster_vars_dict = {}
# set cluster basic information to vars_dict
cluster_baseinfo = self.config_manager.get_cluster_baseinfo()
cluster_vars_dict[const.BASEINFO] = cluster_baseinfo
# get and set template variables from cluster package config.
pk_metadata = self.config_manager.get_pk_config_meatadata()
pk_config = self.config_manager.get_cluster_package_config()
# get os config as ansible needs them
os_metadata = self.config_manager.get_os_config_metadata()
os_config = self.config_manager.get_cluster_os_config()
pk_meta_dict = self.get_tmpl_vars_from_metadata(pk_metadata, pk_config)
os_meta_dict = self.get_tmpl_vars_from_metadata(os_metadata, os_config)
util.merge_dict(pk_meta_dict, os_meta_dict)
cluster_vars_dict[const.PK_CONFIG] = pk_meta_dict
# get and set roles_mapping to vars_dict
mapping = self.config_manager.get_cluster_roles_mapping()
logging.info("cluster role mapping is %s", mapping)
cluster_vars_dict[const.ROLES_MAPPING] = mapping
# get ip settings to vars_dict
hosts_ip_settings = self.config_manager.get_hosts_ip_settings(
pk_meta_dict["network_cfg"]["ip_settings"],
pk_meta_dict["network_cfg"]["sys_intf_mappings"]
)
logging.info("hosts_ip_settings is %s", hosts_ip_settings)
cluster_vars_dict["ip_settings"] = hosts_ip_settings
return byteify(cluster_vars_dict)
def test_update(self):
config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8}
expected_config = deepcopy(config)
ref = config_reference.ConfigReference(config)
config2 = {'9': 9, '10': {'10': 10}}
util.merge_dict(expected_config, config2)
ref.update(config2)
self.assertEqual(ref.config, expected_config)
ref.update(10, False)
self.assertEqual(ref.config, expected_config)
ref.update(10)
self.assertEqual(ref.config, 10)
def test_init(self):
"""test init function."""
config = {'1': {'2': 3, '10': {}}, '4': [5, 6, 7], '8': 8}
ref = config_reference.ConfigReference(config)
config2 = {'5': {'6': 6}}
ref2 = config_reference.ConfigReference(config2['5'], ref, '5')
expected_config = copy.deepcopy(config)
util.merge_dict(expected_config, config2)
self.assertEqual(ref.config, expected_config)
self.assertEqual(id(ref.config['5']), id(ref2.config))
config3 = {'5': {'7': 7}}
ref3 = config_reference.ConfigReference(config3['5'], ref, '5')
self.assertEqual(id(ref.config['5']), id(ref3.config))
def update_cluster_config(self, clusterid, config, target_system,
**kwargs):
"""update cluster config."""
self.clean_cluster_config(clusterid, config, target_system, **kwargs)
global_bag_item = self._get_global_databag_item(target_system)
cluster_bag_item = self._get_cluster_databag_item(
target_system, clusterid)
util.merge_dict(cluster_bag_item, global_bag_item, False)
translated_config = TO_CLUSTER_TRANSLATORS[target_system].translate(
config)
util.merge_dict(cluster_bag_item, translated_config, True)
self._update_cluster_databag_item(target_system, clusterid,
cluster_bag_item)
def _add_flavors_from_configuration():
"""Get flavors from flavor config dir and update to adapters."""
configs = util.load_configs(setting.ADAPTER_FLAVOR_DIR)
for config in configs:
logging.info('add config %s to flavor', config)
adapter_name = config['ADAPTER_NAME']
adapter = ADAPTERS[adapter_name]
adapter_flavors = ADAPTERS_FLAVORS.setdefault(adapter_name, {})
adapter_roles = ADAPTERS_ROLES[adapter_name]
for flavor_dict in config['FLAVORS']:
flavor_name = flavor_dict['flavor']
flavor_id = '%s:%s' % (adapter_name, flavor_name)
flavor = {
'name': flavor_name,
'id': flavor_id,
'adapter_id': adapter_name,
'adapter_name': adapter_name,
'display_name': flavor_dict.get('display_name', flavor_name),
'template': flavor_dict.get('template', None)
}
flavor_roles = flavor_dict.get('roles', [])
roles_in_flavor = []
for flavor_role in flavor_roles:
if isinstance(flavor_role, basestring):
role_name = flavor_role
role_in_flavor = {
'name': role_name,
'flavor_id': flavor_id
}
else:
role_in_flavor = flavor_role
role_in_flavor['flavor_id'] = flavor_id
if 'role' in role_in_flavor:
role_in_flavor['name'] = role_in_flavor['role']
del role_in_flavor['role']
role_name = role_in_flavor['name']
role = adapter_roles[role_name]
util.merge_dict(role_in_flavor, role, override=False)
roles_in_flavor.append(role_in_flavor)
flavor['roles'] = roles_in_flavor
adapter_flavors[flavor_name] = flavor
parents = {}
for name, adapter in ADAPTERS.items():
parent = adapter.get('parent', None)
parents[name] = parent
for adapter_name, adapter_roles in ADAPTERS_FLAVORS.items():
util.recursive_merge_dict(adapter_name, ADAPTERS_FLAVORS, parents)
for adapter_name, adapter_flavors in ADAPTERS_FLAVORS.items():
adapter = ADAPTERS[adapter_name]
adapter['flavors'] = adapter_flavors.values()
def assign_roles_by_host_numbers(upper_ref, from_key, lower_refs, to_key,
policy_by_host_numbers={}, default={},
**_kwargs):
"""Assign roles by role assign policy."""
host_numbers = str(len(lower_refs))
policy_kwargs = deepcopy(default)
if host_numbers in policy_by_host_numbers:
util.merge_dict(policy_kwargs, policy_by_host_numbers[host_numbers])
else:
logging.debug('didnot find policy %s by host numbers %s',
policy_by_host_numbers, host_numbers)
return assign_roles(upper_ref, from_key, lower_refs,
to_key, **policy_kwargs)
def test_update(self):
"""test update function."""
config = {'1': {'2': '3', '10': {}}, '4': [5, 6, 7], '8': 8}
expected_config = copy.deepcopy(config)
ref = config_reference.ConfigReference(config)
config2 = {'9': 9, '10': {'10': 10}}
util.merge_dict(expected_config, config2)
ref.update(config2)
self.assertEqual(ref.config, expected_config)
ref.update(10, False)
self.assertEqual(ref.config, expected_config)
ref.update(10)
self.assertEqual(ref.config, 10)
def update_cluster_config(self, clusterid, config,
target_system, **kwargs):
"""update cluster config."""
self.clean_cluster_config(clusterid, config,
target_system, **kwargs)
global_bag_item = self._get_global_databag_item(target_system)
cluster_bag_item = self._get_cluster_databag_item(
target_system, clusterid)
util.merge_dict(cluster_bag_item, global_bag_item, False)
translated_config = TO_CLUSTER_TRANSLATORS[target_system].translate(
config)
util.merge_dict(cluster_bag_item, translated_config, True)
self._update_cluster_databag_item(
target_system, clusterid, cluster_bag_item)
def _get_cluster_tmpl_vars(self):
vars_dict = {}
cluster_baseinfo = self.config_manager.get_cluster_baseinfo()
util.merge_dict(vars_dict, cluster_baseinfo)
pk_metadata = self.config_manager.get_pk_config_meatadata()
pk_config = self.config_manager.get_cluster_package_config()
meta_dict = self.get_tmpl_vars_from_metadata(pk_metadata, pk_config)
vars_dict[const.DEPLOYED_PK_CONFIG] = meta_dict
mapping = self.config_manager.get_cluster_roles_mapping()
vars_dict[const.DEPLOYED_PK_CONFIG][const.ROLES_MAPPING] = mapping
return {const.CLUSTER: vars_dict}
def _add_flavors_from_configuration():
"""Get flavors from flavor config dir and update to adapters."""
configs = util.load_configs(setting.ADAPTER_FLAVOR_DIR)
for config in configs:
logging.info('add config %s to flavor', config)
adapter_name = config['ADAPTER_NAME']
adapter = ADAPTERS[adapter_name]
adapter_flavors = ADAPTERS_FLAVORS.setdefault(adapter_name, {})
adapter_roles = ADAPTERS_ROLES[adapter_name]
for flavor_dict in config['FLAVORS']:
flavor_name = flavor_dict['flavor']
flavor_id = '%s:%s' % (adapter_name, flavor_name)
flavor = {
'name': flavor_name,
'id': flavor_id,
'adapter_id': adapter_name,
'adapter_name': adapter_name,
'display_name': flavor_dict.get('display_name', flavor_name),
'template': flavor_dict.get('template', None)
}
flavor_roles = flavor_dict.get('roles', [])
roles_in_flavor = []
for flavor_role in flavor_roles:
if isinstance(flavor_role, basestring):
role_name = flavor_role
role_in_flavor = {
'name': role_name,
'flavor_id': flavor_id
}
else:
role_in_flavor = flavor_role
role_in_flavor['flavor_id'] = flavor_id
if 'role' in role_in_flavor:
role_in_flavor['name'] = role_in_flavor['role']
del role_in_flavor['role']
role_name = role_in_flavor['name']
role = adapter_roles[role_name]
util.merge_dict(role_in_flavor, role, override=False)
roles_in_flavor.append(role_in_flavor)
flavor['roles'] = roles_in_flavor
adapter_flavors[flavor_name] = flavor
parents = {}
for name, adapter in ADAPTERS.items():
parent = adapter.get('parent', None)
parents[name] = parent
for adapter_name, adapter_roles in ADAPTERS_FLAVORS.items():
util.recursive_merge_dict(adapter_name, ADAPTERS_FLAVORS, parents)
for adapter_name, adapter_flavors in ADAPTERS_FLAVORS.items():
adapter = ADAPTERS[adapter_name]
adapter['flavors'] = adapter_flavors.values()
def assign_roles_by_host_numbers(upper_ref,
from_key,
lower_refs,
to_key,
policy_by_host_numbers={},
default={},
**_kwargs):
"""Assign roles by role assign policy."""
host_numbers = str(len(lower_refs))
policy_kwargs = deepcopy(default)
if host_numbers in policy_by_host_numbers:
util.merge_dict(policy_kwargs, policy_by_host_numbers[host_numbers])
return assign_roles(upper_ref, from_key, lower_refs, to_key,
**policy_kwargs)
def get_key_from_pattern(
_ref, path, from_pattern='.*',
to_pattern='', **kwargs):
"""Get translated key from pattern"""
match = re.match(from_pattern, path)
if not match:
return None
group = match.groupdict()
util.merge_dict(group, kwargs)
try:
translated_key = to_pattern % group
except Exception as error:
logging.error('failed to get translated key from %s %% %s',
to_pattern, group)
raise error
return translated_key
def get_config_from_template(self, tmpl_path, vars_dict):
logging.debug("template path is %s", tmpl_path)
logging.debug("vars_dict is %s", vars_dict)
if not os.path.exists(tmpl_path) or not vars_dict:
logging.info("Template dir or vars_dict is None!")
return {}
searchList = []
copy_vars_dict = deepcopy(vars_dict)
for key, value in vars_dict.iteritems():
if isinstance(value, dict):
temp = copy_vars_dict[key]
del copy_vars_dict[key]
searchList.append(temp)
searchList.append(copy_vars_dict)
# Load base template first if it exists
base_config = {}
base_tmpl_path = os.path.join(os.path.dirname(tmpl_path), 'base.tmpl')
if os.path.isfile(base_tmpl_path) and base_tmpl_path != tmpl_path:
base_tmpl = Template(file=base_tmpl_path, searchList=searchList)
base_config = json.loads(base_tmpl.respond(), encoding='utf-8')
base_config = json.loads(json.dumps(base_config), encoding='utf-8')
# Load specific template for current adapter
tmpl = Template(file=open(tmpl_path, "r"), searchList=searchList)
config = json.loads(tmpl.respond(), encoding='utf-8')
config = json.loads(json.dumps(config), encoding='utf-8')
# Merge the two outputs
config = util.merge_dict(base_config, config)
logging.debug("get_config_from_template resulting %s", config)
return config
def config(self, value):
"""config setter"""
if not self.config_data:
config = {}
self.config_data = json.dumps(config)
if value:
try:
config = json.loads(self.config_data)
util.merge_dict(config, value)
self.config_data = json.dumps(config)
except Exception as error:
logging.error('failed to dump config %s: %s', self.hostname,
value)
logging.exception(error)
raise error
def update(self, config, override=True):
"""Update with config.
:param config: config to update.
:param override: if the instance config should be overrided
:type override: bool
"""
if (self.config is not None and
isinstance(self.config, dict) and
isinstance(config, dict)):
util.merge_dict(self.config, config, override)
elif self.config is None or override:
self.config = copy.deepcopy(config)
else:
return
self.__init__(self.config, self.parent_, self.parent_key_)
def get_global_config(self, os_version, target_system):
"""Get global config."""
config = self.config_provider_.get_global_config()
logging.debug('got global provider config from %s: %s',
self.config_provider_, config)
os_config = self.os_installer_.get_global_config(
os_version=os_version, target_system=target_system)
logging.debug('got global os config from %s: %s', self.os_installer_,
os_config)
package_config = self.package_installer_.get_global_config(
os_version=os_version, target_system=target_system)
logging.debug('got global package config from %s: %s',
self.package_installer_, package_config)
util.merge_dict(config, os_config)
util.merge_dict(config, package_config)
return config