def test_appending_present(self):
base = ['foo', 'bar', 'baz']
target = 'baz'
retval = du.append_if(base, target)
self.assertFalse(retval)
def test_appending_present(self):
base = ['foo', 'bar', 'baz']
target = 'baz'
retval = du.append_if(base, target)
self.assertFalse(retval)
def test_appending_present(self):
base = ["foo", "bar", "baz"]
target = "baz"
retval = du.append_if(base, target)
self.assertFalse(retval)
def test_appending_not_present(self):
base = ['foo', 'bar']
target = 'baz'
retval = du.append_if(base, target)
self.assertIn(target, base)
self.assertTrue(retval)
def test_appending_not_present(self):
base = ['foo', 'bar']
target = 'baz'
retval = du.append_if(base, target)
self.assertIn(target, base)
self.assertTrue(retval)
def test_appending_not_present(self):
base = ["foo", "bar"]
target = "baz"
retval = du.append_if(base, target)
self.assertIn(target, base)
self.assertTrue(retval)
def _parse_belongs_to(key, belongs_to, inventory):
"""Parse all items in a `belongs_to` list.
:param key: ``str`` Name of key to append to a given entry
:param belongs_to: ``list`` List of items to iterate over
:param inventory: ``dict`` Living dictionary of inventory
"""
for item in belongs_to:
if key not in inventory[item]['children']:
appended = append_if(array=inventory[item]['children'], item=key)
if appended:
logger.debug("Added %s to %s", key, item)
def _parse_belongs_to(key, belongs_to, inventory):
"""Parse all items in a `belongs_to` list.
:param key: ``str`` Name of key to append to a given entry
:param belongs_to: ``list`` List of items to iterate over
:param inventory: ``dict`` Living dictionary of inventory
"""
for item in belongs_to:
if key not in inventory[item]['children']:
appended = append_if(array=inventory[item]['children'], item=key)
if appended:
logger.debug("Added %s to %s", key, item)
def _parse_belongs_to(key, belongs_to, inventory):
"""Parse all items in a `belongs_to` list.
This function assumes the key defined is a group that has child subgroups,
*not* a group with hosts defined in the group configuration.
:param key: ``str`` Name of key to append to a given entry
:param belongs_to: ``list`` List of items to iterate over
:param inventory: ``dict`` Living dictionary of inventory
"""
for item in belongs_to:
if key not in inventory[item]['children']:
appended = du.append_if(array=inventory[item]['children'],
item=key)
if appended:
logger.debug("Added %s to %s", key, item)
def _parse_belongs_to(key, belongs_to, inventory):
"""Parse all items in a `belongs_to` list.
This function assumes the key defined is a group that has child subgroups,
*not* a group with hosts defined in the group configuration.
:param key: ``str`` Name of key to append to a given entry
:param belongs_to: ``list`` List of items to iterate over
:param inventory: ``dict`` Living dictionary of inventory
"""
for item in belongs_to:
if key not in inventory[item]['children']:
appended = du.append_if(array=inventory[item]['children'],
item=key)
if appended:
logger.debug("Added %s to %s", key, item)
def user_defined_setup(config, inventory):
"""Apply user defined entries from config into inventory.
:param config: ``dict`` User defined information
:param inventory: ``dict`` Living dictionary of inventory
"""
hvs = inventory['_meta']['hostvars']
for key, value in config.iteritems():
if key.endswith('hosts'):
if key not in inventory:
logger.debug("Key %s was added to inventory", key)
inventory[key] = {'hosts': []}
if value is None:
logger.debug("Key %s had no value", key)
return
for _key, _value in value.iteritems():
if _key not in hvs:
hvs[_key] = {}
hvs[_key].update({
'ansible_host': _value['ip'],
'container_address': _value['ip'],
'is_metal': True,
'physical_host_group': key
})
logger.debug("Hostvars info updated for %s", key)
# If the entry is missing the properties key add it.
properties = hvs[_key].get('properties')
if not properties or not isinstance(properties, dict):
hvs[_key]['properties'] = dict()
hvs[_key]['properties'].update({'is_metal': True})
if 'host_vars' in _value:
for _k, _v in _value['host_vars'].items():
hvs[_key][_k] = _v
ip.USED_IPS.add(_value['ip'])
appended = du.append_if(array=inventory[key]['hosts'],
item=_key)
if appended:
logger.debug("Added host %s to group %s",
_key, key)
def user_defined_setup(config, inventory):
"""Apply user defined entries from config into inventory.
:param config: ``dict`` User defined information
:param inventory: ``dict`` Living dictionary of inventory
"""
hvs = inventory['_meta']['hostvars']
for key, value in config.iteritems():
if key.endswith('hosts'):
if key not in inventory:
logger.debug("Key %s was added to inventory", key)
inventory[key] = {'hosts': []}
if value is None:
logger.debug("Key %s had no value", key)
return
for _key, _value in value.iteritems():
if _key not in hvs:
hvs[_key] = {}
hvs[_key].update({
'ansible_host': _value['ip'],
'container_address': _value['ip'],
'is_metal': True,
'physical_host_group': key
})
logger.debug("Hostvars info updated for %s", key)
# If the entry is missing the properties key add it.
properties = hvs[_key].get('properties')
if not properties or not isinstance(properties, dict):
hvs[_key]['properties'] = dict()
hvs[_key]['properties'].update({'is_metal': True})
if 'host_vars' in _value:
for _k, _v in _value['host_vars'].items():
hvs[_key][_k] = _v
ip.USED_IPS.add(_value['ip'])
appended = du.append_if(array=inventory[key]['hosts'],
item=_key)
if appended:
logger.debug("Added host %s to group %s",
_key, key)
def _find_lxc_hosts(inventory):
"""Build the lxc_hosts dynamic group
Inspect the generated inventory for nodes that host LXC containers.
Return a list of those that match for insertion into the inventory.
Populate the 'lxc_hosts' group with any node that matches.
This and the populate_lxc_hosts function are split in order to be less
coupled and more testable.
:param inventory: The dictionary containing the Ansible inventory
:returns: List of hostnames that are LXC hosts
:rtype: list
"""
host_nodes = []
for host, hostvars in inventory['_meta']['hostvars'].items():
physical_host = hostvars.get('physical_host', None)
# We want this node's "parent", so append the physical host
if not host == physical_host:
appended = du.append_if(array=host_nodes, item=physical_host)
if appended:
logger.debug("%s added to lxc_hosts group", physical_host)
return host_nodes
def _find_lxc_hosts(inventory):
"""Build the lxc_hosts dynamic group
Inspect the generated inventory for nodes that host LXC containers.
Return a list of those that match for insertion into the inventory.
Populate the 'lxc_hosts' group with any node that matches.
This and the populate_lxc_hosts function are split in order to be less
coupled and more testable.
:param inventory: The dictionary containing the Ansible inventory
:returns: List of hostnames that are LXC hosts
:rtype: list
"""
host_nodes = []
for host, hostvars in inventory['_meta']['hostvars'].items():
physical_host = hostvars.get('physical_host', None)
# We want this node's "parent", so append the physical host
if not host == physical_host:
appended = du.append_if(array=host_nodes, item=physical_host)
if appended:
logger.debug("%s added to lxc_hosts group", physical_host)
return host_nodes
def _add_container_hosts(assignment, config, container_name, container_type,
inventory, properties):
"""Add a given container name and type to the hosts.
:param assignment: ``str`` Name of container component target
:param config: ``dict`` User defined information
:param container_name: ``str`` Name fo container
:param container_type: ``str`` Type of container
:param inventory: ``dict`` Living dictionary of inventory
:param properties: ``dict`` Dict of container properties
"""
physical_host_type = '{}_hosts'.format(container_type.split('_')[0])
# If the physical host type is not in config return
if physical_host_type not in config:
return
for host_type in inventory[physical_host_type]['hosts']:
container_hosts = inventory[container_name]['hosts']
# If host_type is not in config do not append containers to it
if host_type not in config[physical_host_type]:
continue
appended = du.append_if(array=inventory['lxc_hosts']['hosts'],
item=host_type)
if appended:
logger.debug("%s added to lxc_hosts group", host_type)
# Get any set host options
host_options = config[physical_host_type][host_type]
affinity = host_options.get('affinity', {})
container_affinity = affinity.get(container_name, 1)
# Ensures that container names are not longer than 63
# This section will ensure that we are not it by the following bug:
# https://bugzilla.mindrot.org/show_bug.cgi?id=2239
type_and_name = '{}_{}'.format(host_type, container_name)
logger.debug("Generated container name %s", type_and_name)
max_hostname_len = 52
if len(type_and_name) > max_hostname_len:
raise SystemExit(
'The resulting combination of [ "{}" + "{}" ] is longer than'
' 52 characters. This combination will result in a container'
' name that is longer than the maximum allowable hostname of'
' 63 characters. Before this process can continue please'
' adjust the host entries in your "openstack_user_config.yml"'
' to use a short hostname. The recommended hostname length is'
' < 20 characters long.'.format(host_type, container_name)
)
physical_host = inventory['_meta']['hostvars'][host_type]
container_host_type = '{}-host_containers'.format(host_type)
if 'container_types' not in physical_host:
physical_host['container_types'] = container_host_type
elif physical_host['container_types'] != container_host_type:
physical_host['container_types'] = container_host_type
# Add all of the containers into the inventory
logger.debug("Building containers for host %s", container_name)
_build_container_hosts(
container_affinity,
container_hosts,
type_and_name,
inventory,
host_type,
container_type,
container_host_type,
physical_host_type,
config,
properties,
assignment,
)
# Add the physical host type to all containers from the built inventory
_append_container_types(inventory, host_type)
_append_to_host_groups(
inventory,
container_type,
assignment,
host_type,
type_and_name,
host_options
)
def _append_to_host_groups(inventory, container_type, assignment, host_type,
type_and_name, host_options):
"""Append all containers to physical (logical) groups based on host types.
:param inventory: ``dict`` Living dictionary of inventory
:param container_type: ``str`` Type of container
:param assignment: ``str`` Name of container component target
:param host_type: ``str`` Name of the host type
:param type_and_name: ``str`` Combined name of host and container name
"""
physical_group_type = '{}_all'.format(container_type.split('_')[0])
if physical_group_type not in inventory:
logger.debug("Added %s group to inventory", physical_group_type)
inventory[physical_group_type] = {'hosts': []}
iph = inventory[physical_group_type]['hosts']
iah = inventory[assignment]['hosts']
for hname, hdata in inventory['_meta']['hostvars'].iteritems():
is_metal = False
properties = hdata.get('properties')
if properties:
is_metal = properties.get('is_metal', False)
if 'container_types' in hdata or 'container_name' in hdata:
if 'container_name' not in hdata:
container = hdata['container_name'] = hname
else:
container = hdata['container_name']
component = hdata.get('component')
if container.startswith(host_type):
if 'physical_host' not in hdata:
hdata['physical_host'] = host_type
if container.startswith('{}-'.format(type_and_name)):
appended = du.append_if(array=iah, item=container)
if appended:
logger.debug("Added host %s to %s hosts",
container, assignment)
elif is_metal is True:
if component == assignment:
appended = du.append_if(array=iah, item=container)
if appended:
logger.debug("Added is_metal host %s to %s hosts",
container, assignment)
if container.startswith('{}-'.format(type_and_name)):
appended = du.append_if(array=iph, item=container)
if appended:
logger.debug("Added host %s to %s hosts",
container, physical_group_type)
elif is_metal is True:
if container.startswith(host_type):
appended = du.append_if(array=iph, item=container)
if appended:
logger.debug("Added is_metal host %s to %s hosts",
container, physical_group_type)
# Append any options in config to the host_vars of a container
container_vars = host_options.get('container_vars')
if isinstance(container_vars, dict):
for _keys, _vars in container_vars.items():
# Copy the options dictionary for manipulation
if isinstance(_vars, dict):
options = _vars.copy()
else:
options = _vars
limit = None
# If a limit is set use the limit string as a filter
# for the container name and see if it matches.
if isinstance(options, (str, dict, list)):
if 'limit_container_types' in options:
limit = options.pop(
'limit_container_types', None
)
if limit is None or (component and limit in component):
logger.debug("Set options for %s", hname)
hdata[_keys] = options
def _build_container_hosts(container_affinity, container_hosts, type_and_name,
inventory, host_type, container_type,
container_host_type, physical_host_type, config,
properties, assignment):
"""Add in all of the host associations into inventory.
This will add in all of the hosts into the inventory based on the given
affinity for a container component and its subsequent type groups.
:param container_affinity: ``int`` Set the number of a given container
:param container_hosts: ``list`` List of containers on an host
:param type_and_name: ``str`` Combined name of host and container name
:param inventory: ``dict`` Living dictionary of inventory
:param host_type: ``str`` Name of the host type
:param container_type: ``str`` Type of container
:param container_host_type: ``str`` Type of host
:param physical_host_type: ``str`` Name of physical host group
:param config: ``dict`` User defined information
:param properties: ``dict`` Container properties
:param assignment: ``str`` Name of container component target
"""
container_list = []
is_metal = False
if properties:
is_metal = properties.get('is_metal', False)
for make_container in range(container_affinity):
for i in container_hosts:
if '{}-'.format(type_and_name) in i:
du.append_if(array=container_list, item=i)
existing_count = len(list(set(container_list)))
if existing_count < container_affinity:
hostvars = inventory['_meta']['hostvars']
container_mapping = inventory[container_type]['children']
address = None
if is_metal is False:
cuuid = str(uuid.uuid4())
cuuid = cuuid.split('-')[0]
container_host_name = '{}-{}'.format(type_and_name, cuuid)
logger.debug("Generated container name %s",
container_host_name)
hostvars_options = hostvars[container_host_name] = {}
if container_host_type not in inventory:
inventory[container_host_type] = {
"hosts": [],
}
appended = du.append_if(
array=inventory[container_host_type]["hosts"],
item=container_host_name
)
if appended:
logger.debug("Added container %s to %s",
container_host_name, container_host_type)
du.append_if(array=container_hosts, item=container_host_name)
else:
if host_type not in hostvars:
hostvars[host_type] = {}
hostvars_options = hostvars[host_type]
container_host_name = host_type
host_type_config = config[physical_host_type][host_type]
address = host_type_config.get('ip')
# Create a host types containers group and append it to inventory
host_type_containers = '{}-host_containers'.format(host_type)
du.append_if(array=container_mapping, item=host_type_containers)
hostvars_options.update({
'properties': properties,
'ansible_host': address,
'container_address': address,
'container_name': container_host_name,
'physical_host': host_type,
'physical_host_group': physical_host_type,
'component': assignment
})
def _add_container_hosts(assignment, config, container_name, container_type,
inventory, properties):
"""Add a given container name and type to the hosts.
:param assignment: ``str`` Name of container component target
:param config: ``dict`` User defined information
:param container_name: ``str`` Name fo container
:param container_type: ``str`` Type of container
:param inventory: ``dict`` Living dictionary of inventory
:param properties: ``dict`` Dict of container properties
"""
physical_host_type = '{}_hosts'.format(container_type.split('_')[0])
# If the physical host type is not in config return
if physical_host_type not in config:
return
for host_type in inventory[physical_host_type]['hosts']:
container_hosts = inventory[container_name]['hosts']
# If host_type is not in config do not append containers to it
if host_type not in config[physical_host_type]:
continue
appended = du.append_if(array=inventory['lxc_hosts']['hosts'],
item=host_type)
if appended:
logger.debug("%s added to lxc_hosts group", host_type)
# Get any set host options
host_options = config[physical_host_type][host_type]
affinity = host_options.get('affinity', {})
container_affinity = affinity.get(container_name, 1)
# Ensures that container names are not longer than 63
# This section will ensure that we are not it by the following bug:
# https://bugzilla.mindrot.org/show_bug.cgi?id=2239
type_and_name = '{}_{}'.format(host_type, container_name)
logger.debug("Generated container name %s", type_and_name)
max_hostname_len = 52
if len(type_and_name) > max_hostname_len:
raise SystemExit(
'The resulting combination of [ "{}" + "{}" ] is longer than'
' 52 characters. This combination will result in a container'
' name that is longer than the maximum allowable hostname of'
' 63 characters. Before this process can continue please'
' adjust the host entries in your "openstack_user_config.yml"'
' to use a short hostname. The recommended hostname length is'
' < 20 characters long.'.format(host_type, container_name))
physical_host = inventory['_meta']['hostvars'][host_type]
container_host_type = '{}-host_containers'.format(host_type)
if 'container_types' not in physical_host:
physical_host['container_types'] = container_host_type
elif physical_host['container_types'] != container_host_type:
physical_host['container_types'] = container_host_type
# Add all of the containers into the inventory
logger.debug("Building containers for host %s", container_name)
_build_container_hosts(
container_affinity,
container_hosts,
type_and_name,
inventory,
host_type,
container_type,
container_host_type,
physical_host_type,
config,
properties,
assignment,
)
# Add the physical host type to all containers from the built inventory
_append_container_types(inventory, host_type)
_append_to_host_groups(inventory, container_type, assignment,
host_type, type_and_name, host_options)
def _append_to_host_groups(inventory, container_type, assignment, host_type,
type_and_name, host_options):
"""Append all containers to physical (logical) groups based on host types.
:param inventory: ``dict`` Living dictionary of inventory
:param container_type: ``str`` Type of container
:param assignment: ``str`` Name of container component target
:param host_type: ``str`` Name of the host type
:param type_and_name: ``str`` Combined name of host and container name
"""
physical_group_type = '{}_all'.format(container_type.split('_')[0])
if physical_group_type not in inventory:
logger.debug("Added %s group to inventory", physical_group_type)
inventory[physical_group_type] = {'hosts': []}
iph = inventory[physical_group_type]['hosts']
iah = inventory[assignment]['hosts']
for hname, hdata in inventory['_meta']['hostvars'].iteritems():
is_metal = False
properties = hdata.get('properties')
if properties:
is_metal = properties.get('is_metal', False)
if 'container_types' in hdata or 'container_name' in hdata:
if 'container_name' not in hdata:
container = hdata['container_name'] = hname
else:
container = hdata['container_name']
component = hdata.get('component')
if container.startswith(host_type):
if 'physical_host' not in hdata:
hdata['physical_host'] = host_type
if container.startswith('{}-'.format(type_and_name)):
appended = du.append_if(array=iah, item=container)
if appended:
logger.debug("Added host %s to %s hosts", container,
assignment)
elif is_metal is True:
if component == assignment:
appended = du.append_if(array=iah, item=container)
if appended:
logger.debug("Added is_metal host %s to %s hosts",
container, assignment)
if container.startswith('{}-'.format(type_and_name)):
appended = du.append_if(array=iph, item=container)
if appended:
logger.debug("Added host %s to %s hosts", container,
physical_group_type)
elif is_metal is True:
if container.startswith(host_type):
appended = du.append_if(array=iph, item=container)
if appended:
logger.debug("Added is_metal host %s to %s hosts",
container, physical_group_type)
# Append any options in config to the host_vars of a container
container_vars = host_options.get('container_vars')
if isinstance(container_vars, dict):
for _keys, _vars in container_vars.items():
# Copy the options dictionary for manipulation
if isinstance(_vars, dict):
options = _vars.copy()
else:
options = _vars
limit = None
# If a limit is set use the limit string as a filter
# for the container name and see if it matches.
if isinstance(options, (str, dict, list)):
if 'limit_container_types' in options:
limit = options.pop('limit_container_types',
None)
if limit is None or (component and limit in component):
logger.debug("Set options for %s", hname)
hdata[_keys] = options
def _build_container_hosts(container_affinity, container_hosts, type_and_name,
inventory, host_type, container_type,
container_host_type, physical_host_type, config,
properties, assignment):
"""Add in all of the host associations into inventory.
This will add in all of the hosts into the inventory based on the given
affinity for a container component and its subsequent type groups.
:param container_affinity: ``int`` Set the number of a given container
:param container_hosts: ``list`` List of containers on an host
:param type_and_name: ``str`` Combined name of host and container name
:param inventory: ``dict`` Living dictionary of inventory
:param host_type: ``str`` Name of the host type
:param container_type: ``str`` Type of container
:param container_host_type: ``str`` Type of host
:param physical_host_type: ``str`` Name of physical host group
:param config: ``dict`` User defined information
:param properties: ``dict`` Container properties
:param assignment: ``str`` Name of container component target
"""
container_list = []
is_metal = False
if properties:
is_metal = properties.get('is_metal', False)
for make_container in range(container_affinity):
for i in container_hosts:
if '{}-'.format(type_and_name) in i:
du.append_if(array=container_list, item=i)
existing_count = len(list(set(container_list)))
if existing_count < container_affinity:
hostvars = inventory['_meta']['hostvars']
container_mapping = inventory[container_type]['children']
address = None
if is_metal is False:
cuuid = str(uuid.uuid4())
cuuid = cuuid.split('-')[0]
container_host_name = '{}-{}'.format(type_and_name, cuuid)
logger.debug("Generated container name %s",
container_host_name)
hostvars_options = hostvars[container_host_name] = {}
if container_host_type not in inventory:
inventory[container_host_type] = {
"hosts": [],
}
appended = du.append_if(
array=inventory[container_host_type]["hosts"],
item=container_host_name)
if appended:
logger.debug("Added container %s to %s",
container_host_name, container_host_type)
du.append_if(array=container_hosts, item=container_host_name)
else:
if host_type not in hostvars:
hostvars[host_type] = {}
hostvars_options = hostvars[host_type]
container_host_name = host_type
host_type_config = config[physical_host_type][host_type]
address = host_type_config.get('ip')
# Create a host types containers group and append it to inventory
host_type_containers = '{}-host_containers'.format(host_type)
du.append_if(array=container_mapping, item=host_type_containers)
hostvars_options.update({
'properties': properties,
'ansible_host': address,
'container_address': address,
'container_name': container_host_name,
'physical_host': host_type,
'physical_host_group': physical_host_type,
'component': assignment
})
