def format_specifiers(self, specifiers):
"""Formats the given specifiers into a list.
If the list of specifiers is given as a comma-separated list, it is
inferred that the user would like a set of queries joined with
logical AND operators.
If the list of specifiers is given as a dict, it is inferred that each
key is a specifier type, and each value is a list of specifier values.
The specifier values inside each list will be joined with logical OR
operators. The lists for each key will be joined with logical AND
operators.
For example, 'name:eth0,hostname:tasty-buscuits' might match interface
eth0 on node 'tasty-biscuits'; that is, both constraints are required.
"""
if isinstance(specifiers, int):
return [str(specifiers)]
elif isinstance(specifiers, str):
return [
'&' + specifier.strip() for specifier in specifiers.split(',')
]
elif isinstance(specifiers, dict):
return specifiers
else:
return list(flatten(specifiers))
def test_returns_empty_when_nothing_provided(self):
self.assertItemsEqual([], flatten([]))
self.assertItemsEqual([], flatten(()))
self.assertItemsEqual([], flatten({}))
self.assertItemsEqual([], flatten(set()))
self.assertItemsEqual([], flatten(([], (), {}, set())))
self.assertItemsEqual([], flatten(([[]], ((), ))))
def test__renders_ntp_servers_as_comma_separated_list(self):
params = make_sample_params(self, ipv6=self.ipv6)
rendered = config.get_config(self.template, **params)
validate_dhcpd_configuration(self, rendered, self.ipv6)
ntp_servers_expected = flatten([
server if is_ip_address(server) else _get_addresses(server)
for network in params['shared_networks']
for subnet in network['subnets']
for server in subnet["ntp_servers"]
])
ntp_servers_observed = [
server for server_line in re.findall(
r"\b(?:ntp-servers|dhcp6[.]sntp-servers)\s+(.+);", rendered)
for server in server_line.split(", ")
]
self.assertItemsEqual(ntp_servers_expected, ntp_servers_observed)
def __init__(self, system_ids, sources, proxy=None):
"""Create a new importer.
:param system_ids: A sequence of rack controller system_id's.
:param sources: A sequence of endpoints; see `ImportBootImages`.
:param proxy: The HTTP/HTTPS proxy to use, or `None`
:type proxy: :class:`urlparse.ParseResult` or string
"""
super(RackControllersImporter, self).__init__()
self.system_ids = tuple(flatten(system_ids))
if isinstance(sources, Sequence):
self.sources = sources
else:
raise TypeError("expected sequence, got: %r" % (sources, ))
if proxy is None or isinstance(proxy, ParseResult):
self.proxy = proxy
else:
self.proxy = urlparse(proxy)
def test_takes_star_args(self):
self.assertItemsEqual("abcdef", flatten("a", "b", "c", "d", "e", "f"))
def test_treats_string_like_objects_as_leaves(self):
# Strings are iterable, but we know they cannot be flattened further.
self.assertItemsEqual(["abcdef"], flatten("abcdef"))
def test_flattens_other_iterables(self):
self.assertItemsEqual([1, 2, 3.3, 4, 5, 6],
flatten([1, 2, {3.3, 4, (5, 6)}]))
def test_flattens_arbitrarily_nested_lists(self):
self.assertItemsEqual(
[1, "two", "three", 4, 5, 6],
flatten([[1], ["two", "three"], [4], [5, 6]]),
)
def test_flattens_nested_lists(self):
self.assertItemsEqual([1, 2, 3, "abc"], flatten([[[1, 2, 3, "abc"]]]))
def test_flattens_list(self):
self.assertItemsEqual([1, 2, 3, "abc"], flatten([1, 2, 3, "abc"]))
def test_returns_iterator(self):
self.assertThat(flatten(()), IsInstance(Iterator))
def update_node_interfaces(node,
interfaces,
topology_hints=None,
create_fabrics=True):
"""Update the interfaces attached to the node
:param interfaces: a dict with interface details.
:param topology_hints: List of dictionaries representing hints
about fabric/VLAN connectivity.
:param create_fabrics: If True, creates fabrics associated with each
VLAN. Otherwise, creates the interfaces but does not create any
links or VLANs.
"""
# Avoid circular imports
from metadataserver.builtin_scripts.hooks import (
parse_interfaces_details,
update_interface_details,
)
# Get all of the current interfaces on this node.
current_interfaces = {
interface.id: interface
for interface in node.interface_set.all().order_by("id")
}
# Update the interfaces in dependency order. This make sure that the
# parent is created or updated before the child. The order inside
# of the sorttop result is ordered so that the modification locks that
# postgres grabs when updating interfaces is always in the same order.
# The ensures that multiple threads can call this method at the
# exact same time. Without this ordering it will deadlock because
# multiple are trying to update the same items in the database in
# a different order.
process_order = sorttop(
{name: config["parents"]
for name, config in interfaces.items()})
process_order = [sorted(list(items)) for items in process_order]
# Cache the neighbour discovery settings, since they will be used for
# every interface on this Controller.
discovery_mode = Config.objects.get_network_discovery_config()
interfaces_details = parse_interfaces_details(node)
for name in flatten(process_order):
settings = interfaces[name]
# Note: the interface that comes back from this call may be None,
# if we decided not to model an interface based on what the rack
# sent.
interface = update_interface(
node,
name,
settings,
create_fabrics=create_fabrics,
hints=topology_hints,
)
if interface is not None:
interface.update_discovery_state(discovery_mode, settings)
if interface.type == INTERFACE_TYPE.PHYSICAL:
update_interface_details(interface, interfaces_details)
if interface.id in current_interfaces:
del current_interfaces[interface.id]
if not create_fabrics:
# This could be an existing rack controller re-registering,
# so don't delete interfaces during this phase.
return
# Remove all the interfaces that no longer exist. We do this in reverse
# order so the child is deleted before the parent.
deletion_order = {}
for nic_id, nic in current_interfaces.items():
deletion_order[nic_id] = [
parent.id for parent in nic.parents.all()
if parent.id in current_interfaces
]
deletion_order = sorttop(deletion_order)
deletion_order = [sorted(list(items)) for items in deletion_order]
deletion_order = reversed(list(flatten(deletion_order)))
for delete_id in deletion_order:
if node.boot_interface_id == delete_id:
node.boot_interface = None
current_interfaces[delete_id].delete()
node.save()
def update_node_interfaces(node, data):
"""Update the interfaces attached to the node
:param data: a dict containing commissioning data
"""
from metadataserver.builtin_scripts.hooks import (
parse_interfaces,
update_interface_details,
)
if "network-extra" in data:
topology_hints = data["network-extra"]["hints"]
monitored_interfaces = data["network-extra"]["monitored-interfaces"]
address_extra = get_address_extra(data["network-extra"]["interfaces"])
else:
topology_hints = None
monitored_interfaces = []
address_extra = {}
current_interfaces = {
interface.id: interface
for interface in node.interface_set.all()
}
# Update the interfaces in dependency order. This make sure that the
# parent is created or updated before the child. The order inside
# of the sorttop result is ordered so that the modification locks that
# postgres grabs when updating interfaces is always in the same order.
# The ensures that multiple threads can call this method at the
# exact same time. Without this ordering it will deadlock because
# multiple are trying to update the same items in the database in
# a different order.
process_order = sorttop(get_interface_dependencies(data))
process_order = [sorted(list(items)) for items in process_order]
# Cache the neighbour discovery settings, since they will be used for
# every interface on this Controller.
discovery_mode = Config.objects.get_network_discovery_config()
interfaces_details = parse_interfaces(node, data)
for name in flatten(process_order):
# Note: the interface that comes back from this call may be None,
# if we decided not to model an interface based on what the rack
# sent.
interface = update_interface(
node,
name,
data,
address_extra,
hints=topology_hints,
)
if interface is None:
continue
interface.update_discovery_state(discovery_mode, name
in monitored_interfaces)
if interface.type == INTERFACE_TYPE.PHYSICAL:
update_interface_details(interface, interfaces_details)
if interface.id in current_interfaces:
del current_interfaces[interface.id]
# Remove all the interfaces that no longer exist. We do this in reverse
# order so the child is deleted before the parent.
deletion_order = {}
for nic_id, nic in current_interfaces.items():
deletion_order[nic_id] = [
parent.id for parent in nic.parents.all()
if parent.id in current_interfaces
]
deletion_order = sorttop(deletion_order)
deletion_order = [sorted(list(items)) for items in deletion_order]
deletion_order = reversed(list(flatten(deletion_order)))
for delete_id in deletion_order:
if node.boot_interface_id == delete_id:
node.boot_interface = None
current_interfaces[delete_id].delete()
node.save()