def convert_ec2_metadata_network_config(network_md, macs_to_nics=None,
fallback_nic=None):
"""Convert ec2 metadata to network config version 1 data dict.
@param: network_md: 'network' portion of EC2 metadata.
generally formed as {"interfaces": {"macs": {}} where
'macs' is a dictionary with mac address as key and contents like:
{"device-number": "0", "interface-id": "...", "local-ipv4s": ...}
@param: macs_to_nics: Optional dict of mac addresses and nic names. If
not provided, get_interfaces_by_mac is called to get it from the OS.
@param: fallback_nic: Optionally provide the primary nic interface name.
This nic will be guaranteed to minimally have a dhcp4 configuration.
@return A dict of network config version 1 based on the metadata and macs.
"""
netcfg = {'version': 1, 'config': []}
if not macs_to_nics:
macs_to_nics = net.get_interfaces_by_mac()
macs_metadata = network_md['interfaces']['macs']
for mac, nic_name in macs_to_nics.items():
nic_metadata = macs_metadata.get(mac)
if not nic_metadata:
continue # Not a physical nic represented in metadata
nic_cfg = {'type': 'physical', 'name': nic_name, 'subnets': []}
nic_cfg['mac_address'] = mac
if (nic_name == fallback_nic or nic_metadata.get('public-ipv4s') or
nic_metadata.get('local-ipv4s')):
nic_cfg['subnets'].append({'type': 'dhcp4'})
if nic_metadata.get('ipv6s'):
nic_cfg['subnets'].append({'type': 'dhcp6'})
netcfg['config'].append(nic_cfg)
return netcfg
def generate_fallback_config(self):
nconf = {'config': [], 'version': 1}
for mac, name in net.get_interfaces_by_mac().items():
nconf['config'].append(
{'type': 'physical', 'name': name,
'mac_address': mac, 'subnets': [{'type': 'dhcp'}]})
return nconf
def convert_ec2_metadata_network_config(network_md, macs_to_nics=None,
fallback_nic=None):
"""Convert ec2 metadata to network config version 1 data dict.
@param: network_md: 'network' portion of EC2 metadata.
generally formed as {"interfaces": {"macs": {}} where
'macs' is a dictionary with mac address as key and contents like:
{"device-number": "0", "interface-id": "...", "local-ipv4s": ...}
@param: macs_to_nics: Optional dict of mac addresses and nic names. If
not provided, get_interfaces_by_mac is called to get it from the OS.
@param: fallback_nic: Optionally provide the primary nic interface name.
This nic will be guaranteed to minimally have a dhcp4 configuration.
@return A dict of network config version 1 based on the metadata and macs.
"""
netcfg = {'version': 1, 'config': []}
if not macs_to_nics:
macs_to_nics = net.get_interfaces_by_mac()
macs_metadata = network_md['interfaces']['macs']
for mac, nic_name in macs_to_nics.items():
nic_metadata = macs_metadata.get(mac)
if not nic_metadata:
continue # Not a physical nic represented in metadata
nic_cfg = {'type': 'physical', 'name': nic_name, 'subnets': []}
nic_cfg['mac_address'] = mac
if (nic_name == fallback_nic or nic_metadata.get('public-ipv4s') or
nic_metadata.get('local-ipv4s')):
nic_cfg['subnets'].append({'type': 'dhcp4'})
if nic_metadata.get('ipv6s'):
nic_cfg['subnets'].append({'type': 'dhcp6'})
netcfg['config'].append(nic_cfg)
return netcfg
def test_get_interfaces_by_mac(self, mock_is_FreeBSD, mock_subp):
mock_is_FreeBSD.return_value = True
mock_subp.return_value = (SAMPLE_FREEBSD_IFCONFIG_OUT, 0)
a = net.get_interfaces_by_mac()
assert a == {
'52:54:00:50:b7:0d': 'vtnet0',
'80:00:73:63:5c:48': 're0.33',
'02:14:39:0e:25:00': 'bridge0',
'02:ff:60:8c:f3:72': 'vnet0:11'
}
def generate_fallback_config(self):
nconf = {"config": [], "version": 1}
for mac, name in net.get_interfaces_by_mac().items():
nconf["config"].append({
"type": "physical",
"name": name,
"mac_address": mac,
"subnets": [{
"type": "dhcp"
}],
})
return nconf
def _ifconfig_entries(self, settings):
ifname_by_mac = net.get_interfaces_by_mac()
for interface in settings.iter_interfaces():
device_name = interface.get("name")
device_mac = interface.get("mac_address")
if device_name and re.match(r"^lo\d+$", device_name):
continue
if device_mac not in ifname_by_mac:
LOG.info("Cannot find any device with MAC %s", device_mac)
elif device_mac and device_name:
cur_name = ifname_by_mac[device_mac]
if cur_name != device_name:
LOG.info(
"netif service will rename interface %s to %s",
cur_name,
device_name,
)
try:
self.rename_interface(cur_name, device_name)
except NotImplementedError:
LOG.error(
"Interface renaming is not supported on this OS"
)
device_name = cur_name
else:
device_name = ifname_by_mac[device_mac]
LOG.info("Configuring interface %s", device_name)
self.interface_configurations[device_name] = "DHCP"
for subnet in interface.get("subnets", []):
if subnet.get("type") == "static":
if not subnet.get("netmask"):
LOG.debug(
"Skipping IP %s, because there is no netmask",
subnet.get("address"),
)
continue
LOG.debug(
"Configuring dev %s with %s / %s",
device_name,
subnet.get("address"),
subnet.get("netmask"),
)
self.interface_configurations[device_name] = {
"address": subnet.get("address"),
"netmask": subnet.get("netmask"),
"mtu": subnet.get("mtu") or interface.get("mtu"),
}
def _ensure_netfailover_safe(network_config):
"""
Search network config physical interfaces to see if any of them are
a netfailover master. If found, we prevent matching by MAC as the other
failover devices have the same MAC but need to be ignored.
Note: we rely on cloudinit.net changes which prevent netfailover devices
from being present in the provided network config. For more details about
netfailover devices, refer to cloudinit.net module.
:param network_config
A v1 or v2 network config dict with the primary NIC, and possibly
secondary nic configured. This dict will be mutated.
"""
# ignore anything that's not an actual network-config
if "version" not in network_config:
return
if network_config["version"] not in [1, 2]:
LOG.debug(
"Ignoring unknown network config version: %s",
network_config["version"],
)
return
mac_to_name = get_interfaces_by_mac()
if network_config["version"] == 1:
for cfg in [c for c in network_config["config"] if "type" in c]:
if cfg["type"] == "physical":
if "mac_address" in cfg:
mac = cfg["mac_address"]
cur_name = mac_to_name.get(mac)
if not cur_name:
continue
elif is_netfail_master(cur_name):
del cfg["mac_address"]
elif network_config["version"] == 2:
for _, cfg in network_config.get("ethernets", {}).items():
if "match" in cfg:
macaddr = cfg.get("match", {}).get("macaddress")
if macaddr:
cur_name = mac_to_name.get(macaddr)
if not cur_name:
continue
elif is_netfail_master(cur_name):
del cfg["match"]["macaddress"]
del cfg["set-name"]
cfg["match"]["name"] = cur_name
def _ifconfig_entries(self, settings, target=None):
ifname_by_mac = net.get_interfaces_by_mac()
for interface in settings.iter_interfaces():
device_name = interface.get("name")
device_mac = interface.get("mac_address")
if device_name and re.match(r'^lo\d+$', device_name):
continue
if device_mac not in ifname_by_mac:
LOG.info('Cannot find any device with MAC %s', device_mac)
elif device_mac and device_name:
cur_name = ifname_by_mac[device_mac]
if cur_name != device_name:
LOG.info('netif service will rename interface %s to %s',
cur_name, device_name)
try:
self.rename_interface(cur_name, device_name)
except NotImplementedError:
LOG.error((
'Interface renaming is '
'not supported on this OS'))
device_name = cur_name
else:
device_name = ifname_by_mac[device_mac]
LOG.info('Configuring interface %s', device_name)
self.interface_configurations[device_name] = 'DHCP'
for subnet in interface.get("subnets", []):
if subnet.get('type') == 'static':
if not subnet.get('netmask'):
LOG.debug(
'Skipping IP %s, because there is no netmask',
subnet.get('address')
)
continue
LOG.debug('Configuring dev %s with %s / %s', device_name,
subnet.get('address'), subnet.get('netmask'))
self.interface_configurations[device_name] = {
'address': subnet.get('address'),
'netmask': subnet.get('netmask'),
'mtu': subnet.get('mtu'),
}
def _write_ifconfig_entries(self, settings, target=None):
ifname_by_mac = net.get_interfaces_by_mac()
for interface in settings.iter_interfaces():
device_name = interface.get("name")
device_mac = interface.get("mac_address")
if device_name and re.match(r'^lo\d+$', device_name):
continue
if device_mac not in ifname_by_mac:
LOG.info('Cannot find any device with MAC %s', device_mac)
elif device_mac and device_name:
cur_name = ifname_by_mac[device_mac]
if cur_name != device_name:
LOG.info('netif service will rename interface %s to %s',
cur_name, device_name)
self._update_rc_conf(
{'ifconfig_%s_name' % cur_name: device_name},
target=target)
else:
device_name = ifname_by_mac[device_mac]
LOG.info('Configuring interface %s', device_name)
ifconfig = 'DHCP' # default
for subnet in interface.get("subnets", []):
if ifconfig != 'DHCP':
LOG.info('The FreeBSD provider only set the first subnet.')
break
if subnet.get('type') == 'static':
if not subnet.get('netmask'):
LOG.debug(
'Skipping IP %s, because there is no netmask',
subnet.get('address'))
continue
LOG.debug('Configuring dev %s with %s / %s', device_name,
subnet.get('address'), subnet.get('netmask'))
# Configure an ipv4 address.
ifconfig = (subnet.get('address') + ' netmask ' +
subnet.get('netmask'))
if ifconfig == 'DHCP':
self.dhcp_interfaces.append(device_name)
self._update_rc_conf({'ifconfig_' + device_name: ifconfig},
target=target)
def convert_net_json(network_json=None, known_macs=None):
"""Return a dictionary of network_config by parsing provided
OpenStack ConfigDrive NetworkData json format
OpenStack network_data.json provides a 3 element dictionary
- "links" (links are network devices, physical or virtual)
- "networks" (networks are ip network configurations for one or more
links)
- services (non-ip services, like dns)
networks and links are combined via network items referencing specific
links via a 'link_id' which maps to a links 'id' field.
To convert this format to network_config yaml, we first iterate over the
links and then walk the network list to determine if any of the networks
utilize the current link; if so we generate a subnet entry for the device
We also need to map network_data.json fields to network_config fields. For
example, the network_data links 'id' field is equivalent to network_config
'name' field for devices. We apply more of this mapping to the various
link types that we encounter.
There are additional fields that are populated in the network_data.json
from OpenStack that are not relevant to network_config yaml, so we
enumerate a dictionary of valid keys for network_yaml and apply filtering
to drop these superflous keys from the network_config yaml.
"""
if network_json is None:
return None
# dict of network_config key for filtering network_json
valid_keys = {
'physical': [
'name',
'type',
'mac_address',
'subnets',
'params',
'mtu',
],
'subnet': [
'type',
'address',
'netmask',
'broadcast',
'metric',
'gateway',
'pointopoint',
'scope',
'dns_nameservers',
'dns_search',
'routes',
],
}
links = network_json.get('links', [])
networks = network_json.get('networks', [])
services = network_json.get('services', [])
config = []
for link in links:
subnets = []
cfg = {k: v for k, v in link.items()
if k in valid_keys['physical']}
# 'name' is not in openstack spec yet, but we will support it if it is
# present. The 'id' in the spec is currently implemented as the host
# nic's name, meaning something like 'tap-adfasdffd'. We do not want
# to name guest devices with such ugly names.
if 'name' in link:
cfg['name'] = link['name']
for network in [n for n in networks
if n['link'] == link['id']]:
subnet = {k: v for k, v in network.items()
if k in valid_keys['subnet']}
if 'dhcp' in network['type']:
t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4'
subnet.update({
'type': t,
})
else:
subnet.update({
'type': 'static',
'address': network.get('ip_address'),
})
if network['type'] == 'ipv4':
subnet['ipv4'] = True
if network['type'] == 'ipv6':
subnet['ipv6'] = True
subnets.append(subnet)
cfg.update({'subnets': subnets})
if link['type'] in ['ethernet', 'vif', 'ovs', 'phy', 'bridge']:
cfg.update({
'type': 'physical',
'mac_address': link['ethernet_mac_address']})
elif link['type'] in ['bond']:
params = {}
for k, v in link.items():
if k == 'bond_links':
continue
elif k.startswith('bond'):
params.update({k: v})
cfg.update({
'bond_interfaces': copy.deepcopy(link['bond_links']),
'params': params,
})
elif link['type'] in ['vlan']:
cfg.update({
'name': "%s.%s" % (link['vlan_link'],
link['vlan_id']),
'vlan_link': link['vlan_link'],
'vlan_id': link['vlan_id'],
'mac_address': link['vlan_mac_address'],
})
else:
raise ValueError(
'Unknown network_data link type: %s' % link['type'])
config.append(cfg)
need_names = [d for d in config
if d.get('type') == 'physical' and 'name' not in d]
if need_names:
if known_macs is None:
known_macs = net.get_interfaces_by_mac()
for d in need_names:
mac = d.get('mac_address')
if not mac:
raise ValueError("No mac_address or name entry for %s" % d)
if mac not in known_macs:
raise ValueError("Unable to find a system nic for %s" % d)
d['name'] = known_macs[mac]
for service in services:
cfg = service
cfg.update({'type': 'nameserver'})
config.append(cfg)
return {'version': 1, 'config': config}
def get_interfaces_by_mac(self) -> dict:
return net.get_interfaces_by_mac()
def _add_network_config_from_opc_imds(self):
"""Generate secondary NIC config from IMDS and merge it.
The primary NIC configuration should not be modified based on the IMDS
values, as it should continue to be configured for DHCP. As such, this
uses the instance's network config dict which is expected to have the
primary NIC configuration already present.
It will mutate the network config to include the secondary VNICs.
:raises:
Exceptions are not handled within this function. Likely
exceptions are KeyError/IndexError
(if the IMDS returns valid JSON with unexpected contents).
"""
if self._vnics_data is None:
LOG.warning("Secondary NIC data is UNSET but should not be")
return
if 'nicIndex' in self._vnics_data[0]:
# TODO: Once configure_secondary_nics defaults to True, lower the
# level of this log message. (Currently, if we're running this
# code at all, someone has explicitly opted-in to secondary
# VNIC configuration, so we should warn them that it didn't
# happen. Once it's default, this would be emitted on every Bare
# Metal Machine launch, which means INFO or DEBUG would be more
# appropriate.)
LOG.warning(
'VNIC metadata indicates this is a bare metal machine; '
'skipping secondary VNIC configuration.')
return
interfaces_by_mac = get_interfaces_by_mac()
for vnic_dict in self._vnics_data[1:]:
# We skip the first entry in the response because the primary
# interface is already configured by iSCSI boot; applying
# configuration from the IMDS is not required.
mac_address = vnic_dict['macAddr'].lower()
if mac_address not in interfaces_by_mac:
LOG.debug('Interface with MAC %s not found; skipping',
mac_address)
continue
name = interfaces_by_mac[mac_address]
if self._network_config['version'] == 1:
subnet = {
'type': 'static',
'address': vnic_dict['privateIp'],
}
self._network_config['config'].append({
'name': name,
'type': 'physical',
'mac_address': mac_address,
'mtu': MTU,
'subnets': [subnet],
})
elif self._network_config['version'] == 2:
self._network_config['ethernets'][name] = {
'addresses': [vnic_dict['privateIp']],
'mtu': MTU,
'dhcp4': False,
'dhcp6': False,
'match': {
'macaddress': mac_address
}
}
def _ifconfig_entries(self, settings, target=None):
ifname_by_mac = net.get_interfaces_by_mac()
for interface in settings.iter_interfaces():
device_name = interface.get("name")
device_mac = interface.get("mac_address")
if device_name and re.match(r'^lo\d+$', device_name):
continue
if device_mac not in ifname_by_mac:
LOG.info('Cannot find any device with MAC %s', device_mac)
elif device_mac and device_name:
cur_name = ifname_by_mac[device_mac]
if cur_name != device_name:
LOG.info('netif service will rename interface %s to %s',
cur_name, device_name)
try:
self.rename_interface(cur_name, device_name)
except NotImplementedError:
LOG.error(('Interface renaming is '
'not supported on this OS'))
device_name = cur_name
else:
device_name = ifname_by_mac[device_mac]
LOG.info('Configuring interface %s', device_name)
self.interface_configurations[device_name] = 'DHCP'
alias_count = -1
alias_count_ipv6 = -1
for subnet in interface.get("subnets", []):
if subnet.get('type') == 'static6':
if not subnet.get('prefix'):
LOG.debug('Skipping IP %s, because there is no prefix',
subnet)
# not sure how to handle this. is our ipv6 implementation
# correct or not? ip/64 or do we need to pass in the
# netmask for ipv6 interfaces
LOG.debug('Configuring dev %s with %s / %s', device_name,
subnet.get('address'), subnet.get('prefix'))
# If we have added in a config for this interface,
# add this subnet in as an alias
if alias_count_ipv6 > -1:
name = f'{device_name}_ipv6_alias{alias_count_ipv6}'
else:
name = f'{device_name}_ipv6'
alias_count_ipv6 += 1
self.interface_configurations[name] = {
'address': subnet.get('address'),
'prefixlen': subnet.get('prefix'),
'type': 'ipv6',
}
elif subnet.get('type') == 'static':
if not subnet.get('netmask'):
LOG.debug(
'Skipping IP %s, because there is no netmask',
subnet.get('address'))
continue
LOG.debug('Configuring dev %s with %s / %s', device_name,
subnet.get('address'), subnet.get('netmask'))
# If we have added in a config for this interface,
# add this subnet in as an alias
if alias_count > -1:
name = f'{device_name}_alias{alias_count}'
else:
name = device_name
alias_count += 1
self.interface_configurations[name] = {
'address': subnet.get('address'),
'netmask': subnet.get('netmask'),
'type': 'ipv4',
}
def _add_network_config_from_opc_imds(self, set_primary: bool = False):
"""Generate primary and/or secondary NIC config from IMDS and merge it.
It will mutate the network config to include the secondary VNICs.
:param set_primary: If True set primary interface.
:raises:
Exceptions are not handled within this function. Likely
exceptions are KeyError/IndexError
(if the IMDS returns valid JSON with unexpected contents).
"""
if self._vnics_data is None:
LOG.warning("NIC data is UNSET but should not be")
return
if not set_primary and ("nicIndex" in self._vnics_data[0]):
# TODO: Once configure_secondary_nics defaults to True, lower the
# level of this log message. (Currently, if we're running this
# code at all, someone has explicitly opted-in to secondary
# VNIC configuration, so we should warn them that it didn't
# happen. Once it's default, this would be emitted on every Bare
# Metal Machine launch, which means INFO or DEBUG would be more
# appropriate.)
LOG.warning(
"VNIC metadata indicates this is a bare metal machine; "
"skipping secondary VNIC configuration.")
return
interfaces_by_mac = get_interfaces_by_mac()
vnics_data = self._vnics_data if set_primary else self._vnics_data[1:]
for vnic_dict in vnics_data:
mac_address = vnic_dict["macAddr"].lower()
if mac_address not in interfaces_by_mac:
LOG.warning(
"Interface with MAC %s not found; skipping",
mac_address,
)
continue
name = interfaces_by_mac[mac_address]
if self._network_config["version"] == 1:
subnet = {
"type": "static",
"address": vnic_dict["privateIp"],
}
self._network_config["config"].append({
"name": name,
"type": "physical",
"mac_address": mac_address,
"mtu": MTU,
"subnets": [subnet],
})
elif self._network_config["version"] == 2:
self._network_config["ethernets"][name] = {
"addresses": [vnic_dict["privateIp"]],
"mtu": MTU,
"dhcp4": False,
"dhcp6": False,
"match": {
"macaddress": mac_address
},
}
def convert_network_configuration(config, dns_servers):
"""Convert the DigitalOcean Network description into Cloud-init's netconfig
format.
Example JSON:
{'public': [
{'mac': '04:01:58:27:7f:01',
'ipv4': {'gateway': '45.55.32.1',
'netmask': '255.255.224.0',
'ip_address': '45.55.50.93'},
'anchor_ipv4': {
'gateway': '10.17.0.1',
'netmask': '255.255.0.0',
'ip_address': '10.17.0.9'},
'type': 'public',
'ipv6': {'gateway': '....',
'ip_address': '....',
'cidr': 64}}
],
'private': [
{'mac': '04:01:58:27:7f:02',
'ipv4': {'gateway': '10.132.0.1',
'netmask': '255.255.0.0',
'ip_address': '10.132.75.35'},
'type': 'private'}
]
}
"""
def _get_subnet_part(pcfg):
subpart = {
"type": "static",
"control": "auto",
"address": pcfg.get("ip_address"),
"gateway": pcfg.get("gateway"),
}
if ":" in pcfg.get("ip_address"):
subpart["address"] = "{0}/{1}".format(pcfg.get("ip_address"),
pcfg.get("cidr"))
else:
subpart["netmask"] = pcfg.get("netmask")
return subpart
nic_configs = []
macs_to_nics = cloudnet.get_interfaces_by_mac()
LOG.debug("nic mapping: %s", macs_to_nics)
for n in config:
nic = config[n][0]
LOG.debug("considering %s", nic)
mac_address = nic.get("mac")
if mac_address not in macs_to_nics:
raise RuntimeError(
"Did not find network interface on system "
"with mac '%s'. Cannot apply configuration: %s" %
(mac_address, nic))
sysfs_name = macs_to_nics.get(mac_address)
nic_type = nic.get("type", "unknown")
if_name = NIC_MAP.get(nic_type, sysfs_name)
if if_name != sysfs_name:
LOG.debug(
"Found %s interface '%s' on '%s', assigned name of '%s'",
nic_type,
mac_address,
sysfs_name,
if_name,
)
else:
msg = ("Found interface '%s' on '%s', which is not a public "
"or private interface. Using default system naming.")
LOG.debug(msg, mac_address, sysfs_name)
ncfg = {
"type": "physical",
"mac_address": mac_address,
"name": if_name,
}
subnets = []
for netdef in ("ipv4", "ipv6", "anchor_ipv4", "anchor_ipv6"):
raw_subnet = nic.get(netdef, None)
if not raw_subnet:
continue
sub_part = _get_subnet_part(raw_subnet)
if nic_type != "public" or "anchor" in netdef:
del sub_part["gateway"]
subnets.append(sub_part)
ncfg["subnets"] = subnets
nic_configs.append(ncfg)
LOG.debug("nic '%s' configuration: %s", if_name, ncfg)
if dns_servers:
LOG.debug("added dns servers: %s", dns_servers)
nic_configs.append({"type": "nameserver", "address": dns_servers})
return {"version": 1, "config": nic_configs}
def get_interfaces_by_mac(self) -> dict:
return net.get_interfaces_by_mac(
blacklist_drivers=self.blacklist_drivers)
def convert_network_configuration(config, dns_servers):
"""Convert the DigitalOcean Network description into Cloud-init's netconfig
format.
Example JSON:
{'public': [
{'mac': '04:01:58:27:7f:01',
'ipv4': {'gateway': '45.55.32.1',
'netmask': '255.255.224.0',
'ip_address': '45.55.50.93'},
'anchor_ipv4': {
'gateway': '10.17.0.1',
'netmask': '255.255.0.0',
'ip_address': '10.17.0.9'},
'type': 'public',
'ipv6': {'gateway': '....',
'ip_address': '....',
'cidr': 64}}
],
'private': [
{'mac': '04:01:58:27:7f:02',
'ipv4': {'gateway': '10.132.0.1',
'netmask': '255.255.0.0',
'ip_address': '10.132.75.35'},
'type': 'private'}
]
}
"""
def _get_subnet_part(pcfg, nameservers=None):
subpart = {'type': 'static',
'control': 'auto',
'address': pcfg.get('ip_address'),
'gateway': pcfg.get('gateway')}
if nameservers:
subpart['dns_nameservers'] = nameservers
if ":" in pcfg.get('ip_address'):
subpart['address'] = "{0}/{1}".format(pcfg.get('ip_address'),
pcfg.get('cidr'))
else:
subpart['netmask'] = pcfg.get('netmask')
return subpart
all_nics = []
for k in ('public', 'private'):
if k in config:
all_nics.extend(config[k])
macs_to_nics = cloudnet.get_interfaces_by_mac()
nic_configs = []
for nic in all_nics:
mac_address = nic.get('mac')
sysfs_name = macs_to_nics.get(mac_address)
nic_type = nic.get('type', 'unknown')
# Note: the entry 'public' above contains a list, but
# the list will only ever have one nic inside it per digital ocean.
# If it ever had more than one nic, then this code would
# assign all 'public' the same name.
if_name = NIC_MAP.get(nic_type, sysfs_name)
LOG.debug("mapped %s interface to %s, assigning name of %s",
mac_address, sysfs_name, if_name)
ncfg = {'type': 'physical',
'mac_address': mac_address,
'name': if_name}
subnets = []
for netdef in ('ipv4', 'ipv6', 'anchor_ipv4', 'anchor_ipv6'):
raw_subnet = nic.get(netdef, None)
if not raw_subnet:
continue
sub_part = _get_subnet_part(raw_subnet)
if nic_type == 'public' and 'anchor' not in netdef:
# add DNS resolvers to the public interfaces only
sub_part = _get_subnet_part(raw_subnet, dns_servers)
else:
# remove the gateway any non-public interfaces
if 'gateway' in sub_part:
del sub_part['gateway']
subnets.append(sub_part)
ncfg['subnets'] = subnets
nic_configs.append(ncfg)
LOG.debug("nic '%s' configuration: %s", if_name, ncfg)
return {'version': 1, 'config': nic_configs}
def convert_net_json(network_json=None, known_macs=None):
"""Return a dictionary of network_config by parsing provided
OpenStack ConfigDrive NetworkData json format
OpenStack network_data.json provides a 3 element dictionary
- "links" (links are network devices, physical or virtual)
- "networks" (networks are ip network configurations for one or more
links)
- services (non-ip services, like dns)
networks and links are combined via network items referencing specific
links via a 'link_id' which maps to a links 'id' field.
To convert this format to network_config yaml, we first iterate over the
links and then walk the network list to determine if any of the networks
utilize the current link; if so we generate a subnet entry for the device
We also need to map network_data.json fields to network_config fields. For
example, the network_data links 'id' field is equivalent to network_config
'name' field for devices. We apply more of this mapping to the various
link types that we encounter.
There are additional fields that are populated in the network_data.json
from OpenStack that are not relevant to network_config yaml, so we
enumerate a dictionary of valid keys for network_yaml and apply filtering
to drop these superflous keys from the network_config yaml.
"""
if network_json is None:
return None
# dict of network_config key for filtering network_json
valid_keys = {
"physical": [
"name",
"type",
"mac_address",
"subnets",
"params",
"mtu",
],
"subnet": [
"type",
"address",
"netmask",
"broadcast",
"metric",
"gateway",
"pointopoint",
"scope",
"dns_nameservers",
"dns_search",
"routes",
],
}
links = network_json.get("links", [])
networks = network_json.get("networks", [])
services = network_json.get("services", [])
link_updates = []
link_id_info = {}
bond_name_fmt = "bond%d"
bond_number = 0
config = []
for link in links:
subnets = []
cfg = dict(
(k, v) for k, v in link.items() if k in valid_keys["physical"])
# 'name' is not in openstack spec yet, but we will support it if it is
# present. The 'id' in the spec is currently implemented as the host
# nic's name, meaning something like 'tap-adfasdffd'. We do not want
# to name guest devices with such ugly names.
if "name" in link:
cfg["name"] = link["name"]
link_mac_addr = None
if link.get("ethernet_mac_address"):
link_mac_addr = link.get("ethernet_mac_address").lower()
link_id_info[link["id"]] = link_mac_addr
curinfo = {
"name": cfg.get("name"),
"mac": link_mac_addr,
"id": link["id"],
"type": link["type"],
}
for network in [n for n in networks if n["link"] == link["id"]]:
subnet = dict((k, v) for k, v in network.items()
if k in valid_keys["subnet"])
if network["type"] == "ipv4_dhcp":
subnet.update({"type": "dhcp4"})
elif network["type"] == "ipv6_dhcp":
subnet.update({"type": "dhcp6"})
elif network["type"] in [
"ipv6_slaac",
"ipv6_dhcpv6-stateless",
"ipv6_dhcpv6-stateful",
]:
subnet.update({"type": network["type"]})
elif network["type"] in ["ipv4", "static"]:
subnet.update({
"type": "static",
"address": network.get("ip_address"),
})
elif network["type"] in ["ipv6", "static6"]:
cfg.update({"accept-ra": False})
subnet.update({
"type": "static6",
"address": network.get("ip_address"),
})
# Enable accept_ra for stateful and legacy ipv6_dhcp types
if network["type"] in ["ipv6_dhcpv6-stateful", "ipv6_dhcp"]:
cfg.update({"accept-ra": True})
if network["type"] == "ipv4":
subnet["ipv4"] = True
if network["type"] == "ipv6":
subnet["ipv6"] = True
subnets.append(subnet)
cfg.update({"subnets": subnets})
if link["type"] in ["bond"]:
params = {}
if link_mac_addr:
params["mac_address"] = link_mac_addr
for k, v in link.items():
if k == "bond_links":
continue
elif k.startswith("bond"):
params.update({k: v})
# openstack does not provide a name for the bond.
# they do provide an 'id', but that is possibly non-sensical.
# so we just create our own name.
link_name = bond_name_fmt % bond_number
bond_number += 1
# bond_links reference links by their id, but we need to add
# to the network config by their nic name.
# store that in bond_links_needed, and update these later.
link_updates.append((
cfg,
"bond_interfaces",
"%s",
copy.deepcopy(link["bond_links"]),
))
cfg.update({"params": params, "name": link_name})
curinfo["name"] = link_name
elif link["type"] in ["vlan"]:
name = "%s.%s" % (link["vlan_link"], link["vlan_id"])
cfg.update({
"name": name,
"vlan_id": link["vlan_id"],
"mac_address": link["vlan_mac_address"],
})
link_updates.append((cfg, "vlan_link", "%s", link["vlan_link"]))
link_updates.append(
(cfg, "name", "%%s.%s" % link["vlan_id"], link["vlan_link"]))
curinfo.update({"mac": link["vlan_mac_address"], "name": name})
else:
if link["type"] not in KNOWN_PHYSICAL_TYPES:
LOG.warning(
"Unknown network_data link type (%s); treating as"
" physical",
link["type"],
)
cfg.update({"type": "physical", "mac_address": link_mac_addr})
config.append(cfg)
link_id_info[curinfo["id"]] = curinfo
need_names = [
d for d in config if d.get("type") == "physical" and "name" not in d
]
if need_names or link_updates:
if known_macs is None:
known_macs = net.get_interfaces_by_mac()
# go through and fill out the link_id_info with names
for _link_id, info in link_id_info.items():
if info.get("name"):
continue
if info.get("mac") in known_macs:
info["name"] = known_macs[info["mac"]]
for d in need_names:
mac = d.get("mac_address")
if not mac:
raise ValueError("No mac_address or name entry for %s" % d)
if mac not in known_macs:
raise ValueError("Unable to find a system nic for %s" % d)
d["name"] = known_macs[mac]
for cfg, key, fmt, targets in link_updates:
if isinstance(targets, (list, tuple)):
cfg[key] = [
fmt % link_id_info[target]["name"] for target in targets
]
else:
cfg[key] = fmt % link_id_info[targets]["name"]
# Infiniband interfaces may be referenced in network_data.json by a 6 byte
# Ethernet MAC-style address, and we use that address to look up the
# interface name above. Now ensure that the hardware address is set to the
# full 20 byte address.
ib_known_hwaddrs = net.get_ib_hwaddrs_by_interface()
if ib_known_hwaddrs:
for cfg in config:
if cfg["name"] in ib_known_hwaddrs:
cfg["mac_address"] = ib_known_hwaddrs[cfg["name"]]
cfg["type"] = "infiniband"
for service in services:
cfg = service
cfg.update({"type": "nameserver"})
config.append(cfg)
return {"version": 1, "config": config}
def _add_network_config_from_opc_imds(network_config):
"""
Fetch data from Oracle's IMDS, generate secondary NIC config, merge it.
The primary NIC configuration should not be modified based on the IMDS
values, as it should continue to be configured for DHCP. As such, this
takes an existing network_config dict which is expected to have the primary
NIC configuration already present. It will mutate the given dict to
include the secondary VNICs.
:param network_config:
A v1 network config dict with the primary NIC already configured. This
dict will be mutated.
:raises:
Exceptions are not handled within this function. Likely exceptions are
those raised by url_helper.readurl (if communicating with the IMDS
fails), ValueError/JSONDecodeError (if the IMDS returns invalid JSON),
and KeyError/IndexError (if the IMDS returns valid JSON with unexpected
contents).
"""
resp = readurl(VNIC_METADATA_URL)
vnics = json.loads(str(resp))
if 'nicIndex' in vnics[0]:
# TODO: Once configure_secondary_nics defaults to True, lower the level
# of this log message. (Currently, if we're running this code at all,
# someone has explicitly opted-in to secondary VNIC configuration, so
# we should warn them that it didn't happen. Once it's default, this
# would be emitted on every Bare Metal Machine launch, which means INFO
# or DEBUG would be more appropriate.)
LOG.warning(
'VNIC metadata indicates this is a bare metal machine; skipping'
' secondary VNIC configuration.')
return
interfaces_by_mac = get_interfaces_by_mac()
for vnic_dict in vnics[1:]:
# We skip the first entry in the response because the primary interface
# is already configured by iSCSI boot; applying configuration from the
# IMDS is not required.
mac_address = vnic_dict['macAddr'].lower()
if mac_address not in interfaces_by_mac:
LOG.debug('Interface with MAC %s not found; skipping', mac_address)
continue
name = interfaces_by_mac[mac_address]
subnet = {
'type': 'static',
'address': vnic_dict['privateIp'],
'netmask': vnic_dict['subnetCidrBlock'].split('/')[1],
'gateway': vnic_dict['virtualRouterIp'],
'control': 'manual',
}
network_config['config'].append({
'name': name,
'type': 'physical',
'mac_address': mac_address,
'mtu': MTU,
'subnets': [subnet],
})
def handle_ethernets(self, command):
"""
ethernets:
eno1:
match:
macaddress: 00:11:22:33:44:55
driver: hv_netsvc
wakeonlan: true
dhcp4: true
dhcp6: false
addresses:
- 192.168.14.2/24
- 2001:1::1/64
gateway4: 192.168.14.1
gateway6: 2001:1::2
nameservers:
search: [foo.local, bar.local]
addresses: [8.8.8.8, 8.8.4.4]
lom:
match:
driver: ixgbe
set-name: lom1
dhcp6: true
accept-ra: true
switchports:
match:
name: enp2*
mtu: 1280
command = {
'type': 'physical',
'mac_address': 'c0:d6:9f:2c:e8:80',
'name': 'eth0',
'subnets': [
{'type': 'dhcp4'}
]
}
"""
# Get the interfaces by MAC address to update an interface's
# device name to the name of the device that matches a provided
# MAC address when the set-name directive is not present.
#
# Please see https://bugs.launchpad.net/cloud-init/+bug/1855945
# for more information.
ifaces_by_mac = get_interfaces_by_mac()
for eth, cfg in command.items():
phy_cmd = {
"type": "physical",
}
match = cfg.get("match", {})
mac_address = match.get("macaddress", None)
if not mac_address:
LOG.debug(
'NetworkState Version2: missing "macaddress" info '
"in config entry: %s: %s",
eth,
str(cfg),
)
phy_cmd["mac_address"] = mac_address
# Determine the name of the interface by using one of the
# following in the order they are listed:
# * set-name
# * interface name looked up by mac
# * value of "eth" key from this loop
name = eth
set_name = cfg.get("set-name", None)
if set_name:
name = set_name
elif mac_address and ifaces_by_mac:
lcase_mac_address = mac_address.lower()
for iface_mac, iface_name in ifaces_by_mac.items():
if lcase_mac_address == iface_mac.lower():
name = iface_name
break
phy_cmd["name"] = name
driver = match.get("driver", None)
if driver:
phy_cmd["params"] = {"driver": driver}
for key in ["mtu", "match", "wakeonlan", "accept-ra"]:
if key in cfg:
phy_cmd[key] = cfg[key]
subnets = self._v2_to_v1_ipcfg(cfg)
if len(subnets) > 0:
phy_cmd.update({"subnets": subnets})
LOG.debug("v2(ethernets) -> v1(physical):\n%s", phy_cmd)
self.handle_physical(phy_cmd)
def convert_to_network_config_v1(config):
"""
Convert the UpCloud network metadata description into
Cloud-init's version 1 netconfig format.
Example JSON:
{
"interfaces": [
{
"index": 1,
"ip_addresses": [
{
"address": "94.237.105.53",
"dhcp": true,
"dns": [
"94.237.127.9",
"94.237.40.9"
],
"family": "IPv4",
"floating": false,
"gateway": "94.237.104.1",
"network": "94.237.104.0/22"
},
{
"address": "94.237.105.50",
"dhcp": false,
"dns": [],
"family": "IPv4",
"floating": true,
"gateway": "",
"network": "94.237.105.50/32"
}
],
"mac": "32:d5:ba:4a:36:e7",
"network_id": "031457f4-0f8c-483c-96f2-eccede02909c",
"type": "public"
},
{
"index": 2,
"ip_addresses": [
{
"address": "10.6.3.27",
"dhcp": true,
"dns": [],
"family": "IPv4",
"floating": false,
"gateway": "10.6.0.1",
"network": "10.6.0.0/22"
}
],
"mac": "32:d5:ba:4a:84:cc",
"network_id": "03d82553-5bea-4132-b29a-e1cf67ec2dd1",
"type": "utility"
},
{
"index": 3,
"ip_addresses": [
{
"address": "2a04:3545:1000:720:38d6:baff:fe4a:63e7",
"dhcp": true,
"dns": [
"2a04:3540:53::1",
"2a04:3544:53::1"
],
"family": "IPv6",
"floating": false,
"gateway": "2a04:3545:1000:720::1",
"network": "2a04:3545:1000:720::/64"
}
],
"mac": "32:d5:ba:4a:63:e7",
"network_id": "03000000-0000-4000-8046-000000000000",
"type": "public"
},
{
"index": 4,
"ip_addresses": [
{
"address": "172.30.1.10",
"dhcp": true,
"dns": [],
"family": "IPv4",
"floating": false,
"gateway": "172.30.1.1",
"network": "172.30.1.0/24"
}
],
"mac": "32:d5:ba:4a:8a:e1",
"network_id": "035a0a4a-77b4-4de5-820d-189fc8135714",
"type": "private"
}
],
"dns": [
"94.237.127.9",
"94.237.40.9"
]
}
"""
def _get_subnet_config(ip_addr, dns):
if ip_addr.get("dhcp"):
dhcp_type = "dhcp"
if ip_addr.get("family") == "IPv6":
# UpCloud currently passes IPv6 addresses via
# StateLess Address Auto Configuration (SLAAC)
dhcp_type = "ipv6_dhcpv6-stateless"
return {"type": dhcp_type}
static_type = "static"
if ip_addr.get("family") == "IPv6":
static_type = "static6"
subpart = {
"type": static_type,
"control": "auto",
"address": ip_addr.get("address"),
}
if ip_addr.get("gateway"):
subpart["gateway"] = ip_addr.get("gateway")
if "/" in ip_addr.get("network"):
subpart["netmask"] = ip_addr.get("network").split("/")[1]
if dns != ip_addr.get("dns") and ip_addr.get("dns"):
subpart["dns_nameservers"] = ip_addr.get("dns")
return subpart
nic_configs = []
macs_to_interfaces = cloudnet.get_interfaces_by_mac()
LOG.debug("NIC mapping: %s", macs_to_interfaces)
for raw_iface in config.get("interfaces"):
LOG.debug("Considering %s", raw_iface)
mac_address = raw_iface.get("mac")
if mac_address not in macs_to_interfaces:
raise RuntimeError(
"Did not find network interface on system "
"with mac '%s'. Cannot apply configuration: %s" %
(mac_address, raw_iface))
iface_type = raw_iface.get("type")
sysfs_name = macs_to_interfaces.get(mac_address)
LOG.debug(
"Found %s interface '%s' with address '%s' (index %d)",
iface_type,
sysfs_name,
mac_address,
raw_iface.get("index"),
)
interface = {
"type": "physical",
"name": sysfs_name,
"mac_address": mac_address
}
subnets = []
for ip_address in raw_iface.get("ip_addresses"):
sub_part = _get_subnet_config(ip_address, config.get("dns"))
subnets.append(sub_part)
interface["subnets"] = subnets
nic_configs.append(interface)
if config.get("dns"):
LOG.debug("Setting DNS nameservers to %s", config.get("dns"))
nic_configs.append({
"type": "nameserver",
"address": config.get("dns")
})
return {"version": 1, "config": nic_configs}
def convert_net_json(network_json=None, known_macs=None):
"""Return a dictionary of network_config by parsing provided
OpenStack ConfigDrive NetworkData json format
OpenStack network_data.json provides a 3 element dictionary
- "links" (links are network devices, physical or virtual)
- "networks" (networks are ip network configurations for one or more
links)
- services (non-ip services, like dns)
networks and links are combined via network items referencing specific
links via a 'link_id' which maps to a links 'id' field.
To convert this format to network_config yaml, we first iterate over the
links and then walk the network list to determine if any of the networks
utilize the current link; if so we generate a subnet entry for the device
We also need to map network_data.json fields to network_config fields. For
example, the network_data links 'id' field is equivalent to network_config
'name' field for devices. We apply more of this mapping to the various
link types that we encounter.
There are additional fields that are populated in the network_data.json
from OpenStack that are not relevant to network_config yaml, so we
enumerate a dictionary of valid keys for network_yaml and apply filtering
to drop these superflous keys from the network_config yaml.
"""
if network_json is None:
return None
# dict of network_config key for filtering network_json
valid_keys = {
'physical': [
'name',
'type',
'mac_address',
'subnets',
'params',
'mtu',
],
'subnet': [
'type',
'address',
'netmask',
'broadcast',
'metric',
'gateway',
'pointopoint',
'scope',
'dns_nameservers',
'dns_search',
'routes',
],
}
links = network_json.get('links', [])
networks = network_json.get('networks', [])
services = network_json.get('services', [])
link_updates = []
link_id_info = {}
bond_name_fmt = "bond%d"
bond_number = 0
config = []
for link in links:
subnets = []
cfg = dict((k, v) for k, v in link.items()
if k in valid_keys['physical'])
# 'name' is not in openstack spec yet, but we will support it if it is
# present. The 'id' in the spec is currently implemented as the host
# nic's name, meaning something like 'tap-adfasdffd'. We do not want
# to name guest devices with such ugly names.
if 'name' in link:
cfg['name'] = link['name']
link_mac_addr = None
if link.get('ethernet_mac_address'):
link_mac_addr = link.get('ethernet_mac_address').lower()
link_id_info[link['id']] = link_mac_addr
curinfo = {'name': cfg.get('name'), 'mac': link_mac_addr,
'id': link['id'], 'type': link['type']}
for network in [n for n in networks
if n['link'] == link['id']]:
subnet = dict((k, v) for k, v in network.items()
if k in valid_keys['subnet'])
if 'dhcp' in network['type']:
t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4'
subnet.update({
'type': t,
})
else:
subnet.update({
'type': 'static',
'address': network.get('ip_address'),
})
if network['type'] == 'ipv4':
subnet['ipv4'] = True
if network['type'] == 'ipv6':
subnet['ipv6'] = True
subnets.append(subnet)
cfg.update({'subnets': subnets})
if link['type'] in PHYSICAL_TYPES:
cfg.update({'type': 'physical', 'mac_address': link_mac_addr})
elif link['type'] in ['bond']:
params = {}
for k, v in link.items():
if k == 'bond_links':
continue
elif k.startswith('bond'):
params.update({k: v})
# openstack does not provide a name for the bond.
# they do provide an 'id', but that is possibly non-sensical.
# so we just create our own name.
link_name = bond_name_fmt % bond_number
bond_number += 1
# bond_links reference links by their id, but we need to add
# to the network config by their nic name.
# store that in bond_links_needed, and update these later.
link_updates.append(
(cfg, 'bond_interfaces', '%s',
copy.deepcopy(link['bond_links']))
)
cfg.update({'params': params, 'name': link_name})
curinfo['name'] = link_name
elif link['type'] in ['vlan']:
name = "%s.%s" % (link['vlan_link'], link['vlan_id'])
cfg.update({
'name': name,
'vlan_id': link['vlan_id'],
'mac_address': link['vlan_mac_address'],
})
link_updates.append((cfg, 'vlan_link', '%s', link['vlan_link']))
link_updates.append((cfg, 'name', "%%s.%s" % link['vlan_id'],
link['vlan_link']))
curinfo.update({'mac': link['vlan_mac_address'],
'name': name})
else:
raise ValueError(
'Unknown network_data link type: %s' % link['type'])
config.append(cfg)
link_id_info[curinfo['id']] = curinfo
need_names = [d for d in config
if d.get('type') == 'physical' and 'name' not in d]
if need_names or link_updates:
if known_macs is None:
known_macs = net.get_interfaces_by_mac()
# go through and fill out the link_id_info with names
for link_id, info in link_id_info.items():
if info.get('name'):
continue
if info.get('mac') in known_macs:
info['name'] = known_macs[info['mac']]
for d in need_names:
mac = d.get('mac_address')
if not mac:
raise ValueError("No mac_address or name entry for %s" % d)
if mac not in known_macs:
raise ValueError("Unable to find a system nic for %s" % d)
d['name'] = known_macs[mac]
for cfg, key, fmt, target in link_updates:
if isinstance(target, (list, tuple)):
cfg[key] = [fmt % link_id_info[l]['name'] for l in target]
else:
cfg[key] = fmt % link_id_info[target]['name']
for service in services:
cfg = service
cfg.update({'type': 'nameserver'})
config.append(cfg)
return {'version': 1, 'config': config}
def convert_net_json(network_json=None, known_macs=None):
"""Return a dictionary of network_config by parsing provided
OpenStack ConfigDrive NetworkData json format
OpenStack network_data.json provides a 3 element dictionary
- "links" (links are network devices, physical or virtual)
- "networks" (networks are ip network configurations for one or more
links)
- services (non-ip services, like dns)
networks and links are combined via network items referencing specific
links via a 'link_id' which maps to a links 'id' field.
To convert this format to network_config yaml, we first iterate over the
links and then walk the network list to determine if any of the networks
utilize the current link; if so we generate a subnet entry for the device
We also need to map network_data.json fields to network_config fields. For
example, the network_data links 'id' field is equivalent to network_config
'name' field for devices. We apply more of this mapping to the various
link types that we encounter.
There are additional fields that are populated in the network_data.json
from OpenStack that are not relevant to network_config yaml, so we
enumerate a dictionary of valid keys for network_yaml and apply filtering
to drop these superflous keys from the network_config yaml.
"""
if network_json is None:
return None
# dict of network_config key for filtering network_json
valid_keys = {
'physical': [
'name',
'type',
'mac_address',
'subnets',
'params',
'mtu',
],
'subnet': [
'type',
'address',
'netmask',
'broadcast',
'metric',
'gateway',
'pointopoint',
'scope',
'dns_nameservers',
'dns_search',
'routes',
],
}
links = network_json.get('links', [])
networks = network_json.get('networks', [])
services = network_json.get('services', [])
link_updates = []
link_id_info = {}
bond_name_fmt = "bond%d"
bond_number = 0
config = []
for link in links:
subnets = []
cfg = dict(
(k, v) for k, v in link.items() if k in valid_keys['physical'])
# 'name' is not in openstack spec yet, but we will support it if it is
# present. The 'id' in the spec is currently implemented as the host
# nic's name, meaning something like 'tap-adfasdffd'. We do not want
# to name guest devices with such ugly names.
if 'name' in link:
cfg['name'] = link['name']
link_mac_addr = None
if link.get('ethernet_mac_address'):
link_mac_addr = link.get('ethernet_mac_address').lower()
link_id_info[link['id']] = link_mac_addr
curinfo = {
'name': cfg.get('name'),
'mac': link_mac_addr,
'id': link['id'],
'type': link['type']
}
for network in [n for n in networks if n['link'] == link['id']]:
subnet = dict((k, v) for k, v in network.items()
if k in valid_keys['subnet'])
if 'dhcp' in network['type']:
t = (network['type']
if network['type'].startswith('ipv6') else 'dhcp4')
subnet.update({
'type': t,
})
else:
subnet.update({
'type': 'static',
'address': network.get('ip_address'),
})
if network['type'] == 'ipv4':
subnet['ipv4'] = True
if network['type'] == 'ipv6':
subnet['ipv6'] = True
subnets.append(subnet)
cfg.update({'subnets': subnets})
if link['type'] in ['bond']:
params = {}
if link_mac_addr:
params['mac_address'] = link_mac_addr
for k, v in link.items():
if k == 'bond_links':
continue
elif k.startswith('bond'):
params.update({k: v})
# openstack does not provide a name for the bond.
# they do provide an 'id', but that is possibly non-sensical.
# so we just create our own name.
link_name = bond_name_fmt % bond_number
bond_number += 1
# bond_links reference links by their id, but we need to add
# to the network config by their nic name.
# store that in bond_links_needed, and update these later.
link_updates.append((cfg, 'bond_interfaces', '%s',
copy.deepcopy(link['bond_links'])))
cfg.update({'params': params, 'name': link_name})
curinfo['name'] = link_name
elif link['type'] in ['vlan']:
name = "%s.%s" % (link['vlan_link'], link['vlan_id'])
cfg.update({
'name': name,
'vlan_id': link['vlan_id'],
'mac_address': link['vlan_mac_address'],
})
link_updates.append((cfg, 'vlan_link', '%s', link['vlan_link']))
link_updates.append(
(cfg, 'name', "%%s.%s" % link['vlan_id'], link['vlan_link']))
curinfo.update({'mac': link['vlan_mac_address'], 'name': name})
else:
if link['type'] not in KNOWN_PHYSICAL_TYPES:
LOG.warning(
'Unknown network_data link type (%s); treating as'
' physical', link['type'])
cfg.update({'type': 'physical', 'mac_address': link_mac_addr})
config.append(cfg)
link_id_info[curinfo['id']] = curinfo
need_names = [
d for d in config if d.get('type') == 'physical' and 'name' not in d
]
if need_names or link_updates:
if known_macs is None:
known_macs = net.get_interfaces_by_mac()
# go through and fill out the link_id_info with names
for _link_id, info in link_id_info.items():
if info.get('name'):
continue
if info.get('mac') in known_macs:
info['name'] = known_macs[info['mac']]
for d in need_names:
mac = d.get('mac_address')
if not mac:
raise ValueError("No mac_address or name entry for %s" % d)
if mac not in known_macs:
raise ValueError("Unable to find a system nic for %s" % d)
d['name'] = known_macs[mac]
for cfg, key, fmt, target in link_updates:
if isinstance(target, (list, tuple)):
cfg[key] = [fmt % link_id_info[l]['name'] for l in target]
else:
cfg[key] = fmt % link_id_info[target]['name']
# Infiniband interfaces may be referenced in network_data.json by a 6 byte
# Ethernet MAC-style address, and we use that address to look up the
# interface name above. Now ensure that the hardware address is set to the
# full 20 byte address.
ib_known_hwaddrs = net.get_ib_hwaddrs_by_interface()
if ib_known_hwaddrs:
for cfg in config:
if cfg['name'] in ib_known_hwaddrs:
cfg['mac_address'] = ib_known_hwaddrs[cfg['name']]
cfg['type'] = 'infiniband'
for service in services:
cfg = service
cfg.update({'type': 'nameserver'})
config.append(cfg)
return {'version': 1, 'config': config}
def network_config(self):
"""Configure the networking. This needs to be done each boot, since
the IP information may have changed due to snapshot and/or
migration.
"""
if self._network_config:
return self._network_config
interfaces = self.metadata.get('interfaces')
if not interfaces:
raise Exception("Unable to get meta-data from server....")
# Convert Vultr network configuration to cloudinit.net format
# Example JSON:
# [
# {
# "ipv4": {
# "additional": [
# {
# "address": "192.0.2.3",
# "netmask": "255.255.255.0"
# }
# ],
# "address": "192.0.2.2",
# "gateway": "192.0.2.1",
# "netmask": "255.255.255.0"
# },
# "ipv6": {
# "additional": [
# {
# "network": "2001:0db8:0:2::",
# "prefix": "64"
# }
# ],
# "address": "2001:0db8:0:1:5428:d5ff:fe28:1910",
# "network": "2001:0db8:0:1::",
# "prefix": "64"
# },
# "mac": "00:00:00:00:00:00",
# "network-type": "public"
# },
# ......
# ]
nic_configs = []
macs_to_nics = cloudnet.get_interfaces_by_mac()
LOG.debug("nic mapping: %s", macs_to_nics)
config = []
for vultr_ip_dict in interfaces:
mac = vultr_ip_dict["mac"]
if mac not in macs_to_nics:
raise ValueError("Did not find network interface on system "
"with mac '%s'. Cannot apply configuration: %s"
% (mac_address, nic))
if_name = macs_to_nics[mac] # if_name = string 'eth0', ...
if_config= {
'type': 'physical',
'mac_address': mac,
'name': if_name,
'subnets': [{
'type': 'dhcp',
'control': 'auto',
}
]
}
config.append(if_config)
LOG.debug("nic '%s' configuration: %s", if_name, if_config)
LOG.debug("added dns servers: %s", self.dns_servers)
config.append({'type': 'nameserver', 'address': self.dns_servers})
return {'version': 1, 'config': config}
def get_physical_nics_by_mac():
devs = net.get_interfaces_by_mac()
return dict([(m, n) for m, n in devs.items() if net.is_physical(n)])
def convert_net_json(network_json=None, known_macs=None):
"""Return a dictionary of network_config by parsing provided
OpenStack ConfigDrive NetworkData json format
OpenStack network_data.json provides a 3 element dictionary
- "links" (links are network devices, physical or virtual)
- "networks" (networks are ip network configurations for one or more
links)
- services (non-ip services, like dns)
networks and links are combined via network items referencing specific
links via a 'link_id' which maps to a links 'id' field.
To convert this format to network_config yaml, we first iterate over the
links and then walk the network list to determine if any of the networks
utilize the current link; if so we generate a subnet entry for the device
We also need to map network_data.json fields to network_config fields. For
example, the network_data links 'id' field is equivalent to network_config
'name' field for devices. We apply more of this mapping to the various
link types that we encounter.
There are additional fields that are populated in the network_data.json
from OpenStack that are not relevant to network_config yaml, so we
enumerate a dictionary of valid keys for network_yaml and apply filtering
to drop these superflous keys from the network_config yaml.
"""
if network_json is None:
return None
# dict of network_config key for filtering network_json
valid_keys = {
"physical": ["name", "type", "mac_address", "subnets", "params", "mtu"],
"subnet": [
"type",
"address",
"netmask",
"broadcast",
"metric",
"gateway",
"pointopoint",
"scope",
"dns_nameservers",
"dns_search",
"routes",
],
}
links = network_json.get("links", [])
networks = network_json.get("networks", [])
services = network_json.get("services", [])
config = []
for link in links:
subnets = []
cfg = dict((k, v) for k, v in link.items() if k in valid_keys["physical"])
# 'name' is not in openstack spec yet, but we will support it if it is
# present. The 'id' in the spec is currently implemented as the host
# nic's name, meaning something like 'tap-adfasdffd'. We do not want
# to name guest devices with such ugly names.
if "name" in link:
cfg["name"] = link["name"]
for network in [n for n in networks if n["link"] == link["id"]]:
subnet = dict((k, v) for k, v in network.items() if k in valid_keys["subnet"])
if "dhcp" in network["type"]:
t = "dhcp6" if network["type"].startswith("ipv6") else "dhcp4"
subnet.update({"type": t})
else:
subnet.update({"type": "static", "address": network.get("ip_address")})
if network["type"] == "ipv4":
subnet["ipv4"] = True
if network["type"] == "ipv6":
subnet["ipv6"] = True
subnets.append(subnet)
cfg.update({"subnets": subnets})
if link["type"] in ["ethernet", "vif", "ovs", "phy", "bridge"]:
cfg.update({"type": "physical", "mac_address": link["ethernet_mac_address"]})
elif link["type"] in ["bond"]:
params = {}
for k, v in link.items():
if k == "bond_links":
continue
elif k.startswith("bond"):
params.update({k: v})
cfg.update({"bond_interfaces": copy.deepcopy(link["bond_links"]), "params": params})
elif link["type"] in ["vlan"]:
cfg.update(
{
"name": "%s.%s" % (link["vlan_link"], link["vlan_id"]),
"vlan_link": link["vlan_link"],
"vlan_id": link["vlan_id"],
"mac_address": link["vlan_mac_address"],
}
)
else:
raise ValueError("Unknown network_data link type: %s" % link["type"])
config.append(cfg)
need_names = [d for d in config if d.get("type") == "physical" and "name" not in d]
if need_names:
if known_macs is None:
known_macs = net.get_interfaces_by_mac()
for d in need_names:
mac = d.get("mac_address")
if not mac:
raise ValueError("No mac_address or name entry for %s" % d)
if mac not in known_macs:
raise ValueError("Unable to find a system nic for %s" % d)
d["name"] = known_macs[mac]
for service in services:
cfg = service
cfg.update({"type": "nameserver"})
config.append(cfg)
return {"version": 1, "config": config}
def get_physical_nics_by_mac(distro):
devs = net.get_interfaces_by_mac()
return dict(
[(m, n) for m, n in devs.items() if distro.networking.is_physical(n)]
)
def convert_ec2_metadata_network_config(network_md,
macs_to_nics=None,
fallback_nic=None,
full_network_config=True):
"""Convert ec2 metadata to network config version 2 data dict.
@param: network_md: 'network' portion of EC2 metadata.
generally formed as {"interfaces": {"macs": {}} where
'macs' is a dictionary with mac address as key and contents like:
{"device-number": "0", "interface-id": "...", "local-ipv4s": ...}
@param: macs_to_nics: Optional dict of mac addresses and nic names. If
not provided, get_interfaces_by_mac is called to get it from the OS.
@param: fallback_nic: Optionally provide the primary nic interface name.
This nic will be guaranteed to minimally have a dhcp4 configuration.
@param: full_network_config: Boolean set True to configure all networking
presented by IMDS. This includes rendering secondary IPv4 and IPv6
addresses on all NICs and rendering network config on secondary NICs.
If False, only the primary nic will be configured and only with dhcp
(IPv4/IPv6).
@return A dict of network config version 2 based on the metadata and macs.
"""
netcfg = {'version': 2, 'ethernets': {}}
if not macs_to_nics:
macs_to_nics = net.get_interfaces_by_mac()
macs_metadata = network_md['interfaces']['macs']
if not full_network_config:
for mac, nic_name in macs_to_nics.items():
if nic_name == fallback_nic:
break
dev_config = {
'dhcp4': True,
'dhcp6': False,
'match': {
'macaddress': mac.lower()
},
'set-name': nic_name
}
nic_metadata = macs_metadata.get(mac)
if nic_metadata.get('ipv6s'): # Any IPv6 addresses configured
dev_config['dhcp6'] = True
netcfg['ethernets'][nic_name] = dev_config
return netcfg
# Apply network config for all nics and any secondary IPv4/v6 addresses
nic_idx = 1
for mac, nic_name in sorted(macs_to_nics.items()):
nic_metadata = macs_metadata.get(mac)
if not nic_metadata:
continue # Not a physical nic represented in metadata
dhcp_override = {'route-metric': nic_idx * 100}
nic_idx += 1
dev_config = {
'dhcp4': True,
'dhcp4-overrides': dhcp_override,
'dhcp6': False,
'match': {
'macaddress': mac.lower()
},
'set-name': nic_name
}
if nic_metadata.get('ipv6s'): # Any IPv6 addresses configured
dev_config['dhcp6'] = True
dev_config['dhcp6-overrides'] = dhcp_override
dev_config['addresses'] = get_secondary_addresses(nic_metadata, mac)
if not dev_config['addresses']:
dev_config.pop('addresses') # Since we found none configured
netcfg['ethernets'][nic_name] = dev_config
# Remove route-metric dhcp overrides if only one nic configured
if len(netcfg['ethernets']) == 1:
for nic_name in netcfg['ethernets'].keys():
netcfg['ethernets'][nic_name].pop('dhcp4-overrides')
netcfg['ethernets'][nic_name].pop('dhcp6-overrides', None)
return netcfg
def convert_ec2_metadata_network_config(network_md,
macs_to_nics=None,
fallback_nic=None,
full_network_config=True):
"""Convert ec2 metadata to network config version 2 data dict.
@param: network_md: 'network' portion of EC2 metadata.
generally formed as {"interfaces": {"macs": {}} where
'macs' is a dictionary with mac address as key and contents like:
{"device-number": "0", "interface-id": "...", "local-ipv4s": ...}
@param: macs_to_nics: Optional dict of mac addresses and nic names. If
not provided, get_interfaces_by_mac is called to get it from the OS.
@param: fallback_nic: Optionally provide the primary nic interface name.
This nic will be guaranteed to minimally have a dhcp4 configuration.
@param: full_network_config: Boolean set True to configure all networking
presented by IMDS. This includes rendering secondary IPv4 and IPv6
addresses on all NICs and rendering network config on secondary NICs.
If False, only the primary nic will be configured and only with dhcp
(IPv4/IPv6).
@return A dict of network config version 2 based on the metadata and macs.
"""
netcfg = {"version": 2, "ethernets": {}}
if not macs_to_nics:
macs_to_nics = net.get_interfaces_by_mac()
macs_metadata = network_md["interfaces"]["macs"]
if not full_network_config:
for mac, nic_name in macs_to_nics.items():
if nic_name == fallback_nic:
break
dev_config = {
"dhcp4": True,
"dhcp6": False,
"match": {
"match": nicname
},
"set-name": nic_name,
}
nic_metadata = macs_metadata.get(mac)
if nic_metadata.get("ipv6s"): # Any IPv6 addresses configured
dev_config["dhcp6"] = True
netcfg["ethernets"][nic_name] = dev_config
return netcfg
# Apply network config for all nics and any secondary IPv4/v6 addresses
nic_idx = 0
for mac, nic_name in sorted(macs_to_nics.items()):
nic_metadata = macs_metadata.get(mac)
if not nic_metadata:
continue # Not a physical nic represented in metadata
# device-number is zero-indexed, we want it 1-indexed for the
# multiplication on the following line
nic_idx = int(nic_metadata.get("device-number", nic_idx)) + 1
dhcp_override = {"route-metric": nic_idx * 100}
dev_config = {
"dhcp4": True,
"dhcp4-overrides": dhcp_override,
"dhcp6": False,
"match": {
"name": nic_name
},
"set-name": nic_name,
}
if nic_metadata.get("ipv6s"): # Any IPv6 addresses configured
dev_config["dhcp6"] = True
dev_config["dhcp6-overrides"] = dhcp_override
dev_config["addresses"] = get_secondary_addresses(nic_metadata, mac)
if not dev_config["addresses"]:
dev_config.pop("addresses") # Since we found none configured
netcfg["ethernets"][nic_name] = dev_config
# Remove route-metric dhcp overrides if only one nic configured
if len(netcfg["ethernets"]) == 1:
for nic_name in netcfg["ethernets"].keys():
netcfg["ethernets"][nic_name].pop("dhcp4-overrides")
netcfg["ethernets"][nic_name].pop("dhcp6-overrides", None)
return netcfg
def convert_network_configuration(config, dns_servers):
"""Convert the DigitalOcean Network description into Cloud-init's netconfig
format.
Example JSON:
{'public': [
{'mac': '04:01:58:27:7f:01',
'ipv4': {'gateway': '45.55.32.1',
'netmask': '255.255.224.0',
'ip_address': '45.55.50.93'},
'anchor_ipv4': {
'gateway': '10.17.0.1',
'netmask': '255.255.0.0',
'ip_address': '10.17.0.9'},
'type': 'public',
'ipv6': {'gateway': '....',
'ip_address': '....',
'cidr': 64}}
],
'private': [
{'mac': '04:01:58:27:7f:02',
'ipv4': {'gateway': '10.132.0.1',
'netmask': '255.255.0.0',
'ip_address': '10.132.75.35'},
'type': 'private'}
]
}
"""
def _get_subnet_part(pcfg):
subpart = {'type': 'static',
'control': 'auto',
'address': pcfg.get('ip_address'),
'gateway': pcfg.get('gateway')}
if ":" in pcfg.get('ip_address'):
subpart['address'] = "{0}/{1}".format(pcfg.get('ip_address'),
pcfg.get('cidr'))
else:
subpart['netmask'] = pcfg.get('netmask')
return subpart
nic_configs = []
macs_to_nics = cloudnet.get_interfaces_by_mac()
LOG.debug("nic mapping: %s", macs_to_nics)
for n in config:
nic = config[n][0]
LOG.debug("considering %s", nic)
mac_address = nic.get('mac')
if mac_address not in macs_to_nics:
raise RuntimeError("Did not find network interface on system "
"with mac '%s'. Cannot apply configuration: %s"
% (mac_address, nic))
sysfs_name = macs_to_nics.get(mac_address)
nic_type = nic.get('type', 'unknown')
if_name = NIC_MAP.get(nic_type, sysfs_name)
if if_name != sysfs_name:
LOG.debug("Found %s interface '%s' on '%s', assigned name of '%s'",
nic_type, mac_address, sysfs_name, if_name)
else:
msg = ("Found interface '%s' on '%s', which is not a public "
"or private interface. Using default system naming.")
LOG.debug(msg, mac_address, sysfs_name)
ncfg = {'type': 'physical',
'mac_address': mac_address,
'name': if_name}
subnets = []
for netdef in ('ipv4', 'ipv6', 'anchor_ipv4', 'anchor_ipv6'):
raw_subnet = nic.get(netdef, None)
if not raw_subnet:
continue
sub_part = _get_subnet_part(raw_subnet)
if nic_type != "public" or "anchor" in netdef:
del sub_part['gateway']
subnets.append(sub_part)
ncfg['subnets'] = subnets
nic_configs.append(ncfg)
LOG.debug("nic '%s' configuration: %s", if_name, ncfg)
if dns_servers:
LOG.debug("added dns servers: %s", dns_servers)
nic_configs.append({'type': 'nameserver', 'address': dns_servers})
return {'version': 1, 'config': nic_configs}
def convert_network_configuration(config, dns_servers):
"""Convert the DigitalOcean Network description into Cloud-init's netconfig
format.
Example JSON:
{'public': [
{'mac': '04:01:58:27:7f:01',
'ipv4': {'gateway': '45.55.32.1',
'netmask': '255.255.224.0',
'ip_address': '45.55.50.93'},
'anchor_ipv4': {
'gateway': '10.17.0.1',
'netmask': '255.255.0.0',
'ip_address': '10.17.0.9'},
'type': 'public',
'ipv6': {'gateway': '....',
'ip_address': '....',
'cidr': 64}}
],
'private': [
{'mac': '04:01:58:27:7f:02',
'ipv4': {'gateway': '10.132.0.1',
'netmask': '255.255.0.0',
'ip_address': '10.132.75.35'},
'type': 'private'}
]
}
"""
def _get_subnet_part(pcfg, nameservers=None):
subpart = {
'type': 'static',
'control': 'auto',
'address': pcfg.get('ip_address'),
'gateway': pcfg.get('gateway')
}
if nameservers:
subpart['dns_nameservers'] = nameservers
if ":" in pcfg.get('ip_address'):
subpart['address'] = "{0}/{1}".format(pcfg.get('ip_address'),
pcfg.get('cidr'))
else:
subpart['netmask'] = pcfg.get('netmask')
return subpart
all_nics = []
for k in ('public', 'private'):
if k in config:
all_nics.extend(config[k])
macs_to_nics = cloudnet.get_interfaces_by_mac()
nic_configs = []
for nic in all_nics:
mac_address = nic.get('mac')
sysfs_name = macs_to_nics.get(mac_address)
nic_type = nic.get('type', 'unknown')
# Note: the entry 'public' above contains a list, but
# the list will only ever have one nic inside it per digital ocean.
# If it ever had more than one nic, then this code would
# assign all 'public' the same name.
if_name = NIC_MAP.get(nic_type, sysfs_name)
LOG.debug("mapped %s interface to %s, assigning name of %s",
mac_address, sysfs_name, if_name)
ncfg = {
'type': 'physical',
'mac_address': mac_address,
'name': if_name
}
subnets = []
for netdef in ('ipv4', 'ipv6', 'anchor_ipv4', 'anchor_ipv6'):
raw_subnet = nic.get(netdef, None)
if not raw_subnet:
continue
sub_part = _get_subnet_part(raw_subnet)
if nic_type == 'public' and 'anchor' not in netdef:
# add DNS resolvers to the public interfaces only
sub_part = _get_subnet_part(raw_subnet, dns_servers)
else:
# remove the gateway any non-public interfaces
if 'gateway' in sub_part:
del sub_part['gateway']
subnets.append(sub_part)
ncfg['subnets'] = subnets
nic_configs.append(ncfg)
LOG.debug("nic '%s' configuration: %s", if_name, ncfg)
return {'version': 1, 'config': nic_configs}
def get_interface_map():
return net.get_interfaces_by_mac()