async def bridge_setup(self, tapname, tap, attach_iface):
if attach_iface is None:
# XXX: backward compatibility prior to 11.1-RELEASE.
try:
attach_iface = netif.RoutingTable(
).default_route_ipv4.interface
except:
return
if_bridge = []
bridge_enabled = False
for brgname, iface in list(netif.list_interfaces().items()):
if brgname.startswith('bridge'):
if_bridge.append(iface)
if if_bridge:
for bridge in if_bridge:
if attach_iface in bridge.members:
bridge_enabled = True
self.set_tap_mtu(bridge, tap)
bridge.add_member(tapname)
break
if bridge_enabled is False:
bridge = netif.get_interface(netif.create_interface('bridge'))
self.set_tap_mtu(bridge, tap)
bridge.add_member(tapname)
bridge.add_member(attach_iface)
bridge.up()
async def bridge_setup(self, tapname, tap, attach_iface):
if attach_iface is None:
# XXX: backward compatibility prior to 11.1-RELEASE.
try:
attach_iface = netif.RoutingTable().default_route_ipv4.interface
except:
return
if_bridge = []
bridge_enabled = False
for brgname, iface in list(netif.list_interfaces().items()):
if brgname.startswith('bridge'):
if_bridge.append(iface)
if if_bridge:
for bridge in if_bridge:
if attach_iface in bridge.members:
bridge_enabled = True
self.set_tap_mtu(bridge, tap)
bridge.add_member(tapname)
break
if bridge_enabled is False:
bridge = netif.get_interface(netif.create_interface('bridge'))
self.set_tap_mtu(bridge, tap)
bridge.add_member(tapname)
bridge.add_member(attach_iface)
bridge.up()
def run(self, bridge_enable=False):
node = ConfigNode('service.openvpn', self.configstore).__getstate__()
interface_list = list(netif.list_interfaces().keys())
default_interface = self.dispatcher.call_sync('networkd.configuration.get_default_interface')
vpn_interface = netif.get_interface(self.configstore.get('service.openvpn.dev'))
vpn_interface.up()
if 'VPNbridge' not in interface_list:
bridge_interface = netif.get_interface(netif.create_interface('bridge'))
bridge_interface.rename('VPNbridge')
bridge_interface.description = 'OpenVPN bridge interface'
else:
bridge_interface = netif.get_interface('VPNbridge')
if node['server_bridge_extended']:
subnet = ipaddress.ip_interface('{0}/{1}'.format(node['server_bridge_ip'], bridge_values['server_bridge_netmask']))
bridge_interface.add_address(netif.InterfaceAddress(
netif.AddressFamily.INET,
subnet
))
try:
bridge_interface.add_member(default_interface)
except FileExistsError as e:
logger.info('Default interface already bridged: {0}'.format(e))
except OSError as e:
raise TaskException(errno.EBUSY,
'Default interface busy - check other bridge interfaces. {0}'.format(e))
try:
bridge_interface.add_member(self.configstore.get('service.openvpn.dev'))
except FileExistsError as e:
logger.info('OpenVPN interface already bridged: {0}'.format(e))
except OSError as e:
raise TaskException(errno.EBUSY,
'VPN interface busy - check other bridge interfaces. {0}'.format(e))
else:
bridge_interface.up()
def up(self):
# Destroy old bridge (if exists)
try:
netif.destroy_interface(self.ifname)
except OSError as err:
if err.errno != errno.ENXIO:
raise RuntimeError('Cannot destroy {0}: {1}'.format(self.ifname, str(err)))
# Setup bridge
self.bridge_if = netif.get_interface(netif.create_interface('bridge'))
self.bridge_if.rename(self.ifname)
self.bridge_if.description = 'containerd management network'
self.bridge_if.add_address(netif.InterfaceAddress(
netif.AddressFamily.INET,
self.subnet
))
self.bridge_if.up()
self.dhcp_server.server_name = 'FreeNAS'
self.dhcp_server.on_request = self.dhcp_request
self.dhcp_server.start(self.ifname, self.subnet.ip)
self.dhcp_server_thread = gevent.spawn(self.dhcp_worker)
async def bridge_setup(self, tapname, tap, attach_iface):
if_bridge = []
bridge_enabled = False
if attach_iface is None:
# XXX: backward compatibility prior to 11.1-RELEASE.
try:
attach_iface = netif.RoutingTable().default_route_ipv4.interface
attach_iface_info = netif.get_interface(attach_iface)
except:
return
else:
attach_iface_info = netif.get_interface(attach_iface)
# If for some reason the main iface is down, we need to up it.
attach_iface_status = netif.InterfaceFlags.UP in attach_iface_info.flags
if attach_iface_status is False:
attach_iface_info.up()
for brgname, iface in list(netif.list_interfaces().items()):
if brgname.startswith('bridge'):
if_bridge.append(iface)
if if_bridge:
for bridge in if_bridge:
if attach_iface in bridge.members:
bridge_enabled = True
self.set_iface_mtu(attach_iface_info, tap)
bridge.add_member(tapname)
if netif.InterfaceFlags.UP not in bridge.flags:
bridge.up()
break
if bridge_enabled is False:
bridge = netif.get_interface(netif.create_interface('bridge'))
self.set_iface_mtu(attach_iface_info, tap)
bridge.add_member(tapname)
bridge.add_member(attach_iface)
bridge.up()
async def bridge_setup(self, tapname, tap, attach_iface):
if_bridge = []
bridge_enabled = False
if attach_iface is None:
# XXX: backward compatibility prior to 11.1-RELEASE.
try:
attach_iface = netif.RoutingTable(
).default_route_ipv4.interface
attach_iface_info = netif.get_interface(attach_iface)
except:
return
else:
attach_iface_info = netif.get_interface(attach_iface)
# If for some reason the main iface is down, we need to up it.
attach_iface_status = netif.InterfaceFlags.UP in attach_iface_info.flags
if attach_iface_status is False:
attach_iface_info.up()
for brgname, iface in list(netif.list_interfaces().items()):
if brgname.startswith('bridge'):
if_bridge.append(iface)
if if_bridge:
for bridge in if_bridge:
if attach_iface in bridge.members:
bridge_enabled = True
self.set_iface_mtu(attach_iface_info, tap)
bridge.add_member(tapname)
break
if bridge_enabled is False:
bridge = netif.get_interface(netif.create_interface('bridge'))
self.set_iface_mtu(attach_iface_info, tap)
bridge.add_member(tapname)
bridge.add_member(attach_iface)
bridge.up()
def init_tap(self, name, nic, mac):
try:
iface = netif.get_interface(netif.create_interface('tap'))
iface.description = 'vm:{0}:{1}'.format(self.name, name)
iface.up()
if nic['type'] == 'BRIDGE':
if nic['bridge']:
bridge = netif.get_interface(nic['bridge'])
bridge.add_member(iface.name)
if nic['type'] == 'MANAGEMENT':
mgmt = netif.get_interface('mgmt0', bridge=True)
mgmt.add_member(iface.name)
if nic['type'] == 'NAT':
mgmt = netif.get_interface('nat0', bridge=True)
mgmt.add_member(iface.name)
self.tap_interfaces[iface] = mac
return iface.name
except (KeyError, OSError) as err:
self.logger.warning('Cannot initialize NIC {0}: {1}'.format(name, str(err)))
return
def sync(self):
"""
Sync interfaces configured in database to the OS.
"""
interfaces = [
i['int_interface'] for i in self.middleware.call(
'datastore.query', 'network.interfaces')
]
cloned_interfaces = []
parent_interfaces = []
# First of all we need to create the virtual interfaces
# LAGG comes first and then VLAN
laggs = self.middleware.call('datastore.query',
'network.lagginterface')
for lagg in laggs:
name = lagg['lagg_interface']['int_interface']
cloned_interfaces.append(name)
self.logger.info('Setting up {}'.format(name))
try:
iface = netif.get_interface(name)
except KeyError:
netif.create_interface(name)
iface = netif.get_interface(name)
if lagg['lagg_protocol'] == 'fec':
protocol = netif.AggregationProtocol.ETHERCHANNEL
else:
protocol = getattr(netif.AggregationProtocol,
lagg['lagg_protocol'].upper())
if iface.protocol != protocol:
self.logger.info('{}: changing protocol to {}'.format(
name, protocol))
iface.protocol = protocol
members_configured = set(p[0] for p in iface.ports)
members_database = set()
for member in self.middleware.call(
'datastore.query', 'network.lagginterfacemembers',
[('lagg_interfacegroup_id', '=', lagg['id'])]):
members_database.add(member['lagg_physnic'])
# Remeve member configured but not in database
for member in (members_configured - members_database):
iface.delete_port(member)
# Add member in database but not configured
for member in (members_database - members_configured):
iface.add_port(member)
for port in iface.ports:
try:
port_iface = netif.get_interface(port[0])
except KeyError:
self.logger.warn('Could not find {} from {}'.format(
port[0], name))
continue
parent_interfaces.append(port[0])
port_iface.up()
vlans = self.middleware.call('datastore.query', 'network.vlan')
for vlan in vlans:
cloned_interfaces.append(vlan['vlan_vint'])
self.logger.info('Setting up {}'.format(vlan['vlan_vint']))
try:
iface = netif.get_interface(vlan['vlan_vint'])
except KeyError:
netif.create_interface(vlan['vlan_vint'])
iface = netif.get_interface(vlan['vlan_vint'])
if iface.parent != vlan['vlan_pint'] or iface.tag != vlan[
'vlan_tag']:
iface.unconfigure()
iface.configure(vlan['vlan_pint'], vlan['vlan_tag'])
try:
parent_iface = netif.get_interface(iface.parent)
except KeyError:
self.logger.warn('Could not find {} from {}'.format(
iface.parent, vlan['vlan_vint']))
continue
parent_interfaces.append(iface.parent)
parent_iface.up()
self.logger.info('Interfaces in database: {}'.format(
', '.join(interfaces) or 'NONE'))
for interface in interfaces:
try:
self.sync_interface(interface)
except:
self.logger.error('Failed to configure {}'.format(interface),
exc_info=True)
internal_interfaces = [
'lo', 'pflog', 'pfsync', 'tun', 'tap', 'bridge', 'epair'
]
if not self.middleware.call('system.is_freenas'):
internal_interfaces.extend(
self.middleware.call('notifier.failover_internal_interfaces')
or [])
internal_interfaces = tuple(internal_interfaces)
# Destroy interfaces which are not in database
for name, iface in list(netif.list_interfaces().items()):
# Skip internal interfaces
if name.startswith(internal_interfaces):
continue
# Skip interfaces in database
if name in interfaces:
continue
# Interface not in database lose addresses
for address in iface.addresses:
iface.remove_address(address)
# Kill dhclient if its running for this interface
dhclient_running, dhclient_pid = dhclient_status(name)
if dhclient_running:
os.kill(dhclient_pid, signal.SIGTERM)
# If we have vlan or lagg not in the database at all
# It gets destroy, otherwise just bring it down
if name not in cloned_interfaces and name.startswith(
('lagg', 'vlan')):
netif.destroy_interface(name)
elif name not in parent_interfaces:
iface.down()
async def sync(self):
"""
Sync interfaces configured in database to the OS.
"""
interfaces = [
i['int_interface'] for i in (await self.middleware.call(
'datastore.query', 'network.interfaces'))
]
cloned_interfaces = []
parent_interfaces = []
# First of all we need to create the virtual interfaces
# LAGG comes first and then VLAN
laggs = await self.middleware.call('datastore.query',
'network.lagginterface')
for lagg in laggs:
name = lagg['lagg_interface']['int_interface']
cloned_interfaces.append(name)
self.logger.info('Setting up {}'.format(name))
try:
iface = netif.get_interface(name)
except KeyError:
netif.create_interface(name)
iface = netif.get_interface(name)
protocol = getattr(netif.AggregationProtocol,
lagg['lagg_protocol'].upper())
if iface.protocol != protocol:
self.logger.info('{}: changing protocol to {}'.format(
name, protocol))
iface.protocol = protocol
members_database = set()
members_configured = set(p[0] for p in iface.ports)
members_changes = []
# In case there are MTU changes we need to use the lowest MTU between
# all members and use that.
lower_mtu = None
for member in (await self.middleware.call(
'datastore.query', 'network.lagginterfacemembers',
[('lagg_interfacegroup_id', '=', lagg['id'])])):
members_database.add(member['lagg_physnic'])
try:
member_iface = netif.get_interface(member['lagg_physnic'])
except KeyError:
self.logger.warn('Could not find {} from {}'.format(
member['lagg_physnic'], name))
continue
# In case there is no MTU in interface options and it is currently
# different than the default of 1500, revert it.
# If there is MTU and its different set it (using member options).
reg_mtu = RE_MTU.search(member['lagg_deviceoptions'])
if (reg_mtu and
(int(reg_mtu.group(1)) != member_iface.mtu
or int(reg_mtu.group(1)) != iface.mtu)) or (
not reg_mtu and
(member_iface.mtu != 1500 or iface.mtu != 1500)):
if not reg_mtu:
if not lower_mtu or lower_mtu > 1500:
lower_mtu = 1500
else:
reg_mtu = int(reg_mtu.group(1))
if not lower_mtu or lower_mtu > reg_mtu:
lower_mtu = reg_mtu
members_changes.append((member_iface, member['lagg_physnic'],
member['lagg_deviceoptions']))
for member_iface, member_name, member_options in members_changes:
# We need to remove interface from LAGG before changing MTU
if lower_mtu and member_iface.mtu != lower_mtu and member_name in members_configured:
iface.delete_port(member_name)
members_configured.remove(member_name)
proc = await Popen(['/sbin/ifconfig', member_name] +
shlex.split(member_options),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
err = (await proc.communicate())[1].decode()
if err:
self.logger.info(f'{member_name}: error applying: {err}')
if lower_mtu and member_iface.mtu != lower_mtu:
member_iface.mtu = lower_mtu
# Remove member configured but not in database
for member in (members_configured - members_database):
iface.delete_port(member)
# Add member in database but not configured
for member in (members_database - members_configured):
iface.add_port(member)
for port in iface.ports:
try:
port_iface = netif.get_interface(port[0])
except KeyError:
self.logger.warn('Could not find {} from {}'.format(
port[0], name))
continue
parent_interfaces.append(port[0])
port_iface.up()
vlans = await self.middleware.call('datastore.query', 'network.vlan')
for vlan in vlans:
cloned_interfaces.append(vlan['vlan_vint'])
self.logger.info('Setting up {}'.format(vlan['vlan_vint']))
try:
iface = netif.get_interface(vlan['vlan_vint'])
except KeyError:
netif.create_interface(vlan['vlan_vint'])
iface = netif.get_interface(vlan['vlan_vint'])
if iface.parent != vlan['vlan_pint'] or iface.tag != vlan[
'vlan_tag'] or iface.pcp != vlan['vlan_pcp']:
iface.unconfigure()
iface.configure(vlan['vlan_pint'], vlan['vlan_tag'],
vlan['vlan_pcp'])
try:
parent_iface = netif.get_interface(iface.parent)
except KeyError:
self.logger.warn('Could not find {} from {}'.format(
iface.parent, vlan['vlan_vint']))
continue
parent_interfaces.append(iface.parent)
parent_iface.up()
self.logger.info('Interfaces in database: {}'.format(
', '.join(interfaces) or 'NONE'))
for interface in interfaces:
try:
await self.sync_interface(interface)
except Exception:
self.logger.error('Failed to configure {}'.format(interface),
exc_info=True)
internal_interfaces = [
'lo', 'pflog', 'pfsync', 'tun', 'tap', 'bridge', 'epair'
]
if not await self.middleware.call('system.is_freenas'):
internal_interfaces.extend(
await
self.middleware.call('notifier.failover_internal_interfaces')
or [])
internal_interfaces = tuple(internal_interfaces)
# Destroy interfaces which are not in database
for name, iface in list(netif.list_interfaces().items()):
# Skip internal interfaces
if name.startswith(internal_interfaces):
continue
# Skip interfaces in database
if name in interfaces:
continue
# Interface not in database lose addresses
for address in iface.addresses:
iface.remove_address(address)
# Kill dhclient if its running for this interface
dhclient_running, dhclient_pid = dhclient_status(name)
if dhclient_running:
os.kill(dhclient_pid, signal.SIGTERM)
# If we have vlan or lagg not in the database at all
# It gets destroy, otherwise just bring it down
if name not in cloned_interfaces and name.startswith(
('lagg', 'vlan')):
netif.destroy_interface(name)
elif name not in parent_interfaces:
iface.down()
def configure_interface(self, name, restart_rtsold=True):
entity = self.datastore.get_one('network.interfaces', ('id', '=', name))
if not entity:
raise RpcException(errno.ENXIO, "Configuration for interface {0} not found".format(name))
try:
iface = netif.get_interface(name)
except KeyError:
if entity.get('cloned'):
netif.create_interface(entity['id'])
iface = netif.get_interface(name)
else:
yield errno.ENOENT, "Interface {0} not found".format(name)
return
if not entity.get('enabled'):
self.logger.info('Interface {0} is disabled'.format(name))
return
# check whether interface is a lagg member
for j in netif.list_interfaces().values():
try:
if isinstance(j, netif.LaggInterface) and name in [p[0] for p in j.ports]:
lagg_member = True
break
except OSError:
continue
else:
lagg_member = False
try:
if netif.InterfaceFlags.UP not in iface.flags:
self.logger.info('Bringing interface {0} up'.format(name))
iface.up()
# If it's VLAN, configure parent and tag
if entity.get('type') == 'VLAN':
vlan = entity.get('vlan')
if vlan:
parent = vlan.get('parent')
tag = vlan.get('tag')
if parent != iface.parent or tag != iface.tag:
try:
tag = int(tag)
iface.unconfigure()
iface.configure(parent, tag)
except OSError as e:
yield e.errno, 'Failed to configure VLAN interface {0}: {1}'.format(name, str(e))
# Configure protocol and member ports for a LAGG
if entity.get('type') == 'LAGG':
lagg = entity.get('lagg')
if lagg:
new_protocol = getattr(netif.AggregationProtocol, lagg.get('protocol', 'FAILOVER'))
old_ports = set(p[0] for p in iface.ports)
new_ports = set(lagg['ports'])
if iface.protocol != new_protocol:
iface.protocol = new_protocol
for port in old_ports - new_ports:
iface.delete_port(port)
for port in lagg['ports']:
if port not in old_ports:
iface.add_port(port)
# Configure member interfaces for a bridge
if entity.get('type') == 'BRIDGE':
bridge = entity.get('bridge')
if bridge:
old_members = set(iface.members)
new_members = set(bridge['members'])
for port in old_members - new_members:
iface.delete_member(port)
for port in new_members - old_members:
iface.add_member(port)
if entity.get('dhcp'):
if name in self.context.dhcp_clients:
self.logger.info('Interface {0} already configured using DHCP'.format(name))
else:
# Remove all existing aliases
for i in iface.addresses:
iface.remove_address(i)
self.logger.info('Trying to acquire DHCP lease on interface {0}...'.format(name))
if not self.context.configure_dhcp(name, True, INITIAL_DHCP_TIMEOUT):
yield errno.ENETUNREACH, 'Failed to configure interface {0} using DHCP'.format(name)
else:
if name in self.context.dhcp_clients:
self.logger.info('Stopping DHCP client on interface {0}'.format(name))
self.context.deconfigure_dhcp(name)
addresses = set(convert_aliases(entity))
existing_addresses = set([a for a in iface.addresses if a.af != netif.AddressFamily.LINK])
# Remove orphaned addresses
for i in existing_addresses - addresses:
if i.af == netif.AddressFamily.INET6 and str(i.address).startswith('fe80::'):
# skip link-local IPv6 addresses
continue
self.logger.info('Removing address from interface {0}: {1}'.format(name, i))
iface.remove_address(i)
# Add new or changed addresses
for i in addresses - existing_addresses:
self.logger.info('Adding new address to interface {0}: {1}'.format(name, i))
iface.add_address(i)
# nd6 stuff
if entity.get('rtadv', False):
iface.nd6_flags = iface.nd6_flags | {netif.NeighborDiscoveryFlags.ACCEPT_RTADV}
if restart_rtsold:
self.client.call_sync('service.restart', 'rtsold', timeout=300)
else:
iface.nd6_flags = iface.nd6_flags - {netif.NeighborDiscoveryFlags.ACCEPT_RTADV}
if entity.get('noipv6', False):
iface.nd6_flags = iface.nd6_flags | {netif.NeighborDiscoveryFlags.IFDISABLED}
iface.nd6_flags = iface.nd6_flags - {netif.NeighborDiscoveryFlags.AUTO_LINKLOCAL}
else:
iface.nd6_flags = iface.nd6_flags - {netif.NeighborDiscoveryFlags.IFDISABLED}
iface.nd6_flags = iface.nd6_flags | {netif.NeighborDiscoveryFlags.AUTO_LINKLOCAL}
if entity.get('mtu') and not isinstance(iface, netif.LaggInterface) and not lagg_member:
try:
iface.mtu = entity['mtu']
except OSError as err:
yield err.errno, 'Cannot set MTU of {0}: {1}'.format(name, str(err))
if entity.get('media'):
iface.media_subtype = entity['media']
# vlan interfaces don't support capabilities
if entity.get('capabilities') and not isinstance(iface, (netif.VlanInterface, netif.BridgeInterface)):
caps = iface.capabilities
for c in entity['capabilities'].get('add'):
caps.add(getattr(netif.InterfaceCapability, c))
for c in entity['capabilities'].get('del'):
caps.remove(getattr(netif.InterfaceCapability, c))
iface.capabilities = caps
except OSError as err:
yield err.errno, err.strerror
self.client.emit_event('network.interface.configured', {
'interface': name,
})
async def run(self):
args = [
'bhyve',
'-H',
'-w',
'-c', str(self.vm['vcpus']),
'-m', str(self.vm['memory']),
'-s', '0:0,hostbridge',
'-s', '31,lpc',
'-l', 'com1,/dev/nmdm{}A'.format(self.vm['id']),
]
if self.vm['bootloader'] in ('UEFI', 'UEFI_CSM'):
args += [
'-l', 'bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI{}.fd'.format('_CSM' if self.vm['bootloader'] == 'UEFI_CSM' else ''),
]
nid = Nid(3)
for device in self.vm['devices']:
if device['dtype'] == 'DISK' or device['dtype'] == 'RAW':
disk_sector_size = device['attributes'].get('sectorsize', 0)
if disk_sector_size > 0:
sectorsize_args = ",sectorsize=" + str(disk_sector_size)
else:
sectorsize_args = ""
if device['attributes'].get('type') == 'AHCI':
args += ['-s', '{},ahci-hd,{}{}'.format(nid(), device['attributes']['path'], sectorsize_args)]
else:
args += ['-s', '{},virtio-blk,{}{}'.format(nid(), device['attributes']['path'], sectorsize_args)]
elif device['dtype'] == 'CDROM':
args += ['-s', '{},ahci-cd,{}'.format(nid(), device['attributes']['path'])]
elif device['dtype'] == 'NIC':
attach_iface = device['attributes'].get('nic_attach')
self.logger.debug('====> NIC_ATTACH: {0}'.format(attach_iface))
tapname = netif.create_interface('tap')
tap = netif.get_interface(tapname)
tap.up()
self.taps.append(tapname)
await self.bridge_setup(tapname, tap, attach_iface)
if device['attributes'].get('type') == 'VIRTIO':
nictype = 'virtio-net'
else:
nictype = 'e1000'
mac_address = device['attributes'].get('mac', None)
# By default we add one NIC and the MAC address is an empty string.
# Issue: 24222
if mac_address == "":
mac_address = None
if mac_address == '00:a0:98:FF:FF:FF' or mac_address is None:
args += ['-s', '{},{},{},mac={}'.format(nid(), nictype, tapname, self.random_mac())]
else:
args += ['-s', '{},{},{},mac={}'.format(nid(), nictype, tapname, mac_address)]
elif device['dtype'] == 'VNC':
if device['attributes'].get('wait'):
wait = 'wait'
else:
wait = ''
vnc_resolution = device['attributes'].get('vnc_resolution', None)
vnc_port = int(device['attributes'].get('vnc_port', 5900 + self.vm['id']))
vnc_bind = device['attributes'].get('vnc_bind', '0.0.0.0')
vnc_password = device['attributes'].get('vnc_password', None)
vnc_password_args = ""
if vnc_password:
vnc_password_args = ",password="******"===> Rebooting VM: {0} ID: {1} BHYVE_CODE: {2}".format(self.vm['name'], self.vm['id'], self.bhyve_error))
await self.manager.restart(self.vm['id'])
await self.manager.start(self.vm['id'])
elif self.bhyve_error == 1:
# XXX: Need a better way to handle the vmm destroy.
self.logger.info("===> Powered off VM: {0} ID: {1} BHYVE_CODE: {2}".format(self.vm['name'], self.vm['id'], self.bhyve_error))
await self.destroy_vm()
elif self.bhyve_error in (2, 3):
self.logger.info("===> Stopping VM: {0} ID: {1} BHYVE_CODE: {2}".format(self.vm['name'], self.vm['id'], self.bhyve_error))
await self.manager.stop(self.vm['id'])
elif self.bhyve_error not in (0, 1, 2, 3, None):
self.logger.info("===> Error VM: {0} ID: {1} BHYVE_CODE: {2}".format(self.vm['name'], self.vm['id'], self.bhyve_error))
await self.destroy_vm()
async def sync(self):
"""
Sync interfaces configured in database to the OS.
"""
interfaces = [i['int_interface'] for i in (await self.middleware.call('datastore.query', 'network.interfaces'))]
cloned_interfaces = []
parent_interfaces = []
# First of all we need to create the virtual interfaces
# LAGG comes first and then VLAN
laggs = await self.middleware.call('datastore.query', 'network.lagginterface')
for lagg in laggs:
name = lagg['lagg_interface']['int_interface']
cloned_interfaces.append(name)
self.logger.info('Setting up {}'.format(name))
try:
iface = netif.get_interface(name)
except KeyError:
netif.create_interface(name)
iface = netif.get_interface(name)
protocol = getattr(netif.AggregationProtocol, lagg['lagg_protocol'].upper())
if iface.protocol != protocol:
self.logger.info('{}: changing protocol to {}'.format(name, protocol))
iface.protocol = protocol
members_database = set()
members_configured = set(p[0] for p in iface.ports)
members_changes = []
# In case there are MTU changes we need to use the lowest MTU between
# all members and use that.
lower_mtu = None
for member in (await self.middleware.call('datastore.query', 'network.lagginterfacemembers', [('lagg_interfacegroup_id', '=', lagg['id'])])):
members_database.add(member['lagg_physnic'])
try:
member_iface = netif.get_interface(member['lagg_physnic'])
except KeyError:
self.logger.warn('Could not find {} from {}'.format(member['lagg_physnic'], name))
continue
# In case there is no MTU in interface options and it is currently
# different than the default of 1500, revert it.
# If there is MTU and its different set it (using member options).
reg_mtu = RE_MTU.search(member['lagg_deviceoptions'])
if (
reg_mtu and (
int(reg_mtu.group(1)) != member_iface.mtu or
int(reg_mtu.group(1)) != iface.mtu
)
) or (not reg_mtu and (member_iface.mtu != 1500 or iface.mtu != 1500)):
if not reg_mtu:
if not lower_mtu or lower_mtu > 1500:
lower_mtu = 1500
else:
reg_mtu = int(reg_mtu.group(1))
if not lower_mtu or lower_mtu > reg_mtu:
lower_mtu = reg_mtu
members_changes.append((member_iface, member['lagg_physnic'], member['lagg_deviceoptions']))
for member_iface, member_name, member_options in members_changes:
# We need to remove interface from LAGG before changing MTU
if lower_mtu and member_iface.mtu != lower_mtu and member_name in members_configured:
iface.delete_port(member_name)
members_configured.remove(member_name)
proc = await Popen(['/sbin/ifconfig', member_name] + shlex.split(member_options), stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
err = (await proc.communicate())[1].decode()
if err:
self.logger.info(f'{member_name}: error applying: {err}')
if lower_mtu and member_iface.mtu != lower_mtu:
member_iface.mtu = lower_mtu
# Remove member configured but not in database
for member in (members_configured - members_database):
iface.delete_port(member)
# Add member in database but not configured
for member in (members_database - members_configured):
iface.add_port(member)
for port in iface.ports:
try:
port_iface = netif.get_interface(port[0])
except KeyError:
self.logger.warn('Could not find {} from {}'.format(port[0], name))
continue
parent_interfaces.append(port[0])
port_iface.up()
vlans = await self.middleware.call('datastore.query', 'network.vlan')
for vlan in vlans:
cloned_interfaces.append(vlan['vlan_vint'])
self.logger.info('Setting up {}'.format(vlan['vlan_vint']))
try:
iface = netif.get_interface(vlan['vlan_vint'])
except KeyError:
netif.create_interface(vlan['vlan_vint'])
iface = netif.get_interface(vlan['vlan_vint'])
if iface.parent != vlan['vlan_pint'] or iface.tag != vlan['vlan_tag'] or iface.pcp != vlan['vlan_pcp']:
iface.unconfigure()
iface.configure(vlan['vlan_pint'], vlan['vlan_tag'], vlan['vlan_pcp'])
try:
parent_iface = netif.get_interface(iface.parent)
except KeyError:
self.logger.warn('Could not find {} from {}'.format(iface.parent, vlan['vlan_vint']))
continue
parent_interfaces.append(iface.parent)
parent_iface.up()
self.logger.info('Interfaces in database: {}'.format(', '.join(interfaces) or 'NONE'))
for interface in interfaces:
try:
await self.sync_interface(interface)
except Exception:
self.logger.error('Failed to configure {}'.format(interface), exc_info=True)
internal_interfaces = ['lo', 'pflog', 'pfsync', 'tun', 'tap', 'bridge', 'epair']
if not await self.middleware.call('system.is_freenas'):
internal_interfaces.extend(await self.middleware.call('notifier.failover_internal_interfaces') or [])
internal_interfaces = tuple(internal_interfaces)
# Destroy interfaces which are not in database
for name, iface in list(netif.list_interfaces().items()):
# Skip internal interfaces
if name.startswith(internal_interfaces):
continue
# Skip interfaces in database
if name in interfaces:
continue
# Interface not in database lose addresses
for address in iface.addresses:
iface.remove_address(address)
# Kill dhclient if its running for this interface
dhclient_running, dhclient_pid = dhclient_status(name)
if dhclient_running:
os.kill(dhclient_pid, signal.SIGTERM)
# If we have vlan or lagg not in the database at all
# It gets destroy, otherwise just bring it down
if name not in cloned_interfaces and name.startswith(('lagg', 'vlan')):
netif.destroy_interface(name)
elif name not in parent_interfaces:
iface.down()
async def run(self):
vnc_web = None # We need to initialize before line 200
args = [
'bhyve',
'-A',
'-H',
'-w',
'-c', str(self.vm['vcpus']),
'-m', str(self.vm['memory']),
'-s', '0:0,hostbridge',
'-s', '31,lpc',
'-l', 'com1,/dev/nmdm{}A'.format(self.vm['id']),
]
if self.vm['bootloader'] in ('UEFI', 'UEFI_CSM'):
args += [
'-l', 'bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI{}.fd'.format('_CSM' if self.vm['bootloader'] == 'UEFI_CSM' else ''),
]
nid = Nid(3)
device_map_file = None
grub_dir = None
grub_boot_device = False
for device in self.vm['devices']:
if device['dtype'] == 'DISK' or device['dtype'] == 'RAW':
disk_sector_size = device['attributes'].get('sectorsize', 0)
if disk_sector_size > 0:
sectorsize_args = ',sectorsize=' + str(disk_sector_size)
else:
sectorsize_args = ''
if device['attributes'].get('type') == 'AHCI':
args += ['-s', '{},ahci-hd,{}{}'.format(nid(), device['attributes']['path'], sectorsize_args)]
else:
args += ['-s', '{},virtio-blk,{}{}'.format(nid(), device['attributes']['path'], sectorsize_args)]
if self.vmutils.is_container(self.vm) and \
device['attributes'].get('boot', False) is True and \
grub_boot_device is False:
shared_fs = await self.middleware.call('vm.get_sharefs')
device_map_file = self.vmutils.ctn_device_map(shared_fs, self.vm['id'], self.vm['name'], device)
grub_dir = self.vmutils.ctn_grub(shared_fs, self.vm['id'], self.vm['name'], device, device['attributes'].get('rootpwd', None), None)
grub_boot_device = True
self.logger.debug('==> Boot Disk: {0}'.format(device))
elif device['dtype'] == 'CDROM':
args += ['-s', '{},ahci-cd,{}'.format(nid(), device['attributes']['path'])]
elif device['dtype'] == 'NIC':
attach_iface = device['attributes'].get('nic_attach')
self.logger.debug('====> NIC_ATTACH: {0}'.format(attach_iface))
tapname = netif.create_interface('tap')
tap = netif.get_interface(tapname)
tap.up()
self.taps.append(tapname)
await self.bridge_setup(tapname, tap, attach_iface)
if device['attributes'].get('type') == 'VIRTIO':
nictype = 'virtio-net'
else:
nictype = 'e1000'
mac_address = device['attributes'].get('mac', None)
# By default we add one NIC and the MAC address is an empty string.
# Issue: 24222
if mac_address == '':
mac_address = None
if mac_address == '00:a0:98:FF:FF:FF' or mac_address is None:
random_mac = await self.middleware.call('vm.random_mac')
args += ['-s', '{},{},{},mac={}'.format(nid(), nictype, tapname, random_mac)]
else:
args += ['-s', '{},{},{},mac={}'.format(nid(), nictype, tapname, mac_address)]
elif device['dtype'] == 'VNC':
if device['attributes'].get('wait'):
wait = 'wait'
else:
wait = ''
vnc_resolution = device['attributes'].get('vnc_resolution', None)
vnc_port = int(device['attributes'].get('vnc_port', 5900 + self.vm['id']))
vnc_bind = device['attributes'].get('vnc_bind', '0.0.0.0')
vnc_password = device['attributes'].get('vnc_password', None)
vnc_web = device['attributes'].get('vnc_web', None)
vnc_password_args = ''
if vnc_password:
vnc_password_args = 'password='******'x')
width = vnc_resolution[0]
height = vnc_resolution[1]
args += ['-s', '29,fbuf,vncserver,tcp={}:{},w={},h={},{},{}'.format(vnc_bind, vnc_port, width,
height, vnc_password_args, wait),
'-s', '30,xhci,tablet', ]
# grub-bhyve support for containers
if self.vmutils.is_container(self.vm):
grub_bhyve_args = [
'grub-bhyve', '-m', device_map_file,
'-r', 'host',
'-M', str(self.vm['memory']),
'-d', grub_dir,
str(self.vm['id']) + '_' + self.vm['name'],
]
# If container has no boot device, we should stop.
if grub_boot_device is False:
raise CallError(f'There is no boot disk for vm: {self.vm["name"]}')
self.logger.debug('Starting grub-bhyve: {}'.format(' '.join(grub_bhyve_args)))
self.grub_proc = await Popen(grub_bhyve_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
while True:
line = await self.grub_proc.stdout.readline()
if line == b'':
break
args.append(str(self.vm['id']) + '_' + self.vm['name'])
self.logger.debug('Starting bhyve: {}'.format(' '.join(args)))
self.proc = await Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if vnc_web:
split_port = int(str(vnc_port)[:2]) - 1
vnc_web_port = str(split_port) + str(vnc_port)[2:]
web_bind = ':{}'.format(vnc_web_port) if vnc_bind is '0.0.0.0' else '{}:{}'.format(vnc_bind, vnc_web_port)
self.web_proc = await Popen(['/usr/local/libexec/novnc/utils/websockify/run', '--web',
'/usr/local/libexec/novnc/', '--wrap-mode=ignore',
web_bind, '{}:{}'.format(vnc_bind, vnc_port)],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
self.logger.debug('==> Start WEBVNC at port {} with pid number {}'.format(vnc_web_port, self.web_proc.pid))
output = deque(maxlen=10)
while True:
line = await self.proc.stdout.readline()
if line == b'':
break
line = line.decode().rstrip()
self.logger.debug('{}: {}'.format(self.vm['name'], line))
output.append(line)
# bhyve returns the following status code:
# 0 - VM has been reset
# 1 - VM has been powered off
# 2 - VM has been halted
# 3 - VM generated a triple fault
# 4 - VM exited due to an error
# all other non-zero status codes are errors
self.bhyve_error = await self.proc.wait()
if self.bhyve_error == 0:
self.logger.info('===> Rebooting VM: {0} ID: {1} BHYVE_CODE: {2}'.format(self.vm['name'], self.vm['id'], self.bhyve_error))
await self.manager.restart(self.vm['id'])
await self.manager.start(self.vm['id'])
elif self.bhyve_error == 1:
# XXX: Need a better way to handle the vmm destroy.
self.logger.info('===> Powered off VM: {0} ID: {1} BHYVE_CODE: {2}'.format(self.vm['name'], self.vm['id'], self.bhyve_error))
await self.__teardown_guest_vmemory(self.vm['id'])
await self.destroy_vm()
elif self.bhyve_error in (2, 3):
self.logger.info('===> Stopping VM: {0} ID: {1} BHYVE_CODE: {2}'.format(self.vm['name'], self.vm['id'], self.bhyve_error))
await self.__teardown_guest_vmemory(self.vm['id'])
await self.manager.stop(self.vm['id'])
elif self.bhyve_error not in (0, 1, 2, 3, None):
self.logger.info('===> Error VM: {0} ID: {1} BHYVE_CODE: {2}'.format(self.vm['name'], self.vm['id'], self.bhyve_error))
await self.__teardown_guest_vmemory(self.vm['id'])
await self.destroy_vm()
output = "\n".join(output)
raise CallError(f'VM {self.vm["name"]} failed to start: {output}')
def configure_interface(self, name, restart_rtsold=True):
entity = self.datastore.get_one('network.interfaces', ('id', '=', name))
if not entity:
raise RpcException(errno.ENXIO, "Configuration for interface {0} not found".format(name))
try:
iface = netif.get_interface(name)
except KeyError:
if entity.get('cloned'):
netif.create_interface(entity['id'])
iface = netif.get_interface(name)
else:
raise RpcException(errno.ENOENT, "Interface {0} not found".format(name))
if not entity.get('enabled'):
self.logger.info('Interface {0} is disabled'.format(name))
return
# If it's VLAN, configure parent and tag
if entity.get('type') == 'VLAN':
vlan = entity.get('vlan')
if vlan:
parent = vlan.get('parent')
tag = vlan.get('tag')
if parent and tag:
try:
tag = int(tag)
iface.unconfigure()
iface.configure(parent, tag)
except Exception as e:
self.logger.warn('Failed to configure VLAN interface {0}: {1}'.format(name, str(e)))
# Configure protocol and member ports for a LAGG
if entity.get('type') == 'LAGG':
lagg = entity.get('lagg')
if lagg:
iface.protocol = getattr(netif.AggregationProtocol, lagg.get('protocol', 'FAILOVER'))
old_ports = set(iface.ports)
new_ports = set(lagg['ports'])
for port in old_ports - new_ports:
iface.delete_port(port)
for port in new_ports - old_ports:
iface.add_port(port)
# Configure member interfaces for a bridge
if entity.get('type') == 'BRIDGE':
bridge = entity.get('bridge')
if bridge:
old_members = set(iface.members)
new_members = set(bridge['members'])
for port in old_members - new_members:
iface.delete_member(port)
for port in new_members - old_members:
iface.add_member(port)
if entity.get('dhcp'):
if self.context.dhclient_running(name):
self.logger.info('Interface {0} already configured using DHCP'.format(name))
else:
# Remove all existing aliases
for i in iface.addresses:
iface.remove_address(i)
self.logger.info('Trying to acquire DHCP lease on interface {0}...'.format(name))
if not self.context.configure_dhcp(name):
self.logger.warn('Failed to configure interface {0} using DHCP'.format(name))
else:
addresses = set(convert_aliases(entity))
existing_addresses = set([a for a in iface.addresses if a.af != netif.AddressFamily.LINK])
# Remove orphaned addresses
for i in existing_addresses - addresses:
if i.af == netif.AddressFamily.INET6 and str(i.address).startswith('fe80::'):
# skip link-local IPv6 addresses
continue
self.logger.info('Removing address from interface {0}: {1}'.format(name, i))
iface.remove_address(i)
# Add new or changed addresses
for i in addresses - existing_addresses:
self.logger.info('Adding new address to interface {0}: {1}'.format(name, i))
iface.add_address(i)
# nd6 stuff
if entity.get('rtadv', False):
iface.nd6_flags = iface.nd6_flags | {netif.NeighborDiscoveryFlags.ACCEPT_RTADV}
if restart_rtsold:
self.client.call_sync('service.restart', 'rtsold')
else:
iface.nd6_flags = iface.nd6_flags - {netif.NeighborDiscoveryFlags.ACCEPT_RTADV}
if entity.get('noipv6', False):
iface.nd6_flags = iface.nd6_flags | {netif.NeighborDiscoveryFlags.IFDISABLED}
iface.nd6_flags = iface.nd6_flags - {netif.NeighborDiscoveryFlags.AUTO_LINKLOCAL}
else:
iface.nd6_flags = iface.nd6_flags - {netif.NeighborDiscoveryFlags.IFDISABLED}
iface.nd6_flags = iface.nd6_flags | {netif.NeighborDiscoveryFlags.AUTO_LINKLOCAL}
if entity.get('mtu'):
iface.mtu = entity['mtu']
if entity.get('media'):
iface.media_subtype = entity['media']
if entity.get('capabilities'):
caps = iface.capabilities
for c in entity['capabilities'].get('add'):
caps.add(getattr(netif.InterfaceCapability, c))
for c in entity['capabilities'].get('del'):
caps.remove(getattr(netif.InterfaceCapability, c))
iface.capabilities = caps
if netif.InterfaceFlags.UP not in iface.flags:
self.logger.info('Bringing interface {0} up'.format(name))
iface.up()
self.client.emit_event('network.interface.configured', {
'interface': name,
})
def run(self):
vpn_state = self.dispatcher.call_sync('service.query', [('name', '=', 'openvpn')], {'single': True})
node = ConfigNode('service.openvpn', self.configstore).__getstate__()
if vpn_state['state'] != 'RUNNING':
raise TaskException(errno.EPERM, 'For bridging VPN capabilities openvpn needs to be running.')
if node['mode'] == 'psk':
raise TaskException(errno.EPERM, 'Bridging VPN capabilities only in pki mode.')
# This feels like a not so good solution - feel free to change
tap_interfaces = netif.get_ifgroup('tap')
for vpn_interface_name in tap_interfaces:
tap_pids = system('/usr/bin/fuser', '/dev/' + vpn_interface_name)[0].split()
if vpn_state['pid'] in tap_pids:
break
try:
vpn_interface = netif.get_interface(vpn_interface_name)
except KeyError:
raise TaskException(
errno.EINVAL,
'{0} interface does not exist - Verify OpenVPN status'.format(vpn_interface_name)
)
else:
vpn_interface.up()
default_interface = self.dispatcher.call_sync('networkd.configuration.get_default_interface')
if not default_interface:
raise TaskException(errno.EINVAL, 'No default interface configured. Verify network setup.')
# Heavily inspired by containterd
available_bridges = list(b for b in netif.list_interfaces().keys()
if isinstance(netif.get_interface(b), netif.BridgeInterface))
for b in available_bridges:
bridge_interface = netif.get_interface(b, bridge=True)
if default_interface in bridge_interface.members:
try:
bridge_interface.add_member(vpn_interface_name)
except FileExistsError:
pass
break
else:
bridge_interface = netif.get_interface(netif.create_interface('bridge'))
bridge_interface.rename('brg{0}'.format(len(available_bridges)))
bridge_interface.description = 'OpenVPN bridge interface'
bridge_interface.up()
bridge_interface.add_member(default_interface)
bridge_interface.add_member(vpn_interface_name)
if node['server_bridge_extended']:
subnet = ipaddress.ip_interface('{0}/{1}'.format(node['server_bridge_ip'],
node['server_bridge_netmask']))
bridge_interface.add_address(netif.InterfaceAddress(
netif.AddressFamily.INET,
subnet
))
async def run(self):
args = [
'bhyve',
'-H',
'-w',
'-c',
str(self.vm['vcpus']),
'-m',
str(self.vm['memory']),
'-s',
'0:0,hostbridge',
'-s',
'31,lpc',
'-l',
'com1,/dev/nmdm{}A'.format(self.vm['id']),
]
if self.vm['bootloader'] in ('UEFI', 'UEFI_CSM'):
args += [
'-l',
'bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI{}.fd'.
format('_CSM' if self.vm['bootloader'] == 'UEFI_CSM' else ''),
]
nid = Nid(3)
for device in self.vm['devices']:
if device['dtype'] == 'DISK' or device['dtype'] == 'RAW':
disk_sector_size = device['attributes'].get('sectorsize', 0)
if disk_sector_size > 0:
sectorsize_args = ",sectorsize=" + str(disk_sector_size)
else:
sectorsize_args = ""
if device['attributes'].get('type') == 'AHCI':
args += [
'-s',
'{},ahci-hd,{}{}'.format(nid(),
device['attributes']['path'],
sectorsize_args)
]
else:
args += [
'-s', '{},virtio-blk,{}{}'.format(
nid(), device['attributes']['path'],
sectorsize_args)
]
elif device['dtype'] == 'CDROM':
args += [
'-s', '{},ahci-cd,{}'.format(nid(),
device['attributes']['path'])
]
elif device['dtype'] == 'NIC':
attach_iface = device['attributes'].get('nic_attach')
self.logger.debug('====> NIC_ATTACH: {0}'.format(attach_iface))
tapname = netif.create_interface('tap')
tap = netif.get_interface(tapname)
tap.up()
self.taps.append(tapname)
await self.bridge_setup(tapname, tap, attach_iface)
if device['attributes'].get('type') == 'VIRTIO':
nictype = 'virtio-net'
else:
nictype = 'e1000'
mac_address = device['attributes'].get('mac', None)
# By default we add one NIC and the MAC address is an empty string.
# Issue: 24222
if mac_address == "":
mac_address = None
if mac_address == '00:a0:98:FF:FF:FF' or mac_address is None:
args += [
'-s', '{},{},{},mac={}'.format(nid(), nictype, tapname,
self.random_mac())
]
else:
args += [
'-s', '{},{},{},mac={}'.format(nid(), nictype, tapname,
mac_address)
]
elif device['dtype'] == 'VNC':
if device['attributes'].get('wait'):
wait = 'wait'
else:
wait = ''
vnc_resolution = device['attributes'].get(
'vnc_resolution', None)
vnc_port = int(device['attributes'].get(
'vnc_port', 5900 + self.vm['id']))
vnc_bind = device['attributes'].get('vnc_bind', '0.0.0.0')
vnc_password = device['attributes'].get('vnc_password', None)
vnc_web = device['attributes'].get('vnc_web', None)
vnc_password_args = ""
if vnc_password:
vnc_password_args = ",password="******"==> Start WEBVNC at port {} with pid number {}".format(
vnc_web_port, self.web_proc.pid))
while True:
line = await self.proc.stdout.readline()
if line == b'':
break
self.logger.debug('{}: {}'.format(self.vm['name'], line.decode()))
# bhyve returns the following status code:
# 0 - VM has been reset
# 1 - VM has been powered off
# 2 - VM has been halted
# 3 - VM generated a triple fault
# all other non-zero status codes are errors
self.bhyve_error = await self.proc.wait()
if self.bhyve_error == 0:
self.logger.info(
"===> Rebooting VM: {0} ID: {1} BHYVE_CODE: {2}".format(
self.vm['name'], self.vm['id'], self.bhyve_error))
await self.manager.restart(self.vm['id'])
await self.manager.start(self.vm['id'])
elif self.bhyve_error == 1:
# XXX: Need a better way to handle the vmm destroy.
self.logger.info(
"===> Powered off VM: {0} ID: {1} BHYVE_CODE: {2}".format(
self.vm['name'], self.vm['id'], self.bhyve_error))
await self.destroy_vm()
elif self.bhyve_error in (2, 3):
self.logger.info(
"===> Stopping VM: {0} ID: {1} BHYVE_CODE: {2}".format(
self.vm['name'], self.vm['id'], self.bhyve_error))
await self.manager.stop(self.vm['id'])
elif self.bhyve_error not in (0, 1, 2, 3, None):
self.logger.info(
"===> Error VM: {0} ID: {1} BHYVE_CODE: {2}".format(
self.vm['name'], self.vm['id'], self.bhyve_error))
await self.destroy_vm()
def run(self):
args = [
'bhyve',
'-A',
'-P',
'-H',
'-c', str(self.vm['vcpus']),
'-m', str(self.vm['memory']),
'-s', '0:0,hostbridge',
'-s', '31,lpc',
'-l', 'com1,/dev/nmdm{}A'.format(self.vm['id']),
]
if self.vm['bootloader'] in ('UEFI', 'UEFI_CSM'):
args += [
'-l', 'bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI{}.fd'.format('_CSM' if self.vm['bootloader'] == 'UEFI_CSM' else ''),
]
nid = Nid(3)
for device in self.vm['devices']:
if device['dtype'] == 'DISK':
if device['attributes'].get('mode') == 'AHCI':
args += ['-s', '{},ahci-hd,{}'.format(nid(), device['attributes']['path'])]
else:
args += ['-s', '{},virtio-blk,{}'.format(nid(), device['attributes']['path'])]
elif device['dtype'] == 'CDROM':
args += ['-s', '{},ahci-cd,{}'.format(nid(), device['attributes']['path'])]
elif device['dtype'] == 'NIC':
tapname = netif.create_interface('tap')
tap = netif.get_interface(tapname)
tap.up()
self.taps.append(tapname)
# If Bridge
if True:
bridge = None
for name, iface in netif.list_interfaces().items():
if name.startswith('bridge'):
bridge = iface
break
if not bridge:
bridge = netif.get_interface(netif.create_interface('bridge'))
bridge.add_member(tapname)
defiface = Popen("route -nv show default|grep -w interface|awk '{ print $2 }'", stdout=subprocess.PIPE, shell=True).communicate()[0].strip()
if defiface and defiface not in bridge.members:
bridge.add_member(defiface)
bridge.up()
if device['attributes'].get('type') == 'VIRTIO':
nictype = 'virtio-net'
else:
nictype = 'e1000'
args += ['-s', '{},{},{}'.format(nid(), nictype, tapname)]
elif device['dtype'] == 'VNC':
if device['attributes'].get('wait'):
wait = 'wait'
else:
wait = ''
args += [
'-s', '29,fbuf,tcp=0.0.0.0:{},w=1024,h=768,{}'.format(5900 + self.vm['id'], wait),
'-s', '30,xhci,tablet',
]
args.append(self.vm['name'])
self.logger.debug('Starting bhyve: {}'.format(' '.join(args)))
self.proc = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
for line in self.proc.stdout:
self.logger.debug('{}: {}'.format(self.vm['name'], line))
self.proc.wait()
self.logger.info('Destroying {}'.format(self.vm['name']))
Popen(['bhyvectl', '--destroy', '--vm={}'.format(self.vm['name'])], stdout=subprocess.PIPE, stderr=subprocess.PIPE).wait()
while self.taps:
netif.destroy_interface(self.taps.pop())
self.manager._vm.pop(self.vm['id'], None)
def configure_interface(self, name, restart_rtsold=True):
entity = self.datastore.get_one('network.interfaces', ('id', '=', name))
if not entity:
raise RpcException(errno.ENXIO, "Configuration for interface {0} not found".format(name))
try:
iface = netif.get_interface(name)
except KeyError:
if entity.get('cloned'):
netif.create_interface(entity['id'])
iface = netif.get_interface(name)
else:
yield errno.ENOENT, "Interface {0} not found".format(name)
return
if not entity.get('enabled'):
self.logger.info('Interface {0} is disabled'.format(name))
return
# check whether interface is a lagg member
for j in netif.list_interfaces().values():
try:
if isinstance(j, netif.LaggInterface) and name in [p[0] for p in j.ports]:
lagg_member = True
break
except OSError:
continue
else:
lagg_member = False
try:
if netif.InterfaceFlags.UP not in iface.flags:
self.logger.info('Bringing interface {0} up'.format(name))
iface.up()
# If it's VLAN, configure parent and tag
if entity.get('type') == 'VLAN':
vlan = entity.get('vlan')
if vlan:
parent = vlan.get('parent')
tag = vlan.get('tag')
if parent != iface.parent or tag != iface.tag:
try:
tag = int(tag)
iface.unconfigure()
iface.configure(parent, tag)
except OSError as e:
yield e.errno, 'Failed to configure VLAN interface {0}: {1}'.format(name, str(e))
# Configure protocol and member ports for a LAGG
if entity.get('type') == 'LAGG':
lagg = entity.get('lagg')
if lagg:
new_protocol = getattr(netif.AggregationProtocol, lagg.get('protocol', 'FAILOVER'))
old_ports = set(p[0] for p in iface.ports)
new_ports = set(lagg['ports'])
if iface.protocol != new_protocol:
iface.protocol = new_protocol
for port in old_ports - new_ports:
iface.delete_port(port)
for port in new_ports - old_ports:
iface.add_port(port)
# Configure member interfaces for a bridge
if entity.get('type') == 'BRIDGE':
bridge = entity.get('bridge')
if bridge:
old_members = set(iface.members)
new_members = set(bridge['members'])
for port in old_members - new_members:
iface.delete_member(port)
for port in new_members - old_members:
iface.add_member(port)
if entity.get('dhcp'):
if name in self.context.dhcp_clients:
self.logger.info('Interface {0} already configured using DHCP'.format(name))
else:
# Remove all existing aliases
for i in iface.addresses:
iface.remove_address(i)
self.logger.info('Trying to acquire DHCP lease on interface {0}...'.format(name))
if not self.context.configure_dhcp(name):
yield errno.ENETUNREACH, 'Failed to configure interface {0} using DHCP'.format(name)
else:
if name in self.context.dhcp_clients:
self.logger.info('Stopping DHCP client on interface {0}'.format(name))
self.context.deconfigure_dhcp(name)
addresses = set(convert_aliases(entity))
existing_addresses = set([a for a in iface.addresses if a.af != netif.AddressFamily.LINK])
# Remove orphaned addresses
for i in existing_addresses - addresses:
if i.af == netif.AddressFamily.INET6 and str(i.address).startswith('fe80::'):
# skip link-local IPv6 addresses
continue
self.logger.info('Removing address from interface {0}: {1}'.format(name, i))
iface.remove_address(i)
# Add new or changed addresses
for i in addresses - existing_addresses:
self.logger.info('Adding new address to interface {0}: {1}'.format(name, i))
iface.add_address(i)
# nd6 stuff
if entity.get('rtadv', False):
iface.nd6_flags = iface.nd6_flags | {netif.NeighborDiscoveryFlags.ACCEPT_RTADV}
if restart_rtsold:
self.client.call_sync('service.restart', 'rtsold')
else:
iface.nd6_flags = iface.nd6_flags - {netif.NeighborDiscoveryFlags.ACCEPT_RTADV}
if entity.get('noipv6', False):
iface.nd6_flags = iface.nd6_flags | {netif.NeighborDiscoveryFlags.IFDISABLED}
iface.nd6_flags = iface.nd6_flags - {netif.NeighborDiscoveryFlags.AUTO_LINKLOCAL}
else:
iface.nd6_flags = iface.nd6_flags - {netif.NeighborDiscoveryFlags.IFDISABLED}
iface.nd6_flags = iface.nd6_flags | {netif.NeighborDiscoveryFlags.AUTO_LINKLOCAL}
if entity.get('mtu') and not isinstance(iface, netif.LaggInterface) and not lagg_member:
try:
iface.mtu = entity['mtu']
except OSError as err:
yield err.errno, 'Cannot set MTU of {0}: {1}'.format(name, str(err))
if entity.get('media'):
iface.media_subtype = entity['media']
# vlan interfaces don't support capabilities
if entity.get('capabilities') and not isinstance(iface, (netif.VlanInterface, netif.BridgeInterface)):
caps = iface.capabilities
for c in entity['capabilities'].get('add'):
caps.add(getattr(netif.InterfaceCapability, c))
for c in entity['capabilities'].get('del'):
caps.remove(getattr(netif.InterfaceCapability, c))
iface.capabilities = caps
except OSError as err:
yield err.errno, err.strerror
self.client.emit_event('network.interface.configured', {
'interface': name,
})
def sync(self):
"""
Sync interfaces configured in database to the OS.
"""
interfaces = [i['int_interface'] for i in self.middleware.call('datastore.query', 'network.interfaces')]
cloned_interfaces = []
# First of all we need to create the virtual interfaces
# LAGG comes first and then VLAN
laggs = self.middleware.call('datastore.query', 'network.lagginterface')
for lagg in laggs:
name = lagg['lagg_interface']['int_name']
cloned_interfaces.append(name)
self.logger.info('Setting up {}'.format(name))
try:
iface = netif.get_interface(name)
except KeyError:
netif.create_interface(name)
iface = netif.get_interface(name)
if lagg['lagg_protocol'] == 'fec':
protocol = netif.AggregationProtocol.ETHERCHANNEL
else:
protocol = getattr(netif.AggregationProtocol, lagg['lagg_protocol'].upper())
if iface.protocol != protocol:
self.logger.info('{}: changing protocol to {}'.format(name, protocol))
iface.protocol = protocol
members_configured = set(p[0] for p in iface.ports)
members_database = set()
for member in self.middleware.call('datastore.query', 'network.lagginterfacemembers', [('lagg_interfacegroup_id', '=', lagg['id'])]):
members_database.add(member['lagg_physnic'])
# Remeve member configured but not in database
for member in (members_configured - members_database):
iface.delete_port(member)
# Add member in database but not configured
for member in (members_database - members_configured):
iface.add_port(member)
for port in iface.ports:
try:
port_iface = netif.get_interface(port[0])
except KeyError:
self.logger.warn('Could not find {} from {}'.format(port[0], name))
continue
port_iface.up()
vlans = self.middleware.call('datastore.query', 'network.vlan')
for vlan in vlans:
cloned_interfaces.append(vlan['vlan_vint'])
self.logger.info('Setting up {}'.format(vlan['vlan_vint']))
try:
iface = netif.get_interface(vlan['vlan_vint'])
except KeyError:
netif.create_interface(vlan['vlan_vint'])
iface = netif.get_interface(vlan['vlan_vint'])
if iface.parent != vlan['vlan_pint'] or iface.tag != vlan['vlan_tag']:
iface.unconfigure()
iface.configure(vlan['vlan_pint'], vlan['vlan_tag'])
try:
parent_iface = netif.get_interface(iface.parent)
except KeyError:
self.logger.warn('Could not find {} from {}'.format(iface.parent, vlan['vlan_vint']))
continue
parent_iface.up()
self.logger.info('Interfaces in database: {}'.format(', '.join(interfaces) or 'NONE'))
for interface in interfaces:
try:
self.sync_interface(interface)
except:
self.logger.error('Failed to configure {}'.format(interface), exc_info=True)
internal_interfaces = ['lo', 'pflog', 'pfsync', 'tun', 'tap']
if not self.middleware.call('system.is_freenas'):
internal_interfaces.extend(self.middleware.call('notifier.failover_internal_interfaces') or [])
internal_interfaces = tuple(internal_interfaces)
# Destroy interfaces which are not in database
for name, iface in list(netif.list_interfaces().items()):
# Skip internal interfaces
if name.startswith(internal_interfaces):
continue
# Skip interfaces in database
if name in interfaces:
continue
# Interface not in database lose addresses
for address in iface.addresses:
iface.remove_address(address)
# Kill dhclient if its running for this interface
dhclient_running, dhclient_pid = dhclient_status(name)
if dhclient_running:
os.kill(dhclient_pid, signal.SIGTERM)
# If we have vlan or lagg not in the database at all
# It gets destroy, otherwise just bring it down
if name not in cloned_interfaces and name.startswith(('lagg', 'vlan')):
netif.destroy_interface(name)
else:
iface.down()
def run(self):
vpn_state = self.dispatcher.call_sync('service.query', [('name', '=', 'openvpn')], {'single': True})
node = ConfigNode('service.openvpn', self.configstore).__getstate__()
if vpn_state['state'] != 'RUNNING':
raise TaskException(errno.EPERM, 'For bridging VPN capabilities openvpn needs to be running.')
if node['mode'] == 'psk':
raise TaskException(errno.EPERM, 'Bridging VPN capabilities only in pki mode.')
# This feels like a not so good solution - feel free to change
tap_interfaces = netif.get_ifgroup('tap')
for vpn_interface_name in tap_interfaces:
tap_pids = system('/usr/bin/fuser', '/dev/' + vpn_interface_name)[0].split()
if vpn_state['pid'] in tap_pids:
break
try:
vpn_interface = netif.get_interface(vpn_interface_name)
except KeyError:
raise TaskException(
errno.EINVAL,
'{0} interface does not exist - Verify OpenVPN status'.format(vpn_interface_name)
)
else:
vpn_interface.up()
default_interface = self.dispatcher.call_sync('networkd.configuration.get_default_interface')
if not default_interface:
raise TaskException(errno.EINVAL, 'No default interface configured. Verify network setup.')
# Heavily inspired by containterd
available_bridges = list(b for b in netif.list_interfaces().keys()
if isinstance(netif.get_interface(b), netif.BridgeInterface))
for b in available_bridges:
bridge_interface = netif.get_interface(b, bridge=True)
if default_interface in bridge_interface.members:
try:
bridge_interface.add_member(vpn_interface_name)
except FileExistsError:
pass
break
else:
bridge_interface = netif.get_interface(netif.create_interface('bridge'))
bridge_interface.rename('brg{0}'.format(len(available_bridges)))
bridge_interface.description = 'OpenVPN bridge interface'
bridge_interface.up()
bridge_interface.add_member(default_interface)
bridge_interface.add_member(vpn_interface_name)
if node['server_bridge_extended']:
subnet = ipaddress.ip_interface('{0}/{1}'.format(node['server_bridge_ip'],
node['server_bridge_netmask']))
bridge_interface.add_address(netif.InterfaceAddress(
netif.AddressFamily.INET,
subnet
))
