def common_validation(self, verrors, options, update=False, jail=None):
if not update:
# Ensure that api call conforms to format set by iocage for props
# Example 'key=value'
for value in options['props']:
if '=' not in value:
verrors.add(
'options.props',
'Please follow the format specified by iocage for api calls'
'e.g "key=value"'
)
break
if verrors:
raise verrors
# normalise vnet mac address
# expected format here is 'vnet0_mac=00-D0-56-F2-B5-12,00-D0-56-F2-B5-13'
vnet_macs = {
f.split('=')[0]: f.split('=')[1] for f in options['props']
if any(f'vnet{i}_mac' in f.split('=')[0] for i in range(0, 4))
}
self.validate_ips(verrors, options)
else:
vnet_macs = {
key: value for key, value in options.items()
if any(f'vnet{i}_mac' in key for i in range(0, 4))
}
exclude_ips = [
ip.split('|')[1].split('/')[0] if '|' in ip else ip.split('/')[0]
for f in ('ip4_addr', 'ip6_addr') for ip in jail[f].split(',')
if ip not in ('none', 'DHCP (not running)')
]
self.validate_ips(
verrors, {'props': [f'{k}={v}' for k, v in options.items()]},
'options', exclude_ips
)
# validate vnetX_mac addresses
for key, value in vnet_macs.items():
if value and value != 'none':
value = value.replace(',', ' ')
try:
for mac in value.split():
MACAddr()(mac)
if (
len(value.split()) != 2 or
any(value.split().count(v) > 1 for v in value.split())
):
raise ValueError('Exception')
except ValueError:
verrors.add(
key,
'Please Enter two valid and different '
f'space/comma-delimited MAC addresses for {key}.'
)
return verrors
def common_validation(middleware,
options,
update=False,
jail=None,
schema='options'):
verrors = ValidationErrors()
if not update:
# Ensure that api call conforms to format set by iocage for props
# Example 'key=value'
for value in options['props']:
if '=' not in value:
verrors.add(
f'{schema}.props',
'Please follow the format specified by iocage for api calls'
'e.g "key=value"')
break
if verrors:
raise verrors
# normalise vnet mac address
# expected format here is 'vnet0_mac=00-D0-56-F2-B5-12,00-D0-56-F2-B5-13'
vnet_macs = {
f.split('=')[0]: f.split('=')[1]
for f in options['props']
if any(f'vnet{i}_mac' in f.split('=')[0] for i in range(0, 4))
}
validate_ips(middleware, verrors, options, schema=f'{schema}.props')
else:
vnet_macs = {
key: value
for key, value in options.items()
if any(f'vnet{i}_mac' in key for i in range(0, 4))
}
exclude_ips = [
ip.split('|')[1].split('/')[0] if '|' in ip else ip.split('/')[0]
for f in ('ip4_addr', 'ip6_addr') for ip in jail[f].split(',')
if ip not in ('none', 'DHCP (not running)')
]
validate_ips(middleware, verrors,
{'props': [f'{k}={v}' for k, v in options.items()]},
schema, exclude_ips)
# validate vnetX_mac addresses
for key, value in vnet_macs.items():
if value and value != 'none':
value = value.replace(',', ' ')
try:
for mac in value.split():
MACAddr()(mac)
if len(value.split()) != 2 or any(value.split().count(v) > 1
for v in value.split()):
raise ValueError('Exception')
except ValueError:
verrors.add(
f'{schema}.{key}', 'Please enter two valid and different '
f'space/comma-delimited MAC addresses for {key}.')
if options.get('uuid'):
valid = True if re.match(r"^[a-zA-Z0-9\._-]+$",
options['uuid']) else False
if not valid:
verrors.add(
f'{schema}.uuid', f'Invalid character in {options["uuid"]}. '
'Alphanumeric, period (.), underscore (_), '
'and dash (-) characters are allowed.')
return verrors
class NIC(Device):
schema = Dict(
'attributes',
Str('type', enum=['E1000', 'VIRTIO'], default='E1000'),
Str('nic_attach', default=None, null=True),
Str('mac',
default=None,
null=True,
validators=[MACAddr(separator=':')]),
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.bridge = self.bridge_created = self.nic_attach = None
@staticmethod
def random_mac():
mac_address = [
0x00, 0xa0, 0x98,
random.randint(0x00, 0x7f),
random.randint(0x00, 0xff),
random.randint(0x00, 0xff)
]
return ':'.join(['%02x' % x for x in mac_address])
def setup_nic_attach(self):
nic_attach = self.data['attributes'].get('nic_attach')
interfaces = netif.list_interfaces()
if nic_attach and nic_attach not in interfaces:
raise CallError(f'{nic_attach} not found.')
else:
if not nic_attach:
try:
nic_attach = netif.RoutingTable(
).default_route_ipv4.interface
nic = netif.get_interface(nic_attach)
except Exception as e:
raise CallError(
f'Unable to retrieve default interface: {e}')
else:
nic = netif.get_interface(nic_attach)
if netif.InterfaceFlags.UP not in nic.flags:
nic.up()
self.nic_attach = nic.name
def pre_start_vm_freebsd(self, *args, **kwargs):
self.setup_nic_attach()
interfaces = netif.list_interfaces()
bridge = None
if self.nic_attach.startswith('bridge'):
bridge = interfaces[self.nic_attach]
if not bridge:
for iface in filter(lambda v: v.startswith('bridge'), interfaces):
if self.nic_attach in interfaces[iface].members:
bridge = interfaces[iface]
break
else:
bridge = netif.get_interface(netif.create_interface('bridge'))
bridge.add_member(self.nic_attach)
self.bridge_created = True
if netif.InterfaceFlags.UP not in bridge.flags:
bridge.up()
self.bridge = bridge.name
def pre_start_vm_rollback_freebsd(self, *args, **kwargs):
if self.bridge_created and self.bridge in netif.list_interfaces():
netif.destroy_interface(self.bridge)
self.bridge = self.bridge_created = None
def xml_children(self):
return [
create_element('model',
type='virtio' if self.data['attributes']['type']
== 'VIRTIO' else 'e1000'),
create_element(
'mac',
address=self.data['attributes']['mac']
if self.data['attributes'].get('mac') else self.random_mac()),
]
def xml_linux(self, *args, **kwargs):
self.setup_nic_attach()
if self.nic_attach.startswith('br'):
return create_element(
'interface',
type='bridge',
attribute_dict={
'children':
[create_element('source', bridge=self.nic_attach)] +
self.xml_children()
})
else:
return create_element(
'interface',
type='direct',
attribute_dict={
'children': [
create_element(
'source', dev=self.nic_attach, mode='bridge')
] + self.xml_children()
})
def xml_freebsd(self, *args, **kwargs):
return create_element(
'interface',
type='bridge',
attribute_dict={
'children': [
create_element('source', bridge=self.bridge or ''),
create_element(
'address', type='pci', slot=str(kwargs['slot'])),
] + self.xml_children()
})
class VMService(Service):
@accepts(Int('id'))
@returns(List(items=[Ref('vm_device_entry')]))
async def get_display_devices(self, id):
"""
Get the display devices from a given guest. If a display device has password configured,
`attributes.password_configured` will be set to `true`.
"""
devices = []
for device in await self.middleware.call(
'vm.device.query',
[['vm', '=', id], ['dtype', '=', 'DISPLAY']]):
device['attributes']['password_configured'] = bool(
device['attributes'].get('password'))
devices.append(device)
return devices
@accepts()
@returns(
Dict(
'available_display_port',
Int('port', required=True, description='Available server port'),
Int('web',
required=True,
description='Web port to be used based on available `port`'),
))
async def port_wizard(self):
"""
It returns the next available Display Server Port and Web Port.
Returns a dict with two keys `port` and `web`.
"""
all_ports = [
d['attributes'].get('port') for d in (await self.middleware.call(
'vm.device.query', [['dtype', '=', 'DISPLAY']]))
] + [6000, 6100]
port = next((i for i in range(5900, 65535) if i not in all_ports))
return {'port': port, 'web': DISPLAY.get_web_port(port)}
@accepts()
@returns(
Dict(
'vmemory_in_use',
Int('RNP',
required=True,
description='Running but not provisioned'),
Int('PRD',
required=True,
description='Provisioned but not running'),
Int('RPRD', required=True, description='Running and provisioned'),
))
async def get_vmemory_in_use(self):
"""
The total amount of virtual memory in MB used by guests
Returns a dict with the following information:
RNP - Running but not provisioned
PRD - Provisioned but not running
RPRD - Running and provisioned
"""
memory_allocation = {'RNP': 0, 'PRD': 0, 'RPRD': 0}
guests = await self.middleware.call('datastore.query', 'vm.vm')
for guest in guests:
status = await self.middleware.call('vm.status', guest['id'])
if status['state'] == 'RUNNING' and guest['autostart'] is False:
memory_allocation['RNP'] += guest['memory'] * 1024 * 1024
elif status['state'] == 'RUNNING' and guest['autostart'] is True:
memory_allocation['RPRD'] += guest['memory'] * 1024 * 1024
elif guest['autostart']:
memory_allocation['PRD'] += guest['memory'] * 1024 * 1024
return memory_allocation
@accepts(Bool('overcommit', default=False))
@returns(Int('available_memory'))
async def get_available_memory(self, overcommit):
"""
Get the current maximum amount of available memory to be allocated for VMs.
If `overcommit` is true only the current used memory of running VMs will be accounted for.
If false all memory (including unused) of runnings VMs will be accounted for.
This will include memory shrinking ZFS ARC to the minimum.
Memory is of course a very "volatile" resource, values may change abruptly between a
second but I deem it good enough to give the user a clue about how much memory is
available at the current moment and if a VM should be allowed to be launched.
"""
# Use 90% of available memory to play safe
free = int(psutil.virtual_memory().available * 0.9)
# swap used space is accounted for used physical memory because
# 1. processes (including VMs) can be swapped out
# 2. we want to avoid using swap
swap_used = psutil.swap_memory().used
# Difference between current ARC total size and the minimum allowed
arc_total = await self.middleware.call('sysctl.get_arcstats_size')
arc_min = await self.middleware.call('sysctl.get_arc_min')
arc_shrink = max(0, arc_total - arc_min)
vms_memory_used = 0
if overcommit is False:
# If overcommit is not wanted its verified how much physical memory
# the vm process is currently using and add the maximum memory its
# supposed to have.
for vm in await self.middleware.call('vm.query'):
if vm['status']['state'] == 'RUNNING':
try:
vms_memory_used += await self.middleware.call(
'vm.get_memory_usage_internal', vm)
except Exception:
self.logger.error(
'Unable to retrieve %r vm memory usage',
vm['name'],
exc_info=True)
continue
return max(0, free + arc_shrink - vms_memory_used - swap_used)
@accepts()
@returns(
Str('mac', validators=[MACAddr(separator=':')]), )
def random_mac(self):
"""
Create a random mac address.
Returns:
str: with six groups of two hexadecimal digits
"""
return NIC.random_mac()
@accepts(Int('id'), Str('host', default=''),
Dict(
'options',
List('devices_passwords',
items=[
Dict('device_password',
Int('device_id', required=True),
Str('password', required=True, empty=False))
])))
@returns(Dict('display_devices_uri', additional_attrs=True))
@pass_app()
async def get_display_web_uri(self, app, id, host, options):
"""
Retrieve Display URI's for a given VM.
Display devices which have a password configured must specify the password explicitly to retrieve display
device web uri. In case a password is not specified, the uri for display device in question will not be
retrieved because of missing password information.
"""
web_uris = {}
host = host or await self.middleware.call(
'interface.websocket_local_ip', app=app)
try:
ipaddress.IPv6Address(host)
except ipaddress.AddressValueError:
pass
else:
host = f'[{host}]'
creds = {
d['device_id']: d['password']
for d in options['devices_passwords']
}
for device in map(lambda d: DISPLAY(d, middleware=self.middleware),
await self.get_display_devices(id)):
uri_data = {'error': None, 'uri': None}
if device.data['attributes'].get('web'):
if device.password_configured():
if creds.get(device.data['id']) and creds[device.data[
'id']] != device.data['attributes']['password']:
uri_data['error'] = 'Incorrect password specified'
elif not creds.get(device.data['id']):
uri_data['error'] = 'Password not specified'
uri_data['uri'] = device.web_uri(host,
creds.get(device.data['id']))
else:
uri_data['error'] = 'Web display is not configured'
web_uris[device.data['id']] = uri_data
return web_uris
@accepts()
@returns(Dict(*[Str(r, enum=[r]) for r in DISPLAY.RESOLUTION_ENUM]))
async def resolution_choices(self):
"""
Retrieve supported resolution choices for VM Display devices.
"""
return {r: r for r in DISPLAY.RESOLUTION_ENUM}
class VMService(Service):
@accepts()
@returns(
Dict(
'vmemory_in_use',
Int('RNP',
required=True,
description='Running but not provisioned'),
Int('PRD',
required=True,
description='Provisioned but not running'),
Int('RPRD', required=True, description='Running and provisioned'),
))
async def get_vmemory_in_use(self):
"""
The total amount of virtual memory in MB used by guests
Returns a dict with the following information:
RNP - Running but not provisioned
PRD - Provisioned but not running
RPRD - Running and provisioned
"""
memory_allocation = {'RNP': 0, 'PRD': 0, 'RPRD': 0}
guests = await self.middleware.call('datastore.query', 'vm.vm')
for guest in guests:
status = await self.middleware.call('vm.status', guest['id'])
if status['state'] == 'RUNNING' and guest['autostart'] is False:
memory_allocation['RNP'] += guest['memory'] * 1024 * 1024
elif status['state'] == 'RUNNING' and guest['autostart'] is True:
memory_allocation['RPRD'] += guest['memory'] * 1024 * 1024
elif guest['autostart']:
memory_allocation['PRD'] += guest['memory'] * 1024 * 1024
return memory_allocation
@accepts(Bool('overcommit', default=False))
@returns(Int('available_memory'))
async def get_available_memory(self, overcommit):
"""
Get the current maximum amount of available memory to be allocated for VMs.
If `overcommit` is true only the current used memory of running VMs will be accounted for.
If false all memory (including unused) of runnings VMs will be accounted for.
This will include memory shrinking ZFS ARC to the minimum.
Memory is of course a very "volatile" resource, values may change abruptly between a
second but I deem it good enough to give the user a clue about how much memory is
available at the current moment and if a VM should be allowed to be launched.
"""
# Use 90% of available memory to play safe
free = int(psutil.virtual_memory().available * 0.9)
# swap used space is accounted for used physical memory because
# 1. processes (including VMs) can be swapped out
# 2. we want to avoid using swap
swap_used = psutil.swap_memory().used
# Difference between current ARC total size and the minimum allowed
arc_total = await self.middleware.call('sysctl.get_arcstats_size')
arc_min = await self.middleware.call('sysctl.get_arc_min')
arc_shrink = max(0, arc_total - arc_min)
vms_memory_used = 0
if overcommit is False:
# If overcommit is not wanted its verified how much physical memory
# the vm process is currently using and add the maximum memory its
# supposed to have.
for vm in await self.middleware.call('vm.query'):
if vm['status']['state'] == 'RUNNING':
try:
vms_memory_used += await self.middleware.call(
'vm.get_memory_usage_internal', vm)
except Exception:
self.logger.error(
'Unable to retrieve %r vm memory usage',
vm['name'],
exc_info=True)
continue
return max(0, free + arc_shrink - vms_memory_used - swap_used)
@accepts()
@returns(
Str('mac', validators=[MACAddr(separator=':')]), )
def random_mac(self):
"""
Create a random mac address.
Returns:
str: with six groups of two hexadecimal digits
"""
return NIC.random_mac()
class NIC(Device):
schema = Dict(
'attributes',
Str('type', enum=['E1000', 'VIRTIO'], default='E1000'),
Str('nic_attach', default=None, null=True),
Str('mac', default=None, null=True, validators=[MACAddr(separator=':')]),
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.bridge = self.bridge_created = self.nic_attach = None
def identity(self):
nic_attach = self.data['attributes'].get('nic_attach')
if not nic_attach:
nic_attach = netif.RoutingTable().default_route_ipv4.interface
return nic_attach
def is_available(self):
return self.identity() in netif.list_interfaces()
@staticmethod
def random_mac():
mac_address = [
0x00, 0xa0, 0x98, random.randint(0x00, 0x7f), random.randint(0x00, 0xff), random.randint(0x00, 0xff)
]
return ':'.join(['%02x' % x for x in mac_address])
def setup_nic_attach(self):
nic_attach = self.data['attributes'].get('nic_attach')
interfaces = netif.list_interfaces()
if nic_attach and nic_attach not in interfaces:
raise CallError(f'{nic_attach} not found.')
else:
if not nic_attach:
try:
nic_attach = netif.RoutingTable().default_route_ipv4.interface
nic = netif.get_interface(nic_attach)
except Exception as e:
raise CallError(f'Unable to retrieve default interface: {e}')
else:
nic = netif.get_interface(nic_attach)
if netif.InterfaceFlags.UP not in nic.flags:
nic.up()
self.nic_attach = nic.name
def xml_children(self):
return [
create_element('model', type='virtio' if self.data['attributes']['type'] == 'VIRTIO' else 'e1000'),
create_element(
'mac', address=self.data['attributes']['mac'] if
self.data['attributes'].get('mac') else self.random_mac()
),
]
def xml_linux(self, *args, **kwargs):
self.setup_nic_attach()
if self.nic_attach.startswith('br'):
return create_element(
'interface', type='bridge', attribute_dict={
'children': [
create_element('source', bridge=self.nic_attach)
] + self.xml_children()
}
)
else:
return create_element(
'interface', type='direct', attribute_dict={
'children': [
create_element('source', dev=self.nic_attach, mode='bridge')
] + self.xml_children()
}
)