def validate(self, value):
verrors = ValidationErrors()
for attr in self.attrs.values():
if attr.name in value:
try:
attr.validate(value[attr.name])
except ValidationErrors as e:
verrors.add_child(self.name, e)
for v in value:
if v not in Cron.FIELDS:
verrors.add(self.name, f'Unexpected {v} value')
if verrors:
raise verrors
cron_expression = ''
for field in Cron.FIELDS:
cron_expression += value.get(field) + ' ' if value.get(field) else '* '
try:
croniter(cron_expression)
except Exception as e:
verrors.add(self.name, 'Please ensure fields match cron syntax - ' + str(e))
if verrors:
raise verrors
def validate(self, value):
if value is None:
return
verrors = ValidationErrors()
if value:
try:
if self.network:
self.factory(value, strict=self.network_strict)
else:
if self.cidr and '/' not in value:
raise ValueError(
'Specified address should be in CIDR notation, e.g. 192.168.0.2/24'
)
has_zone_index = False
if self.allow_zone_index and "%" in value:
has_zone_index = True
value = value[:value.rindex("%")]
addr = self.factory(value)
if has_zone_index and not isinstance(addr, ipaddress.IPv6Address):
raise ValueError("Zone index is allowed only for IPv6 addresses")
except ValueError as e:
verrors.add(self.name, str(e), errno.EINVAL)
if verrors:
raise verrors
return super().validate(value)
def validate(self, value):
verrors = ValidationErrors()
for validator in self.validators:
try:
validator(value)
except ShouldBe as e:
verrors.add(self.name, f"Should be {e.what}")
if verrors:
raise verrors
def validate(self, value):
verrors = ValidationErrors()
for validator in self.validators:
try:
validator(value)
except ValueError as e:
verrors.add(self.name, str(e))
if verrors:
raise verrors
def validate(self, value):
verrors = ValidationErrors()
if value:
if not os.path.exists(value):
verrors.add(self.name, "This path does not exist.", errno.ENOENT)
elif not os.path.isfile(value):
verrors.add(self.name, "This path is not a file.", errno.EISDIR)
if verrors:
raise verrors
return super().validate(value)
def validate(self, value):
if value is None:
return
verrors = ValidationErrors()
if value:
if not os.path.exists(value):
verrors.add(self.name, "This path does not exist.", errno.ENOENT)
elif not os.path.isdir(value):
verrors.add(self.name, "This path is not a directory.", errno.ENOTDIR)
if verrors:
raise verrors
return super().validate(value)
def validate(self, value):
if value is None:
return
verrors = ValidationErrors()
s = set()
for i, v in enumerate(value):
if self.unique:
if isinstance(v, dict):
v = tuple(sorted(list(v.items())))
if v in s:
verrors.add(f"{self.name}.{i}", "This value is not unique.")
s.add(v)
for attr in self.items:
try:
attr.validate(v)
except ValidationErrors as e:
verrors.add_child(f"{self.name}.{i}", e)
if verrors:
raise verrors
super().validate(value)
def validate(self, value):
verrors = ValidationErrors()
if value:
try:
if self.cidr:
self.factory(value, strict=self.cidr_strict)
else:
has_zone_index = False
if self.allow_zone_index and "%" in value:
has_zone_index = True
value = value[:value.rindex("%")]
addr = self.factory(value)
if has_zone_index and not isinstance(addr, ipaddress.IPv6Address):
raise ValueError("Zone index is allowed only for IPv6 addresses")
except ValueError as e:
verrors.add(self.name, str(e), errno.EINVAL)
if verrors:
raise verrors
return super().validate(value)
def ping_remote(self, options):
"""
Method that will send an ICMP echo request to "hostname"
and will wait up to "timeout" for a reply.
"""
ip = None
ip_found = True
verrors = ValidationErrors()
try:
ip = IpAddress()
ip(options['hostname'])
ip = options['hostname']
except ValueError:
ip_found = False
if not ip_found:
try:
if options['type'] == 'ICMP':
ip = socket.getaddrinfo(options['hostname'], None)[0][4][0]
elif options['type'] == 'ICMPV4':
ip = socket.getaddrinfo(options['hostname'], None,
socket.AF_INET)[0][4][0]
elif options['type'] == 'ICMPV6':
ip = socket.getaddrinfo(options['hostname'], None,
socket.AF_INET6)[0][4][0]
except socket.gaierror:
verrors.add(
'options.hostname',
f'{options["hostname"]} cannot be resolved to an IP address.'
)
verrors.check()
addr = ipaddress.ip_address(ip)
if not addr.version == 4 and (options['type'] == 'ICMP'
or options['type'] == 'ICMPV4'):
verrors.add(
'options.type',
f'Requested ICMPv4 protocol, but the address provided "{addr}" is not a valid IPv4 address.'
)
if not addr.version == 6 and options['type'] == 'ICMPV6':
verrors.add(
'options.type',
f'Requested ICMPv6 protocol, but the address provided "{addr}" is not a valid IPv6 address.'
)
verrors.check()
ping_host = False
if addr.version == 4:
ping_host = self._ping_host(ip, options['timeout'])
elif addr.version == 6:
ping_host = self._ping6_host(ip, options['timeout'])
return ping_host
async def check_disks_availability(self, disks, allow_duplicate_serials):
"""
Makes sure the disks are present in the system and not reserved
by anything else (boot, pool, iscsi, etc).
Returns:
verrors, dict - validation errors (if any) and disk.query for all disks
"""
verrors = ValidationErrors()
disks_cache = dict(map(lambda x: (x['devname'], x), await self.middleware.call('disk.query')))
disks_set = set(disks)
disks_not_in_cache = disks_set - set(disks_cache.keys())
if disks_not_in_cache:
verrors.add(
'topology',
f'The following disks were not found in system: {"," .join(disks_not_in_cache)}.'
)
disks_reserved = await self.middleware.call('disk.get_reserved')
already_used = disks_set - (disks_set - set(disks_reserved))
if already_used:
verrors.add(
'topology',
f'The following disks are already in use: {"," .join(already_used)}.'
)
if not allow_duplicate_serials and not verrors:
serial_to_disk = defaultdict(set)
for disk in disks:
serial_to_disk[(disks_cache[disk]['serial'], disks_cache[disk]['lunid'])].add(disk)
for reserved_disk in disks_reserved:
reserved_disk_cache = disks_cache.get(reserved_disk)
if not reserved_disk_cache:
continue
serial_to_disk[(reserved_disk_cache['serial'], reserved_disk_cache['lunid'])].add(reserved_disk)
if duplicate_serials := {serial for serial, serial_disks in serial_to_disk.items()
if len(serial_disks) > 1}:
error = ', '.join(map(lambda serial: f'{serial[0]!r} ({", ".join(sorted(serial_to_disk[serial]))})',
duplicate_serials))
verrors.add('topology', f'Disks have duplicate serial numbers: {error}.')
def validate(self, value):
if value is None:
return
verrors = ValidationErrors()
for attr in self.attrs.values():
if attr.name in value:
try:
attr.validate(value[attr.name])
except ValidationErrors as e:
verrors.add_child(self.name, e)
for v in value:
if self.begin_end and v in ['begin', 'end']:
continue
if v not in Cron.FIELDS:
verrors.add(self.name, f'Unexpected {v} value')
if verrors:
raise verrors
cron_expression = ''
for field in Cron.FIELDS:
cron_expression += value.get(field) + ' ' if value.get(
field) else '* '
try:
croniter(cron_expression)
except Exception as e:
verrors.add(self.name,
'Please ensure fields match cron syntax - ' + str(e))
if value.get('begin') and value.get('end') and not (
value.get('begin') <= value.get('end')):
verrors.add(self.name,
'Begin time should be less or equal than end time')
if verrors:
raise verrors
def validate(self, value):
if value is None:
return
verrors = ValidationErrors()
for attr in self.attrs.values():
if attr.name in value:
try:
attr.validate(value[attr.name])
except ValidationErrors as e:
verrors.add_child(self.name, e)
for v in value:
if self.begin_end and v in ['begin', 'end']:
continue
if v not in Cron.FIELDS:
verrors.add(self.name, f'Unexpected {v} value')
if verrors:
raise verrors
cron_expression = ''
for field in Cron.FIELDS:
cron_expression += value.get(field) + ' ' if value.get(field) else '* '
try:
croniter(cron_expression)
except Exception as e:
verrors.add(self.name, 'Please ensure fields match cron syntax - ' + str(e))
if value.get('begin') and value.get('end') and not (value.get('begin') <= value.get('end')):
verrors.add(self.name, 'Begin time should be less or equal than end time')
if verrors:
raise verrors
def fstab(self, jail, options):
"""Adds an fstab mount to the jail"""
uuid, _, iocage = self.check_jail_existence(jail, skip=False)
status, jid = IOCList.list_get_jid(uuid)
action = options['action'].lower()
if status and action != 'list':
raise CallError(
f'{jail} should not be running when adding a mountpoint')
verrors = ValidationErrors()
source = options.get('source')
if source:
if not os.path.exists(source):
verrors.add(
'options.source',
'Provided path for source does not exist'
)
source = source.replace(' ', r'\040') # fstab hates spaces ;)
destination = options.get('destination')
if destination:
destination = f'/{destination}' if destination[0] != '/' else destination
dst = f'{self.get_iocroot()}/jails/{jail}/root'
if dst not in destination:
destination = f'{dst}{destination}'
if os.path.exists(destination):
if not os.path.isdir(destination):
verrors.add(
'options.destination',
'Destination is not a directory, please provide a valid destination'
)
elif os.listdir(destination):
verrors.add(
'options.destination',
'Destination directory should be empty'
)
else:
os.makedirs(destination)
# fstab hates spaces ;)
destination = destination.replace(' ', r'\040')
if action != 'list':
for f in options:
if not options.get(f) and f not in ('index',):
verrors.add(
f'options.{f}',
'This field is required'
)
fstype = options.get('fstype')
fsoptions = options.get('fsoptions')
dump = options.get('dump')
_pass = options.get('pass')
index = options.get('index')
if action == 'replace' and index is None:
verrors.add(
'options.index',
'Index must not be None when replacing fstab entry'
)
if verrors:
raise verrors
_list = iocage.fstab(action, source, destination, fstype, fsoptions,
dump, _pass, index=index)
if action == "list":
split_list = {}
system_mounts = (
'/root/bin', '/root/boot', '/root/lib', '/root/libexec',
'/root/rescue', '/root/sbin', '/root/usr/bin',
'/root/usr/include', '/root/usr/lib', '/root/usr/libexec',
'/root/usr/sbin', '/root/usr/share', '/root/usr/libdata',
'/root/usr/lib32'
)
for i in _list:
fstab_entry = i[1].split()
_fstab_type = 'SYSTEM' if fstab_entry[0].endswith(
system_mounts) else 'USER'
split_list[i[0]] = {'entry': fstab_entry, 'type': _fstab_type}
return split_list
return True
def create_job(self, job, options):
verrors = ValidationErrors()
uuid = options["uuid"]
job.set_progress(0, f'Creating: {uuid}')
try:
self.check_jail_existence(uuid, skip=False)
verrors.add(
'uuid',
f'A jail with name {uuid} already exists'
)
raise verrors
except CallError:
# A jail does not exist with the provided name, we can create one
# now
verrors = self.common_validation(verrors, options)
if verrors:
raise verrors
job.set_progress(20, 'Initial validation complete')
iocage = ioc.IOCage(skip_jails=True)
release = options["release"]
template = options.get("template", False)
pkglist = options.get("pkglist", None)
basejail = options["basejail"]
empty = options["empty"]
short = options["short"]
props = options["props"]
pool = IOCJson().json_get_value("pool")
iocroot = IOCJson(pool).json_get_value("iocroot")
if template:
release = template
if (
not os.path.isdir(f'{iocroot}/releases/{release}') and
not template and
not empty
):
job.set_progress(50, f'{release} missing, calling fetch')
self.middleware.call_sync(
'jail.fetch', {"release": release}, job=True
)
err, msg = iocage.create(
release,
props,
0,
pkglist,
template=template,
short=short,
_uuid=uuid,
basejail=basejail,
empty=empty
)
if err:
raise CallError(msg)
job.set_progress(100, f'Created: {uuid}')
return True
async def do_update(self, id, data):
"""
Update disk of `id`.
If extra options need to be passed to SMART which we don't already support, they can be passed by
`smartoptions`.
`critical`, `informational` and `difference` are integer values on which alerts for SMART are configured
if the disk temperature crosses the assigned threshold for each respective attribute.
If they are set to null, then SMARTD config values are used as defaults.
Email of log level LOG_CRIT is issued when disk temperature crosses `critical`.
Email of log level LOG_INFO is issued when disk temperature crosses `informational`.
If temperature of a disk changes by `difference` degree Celsius since the last report, SMART reports this.
"""
old = await self.middleware.call(
'datastore.query', 'storage.disk', [['identifier', '=', id]], {
'get': True, 'prefix': self._config.datastore_prefix
}
)
old.pop('enabled', None)
self._expand_enclosure(old)
new = old.copy()
new.update(data)
verrors = ValidationErrors()
if new['hddstandby_force']:
if new['hddstandby'] == 'ALWAYS ON':
verrors.add(
'disk_update.hddstandby_force',
'This option does not have sense when HDD Standby is not set'
)
if verrors:
raise verrors
if not new['passwd'] and old['passwd'] != new['passwd']:
# We want to make sure kmip uid is None in this case
if new['kmip_uid']:
asyncio.ensure_future(self.middleware.call('kmip.reset_sed_disk_password', id, new['kmip_uid']))
new['kmip_uid'] = None
for key in ['acousticlevel', 'advpowermgmt', 'hddstandby']:
new[key] = new[key].title()
self._compress_enclosure(new)
await self.middleware.call(
'datastore.update',
self._config.datastore,
id,
new,
{'prefix': self._config.datastore_prefix}
)
if any(new[key] != old[key] for key in ['hddstandby', 'advpowermgmt', 'acousticlevel']):
await self.middleware.call('disk.power_management', new['name'])
if any(
new[key] != old[key]
for key in [
'togglesmart', 'smartoptions', 'hddstandby', 'hddstandby_force',
'critical', 'difference', 'informational',
]
):
if new['togglesmart']:
await self.middleware.call('disk.toggle_smart_on', new['name'])
else:
await self.middleware.call('disk.toggle_smart_off', new['name'])
await self.middleware.call('disk.update_hddstandby_force')
await self.middleware.call('disk.update_smartctl_args_for_disks')
await self.middleware.call('service.restart', 'collectd')
await self._service_change('smartd', 'restart')
await self._service_change('snmp', 'restart')
if new['passwd'] and old['passwd'] != new['passwd']:
await self.middleware.call('kmip.sync_sed_keys', [id])
return await self.query([['identifier', '=', id]], {'get': True})
def common_validation(middleware,
options,
update=False,
jail=None,
schema='options'):
verrors = ValidationErrors()
if not update:
# Ensure that api call conforms to format set by iocage for props
# Example 'key=value'
for value in options['props']:
if '=' not in value:
verrors.add(
f'{schema}.props',
'Please follow the format specified by iocage for api calls'
'e.g "key=value"')
break
if verrors:
raise verrors
# normalise vnet mac address
# expected format here is 'vnet0_mac=00-D0-56-F2-B5-12,00-D0-56-F2-B5-13'
vnet_macs = {
f.split('=')[0]: f.split('=')[1]
for f in options['props']
if any(f'vnet{i}_mac' in f.split('=')[0] for i in range(0, 4))
}
validate_ips(middleware, verrors, options, schema=f'{schema}.props')
else:
vnet_macs = {
key: value
for key, value in options.items()
if any(f'vnet{i}_mac' in key for i in range(0, 4))
}
exclude_ips = [
ip.split('|')[1].split('/')[0] if '|' in ip else ip.split('/')[0]
for f in ('ip4_addr', 'ip6_addr') for ip in jail[f].split(',')
if ip not in ('none', 'DHCP (not running)')
]
validate_ips(middleware, verrors,
{'props': [f'{k}={v}' for k, v in options.items()]},
schema, exclude_ips)
# validate vnetX_mac addresses
for key, value in vnet_macs.items():
if value and value != 'none':
value = value.replace(',', ' ')
try:
for mac in value.split():
MACAddr()(mac)
if len(value.split()) != 2 or any(value.split().count(v) > 1
for v in value.split()):
raise ValueError('Exception')
except ValueError:
verrors.add(
f'{schema}.{key}', 'Please enter two valid and different '
f'space/comma-delimited MAC addresses for {key}.')
if options.get('uuid'):
valid = True if re.match(r"^[a-zA-Z0-9\._-]+$",
options['uuid']) else False
if not valid:
verrors.add(
f'{schema}.uuid', f'Invalid character in {options["uuid"]}. '
'Alphanumeric, period (.), underscore (_), '
'and dash (-) characters are allowed.')
return verrors
def fstab(self, jail, options):
"""Manipulate a jails fstab"""
uuid, _, iocage = self.check_jail_existence(jail, skip=False)
status, jid = IOCList.list_get_jid(uuid)
action = options['action'].lower()
index = options.get('index')
if status and action != 'list':
raise CallError(
f'{jail} should not be running when adding a mountpoint')
verrors = ValidationErrors()
if action in ('add', 'replace', 'remove'):
if action != 'remove' or index is None:
# For remove we allow removing by index or mount, so if index is not specified
# we should validate that rest of the fields exist.
for f in ('source', 'destination', 'fstype', 'fsoptions',
'dump', 'pass'):
if not options.get(f):
verrors.add(
f'options.{f}',
f'This field is required with "{action}" action.')
if action == 'replace' and index is None:
verrors.add(
'options.index',
'Index cannot be "None" when replacing an fstab entry.')
verrors.check()
source = options.get('source')
if action in ('add', 'replace') and not os.path.exists(source):
verrors.add('options.source',
'The provided path for the source does not exist.')
destination = options.get('destination')
if destination:
destination = f'/{destination}' if destination[0] != '/' else \
destination
dst = f'{self.get_iocroot()}/jails/{jail}/root'
if dst not in destination:
destination = f'{dst}{destination}'
if os.path.exists(destination):
if not os.path.isdir(destination):
verrors.add(
'options.destination',
'Destination is not a directory. Please provide a '
'empty directory for the destination.')
elif os.listdir(destination):
verrors.add('options.destination',
'Destination directory must be empty.')
else:
os.makedirs(destination)
# Setup defaults for library
source = source or ''
destination = destination or ''
fstype = options.get('fstype')
fsoptions = options.get('fsoptions')
dump = options.get('dump')
_pass = options.get('pass')
if verrors:
raise verrors
try:
_list = iocage.fstab(action,
source,
destination,
fstype,
fsoptions,
dump,
_pass,
index=index)
except ioc_exceptions.ValidationFailed as e:
# CallError uses strings, the exception message may not always be a
# list.
if not isinstance(e.message, str) and isinstance(
e.message, Iterable):
e.message = '\n'.join(e.message)
self.logger.error(f'{e!r}')
raise CallError(e.message)
if action == "list":
split_list = {}
system_mounts = ('/root/bin', '/root/boot', '/root/lib',
'/root/libexec', '/root/rescue', '/root/sbin',
'/root/usr/bin', '/root/usr/include',
'/root/usr/lib', '/root/usr/libexec',
'/root/usr/sbin', '/root/usr/share',
'/root/usr/libdata', '/root/usr/lib32')
for i in _list:
fstab_entry = i[1]
_fstab_type = 'SYSTEM' if fstab_entry[0].endswith(
system_mounts) else 'USER'
split_list[i[0]] = {'entry': fstab_entry, 'type': _fstab_type}
return split_list
return True
def create_job(self, job, options):
verrors = ValidationErrors()
uuid = options["uuid"]
job.set_progress(0, f'Creating: {uuid}')
try:
self.check_jail_existence(uuid, skip=False)
verrors.add('uuid', f'A jail with name {uuid} already exists')
raise verrors
except CallError:
# A jail does not exist with the provided name, we can create one
# now
verrors = self.common_validation(verrors, options)
if verrors:
raise verrors
job.set_progress(20, 'Initial validation complete')
if not any('resolver' in p for p in options['props']):
dc = self.middleware.call_sync(
'service.query', [('service', '=', 'domaincontroller')])[0]
dc_config = self.middleware.call_sync('domaincontroller.config')
if dc['enable'] and (dc_config['dns_forwarder'] and
dc_config['dns_backend'] == 'SAMBA_INTERNAL'):
options['props'].append(
f'resolver=nameserver {dc_config["dns_forwarder"]}')
iocage = ioc.IOCage(skip_jails=True)
release = options["release"]
template = options.get("template", False)
pkglist = options.get("pkglist", None)
basejail = options["basejail"]
empty = options["empty"]
short = options["short"]
props = options["props"]
pool = IOCJson().json_get_value("pool")
iocroot = IOCJson(pool).json_get_value("iocroot")
if template:
release = template
if (not os.path.isdir(f'{iocroot}/releases/{release}') and not template
and not empty):
job.set_progress(50, f'{release} missing, calling fetch')
self.middleware.call_sync('jail.fetch', {"release": release},
job=True)
err, msg = iocage.create(release,
props,
0,
pkglist,
template=template,
short=short,
_uuid=uuid,
basejail=basejail,
empty=empty)
if err:
raise CallError(msg)
job.set_progress(100, f'Created: {uuid}')
return True