class PoolResilverService(ConfigService):
class Config:
namespace = 'pool.resilver'
datastore = 'storage.resilver'
datastore_extend = 'pool.resilver.resilver_extend'
async def resilver_extend(self, data):
data['begin'] = data['begin'].strftime('%H:%M')
data['end'] = data['end'].strftime('%H:%M')
data['weekday'] = [int(v) for v in data['weekday'].split(',')]
return data
async def validate_fields_and_update(self, data, schema):
verrors = ValidationErrors()
begin = data.get('begin')
if begin:
data['begin'] = time(int(begin.split(':')[0]),
int(begin.split(':')[1]))
end = data.get('end')
if end:
data['end'] = time(int(end.split(':')[0]), int(end.split(':')[1]))
weekdays = data.get('weekday')
if weekdays:
if len([day for day in weekdays if day not in range(1, 8)]) > 0:
verrors.add(
f'{schema}.weekday',
'The week days should be in range of 1-7 inclusive')
else:
data['weekday'] = ','.join([str(day) for day in weekdays])
return verrors, data
@accepts(
Dict('pool_resilver', Str('begin', validators=[Time()]),
Str('end', validators=[Time()]), Bool('enabled'),
List('weekday', items=[Int('weekday')])))
async def do_update(self, data):
config = await self.config()
original_config = config.copy()
config.update(data)
verrors, new_config = await self.validate_fields_and_update(
config, 'pool_resilver_update')
if verrors:
raise verrors
# before checking if any changes have been made, original_config needs to be mapped to new_config
original_config['weekday'] = ','.join(
[str(day) for day in original_config['weekday']])
original_config['begin'] = time(
*(int(value) for value in original_config['begin'].split(':')))
original_config['end'] = time(
*(int(value) for value in original_config['end'].split(':')))
if len(set(original_config.items()) ^ set(new_config.items())) > 0:
# data has changed
await self.middleware.call('datastore.update',
self._config.datastore,
new_config['id'], new_config)
await self.middleware.call('service.restart', 'cron',
{'onetime': False})
await self.middleware.call('pool.configure_resilver_priority')
return await self.config()
class NetworkGeneralService(Service):
class Config:
namespace = 'network.general'
cli_namespace = 'network.general'
@accepts()
@returns(
Dict(
'network_summary',
Dict('ips', additional_attrs=True, required=True),
List('default_routes', items=[IPAddr('default_route')], required=True),
List('nameservers', items=[IPAddr('nameserver')], required=True),
)
)
async def summary(self):
"""
Retrieve general information for current Network.
Returns a dictionary. For example:
.. examples(websocket)::
:::javascript
{
"ips": {
"vtnet0": {
"IPV4": [
"192.168.0.15/24"
]
}
},
"default_routes": [
"192.168.0.1"
],
"nameservers": [
"192.168.0.1"
]
}
"""
ips = defaultdict(lambda: defaultdict(list))
for iface in await self.middleware.call('interface.query'):
for alias in iface['state']['aliases']:
if alias['type'] == 'INET':
key = 'IPV4'
elif alias['type'] == 'INET6':
key = 'IPV6'
else:
continue
ips[iface['name']][key].append(f'{alias["address"]}/{alias["netmask"]}')
default_routes = []
for route in await self.middleware.call('route.system_routes', [('netmask', 'in', ['0.0.0.0', '::'])]):
# IPv6 have local addresses that don't have gateways. Make sure we only return a gateway
# if there is one.
if route['gateway']:
default_routes.append(route['gateway'])
nameservers = []
for ns in await self.middleware.call('dns.query'):
nameservers.append(ns['nameserver'])
return {
'ips': ips,
'default_routes': default_routes,
'nameservers': nameservers,
}
class CoreService(Service):
@filterable
def get_jobs(self, filters=None, options=None):
"""Get the long running jobs."""
jobs = filter_list([
i.__encode__() for i in list(self.middleware.jobs.all().values())
], filters, options)
return jobs
@accepts(Int('id'))
@job()
def job_wait(self, job, id):
target_job = self.middleware.jobs.get(id)
target_job.wait_sync()
if target_job.error:
raise CallError(target_job.error)
else:
return target_job.result
@accepts(Int('id'),
Dict(
'job-update',
Dict('progress', additional_attrs=True),
))
def job_update(self, id, data):
job = self.middleware.jobs.all()[id]
progress = data.get('progress')
if progress:
job.set_progress(
progress['percent'],
description=progress.get('description'),
extra=progress.get('extra'),
)
@private
def notify_postinit(self):
self.middleware.call_sync('migration.run')
# Sentinel file to tell we have gone far enough in the boot process.
# See #17508
open('/tmp/.bootready', 'w').close()
# Send event to middlewared saying we are late enough in the process to call it ready
self.middleware.call_sync('core.event_send', 'system', 'ADDED',
{'id': 'ready'})
@accepts(Int('id'))
def job_abort(self, id):
job = self.middleware.jobs.all()[id]
return job.abort()
@accepts()
def get_services(self):
"""Returns a list of all registered services."""
services = {}
for k, v in list(self.middleware.get_services().items()):
if v._config.private is True:
continue
if isinstance(v, CRUDService):
_typ = 'crud'
elif isinstance(v, ConfigService):
_typ = 'config'
else:
_typ = 'service'
services[k] = {
'config': {
k: v
for k, v in list(v._config.__dict__.items())
if not k.startswith(('_', 'process_pool', 'thread_pool'))
},
'type': _typ,
}
return services
@accepts(Str('service', default=None, null=True))
def get_methods(self, service=None):
"""Return methods metadata of every available service.
`service` parameter is optional and filters the result for a single service."""
data = {}
for name, svc in list(self.middleware.get_services().items()):
if service is not None and name != service:
continue
# Skip private services
if svc._config.private:
continue
for attr in dir(svc):
if attr.startswith('_'):
continue
method = None
# For CRUD.do_{update,delete} they need to be accounted
# as "item_method", since they are just wrapped.
item_method = None
if isinstance(svc, CRUDService):
"""
For CRUD the create/update/delete are special.
The real implementation happens in do_create/do_update/do_delete
so thats where we actually extract pertinent information.
"""
if attr in ('create', 'update', 'delete'):
method = getattr(svc, 'do_{}'.format(attr), None)
if method is None:
continue
if attr in ('update', 'delete'):
item_method = True
elif attr in ('do_create', 'do_update', 'do_delete'):
continue
elif isinstance(svc, ConfigService):
"""
For Config the update is special.
The real implementation happens in do_update
so thats where we actually extract pertinent information.
"""
if attr == 'update':
original_name = 'do_{}'.format(attr)
if hasattr(svc, original_name):
method = getattr(svc, original_name, None)
else:
method = getattr(svc, attr)
if method is None:
continue
elif attr in ('do_update'):
continue
if method is None:
method = getattr(svc, attr, None)
if method is None or not callable(method):
continue
# Skip private methods
if hasattr(method, '_private'):
continue
# terminate is a private method used to clean up a service on shutdown
if attr == 'terminate':
continue
examples = defaultdict(list)
doc = inspect.getdoc(method)
if doc:
"""
Allow method docstring to have sections in the format of:
.. section_name::
Currently the following sections are available:
.. examples:: - goes into `__all__` list in examples
.. examples(rest):: - goes into `rest` list in examples
.. examples(websocket):: - goes into `websocket` list in examples
"""
sections = re.split(r'^.. (.+?)::$', doc, flags=re.M)
doc = sections[0]
for i in range(int((len(sections) - 1) / 2)):
idx = (i + 1) * 2 - 1
reg = re.search(r'examples(?:\((.+)\))?',
sections[idx])
if reg is None:
continue
exname = reg.groups()[0]
if exname is None:
exname = '__all__'
examples[exname].append(sections[idx + 1])
accepts = getattr(method, 'accepts', None)
if accepts:
accepts = [
i.to_json_schema() for i in accepts
if not getattr(i, 'hidden', False)
]
data['{0}.{1}'.format(name, attr)] = {
'description':
doc,
'examples':
examples,
'accepts':
accepts,
'item_method':
True if item_method else hasattr(method, '_item_method'),
'no_auth_required':
hasattr(method, '_no_auth_required'),
'filterable':
hasattr(method, '_filterable'),
'require_websocket':
hasattr(method, '_pass_app'),
'job':
hasattr(method, '_job'),
'downloadable':
hasattr(method, '_job')
and method._job['pipes'] == ['output'],
}
return data
@accepts()
def get_events(self):
"""
Returns metadata for every possible event emitted from websocket server.
"""
events = {}
for name, attrs in self.middleware.get_events():
events[name] = {
'description': attrs['description'],
}
return events
@private
async def call_hook(self, name, args, kwargs=None):
kwargs = kwargs or {}
await self.middleware.call_hook(name, *args, **kwargs)
@private
async def event_send(self, name, event_type, kwargs):
self.middleware.send_event(name, event_type, **kwargs)
@accepts()
def ping(self):
"""
Utility method which just returns "pong".
Can be used to keep connection/authtoken alive instead of using
"ping" protocol message.
"""
return 'pong'
@accepts(
Str('method'),
List('args', default=[]),
Str('filename'),
)
async def download(self, method, args, filename):
"""
Core helper to call a job marked for download.
Returns the job id and the URL for download.
"""
job = await self.middleware.call(
method, *args, pipes=Pipes(output=self.middleware.pipe()))
token = await self.middleware.call('auth.generate_token', 300, {
'filename': filename,
'job': job.id
})
self.middleware.fileapp.register_job(job.id)
return job.id, f'/_download/{job.id}?auth_token={token}'
@private
def reconfigure_logging(self):
"""
When /var/log gets moved because of system dataset
we need to make sure the log file is reopened because
of the new location
"""
handler = logging._handlers.get('file')
if handler:
stream = handler.stream
handler.stream = handler._open()
if sys.stdout is stream:
sys.stdout = handler.stream
sys.stderr = handler.stream
try:
stream.close()
except Exception:
pass
@private
@accepts(Dict(
'core-job',
Int('sleep'),
))
@job()
def job_test(self, job, data=None):
"""
Private no-op method to test a job, simply returning `true`.
"""
if data is None:
data = {}
sleep = data.get('sleep')
if sleep is not None:
def sleep_fn():
i = 0
while i < sleep:
job.set_progress((i / sleep) * 100)
time.sleep(1)
i += 1
job.set_progress(100)
t = threading.Thread(target=sleep_fn, daemon=True)
t.start()
t.join()
return True
@accepts(
Str('engine', enum=['PTVS', 'PYDEV']),
Dict(
'options',
Str('secret'),
Str('bind_address', default='0.0.0.0'),
Int('bind_port', default=3000),
Str('host'),
Bool('wait_attach', default=False),
Str('local_path'),
),
)
async def debug(self, engine, options):
"""
Setup middlewared for remote debugging.
engines:
- PTVS: Python Visual Studio
- PYDEV: Python Dev (Eclipse/PyCharm)
options:
- secret: password for PTVS
- host: required for PYDEV, hostname of local computer (developer workstation)
- local_path: required for PYDEV, path for middlewared source in local computer (e.g. /home/user/freenas/src/middlewared/middlewared
"""
if engine == 'PTVS':
import ptvsd
if 'secret' not in options:
raise ValidationError('secret', 'secret is required for PTVS')
ptvsd.enable_attach(
options['secret'],
address=(options['bind_address'], options['bind_port']),
)
if options['wait_attach']:
ptvsd.wait_for_attach()
elif engine == 'PYDEV':
for i in ('host', 'local_path'):
if i not in options:
raise ValidationError(i, f'{i} is required for PYDEV')
os.environ['PATHS_FROM_ECLIPSE_TO_PYTHON'] = json.dumps([
[
options['local_path'],
'/usr/local/lib/python3.7/site-packages/middlewared'
],
])
import pydevd
pydevd.stoptrace()
pydevd.settrace(host=options['host'])
@private
def threads_stacks(self):
return get_threads_stacks()
@accepts(Str("method"), List("params", default=[]))
@job(lock=lambda args: f"bulk:{args[0]}")
async def bulk(self, job, method, params):
"""
Will loop on a list of items for the given method, returning a list of
dicts containing a result and error key.
Result will be the message returned by the method being called,
or a string of an error, in which case the error key will be the
exception
"""
statuses = []
progress_step = 100 / len(params)
current_progress = 0
for p in params:
try:
msg = await self.middleware.call(method, *p)
error = None
if isinstance(msg, Job):
job = msg
msg = await msg.wait()
if job.error:
error = job.error
statuses.append({"result": msg, "error": error})
except Exception as e:
statuses.append({"result": None, "error": str(e)})
current_progress += progress_step
job.set_progress(current_progress)
return statuses
class ZFSDatasetService(CRUDService):
class Config:
namespace = 'zfs.dataset'
private = True
process_pool = True
def flatten_datasets(self, datasets):
return sum([[deepcopy(ds)] + self.flatten_datasets(ds['children']) for ds in datasets], [])
@filterable
def query(self, filters=None, options=None):
"""
In `query-options` we can provide `extra` arguments which control which data should be retrieved
for a dataset.
`query-options.extra.top_level_properties` is a list of properties which we will like to include in the
top level dict of dataset. It defaults to adding only mountpoint key keeping legacy behavior. If none are
desired in top level dataset, an empty list should be passed else if null is specified it will add mountpoint
key to the top level dict if it's present in `query-options.extra.properties` or it's null as well.
`query-options.extra.properties` is a list of properties which should be retrieved. If null ( by default ),
it would retrieve all properties, if empty, it will retrieve no property ( `mountpoint` is special in this
case and is controlled by `query-options.extra.mountpoint` attribute ).
We provide 2 ways how zfs.dataset.query returns dataset's data. First is a flat structure ( default ), which
means that all the datasets in the system are returned as separate objects which also contain all the data
their is for their children. This retrieval type is slightly slower because of duplicates which exist in
each object.
Second type is hierarchical where only top level datasets are returned in the list and they contain all the
children there are for them in `children` key. This retrieval type is slightly faster.
These options are controlled by `query-options.extra.flat` attribute which defaults to true.
`query-options.extra.user_properties` controls if user defined properties of datasets should be retrieved
or not.
While we provide a way to exclude all properties from data retrieval, we introduce a single attribute
`query-options.extra.retrieve_properties` which if set to false will make sure that no property is retrieved
whatsoever and overrides any other property retrieval attribute.
"""
options = options or {}
extra = options.get('extra', {}).copy()
top_level_props = None if extra.get('top_level_properties') is None else extra['top_level_properties'].copy()
props = extra.get('properties', None)
flat = extra.get('flat', True)
user_properties = extra.get('user_properties', True)
retrieve_properties = extra.get('retrieve_properties', True)
if not retrieve_properties:
# This is a short hand version where consumer can specify that they don't want any property to
# be retrieved
user_properties = False
props = []
with libzfs.ZFS() as zfs:
# Handle `id` filter specially to avoiding getting all datasets
if filters and len(filters) == 1 and list(filters[0][:2]) == ['id', '=']:
state_options = {
'snapshots': extra.get('snapshots', False),
'recursive': extra.get('recursive', True),
'snapshots_recursive': extra.get('snapshots_recursive', False)
}
try:
datasets = [zfs.get_dataset(filters[0][2]).__getstate__(**state_options)]
except libzfs.ZFSException:
datasets = []
else:
datasets = zfs.datasets_serialized(
props=props, top_level_props=top_level_props, user_props=user_properties
)
if flat:
datasets = self.flatten_datasets(datasets)
else:
datasets = list(datasets)
return filter_list(datasets, filters, options)
def query_for_quota_alert(self):
return [
{
k: v for k, v in dataset['properties'].items()
if k in [
"name", "quota", "available", "refquota", "usedbydataset", "mounted", "mountpoint",
"org.freenas:quota_warning", "org.freenas:quota_critical",
"org.freenas:refquota_warning", "org.freenas:refquota_critical"
]
}
for dataset in self.query()
]
def common_load_dataset_checks(self, ds):
self.common_encryption_checks(ds)
if ds.key_loaded:
raise CallError(f'{id} key is already loaded')
def common_encryption_checks(self, ds):
if not ds.encrypted:
raise CallError(f'{id} is not encrypted')
def get_quota(self, ds, quota_type):
if quota_type == 'dataset':
dataset = self.query([('id', '=', ds)], {'get': True})
return [{
'quota_type': 'DATASET',
'id': ds,
'name': ds,
'quota': int(dataset['properties']['quota']['rawvalue']),
'refquota': int(dataset['properties']['refquota']['rawvalue']),
'used_bytes': int(dataset['properties']['used']['rawvalue']),
}]
quota_list = []
quota_get = subprocess.run(
['zfs', f'{quota_type}space', '-H', '-n', '-p', '-o', 'name,used,quota,objquota,objused', ds],
capture_output=True,
check=False,
)
if quota_get.returncode != 0:
raise CallError(
f'Failed to get {quota_type} quota for {ds}: [{quota_get.stderr.decode()}]'
)
for quota in quota_get.stdout.decode().splitlines():
m = quota.split('\t')
if len(m) != 5:
self.logger.debug('Invalid %s quota: %s',
quota_type.lower(), quota)
continue
entry = {
'quota_type': quota_type.upper(),
'id': int(m[0]),
'name': None,
'quota': int(m[2]),
'used_bytes': int(m[1]),
'used_percent': 0,
'obj_quota': int(m[3]) if m[3] != '-' else 0,
'obj_used': int(m[4]) if m[4] != '-' else 0,
'obj_used_percent': 0,
}
if entry['quota'] > 0:
entry['used_percent'] = entry['used_bytes'] / entry['quota'] * 100
if entry['obj_quota'] > 0:
entry['obj_used_percent'] = entry['obj_used'] / entry['obj_quota'] * 100
try:
if quota_type == 'USER':
entry['name'] = (
self.middleware.call_sync('user.get_user_obj',
{'uid': entry['id']})
)['pw_name']
else:
entry['name'] = (
self.middleware.call_sync('group.get_group_obj',
{'gid': entry['id']})
)['gr_name']
except Exception:
self.logger.debug('Unable to resolve %s id %d to name',
quota_type.lower(), entry['id'])
pass
quota_list.append(entry)
return quota_list
def set_quota(self, ds, quota_list):
cmd = ['zfs', 'set']
cmd.extend(quota_list)
cmd.append(ds)
quota_set = subprocess.run(cmd, check=False)
if quota_set.returncode != 0:
raise CallError(f'Failed to set userspace quota on {ds}: [{quota_set.stderr.decode()}]')
@accepts(
Str('id'),
Dict(
'load_key_options',
Bool('mount', default=True),
Bool('recursive', default=False),
Any('key', default=None, null=True),
Str('key_location', default=None, null=True),
),
)
def load_key(self, id, options):
mount_ds = options.pop('mount')
recursive = options.pop('recursive')
try:
with libzfs.ZFS() as zfs:
ds = zfs.get_dataset(id)
self.common_load_dataset_checks(ds)
ds.load_key(**options)
except libzfs.ZFSException as e:
self.logger.error(f'Failed to load key for {id}', exc_info=True)
raise CallError(f'Failed to load key for {id}: {e}')
else:
if mount_ds:
self.mount(id, {'recursive': recursive})
@accepts(Str('name'), List('params', default=[], private=True))
@job()
def bulk_process(self, job, name, params):
f = getattr(self, name, None)
if not f:
raise CallError(f'{name} method not found in zfs.dataset')
statuses = []
for i in params:
result = error = None
try:
result = f(*i)
except Exception as e:
error = str(e)
finally:
statuses.append({'result': result, 'error': error})
return statuses
@accepts(
Str('id'),
Dict(
'check_key',
Any('key', default=None, null=True),
Str('key_location', default=None, null=True),
)
)
def check_key(self, id, options):
"""
Returns `true` if the `key` is valid, `false` otherwise.
"""
try:
with libzfs.ZFS() as zfs:
ds = zfs.get_dataset(id)
self.common_encryption_checks(ds)
return ds.check_key(**options)
except libzfs.ZFSException as e:
self.logger.error(f'Failed to check key for {id}', exc_info=True)
raise CallError(f'Failed to check key for {id}: {e}')
@accepts(
Str('id'),
Dict(
'unload_key_options',
Bool('recursive', default=False),
Bool('force_umount', default=False),
Bool('umount', default=False),
)
)
def unload_key(self, id, options):
force = options.pop('force_umount')
if options.pop('umount') and self.middleware.call_sync('zfs.dataset.get_instance', id)['mountpoint']:
self.umount(id, {'force': force})
try:
with libzfs.ZFS() as zfs:
ds = zfs.get_dataset(id)
self.common_encryption_checks(ds)
if not ds.key_loaded:
raise CallError(f'{id}\'s key is not loaded')
ds.unload_key(**options)
except libzfs.ZFSException as e:
self.logger.error(f'Failed to unload key for {id}', exc_info=True)
raise CallError(f'Failed to unload key for {id}: {e}')
@accepts(
Str('id'),
Dict(
'change_key_options',
Dict(
'encryption_properties',
Str('keyformat'),
Str('keylocation'),
Int('pbkdf2iters')
),
Bool('load_key', default=True),
Any('key', default=None, null=True),
),
)
def change_key(self, id, options):
try:
with libzfs.ZFS() as zfs:
ds = zfs.get_dataset(id)
self.common_encryption_checks(ds)
ds.change_key(props=options['encryption_properties'], load_key=options['load_key'], key=options['key'])
except libzfs.ZFSException as e:
self.logger.error(f'Failed to change key for {id}', exc_info=True)
raise CallError(f'Failed to change key for {id}: {e}')
@accepts(
Str('id'),
Dict(
'change_encryption_root_options',
Bool('load_key', default=True),
)
)
def change_encryption_root(self, id, options):
try:
with libzfs.ZFS() as zfs:
ds = zfs.get_dataset(id)
ds.change_key(load_key=options['load_key'], inherit=True)
except libzfs.ZFSException as e:
raise CallError(f'Failed to change encryption root for {id}: {e}')
@accepts(Dict(
'dataset_create',
Str('name', required=True),
Str('type', enum=['FILESYSTEM', 'VOLUME'], default='FILESYSTEM'),
Dict(
'properties',
Bool('sparse'),
additional_attrs=True,
),
))
def do_create(self, data):
"""
Creates a ZFS dataset.
"""
verrors = ValidationErrors()
if '/' not in data['name']:
verrors.add('name', 'You need a full name, e.g. pool/newdataset')
if verrors:
raise verrors
properties = data.get('properties') or {}
sparse = properties.pop('sparse', False)
params = {}
for k, v in data['properties'].items():
params[k] = v
try:
with libzfs.ZFS() as zfs:
pool = zfs.get(data['name'].split('/')[0])
pool.create(data['name'], params, fstype=getattr(libzfs.DatasetType, data['type']), sparse_vol=sparse)
except libzfs.ZFSException as e:
self.logger.error('Failed to create dataset', exc_info=True)
raise CallError(f'Failed to create dataset: {e}')
@accepts(
Str('id'),
Dict(
'dataset_update',
Dict(
'properties',
additional_attrs=True,
),
),
)
def do_update(self, id, data):
try:
with libzfs.ZFS() as zfs:
dataset = zfs.get_dataset(id)
if 'properties' in data:
properties = data['properties'].copy()
# Set these after reservations
for k in ['quota', 'refquota']:
if k in properties:
properties[k] = properties.pop(k) # Set them last
for k, v in properties.items():
# If prop already exists we just update it,
# otherwise create a user property
prop = dataset.properties.get(k)
try:
if prop:
if v.get('source') == 'INHERIT':
prop.inherit(recursive=v.get('recursive', False))
elif 'value' in v and (
prop.value != v['value'] or prop.source.name == 'INHERITED'
):
prop.value = v['value']
elif 'parsed' in v and (
prop.parsed != v['parsed'] or prop.source.name == 'INHERITED'
):
prop.parsed = v['parsed']
else:
if v.get('source') == 'INHERIT':
pass
else:
if 'value' not in v:
raise ValidationError(
'properties', f'properties.{k} needs a "value" attribute'
)
if ':' not in k:
raise ValidationError(
'properties', f'User property needs a colon (:) in its name`'
)
prop = libzfs.ZFSUserProperty(v['value'])
dataset.properties[k] = prop
except libzfs.ZFSException as e:
raise ZFSSetPropertyError(k, str(e))
except libzfs.ZFSException as e:
self.logger.error('Failed to update dataset', exc_info=True)
raise CallError(f'Failed to update dataset: {e}')
def do_delete(self, id, options=None):
options = options or {}
force = options.get('force', False)
recursive = options.get('recursive', False)
args = []
if force:
args += ['-f']
if recursive:
args += ['-r']
# If dataset is mounted and has receive_resume_token, we should destroy it or ZFS will say
# "cannot destroy 'pool/dataset': dataset already exists"
recv_run = subprocess.run(['zfs', 'recv', '-A', id], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
# Destroying may take a long time, lets not use py-libzfs as it will block
# other ZFS operations.
try:
subprocess.run(
['zfs', 'destroy'] + args + [id], text=True, capture_output=True, check=True,
)
except subprocess.CalledProcessError as e:
if recv_run.returncode == 0 and e.stderr.strip().endswith('dataset does not exist'):
# This operation might have deleted this dataset if it was created by `zfs recv` operation
return
self.logger.error('Failed to delete dataset', exc_info=True)
error = e.stderr.strip()
errno_ = errno.EFAULT
if "Device busy" in error:
errno_ = errno.EBUSY
raise CallError(f'Failed to delete dataset: {error}', errno_)
@accepts(Str('name'), Dict('options', Bool('recursive', default=False)))
def mount(self, name, options):
try:
with libzfs.ZFS() as zfs:
dataset = zfs.get_dataset(name)
if options['recursive']:
dataset.mount_recursive()
else:
dataset.mount()
except libzfs.ZFSException as e:
self.logger.error('Failed to mount dataset', exc_info=True)
raise CallError(f'Failed to mount dataset: {e}')
@accepts(Str('name'), Dict('options', Bool('force', default=False)))
def umount(self, name, options):
try:
with libzfs.ZFS() as zfs:
dataset = zfs.get_dataset(name)
dataset.umount(force=options['force'])
except libzfs.ZFSException as e:
self.logger.error('Failed to umount dataset', exc_info=True)
raise CallError(f'Failed to umount dataset: {e}')
@accepts(
Str('dataset'),
Dict(
'options',
Str('new_name', required=True, empty=False),
Bool('recursive', default=False)
)
)
def rename(self, name, options):
try:
with libzfs.ZFS() as zfs:
dataset = zfs.get_dataset(name)
dataset.rename(options['new_name'], recursive=options['recursive'])
except libzfs.ZFSException as e:
self.logger.error('Failed to rename dataset', exc_info=True)
raise CallError(f'Failed to rename dataset: {e}')
def promote(self, name):
try:
with libzfs.ZFS() as zfs:
dataset = zfs.get_dataset(name)
dataset.promote()
except libzfs.ZFSException as e:
self.logger.error('Failed to promote dataset', exc_info=True)
raise CallError(f'Failed to promote dataset: {e}')
def inherit(self, name, prop, recursive=False):
try:
with libzfs.ZFS() as zfs:
dataset = zfs.get_dataset(name)
zprop = dataset.properties.get(prop)
if not zprop:
raise CallError(f'Property {prop!r} not found.', errno.ENOENT)
zprop.inherit(recursive=recursive)
except libzfs.ZFSException as e:
raise CallError(str(e))
class MailService(ConfigService):
class Config:
datastore = 'system.email'
datastore_prefix = 'em_'
datastore_extend = 'mail.mail_extend'
@private
async def mail_extend(self, cfg):
if cfg['security']:
cfg['security'] = cfg['security'].upper()
return cfg
@accepts(Dict(
'mail_update',
Str('fromemail', validators=[Email()]),
Str('fromname'),
Str('outgoingserver'),
Int('port'),
Str('security', enum=['PLAIN', 'SSL', 'TLS']),
Bool('smtp'),
Str('user'),
Str('pass', private=True),
register=True,
update=True,
))
async def do_update(self, data):
"""
Update Mail Service Configuration.
`fromemail` is used as a sending address which the mail server will use for sending emails.
`outgoingserver` is the hostname or IP address of SMTP server used for sending an email.
`security` is type of encryption desired.
`smtp` is a boolean value which when set indicates that SMTP authentication has been enabled and `user`/`pass`
are required attributes now.
"""
config = await self.config()
new = config.copy()
new.update(data)
new['security'] = new['security'].lower() # Django Model compatibility
verrors = ValidationErrors()
if new['smtp'] and new['user'] == '':
verrors.add(
'mail_update.user',
'This field is required when SMTP authentication is enabled',
)
self.__password_verify(new['pass'], 'mail_update.pass', verrors)
if verrors:
raise verrors
await self.middleware.call('datastore.update', 'system.email', config['id'], new, {'prefix': 'em_'})
return await self.config()
def __password_verify(self, password, schema, verrors=None):
if not password:
return
if verrors is None:
verrors = ValidationErrors()
# FIXME: smtplib does not support non-ascii password yet
# https://github.com/python/cpython/pull/8938
try:
password.encode('ascii')
except UnicodeEncodeError:
verrors.add(
schema,
'Only plain text characters (7-bit ASCII) are allowed in passwords. '
'UTF or composed characters are not allowed.'
)
return verrors
@accepts(Dict(
'mail_message',
Str('subject', required=True),
Str('text', required=True, max_length=None),
Str('html', null=True, max_length=None),
List('to', items=[Str('email')]),
List('cc', items=[Str('email')]),
Int('interval', null=True),
Str('channel', null=True),
Int('timeout', default=300),
Bool('attachments', default=False),
Bool('queue', default=True),
Dict('extra_headers', additional_attrs=True),
register=True,
), Ref('mail_update'))
@job(pipes=['input'], check_pipes=False)
def send(self, job, message, config):
"""
Sends mail using configured mail settings.
`text` will be formatted to HTML using Markdown and rendered using default E-Mail template.
You can put your own HTML using `html`. If `html` is null, no HTML MIME part will be added to E-Mail.
If `attachments` is true, a list compromised of the following dict is required
via HTTP upload:
- headers(list)
- name(str)
- value(str)
- params(dict)
- content (str)
[
{
"headers": [
{
"name": "Content-Transfer-Encoding",
"value": "base64"
},
{
"name": "Content-Type",
"value": "application/octet-stream",
"params": {
"name": "test.txt"
}
}
],
"content": "dGVzdAo="
}
]
"""
product_name = self.middleware.call_sync('system.product_name')
gc = self.middleware.call_sync('datastore.config', 'network.globalconfiguration')
hostname = f'{gc["gc_hostname"]}.{gc["gc_domain"]}'
message['subject'] = f'{product_name} {hostname}: {message["subject"]}'
if 'html' in message and message['html'] is None:
message.pop('html')
elif 'html' not in message:
lookup = TemplateLookup(
directories=[os.path.join(os.path.dirname(os.path.realpath(__file__)), '../assets/templates')],
module_directory="/tmp/mako/templates")
tmpl = lookup.get_template('mail.html')
message['html'] = tmpl.render(body=html.escape(message['text']).replace('\n', '<br>\n'))
return self.send_raw(job, message, config)
@accepts(Ref('mail_message'), Ref('mail_update'))
@job(pipes=['input'], check_pipes=False)
@private
def send_raw(self, job, message, config):
config = dict(self.middleware.call_sync('mail.config'), **config)
if config['fromname']:
from_addr = Header(config['fromname'], 'utf-8')
try:
config['fromemail'].encode('ascii')
except UnicodeEncodeError:
from_addr.append(f'<{config["fromemail"]}>', 'utf-8')
else:
from_addr.append(f'<{config["fromemail"]}>', 'ascii')
else:
try:
config['fromemail'].encode('ascii')
except UnicodeEncodeError:
from_addr = Header(config['fromemail'], 'utf-8')
else:
from_addr = Header(config['fromemail'], 'ascii')
interval = message.get('interval')
if interval is None:
interval = timedelta()
else:
interval = timedelta(seconds=interval)
sw_name = self.middleware.call_sync('system.info')['version'].split('-', 1)[0]
channel = message.get('channel')
if not channel:
channel = sw_name.lower()
if interval > timedelta():
channelfile = '/tmp/.msg.%s' % (channel)
last_update = datetime.now() - interval
try:
last_update = datetime.fromtimestamp(os.stat(channelfile).st_mtime)
except OSError:
pass
timediff = datetime.now() - last_update
if (timediff >= interval) or (timediff < timedelta()):
# Make sure mtime is modified
# We could use os.utime but this is simpler!
with open(channelfile, 'w') as f:
f.write('!')
else:
raise CallError('This message was already sent in the given interval')
verrors = self.__password_verify(config['pass'], 'mail-config.pass')
if verrors:
raise verrors
to = message.get('to')
if not to:
to = [
self.middleware.call_sync(
'user.query', [('username', '=', 'root')], {'get': True}
)['email']
]
if not to[0]:
raise CallError('Email address for root is not configured')
if message.get('attachments'):
job.check_pipe("input")
def read_json():
f = job.pipes.input.r
data = b''
i = 0
while True:
read = f.read(1048576) # 1MiB
if read == b'':
break
data += read
i += 1
if i > 50:
raise ValueError('Attachments bigger than 50MB not allowed yet')
if data == b'':
return None
return json.loads(data)
attachments = read_json()
else:
attachments = None
if 'html' in message or attachments:
msg = MIMEMultipart()
msg.preamble = message['text']
if 'html' in message:
msg2 = MIMEMultipart('alternative')
msg2.attach(MIMEText(message['text'], 'plain', _charset='utf-8'))
msg2.attach(MIMEText(message['html'], 'html', _charset='utf-8'))
msg.attach(msg2)
if attachments:
for attachment in attachments:
m = Message()
m.set_payload(attachment['content'])
for header in attachment.get('headers'):
m.add_header(header['name'], header['value'], **(header.get('params') or {}))
msg.attach(m)
else:
msg = MIMEText(message['text'], _charset='utf-8')
msg['Subject'] = message['subject']
msg['From'] = from_addr
msg['To'] = ', '.join(to)
if message.get('cc'):
msg['Cc'] = ', '.join(message.get('cc'))
msg['Date'] = formatdate()
local_hostname = socket.gethostname()
msg['Message-ID'] = "<%s-%s.%s@%s>" % (sw_name.lower(), datetime.utcnow().strftime("%Y%m%d.%H%M%S.%f"), base64.urlsafe_b64encode(os.urandom(3)), local_hostname)
extra_headers = message.get('extra_headers') or {}
for key, val in list(extra_headers.items()):
# We already have "Content-Type: multipart/mixed" and setting "Content-Type: text/plain" like some scripts
# do will break python e-mail module.
if key.lower() == "сontent-type":
continue
if key in msg:
msg.replace_header(key, val)
else:
msg[key] = val
syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_MAIL)
try:
server = self._get_smtp_server(config, message['timeout'], local_hostname=local_hostname)
# NOTE: Don't do this.
#
# If smtplib.SMTP* tells you to run connect() first, it's because the
# mailserver it tried connecting to via the outgoing server argument
# was unreachable and it tried to connect to 'localhost' and barfed.
# This is because FreeNAS doesn't run a full MTA.
# else:
# server.connect()
headers = '\n'.join([f'{k}: {v}' for k, v in msg._headers])
syslog.syslog(f"sending mail to {', '.join(to)}\n{headers}")
server.sendmail(from_addr.encode(), to, msg.as_string())
server.quit()
except Exception as e:
# Don't spam syslog with these messages. They should only end up in the
# test-email pane.
# We are only interested in ValueError, not subclasses.
if e.__class__ is ValueError:
raise CallError(str(e))
syslog.syslog(f'Failed to send email to {", ".join(to)}: {str(e)}')
if isinstance(e, smtplib.SMTPAuthenticationError):
raise CallError(f'Authentication error ({e.smtp_code}): {e.smtp_error}', errno.EAUTH)
self.logger.warn('Failed to send email: %s', str(e), exc_info=True)
if message['queue']:
with MailQueue() as mq:
mq.append(msg)
raise CallError(f'Failed to send email: {e}')
return True
def _get_smtp_server(self, config, timeout=300, local_hostname=None):
if local_hostname is None:
local_hostname = socket.gethostname()
if not config['outgoingserver'] or not config['port']:
# See NOTE below.
raise ValueError('you must provide an outgoing mailserver and mail'
' server port when sending mail')
if config['security'] == 'SSL':
server = smtplib.SMTP_SSL(
config['outgoingserver'],
config['port'],
timeout=timeout,
local_hostname=local_hostname)
else:
server = smtplib.SMTP(
config['outgoingserver'],
config['port'],
timeout=timeout,
local_hostname=local_hostname)
if config['security'] == 'TLS':
server.starttls()
if config['smtp']:
server.login(config['user'], config['pass'])
return server
@periodic(600, run_on_start=False)
@private
def send_mail_queue(self):
with MailQueue() as mq:
for queue in list(mq.queue):
try:
config = self.middleware.call_sync('mail.config')
server = self._get_smtp_server(config)
server.sendmail(queue.message['From'].encode(), queue.message['To'].split(', '), queue.message.as_string())
server.quit()
except Exception:
self.logger.debug('Sending message from queue failed', exc_info=True)
queue.attempts += 1
if queue.attempts >= mq.MAX_ATTEMPTS:
mq.queue.remove(queue)
else:
mq.queue.remove(queue)
class UPSService(SystemServiceService):
DRIVERS_AVAILABLE = set(os.listdir(DRIVER_BIN_DIR))
class Config:
datastore = 'services.ups'
datastore_prefix = 'ups_'
datastore_extend = 'ups.ups_config_extend'
service = 'ups'
service_verb = 'restart'
@private
async def ups_config_extend(self, data):
data['mode'] = data['mode'].upper()
data['shutdown'] = data['shutdown'].upper()
data['toemail'] = [v for v in data['toemail'].split(';') if v]
return data
@accepts()
async def port_choices(self):
ports = [x for x in glob.glob('/dev/cua*') if x.find('.') == -1]
ports.extend(glob.glob('/dev/ugen*'))
ports.extend(glob.glob('/dev/uhid*'))
return ports
@accepts()
def driver_choices(self):
"""
Returns choices of UPS drivers supported by the system.
"""
ups_choices = {}
if os.path.exists("/conf/base/etc/local/nut/driver.list"):
with open('/conf/base/etc/local/nut/driver.list', 'rb') as f:
d = f.read().decode('utf8', 'ignore')
r = io.StringIO()
for line in re.sub(r'[ \t]+', ' ', d, flags=re.M).split('\n'):
r.write(line.strip() + '\n')
r.seek(0)
reader = csv.reader(r, delimiter=' ', quotechar='"')
for row in reader:
if len(row) == 0 or row[0].startswith('#'):
continue
if row[-2] == '#':
last = -3
else:
last = -1
driver_str = row[last]
driver_annotation = ''
m = re.match(r'(.+) \((.+)\)',
driver_str) # "blazer_usb (USB ID 0665:5161)"
if m:
driver_str, driver_annotation = m.group(1), m.group(2)
for driver in driver_str.split(
' or '): # can be "blazer_ser or blazer_usb"
driver = driver.strip()
if driver not in self.DRIVERS_AVAILABLE:
continue
for i, field in enumerate(list(row)):
row[i] = field
ups_choices['$'.join(
[driver, row[3]])] = '%s (%s)' % (' '.join(
filter(None, row[0:last])), ', '.join(
filter(None, [driver, driver_annotation])))
return ups_choices
@private
async def validate_data(self, data, schema):
verrors = ValidationErrors()
driver = data.get('driver')
if driver:
if driver not in (
await self.middleware.call('ups.driver_choices')).keys():
verrors.add(
f'{schema}.driver',
'Driver selected does not match local machine\'s driver list'
)
identifier = data['identifier']
if identifier:
if not re.search(r'^[a-z0-9\.\-_]+$', identifier, re.I):
verrors.add(f'{schema}.identifier',
'Use alphanumeric characters, ".", "-" and "_"')
for field in [
field for field in ['monpwd', 'monuser'] if data.get(field)
]:
if re.search(r'[ #]', data[field], re.I):
verrors.add(f'{schema}.{field}',
'Spaces or number signs are not allowed')
mode = data.get('mode')
if mode == 'MASTER':
for field in filter(lambda f: not data[f], ['port', 'driver']):
verrors.add(f'{schema}.{field}', 'This field is required')
else:
if not data.get('remotehost'):
verrors.add(f'{schema}.remotehost', 'This field is required')
to_emails = data.get('toemail')
if to_emails:
data['toemail'] = ';'.join(to_emails)
else:
data['toemail'] = ''
data['mode'] = data['mode'].lower()
data['shutdown'] = data['shutdown'].lower()
return verrors, data
@accepts(
Dict('ups_update',
Bool('emailnotify'),
Bool('powerdown'),
Bool('rmonitor'),
Int('nocommwarntime', null=True),
Int('remoteport'),
Int('shutdowntimer'),
Int('hostsync', validators=[Range(min=0)]),
Str('description'),
Str('driver'),
Str('extrausers', max_length=None),
Str('identifier', empty=False),
Str('mode', enum=['MASTER', 'SLAVE']),
Str('monpwd', empty=False),
Str('monuser', empty=False),
Str('options', max_length=None),
Str('optionsupsd', max_length=None),
Str('port'),
Str('remotehost'),
Str('shutdown', enum=['LOWBATT', 'BATT']),
Str('shutdowncmd', empty=False),
Str('subject'),
List('toemail', items=[Str('email', validators=[Email()])]),
update=True))
async def do_update(self, data):
"""
Update UPS Service Configuration.
`emailnotify` when enabled, sends out notifications of different UPS events via email.
`powerdown` when enabled, sets UPS to power off after shutting down the system.
`nocommwarntime` is a value in seconds which makes UPS Service wait the specified seconds before alerting that
the Service cannot reach configured UPS.
`shutdowntimer` is a value in seconds which tells the Service to wait specified seconds for the UPS before
initiating a shutdown. This only applies when `shutdown` is set to "BATT".
`shutdowncmd` is the command which is executed to initiate a shutdown. It defaults to "poweroff".
`toemail` is a list of valid email id's on which notification emails are sent.
"""
config = await self.config()
old_config = config.copy()
config.update(data)
verros, config = await self.validate_data(config, 'ups_update')
if verros:
raise verros
old_config['mode'] = old_config['mode'].lower()
old_config['shutdown'] = old_config['shutdown'].lower()
old_config['toemail'] = ';'.join(
old_config['toemail']) if old_config['toemail'] else ''
if len(set(old_config.items()) ^ set(config.items())) > 0:
if config['identifier'] != old_config['identifier']:
await self.dismiss_alerts()
await self._update_service(old_config, config)
return await self.config()
@private
async def alerts_mapping(self):
return {
'LOWBATT': 'UPSBatteryLow',
'COMMBAD': 'UPSCommbad',
'COMMOK': 'UPSCommok',
'ONBATT': 'UPSOnBattery',
'ONLINE': 'UPSOnline',
'REPLBATT': 'UPSReplbatt'
}
@private
async def dismiss_alerts(self):
config = await self.config()
for alert in (await self.alerts_mapping()).values():
await self.middleware.call('alert.oneshot_delete', alert,
{'ups': config['identifier']})
@private
@accepts(Str('notify_type'))
async def upssched_event(self, notify_type):
config = await self.config()
if config['mode'] == 'MASTER':
upsc_identifier = f'{config["identifier"]}@localhost:{config["remoteport"]}'
else:
upsc_identifier = f'{config["identifier"]}@{config["remotehost"]}:{config["remoteport"]}'
if notify_type.lower() == 'shutdown':
# Before we start FSD with upsmon, lets ensure that ups is not ONLINE (OL).
# There are cases where battery/charger issues can result in ups.status being "OL LB" at the
# same time. This will ensure that we don't initiate a shutdown if ups is OL.
stats_output = (await run('/usr/local/bin/upsc',
upsc_identifier,
check=False)).stdout
ups_status = re.findall(
fr'ups.status: (.*)',
'' if not stats_output else stats_output.decode())
if ups_status and 'ol' in ups_status[0].lower():
self.middleware.logger.debug(
f'Shutdown not initiated as ups.status ({ups_status[0]}) indicates '
f'{config["identifier"]} is ONLINE (OL).')
else:
syslog.syslog(syslog.LOG_NOTICE,
'upssched-cmd "issuing shutdown"')
await run('/usr/local/sbin/upsmon', '-c', 'fsd', check=False)
elif 'notify' in notify_type.lower():
# notify_type is expected to be of the following format
# NOTIFY-EVENT i.e NOTIFY-LOWBATT
notify_type = notify_type.split('-')[-1]
# We would like to send alerts for the following events
alert_mapping = await self.alerts_mapping()
await self.dismiss_alerts()
if notify_type in alert_mapping:
await self.middleware.call('alert.oneshot_create',
alert_mapping[notify_type],
{'ups': config['identifier']})
if config['emailnotify']:
# Email user with the notification event and details
# We send the email in the following format ( inclusive line breaks )
# NOTIFICATION: 'LOWBATT'
# UPS: 'ups'
#
# Statistics recovered:
#
# 1) Battery charge (percent)
# battery.charge: 5
#
# 2) Remaining battery level when UPS switches to LB (percent)
# battery.charge.low: 10
#
# 3) Battery runtime (seconds)
# battery.runtime: 1860
#
# 4) Remaining battery runtime when UPS switches to LB (seconds)
# battery.runtime.low: 900
ups_name = config['identifier']
hostname = (await
self.middleware.call('system.info'))['hostname']
current_time = datetime.datetime.now(tz=dateutil.tz.tzlocal(
)).strftime('%a %b %d %H:%M:%S %Z %Y')
ups_subject = config['subject'].replace('%d',
current_time).replace(
'%h', hostname)
body = f'NOTIFICATION: {notify_type!r}<br>UPS: {ups_name!r}<br><br>'
# Let's gather following stats
data_points = {
'battery.charge':
'Battery charge (percent)',
'battery.charge.low':
'Battery level remaining (percent) when UPS switches to Low Battery (LB)',
'battery.charge.status':
'Battery charge status',
'battery.runtime':
'Battery runtime (seconds)',
'battery.runtime.low':
'Battery runtime remaining (seconds) when UPS switches to Low Battery (LB)',
'battery.runtime.restart':
'Minimum battery runtime (seconds) to allow UPS restart after power-off',
}
stats_output = (await run('/usr/local/bin/upsc',
upsc_identifier,
check=False)).stdout
recovered_stats = re.findall(
fr'({"|".join(data_points)}): (.*)',
'' if not stats_output else stats_output.decode())
if recovered_stats:
body += 'Statistics recovered:<br><br>'
# recovered_stats is expected to be a list in this format
# [('battery.charge', '5'), ('battery.charge.low', '10'), ('battery.runtime', '1860')]
for index, stat in enumerate(recovered_stats):
body += f'{index + 1}) {data_points[stat[0]]}<br> {stat[0]}: {stat[1]}<br><br>'
else:
body += 'Statistics could not be recovered<br>'
# Subject and body defined, send email
job = await self.middleware.call('mail.send', {
'subject': ups_subject,
'text': body,
'to': config['toemail']
})
await job.wait()
if job.error:
self.middleware.logger.debug(
f'Failed to send UPS status email: {job.error}')
else:
self.middleware.logger.debug(
f'Unrecognized UPS notification event: {notify_type}')
class KerberosRealmService(CRUDService):
class Config:
datastore = 'directoryservice.kerberosrealm'
datastore_prefix = 'krb_'
datastore_extend = 'kerberos.realm.kerberos_extend'
namespace = 'kerberos.realm'
@private
async def kerberos_extend(self, data):
for param in ['kdc', 'admin_server', 'kpasswd_server']:
data[param] = data[param].split(' ') if data[param] else []
return data
@private
async def kerberos_compress(self, data):
for param in ['kdc', 'admin_server', 'kpasswd_server']:
data[param] = ' '.join(data[param])
return data
@accepts(
Dict('kerberos_realm_create',
Str('realm', required=True),
List('kdc', default=[]),
List('admin_server', default=[]),
List('kpasswd_server', default=[]),
register=True))
async def do_create(self, data):
"""
Create a new kerberos realm. This will be automatically populated during the
domain join process in an Active Directory environment. Kerberos realm names
are case-sensitive, but convention is to only use upper-case.
Entries for kdc, admin_server, and kpasswd_server are not required.
If they are unpopulated, then kerberos will use DNS srv records to
discover the correct servers. The option to hard-code them is provided
due to AD site discovery. Kerberos has no concept of Active Directory
sites. This means that middleware performs the site discovery and
sets the kerberos configuration based on the AD site.
"""
verrors = ValidationErrors()
verrors.add_child('kerberos_realm_create', await self._validate(data))
if verrors:
raise verrors
data = await self.kerberos_compress(data)
data["id"] = await self.middleware.call(
"datastore.insert",
self._config.datastore,
data,
{"prefix": self._config.datastore_prefix},
)
await self.middleware.call('etc.generate', 'kerberos')
await self.middleware.call('service.restart', 'cron')
return await self._get_instance(data['id'])
@accepts(Int('id', required=True),
Patch("kerberos_realm_create", "kerberos_realm_update",
("attr", {
"update": True
})))
async def do_update(self, id, data):
"""
Update a kerberos realm by id. This will be automatically populated during the
domain join process in an Active Directory environment. Kerberos realm names
are case-sensitive, but convention is to only use upper-case.
"""
old = await self._get_instance(id)
new = old.copy()
new.update(data)
data = await self.kerberos_compress(new)
await self.middleware.call('datastore.update', self._config.datastore,
id, new,
{'prefix': self._config.datastore_prefix})
await self.middleware.call('etc.generate', 'kerberos')
return await self._get_instance(id)
@accepts(Int('id'))
async def do_delete(self, id):
"""
Delete a kerberos realm by ID.
"""
await self.middleware.call("datastore.delete", self._config.datastore,
id)
await self.middleware.call('etc.generate', 'kerberos')
@private
async def _validate(self, data):
verrors = ValidationErrors()
realms = await self.query()
for realm in realms:
if realm['realm'].upper() == data['realm'].upper():
verrors.add(
f'kerberos_realm',
f'kerberos realm with name {realm["realm"]} already exists.'
)
return verrors
class SMBService(SystemServiceService):
class Config:
service = 'cifs'
service_verb = 'restart'
datastore = 'services.cifs'
datastore_extend = 'smb.smb_extend'
datastore_prefix = 'cifs_srv_'
@private
async def smb_extend(self, smb):
"""Extend smb for netbios."""
smb['netbiosname_local'] = smb['netbiosname']
if not await self.middleware.call('system.is_freenas'
) and await self.middleware.call(
'failover.node') == 'B':
smb['netbiosname_local'] = smb['netbiosname_b']
smb['netbiosalias'] = (smb['netbiosalias'] or '').split()
smb['loglevel'] = LOGLEVEL_MAP.get(smb['loglevel'])
smb.pop('secrets')
return smb
async def __validate_netbios_name(self, name):
return RE_NETBIOSNAME.match(name)
async def unixcharset_choices(self):
return await self.generate_choices([
'UTF-8', 'ISO-8859-1', 'ISO-8859-15', 'GB2312', 'EUC-JP', 'ASCII'
])
@private
async def generate_choices(self, initial):
def key_cp(encoding):
cp = re.compile(
r"(?P<name>CP|GB|ISO-8859-|UTF-)(?P<num>\d+)").match(encoding)
if cp:
return tuple((cp.group('name'), int(cp.group('num'), 10)))
else:
return tuple((encoding, float('inf')))
charset = await self.common_charset_choices()
return {
v: v
for v in
[c for c in sorted(charset, key=key_cp) if c not in initial] +
initial
}
@accepts()
async def bindip_choices(self):
"""
List of valid choices for IP addresses to which to bind the SMB service.
Addresses assigned by DHCP are excluded from the results.
"""
choices = {}
for i in await self.middleware.call('interface.ip_in_use'):
choices[i['address']] = i['address']
return choices
@accepts()
async def domain_choices(self):
"""
List of domains visible to winbindd. Returns empty list if winbindd is
stopped.
"""
domains = []
wb = await run([SMBCmd.WBINFO.value, '-m'], check=False)
if wb.returncode == 0:
domains = wb.stdout.decode().splitlines()
return domains
@private
async def common_charset_choices(self):
def check_codec(encoding):
try:
return encoding.upper() if codecs.lookup(encoding) else False
except LookupError:
return False
proc = await Popen(['/usr/bin/iconv', '-l'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = (await proc.communicate())[0].decode()
encodings = set()
for line in output.splitlines():
enc = [e for e in line.split() if check_codec(e)]
if enc:
cp = enc[0]
for e in enc:
if e in ('UTF-8', 'ASCII', 'GB2312', 'HZ-GB-2312',
'CP1361'):
cp = e
break
encodings.add(cp)
return encodings
@private
async def store_ldap_admin_password(self):
"""
This is required if the LDAP directory service is enabled. The ldap admin dn and
password are stored in private/secrets.tdb file.
"""
ldap = await self.middleware.call('datastore.config',
'directoryservice.ldap')
if not ldap['ldap_enable']:
return True
set_pass = await run(
['usr/local/bin/smbpasswd', '-w', ldap['ldap_bindpw']],
check=False)
if set_pass.returncode != 0:
self.logger.debug(
f"Failed to set set ldap bindpw in secrets.tdb: {set_pass.stdout.decode()}"
)
return False
return True
@private
def getparm(self, parm, section):
"""
Get a parameter from the smb4.conf file. This is more reliable than
'testparm --parameter-name'. testparm will fail in a variety of
conditions without returning the parameter's value.
"""
try:
if section.upper() == 'GLOBAL':
try:
LP_CTX.load(SMBPath.GLOBALCONF.platform())
except Exception as e:
self.logger.warning("Failed to reload smb.conf: %s", e)
return LP_CTX.get(parm)
else:
return self.middleware.call_sync('sharing.smb.reg_getparm',
section, parm)
except Exception as e:
raise CallError(
f'Attempt to query smb4.conf parameter [{parm}] failed with error: {e}'
)
@private
async def get_next_rid(self):
next_rid = (await self.config())['next_rid']
if next_rid == 0:
try:
private_dir = await self.middleware.call(
"smb.getparm", "private directory", "GLOBAL")
next_rid = passdb.PDB(
f"tdbsam:{private_dir}/passdb.tdb").new_rid()
except Exception:
self.logger.warning(
"Failed to initialize RID counter from passdb. "
"Using default value for initialization.",
exc_info=True)
next_rid = 5000
await self.middleware.call('datastore.update', 'services.cifs', 1,
{'next_rid': next_rid + 1},
{'prefix': 'cifs_srv_'})
return next_rid
@private
async def setup_directories(self):
for p in SMBPath:
if p == SMBPath.STATEDIR:
path = await self.middleware.call("smb.getparm",
"state directory", "global")
elif p == SMBPath.PRIVATEDIR:
path = await self.middleware.call("smb.getparm", "privatedir",
"global")
else:
path = p.platform()
try:
if not await self.middleware.call('filesystem.acl_is_trivial',
path):
self.logger.warning(
"Inappropriate ACL detected on path [%s] stripping ACL",
path)
stripacl = await run(['setfacl', '-b', path], check=False)
if stripacl.returncode != 0:
self.logger.warning(
"Failed to strip ACL from path %s: %s", path,
stripacl.stderr.decode())
except CallError:
# Currently only time CallError is raise here is on ENOENT, which may be expected
pass
if not os.path.exists(path):
if p.is_dir():
os.mkdir(path, p.mode())
else:
os.chmod(path, p.mode())
@private
async def import_conf_to_registry(self):
drop = await run([SMBCmd.NET.value, 'conf', 'drop'], check=False)
if drop.returncode != 0:
self.logger.warning('failed to drop existing share config: %s',
drop.stderr.decode())
load = await run(
[SMBCmd.NET.value, 'conf', 'import',
SMBPath.SHARECONF.platform()],
check=False)
if load.returncode != 0:
self.logger.warning('failed to load share config: %s',
load.stderr.decode())
@private
@job(lock="smb_configure")
async def configure(self, job):
await self.reset_smb_ha_mode()
job.set_progress(0, 'Preparing to configure SMB.')
data = await self.config()
job.set_progress(10, 'Generating SMB config.')
await self.middleware.call('etc.generate', 'smb')
# Following hack will be removed once we make our own samba package
if osc.IS_LINUX:
os.remove("/etc/samba/smb.conf")
os.symlink("/etc/smb4.conf", "etc/samba/smb.conf")
"""
Many samba-related tools will fail if they are unable to initialize
a messaging context, which will happen if the samba-related directories
do not exist or have incorrect permissions.
"""
job.set_progress(20, 'Setting up SMB directories.')
await self.setup_directories()
job.set_progress(30, 'Setting up server SID.')
await self.middleware.call('smb.set_sid', data['cifs_SID'])
"""
If the ldap passdb backend is being used, then the remote LDAP server
will provide the SMB users and groups. We skip these steps to avoid having
samba potentially try to write our local users and groups to the remote
LDAP server.
"""
if await self.middleware.call("smb.getparm", "passdb backend",
"global") == "tdbsam":
job.set_progress(40, 'Synchronizing passdb and groupmap.')
await self.middleware.call('etc.generate', 'user')
pdb_job = await self.middleware.call("smb.synchronize_passdb")
grp_job = await self.middleware.call(
"smb.synchronize_group_mappings")
await pdb_job.wait()
await grp_job.wait()
await self.middleware.call("admonitor.start")
"""
The following steps ensure that we cleanly import our SMB shares
into the registry.
"""
job.set_progress(60, 'generating SMB share configuration.')
await self.middleware.call('cache.put', 'SMB_REG_INITIALIZED', False)
await self.middleware.call("etc.generate", "smb_share")
await self.middleware.call("smb.import_conf_to_registry")
await self.middleware.call('cache.put', 'SMB_REG_INITIALIZED', True)
os.unlink(SMBPath.SHARECONF.platform())
"""
It is possible that system dataset was migrated or an upgrade
wiped our secrets.tdb file. Re-import directory service secrets
if they are missing from the current running configuration.
"""
job.set_progress(65, 'Initializing directory services')
await self.middleware.call("directoryservices.initialize")
job.set_progress(70, 'Checking SMB server status.')
if await self.middleware.call("service.started", "cifs"):
job.set_progress(80, 'Restarting SMB service.')
await self.middleware.call("service.restart", "cifs")
job.set_progress(100, 'Finished configuring SMB.')
@private
async def get_smb_ha_mode(self):
if await self.middleware.call('cache.has_key', 'SMB_HA_MODE'):
return await self.middleware.call('cache.get', 'SMB_HA_MODE')
if not await self.middleware.call('system.is_freenas'
) and await self.middleware.call(
'failover.licensed'):
system_dataset = await self.middleware.call('systemdataset.config')
if system_dataset['pool'] != await self.middleware.call(
'boot.pool_name'):
hamode = SMBHAMODE['UNIFIED'].name
else:
hamode = SMBHAMODE['LEGACY'].name
else:
hamode = SMBHAMODE['STANDALONE'].name
await self.middleware.call('cache.put', 'SMB_HA_MODE', hamode)
return hamode
@private
async def reset_smb_ha_mode(self):
await self.middleware.call('cache.pop', 'SMB_HA_MODE')
return await self.get_smb_ha_mode()
@private
async def validate_smb(self, new, verrors):
try:
await self.middleware.call('sharing.smb.validate_aux_params',
new['smb_options'],
'smb_update.smb_options')
except ValidationErrors as errs:
verrors.add_child('smb_update.smb_options', errs)
if new.get('unixcharset') and new[
'unixcharset'] not in await self.unixcharset_choices():
verrors.add('smb_update.unixcharset',
'Please provide a valid value for unixcharset')
for i in ('workgroup', 'netbiosname', 'netbiosname_b', 'netbiosalias'):
"""
There are two cases where NetBIOS names must be rejected:
1. They contain invalid characters for NetBIOS protocol
2. The name is identical to the NetBIOS workgroup.
"""
if not i:
continue
if i == 'netbiosalias':
for idx, item in enumerate(new[i]):
if not await self.__validate_netbios_name(item):
verrors.add(f'smb_update.{i}.{idx}',
f'Invalid NetBIOS name: {item}')
if item.casefold() == new['workgroup'].casefold():
verrors.add(
f'smb_update.{i}.{idx}',
f'NetBIOS alias [{item}] conflicts with workgroup name.'
)
else:
if not await self.__validate_netbios_name(new[i]):
verrors.add(f'smb_update.{i}',
f'Invalid NetBIOS name: {new[i]}')
if i != 'workgroup' and new[i].casefold(
) == new['workgroup'].casefold():
verrors.add(
f'smb_update.{i}',
f'NetBIOS name [{new[i]}] conflicts with workgroup name.'
)
if new['guest'] == 'root':
verrors.add('smb_update.guest',
'"root" is not a permitted guest account')
if new.get('bindip'):
bindip_choices = list((await self.bindip_choices()).keys())
for idx, item in enumerate(new['bindip']):
if item not in bindip_choices:
verrors.add(
f'smb_update.bindip.{idx}',
f'IP address [{item}] is not a configured address for this server'
)
for i in ('filemask', 'dirmask'):
if not new[i]:
continue
try:
if int(new[i], 8) & ~0o11777:
raise ValueError('Not an octet')
except (ValueError, TypeError):
verrors.add(f'smb_update.{i}', 'Not a valid mask')
@accepts(
Dict(
'smb_update',
Str('netbiosname', max_length=15),
Str('netbiosname_b', max_length=15),
List('netbiosalias',
default=[],
items=[Str('netbios_alias', max_length=15)]),
Str('workgroup'),
Str('description'),
Bool('enable_smb1'),
Str('unixcharset'),
Str('loglevel',
enum=['NONE', 'MINIMUM', 'NORMAL', 'FULL', 'DEBUG']),
Bool('syslog'),
Bool('aapl_extensions'),
Bool('localmaster'),
Str('guest'),
Str('admin_group', required=False, default=None, null=True),
Str('filemask'),
Str('dirmask'),
Bool('ntlmv1_auth'),
List('bindip', items=[IPAddr('ip')], default=[]),
Str('smb_options', max_length=None),
update=True,
))
async def do_update(self, data):
"""
Update SMB Service Configuration.
`netbiosname` defaults to the original hostname of the system.
`workgroup` and `netbiosname` should have different values.
`enable_smb1` allows legacy SMB clients to connect to the server when enabled.
`localmaster` when set, determines if the system participates in a browser election.
`domain_logons` is used to provide netlogin service for older Windows clients if enabled.
`guest` attribute is specified to select the account to be used for guest access. It defaults to "nobody".
`nullpw` when enabled allows the users to authorize access without a password.
`hostlookup` when enabled, allows using hostnames rather then IP addresses in "hostsallow"/"hostsdeny" fields
of SMB Shares.
"""
old = await self.config()
new = old.copy()
new.update(data)
verrors = ValidationErrors()
await self.validate_smb(new, verrors)
verrors.check()
if new['admin_group'] and new['admin_group'] != old['admin_group']:
await self.middleware.call('smb.add_admin_group',
new['admin_group'])
# TODO: consider using bidict
for k, v in LOGLEVEL_MAP.items():
if new['loglevel'] == v:
new['loglevel'] = k
break
await self.compress(new)
await self._update_service(old, new)
await self.reset_smb_ha_mode()
return await self.config()
@private
async def compress(self, data):
data['netbiosalias'] = ' '.join(data['netbiosalias'])
data.pop('netbiosname_local', None)
data.pop('next_rid')
return data
class SharingSMBService(SharingService):
share_task_type = 'SMB'
class Config:
namespace = 'sharing.smb'
datastore = 'sharing.cifs_share'
datastore_prefix = 'cifs_'
datastore_extend = 'sharing.smb.extend'
@private
async def strip_comments(self, data):
parsed_config = ""
for entry in data['auxsmbconf'].splitlines():
if entry == "" or entry.startswith(('#', ';')):
continue
parsed_config += entry if len(parsed_config) == 0 else f'\n{entry}'
data['auxsmbconf'] = parsed_config
@accepts(
Dict(
'sharingsmb_create',
Str('purpose',
enum=[x.name for x in SMBSharePreset],
default=SMBSharePreset.DEFAULT_SHARE.name),
Str('path', required=True),
Str('path_suffix', default=''),
Bool('home', default=False),
Str('name', max_length=80),
Str('comment', default=''),
Bool('ro', default=False),
Bool('browsable', default=True),
Bool('timemachine', default=False),
Bool('recyclebin', default=False),
Bool('guestok', default=False),
Bool('abe', default=False),
List('hostsallow', default=[]),
List('hostsdeny', default=[]),
Bool('aapl_name_mangling', default=False),
Bool('acl', default=True),
Bool('durablehandle', default=True),
# shadowcopy only available for FreeBSD (for now)
Bool('shadowcopy', default=osc.IS_FREEBSD),
Bool('streams', default=True),
Bool('fsrvp', default=False),
Str('auxsmbconf', max_length=None, default=''),
Bool('enabled', default=True),
register=True))
async def do_create(self, data):
"""
Create a SMB Share.
`purpose` applies common configuration presets depending on intended purpose.
`timemachine` when set, enables Time Machine backups for this share.
`ro` when enabled, prohibits write access to the share.
`guestok` when enabled, allows access to this share without a password.
`hostsallow` is a list of hostnames / IP addresses which have access to this share.
`hostsdeny` is a list of hostnames / IP addresses which are not allowed access to this share. If a handful
of hostnames are to be only allowed access, `hostsdeny` can be passed "ALL" which means that it will deny
access to ALL hostnames except for the ones which have been listed in `hostsallow`.
`acl` enables support for storing the SMB Security Descriptor as a Filesystem ACL.
`streams` enables support for storing alternate datastreams as filesystem extended attributes.
`fsrvp` enables support for the filesystem remote VSS protocol. This allows clients to create
ZFS snapshots through RPC.
`shadowcopy` enables support for the volume shadow copy service.
`auxsmbconf` is a string of additional smb4.conf parameters not covered by the system's API.
"""
verrors = ValidationErrors()
path = data['path']
await self.clean(data, 'sharingsmb_create', verrors)
await self.validate(data, 'sharingsmb_create', verrors)
verrors.check()
if path and not os.path.exists(path):
try:
os.makedirs(path)
except OSError as e:
raise CallError(f'Failed to create {path}: {e}')
await self.apply_presets(data)
await self.compress(data)
vuid = await self.generate_vuid(data['timemachine'])
data.update({'vuid': vuid})
data['id'] = await self.middleware.call(
'datastore.insert', self._config.datastore, data,
{'prefix': self._config.datastore_prefix})
await self.strip_comments(data)
await self.middleware.call('sharing.smb.reg_addshare', data)
enable_aapl = await self.check_aapl(data)
if enable_aapl:
await self._service_change('cifs', 'restart')
else:
await self._service_change('cifs', 'reload')
return await self.get_instance(data['id'])
@accepts(Int('id'),
Patch('sharingsmb_create', 'sharingsmb_update', ('attr', {
'update': True
})))
async def do_update(self, id, data):
"""
Update SMB Share of `id`.
"""
verrors = ValidationErrors()
path = data.get('path')
old = await self.middleware.call(
'datastore.query', self._config.datastore, [('id', '=', id)], {
'extend': self._config.datastore_extend,
'prefix': self._config.datastore_prefix,
'get': True
})
new = old.copy()
new.update(data)
oldname = 'homes' if old['home'] else old['name']
newname = 'homes' if new['home'] else new['name']
new['vuid'] = await self.generate_vuid(new['timemachine'], new['vuid'])
await self.clean(new, 'sharingsmb_update', verrors, id=id)
await self.validate(new, 'sharingsmb_update', verrors, old=old)
if verrors:
raise verrors
if path and not os.path.exists(path):
try:
os.makedirs(path)
except OSError as e:
raise CallError(f'Failed to create {path}: {e}')
if old['purpose'] != new['purpose']:
await self.apply_presets(new)
old_is_locked = (await self.get_instance(id))['locked']
if old['path'] != new['path']:
new_is_locked = await self.middleware.call(
'pool.dataset.path_in_locked_datasets', new['path'])
else:
new_is_locked = old_is_locked
await self.compress(new)
await self.middleware.call('datastore.update', self._config.datastore,
id, new,
{'prefix': self._config.datastore_prefix})
await self.strip_comments(new)
if not new_is_locked:
"""
Enabling AAPL SMB2 extensions globally affects SMB shares. If this
happens, the SMB service _must_ be restarted. Skip this step if dataset
underlying the new path is encrypted.
"""
enable_aapl = await self.check_aapl(new)
else:
enable_aapl = False
"""
OLD NEW = dataset path is encrypted
----------
- - = pre-12 behavior. Remove and replace if name changed, else update.
- X = Delete share from running configuration
X - = Add share to running configuration
X X = no-op
"""
if old_is_locked and new_is_locked:
"""
Configuration change only impacts a locked SMB share. From standpoint of
running config, this is a no-op. No need to restart or reload service.
"""
return await self.get_instance(id)
elif not old_is_locked and not new_is_locked:
"""
Default behavior before changes for locked datasets.
"""
if newname != oldname:
# This is disruptive change. Share is actually being removed and replaced.
# Forcibly closes any existing SMB sessions.
await self.close_share(oldname)
try:
await self.middleware.call('sharing.smb.reg_delshare',
oldname)
except Exception:
self.logger.warning('Failed to remove stale share [%s]',
old['name'],
exc_info=True)
await self.middleware.call('sharing.smb.reg_addshare', new)
else:
diff = await self.middleware.call(
'sharing.smb.diff_middleware_and_registry', new['name'],
new)
if diff is None:
await self.middleware.call('sharing.smb.reg_addshare', new)
else:
share_name = new['name'] if not new['home'] else 'homes'
await self.middleware.call('sharing.smb.apply_conf_diff',
'REGISTRY', share_name, diff)
elif old_is_locked and not new_is_locked:
"""
Since the old share was not in our running configuration, we need
to add it.
"""
await self.middleware.call('sharing.smb.reg_addshare', new)
elif not old_is_locked and new_is_locked:
try:
await self.middleware.call('sharing.smb.reg_delshare', oldname)
except Exception:
self.logger.warning('Failed to remove locked share [%s]',
old['name'],
exc_info=True)
if enable_aapl:
await self._service_change('cifs', 'restart')
else:
await self._service_change('cifs', 'reload')
return await self.get_instance(id)
@accepts(Int('id'))
async def do_delete(self, id):
"""
Delete SMB Share of `id`. This will forcibly disconnect SMB clients
that are accessing the share.
"""
share = await self._get_instance(id)
result = await self.middleware.call('datastore.delete',
self._config.datastore, id)
await self.close_share(share['name'])
try:
await self.middleware.call(
'smb.sharesec._delete',
share['name'] if not share['home'] else 'homes')
except Exception:
self.logger.debug('Failed to delete share ACL for [%s].',
share['name'],
exc_info=True)
try:
await self.middleware.call(
'sharing.smb.reg_delshare',
share['name'] if not share['home'] else 'homes')
except Exception:
self.logger.warn('Failed to remove registry entry for [%s].',
share['name'],
exc_info=True)
if share['timemachine']:
await self.middleware.call('service.restart', 'mdns')
return result
@private
async def check_aapl(self, data):
"""
Returns whether we changed the global aapl support settings.
"""
aapl_extensions = (
await self.middleware.call('smb.config'))['aapl_extensions']
if not aapl_extensions and data['timemachine']:
await self.middleware.call('datastore.update', 'services_cifs', 1,
{'cifs_srv_aapl_extensions': True})
return True
return False
@private
async def close_share(self, share_name):
c = await run(
[SMBCmd.SMBCONTROL.value, 'smbd', 'close-share', share_name],
check=False)
if c.returncode != 0:
self.logger.warn('Failed to close smb share [%s]: [%s]',
share_name,
c.stderr.decode().strip())
@private
async def clean(self, data, schema_name, verrors, id=None):
data['name'] = await self.name_exists(data, schema_name, verrors, id)
@private
async def validate_aux_params(self, data, schema_name):
"""
libsmbconf expects to be provided with key-value pairs.
"""
verrors = ValidationErrors()
for entry in data.splitlines():
if entry == '' or entry.startswith(('#', ';')):
continue
kv = entry.split('=', 1)
if len(kv) != 2:
verrors.add(
f'{schema_name}.auxsmbconf',
f'Auxiliary parameters must be in the format of "key = value": {entry}'
)
continue
verrors.check()
@private
async def validate(self, data, schema_name, verrors, old=None):
home_result = await self.home_exists(data['home'], schema_name,
verrors, old)
if home_result:
verrors.add(f'{schema_name}.home',
'Only one share is allowed to be a home share.')
elif not home_result and not data['path']:
verrors.add(f'{schema_name}.path', 'This field is required.')
if data['path']:
await self.validate_path_field(data, schema_name, verrors)
if data['auxsmbconf']:
try:
await self.validate_aux_params(data['auxsmbconf'],
f'{schema_name}.auxsmbconf')
except ValidationErrors as errs:
verrors.add_child(f'{schema_name}.auxsmbconf', errs)
if not data['acl'] and not await self.middleware.call(
'filesystem.acl_is_trivial', data['path']):
verrors.add(
f'{schema_name}.acl',
f'ACL detected on {data["path"]}. ACLs must be stripped prior to creation '
'of SMB share.')
if data.get('name') and data['name'].lower() in [
'global', 'homes', 'printers'
]:
verrors.add(
f'{schema_name}.name',
f'{data["name"]} is a reserved section name, please select another one'
)
if osc.IS_LINUX:
if data['shadowcopy']:
verrors.add(
f'{schema_name}.shadowcopy',
'ZFS shadow copy support is not yet implemented in TrueNAS scale'
)
if data['fsrvp']:
verrors.add(
f'{schema_name}.fsrvp',
'ZFS fsrvp support is not yet implemented in TrueNAS scale'
)
if data.get('path_suffix') and len(data['path_suffix'].split('/')) > 2:
verrors.add(
f'{schema_name}.name',
'Path suffix may not contain more than two components.')
@private
async def home_exists(self, home, schema_name, verrors, old=None):
home_filters = [('home', '=', True)]
home_result = None
if home:
if old and old['id'] is not None:
id = old['id']
if not old['home']:
home_filters.append(('id', '!=', id))
# The user already had this set as the home share
home_result = await self.middleware.call(
'datastore.query', self._config.datastore,
home_filters,
{'prefix': self._config.datastore_prefix})
return home_result
@private
async def auxsmbconf_dict(self, aux, direction="TO"):
ret = None
if direction == 'TO':
ret = {}
for entry in aux.splitlines():
if entry == '':
continue
if entry.startswith(('#', ';')):
# Special handling for comments
ret[entry] = None
continue
kv = entry.split('=', 1)
ret[kv[0].strip()] = kv[1].strip()
return ret
if direction == 'FROM':
return '\n'.join(
[f'{k}={v}' if v is not None else k for k, v in aux.items()])
@private
async def name_exists(self, data, schema_name, verrors, id=None):
name = data['name']
path = data['path']
if path and not name:
name = path.rsplit('/', 1)[-1]
name_filters = [('name', '=', name)]
if id is not None:
name_filters.append(('id', '!=', id))
name_result = await self.middleware.call(
'datastore.query', self._config.datastore, name_filters,
{'prefix': self._config.datastore_prefix})
if name_result:
verrors.add(f'{schema_name}.name',
'A share with this name already exists.')
return name
@private
async def extend(self, data):
data['hostsallow'] = data['hostsallow'].split()
data['hostsdeny'] = data['hostsdeny'].split()
if data['fsrvp']:
data['shadowcopy'] = True
if 'share_acl' in data:
data.pop('share_acl')
return data
@private
async def compress(self, data):
data['hostsallow'] = ' '.join(data['hostsallow'])
data['hostsdeny'] = ' '.join(data['hostsdeny'])
data.pop(self.locked_field, None)
return data
@private
async def generate_vuid(self, timemachine, vuid=""):
try:
if timemachine and vuid:
uuid.UUID(vuid, version=4)
except ValueError:
self.logger.debug(
f"Time machine VUID string ({vuid}) is invalid. Regenerating.")
vuid = ""
if timemachine and not vuid:
vuid = str(uuid.uuid4())
return vuid
@private
async def apply_presets(self, data):
"""
Apply settings from presets. Only include auxiliary parameters
from preset if user-defined aux parameters already exist. In this
case user-defined takes precedence.
"""
params = (SMBSharePreset[data["purpose"]].value)["params"].copy()
aux = params.pop("auxsmbconf")
data.update(params)
if data["auxsmbconf"]:
preset_aux = await self.auxsmbconf_dict(aux, direction="TO")
data_aux = await self.auxsmbconf_dict(data["auxsmbconf"],
direction="TO")
preset_aux.update(data_aux)
data["auxsmbconf"] = await self.auxsmbconf_dict(preset_aux,
direction="FROM")
return data
@accepts()
async def presets(self):
"""
Retrieve pre-defined configuration sets for specific use-cases. These parameter
combinations are often non-obvious, but beneficial in these scenarios.
"""
return {x.name: x.value for x in SMBSharePreset}
@private
async def sync_registry(self):
"""
Synchronize registry config with the share configuration in the truenas config
file. This method simply reconciles lists of shares, removing from and adding to
the registry as-needed.
"""
if not os.path.exists(SMBPath.GLOBALCONF.platform()):
self.logger.warning(
"smb.conf does not exist. Skipping registry synchronization."
"This may indicate that SMB service has not completed initialization."
)
return
active_shares = await self.query([('locked', '=', False),
('enabled', '=', True)])
registry_shares = await self.middleware.call(
'sharing.smb.reg_listshares')
cf_active = set([x['name'].casefold() for x in active_shares])
cf_reg = set([x.casefold() for x in registry_shares])
to_add = cf_active - cf_reg
to_del = cf_reg - cf_active
for share in to_add:
share_conf = list(
filter(lambda x: x['name'].casefold() == share.casefold(),
active_shares))
if not os.path.exists(share_conf[0]['path']):
self.logger.warning(
"Path [%s] for share [%s] does not exist. "
"Refusing to add share to SMB configuration.",
share_conf[0]['path'], share_conf[0]['name'])
continue
try:
await self.middleware.call('sharing.smb.reg_addshare',
share_conf[0])
except Exception:
self.logger.warning(
"Failed to add SMB share [%] while synchronizing registry config",
share,
exc_info=True)
for share in to_del:
await self.middleware.call('sharing.smb.close_share', share)
try:
await self.middleware.call('sharing.smb.reg_delshare', share)
except Exception:
self.middleware.logger.warning(
'Failed to remove stale share [%s]', share, exc_info=True)
class RsyncTaskService(TaskPathService):
share_task_type = 'Rsync'
class Config:
datastore = 'tasks.rsync'
datastore_prefix = 'rsync_'
datastore_extend = 'rsynctask.rsync_task_extend'
datastore_extend_context = 'rsynctask.rsync_task_extend_context'
cli_namespace = 'task.rsync'
ENTRY = Patch(
'rsync_task_create',
'rsync_task_entry',
('rm', {
'name': 'validate_rpath'
}),
('add', Int('id')),
('add', Bool('locked')),
('add', Dict('job', null=True, additional_attrs=True)),
)
@private
async def rsync_task_extend(self, data, context):
try:
data['extra'] = shlex.split(data['extra'].replace('"',
r'"\"').replace(
"'", r'"\"'))
except ValueError:
# This is to handle the case where the extra value is misconfigured for old cases
# Moving on, we are going to verify that it can be split successfully using shlex
data['extra'] = data['extra'].split()
for field in ('mode', 'direction'):
data[field] = data[field].upper()
Cron.convert_db_format_to_schedule(data)
data['job'] = context['jobs'].get(data['id'])
return data
@private
async def rsync_task_extend_context(self, rows, extra):
jobs = {}
for j in await self.middleware.call("core.get_jobs",
[("method", "=", "rsynctask.run")],
{"order_by": ["id"]}):
try:
task_id = int(j["arguments"][0])
except (IndexError, ValueError):
continue
if task_id in jobs and jobs[task_id]["state"] == "RUNNING":
continue
jobs[task_id] = j
return {
"jobs": jobs,
}
@private
async def validate_rsync_task(self, data, schema):
verrors = ValidationErrors()
# Windows users can have spaces in their usernames
# http://www.freebsd.org/cgi/query-pr.cgi?pr=164808
username = data.get('user')
if ' ' in username:
verrors.add(f'{schema}.user', 'User names cannot have spaces')
raise verrors
user = None
with contextlib.suppress(KeyError):
user = await self.middleware.call('dscache.get_uncached_user',
username)
if not user:
verrors.add(f'{schema}.user',
f'Provided user "{username}" does not exist')
raise verrors
await self.validate_path_field(data, schema, verrors)
remote_host = data.get('remotehost')
if not remote_host:
verrors.add(f'{schema}.remotehost', 'Please specify a remote host')
data['extra'] = ' '.join(data['extra'])
try:
shlex.split(data['extra'].replace('"',
r'"\"').replace("'", r'"\"'))
except ValueError as e:
verrors.add(f'{schema}.extra', f'Please specify valid value: {e}')
mode = data.get('mode')
if not mode:
verrors.add(f'{schema}.mode', 'This field is required')
remote_module = data.get('remotemodule')
if mode == 'MODULE' and not remote_module:
verrors.add(f'{schema}.remotemodule', 'This field is required')
if mode == 'SSH':
remote_port = data.get('remoteport')
if not remote_port:
verrors.add(f'{schema}.remoteport', 'This field is required')
remote_path = data.get('remotepath')
if not remote_path:
verrors.add(f'{schema}.remotepath', 'This field is required')
search = os.path.join(user['pw_dir'], '.ssh', 'id_[edr]*')
exclude_from_search = os.path.join(user['pw_dir'], '.ssh',
'id_[edr]*pub')
key_files = set(glob.glob(search)) - set(
glob.glob(exclude_from_search))
if not key_files:
verrors.add(
f'{schema}.user',
'In order to use rsync over SSH you need a user'
' with a private key (DSA/ECDSA/RSA) set up in home dir.')
else:
for file in glob.glob(search):
if '.pub' not in file:
# file holds a private key and it's permissions should be 600
if os.stat(file).st_mode & 0o077 != 0:
verrors.add(
f'{schema}.user',
f'Permissions {oct(os.stat(file).st_mode & 0o777)} for {file} are too open. Please '
f'correct them by running chmod 600 {file}')
if (data['enabled'] and data['validate_rpath'] and remote_path
and remote_host and remote_port):
if '@' in remote_host:
remote_username, remote_host = remote_host.rsplit('@', 1)
else:
remote_username = username
try:
async with await asyncio.wait_for(
asyncssh.connect(remote_host,
port=remote_port,
username=remote_username,
client_keys=key_files,
known_hosts=None),
timeout=5,
) as conn:
await conn.run(f'test -d {shlex.quote(remote_path)}',
check=True)
except asyncio.TimeoutError:
verrors.add(
f'{schema}.remotehost',
'SSH timeout occurred. Remote path cannot be validated.'
)
except OSError as e:
if e.errno == 113:
verrors.add(
f'{schema}.remotehost',
f'Connection to the remote host {remote_host} on port {remote_port} failed.'
)
else:
verrors.add(f'{schema}.remotehost', e.__str__())
except asyncssh.DisconnectError as e:
verrors.add(
f'{schema}.remotehost',
f'Disconnect Error[ error code {e.code} ] was generated when trying to '
f'communicate with remote host {remote_host} and remote user {remote_username}.'
)
except asyncssh.ProcessError as e:
if e.code == '1':
verrors.add(
f'{schema}.remotepath',
'The Remote Path you specified does not exist or is not a directory.'
'Either create one yourself on the remote machine or uncheck the '
'validate_rpath field')
else:
verrors.add(
f'{schema}.remotepath',
f'Connection to Remote Host was successful but failed to verify '
f'Remote Path. {e.__str__()}')
except asyncssh.Error as e:
if e.__class__.__name__ in e.__str__():
exception_reason = e.__str__()
else:
exception_reason = e.__class__.__name__ + ' ' + e.__str__(
)
verrors.add(
f'{schema}.remotepath',
f'Remote Path could not be validated. An exception was raised. {exception_reason}'
)
elif data['enabled'] and data['validate_rpath']:
verrors.add(
f'{schema}.remotepath',
'Remote path could not be validated because of missing fields'
)
data.pop('validate_rpath', None)
# Keeping compatibility with legacy UI
for field in ('mode', 'direction'):
data[field] = data[field].lower()
return verrors, data
@accepts(
Dict(
'rsync_task_create',
Str('path', required=True, max_length=RSYNC_PATH_LIMIT),
Str('user', required=True),
Str('remotehost'),
Int('remoteport'),
Str('mode', enum=['MODULE', 'SSH'], default='MODULE'),
Str('remotemodule'),
Str('remotepath'),
Bool('validate_rpath', default=True),
Str('direction', enum=['PULL', 'PUSH'], default='PUSH'),
Str('desc'),
Cron(
'schedule',
defaults={'minute': '00'},
),
Bool('recursive'),
Bool('times'),
Bool('compress'),
Bool('archive'),
Bool('delete'),
Bool('quiet'),
Bool('preserveperm'),
Bool('preserveattr'),
Bool('delayupdates'),
List('extra', items=[Str('extra')]),
Bool('enabled', default=True),
register=True,
))
async def do_create(self, data):
"""
Create a Rsync Task.
See the comment in Rsyncmod about `path` length limits.
`remotehost` is ip address or hostname of the remote system. If username differs on the remote host,
"username@remote_host" format should be used.
`mode` represents different operating mechanisms for Rsync i.e Rsync Module mode / Rsync SSH mode.
`remotemodule` is the name of remote module, this attribute should be specified when `mode` is set to MODULE.
`remotepath` specifies the path on the remote system.
`validate_rpath` is a boolean which when sets validates the existence of the remote path.
`direction` specifies if data should be PULLED or PUSHED from the remote system.
`compress` when set reduces the size of the data which is to be transmitted.
`archive` when set makes rsync run recursively, preserving symlinks, permissions, modification times, group,
and special files.
`delete` when set deletes files in the destination directory which do not exist in the source directory.
`preserveperm` when set preserves original file permissions.
.. examples(websocket)::
Create a Rsync Task which pulls data from a remote system every 5 minutes.
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "rsynctask.create",
"params": [{
"enabled": true,
"schedule": {
"minute": "5",
"hour": "*",
"dom": "*",
"month": "*",
"dow": "*"
},
"desc": "Test rsync task",
"user": "******",
"mode": "MODULE",
"remotehost": "[email protected]",
"compress": true,
"archive": true,
"direction": "PULL",
"path": "/mnt/vol1/rsync_dataset",
"remotemodule": "remote_module1"
}]
}
"""
verrors, data = await self.validate_rsync_task(data,
'rsync_task_create')
if verrors:
raise verrors
Cron.convert_schedule_to_db_format(data)
data['id'] = await self.middleware.call(
'datastore.insert', self._config.datastore, data,
{'prefix': self._config.datastore_prefix})
await self.middleware.call('service.restart', 'cron')
return await self.get_instance(data['id'])
@accepts(Int('id', validators=[Range(min=1)]),
Patch('rsync_task_create', 'rsync_task_update', ('attr', {
'update': True
})))
async def do_update(self, id, data):
"""
Update Rsync Task of `id`.
"""
old = await self.query(filters=[('id', '=', id)],
options={'get': True})
old.pop('job')
new = old.copy()
new.update(data)
verrors, data = await self.validate_rsync_task(new,
'rsync_task_update')
if verrors:
raise verrors
Cron.convert_schedule_to_db_format(new)
new.pop(self.locked_field)
await self.middleware.call('datastore.update', self._config.datastore,
id, new,
{'prefix': self._config.datastore_prefix})
await self.middleware.call('service.restart', 'cron')
return await self.get_instance(id)
async def do_delete(self, id):
"""
Delete Rsync Task of `id`.
"""
res = await self.middleware.call('datastore.delete',
self._config.datastore, id)
await self.middleware.call('service.restart', 'cron')
return res
@private
async def commandline(self, id):
"""
Helper method to generate the rsync command avoiding code duplication.
"""
rsync = await self.get_instance(id)
path = shlex.quote(rsync['path'])
line = ['rsync']
for name, flag in (
('archive', '-a'),
('compress', '-zz'),
('delayupdates', '--delay-updates'),
('delete', '--delete-delay'),
('preserveattr', '-X'),
('preserveperm', '-p'),
('recursive', '-r'),
('times', '-t'),
):
if rsync[name]:
line.append(flag)
if rsync['extra']:
line.append(' '.join(rsync['extra']))
# Do not use username if one is specified in host field
# See #5096 for more details
if '@' in rsync['remotehost']:
remote = rsync['remotehost']
else:
remote = f'"{rsync["user"]}"@{rsync["remotehost"]}'
if rsync['mode'] == 'MODULE':
module_args = [path, f'{remote}::"{rsync["remotemodule"]}"']
if rsync['direction'] != 'PUSH':
module_args.reverse()
line += module_args
else:
line += [
'-e',
f'"ssh -p {rsync["remoteport"]} -o BatchMode=yes -o StrictHostKeyChecking=yes"'
]
path_args = [
path, f'{remote}:"{shlex.quote(rsync["remotepath"])}"'
]
if rsync['direction'] != 'PUSH':
path_args.reverse()
line += path_args
if rsync['quiet']:
line += ['>', '/dev/null', '2>&1']
return ' '.join(line)
@item_method
@accepts(Int('id'))
@returns()
@job(lock=lambda args: args[-1], lock_queue_size=1, logs=True)
def run(self, job, id):
"""
Job to run rsync task of `id`.
Output is saved to job log excerpt (not syslog).
"""
self.middleware.call_sync('network.general.will_perform_activity',
'rsync')
rsync = self.middleware.call_sync('rsynctask.get_instance', id)
if rsync['locked']:
self.middleware.call_sync('rsynctask.generate_locked_alert', id)
return
commandline = self.middleware.call_sync('rsynctask.commandline', id)
cp = run_command_with_user_context(commandline, rsync['user'],
lambda v: job.logs_fd.write(v))
for klass in ('RsyncSuccess',
'RsyncFailed') if not rsync['quiet'] else ():
self.middleware.call_sync('alert.oneshot_delete', klass,
rsync['id'])
if cp.returncode not in RsyncReturnCode.nonfatals():
err = None
if cp.returncode == RsyncReturnCode.STREAMIO and rsync['compress']:
err = (
"rsync command with compression enabled failed with STREAMIO error. "
"This may indicate that remote server lacks support for the new-style "
"compression used by TrueNAS.")
if not rsync['quiet']:
self.middleware.call_sync(
'alert.oneshot_create', 'RsyncFailed', {
'id': rsync['id'],
'direction': rsync['direction'],
'path': rsync['path'],
})
if err:
msg = f'{err} Check logs for further information'
else:
try:
rc_name = RsyncReturnCode(cp.returncode).name
except ValueError:
rc_name = 'UNKNOWN'
msg = (f'rsync command returned {cp.returncode} - {rc_name}. '
'Check logs for further information.')
raise CallError(msg)
elif not rsync['quiet']:
self.middleware.call_sync(
'alert.oneshot_create', 'RsyncSuccess', {
'id': rsync['id'],
'direction': rsync['direction'],
'path': rsync['path'],
})
class CloudSyncService(CRUDService):
local_fs_lock_manager = FsLockManager()
remote_fs_lock_manager = FsLockManager()
class Config:
datastore = "tasks.cloudsync"
datastore_extend = "cloudsync._extend"
@filterable
async def query(self, filters=None, options=None):
"""
Query all Cloud Sync Tasks with `query-filters` and `query-options`.
"""
tasks_or_task = await super().query(filters, options)
jobs = {}
for j in await self.middleware.call("core.get_jobs", [("method", "=", "cloudsync.sync")],
{"order_by": ["id"]}):
try:
task_id = int(j["arguments"][0])
except (IndexError, ValueError):
continue
if task_id in jobs and jobs[task_id]["state"] == "RUNNING":
continue
jobs[task_id] = j
if isinstance(tasks_or_task, list):
for task in tasks_or_task:
task["job"] = jobs.get(task["id"])
else:
tasks_or_task["job"] = jobs.get(tasks_or_task["id"])
return tasks_or_task
@private
async def _extend(self, cloud_sync):
cloud_sync["credentials"] = cloud_sync.pop("credential")
cloud_sync["encryption_password"] = await self.middleware.call(
"pwenc.decrypt", cloud_sync["encryption_password"]
)
cloud_sync["encryption_salt"] = await self.middleware.call(
"pwenc.decrypt", cloud_sync["encryption_salt"]
)
Cron.convert_db_format_to_schedule(cloud_sync)
return cloud_sync
@private
async def _compress(self, cloud_sync):
cloud_sync["credential"] = cloud_sync.pop("credentials")
cloud_sync["encryption_password"] = await self.middleware.call(
"pwenc.encrypt", cloud_sync["encryption_password"]
)
cloud_sync["encryption_salt"] = await self.middleware.call(
"pwenc.encrypt", cloud_sync["encryption_salt"]
)
Cron.convert_schedule_to_db_format(cloud_sync)
cloud_sync.pop('job', None)
return cloud_sync
@private
async def _get_credentials(self, credentials_id):
try:
return await self.middleware.call("datastore.query", "system.cloudcredentials",
[("id", "=", credentials_id)], {"get": True})
except IndexError:
return None
@private
async def _basic_validate(self, verrors, name, data):
if data["encryption"]:
if not data["encryption_password"]:
verrors.add(f"{name}.encryption_password", "This field is required when encryption is enabled")
credentials = await self._get_credentials(data["credentials"])
if not credentials:
verrors.add(f"{name}.credentials", "Invalid credentials")
try:
shlex.split(data["args"])
except ValueError as e:
verrors.add(f"{name}.args", f"Parse error: {e.args[0]}")
if verrors:
raise verrors
provider = REMOTES[credentials["provider"]]
schema = []
if provider.buckets:
schema.append(Str("bucket", required=True, empty=False))
schema.append(Str("folder", required=True))
schema.extend(provider.task_schema)
schema.extend(self.common_task_schema(provider))
attributes_verrors = validate_attributes(schema, data, additional_attrs=True)
if not attributes_verrors:
await provider.pre_save_task(data, credentials, verrors)
verrors.add_child(f"{name}.attributes", attributes_verrors)
@private
async def _validate(self, verrors, name, data):
await self._basic_validate(verrors, name, data)
for i, (limit1, limit2) in enumerate(zip(data["bwlimit"], data["bwlimit"][1:])):
if limit1["time"] >= limit2["time"]:
verrors.add(f"{name}.bwlimit.{i + 1}.time", f"Invalid time order: {limit1['time']}, {limit2['time']}")
if data["snapshot"]:
if data["direction"] != "PUSH":
verrors.add(f"{name}.snapshot", "This option can only be enabled for PUSH tasks")
@private
async def _validate_folder(self, verrors, name, data):
if data["direction"] == "PULL":
folder = data["attributes"]["folder"].rstrip("/")
if folder:
folder_parent = os.path.normpath(os.path.join(folder, ".."))
if folder_parent == ".":
folder_parent = ""
folder_basename = os.path.basename(folder)
ls = await self.list_directory(dict(
credentials=data["credentials"],
encryption=data["encryption"],
filename_encryption=data["filename_encryption"],
encryption_password=data["encryption_password"],
encryption_salt=data["encryption_salt"],
attributes=dict(data["attributes"], folder=folder_parent),
args=data["args"],
))
for item in ls:
if item["Name"] == folder_basename:
if not item["IsDir"]:
verrors.add(f"{name}.attributes.folder", "This is not a directory")
break
else:
verrors.add(f"{name}.attributes.folder", "Directory does not exist")
if data["direction"] == "PUSH":
credentials = await self._get_credentials(data["credentials"])
provider = REMOTES[credentials["provider"]]
if provider.readonly:
verrors.add(f"{name}.direction", "This remote is read-only")
@accepts(Dict(
"cloud_sync_create",
Str("description", default=""),
Str("direction", enum=["PUSH", "PULL"], required=True),
Str("transfer_mode", enum=["SYNC", "COPY", "MOVE"], required=True),
Str("path", required=True),
Int("credentials", required=True),
Bool("encryption", default=False),
Bool("filename_encryption", default=False),
Str("encryption_password", default=""),
Str("encryption_salt", default=""),
Cron(
"schedule",
defaults={"minute": "00"},
required=True
),
Bool("follow_symlinks", default=False),
Int("transfers", null=True, default=None, validators=[Range(min=1)]),
List("bwlimit", default=[], items=[Dict("cloud_sync_bwlimit",
Str("time", validators=[Time()]),
Int("bandwidth", validators=[Range(min=1)], null=True))]),
List("exclude", default=[], items=[Str("path", empty=False)]),
Dict("attributes", additional_attrs=True, required=True),
Bool("snapshot", default=False),
Str("pre_script", default=""),
Str("post_script", default=""),
Str("args", default=""),
Bool("enabled", default=True),
register=True,
))
async def do_create(self, cloud_sync):
"""
Creates a new cloud_sync entry.
.. examples(websocket)::
Create a new cloud_sync using amazon s3 attributes, which is supposed to run every hour.
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "cloudsync.create",
"params": [{
"description": "s3 sync",
"path": "/mnt/tank",
"credentials": 1,
"minute": "00",
"hour": "*",
"daymonth": "*",
"month": "*",
"attributes": {
"bucket": "mybucket",
"folder": ""
},
"enabled": true
}]
}
"""
verrors = ValidationErrors()
await self._validate(verrors, "cloud_sync", cloud_sync)
if verrors:
raise verrors
await self._validate_folder(verrors, "cloud_sync", cloud_sync)
if verrors:
raise verrors
cloud_sync = await self._compress(cloud_sync)
cloud_sync["id"] = await self.middleware.call("datastore.insert", "tasks.cloudsync", cloud_sync)
await self.middleware.call("service.restart", "cron")
cloud_sync = await self._extend(cloud_sync)
return cloud_sync
@accepts(Int("id"), Patch("cloud_sync_create", "cloud_sync_update", ("attr", {"update": True})))
async def do_update(self, id, data):
"""
Updates the cloud_sync entry `id` with `data`.
"""
cloud_sync = await self._get_instance(id)
# credentials is a foreign key for now
if cloud_sync["credentials"]:
cloud_sync["credentials"] = cloud_sync["credentials"]["id"]
cloud_sync.update(data)
verrors = ValidationErrors()
await self._validate(verrors, "cloud_sync_update", cloud_sync)
if verrors:
raise verrors
await self._validate_folder(verrors, "cloud_sync_update", cloud_sync)
if verrors:
raise verrors
cloud_sync = await self._compress(cloud_sync)
await self.middleware.call("datastore.update", "tasks.cloudsync", id, cloud_sync)
await self.middleware.call("service.restart", "cron")
cloud_sync = await self._extend(cloud_sync)
return cloud_sync
@accepts(Int("id"))
async def do_delete(self, id):
"""
Deletes cloud_sync entry `id`.
"""
await self.middleware.call("datastore.delete", "tasks.cloudsync", id)
await self.middleware.call("alert.oneshot_delete", "CloudSyncTaskFailed", id)
await self.middleware.call("service.restart", "cron")
@accepts(Int("credentials_id"))
async def list_buckets(self, credentials_id):
credentials = await self._get_credentials(credentials_id)
if not credentials:
raise CallError("Invalid credentials")
provider = REMOTES[credentials["provider"]]
if not provider.buckets:
raise CallError("This provider does not use buckets")
return await self.ls({"credentials": credentials}, "")
@accepts(Dict(
"cloud_sync_ls",
Int("credentials", required=True),
Bool("encryption", default=False),
Bool("filename_encryption", default=False),
Str("encryption_password", default=""),
Str("encryption_salt", default=""),
Dict("attributes", required=True, additional_attrs=True),
Str("args", default=""),
))
async def list_directory(self, cloud_sync):
"""
List contents of a remote bucket / directory.
If remote supports buckets, path is constructed by two keys "bucket"/"folder" in `attributes`.
If remote does not support buckets, path is constructed using "folder" key only in `attributes`.
"folder" is directory name and "bucket" is bucket name for remote.
Path examples:
S3 Service
`bucketname/directory/name`
Dropbox Service
`directory/name`
`credentials` is a valid id of a Cloud Sync Credential which will be used to connect to the provider.
"""
verrors = ValidationErrors()
await self._basic_validate(verrors, "cloud_sync", dict(cloud_sync))
if verrors:
raise verrors
credentials = await self._get_credentials(cloud_sync["credentials"])
path = get_remote_path(REMOTES[credentials["provider"]], cloud_sync["attributes"])
return await self.ls(dict(cloud_sync, credentials=credentials), path)
@private
async def ls(self, config, path):
async with RcloneConfig(config) as config:
proc = await run(["rclone", "--config", config.config_path, "lsjson", "remote:" + path],
check=False, encoding="utf8")
if proc.returncode == 0:
return json.loads(proc.stdout)
else:
raise CallError(proc.stderr)
@item_method
@accepts(Int("id"))
@job(lock=lambda args: "cloud_sync:{}".format(args[-1]), lock_queue_size=1, logs=True)
async def sync(self, job, id):
"""
Run the cloud_sync job `id`, syncing the local data to remote.
"""
cloud_sync = await self._get_instance(id)
credentials = cloud_sync["credentials"]
local_path = cloud_sync["path"]
local_direction = FsLockDirection.READ if cloud_sync["direction"] == "PUSH" else FsLockDirection.WRITE
remote_path = get_remote_path(REMOTES[credentials["provider"]], cloud_sync["attributes"])
remote_direction = FsLockDirection.READ if cloud_sync["direction"] == "PULL" else FsLockDirection.WRITE
directions = {
FsLockDirection.READ: "reading",
FsLockDirection.WRITE: "writing",
}
job.set_progress(0, f"Locking local path {local_path!r} for {directions[local_direction]}")
async with self.local_fs_lock_manager.lock(local_path, local_direction):
job.set_progress(0, f"Locking remote path {remote_path!r} for {directions[remote_direction]}")
async with self.remote_fs_lock_manager.lock(f"{credentials['id']}/{remote_path}", remote_direction):
job.set_progress(0, "Starting")
try:
await rclone(self.middleware, job, cloud_sync)
await self.middleware.call("alert.oneshot_delete", "CloudSyncTaskFailed", cloud_sync["id"])
except Exception:
await self.middleware.call("alert.oneshot_create", "CloudSyncTaskFailed", {
"id": cloud_sync["id"],
"name": cloud_sync["description"],
})
raise
@accepts()
async def providers(self):
"""
Returns a list of dictionaries of supported providers for Cloud Sync Tasks.
`credentials_schema` is JSON schema for credentials attributes.
`task_schema` is JSON schema for task attributes.
`buckets` is a boolean value which is set to "true" if provider supports buckets.
Example of a single provider:
[
{
"name": "AMAZON_CLOUD_DRIVE",
"title": "Amazon Cloud Drive",
"credentials_schema": [
{
"property": "client_id",
"schema": {
"title": "Amazon Application Client ID",
"_required_": true,
"type": "string"
}
},
{
"property": "client_secret",
"schema": {
"title": "Application Key",
"_required_": true,
"type": "string"
}
}
],
"credentials_oauth": null,
"buckets": false,
"bucket_title": "Bucket",
"task_schema": []
}
]
"""
return sorted(
[
{
"name": provider.name,
"title": provider.title,
"credentials_schema": [
{
"property": field.name,
"schema": field.to_json_schema()
}
for field in provider.credentials_schema
],
"credentials_oauth": f"{OAUTH_URL}/{provider.name.lower()}" if provider.credentials_oauth else None,
"buckets": provider.buckets,
"bucket_title": provider.bucket_title,
"task_schema": [
{
"property": field.name,
"schema": field.to_json_schema()
}
for field in provider.task_schema + self.common_task_schema(provider)
],
}
for provider in REMOTES.values()
],
key=lambda provider: provider["title"].lower()
)
def common_task_schema(self, provider):
schema = []
if provider.fast_list:
schema.append(Bool("fast_list", default=False, title="Use --fast-list", description=textwrap.dedent("""\
Use fewer transactions in exchange for more RAM. This may also speed up or slow down your
transfer. See [rclone documentation](https://rclone.org/docs/#fast-list) for more details.
""").rstrip()))
return schema
class RsyncModService(SharingService):
share_task_type = 'Rsync Module'
class Config:
datastore = 'services.rsyncmod'
datastore_prefix = 'rsyncmod_'
datastore_extend = 'rsyncmod.rsync_mod_extend'
cli_namespace = 'service.rsync_mod'
ENTRY = Patch(
'rsyncmod_create',
'rsyncmod_entry',
('add', Bool('locked')),
('add', Int('id')),
)
@private
async def rsync_mod_extend(self, data):
data['hostsallow'] = data['hostsallow'].split()
data['hostsdeny'] = data['hostsdeny'].split()
data['mode'] = data['mode'].upper()
return data
@private
async def common_validation(self, data, schema_name):
verrors = ValidationErrors()
await self.validate_path_field(data, schema_name, verrors)
for entity in ('user', 'group'):
value = data.get(entity)
try:
await self.middleware.call(f'{entity}.get_{entity}_obj',
{f'{entity}name': value})
except Exception:
verrors.add(f'{schema_name}.{entity}',
f'Please specify a valid {entity}')
verrors.check()
data['hostsallow'] = ' '.join(data['hostsallow'])
data['hostsdeny'] = ' '.join(data['hostsdeny'])
data['mode'] = data['mode'].lower()
return data
@accepts(
Dict(
'rsyncmod_create',
Bool('enabled', default=True),
Str('name', validators=[Match(r'[^/\]]')]),
Str('comment'),
Str('path', required=True, max_length=RSYNC_PATH_LIMIT),
Str('mode', enum=['RO', 'RW', 'WO'], required=True),
Int('maxconn'),
Str('user', default='nobody'),
Str('group', default='nobody'),
List('hostsallow', items=[Str('hostsallow')]),
List('hostsdeny', items=[Str('hostdeny')]),
Str('auxiliary', max_length=None),
register=True,
))
async def do_create(self, data):
"""
Create a Rsyncmod module.
`path` represents the path to a dataset. Path length is limited to 1023 characters maximum as per the limit
enforced by FreeBSD. It is possible that we reach this max length recursively while transferring data. In that
case, the user must ensure the maximum path will not be too long or modify the recursed path to shorter
than the limit.
`maxconn` is an integer value representing the maximum number of simultaneous connections. Zero represents
unlimited.
`hostsallow` is a list of patterns to match hostname/ip address of a connecting client. If list is empty,
all hosts are allowed.
`hostsdeny` is a list of patterns to match hostname/ip address of a connecting client. If the pattern is
matched, access is denied to the client. If no client should be denied, this should be left empty.
`auxiliary` attribute can be used to pass on any additional parameters from rsyncd.conf(5).
"""
data = await self.common_validation(data, 'rsyncmod_create')
data['id'] = await self.middleware.call(
'datastore.insert', self._config.datastore, data,
{'prefix': self._config.datastore_prefix})
await self._service_change('rsync', 'reload')
return await self.get_instance(data['id'])
@accepts(Int('id'),
Patch('rsyncmod_create', 'rsyncmod_update', ('attr', {
'update': True
})))
async def do_update(self, id, data):
"""
Update Rsyncmod module of `id`.
"""
module = await self.get_instance(id)
module.update(data)
module = await self.common_validation(module, 'rsyncmod_update')
module.pop(self.locked_field)
await self.middleware.call('datastore.update', self._config.datastore,
id, module,
{'prefix': self._config.datastore_prefix})
await self._service_change('rsync', 'reload')
return await self.get_instance(id)
async def do_delete(self, id):
"""
Delete Rsyncmod module of `id`.
"""
return await self.middleware.call('datastore.delete',
self._config.datastore, id)
class ACLBase(ServicePartBase):
@accepts(
Dict(
'filesystem_acl',
Str('path', required=True),
Int('uid', null=True, default=None),
Int('gid', null=True, default=None),
List(
'dacl',
items=[
Dict(
'aclentry',
Str('tag', enum=['owner@', 'group@', 'everyone@', 'USER', 'GROUP']),
Int('id', null=True),
Str('type', enum=['ALLOW', 'DENY']),
Dict(
'perms',
Bool('READ_DATA'),
Bool('WRITE_DATA'),
Bool('APPEND_DATA'),
Bool('READ_NAMED_ATTRS'),
Bool('WRITE_NAMED_ATTRS'),
Bool('EXECUTE'),
Bool('DELETE_CHILD'),
Bool('READ_ATTRIBUTES'),
Bool('WRITE_ATTRIBUTES'),
Bool('DELETE'),
Bool('READ_ACL'),
Bool('WRITE_ACL'),
Bool('WRITE_OWNER'),
Bool('SYNCHRONIZE'),
Str('BASIC', enum=['FULL_CONTROL', 'MODIFY', 'READ', 'TRAVERSE']),
),
Dict(
'flags',
Bool('FILE_INHERIT'),
Bool('DIRECTORY_INHERIT'),
Bool('NO_PROPAGATE_INHERIT'),
Bool('INHERIT_ONLY'),
Bool('INHERITED'),
Str('BASIC', enum=['INHERIT', 'NOINHERIT']),
),
),
Dict(
'posix1e_ace',
Bool('default', default=False),
Str('tag', enum=['USER_OBJ', 'GROUP_OBJ', 'USER', 'GROUP', 'OTHER', 'MASK']),
Int('id', default=-1),
Dict(
'perms',
Bool('READ', default=False),
Bool('WRITE', default=False),
Bool('EXECUTE', default=False),
),
)
],
default=[]
),
Dict(
'nfs41_flags',
Bool('autoinherit', default=False),
Bool('protected', default=False),
),
Str('acltype', enum=[x.name for x in ACLType], default=ACLType.NFS4.name),
Dict(
'options',
Bool('stripacl', default=False),
Bool('recursive', default=False),
Bool('traverse', default=False),
Bool('canonicalize', default=True)
)
)
)
@job(lock="perm_change")
def setacl(self, job, data):
"""
Set ACL of a given path. Takes the following parameters:
`path` full path to directory or file.
`dacl` ACL entries. Formatting depends on the underlying `acltype`. NFS4ACL requires
NFSv4 entries. POSIX1e requires POSIX1e entries.
`uid` the desired UID of the file user. If set to None (the default), then user is not changed.
`gid` the desired GID of the file group. If set to None (the default), then group is not changed.
`recursive` apply the ACL recursively
`traverse` traverse filestem boundaries (ZFS datasets)
`strip` convert ACL to trivial. ACL is trivial if it can be expressed as a file mode without
losing any access rules.
`canonicalize` reorder ACL entries so that they are in concanical form as described
in the Microsoft documentation MS-DTYP 2.4.5 (ACL). This only applies to NFSv4 ACLs.
For case of NFSv4 ACLs USER_OBJ, GROUP_OBJ, and EVERYONE with owner@, group@, everyone@ for
consistency with getfacl and setfacl. If one of aforementioned special tags is used, 'id' must
be set to None.
An inheriting empty everyone@ ACE is appended to non-trivial ACLs in order to enforce Windows
expectations regarding permissions inheritance. This entry is removed from NT ACL returned
to SMB clients when 'ixnas' samba VFS module is enabled.
"""
@accepts(
Str('path'),
Bool('simplified', default=True),
)
def getacl(self, path, simplified=True):
"""
Return ACL of a given path. This may return a POSIX1e ACL or a NFSv4 ACL. The acl type is indicated
by the `ACLType` key.
Errata about ACLType NFSv4:
`simplified` returns a shortened form of the ACL permset and flags.
`TRAVERSE` sufficient rights to traverse a directory, but not read contents.
`READ` sufficient rights to traverse a directory, and read file contents.
`MODIFIY` sufficient rights to traverse, read, write, and modify a file. Equivalent to modify_set.
`FULL_CONTROL` all permissions.
If the permisssions do not fit within one of the pre-defined simplified permissions types, then
the full ACL entry will be returned.
In all cases we replace USER_OBJ, GROUP_OBJ, and EVERYONE with owner@, group@, everyone@ for
consistency with getfacl and setfacl. If one of aforementioned special tags is used, 'id' must
be set to None.
An inheriting empty everyone@ ACE is appended to non-trivial ACLs in order to enforce Windows
expectations regarding permissions inheritance. This entry is removed from NT ACL returned
to SMB clients when 'ixnas' samba VFS module is enabled. We also remove it here to avoid confusion.
"""
@accepts(
Dict(
'filesystem_ownership',
Str('path', required=True),
Int('uid', null=True, default=None),
Int('gid', null=True, default=None),
Dict(
'options',
Bool('recursive', default=False),
Bool('traverse', default=False)
)
)
)
@job(lock="perm_change")
def chown(self, job, data):
"""
Change owner or group of file at `path`.
`uid` and `gid` specify new owner of the file. If either
key is absent or None, then existing value on the file is not
changed.
`recursive` performs action recursively, but does
not traverse filesystem mount points.
If `traverse` and `recursive` are specified, then the chown
operation will traverse filesystem mount points.
"""
@accepts(
Dict(
'filesystem_permission',
Str('path', required=True),
UnixPerm('mode', null=True),
Int('uid', null=True, default=None),
Int('gid', null=True, default=None),
Dict(
'options',
Bool('stripacl', default=False),
Bool('recursive', default=False),
Bool('traverse', default=False),
)
)
)
@job(lock="perm_change")
def setperm(self, job, data):
"""
Remove extended ACL from specified path.
If `mode` is specified then the mode will be applied to the
path and files and subdirectories depending on which `options` are
selected. Mode should be formatted as string representation of octal
permissions bits.
`uid` the desired UID of the file user. If set to None (the default), then user is not changed.
`gid` the desired GID of the file group. If set to None (the default), then group is not changed.
`stripacl` setperm will fail if an extended ACL is present on `path`,
unless `stripacl` is set to True.
`recursive` remove ACLs recursively, but do not traverse dataset
boundaries.
`traverse` remove ACLs from child datasets.
If no `mode` is set, and `stripacl` is True, then non-trivial ACLs
will be converted to trivial ACLs. An ACL is trivial if it can be
expressed as a file mode without losing any access rules.
"""
@accepts()
async def default_acl_choices(self):
"""
Get list of default ACL types.
"""
@accepts(
Str('acl_type', default='OPEN', enum=ACLDefault.options()),
Str('share_type', default='NONE', enum=['NONE', 'AFP', 'SMB', 'NFS']),
)
async def get_default_acl(self, acl_type, share_type):
"""
'options.type',
f'Requested ICMPv6 protocol, but the address provided "{addr}" is not a valid IPv6 address.'
)
verrors.check()
ping_host = False
if addr.version == 4:
ping_host = self._ping_host(ip, options['timeout'])
elif addr.version == 6:
ping_host = self._ping6_host(ip, options['timeout'])
return ping_host
@accepts(
Str('method'),
List('args'),
Str('filename'),
)
async def download(self, method, args, filename):
"""
Core helper to call a job marked for download.
Returns the job id and the URL for download.
"""
job = await self.middleware.call(method, *args, pipes=Pipes(output=self.middleware.pipe()))
token = await self.middleware.call('auth.generate_token', 300, {'filename': filename, 'job': job.id})
self.middleware.fileapp.register_job(job.id)
return job.id, f'/_download/{job.id}?auth_token={token}'
def __kill_multiprocessing(self):
# We need to kill this because multiprocessing has passed it stderr fd which is /var/log/middlewared.log
class KubernetesNodeService(ConfigService):
class Config:
namespace = 'k8s.node'
private = True
async def config(self):
try:
async with api_client({'node': True}) as (api, context):
return {
'node_configured': True,
'events': await self.middleware.call('k8s.event.query', [], {
'extra': {'field_selector': f'involvedObject.uid={NODE_NAME}'}
}),
**(context['node'].to_dict())
}
except Exception as e:
return {'node_configured': False, 'error': str(e)}
@accepts(
List(
'add_taints',
items=[Dict(
'taint',
Str('key', required=True, empty=False),
Str('value', null=True, default=None),
Str('effect', required=True, empty=False, enum=['NoSchedule', 'NoExecute'])
)],
)
)
async def add_taints(self, taints):
async with api_client({'node': True}) as (api, context):
for taint in taints:
await nodes.add_taint(context['core_api'], taint, context['node'])
remaining_taints = {t['key'] for t in taints}
timeout = 600
while remaining_taints and timeout > 0:
await asyncio.sleep(3)
timeout -= 3
config = await self.config()
if not config['node_configured']:
break
remaining_taints -= {t['key'] for t in (config['spec']['taints'] or [])}
@accepts(
List('remove_taints', items=[Str('taint_key')]),
)
async def remove_taints(self, taint_keys):
async with api_client({'node': True}) as (api, context):
for taint_key in taint_keys:
await nodes.remove_taint(context['core_api'], taint_key, context['node'])
@accepts()
async def delete_node(self):
async with api_client({'node': True}) as (api, context):
await context['core_api'].delete_node(NODE_NAME)
@accepts()
async def worker_node_password(self):
return KUBERNETES_WORKER_NODE_PASSWORD
class StatsService(Service):
@accepts()
def get_sources(self):
"""
Returns an object with all available sources tried with metric datasets.
"""
sources = {}
if not os.path.exists(RRD_PATH):
return {}
for i in glob.glob('{}/*/*.rrd'.format(RRD_PATH)):
source, metric = i.replace(RRD_PATH, '').split('/', 1)
if metric.endswith('.rrd'):
metric = metric[:-4]
if source not in sources:
sources[source] = []
sources[source].append(metric)
return sources
@accepts(Str('source'), Str('type'))
async def get_dataset_info(self, source, _type):
"""
Returns info about a given dataset from some source.
"""
rrdfile = '{}/{}/{}.rrd'.format(RRD_PATH, source, _type)
proc = await Popen(
['/usr/local/bin/rrdtool', 'info', rrdfile],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
data, err = await proc.communicate()
if proc.returncode != 0:
raise ValueError('rrdtool failed: {}'.format(err.decode()))
data = data.decode()
info = {'source': source, 'type': _type, 'datasets': {}}
for dataset, _type in RE_DSTYPE.findall(data):
info['datasets'][dataset] = {'type': _type}
reg = RE_STEP.search(data)
if reg:
info['step'] = int(reg.group(1))
reg = RE_LAST_UPDATE.search(data)
if reg:
info['last_update'] = int(reg.group(1))
return info
@accepts(
List('stats_list',
items=[
Dict(
'stats-data',
Str('source'),
Str('type'),
Str('dataset'),
Str('cf', default='AVERAGE'),
additional_attrs=False,
)
]),
Dict(
'stats-filter',
Int('step', default=10),
Str('start', default='now-1h'),
Str('end', default='now'),
),
)
async def get_data(self, data_list, stats):
"""
Get data points from rrd files.
"""
if not data_list:
raise ValidationError('stats_list',
'This parameter cannot be empty')
defs = []
names_pair = []
for i, data in enumerate(data_list):
names_pair.append([data['source'], data['type']])
rrdfile = '{}/{}/{}.rrd'.format(RRD_PATH, data['source'],
data['type'])
defs.extend([
'DEF:xxx{}={}:{}:{}'.format(i, rrdfile, data['dataset'],
data['cf']),
'XPORT:xxx{}:{}/{}'.format(i, data['source'], data['type']),
])
proc = await Popen(
[
'/usr/local/bin/rrdtool',
'xport',
'--json',
'--start',
stats['start'],
'--end',
stats['end'],
] + (['--step', str(stats['step'])] if stats.get('step') else []) +
defs,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
data, err = await proc.communicate()
if proc.returncode != 0:
raise CallError('rrdtool failed: {}'.format(err.decode()))
data = json.loads(data.decode())
# Custom about property
data['about'] = 'Data for ' + ','.join(
['/'.join(i) for i in names_pair])
return data
class FilesystemService(Service):
@accepts(Str('path', required=True), Ref('query-filters'), Ref('query-options'))
def listdir(self, path, filters=None, options=None):
"""
Get the contents of a directory.
Each entry of the list consists of:
name(str): name of the file
path(str): absolute path of the entry
realpath(str): absolute real path of the entry (if SYMLINK)
type(str): DIRECTORY | FILESYSTEM | SYMLINK | OTHER
size(int): size of the entry
mode(int): file mode/permission
uid(int): user id of entry owner
gid(int): group id of entry onwer
"""
if not os.path.exists(path):
raise CallError(f'Directory {path} does not exist', errno.ENOENT)
if not os.path.isdir(path):
raise CallError(f'Path {path} is not a directory', errno.ENOTDIR)
rv = []
for entry in os.scandir(path):
if entry.is_dir():
etype = 'DIRECTORY'
elif entry.is_file():
etype = 'FILE'
elif entry.is_symlink():
etype = 'SYMLINK'
else:
etype = 'OTHER'
data = {
'name': entry.name,
'path': entry.path,
'realpath': os.path.realpath(entry.path) if etype == 'SYMLINK' else entry.path,
'type': etype,
}
try:
stat = entry.stat()
data.update({
'size': stat.st_size,
'mode': stat.st_mode,
'uid': stat.st_uid,
'gid': stat.st_gid,
})
except FileNotFoundError:
data.update({'size': None, 'mode': None, 'uid': None, 'gid': None})
rv.append(data)
return filter_list(rv, filters=filters or [], options=options or {})
@accepts(Str('path'))
def stat(self, path):
"""
Return the filesystem stat(2) for a given `path`.
"""
try:
stat = os.stat(path, follow_symlinks=False)
except FileNotFoundError:
raise CallError(f'Path {path} not found', errno.ENOENT)
stat = {
'size': stat.st_size,
'mode': stat.st_mode,
'uid': stat.st_uid,
'gid': stat.st_gid,
'atime': stat.st_atime,
'mtime': stat.st_mtime,
'ctime': stat.st_ctime,
'dev': stat.st_dev,
'inode': stat.st_ino,
'nlink': stat.st_nlink,
}
try:
stat['user'] = pwd.getpwuid(stat['uid']).pw_name
except KeyError:
stat['user'] = None
try:
stat['group'] = grp.getgrgid(stat['gid']).gr_name
except KeyError:
stat['group'] = None
if os.path.exists(os.path.join(path, ".windows")):
stat["acl"] = "windows"
elif os.path.exists(os.path.join(path, ".mac")):
stat["acl"] = "mac"
else:
stat["acl"] = "unix"
return stat
@private
@accepts(
Str('path'),
Str('content'),
Dict(
'options',
Bool('append', default=False),
Int('mode'),
),
)
def file_receive(self, path, content, options=None):
"""
Simplified file receiving method for small files.
`content` must be a base 64 encoded file content.
DISCLAIMER: DO NOT USE THIS FOR BIG FILES (> 500KB).
"""
options = options or {}
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname)
if options.get('append'):
openmode = 'ab'
else:
openmode = 'wb+'
with open(path, openmode) as f:
f.write(binascii.a2b_base64(content))
mode = options.get('mode')
if mode:
os.chmod(path, mode)
return True
@private
@accepts(
Str('path'),
Dict(
'options',
Int('offset'),
Int('maxlen'),
),
)
def file_get_contents(self, path, options=None):
"""
Get contents of a file `path` in base64 encode.
DISCLAIMER: DO NOT USE THIS FOR BIG FILES (> 500KB).
"""
options = options or {}
if not os.path.exists(path):
return None
with open(path, 'rb') as f:
if options.get('offset'):
f.seek(options['offset'])
data = binascii.b2a_base64(f.read(options.get('maxlen'))).decode().strip()
return data
@accepts(Str('path'))
@job(pipes=["output"])
async def get(self, job, path):
"""
Job to get contents of `path`.
"""
if not os.path.isfile(path):
raise CallError(f'{path} is not a file')
with open(path, 'rb') as f:
await self.middleware.run_in_thread(shutil.copyfileobj, f, job.pipes.output.w)
@accepts(
Str('path'),
Dict(
'options',
Bool('append', default=False),
Int('mode'),
),
)
@job(pipes=["input"])
async def put(self, job, path, options=None):
"""
Job to put contents to `path`.
"""
options = options or {}
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname)
if options.get('append'):
openmode = 'ab'
else:
openmode = 'wb+'
with open(path, openmode) as f:
await self.middleware.run_in_thread(shutil.copyfileobj, job.pipes.input.r, f)
mode = options.get('mode')
if mode:
os.chmod(path, mode)
return True
@accepts(Str('path'))
def statfs(self, path):
"""
Return stats from the filesystem of a given path.
Raises:
CallError(ENOENT) - Path not found
"""
try:
statfs = bsd.statfs(path)
except FileNotFoundError:
raise CallError('Path not found.', errno.ENOENT)
return {
**statfs.__getstate__(),
'total_bytes': statfs.total_blocks * statfs.blocksize,
'free_bytes': statfs.free_blocks * statfs.blocksize,
'avail_bytes': statfs.avail_blocks * statfs.blocksize,
}
def __convert_to_basic_permset(self, permset):
"""
Convert "advanced" ACL permset format to basic format using
bitwise operation and constants defined in py-bsd/bsd/acl.pyx,
py-bsd/defs.pxd and acl.h.
If the advanced ACL can't be converted without losing
information, we return 'OTHER'.
Reverse process converts the constant's value to a dictionary
using a bitwise operation.
"""
perm = 0
for k, v, in permset.items():
if v:
perm |= acl.NFS4Perm[k]
try:
SimplePerm = (acl.NFS4BasicPermset(perm)).name
except Exception:
SimplePerm = 'OTHER'
return SimplePerm
def __convert_to_basic_flagset(self, flagset):
flags = 0
for k, v, in flagset.items():
if k == "INHERITED":
continue
if v:
flags |= acl.NFS4Flag[k]
try:
SimpleFlag = (acl.NFS4BasicFlagset(flags)).name
except Exception:
SimpleFlag = 'OTHER'
return SimpleFlag
def __convert_to_adv_permset(self, basic_perm):
permset = {}
perm_mask = acl.NFS4BasicPermset[basic_perm].value
for name, member in acl.NFS4Perm.__members__.items():
if perm_mask & member.value:
permset.update({name: True})
else:
permset.update({name: False})
return permset
def __convert_to_adv_flagset(self, basic_flag):
flagset = {}
flag_mask = acl.NFS4BasicFlagset[basic_flag].value
for name, member in acl.NFS4Flag.__members__.items():
if flag_mask & member.value:
flagset.update({name: True})
else:
flagset.update({name: False})
return flagset
@accepts(Str('path'))
def acl_is_trivial(self, path):
"""
ACL is trivial if it can be fully expressed as a file mode without losing
any access rules. This is intended to be used as a check before allowing
users to chmod() through the webui
"""
if not os.path.exists(path):
raise CallError('Path not found.', errno.ENOENT)
a = acl.ACL(file=path)
return a.is_trivial
@accepts(
Str('path'),
Bool('simplified', default=True),
)
def getacl(self, path, simplified=True):
"""
Return ACL of a given path.
Simplified returns a shortened form of the ACL permset and flags
- TRAVERSE = sufficient rights to traverse a directory, but not read contents.
- READ = sufficient rights to traverse a directory, and read file contents.
- MODIFIY = sufficient rights to traverse, read, write, and modify a file. Equivalent to modify_set.
- FULL_CONTROL = all permissions.
- OTHER = does not fit into any of the above categories without losing information.
In all cases we replace USER_OBJ, GROUP_OBJ, and EVERYONE with owner@, group@, everyone@ for
consistency with getfacl and setfacl. If one of aforementioned special tags is used, 'id' must
be set to None.
An inheriting empty everyone@ ACE is appended to non-trivial ACLs in order to enforce Windows
expectations regarding permissions inheritance. This entry is removed from NT ACL returned
to SMB clients when 'ixnas' samba VFS module is enabled. We also remove it here to avoid confusion.
"""
if not os.path.exists(path):
raise CallError('Path not found.', errno.ENOENT)
a = acl.ACL(file=path)
fs_acl = a.__getstate__()
if not simplified:
advanced_acl = []
for entry in fs_acl:
ace = {
'tag': (acl.ACLWho[entry['tag']]).value,
'id': entry['id'],
'type': entry['type'],
'perms': entry['perms'],
'flags': entry['flags'],
}
if ace['tag'] == 'everyone@' and self.__convert_to_basic_permset(ace['perms']) == 'NOPERMS':
self.logger.debug('detected hidden ace')
continue
advanced_acl.append(ace)
return advanced_acl
if simplified:
simple_acl = []
for entry in fs_acl:
ace = {
'tag': (acl.ACLWho[entry['tag']]).value,
'id': entry['id'],
'type': entry['type'],
'perms': {'BASIC': self.__convert_to_basic_permset(entry['perms'])},
'flags': {'BASIC': self.__convert_to_basic_flagset(entry['flags'])},
}
if ace['tag'] == 'everyone@' and ace['perms']['BASIC'] == 'NOPERMS':
continue
simple_acl.append(ace)
return simple_acl
@accepts(
Str('path'),
List(
'dacl',
items=[
Dict(
'aclentry',
Str('tag', enum=['owner@', 'group@', 'everyone@', 'USER', 'GROUP']),
Int('id', null=True),
Str('type', enum=['ALLOW', 'DENY']),
Dict(
'perms',
Bool('READ_DATA'),
Bool('WRITE_DATA'),
Bool('APPEND_DATA'),
Bool('READ_NAMED_ATTRS'),
Bool('WRITE_NAMED_ATTRS'),
Bool('EXECUTE'),
Bool('DELETE_CHILD'),
Bool('READ_ATTRIBUTES'),
Bool('WRITE_ATTRIBUTES'),
Bool('DELETE'),
Bool('READ_ACL'),
Bool('WRITE_ACL'),
Bool('WRITE_OWNER'),
Bool('SYNCHRONIZE'),
Str('BASIC', enum=['FULL_CONTROL', 'MODIFY', 'READ', 'OTHER']),
),
Dict(
'flags',
Bool('FILE_INHERIT'),
Bool('DIRECTORY_INHERIT'),
Bool('NO_PROPAGATE_INHERIT'),
Bool('INHERIT_ONLY'),
Bool('INHERITED'),
Str('BASIC', enum=['INHERIT', 'NOINHERIT', 'OTHER']),
),
)
],
default=[]
),
Dict(
'options',
Bool('stripacl', default=False),
Bool('recursive', default=False),
Bool('traverse', default=False),
)
)
@job(lock=lambda args: f'setacl:{args[0]}')
def setacl(self, job, path, dacl, options):
"""
Set ACL of a given path. Takes the following parameters:
:path: realpath or relative path. We make a subsequent realpath call to resolve it.
:dacl: Accept a "simplified" ACL here or a full ACL. If the simplified ACL
contains ACE perms or flags that are "SPECIAL", then raise a validation error.
:recursive: apply the ACL recursively
:traverse: traverse filestem boundaries (ZFS datasets)
:strip: convert ACL to trivial. ACL is trivial if it can be expressed as a file mode without
losing any access rules.
In all cases we replace USER_OBJ, GROUP_OBJ, and EVERYONE with owner@, group@, everyone@ for
consistency with getfacl and setfacl. If one of aforementioned special tags is used, 'id' must
be set to None.
An inheriting empty everyone@ ACE is appended to non-trivial ACLs in order to enforce Windows
expectations regarding permissions inheritance. This entry is removed from NT ACL returned
to SMB clients when 'ixnas' samba VFS module is enabled.
"""
if not os.path.exists(path):
raise CallError('Path not found.', errno.ENOENT)
if dacl and options['stripacl']:
raise CallError('Setting ACL and stripping ACL are not permitted simultaneously.', errno.EINVAL)
if options['stripacl']:
a = acl.ACL(file=path)
a.strip()
a.apply(path)
else:
cleaned_acl = []
lockace_is_present = False
for entry in dacl:
if entry['perms'].get('BASIC') == 'OTHER' or entry['flags'].get('BASIC') == 'OTHER':
raise CallError('Unable to apply simplified ACL due to OTHER entry. Use full ACL.', errno.EINVAL)
ace = {
'tag': (acl.ACLWho(entry['tag'])).name,
'id': entry['id'],
'type': entry['type'],
'perms': self.__convert_to_adv_permset(entry['perms']['BASIC']) if 'BASIC' in entry['perms'] else entry['perms'],
'flags': self.__convert_to_adv_flagset(entry['flags']['BASIC']) if 'BASIC' in entry['perms'] else entry['flags'],
}
if ace['tag'] == 'EVERYONE' and self.__convert_to_basic_permset(ace['perms']) == 'NOPERMS':
lockace_is_present = True
cleaned_acl.append(ace)
if not lockace_is_present:
locking_ace = {
'tag': 'EVERYONE',
'id': None,
'type': 'ALLOW',
'perms': self.__convert_to_adv_permset('NOPERMS'),
'flags': self.__convert_to_adv_flagset('INHERIT')
}
cleaned_acl.append(locking_ace)
a = acl.ACL()
a.__setstate__(cleaned_acl)
a.apply(path)
if not options['recursive']:
self.logger.debug('exiting early on non-recursive task')
return True
winacl = subprocess.run([
'/usr/local/bin/winacl',
'-a', 'clone',
f"{'-rx' if options['traverse'] else '-r'}",
'-p', path], check=False
)
if winacl.returncode != 0:
raise CallError(f"Failed to recursively apply ACL: {winacl.stderr.decode()}")
return True
class SharingAFPService(CRUDService):
class Config:
namespace = 'sharing.afp'
datastore = 'sharing.afp_share'
datastore_prefix = 'afp_'
datastore_extend = 'sharing.afp.extend'
@accepts(
Dict('sharingafp_create',
Str('path'),
Bool('home', default=False),
Str('name'),
Str('comment'),
List('allow'),
List('deny'),
List('ro'),
List('rw'),
Bool('timemachine', default=False),
Int('timemachine_quota', default=0),
Bool('nodev', default=False),
Bool('nostat', default=False),
Bool('upriv', default=True),
UnixPerm('fperm', default='644'),
UnixPerm('dperm', default='755'),
UnixPerm('umask', default='000'),
List('hostsallow', items=[IPAddr('ip', cidr=True)]),
List('hostsdeny', items=[IPAddr('ip', cidr=True)]),
Str('auxparams'),
register=True))
async def do_create(self, data):
verrors = ValidationErrors()
path = data['path']
await self.clean(data, 'sharingafp_create', verrors)
await self.validate(data, 'sharingafp_create', verrors)
await check_path_resides_within_volume(verrors, self.middleware,
"sharingafp_create.path", path)
if verrors:
raise verrors
if path and not os.path.exists(path):
try:
os.makedirs(path)
except OSError as e:
raise CallError(f'Failed to create {path}: {e}')
await self.compress(data)
data['id'] = await self.middleware.call(
'datastore.insert', self._config.datastore, data,
{'prefix': self._config.datastore_prefix})
await self.extend(data)
await self.middleware.call('service.reload', 'afp')
return data
@accepts(Int('id'),
Patch('sharingafp_create', 'sharingafp_update', ('attr', {
'update': True
})))
async def do_update(self, id, data):
verrors = ValidationErrors()
old = await self.middleware.call(
'datastore.query', self._config.datastore, [('id', '=', id)], {
'extend': self._config.datastore_extend,
'prefix': self._config.datastore_prefix,
'get': True
})
path = data['path']
new = old.copy()
new.update(data)
await self.clean(new, 'sharingafp_update', verrors, id=id)
await self.validate(new, 'sharingafp_update', verrors, old=old)
await check_path_resides_within_volume(verrors, self.middleware,
"sharingafp_create.path", path)
if verrors:
raise verrors
if path and not os.path.exists(path):
try:
os.makedirs(path)
except OSError as e:
raise CallError(f'Failed to create {path}: {e}')
await self.compress(new)
await self.middleware.call('datastore.update', self._config.datastore,
id, new,
{'prefix': self._config.datastore_prefix})
await self.extend(new)
await self.middleware.call('service.reload', 'afp')
return new
@accepts(Int('id'))
async def do_delete(self, id):
return await self.middleware.call('datastore.delete',
self._config.datastore, id)
@private
async def clean(self, data, schema_name, verrors, id=None):
data['name'] = await self.name_exists(data, schema_name, verrors, id)
@private
async def validate(self, data, schema_name, verrors, old=None):
await self.home_exists(data['home'], schema_name, verrors, old)
@private
async def home_exists(self, home, schema_name, verrors, old=None):
home_filters = [('home', '=', True)]
home_result = None
if home:
if old and old['id'] is not None:
id = old['id']
if not old['home']:
home_filters.append(('id', '!=', id))
# The user already had this set as the home share
home_result = await self.middleware.call(
'datastore.query', self._config.datastore,
home_filters,
{'prefix': self._config.datastore_prefix})
if home_result:
verrors.add(f'{schema_name}.home',
'Only one share is allowed to be a home share.')
async def name_exists(self, data, schema_name, verrors, id=None):
name = data['name']
path = data['path']
home = data['home']
name_filters = [('name', '=', name)]
path_filters = [('path', '=', path)]
if not name:
if home:
name = 'Homes'
else:
name = path.rsplit('/', 1)[-1]
if id is not None:
name_filters.append(('id', '!=', id))
path_filters.append(('id', '!=', id))
name_result = await self.middleware.call(
'datastore.query', self._config.datastore, name_filters,
{'prefix': self._config.datastore_prefix})
path_result = await self.middleware.call(
'datastore.query', self._config.datastore, path_filters,
{'prefix': self._config.datastore_prefix})
if name_result:
verrors.add(f'{schema_name}.name',
'A share with this name already exists.')
if path_result:
verrors.add(f'{schema_name}.path',
'A share with this path already exists.')
return name
@private
async def extend(self, data):
data['allow'] = data['allow'].split()
data['deny'] = data['deny'].split()
data['ro'] = data['ro'].split()
data['rw'] = data['rw'].split()
data['hostsallow'] = data['hostsallow'].split()
data['hostsdeny'] = data['hostsdeny'].split()
return data
@private
async def compress(self, data):
data['allow'] = ' '.join(data['allow'])
data['deny'] = ' '.join(data['deny'])
data['ro'] = ' '.join(data['ro'])
data['rw'] = ' '.join(data['rw'])
data['hostsallow'] = ' '.join(data['hostsallow'])
data['hostsdeny'] = ' '.join(data['hostsdeny'])
return data
class ChartReleaseService(Service):
class Config:
namespace = 'chart.release'
@accepts()
@returns(Dict(*[Str(r.name, enum=[r.name])
for r in SCALEABLE_RESOURCES], ))
async def scaleable_resources(self):
"""
Returns choices for types of workloads which can be scaled up/down.
"""
return {r.name: r.name for r in SCALEABLE_RESOURCES}
@accepts(Str('release_name'),
Dict(
'scale_options',
Int('replica_count', required=True,
validators=[Range(min=0)]),
))
@returns(
Dict(
'scale_chart_release',
Dict('before_scale',
*[
Dict(r.value, additional_attrs=True)
for r in SCALEABLE_RESOURCES
],
required=True),
Dict('after_scale',
*[
Dict(r.value, additional_attrs=True)
for r in SCALEABLE_RESOURCES
],
required=True),
))
@job(lock=lambda args: f'{args[0]}_chart_release_scale')
async def scale(self, job, release_name, options):
"""
Scale a `release_name` chart release to `scale_options.replica_count` specified.
This will scale deployments/statefulset to replica count specified.
"""
await self.middleware.call('kubernetes.validate_k8s_setup')
release = await self.middleware.call(
'chart.release.query', [['id', '=', release_name]], {
'get': True,
'extra': {
'retrieve_resources': True,
'retrieve_locked_paths': True
},
})
if options['replica_count']:
# This means we have a number higher then 0 - we would like to make sure in this case that we
# are not going to start an app which might be consuming a locked path
if release['resources']['locked_host_paths']:
raise CallError(
f'{release_name!r} cannot be started as it is consuming following host path(s) '
f'which are locked: {", ".join(release["resources"]["locked_host_paths"])}'
)
resources = release['resources']
replica_counts = await self.get_replica_count_for_resources(resources)
job.set_progress(
20,
f'Scaling workload(s) to {options["replica_count"]!r} replica(s)')
try:
await self.scale_release_internal(resources,
options['replica_count'])
except Exception:
# This is a best effort to get relevant workloads back to replica count which they were on before
await self.scale_release_internal(resources,
replica_counts=replica_counts)
raise
else:
desired_pods_count = sum(
len(replica_counts[r.value]) * options['replica_count']
for r in SCALEABLE_RESOURCES)
job.set_progress(
40,
f'Waiting for pods to be scaled to {desired_pods_count!r} replica(s)'
)
while await self.middleware.call('k8s.pod.query', [
['metadata.namespace', '=', release['namespace']],
['status.phase', 'in', ['Running', 'Pending']],
], {'count': True}) != desired_pods_count:
await asyncio.sleep(5)
job.set_progress(
100,
f'Scaled workload(s) successfully to {options["replica_count"]!r} replica(s)'
)
return {
'before_scale':
replica_counts,
'after_scale':
await self.get_replica_count_for_resources(
(await self.middleware.call('chart.release.query',
[['id', '=', release_name]], {
'get': True,
'extra': {
'retrieve_resources': True
}
}))['resources'])
}
@private
async def get_replica_count_for_resources(self, resources):
replica_counts = {r.value: {} for r in SCALEABLE_RESOURCES}
for resource in SCALEABLE_RESOURCES:
for workload in resources[resource.value]:
replica_counts[resource.value][
workload['metadata']['name']] = {
'replicas': workload['spec']['replicas'],
}
return replica_counts
@private
async def scale_release_internal(self,
resources,
replicas=None,
replica_counts=None,
resource_check=False):
if replicas is not None and replica_counts:
raise CallError(
'Only one of "replicas" or "replica_counts" should be specified'
)
elif replicas is None and not replica_counts:
raise CallError(
'Either one of "replicas" or "replica_counts" must be specified'
)
assert bool(resources or replica_counts) is True
replica_counts = replica_counts or {
r.value: {}
for r in SCALEABLE_RESOURCES
}
if resource_check:
resources_data = {
r.name.lower(): {
w['metadata']['name']
for w in await self.middleware.call(
f'k8s.{r.name.lower()}.query')
}
for r in SCALEABLE_RESOURCES
}
for resource in SCALEABLE_RESOURCES:
for workload in resources[resource.value]:
replica_count = replica_counts[resource.value].get(
workload['metadata']['name'],
{}).get('replicas') or replicas
if resource_check:
if workload['metadata']['name'] not in resources_data[
resource.name.lower()]:
continue
await self.middleware.call(
f'k8s.{resource.name.lower()}.update',
workload['metadata']['name'], {
'namespace': workload['metadata']['namespace'],
'body': {
'spec': {
'replicas': replica_count,
}
}
})
@accepts(
Str('release_name'),
List(
'workloads',
items=[
Dict(
'scale_workload',
Int('replica_count', required=True),
Str('type',
enum=[r.name for r in SCALEABLE_RESOURCES],
required=True),
Str('name', required=True),
)
],
empty=False,
),
)
@returns()
async def scale_workloads(self, release_name, workloads):
"""
Scale workloads in a chart release to specified `replica_count`.
"""
release = await self.middleware.call('chart.release.query',
[['id', '=', release_name]], {
'get': True,
'extra': {
'retrieve_resources': True
}
})
not_found = {}
scale_resources = {r.name: [] for r in SCALEABLE_RESOURCES}
to_scale_resources = defaultdict(dict)
for workload in workloads:
to_scale_resources[workload['type']][workload['name']] = workload
for scaleable_resource in SCALEABLE_RESOURCES:
to_scale = to_scale_resources[scaleable_resource.name]
if not to_scale:
continue
for resource in map(
lambda r: r['metadata']['name'], release['resources']
[f'{scaleable_resource.name.lower()}s']):
if resource in to_scale:
scale_resources[scaleable_resource.name].append(
to_scale[resource])
to_scale.pop(resource)
not_found.update(to_scale)
if not_found:
raise CallError(
f'Unable to find {", ".join(not_found)} workload(s) for {release_name} release',
errno=errno.ENOENT)
for resource_type in scale_resources:
for workload in scale_resources[resource_type]:
await self.middleware.call(
f'k8s.{resource_type.lower()}.update', workload['name'], {
'namespace': release['namespace'],
'body': {
'spec': {
'replicas': workload['replica_count']
}
},
})
@private
async def wait_for_pods_to_terminate(self, namespace, extra_filters=None):
# wait for release to uninstall properly, helm right now does not support a flag for this but
# a feature request is open in the community https://github.com/helm/helm/issues/2378
while await self.middleware.call('k8s.pod.query', [
['metadata.namespace', '=', namespace],
['status.phase', 'in', ['Running', 'Pending']],
] + (extra_filters or [])):
await asyncio.sleep(5)
@private
async def get_workload_to_pod_mapping(self, namespace):
mapping = {'replicaset': defaultdict(dict), 'pod': defaultdict(dict)}
for key in ('replicaset', 'pod'):
for r in await self.middleware.call(f'k8s.{key}.query', [
['metadata.namespace', '=', namespace],
['metadata', 'rin', 'owner_references'],
], {'select': ['metadata']}):
for owner_reference in filter(
lambda o: o.get('uid'),
r['metadata']['owner_references'] or []):
mapping[key][owner_reference['uid']][r['metadata']
['uid']] = r
pod_mapping = defaultdict(list)
for parent, replicasets in mapping['replicaset'].items():
for replicaset in map(lambda r: mapping['replicaset'][parent][r],
replicasets):
if replicaset['metadata']['uid'] not in mapping['pod']:
continue
pod_mapping[parent].extend([
p['metadata']['name'] for p in mapping['pod'][
replicaset['metadata']['uid']].values()
])
return pod_mapping
@private
async def scale_down_resources_consuming_locked_paths(self):
args = [[r['id'], {
'replica_count': 0
}] for r in await self.middleware.call(
'chart.release.query', [['status', '!=', 'STOPPED']], {
'extra': {
'retrieve_resources': True,
'retrieve_locked_paths': True,
}
}) if r['resources']['locked_host_paths']]
if args:
await self.middleware.call('core.bulk', 'chart.release.scale',
args)
class FilesystemService(Service):
@accepts(Str('path', required=True), Ref('query-filters'),
Ref('query-options'))
def listdir(self, path, filters=None, options=None):
"""
Get the contents of a directory.
Each entry of the list consists of:
name(str): name of the file
path(str): absolute path of the entry
realpath(str): absolute real path of the entry (if SYMLINK)
type(str): DIRECTORY | FILESYSTEM | SYMLINK | OTHER
size(int): size of the entry
mode(int): file mode/permission
uid(int): user id of entry owner
gid(int): group id of entry onwer
acl(bool): extended ACL is present on file
"""
if not os.path.exists(path):
raise CallError(f'Directory {path} does not exist', errno.ENOENT)
if not os.path.isdir(path):
raise CallError(f'Path {path} is not a directory', errno.ENOTDIR)
rv = []
for entry in os.scandir(path):
if entry.is_symlink():
etype = 'SYMLINK'
elif entry.is_dir():
etype = 'DIRECTORY'
elif entry.is_file():
etype = 'FILE'
else:
etype = 'OTHER'
data = {
'name':
entry.name,
'path':
entry.path,
'realpath':
os.path.realpath(entry.path)
if etype == 'SYMLINK' else entry.path,
'type':
etype,
}
try:
stat = entry.stat()
data.update({
'size':
stat.st_size,
'mode':
stat.st_mode,
'acl':
False if self.acl_is_trivial(data["realpath"]) else True,
'uid':
stat.st_uid,
'gid':
stat.st_gid,
})
except FileNotFoundError:
data.update({
'size': None,
'mode': None,
'acl': None,
'uid': None,
'gid': None
})
rv.append(data)
return filter_list(rv, filters=filters or [], options=options or {})
@accepts(Str('path'))
def stat(self, path):
"""
Return the filesystem stat(2) for a given `path`.
"""
try:
stat = os.stat(path, follow_symlinks=False)
except FileNotFoundError:
raise CallError(f'Path {path} not found', errno.ENOENT)
stat = {
'size': stat.st_size,
'mode': stat.st_mode,
'uid': stat.st_uid,
'gid': stat.st_gid,
'atime': stat.st_atime,
'mtime': stat.st_mtime,
'ctime': stat.st_ctime,
'dev': stat.st_dev,
'inode': stat.st_ino,
'nlink': stat.st_nlink,
}
try:
stat['user'] = pwd.getpwuid(stat['uid']).pw_name
except KeyError:
stat['user'] = None
try:
stat['group'] = grp.getgrgid(stat['gid']).gr_name
except KeyError:
stat['group'] = None
stat['acl'] = False if self.middleware.call_sync(
'filesystem.acl_is_trivial', path) else True
return stat
@private
@accepts(
Str('path'),
Str('content', max_length=2048000),
Dict(
'options',
Bool('append', default=False),
Int('mode'),
),
)
def file_receive(self, path, content, options=None):
"""
Simplified file receiving method for small files.
`content` must be a base 64 encoded file content.
"""
options = options or {}
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname)
if options.get('append'):
openmode = 'ab'
else:
openmode = 'wb+'
with open(path, openmode) as f:
f.write(binascii.a2b_base64(content))
mode = options.get('mode')
if mode:
os.chmod(path, mode)
return True
@private
@accepts(
Str('path'),
Dict(
'options',
Int('offset'),
Int('maxlen'),
),
)
def file_get_contents(self, path, options=None):
"""
Get contents of a file `path` in base64 encode.
DISCLAIMER: DO NOT USE THIS FOR BIG FILES (> 500KB).
"""
options = options or {}
if not os.path.exists(path):
return None
with open(path, 'rb') as f:
if options.get('offset'):
f.seek(options['offset'])
data = binascii.b2a_base64(f.read(
options.get('maxlen'))).decode().strip()
return data
@accepts(Str('path'))
@job(pipes=["output"])
async def get(self, job, path):
"""
Job to get contents of `path`.
"""
if not os.path.isfile(path):
raise CallError(f'{path} is not a file')
with open(path, 'rb') as f:
await self.middleware.run_in_thread(shutil.copyfileobj, f,
job.pipes.output.w)
@accepts(
Str('path'),
Dict(
'options',
Bool('append', default=False),
Int('mode'),
),
)
@job(pipes=["input"])
async def put(self, job, path, options=None):
"""
Job to put contents to `path`.
"""
options = options or {}
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname)
if options.get('append'):
openmode = 'ab'
else:
openmode = 'wb+'
with open(path, openmode) as f:
await self.middleware.run_in_thread(shutil.copyfileobj,
job.pipes.input.r, f)
mode = options.get('mode')
if mode:
os.chmod(path, mode)
return True
@accepts(Str('path'))
def statfs(self, path):
"""
Return stats from the filesystem of a given path.
Raises:
CallError(ENOENT) - Path not found
"""
try:
statfs = bsd.statfs(path)
except FileNotFoundError:
raise CallError('Path not found.', errno.ENOENT)
return {
**statfs.__getstate__(),
'total_bytes': statfs.total_blocks * statfs.blocksize,
'free_bytes': statfs.free_blocks * statfs.blocksize,
'avail_bytes': statfs.avail_blocks * statfs.blocksize,
}
def __convert_to_basic_permset(self, permset):
"""
Convert "advanced" ACL permset format to basic format using
bitwise operation and constants defined in py-bsd/bsd/acl.pyx,
py-bsd/defs.pxd and acl.h.
If the advanced ACL can't be converted without losing
information, we return 'OTHER'.
Reverse process converts the constant's value to a dictionary
using a bitwise operation.
"""
perm = 0
for k, v, in permset.items():
if v:
perm |= acl.NFS4Perm[k]
try:
SimplePerm = (acl.NFS4BasicPermset(perm)).name
except Exception:
SimplePerm = 'OTHER'
return SimplePerm
def __convert_to_basic_flagset(self, flagset):
flags = 0
for k, v, in flagset.items():
if k == "INHERITED":
continue
if v:
flags |= acl.NFS4Flag[k]
try:
SimpleFlag = (acl.NFS4BasicFlagset(flags)).name
except Exception:
SimpleFlag = 'OTHER'
return SimpleFlag
def __convert_to_adv_permset(self, basic_perm):
permset = {}
perm_mask = acl.NFS4BasicPermset[basic_perm].value
for name, member in acl.NFS4Perm.__members__.items():
if perm_mask & member.value:
permset.update({name: True})
else:
permset.update({name: False})
return permset
def __convert_to_adv_flagset(self, basic_flag):
flagset = {}
flag_mask = acl.NFS4BasicFlagset[basic_flag].value
for name, member in acl.NFS4Flag.__members__.items():
if flag_mask & member.value:
flagset.update({name: True})
else:
flagset.update({name: False})
return flagset
def _winacl(self, path, action, uid, gid, options):
chroot_dir = os.path.dirname(path)
target = os.path.basename(path)
winacl = subprocess.run([
'/usr/local/bin/winacl', '-a', action, '-O',
str(uid), '-G',
str(gid), '-rx' if options['traverse'] else '-r', '-c', chroot_dir,
'-p', target
],
check=False,
capture_output=True)
if winacl.returncode != 0:
CallError(
f"Winacl {action} on path {path} failed with error: [{winacl.stderr.decode().strip()}]"
)
def _common_perm_path_validate(self, path):
if not os.path.exists(path):
raise CallError(f"Path not found: {path}", errno.ENOENT)
if not os.path.realpath(path).startswith('/mnt/'):
raise CallError(
f"Changing permissions on paths outside of /mnt is not permitted: {path}",
errno.EPERM)
if os.path.realpath(path) in [
x['path'] for x in self.middleware.call_sync('pool.query')
]:
raise CallError(
f"Changing permissions of root level dataset is not permitted: {path}",
errno.EPERM)
@accepts(Str('path'))
def acl_is_trivial(self, path):
"""
Returns True if the ACL can be fully expressed as a file mode without losing
any access rules, or if the path does not support NFSv4 ACLs (for example
a path on a tmpfs filesystem).
"""
if not os.path.exists(path):
raise CallError('Path not found.', errno.ENOENT)
has_nfs4_acl_support = os.pathconf(path, 64)
if not has_nfs4_acl_support:
return True
return acl.ACL(file=path).is_trivial
@accepts(
Dict(
'filesystem_ownership', Str('path', required=True),
Int('uid', null=True, default=None),
Int('gid', null=True, default=None),
Dict('options', Bool('recursive', default=False),
Bool('traverse', default=False))))
@job(lock="perm_change")
def chown(self, job, data):
"""
Change owner or group of file at `path`.
`uid` and `gid` specify new owner of the file. If either
key is absent or None, then existing value on the file is not
changed.
`recursive` performs action recursively, but does
not traverse filesystem mount points.
If `traverse` and `recursive` are specified, then the chown
operation will traverse filesystem mount points.
"""
job.set_progress(0, 'Preparing to change owner.')
self._common_perm_path_validate(data['path'])
uid = -1 if data['uid'] is None else data['uid']
gid = -1 if data['gid'] is None else data['gid']
options = data['options']
if not options['recursive']:
job.set_progress(100, 'Finished changing owner.')
os.chown(data['path'], uid, gid)
else:
job.set_progress(10,
f'Recursively changing owner of {data["path"]}.')
self._winacl(data['path'], 'chown', uid, gid, options)
job.set_progress(100, 'Finished changing owner.')
@accepts(
Dict(
'filesystem_permission', Str('path', required=True),
UnixPerm('mode', null=True), Int('uid', null=True, default=None),
Int('gid', null=True, default=None),
Dict(
'options',
Bool('stripacl', default=False),
Bool('recursive', default=False),
Bool('traverse', default=False),
)))
@job(lock="perm_change")
def setperm(self, job, data):
"""
Remove extended ACL from specified path.
If `mode` is specified then the mode will be applied to the
path and files and subdirectories depending on which `options` are
selected. Mode should be formatted as string representation of octal
permissions bits.
`uid` the desired UID of the file user. If set to None (the default), then user is not changed.
`gid` the desired GID of the file group. If set to None (the default), then group is not changed.
`stripacl` setperm will fail if an extended ACL is present on `path`,
unless `stripacl` is set to True.
`recursive` remove ACLs recursively, but do not traverse dataset
boundaries.
`traverse` remove ACLs from child datasets.
If no `mode` is set, and `stripacl` is True, then non-trivial ACLs
will be converted to trivial ACLs. An ACL is trivial if it can be
expressed as a file mode without losing any access rules.
"""
job.set_progress(0, 'Preparing to set permissions.')
options = data['options']
mode = data.get('mode', None)
uid = -1 if data['uid'] is None else data['uid']
gid = -1 if data['gid'] is None else data['gid']
self._common_perm_path_validate(data['path'])
acl_is_trivial = self.middleware.call_sync('filesystem.acl_is_trivial',
data['path'])
if not acl_is_trivial and not options['stripacl']:
raise CallError(
f'Non-trivial ACL present on [{data["path"]}]. Option "stripacl" required to change permission.',
errno.EINVAL)
if mode is not None:
mode = int(mode, 8)
a = acl.ACL(file=data['path'])
a.strip()
a.apply(data['path'])
if mode:
os.chmod(data['path'], mode)
if uid or gid:
os.chown(data['path'], uid, gid)
if not options['recursive']:
job.set_progress(100, 'Finished setting permissions.')
return
action = 'clone' if mode else 'strip'
job.set_progress(
10, f'Recursively setting permissions on {data["path"]}.')
self._winacl(data['path'], action, uid, gid, options)
job.set_progress(100, 'Finished setting permissions.')
@accepts()
async def default_acl_choices(self):
"""
Get list of default ACL types.
"""
acl_choices = []
for x in ACLDefault:
if x.value['visible']:
acl_choices.append(x.name)
return acl_choices
@accepts(Str('acl_type', default='OPEN',
enum=[x.name for x in ACLDefault]))
async def get_default_acl(self, acl_type):
"""
Returns a default ACL depending on the usage specified by `acl_type`.
If an admin group is defined, then an entry granting it full control will
be placed at the top of the ACL.
"""
acl = []
admin_group = (await self.middleware.call('smb.config'))['admin_group']
if acl_type == 'HOME' and (await self.middleware.call(
'activedirectory.get_state')) == 'HEALTHY':
acl_type = 'DOMAIN_HOME'
if admin_group:
acl.append({
'tag':
'GROUP',
'id': (await self.middleware.call('dscache.get_uncached_group',
admin_group))['gr_gid'],
'perms': {
'BASIC': 'FULL_CONTROL'
},
'flags': {
'BASIC': 'INHERIT'
},
'type':
'ALLOW'
})
acl.extend((ACLDefault[acl_type].value)['acl'])
return acl
def _is_inheritable(self, flags):
"""
Takes ACE flags and return True if any inheritance bits are set.
"""
inheritance_flags = [
'FILE_INHERIT', 'DIRECTORY_INHERIT', 'NO_PROPAGATE_INHERIT',
'INHERIT_ONLY'
]
for i in inheritance_flags:
if flags.get(i):
return True
return False
@private
def canonicalize_acl_order(self, acl):
"""
Convert flags to advanced, then separate the ACL into two lists. One for ACEs that have been inherited,
one for aces that have not been inherited. Non-inherited ACEs take precedence
and so they are placed first in finalized combined list. Within each list, the
ACEs are orderd according to the following:
1) Deny ACEs that apply to the object itself (NOINHERIT)
2) Deny ACEs that apply to a subobject of the object (INHERIT)
3) Allow ACEs that apply to the object itself (NOINHERIT)
4) Allow ACEs that apply to a subobject of the object (INHERIT)
See http://docs.microsoft.com/en-us/windows/desktop/secauthz/order-of-aces-in-a-dacl
The "INHERITED" bit is stripped in filesystem.getacl when generating a BASIC flag type.
It is best practice to use a non-simplified ACL for canonicalization.
"""
inherited_aces = []
final_acl = []
non_inherited_aces = []
for entry in acl:
entry['flags'] = self.__convert_to_adv_flagset(
entry['flags']
['BASIC']) if 'BASIC' in entry['flags'] else entry['flags']
if entry['flags'].get('INHERITED'):
inherited_aces.append(entry)
else:
non_inherited_aces.append(entry)
if inherited_aces:
inherited_aces = sorted(
inherited_aces,
key=lambda x:
(x['type'] == 'ALLOW', self._is_inheritable(x['flags'])),
)
if non_inherited_aces:
non_inherited_aces = sorted(
non_inherited_aces,
key=lambda x:
(x['type'] == 'ALLOW', self._is_inheritable(x['flags'])),
)
final_acl = non_inherited_aces + inherited_aces
return final_acl
@accepts(
Str('path'),
Bool('simplified', default=True),
)
def getacl(self, path, simplified=True):
"""
Return ACL of a given path.
Simplified returns a shortened form of the ACL permset and flags
`TRAVERSE` sufficient rights to traverse a directory, but not read contents.
`READ` sufficient rights to traverse a directory, and read file contents.
`MODIFIY` sufficient rights to traverse, read, write, and modify a file. Equivalent to modify_set.
`FULL_CONTROL` all permissions.
If the permisssions do not fit within one of the pre-defined simplified permissions types, then
the full ACL entry will be returned.
In all cases we replace USER_OBJ, GROUP_OBJ, and EVERYONE with owner@, group@, everyone@ for
consistency with getfacl and setfacl. If one of aforementioned special tags is used, 'id' must
be set to None.
An inheriting empty everyone@ ACE is appended to non-trivial ACLs in order to enforce Windows
expectations regarding permissions inheritance. This entry is removed from NT ACL returned
to SMB clients when 'ixnas' samba VFS module is enabled. We also remove it here to avoid confusion.
"""
if not os.path.exists(path):
raise CallError('Path not found.', errno.ENOENT)
stat = os.stat(path)
a = acl.ACL(file=path)
fs_acl = a.__getstate__()
if not simplified:
advanced_acl = []
for entry in fs_acl:
ace = {
'tag': (acl.ACLWho[entry['tag']]).value,
'id': entry['id'],
'type': entry['type'],
'perms': entry['perms'],
'flags': entry['flags'],
}
if ace['tag'] == 'everyone@' and self.__convert_to_basic_permset(
ace['perms']) == 'NOPERMS':
continue
advanced_acl.append(ace)
return {
'uid': stat.st_uid,
'gid': stat.st_gid,
'acl': advanced_acl
}
if simplified:
simple_acl = []
for entry in fs_acl:
ace = {
'tag': (acl.ACLWho[entry['tag']]).value,
'id': entry['id'],
'type': entry['type'],
'perms': {
'BASIC':
self.__convert_to_basic_permset(entry['perms'])
},
'flags': {
'BASIC':
self.__convert_to_basic_flagset(entry['flags'])
},
}
if ace['tag'] == 'everyone@' and ace['perms'][
'BASIC'] == 'NOPERMS':
continue
for key in ['perms', 'flags']:
if ace[key]['BASIC'] == 'OTHER':
ace[key] = entry[key]
simple_acl.append(ace)
return {'uid': stat.st_uid, 'gid': stat.st_gid, 'acl': simple_acl}
@accepts(
Dict(
'filesystem_acl', Str('path', required=True),
Int('uid', null=True, default=None),
Int('gid', null=True, default=None),
List('dacl',
items=[
Dict(
'aclentry',
Str('tag',
enum=[
'owner@', 'group@', 'everyone@', 'USER',
'GROUP'
]),
Int('id', null=True),
Str('type', enum=['ALLOW', 'DENY']),
Dict(
'perms',
Bool('READ_DATA'),
Bool('WRITE_DATA'),
Bool('APPEND_DATA'),
Bool('READ_NAMED_ATTRS'),
Bool('WRITE_NAMED_ATTRS'),
Bool('EXECUTE'),
Bool('DELETE_CHILD'),
Bool('READ_ATTRIBUTES'),
Bool('WRITE_ATTRIBUTES'),
Bool('DELETE'),
Bool('READ_ACL'),
Bool('WRITE_ACL'),
Bool('WRITE_OWNER'),
Bool('SYNCHRONIZE'),
Str('BASIC',
enum=[
'FULL_CONTROL', 'MODIFY', 'READ',
'TRAVERSE'
]),
),
Dict(
'flags',
Bool('FILE_INHERIT'),
Bool('DIRECTORY_INHERIT'),
Bool('NO_PROPAGATE_INHERIT'),
Bool('INHERIT_ONLY'),
Bool('INHERITED'),
Str('BASIC', enum=['INHERIT', 'NOINHERIT']),
),
)
],
default=[]),
Dict('options', Bool('stripacl', default=False),
Bool('recursive', default=False),
Bool('traverse', default=False),
Bool('canonicalize', default=True))))
@job(lock="perm_change")
def setacl(self, job, data):
"""
Set ACL of a given path. Takes the following parameters:
`path` full path to directory or file.
`dacl` "simplified" ACL here or a full ACL.
`uid` the desired UID of the file user. If set to None (the default), then user is not changed.
`gid` the desired GID of the file group. If set to None (the default), then group is not changed.
`recursive` apply the ACL recursively
`traverse` traverse filestem boundaries (ZFS datasets)
`strip` convert ACL to trivial. ACL is trivial if it can be expressed as a file mode without
losing any access rules.
`canonicalize` reorder ACL entries so that they are in concanical form as described
in the Microsoft documentation MS-DTYP 2.4.5 (ACL)
In all cases we replace USER_OBJ, GROUP_OBJ, and EVERYONE with owner@, group@, everyone@ for
consistency with getfacl and setfacl. If one of aforementioned special tags is used, 'id' must
be set to None.
An inheriting empty everyone@ ACE is appended to non-trivial ACLs in order to enforce Windows
expectations regarding permissions inheritance. This entry is removed from NT ACL returned
to SMB clients when 'ixnas' samba VFS module is enabled.
"""
job.set_progress(0, 'Preparing to set acl.')
options = data['options']
dacl = data.get('dacl', [])
self._common_perm_path_validate(data['path'])
if dacl and options['stripacl']:
raise CallError(
'Setting ACL and stripping ACL are not permitted simultaneously.',
errno.EINVAL)
uid = -1 if data.get('uid', None) is None else data['uid']
gid = -1 if data.get('gid', None) is None else data['gid']
if options['stripacl']:
a = acl.ACL(file=data['path'])
a.strip()
a.apply(data['path'])
else:
cleaned_acl = []
lockace_is_present = False
for entry in dacl:
ace = {
'tag': (acl.ACLWho(entry['tag'])).name,
'id':
entry['id'],
'type':
entry['type'],
'perms':
self.__convert_to_adv_permset(entry['perms']['BASIC'])
if 'BASIC' in entry['perms'] else entry['perms'],
'flags':
self.__convert_to_adv_flagset(entry['flags']['BASIC'])
if 'BASIC' in entry['flags'] else entry['flags'],
}
if ace['flags']['INHERIT_ONLY'] and not ace['flags'].get(
'DIRECTORY_INHERIT', False) and not ace['flags'].get(
'FILE_INHERIT', False):
raise CallError(
'Invalid flag combination. DIRECTORY_INHERIT or FILE_INHERIT must be set if INHERIT_ONLY is set.',
errno.EINVAL)
if ace['tag'] == 'EVERYONE' and self.__convert_to_basic_permset(
ace['perms']) == 'NOPERMS':
lockace_is_present = True
cleaned_acl.append(ace)
if options['canonicalize']:
cleaned_acl = self.canonicalize_acl_order(cleaned_acl)
if not lockace_is_present:
locking_ace = {
'tag': 'EVERYONE',
'id': None,
'type': 'ALLOW',
'perms': self.__convert_to_adv_permset('NOPERMS'),
'flags': self.__convert_to_adv_flagset('INHERIT')
}
cleaned_acl.append(locking_ace)
a = acl.ACL()
a.__setstate__(cleaned_acl)
a.apply(data['path'])
if not options['recursive']:
os.chown(data['path'], uid, gid)
job.set_progress(100, 'Finished setting ACL.')
return
job.set_progress(10, f'Recursively setting ACL on {data["path"]}.')
self._winacl(data['path'], 'clone', uid, gid, options)
job.set_progress(100, 'Finished setting ACL.')
class SSHService(SystemServiceService):
class Config:
service = "ssh"
service_model = "ssh"
datastore_prefix = "ssh_"
@accepts()
def bindiface_choices(self):
"""
Available choices for the bindiface attribute of SSH service.
"""
return self.middleware.call_sync('interface.choices')
@accepts(Dict(
'ssh_update',
List('bindiface', items=[Str('iface')]),
Int('tcpport', validators=[Range(min=1, max=65535)]),
Bool('rootlogin'),
Bool('passwordauth'),
Bool('kerberosauth'),
Bool('tcpfwd'),
Bool('compression'),
Str('sftp_log_level', enum=["", "QUIET", "FATAL", "ERROR", "INFO", "VERBOSE", "DEBUG", "DEBUG2", "DEBUG3"]),
Str('sftp_log_facility', enum=["", "DAEMON", "USER", "AUTH", "LOCAL0", "LOCAL1", "LOCAL2", "LOCAL3", "LOCAL4",
"LOCAL5", "LOCAL6", "LOCAL7"]),
Str('options'),
update=True
))
async def do_update(self, data):
"""
Update settings of SSH daemon service.
If `bindiface` is empty it will listen for all available addresses.
.. examples(websocket)::
Make sshd listen only to igb0 interface.
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "ssh.update",
"params": [{
"bindiface": ["igb0"]
}]
}
"""
old = await self.config()
new = old.copy()
new.update(data)
if new['bindiface']:
verrors = ValidationErrors()
iface_choices = await self.middleware.call('ssh.bindiface_choices')
invalid_ifaces = list(filter(lambda x: x not in iface_choices, new['bindiface']))
if invalid_ifaces:
verrors.add(
'ssh_update.bindiface',
f'The following interfaces are not valid: {", ".join(invalid_ifaces)}',
)
verrors.check()
await self._update_service(old, new)
keyfile = "/usr/local/etc/ssh/ssh_host_ecdsa_key.pub"
if os.path.exists(keyfile):
with open(keyfile, "rb") as f:
pubkey = f.read().strip().split(None, 3)[1]
decoded_key = base64.b64decode(pubkey)
key_digest = hashlib.sha256(decoded_key).digest()
ssh_fingerprint = (b"SHA256:" + base64.b64encode(key_digest).replace(b"=", b"")).decode("utf-8")
syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER)
syslog.syslog(syslog.LOG_ERR, 'ECDSA Fingerprint of the SSH KEY: ' + ssh_fingerprint)
syslog.closelog()
return new
class ZFSPoolService(Service):
class Config:
namespace = 'zfs.pool'
private = True
@filterable
def query(self, filters, options):
with libzfs.ZFS() as zfs:
# Handle `id` filter specially to avoiding getting all pool
if filters and len(filters) == 1 and list(
filters[0][:2]) == ['id', '=']:
try:
pools = [zfs.get(filters[0][2]).__getstate__()]
except libzfs.ZFSException:
pools = []
else:
pools = [i.__getstate__() for i in zfs.pools]
return filter_list(pools, filters, options)
@accepts(Str('pool'))
async def get_disks(self, name):
try:
with libzfs.ZFS() as zfs:
disks = list(zfs.get(name).disks)
except libzfs.ZFSException as e:
raise CallError(str(e), errno.ENOENT)
await self.middleware.run_in_thread(geom.scan)
labelclass = geom.class_by_name('LABEL')
for absdev in disks:
dev = absdev.replace('/dev/', '').replace('.eli', '')
find = labelclass.xml.findall(
f".//provider[name='{dev}']/../consumer/provider")
name = None
if find:
name = geom.provider_by_id(find[0].get('ref')).geom.name
else:
g = geom.geom_by_name('DEV', dev)
if g:
name = g.consumer.provider.geom.name
if name and geom.geom_by_name('DISK', name):
yield name
else:
self.logger.debug(f'Could not find disk for {dev}')
@accepts(
Str('name'),
List('new'),
List('existing',
items=[
Dict(
'attachvdev',
Str('target'),
Str('type', enum=['DISK']),
Str('path'),
),
]),
)
@job()
def extend(self, job, name, new=None, existing=None):
"""
Extend a zfs pool `name` with `new` vdevs or attach to `existing` vdevs.
"""
if new is None and existing is None:
raise CallError('New or existing vdevs must be provided',
errno.EINVAL)
if new:
raise CallError('Adding new vdev is not implemented yet')
try:
with libzfs.ZFS() as zfs:
pool = zfs.get(name)
# Make sure we can find all target vdev
for i in (existing or []):
target = find_vdev(pool, i['target'])
if target is None:
raise CallError(f'Failed to find vdev for {target}',
errno.EINVAL)
i['target'] = target
for i in (existing or []):
newvdev = libzfs.ZFSVdev(zfs, i['type'].lower())
newvdev.path = i['path']
i['target'].attach(newvdev)
except libzfs.ZFSException as e:
raise CallError(str(e), e.code)
@accepts(Str('pool'), Str('label'))
def detach(self, name, label):
"""
Detach device `label` from the pool `pool`.
"""
try:
with libzfs.ZFS() as zfs:
pool = zfs.get(name)
target = find_vdev(pool, label)
if target is None:
raise CallError(f'Failed to find vdev for {label}',
errno.EINVAL)
target.detach()
except libzfs.ZFSException as e:
raise CallError(str(e), e.code)
@accepts(Str('pool'), Str('label'), Str('dev'))
def replace(self, name, label, dev):
"""
Replace device `label` with `dev` in pool `name`.
"""
try:
with libzfs.ZFS() as zfs:
pool = zfs.get(name)
target = find_vdev(pool, label)
if target is None:
raise CallError(f'Failed to find vdev for {label}',
errno.EINVAL)
newvdev = libzfs.ZFSVdev(zfs, 'disk')
newvdev.path = f'/dev/{dev}'
target.replace(newvdev)
except libzfs.ZFSException as e:
raise CallError(str(e), e.code)
@accepts(Str('name'))
@job(lock=lambda i: i[0])
def scrub(self, job, name):
"""
Start a scrub on pool `name`.
"""
try:
with libzfs.ZFS() as zfs:
pool = zfs.get(name)
pool.start_scrub()
except libzfs.ZFSException as e:
raise CallError(str(e), e.code)
def watch():
while True:
with libzfs.ZFS() as zfs:
scrub = zfs.get(name).scrub.__getstate__()
if scrub['function'] != 'SCRUB':
break
if scrub['state'] == 'FINISHED':
job.set_progress(100, 'Scrub finished')
break
if scrub['state'] == 'CANCELED':
break
if scrub['state'] == 'SCANNING':
job.set_progress(scrub['percentage'], 'Scrubbing')
time.sleep(1)
t = threading.Thread(target=watch, daemon=True)
t.start()
t.join()
@accepts()
def find_import(self):
with libzfs.ZFS() as zfs:
return [i.__getstate__() for i in zfs.find_import()]
class PeriodicSnapshotTaskService(CRUDService):
class Config:
datastore = 'storage.task'
datastore_prefix = 'task_'
datastore_extend = 'pool.snapshottask.extend'
datastore_extend_context = 'pool.snapshottask.extend_context'
namespace = 'pool.snapshottask'
@private
async def extend_context(self, extra):
return {
'state': await self.middleware.call('zettarepl.get_state'),
'vmware': await self.middleware.call('vmware.query'),
}
@private
async def extend(self, data, context):
Cron.convert_db_format_to_schedule(data, begin_end=True)
data['vmware_sync'] = any((vmware['filesystem'] == data['dataset'] or (
data['recursive']
and is_child(vmware['filesystem'], data['dataset'])))
for vmware in context['vmware'])
if 'error' in context['state']:
data['state'] = context['state']['error']
else:
data['state'] = context['state']['tasks'].get(
f'periodic_snapshot_task_{data["id"]}', {
'state': 'PENDING',
})
return data
@accepts(
Dict('periodic_snapshot_create',
Path('dataset', required=True),
Bool('recursive', required=True),
List('exclude', items=[Path('item', empty=False)], default=[]),
Int('lifetime_value', required=True),
Str('lifetime_unit',
enum=['HOUR', 'DAY', 'WEEK', 'MONTH', 'YEAR'],
required=True),
Str('naming_schema',
required=True,
validators=[ReplicationSnapshotNamingSchema()]),
Cron('schedule',
defaults={
'minute': '00',
'begin': '00:00',
'end': '23:59',
},
required=True,
begin_end=True),
Bool('allow_empty', default=True),
Bool('enabled', default=True),
register=True))
async def do_create(self, data):
"""
Create a Periodic Snapshot Task
Create a Periodic Snapshot Task that will take snapshots of specified `dataset` at specified `schedule`.
Recursive snapshots can be created if `recursive` flag is enabled. You can `exclude` specific child datasets
from snapshot.
Snapshots will be automatically destroyed after a certain amount of time, specified by
`lifetime_value` and `lifetime_unit`.
Snapshots will be named according to `naming_schema` which is a `strftime`-like template for snapshot name
and must contain `%Y`, `%m`, `%d`, `%H` and `%M`.
.. examples(websocket)::
Create a recursive Periodic Snapshot Task for dataset `data/work` excluding `data/work/temp`. Snapshots
will be created on weekdays every hour from 09:00 to 18:00 and will be stored for two weeks.
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "pool.snapshottask.create",
"params": [{
"dataset": "data/work",
"recursive": true,
"exclude": ["data/work/temp"],
"lifetime_value": 2,
"lifetime_unit": "WEEK",
"naming_schema": "auto_%Y-%m-%d_%H-%M",
"schedule": {
"minute": "0",
"hour": "*",
"dom": "*",
"month": "*",
"dow": "1,2,3,4,5",
"begin": "09:00",
"end": "18:00"
}
}]
}
"""
verrors = ValidationErrors()
verrors.add_child('periodic_snapshot_create', await
self._validate(data))
if verrors:
raise verrors
Cron.convert_schedule_to_db_format(data, begin_end=True)
data['id'] = await self.middleware.call(
'datastore.insert', self._config.datastore, data,
{'prefix': self._config.datastore_prefix})
await self.middleware.call('zettarepl.update_tasks')
return await self._get_instance(data['id'])
@accepts(Int('id', required=True),
Patch('periodic_snapshot_create', 'periodic_snapshot_update',
('attr', {
'update': True
})))
async def do_update(self, id, data):
"""
Update a Periodic Snapshot Task with specific `id`
See the documentation for `create` method for information on payload contents
.. examples(websocket)::
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "pool.snapshottask.update",
"params": [
1,
{
"dataset": "data/work",
"recursive": true,
"exclude": ["data/work/temp"],
"lifetime_value": 2,
"lifetime_unit": "WEEK",
"naming_schema": "auto_%Y-%m-%d_%H-%M",
"schedule": {
"minute": "0",
"hour": "*",
"dom": "*",
"month": "*",
"dow": "1,2,3,4,5",
"begin": "09:00",
"end": "18:00"
}
}
]
}
"""
old = await self._get_instance(id)
new = old.copy()
new.update(data)
verrors = ValidationErrors()
verrors.add_child('periodic_snapshot_update', await
self._validate(new))
if not new['enabled']:
for replication_task in await self.middleware.call(
'replication.query', [['enabled', '=', True]]):
if any(periodic_snapshot_task['id'] == id
for periodic_snapshot_task in
replication_task['periodic_snapshot_tasks']):
verrors.add('periodic_snapshot_update.enabled', (
f'You can\'t disable this periodic snapshot task because it is bound to enabled replication '
f'task {replication_task["id"]!r}'))
break
if verrors:
raise verrors
Cron.convert_schedule_to_db_format(new, begin_end=True)
for key in ('vmware_sync', 'state'):
new.pop(key, None)
await self.middleware.call('datastore.update', self._config.datastore,
id, new,
{'prefix': self._config.datastore_prefix})
await self.middleware.call('zettarepl.update_tasks')
return await self._get_instance(id)
@accepts(Int('id'))
async def do_delete(self, id):
"""
Delete a Periodic Snapshot Task with specific `id`
.. examples(websocket)::
:::javascript
{
"id": "6841f242-840a-11e6-a437-00e04d680384",
"msg": "method",
"method": "pool.snapshottask.delete",
"params": [
1
]
}
"""
for replication_task in await self.middleware.call(
'replication.query', [
['direction', '=', 'PUSH'],
['also_include_naming_schema', '=', []],
['enabled', '=', True],
]):
if len(replication_task['periodic_snapshot_tasks']) == 1:
if replication_task['periodic_snapshot_tasks'][0]['id'] == id:
raise CallError(
f'You are deleting the last periodic snapshot task bound to enabled replication task '
f'{replication_task["name"]!r} which will break it. Please, disable that replication task '
f'first.', )
response = await self.middleware.call('datastore.delete',
self._config.datastore, id)
await self.middleware.call('zettarepl.update_tasks')
return response
@item_method
@accepts(Int("id"))
async def run(self, id):
"""
Execute a Periodic Snapshot Task of `id`.
"""
task = await self._get_instance(id)
if not task["enabled"]:
raise CallError("Task is not enabled")
await self.middleware.call("zettarepl.run_periodic_snapshot_task",
task["id"])
async def _validate(self, data):
verrors = ValidationErrors()
if data['dataset'] not in (
await self.middleware.call('pool.filesystem_choices')):
verrors.add('dataset', 'Invalid ZFS dataset')
if not data['recursive'] and data['exclude']:
verrors.add(
'exclude',
'Excluding datasets has no sense for non-recursive periodic snapshot tasks'
)
for i, v in enumerate(data['exclude']):
if not v.startswith(f'{data["dataset"]}/'):
verrors.add(
f'exclude.{i}',
'Excluded dataset should be a child of selected dataset')
return verrors
class NFSService(SystemServiceService):
class Config:
service = "nfs"
datastore_prefix = "nfs_srv_"
datastore_extend = 'nfs.nfs_extend'
@private
def nfs_extend(self, nfs):
nfs["userd_manage_gids"] = nfs.pop("16")
return nfs
@private
def nfs_compress(self, nfs):
nfs["16"] = nfs.pop("userd_manage_gids")
return nfs
@accepts(
Dict(
'nfs_update',
Int('servers', validators=[Range(min=1, max=256)]),
Bool('udp'),
Bool('allow_nonroot'),
Bool('v4'),
Bool('v4_v3owner'),
Bool('v4_krb'),
List('bindip', items=[IPAddr('ip')]),
Int('mountd_port',
required=False,
validators=[Range(min=1, max=65535)]),
Int('rpcstatd_port',
required=False,
validators=[Range(min=1, max=65535)]),
Int('rpclockd_port',
required=False,
validators=[Range(min=1, max=65535)]),
Bool('userd_manage_gids'),
Bool('mountd_log'),
Bool('statd_lockd_log'),
))
async def do_update(self, data):
old = await self.config()
new = old.copy()
new.update(data)
verrors = ValidationErrors()
if not new["v4"] and new["v4_v3owner"]:
verrors.add("nfs_update.v4_v3owner",
"This option requires enabling NFSv4")
if new["v4_v3owner"] and new["userd_manage_gids"]:
verrors.add(
"nfs_update.userd_manage_gids",
"This option is incompatible with NFSv3 ownership model for NFSv4"
)
if verrors:
raise verrors
self.nfs_compress(new)
await self._update_service(old, new)
self.nfs_extend(new)
return new
class CatalogService(Service):
class Config:
cli_namespace = 'app.catalog'
@private
def cached(self, label):
return self.middleware.call_sync('cache.has_key', get_cache_key(label))
@accepts(Str('label'),
Dict(
'options',
Bool('cache', default=True),
Bool('cache_only', default=False),
Bool('retrieve_all_trains', default=True),
List('trains', items=[Str('train_name')]),
))
@returns(
Dict(
'trains',
additional_attrs=True,
example={
'charts': {
'chia': {
'name': 'chia',
'categories': ['storage', 'crypto'],
'app_readme': 'app readme here',
'location':
'/mnt/evo/ix-applications/catalogs/github_com_truenas_charts_git_master/charts/chia',
'healthy': True,
'healthy_error': False,
'latest_version': '1.2.0',
'latest_app_version': '1.1.6',
'icon_url': 'https://www.chia.net/img/chia_logo.svg',
}
}
}))
@job(lock=lambda args: f'catalog_item_retrieval_{json.dumps(args)}',
lock_queue_size=1)
def items(self, job, label, options):
"""
Retrieve item details for `label` catalog.
`options.cache` is a boolean which when set will try to get items details for `label` catalog from cache
if available.
`options.cache_only` is a boolean which when set will force usage of cache only for retrieving catalog
information. If the content for the catalog in question is not cached, no content would be returned. If
`options.cache` is unset, this attribute has no effect.
`options.retrieve_all_trains` is a boolean value which when set will retrieve information for all the trains
present in the catalog ( it is set by default ).
`options.trains` is a list of train name(s) which will allow selective filtering to retrieve only information
of desired trains in a catalog. If `options.retrieve_all_trains` is set, it has precedence over `options.train`.
"""
catalog = self.middleware.call_sync('catalog.get_instance', label)
all_trains = options['retrieve_all_trains']
cache_key = get_cache_key(label)
cache_available = self.middleware.call_sync('cache.has_key', cache_key)
if options['cache'] and options['cache_only'] and not cache_available:
return {}
if options['cache'] and cache_available:
job.set_progress(10, 'Retrieving cached content')
orig_data = self.middleware.call_sync('cache.get', cache_key)
job.set_progress(60, 'Normalizing cached content')
cached_data = {}
for train in orig_data:
if not all_trains and train not in options['trains']:
continue
train_data = {}
for catalog_item in orig_data[train]:
train_data[catalog_item] = {
k: v
for k, v in orig_data[train][catalog_item].items()
}
cached_data[train] = train_data
job.set_progress(100,
'Retrieved catalog item(s) details successfully')
self.middleware.loop.call_later(
30, functools.partial(job.set_result, None))
return cached_data
elif not os.path.exists(catalog['location']):
job.set_progress(5, f'Cloning {label!r} catalog repository')
self.middleware.call_sync('catalog.update_git_repository', catalog)
if all_trains:
# We can only safely say that the catalog is healthy if we retrieve data for all trains
self.middleware.call_sync('alert.oneshot_delete',
'CatalogNotHealthy', label)
trains = self.get_trains(job, catalog, options)
if all_trains:
# We will only update cache if we are retrieving data of all trains for a catalog
# which happens when we sync catalog(s) periodically or manually
# We cache for 90000 seconds giving system an extra 1 hour to refresh it's cache which
# happens after 24h - which means that for a small amount of time it's possible that user
# come with a case where system is trying to access cached data but it has expired and it's
# reading again from disk hence the extra 1 hour.
self.middleware.call_sync('cache.put', get_cache_key(label),
trains, 90000)
if label == self.middleware.call_sync(
'catalog.official_catalog_label'):
# Update feature map cache whenever official catalog is updated
self.middleware.call_sync('catalog.get_feature_map', False)
job.set_progress(
100, f'Successfully retrieved {label!r} catalog information')
self.middleware.loop.call_later(
30, functools.partial(job.set_result, None))
return trains
@private
def retrieve_train_names(self,
location,
all_trains=True,
trains_filter=None):
train_names = []
trains_filter = trains_filter or []
for train in os.listdir(location):
if (not (all_trains or train in trains_filter)
or not os.path.isdir(os.path.join(location, train))
or train.startswith('.') or train in ('library', 'docs')
or not VALID_TRAIN_REGEX.match(train)):
continue
train_names.append(train)
return train_names
@private
def get_trains(self, job, catalog, options):
# We make sure we do not dive into library and docs folders and not consider those a train
# This allows us to use these folders for placing helm library charts and docs respectively
trains = {'charts': {}, 'test': {}}
location = catalog['location']
questions_context = self.middleware.call_sync(
'catalog.get_normalised_questions_context')
unhealthy_apps = set()
preferred_trains = catalog['preferred_trains']
trains_to_traverse = self.retrieve_train_names(
location, options['retrieve_all_trains'], options['trains'])
# In order to calculate job progress, we need to know number of items we would be traversing
items = {}
for train in trains_to_traverse:
trains[train] = {}
items.update({
f'{i}_{train}': train
for i in os.listdir(os.path.join(location, train))
if os.path.isdir(os.path.join(location, train, i))
})
job.set_progress(
8,
f'Retrieving {", ".join(trains_to_traverse)!r} train(s) information'
)
total_items = len(items)
with concurrent.futures.ProcessPoolExecutor(
max_workers=(5 if total_items > 10 else 2)) as exc:
for index, result in enumerate(
zip(
items,
exc.map(functools.partial(item_details, items,
location, questions_context),
items,
chunksize=(10 if total_items > 10 else 5)))):
item_key = result[0]
item_info = result[1]
train = items[item_key]
item = item_key.removesuffix(f'_{train}')
job.set_progress(
int((index / total_items) * 80) + 10,
f'Retrieved information of {item!r} item from {train!r} train'
)
trains[train][item] = item_info
if train in preferred_trains and not trains[train][item][
'healthy']:
unhealthy_apps.add(f'{item} ({train} train)')
if unhealthy_apps:
self.middleware.call_sync('alert.oneshot_create',
'CatalogNotHealthy', {
'catalog': catalog['id'],
'apps': ', '.join(unhealthy_apps)
})
job.set_progress(
90,
f'Retrieved {", ".join(trains_to_traverse)} train(s) information')
return trains
@private
def item_version_details(self, version_path, questions_context=None):
if not questions_context:
questions_context = self.middleware.call_sync(
'catalog.get_normalised_questions_context')
return get_item_version_details(version_path, questions_context)
@private
async def get_normalised_questions_context(self):
k8s_started = await self.middleware.call(
'kubernetes.validate_k8s_setup', False)
return {
'nic_choices':
await self.middleware.call('chart.release.nic_choices'),
'gpus':
await self.middleware.call('k8s.gpu.available_gpus')
if k8s_started else {},
'timezones':
await self.middleware.call('system.general.timezone_choices'),
'node_ip':
await self.middleware.call('kubernetes.node_ip'),
'certificates':
await self.middleware.call('chart.release.certificate_choices'),
'certificate_authorities':
await self.middleware.call(
'chart.release.certificate_authority_choices'),
'system.general.config':
await self.middleware.call('system.general.config'),
}
class SharingNFSService(CRUDService):
class Config:
namespace = "sharing.nfs"
datastore = "sharing.nfs_share"
datastore_prefix = "nfs_"
datastore_extend = "sharing.nfs.extend"
@accepts(
Dict(
"sharingnfs_create",
List("paths", items=[Dir("path")]),
Str("comment"),
List("networks", items=[IPAddr("network", cidr=True)]),
List("hosts", items=[Str("host")]),
Bool("alldirs"),
Bool("ro"),
Bool("quiet"),
Str("maproot_user", required=False, default=None),
Str("maproot_group", required=False, default=None),
Str("mapall_user", required=False, default=None),
Str("mapall_group", required=False, default=None),
List("security",
items=[
Str("provider", enum=["SYS", "KRB5", "KRB5I", "KRB5P"])
]),
register=True,
))
async def do_create(self, data):
verrors = ValidationErrors()
await self.validate(data, "sharingnfs_create", verrors)
if verrors:
raise verrors
await self.compress(data)
paths = data.pop("paths")
data["id"] = await self.middleware.call(
"datastore.insert",
self._config.datastore,
data,
{"prefix": self._config.datastore_prefix},
)
for path in paths:
await self.middleware.call(
"datastore.insert",
"sharing.nfs_share_path",
{
"share_id": data["id"],
"path": path,
},
)
await self.extend(data)
await self.middleware.call("service.reload", "nfs")
return data
@accepts(Int("id"),
Patch("sharingnfs_create", "sharingnfs_update", ("attr", {
"update": True
})))
async def do_update(self, id, data):
verrors = ValidationErrors()
old = await self.middleware.call(
"datastore.query",
self._config.datastore,
[("id", "=", id)],
{
"extend": self._config.datastore_extend,
"prefix": self._config.datastore_prefix,
"get": True
},
)
new = old.copy()
new.update(data)
await self.validate(new, "sharingnfs_update", verrors, old=old)
if verrors:
raise verrors
await self.compress(new)
paths = new.pop("paths")
await self.middleware.call("datastore.update", self._config.datastore,
id, new,
{"prefix": self._config.datastore_prefix})
await self.middleware.call("datastore.delete",
"sharing.nfs_share_path",
[["share_id", "=", id]])
for path in paths:
await self.middleware.call(
"datastore.insert",
"sharing.nfs_share_path",
{
"share_id": id,
"path": path,
},
)
await self.extend(new)
new["paths"] = paths
await self.middleware.call("service.reload", "nfs")
return new
@accepts(Int("id"))
async def do_delete(self, id):
await self.middleware.call("datastore.delete",
"sharing.nfs_share_path",
[["share_id", "=", id]])
await self.middleware.call("datastore.delete", self._config.datastore,
id)
@private
async def validate(self, data, schema_name, verrors, old=None):
if not data["paths"]:
verrors.add(f"{schema_name}.paths",
"At least one path is required")
await self.middleware.run_in_io_thread(self.validate_paths, data,
schema_name, verrors)
filters = []
if old:
filters.append(["id", "!=", old["id"]])
other_shares = await self.middleware.call("sharing.nfs.query", filters)
dns_cache = await self.resolve_hostnames(
sum([share["hosts"]
for share in other_shares], []) + data["hosts"])
await self.middleware.run_in_io_thread(
self.validate_hosts_and_networks, other_shares, data, schema_name,
verrors, dns_cache)
for k in ["maproot", "mapall"]:
if not data[f"{k}_user"] and not data[f"{k}_group"]:
pass
elif not data[f"{k}_user"] and data[f"{k}_group"]:
verrors.add(
f"{schema_name}.{k}_user",
"This field is required when map group is specified")
else:
user = await self.middleware.call(
"user.query", [("username", "=", data[f"{k}_user"])])
if not user:
verrors.add(f"{schema_name}.{k}_user", "User not found")
if data[f"{k}_group"]:
group = await self.middleware.call(
"group.query", [("group", "=", data[f"{k}_group"])])
if not group:
verrors.add(f"{schema_name}.{k}_group",
"Group not found")
if data["maproot_user"] and data["mapall_user"]:
verrors.add(f"{schema_name}.mapall_user",
"maproot_user disqualifies mapall_user")
if data["security"]:
nfs_config = await self.middleware.call("nfs.config")
if not nfs_config["v4"]:
verrors.add(f"{schema_name}.security",
"This is not allowed when NFS v4 is disabled")
@private
def validate_paths(self, data, schema_name, verrors):
dev = None
is_mountpoint = False
for i, path in enumerate(data["paths"]):
stat = os.stat(path)
if dev is None:
dev = stat.st_dev
else:
if dev != stat.st_dev:
verrors.add(
f"{schema_name}.paths.{i}",
"Paths for a NFS share must reside within the same filesystem"
)
parent = os.path.abspath(os.path.join(path, ".."))
if os.stat(parent).st_dev != dev:
is_mountpoint = True
if any(
os.path.abspath(p).startswith(parent + "/")
for p in data["paths"] if p != path):
verrors.add(
f"{schema_name}.paths.{i}",
"You cannot share a mount point and subdirectories all at once"
)
if not is_mountpoint and data["alldirs"]:
verrors.add(f"{schema_name}.alldirs",
"This option can only be used for datasets")
@private
async def resolve_hostnames(self, hostnames):
hostnames = list(set(hostnames))
async def resolve(hostname):
try:
return (await asyncio.wait_for(
self.middleware.run_in_io_thread(socket.getaddrinfo,
hostname, None),
5))[0][4][0]
except Exception as e:
self.logger.warning("Unable to resolve host %r: %r", hostname,
e)
return None
resolved_hostnames = await asyncio_map(resolve, hostnames, 8)
return dict(zip(hostnames, resolved_hostnames))
@private
def validate_hosts_and_networks(self, other_shares, data, schema_name,
verrors, dns_cache):
explanation = (
". This is so because /etc/exports does not act like ACL and it is undefined which rule among "
"all overlapping networks will be applied.")
dev = os.stat(data["paths"][0]).st_dev
used_networks = set()
for share in other_shares:
try:
share_dev = os.stat(share["paths"][0]).st_dev
except Exception:
self.logger.warning("Failed to stat first path for %r",
share,
exc_info=True)
continue
if share_dev == dev:
for host in share["hosts"]:
host = dns_cache[host]
if host is None:
continue
try:
network = ipaddress.ip_network(host)
except Exception:
self.logger.warning("Got invalid host %r", host)
continue
else:
used_networks.add(network)
for network in share["networks"]:
try:
network = ipaddress.ip_network(network, strict=False)
except Exception:
self.logger.warning("Got invalid network %r", network)
continue
else:
used_networks.add(network)
if not share["hosts"] and not share["networks"]:
used_networks.add(ipaddress.ip_network("0.0.0.0/0"))
used_networks.add(ipaddress.ip_network("::/0"))
if share["alldirs"] and data["alldirs"]:
verrors.add(
f"{schema_name}.alldirs",
"This option is only available once per mountpoint")
had_explanation = False
for i, host in enumerate(data["hosts"]):
host = dns_cache[host]
if host is None:
verrors.add(f"{schema_name}.hosts.{i}",
"Unable to resolve host")
continue
network = ipaddress.ip_network(host)
for another_network in used_networks:
if network.overlaps(another_network):
verrors.add(f"{schema_name}.hosts.{i}", (
f"You can't share same filesystem with overlapping networks {network} and {another_network}"
+ ("" if had_explanation else explanation)))
had_explanation = True
used_networks.add(network)
had_explanation = False
for i, network in enumerate(data["networks"]):
network = ipaddress.ip_network(network, strict=False)
for another_network in used_networks:
if network.overlaps(another_network):
verrors.add(f"{schema_name}.networks.{i}", (
f"You can't share same filesystem with overlapping networks {network} and {another_network}"
+ ("" if had_explanation else explanation)))
had_explanation = True
used_networks.add(network)
if not data["hosts"] and not data["networks"]:
if used_networks:
verrors.add(
f"{schema_name}.networks",
(f"You can't share same filesystem with all hosts twice" +
("" if had_explanation else explanation)))
@private
async def extend(self, data):
data["paths"] = [
path["path"] for path in await self.middleware.call(
"datastore.query", "sharing.nfs_share_path",
[["share_id", "=", data["id"]]])
]
data["networks"] = data.pop("network").split()
data["hosts"] = data["hosts"].split()
data["security"] = [s.upper() for s in data["security"]]
return data
@private
async def compress(self, data):
data["network"] = " ".join(data.pop("networks"))
data["hosts"] = " ".join(data["hosts"])
data["security"] = [s.lower() for s in data["security"]]
return data
class ZFSPoolService(CRUDService):
class Config:
namespace = 'zfs.pool'
private = True
process_pool = True
@filterable
def query(self, filters, options):
# We should not get datasets, there is zfs.dataset.query for that
state_kwargs = {'datasets_recursive': False}
with libzfs.ZFS() as zfs:
# Handle `id` filter specially to avoiding getting all pool
if filters and len(filters) == 1 and list(
filters[0][:2]) == ['id', '=']:
try:
pools = [
zfs.get(filters[0][2]).__getstate__(**state_kwargs)
]
except libzfs.ZFSException:
pools = []
else:
pools = [i.__getstate__(**state_kwargs) for i in zfs.pools]
return filter_list(pools, filters, options)
@accepts(
Dict(
'zfspool_create',
Str('name', required=True),
List('vdevs',
items=[
Dict(
'vdev',
Str('root',
enum=['DATA', 'CACHE', 'LOG', 'SPARE'],
required=True),
Str('type',
enum=[
'RAIDZ1', 'RAIDZ2', 'RAIDZ3', 'MIRROR',
'STRIPE'
],
required=True),
List('devices', items=[Str('disk')], required=True),
),
],
required=True),
Dict('options', additional_attrs=True),
Dict('fsoptions', additional_attrs=True),
), )
def do_create(self, data):
with libzfs.ZFS() as zfs:
topology = convert_topology(zfs, data['vdevs'])
zfs.create(data['name'], topology, data['options'],
data['fsoptions'])
return self.middleware.call_sync('zfs.pool._get_instance',
data['name'])
@accepts(Str('pool'),
Dict(
'options',
Dict('properties', additional_attrs=True),
))
def do_update(self, name, options):
try:
with libzfs.ZFS() as zfs:
pool = zfs.get(name)
for k, v in options['properties'].items():
prop = pool.properties[k]
if 'value' in v:
prop.value = v['value']
elif 'parsed' in v:
prop.parsed = v['parsed']
except libzfs.ZFSException as e:
raise CallError(str(e))
@accepts(Str('pool'), Dict(
'options',
Bool('force', default=False),
))
def do_delete(self, name, options):
try:
with libzfs.ZFS() as zfs:
zfs.destroy(name, force=options['force'])
except libzfs.ZFSException as e:
errno_ = errno.EFAULT
if e.code == libzfs.Error.UMOUNTFAILED:
errno_ = errno.EBUSY
raise CallError(str(e), errno_)
@accepts(Str('pool', required=True))
def upgrade(self, pool):
try:
with libzfs.ZFS() as zfs:
zfs.get(pool).upgrade()
except libzfs.ZFSException as e:
raise CallError(str(e))
@accepts(Str('pool'), Dict(
'options',
Bool('force', default=False),
))
def export(self, name, options):
try:
with libzfs.ZFS() as zfs:
# FIXME: force not yet implemented
pool = zfs.get(name)
zfs.export_pool(pool)
except libzfs.ZFSException as e:
raise CallError(str(e))
@accepts(Str('pool'))
def get_devices(self, name):
try:
with libzfs.ZFS() as zfs:
return [i.replace('/dev/', '') for i in zfs.get(name).disks]
except libzfs.ZFSException as e:
raise CallError(str(e), errno.ENOENT)
@accepts(Str('pool'))
def get_disks(self, name):
disks = self.get_devices(name)
geom.scan()
labelclass = geom.class_by_name('LABEL')
for dev in disks:
dev = dev.replace('.eli', '')
find = labelclass.xml.findall(
f".//provider[name='{dev}']/../consumer/provider")
name = None
if find:
name = geom.provider_by_id(find[0].get('ref')).geom.name
else:
g = geom.geom_by_name('DEV', dev)
if g:
name = g.consumer.provider.geom.name
if name and (name.startswith('multipath/')
or geom.geom_by_name('DISK', name)):
yield name
else:
self.logger.debug(f'Could not find disk for {dev}')
@accepts(
Str('name'),
List('new', default=None, null=True),
List('existing',
items=[
Dict(
'attachvdev',
Str('target'),
Str('type', enum=['DISK']),
Str('path'),
),
],
null=True,
default=None),
)
@job()
def extend(self, job, name, new=None, existing=None):
"""
Extend a zfs pool `name` with `new` vdevs or attach to `existing` vdevs.
"""
if new is None and existing is None:
raise CallError('New or existing vdevs must be provided',
errno.EINVAL)
try:
with libzfs.ZFS() as zfs:
pool = zfs.get(name)
if new:
topology = convert_topology(zfs, new)
pool.attach_vdevs(topology)
# Make sure we can find all target vdev
for i in (existing or []):
target = find_vdev(pool, i['target'])
if target is None:
raise CallError(
f"Failed to find vdev for {i['target']}",
errno.EINVAL)
i['target'] = target
for i in (existing or []):
newvdev = libzfs.ZFSVdev(zfs, i['type'].lower())
newvdev.path = i['path']
i['target'].attach(newvdev)
except libzfs.ZFSException as e:
raise CallError(str(e), e.code)
def __zfs_vdev_operation(self, name, label, op, *args):
try:
with libzfs.ZFS() as zfs:
pool = zfs.get(name)
target = find_vdev(pool, label)
if target is None:
raise CallError(f'Failed to find vdev for {label}',
errno.EINVAL)
op(target, *args)
except libzfs.ZFSException as e:
raise CallError(str(e), e.code)
@accepts(Str('pool'), Str('label'))
def detach(self, name, label):
"""
Detach device `label` from the pool `pool`.
"""
self.__zfs_vdev_operation(name, label, lambda target: target.detach())
@accepts(Str('pool'), Str('label'))
def offline(self, name, label):
"""
Offline device `label` from the pool `pool`.
"""
self.__zfs_vdev_operation(name, label, lambda target: target.offline())
@accepts(Str('pool'), Str('label'), Bool('expand', default=False))
def online(self, name, label, expand=False):
"""
Online device `label` from the pool `pool`.
"""
self.__zfs_vdev_operation(name, label,
lambda target, *args: target.online(*args),
expand)
@accepts(Str('pool'), Str('label'))
def remove(self, name, label):
"""
Remove device `label` from the pool `pool`.
"""
self.__zfs_vdev_operation(name, label, lambda target: target.remove())
@accepts(Str('pool'), Str('label'), Str('dev'))
def replace(self, name, label, dev):
"""
Replace device `label` with `dev` in pool `name`.
"""
try:
with libzfs.ZFS() as zfs:
pool = zfs.get(name)
target = find_vdev(pool, label)
if target is None:
raise CallError(f'Failed to find vdev for {label}',
errno.EINVAL)
newvdev = libzfs.ZFSVdev(zfs, 'disk')
newvdev.path = f'/dev/{dev}'
# FIXME: Replace using old path is not working for some reason
# Lets use guid for now.
target.path = str(target.guid)
target.replace(newvdev)
except libzfs.ZFSException as e:
raise CallError(str(e), e.code)
@accepts(Str('name', required=True),
Str('action', enum=['START', 'STOP', 'PAUSE'], default='START'))
@job(lock=lambda i: f'{i[0]}-{i[1] if len(i) >= 2 else "START"}')
def scrub(self, job, name, action=None):
"""
Start/Stop/Pause a scrub on pool `name`.
"""
if action != 'PAUSE':
try:
with libzfs.ZFS() as zfs:
pool = zfs.get(name)
if action == 'START':
pool.start_scrub()
else:
pool.stop_scrub()
except libzfs.ZFSException as e:
raise CallError(str(e), e.code)
else:
proc = subprocess.Popen(f'zpool scrub -p {name}'.split(' '),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc.communicate()
if proc.returncode != 0:
raise CallError('Unable to pause scrubbing')
def watch():
while True:
with libzfs.ZFS() as zfs:
scrub = zfs.get(name).scrub.__getstate__()
if scrub['pause']:
job.set_progress(100, 'Scrub paused')
break
if scrub['function'] != 'SCRUB':
break
if scrub['state'] == 'FINISHED':
job.set_progress(100, 'Scrub finished')
break
if scrub['state'] == 'CANCELED':
break
if scrub['state'] == 'SCANNING':
job.set_progress(scrub['percentage'], 'Scrubbing')
time.sleep(1)
if action == 'START':
t = threading.Thread(target=watch, daemon=True)
t.start()
t.join()
@accepts()
def find_import(self):
with libzfs.ZFS() as zfs:
return [i.__getstate__() for i in zfs.find_import()]
@accepts(
Str('name_or_guid'),
Dict('options', additional_attrs=True),
Bool('any_host', default=True),
Str('cachefile', null=True, default=None),
)
def import_pool(self, name_or_guid, options, any_host, cachefile):
found = False
with libzfs.ZFS() as zfs:
for pool in zfs.find_import(cachefile=cachefile):
if pool.name == name_or_guid or str(pool.guid) == name_or_guid:
found = pool
break
if not found:
raise CallError(f'Pool {name_or_guid} not found.',
errno.ENOENT)
zfs.import_pool(found, found.name, options, any_host=any_host)
@accepts(Str('pool'))
async def find_not_online(self, pool):
pool = await self.middleware.call('zfs.pool.query',
[['id', '=', pool]], {'get': True})
unavails = []
for nodes in pool['groups'].values():
for node in nodes:
unavails.extend(self.__find_not_online(node))
return unavails
def __find_not_online(self, node):
if len(node['children']) == 0 and node['status'] not in ('ONLINE',
'AVAIL'):
return [node]
unavails = []
for child in node['children']:
unavails.extend(self.__find_not_online(child))
return unavails
def get_vdev(self, name, vname):
try:
with libzfs.ZFS() as zfs:
pool = zfs.get(name)
vdev = find_vdev(pool, vname)
if not vdev:
raise CallError(f'{vname} not found in {name}',
errno.ENOENT)
return vdev.__getstate__()
except libzfs.ZFSException as e:
raise CallError(str(e))
class DatastoreService(Service):
def _filters_to_queryset(self, filters, field_prefix=None):
opmap = {
'=': 'exact',
'!=': 'exact',
'>': 'gt',
'>=': 'gte',
'<': 'lt',
'<=': 'lte',
'~': 'regex',
}
rv = []
for f in filters:
if not isinstance(f, (list, tuple)):
raise ValueError('Filter must be a list: {0}'.format(f))
if len(f) == 3:
name, op, value = f
if field_prefix:
name = field_prefix + name
if op not in opmap:
raise Exception("Invalid operation: {0}".format(op))
q = Q(**{'{0}__{1}'.format(name, opmap[op]): value})
if op == '!=':
q.negate()
rv.append(q)
elif len(f) == 2:
op, value = f
if op == 'OR':
or_value = None
for value in self._filters_to_queryset(
value, field_prefix=field_prefix):
if or_value is None:
or_value = value
else:
or_value |= value
rv.append(or_value)
else:
raise ValueError('Invalid operation: {0}'.format(op))
else:
raise Exception("Invalid filter {0}".format(f))
return rv
def __get_model(self, name):
"""Helper method to get Model for given name
e.g. network.interfaces -> Interfaces
"""
app, model = name.split('.', 1)
return apps.get_model(app, model)
def __queryset_serialize(self, qs, extend=None, field_prefix=None):
for i in self.middleware.threaded(lambda: list(qs)):
yield django_modelobj_serialize(self.middleware,
i,
extend=extend,
field_prefix=field_prefix)
@accepts(
Str('name'),
List('query-filters', register=True),
Dict(
'query-options',
Str('extend'),
Dict('extra', additional_attrs=True),
List('order_by'),
Bool('count'),
Bool('get'),
Str('prefix'),
register=True,
),
)
def query(self, name, filters=None, options=None):
"""Query for items in a given collection `name`.
`filters` is a list which each entry can be in one of the following formats:
entry: simple_filter | conjuntion
simple_filter: '[' attribute_name, OPERATOR, value ']'
conjunction: '[' CONJUNTION, '[' simple_filter (',' simple_filter)* ']]'
OPERATOR: ('=' | '!=' | '>' | '>=' | '<' | '<=' | '~' )
CONJUNCTION: 'OR'
e.g.
`['OR', [ ['username', '=', 'root' ], ['uid', '=', 0] ] ]`
`[ ['username', '=', 'root' ] ]`
.. examples(websocket)::
Querying for username "root" and returning a single item:
:::javascript
{
"id": "d51da71b-bb48-4b8b-a8f7-6046fcc892b4",
"msg": "method",
"method": "datastore.query",
"params": ["account.bsdusers", [ ["username", "=", "root" ] ], {"get": true}]
}
"""
model = self.__get_model(name)
if options is None:
options = {}
else:
# We do not want to make changes to original options
# which might happen with "prefix"
options = options.copy()
qs = model.objects.all()
extra = options.get('extra')
if extra:
qs = qs.extra(**extra)
prefix = options.get('prefix')
if filters:
qs = qs.filter(*self._filters_to_queryset(filters, prefix))
order_by = options.get('order_by')
if order_by:
if prefix:
# Do not change original order_by
order_by = order_by[:]
for i, order in enumerate(order_by):
if order.startswith('-'):
order_by[i] = '-' + prefix + order[1:]
else:
order_by[i] = prefix + order
qs = qs.order_by(*order_by)
if options.get('count') is True:
return qs.count()
result = list(
self.__queryset_serialize(qs,
extend=options.get('extend'),
field_prefix=options.get('prefix')))
if options.get('get') is True:
return result[0]
return result
@accepts(Str('name'), Ref('query-options'))
def config(self, name, options=None):
"""
Get configuration settings object for a given `name`.
This is a shortcut for `query(name, {"get": true})`.
"""
if options is None:
options = {}
options['get'] = True
return self.query(name, None, options)
@accepts(Str('name'), Dict('data', additional_attrs=True))
def insert(self, name, data):
"""
Insert a new entry to `name`.
"""
model = self.__get_model(name)
for field in model._meta.fields:
if field.name not in data:
continue
if isinstance(field, ForeignKey):
data[field.name] = field.rel.to.objects.get(
pk=data[field.name])
obj = model(**data)
obj.save()
return obj.pk
@accepts(Str('name'), Int('id'), Dict('data', additional_attrs=True))
def update(self, name, id, data):
"""
Update an entry `id` in `name`.
"""
model = self.__get_model(name)
obj = model.objects.get(pk=id)
for field in model._meta.fields:
if field.name not in data:
continue
if isinstance(field, ForeignKey):
data[field.name] = field.rel.to.objects.get(
pk=data[field.name])
for k, v in list(data.items()):
setattr(obj, k, v)
obj.save()
return obj.pk
@accepts(Str('name'), Int('id'))
def delete(self, name, id):
"""
Delete an entry `id` in `name`.
"""
model = self.__get_model(name)
model.objects.get(pk=id).delete()
return True
@private
def sql(self, query, params=None):
cursor = connection.cursor()
rv = None
try:
if params is None:
cursor.executelocal(query)
else:
cursor.executelocal(query, params)
rv = cursor.fetchall()
finally:
cursor.close()
return rv
class ChartReleaseService(CRUDService):
class Config:
datastore_primary_key_type = 'string'
namespace = 'chart.release'
cli_namespace = 'app.chart_release'
ENTRY = Dict(
'chart_release_entry',
Str('name', required=True),
Dict('info', additional_attrs=True),
Dict('config', additional_attrs=True),
List('hooks'),
Int('version', required=True, description='Version of chart release'),
Str('namespace', required=True),
Dict(
'chart_metadata',
Str('name', required=True, description='Name of application'),
Str('version', required=True,
description='Version of application'),
Str('latest_chart_version',
required=True,
description='Latest available version of application'),
additional_attrs=True,
),
Str('id', required=True),
Str('catalog', required=True),
Str('catalog_train', required=True),
Str('path', required=True),
Str('dataset', required=True),
Str('status', required=True),
List('used_ports',
items=[
Dict(
'port',
Int('port', required=True),
Str('protocol', required=True),
)
],
required=True),
Dict(
'pod_status',
Int('available', required=True),
Int('desired', required=True),
required=True,
),
Bool('update_available', required=True),
Str('human_version',
required=True,
description='Human friendly version identifier for chart release'),
Str('human_latest_version',
required=True,
description=
'Human friendly latest available version identifier for chart release'
),
Bool(
'container_images_update_available',
required=True,
description=
'Will be set when any image(s) being used in the chart release has a newer version available'
),
Dict('portals', additional_attrs=True),
Dict('chart_schema', null=True, additional_attrs=True),
Dict('history', additional_attrs=True),
Dict(
'resources',
Dict('storage_class', additional_attrs=True),
List('persistent_volumes'),
List('host_path_volumes'),
List('locked_host_paths'),
Dict('container_images', additional_attrs=True),
List('truenas_certificates', items=[Int('certificate_id')]),
List('truenas_certificate_authorities',
items=[Int('certificate_authority_id')]),
*[List(r.value) for r in Resources],
),
)
@filterable
async def query(self, filters, options):
"""
Query available chart releases.
`query-options.extra.retrieve_resources` is a boolean when set will retrieve existing kubernetes resources
in the chart namespace.
`query-options.extra.history` is a boolean when set will retrieve all chart version upgrades
for a chart release.
`query-options.extra.include_chart_schema` is a boolean when set will retrieve the schema being used by
the chart release in question.
`query-options.extra.resource_events` is a boolean when set will retrieve individual events of each resource.
This only has effect if `query-options.extra.retrieve_resources` is set.
"""
if not await self.middleware.call('kubernetes.validate_k8s_setup',
False):
# We use filter_list here to ensure that `options` are respected, options like get: true
return filter_list([], filters, options)
k8s_config = await self.middleware.call('kubernetes.config')
update_catalog_config = {}
catalogs = await self.middleware.call(
'catalog.query', [], {'extra': {
'item_details': True
}})
container_images = {}
for image in await self.middleware.call('container.image.query'):
for tag in image['repo_tags']:
if not container_images.get(tag):
container_images[tag] = image
for catalog in catalogs:
update_catalog_config[catalog['label']] = {}
for train in catalog['trains']:
train_data = {}
for catalog_item in catalog['trains'][train]:
max_version = catalog['trains'][train][catalog_item][
'latest_version'] or '0.0.0'
app_version = catalog['trains'][train][catalog_item][
'latest_app_version'] or '0.0.0'
train_data[catalog_item] = {
'chart_version': parse_version(max_version),
'app_version': app_version,
}
update_catalog_config[catalog['label']][train] = train_data
k8s_node_ip = await self.middleware.call('kubernetes.node_ip')
options = options or {}
extra = copy.deepcopy(options.get('extra', {}))
retrieve_schema = extra.get('include_chart_schema')
get_resources = extra.get('retrieve_resources')
get_locked_paths = extra.get('retrieve_locked_paths')
locked_datasets = await self.middleware.call(
'zfs.dataset.locked_datasets') if get_locked_paths else []
get_history = extra.get('history')
if retrieve_schema:
questions_context = await self.middleware.call(
'catalog.get_normalised_questions_context')
else:
questions_context = None
if filters and len(filters) == 1 and filters[0][:2] == ['id', '=']:
extra['namespace_filter'] = [
'metadata.namespace', '=',
f'{CHART_NAMESPACE_PREFIX}{filters[0][-1]}'
]
resources_filters = [extra['namespace_filter']]
else:
resources_filters = [[
'metadata.namespace', '^', CHART_NAMESPACE_PREFIX
]]
ports_used = collections.defaultdict(list)
for node_port_svc in await self.middleware.call(
'k8s.service.query',
[['spec.type', '=', 'NodePort']] + resources_filters):
release_name = node_port_svc['metadata']['namespace'][
len(CHART_NAMESPACE_PREFIX):]
ports_used[release_name].extend([{
'port': p['node_port'],
'protocol': p['protocol']
} for p in node_port_svc['spec']['ports']])
if get_resources:
storage_mapping = await self.middleware.call(
'chart.release.get_workload_storage_details')
resources_mapping = await self.middleware.call(
'chart.release.get_resources_with_workload_mapping', {
'resource_events':
extra.get('resource_events', False),
'resource_filters':
resources_filters,
'resources': [
r.name for r in (Resources if get_resources else [
Resources.POD, Resources.DEPLOYMENT, Resources.
STATEFULSET
])
],
})
resources = resources_mapping['resources']
release_secrets = await self.middleware.call(
'chart.release.releases_secrets', extra)
releases = []
for name, release in release_secrets.items():
config = {}
release_data = release['releases'].pop(0)
cur_version = release_data['chart_metadata']['version']
for rel_data in filter(
lambda r: r['chart_metadata']['version'] == cur_version,
itertools.chain(reversed(release['releases']),
[release_data])):
config.update(rel_data['config'])
pods_status = resources_mapping['workload_status'][name]
pod_diff = pods_status['available'] - pods_status['desired']
status = 'ACTIVE'
if pod_diff == 0 and pods_status['desired'] == 0:
status = 'STOPPED'
elif pod_diff < 0:
status = 'DEPLOYING'
# We will retrieve all host ports being used
for pod in filter(lambda p: p['status']['phase'] == 'Running',
resources[Resources.POD.value][name]):
for container in pod['spec']['containers']:
ports_used[name].extend([{
'port': p['host_port'],
'protocol': p['protocol']
} for p in (container['ports'] or []) if p['host_port']])
release_data.update({
'path':
os.path.join('/mnt', k8s_config['dataset'], 'releases', name),
'dataset':
os.path.join(k8s_config['dataset'], 'releases', name),
'config':
config,
'status':
status,
'used_ports':
ports_used[name],
'pod_status':
pods_status,
})
container_images_normalized = {
i_name: {
'id':
image_details.get('id'),
'update_available':
image_details.get('update_available', False)
}
for i_name, image_details in map(
lambda i: (i, container_images.get(i, {})),
list(
set(c['image'] for workload_type in ('deployments',
'statefulsets')
for workload in resources[workload_type][name]
for c in workload['spec']['template']['spec']
['containers'])))
}
if get_resources:
release_resources = {
'storage_class':
storage_mapping['storage_classes'][get_storage_class_name(
name)],
'persistent_volumes':
storage_mapping['persistent_volumes'][name],
'host_path_volumes':
await self.host_path_volumes(
itertools.chain(*[
resources[getattr(Resources, k).value][name]
for k in ('DEPLOYMENT', 'STATEFULSET')
])),
**{r.value: resources[r.value][name]
for r in Resources},
'container_images':
container_images_normalized,
'truenas_certificates': [
v['id'] for v in release_data['config'].get(
'ixCertificates', {}).values()
],
'truenas_certificate_authorities': [
v['id'] for v in release_data['config'].get(
'ixCertificateAuthorities', {}).values()
],
}
if get_locked_paths:
release_resources['locked_host_paths'] = [
v['host_path']['path']
for v in release_resources['host_path_volumes']
if await self.middleware.call(
'pool.dataset.path_in_locked_datasets',
v['host_path']['path'], locked_datasets)
]
release_data['resources'] = release_resources
if get_history:
release_data['history'] = release['history']
for k, v in release_data['history'].items():
r_app_version = self.normalize_app_version_of_chart_release(
v)
release_data['history'][k].update({
'human_version':
f'{r_app_version}_{parse_version(v["chart_metadata"]["version"])}',
})
current_version = parse_version(
release_data['chart_metadata']['version'])
catalog_version_dict = update_catalog_config.get(
release_data['catalog'],
{}).get(release_data['catalog_train'],
{}).get(release_data['chart_metadata']['name'], {})
latest_version = catalog_version_dict.get('chart_version',
current_version)
latest_app_version = catalog_version_dict.get('app_version')
release_data['update_available'] = latest_version > current_version
app_version = self.normalize_app_version_of_chart_release(
release_data)
if release_data['chart_metadata']['name'] == 'ix-chart':
# Latest app version for ix-chart remains same
latest_app_version = app_version
for key, app_v, c_v in (
('human_version', app_version, current_version),
('human_latest_version', latest_app_version, latest_version),
):
if app_v:
release_data[key] = f'{app_v}_{c_v}'
else:
release_data[key] = str(c_v)
if retrieve_schema:
chart_path = os.path.join(
release_data['path'], 'charts',
release_data['chart_metadata']['version'])
if os.path.exists(chart_path):
release_data['chart_schema'] = await self.middleware.call(
'catalog.item_version_details', chart_path,
questions_context)
else:
release_data['chart_schema'] = None
release_data['container_images_update_available'] = any(
details['update_available']
for details in container_images_normalized.values())
release_data['chart_metadata']['latest_chart_version'] = str(
latest_version)
release_data['portals'] = await self.middleware.call(
'chart.release.retrieve_portals_for_chart_release',
release_data, k8s_node_ip)
if 'icon' not in release_data['chart_metadata']:
release_data['chart_metadata']['icon'] = None
releases.append(release_data)
return filter_list(releases, filters, options)
@private
def normalize_app_version_of_chart_release(self, release_data):
app_version = None
if release_data['chart_metadata']['name'] == 'ix-chart':
image_config = release_data['config'].get('image') or {}
if all(k in image_config for k in ('tag', 'repository')):
# TODO: Let's see if we can find sane versioning for `latest` from upstream
if image_config['tag'] == 'latest':
app_version = f'{image_config["repository"]}:{image_config["tag"]}'
else:
app_version = image_config['tag']
else:
app_version = release_data['chart_metadata'].get('appVersion')
return app_version
@private
async def host_path_volumes(self, resources):
host_path_volumes = []
for resource in resources:
for volume in filter(
lambda v: (v.get('host_path') or {}).get('path'),
resource['spec']['template']['spec']['volumes'] or []):
host_path_volumes.append(copy.deepcopy(volume))
return host_path_volumes
@private
async def normalise_and_validate_values(self,
item_details,
values,
update,
release_ds,
release_data=None):
dict_obj = await self.middleware.call(
'chart.release.validate_values',
item_details,
values,
update,
release_data,
)
return await self.middleware.call(
'chart.release.get_normalised_values', dict_obj, values, update, {
'release': {
'name': release_ds.split('/')[-1],
'dataset': release_ds,
'path': os.path.join('/mnt', release_ds),
},
'actions': [],
})
@private
async def perform_actions(self, context):
for action in context['actions']:
await self.middleware.call(f'chart.release.{action["method"]}',
*action['args'])
@accepts(
Dict(
'chart_release_create',
Dict('values', additional_attrs=True),
Str('catalog', required=True),
Str('item', required=True),
Str('release_name',
required=True,
validators=[Match(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$')]),
Str('train', default='charts'),
Str('version', default='latest'),
))
@job(lock=lambda args: f'chart_release_create_{args[0]["release_name"]}')
async def do_create(self, job, data):
"""
Create a chart release for a catalog item.
`release_name` is the name which will be used to identify the created chart release.
`catalog` is a valid catalog id where system will look for catalog `item` details.
`train` is which train to look for under `catalog` i.e stable / testing etc.
`version` specifies the catalog `item` version.
`values` is configuration specified for the catalog item version in question which will be used to
create the chart release.
"""
await self.middleware.call('kubernetes.validate_k8s_setup')
if await self.query([['id', '=', data['release_name']]]):
raise CallError(
f'Chart release with {data["release_name"]} already exists.',
errno=errno.EEXIST)
catalog = await self.middleware.call('catalog.get_instance',
data['catalog'])
item_details = await self.middleware.call(
'catalog.get_item_details', data['item'], {
'catalog': data['catalog'],
'train': data['train'],
})
version = data['version']
if version == 'latest':
version = await self.middleware.call(
'chart.release.get_latest_version_from_item_versions',
item_details['versions'])
if version not in item_details['versions']:
raise CallError(
f'Unable to locate "{data["version"]}" catalog item version.',
errno=errno.ENOENT)
item_details = item_details['versions'][version]
await self.middleware.call('catalog.version_supported_error_check',
item_details)
k8s_config = await self.middleware.call('kubernetes.config')
release_ds = os.path.join(k8s_config['dataset'], 'releases',
data['release_name'])
# The idea is to validate the values provided first and if it passes our validation test, we
# can move forward with setting up the datasets and installing the catalog item
new_values = data['values']
new_values, context = await self.normalise_and_validate_values(
item_details, new_values, False, release_ds)
job.set_progress(25, 'Initial Validation completed')
# Now that we have completed validation for the item in question wrt values provided,
# we will now perform the following steps
# 1) Create release datasets
# 2) Copy chart version into release/charts dataset
# 3) Install the helm chart
# 4) Create storage class
try:
job.set_progress(30, 'Creating chart release datasets')
for dataset in await self.release_datasets(release_ds):
if not await self.middleware.call('zfs.dataset.query',
[['id', '=', dataset]]):
await self.middleware.call('zfs.dataset.create', {
'name': dataset,
'type': 'FILESYSTEM'
})
await self.middleware.call('zfs.dataset.mount', dataset)
job.set_progress(45, 'Created chart release datasets')
chart_path = os.path.join('/mnt', release_ds, 'charts', version)
await self.middleware.run_in_thread(
lambda: shutil.copytree(item_details['location'], chart_path))
job.set_progress(55, 'Completed setting up chart release')
# Before finally installing the release, we will perform any actions which might be required
# for the release to function like creating/deleting ix-volumes
await self.perform_actions(context)
namespace_name = get_namespace(data['release_name'])
job.set_progress(65,
f'Creating {namespace_name} for chart release')
namespace_body = {
'metadata': {
'labels': {
'catalog': data['catalog'],
'catalog_train': data['train'],
'catalog_branch': catalog['branch'],
},
'name': namespace_name,
}
}
if not await self.middleware.call(
'k8s.namespace.query',
[['metadata.name', '=', namespace_name]]):
await self.middleware.call('k8s.namespace.create',
{'body': namespace_body})
else:
await self.middleware.call('k8s.namespace.update',
namespace_name,
{'body': namespace_body})
job.set_progress(75, 'Installing Catalog Item')
new_values = await add_context_to_configuration(
new_values, {
CONTEXT_KEY_NAME: {
**get_action_context(data['release_name']),
'operation': 'INSTALL',
'isInstall': True,
}
}, self.middleware)
await self.middleware.call(
'chart.release.create_update_storage_class_for_chart_release',
data['release_name'], os.path.join(release_ds, 'volumes'))
# We will install the chart now and force the installation in an ix based namespace
# https://github.com/helm/helm/issues/5465#issuecomment-473942223
await self.middleware.call('chart.release.helm_action',
data['release_name'], chart_path,
new_values, 'install')
except Exception:
# Do a rollback here
# Let's uninstall the release as well if it did get installed ( it is possible this might have happened )
if await self.middleware.call('chart.release.query',
[['id', '=', data['release_name']]]):
delete_job = await self.middleware.call(
'chart.release.delete', data['release_name'])
await delete_job.wait()
if delete_job.error:
self.logger.error(
'Failed to uninstall helm chart release: %s',
delete_job.error)
else:
await self.post_remove_tasks(data['release_name'])
raise
else:
await self.middleware.call('chart.release.refresh_events_state',
data['release_name'])
job.set_progress(100, 'Chart release created')
return await self.get_instance(data['release_name'])
@accepts(Str('chart_release'),
Dict(
'chart_release_update',
Dict('values', additional_attrs=True),
))
@job(lock=lambda args: f'chart_release_update_{args[0]}')
async def do_update(self, job, chart_release, data):
"""
Update an existing chart release.
`values` is configuration specified for the catalog item version in question which will be used to
create the chart release.
"""
release = await self.get_instance(chart_release)
release_orig = copy.deepcopy(release)
chart_path = os.path.join(release['path'], 'charts',
release['chart_metadata']['version'])
if not os.path.exists(chart_path):
raise CallError(
f'Unable to locate {chart_path!r} chart version for updating {chart_release!r} chart release',
errno=errno.ENOENT)
version_details = await self.middleware.call(
'catalog.item_version_details', chart_path)
config = release['config']
config.update(data['values'])
# We use update=False because we want defaults to be populated again if they are not present in the payload
# Why this is not dangerous is because the defaults will be added only if they are not present/configured for
# the chart release.
config, context = await self.normalise_and_validate_values(
version_details,
config,
False,
release['dataset'],
release_orig,
)
job.set_progress(25, 'Initial Validation complete')
await self.perform_actions(context)
config = await add_context_to_configuration(
config, {
CONTEXT_KEY_NAME: {
**get_action_context(chart_release),
'operation': 'UPDATE',
'isUpdate': True,
}
}, self.middleware)
await self.middleware.call('chart.release.helm_action', chart_release,
chart_path, config, 'update')
job.set_progress(90, 'Syncing secrets for chart release')
await self.middleware.call('chart.release.sync_secrets_for_release',
chart_release)
await self.middleware.call('chart.release.refresh_events_state',
chart_release)
job.set_progress(100, 'Update completed for chart release')
return await self.get_instance(chart_release)
@accepts(Str('release_name'),
Dict(
'options',
Bool('delete_unused_images', default=False),
))
@job(lock=lambda args: f'chart_release_delete_{args[0]}')
async def do_delete(self, job, release_name, options):
"""
Delete existing chart release.
This will delete the chart release from the kubernetes cluster and also remove any associated volumes / data.
To clarify, host path volumes will not be deleted which live outside the chart release dataset.
"""
# For delete we will uninstall the release first and then remove the associated datasets
await self.middleware.call('kubernetes.validate_k8s_setup')
chart_release = await self.get_instance(
release_name, {'extra': {
'retrieve_resources': True
}})
cp = await run([
'helm', 'uninstall', release_name, '-n',
get_namespace(release_name)
],
check=False)
if cp.returncode:
raise CallError(
f'Unable to uninstall "{release_name}" chart release: {cp.stderr}'
)
job.set_progress(50, f'Uninstalled {release_name}')
job.set_progress(75, f'Waiting for {release_name!r} pods to terminate')
await self.middleware.call('chart.release.wait_for_pods_to_terminate',
get_namespace(release_name))
await self.post_remove_tasks(release_name, job)
await self.middleware.call(
'chart.release.remove_chart_release_from_events_state',
release_name)
await self.middleware.call(
'chart.release.clear_chart_release_portal_cache', release_name)
await self.middleware.call('alert.oneshot_delete',
'ChartReleaseUpdate', release_name)
if options['delete_unused_images']:
job.set_progress(97, 'Deleting unused container images')
failed = await self.middleware.call(
'chart.release.delete_unused_app_images', chart_release)
if failed:
msg = '\n'
for i, v in failed.items():
msg += f'{i+1}) {v[0]} ({v[1]})\n'
raise CallError(
f'{release_name!r} was deleted but unable to delete following images:{msg}'
)
job.set_progress(100, f'{release_name!r} chart release deleted')
return True
@private
async def post_remove_tasks(self, release_name, job=None):
await self.remove_storage_class_and_dataset(release_name, job)
await self.middleware.call('k8s.namespace.delete',
get_namespace(release_name))
@private
async def remove_storage_class_and_dataset(self, release_name, job=None):
storage_class_name = get_storage_class_name(release_name)
if await self.middleware.call(
'k8s.storage_class.query',
[['metadata.name', '=', storage_class_name]]):
if job:
job.set_progress(85,
f'Removing {release_name!r} storage class')
try:
await self.middleware.call('k8s.storage_class.delete',
storage_class_name)
except Exception as e:
self.logger.error('Failed to remove %r storage class: %s',
storage_class_name, e)
k8s_config = await self.middleware.call('kubernetes.config')
release_ds = os.path.join(k8s_config['dataset'], 'releases',
release_name)
# If the chart release was consuming any PV's, they would have to be manually removed from k8s database
# because of chart release reclaim policy being retain
pvc_volume_ds = os.path.join(release_ds, 'volumes')
for pv in await self.middleware.call('k8s.pv.query', [[
'spec.csi.volume_attributes.openebs\\.io/poolname', '=',
pvc_volume_ds
]]):
await self.middleware.call('k8s.pv.delete', pv['metadata']['name'])
failed_zfs_volumes = []
# We would like to delete openebs zfs volumes ( not actual zfs volumes ) in openebs namespace
for zfs_volume in await self.middleware.call(
'k8s.zv.query', [['spec.poolName', '=', pvc_volume_ds]]):
try:
await self.middleware.call('k8s.zv.delete',
zfs_volume['metadata']['name'])
except Exception:
# It's perfectly fine if this fails as functionality wise this change is just cosmetic
# and is essentially cleaning up leftover zfs volume entries from k8s db
failed_zfs_volumes.append(zfs_volume['metadata']['name'])
if failed_zfs_volumes:
self.logger.error(
'Failed to delete %r zfs volumes when deleting %r chart release',
', '.join(failed_zfs_volumes), release_name)
if await self.middleware.call('zfs.dataset.query',
[['id', '=', release_ds]]):
if job:
job.set_progress(95, f'Removing {release_ds!r} dataset')
await self.middleware.call('zfs.dataset.delete', release_ds, {
'recursive': True,
'force': True
})
@private
async def release_datasets(self, release_dataset):
return [release_dataset] + [
os.path.join(release_dataset, k)
for k in ('charts', 'volumes', 'volumes/ix_volumes')
]
@private
async def get_chart_namespace_prefix(self):
return CHART_NAMESPACE_PREFIX
class PoolService(CRUDService):
GELI_KEYPATH = '/data/geli'
class Config:
datastore = 'storage.volume'
datastore_extend = 'pool.pool_extend'
datastore_prefix = 'vol_'
@accepts()
async def filesystem_choices(self):
vol_names = [vol['name'] for vol in (await self.query())]
return [
y['name']
for y in await self.middleware.call('zfs.dataset.query', [(
'name', 'rnin', '.system'), ('pool', 'in', vol_names)])
]
def _topology(self, x, geom_scan=True):
"""
Transform topology output from libzfs to add `device` and make `type` uppercase.
"""
if isinstance(x, dict):
path = x.get('path')
if path is not None:
device = None
if path.startswith('/dev/'):
device = self.middleware.call_sync('disk.label_to_dev',
path[5:], geom_scan)
x['device'] = device
for key in x:
if key == 'type' and isinstance(x[key], str):
x[key] = x[key].upper()
else:
x[key] = self._topology(x[key], False)
elif isinstance(x, list):
for i, entry in enumerate(x):
x[i] = self._topology(x[i], False)
return x
@private
def pool_extend(self, pool):
"""
If pool is encrypted we need to check if the pool is imported
or if all geli providers exist.
"""
try:
zpool = self.middleware.call_sync('zfs.pool.query',
[('id', '=', pool['name'])])[0]
except Exception:
zpool = None
if zpool:
pool['status'] = zpool['status']
pool['scan'] = zpool['scan']
pool['topology'] = self._topology(zpool['groups'])
else:
pool.update({
'status': 'OFFLINE',
'scan': None,
'topology': None,
})
if pool['encrypt'] > 0:
if zpool:
pool['is_decrypted'] = True
else:
decrypted = True
for ed in self.middleware.call_sync(
'datastore.query', 'storage.encrypteddisk',
[('encrypted_volume', '=', pool['id'])]):
if not os.path.exists(
f'/dev/{ed["encrypted_provider"]}.eli'):
decrypted = False
break
pool['is_decrypted'] = decrypted
else:
pool['is_decrypted'] = True
return pool
@item_method
@accepts(Int('id', required=False))
async def get_disks(self, oid=None):
"""
Get all disks in use by pools.
If `id` is provided only the disks from the given pool `id` will be returned.
"""
filters = []
if oid:
filters.append(('id', '=', oid))
for pool in await self.query(filters):
if pool['is_decrypted']:
async for i in await self.middleware.call(
'zfs.pool.get_disks', pool['name']):
yield i
else:
for encrypted_disk in await self.middleware.call(
'datastore.query', 'storage.encrypteddisk',
[('encrypted_volume', '=', pool['id'])]):
disk = {
k[len("disk_"):]: v
for k, v in encrypted_disk["encrypted_disk"].items()
}
name = await self.middleware.call("disk.get_name", disk)
if os.path.exists(os.path.join("/dev", name)):
yield name
@item_method
@accepts(Int('id'))
async def download_encryption_key(self, oid):
"""
Download encryption key for a given pool `id`.
"""
pool = await self.query([('id', '=', oid)], {'get': True})
if not pool['encryptkey']:
return None
job_id, url = await self.middleware.call(
'core.download', 'filesystem.get',
[os.path.join(self.GELI_KEYPATH, f"{pool['encryptkey']}.key")],
'geli.key')
return url
@private
def configure_resilver_priority(self):
"""
Configure resilver priority based on user selected off-peak hours.
"""
resilver = self.middleware.call_sync('datastore.config',
'storage.resilver')
if not resilver['enabled'] or not resilver['weekday']:
return
higher_prio = False
weekdays = map(lambda x: int(x), resilver['weekday'].split(','))
now = datetime.now()
now_t = now.time()
# end overlaps the day
if resilver['begin'] > resilver['end']:
if now.isoweekday() in weekdays and now_t >= resilver['begin']:
higher_prio = True
else:
lastweekday = now.isoweekday() - 1
if lastweekday == 0:
lastweekday = 7
if lastweekday in weekdays and now_t < resilver['end']:
higher_prio = True
# end does not overlap the day
else:
if now.isoweekday() in weekdays and now_t >= resilver[
'begin'] and now_t < resilver['end']:
higher_prio = True
if higher_prio:
resilver_delay = 0
resilver_min_time_ms = 9000
scan_idle = 0
else:
resilver_delay = 2
resilver_min_time_ms = 3000
scan_idle = 50
sysctl.filter('vfs.zfs.resilver_delay')[0].value = resilver_delay
sysctl.filter(
'vfs.zfs.resilver_min_time_ms')[0].value = resilver_min_time_ms
sysctl.filter('vfs.zfs.scan_idle')[0].value = scan_idle
@accepts()
async def import_find(self):
"""
Get a list of pools available for import with the following details:
name, guid, status, hostname.
"""
existing_guids = [
i['guid'] for i in await self.middleware.call('pool.query')
]
for pool in await self.middleware.call('zfs.pool.find_import'):
if pool['status'] == 'UNAVAIL':
continue
# Exclude pools with same guid as existing pools (in database)
# It could be the pool is in the database but was exported/detached for some reason
# See #6808
if pool['guid'] in existing_guids:
continue
entry = {}
for i in ('name', 'guid', 'status', 'hostname'):
entry[i] = pool[i]
yield entry
@accepts(
Dict(
'pool_import',
Str('guid', required=True),
Str('name'),
Str('passphrase', private=True),
List('devices', items=[Str('device')]),
))
@job(lock='import_pool', pipes=['input'], check_pipes=False)
async def import_pool(self, job, data):
"""
Import a pool.
Errors:
ENOENT - Pool not found
"""
pool = None
for p in await self.middleware.call('zfs.pool.find_import'):
if p['guid'] == data['guid']:
pool = p
break
if pool is None:
raise CallError(f'Pool with guid "{data["guid"]}" not found',
errno.ENOENT)
if data['devices']:
job.check_pipe("input")
args = [job.pipes.input.r, data['passphrase'], data['devices']]
else:
args = []
await self.middleware.call('notifier.volume_import',
data.get('name') or pool['name'],
data['guid'], *args)
return True
@accepts(Str('volume'), Str('fs_type'),
Dict('fs_options', additional_attrs=True), Str('dst_path'))
@job(lock=lambda args: 'volume_import', logs=True)
async def import_disk(self, job, volume, fs_type, fs_options, dst_path):
job.set_progress(None, description="Mounting")
src = os.path.join('/var/run/importcopy/tmpdir',
os.path.relpath(volume, '/'))
if os.path.exists(src):
os.rmdir(src)
try:
os.makedirs(src)
async with KernelModuleContextManager({
"msdosfs": "msdosfs_iconv",
"ntfs": "fuse"
}.get(fs_type)):
async with MountFsContextManager(self.middleware, volume, src,
fs_type, fs_options, ["ro"]):
job.set_progress(None, description="Importing")
line = [
'/usr/local/bin/rsync', '--info=progress2',
'--modify-window=1', '-rltvh', '--no-perms', src + '/',
dst_path
]
rsync_proc = await Popen(
line,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=0,
preexec_fn=os.setsid,
)
try:
progress_buffer = JobProgressBuffer(job)
while True:
line = await rsync_proc.stdout.readline()
job.logs_fd.write(line)
if line:
try:
line = line.decode("utf-8",
"ignore").strip()
bits = re.split("\s+", line)
if len(bits) == 6 and bits[1].endswith(
"%") and bits[1][:-1].isdigit():
progress_buffer.set_progress(
int(bits[1][:-1]))
elif not line.endswith('/'):
if (line not in [
'sending incremental file list'
] and 'xfr#' not in line):
progress_buffer.set_progress(
None, extra=line)
except Exception:
logger.warning(
'Parsing error in rsync task',
exc_info=True)
else:
break
progress_buffer.flush()
await rsync_proc.wait()
if rsync_proc.returncode != 0:
raise Exception("rsync failed with exit code %r" %
rsync_proc.returncode)
except asyncio.CancelledError:
rsync_proc.kill()
raise
job.set_progress(100, description="Done", extra="")
finally:
os.rmdir(src)
@accepts()
def import_disk_msdosfs_locales(self):
return [
locale.strip()
for locale in subprocess.check_output(["locale", "-a"],
encoding="utf-8").split("\n")
if locale.strip()
]
"""
These methods are hacks for old UI which supports only one volume import at a time
"""
dismissed_import_disk_jobs = set()
@private
async def get_current_import_disk_job(self):
import_jobs = await self.middleware.call(
'core.get_jobs', [('method', '=', 'pool.import_disk')])
not_dismissed_import_jobs = [
job for job in import_jobs
if job["id"] not in self.dismissed_import_disk_jobs
]
if not_dismissed_import_jobs:
return not_dismissed_import_jobs[0]
@private
async def dismiss_current_import_disk_job(self):
current_import_job = await self.get_current_import_disk_job()
if current_import_job:
self.dismissed_import_disk_jobs.add(current_import_job["id"])
