class DiskService(Service): @private @lock('swaps_configure') @accepts(List('disks', items=[Str('disk')]), Dict('swap_removal_options', Bool('configure_swap', default=True), register=True)) async def swaps_remove_disks(self, disks, options): """ Remove a given disk (e.g. ["da0", "da1"]) from swap. It will offline if from swap, removing encryption and destroying the mirror ( if part of one ). """ return await self.swaps_remove_disks_unlocked(disks, options) @private async def swaps_remove_disks_unlocked(self, disks, options=None): """ We have a separate endpoint for this to ensure that no other swap related operations not do swap devices removal while swap configuration is in progress - however we still need to allow swap configuration process to remove swap devices and it can use this endpoint directly for that purpose. """ options = options or {} providers = {} for disk in disks: partitions = await self.middleware.call('disk.list_partitions', disk) if not partitions: continue for p in partitions: if p['partition_type'] in await self.middleware.call( 'disk.get_valid_swap_partition_type_uuids'): providers[p['id']] = p break if not providers: return swap_devices = await self.middleware.call('disk.get_swap_devices') for mirror in await self.middleware.call('disk.get_swap_mirrors'): devname = mirror['encrypted_provider'] or mirror['real_path'] if devname in swap_devices: await run('swapoff', devname) if mirror['encrypted_provider']: await self.middleware.call('disk.remove_encryption', mirror['encrypted_provider']) await self.middleware.call('disk.destroy_swap_mirror', mirror['name']) configure_swap = False for p in providers.values(): devname = p['encrypted_provider'] or p['path'] if devname in swap_devices: await run('swapoff', devname) if p['encrypted_provider']: await self.middleware.call('disk.remove_encryption', p['encrypted_provider']) if not IS_LINUX and os.path.realpath('/dev/dumpdev') == p['path']: configure_swap = True try: os.unlink('/dev/dumpdev') except OSError: pass # Let consumer explicitly deny swap configuration if desired if configure_swap and options.get('configure_swap', True): asyncio.ensure_future(self.middleware.call('disk.swaps_configure'))
class ServiceService(CRUDService): SERVICE_DEFS = { 's3': ServiceDefinition('minio', '/var/run/minio.pid'), 'ssh': ServiceDefinition('sshd', '/var/run/sshd.pid'), 'rsync': ServiceDefinition('rsync', '/var/run/rsyncd.pid'), 'nfs': ServiceDefinition('nfsd', None), 'afp': ServiceDefinition('netatalk', None), 'cifs': ServiceDefinition('smbd', '/var/run/samba4/smbd.pid'), 'dynamicdns': ServiceDefinition('inadyn', None), 'snmp': ServiceDefinition('snmpd', '/var/run/net_snmpd.pid'), 'ftp': ServiceDefinition('proftpd', '/var/run/proftpd.pid'), 'tftp': ServiceDefinition('inetd', '/var/run/inetd.pid'), 'iscsitarget': ServiceDefinition('ctld', '/var/run/ctld.pid'), 'lldp': ServiceDefinition('ladvd', '/var/run/ladvd.pid'), 'ups': ServiceDefinition('upsd', '/var/db/nut/upsd.pid'), 'upsmon': ServiceDefinition('upsmon', '/var/db/nut/upsmon.pid'), 'smartd': ServiceDefinition('smartd', 'smartd-daemon', '/var/run/smartd-daemon.pid'), 'webshell': ServiceDefinition(None, '/var/run/webshell.pid'), 'webdav': ServiceDefinition('httpd', '/var/run/httpd.pid'), 'netdata': ServiceDefinition('netdata', '/var/db/netdata/netdata.pid') } @filterable async def query(self, filters=None, options=None): if options is None: options = {} options['prefix'] = 'srv_' services = await self.middleware.call('datastore.query', 'services.services', filters, options) # In case a single service has been requested if not isinstance(services, list): services = [services] jobs = { asyncio.ensure_future(self._get_status(entry)): entry for entry in services } await asyncio.wait(list(jobs.keys()), timeout=15) def result(task): """ Method to handle results of the greenlets. In case a greenlet has timed out, provide UNKNOWN state """ try: result = task.result() except Exception: result = None self.logger.warn('Failed to get status', exc_info=True) if result is None: entry = jobs.get(task) entry['state'] = 'UNKNOWN' entry['pids'] = [] return entry else: return result services = list(map(result, jobs)) return filter_list(services, filters, options) @accepts( Int('id'), Dict( 'service-update', Bool('enable', default=False), ), ) async def do_update(self, id, data): """ Update service entry of `id`. Currently it only accepts `enable` option which means whether the service should start on boot. """ return await self.middleware.call('datastore.update', 'services.services', id, {'srv_enable': data['enable']}) @accepts( Str('service'), Dict( 'service-control', Bool('onetime', default=True), register=True, ), ) async def start(self, service, options=None): """ Start the service specified by `service`. The helper will use method self._start_[service]() to start the service. If the method does not exist, it would fallback using service(8).""" await self.middleware.call_hook('service.pre_start', service) sn = self._started_notify("start", service) await self._simplecmd("start", service, options) return await self.started(service, sn) async def started(self, service, sn=None): """ Test if service specified by `service` has been started. """ if sn: await self.middleware.run_in_thread(sn.join) try: svc = await self.query([('service', '=', service)], {'get': True}) self.middleware.send_event('service.query', 'CHANGED', fields=svc) return svc['state'] == 'RUNNING' except IndexError: f = getattr(self, '_started_' + service, None) if callable(f): if inspect.iscoroutinefunction(f): return (await f())[0] else: return f()[0] else: return (await self._started(service))[0] @accepts( Str('service'), Ref('service-control'), ) async def stop(self, service, options=None): """ Stop the service specified by `service`. The helper will use method self._stop_[service]() to stop the service. If the method does not exist, it would fallback using service(8).""" await self.middleware.call_hook('service.pre_stop', service) sn = self._started_notify("stop", service) await self._simplecmd("stop", service, options) return await self.started(service, sn) @accepts( Str('service'), Ref('service-control'), ) async def restart(self, service, options=None): """ Restart the service specified by `service`. The helper will use method self._restart_[service]() to restart the service. If the method does not exist, it would fallback using service(8).""" await self.middleware.call_hook('service.pre_restart', service) sn = self._started_notify("restart", service) await self._simplecmd("restart", service, options) return await self.started(service, sn) @accepts( Str('service'), Ref('service-control'), ) async def reload(self, service, options=None): """ Reload the service specified by `service`. The helper will use method self._reload_[service]() to reload the service. If the method does not exist, the helper will try self.restart of the service instead.""" await self.middleware.call_hook('service.pre_reload', service) try: await self._simplecmd("reload", service, options) except Exception as e: await self.restart(service, options) return await self.started(service) async def _get_status(self, service): f = getattr(self, '_started_' + service['service'], None) if callable(f): if inspect.iscoroutinefunction(f): running, pids = await f() else: running, pids = f() else: running, pids = await self._started(service['service']) if running: state = 'RUNNING' else: if service['enable']: state = 'CRASHED' else: state = 'STOPPED' service['state'] = state service['pids'] = pids return service async def _simplecmd(self, action, what, options=None): self.logger.debug("Calling: %s(%s) ", action, what) f = getattr(self, '_' + action + '_' + what, None) if f is None: # Provide generic start/stop/restart verbs for rc.d scripts if what in self.SERVICE_DEFS: if self.SERVICE_DEFS[what].rc_script: what = self.SERVICE_DEFS[what].rc_script if action in ("start", "stop", "restart", "reload"): if action == 'restart': await self._system("/usr/sbin/service " + what + " forcestop ") await self._service(what, action, **options) else: raise ValueError("Internal error: Unknown command") else: call = f(**(options or {})) if inspect.iscoroutinefunction(f): await call async def _system(self, cmd, options=None): stdout = DEVNULL if options and 'stdout' in options: stdout = options['stdout'] stderr = DEVNULL if options and 'stderr' in options: stderr = options['stderr'] proc = await Popen(cmd, stdout=stdout, stderr=stderr, shell=True, close_fds=True) await proc.communicate() return proc.returncode async def _service(self, service, verb, **options): onetime = options.pop('onetime', None) force = options.pop('force', None) quiet = options.pop('quiet', None) extra = options.pop('extra', '') # force comes before one which comes before quiet # they are mutually exclusive preverb = '' if force: preverb = 'force' elif onetime: preverb = 'one' elif quiet: preverb = 'quiet' return await self._system('/usr/sbin/service {} {}{} {}'.format( service, preverb, verb, extra, ), options) def _started_notify(self, verb, what): """ The check for started [or not] processes is currently done in 2 steps This is the first step which involves a thread StartNotify that watch for event before actually start/stop rc.d scripts Returns: StartNotify object if the service is known or None otherwise """ if what in self.SERVICE_DEFS: sn = StartNotify(verb=verb, pidfile=self.SERVICE_DEFS[what].pidfile) sn.start() return sn else: return None async def _started(self, what, notify=None): """ This is the second step:: Wait for the StartNotify thread to finish and then check for the status of pidfile/procname using pgrep Returns: True whether the service is alive, False otherwise """ if what in self.SERVICE_DEFS: if notify: await self.middleware.run_in_thread(notify.join) if self.SERVICE_DEFS[what].pidfile: pgrep = "/bin/pgrep -F {}{}".format( self.SERVICE_DEFS[what].pidfile, ' ' + self.SERVICE_DEFS[what].procname if self.SERVICE_DEFS[what].procname else '', ) else: pgrep = "/bin/pgrep {}".format(self.SERVICE_DEFS[what].procname) proc = await Popen(pgrep, shell=True, stdout=PIPE, stderr=PIPE, close_fds=True) data = (await proc.communicate())[0].decode() if proc.returncode == 0: return True, [ int(i) for i in data.strip().split('\n') if i.isdigit() ] return False, [] async def _start_webdav(self, **kwargs): await self._service("ix-apache", "start", force=True, **kwargs) await self._service("apache24", "start", **kwargs) async def _stop_webdav(self, **kwargs): await self._service("apache24", "stop", **kwargs) async def _restart_webdav(self, **kwargs): await self._service("apache24", "stop", force=True, **kwargs) await self._service("ix-apache", "start", force=True, **kwargs) await self._service("apache24", "restart", **kwargs) async def _reload_webdav(self, **kwargs): await self._service("ix-apache", "start", force=True, **kwargs) await self._service("apache24", "reload", **kwargs) async def _restart_django(self, **kwargs): await self._service("django", "restart", **kwargs) async def _start_webshell(self, **kwargs): await self._system("/usr/local/bin/python /usr/local/www/freenasUI/tools/webshell.py") async def _restart_webshell(self, **kwargs): try: with open('/var/run/webshell.pid', 'r') as f: pid = f.read() os.kill(int(pid), signal.SIGTERM) time.sleep(0.2) os.kill(int(pid), signal.SIGKILL) except: pass await self._system("ulimit -n 1024 && /usr/local/bin/python /usr/local/www/freenasUI/tools/webshell.py") async def _restart_iscsitarget(self, **kwargs): await self._service("ix-ctld", "start", force=True, **kwargs) await self._service("ctld", "stop", force=True, **kwargs) await self._service("ix-ctld", "start", quiet=True, **kwargs) await self._service("ctld", "restart", **kwargs) async def _start_iscsitarget(self, **kwargs): await self._service("ix-ctld", "start", quiet=True, **kwargs) await self._service("ctld", "start", **kwargs) async def _stop_iscsitarget(self, **kwargs): await self._service("ix-ctld", "stop", force=True, **kwargs) await self._service("ctld", "stop", force=True, **kwargs) async def _reload_iscsitarget(self, **kwargs): await self._service("ix-ctld", "start", quiet=True, **kwargs) await self._service("ctld", "reload", **kwargs) async def _start_collectd(self, **kwargs): await self._service("ix-collectd", "start", quiet=True, **kwargs) await self._service("collectd", "restart", **kwargs) async def _restart_collectd(self, **kwargs): await self._service("collectd", "stop", **kwargs) await self._service("ix-collectd", "start", quiet=True, **kwargs) await self._service("collectd", "start", **kwargs) async def _start_sysctl(self, **kwargs): await self._service("sysctl", "start", **kwargs) await self._service("ix-sysctl", "start", quiet=True, **kwargs) async def _reload_sysctl(self, **kwargs): await self._service("sysctl", "start", **kwargs) await self._service("ix-sysctl", "reload", **kwargs) async def _start_network(self, **kwargs): await self.middleware.call('interfaces.sync') await self.middleware.call('routes.sync') async def _stop_jails(self, **kwargs): for jail in await self.middleware.call('datastore.query', 'jails.jails'): try: await self.middleware.call('notifier.warden', 'stop', [], {'jail': jail['jail_host']}) except Exception as e: self.logger.debug(f'Failed to stop jail {jail["jail_host"]}', exc_info=True) async def _start_jails(self, **kwargs): await self._service("ix-warden", "start", **kwargs) for jail in await self.middleware.call('datastore.query', 'jails.jails'): if jail['jail_autostart']: try: await self.middleware.call('notifier.warden', 'start', [], {'jail': jail['jail_host']}) except Exception as e: self.logger.debug(f'Failed to start jail {jail["jail_host"]}', exc_info=True) await self._service("ix-plugins", "start", **kwargs) await self.reload("http", kwargs) async def _restart_jails(self, **kwargs): await self._stop_jails() await self._start_jails() async def _stop_pbid(self, **kwargs): await self._service("pbid", "stop", **kwargs) async def _start_pbid(self, **kwargs): await self._service("pbid", "start", **kwargs) async def _restart_pbid(self, **kwargs): await self._service("pbid", "restart", **kwargs) async def _reload_named(self, **kwargs): await self._service("named", "reload", **kwargs) async def _reload_hostname(self, **kwargs): await self._system('/bin/hostname ""') await self._service("ix-hostname", "start", quiet=True, **kwargs) await self._service("hostname", "start", quiet=True, **kwargs) await self._service("mdnsd", "restart", quiet=True, **kwargs) await self._service("collectd", "stop", **kwargs) await self._service("ix-collectd", "start", quiet=True, **kwargs) await self._service("collectd", "start", **kwargs) async def _reload_resolvconf(self, **kwargs): await self._reload_hostname() await self._service("ix-resolv", "start", quiet=True, **kwargs) async def _reload_networkgeneral(self, **kwargs): await self._reload_resolvconf() await self._service("routing", "restart", **kwargs) async def _reload_timeservices(self, **kwargs): await self._service("ix-localtime", "start", quiet=True, **kwargs) await self._service("ix-ntpd", "start", quiet=True, **kwargs) await self._service("ntpd", "restart", **kwargs) os.environ['TZ'] = await self.middleware.call('datastore.query', 'system.settings', [], {'order_by': ['-id'], 'get': True})['stg_timezone'] time.tzset() async def _restart_smartd(self, **kwargs): await self._service("ix-smartd", "start", quiet=True, **kwargs) await self._service("smartd-daemon", "stop", force=True, **kwargs) await self._service("smartd-daemon", "restart", **kwargs) async def _reload_ssh(self, **kwargs): await self._service("ix-sshd", "start", quiet=True, **kwargs) await self._service("ix_register", "reload", **kwargs) await self._service("openssh", "reload", **kwargs) await self._service("ix_sshd_save_keys", "start", quiet=True, **kwargs) async def _start_ssh(self, **kwargs): await self._service("ix-sshd", "start", quiet=True, **kwargs) await self._service("ix_register", "reload", **kwargs) await self._service("openssh", "start", **kwargs) await self._service("ix_sshd_save_keys", "start", quiet=True, **kwargs) async def _stop_ssh(self, **kwargs): await self._service("openssh", "stop", force=True, **kwargs) await self._service("ix_register", "reload", **kwargs) async def _restart_ssh(self, **kwargs): await self._service("ix-sshd", "start", quiet=True, **kwargs) await self._service("openssh", "stop", force=True, **kwargs) await self._service("ix_register", "reload", **kwargs) await self._service("openssh", "restart", **kwargs) await self._service("ix_sshd_save_keys", "start", quiet=True, **kwargs) async def _start_ssl(self, what=None): if what is not None: await self._service("ix-ssl", "start", quiet=True, extra=what) else: await self._service("ix-ssl", "start", quiet=True) async def _start_s3(self, **kwargs): await self.middleware.call('etc.generate', 's3') await self._service("minio", "start", quiet=True, stdout=None, stderr=None, **kwargs) async def _reload_s3(self, **kwargs): await self.middleware.call('etc.generate', 's3') await self._service("minio", "restart", quiet=True, stdout=None, stderr=None, **kwargs) async def _reload_rsync(self, **kwargs): await self._service("ix-rsyncd", "start", quiet=True, **kwargs) await self._service("rsyncd", "restart", **kwargs) async def _restart_rsync(self, **kwargs): await self._stop_rsync() await self._start_rsync() async def _start_rsync(self, **kwargs): await self._service("ix-rsyncd", "start", quiet=True, **kwargs) await self._service("rsyncd", "start", **kwargs) async def _stop_rsync(self, **kwargs): await self._service("rsyncd", "stop", force=True, **kwargs) async def _started_nis(self, **kwargs): res = False if not await self._system("/etc/directoryservice/NIS/ctl status"): res = True return res, [] async def _start_nis(self, **kwargs): res = False if not await self._system("/etc/directoryservice/NIS/ctl start"): res = True return res async def _restart_nis(self, **kwargs): res = False if not await self._system("/etc/directoryservice/NIS/ctl restart"): res = True return res async def _stop_nis(self, **kwargs): res = False if not await self._system("/etc/directoryservice/NIS/ctl stop"): res = True return res async def _started_ldap(self, **kwargs): if (await self._system('/usr/sbin/service ix-ldap status') != 0): return False, [] return await self.middleware.call('notifier.ldap_status'), [] async def _start_ldap(self, **kwargs): res = False if not await self._system("/etc/directoryservice/LDAP/ctl start"): res = True return res async def _stop_ldap(self, **kwargs): res = False if not await self._system("/etc/directoryservice/LDAP/ctl stop"): res = True return res async def _restart_ldap(self, **kwargs): res = False if not await self._system("/etc/directoryservice/LDAP/ctl restart"): res = True return res async def _start_lldp(self, **kwargs): await self._service("ladvd", "start", **kwargs) async def _stop_lldp(self, **kwargs): await self._service("ladvd", "stop", force=True, **kwargs) async def _restart_lldp(self, **kwargs): await self._service("ladvd", "stop", force=True, **kwargs) await self._service("ladvd", "restart", **kwargs) async def _clear_activedirectory_config(self): await self._system("/bin/rm -f /etc/directoryservice/ActiveDirectory/config") async def _started_activedirectory(self, **kwargs): for srv in ('kinit', 'activedirectory', ): if await self._system('/usr/sbin/service ix-%s status' % (srv, )) != 0: return False, [] if await self._system('/usr/local/bin/wbinfo -p') != 0: return False, [] if await self._system('/usr/local/bin/wbinfo -t') != 0: return False, [] return True, [] async def _start_activedirectory(self, **kwargs): res = False if not await self._system("/etc/directoryservice/ActiveDirectory/ctl start"): res = True return res async def _stop_activedirectory(self, **kwargs): res = False if not await self._system("/etc/directoryservice/ActiveDirectory/ctl stop"): res = True return res async def _restart_activedirectory(self, **kwargs): res = False if not await self._system("/etc/directoryservice/ActiveDirectory/ctl restart"): res = True return res async def _started_domaincontroller(self, **kwargs): res = False if not await self._system("/etc/directoryservice/DomainController/ctl status"): res = True return res, [] async def _start_domaincontroller(self, **kwargs): res = False if not await self._system("/etc/directoryservice/DomainController/ctl start"): res = True return res async def _stop_domaincontroller(self, **kwargs): res = False if not await self._system("/etc/directoryservice/DomainController/ctl stop"): res = True return res async def _restart_domaincontroller(self, **kwargs): res = False if not await self._system("/etc/directoryservice/DomainController/ctl restart"): res = True return res async def _restart_syslogd(self, **kwargs): await self._service("ix-syslogd", "start", quiet=True, **kwargs) await self._system("/etc/local/rc.d/syslog-ng restart") async def _start_syslogd(self, **kwargs): await self._service("ix-syslogd", "start", quiet=True, **kwargs) await self._system("/etc/local/rc.d/syslog-ng start") async def _stop_syslogd(self, **kwargs): await self._system("/etc/local/rc.d/syslog-ng stop") async def _reload_syslogd(self, **kwargs): await self._service("ix-syslogd", "start", quiet=True, **kwargs) await self._system("/etc/local/rc.d/syslog-ng reload") async def _start_tftp(self, **kwargs): await self._service("ix-inetd", "start", quiet=True, **kwargs) await self._service("inetd", "start", **kwargs) async def _reload_tftp(self, **kwargs): await self._service("ix-inetd", "start", quiet=True, **kwargs) await self._service("inetd", "stop", force=True, **kwargs) await self._service("inetd", "restart", **kwargs) async def _restart_tftp(self, **kwargs): await self._service("ix-inetd", "start", quiet=True, **kwargs) await self._service("inetd", "stop", force=True, **kwargs) await self._service("inetd", "restart", **kwargs) async def _restart_cron(self, **kwargs): await self._service("ix-crontab", "start", quiet=True, **kwargs) async def _start_motd(self, **kwargs): await self._service("ix-motd", "start", quiet=True, **kwargs) await self._service("motd", "start", quiet=True, **kwargs) async def _start_ttys(self, **kwargs): await self._service("ix-ttys", "start", quiet=True, **kwargs) async def _reload_ftp(self, **kwargs): await self._service("ix-proftpd", "start", quiet=True, **kwargs) await self._service("proftpd", "restart", **kwargs) async def _restart_ftp(self, **kwargs): await self._stop_ftp() await self._start_ftp() async def _start_ftp(self, **kwargs): await self._service("ix-proftpd", "start", quiet=True, **kwargs) await self._service("proftpd", "start", **kwargs) async def _stop_ftp(self, **kwargs): await self._service("proftpd", "stop", force=True, **kwargs) async def _start_ups(self, **kwargs): await self._service("ix-ups", "start", quiet=True, **kwargs) await self._service("nut", "start", **kwargs) await self._service("nut_upsmon", "start", **kwargs) await self._service("nut_upslog", "start", **kwargs) async def _stop_ups(self, **kwargs): await self._service("nut_upslog", "stop", force=True, **kwargs) await self._service("nut_upsmon", "stop", force=True, **kwargs) await self._service("nut", "stop", force=True, **kwargs) async def _restart_ups(self, **kwargs): await self._service("ix-ups", "start", quiet=True, **kwargs) await self._service("nut", "stop", force=True, **kwargs) await self._service("nut_upsmon", "stop", force=True, **kwargs) await self._service("nut_upslog", "stop", force=True, **kwargs) await self._service("nut", "restart", **kwargs) await self._service("nut_upsmon", "restart", **kwargs) await self._service("nut_upslog", "restart", **kwargs) async def _started_ups(self, **kwargs): mode = (await self.middleware.call('datastore.query', 'services.ups', [], {'order_by': ['-id'], 'get': True}))['ups_mode'] if mode == "master": svc = "ups" else: svc = "upsmon" return await self._started(svc) async def _start_afp(self, **kwargs): await self._service("ix-afpd", "start", **kwargs) await self._service("netatalk", "start", **kwargs) async def _stop_afp(self, **kwargs): await self._service("netatalk", "stop", force=True, **kwargs) # when netatalk stops if afpd or cnid_metad is stuck # they'll get left behind, which can cause issues # restarting netatalk. await self._system("pkill -9 afpd") await self._system("pkill -9 cnid_metad") async def _restart_afp(self, **kwargs): await self._stop_afp() await self._start_afp() async def _reload_afp(self, **kwargs): await self._service("ix-afpd", "start", quiet=True, **kwargs) await self._system("killall -1 netatalk") async def _reload_nfs(self, **kwargs): await self._service("ix-nfsd", "start", quiet=True, **kwargs) async def _restart_nfs(self, **kwargs): await self._stop_nfs(**kwargs) await self._start_nfs(**kwargs) async def _stop_nfs(self, **kwargs): await self._service("lockd", "stop", force=True, **kwargs) await self._service("statd", "stop", force=True, **kwargs) await self._service("nfsd", "stop", force=True, **kwargs) await self._service("mountd", "stop", force=True, **kwargs) await self._service("nfsuserd", "stop", force=True, **kwargs) await self._service("gssd", "stop", force=True, **kwargs) await self._service("rpcbind", "stop", force=True, **kwargs) if not await self.middleware.call('system.is_freenas'): await self._service("vaaiserver", "stop", force=True, **kwargs) async def _start_nfs(self, **kwargs): nfs = await self.middleware.call('datastore.config', 'services.nfs') await self._service("ix-nfsd", "start", quiet=True, **kwargs) await self._service("rpcbind", "start", quiet=True, **kwargs) await self._service("gssd", "start", quiet=True, **kwargs) # Workaround to work with "onetime", since the rc scripts depend on rc flags. if nfs['nfs_srv_v4']: sysctl.filter('vfs.nfsd.server_max_nfsvers')[0].value = 4 if nfs['nfs_srv_v4_v3owner']: # Per RFC7530, sending NFSv3 style UID/GIDs across the wire is now allowed # You must have both of these sysctl's set to allow the desired functionality sysctl.filter('vfs.nfsd.enable_stringtouid')[0].value = 1 sysctl.filter('vfs.nfs.enable_uidtostring')[0].value = 1 await self._service("nfsuserd", "stop", force=True, **kwargs) else: sysctl.filter('vfs.nfsd.enable_stringtouid')[0].value = 0 sysctl.filter('vfs.nfs.enable_uidtostring')[0].value = 0 await self._service("nfsuserd", "start", quiet=True, **kwargs) else: sysctl.filter('vfs.nfsd.server_max_nfsvers')[0].value = 3 if nfs['nfs_srv_16']: await self._service("nfsuserd", "start", quiet=True, **kwargs) await self._service("mountd", "start", quiet=True, **kwargs) await self._service("nfsd", "start", quiet=True, **kwargs) await self._service("statd", "start", quiet=True, **kwargs) await self._service("lockd", "start", quiet=True, **kwargs) if not await self.middleware.call('system.is_freenas'): await self._service("vaaiserver", "start", quiet=True, **kwargs) async def _force_stop_jail(self, **kwargs): await self._service("jail", "stop", force=True, **kwargs) async def _start_plugins(self, jail=None, plugin=None, **kwargs): if jail and plugin: await self._system("/usr/sbin/service ix-plugins forcestart %s:%s" % (jail, plugin)) else: await self._service("ix-plugins", "start", force=True, **kwargs) async def _stop_plugins(self, jail=None, plugin=None, **kwargs): if jail and plugin: await self._system("/usr/sbin/service ix-plugins forcestop %s:%s" % (jail, plugin)) else: await self._service("ix-plugins", "stop", force=True, **kwargs) async def _restart_plugins(self, jail=None, plugin=None): await self._stop_plugins(jail=jail, plugin=plugin) await self._start_plugins(jail=jail, plugin=plugin) async def _started_plugins(self, jail=None, plugin=None, **kwargs): res = False if jail and plugin: if self._system("/usr/sbin/service ix-plugins status %s:%s" % (jail, plugin)) == 0: res = True else: if await self._service("ix-plugins", "status", **kwargs) == 0: res = True return res, [] async def _restart_dynamicdns(self, **kwargs): await self._service("ix-inadyn", "start", quiet=True, **kwargs) await self._service("inadyn", "stop", force=True, **kwargs) await self._service("inadyn", "restart", **kwargs) async def _reload_dynamicdns(self, **kwargs): await self._service("ix-inadyn", "start", quiet=True, **kwargs) await self._service("inadyn", "stop", force=True, **kwargs) await self._service("inadyn", "restart", **kwargs) async def _restart_system(self, **kwargs): asyncio.ensure_future(self._system("/bin/sleep 3 && /sbin/shutdown -r now")) async def _stop_system(self, **kwargs): asyncio.ensure_future(self._system("/bin/sleep 3 && /sbin/shutdown -p now")) async def _reload_cifs(self, **kwargs): await self._service("ix-pre-samba", "start", quiet=True, **kwargs) await self._service("samba_server", "reload", force=True, **kwargs) await self._service("ix-post-samba", "start", quiet=True, **kwargs) await self._service("mdnsd", "restart", **kwargs) # After mdns is restarted we need to reload netatalk to have it rereregister # with mdns. Ticket #7133 await self._service("netatalk", "reload", **kwargs) async def _restart_cifs(self, **kwargs): await self._service("ix-pre-samba", "start", quiet=True, **kwargs) await self._service("samba_server", "stop", force=True, **kwargs) await self._service("samba_server", "restart", quiet=True, **kwargs) await self._service("ix-post-samba", "start", quiet=True, **kwargs) await self._service("mdnsd", "restart", **kwargs) # After mdns is restarted we need to reload netatalk to have it rereregister # with mdns. Ticket #7133 await self._service("netatalk", "reload", **kwargs) async def _start_cifs(self, **kwargs): await self._service("ix-pre-samba", "start", quiet=True, **kwargs) await self._service("samba_server", "start", quiet=True, **kwargs) await self._service("ix-post-samba", "start", quiet=True, **kwargs) async def _stop_cifs(self, **kwargs): await self._service("samba_server", "stop", force=True, **kwargs) await self._service("ix-post-samba", "start", quiet=True, **kwargs) async def _start_snmp(self, **kwargs): await self._service("ix-snmpd", "start", quiet=True, **kwargs) await self._service("snmpd", "start", quiet=True, **kwargs) await self._service("snmp-agent", "start", quiet=True, **kwargs) async def _stop_snmp(self, **kwargs): await self._service("snmp-agent", "stop", quiet=True, **kwargs) await self._service("snmpd", "stop", quiet=True, **kwargs) async def _restart_snmp(self, **kwargs): await self._service("snmp-agent", "stop", quiet=True, **kwargs) await self._service("snmpd", "stop", force=True, **kwargs) await self._service("ix-snmpd", "start", quiet=True, **kwargs) await self._service("snmpd", "start", quiet=True, **kwargs) await self._service("snmp-agent", "start", quiet=True, **kwargs) async def _reload_snmp(self, **kwargs): await self._service("snmp-agent", "stop", quiet=True, **kwargs) await self._service("snmpd", "stop", force=True, **kwargs) await self._service("ix-snmpd", "start", quiet=True, **kwargs) await self._service("snmpd", "start", quiet=True, **kwargs) await self._service("snmp-agent", "start", quiet=True, **kwargs) async def _restart_http(self, **kwargs): await self._service("ix-nginx", "start", quiet=True, **kwargs) await self._service("ix_register", "reload", **kwargs) await self._service("nginx", "restart", **kwargs) async def _reload_http(self, **kwargs): await self._service("ix-nginx", "start", quiet=True, **kwargs) await self._service("ix_register", "reload", **kwargs) await self._service("nginx", "reload", **kwargs) async def _reload_loader(self, **kwargs): await self._service("ix-loader", "reload", **kwargs) async def _start_loader(self, **kwargs): await self._service("ix-loader", "start", quiet=True, **kwargs) async def __saver_loaded(self): pipe = os.popen("kldstat|grep daemon_saver") out = pipe.read().strip('\n') pipe.close() return (len(out) > 0) async def _start_saver(self, **kwargs): if not self.__saver_loaded(): await self._system("kldload daemon_saver") async def _stop_saver(self, **kwargs): if self.__saver_loaded(): await self._system("kldunload daemon_saver") async def _restart_saver(self, **kwargs): await self._stop_saver() await self._start_saver() async def _reload_disk(self, **kwargs): await self._service("ix-fstab", "start", quiet=True, **kwargs) await self._service("ix-swap", "start", quiet=True, **kwargs) await self._service("swap", "start", quiet=True, **kwargs) await self._service("mountlate", "start", quiet=True, **kwargs) # Restarting collectd may take a long time and there is no # benefit in waiting for it since even if it fails it wont # tell the user anything useful. asyncio.ensure_future(self.restart("collectd", kwargs)) async def _reload_user(self, **kwargs): await self._service("ix-passwd", "start", quiet=True, **kwargs) await self._service("ix-aliases", "start", quiet=True, **kwargs) await self._service("ix-sudoers", "start", quiet=True, **kwargs) await self.reload("cifs", kwargs) async def _restart_system_datasets(self, **kwargs): systemdataset = await self.middleware.call('systemdataset.setup') if not systemdataset: return None if systemdataset['syslog']: await self.restart("syslogd", kwargs) await self.restart("cifs", kwargs) if systemdataset['rrd']: # Restarting collectd may take a long time and there is no # benefit in waiting for it since even if it fails it wont # tell the user anything useful. asyncio.ensure_future(self.restart("collectd", kwargs))
class AlertServiceService(CRUDService): class Config: datastore = "system.alertservice" datastore_extend = "alertservice._extend" datastore_order_by = ["name"] cli_namespace = "system.alert.service" @accepts() async def list_types(self): """ List all types of supported Alert services which can be configured with the system. """ return [{ "name": name, "title": factory.title, } for name, factory in sorted(ALERT_SERVICES_FACTORIES.items(), key=lambda i: i[1].title.lower())] @private async def _extend(self, service): try: service["type__title"] = ALERT_SERVICES_FACTORIES[ service["type"]].title except KeyError: service["type__title"] = "<Unknown>" return service @private async def _compress(self, service): service.pop("type__title") return service @private async def _validate(self, service, schema_name): verrors = ValidationErrors() factory = ALERT_SERVICES_FACTORIES.get(service["type"]) if factory is None: verrors.add(f"{schema_name}.type", "This field has invalid value") raise verrors verrors.add_child( f"{schema_name}.attributes", validate_attributes(list(factory.schema.attrs.values()), service)) if verrors: raise verrors @accepts( Dict( "alert_service_create", Str("name"), Str("type", required=True), Dict("attributes", additional_attrs=True), Str("level", enum=list(AlertLevel.__members__)), Bool("enabled"), register=True, )) async def do_create(self, data): """ Create an Alert Service of specified `type`. If `enabled`, it sends alerts to the configured `type` of Alert Service. .. examples(websocket):: Create an Alert Service of Mail `type` :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "alertservice.create", "params": [{ "name": "Test Email Alert", "enabled": true, "type": "Mail", "attributes": { "email": "*****@*****.**" }, "settings": { "VolumeVersion": "HOURLY" } }] } """ await self._validate(data, "alert_service_create") data["id"] = await self.middleware.call("datastore.insert", self._config.datastore, data) await self._extend(data) return data @accepts(Int("id"), Patch( "alert_service_create", "alert_service_update", ("attr", { "update": True }), )) async def do_update(self, id, data): """ Update Alert Service of `id`. """ old = await self.middleware.call( "datastore.query", self._config.datastore, [("id", "=", id)], { "extend": self._config.datastore_extend, "get": True }) new = old.copy() new.update(data) await self._validate(new, "alert_service_update") await self._compress(new) await self.middleware.call("datastore.update", self._config.datastore, id, new) await self._extend(new) return new @accepts(Int("id")) async def do_delete(self, id): """ Delete Alert Service of `id`. """ return await self.middleware.call("datastore.delete", self._config.datastore, id) @accepts(Ref('alert_service_create')) async def test(self, data): """ Send a test alert using `type` of Alert Service. .. examples(websocket):: Send a test alert using Alert Service of Mail `type`. :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "alertservice.test", "params": [{ "name": "Test Email Alert", "enabled": true, "type": "Mail", "attributes": { "email": "*****@*****.**" }, "settings": {} }] } """ await self._validate(data, "alert_service_test") factory = ALERT_SERVICES_FACTORIES.get(data["type"]) if factory is None: self.logger.error("Alert service %r does not exist", data["type"]) return False try: alert_service = factory(self.middleware, data["attributes"]) except Exception: self.logger.error( "Error creating alert service %r with parameters=%r", data["type"], data["attributes"], exc_info=True) return False master_node = "A" if await self.middleware.call("failover.licensed"): master_node = await self.middleware.call("failover.node") test_alert = Alert( TestAlertClass, node=master_node, datetime=datetime.utcnow(), last_occurrence=datetime.utcnow(), _uuid="test", ) try: await alert_service.send([test_alert], [], [test_alert]) except Exception: self.logger.error("Error in alert service %r", data["type"], exc_info=True) return False return True
class JailService(CRUDService): @filterable async def query(self, filters=None, options=None): options = options or {} jails = [] try: jails = [ list(jail.values())[0] for jail in ioc.IOCage().get("all", recursive=True) ] except BaseException: # Brandon is working on fixing this generic except, till then I # am not going to make the perfect the enemy of the good enough! self.logger.debug("iocage failed to fetch jails", exc_info=True) pass return filter_list(jails, filters, options) @accepts( Dict("options", Str("release"), Str("template"), Str("pkglist"), Str("uuid"), Bool("basejail", default=False), Bool("empty", default=False), Bool("short", default=False), List("props"))) async def do_create(self, options): """Creates a jail.""" # Typically one would return the created jail's id in this # create call BUT since jail creation may or may not involve # fetching a release, which in turn could be time consuming # and could then block for a long time. This dictates that we # make it a job, but that violates the principle that CRUD methods # are not jobs as yet, so I settle on making this a wrapper around # the main job that calls this and return said job's id instead of # the created jail's id return await self.middleware.call('jail.create_job', options) @private @accepts( Dict("options", Str("release"), Str("template"), Str("pkglist"), Str("uuid"), Bool("basejail", default=False), Bool("empty", default=False), Bool("short", default=False), List("props"))) @job() def create_job(self, job, options): iocage = ioc.IOCage(skip_jails=True) release = options["release"] template = options.get("template", False) pkglist = options.get("pkglist", None) uuid = options.get("uuid", None) basejail = options["basejail"] empty = options["empty"] short = options["short"] props = options["props"] pool = IOCJson().json_get_value("pool") iocroot = IOCJson(pool).json_get_value("iocroot") if template: release = template if not os.path.isdir(f"{iocroot}/releases/{release}") and not \ template and not empty: self.middleware.call_sync('jail.fetch', { "release": release }).wait_sync() err, msg = iocage.create(release, props, 0, pkglist, template=template, short=short, _uuid=uuid, basejail=basejail, empty=empty) if err: raise CallError(msg) return True @accepts(Str("jail"), Dict( "options", Bool("plugin", default=False), additional_attrs=True, )) def do_update(self, jail, options): """Sets a jail property.""" plugin = options.pop("plugin") _, _, iocage = self.check_jail_existence(jail) name = options.pop("name", None) for prop, val in options.items(): p = f"{prop}={val}" iocage.set(p, plugin) if name: iocage.rename(name) return True @accepts(Str("jail")) def do_delete(self, jail): """Takes a jail and destroys it.""" _, _, iocage = self.check_jail_existence(jail) # TODO: Port children checking, release destroying. iocage.destroy_jail() return True @private def check_dataset_existence(self): IOCCheck() @private def check_jail_existence(self, jail, skip=True): """Wrapper for iocage's API, as a few commands aren't ported to it""" try: iocage = ioc.IOCage(skip_jails=skip, jail=jail) jail, path = iocage.__check_jail_existence__() except SystemExit: raise CallError(f"jail '{jail}' not found!") return jail, path, iocage @accepts() def get_activated_pool(self): """Returns the activated pool if there is one, or None""" try: pool = ioc.IOCage(skip_jails=True).get("", pool=True) except Exception: pool = None return pool @accepts( Dict( "options", Str("release"), Str("server", default="download.freebsd.org"), Str("user", default="anonymous"), Str("password", default="anonymous@"), Str("name", default=None), Bool("accept", default=False), List("props"), List("files", default=["MANIFEST", "base.txz", "lib32.txz", "doc.txz"]))) @job(lock=lambda args: f"jail_fetch:{args[-1]}") def fetch(self, job, options): """Fetches a release or plugin.""" self.check_dataset_existence() # Make sure our datasets exist. if options["name"] is not None: options["plugins"] = True iocage = ioc.IOCage(silent=True) iocage.fetch(**options) return True @accepts(Str("resource", enum=["RELEASE", "TEMPLATE", "PLUGIN"]), Bool("remote", default=False)) def list_resource(self, resource, remote): """Returns a JSON list of the supplied resource on the host""" self.check_dataset_existence() # Make sure our datasets exist. iocage = ioc.IOCage(skip_jails=True) resource = "base" if resource == "RELEASE" else resource.lower() if resource == "plugin": if remote: resource_list = iocage.fetch(list=True, plugins=True, header=False) else: resource_list = iocage.list("all", plugin=True) for plugin in resource_list: for i, elem in enumerate(plugin): # iocage returns - for None plugin[i] = elem if elem != "-" else None elif resource == "base": resource_list = iocage.fetch(list=True, remote=remote, http=True) else: resource_list = iocage.list(resource) return resource_list @accepts(Str("jail")) def start(self, jail): """Takes a jail and starts it.""" _, _, iocage = self.check_jail_existence(jail) iocage.start() return True @accepts(Str("jail")) def stop(self, jail): """Takes a jail and stops it.""" _, _, iocage = self.check_jail_existence(jail) iocage.stop() return True @accepts(Str("jail"), Dict( "options", Str("action", enum=["ADD", "EDIT", "REMOVE", "REPLACE", "LIST"]), Str("source"), Str("destination"), Str("fstype"), Str("fsoptions"), Str("dump"), Str("pass"), Int("index", default=None), )) def fstab(self, jail, options): """ Adds an fstab mount to the jail, mounts if the jail is running. """ _, _, iocage = self.check_jail_existence(jail, skip=False) action = options["action"].lower() source = options["source"] destination = options["destination"] fstype = options["fstype"] fsoptions = options["fsoptions"] dump = options["dump"] _pass = options["pass"] index = options["index"] if action == "replace" and index is None: raise ValueError( "index must not be None when replacing fstab entry") _list = iocage.fstab(action, source, destination, fstype, fsoptions, dump, _pass, index=index) if action == "list": split_list = {} for i in _list: split_list[i[0]] = i[1].split() return split_list return True @accepts(Str("pool")) def activate(self, pool): """Activates a pool for iocage usage, and deactivates the rest.""" zfs = libzfs.ZFS(history=True, history_prefix="<iocage>") pools = zfs.pools prop = "org.freebsd.ioc:active" for _pool in pools: if _pool.name == pool: ds = zfs.get_dataset(_pool.name) ds.properties[prop] = libzfs.ZFSUserProperty("yes") else: ds = zfs.get_dataset(_pool.name) ds.properties[prop] = libzfs.ZFSUserProperty("no") return True @accepts(Str("ds_type", enum=["ALL", "JAIL", "TEMPLATE", "RELEASE"])) def clean(self, ds_type): """Cleans all iocage datasets of ds_type""" if ds_type == "JAIL": IOCClean().clean_jails() elif ds_type == "ALL": IOCClean().clean_all() elif ds_type == "RELEASE": pass elif ds_type == "TEMPLATE": IOCClean().clean_templates() return True @accepts(Str("jail"), List("command"), Dict("options", Str("host_user", default="root"), Str("jail_user"))) def exec(self, jail, command, options): """Issues a command inside a jail.""" _, _, iocage = self.check_jail_existence(jail, skip=False) host_user = options["host_user"] jail_user = options.get("jail_user", None) # We may be getting ';', '&&' and so forth. Adding the shell for # safety. if len(command) == 1: command = ["/bin/sh", "-c"] + command host_user = "" if jail_user and host_user == "root" else host_user msg = iocage.exec(command, host_user, jail_user, return_msg=True) return msg.decode("utf-8") @accepts(Str("jail")) @job(lock=lambda args: f"jail_update:{args[-1]}") def update_to_latest_patch(self, job, jail): """Updates specified jail to latest patch level.""" uuid, path, _ = self.check_jail_existence(jail) status, jid = IOCList.list_get_jid(uuid) conf = IOCJson(path).json_load() # Sometimes if they don't have an existing patch level, this # becomes 11.1 instead of 11.1-RELEASE _release = conf["release"].rsplit("-", 1)[0] release = _release if "-RELEASE" in _release else conf["release"] started = False if conf["type"] == "jail": if not status: self.start(jail) started = True else: return False if conf["basejail"] != "yes": IOCFetch(release).fetch_update(True, uuid) else: # Basejails only need their base RELEASE updated IOCFetch(release).fetch_update() if started: self.stop(jail) return True @accepts(Str("jail"), Str("release")) @job(lock=lambda args: f"jail_upgrade:{args[-1]}") def upgrade(self, job, jail, release): """Upgrades specified jail to specified RELEASE.""" uuid, path, _ = self.check_jail_existence(jail) status, jid = IOCList.list_get_jid(uuid) conf = IOCJson(path).json_load() root_path = f"{path}/root" started = False if conf["type"] == "jail": if not status: self.start(jail) started = True else: return False IOCUpgrade(conf, release, root_path).upgrade_jail() if started: self.stop(jail) return True @accepts(Str("jail")) @job(lock=lambda args: f"jail_export:{args[-1]}") def export(self, job, jail): """Exports jail to zip file""" uuid, path, _ = self.check_jail_existence(jail) status, jid = IOCList.list_get_jid(uuid) started = False if status: self.stop(jail) started = True IOCImage().export_jail(uuid, path) if started: self.start(jail) return True @accepts(Str("jail")) @job(lock=lambda args: f"jail_import:{args[-1]}") def _import(self, job, jail): """Imports jail from zip file""" IOCImage().import_jail(jail) return True
class ZFSDatasetService(CRUDService): class Config: namespace = 'zfs.dataset' private = True process_pool = True def locked_datasets(self): try: about_to_lock_dataset = self.middleware.call_sync( 'cache.get', 'about_to_lock_dataset') except KeyError: about_to_lock_dataset = None or_filters = [ 'OR', [['key_loaded', '=', False]] + ([['id', '=', about_to_lock_dataset], ['id', '^', f'{about_to_lock_dataset}/']] if about_to_lock_dataset else []) ] return self.query( [['encrypted', '=', True], or_filters], { 'extra': { 'properties': ['encryption', 'keystatus', 'mountpoint'] }, 'select': ['id', 'mountpoint'] }) def flatten_datasets(self, datasets): return sum([[deepcopy(ds)] + self.flatten_datasets(ds['children']) for ds in datasets], []) @filterable def query(self, filters=None, options=None): """ In `query-options` we can provide `extra` arguments which control which data should be retrieved for a dataset. `query-options.extra.top_level_properties` is a list of properties which we will like to include in the top level dict of dataset. It defaults to adding only mountpoint key keeping legacy behavior. If none are desired in top level dataset, an empty list should be passed else if null is specified it will add mountpoint key to the top level dict if it's present in `query-options.extra.properties` or it's null as well. `query-options.extra.properties` is a list of properties which should be retrieved. If null ( by default ), it would retrieve all properties, if empty, it will retrieve no property ( `mountpoint` is special in this case and is controlled by `query-options.extra.mountpoint` attribute ). We provide 2 ways how zfs.dataset.query returns dataset's data. First is a flat structure ( default ), which means that all the datasets in the system are returned as separate objects which also contain all the data their is for their children. This retrieval type is slightly slower because of duplicates which exist in each object. Second type is hierarchical where only top level datasets are returned in the list and they contain all the children there are for them in `children` key. This retrieval type is slightly faster. These options are controlled by `query-options.extra.flat` attribute which defaults to true. `query-options.extra.user_properties` controls if user defined properties of datasets should be retrieved or not. While we provide a way to exclude all properties from data retrieval, we introduce a single attribute `query-options.extra.retrieve_properties` which if set to false will make sure that no property is retrieved whatsoever and overrides any other property retrieval attribute. """ options = options or {} extra = options.get('extra', {}).copy() top_level_props = None if extra.get( 'top_level_properties' ) is None else extra['top_level_properties'].copy() props = extra.get('properties', None) flat = extra.get('flat', True) user_properties = extra.get('user_properties', True) retrieve_properties = extra.get('retrieve_properties', True) if not retrieve_properties: # This is a short hand version where consumer can specify that they don't want any property to # be retrieved user_properties = False props = [] with libzfs.ZFS() as zfs: # Handle `id` filter specially to avoiding getting all datasets if filters and len(filters) == 1 and list( filters[0][:2]) == ['id', '=']: state_options = { 'snapshots': extra.get('snapshots', False), 'recursive': extra.get('recursive', True), 'snapshots_recursive': extra.get('snapshots_recursive', False) } try: datasets = [ zfs.get_dataset( filters[0][2]).__getstate__(**state_options) ] except libzfs.ZFSException: datasets = [] else: datasets = zfs.datasets_serialized( props=props, top_level_props=top_level_props, user_props=user_properties) if flat: datasets = self.flatten_datasets(datasets) else: datasets = list(datasets) return filter_list(datasets, filters, options) def query_for_quota_alert(self): return [{ k: v for k, v in dataset['properties'].items() if k in [ "name", "quota", "available", "refquota", "usedbydataset", "mounted", "mountpoint", "org.freenas:quota_warning", "org.freenas:quota_critical", "org.freenas:refquota_warning", "org.freenas:refquota_critical" ] } for dataset in self.query()] def common_load_dataset_checks(self, ds): self.common_encryption_checks(ds) if ds.key_loaded: raise CallError(f'{id} key is already loaded') def common_encryption_checks(self, ds): if not ds.encrypted: raise CallError(f'{id} is not encrypted') def path_to_dataset(self, path): with libzfs.ZFS() as zfs: try: zh = zfs.get_dataset_by_path(path) ds_name = zh.name except libzfs.ZFSException: ds_name = None return ds_name def get_quota(self, ds, quota_type): if quota_type == 'dataset': dataset = self.query([('id', '=', ds)], {'get': True}) return [{ 'quota_type': 'DATASET', 'id': ds, 'name': ds, 'quota': int(dataset['properties']['quota']['rawvalue']), 'refquota': int(dataset['properties']['refquota']['rawvalue']), 'used_bytes': int(dataset['properties']['used']['rawvalue']), }] quota_list = [] quota_get = subprocess.run( [ 'zfs', f'{quota_type}space', '-H', '-n', '-p', '-o', 'name,used,quota,objquota,objused', ds ], capture_output=True, check=False, ) if quota_get.returncode != 0: raise CallError( f'Failed to get {quota_type} quota for {ds}: [{quota_get.stderr.decode()}]' ) for quota in quota_get.stdout.decode().splitlines(): m = quota.split('\t') if len(m) != 5: self.logger.debug('Invalid %s quota: %s', quota_type.lower(), quota) continue entry = { 'quota_type': quota_type.upper(), 'id': int(m[0]), 'name': None, 'quota': int(m[2]), 'used_bytes': int(m[1]), 'used_percent': 0, 'obj_quota': int(m[3]) if m[3] != '-' else 0, 'obj_used': int(m[4]) if m[4] != '-' else 0, 'obj_used_percent': 0, } if entry['quota'] > 0: entry['used_percent'] = entry['used_bytes'] / entry[ 'quota'] * 100 if entry['obj_quota'] > 0: entry['obj_used_percent'] = entry['obj_used'] / entry[ 'obj_quota'] * 100 try: if entry['quota_type'] == 'USER': entry['name'] = (self.middleware.call_sync( 'user.get_user_obj', {'uid': entry['id']}))['pw_name'] else: entry['name'] = (self.middleware.call_sync( 'group.get_group_obj', {'gid': entry['id']}))['gr_name'] except Exception: self.logger.debug('Unable to resolve %s id %d to name', quota_type.lower(), entry['id']) pass quota_list.append(entry) return quota_list def set_quota(self, ds, quota_list): cmd = ['zfs', 'set'] cmd.extend(quota_list) cmd.append(ds) quota_set = subprocess.run(cmd, check=False) if quota_set.returncode != 0: raise CallError( f'Failed to set userspace quota on {ds}: [{quota_set.stderr.decode()}]' ) @accepts( Str('id'), Dict( 'load_key_options', Bool('mount', default=True), Bool('recursive', default=False), Any('key', default=None, null=True), Str('key_location', default=None, null=True), ), ) def load_key(self, id, options): mount_ds = options.pop('mount') recursive = options.pop('recursive') try: with libzfs.ZFS() as zfs: ds = zfs.get_dataset(id) self.common_load_dataset_checks(ds) ds.load_key(**options) except libzfs.ZFSException as e: self.logger.error(f'Failed to load key for {id}', exc_info=True) raise CallError(f'Failed to load key for {id}: {e}') else: if mount_ds: self.mount(id, {'recursive': recursive}) @accepts(Str('name'), List('params', default=[], private=True)) @job() def bulk_process(self, job, name, params): f = getattr(self, name, None) if not f: raise CallError(f'{name} method not found in zfs.dataset') statuses = [] for i in params: result = error = None try: result = f(*i) except Exception as e: error = str(e) finally: statuses.append({'result': result, 'error': error}) return statuses @accepts(Str('id'), Dict( 'check_key', Any('key', default=None, null=True), Str('key_location', default=None, null=True), )) def check_key(self, id, options): """ Returns `true` if the `key` is valid, `false` otherwise. """ try: with libzfs.ZFS() as zfs: ds = zfs.get_dataset(id) self.common_encryption_checks(ds) return ds.check_key(**options) except libzfs.ZFSException as e: self.logger.error(f'Failed to check key for {id}', exc_info=True) raise CallError(f'Failed to check key for {id}: {e}') @accepts(Str('id'), Dict( 'unload_key_options', Bool('recursive', default=False), Bool('force_umount', default=False), Bool('umount', default=False), )) def unload_key(self, id, options): force = options.pop('force_umount') if options.pop('umount') and self.middleware.call_sync( 'zfs.dataset.get_instance', id)['mountpoint']: self.umount(id, {'force': force}) try: with libzfs.ZFS() as zfs: ds = zfs.get_dataset(id) self.common_encryption_checks(ds) if not ds.key_loaded: raise CallError(f'{id}\'s key is not loaded') ds.unload_key(**options) except libzfs.ZFSException as e: self.logger.error(f'Failed to unload key for {id}', exc_info=True) raise CallError(f'Failed to unload key for {id}: {e}') @accepts( Str('id'), Dict( 'change_key_options', Dict('encryption_properties', Str('keyformat'), Str('keylocation'), Int('pbkdf2iters')), Bool('load_key', default=True), Any('key', default=None, null=True), ), ) def change_key(self, id, options): try: with libzfs.ZFS() as zfs: ds = zfs.get_dataset(id) self.common_encryption_checks(ds) ds.change_key(props=options['encryption_properties'], load_key=options['load_key'], key=options['key']) except libzfs.ZFSException as e: self.logger.error(f'Failed to change key for {id}', exc_info=True) raise CallError(f'Failed to change key for {id}: {e}') @accepts(Str('id'), Dict( 'change_encryption_root_options', Bool('load_key', default=True), )) def change_encryption_root(self, id, options): try: with libzfs.ZFS() as zfs: ds = zfs.get_dataset(id) ds.change_key(load_key=options['load_key'], inherit=True) except libzfs.ZFSException as e: raise CallError(f'Failed to change encryption root for {id}: {e}') @accepts( Dict( 'dataset_create', Str('name', required=True), Str('type', enum=['FILESYSTEM', 'VOLUME'], default='FILESYSTEM'), Dict( 'properties', Bool('sparse'), additional_attrs=True, ), )) def do_create(self, data): """ Creates a ZFS dataset. """ verrors = ValidationErrors() if '/' not in data['name']: verrors.add('name', 'You need a full name, e.g. pool/newdataset') if verrors: raise verrors properties = data.get('properties') or {} sparse = properties.pop('sparse', False) params = {} for k, v in data['properties'].items(): params[k] = v try: with libzfs.ZFS() as zfs: pool = zfs.get(data['name'].split('/')[0]) pool.create(data['name'], params, fstype=getattr(libzfs.DatasetType, data['type']), sparse_vol=sparse) except libzfs.ZFSException as e: self.logger.error('Failed to create dataset', exc_info=True) raise CallError(f'Failed to create dataset: {e}') @accepts( Str('id'), Dict( 'dataset_update', Dict( 'properties', additional_attrs=True, ), ), ) def do_update(self, id, data): try: with libzfs.ZFS() as zfs: dataset = zfs.get_dataset(id) if 'properties' in data: properties = data['properties'].copy() # Set these after reservations for k in ['quota', 'refquota']: if k in properties: properties[k] = properties.pop(k) # Set them last for k, v in properties.items(): # If prop already exists we just update it, # otherwise create a user property prop = dataset.properties.get(k) try: if prop: if v.get('source') == 'INHERIT': prop.inherit( recursive=v.get('recursive', False)) elif 'value' in v and (prop.value != v['value'] or prop.source.name == 'INHERITED'): prop.value = v['value'] elif 'parsed' in v and ( prop.parsed != v['parsed'] or prop.source.name == 'INHERITED'): prop.parsed = v['parsed'] else: if v.get('source') == 'INHERIT': pass else: if 'value' not in v: raise ValidationError( 'properties', f'properties.{k} needs a "value" attribute' ) if ':' not in k: raise ValidationError( 'properties', f'User property needs a colon (:) in its name`' ) prop = libzfs.ZFSUserProperty(v['value']) dataset.properties[k] = prop except libzfs.ZFSException as e: raise ZFSSetPropertyError(k, str(e)) except libzfs.ZFSException as e: self.logger.error('Failed to update dataset', exc_info=True) raise CallError(f'Failed to update dataset: {e}') def do_delete(self, id, options=None): options = options or {} force = options.get('force', False) recursive = options.get('recursive', False) args = [] if force: args += ['-f'] if recursive: args += ['-r'] # If dataset is mounted and has receive_resume_token, we should destroy it or ZFS will say # "cannot destroy 'pool/dataset': dataset already exists" recv_run = subprocess.run(['zfs', 'recv', '-A', id], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) # Destroying may take a long time, lets not use py-libzfs as it will block # other ZFS operations. try: subprocess.run( ['zfs', 'destroy'] + args + [id], text=True, capture_output=True, check=True, ) except subprocess.CalledProcessError as e: if recv_run.returncode == 0 and e.stderr.strip().endswith( 'dataset does not exist'): # This operation might have deleted this dataset if it was created by `zfs recv` operation return self.logger.error('Failed to delete dataset', exc_info=True) error = e.stderr.strip() errno_ = errno.EFAULT if "Device busy" in error: errno_ = errno.EBUSY raise CallError(f'Failed to delete dataset: {error}', errno_) @accepts(Str('name'), Dict('options', Bool('recursive', default=False))) def mount(self, name, options): try: with libzfs.ZFS() as zfs: dataset = zfs.get_dataset(name) if options['recursive']: dataset.mount_recursive() else: dataset.mount() except libzfs.ZFSException as e: self.logger.error('Failed to mount dataset', exc_info=True) raise CallError(f'Failed to mount dataset: {e}') @accepts(Str('name'), Dict('options', Bool('force', default=False))) def umount(self, name, options): try: with libzfs.ZFS() as zfs: dataset = zfs.get_dataset(name) dataset.umount(force=options['force']) except libzfs.ZFSException as e: self.logger.error('Failed to umount dataset', exc_info=True) raise CallError(f'Failed to umount dataset: {e}') @accepts(Str('dataset'), Dict('options', Str('new_name', required=True, empty=False), Bool('recursive', default=False))) def rename(self, name, options): try: with libzfs.ZFS() as zfs: dataset = zfs.get_dataset(name) dataset.rename(options['new_name'], recursive=options['recursive']) except libzfs.ZFSException as e: self.logger.error('Failed to rename dataset', exc_info=True) raise CallError(f'Failed to rename dataset: {e}') def promote(self, name): try: with libzfs.ZFS() as zfs: dataset = zfs.get_dataset(name) dataset.promote() except libzfs.ZFSException as e: self.logger.error('Failed to promote dataset', exc_info=True) raise CallError(f'Failed to promote dataset: {e}') def inherit(self, name, prop, recursive=False): try: with libzfs.ZFS() as zfs: dataset = zfs.get_dataset(name) zprop = dataset.properties.get(prop) if not zprop: raise CallError(f'Property {prop!r} not found.', errno.ENOENT) zprop.inherit(recursive=recursive) except libzfs.ZFSException as e: raise CallError(str(e))
class PoolService(Service): @item_method @accepts(Int('id'), Dict( 'options', Str('label', required=True), Str('disk', required=True), Bool('force', default=False), Str('passphrase', private=True), Bool('preserve_settings', default=True), )) @job(lock='pool_replace') async def replace(self, job, oid, options): """ Replace a disk on a pool. `label` is the ZFS guid or a device name `disk` is the identifier of a disk `passphrase` is only valid for TrueNAS Core/Enterprise platform where pool is GELI encrypted If `preserve_settings` is true, then settings (power management, S.M.A.R.T., etc.) of a disk being replaced will be applied to a new disk. .. examples(websocket):: Replace missing ZFS device with disk {serial}FOO. :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "pool.replace", "params": [1, { "label": "80802394992848654", "disk": "{serial}FOO" }] } """ pool = await self.middleware.call('pool.get_instance', oid) verrors = ValidationErrors() unused_disks = await self.middleware.call('disk.get_unused') disk = list( filter(lambda x: x['identifier'] == options['disk'], unused_disks)) if not disk: verrors.add('options.disk', 'Disk not found.', errno.ENOENT) else: disk = disk[0] if not options['force'] and not await self.middleware.call( 'disk.check_clean', disk['devname']): verrors.add('options.force', 'Disk is not clean, partitions were found.') if osc.IS_FREEBSD and pool['encrypt'] == 2: if not options.get('passphrase'): verrors.add('options.passphrase', 'Passphrase is required for encrypted pool.') elif not await self.middleware.call('disk.geli_testkey', pool, options['passphrase']): verrors.add('options.passphrase', 'Passphrase is not valid.') if osc.IS_LINUX and options.get('passphrase'): verrors.add('options.passphrase', 'This field is not valid on this platform.') found = await self.middleware.call('pool.find_disk_from_topology', options['label'], pool) if not found: verrors.add('options.label', f'Label {options["label"]} not found.', errno.ENOENT) if verrors: raise verrors old_disk = None if options['preserve_settings']: try: old_disk = await self.middleware.call( 'disk.query', [['zfs_guid', '=', options['label']]], { 'extra': { 'include_expired': True }, 'get': True }) except MatchNotFound: pass create_swap = found[0] in ('data', 'spare') swap_disks = [disk['devname']] # If the disk we are replacing is still available, remove it from swap as well if found[1] and os.path.exists(found[1]['path']): from_disk = await self.middleware.call( 'disk.label_to_disk', found[1]['path'].replace('/dev/', '')) if from_disk: swap_disks.append(from_disk) await self.middleware.call('disk.swaps_remove_disks', swap_disks) vdev = [] enc_disks = await self.middleware.call( 'pool.format_disks', job, {disk['devname']: { 'vdev': vdev, 'create_swap': create_swap }}, { 'enc_keypath': pool['encryptkey_path'], 'passphrase': options.get('passphrase') }, ) new_devname = vdev[0].replace('/dev/', '') job.set_progress(30, 'Replacing disk') try: await self.middleware.call('zfs.pool.replace', pool['name'], options['label'], new_devname) # If we are replacing a faulted disk, kick it right after replace # is initiated. try: vdev = await self.middleware.call( 'zfs.pool.get_vdev', pool['name'], options['label'], ) if vdev['status'] not in ('ONLINE', 'DEGRADED'): await self.middleware.call('zfs.pool.detach', pool['name'], options['label']) except Exception: self.logger.warn('Failed to detach device', exc_info=True) except Exception as e: if osc.IS_FREEBSD: try: # If replace has failed lets detach geli to not keep disk busy await self.middleware.call('disk.geli_detach_single', new_devname) except Exception: self.logger.warn(f'Failed to geli detach {new_devname}', exc_info=True) raise e finally: # Needs to happen even if replace failed to put back disk that had been # removed from swap prior to replacement asyncio.ensure_future(self.middleware.call('disk.swaps_configure')) if osc.IS_FREEBSD: await self.middleware.call('pool.save_encrypteddisks', oid, enc_disks, {disk['devname']: disk}) if old_disk: await self.middleware.call('disk.copy_settings', old_disk, disk) return True
class RsyncModService(SharingService): share_task_type = 'Rsync Module' class Config: datastore = 'services.rsyncmod' datastore_prefix = 'rsyncmod_' datastore_extend = 'rsyncmod.rsync_mod_extend' cli_namespace = 'service.rsync_mod' ENTRY = Patch( 'rsyncmod_create', 'rsyncmod_entry', ('add', Bool('locked')), ('add', Int('id')), ) @private async def rsync_mod_extend(self, data): data['hostsallow'] = data['hostsallow'].split() data['hostsdeny'] = data['hostsdeny'].split() data['mode'] = data['mode'].upper() return data @private async def common_validation(self, data, schema_name): verrors = ValidationErrors() await self.validate_path_field(data, schema_name, verrors) for entity in ('user', 'group'): value = data.get(entity) try: await self.middleware.call(f'{entity}.get_{entity}_obj', {f'{entity}name': value}) except Exception: verrors.add(f'{schema_name}.{entity}', f'Please specify a valid {entity}') verrors.check() data['hostsallow'] = ' '.join(data['hostsallow']) data['hostsdeny'] = ' '.join(data['hostsdeny']) data['mode'] = data['mode'].lower() return data @accepts( Dict( 'rsyncmod_create', Bool('enabled', default=True), Str('name', validators=[Match(r'[^/\]]')]), Str('comment'), Str('path', required=True, max_length=RSYNC_PATH_LIMIT), Str('mode', enum=['RO', 'RW', 'WO'], required=True), Int('maxconn'), Str('user', default='nobody'), Str('group', default='nobody'), List('hostsallow', items=[Str('hostsallow')]), List('hostsdeny', items=[Str('hostdeny')]), Str('auxiliary', max_length=None), register=True, )) async def do_create(self, data): """ Create a Rsyncmod module. `path` represents the path to a dataset. Path length is limited to 1023 characters maximum as per the limit enforced by FreeBSD. It is possible that we reach this max length recursively while transferring data. In that case, the user must ensure the maximum path will not be too long or modify the recursed path to shorter than the limit. `maxconn` is an integer value representing the maximum number of simultaneous connections. Zero represents unlimited. `hostsallow` is a list of patterns to match hostname/ip address of a connecting client. If list is empty, all hosts are allowed. `hostsdeny` is a list of patterns to match hostname/ip address of a connecting client. If the pattern is matched, access is denied to the client. If no client should be denied, this should be left empty. `auxiliary` attribute can be used to pass on any additional parameters from rsyncd.conf(5). """ data = await self.common_validation(data, 'rsyncmod_create') data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self._service_change('rsync', 'reload') return await self._get_instance(data['id']) @accepts(Int('id'), Patch('rsyncmod_create', 'rsyncmod_update', ('attr', { 'update': True }))) async def do_update(self, id, data): """ Update Rsyncmod module of `id`. """ module = await self.get_instance(id) module.update(data) module = await self.common_validation(module, 'rsyncmod_update') module.pop(self.locked_field) await self.middleware.call('datastore.update', self._config.datastore, id, module, {'prefix': self._config.datastore_prefix}) await self._service_change('rsync', 'reload') return await self.get_instance(id) async def do_delete(self, id): """ Delete Rsyncmod module of `id`. """ return await self.middleware.call('datastore.delete', self._config.datastore, id)
class NFSService(SystemServiceService): class Config: service = "nfs" service_verb = "restart" datastore_prefix = "nfs_srv_" datastore_extend = 'nfs.nfs_extend' @private async def nfs_extend(self, nfs): keytab_has_nfs = await self.middleware.call( "kerberos.keytab.has_nfs_principal") nfs["v4_krb_enabled"] = (nfs["v4_krb"] or keytab_has_nfs) nfs["userd_manage_gids"] = nfs.pop("16") return nfs @private async def nfs_compress(self, nfs): nfs.pop("v4_krb_enabled") nfs["16"] = nfs.pop("userd_manage_gids") return nfs @accepts() async def bindip_choices(self): """ Returns ip choices for NFS service to use """ return { d['address']: d['address'] for d in await self.middleware.call('interface.ip_in_use', { 'static': True, 'any': True }) } @private async def bindip(self, config): bindip = config['bindip'] if osc.IS_LINUX: bindip = bindip[:1] if bindip: found = False for iface in await self.middleware.call('interface.query'): for alias in iface['state']['aliases']: if alias['address'] in bindip: found = True break if found: break else: found = True if found: await self.middleware.call('alert.oneshot_delete', 'NFSBindAddress', None) return bindip else: if await self.middleware.call('cache.has_key', 'interfaces_are_set_up'): await self.middleware.call('alert.oneshot_create', 'NFSBindAddress', None) return [] @accepts( Dict('nfs_update', Int('servers', validators=[Range(min=1, max=256)]), Bool('udp'), Bool('allow_nonroot'), Bool('v4'), Bool('v4_v3owner'), Bool('v4_krb'), Str('v4_domain'), List('bindip', items=[IPAddr('ip')]), Int('mountd_port', null=True, validators=[Range(min=1, max=65535)]), Int('rpcstatd_port', null=True, validators=[Range(min=1, max=65535)]), Int('rpclockd_port', null=True, validators=[Range(min=1, max=65535)]), Bool('userd_manage_gids'), Bool('mountd_log'), Bool('statd_lockd_log'), update=True)) async def do_update(self, data): """ Update NFS Service Configuration. `servers` represents number of servers to create. When `allow_nonroot` is set, it allows non-root mount requests to be served. `bindip` is a list of IP's on which NFS will listen for requests. When it is unset/empty, NFS listens on all available addresses. `v4` when set means that we switch from NFSv3 to NFSv4. `v4_v3owner` when set means that system will use NFSv3 ownership model for NFSv4. `v4_krb` will force NFS shares to fail if the Kerberos ticket is unavailable. `v4_domain` overrides the default DNS domain name for NFSv4. `mountd_port` specifies the port mountd(8) binds to. `rpcstatd_port` specifies the port rpc.statd(8) binds to. `rpclockd_port` specifies the port rpclockd_port(8) binds to. .. examples(websocket):: Update NFS Service Configuration to listen on 192.168.0.10 and use NFSv4 :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "pool.resilver.update", "params": [{ "bindip": [ "192.168.0.10" ], "v4": true }] } """ if data.get("v4") is False: data.setdefault("v4_v3owner", False) old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() keytab_has_nfs = await self.middleware.call( "kerberos.keytab.has_nfs_principal") new_v4_krb_enabled = new["v4_krb"] or keytab_has_nfs if new["v4"] and new_v4_krb_enabled and not await self.middleware.call( "system.is_freenas"): if await self.middleware.call("failover.licensed"): gc = await self.middleware.call("datastore.config", "network.globalconfiguration") if not gc["gc_hostname_virtual"] or not gc["gc_domain"]: verrors.add( "nfs_update.v4", "Enabling kerberos authentication on TrueNAS HA requires setting the virtual hostname and " "domain") if osc.IS_LINUX: if len(new['bindip']) > 1: verrors.add( 'nfs_update.bindip', 'Listening on more than one address is not supported') bindip_choices = await self.bindip_choices() for i, bindip in enumerate(new['bindip']): if bindip not in bindip_choices: verrors.add(f'nfs_update.bindip.{i}', 'Please provide a valid ip address') if new["v4"] and new_v4_krb_enabled and await self.middleware.call( 'activedirectory.get_state') != "DISABLED": """ In environments with kerberized NFSv4 enabled, we need to tell winbindd to not prefix usernames with the short form of the AD domain. Directly update the db and regenerate the smb.conf to avoid having a service disruption due to restarting the samba server. """ if await self.middleware.call('smb.get_smb_ha_mode') == 'LEGACY': raise ValidationError( 'nfs_update.v4', 'Enabling kerberos authentication on TrueNAS HA requires ' 'the system dataset to be located on a data pool.') ad = await self.middleware.call('activedirectory.config') await self.middleware.call('datastore.update', 'directoryservice.activedirectory', ad['id'], {'ad_use_default_domain': True}) await self.middleware.call('etc.generate', 'smb') await self.middleware.call('service.reload', 'cifs') if not new["v4"] and new["v4_v3owner"]: verrors.add("nfs_update.v4_v3owner", "This option requires enabling NFSv4") if new["v4_v3owner"] and new["userd_manage_gids"]: verrors.add( "nfs_update.userd_manage_gids", "This option is incompatible with NFSv3 ownership model for NFSv4" ) if not new["v4"] and new["v4_domain"]: verrors.add("nfs_update.v4_domain", "This option does not apply to NFSv3") if verrors: raise verrors await self.nfs_compress(new) await self._update_service(old, new) await self.nfs_extend(new) return new
class DiskService(Service): RE_IDENTIFIER = re.compile(r'^\{(?P<type>.+?)\}(?P<value>.+)$') @private @accepts(Str('name'), Dict('disks', additional_attrs=True)) async def device_to_identifier(self, name, disks): """ Given a device `name` (e.g. sda) returns an unique identifier string for this device. This identifier is in the form of {type}string, "type" can be one of the following: - serial_lunid - for disk serial concatenated with the lunid - serial - disk serial - uuid - uuid of a ZFS GPT partition - label - label name from geom label - devicename - name of the device if any other could not be used/found `disks` is value returned by `device.get_disks`. This can be passed to avoid collecting system data again if the consumer already has it. Returns: str - identifier """ disks = disks or await self.middleware.call('device.get_disks') if name not in disks: return '' else: block_device = disks[name] if block_device['serial_lunid']: return f'{{serial_lunid}}{block_device["serial_lunid"]}' elif block_device['serial']: return f'{{serial}}{block_device["serial"]}' dev = pyudev.Devices.from_name(pyudev.Context(), 'block', name) for partition in filter( lambda p: all( p.get(k) for k in ('ID_PART_ENTRY_TYPE', 'ID_PART_ENTRY_UUID')), dev.children): if partition[ 'ID_PART_ENTRY_TYPE'] not in await self.middleware.call( 'disk.get_valid_zfs_partition_type_uuids'): continue return f'{{uuid}}{partition["ID_PART_ENTRY_UUID"]}' return f'{{devicename}}{name}' @private async def identifier_to_device(self, ident, disks): if not ident: return None search = self.RE_IDENTIFIER.search(ident) if not search: return None tp = search.group('type') value = search.group('value') mapping = { 'uuid': 'uuid', 'devicename': 'name', 'serial_lunid': 'serial_lunid', 'serial': 'serial' } if tp not in mapping: return None elif tp == 'uuid': partition = await self.middleware.call( 'disk.list_all_partitions', [['partition_uuid', '=', value]]) if partition: return partition[0]['disk'] else: disk = next((b for b in (disks or await self.middleware.call( 'device.get_disks')).values() if b[mapping[tp]] == value), None) return disk['name'] if disk else None
class ImageService(CRUDService): class Config: namespace = 'webui.image' datastore = 'system.filesystem' datastore_extend = 'webui.image.url_extend' cli_private = True @private async def url_extend(self, image): """ Adds the URL field to the image which is /images/ID """ image["url"] = f"/images/{image['id']}.png" return image @accepts(Dict("options", Str("identifier"))) @job(pipes=["input"]) async def do_create(self, job, options): """ Create a new database entry with identifier as the tag, all entries are lowercased Then puts the file in the /var/db/system/webui/images directory """ identifier = options.get('identifier') self.__ensure_dir() try: id = await self.middleware.call('datastore.insert', 'system.filesystem', {'identifier': identifier.lower()}) except IntegrityError as e: # Likely a duplicate entry raise CallError(e) final_location = f"/var/db/system/webui/images/{id}.png" put_job = await self.middleware.call( 'filesystem.put', final_location, {"mode": 0o755}, pipes=Pipes(input=job.pipes.input)) await put_job.wait() return id @accepts(Int("id")) def do_delete(self, id): """ Remove the database entry, and then the item if it exists """ self.__ensure_dir() item = f"/var/db/system/webui/images/{id}.png" self.middleware.call_sync('datastore.delete', 'system.filesystem', id) if os.path.exists(item): os.remove(item) return True def __ensure_dir(self): """ Ensure that the images directory exists """ dirname = "/var/db/system/webui/images" if not os.path.isdir(dirname): if os.path.exists(dirname): # This is an imposter! Nuke it. os.remove(dirname) if not os.path.exists(dirname): os.makedirs(dirname)
class SharingNFSService(SharingService): path_field = 'paths' share_task_type = 'NFS' class Config: namespace = "sharing.nfs" datastore = "sharing.nfs_share" datastore_prefix = "nfs_" datastore_extend = "sharing.nfs.extend" async def human_identifier(self, share_task): return ', '.join(share_task[self.path_field]) @private async def sharing_task_determine_locked(self, data, locked_datasets): for path in data[self.path_field]: if await self.middleware.call( 'pool.dataset.path_in_locked_datasets', path, locked_datasets): return True else: return False @accepts( Dict( "sharingnfs_create", List("paths", items=[Dir("path")], required=True, empty=False), List("aliases", items=[Str("path", validators=[Match(r"^/.*")])], default=[]), Str("comment", default=""), List("networks", items=[IPAddr("network", network=True)], default=[]), List("hosts", items=[Str("host")], default=[]), Bool("alldirs", default=False), Bool("ro", default=False), Bool("quiet", default=False), Str("maproot_user", required=False, default=None, null=True), Str("maproot_group", required=False, default=None, null=True), Str("mapall_user", required=False, default=None, null=True), Str("mapall_group", required=False, default=None, null=True), List( "security", default=[], items=[ Str("provider", enum=["SYS", "KRB5", "KRB5I", "KRB5P"]) ], ), Bool("enabled", default=True), register=True, strict=True, )) async def do_create(self, data): """ Create a NFS Share. `paths` is a list of valid paths which are configured to be shared on this share. `aliases` is a list of aliases for each path (or an empty list if aliases are not used). `networks` is a list of authorized networks that are allowed to access the share having format "network/mask" CIDR notation. If empty, all networks are allowed. `hosts` is a list of IP's/hostnames which are allowed to access the share. If empty, all IP's/hostnames are allowed. `alldirs` is a boolean value which when set indicates that the client can mount any subdirectories of the selected pool or dataset. """ verrors = ValidationErrors() await self.validate(data, "sharingnfs_create", verrors) if verrors: raise verrors await self.compress(data) data["id"] = await self.middleware.call( "datastore.insert", self._config.datastore, data, {"prefix": self._config.datastore_prefix}, ) await self.extend(data) await self._service_change("nfs", "reload") return await self.get_instance(data["id"]) @accepts(Int("id"), Patch("sharingnfs_create", "sharingnfs_update", ("attr", { "update": True }))) async def do_update(self, id, data): """ Update NFS Share of `id`. """ verrors = ValidationErrors() old = await self.get_instance(id) new = old.copy() new.update(data) await self.validate(new, "sharingnfs_update", verrors, old=old) if verrors: raise verrors await self.compress(new) await self.middleware.call("datastore.update", self._config.datastore, id, new, {"prefix": self._config.datastore_prefix}) await self._service_change("nfs", "reload") return await self.get_instance(id) @accepts(Int("id")) async def do_delete(self, id): """ Delete NFS Share of `id`. """ await self.middleware.call("datastore.delete", self._config.datastore, id) await self._service_change("nfs", "reload") @private async def validate(self, data, schema_name, verrors, old=None): if len(data["aliases"]): if not osc.IS_LINUX: verrors.add( f"{schema_name}.aliases", "This field is only supported on SCALE", ) if len(data["aliases"]) != len(data["paths"]): verrors.add( f"{schema_name}.aliases", "This field should be either empty of have the same number of elements as paths", ) if data["alldirs"] and len(data["paths"]) > 1: verrors.add( f"{schema_name}.alldirs", "This option can only be used for shares that contain single path" ) # need to make sure that the nfs share is within the zpool mountpoint for idx, i in enumerate(data["paths"]): await check_path_resides_within_volume( verrors, self.middleware, f'{schema_name}.paths.{idx}', i) await self.middleware.run_in_thread(self.validate_paths, data, schema_name, verrors) filters = [] if old: filters.append(["id", "!=", old["id"]]) other_shares = await self.middleware.call("sharing.nfs.query", filters) dns_cache = await self.resolve_hostnames( sum([share["hosts"] for share in other_shares], []) + data["hosts"]) await self.middleware.run_in_thread(self.validate_hosts_and_networks, other_shares, data, schema_name, verrors, dns_cache) for k in ["maproot", "mapall"]: if not data[f"{k}_user"] and not data[f"{k}_group"]: pass elif not data[f"{k}_user"] and data[f"{k}_group"]: verrors.add( f"{schema_name}.{k}_user", "This field is required when map group is specified") else: user = group = None with contextlib.suppress(KeyError): user = await self.middleware.call( 'dscache.get_uncached_user', data[f'{k}_user']) if not user: verrors.add(f"{schema_name}.{k}_user", "User not found") if data[f'{k}_group']: with contextlib.suppress(KeyError): group = await self.middleware.call( 'dscache.get_uncached_group', data[f'{k}_group']) if not group: verrors.add(f"{schema_name}.{k}_group", "Group not found") if data["maproot_user"] and data["mapall_user"]: verrors.add(f"{schema_name}.mapall_user", "maproot_user disqualifies mapall_user") if data["security"]: nfs_config = await self.middleware.call("nfs.config") if not nfs_config["v4"]: verrors.add(f"{schema_name}.security", "This is not allowed when NFS v4 is disabled") @private def validate_paths(self, data, schema_name, verrors): if osc.IS_LINUX: # Ganesha does not have such a restriction, each path is a different share return dev = None for i, path in enumerate(data["paths"]): stat = os.stat(path) if dev is None: dev = stat.st_dev else: if dev != stat.st_dev: verrors.add( f'{schema_name}.paths.{i}', 'Paths for a NFS share must reside within the same filesystem' ) @private async def resolve_hostnames(self, hostnames): hostnames = list(set(hostnames)) async def resolve(hostname): try: return (await asyncio.wait_for( self.middleware.run_in_thread(socket.getaddrinfo, hostname, None), 5))[0][4][0] except Exception as e: self.logger.warning("Unable to resolve host %r: %r", hostname, e) return None resolved_hostnames = await asyncio_map(resolve, hostnames, 8) return dict(zip(hostnames, resolved_hostnames)) @private def validate_hosts_and_networks(self, other_shares, data, schema_name, verrors, dns_cache): dev = os.stat(data["paths"][0]).st_dev used_networks = set() for share in other_shares: try: share_dev = os.stat(share["paths"][0]).st_dev except Exception: self.logger.warning("Failed to stat first path for %r", share, exc_info=True) continue if share_dev == dev: for host in share["hosts"]: host = dns_cache[host] if host is None: continue try: network = ipaddress.ip_network(host) except Exception: self.logger.warning("Got invalid host %r", host) continue else: used_networks.add(network) for network in share["networks"]: try: network = ipaddress.ip_network(network, strict=False) except Exception: self.logger.warning("Got invalid network %r", network) continue else: used_networks.add(network) if not share["hosts"] and not share["networks"]: used_networks.add(ipaddress.ip_network("0.0.0.0/0")) used_networks.add(ipaddress.ip_network("::/0")) for host in set(data["hosts"]): host = dns_cache[host] if host is None: continue network = ipaddress.ip_network(host) if network in used_networks: verrors.add( f"{schema_name}.hosts", f"Another NFS share already exports this dataset for {host}" ) used_networks.add(network) for network in set(data["networks"]): network = ipaddress.ip_network(network, strict=False) if network in used_networks: verrors.add( f"{schema_name}.networks", f"Another NFS share already exports this dataset for {network}" ) used_networks.add(network) if not data["hosts"] and not data["networks"]: if used_networks: verrors.add( f"{schema_name}.networks", "Another NFS share already exports this dataset for some network" ) @private async def extend(self, data): data["networks"] = data.pop("network").split() data["hosts"] = data["hosts"].split() data["security"] = [s.upper() for s in data["security"]] return data @private async def compress(self, data): data["network"] = " ".join(data.pop("networks")) data["hosts"] = " ".join(data["hosts"]) data["security"] = [s.lower() for s in data["security"]] data.pop(self.locked_field, None) return data
class KubernetesSnapshotService(CRUDService): GROUP = 'snapshot.storage.k8s.io' PLURAL = 'volumesnapshots' VERSION = 'v1' class Config: namespace = 'k8s.volume.snapshot' private = True @filterable async def query(self, filters, options): async with api_client() as (api, context): return filter_list( list( itertools.chain(*[[ d for d in (await context['custom_object_api']. list_namespaced_custom_object( group=self.GROUP, version=self.VERSION, plural=self.PLURAL, namespace=namespace))['items'] ] for namespace in await self.middleware.call( 'k8s.namespace.namespace_names')])), filters, options) @accepts( Dict( 'zfs_snapshot_create', Str('namespace', required=True), Dict( 'metadata', Str('name', required=True), additional_attrs=True, ), Dict( 'spec', Str('volumeSnapshotClassName', required=True), Dict( 'source', Str('persistentVolumeClaimName', required=True), ), additional_attrs=True, ), additional_attrs=True, )) async def do_create(self, data): data.update({ 'kind': 'VolumeSnapshot', 'apiVersion': f'snapshot.storage.k8s.io/{self.VERSION}' }) namespace = data.pop('namespace') verrors = ValidationErrors() if not await self.middleware.call( 'k8s.zfs.snapshotclass.query', [['metadata.name', '=', data['spec']['volumeSnapshotClassName']]]): verrors.add('zfs_snapshot_create.spec.volumeSnapshotClassName', 'Specified volumeSnapshotClassName does not exist.') if not await self.middleware.call('k8s.pvc.query', [[ 'metadata.name', '=', data['spec']['source']['persistentVolumeClaimName'] ], ['metadata.namespace', '=', namespace]]): verrors.add( 'zfs_snapshot_create.spec.source.persistentVolumeClaimName', f'Specified persistentVolumeClaimName does not exist in {namespace}.' ) verrors.check() async with api_client() as (api, context): await context['custom_object_api'].create_namespaced_custom_object( group=self.GROUP, version=self.VERSION, plural=self.PLURAL, namespace=namespace, body=data) @accepts(Str('snapshot_name'), Dict( 'zfs_snapshot_delete', Str('namespace', required=True), )) async def do_delete(self, snapshot_name, options): async with api_client() as (api, context): await context['custom_object_api'].delete_namespaced_custom_object( group=self.GROUP, version=self.VERSION, plural=self.PLURAL, namespace=options['namespace'], name=snapshot_name, )
class ACLTemplateService(CRUDService): class Config: cli_namespace = 'filesystem.acltemplate' datastore = 'filesystem.acltemplate' datastore_prefix = 'acltemplate_' namespace = 'filesystem.acltemplate' ENTRY = Patch( 'acltemplate_create', 'acltemplate_entry', ('add', Int('id')), ('add', Bool('builtin')), ) @private async def validate_acl(self, data, schema, verrors): acltype = ACLType[data['acltype']] aclcheck = acltype.validate({'dacl': data['acl']}) if not aclcheck['is_valid']: for err in aclcheck['errors']: if err[2]: v = f'{schema}.{err[0]}.{err[2]}' else: v = f'{schema}.{err[0]}' verrors.add(v, err[1]) if acltype is ACLType.POSIX1E: await self.middleware.call( "filesystem.gen_aclstring_posix1e", copy.deepcopy(data["acl"]), False, verrors ) for idx, ace in enumerate(data['acl']): if ace['id'] is None: verrors.add(f'{schema}.{idx}.id', 'null id is not permitted.') @accepts(Dict( "acltemplate_create", Str("name", required=True), Str("acltype", required=True, enum=["NFS4", "POSIX1E"]), OROperator(Ref('nfs4_acl'), Ref('posix1e_acl'), name='acl', required=True), register=True )) async def do_create(self, data): """ Create a new filesystem ACL template. """ verrors = ValidationErrors() if len(data['acl']) == 0: verrors.add( "filesystem_acltemplate_create.acl", "At least one ACL entry must be specified." ) await self.validate_acl(data, "filesystem_acltemplate_create.acl", verrors) verrors.check() data['builtin'] = False data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix} ) return await self._get_instance(data['id']) @accepts( Int('id'), Patch( 'acltemplate_create', 'acltemplate_update', ('attr', {'update': True}) ) ) async def do_update(self, id, data): """ update filesystem ACL template with `id`. """ old = await self._get_instance(id) new = old.copy() new.update(data) verrors = ValidationErrors() if old['builtin']: verrors.add("filesystem_acltemplate_update.builtin", "built-in ACL templates may not be changed") if new['name'] != old['name']: name_exists = bool(await self.query([('name', '=', new['name'])])) if name_exists: verrors.add("filesystem_acltemplate_update.name", f"{data['name']}: name is not unique") if len(new['acl']) == 0: verrors.add( "filesystem_acltemplate_update.acl", "At least one ACL entry must be specified." ) await self.validate_acl(new, "filesystem_acltemplate_update.acl", verrors) verrors.check() await self.middleware.call( 'datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix} ) return await self.get_instance(id) @accepts(Int('id')) async def do_delete(self, id): entry = await self.get_instance(id) if entry['builtin']: raise CallError("Deletion of builtin templates is not permitted", errno.EPERM) return await self.middleware.call( 'datastore.delete', self._config.datastore, id ) @private async def append_builtins(self, data): """ This method ensures that ACL grants some minimum level of permissions to our builtin users or builtin admins accounts. """ bu_id = int(SMBBuiltin.USERS.value[1][9:]) ba_id = int(SMBBuiltin.USERS.value[1][9:]) has_builtins = any(filter(lambda x: x["id"] in [bu_id, ba_id], data['acl'])) if has_builtins: return if data['acltype'] == ACLType.NFS4.name: data['acl'].extend([ {"tag": "GROUP", "id": bu_id, "perms": {"BASIC": "MODIFY"}, "flags": {"BASIC": "INHERIT"}, "type": "ALLOW"}, {"tag": "GROUP", "id": ba_id, "perms": {"BASIC": "FULL_CONTROL"}, "flags": {"BASIC": "INHERIT"}, "type": "ALLOW"}, ]) return has_default_mask = any(filter(lambda x: x["tag"] == "MASK" and x["default"], data['acl'])) has_access_mask = any(filter(lambda x: x["tag"] == "MASK" and x["default"], data['acl'])) all_perms = {"READ": True, "WRITE": True, "EXECUTE": True} data['acl'].extend([ {"tag": "GROUP", "id": bu_id, "perms": all_perms, "default": False}, {"tag": "GROUP", "id": bu_id, "perms": all_perms, "default": True}, {"tag": "GROUP", "id": ba_id, "perms": all_perms, "default": False}, {"tag": "GROUP", "id": ba_id, "perms": all_perms, "default": True}, ]) if not has_default_mask: data['acl'].append({"tag": "MASK", "id": -1, "perms": all_perms, "default": False}) if not has_access_mask: data['acl'].append({"tag": "MASK", "id": -1, "perms": all_perms, "default": True}) return @private async def resolve_names(self, uid, gid, data): for ace in data['acl']: if ace['id'] != -1: ace['who'] = await self.middleware.call( 'idmap.id_to_name', ace['id'], ace['tag'] ) elif ace['tag'] in ('group@', 'GROUP_OBJ'): ace['who'] = await self.middleware.call( 'idmap.id_to_name', gid, 'GROUP' ) elif ace['tag'] in ('owner@', 'USER_OBJ'): ace['who'] = await self.middleware.call( 'idmap.id_to_name', uid, 'USER' ) else: ace['who'] = None return @accepts(Dict( "acltemplate_by_path", Str("path", default=""), Ref('query-filters'), Ref('query-options'), Dict( "format-options", Bool("canonicalize", default=False), Bool("ensure_builtins", default=False), Bool("resolve_names", default=False), ) )) @returns(List( 'templates', items=[Ref('acltemplate_entry')] )) async def by_path(self, data): """ Retrieve list of available ACL templates for a given `path`. Supports `query-filters` and `query-options`. `format-options` gives additional options to alter the results of the template query: `canonicalize` - place ACL entries for NFSv4 ACLs in Microsoft canonical order. `ensure_builtins` - ensure all results contain entries for `builtin_users` and `builtin_administrators` groups. `resolve_names` - convert ids in ACL entries into names. """ verrors = ValidationErrors() filters = data.get('query-filters') if data['path']: path = await self.middleware.call( "filesystem.resolve_cluster_path", data['path'] ) acltype = await self.middleware.call( 'filesystem.path_get_acltype', path ) if acltype == ACLType.DISABLED.name: return [] if acltype == ACLType.POSIX1E.name and data['format-options']['canonicalize']: verrors.add( "filesystem.acltemplate_by_path.format-options.canonicalize", "POSIX1E ACLs may not be sorted into Windows canonical order." ) filters.append(("acltype", "=", acltype)) if not data['path'] and data['format-options']['resolve_names']: verrors.add( "filesystem.acltemplate_by_path.format-options.canonicalize", "ACL entry ids may not be resolved into names unless path is provided." ) verrors.check() templates = await self.query(filters, data['query-options']) for t in templates: if data['format-options']['ensure_builtins']: await self.append_builtins(t) if data['format-options']['resolve_names']: st = await self.middleware.run_in_thread(os.stat(path)) await self.resolve_names(st.st_uid, st.st_gid, t) if data['format-options']['canonicalize'] and t['acltype'] == ACLType.NFS4.name: canonicalized = ACLType[t['acltype']].canonicalize(t['acl']) t['acl'] = canonicalized return templates
class UpdateService(Service): @accepts() def get_trains(self): """ Returns available trains dict and the currently configured train as well as the train of currently booted environment. """ data = self.middleware.call_sync('datastore.config', 'system.update') conf = Configuration.Configuration() conf.LoadTrainsConfig() selected = None trains = {} for name, descr in (conf.AvailableTrains() or {}).items(): train = conf._trains.get(name) if train is None: train = Train.Train(name, descr) if not selected and data['upd_train'] == train.Name(): selected = data['upd_train'] trains[train.Name()] = { 'description': train.Description(), 'sequence': train.LastSequence(), } if not data['upd_train'] or not selected: selected = conf.CurrentTrain() return { 'trains': trains, 'current': conf.CurrentTrain(), 'selected': selected, } @accepts( Dict( 'update-check-available', Str('train', required=False), required=False, )) def check_available(self, attrs=None): """ Checks if there is an update available from update server. status: - REBOOT_REQUIRED: an update has already been applied - AVAILABLE: an update is available - UNAVAILABLE: no update available .. examples(websocket):: Check available update using default train: :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "update.check_available" } """ try: applied = self.middleware.call_sync('cache.get', 'update.applied') except Exception: applied = False if applied is True: return {'status': 'REBOOT_REQUIRED'} train = (attrs or {}).get('train') or self.middleware.call_sync( 'update.get_trains')['selected'] handler = CheckUpdateHandler() manifest = CheckForUpdates( diff_handler=handler.diff_call, handler=handler.call, train=train, ) if not manifest: return {'status': 'UNAVAILABLE'} data = { 'status': 'AVAILABLE', 'changes': handler.changes, 'notice': manifest.Notice(), 'notes': manifest.Notes(), } conf = Configuration.Configuration() sys_mani = conf.SystemManifest() if sys_mani: sequence = sys_mani.Sequence() else: sequence = '' data['changelog'] = get_changelog(train, start=sequence, end=manifest.Sequence()) data['version'] = manifest.Version() return data @accepts(Str('path')) async def get_pending(self, path=None): """ Gets a list of packages already downloaded and ready to be applied. Each entry of the lists consists of type of operation and name of it, e.g. { "operation": "upgrade", "name": "baseos-11.0 -> baseos-11.1" } """ if path is None: path = await self.middleware.call('notifier.get_update_location') data = [] try: changes = await self.middleware.run_in_thread( Update.PendingUpdatesChanges, path) except ( UpdateIncompleteCacheException, UpdateInvalidCacheException, UpdateBusyCacheException, ): changes = [] if changes: if changes.get("Reboot", True) is False: for svc in changes.get("Restart", []): data.append({ 'operation': svc, 'name': Update.GetServiceDescription(svc), }) for new, op, old in changes['Packages']: if op == 'upgrade': name = '%s-%s -> %s-%s' % ( old.Name(), old.Version(), new.Name(), new.Version(), ) elif op == 'install': name = '%s-%s' % (new.Name(), new.Version()) else: # Its unclear why "delete" would feel out new # instead of old, sounds like a pkgtools bug? if old: name = '%s-%s' % (old.Name(), old.Version()) else: name = '%s-%s' % (new.Name(), new.Version()) data.append({ 'operation': op, 'name': name, }) return data @accepts( Dict( 'update', Str('train', required=False), Bool('reboot', default=False), required=False, )) @job(lock='update', process=True) async def update(self, job, attrs=None): """ Downloads (if not already in cache) and apply an update. """ attrs = attrs or {} train = attrs.get('train') or ( await self.middleware.call('update.get_trains'))['selected'] location = await self.middleware.call('notifier.get_update_location') job.set_progress(0, 'Retrieving update manifest') handler = UpdateHandler(self, job) update = Update.DownloadUpdate( train, location, check_handler=handler.check_handler, get_handler=handler.get_handler, ) if update is False: raise ValueError('No update available') new_manifest = Manifest.Manifest(require_signature=True) new_manifest.LoadPath('{}/MANIFEST'.format(location)) Update.ApplyUpdate( location, install_handler=handler.install_handler, ) await self.middleware.call('cache.put', 'update.applied', True) if attrs.get('reboot'): await self.middleware.call('system.reboot', {'delay': 10}) return True @accepts() @job(lock='updatedownload') def download(self, job): train = self.middleware.call_sync('update.get_trains')['selected'] location = self.middleware.call_sync('notifier.get_update_location') job.set_progress(0, 'Retrieving update manifest') handler = UpdateHandler(self, job, 100) Update.DownloadUpdate( train, location, check_handler=handler.check_handler, get_handler=handler.get_handler, ) update = Update.CheckForUpdates(train=train, cache_dir=location) if not update: return False notified = False try: if self.middleware.call_sync('cache.has_key', 'update.notified'): notified = self.middleware.call_sync('cache.get', 'update.notified') except Exception: pass if not notified: self.middleware.call_sync('cache.put', 'update.notified', True) conf = Configuration.Configuration() sys_mani = conf.SystemManifest() if sys_mani: sequence = sys_mani.Sequence() else: sequence = '' changelog = get_changelog(train, start=sequence, end=update.Sequence()) hostname = socket.gethostname() try: # FIXME: Translation self.middleware.call_sync( 'mail.send', { 'subject': '{}: {}'.format(hostname, 'Update Available'), 'text': '''A new update is available for the %(train)s train. Version: %(version)s Changelog: %(changelog)s ''' % { 'train': train, 'version': update.Version(), 'changelog': changelog, }, }).wait_sync() except Exception: self.logger.warn('Failed to send email about new update', exc_info=True) return True @accepts(Str('path')) @job(lock='updatemanual', process=True) async def manual(self, job, path): """ Apply manual update of file `path`. """ rv = await self.middleware.call('notifier.validate_update', path) if not rv: raise CallError('Invalid update file', errno.EINVAL) await self.middleware.call('notifier.apply_update', path, timeout=None) try: await self.middleware.call('notifier.destroy_upload_location') except Exception: self.logger.warn('Failed to destroy upload location', exc_info=True) @accepts(Dict( 'updatefile', Str('destination'), )) @job(lock='updatemanual', pipes=['input']) async def file(self, job, options): """ Updates the system using the uploaded .tar file. Use null `destination` to create a temporary location. """ dest = options.get('destination') if not dest: try: await self.middleware.call('notifier.create_upload_location') dest = '/var/tmp/firmware' except Exception as e: raise CallError(str(e)) elif not dest.startswith('/mnt/'): raise CallError('Destination must reside within a pool') if not os.path.isdir(dest): raise CallError('Destination is not a directory') destfile = os.path.join(dest, 'manualupdate.tar') dest_extracted = os.path.join(dest, '.update') try: job.set_progress(10, 'Writing uploaded file to disk') with open(destfile, 'wb') as f: await self.middleware.run_in_io_thread( shutil.copyfileobj, job.pipes.input.r, f, 1048576, ) def do_update(): try: job.set_progress(30, 'Extracting uploaded file') ExtractFrozenUpdate(destfile, dest_extracted, verbose=True) job.set_progress(50, 'Applying update') ApplyUpdate(dest_extracted) except Exception as e: raise CallError(str(e)) await self.middleware.run_in_io_thread(do_update) job.set_progress(95, 'Cleaning up') finally: if os.path.exists(destfile): os.unlink(destfile) if os.path.exists(dest_extracted): shutil.rmtree(dest_extracted, ignore_errors=True) if dest == '/var/tmp/firmware': await self.middleware.call('notifier.destroy_upload_location') job.set_progress(100, 'Update completed')
class RsyncTaskService(CRUDService): class Config: datastore = 'tasks.rsync' datastore_prefix = 'rsync_' datastore_extend = 'rsynctask.rsync_task_extend' @private async def rsync_task_extend(self, data): data['extra'] = list(filter(None, re.split(r"\s+", data["extra"]))) for field in ('mode', 'direction'): data[field] = data[field].upper() Cron.convert_db_format_to_schedule(data) return data @private async def validate_rsync_task(self, data, schema): verrors = ValidationErrors() # Windows users can have spaces in their usernames # http://www.freebsd.org/cgi/query-pr.cgi?pr=164808 username = data.get('user') if ' ' in username: verrors.add(f'{schema}.user', 'User names cannot have spaces') raise verrors user = None with contextlib.suppress(KeyError): user = await self.middleware.call('dscache.get_uncached_user', username) if not user: verrors.add(f'{schema}.user', f'Provided user "{username}" does not exist') raise verrors remote_host = data.get('remotehost') if not remote_host: verrors.add(f'{schema}.remotehost', 'Please specify a remote host') if data.get('extra'): data['extra'] = ' '.join(data['extra']) else: data['extra'] = '' mode = data.get('mode') if not mode: verrors.add(f'{schema}.mode', 'This field is required') remote_module = data.get('remotemodule') if mode == 'MODULE' and not remote_module: verrors.add(f'{schema}.remotemodule', 'This field is required') if mode == 'SSH': remote_port = data.get('remoteport') if not remote_port: verrors.add(f'{schema}.remoteport', 'This field is required') remote_path = data.get('remotepath') if not remote_path: verrors.add(f'{schema}.remotepath', 'This field is required') search = os.path.join(user['pw_dir'], '.ssh', 'id_[edr]*') exclude_from_search = os.path.join(user['pw_dir'], '.ssh', 'id_[edr]*pub') key_files = set(glob.glob(search)) - set( glob.glob(exclude_from_search)) if not key_files: verrors.add( f'{schema}.user', 'In order to use rsync over SSH you need a user' ' with a private key (DSA/ECDSA/RSA) set up in home dir.') else: for file in glob.glob(search): if '.pub' not in file: # file holds a private key and it's permissions should be 600 if os.stat(file).st_mode & 0o077 != 0: verrors.add( f'{schema}.user', f'Permissions {oct(os.stat(file).st_mode & 0o777)} for {file} are too open. Please ' f'correct them by running chmod 600 {file}') if (data.get('validate_rpath') and remote_path and remote_host and remote_port): if '@' in remote_host: remote_username, remote_host = remote_host.rsplit('@', 1) else: remote_username = username try: with (await asyncio.wait_for(asyncssh.connect( remote_host, port=remote_port, username=remote_username, client_keys=key_files, known_hosts=None), timeout=5)) as conn: await conn.run(f'test -d {shlex.quote(remote_path)}', check=True) except asyncio.TimeoutError: verrors.add( f'{schema}.remotehost', 'SSH timeout occurred. Remote path cannot be validated.' ) except OSError as e: if e.errno == 113: verrors.add( f'{schema}.remotehost', f'Connection to the remote host {remote_host} on port {remote_port} failed.' ) else: verrors.add(f'{schema}.remotehost', e.__str__()) except asyncssh.DisconnectError as e: verrors.add( f'{schema}.remotehost', f'Disconnect Error[ error code {e.code} ] was generated when trying to ' f'communicate with remote host {remote_host} and remote user {remote_username}.' ) except asyncssh.ProcessError as e: if e.code == '1': verrors.add( f'{schema}.remotepath', 'The Remote Path you specified does not exist or is not a directory.' 'Either create one yourself on the remote machine or uncheck the ' 'validate_rpath field') else: verrors.add( f'{schema}.remotepath', f'Connection to Remote Host was successful but failed to verify ' f'Remote Path. {e.__str__()}') except asyncssh.Error as e: if e.__class__.__name__ in e.__str__(): exception_reason = e.__str__() else: exception_reason = e.__class__.__name__ + ' ' + e.__str__( ) verrors.add( f'{schema}.remotepath', f'Remote Path could not be validated. An exception was raised. {exception_reason}' ) elif data.get('validate_rpath'): verrors.add( f'{schema}.remotepath', 'Remote path could not be validated because of missing fields' ) data.pop('validate_rpath', None) # Keeping compatibility with legacy UI for field in ('mode', 'direction'): data[field] = data[field].lower() return verrors, data @accepts( Dict( 'rsync_task_create', Str('path', required=True), Str('user', required=True), Str('remotehost'), Int('remoteport'), Str('mode', enum=['MODULE', 'SSH'], default='MODULE'), Str('remotemodule'), Str('remotepath'), Bool('validate_rpath'), Str('direction', enum=['PULL', 'PUSH'], default='PUSH'), Str('desc'), Cron( 'schedule', defaults={'minute': '00'}, ), Bool('recursive'), Bool('times'), Bool('compress'), Bool('archive'), Bool('delete'), Bool('quiet'), Bool('preserveperm'), Bool('preserveattr'), Bool('delayupdates'), List('extra', items=[Str('extra')]), Bool('enabled'), register=True, )) async def do_create(self, data): """ Create a Rsync Task. `path` represents the path to pool/dataset. `remotehost` is ip address or hostname of the remote system. If username differs on the remote host, "username@remote_host" format should be used. `mode` represents different operating mechanisms for Rsync i.e Rsync Module mode / Rsync SSH mode. `remotemodule` is the name of remote module, this attribute should be specified when `mode` is set to MODULE. `remotepath` specifies the path on the remote system. `validate_rpath` is a boolean which when sets validates the existence of the remote path. `direction` specifies if data should be PULLED or PUSHED from the remote system. `compress` when set reduces the size of the data which is to be transmitted. `archive` when set makes rsync run recursively, preserving symlinks, permissions, modification times, group, and special files. `delete` when set deletes files in the destination directory which do not exist in the source directory. `preserveperm` when set preserves original file permissions. .. examples(websocket):: Create a Rsync Task which pulls data from a remote system every 5 minutes. :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "rsynctask.create", "params": [{ "enabled": true, "schedule": { "minute": "5", "hour": "*", "dom": "*", "month": "*", "dow": "*" }, "desc": "Test rsync task", "user": "******", "mode": "MODULE", "remotehost": "[email protected]", "compress": true, "archive": true, "direction": "PULL", "path": "/mnt/vol1/rsync_dataset", "remotemodule": "remote_module1" }] } """ verrors, data = await self.validate_rsync_task(data, 'rsync_task_create') if verrors: raise verrors Cron.convert_schedule_to_db_format(data) data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self.middleware.call('service.restart', 'cron') return await self._get_instance(data['id']) @accepts(Int('id', validators=[Range(min=1)]), Patch('rsync_task_create', 'rsync_task_update', ('attr', { 'update': True }))) async def do_update(self, id, data): """ Update Rsync Task of `id`. """ old = await self.query(filters=[('id', '=', id)], options={'get': True}) new = old.copy() new.update(data) verrors, data = await self.validate_rsync_task(new, 'rsync_task_update') if verrors: raise verrors Cron.convert_schedule_to_db_format(new) await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) await self.middleware.call('service.restart', 'cron') return await self.query(filters=[('id', '=', id)], options={'get': True}) @accepts(Int('id')) async def do_delete(self, id): """ Delete Rsync Task of `id`. """ res = await self.middleware.call('datastore.delete', self._config.datastore, id) await self.middleware.call('service.restart', 'cron') return res @private async def commandline(self, id): """ Helper method to generate the rsync command avoiding code duplication. """ rsync = await self._get_instance(id) path = shlex.quote(rsync['path']) line = [ '/usr/bin/lockf', '-s', '-t', '0', '-k', path, '/usr/local/bin/rsync' ] for name, flag in ( ('archive', '-a'), ('compress', '-z'), ('delayupdates', '--delay-updates'), ('delete', '--delete-delay'), ('preserveattr', '-X'), ('preserveperm', '-p'), ('recursive', '-r'), ('times', '-t'), ): if rsync[name]: line.append(flag) if rsync['extra']: line.append(' '.join(rsync['extra'])) # Do not use username if one is specified in host field # See #5096 for more details if '@' in rsync['remotehost']: remote = rsync['remotehost'] else: remote = f'"{rsync["user"]}"@{rsync["remotehost"]}' if rsync['mode'] == 'MODULE': module_args = [path, f'{remote}::"{rsync["remotemodule"]}"'] if rsync['direction'] != 'PUSH': module_args.reverse() line += module_args else: line += [ '-e', f'"ssh -p {rsync["remoteport"]} -o BatchMode=yes -o StrictHostKeyChecking=yes"' ] path_args = [ path, f'{remote}:"{shlex.quote(rsync["remotepath"])}"' ] if rsync['direction'] != 'PUSH': path_args.reverse() line += path_args if rsync['quiet']: line += ['>', '/dev/null', '2>&1'] return ' '.join(line) @item_method @accepts(Int('id')) @job(lock=lambda args: args[-1], logs=True) def run(self, job, id): """ Job to run rsync task of `id`. Output is saved to job log excerpt as well as syslog. """ rsync = self.middleware.call_sync('rsynctask._get_instance', id) commandline = self.middleware.call_sync('rsynctask.commandline', id) cp = run_command_with_user_context(commandline, rsync['user'], lambda v: job.logs_fd.write(v)) for klass in ('RsyncSuccess', 'RsyncFailed') if not rsync['quiet'] else (): self.middleware.call_sync('alert.oneshot_delete', klass, rsync['id']) if cp.returncode != 0: if not rsync['quiet']: self.middleware.call_sync('alert.oneshot_create', 'RsyncFailed', rsync) raise CallError( f'rsync command returned {cp.returncode}. Check logs for further information.' ) elif not rsync['quiet']: self.middleware.call_sync('alert.oneshot_create', 'RsyncSuccess', rsync)
class SharingAFPService(CRUDService): class Config: namespace = 'sharing.afp' datastore = 'sharing.afp_share' datastore_prefix = 'afp_' datastore_extend = 'sharing.afp.extend' @accepts(Dict( 'sharingafp_create', Str('path', required=True), Bool('home', default=False), Str('name'), Str('comment'), List('allow', default=[]), List('deny', default=[]), List('ro', default=[]), List('rw', default=[]), Bool('timemachine', default=False), Int('timemachine_quota', default=0), Bool('nodev', default=False), Bool('nostat', default=False), Bool('upriv', default=True), UnixPerm('fperm', default='644'), UnixPerm('dperm', default='755'), UnixPerm('umask', default='000'), List('hostsallow', items=[], default=[]), List('hostsdeny', items=[], default=[]), Str('vuid', null=True, default=''), Str('auxparams', max_length=None), Bool('enabled', default=True), register=True )) async def do_create(self, data): """ Create AFP share. `allow`, `deny`, `ro`, and `rw` are lists of users and groups. Groups are designated by an @ prefix. `hostsallow` and `hostsdeny` are lists of hosts and/or networks. """ verrors = ValidationErrors() path = data['path'] await self.clean(data, 'sharingafp_create', verrors) await self.validate(data, 'sharingafp_create', verrors) await check_path_resides_within_volume( verrors, self.middleware, 'sharingafp_create.path', path) verrors.check() if path and not os.path.exists(path): try: os.makedirs(path) except OSError as e: raise CallError(f'Failed to create {path}: {e}') await self.compress(data) data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self.extend(data) await self._service_change('afp', 'reload') return data @accepts( Int('id'), Patch( 'sharingafp_create', 'sharingafp_update', ('attr', {'update': True}) ) ) async def do_update(self, id, data): """ Update AFP share `id`. """ verrors = ValidationErrors() old = await self.middleware.call( 'datastore.query', self._config.datastore, [('id', '=', id)], {'extend': self._config.datastore_extend, 'prefix': self._config.datastore_prefix, 'get': True}) path = data.get('path') new = old.copy() new.update(data) await self.clean(new, 'sharingafp_update', verrors, id=id) await self.validate(new, 'sharingafp_update', verrors, old=old) if path: await check_path_resides_within_volume( verrors, self.middleware, 'sharingafp_create.path', path) if verrors: raise verrors if path and not os.path.exists(path): try: os.makedirs(path) except OSError as e: raise CallError(f'Failed to create {path}: {e}') await self.compress(new) await self.middleware.call( 'datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) await self.extend(new) await self._service_change('afp', 'reload') return new @accepts(Int('id')) async def do_delete(self, id): """ Delete AFP share `id`. """ result = await self.middleware.call('datastore.delete', self._config.datastore, id) await self._service_change('afp', 'reload') return result @private async def clean(self, data, schema_name, verrors, id=None): data['name'] = await self.name_exists(data, schema_name, verrors, id) @private async def validate(self, data, schema_name, verrors, old=None): await self.home_exists(data['home'], schema_name, verrors, old) if data['vuid']: try: uuid.UUID(data['vuid'], version=4) except ValueError: verrors.add(f'{schema_name}.vuid', 'vuid must be a valid UUID.') @private async def home_exists(self, home, schema_name, verrors, old=None): home_filters = [('home', '=', True)] home_result = None if home: if old and old['id'] is not None: id = old['id'] if not old['home']: home_filters.append(('id', '!=', id)) # The user already had this set as the home share home_result = await self.middleware.call( 'datastore.query', self._config.datastore, home_filters, {'prefix': self._config.datastore_prefix}) if home_result: verrors.add(f'{schema_name}.home', 'Only one share is allowed to be a home share.') @private async def name_exists(self, data, schema_name, verrors, id=None): name = data['name'] path = data['path'] home = data['home'] name_filters = [('name', '=', name)] path_filters = [('path', '=', path)] if not name: if home: name = 'Homes' else: name = path.rsplit('/', 1)[-1] if id is not None: name_filters.append(('id', '!=', id)) path_filters.append(('id', '!=', id)) name_result = await self.middleware.call( 'datastore.query', self._config.datastore, name_filters, {'prefix': self._config.datastore_prefix}) path_result = await self.middleware.call( 'datastore.query', self._config.datastore, path_filters, {'prefix': self._config.datastore_prefix}) if name_result: verrors.add(f'{schema_name}.name', 'A share with this name already exists.') if path_result: verrors.add(f'{schema_name}.path', 'A share with this path already exists.') return name @private async def extend(self, data): data['allow'] = data['allow'].split() data['deny'] = data['deny'].split() data['ro'] = data['ro'].split() data['rw'] = data['rw'].split() data['hostsallow'] = data['hostsallow'].split() data['hostsdeny'] = data['hostsdeny'].split() return data @private async def compress(self, data): data['allow'] = ' '.join(data['allow']) data['deny'] = ' '.join(data['deny']) data['ro'] = ' '.join(data['ro']) data['rw'] = ' '.join(data['rw']) data['hostsallow'] = ' '.join(data['hostsallow']) data['hostsdeny'] = ' '.join(data['hostsdeny']) if not data['vuid'] and data['timemachine']: data['vuid'] = str(uuid.uuid4()) return data
class RsyncModService(CRUDService): class Config: datastore = 'services.rsyncmod' datastore_prefix = 'rsyncmod_' datastore_extend = 'rsyncmod.rsync_mod_extend' @private async def rsync_mod_extend(self, data): data['hostsallow'] = data['hostsallow'].split() data['hostsdeny'] = data['hostsdeny'].split() data['mode'] = data['mode'].upper() return data @private async def common_validation(self, data, schema_name): verrors = ValidationErrors() await check_path_resides_within_volume(verrors, self.middleware, f'{schema_name}.path', data.get('path')) for entity in ('user', 'group'): value = data.get(entity) if value not in map( lambda e: e[entity if entity == 'group' else 'username'], await self.middleware.call(f'{entity}.query')): verrors.add(f'{schema_name}.{entity}', f'Please specify a valid {entity}') verrors.check() data['hostsallow'] = ' '.join(data['hostsallow']) data['hostsdeny'] = ' '.join(data['hostsdeny']) data['mode'] = data['mode'].lower() return data @accepts( Dict( 'rsyncmod_create', Str('name', validators=[Match(r'[^/\]]')]), Str('comment'), Str('path', required=True), Str('mode', enum=['RO', 'RW', 'WO']), Int('maxconn'), Str('user', default='nobody'), Str('group', default='nobody'), List('hostsallow', items=[Str('hostsallow')], default=[]), List('hostsdeny', items=[Str('hostdeny')], default=[]), Str('auxiliary', max_length=None), register=True, )) async def do_create(self, data): """ Create a Rsyncmod module. `path` represents the path to pool/dataset. `maxconn` is an integer value representing the maximum number of simultaneous connections. Zero represents unlimited. `hostsallow` is a list of patterns to match hostname/ip address of a connecting client. If list is empty, all hosts are allowed. `hostsdeny` is a list of patterns to match hostname/ip address of a connecting client. If the pattern is matched, access is denied to the client. If no client should be denied, this should be left empty. `auxiliary` attribute can be used to pass on any additional parameters from rsyncd.conf(5). """ data = await self.common_validation(data, 'rsyncmod_create') data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self._service_change('rsync', 'reload') return await self._get_instance(data['id']) @accepts(Int('id'), Patch('rsyncmod_create', 'rsyncmod_update', ('attr', { 'update': True }))) async def do_update(self, id, data): """ Update Rsyncmod module of `id`. """ module = await self._get_instance(id) module.update(data) module = await self.common_validation(module, 'rsyncmod_update') await self.middleware.call('datastore.update', self._config.datastore, id, module, {'prefix': self._config.datastore_prefix}) await self._service_change('rsync', 'reload') return await self._get_instance(id) @accepts(Int('id')) async def do_delete(self, id): """ Delete Rsyncmod module of `id`. """ return await self.middleware.call('datastore.delete', self._config.datastore, id)
class AFPService(SystemServiceService): class Config: service = 'afp' datastore_extend = 'afp.extend' datastore_prefix = 'afp_srv_' @private async def extend(self, afp): for i in ('map_acls', 'chmod_request'): afp[i] = afp[i].upper() return afp @private async def compress(self, afp): for i in ('map_acls', 'chmod_request'): value = afp.get(i) if value: afp[i] = value.lower() return afp @accepts(Dict( 'afp_update', Bool('guest'), Str('guest_user'), List('bindip', items=[Str('ip', validators=[IpAddress()])]), Int('connections_limit', validators=[Range(min=1, max=65535)]), Dir('dbpath'), Str('global_aux', max_length=None), Str('map_acls', enum=['RIGHTS', 'MODE', 'NONE']), Str('chmod_request', enum=['PRESERVE', 'SIMPLE', 'IGNORE']), Str('loglevel', enum=[x.name for x in AFPLogLevel]), update=True )) async def do_update(self, data): """ Update AFP service settings. `bindip` is a list of IPs to bind AFP to. Leave blank (empty list) to bind to all available IPs. `map_acls` defines how to map the effective permissions of authenticated users. RIGHTS - Unix-style permissions MODE - ACLs NONE - Do not map `chmod_request` defines advanced permission control that deals with ACLs. PRESERVE - Preserve ZFS ACEs for named users and groups or POSIX ACL group mask SIMPLE - Change permission as requested without any extra steps IGNORE - Permission change requests are ignored """ old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() if new['dbpath']: await check_path_resides_within_volume( verrors, self.middleware, 'afp_update.dbpath', new['dbpath'], ) verrors.check() new = await self.compress(new) await self._update_service(old, new) return await self.config() @accepts() async def bindip_choices(self): """ List of valid choices for IP addresses to which to bind the AFP service. """ return { d['address']: d['address'] for d in await self.middleware.call('interface.ip_in_use') }
class CronJobService(CRUDService): class Config: datastore = 'tasks.cronjob' datastore_prefix = 'cron_' datastore_extend = 'cronjob.cron_extend' namespace = 'cronjob' @private def cron_extend(self, data): Cron.convert_db_format_to_schedule(data) return data @private async def validate_data(self, data, schema): verrors = ValidationErrors() user = data.get('user') if user: # Windows users can have spaces in their usernames # http://www.freebsd.org/cgi/query-pr.cgi?pr=164808 if ' ' in user: verrors.add(f'{schema}.user', 'Usernames cannot have spaces') elif not (await self.middleware.call('notifier.get_user_object', user)): verrors.add(f'{schema}.user', 'Specified user does not exist') return verrors, data @accepts( Dict('cron_job_create', Bool('enabled'), Bool('stderr'), Bool('stdout'), Cron('schedule'), Str('command', required=True), Str('description'), Str('user', required=True), register=True)) async def do_create(self, data): """ Create a new cron job. `stderr` and `stdout` are boolean values which if `true`, represent that we would like to suppress standard error / standard output respectively. .. examples(websocket):: Create a cron job which executes `touch /tmp/testfile` after every 5 minutes. :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "cronjob.create", "params": [{ "enabled": true, "schedule": { "minute": "5", "hour": "*", "dom": "*", "month": "*", "dow": "*" }, "command": "touch /tmp/testfile", "description": "Test command", "user": "******", "stderr": true, "stdout": true }] } """ verrors, data = await self.validate_data(data, 'cron_job_create') if verrors: raise verrors Cron.convert_schedule_to_db_format(data) data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self.middleware.call('service.restart', 'cron') return data @accepts(Int('id', validators=[Range(min=1)]), Patch('cron_job_create', 'cron_job_update', ('attr', { 'update': True }))) async def do_update(self, id, data): """ Update cronjob of `id`. """ task_data = await self.query(filters=[('id', '=', id)], options={'get': True}) original_data = task_data.copy() task_data.update(data) verrors, task_data = await self.validate_data(task_data, 'cron_job_update') if verrors: raise verrors Cron.convert_schedule_to_db_format(task_data) Cron.convert_schedule_to_db_format(original_data) if len(set(task_data.items()) ^ set(original_data.items())) > 0: await self.middleware.call( 'datastore.update', self._config.datastore, id, task_data, {'prefix': self._config.datastore_prefix}) await self.middleware.call('service.restart', 'cron') return await self.query(filters=[('id', '=', id)], options={'get': True}) @accepts(Int('id')) async def do_delete(self, id): """ Delete cronjob of `id`. """ response = await self.middleware.call('datastore.delete', self._config.datastore, id) await self.middleware.call('service.restart', 'cron') return response
class SSHService(SystemServiceService): class Config: service = "ssh" service_model = "ssh" datastore_prefix = "ssh_" @accepts() def bindiface_choices(self): """ Available choices for the bindiface attribute of SSH service. """ return self.middleware.call_sync('interface.choices') @accepts( Dict('ssh_update', List('bindiface', items=[Str('iface')]), Int('tcpport', validators=[Range(min=1, max=65535)]), Bool('rootlogin'), Bool('passwordauth'), Bool('kerberosauth'), Bool('tcpfwd'), Bool('compression'), Str('sftp_log_level', enum=[ "", "QUIET", "FATAL", "ERROR", "INFO", "VERBOSE", "DEBUG", "DEBUG2", "DEBUG3" ]), Str('sftp_log_facility', enum=[ "", "DAEMON", "USER", "AUTH", "LOCAL0", "LOCAL1", "LOCAL2", "LOCAL3", "LOCAL4", "LOCAL5", "LOCAL6", "LOCAL7" ]), List('weak_ciphers', items=[Str('cipher', enum=['AES128-CBC', 'NONE'])]), Str('options', max_length=None), update=True)) async def do_update(self, data): """ Update settings of SSH daemon service. If `bindiface` is empty it will listen for all available addresses. .. examples(websocket):: Make sshd listen only to igb0 interface. :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "ssh.update", "params": [{ "bindiface": ["igb0"] }] } """ old = await self.config() new = old.copy() new.update(data) if new['bindiface']: verrors = ValidationErrors() iface_choices = await self.middleware.call('ssh.bindiface_choices') invalid_ifaces = list( filter(lambda x: x not in iface_choices, new['bindiface'])) if invalid_ifaces: verrors.add( 'ssh_update.bindiface', f'The following interfaces are not valid: {", ".join(invalid_ifaces)}', ) verrors.check() await self._update_service(old, new) keyfile = "/usr/local/etc/ssh/ssh_host_ecdsa_key.pub" if os.path.exists(keyfile): with open(keyfile, "rb") as f: pubkey = f.read().strip().split(None, 3)[1] decoded_key = base64.b64decode(pubkey) key_digest = hashlib.sha256(decoded_key).digest() ssh_fingerprint = (b"SHA256:" + base64.b64encode(key_digest).replace( b"=", b"")).decode("utf-8") syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER) syslog.syslog( syslog.LOG_ERR, 'ECDSA Fingerprint of the SSH KEY: ' + ssh_fingerprint) syslog.closelog() return new @private def save_keys(self): update = {} for i in [ "ssh_host_key", "ssh_host_key.pub", "ssh_host_dsa_key", "ssh_host_dsa_key.pub", "ssh_host_dsa_key-cert.pub", "ssh_host_ecdsa_key", "ssh_host_ecdsa_key.pub", "ssh_host_ecdsa_key-cert.pub", "ssh_host_rsa_key", "ssh_host_rsa_key.pub", "ssh_host_rsa_key-cert.pub", "ssh_host_ed25519_key", "ssh_host_ed25519_key.pub", "ssh_host_ed25519_key-cert.pub", ]: path = os.path.join("/usr/local/etc/ssh", i) if os.path.exists(path): with open(path, "rb") as f: data = base64.b64encode(f.read()).decode("ascii") column = i.replace( ".", "_", ).replace("-", "_") update[column] = data old = self.middleware.call_sync('ssh.config') self.middleware.call_sync('datastore.update', 'services.ssh', old['id'], update)
class RsyncTaskService(TaskPathService): share_task_type = 'Rsync' class Config: datastore = 'tasks.rsync' datastore_prefix = 'rsync_' datastore_extend = 'rsynctask.rsync_task_extend' datastore_extend_context = 'rsynctask.rsync_task_extend_context' cli_namespace = 'task.rsync' ENTRY = Patch( 'rsync_task_create', 'rsync_task_entry', ('rm', { 'name': 'validate_rpath' }), ('add', Int('id')), ('add', Bool('locked')), ('add', Dict('job', null=True, additional_attrs=True)), ) @private async def rsync_task_extend(self, data, context): try: data['extra'] = shlex.split(data['extra'].replace('"', r'"\"').replace( "'", r'"\"')) except ValueError: # This is to handle the case where the extra value is misconfigured for old cases # Moving on, we are going to verify that it can be split successfully using shlex data['extra'] = data['extra'].split() for field in ('mode', 'direction'): data[field] = data[field].upper() Cron.convert_db_format_to_schedule(data) data['job'] = context['jobs'].get(data['id']) return data @private async def rsync_task_extend_context(self, rows, extra): jobs = {} for j in await self.middleware.call("core.get_jobs", [("method", "=", "rsynctask.run")], {"order_by": ["id"]}): try: task_id = int(j["arguments"][0]) except (IndexError, ValueError): continue if task_id in jobs and jobs[task_id]["state"] == "RUNNING": continue jobs[task_id] = j return { "jobs": jobs, } @private async def validate_rsync_task(self, data, schema): verrors = ValidationErrors() # Windows users can have spaces in their usernames # http://www.freebsd.org/cgi/query-pr.cgi?pr=164808 username = data.get('user') if ' ' in username: verrors.add(f'{schema}.user', 'User names cannot have spaces') raise verrors user = None with contextlib.suppress(KeyError): user = await self.middleware.call('dscache.get_uncached_user', username) if not user: verrors.add(f'{schema}.user', f'Provided user "{username}" does not exist') raise verrors await self.validate_path_field(data, schema, verrors) remote_host = data.get('remotehost') if not remote_host: verrors.add(f'{schema}.remotehost', 'Please specify a remote host') data['extra'] = ' '.join(data['extra']) try: shlex.split(data['extra'].replace('"', r'"\"').replace("'", r'"\"')) except ValueError as e: verrors.add(f'{schema}.extra', f'Please specify valid value: {e}') mode = data.get('mode') if not mode: verrors.add(f'{schema}.mode', 'This field is required') remote_module = data.get('remotemodule') if mode == 'MODULE' and not remote_module: verrors.add(f'{schema}.remotemodule', 'This field is required') if mode == 'SSH': remote_port = data.get('remoteport') if not remote_port: verrors.add(f'{schema}.remoteport', 'This field is required') remote_path = data.get('remotepath') if not remote_path: verrors.add(f'{schema}.remotepath', 'This field is required') search = os.path.join(user['pw_dir'], '.ssh', 'id_[edr]*') exclude_from_search = os.path.join(user['pw_dir'], '.ssh', 'id_[edr]*pub') key_files = set(glob.glob(search)) - set( glob.glob(exclude_from_search)) if not key_files: verrors.add( f'{schema}.user', 'In order to use rsync over SSH you need a user' ' with a private key (DSA/ECDSA/RSA) set up in home dir.') else: for file in glob.glob(search): if '.pub' not in file: # file holds a private key and it's permissions should be 600 if os.stat(file).st_mode & 0o077 != 0: verrors.add( f'{schema}.user', f'Permissions {oct(os.stat(file).st_mode & 0o777)} for {file} are too open. Please ' f'correct them by running chmod 600 {file}') if (data['enabled'] and data['validate_rpath'] and remote_path and remote_host and remote_port): if '@' in remote_host: remote_username, remote_host = remote_host.rsplit('@', 1) else: remote_username = username try: async with await asyncio.wait_for( asyncssh.connect(remote_host, port=remote_port, username=remote_username, client_keys=key_files, known_hosts=None), timeout=5, ) as conn: await conn.run(f'test -d {shlex.quote(remote_path)}', check=True) except asyncio.TimeoutError: verrors.add( f'{schema}.remotehost', 'SSH timeout occurred. Remote path cannot be validated.' ) except OSError as e: if e.errno == 113: verrors.add( f'{schema}.remotehost', f'Connection to the remote host {remote_host} on port {remote_port} failed.' ) else: verrors.add(f'{schema}.remotehost', e.__str__()) except asyncssh.DisconnectError as e: verrors.add( f'{schema}.remotehost', f'Disconnect Error[ error code {e.code} ] was generated when trying to ' f'communicate with remote host {remote_host} and remote user {remote_username}.' ) except asyncssh.ProcessError as e: if e.code == '1': verrors.add( f'{schema}.remotepath', 'The Remote Path you specified does not exist or is not a directory.' 'Either create one yourself on the remote machine or uncheck the ' 'validate_rpath field') else: verrors.add( f'{schema}.remotepath', f'Connection to Remote Host was successful but failed to verify ' f'Remote Path. {e.__str__()}') except asyncssh.Error as e: if e.__class__.__name__ in e.__str__(): exception_reason = e.__str__() else: exception_reason = e.__class__.__name__ + ' ' + e.__str__( ) verrors.add( f'{schema}.remotepath', f'Remote Path could not be validated. An exception was raised. {exception_reason}' ) elif data['enabled'] and data['validate_rpath']: verrors.add( f'{schema}.remotepath', 'Remote path could not be validated because of missing fields' ) data.pop('validate_rpath', None) # Keeping compatibility with legacy UI for field in ('mode', 'direction'): data[field] = data[field].lower() return verrors, data @accepts( Dict( 'rsync_task_create', Str('path', required=True, max_length=RSYNC_PATH_LIMIT), Str('user', required=True), Str('remotehost'), Int('remoteport'), Str('mode', enum=['MODULE', 'SSH'], default='MODULE'), Str('remotemodule'), Str('remotepath'), Bool('validate_rpath', default=True), Str('direction', enum=['PULL', 'PUSH'], default='PUSH'), Str('desc'), Cron( 'schedule', defaults={'minute': '00'}, ), Bool('recursive'), Bool('times'), Bool('compress'), Bool('archive'), Bool('delete'), Bool('quiet'), Bool('preserveperm'), Bool('preserveattr'), Bool('delayupdates'), List('extra', items=[Str('extra')]), Bool('enabled', default=True), register=True, )) async def do_create(self, data): """ Create a Rsync Task. See the comment in Rsyncmod about `path` length limits. `remotehost` is ip address or hostname of the remote system. If username differs on the remote host, "username@remote_host" format should be used. `mode` represents different operating mechanisms for Rsync i.e Rsync Module mode / Rsync SSH mode. `remotemodule` is the name of remote module, this attribute should be specified when `mode` is set to MODULE. `remotepath` specifies the path on the remote system. `validate_rpath` is a boolean which when sets validates the existence of the remote path. `direction` specifies if data should be PULLED or PUSHED from the remote system. `compress` when set reduces the size of the data which is to be transmitted. `archive` when set makes rsync run recursively, preserving symlinks, permissions, modification times, group, and special files. `delete` when set deletes files in the destination directory which do not exist in the source directory. `preserveperm` when set preserves original file permissions. .. examples(websocket):: Create a Rsync Task which pulls data from a remote system every 5 minutes. :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "rsynctask.create", "params": [{ "enabled": true, "schedule": { "minute": "5", "hour": "*", "dom": "*", "month": "*", "dow": "*" }, "desc": "Test rsync task", "user": "******", "mode": "MODULE", "remotehost": "[email protected]", "compress": true, "archive": true, "direction": "PULL", "path": "/mnt/vol1/rsync_dataset", "remotemodule": "remote_module1" }] } """ verrors, data = await self.validate_rsync_task(data, 'rsync_task_create') if verrors: raise verrors Cron.convert_schedule_to_db_format(data) data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self.middleware.call('service.restart', 'cron') return await self.get_instance(data['id']) @accepts(Int('id', validators=[Range(min=1)]), Patch('rsync_task_create', 'rsync_task_update', ('attr', { 'update': True }))) async def do_update(self, id, data): """ Update Rsync Task of `id`. """ old = await self.query(filters=[('id', '=', id)], options={'get': True}) old.pop('job') new = old.copy() new.update(data) verrors, data = await self.validate_rsync_task(new, 'rsync_task_update') if verrors: raise verrors Cron.convert_schedule_to_db_format(new) new.pop(self.locked_field) await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) await self.middleware.call('service.restart', 'cron') return await self.get_instance(id) async def do_delete(self, id): """ Delete Rsync Task of `id`. """ res = await self.middleware.call('datastore.delete', self._config.datastore, id) await self.middleware.call('service.restart', 'cron') return res @private async def commandline(self, id): """ Helper method to generate the rsync command avoiding code duplication. """ rsync = await self._get_instance(id) path = shlex.quote(rsync['path']) line = ['rsync'] for name, flag in ( ('archive', '-a'), ('compress', '-zz'), ('delayupdates', '--delay-updates'), ('delete', '--delete-delay'), ('preserveattr', '-X'), ('preserveperm', '-p'), ('recursive', '-r'), ('times', '-t'), ): if rsync[name]: line.append(flag) if rsync['extra']: line.append(' '.join(rsync['extra'])) # Do not use username if one is specified in host field # See #5096 for more details if '@' in rsync['remotehost']: remote = rsync['remotehost'] else: remote = f'"{rsync["user"]}"@{rsync["remotehost"]}' if rsync['mode'] == 'MODULE': module_args = [path, f'{remote}::"{rsync["remotemodule"]}"'] if rsync['direction'] != 'PUSH': module_args.reverse() line += module_args else: line += [ '-e', f'"ssh -p {rsync["remoteport"]} -o BatchMode=yes -o StrictHostKeyChecking=yes"' ] path_args = [ path, f'{remote}:"{shlex.quote(rsync["remotepath"])}"' ] if rsync['direction'] != 'PUSH': path_args.reverse() line += path_args if rsync['quiet']: line += ['>', '/dev/null', '2>&1'] return ' '.join(line) @item_method @accepts(Int('id')) @returns() @job(lock=lambda args: args[-1], lock_queue_size=1, logs=True) def run(self, job, id): """ Job to run rsync task of `id`. Output is saved to job log excerpt (not syslog). """ self.middleware.call_sync('network.general.will_perform_activity', 'rsync') rsync = self.middleware.call_sync('rsynctask.get_instance', id) if rsync['locked']: self.middleware.call_sync('rsynctask.generate_locked_alert', id) return commandline = self.middleware.call_sync('rsynctask.commandline', id) cp = run_command_with_user_context(commandline, rsync['user'], lambda v: job.logs_fd.write(v)) for klass in ('RsyncSuccess', 'RsyncFailed') if not rsync['quiet'] else (): self.middleware.call_sync('alert.oneshot_delete', klass, rsync['id']) if cp.returncode not in RsyncReturnCode.nonfatals(): err = None if cp.returncode == RsyncReturnCode.STREAMIO and rsync['compress']: err = ( "rsync command with compression enabled failed with STREAMIO error. " "This may indicate that remote server lacks support for the new-style " "compression used by TrueNAS.") if not rsync['quiet']: self.middleware.call_sync( 'alert.oneshot_create', 'RsyncFailed', { 'id': rsync['id'], 'direction': rsync['direction'], 'path': rsync['path'], }) if err: msg = f'{err} Check logs for further information' else: try: rc_name = RsyncReturnCode(cp.returncode).name except ValueError: rc_name = 'UNKNOWN' msg = (f'rsync command returned {cp.returncode} - {rc_name}. ' 'Check logs for further information.') raise CallError(msg) elif not rsync['quiet']: self.middleware.call_sync( 'alert.oneshot_create', 'RsyncSuccess', { 'id': rsync['id'], 'direction': rsync['direction'], 'path': rsync['path'], })
class PoolService(CRUDService): GELI_KEYPATH = '/data/geli' @filterable async def query(self, filters=None, options=None): filters = filters or [] options = options or {} options['extend'] = 'pool.pool_extend' options['prefix'] = 'vol_' return await self.middleware.call('datastore.query', 'storage.volume', filters, options) @private async def pool_extend(self, pool): pool.pop('fstype', None) """ If pool is encrypted we need to check if the pool is imported or if all geli providers exist. """ try: zpool = (await self.middleware.call('zfs.pool.query', [('id', '=', pool['name'])]))[0] except Exception: zpool = None if zpool: pool['status'] = zpool['status'] pool['scan'] = zpool['scan'] else: pool.update({ 'status': 'OFFLINE', 'scan': None, }) if pool['encrypt'] > 0: if zpool: pool['is_decrypted'] = True else: decrypted = True for ed in await self.middleware.call( 'datastore.query', 'storage.encrypteddisk', [('encrypted_volume', '=', pool['id'])]): if not os.path.exists( f'/dev/{ed["encrypted_provider"]}.eli'): decrypted = False break pool['is_decrypted'] = decrypted else: pool['is_decrypted'] = True return pool @item_method @accepts(Int('id', required=False)) async def get_disks(self, oid=None): """ Get all disks in use by pools. If `id` is provided only the disks from the given pool `id` will be returned. """ filters = [] if oid: filters.append(('id', '=', oid)) for pool in await self.query(filters): if pool['is_decrypted']: async for i in await self.middleware.call( 'zfs.pool.get_disks', pool['name']): yield i else: for encrypted_disk in await self.middleware.call( 'datastore.query', 'storage.encrypteddisk', [('encrypted_volume', '=', pool['id'])]): disk = { k[len("disk_"):]: v for k, v in encrypted_disk["encrypted_disk"].items() } name = await self.middleware.call("disk.get_name", disk) if os.path.exists(os.path.join("/dev", name)): yield name @item_method @accepts(Int('id')) async def download_encryption_key(self, oid): """ Download encryption key for a given pool `id`. """ pool = await self.query([('id', '=', oid)], {'get': True}) if not pool['encryptkey']: return None job_id, url = await self.middleware.call( 'core.download', 'filesystem.get', [os.path.join(self.GELI_KEYPATH, f"{pool['encryptkey']}.key")], 'geli.key') return url @private def configure_resilver_priority(self): """ Configure resilver priority based on user selected off-peak hours. """ resilver = self.middleware.call_sync('datastore.config', 'storage.resilver') if not resilver['enabled'] or not resilver['weekday']: return higher_prio = False weekdays = map(lambda x: int(x), resilver['weekday'].split(',')) now = datetime.now() now_t = now.time() # end overlaps the day if resilver['begin'] > resilver['end']: if now.isoweekday() in weekdays and now_t >= resilver['begin']: higher_prio = True else: lastweekday = now.isoweekday() - 1 if lastweekday == 0: lastweekday = 7 if lastweekday in weekdays and now_t < resilver['end']: higher_prio = True # end does not overlap the day else: if now.isoweekday() in weekdays and now_t >= resilver[ 'begin'] and now_t < resilver['end']: higher_prio = True if higher_prio: resilver_delay = 0 resilver_min_time_ms = 9000 scan_idle = 0 else: resilver_delay = 2 resilver_min_time_ms = 3000 scan_idle = 50 sysctl.filter('vfs.zfs.resilver_delay')[0].value = resilver_delay sysctl.filter( 'vfs.zfs.resilver_min_time_ms')[0].value = resilver_min_time_ms sysctl.filter('vfs.zfs.scan_idle')[0].value = scan_idle @accepts() async def import_find(self): """ Get a list of pools available for import with the following details: name, guid, status, hostname. """ existing_guids = [ i['guid'] for i in await self.middleware.call('pool.query') ] for pool in await self.middleware.call('zfs.pool.find_import'): if pool['status'] == 'UNAVAIL': continue # Exclude pools with same guid as existing pools (in database) # It could be the pool is in the database but was exported/detached for some reason # See #6808 if pool['guid'] in existing_guids: continue entry = {} for i in ('name', 'guid', 'status', 'hostname'): entry[i] = pool[i] yield entry @accepts( Dict( 'pool_import', Str('guid', required=True), Str('name'), Str('passphrase', private=True), List('devices', items=[Str('device')]), )) @job(lock='import_pool', pipes=['input'], check_pipes=False) async def import_pool(self, job, data): """ Import a pool. Errors: ENOENT - Pool not found """ pool = None for p in await self.middleware.call('zfs.pool.find_import'): if p['guid'] == data['guid']: pool = p break if pool is None: raise CallError(f'Pool with guid "{data["guid"]}" not found', errno.ENOENT) if data['devices']: job.check_pipe("input") args = [job.pipes.input.r, data['passphrase'], data['devices']] else: args = [] await self.middleware.call('notifier.volume_import', data.get('name') or pool['name'], data['guid'], *args) return True @accepts(Str('volume'), Str('fs_type'), Dict('fs_options', additional_attrs=True), Str('dst_path')) @job(lock=lambda args: 'volume_import', logs=True) async def import_disk(self, job, volume, fs_type, fs_options, dst_path): job.set_progress(None, description="Mounting") src = os.path.join('/var/run/importcopy/tmpdir', os.path.relpath(volume, '/')) if os.path.exists(src): os.rmdir(src) try: os.makedirs(src) async with KernelModuleContextManager({ "msdosfs": "msdosfs_iconv", "ntfs": "fuse" }.get(fs_type)): async with MountFsContextManager(self.middleware, volume, src, fs_type, fs_options, ["ro"]): job.set_progress(None, description="Importing") line = [ '/usr/local/bin/rsync', '--info=progress2', '--modify-window=1', '-rltvh', '--no-perms', src + '/', dst_path ] rsync_proc = await Popen( line, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0, preexec_fn=os.setsid, ) try: progress_buffer = JobProgressBuffer(job) while True: line = await rsync_proc.stdout.readline() job.logs_fd.write(line) if line: try: line = line.decode("utf-8", "ignore").strip() bits = re.split("\s+", line) if len(bits) == 6 and bits[1].endswith( "%") and bits[1][:-1].isdigit(): progress_buffer.set_progress( int(bits[1][:-1])) elif not line.endswith('/'): if (line not in [ 'sending incremental file list' ] and 'xfr#' not in line): progress_buffer.set_progress( None, extra=line) except Exception: logger.warning( 'Parsing error in rsync task', exc_info=True) else: break progress_buffer.flush() await rsync_proc.wait() if rsync_proc.returncode != 0: raise Exception("rsync failed with exit code %r" % rsync_proc.returncode) except asyncio.CancelledError: rsync_proc.kill() raise job.set_progress(100, description="Done", extra="") finally: os.rmdir(src) @accepts() def import_disk_msdosfs_locales(self): return [ locale.strip() for locale in subprocess.check_output(["locale", "-a"], encoding="utf-8").split("\n") if locale.strip() ] """ These methods are hacks for old UI which supports only one volume import at a time """ dismissed_import_disk_jobs = set() @private async def get_current_import_disk_job(self): import_jobs = await self.middleware.call( 'core.get_jobs', [('method', '=', 'pool.import_disk')]) not_dismissed_import_jobs = [ job for job in import_jobs if job["id"] not in self.dismissed_import_disk_jobs ] if not_dismissed_import_jobs: return not_dismissed_import_jobs[0] @private async def dismiss_current_import_disk_job(self): current_import_job = await self.get_current_import_disk_job() if current_import_job: self.dismissed_import_disk_jobs.add(current_import_job["id"])
class ZFSSnapshot(CRUDService): class Config: namespace = 'zfs.snapshot' process_pool = True @filterable def query(self, filters=None, options=None): """ Query all ZFS Snapshots with `query-filters` and `query-options`. """ # Special case for faster listing of snapshot names (#53149) if (options and options.get('select') == ['name'] and (not filters or filter_getattrs(filters).issubset({'name', 'pool'}))): # Using zfs list -o name is dozens of times faster than py-libzfs cmd = ['zfs', 'list', '-H', '-o', 'name', '-t', 'snapshot'] order_by = options.get('order_by') # -s name makes it even faster if not order_by or order_by == ['name']: cmd += ['-s', 'name'] cp = subprocess.run( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, ) if cp.returncode != 0: raise CallError(f'Failed to retrieve snapshots: {cp.stderr}') stdout = cp.stdout.strip() if not stdout: return [] snaps = [{ 'name': i, 'pool': i.split('/', 1)[0] } for i in stdout.split('\n')] if filters: return filter_list(snaps, filters, options) return snaps with libzfs.ZFS() as zfs: # Handle `id` filter to avoid getting all snapshots first snapshots = [] if filters and len(filters) == 1 and list( filters[0][:2]) == ['id', '=']: try: snapshots.append( zfs.get_snapshot(filters[0][2]).__getstate__()) except libzfs.ZFSException as e: if e.code != libzfs.Error.NOENT: raise else: for i in zfs.snapshots: try: snapshots.append(i.__getstate__()) except libzfs.ZFSException as e: # snapshot may have been deleted while this is running if e.code != libzfs.Error.NOENT: raise # FIXME: awful performance with hundreds/thousands of snapshots return filter_list(snapshots, filters, options) @accepts( Dict( 'snapshot_create', Str('dataset', required=True, empty=False), Str('name', empty=False), Str('naming_schema', empty=False, validators=[ReplicationSnapshotNamingSchema()]), Bool('recursive', default=False), Bool('vmware_sync', default=False), Dict('properties', additional_attrs=True), )) def do_create(self, data): """ Take a snapshot from a given dataset. """ dataset = data['dataset'] recursive = data['recursive'] properties = data['properties'] verrors = ValidationErrors() if 'name' in data and 'naming_schema' in data: verrors.add( 'snapshot_create.naming_schema', 'You can\'t specify name and naming schema at the same time') elif 'name' in data: name = data['name'] elif 'naming_schema' in data: # We can't do `strftime` here because we are in the process pool and `TZ` environment variable update # is not propagated here. name = self.middleware.call_sync('replication.new_snapshot_name', data['naming_schema']) else: verrors.add('snapshot_create.naming_schema', 'You must specify either name or naming schema') if verrors: raise verrors vmware_context = None if data['vmware_sync']: vmware_context = self.middleware.call_sync('vmware.snapshot_begin', dataset, recursive) try: with libzfs.ZFS() as zfs: ds = zfs.get_dataset(dataset) ds.snapshot(f'{dataset}@{name}', recursive=recursive, fsopts=properties) if vmware_context and vmware_context['vmsynced']: ds.properties['freenas:vmsynced'] = libzfs.ZFSUserProperty( 'Y') self.logger.info(f"Snapshot taken: {dataset}@{name}") except libzfs.ZFSException as err: self.logger.error(f'Failed to snapshot {dataset}@{name}: {err}') raise CallError(f'Failed to snapshot {dataset}@{name}: {err}') else: return self.middleware.call_sync('zfs.snapshot.get_instance', f'{dataset}@{name}') finally: if vmware_context: self.middleware.call_sync('vmware.snapshot_end', vmware_context) @accepts( Dict('snapshot_remove', Str('dataset', required=True), Str('name', required=True), Bool('defer_delete'))) def remove(self, data): """ Remove a snapshot from a given dataset. Returns: bool: True if succeed otherwise False. """ self.logger.debug( 'zfs.snapshot.remove is deprecated, use zfs.snapshot.delete') snapshot_name = data['dataset'] + '@' + data['name'] try: self.do_delete(snapshot_name, {'defer': data.get('defer_delete') or False}) except Exception: return False return True @accepts( Str('id'), Dict('options', Bool('defer', default=False)), ) def do_delete(self, id, options): """ Delete snapshot of name `id`. `options.defer` will defer the deletion of snapshot. """ try: with libzfs.ZFS() as zfs: snap = zfs.get_snapshot(id) snap.delete(defer=options['defer']) except libzfs.ZFSException as e: raise CallError(str(e)) else: return True @accepts(Dict( 'snapshot_clone', Str('snapshot'), Str('dataset_dst'), )) def clone(self, data): """ Clone a given snapshot to a new dataset. Returns: bool: True if succeed otherwise False. """ snapshot = data.get('snapshot', '') dataset_dst = data.get('dataset_dst', '') if not snapshot or not dataset_dst: return False try: with libzfs.ZFS() as zfs: snp = zfs.get_snapshot(snapshot) snp.clone(dataset_dst) dataset = zfs.get_dataset(dataset_dst) if dataset.type.name == 'FILESYSTEM': dataset.mount_recursive() self.logger.info("Cloned snapshot {0} to dataset {1}".format( snapshot, dataset_dst)) return True except libzfs.ZFSException as err: self.logger.error("{0}".format(err)) return False @accepts( Str('id'), Dict( 'options', Bool('recursive', default=False), Bool('recursive_clones', default=False), Bool('force', default=False), ), ) def rollback(self, id, options): """ Rollback to a given snapshot `id`. `options.recursive` will destroy any snapshots and bookmarks more recent than the one specified. `options.recursive_clones` is just like `recursive` but will also destroy any clones. `options.force` will force unmount of any clones. """ args = [] if options['force']: args += ['-f'] if options['recursive']: args += ['-r'] if options['recursive_clones']: args += ['-R'] try: subprocess.run( ['zfs', 'rollback'] + args + [id], text=True, capture_output=True, check=True, ) except subprocess.CalledProcessError as e: raise CallError(f'Failed to rollback snapshot: {e.stderr.strip()}')
class PoolDatasetService(CRUDService): class Config: namespace = 'pool.dataset' @filterable def query(self, filters, options): # Otimization for cases in which they can be filtered at zfs.dataset.query zfsfilters = [] for f in filters: if len(f) == 3: if f[0] in ('id', 'name', 'pool', 'type'): zfsfilters.append(f) datasets = self.middleware.call_sync('zfs.dataset.query', zfsfilters, None) return filter_list(self.__transform(datasets), filters, options) def __transform(self, datasets): """ We need to transform the data zfs gives us to make it consistent/user-friendly, making it match whatever pool.dataset.{create,update} uses as input. """ def transform(dataset): for orig_name, new_name, method in ( ('org.freenas:description', 'comments', None), ('dedup', 'deduplication', str.upper), ('atime', None, str.upper), ('casesensitivity', None, str.upper), ('exec', None, str.upper), ('sync', None, str.upper), ('compression', None, str.upper), ('origin', None, None), ('quota', None, _null), ('refquota', None, _null), ('reservation', None, _null), ('refreservation', None, _null), ('copies', None, None), ('snapdir', None, str.upper), ('readonly', None, str.upper), ('recordsize', None, None), ('sparse', None, None), ('volsize', None, None), ('volblocksize', None, None), ): if orig_name not in dataset['properties']: continue i = new_name or orig_name dataset[i] = dataset['properties'][orig_name] if method: dataset[i]['value'] = method(dataset[i]['value']) del dataset['properties'] rv = [] for child in dataset['children']: rv.append(transform(child)) dataset['children'] = rv return dataset rv = [] for dataset in datasets: rv.append(transform(dataset)) return rv @accepts( Dict( 'pool_dataset_create', Str('name', required=True), Str('type', enum=['FILESYSTEM', 'VOLUME'], default='FILESYSTEM'), Int('volsize'), Str('volblocksize', enum=[ '512', '1K', '2K', '4K', '8K', '16K', '32K', '64K', '128K', ]), Bool('sparse'), Str('comments'), Str('sync', enum=[ 'STANDARD', 'ALWAYS', 'DISABLED', ]), Str('compression', enum=[ 'OFF', 'LZ4', 'GZIP-1', 'GZIP-6', 'GZIP-9', 'ZLE', 'LZJB', ]), Str('atime', enum=['ON', 'OFF']), Str('exec', enum=['ON', 'OFF']), Int('quota'), Int('refquota'), Int('reservation'), Int('refreservation'), Int('copies'), Str('snapdir', enum=['VISIBLE', 'HIDDEN']), Str('deduplication', enum=['ON', 'VERIFY', 'OFF']), Str('readonly', enum=['ON', 'OFF']), Str('recordsize', enum=[ '512', '1K', '2K', '4K', '8K', '16K', '32K', '64K', '128K', '256K', '512K', '1024K', ]), Str('casesensitivity', enum=['SENSITIVE', 'INSENSITIVE', 'MIXED']), register=True, )) async def do_create(self, data): """ Creates a dataset/zvol. `volsize` is required for type=VOLUME and is supposed to be a multiple of the block size. """ verrors = ValidationErrors() await self.__common_validation(verrors, 'pool_dataset_create', data, 'CREATE') if verrors: raise verrors props = {} for i, real_name, transform in ( ('atime', None, str.lower), ('casesensitivity', None, str.lower), ('comments', 'org.freenas:description', None), ('compression', None, str.lower), ('copies', None, lambda x: str(x)), ('deduplication', 'dedup', str.lower), ('exec', None, str.lower), ('quota', None, _none), ('readonly', None, str.lower), ('recordsize', None, None), ('refquota', None, _none), ('refreservation', None, _none), ('reservation', None, _none), ('snapdir', None, str.lower), ('sparse', None, None), ('sync', None, str.lower), ('volblocksize', None, None), ('volsize', None, lambda x: str(x)), ): if i not in data: continue name = real_name or i props[name] = data[i] if not transform else transform(data[i]) await self.middleware.call('zfs.dataset.create', { 'name': data['name'], 'type': data['type'], 'properties': props, }) await self.middleware.call('zfs.dataset.mount', data['name']) def _add_inherit(name): def add(attr): attr.enum.append('INHERIT') return {'name': name, 'method': add} @accepts( Str('id'), Patch( 'pool_dataset_create', 'pool_dataset_update', ('rm', { 'name': 'name' }), ('rm', { 'name': 'type' }), ('rm', { 'name': 'casesensitivity' }), # Its a readonly attribute ('rm', { 'name': 'sparse' }), # Create time only attribute ('rm', { 'name': 'volblocksize' }), # Create time only attribute ('edit', _add_inherit('atime')), ('edit', _add_inherit('exec')), ('edit', _add_inherit('sync')), ('edit', _add_inherit('compression')), ('edit', _add_inherit('deduplication')), ('edit', _add_inherit('readonly')), ('edit', _add_inherit('recordsize')), ('edit', _add_inherit('snapdir')), )) async def do_update(self, id, data): """ Updates a dataset/zvol `id`. """ verrors = ValidationErrors() dataset = await self.middleware.call('pool.dataset.query', [('id', '=', id)]) if not dataset: verrors.add('id', f'{id} does not exist', errno.ENOENT) else: data['type'] = dataset[0]['type'] await self.__common_validation(verrors, 'pool_dataset_update', data, 'UPDATE') if verrors: raise verrors props = {} for i, real_name, transform, inheritable in ( ('atime', None, str.lower, True), ('comments', 'org.freenas:description', None, False), ('sync', None, str.lower, True), ('compression', None, str.lower, True), ('deduplication', 'dedup', str.lower, True), ('exec', None, str.lower, True), ('quota', None, _none, False), ('refquota', None, _none, False), ('reservation', None, _none, False), ('refreservation', None, _none, False), ('copies', None, None, False), ('snapdir', None, str.lower, True), ('readonly', None, str.lower, True), ('recordsize', None, None, True), ('volsize', None, lambda x: str(x), False), ): if i not in data: continue name = real_name or i if inheritable and data[i] == 'INHERIT': props[name] = {'source': 'INHERIT'} else: props[name] = { 'value': data[i] if not transform else transform(data[i]) } return await self.middleware.call('zfs.dataset.update', id, {'properties': props}) async def __common_validation(self, verrors, schema, data, mode): assert mode in ('CREATE', 'UPDATE') if data['type'] == 'FILESYSTEM': for i in ('sparse', 'volsize', 'volblocksize'): if i in data: verrors.add(f'{schema}.{i}', 'This field is not valid for FILESYSTEM') elif data['type'] == 'VOLUME': if mode == 'CREATE' and 'volsize' not in data: verrors.add(f'{schema}.volsize', 'This field is required for VOLUME') for i in ( 'atime', 'casesensitivity', 'quota', 'refquota', 'recordsize', ): if i in data: verrors.add(f'{schema}.{i}', 'This field is not valid for VOLUME') @accepts(Str('id')) async def do_delete(self, id): return await self.middleware.call('zfs.dataset.delete', id) @item_method @accepts(Str('id')) async def promote(self, id): """ Promote the cloned dataset `id` """ dataset = await self.middleware.call('zfs.dataset.query', [('id', '=', id)]) if not dataset: raise CallError(f'Dataset "{id}" does not exist.', errno.ENOENT) if not dataset[0]['properties']['origin']['value']: raise CallError('Only cloned datasets can be promoted.', errno.EBADMSG) return await self.middleware.call('zfs.dataset.promote', id)
class ZFSPoolService(CRUDService): class Config: namespace = 'zfs.pool' private = True process_pool = True @filterable def query(self, filters, options): # We should not get datasets, there is zfs.dataset.query for that state_kwargs = {'datasets_recursive': False} with libzfs.ZFS() as zfs: # Handle `id` filter specially to avoiding getting all pool if filters and len(filters) == 1 and list( filters[0][:2]) == ['id', '=']: try: pools = [ zfs.get(filters[0][2]).__getstate__(**state_kwargs) ] except libzfs.ZFSException: pools = [] else: pools = [i.__getstate__(**state_kwargs) for i in zfs.pools] return filter_list(pools, filters, options) @accepts( Dict( 'zfspool_create', Str('name', required=True), List('vdevs', items=[ Dict( 'vdev', Str('root', enum=[ 'DATA', 'CACHE', 'LOG', 'SPARE', 'SPECIAL', 'DEDUP' ], required=True), Str('type', enum=[ 'RAIDZ1', 'RAIDZ2', 'RAIDZ3', 'MIRROR', 'STRIPE' ], required=True), List('devices', items=[Str('disk')], required=True), ), ], required=True), Dict('options', additional_attrs=True), Dict('fsoptions', additional_attrs=True), ), ) def do_create(self, data): with libzfs.ZFS() as zfs: topology = convert_topology(zfs, data['vdevs']) zfs.create(data['name'], topology, data['options'], data['fsoptions']) return self.middleware.call_sync('zfs.pool.get_instance', data['name']) @accepts(Str('pool'), Dict( 'options', Dict('properties', additional_attrs=True), )) def do_update(self, name, options): try: with libzfs.ZFS() as zfs: pool = zfs.get(name) for k, v in options['properties'].items(): prop = pool.properties[k] if 'value' in v: prop.value = v['value'] elif 'parsed' in v: prop.parsed = v['parsed'] except libzfs.ZFSException as e: raise CallError(str(e)) @accepts(Str('pool'), Dict( 'options', Bool('force', default=False), )) def do_delete(self, name, options): try: with libzfs.ZFS() as zfs: zfs.destroy(name, force=options['force']) except libzfs.ZFSException as e: errno_ = errno.EFAULT if e.code == libzfs.Error.UMOUNTFAILED: errno_ = errno.EBUSY raise CallError(str(e), errno_) @accepts(Str('pool', required=True)) def upgrade(self, pool): try: with libzfs.ZFS() as zfs: zfs.get(pool).upgrade() except libzfs.ZFSException as e: raise CallError(str(e)) @accepts(Str('pool'), Dict( 'options', Bool('force', default=False), )) def export(self, name, options): try: with libzfs.ZFS() as zfs: # FIXME: force not yet implemented pool = zfs.get(name) zfs.export_pool(pool) except libzfs.ZFSException as e: raise CallError(str(e)) @accepts(Str('pool')) def get_devices(self, name): try: with libzfs.ZFS() as zfs: return [i.replace('/dev/', '') for i in zfs.get(name).disks] except libzfs.ZFSException as e: raise CallError(str(e), errno.ENOENT) @accepts( Str('name'), List('new', default=None, null=True), List('existing', items=[ Dict( 'attachvdev', Str('target'), Str('type', enum=['DISK']), Str('path'), ), ], null=True, default=None), ) @job() def extend(self, job, name, new=None, existing=None): """ Extend a zfs pool `name` with `new` vdevs or attach to `existing` vdevs. """ if new is None and existing is None: raise CallError('New or existing vdevs must be provided', errno.EINVAL) try: with libzfs.ZFS() as zfs: pool = zfs.get(name) if new: topology = convert_topology(zfs, new) pool.attach_vdevs(topology) # Make sure we can find all target vdev for i in (existing or []): target = find_vdev(pool, i['target']) if target is None: raise CallError( f"Failed to find vdev for {i['target']}", errno.EINVAL) i['target'] = target for i in (existing or []): newvdev = libzfs.ZFSVdev(zfs, i['type'].lower()) newvdev.path = i['path'] i['target'].attach(newvdev) except libzfs.ZFSException as e: raise CallError(str(e), e.code) def __zfs_vdev_operation(self, name, label, op, *args): try: with libzfs.ZFS() as zfs: pool = zfs.get(name) target = find_vdev(pool, label) if target is None: raise CallError(f'Failed to find vdev for {label}', errno.EINVAL) op(target, *args) except libzfs.ZFSException as e: raise CallError(str(e), e.code) @accepts(Str('pool'), Str('label'), Dict('options', Bool('clear_label', default=False))) def detach(self, name, label, options): """ Detach device `label` from the pool `pool`. """ self.detach_remove_impl('detach', name, label, options) def detach_remove_impl(self, op, name, label, options): def impl(target): getattr(target, op)() if options['clear_label']: self.clear_label(target.path) self.__zfs_vdev_operation(name, label, impl) @accepts(Str('device')) def clear_label(self, device): """ Clear label from `device`. """ try: libzfs.clear_label(device) except (libzfs.ZFSException, OSError) as e: raise CallError(str(e)) @accepts(Str('pool'), Str('label')) def offline(self, name, label): """ Offline device `label` from the pool `pool`. """ self.__zfs_vdev_operation(name, label, lambda target: target.offline()) @accepts(Str('pool'), Str('label'), Bool('expand', default=False)) def online(self, name, label, expand=False): """ Online device `label` from the pool `pool`. """ self.__zfs_vdev_operation(name, label, lambda target, *args: target.online(*args), expand) @accepts(Str('pool'), Str('label'), Dict('options', Bool('clear_label', default=False))) def remove(self, name, label, options): """ Remove device `label` from the pool `pool`. """ self.detach_remove_impl('remove', name, label, options) @accepts(Str('pool'), Str('label'), Str('dev')) def replace(self, name, label, dev): """ Replace device `label` with `dev` in pool `name`. """ try: with libzfs.ZFS() as zfs: pool = zfs.get(name) target = find_vdev(pool, label) if target is None: raise CallError(f'Failed to find vdev for {label}', errno.EINVAL) newvdev = libzfs.ZFSVdev(zfs, 'disk') newvdev.path = f'/dev/{dev}' # FIXME: Replace using old path is not working for some reason # Lets use guid for now. target.path = str(target.guid) target.replace(newvdev) except libzfs.ZFSException as e: raise CallError(str(e), e.code) @accepts(Str('name', required=True), Str('action', enum=['START', 'STOP', 'PAUSE'], default='START')) @job(lock=lambda i: f'{i[0]}-{i[1] if len(i) >= 2 else "START"}') def scrub(self, job, name, action=None): """ Start/Stop/Pause a scrub on pool `name`. """ if action != 'PAUSE': try: with libzfs.ZFS() as zfs: pool = zfs.get(name) if action == 'START': pool.start_scrub() else: pool.stop_scrub() except libzfs.ZFSException as e: raise CallError(str(e), e.code) else: proc = subprocess.Popen(f'zpool scrub -p {name}'.split(' '), stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc.communicate() if proc.returncode != 0: raise CallError('Unable to pause scrubbing') def watch(): while True: with libzfs.ZFS() as zfs: scrub = zfs.get(name).scrub.__getstate__() if scrub['pause']: job.set_progress(100, 'Scrub paused') break if scrub['function'] != 'SCRUB': break if scrub['state'] == 'FINISHED': job.set_progress(100, 'Scrub finished') break if scrub['state'] == 'CANCELED': break if scrub['state'] == 'SCANNING': job.set_progress(scrub['percentage'], 'Scrubbing') time.sleep(1) if action == 'START': t = threading.Thread(target=watch, daemon=True) t.start() t.join() @accepts() def find_import(self): with libzfs.ZFS() as zfs: return [i.__getstate__() for i in zfs.find_import()] @accepts( Str('name_or_guid'), Dict('options', additional_attrs=True), Bool('any_host', default=True), Str('cachefile', null=True, default=None), Str('new_name', null=True, default=None), ) def import_pool(self, name_or_guid, options, any_host, cachefile, new_name): found = False with libzfs.ZFS() as zfs: for pool in zfs.find_import(cachefile=cachefile, search_paths=['/dev/disk/by-partuuid'] if osc.IS_LINUX else None): if pool.name == name_or_guid or str(pool.guid) == name_or_guid: found = pool break if not found: raise CallError(f'Pool {name_or_guid} not found.', errno.ENOENT) try: zfs.import_pool(found, new_name or found.name, options, any_host=any_host) except libzfs.ZFSException as e: # We only log if some datasets failed to mount after pool import if e.code != libzfs.Error.MOUNTFAILED: raise else: self.logger.error( 'Failed to mount datasets after importing "%s" pool: %s', name_or_guid, str(e), exc_info=True) @accepts(Str('pool')) def find_not_online(self, pool): pool = self.middleware.call_sync('zfs.pool.query', [['id', '=', pool]], {'get': True}) unavails = [] for nodes in pool['groups'].values(): for node in nodes: unavails.extend(self.__find_not_online(node)) return unavails def __find_not_online(self, node): if len(node['children']) == 0 and node['status'] not in ('ONLINE', 'AVAIL'): return [node] unavails = [] for child in node['children']: unavails.extend(self.__find_not_online(child)) return unavails def get_vdev(self, name, vname): try: with libzfs.ZFS() as zfs: pool = zfs.get(name) vdev = find_vdev(pool, vname) if not vdev: raise CallError(f'{vname} not found in {name}', errno.ENOENT) return vdev.__getstate__() except libzfs.ZFSException as e: raise CallError(str(e))
class ChartReleaseService(Service): class Config: namespace = 'chart.release' @accepts(Str('chart_release_name')) @returns( Dict(example={'minio2': ['minio/minio:RELEASE.2022-03-05T06-32-39Z']})) async def get_chart_releases_using_chart_release_images( self, chart_release_name): """ Retrieve chart releases which are consuming any images in use by `chart_release_name`. """ chart_releases = await self.middleware.call( 'chart.release.query', [], {'extra': { 'retrieve_resources': True }}) idx = next((idx for (idx, d) in enumerate(chart_releases) if d['name'] == chart_release_name), None) if idx is None: raise CallError(f'{chart_release_name!r} not found', errno=errno.ENOENT) chart_release = chart_releases.pop(idx) return get_chart_releases_consuming_image( chart_release['resources']['container_images'], chart_releases, True) @private async def delete_unused_app_images(self, chart_release): failed_to_delete = {} to_delete_tags = await self.get_to_delete_unused_app_images( chart_release) for image in await self.middleware.call('container.image.query', [[ 'OR', [['complete_tags', 'rin', tag] for tag in to_delete_tags] ]], {'extra': { 'complete_tags': True }}) if to_delete_tags else []: try: await self.middleware.call('container.image.delete', image['id']) except Exception as e: failed_to_delete[', '.join(image['complete_tags'])] = str(e) return failed_to_delete @private async def get_to_delete_unused_app_images(self, chart_release): to_delete = { normalize_reference(i)['complete_tag'] for i in chart_release['resources']['container_images'] } in_use = get_chart_releases_consuming_image( to_delete, await self.middleware.call('chart.release.query', [['id', '!=', chart_release['name']]], {'extra': { 'retrieve_resources': True }}), True) for image_list in in_use.values(): for image in filter(lambda i: i['complete_tag'] in to_delete, map(normalize_reference, image_list)): to_delete.remove(image['complete_tag']) return list(to_delete)
class FilesystemService(Service): @accepts(Str('path', required=True), Ref('query-filters'), Ref('query-options')) def listdir(self, path, filters=None, options=None): """ Get the contents of a directory. Each entry of the list consists of: name(str): name of the file path(str): absolute path of the entry realpath(str): absolute real path of the entry (if SYMLINK) type(str): DIRECTORY | FILESYSTEM | SYMLINK | OTHER size(int): size of the entry mode(int): file mode/permission uid(int): user id of entry owner gid(int): group id of entry onwer acl(bool): extended ACL is present on file """ if not os.path.exists(path): raise CallError(f'Directory {path} does not exist', errno.ENOENT) if not os.path.isdir(path): raise CallError(f'Path {path} is not a directory', errno.ENOTDIR) rv = [] for entry in os.scandir(path): if entry.is_symlink(): etype = 'SYMLINK' elif entry.is_dir(): etype = 'DIRECTORY' elif entry.is_file(): etype = 'FILE' else: etype = 'OTHER' data = { 'name': entry.name, 'path': entry.path, 'realpath': os.path.realpath(entry.path) if etype == 'SYMLINK' else entry.path, 'type': etype, } try: stat = entry.stat() data.update({ 'size': stat.st_size, 'mode': stat.st_mode, 'acl': False if self.acl_is_trivial(data["realpath"]) else True, 'uid': stat.st_uid, 'gid': stat.st_gid, }) except FileNotFoundError: data.update({ 'size': None, 'mode': None, 'acl': None, 'uid': None, 'gid': None }) rv.append(data) return filter_list(rv, filters=filters or [], options=options or {}) @accepts(Str('path')) def stat(self, path): """ Return the filesystem stat(2) for a given `path`. """ try: stat = os.stat(path, follow_symlinks=False) except FileNotFoundError: raise CallError(f'Path {path} not found', errno.ENOENT) stat = { 'size': stat.st_size, 'mode': stat.st_mode, 'uid': stat.st_uid, 'gid': stat.st_gid, 'atime': stat.st_atime, 'mtime': stat.st_mtime, 'ctime': stat.st_ctime, 'dev': stat.st_dev, 'inode': stat.st_ino, 'nlink': stat.st_nlink, } try: stat['user'] = pwd.getpwuid(stat['uid']).pw_name except KeyError: stat['user'] = None try: stat['group'] = grp.getgrgid(stat['gid']).gr_name except KeyError: stat['group'] = None stat['acl'] = False if self.acl_is_trivial(path) else True return stat @private @accepts( Str('path'), Str('content', max_length=2048000), Dict( 'options', Bool('append', default=False), Int('mode'), ), ) def file_receive(self, path, content, options=None): """ Simplified file receiving method for small files. `content` must be a base 64 encoded file content. """ options = options or {} dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname) if options.get('append'): openmode = 'ab' else: openmode = 'wb+' with open(path, openmode) as f: f.write(binascii.a2b_base64(content)) mode = options.get('mode') if mode: os.chmod(path, mode) return True @private @accepts( Str('path'), Dict( 'options', Int('offset'), Int('maxlen'), ), ) def file_get_contents(self, path, options=None): """ Get contents of a file `path` in base64 encode. DISCLAIMER: DO NOT USE THIS FOR BIG FILES (> 500KB). """ options = options or {} if not os.path.exists(path): return None with open(path, 'rb') as f: if options.get('offset'): f.seek(options['offset']) data = binascii.b2a_base64(f.read( options.get('maxlen'))).decode().strip() return data @accepts(Str('path')) @job(pipes=["output"]) async def get(self, job, path): """ Job to get contents of `path`. """ if not os.path.isfile(path): raise CallError(f'{path} is not a file') with open(path, 'rb') as f: await self.middleware.run_in_thread(shutil.copyfileobj, f, job.pipes.output.w) @accepts( Str('path'), Dict( 'options', Bool('append', default=False), Int('mode'), ), ) @job(pipes=["input"]) async def put(self, job, path, options=None): """ Job to put contents to `path`. """ options = options or {} dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname) if options.get('append'): openmode = 'ab' else: openmode = 'wb+' with open(path, openmode) as f: await self.middleware.run_in_thread(shutil.copyfileobj, job.pipes.input.r, f) mode = options.get('mode') if mode: os.chmod(path, mode) return True @accepts(Str('path')) def statfs(self, path): """ Return stats from the filesystem of a given path. Raises: CallError(ENOENT) - Path not found """ try: statfs = bsd.statfs(path) except FileNotFoundError: raise CallError('Path not found.', errno.ENOENT) return { **statfs.__getstate__(), 'total_bytes': statfs.total_blocks * statfs.blocksize, 'free_bytes': statfs.free_blocks * statfs.blocksize, 'avail_bytes': statfs.avail_blocks * statfs.blocksize, } def __convert_to_basic_permset(self, permset): """ Convert "advanced" ACL permset format to basic format using bitwise operation and constants defined in py-bsd/bsd/acl.pyx, py-bsd/defs.pxd and acl.h. If the advanced ACL can't be converted without losing information, we return 'OTHER'. Reverse process converts the constant's value to a dictionary using a bitwise operation. """ perm = 0 for k, v, in permset.items(): if v: perm |= acl.NFS4Perm[k] try: SimplePerm = (acl.NFS4BasicPermset(perm)).name except Exception: SimplePerm = 'OTHER' return SimplePerm def __convert_to_basic_flagset(self, flagset): flags = 0 for k, v, in flagset.items(): if k == "INHERITED": continue if v: flags |= acl.NFS4Flag[k] try: SimpleFlag = (acl.NFS4BasicFlagset(flags)).name except Exception: SimpleFlag = 'OTHER' return SimpleFlag def __convert_to_adv_permset(self, basic_perm): permset = {} perm_mask = acl.NFS4BasicPermset[basic_perm].value for name, member in acl.NFS4Perm.__members__.items(): if perm_mask & member.value: permset.update({name: True}) else: permset.update({name: False}) return permset def __convert_to_adv_flagset(self, basic_flag): flagset = {} flag_mask = acl.NFS4BasicFlagset[basic_flag].value for name, member in acl.NFS4Flag.__members__.items(): if flag_mask & member.value: flagset.update({name: True}) else: flagset.update({name: False}) return flagset def _winacl(self, path, action, uid, gid, options): chroot_dir = os.path.dirname(path) target = os.path.basename(path) winacl = subprocess.run([ '/usr/local/bin/winacl', '-a', action, '-O', str(uid), '-G', str(gid), '-rx' if options['traverse'] else '-r', '-c', chroot_dir, '-p', target ], check=False, capture_output=True) if winacl.returncode != 0: CallError( f"Winacl {action} on path {path} failed with error: [{winacl.stderr.decode().strip()}]" ) def _common_perm_path_validate(self, path): if not os.path.exists(path): raise CallError(f"Path not found: {path}", errno.ENOENT) if not os.path.realpath(path).startswith('/mnt/'): raise CallError( f"Changing permissions on paths outside of /mnt is not permitted: {path}", errno.EPERM) if os.path.realpath(path) in [ x['path'] for x in self.middleware.call_sync('pool.query') ]: raise CallError( f"Changing permissions of root level dataset is not permitted: {path}", errno.EPERM) @accepts(Str('path')) def acl_is_trivial(self, path): """ Returns True if the ACL can be fully expressed as a file mode without losing any access rules, or if the path does not support NFSv4 ACLs (for example a path on a tmpfs filesystem). """ if not os.path.exists(path): raise CallError(f'Path not found [{path}].', errno.ENOENT) if osc.IS_LINUX: posix1e_acl = self.getacl_posix1e(path, True) return True if len(posix1e_acl['acl']) == 3 else False if not os.pathconf(path, 64): return True return acl.ACL(file=path).is_trivial @accepts( Dict( 'filesystem_ownership', Str('path', required=True), Int('uid', null=True, default=None), Int('gid', null=True, default=None), Dict('options', Bool('recursive', default=False), Bool('traverse', default=False)))) @job(lock="perm_change") def chown(self, job, data): """ Change owner or group of file at `path`. `uid` and `gid` specify new owner of the file. If either key is absent or None, then existing value on the file is not changed. `recursive` performs action recursively, but does not traverse filesystem mount points. If `traverse` and `recursive` are specified, then the chown operation will traverse filesystem mount points. """ job.set_progress(0, 'Preparing to change owner.') self._common_perm_path_validate(data['path']) uid = -1 if data['uid'] is None else data['uid'] gid = -1 if data['gid'] is None else data['gid'] options = data['options'] if not options['recursive']: job.set_progress(100, 'Finished changing owner.') os.chown(data['path'], uid, gid) else: job.set_progress(10, f'Recursively changing owner of {data["path"]}.') self._winacl(data['path'], 'chown', uid, gid, options) job.set_progress(100, 'Finished changing owner.') @accepts( Dict( 'filesystem_permission', Str('path', required=True), UnixPerm('mode', null=True), Int('uid', null=True, default=None), Int('gid', null=True, default=None), Dict( 'options', Bool('stripacl', default=False), Bool('recursive', default=False), Bool('traverse', default=False), ))) @job(lock="perm_change") def setperm(self, job, data): """ Remove extended ACL from specified path. If `mode` is specified then the mode will be applied to the path and files and subdirectories depending on which `options` are selected. Mode should be formatted as string representation of octal permissions bits. `uid` the desired UID of the file user. If set to None (the default), then user is not changed. `gid` the desired GID of the file group. If set to None (the default), then group is not changed. `stripacl` setperm will fail if an extended ACL is present on `path`, unless `stripacl` is set to True. `recursive` remove ACLs recursively, but do not traverse dataset boundaries. `traverse` remove ACLs from child datasets. If no `mode` is set, and `stripacl` is True, then non-trivial ACLs will be converted to trivial ACLs. An ACL is trivial if it can be expressed as a file mode without losing any access rules. """ job.set_progress(0, 'Preparing to set permissions.') options = data['options'] mode = data.get('mode', None) uid = -1 if data['uid'] is None else data['uid'] gid = -1 if data['gid'] is None else data['gid'] self._common_perm_path_validate(data['path']) acl_is_trivial = self.middleware.call_sync('filesystem.acl_is_trivial', data['path']) if not acl_is_trivial and not options['stripacl']: raise CallError( f'Non-trivial ACL present on [{data["path"]}]. Option "stripacl" required to change permission.', errno.EINVAL) if mode is not None: mode = int(mode, 8) a = acl.ACL(file=data['path']) a.strip() a.apply(data['path']) if mode: os.chmod(data['path'], mode) if uid or gid: os.chown(data['path'], uid, gid) if not options['recursive']: job.set_progress(100, 'Finished setting permissions.') return action = 'clone' if mode else 'strip' job.set_progress( 10, f'Recursively setting permissions on {data["path"]}.') self._winacl(data['path'], action, uid, gid, options) job.set_progress(100, 'Finished setting permissions.') @accepts() async def default_acl_choices(self): """ Get list of default ACL types. """ acl_choices = [] for x in ACLDefault: if x.value['visible']: acl_choices.append(x.name) return acl_choices @accepts( Str('acl_type', default='OPEN', enum=[x.name for x in ACLDefault]), Str('share_type', default='NONE', enum=['NONE', 'AFP', 'SMB', 'NFS']), ) async def get_default_acl(self, acl_type, share_type): """ Returns a default ACL depending on the usage specified by `acl_type`. If an admin group is defined, then an entry granting it full control will be placed at the top of the ACL. Optionally may pass `share_type` to argument to get share-specific template ACL. """ acl = [] admin_group = (await self.middleware.call('smb.config'))['admin_group'] if acl_type == 'HOME' and (await self.middleware.call( 'activedirectory.get_state')) == 'HEALTHY': acl_type = 'DOMAIN_HOME' if admin_group: acl.append({ 'tag': 'GROUP', 'id': (await self.middleware.call('dscache.get_uncached_group', admin_group))['gr_gid'], 'perms': { 'BASIC': 'FULL_CONTROL' }, 'flags': { 'BASIC': 'INHERIT' }, 'type': 'ALLOW' }) if share_type == 'SMB': acl.append({ 'tag': 'GROUP', 'id': int(SMBBuiltin['USERS'].value[1][9:]), 'perms': { 'BASIC': 'MODIFY' }, 'flags': { 'BASIC': 'INHERIT' }, 'type': 'ALLOW' }) acl.extend((ACLDefault[acl_type].value)['acl']) return acl def _is_inheritable(self, flags): """ Takes ACE flags and return True if any inheritance bits are set. """ inheritance_flags = [ 'FILE_INHERIT', 'DIRECTORY_INHERIT', 'NO_PROPAGATE_INHERIT', 'INHERIT_ONLY' ] for i in inheritance_flags: if flags.get(i): return True return False @private def canonicalize_acl_order(self, acl): """ Convert flags to advanced, then separate the ACL into two lists. One for ACEs that have been inherited, one for aces that have not been inherited. Non-inherited ACEs take precedence and so they are placed first in finalized combined list. Within each list, the ACEs are orderd according to the following: 1) Deny ACEs that apply to the object itself (NOINHERIT) 2) Deny ACEs that apply to a subobject of the object (INHERIT) 3) Allow ACEs that apply to the object itself (NOINHERIT) 4) Allow ACEs that apply to a subobject of the object (INHERIT) See http://docs.microsoft.com/en-us/windows/desktop/secauthz/order-of-aces-in-a-dacl The "INHERITED" bit is stripped in filesystem.getacl when generating a BASIC flag type. It is best practice to use a non-simplified ACL for canonicalization. """ inherited_aces = [] final_acl = [] non_inherited_aces = [] for entry in acl: entry['flags'] = self.__convert_to_adv_flagset( entry['flags'] ['BASIC']) if 'BASIC' in entry['flags'] else entry['flags'] if entry['flags'].get('INHERITED'): inherited_aces.append(entry) else: non_inherited_aces.append(entry) if inherited_aces: inherited_aces = sorted( inherited_aces, key=lambda x: (x['type'] == 'ALLOW', self._is_inheritable(x['flags'])), ) if non_inherited_aces: non_inherited_aces = sorted( non_inherited_aces, key=lambda x: (x['type'] == 'ALLOW', self._is_inheritable(x['flags'])), ) final_acl = non_inherited_aces + inherited_aces return final_acl @private def getacl_posix1e(self, path, simplified): st = os.stat(path) ret = { 'uid': st.st_uid, 'gid': st.st_gid, 'acl': [], 'flags': { 'setuid': bool(st.st_mode & pystat.S_ISUID), 'setgid': bool(st.st_mode & pystat.S_ISGID), 'sticky': bool(st.st_mode & pystat.S_ISVTX), }, 'acltype': ACLType.POSIX1E.name } ret['uid'] = st.st_uid ret['gid'] = st.st_gid gfacl = subprocess.run( ['getfacl', '-c' if osc.IS_LINUX else '-q', '-n', path], check=False, capture_output=True) if gfacl.returncode != 0: raise CallError( f"Failed to get POSIX1e ACL on path [{path}]: {gfacl.stderr.decode()}" ) # linux output adds extra line to output if it's an absolute path and extra newline at end. entries = gfacl.stdout.decode().splitlines() if osc.IS_LINUX: entries = entries[:-1] for entry in entries: if entry.startswith("#"): continue ace = { "default": False, "tag": None, "id": -1, "perms": { "READ": False, "WRITE": False, "EXECUTE": False, } } tag, id, perms = entry.rsplit(":", 3) ace['perms'].update({ "READ": perms[0].casefold() == "r", "WRITE": perms[1].casefold() == "w", "EXECUTE": perms[2].casefold() == "x", }) if tag.startswith('default'): ace['default'] = True tag = tag[8:] ace['tag'] = tag.upper() if id.isdigit(): ace['id'] = int(id) ret['acl'].append(ace) return ret @private def getacl_nfs4(self, path, simplified): stat = os.stat(path) a = acl.ACL(file=path) fs_acl = a.__getstate__() if not simplified: advanced_acl = [] for entry in fs_acl: ace = { 'tag': (acl.ACLWho[entry['tag']]).value, 'id': entry['id'], 'type': entry['type'], 'perms': entry['perms'], 'flags': entry['flags'], } if ace['tag'] == 'everyone@' and self.__convert_to_basic_permset( ace['perms']) == 'NOPERMS': continue advanced_acl.append(ace) return { 'uid': stat.st_uid, 'gid': stat.st_gid, 'acl': advanced_acl, 'acltype': ACLType.NFS4.name } simple_acl = [] for entry in fs_acl: ace = { 'tag': (acl.ACLWho[entry['tag']]).value, 'id': entry['id'], 'type': entry['type'], 'perms': { 'BASIC': self.__convert_to_basic_permset(entry['perms']) }, 'flags': { 'BASIC': self.__convert_to_basic_flagset(entry['flags']) }, } if ace['tag'] == 'everyone@' and ace['perms']['BASIC'] == 'NOPERMS': continue for key in ['perms', 'flags']: if ace[key]['BASIC'] == 'OTHER': ace[key] = entry[key] simple_acl.append(ace) return { 'uid': stat.st_uid, 'gid': stat.st_gid, 'acl': simple_acl, 'acltype': ACLType.NFS4.name } @accepts( Str('path'), Bool('simplified', default=True), ) def getacl(self, path, simplified=True): """ Return ACL of a given path. This may return a POSIX1e ACL or a NFSv4 ACL. The acl type is indicated by the `ACLType` key. Errata about ACLType NFSv4: `simplified` returns a shortened form of the ACL permset and flags. `TRAVERSE` sufficient rights to traverse a directory, but not read contents. `READ` sufficient rights to traverse a directory, and read file contents. `MODIFIY` sufficient rights to traverse, read, write, and modify a file. Equivalent to modify_set. `FULL_CONTROL` all permissions. If the permisssions do not fit within one of the pre-defined simplified permissions types, then the full ACL entry will be returned. In all cases we replace USER_OBJ, GROUP_OBJ, and EVERYONE with owner@, group@, everyone@ for consistency with getfacl and setfacl. If one of aforementioned special tags is used, 'id' must be set to None. An inheriting empty everyone@ ACE is appended to non-trivial ACLs in order to enforce Windows expectations regarding permissions inheritance. This entry is removed from NT ACL returned to SMB clients when 'ixnas' samba VFS module is enabled. We also remove it here to avoid confusion. """ if not os.path.exists(path): raise CallError('Path not found.', errno.ENOENT) if osc.IS_LINUX or not os.pathconf(path, 64): return self.getacl_posix1e(path, simplified) return self.getacl_nfs4(path, simplified) @private def setacl_posix1e(self, job, data): job.set_progress(0, 'Preparing to set acl.') if osc.IS_FREEBSD: raise CallError( "POSIX1e brand ACLs not supported on the FreeBSD-based TrueNAS platform", errno.EOPNOTSUPP) options = data['options'] recursive = options.get('recursive') dacl = data.get('dacl', []) path = data['path'] aclcheck = ACLType.POSIX1E.validate(data) if not aclcheck['is_valid']: raise CallError( f"POSIX1e ACL is invalid: {' '.join(aclcheck['errors'])}") stripacl = subprocess.run( ['setfacl', '-bR' if recursive else '-b', path], check=False, capture_output=True) if stripacl.returncode != 0: raise CallError(f"Failed to remove POSIX1e ACL from [{path}]: " f"{stripacl.stderr.decode()}") if options['stripacl']: job.set_progress(100, "Finished removing POSIX1e ACL") return job.set_progress(50, 'Reticulating splines.') for idx, ace in enumerate(dacl): if idx == 0: aclstring = "" else: aclstring += "," if ace['id'] == -1: ace['id'] = '' ace['tag'] = ace['tag'].rstrip('_OBJ').lower() if ace['default']: aclstring += "default:" aclstring += f"{ace['tag']}:{ace['id']}:" aclstring += 'r' if ace['perms']['READ'] else '-' aclstring += 'w' if ace['perms']['WRITE'] else '-' aclstring += 'x' if ace['perms']['EXECUTE'] else '-' self.logger.debug("acl string: %s", aclstring) setacl = subprocess.run( ['setfacl', '-mR' if recursive else '-m', aclstring, path], check=False, capture_output=True) if setacl.returncode != 0: return CallError(f'Failed to set ACL on path [{path}]: ', f'{setacl.stderr.decode()}') job.set_progress(100, 'Finished setting POSIX1e ACL.') @private def setacl_nfs4(self, job, data): job.set_progress(0, 'Preparing to set acl.') options = data['options'] dacl = data.get('dacl', []) if osc.IS_LINUX or not os.pathconf(data['path'], 64): raise CallError( f"NFSv4 ACLS are not supported on path {data['path']}", errno.EOPNOTSUPP) self._common_perm_path_validate(data['path']) if dacl and options['stripacl']: raise CallError( 'Setting ACL and stripping ACL are not permitted simultaneously.', errno.EINVAL) uid = -1 if data.get('uid', None) is None else data['uid'] gid = -1 if data.get('gid', None) is None else data['gid'] if options['stripacl']: a = acl.ACL(file=data['path']) a.strip() a.apply(data['path']) else: inheritable_is_present = False cleaned_acl = [] lockace_is_present = False for entry in dacl: ace = { 'tag': (acl.ACLWho(entry['tag'])).name, 'id': entry['id'], 'type': entry['type'], 'perms': self.__convert_to_adv_permset(entry['perms']['BASIC']) if 'BASIC' in entry['perms'] else entry['perms'], 'flags': self.__convert_to_adv_flagset(entry['flags']['BASIC']) if 'BASIC' in entry['flags'] else entry['flags'], } if ace['flags'].get('INHERIT_ONLY') and not ace['flags'].get( 'DIRECTORY_INHERIT', False) and not ace['flags'].get( 'FILE_INHERIT', False): raise CallError( 'Invalid flag combination. DIRECTORY_INHERIT or FILE_INHERIT must be set if INHERIT_ONLY is set.', errno.EINVAL) if ace['tag'] == 'EVERYONE' and self.__convert_to_basic_permset( ace['perms']) == 'NOPERMS': lockace_is_present = True elif ace['flags'].get('DIRECTORY_INHERIT') or ace['flags'].get( 'FILE_INHERIT'): inheritable_is_present = True cleaned_acl.append(ace) if not inheritable_is_present: raise CallError( 'At least one inheritable ACL entry is required', errno.EINVAL) if options['canonicalize']: cleaned_acl = self.canonicalize_acl_order(cleaned_acl) if not lockace_is_present: locking_ace = { 'tag': 'EVERYONE', 'id': None, 'type': 'ALLOW', 'perms': self.__convert_to_adv_permset('NOPERMS'), 'flags': self.__convert_to_adv_flagset('INHERIT') } cleaned_acl.append(locking_ace) a = acl.ACL() a.__setstate__(cleaned_acl) a.apply(data['path']) if not options['recursive']: os.chown(data['path'], uid, gid) job.set_progress(100, 'Finished setting NFS4 ACL.') return job.set_progress(10, f'Recursively setting ACL on {data["path"]}.') self._winacl(data['path'], 'clone', uid, gid, options) job.set_progress(100, 'Finished setting NFS4 ACL.') @accepts( Dict( 'filesystem_acl', Str('path', required=True), Int('uid', null=True, default=None), Int('gid', null=True, default=None), List('dacl', items=[ Dict( 'aclentry', Str('tag', enum=[ 'owner@', 'group@', 'everyone@', 'USER', 'GROUP' ]), Int('id', null=True), Str('type', enum=['ALLOW', 'DENY']), Dict( 'perms', Bool('READ_DATA'), Bool('WRITE_DATA'), Bool('APPEND_DATA'), Bool('READ_NAMED_ATTRS'), Bool('WRITE_NAMED_ATTRS'), Bool('EXECUTE'), Bool('DELETE_CHILD'), Bool('READ_ATTRIBUTES'), Bool('WRITE_ATTRIBUTES'), Bool('DELETE'), Bool('READ_ACL'), Bool('WRITE_ACL'), Bool('WRITE_OWNER'), Bool('SYNCHRONIZE'), Str('BASIC', enum=[ 'FULL_CONTROL', 'MODIFY', 'READ', 'TRAVERSE' ]), ), Dict( 'flags', Bool('FILE_INHERIT'), Bool('DIRECTORY_INHERIT'), Bool('NO_PROPAGATE_INHERIT'), Bool('INHERIT_ONLY'), Bool('INHERITED'), Str('BASIC', enum=['INHERIT', 'NOINHERIT']), ), ), Dict( 'posix1e_ace', Bool('default', default=False), Str('tag', enum=[ 'USER_OBJ', 'GROUP_OBJ', 'USER', 'GROUP', 'OTHER', 'MASK' ]), Int('id', default=-1), Dict( 'perms', Bool('READ', default=False), Bool('WRITE', default=False), Bool('EXECUTE', default=False), ), ) ], default=[]), Dict( 'nfs41_flags', Bool('autoinherit', default=False), Bool('protected', default=False), ), Str('acltype', enum=[x.name for x in ACLType], default=ACLType.NFS4.name), Dict('options', Bool('stripacl', default=False), Bool('recursive', default=False), Bool('traverse', default=False), Bool('canonicalize', default=True)))) @job(lock="perm_change") def setacl(self, job, data): """ Set ACL of a given path. Takes the following parameters: `path` full path to directory or file. `dacl` "simplified" ACL here or a full ACL. `uid` the desired UID of the file user. If set to None (the default), then user is not changed. `gid` the desired GID of the file group. If set to None (the default), then group is not changed. `recursive` apply the ACL recursively `traverse` traverse filestem boundaries (ZFS datasets) `strip` convert ACL to trivial. ACL is trivial if it can be expressed as a file mode without losing any access rules. `canonicalize` reorder ACL entries so that they are in concanical form as described in the Microsoft documentation MS-DTYP 2.4.5 (ACL) In all cases we replace USER_OBJ, GROUP_OBJ, and EVERYONE with owner@, group@, everyone@ for consistency with getfacl and setfacl. If one of aforementioned special tags is used, 'id' must be set to None. An inheriting empty everyone@ ACE is appended to non-trivial ACLs in order to enforce Windows expectations regarding permissions inheritance. This entry is removed from NT ACL returned to SMB clients when 'ixnas' samba VFS module is enabled. """ acltype = ACLType[data['acltype']] if acltype == ACLType.NFS4: return self.setacl_nfs4(job, data) else: return self.setacl_posix1e(job, data) @private async def children_are_locked(self, path, child): if child["locked"] and path.startswith(child["mountpoint"]): return True if not path.startswith(child["mountpoint"]): return False if child.get("children"): for c in child["children"]: is_locked = await self.children_are_locked(path, c) if is_locked: return True return False @private async def path_is_encrypted(self, path): ds = await self.middleware.call("pool.dataset.from_path", path, True) if ds["locked"]: return True if not ds["children"]: return False return await self.children_are_locked(path, ds)
class ServiceService(CRUDService): SERVICE_DEFS = { 's3': ServiceDefinition('minio', '/var/run/minio.pid'), 'ssh': ServiceDefinition('sshd', '/var/run/sshd.pid'), 'rsync': ServiceDefinition('rsync', '/var/run/rsyncd.pid'), 'nfs': ServiceDefinition('nfsd', None), 'afp': ServiceDefinition('netatalk', None), 'cifs': ServiceDefinition('smbd', '/var/run/samba4/smbd.pid'), 'dynamicdns': ServiceDefinition('inadyn', None), 'snmp': ServiceDefinition('snmpd', '/var/run/net_snmpd.pid'), 'ftp': ServiceDefinition('proftpd', '/var/run/proftpd.pid'), 'tftp': ServiceDefinition('inetd', '/var/run/inetd.pid'), 'iscsitarget': ServiceDefinition('ctld', '/var/run/ctld.pid'), 'lldp': ServiceDefinition('ladvd', '/var/run/ladvd.pid'), 'ups': ServiceDefinition('upsd', '/var/db/nut/upsd.pid'), 'upsmon': ServiceDefinition('upsmon', '/var/db/nut/upsmon.pid'), 'smartd': ServiceDefinition('smartd', 'smartd-daemon', '/var/run/smartd-daemon.pid'), 'webshell': ServiceDefinition(None, '/var/run/webshell.pid'), 'webdav': ServiceDefinition('httpd', '/var/run/httpd.pid'), 'netdata': ServiceDefinition('netdata', '/var/db/netdata/netdata.pid'), 'asigra': ServiceDefinition('asigra', '/var/run/dssystem.pid') } @filterable async def query(self, filters=None, options=None): if options is None: options = {} options['prefix'] = 'srv_' services = await self.middleware.call('datastore.query', 'services.services', filters, options) # In case a single service has been requested if not isinstance(services, list): services = [services] jobs = { asyncio.ensure_future(self._get_status(entry)): entry for entry in services } if jobs: done, pending = await asyncio.wait(list(jobs.keys()), timeout=15) def result(task): """ Method to handle results of the coroutines. In case of error or timeout, provide UNKNOWN state. """ result = None try: if task in done: result = task.result() except Exception: pass if result is None: entry = jobs.get(task) self.logger.warn('Failed to get status for %s', entry['service']) entry['state'] = 'UNKNOWN' entry['pids'] = [] return entry else: return result services = list(map(result, jobs)) return filter_list(services, filters, options) @accepts( Str('id_or_name'), Dict( 'service-update', Bool('enable', default=False), ), ) async def do_update(self, id_or_name, data): """ Update service entry of `id_or_name`. Currently it only accepts `enable` option which means whether the service should start on boot. """ if not id_or_name.isdigit(): svc = await self.middleware.call( 'datastore.query', 'services.services', [('srv_service', '=', id_or_name)]) if not svc: raise CallError(f'Service {id_or_name} not found.', errno.ENOENT) id_or_name = svc[0]['id'] rv = await self.middleware.call('datastore.update', 'services.services', id_or_name, {'srv_enable': data['enable']}) await self.middleware.call('etc.generate', 'rc') return rv @accepts( Str('service'), Dict( 'service-control', Bool('onetime', default=True), Bool('wait', default=None, null=True), Bool('sync', default=None, null=True), register=True, ), ) async def start(self, service, options=None): """ Start the service specified by `service`. The helper will use method self._start_[service]() to start the service. If the method does not exist, it would fallback using service(8).""" await self.middleware.call_hook('service.pre_action', service, 'start', options) sn = self._started_notify("start", service) await self._simplecmd("start", service, options) return await self.started(service, sn) async def started(self, service, sn=None): """ Test if service specified by `service` has been started. """ if sn: await self.middleware.run_in_thread(sn.join) try: svc = await self.query([('service', '=', service)], {'get': True}) self.middleware.send_event('service.query', 'CHANGED', fields=svc) return svc['state'] == 'RUNNING' except IndexError: f = getattr(self, '_started_' + service, None) if callable(f): if inspect.iscoroutinefunction(f): return (await f())[0] else: return f()[0] else: return (await self._started(service))[0] @accepts( Str('service'), Ref('service-control'), ) async def stop(self, service, options=None): """ Stop the service specified by `service`. The helper will use method self._stop_[service]() to stop the service. If the method does not exist, it would fallback using service(8).""" await self.middleware.call_hook('service.pre_action', service, 'stop', options) sn = self._started_notify("stop", service) await self._simplecmd("stop", service, options) return await self.started(service, sn) @accepts( Str('service'), Ref('service-control'), ) async def restart(self, service, options=None): """ Restart the service specified by `service`. The helper will use method self._restart_[service]() to restart the service. If the method does not exist, it would fallback using service(8).""" await self.middleware.call_hook('service.pre_action', service, 'restart', options) sn = self._started_notify("restart", service) await self._simplecmd("restart", service, options) return await self.started(service, sn) @accepts( Str('service'), Ref('service-control'), ) async def reload(self, service, options=None): """ Reload the service specified by `service`. The helper will use method self._reload_[service]() to reload the service. If the method does not exist, the helper will try self.restart of the service instead.""" await self.middleware.call_hook('service.pre_action', service, 'reload', options) try: await self._simplecmd("reload", service, options) except Exception as e: await self.restart(service, options) return await self.started(service) async def _get_status(self, service): f = getattr(self, '_started_' + service['service'], None) if callable(f): if inspect.iscoroutinefunction(f): running, pids = await f() else: running, pids = f() else: running, pids = await self._started(service['service']) if running: state = 'RUNNING' else: state = 'STOPPED' service['state'] = state service['pids'] = pids return service async def _simplecmd(self, action, what, options=None): self.logger.debug("Calling: %s(%s) ", action, what) f = getattr(self, '_' + action + '_' + what, None) if f is None: # Provide generic start/stop/restart verbs for rc.d scripts if what in self.SERVICE_DEFS: if self.SERVICE_DEFS[what].rc_script: what = self.SERVICE_DEFS[what].rc_script if action in ("start", "stop", "restart", "reload"): if action == 'restart': await self._system("/usr/sbin/service " + what + " forcestop ") await self._service(what, action, **options) else: raise ValueError("Internal error: Unknown command") else: call = f(**(options or {})) if inspect.iscoroutinefunction(f): await call async def _system(self, cmd, options=None): stdout = DEVNULL if options and 'stdout' in options: stdout = options['stdout'] stderr = DEVNULL if options and 'stderr' in options: stderr = options['stderr'] proc = await Popen(cmd, stdout=stdout, stderr=stderr, shell=True, close_fds=True) await proc.communicate() return proc.returncode async def _service(self, service, verb, **options): onetime = options.pop('onetime', None) force = options.pop('force', None) quiet = options.pop('quiet', None) extra = options.pop('extra', '') # force comes before one which comes before quiet # they are mutually exclusive preverb = '' if force: preverb = 'force' elif onetime: preverb = 'one' elif quiet: preverb = 'quiet' return await self._system( '/usr/sbin/service {} {}{} {}'.format( service, preverb, verb, extra, ), options) def _started_notify(self, verb, what): """ The check for started [or not] processes is currently done in 2 steps This is the first step which involves a thread StartNotify that watch for event before actually start/stop rc.d scripts Returns: StartNotify object if the service is known or None otherwise """ if what in self.SERVICE_DEFS: sn = StartNotify(verb=verb, pidfile=self.SERVICE_DEFS[what].pidfile) sn.start() return sn else: return None async def _started(self, what, notify=None): """ This is the second step:: Wait for the StartNotify thread to finish and then check for the status of pidfile/procname using pgrep Returns: True whether the service is alive, False otherwise """ if what in self.SERVICE_DEFS: if notify: await self.middleware.run_in_thread(notify.join) if self.SERVICE_DEFS[what].pidfile: pgrep = "/bin/pgrep -F {}{}".format( self.SERVICE_DEFS[what].pidfile, ' ' + self.SERVICE_DEFS[what].procname if self.SERVICE_DEFS[what].procname else '', ) else: pgrep = "/bin/pgrep {}".format( self.SERVICE_DEFS[what].procname) proc = await Popen(pgrep, shell=True, stdout=PIPE, stderr=PIPE, close_fds=True) data = (await proc.communicate())[0].decode() if proc.returncode == 0: return True, [ int(i) for i in data.strip().split('\n') if i.isdigit() ] return False, [] async def _start_asigra(self, **kwargs): await self.middleware.call('asigra.setup_filesystems') await self.middleware.call('asigra.setup_postgresql') await self._service("postgresql", "start", force=True, **kwargs) await self.middleware.call('asigra.setup_asigra') await self.middleware.call('etc.generate', 'asigra') await self._service("dssystem", "start", force=True, **kwargs) async def _stop_asigra(self, **kwargs): await self._service("dssystem", "stop", force=True, **kwargs) await self._service("postgresql", "stop", force=True, **kwargs) async def _restart_asigra(self, **kwargs): await self._stop_asigra(**kwargs) await self._start_asigra(**kwargs) async def _started_asigra(self, **kwargs): if await self._service("dssystem", "status", force=True, ** kwargs) != 0: return False, [] return True, [] async def _start_webdav(self, **kwargs): await self.middleware.call('etc.generate', 'webdav') await self._service("apache24", "start", **kwargs) async def _stop_webdav(self, **kwargs): await self._service("apache24", "stop", **kwargs) async def _restart_webdav(self, **kwargs): await self._service("apache24", "stop", force=True, **kwargs) await self.middleware.call('etc.generate', 'webdav') await self._service("apache24", "restart", **kwargs) async def _reload_webdav(self, **kwargs): await self.middleware.call('etc.generate', 'webdav') await self._service("apache24", "reload", **kwargs) async def _restart_django(self, **kwargs): await self._service("django", "restart", **kwargs) async def _start_webshell(self, **kwargs): await self._system( "/usr/local/bin/python /usr/local/www/freenasUI/tools/webshell.py") async def _restart_webshell(self, **kwargs): try: with open('/var/run/webshell.pid', 'r') as f: pid = f.read() os.kill(int(pid), signal.SIGTERM) time.sleep(0.2) os.kill(int(pid), signal.SIGKILL) except Exception: pass await self._system( "ulimit -n 1024 && /usr/local/bin/python /usr/local/www/freenasUI/tools/webshell.py" ) async def _restart_iscsitarget(self, **kwargs): await self.middleware.call("etc.generate", "ctld") await self._service("ctld", "stop", force=True, **kwargs) await self.middleware.call("etc.generate", "ctld") await self._service("ctld", "restart", **kwargs) async def _start_iscsitarget(self, **kwargs): await self.middleware.call("etc.generate", "ctld") await self._service("ctld", "start", **kwargs) async def _stop_iscsitarget(self, **kwargs): with contextlib.suppress(IndexError): sysctl.filter("kern.cam.ctl.ha_peer")[0].value = "" await self._service("ctld", "stop", force=True, **kwargs) async def _reload_iscsitarget(self, **kwargs): await self.middleware.call("etc.generate", "ctld") await self._service("ctld", "reload", **kwargs) async def _start_collectd(self, **kwargs): if not await self.started('rrdcached'): # Let's ensure that before we start collectd, rrdcached is always running await self.start('rrdcached') await self.middleware.call('etc.generate', 'collectd') await self._service("collectd", "restart", **kwargs) async def _restart_collectd(self, **kwargs): await self._service("collectd", "stop", **kwargs) await self._start_collectd(**kwargs) async def _started_collectd(self, **kwargs): if await self._service('collectd', 'status', quiet=True, **kwargs): return False, [] else: return True, [] async def _started_rrdcached(self, **kwargs): if await self._service('rrdcached', 'status', quiet=True, **kwargs): return False, [] else: return True, [] async def _stop_rrdcached(self, **kwargs): await self._service("collectd", "stop", **kwargs) await self._service('rrdcached', 'stop', **kwargs) async def _restart_rrdcached(self, **kwargs): await self._stop_rrdcached(**kwargs) await self.start('rrdcached') await self.start('collectd') async def _reload_rc(self, **kwargs): await self.middleware.call('etc.generate', 'rc') async def _restart_powerd(self, **kwargs): await self.middleware.call('etc.generate', 'rc') await self._service('powerd', 'restart', **kwargs) async def _reload_sysctl(self, **kwargs): await self.middleware.call('etc.generate', 'sysctl') async def _start_network(self, **kwargs): await self.middleware.call('interface.sync') await self.middleware.call('route.sync') async def _reload_named(self, **kwargs): await self._service("named", "reload", **kwargs) async def _restart_syscons(self, **kwargs): await self.middleware.call('etc.generate', 'rc') await self._service('syscons', 'restart', **kwargs) async def _reload_hostname(self, **kwargs): await self._system('/bin/hostname ""') await self.middleware.call('etc.generate', 'hostname') await self.middleware.call('etc.generate', 'rc') await self._service("hostname", "start", quiet=True, **kwargs) await self._service("mdnsd", "restart", quiet=True, **kwargs) await self._restart_collectd(**kwargs) async def _reload_resolvconf(self, **kwargs): await self._reload_hostname() await self.middleware.call('dns.sync') async def _reload_networkgeneral(self, **kwargs): await self._reload_resolvconf() await self._service("routing", "restart", **kwargs) async def _start_routing(self, **kwargs): await self.middleware.call('etc.generate', 'rc') await self._service('routing', 'start', **kwargs) async def _reload_timeservices(self, **kwargs): await self.middleware.call('etc.generate', 'localtime') await self.middleware.call('etc.generate', 'ntpd') await self._service("ntpd", "restart", **kwargs) settings = await self.middleware.call('datastore.query', 'system.settings', [], { 'order_by': ['-id'], 'get': True }) os.environ['TZ'] = settings['stg_timezone'] time.tzset() async def _restart_ntpd(self, **kwargs): await self.middleware.call('etc.generate', 'ntpd') await self._service('ntpd', 'restart', **kwargs) async def _start_smartd(self, **kwargs): await self.middleware.call("etc.generate", "smartd") await self._service("smartd-daemon", "start", **kwargs) def _initializing_smartd_pid(self): """ smartd initialization can take a long time if lots of disks are present It only writes pidfile at the end of the initialization but forks immediately This method returns PID of smartd process that is still initializing and has not written pidfile yet """ if os.path.exists(self.SERVICE_DEFS["smartd"].pidfile): # Already started, no need for special handling return for process in psutil.process_iter(attrs=["cmdline", "create_time"]): if process.info["cmdline"][:1] == ["/usr/local/sbin/smartd"]: break else: # No smartd process present return lifetime = time.time() - process.info["create_time"] if lifetime < 300: # Looks like just the process we need return process.pid self.logger.warning( "Got an orphan smartd process: pid=%r, lifetime=%r", process.pid, lifetime) async def _started_smartd(self, **kwargs): result = await self._started("smartd") if result[0]: return result if await self.middleware.run_in_thread(self._initializing_smartd_pid ) is not None: return True, [] return False, [] async def _reload_smartd(self, **kwargs): await self.middleware.call("etc.generate", "smartd") pid = await self.middleware.run_in_thread(self._initializing_smartd_pid ) if pid is None: await self._service("smartd-daemon", "reload", **kwargs) return os.kill(pid, signal.SIGKILL) await self._service("smartd-daemon", "start", **kwargs) async def _restart_smartd(self, **kwargs): await self.middleware.call("etc.generate", "smartd") pid = await self.middleware.run_in_thread(self._initializing_smartd_pid ) if pid is None: await self._service("smartd-daemon", "stop", force=True, **kwargs) await self._service("smartd-daemon", "restart", **kwargs) return os.kill(pid, signal.SIGKILL) await self._service("smartd-daemon", "start", **kwargs) async def _stop_smartd(self, **kwargs): pid = await self.middleware.run_in_thread(self._initializing_smartd_pid ) if pid is None: await self._service("smartd-daemon", "stop", force=True, **kwargs) return os.kill(pid, signal.SIGKILL) async def _reload_ssh(self, **kwargs): await self.middleware.call('etc.generate', 'ssh') await self.middleware.call('mdnsadvertise.restart') await self._service("openssh", "reload", **kwargs) await self._service("ix_sshd_save_keys", "start", quiet=True, **kwargs) async def _start_ssh(self, **kwargs): await self.middleware.call('etc.generate', 'ssh') await self.middleware.call('mdnsadvertise.restart') await self._service("openssh", "start", **kwargs) await self._service("ix_sshd_save_keys", "start", quiet=True, **kwargs) async def _stop_ssh(self, **kwargs): await self._service("openssh", "stop", force=True, **kwargs) await self.middleware.call('mdnsadvertise.restart') async def _restart_ssh(self, **kwargs): await self.middleware.call('etc.generate', 'ssh') await self._service("openssh", "stop", force=True, **kwargs) await self.middleware.call('mdnsadvertise.restart') await self._service("openssh", "restart", **kwargs) await self._service("ix_sshd_save_keys", "start", quiet=True, **kwargs) async def _start_ssl(self, **kwargs): await self.middleware.call('etc.generate', 'ssl') async def _start_s3(self, **kwargs): await self.middleware.call('etc.generate', 's3') await self._service("minio", "start", quiet=True, stdout=None, stderr=None, **kwargs) async def _reload_s3(self, **kwargs): await self.middleware.call('etc.generate', 's3') await self._service("minio", "restart", quiet=True, stdout=None, stderr=None, **kwargs) async def _reload_rsync(self, **kwargs): await self.middleware.call('etc.generate', 'rsync') await self._service("rsyncd", "restart", **kwargs) async def _restart_rsync(self, **kwargs): await self._stop_rsync() await self._start_rsync() async def _start_rsync(self, **kwargs): await self.middleware.call('etc.generate', 'rsync') await self._service("rsyncd", "start", **kwargs) async def _stop_rsync(self, **kwargs): await self._service("rsyncd", "stop", force=True, **kwargs) async def _started_nis(self, **kwargs): return (await self.middleware.call('nis.started')), [] async def _start_nis(self, **kwargs): return (await self.middleware.call('nis.start')), [] async def _restart_nis(self, **kwargs): await self.middleware.call('nis.stop') return (await self.middleware.call('nis.start')), [] async def _stop_nis(self, **kwargs): return (await self.middleware.call('nis.stop')), [] async def _started_ldap(self, **kwargs): if (await self._system('/usr/sbin/service ix-ldap status') != 0): return False, [] return await self.middleware.call('notifier.ldap_status'), [] async def _start_ldap(self, **kwargs): await self.middleware.call('etc.generate', 'rc') res = False if not await self._system("/etc/directoryservice/LDAP/ctl start"): res = True return res async def _stop_ldap(self, **kwargs): await self.middleware.call('etc.generate', 'rc') res = False if not await self._system("/etc/directoryservice/LDAP/ctl stop"): res = True return res async def _restart_ldap(self, **kwargs): await self.middleware.call('etc.generate', 'rc') res = False if not await self._system("/etc/directoryservice/LDAP/ctl restart"): res = True return res async def _start_lldp(self, **kwargs): await self._service("ladvd", "start", **kwargs) async def _stop_lldp(self, **kwargs): await self._service("ladvd", "stop", force=True, **kwargs) async def _restart_lldp(self, **kwargs): await self._service("ladvd", "stop", force=True, **kwargs) await self._service("ladvd", "restart", **kwargs) async def _clear_activedirectory_config(self): await self._system( "/bin/rm -f /etc/directoryservice/ActiveDirectory/config") async def _started_activedirectory(self, **kwargs): # Perform a wbinfo -t because it's the most accurate single test we have to # detect problems with AD join. The default winbind timeout is 60 seconds (as of Samba 4.7). # This can be controlled by the smb4.conf parameter "winbind request timeout = " if await self._system('/usr/local/bin/wbinfo -t') != 0: self.logger.debug('AD status check: wbinfo -t failed') return False, [] return True, [] async def _start_activedirectory(self, **kwargs): res = False if not await self._system( "/etc/directoryservice/ActiveDirectory/ctl start"): res = True return res async def _stop_activedirectory(self, **kwargs): res = False if not await self._system( "/etc/directoryservice/ActiveDirectory/ctl stop"): res = True return res async def _restart_activedirectory(self, **kwargs): res = False if not await self._system( "/etc/directoryservice/ActiveDirectory/ctl restart"): res = True return res async def _reload_activedirectory(self, **kwargs): # Steps required to force winbind to connect to new DC if DC it's connected to goes down # We may need to expand the list of operations below to include fresh kinit. Some # information about winbind connection is stored in samba's gencache. In test cases, flushing # gencache (net cache flush) was not required to do this. await self._service("samba_server", "stop", force=True, **kwargs) await self._service("samba_server", "start", quiet=True, **kwargs) async def _restart_syslogd(self, **kwargs): await self.middleware.call("etc.generate", "syslogd") await self._system("/etc/local/rc.d/syslog-ng restart") async def _start_syslogd(self, **kwargs): await self.middleware.call("etc.generate", "syslogd") await self._system("/etc/local/rc.d/syslog-ng start") async def _stop_syslogd(self, **kwargs): await self._system("/etc/local/rc.d/syslog-ng stop") async def _reload_syslogd(self, **kwargs): await self.middleware.call("etc.generate", "syslogd") await self._system("/etc/local/rc.d/syslog-ng reload") async def _start_tftp(self, **kwargs): await self.middleware.call('etc.generate', 'inetd') await self._service("inetd", "start", **kwargs) async def _reload_tftp(self, **kwargs): await self.middleware.call('etc.generate', 'inetd') await self._service("inetd", "stop", force=True, **kwargs) await self._service("inetd", "restart", **kwargs) async def _restart_tftp(self, **kwargs): await self.middleware.call('etc.generate', 'inetd') await self._service("inetd", "stop", force=True, **kwargs) await self._service("inetd", "restart", **kwargs) async def _restart_cron(self, **kwargs): await self.middleware.call('etc.generate', 'cron') async def _start_motd(self, **kwargs): await self.middleware.call('etc.generate', 'motd') await self._service("motd", "start", quiet=True, **kwargs) async def _start_ttys(self, **kwargs): await self.middleware.call('etc.generate', 'ttys') async def _reload_ftp(self, **kwargs): await self.middleware.call("etc.generate", "ftp") await self._service("proftpd", "restart", **kwargs) async def _restart_ftp(self, **kwargs): await self._stop_ftp() await self._start_ftp() async def _start_ftp(self, **kwargs): await self.middleware.call("etc.generate", "ftp") await self._service("proftpd", "start", **kwargs) async def _stop_ftp(self, **kwargs): await self._service("proftpd", "stop", force=True, **kwargs) async def _start_ups(self, **kwargs): await self.middleware.call('etc.generate', 'ups') await self._service("nut", "start", **kwargs) await self._service("nut_upsmon", "start", **kwargs) await self._service("nut_upslog", "start", **kwargs) if await self.started('collectd'): asyncio.ensure_future(self.restart('collectd')) async def _stop_ups(self, **kwargs): await self._service("nut_upslog", "stop", force=True, **kwargs) await self._service("nut_upsmon", "stop", force=True, **kwargs) await self._service("nut", "stop", force=True, **kwargs) if await self.started('collectd'): asyncio.ensure_future(self.restart('collectd')) async def _restart_ups(self, **kwargs): await self.middleware.call('etc.generate', 'ups') await self._service("nut", "stop", force=True, **kwargs) await self._service("nut_upsmon", "stop", force=True, **kwargs) await self._service("nut_upslog", "stop", force=True, **kwargs) await self._service("nut", "restart", **kwargs) await self._service("nut_upsmon", "restart", **kwargs) await self._service("nut_upslog", "restart", **kwargs) if await self.started('collectd'): asyncio.ensure_future(self.restart('collectd')) async def _started_ups(self, **kwargs): return await self._started('upsmon') async def _start_afp(self, **kwargs): await self.middleware.call("etc.generate", "afpd") await self._service("netatalk", "start", **kwargs) async def _stop_afp(self, **kwargs): await self._service("netatalk", "stop", force=True, **kwargs) # when netatalk stops if afpd or cnid_metad is stuck # they'll get left behind, which can cause issues # restarting netatalk. await self._system("pkill -9 afpd") await self._system("pkill -9 cnid_metad") async def _restart_afp(self, **kwargs): await self._stop_afp() await self._start_afp() async def _reload_afp(self, **kwargs): await self.middleware.call("etc.generate", "afpd") await self._system("killall -1 netatalk") async def _reload_nfs(self, **kwargs): await self.middleware.call("etc.generate", "nfsd") async def _restart_nfs(self, **kwargs): await self._stop_nfs(**kwargs) await self._start_nfs(**kwargs) async def _stop_nfs(self, **kwargs): await self._service("lockd", "stop", force=True, **kwargs) await self._service("statd", "stop", force=True, **kwargs) await self._service("nfsd", "stop", force=True, **kwargs) await self._service("mountd", "stop", force=True, **kwargs) await self._service("nfsuserd", "stop", force=True, **kwargs) await self._service("gssd", "stop", force=True, **kwargs) await self._service("rpcbind", "stop", force=True, **kwargs) async def _start_nfs(self, **kwargs): nfs = await self.middleware.call('datastore.config', 'services.nfs') await self.middleware.call("etc.generate", "nfsd") await self._service("rpcbind", "start", quiet=True, **kwargs) await self._service("gssd", "start", quiet=True, **kwargs) # Workaround to work with "onetime", since the rc scripts depend on rc flags. if nfs['nfs_srv_v4']: sysctl.filter('vfs.nfsd.server_max_nfsvers')[0].value = 4 if nfs['nfs_srv_v4_v3owner']: # Per RFC7530, sending NFSv3 style UID/GIDs across the wire is now allowed # You must have both of these sysctl's set to allow the desired functionality sysctl.filter('vfs.nfsd.enable_stringtouid')[0].value = 1 sysctl.filter('vfs.nfs.enable_uidtostring')[0].value = 1 await self._service("nfsuserd", "stop", force=True, **kwargs) else: sysctl.filter('vfs.nfsd.enable_stringtouid')[0].value = 0 sysctl.filter('vfs.nfs.enable_uidtostring')[0].value = 0 await self._service("nfsuserd", "start", quiet=True, **kwargs) else: sysctl.filter('vfs.nfsd.server_max_nfsvers')[0].value = 3 if nfs['nfs_srv_16']: await self._service("nfsuserd", "start", quiet=True, **kwargs) await self._service("mountd", "start", quiet=True, **kwargs) await self._service("nfsd", "start", quiet=True, **kwargs) await self._service("statd", "start", quiet=True, **kwargs) await self._service("lockd", "start", quiet=True, **kwargs) async def _start_dynamicdns(self, **kwargs): await self.middleware.call('etc.generate', 'inadyn') await self._service("inadyn", "start", **kwargs) async def _restart_dynamicdns(self, **kwargs): await self.middleware.call('etc.generate', 'inadyn') await self._service("inadyn", "stop", force=True, **kwargs) await self._service("inadyn", "restart", **kwargs) async def _reload_dynamicdns(self, **kwargs): await self.middleware.call('etc.generate', 'inadyn') await self._service("inadyn", "stop", force=True, **kwargs) await self._service("inadyn", "restart", **kwargs) async def _restart_system(self, **kwargs): asyncio.ensure_future( self.middleware.call('system.reboot', {'delay': 3})) async def _stop_system(self, **kwargs): asyncio.ensure_future( self.middleware.call('system.shutdown', {'delay': 3})) async def _reload_cifs(self, **kwargs): await self.middleware.call("etc.generate", "smb_share") await self._service("samba_server", "reload", force=True, **kwargs) await self._service("mdnsd", "restart", **kwargs) # After mdns is restarted we need to reload netatalk to have it rereregister # with mdns. Ticket #7133 await self._service("netatalk", "reload", **kwargs) async def _restart_cifs(self, **kwargs): await self.middleware.call("etc.generate", "smb") await self.middleware.call("etc.generate", "smb_share") await self._service("samba_server", "stop", force=True, **kwargs) await self._service("samba_server", "restart", quiet=True, **kwargs) await self._service("mdnsd", "restart", **kwargs) # After mdns is restarted we need to reload netatalk to have it rereregister # with mdns. Ticket #7133 await self._service("netatalk", "reload", **kwargs) async def _start_cifs(self, **kwargs): await self.middleware.call("etc.generate", "smb") await self.middleware.call("etc.generate", "smb_share") await self._service("samba_server", "start", quiet=True, **kwargs) async def _stop_cifs(self, **kwargs): await self._service("samba_server", "stop", force=True, **kwargs) async def _started_cifs(self, **kwargs): if await self._service("samba_server", "status", quiet=True, onetime=True, **kwargs): return False, [] else: return True, [] async def _start_snmp(self, **kwargs): await self.middleware.call("etc.generate", "snmpd") await self._service("snmpd", "start", quiet=True, **kwargs) await self._service("snmp-agent", "start", quiet=True, **kwargs) async def _stop_snmp(self, **kwargs): await self._service("snmp-agent", "stop", quiet=True, **kwargs) await self._service("snmpd", "stop", quiet=True, **kwargs) async def _restart_snmp(self, **kwargs): await self._service("snmp-agent", "stop", quiet=True, **kwargs) await self._service("snmpd", "stop", force=True, **kwargs) await self.middleware.call("etc.generate", "snmpd") await self._service("snmpd", "start", quiet=True, **kwargs) await self._service("snmp-agent", "start", quiet=True, **kwargs) async def _reload_snmp(self, **kwargs): await self._service("snmp-agent", "stop", quiet=True, **kwargs) await self._service("snmpd", "stop", force=True, **kwargs) await self.middleware.call("etc.generate", "snmpd") await self._service("snmpd", "start", quiet=True, **kwargs) await self._service("snmp-agent", "start", quiet=True, **kwargs) async def _restart_http(self, **kwargs): await self.middleware.call("etc.generate", "nginx") await self.middleware.call('mdnsadvertise.restart') await self._service("nginx", "restart", **kwargs) async def _reload_http(self, **kwargs): await self.middleware.call("etc.generate", "nginx") await self.middleware.call('mdnsadvertise.restart') await self._service("nginx", "reload", **kwargs) async def _reload_loader(self, **kwargs): await self.middleware.call("etc.generate", "loader") async def _restart_disk(self, **kwargs): await self._reload_disk(**kwargs) async def _reload_disk(self, **kwargs): await self.middleware.call('etc.generate', 'fstab') await self._service("mountlate", "start", quiet=True, **kwargs) # Restarting rrdcached can take a long time. There is no # benefit in waiting for it, since even if it fails it will not # tell the user anything useful. asyncio.ensure_future(self.restart("collectd", kwargs)) async def _reload_user(self, **kwargs): await self.middleware.call("etc.generate", "user") await self.middleware.call('etc.generate', 'aliases') await self.middleware.call('etc.generate', 'sudoers') await self.reload("cifs", kwargs) async def _restart_system_datasets(self, **kwargs): systemdataset = await self.middleware.call('systemdataset.setup') if not systemdataset: return None if systemdataset['syslog']: await self.restart("syslogd", kwargs) await self.restart("cifs", kwargs) # Restarting rrdcached can take a long time. There is no # benefit in waiting for it, since even if it fails it will not # tell the user anything useful. # Restarting rrdcached will make sure that we start/restart collectd as well asyncio.ensure_future(self.restart("rrdcached", kwargs)) async def _start_netdata(self, **kwargs): await self.middleware.call('etc.generate', 'netdata') await self._service('netdata', 'start', **kwargs) async def _restart_netdata(self, **kwargs): await self._service('netdata', 'stop') await self._start_netdata(**kwargs)
class IdmapTDBService(CRUDService): class Config: datastore = 'directoryservice.idmap_tdb' datastore_prefix = 'idmap_tdb_' datastore_extend = 'idmap.common_backend_extend' namespace = 'idmap.tdb' @accepts( Dict('idmap_tdb_create', Dict( 'domain', Int('id'), Str('idmap_domain_name'), Str('idmap_domain_dns_domain_name'), ), Int('range_low', required=True, validators=[Range(min=1000, max=2147483647)]), Int('range_high', required=True, validators=[Range(min=1000, max=2147483647)]), register=True)) async def do_create(self, data): """ Create an entry in the idmap backend table. """ verrors = ValidationErrors() verrors.add_child( 'idmap_tdb_create', await self.middleware.call('idmap._common_validate', 'tdb', data)) if verrors: raise verrors data = await self.middleware.call('idmap.common_backend_compress', data) data["id"] = await self.middleware.call( "datastore.insert", self._config.datastore, data, {"prefix": self._config.datastore_prefix}, ) return await self._get_instance(data['id']) @accepts(Int('id', required=True), Patch("idmap_tdb_create", "idmap_tdb_update", ("attr", { "update": True }))) async def do_update(self, id, data): """ Update an entry in the idmap backend table by id. """ old = await self._get_instance(id) new = old.copy() new.update(data) verrors = ValidationErrors() verrors.add_child( 'idmap_tdb_update', await self.middleware.call('idmap._common_validate', 'tdb', new)) if verrors: raise verrors new = await self.middleware.call('idmap.common_backend_compress', new) await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) return await self._get_instance(id) @accepts(Int('id')) async def do_delete(self, id): """ Delete idmap to backend mapping by id """ await self.middleware.call("datastore.delete", self._config.datastore, id)
class RsyncService(Service): def __rsync_worker(self, line, user, job): proc_stdout = tempfile.TemporaryFile(mode='w+b', buffering=0) try: rsync_proc = subprocess.Popen( line, shell=True, stdout=proc_stdout.fileno(), stderr=subprocess.PIPE, bufsize=0, preexec_fn=demote(user) ) seek = 0 old_seek = 0 progress = 0 message = 'Starting rsync copy job...' while rsync_proc.poll() is None: job.set_progress(progress, message) proc_op = '' proc_stdout.seek(seek) try: while True: op_byte = proc_stdout.read(1).decode('utf8') if op_byte == '': # In this case break before incrementing `seek` break seek += 1 if op_byte == '\r': break proc_op += op_byte seek += 1 if old_seek != seek: old_seek = seek message = proc_op.strip() try: progress = int([x for x in message.split(' ') if '%' in x][0][:-1]) except (IndexError, ValueError): pass except BaseException as err: # Catch IOERROR Errno 9 which usually arises because # of already closed fileobject being used here therby # raising Bad File Descriptor error. In this case break # and the outer while loop will check for rsync_proc.poll() # to be None or not and DTRT if hasattr(err, 'errno') and err.errno == 9: break logger.debug('Error whilst parsing rsync progress', exc_info=True) except BaseException as e: raise CallError(f'Rsync copy job id: {job.id} failed due to: {e}', errno.EIO) if rsync_proc.returncode != 0: job.set_progress(None, 'Rsync copy job failed') raise CallError( f'Rsync copy job id: {job.id} returned non-zero exit code. Command used was: {line}. Error: {rsync_proc.stderr.read()}' ) @accepts(Dict( 'rsync-copy', Str('user', required=True), Str('path', required=True), Str('remote_user'), Str('remote_host', required=True), Str('remote_path'), Int('remote_ssh_port'), Str('remote_module'), Str('direction', enum=['PUSH', 'PULL'], required=True), Str('mode', enum=['MODULE', 'SSH'], required=True), Str('remote_password'), Dict( 'properties', Bool('recursive'), Bool('compress'), Bool('times'), Bool('archive'), Bool('delete'), Bool('preserve_permissions'), Bool('preserve_attributes'), Bool('delay_updates') ), required=True )) @job() def copy(self, job, rcopy): """ Starts an rsync copy task between current freenas machine and specified remote host (or local copy too). It reports the progress of the copy task. """ # Assigning variables and such user = rcopy.get('user') path = rcopy.get('path') mode = rcopy.get('mode') remote_path = rcopy.get('remote_path') remote_host = rcopy.get('remote_host') remote_module = rcopy.get('remote_module') remote_user = rcopy.get('remote_user', rcopy.get('user')) remote_address = remote_host if '@' in remote_host else f'"{remote_user}"@{remote_host}' remote_password = rcopy.get('remote_password', None) password_file = None properties = rcopy.get('properties', defaultdict(bool)) # Let's do a brief check of all the user provided parameters if not path: raise ValueError('The path is required') elif not os.path.exists(path): raise CallError(f'The specified path: {path} does not exist', errno.ENOENT) if not remote_host: raise ValueError('The remote host is required') if mode == 'SSH' and not remote_path: raise ValueError('The remote path is required') elif mode == 'MODULE' and not remote_module: raise ValueError('The remote module is required') try: pwd.getpwnam(user) except KeyError: raise CallError(f'User: {user} does not exist', errno.ENOENT) if ( mode == 'SSH' and rcopy.get('remote_host') in ['127.0.0.1', 'localhost'] and not os.path.exists(remote_path) ): raise CallError(f'The specified path: {remote_path} does not exist', errno.ENOENT) # Phew! with that out of the let's begin the transfer line = f'{RSYNC_PATH} --info=progress2 -h' if properties: if properties.get('recursive'): line += ' -r' if properties.get('times'): line += ' -t' if properties.get('compress'): line += ' -z' if properties.get('archive'): line += ' -a' if properties.get('preserve_permissions'): line += ' -p' if properties.get('preserve_attributes'): line += ' -X' if properties.get('delete'): line += ' --delete-delay' if properties.get('delay_updates'): line += ' --delay-updates' if mode == 'MODULE': if rcopy.get('direction') == 'PUSH': line += f' "{path}" {remote_address}::"{remote_module}"' else: line += f' {remote_address}::"{remote_module}" "{path}"' if remote_password: password_file = tempfile.NamedTemporaryFile(mode='w') password_file.write(remote_password) password_file.flush() shutil.chown(password_file.name, user=user) os.chmod(password_file.name, 0o600) line += f' --password-file={password_file.name}' else: # there seems to be some code duplication here but hey its simple # if you find a way (THAT DOES NOT BREAK localhost based rsync copies) # then please go for it if rcopy.get('remote_host') in ['127.0.0.1', 'localhost']: if rcopy['direction'] == 'PUSH': line += f' "{path}" "{remote_path}"' else: line += f' "{remote_path}" "{path}"' else: line += ' -e "ssh -p {0} -o BatchMode=yes -o StrictHostKeyChecking=yes"'.format( rcopy.get('remote_ssh_port', 22) ) if rcopy['direction'] == 'PUSH': line += f' "{path}" {remote_address}:\\""{remote_path}"\\"' else: line += f' {remote_address}:\\""{remote_path}"\\" "{path}"' logger.debug(f'Executing rsync job id: {job.id} with the following command {line}') try: t = threading.Thread(target=self.__rsync_worker, args=(line, user, job), daemon=True) t.start() t.join() finally: if password_file: password_file.close() job.set_progress(100, 'Rsync copy job successfully completed')