class StaticRouteService(CRUDService): class Config: datastore = 'network.staticroute' datastore_prefix = 'sr_' datastore_extend = 'staticroute.upper' @accepts(Dict( 'staticroute_create', IPAddr('destination', cidr=True), IPAddr('gateway'), Str('description'), register=True )) async def do_create(self, data): await self.lower(data) id = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self.middleware.call('service.start', 'routing') return await self._get_instance(id) @accepts( Int('id'), Patch( 'staticroute_create', 'staticroute_update', ('attr', {'update': True}) ) ) async def do_update(self, id, data): old = await self._get_instance(id) new = old.copy() new.update(data) await self.lower(data) await self.middleware.call( 'datastore.update', self._config.datastore, id, data, {'prefix': self._config.datastore_prefix}) await self.middleware.call('service.start', 'routing') return await self._get_instance(id) @accepts(Int('id')) async def do_delete(self, id): return await self.middleware.call( 'datastore.delete', self._config.datastore, id) @private async def lower(self, data): data['description'] = data['description'].lower() return data @private async def upper(self, data): data['description'] = data['description'].upper() return data
class NFSService(SystemServiceService): class Config: service = "nfs" datastore_prefix = "nfs_srv_" datastore_extend = 'nfs.nfs_extend' @private def nfs_extend(self, nfs): nfs["userd_manage_gids"] = nfs.pop("16") return nfs @private def nfs_compress(self, nfs): nfs["16"] = nfs.pop("userd_manage_gids") return nfs @accepts(Dict( 'nfs_update', Int('servers', validators=[Range(min=1, max=256)]), Bool('udp'), Bool('allow_nonroot'), Bool('v4'), Bool('v4_v3owner'), Bool('v4_krb'), List('bindip', items=[IPAddr('ip')]), Int('mountd_port', required=False, validators=[Range(min=1, max=65535)]), Int('rpcstatd_port', required=False, validators=[Range(min=1, max=65535)]), Int('rpclockd_port', required=False, validators=[Range(min=1, max=65535)]), Bool('userd_manage_gids'), Bool('mountd_log'), Bool('statd_lockd_log'), )) async def do_update(self, data): old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() if not new["v4"] and new["v4_v3owner"]: verrors.add("nfs_update.v4_v3owner", "This option requires enabling NFSv4") if new["v4_v3owner"] and new["userd_manage_gids"]: verrors.add( "nfs_update.userd_manage_gids", "This option is incompatible with NFSv3 ownership model for NFSv4") if verrors: raise verrors self.nfs_compress(new) await self._update_service(old, new) self.nfs_extend(new) return new
def test__schema_ipaddr_cidr(value, expected): @accepts(IPAddr('data', cidr=True)) def ipaddrv(self, data): return data self = Mock() if expected is ValidationErrors: with pytest.raises(ValidationErrors): ipaddrv(self, value) else: assert ipaddrv(self, value) == expected
def query(self, filters, options): """ Query Name Servers with `query-filters` and `query-options`. """ ips = set() with contextlib.suppress(Exception): with open('/etc/resolv.conf') as f: for line in f: if line.startswith('nameserver'): ip = line[len('nameserver'):].strip() try: IPAddr().validate( ip ) # make sure it's a valid IP (better safe than sorry) ips.add(ip) except ValidationErrors: self.logger.warning( 'IP %r in resolv.conf does not seem to be valid', ip) continue return filter_list([{'nameserver': i} for i in ips], filters, options)
class TruecommandService(ConfigService): STATUS = Status.DISABLED class Config: datastore = 'system.truecommand' datastore_extend = 'truecommand.tc_extend' cli_namespace = 'system.truecommand' ENTRY = Dict( 'truecommand_entry', Int('id', required=True), Str('api_key', required=True, null=True), Str('status', required=True, enum=[s.value for s in Status]), Str('status_reason', required=True, enum=[s.value for s in StatusReason]), Str('remote_url', required=True, null=True), IPAddr('remote_ip_address', required=True, null=True), Bool('enabled', required=True), ) @private async def tc_extend(self, config): for key in ('wg_public_key', 'wg_private_key', 'tc_public_key', 'endpoint', 'wg_address'): config.pop(key) # In database we will have CONNECTED when the portal has approved the key # Connecting basically represents 2 phases - where we wait for TC to connect to # NAS and where we are waiting to hear back from the portal after registration status_reason = None if Status( config.pop('api_key_state') ) == self.STATUS.CONNECTED and self.STATUS == Status.CONNECTING: if await self.middleware.call( 'truecommand.wireguard_connection_health'): await self.set_status(Status.CONNECTED.value) else: status_reason = 'Waiting for connection from Truecommand.' config['remote_ip_address'] = config['remote_url'] = config.pop( 'remote_address') if config['remote_ip_address']: config['remote_ip_address'] = config.pop( 'remote_ip_address').split('/', 1)[0] config['remote_url'] = f'http://{config["remote_ip_address"]}/' config.update({ 'status': self.STATUS.value, 'status_reason': status_reason or StatusReason.__members__[self.STATUS.value].value }) return config @accepts( Dict( 'truecommand_update', Bool('enabled'), Str('api_key', null=True, validators=[Range(min=16, max=16)]), )) async def do_update(self, data): """ Update Truecommand service settings. `api_key` is a valid API key generated by iX Portal. """ # We have following cases worth mentioning wrt updating TC credentials # 1) User enters API Key and enables the service # 2) User disables the service # 3) User changes API Key and service is enabled # # Another point to document is how we intend to poll, we are going to send a request to iX Portal # and if it returns active state with the data we require for wireguard connection, we mark the # API Key as connected. As long as we keep polling iX portal, we are going to be in a connecting state, # no matter what errors we are getting from the polling bits. The failure case is when iX Portal sends # us the state "unknown", which after confirming with Ken means that the portal has revoked the api key # in question and we no longer use it. In this case we are going to stop polling and mark the connection # as failed. # # For case (1), when user enters API key and enables the service, we are first going to generate wg keys # if they haven't been generated already. Then we are going to register the new api key with ix portal. # Once done, we are going to start polling. If polling gets us in success state, we are going to start # wireguard connection, for the other case, we are going to emit an event with truecommand failure status. # # For case (2), if the service was running previously, we do nothing except for stopping wireguard and # ensuring it is not started at boot as well. The connection details remain secure in the database. # # For case (3), everything is similar to how we handle case (1), however we are going to stop wireguard # if it was running with previous api key credentials. async with TRUECOMMAND_UPDATE_LOCK: old = await self.middleware.call('datastore.config', self._config.datastore) new = old.copy() new.update(data) verrors = ValidationErrors() if new['enabled'] and not new['api_key']: verrors.add( 'truecommand_update.api_key', 'API Key must be provided when Truecommand service is enabled.' ) verrors.check() if all(old[k] == new[k] for k in ('enabled', 'api_key')): # Nothing changed return await self.config() polling_jobs = await self.middleware.call( 'core.get_jobs', [['method', '=', 'truecommand.poll_api_for_status'], ['state', 'in', ['WAITING', 'RUNNING']]]) for polling_job in polling_jobs: await self.middleware.call('core.job_abort', polling_job['id']) await self.set_status(Status.DISABLED.value) new['api_key_state'] = Status.DISABLED.value if new['enabled']: if not old['wg_public_key'] or not old['wg_private_key']: new.update(**(await self.middleware.call( 'truecommand.generate_wg_keys'))) if old['api_key'] != new['api_key']: await self.middleware.call( 'truecommand.register_with_portal', new) # Registration succeeded, we are good to poll now elif all(new[k] for k in ('wg_address', 'wg_private_key', 'remote_address', 'endpoint', 'tc_public_key')): # Api key hasn't changed and we have wireguard details, let's please start wireguard in this case await self.set_status(Status.CONNECTING.value) new['api_key_state'] = Status.CONNECTED.value if old['api_key'] != new['api_key']: new.update({ 'remote_address': None, 'endpoint': None, 'tc_public_key': None, 'wg_address': None, 'api_key_state': Status.DISABLED.value, }) await self.dismiss_alerts(True) await self.middleware.call('datastore.update', self._config.datastore, old['id'], new) self.middleware.send_event('truecommand.config', 'CHANGED', fields=(await self.config())) # We are going to stop truecommand service with this update anyways as only 2 possible actions # can happen on update # 1) Service enabled/disabled # 2) Api Key changed await self.middleware.call('truecommand.stop_truecommand_service') if new['enabled']: if new['api_key'] != old['api_key'] or any( not new[k] for k in ('wg_address', 'wg_private_key', 'remote_address', 'endpoint', 'tc_public_key')): # We are going to start polling here await self.middleware.call( 'truecommand.poll_api_for_status') else: # User just enabled the service after disabling it - we have wireguard details and # we can initiate the connection. If it is not good, health check will fail and we will # poll iX Portal to see what's up. Let's just start wireguard now await self.middleware.call( 'truecommand.start_truecommand_service') return await self.config() @private async def set_status(self, new_status): assert new_status in Status.__members__ self.STATUS = Status(new_status) self.middleware.send_event('truecommand.config', 'CHANGED', fields=(await self.config())) @private async def dismiss_alerts(self, dismiss_health=False): # We do not dismiss health by default because it's possible that the key has not been revoked # and it's just that TC has not connected to TN in 30 minutes, so we only should dismiss it when # we update TC service or the health is okay now with the service running or when service is not running for klass in [ 'TruecommandConnectionDisabled', 'TruecommandConnectionPending' ] + (['TruecommandConnectionHealth'] if dismiss_health else []): await self.middleware.call('alert.oneshot_delete', klass, None)
class KubernetesService(ConfigService): class Config: datastore = 'services.kubernetes' datastore_extend = 'kubernetes.k8s_extend' cli_namespace = 'app.kubernetes' ENTRY = Dict( 'kubernetes_entry', Bool('configure_gpus', required=True), Str('pool', required=True, null=True), IPAddr('cluster_cidr', required=True, cidr=True, empty=True), IPAddr('service_cidr', required=True, cidr=True, empty=True), IPAddr('cluster_dns_ip', required=True, empty=True), IPAddr('node_ip', required=True), Str('route_v4_interface', required=True, null=True), IPAddr('route_v4_gateway', required=True, null=True, v6=False), Str('route_v6_interface', required=True, null=True), IPAddr('route_v6_gateway', required=True, null=True, v4=False), Str('dataset', required=True, null=True), Int('id', required=True), update=True, ) @private async def k8s_extend(self, data): data['dataset'] = applications_ds_name( data['pool']) if data['pool'] else None data.pop('cni_config') return data @private async def unused_cidrs(self, network_cidrs): return [ str(network) for network in itertools.chain( ipaddress.ip_network('172.16.0.0/12', False).subnets(4), ipaddress.ip_network('10.0.0.0/8', False).subnets(8), ipaddress.ip_network('192.168.0.0/16', False).subnets(1), ) if not any( network.overlaps(used_network) for used_network in network_cidrs) ] @private async def validate_data(self, data, schema, old_data): verrors = ValidationErrors() if data.pop('migrate_applications', False): if data['pool'] == old_data['pool']: verrors.add( f'{schema}.migrate_applications', 'Migration of applications dataset only happens when a new pool is configured.' ) elif not data['pool']: verrors.add( f'{schema}.migrate_applications', 'Pool must be specified when migration of ix-application dataset is desired.' ) elif not old_data['pool']: verrors.add( f'{schema}.migrate_applications', 'A pool must have been configured previously for ix-application dataset migration.' ) else: if await self.middleware.call( 'zfs.dataset.query', [['id', '=', applications_ds_name(data['pool'])]]): verrors.add( f'{schema}.migrate_applications', f'Migration of {applications_ds_name(old_data["pool"])!r} to {data["pool"]!r} not ' f'possible as {applications_ds_name(data["pool"])} already exists.' ) if not await self.middleware.call( 'zfs.dataset.query', [['id', '=', applications_ds_name(old_data['pool'])]]): # Edge case but handled just to be sure verrors.add( f'{schema}.migrate_applications', f'{applications_ds_name(old_data["pool"])!r} does not exist, migration not possible.' ) network_cidrs = set([ ipaddress.ip_network( f'{ip_config["address"]}/{ip_config["netmask"]}', False) for interface in await self.middleware.call('interface.query') for ip_config in itertools.chain(interface['aliases'], interface['state']['aliases']) if ip_config['type'] != 'LINK' ]) unused_cidrs = [] if not data['cluster_cidr'] or not data['service_cidr']: unused_cidrs = await self.unused_cidrs(network_cidrs) # If index 0,1 belong to different classes, let's make sure that is not the case anymore if len(unused_cidrs) > 2 and unused_cidrs[0].split( '.')[0] != unused_cidrs[1].split('.')[0]: unused_cidrs.pop(0) if unused_cidrs and not data['cluster_cidr']: data['cluster_cidr'] = unused_cidrs.pop(0) if unused_cidrs and not data['service_cidr']: data['service_cidr'] = unused_cidrs.pop(0) if not data['cluster_dns_ip']: if data['service_cidr']: # Picking 10th ip ( which is the usual default ) from service cidr data['cluster_dns_ip'] = str( list( ipaddress.ip_network(data['service_cidr'], False).hosts())[9]) else: verrors.add(f'{schema}.cluster_dns_ip', 'Please specify cluster_dns_ip.') if data['pool'] and not await self.middleware.call( 'pool.query', [['name', '=', data['pool']]]): verrors.add( f'{schema}.pool', 'Please provide a valid pool configured in the system.') for k in ('cluster_cidr', 'service_cidr'): if not data[k]: verrors.add(f'{schema}.{k}', f'Please specify a {k.split("_")[0]} CIDR.') elif any( ipaddress.ip_network(data[k], False).overlaps(cidr) for cidr in network_cidrs): verrors.add(f'{schema}.{k}', 'Requested CIDR is already in use.') if data['cluster_cidr'] and data[ 'service_cidr'] and ipaddress.ip_network( data['cluster_cidr'], False).overlaps( ipaddress.ip_network(data['service_cidr'], False)): verrors.add(f'{schema}.cluster_cidr', 'Must not overlap with service CIDR.') if data['service_cidr'] and data[ 'cluster_dns_ip'] and ipaddress.ip_address( data['cluster_dns_ip']) not in ipaddress.ip_network( data['service_cidr']): verrors.add(f'{schema}.cluster_dns_ip', 'Must be in range of "service_cidr".') if data['node_ip'] not in await self.bindip_choices(): verrors.add(f'{schema}.node_ip', 'Please provide a valid IP address.') if not await self.middleware.call('route.configured_default_ipv4_route' ): verrors.add( f'{schema}.route_v4_interface', 'Please, set IPv4 Default Gateway (it can be fake) in Network → Global Configuration and then ' 'update Kubernetes settings. Currently, k3s cannot be used without a default route.' ) for k, _ in await self.validate_interfaces(data): verrors.add(f'{schema}.{k}', 'Please specify a valid interface.') for k in ('route_v4', 'route_v6'): gateway = data[f'{k}_gateway'] interface = data[f'{k}_interface'] if (not gateway and not interface) or (gateway and interface): continue for k2 in ('gateway', 'interface'): verrors.add( f'{schema}.{k}_{k2}', f'{k}_gateway and {k}_interface must be specified together.' ) verrors.check() @private async def validate_interfaces(self, data): errors = [] interfaces = { i['name']: i for i in await self.middleware.call('interface.query') } for k in filter(lambda k: data[k] and data[k] not in interfaces, ('route_v4_interface', 'route_v6_interface')): errors.append((k, data[k])) return errors @private async def validate_config(self): data = await self.middleware.call('kubernetes.config') data.pop('id') data.pop('dataset') try: await self.validate_data(data, 'kubernetes', data) except ValidationErrors as e: return e @accepts( Patch( 'kubernetes_entry', 'kubernetes_update', ('add', Bool('migrate_applications')), ('rm', { 'name': 'id' }), ('rm', { 'name': 'dataset' }), ('attr', { 'update': True }), )) @job(lock='kubernetes_update') async def do_update(self, job, data): """ `pool` must be a valid ZFS pool configured in the system. Kubernetes service will initialise the pool by creating datasets under `pool_name/ix-applications`. `cluster_cidr` is the CIDR to be used for default NAT network between workloads. `service_cidr` is the CIDR to be used for kubernetes services which are an abstraction and refer to a logically set of kubernetes pods. `cluster_dns_ip` is the IP of the DNS server running for the kubernetes cluster. It must be in the range of `service_cidr`. Specifying values for `cluster_cidr`, `service_cidr` and `cluster_dns_ip` are permanent and a subsequent change requires re-initialisation of the applications. To clarify, system will destroy old `ix-applications` dataset and any data within it when any of the values for the above configuration change. `node_ip` is the IP address which the kubernetes cluster will assign to the TrueNAS node. It defaults to 0.0.0.0 and the cluster in this case will automatically manage which IP address to use for managing traffic for default NAT network. By default kubernetes pods will be using default gateway of the system for outward traffic. This might not be desirable for certain users who want to separate NAT traffic over a specific interface / route. System will create a L3 network which will be routing the traffic towards default gateway for NAT. If users want to restrict traffic over a certain gateway / interface, they can specify a default route for the NAT traffic. `route_v4_interface` and `route_v4_gateway` will set a default route for the kubernetes cluster IPv4 traffic. Similarly `route_v6_interface` and 'route_v6_gateway` can be used to specify default route for IPv6 traffic. In case user is switching pools and the new desired pool has not been configured for kubernetes before, it is possible to replicate data from old pool to new pool with setting `migrate_applications` attribute. This will replicate contents of old pool's ix-applications dataset to the new pool. """ old_config = await self.config() old_config.pop('dataset') config = old_config.copy() config.update(data) migrate = config.get('migrate_applications') await self.validate_data(config, 'kubernetes_update', old_config) if migrate and config['pool'] != old_config['pool']: job.set_progress( 25, f'Migrating {applications_ds_name(old_config["pool"])} to {applications_ds_name(config["pool"])}' ) await self.migrate_ix_applications_dataset(config['pool'], old_config['pool']) job.set_progress(40, 'Migration complete for ix-applications dataset') if len(set(old_config.items()) ^ set(config.items())) > 0: await self.middleware.call( 'chart.release.clear_update_alerts_for_all_chart_releases') config['cni_config'] = {} await self.middleware.call('datastore.update', self._config.datastore, old_config['id'], config) await self.middleware.call('kubernetes.status_change') if not config['pool'] and config['pool'] != old_config['pool']: # We only want to do this when we don't have any pool configured and would like to use # host catalog repos temporarily. Otherwise, we should call this after k8s datasets have # been initialised await self.middleware.call('catalog.sync_all') return await self.config() @private async def migrate_ix_applications_dataset(self, new_pool, old_pool): snap_details = await self.middleware.call( 'zfs.snapshot.create', { 'dataset': applications_ds_name(old_pool), 'naming_schema': MIGRATION_NAMING_SCHEMA, 'recursive': True, }) try: old_ds = applications_ds_name(old_pool) new_ds = applications_ds_name(new_pool) migrate_job = await self.middleware.call( 'replication.run_onetime', { 'direction': 'PUSH', 'transport': 'LOCAL', 'source_datasets': [old_ds], 'target_dataset': new_ds, 'recursive': True, 'also_include_naming_schema': [MIGRATION_NAMING_SCHEMA], 'retention_policy': 'NONE', 'replicate': True, 'readonly': 'IGNORE', }) await migrate_job.wait() if migrate_job.error: raise CallError( f'Failed to migrate {old_ds} to {new_ds}: {migrate_job.error}' ) finally: await self.middleware.call('zfs.snapshot.delete', snap_details['id'], {'recursive': True}) snap_name = f'{applications_ds_name(new_pool)}@{snap_details["snapshot_name"]}' if await self.middleware.call('zfs.snapshot.query', [['id', '=', snap_name]]): await self.middleware.call('zfs.snapshot.delete', snap_name, {'recursive': True}) @accepts() @returns(Dict( 'kubernetes_bind_ip_choices', additional_attrs=True, )) async def bindip_choices(self): """ Returns ip choices for Kubernetes service to use. """ return { d['address']: d['address'] for d in await self.middleware.call('interface.ip_in_use', { 'static': True, 'any': True }) } @private async def validate_k8s_setup(self): k8s_config = await self.middleware.call('kubernetes.config') if not k8s_config['dataset']: raise CallError('Please configure kubernetes pool.') if not await self.middleware.call('service.started', 'kubernetes'): raise CallError('Kubernetes service is not running.') @accepts() @returns(Str('kubernetes_node_ip', null=True)) async def node_ip(self): """ Returns IP used by kubernetes which kubernetes uses to allow incoming connections. """ node_ip = None if await self.middleware.call('service.started', 'kubernetes'): k8s_node_config = await self.middleware.call('k8s.node.config') if k8s_node_config['node_configured']: node_ip = next( (addr['address'] for addr in k8s_node_config['status']['addresses'] if addr['type'] == 'InternalIP'), None) if not node_ip: node_ip = (await self.middleware.call('kubernetes.config'))['node_ip'] return node_ip
class SharingNFSService(CRUDService): class Config: namespace = "sharing.nfs" datastore = "sharing.nfs_share" datastore_prefix = "nfs_" datastore_extend = "sharing.nfs.extend" @accepts(Dict( "sharingnfs_create", List("paths", items=[Dir("path")], empty=False), Str("comment", default=""), List("networks", items=[IPAddr("network", network=True)], default=[]), List("hosts", items=[Str("host")], default=[]), Bool("alldirs", default=False), Bool("ro", default=False), Bool("quiet", default=False), Str("maproot_user", required=False, default=None, null=True), Str("maproot_group", required=False, default=None, null=True), Str("mapall_user", required=False, default=None, null=True), Str("mapall_group", required=False, default=None, null=True), List( "security", default=[], items=[Str("provider", enum=["SYS", "KRB5", "KRB5I", "KRB5P"])], ), Bool("enabled", default=True), register=True, )) async def do_create(self, data): """ Create a NFS Share. `paths` is a list of valid paths which are configured to be shared on this share. `networks` is a list of authorized networks that are allowed to access the share having format "network/mask" CIDR notation. If empty, all networks are allowed. `hosts` is a list of IP's/hostnames which are allowed to access the share. If empty, all IP's/hostnames are allowed. `alldirs` is a boolean value which when set indicates that the client can mount any subdirectories of the selected pool or dataset. """ verrors = ValidationErrors() await self.validate(data, "sharingnfs_create", verrors) if verrors: raise verrors await self.compress(data) paths = data.pop("paths") data["id"] = await self.middleware.call( "datastore.insert", self._config.datastore, data, { "prefix": self._config.datastore_prefix }, ) for path in paths: await self.middleware.call( "datastore.insert", "sharing.nfs_share_path", { "share_id": data["id"], "path": path, }, ) await self.extend(data) await self._service_change("nfs", "reload") return data @accepts( Int("id"), Patch( "sharingnfs_create", "sharingnfs_update", ("attr", {"update": True}) ) ) async def do_update(self, id, data): """ Update NFS Share of `id`. """ verrors = ValidationErrors() old = await self._get_instance(id) new = old.copy() new.update(data) await self.validate(new, "sharingnfs_update", verrors, old=old) if verrors: raise verrors await self.compress(new) paths = new.pop("paths") await self.middleware.call( "datastore.update", self._config.datastore, id, new, { "prefix": self._config.datastore_prefix } ) await self.middleware.call("datastore.delete", "sharing.nfs_share_path", [["share_id", "=", id]]) for path in paths: await self.middleware.call( "datastore.insert", "sharing.nfs_share_path", { "share_id": id, "path": path, }, ) await self.extend(new) new["paths"] = paths await self._service_change("nfs", "reload") return new @accepts(Int("id")) async def do_delete(self, id): """ Delete NFS Share of `id`. """ await self.middleware.call("datastore.delete", self._config.datastore, id) await self._service_change("nfs", "reload") @private async def validate(self, data, schema_name, verrors, old=None): if data["alldirs"] and len(data["paths"]) > 1: verrors.add(f"{schema_name}.alldirs", "This option can only be used for shares that contain single path") await self.middleware.run_in_thread(self.validate_paths, data, schema_name, verrors) filters = [] if old: filters.append(["id", "!=", old["id"]]) other_shares = await self.middleware.call("sharing.nfs.query", filters) dns_cache = await self.resolve_hostnames( sum([share["hosts"] for share in other_shares], []) + data["hosts"] ) await self.middleware.run_in_thread( self.validate_hosts_and_networks, other_shares, data, schema_name, verrors, dns_cache ) for k in ["maproot", "mapall"]: if not data[f"{k}_user"] and not data[f"{k}_group"]: pass elif not data[f"{k}_user"] and data[f"{k}_group"]: verrors.add(f"{schema_name}.{k}_user", "This field is required when map group is specified") else: user = group = None with contextlib.suppress(KeyError): user = await self.middleware.call('dscache.get_uncached_user', data[f'{k}_user']) if not user: verrors.add(f"{schema_name}.{k}_user", "User not found") if data[f'{k}_group']: with contextlib.suppress(KeyError): group = await self.middleware.call('dscache.get_uncached_group', data[f'{k}_group']) if not group: verrors.add(f"{schema_name}.{k}_group", "Group not found") if data["maproot_user"] and data["mapall_user"]: verrors.add(f"{schema_name}.mapall_user", "maproot_user disqualifies mapall_user") if data["security"]: nfs_config = await self.middleware.call("nfs.config") if not nfs_config["v4"]: verrors.add(f"{schema_name}.security", "This is not allowed when NFS v4 is disabled") @private def validate_paths(self, data, schema_name, verrors): dev = None for i, path in enumerate(data["paths"]): stat = os.stat(path) if dev is None: dev = stat.st_dev else: if dev != stat.st_dev: verrors.add(f"{schema_name}.paths.{i}", "Paths for a NFS share must reside within the same filesystem") @private async def resolve_hostnames(self, hostnames): hostnames = list(set(hostnames)) async def resolve(hostname): try: return ( await asyncio.wait_for(self.middleware.run_in_thread(socket.getaddrinfo, hostname, None), 5) )[0][4][0] except Exception as e: self.logger.warning("Unable to resolve host %r: %r", hostname, e) return None resolved_hostnames = await asyncio_map(resolve, hostnames, 8) return dict(zip(hostnames, resolved_hostnames)) @private def validate_hosts_and_networks(self, other_shares, data, schema_name, verrors, dns_cache): dev = os.stat(data["paths"][0]).st_dev used_networks = set() for share in other_shares: try: share_dev = os.stat(share["paths"][0]).st_dev except Exception: self.logger.warning("Failed to stat first path for %r", share, exc_info=True) continue if share_dev == dev: for host in share["hosts"]: host = dns_cache[host] if host is None: continue try: network = ipaddress.ip_network(host) except Exception: self.logger.warning("Got invalid host %r", host) continue else: used_networks.add(network) for network in share["networks"]: try: network = ipaddress.ip_network(network, strict=False) except Exception: self.logger.warning("Got invalid network %r", network) continue else: used_networks.add(network) if not share["hosts"] and not share["networks"]: used_networks.add(ipaddress.ip_network("0.0.0.0/0")) used_networks.add(ipaddress.ip_network("::/0")) for i, host in enumerate(data["hosts"]): host = dns_cache[host] if host is None: continue network = ipaddress.ip_network(host) if network in used_networks: verrors.add( f"{schema_name}.hosts.{i}", "Another NFS share already exports this dataset for this host" ) used_networks.add(network) for i, network in enumerate(data["networks"]): network = ipaddress.ip_network(network, strict=False) if network in used_networks: verrors.add( f"{schema_name}.networks.{i}", "Another NFS share already exports this dataset for this network" ) used_networks.add(network) if not data["hosts"] and not data["networks"]: if used_networks: verrors.add( f"{schema_name}.networks", "Another NFS share already exports this dataset for some network" ) @private async def extend(self, data): data["paths"] = [path["path"] for path in await self.middleware.call("datastore.query", "sharing.nfs_share_path", [["share_id", "=", data["id"]]])] data["networks"] = data.pop("network").split() data["hosts"] = data["hosts"].split() data["security"] = [s.upper() for s in data["security"]] return data @private async def compress(self, data): data["network"] = " ".join(data.pop("networks")) data["hosts"] = " ".join(data["hosts"]) data["security"] = [s.lower() for s in data["security"]] return data
class SharingNFSService(SharingService): path_field = 'paths' share_task_type = 'NFS' class Config: namespace = "sharing.nfs" datastore = "sharing.nfs_share" datastore_prefix = "nfs_" datastore_extend = "sharing.nfs.extend" cli_namespace = "sharing.nfs" async def human_identifier(self, share_task): return ', '.join(share_task[self.path_field]) @private async def sharing_task_determine_locked(self, data, locked_datasets): for path in data[self.path_field]: if await self.middleware.call( 'pool.dataset.path_in_locked_datasets', path, locked_datasets): return True else: return False @accepts( Dict( "sharingnfs_create", List("paths", items=[Dir("path")], empty=False), List("aliases", items=[Str("path", validators=[Match(r"^/.*")])]), Str("comment", default=""), List("networks", items=[IPAddr("network", network=True)]), List("hosts", items=[Str("host")]), Bool("alldirs", default=False), Bool("ro", default=False), Bool("quiet", default=False), Str("maproot_user", required=False, default=None, null=True), Str("maproot_group", required=False, default=None, null=True), Str("mapall_user", required=False, default=None, null=True), Str("mapall_group", required=False, default=None, null=True), List( "security", items=[ Str("provider", enum=["SYS", "KRB5", "KRB5I", "KRB5P"]) ], ), Bool("enabled", default=True), register=True, strict=True, )) async def do_create(self, data): """ Create a NFS Share. `paths` is a list of valid paths which are configured to be shared on this share. `aliases` is a list of aliases for each path (or an empty list if aliases are not used). `networks` is a list of authorized networks that are allowed to access the share having format "network/mask" CIDR notation. If empty, all networks are allowed. `hosts` is a list of IP's/hostnames which are allowed to access the share. If empty, all IP's/hostnames are allowed. `alldirs` is a boolean value which when set indicates that the client can mount any subdirectories of the selected pool or dataset. """ verrors = ValidationErrors() await self.validate(data, "sharingnfs_create", verrors) if verrors: raise verrors await self.compress(data) data["id"] = await self.middleware.call( "datastore.insert", self._config.datastore, data, {"prefix": self._config.datastore_prefix}, ) await self.extend(data) await self._service_change("nfs", "reload") return await self.get_instance(data["id"]) @accepts(Int("id"), Patch("sharingnfs_create", "sharingnfs_update", ("attr", { "update": True }))) async def do_update(self, id, data): """ Update NFS Share of `id`. """ verrors = ValidationErrors() old = await self.get_instance(id) new = old.copy() new.update(data) await self.validate(new, "sharingnfs_update", verrors, old=old) if verrors: raise verrors await self.compress(new) await self.middleware.call("datastore.update", self._config.datastore, id, new, {"prefix": self._config.datastore_prefix}) await self._service_change("nfs", "reload") return await self.get_instance(id) @accepts(Int("id")) async def do_delete(self, id): """ Delete NFS Share of `id`. """ await self.middleware.call("datastore.delete", self._config.datastore, id) await self._service_change("nfs", "reload") @private async def validate(self, data, schema_name, verrors, old=None): if len(data["aliases"]): if not osc.IS_LINUX: verrors.add( f"{schema_name}.aliases", "This field is only supported on SCALE", ) if len(data["aliases"]) != len(data["paths"]): verrors.add( f"{schema_name}.aliases", "This field should be either empty of have the same number of elements as paths", ) if data["alldirs"] and len(data["paths"]) > 1: verrors.add( f"{schema_name}.alldirs", "This option can only be used for shares that contain single path" ) # if any of the `paths` that were passed to us by user are within the gluster volume # mountpoint then we need to pass the `gluster_bypass` kwarg so that we don't raise a # validation error complaining about using a gluster path within the zpool mountpoint bypass = any('.glusterfs' in i for i in data["paths"] + data["aliases"]) # need to make sure that the nfs share is within the zpool mountpoint for idx, i in enumerate(data["paths"]): await check_path_resides_within_volume( verrors, self.middleware, f'{schema_name}.paths.{idx}', i, gluster_bypass=bypass) await self.middleware.run_in_thread(self.validate_paths, data, schema_name, verrors) filters = [] if old: filters.append(["id", "!=", old["id"]]) other_shares = await self.middleware.call("sharing.nfs.query", filters) dns_cache = await self.resolve_hostnames( sum([share["hosts"] for share in other_shares], []) + data["hosts"]) await self.middleware.run_in_thread(self.validate_hosts_and_networks, other_shares, data, schema_name, verrors, dns_cache) for k in ["maproot", "mapall"]: if not data[f"{k}_user"] and not data[f"{k}_group"]: pass elif not data[f"{k}_user"] and data[f"{k}_group"]: verrors.add( f"{schema_name}.{k}_user", "This field is required when map group is specified") else: user = group = None with contextlib.suppress(KeyError): user = await self.middleware.call( 'dscache.get_uncached_user', data[f'{k}_user']) if not user: verrors.add(f"{schema_name}.{k}_user", "User not found") if data[f'{k}_group']: with contextlib.suppress(KeyError): group = await self.middleware.call( 'dscache.get_uncached_group', data[f'{k}_group']) if not group: verrors.add(f"{schema_name}.{k}_group", "Group not found") if data["maproot_user"] and data["mapall_user"]: verrors.add(f"{schema_name}.mapall_user", "maproot_user disqualifies mapall_user") if data["security"]: nfs_config = await self.middleware.call("nfs.config") if not nfs_config["v4"]: verrors.add(f"{schema_name}.security", "This is not allowed when NFS v4 is disabled") @private def validate_paths(self, data, schema_name, verrors): if osc.IS_LINUX: # Ganesha does not have such a restriction, each path is a different share return dev = None for i, path in enumerate(data["paths"]): stat = os.stat(path) if dev is None: dev = stat.st_dev else: if dev != stat.st_dev: verrors.add( f'{schema_name}.paths.{i}', 'Paths for a NFS share must reside within the same filesystem' ) @private async def resolve_hostnames(self, hostnames): hostnames = list(set(hostnames)) async def resolve(hostname): try: return (await asyncio.wait_for( self.middleware.run_in_thread(socket.getaddrinfo, hostname, None), 5))[0][4][0] except Exception as e: self.logger.warning("Unable to resolve host %r: %r", hostname, e) return None resolved_hostnames = await asyncio_map(resolve, hostnames, 8) return dict(zip(hostnames, resolved_hostnames)) @private def validate_hosts_and_networks(self, other_shares, data, schema_name, verrors, dns_cache): dev = os.stat(data["paths"][0]).st_dev used_networks = set() for share in other_shares: try: share_dev = os.stat(share["paths"][0]).st_dev except Exception: self.logger.warning("Failed to stat first path for %r", share, exc_info=True) continue if share_dev == dev: for host in share["hosts"]: host = dns_cache[host] if host is None: continue try: network = ipaddress.ip_network(host) except Exception: self.logger.warning("Got invalid host %r", host) continue else: used_networks.add(network) for network in share["networks"]: try: network = ipaddress.ip_network(network, strict=False) except Exception: self.logger.warning("Got invalid network %r", network) continue else: used_networks.add(network) if not share["hosts"] and not share["networks"]: used_networks.add(ipaddress.ip_network("0.0.0.0/0")) used_networks.add(ipaddress.ip_network("::/0")) for host in set(data["hosts"]): host = dns_cache[host] if host is None: continue network = ipaddress.ip_network(host) if network in used_networks: verrors.add( f"{schema_name}.hosts", f"Another NFS share already exports this dataset for {host}" ) used_networks.add(network) for network in set(data["networks"]): network = ipaddress.ip_network(network, strict=False) if network in used_networks: verrors.add( f"{schema_name}.networks", f"Another NFS share already exports this dataset for {network}" ) used_networks.add(network) if not data["hosts"] and not data["networks"]: if used_networks: verrors.add( f"{schema_name}.networks", "Another NFS share already exports this dataset for some network" ) @private async def extend(self, data): data["networks"] = data.pop("network").split() data["hosts"] = data["hosts"].split() data["security"] = [s.upper() for s in data["security"]] return data @private async def compress(self, data): data["network"] = " ".join(data.pop("networks")) data["hosts"] = " ".join(data["hosts"]) data["security"] = [s.lower() for s in data["security"]] data.pop(self.locked_field, None) return data
class SharingAFPService(CRUDService): class Config: namespace = 'sharing.afp' datastore = 'sharing.afp_share' datastore_prefix = 'afp_' datastore_extend = 'sharing.afp.extend' @accepts(Dict( 'sharingafp_create', Str('path'), Bool('home', default=False), Str('name'), Str('comment'), List('allow', default=[]), List('deny', default=[]), List('ro', default=[]), List('rw', default=[]), Bool('timemachine', default=False), Int('timemachine_quota', default=0), Bool('nodev', default=False), Bool('nostat', default=False), Bool('upriv', default=True), UnixPerm('fperm', default='644'), UnixPerm('dperm', default='755'), UnixPerm('umask', default='000'), List('hostsallow', items=[IPAddr('ip', network=True)], default=[]), List('hostsdeny', items=[IPAddr('ip', network=True)], default=[]), Str('auxparams'), register=True )) async def do_create(self, data): verrors = ValidationErrors() path = data['path'] await self.clean(data, 'sharingafp_create', verrors) await self.validate(data, 'sharingafp_create', verrors) await check_path_resides_within_volume( verrors, self.middleware, "sharingafp_create.path", path) if verrors: raise verrors if path and not os.path.exists(path): try: os.makedirs(path) except OSError as e: raise CallError(f'Failed to create {path}: {e}') await self.compress(data) data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self.extend(data) await self._service_change('afp', 'reload') return data @accepts( Int('id'), Patch( 'sharingafp_create', 'sharingafp_update', ('attr', {'update': True}) ) ) async def do_update(self, id, data): verrors = ValidationErrors() old = await self.middleware.call( 'datastore.query', self._config.datastore, [('id', '=', id)], {'extend': self._config.datastore_extend, 'prefix': self._config.datastore_prefix, 'get': True}) path = data.get('path') new = old.copy() new.update(data) await self.clean(new, 'sharingafp_update', verrors, id=id) await self.validate(new, 'sharingafp_update', verrors, old=old) if path: await check_path_resides_within_volume( verrors, self.middleware, "sharingafp_create.path", path) if verrors: raise verrors if path and not os.path.exists(path): try: os.makedirs(path) except OSError as e: raise CallError(f'Failed to create {path}: {e}') await self.compress(new) await self.middleware.call( 'datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) await self.extend(new) await self._service_change('afp', 'reload') return new @accepts(Int('id')) async def do_delete(self, id): result = await self.middleware.call('datastore.delete', self._config.datastore, id) await self._service_change('afp', 'reload') return result @private async def clean(self, data, schema_name, verrors, id=None): data['name'] = await self.name_exists(data, schema_name, verrors, id) @private async def validate(self, data, schema_name, verrors, old=None): await self.home_exists(data['home'], schema_name, verrors, old) @private async def home_exists(self, home, schema_name, verrors, old=None): home_filters = [('home', '=', True)] home_result = None if home: if old and old['id'] is not None: id = old['id'] if not old['home']: home_filters.append(('id', '!=', id)) # The user already had this set as the home share home_result = await self.middleware.call( 'datastore.query', self._config.datastore, home_filters, {'prefix': self._config.datastore_prefix}) if home_result: verrors.add(f'{schema_name}.home', 'Only one share is allowed to be a home share.') @private async def name_exists(self, data, schema_name, verrors, id=None): name = data['name'] path = data['path'] home = data['home'] name_filters = [('name', '=', name)] path_filters = [('path', '=', path)] if not name: if home: name = 'Homes' else: name = path.rsplit('/', 1)[-1] if id is not None: name_filters.append(('id', '!=', id)) path_filters.append(('id', '!=', id)) name_result = await self.middleware.call( 'datastore.query', self._config.datastore, name_filters, {'prefix': self._config.datastore_prefix}) path_result = await self.middleware.call( 'datastore.query', self._config.datastore, path_filters, {'prefix': self._config.datastore_prefix}) if name_result: verrors.add(f'{schema_name}.name', 'A share with this name already exists.') if path_result: verrors.add(f'{schema_name}.path', 'A share with this path already exists.') return name @private async def extend(self, data): data['allow'] = data['allow'].split() data['deny'] = data['deny'].split() data['ro'] = data['ro'].split() data['rw'] = data['rw'].split() data['hostsallow'] = data['hostsallow'].split() data['hostsdeny'] = data['hostsdeny'].split() return data @private async def compress(self, data): data['allow'] = ' '.join(data['allow']) data['deny'] = ' '.join(data['deny']) data['ro'] = ' '.join(data['ro']) data['rw'] = ' '.join(data['rw']) data['hostsallow'] = ' '.join(data['hostsallow']) data['hostsdeny'] = ' '.join(data['hostsdeny']) return data
class ISCSIPortalService(CRUDService): class Config: datastore = 'services.iscsitargetportal' datastore_extend = 'iscsi.portal.config_extend' datastore_prefix = 'iscsi_target_portal_' namespace = 'iscsi.portal' @private async def config_extend(self, data): data['listen'] = [] for portalip in await self.middleware.call( 'datastore.query', 'services.iscsitargetportalip', [('portal', '=', data['id'])], {'prefix': 'iscsi_target_portalip_'}): data['listen'].append({ 'ip': portalip['ip'], 'port': portalip['port'], }) data['discovery_authmethod'] = AUTHMETHOD_LEGACY_MAP.get( data.pop('discoveryauthmethod')) data['discovery_authgroup'] = data.pop('discoveryauthgroup') return data async def __validate(self, verrors, data, schema, old=None): if not data['listen']: verrors.add(f'{schema}.listen', 'At least one listen entry is required.') else: system_ips = [ ip['address'] for ip in await self.middleware.call('interfaces.ip_in_use') ] system_ips.extend(['0.0.0.0', '::']) new_ips = set(i['ip'] for i in data['listen']) - set( i['ip'] for i in old['listen']) if old else set() for i in data['listen']: filters = [ ('iscsi_target_portalip_ip', '=', i['ip']), ('iscsi_target_portalip_port', '=', i['port']), ] if schema == 'iscsiportal_update': filters.append( ('iscsi_target_portalip_portal', '!=', data['id'])) if await self.middleware.call('datastore.query', 'services.iscsitargetportalip', filters): verrors.add(f'{schema}.listen', f'{i["ip"]}:{i["port"]} already in use.') if ((i['ip'] in new_ips or not new_ips) and i['ip'] not in system_ips): verrors.add( f'{schema}.listen', f'IP {i["ip"]} not configured on this system.') if data['discovery_authgroup']: if not await self.middleware.call( 'datastore.query', 'services.iscsitargetauthcredential', [('iscsi_target_auth_tag', '=', data['discovery_authgroup'])]): verrors.add( f'{schema}.discovery_authgroup', f'Auth Group "{data["discovery_authgroup"]}" not found.', errno.ENOENT, ) elif data['discovery_authmethod'] in ('CHAP', 'CHAP_MUTUAL'): verrors.add( f'{schema}.discovery_authgroup', 'This field is required if discovery method is ' 'set to CHAP or CHAP Mutual.') @accepts( Dict( 'iscsiportal_create', Str('comment'), Str('discovery_authmethod', default='NONE', enum=['NONE', 'CHAP', 'CHAP_MUTUAL']), Int('discovery_authgroup', default=None, null=True), List('listen', required=True, items=[ Dict( 'listen', IPAddr('ip', required=True), Int('port', default=3260, validators=[Range(min=1, max=65535)]), ), ], default=[]), register=True, )) async def do_create(self, data): """ Create a new iSCSI Portal. `discovery_authgroup` is required for CHAP and CHAP_MUTUAL. """ verrors = ValidationErrors() await self.__validate(verrors, data, 'iscsiportal_create') if verrors: raise verrors # tag attribute increments sequentially data['tag'] = (await self.middleware.call('datastore.query', self._config.datastore, [], {'count': True})) + 1 listen = data.pop('listen') data['discoveryauthgroup'] = data.pop('discovery_authgroup', None) data['discoveryauthmethod'] = AUTHMETHOD_LEGACY_MAP.inv.get( data.pop('discovery_authmethod'), 'None') pk = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) try: await self.__save_listen(pk, listen) except Exception as e: await self.middleware.call('datastore.delete', self._config.datastore, pk) raise e await self._service_change('iscsitarget', 'reload') return await self._get_instance(pk) async def __save_listen(self, pk, new, old=None): """ Update database with a set new listen IP:PORT tuples. It will delete no longer existing addresses and add new ones. """ new_listen_set = set([tuple(i.items()) for i in new]) old_listen_set = set([tuple(i.items()) for i in old]) if old else set() for i in new_listen_set - old_listen_set: i = dict(i) await self.middleware.call('datastore.insert', 'services.iscsitargetportalip', { 'portal': pk, 'ip': i['ip'], 'port': i['port'] }, {'prefix': 'iscsi_target_portalip_'}) for i in old_listen_set - new_listen_set: i = dict(i) portalip = await self.middleware.call( 'datastore.query', 'services.iscsitargetportalip', [('portal', '=', pk), ('ip', '=', i['ip']), ('port', '=', i['port'])], {'prefix': 'iscsi_target_portalip_'}) if portalip: await self.middleware.call('datastore.delete', 'services.iscsitargetportalip', portalip[0]['id']) @accepts(Int('id'), Patch('iscsiportal_create', 'iscsiportal_update', ('attr', { 'update': True }))) async def do_update(self, pk, data): """ Update iSCSI Portal `id`. """ old = await self._get_instance(pk) new = old.copy() new.update(data) verrors = ValidationErrors() await self.__validate(verrors, new, 'iscsiportal_update', old) if verrors: raise verrors listen = new.pop('listen') new['discoveryauthgroup'] = new.pop('discovery_authgroup', None) new['discoveryauthmethod'] = AUTHMETHOD_LEGACY_MAP.inv.get( new.pop('discovery_authmethod'), 'None') await self.__save_listen(pk, listen, old['listen']) await self.middleware.call('datastore.update', self._config.datastore, pk, new, {'prefix': self._config.datastore_prefix}) await self._service_change('iscsitarget', 'reload') return await self._get_instance(pk) @accepts(Int('id')) async def do_delete(self, id): """ Delete iSCSI Portal `id`. """ result = await self.middleware.call('datastore.delete', self._config.datastore, id) for i, portal in enumerate(await self.middleware.call( 'iscsi.portal.query', [], {'order_by': ['tag']})): await self.middleware.call( 'datastore.update', self._config.datastore, portal['id'], {'tag': i + 1}, {'prefix': self._config.datastore_prefix}) await self._service_change('iscsitarget', 'reload') return result
class CtdbPublicIpService(CRUDService): class Config: namespace = 'ctdb.public.ips' cli_private = True @filterable def query(self, filters, options): ctdb_ips = [] if self.middleware.call_sync('service.started', 'ctdb'): ips = self.middleware.call_sync('ctdb.general.ips') ctdb_ips = list(map(lambda i: dict(i, id=i['public_ip'], enabled=(i['pnn'] >= 0)), ips)) try: shared_vol = Path(CTDBConfig.CTDB_LOCAL_MOUNT.value) mounted = shared_vol.is_mount() except Exception: # can happen when mounted but glusterd service # is stopped/crashed etc mounted = False etc_ips = [] if mounted: pub_ip_file = Path(CTDBConfig.GM_PUB_IP_FILE.value) etc_ip_file = Path(CTDBConfig.ETC_PUB_IP_FILE.value) if pub_ip_file.exists(): if etc_ip_file.is_symlink() and etc_ip_file.resolve() == pub_ip_file: with open(pub_ip_file) as f: for i in f.read().splitlines(): if not i.startswith('#'): enabled = True public_ip = i.split('/')[0] else: enabled = False public_ip = i.split('#')[1].split('/')[0] etc_ips.append({ 'id': public_ip, 'pnn': -1 if not enabled else 'N/A', 'enabled': enabled, 'public_ip': public_ip, 'interfaces': [{ 'name': i.split()[-1], 'active': False, 'available': False, }] }) # if the public ip was gracefully disabled and ctdb daemon is running # then it will report the public ip address information, however, # if the ctdb daemon was restarted after it was disabled then it # won't report it at all, yet, it's still written to the config # file prepended with a "#". This is by design so we need to # make sure we normalize the output of what ctdb daemon reports # and what's been written to the public address config file normalized = [] if not ctdb_ips: # means the ctdb daemon didn't return any type of # public address information so just return the # contents of etc_ips # NOTE: the contents of the etc file could be empty # (or not there) because it's a symlink pointed to # the cluster shared volume. In this case, there # isn't much we can do normalized = etc_ips else: if not etc_ips: # means the ctdb daemon is reporting public address(es) # however we're unable to read the config file which # could happen if the ctdb shared volume was umounted # while the ctdb daemon is running so we just return # what the daemon sees normalized = ctdb_ips else: # means the ctdb daemon is reporting public address(es) # and we have public addresses written to the config file # but it doesn't mean they necessarily match each other # so we need to normalize the output so the returned output # is always the same between the 2 normalized.extend([i for i in ctdb_ips if i['public_ip'] not in [j.keys() for j in etc_ips]]) return filter_list(normalized, filters, options) @private async def reload(self): """ Reload the public addresses configuration file on the ctdb nodes. When it completes the public addresses will be reconfigured and reassigned across the cluster as necessary. """ if await self.middleware.call('service.started', 'ctdb'): re = await run(['ctdb', 'reloadips'], encoding='utf8', errors='ignore', check=False) if re.returncode: # this isn't fatal it just means the newly added public ip won't show # up until the ctdb service has been restarted so just log a message self.logger.warning('Failed to reload public ip addresses %r', re.stderr) @accepts(Dict( 'public_create', IPAddr('ip', required=True), Int('netmask', required=True), Str('interface', required=True), )) @job(lock=PUB_LOCK) async def do_create(self, job, data): """ Add a ctdb public address to the cluster `ip` string representing an IP v4/v6 address `netmask` integer representing a cidr notated netmask (i.e. 16/24/48/64 etc) `interface` string representing a network interface to apply the `ip` """ schema_name = 'public_create' verrors = ValidationErrors() await self.middleware.call('ctdb.ips.common_validation', data, schema_name, verrors) await self.middleware.call('ctdb.ips.update_file', data, schema_name) await self.middleware.call('ctdb.public.ips.reload') return await self.middleware.call('ctdb.public.ips.query', [('public_ip', '=', data['ip'])]) @accepts( Str('ip', validators=[IpAddress()], required=True), Dict( 'public_update', Bool('enable', required=True), ) ) @job(lock=PUB_LOCK) async def do_update(self, job, id, option): """ Update Public IP address in the ctdb cluster. `ip` string representing the public ip address `enable` boolean. When True, enable the node else disable the node. """ schema_name = 'public_update' verrors = ValidationErrors() data = await self.get_instance(id) data['enable'] = option['enable'] await self.middleware.call('ctdb.ips.common_validation', data, schema_name, verrors) await self.middleware.call('ctdb.ips.update_file', data, schema_name) await self.middleware.call('ctdb.public.ips.reload') return await self.get_instance(id)
class TruecommandService(Service): @private async def generate_wg_keys(self): cp = await Popen(['wg', 'genkey'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) private_key, stderr = await cp.communicate() if cp.returncode: raise CallError( f'Failed to generate key for wireguard with exit code ({cp.returncode}): {stderr.decode()}' ) cp = await Popen(['wg', 'pubkey'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) public_key, stderr = await cp.communicate(input=private_key) if cp.returncode: raise CallError( f'Failed to generate public key for wireguard with exit code ({cp.returncode}): {stderr.decode()}' ) return { 'wg_public_key': public_key.decode().strip(), 'wg_private_key': private_key.decode().strip() } @private @periodic(HEALTH_CHECK_SECONDS, run_on_start=False) async def health_check(self): # The purpose of this method is to ensure that the wireguard connection # is active. If wireguard service is running, we want to make sure that the last # handshake we have had was under 30 minutes. if Status((await self.middleware.call( 'datastore.config', 'system.truecommand'))['api_key_state']) != Status.CONNECTED: await self.middleware.call('alert.oneshot_delete', 'TruecommandConnectionHealth', None) return if not await self.wireguard_connection_health(): # Stop wireguard if it's running and start polling the api to see what's up await self.middleware.call('truecommand.set_status', Status.CONNECTING.value) await self.stop_truecommand_service() await self.middleware.call('alert.oneshot_create', 'TruecommandConnectionHealth', None) await self.middleware.call('truecommand.poll_api_for_status') else: # Mark the connection as connected - we do this for just in case user never called # truecommand.config and is in WAITING state right now assuming that an event will be # raised when TC finally connects await self.middleware.call('truecommand.set_status', Status.CONNECTED.value) await self.middleware.call('alert.oneshot_delete', 'TruecommandConnectionHealth', None) @private async def wireguard_connection_health(self): """ Returns true if we are connected and wireguard connection has have had a handshake within last HEALTH_CHECK_SECONDS """ health_error = not (await self.middleware.call('service.started', 'truecommand')) if not health_error: cp = await run(['wg', 'show', 'wg0', 'latest-handshakes'], encoding='utf8', check=False) if cp.returncode: health_error = True else: timestamp = WIREGUARD_HEALTH_RE.findall(cp.stdout) if not timestamp: health_error = True else: timestamp = timestamp[0].strip() if timestamp == '0' or not timestamp.isdigit() or (int( time.time()) - int(timestamp)) > HEALTH_CHECK_SECONDS: # We never established handshake with TC if timestamp is 0, otherwise it's been more # then 30 minutes, error out please health_error = True else: # It's possible that IP of TC changed and we just need to get up to speed with the # new IP. So if we have a correct handshake, we should ping the TC IP to see if it's # still reachable config = await self.middleware.call( 'datastore.config', 'system.truecommand') cp = await run([ 'ping', '-t' if osc.IS_FREEBSD else '-w', '5', '-q', str(config['remote_address'].split('/', 1)[0]) ], check=False) if cp.returncode: # We have return code of 0 if we heard at least one response from the host health_error = True return not health_error @no_auth_required @throttle(seconds=2, condition=throttle_condition) @accepts() @returns( Dict( 'truecommand_connected', Bool('connected', required=True), IPAddr('truecommand_ip', null=True, required=True), Str('truecommand_url', null=True, required=True), Str('status', required=True), Str('status_reason', required=True), )) @pass_app() async def connected(self, app): """ Returns information which shows if system has an authenticated api key and has initiated a VPN connection with TrueCommand. """ tc_config = await self.middleware.call('truecommand.config') connected = Status(tc_config['status']) == Status.CONNECTED return { 'connected': connected, 'truecommand_ip': tc_config['remote_ip_address'] if connected else None, 'truecommand_url': tc_config['remote_url'] if connected else None, 'status': tc_config['status'], 'status_reason': tc_config['status_reason'], } @private async def start_truecommand_service(self): config = await self.middleware.call('datastore.config', 'system.truecommand') if config['enabled']: if Status(config['api_key_state']) == Status.CONNECTED and all( config[k] for k in ('wg_private_key', 'remote_address', 'endpoint', 'tc_public_key', 'wg_address')): await self.middleware.call('service.start', 'truecommand') await self.middleware.call('service.reload', 'http') else: # start polling iX Portal to see what's up and why we don't have these values set # This can happen in instances where system was polling and then was rebooted, # So we should continue polling in this case await self.middleware.call('truecommand.poll_api_for_status') @private async def stop_truecommand_service(self): await self.middleware.call('service.reload', 'http') if await self.middleware.call('service.started', 'truecommand'): await self.middleware.call('service.stop', 'truecommand')
class iSCSITargetAuthorizedInitiator(CRUDService): class Config: namespace = 'iscsi.initiator' datastore = 'services.iscsitargetauthorizedinitiator' datastore_prefix = 'iscsi_target_initiator_' datastore_extend = 'iscsi.initiator.extend' @accepts( Dict('iscsi_initiator_create', Int('tag', default=0), List('initiators', default=[]), List('auth_network', items=[IPAddr('ip', network=True)], default=[]), Str('comment'), register=True)) async def do_create(self, data): if data['tag'] == 0: i = len((await self.query())) + 1 while True: tag_result = await self.query([('tag', '=', i)]) if not tag_result: break i += 1 data['tag'] = i await self.compress(data) data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self._service_change('iscsitarget', 'reload') return await self._get_instance(data['id']) @accepts(Int('id'), Patch('iscsi_initiator_create', 'iscsi_initiator_update', ('attr', { 'update': True }))) async def do_update(self, id, data): old = await self._get_instance(id) new = old.copy() new.update(data) await self.compress(new) await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) await self._service_change('iscsitarget', 'reload') return await self._get_instance(id) @accepts(Int('id')) async def do_delete(self, id): result = await self.middleware.call('datastore.delete', self._config.datastore, id) for i, initiator in enumerate(await self.middleware.call( 'iscsi.initiator.query', [], {'order_by': ['tag']})): await self.middleware.call( 'datastore.update', self._config.datastore, initiator['id'], {'tag': i + 1}, {'prefix': self._config.datastore_prefix}) await self._service_change('iscsitarget', 'reload') return result @private async def compress(self, data): initiators = data['initiators'] auth_network = data['auth_network'] initiators = 'ALL' if not initiators else '\n'.join(initiators) auth_network = 'ALL' if not auth_network else '\n'.join(auth_network) data['initiators'] = initiators data['auth_network'] = auth_network return data @private async def extend(self, data): initiators = data['initiators'] auth_network = data['auth_network'] initiators = [] if initiators == 'ALL' else initiators.split() auth_network = [] if auth_network == 'ALL' else auth_network.split() data['initiators'] = initiators data['auth_network'] = auth_network return data
if await self.middleware.call('service.started', 'ctdb'): re = await run(['ctdb', 'reloadips'], encoding='utf8', errors='ignore', check=False) if re.returncode: # this isn't fatal it just means the newly added public ip won't show # up until the ctdb service has been restarted so just log a message self.logger.warning('Failed to reload public ip addresses %r', re.stderr) @accepts( Dict( 'public_create', Int('pnn', required=True), IPAddr('ip', required=True), Int('netmask', required=True), Str('interface', required=True), )) @job(lock=PUB_LOCK) async def do_create(self, job, data): """ Add a ctdb public address to the cluster `pnn` node number of record to adjust `ip` string representing an IP v4/v6 address `netmask` integer representing a cidr notated netmask (i.e. 16/24/48/64 etc) `interface` string representing a network interface to apply the `ip` """ schema_name = 'public_create'
class IPMIService(CRUDService): @accepts() async def is_loaded(self): """ Returns a boolean true value indicating if ipmi device is loaded. """ return os.path.exists('/dev/ipmi0') @accepts() async def channels(self): """ Return a list with the IPMI channels available. """ return channels @filterable async def query(self, filters=None, options=None): """ Query all IPMI Channels with `query-filters` and `query-options`. """ result = [] for channel in await self.channels(): try: cp = await run('ipmitool', 'lan', 'print', str(channel)) except subprocess.CalledProcessError as e: raise CallError(f'Failed to get details from channel {channel}: {e}') output = cp.stdout.decode() data = {'channel': channel, 'id': channel} for line in output.split('\n'): if ':' not in line: continue name, value = line.split(':', 1) if not name: continue name = name.strip() value = value.strip() if name == 'IP Address': data['ipaddress'] = value elif name == 'Subnet Mask': data['netmask'] = value elif name == 'Default Gateway IP': data['gateway'] = value elif name == '802.1q VLAN ID': if value == 'Disabled': data['vlan'] = None else: data['vlan'] = value elif name == 'IP Address Source': data['dhcp'] = False if value == 'Static Address' else True result.append(data) return filter_list(result, filters, options) @accepts(Int('channel'), Dict( 'ipmi', IPAddr('ipaddress', v6=False), Str('netmask', validators=[Netmask(ipv6=False, prefix_length=False)]), IPAddr('gateway', v6=False), Str('password', private=True), Bool('dhcp'), Int('vlan', null=True), )) async def do_update(self, id, data): """ Update `id` IPMI Configuration. `ipaddress` is a valid ip which will be used to connect to the IPMI interface. `netmask` is the subnet mask associated with `ipaddress`. `dhcp` is a boolean value which if unset means that `ipaddress`, `netmask` and `gateway` must be set. """ if not await self.is_loaded(): raise CallError('The ipmi device could not be found') verrors = ValidationErrors() if data.get('password') and len(data.get('password')) > 20: verrors.add( 'ipmi_update.password', 'A maximum of 20 characters are allowed' ) if not data.get('dhcp'): for k in ['ipaddress', 'netmask', 'gateway']: if not data.get(k): verrors.add( f'ipmi_update.{k}', 'This field is required when dhcp is not given' ) if verrors: raise verrors args = ['ipmitool', 'lan', 'set', str(id)] rv = 0 if data.get('dhcp'): rv |= (await run(*args, 'ipsrc', 'dhcp', check=False)).returncode else: rv |= (await run(*args, 'ipsrc', 'static', check=False)).returncode rv |= (await run(*args, 'ipaddr', data['ipaddress'], check=False)).returncode rv |= (await run(*args, 'netmask', data['netmask'], check=False)).returncode rv |= (await run(*args, 'defgw', 'ipaddr', data['gateway'], check=False)).returncode rv |= (await run( *args, 'vlan', 'id', str(data['vlan']) if data.get('vlan') else 'off' )).returncode rv |= (await run(*args, 'access', 'on', check=False)).returncode rv |= (await run(*args, 'auth', 'USER', 'MD2,MD5', check=False)).returncode rv |= (await run(*args, 'auth', 'OPERATOR', 'MD2,MD5', check=False)).returncode rv |= (await run(*args, 'auth', 'ADMIN', 'MD2,MD5', check=False)).returncode rv |= (await run(*args, 'auth', 'CALLBACK', 'MD2,MD5', check=False)).returncode # Setting arp have some issues in some hardwares # Do not fail if setting these couple settings do not work # See #15578 await run(*args, 'arp', 'respond', 'on', check=False) await run(*args, 'arp', 'generate', 'on', check=False) if data.get('password'): rv |= (await run( 'ipmitool', 'user', 'set', 'password', '2', data.get('password'), )).returncode rv |= (await run('ipmitool', 'user', 'enable', '2')).returncode # XXX: according to dwhite, this needs to be executed off the box via # the lanplus interface. # rv |= (await run('ipmitool', 'sol', 'set', 'enabled', 'true', '1')).returncode # ) return rv @accepts(Dict( 'options', Int('seconds'), Bool('force'), )) async def identify(self, options=None): """ Turn on IPMI chassis identify light. To turn off specify 0 as `seconds`. """ options = options or {} if options.get('force') and options.get('seconds'): raise CallError('You have to use either "seconds" or "force" option, not both') if options.get('force'): cmd = 'force' else: cmd = str(options.get('seconds')) await run('ipmitool', 'chassis', 'identify', cmd)
class NFSService(SystemServiceService): class Config: service = "nfs" service_verb = "restart" datastore_prefix = "nfs_srv_" datastore_extend = 'nfs.nfs_extend' @private def nfs_extend(self, nfs): nfs["userd_manage_gids"] = nfs.pop("16") return nfs @private def nfs_compress(self, nfs): nfs["16"] = nfs.pop("userd_manage_gids") return nfs @accepts(Dict( 'nfs_update', Int('servers', validators=[Range(min=1, max=256)]), Bool('udp'), Bool('allow_nonroot'), Bool('v4'), Bool('v4_v3owner'), Bool('v4_krb'), List('bindip', items=[IPAddr('ip')]), Int('mountd_port', null=True, validators=[Range(min=1, max=65535)]), Int('rpcstatd_port', null=True, validators=[Range(min=1, max=65535)]), Int('rpclockd_port', null=True, validators=[Range(min=1, max=65535)]), Bool('userd_manage_gids'), Bool('mountd_log'), Bool('statd_lockd_log'), update=True )) async def do_update(self, data): """ Update NFS Service Configuration. `servers` represents number of servers to create. When `allow_nonroot` is set, it allows non-root mount requests to be served. `bindip` is a list of IP's on which NFS will listen for requests. When it is unset/empty, NFS listens on all available addresses. `v4` when set means that we switch from NFSv3 to NFSv4. `v4_v3owner` when set means that system will use NFSv3 ownership model for NFSv4. `mountd_port` specifies the port mountd(8) binds to. `rpcstatd_port` specifies the port rpc.statd(8) binds to. `rpclockd_port` specifies the port rpclockd_port(8) binds to. .. examples(websocket):: Update NFS Service Configuration to listen on 192.168.0.10 and use NFSv4 :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "pool.resilver.update", "params": [{ "bindip": [ "192.168.0.10" ], "v4": true }] } """ if data.get("v4") is False: data.setdefault("v4_v3owner", False) old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() if new["v4"] and new["v4_krb"] and not await self.middleware.call("system.is_freenas"): if await self.middleware.call("failover.licensed"): gc = await self.middleware.call("datastore.config", "network.globalconfiguration") if not gc["gc_hostname_virtual"] or gc["gc_domain"]: verrors.add( "nfs_update.v4", "Enabling kerberos authentication on TrueNAS HA requires setting the virtual hostname and " "domain" ) if not new["v4"] and new["v4_v3owner"]: verrors.add("nfs_update.v4_v3owner", "This option requires enabling NFSv4") if new["v4_v3owner"] and new["userd_manage_gids"]: verrors.add( "nfs_update.userd_manage_gids", "This option is incompatible with NFSv3 ownership model for NFSv4") if verrors: raise verrors self.nfs_compress(new) await self._update_service(old, new) self.nfs_extend(new) return new @private def setup_v4(self): config = self.middleware.call_sync("nfs.config") if config["v4_krb"]: subprocess.run(["service", "gssd", "onerestart"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: subprocess.run(["service", "gssd", "forcestop"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) if config["v4"]: sysctl.filter("vfs.nfsd.server_max_nfsvers")[0].value = 4 if config["v4_v3owner"]: # Per RFC7530, sending NFSv3 style UID/GIDs across the wire is now allowed # You must have both of these sysctl"s set to allow the desired functionality sysctl.filter("vfs.nfsd.enable_stringtouid")[0].value = 1 sysctl.filter("vfs.nfs.enable_uidtostring")[0].value = 1 subprocess.run(["service", "nfsuserd", "forcestop"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: sysctl.filter("vfs.nfsd.enable_stringtouid")[0].value = 0 sysctl.filter("vfs.nfs.enable_uidtostring")[0].value = 0 subprocess.run(["service", "nfsuserd", "onerestart"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: sysctl.filter("vfs.nfsd.server_max_nfsvers")[0].value = 3 if config["userd_manage_gids"]: subprocess.run(["service", "nfsuserd", "onerestart"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: subprocess.run(["service", "nfsuserd", "forcestop"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
class NetworkConfigurationService(ConfigService): class Config: namespace = 'network.configuration' datastore = 'network.globalconfiguration' datastore_prefix = 'gc_' datastore_extend = 'network.configuration.network_config_extend' cli_namespace = 'network.configuration' ENTRY = Dict( 'network_configuration_entry', Int('id', required=True), Str('hostname', required=True, validators=[Hostname()]), Str( 'domain', validators=[Match(r'^[a-zA-Z\.\-\0-9]*$')], ), IPAddr('ipv4gateway', required=True), IPAddr('ipv6gateway', required=True, allow_zone_index=True), IPAddr('nameserver1', required=True), IPAddr('nameserver2', required=True), IPAddr('nameserver3', required=True), Str('httpproxy', required=True), Bool('netwait_enabled', required=True), List('netwait_ip', required=True, items=[Str('netwait_ip')]), Str('hosts', required=True), List('domains', required=True, items=[Str('domain')]), Dict( 'service_announcement', Bool('netbios'), Bool('mdns'), Bool('wsd'), register=True, ), Dict('activity', Str('type', enum=['ALLOW', 'DENY'], required=True), List('activities', items=[Str('activity')]), strict=True), Str('hostname_local', required=True, validators=[Hostname()]), Str('hostname_b', validators=[Hostname()], null=True), Str('hostname_virtual', validators=[Hostname()], null=True), Dict( 'state', IPAddr('ipv4gateway', required=True), IPAddr('ipv6gateway', required=True, allow_zone_index=True), IPAddr('nameserver1', required=True), IPAddr('nameserver2', required=True), IPAddr('nameserver3', required=True), ), ) @private def network_config_extend(self, data): # hostname_local will be used when the hostname of the current machine # needs to be used so it works with either TrueNAS SCALE or SCALE_ENTERPRISE data['hostname_local'] = data['hostname'] if not self.middleware.call_sync('system.is_enterprise'): data.pop('hostname_b') data.pop('hostname_virtual') else: if self.middleware.call_sync('failover.node') == 'B': data['hostname_local'] = data['hostname_b'] data['domains'] = data['domains'].split() data['netwait_ip'] = data['netwait_ip'].split() data['state'] = { 'ipv4gateway': '', 'ipv6gateway': '', 'nameserver1': '', 'nameserver2': '', 'nameserver3': '', } summary = self.middleware.call_sync('network.general.summary') for default_route in summary['default_routes']: try: ipaddress.IPv4Address(default_route) except ValueError: if not data['state']['ipv6gateway']: data['state']['ipv6gateway'] = default_route else: if not data['state']['ipv4gateway']: data['state']['ipv4gateway'] = default_route for i, nameserver in enumerate(summary['nameservers'][:3]): data['state'][f'nameserver{i + 1}'] = nameserver return data @private async def validate_general_settings(self, data, schema): verrors = ValidationErrors() for key in [key for key in data.keys() if 'nameserver' in key]: nameserver_value = data.get(key) if nameserver_value: try: nameserver_ip = ipaddress.ip_address(nameserver_value) except ValueError as e: verrors.add(f'{schema}.{key}', str(e)) else: if nameserver_ip.is_loopback: verrors.add(f'{schema}.{key}', 'Loopback is not a valid nameserver') elif nameserver_ip.is_unspecified: verrors.add( f'{schema}.{key}', 'Unspecified addresses are not valid as nameservers' ) elif nameserver_ip.version == 4: if nameserver_value == '255.255.255.255': verrors.add( f'{schema}.{key}', 'This is not a valid nameserver address') elif nameserver_value.startswith('169.254'): verrors.add( f'{schema}.{key}', '169.254/16 subnet is not valid for nameserver' ) nameserver_number = int(key[-1]) for i in range(nameserver_number - 1, 0, -1): if f'nameserver{i}' in data.keys( ) and not data[f'nameserver{i}']: verrors.add( f'{schema}.{key}', f'Must fill out namserver{i} before filling out {key}' ) ipv4_gateway_value = data.get('ipv4gateway') if ipv4_gateway_value: if not await self.middleware.call( 'route.ipv4gw_reachable', ipaddress.ip_address(ipv4_gateway_value).exploded): verrors.add(f'{schema}.ipv4gateway', f'Gateway {ipv4_gateway_value} is unreachable') netwait_ip = data.get('netwait_ip') if netwait_ip: for ip in netwait_ip: try: ipaddress.ip_address(ip) except ValueError as e: verrors.add(f'{schema}.netwait_ip', f'{e.__str__()}') if data.get('domains'): if len(data.get('domains')) > 5: verrors.add(f'{schema}.domains', 'No more than 5 additional domains are allowed') return verrors @accepts(Ref('service_announcement')) @private async def toggle_announcement(self, data): announce_srv = {'mdns': 'mdns', 'netbios': 'nmbd', 'wsd': 'wsdd'} for srv, enabled in data.items(): service_name = announce_srv[srv] started = await self.middleware.call('service.started', service_name) verb = None if enabled: verb = 'restart' if started else 'start' else: verb = 'stop' if started else None if not verb: continue await self.middleware.call(f'service.{verb}', service_name) @accepts( Patch( 'network_configuration_entry', 'global_configuration_update', ('rm', { 'name': 'id' }), ('rm', { 'name': 'hostname_local' }), ('rm', { 'name': 'state' }), ('attr', { 'update': True }), ), ) async def do_update(self, data): """ Update Network Configuration Service configuration. `ipv4gateway` if set is used instead of the default gateway provided by DHCP. `nameserver1` is primary DNS server. `nameserver2` is secondary DNS server. `nameserver3` is tertiary DNS server. `httpproxy` attribute must be provided if a proxy is to be used for network operations. `netwait_enabled` is a boolean attribute which when set indicates that network services will not start at boot unless they are able to ping the addresses listed in `netwait_ip` list. `service_announcement` determines the broadcast protocols that will be used to advertise the server. `netbios` enables the NetBIOS name server (NBNS), which starts concurrently with the SMB service. SMB clients will only perform NBNS lookups if SMB1 is enabled. NBNS may be required for legacy SMB clients. `mdns` enables multicast DNS service announcements for enabled services. `wsd` enables Web Service Discovery support. """ config = await self.config() config.pop('state') new_config = config.copy() new_config.update(data) new_config['service_announcement'] = config[ 'service_announcement'] | data.get('service_announcement', {}) if new_config == config: # nothing changed so return early return await self.config() verrors = await self.validate_general_settings( data, 'global_configuration_update') filters = [('timemachine', '=', True), ('enabled', '=', True)] if not new_config[ 'service_announcement']['mdns'] and await self.middleware.call( 'sharing.smb.query', filters): verrors.add( 'global_configuration_update.service_announcement.mdns', 'NAS is configured as a time machine target. mDNS is required.' ) lhost_changed = rhost_changed = False this_node = await self.middleware.call('failover.node') if this_node in ('MANUAL', 'A'): lhost_changed = config['hostname'] != new_config['hostname'] rhost_changed = config.get('hostname_b') and config[ 'hostname_b'] != new_config['hostname_b'] elif this_node == 'B': lhost_changed = config['hostname_b'] != new_config['hostname_b'] rhost_changed = config['hostname'] != new_config['hostname'] vhost_changed = config.get('hostname_virtual') and config[ 'hostname_virtual'] != new_config['hostname_virtual'] if vhost_changed and await self.middleware.call( 'activedirectory.get_state') != "DISABLED": verrors.add( 'global_configuration_update.hostname_virtual', 'This parameter may not be changed after joining Active Directory (AD). ' 'If it must be changed, the proper procedure is to leave the AD domain ' 'and then alter the parameter before re-joining the domain.') verrors.check() # pop the `hostname_local` key since that's created in the _extend method # and doesn't exist in the database new_config.pop('hostname_local', None) # normalize the `domains` and `netwait_ip` keys new_config['domains'] = ' '.join(new_config.get('domains', [])) new_config['netwait_ip'] = ' '.join(new_config.get('netwait_ip', [])) # update the db await self.middleware.call('datastore.update', 'network.globalconfiguration', config['id'], new_config, {'prefix': 'gc_'}) service_actions = set() if lhost_changed: await self.middleware.call('etc.generate', 'hostname') service_actions.add(('collectd', 'restart')) service_actions.add(('nscd', 'reload')) if rhost_changed: try: await self.middleware.call('failover.call_remote', 'etc.generate', ['hostname']) except Exception: self.logger.warning( 'Failed to set hostname on standby storage controller', exc_info=True) # dns domain name changed licensed = await self.middleware.call('failover.licensed') domainname_changed = new_config['domain'] != config['domain'] if domainname_changed: await self.middleware.call('etc.generate', 'hosts') service_actions.add(('collectd', 'restart')) service_actions.add(('nscd', 'reload')) if licensed: try: await self.middleware.call('failover.call_remote', 'etc.generate', ['hosts']) except Exception: self.logger.warning( 'Failed to set domain name on standby storage controller', exc_info=True) # anything related to resolv.conf changed dnssearch_changed = new_config['domains'] != config['domains'] dns1_changed = new_config['nameserver1'] != config['nameserver1'] dns2_changed = new_config['nameserver2'] != config['nameserver2'] dns3_changed = new_config['nameserver3'] != config['nameserver3'] dnsservers_changed = any((dns1_changed, dns2_changed, dns3_changed)) if dnssearch_changed or dnsservers_changed: await self.middleware.call('dns.sync') service_actions.add(('nscd', 'reload')) if licensed: try: await self.middleware.call('failover.call_remote', 'dns.sync') except Exception: self.logger.warning( 'Failed to generate resolv.conf on standby storage controller', exc_info=True) def reload_cli(): for process in psutil.process_iter(['pid', 'cmdline']): cmdline = process.cmdline() if len(cmdline) >= 2 and cmdline[1] == '/usr/bin/cli': with contextlib.suppress(Exception): process.send_signal(signal.SIGUSR1) await self.middleware.run_in_thread(reload_cli) # default gateways changed ipv4gw_changed = new_config['ipv4gateway'] != config['ipv4gateway'] ipv6gw_changed = new_config['ipv6gateway'] != config['ipv6gateway'] if ipv4gw_changed or ipv6gw_changed: await self.middleware.call('route.sync') if licensed: try: await self.middleware.call('failover.call_remote', 'route.sync') except Exception: self.logger.warning( 'Failed to generate routes on standby storage controller', exc_info=True) # kerberized NFS needs to be restarted if these change if lhost_changed or vhost_changed or domainname_changed: if await self.middleware.call('kerberos.keytab.has_nfs_principal'): service_actions.add(('nfs', 'restart')) # proxy server has changed if new_config['httpproxy'] != config['httpproxy']: await self.middleware.call( 'core.event_send', 'network.config', 'CHANGED', {'data': { 'httpproxy': new_config['httpproxy'] }}) # allowing outbound network activity has been changed if new_config['activity'] != config['activity']: await self.middleware.call('zettarepl.update_tasks') # handle the various service announcement daemons announce_changed = new_config['service_announcement'] != config[ 'service_announcement'] announce_srv = {'mdns': 'mdns', 'netbios': 'nmbd', 'wsd': 'wsdd'} if any((lhost_changed, vhost_changed)) or announce_changed: # lhost_changed is the local hostname and vhost_changed is the virtual hostname # and if either of these change then we need to toggle the service announcement # daemons irregardless whether or not these were toggled on their own for srv, enabled in new_config['service_announcement'].items(): service_name = announce_srv[srv] started = await self.middleware.call('service.started', service_name) verb = None if enabled: verb = 'restart' if started else 'start' else: verb = 'stop' if started else None if not verb: continue service_actions.add((service_name, verb)) for service, verb in service_actions: await self.middleware.call(f'service.{verb}', service) await self.middleware.call('network.configuration.toggle_announcement', new_config['service_announcement']) return await self.config()
class NFSService(SystemServiceService): class Config: service = "nfs" service_verb = "restart" datastore_prefix = "nfs_srv_" datastore_extend = 'nfs.nfs_extend' @private def nfs_extend(self, nfs): nfs["userd_manage_gids"] = nfs.pop("16") return nfs @private def nfs_compress(self, nfs): nfs["16"] = nfs.pop("userd_manage_gids") return nfs @accepts() async def bindip_choices(self): """ Returns ip choices for NFS service to use """ return { d['address']: d['address'] for d in await self.middleware.call( 'interface.ip_in_use', {'static': True, 'any': True} ) } @accepts(Dict( 'nfs_update', Int('servers', validators=[Range(min=1, max=256)]), Bool('udp'), Bool('allow_nonroot'), Bool('v4'), Bool('v4_v3owner'), Bool('v4_krb'), Str('v4_domain'), List('bindip', items=[IPAddr('ip')]), Int('mountd_port', null=True, validators=[Range(min=1, max=65535)]), Int('rpcstatd_port', null=True, validators=[Range(min=1, max=65535)]), Int('rpclockd_port', null=True, validators=[Range(min=1, max=65535)]), Bool('userd_manage_gids'), Bool('mountd_log'), Bool('statd_lockd_log'), update=True )) async def do_update(self, data): """ Update NFS Service Configuration. `servers` represents number of servers to create. When `allow_nonroot` is set, it allows non-root mount requests to be served. `bindip` is a list of IP's on which NFS will listen for requests. When it is unset/empty, NFS listens on all available addresses. `v4` when set means that we switch from NFSv3 to NFSv4. `v4_v3owner` when set means that system will use NFSv3 ownership model for NFSv4. `v4_domain` overrides the default DNS domain name for NFSv4. `mountd_port` specifies the port mountd(8) binds to. `rpcstatd_port` specifies the port rpc.statd(8) binds to. `rpclockd_port` specifies the port rpclockd_port(8) binds to. .. examples(websocket):: Update NFS Service Configuration to listen on 192.168.0.10 and use NFSv4 :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "pool.resilver.update", "params": [{ "bindip": [ "192.168.0.10" ], "v4": true }] } """ if data.get("v4") is False: data.setdefault("v4_v3owner", False) old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() if new["v4"] and new["v4_krb"] and not await self.middleware.call("system.is_freenas"): if await self.middleware.call("failover.licensed"): gc = await self.middleware.call("datastore.config", "network.globalconfiguration") if not gc["gc_hostname_virtual"] or gc["gc_domain"]: verrors.add( "nfs_update.v4", "Enabling kerberos authentication on TrueNAS HA requires setting the virtual hostname and " "domain" ) bindip_choices = await self.bindip_choices() for i, bindip in enumerate(new['bindip']): if bindip not in bindip_choices: verrors.add(f'nfs_update.bindip.{i}', 'Please provide a valid ip address') if new["v4"] and new["v4_krb"] and await self.middleware.call('activedirectory.get_state') != "DISABLED": """ In environments with kerberized NFSv4 enabled, we need to tell winbindd to not prefix usernames with the short form of the AD domain. Directly update the db and regenerate the smb.conf to avoid having a service disruption due to restarting the samba server. """ if await self.middleware.call('smb.get_smb_ha_mode') == 'LEGACY': raise ValidationError( 'nfs_update.v4_krb', 'Enabling kerberos authentication on TrueNAS HA requires ' 'the system dataset to be located on a data pool.' ) ad = await self.middleware.call('activedirectory.config') await self.middleware.call( 'datastore.update', 'directoryservice.activedirectory', ad['id'], {'ad_use_default_domain': True} ) await self.middleware.call('etc.generate', 'smb') await self.middleware.call('service.reload', 'cifs') if not new["v4"] and new["v4_v3owner"]: verrors.add("nfs_update.v4_v3owner", "This option requires enabling NFSv4") if new["v4_v3owner"] and new["userd_manage_gids"]: verrors.add( "nfs_update.userd_manage_gids", "This option is incompatible with NFSv3 ownership model for NFSv4") if not new["v4"] and new["v4_domain"]: verrors.add("nfs_update.v4_domain", "This option does not apply to NFSv3") if verrors: raise verrors self.nfs_compress(new) await self._update_service(old, new) self.nfs_extend(new) return new @private def setup_v4(self): config = self.middleware.call_sync("nfs.config") if config["v4_krb"]: subprocess.run(["service", "gssd", "onerestart"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: subprocess.run(["service", "gssd", "forcestop"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) if config["v4"]: sysctl.filter("vfs.nfsd.server_max_nfsvers")[0].value = 4 if config["v4_v3owner"]: # Per RFC7530, sending NFSv3 style UID/GIDs across the wire is now allowed # You must have both of these sysctl"s set to allow the desired functionality sysctl.filter("vfs.nfsd.enable_stringtouid")[0].value = 1 sysctl.filter("vfs.nfs.enable_uidtostring")[0].value = 1 subprocess.run(["service", "nfsuserd", "forcestop"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: sysctl.filter("vfs.nfsd.enable_stringtouid")[0].value = 0 sysctl.filter("vfs.nfs.enable_uidtostring")[0].value = 0 subprocess.run(["service", "nfsuserd", "onerestart"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: sysctl.filter("vfs.nfsd.server_max_nfsvers")[0].value = 3 if config["userd_manage_gids"]: subprocess.run(["service", "nfsuserd", "onerestart"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) else: subprocess.run(["service", "nfsuserd", "forcestop"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
class SMBService(SystemServiceService): class Config: service = 'cifs' service_verb = 'restart' datastore = 'services.cifs' datastore_extend = 'smb.smb_extend' datastore_prefix = 'cifs_srv_' @private async def smb_extend(self, smb): """Extend smb for netbios.""" smb['netbiosname_local'] = smb['netbiosname'] if not await self.middleware.call('system.is_freenas' ) and await self.middleware.call( 'failover.node') == 'B': smb['netbiosname_local'] = smb['netbiosname_b'] smb['netbiosalias'] = (smb['netbiosalias'] or '').split() smb['loglevel'] = LOGLEVEL_MAP.get(smb['loglevel']) return smb async def __validate_netbios_name(self, name): return RE_NETBIOSNAME.match(name) async def unixcharset_choices(self): return await self.generate_choices([ 'UTF-8', 'ISO-8859-1', 'ISO-8859-15', 'GB2312', 'EUC-JP', 'ASCII' ]) @private async def generate_choices(self, initial): def key_cp(encoding): cp = re.compile( r"(?P<name>CP|GB|ISO-8859-|UTF-)(?P<num>\d+)").match(encoding) if cp: return tuple((cp.group('name'), int(cp.group('num'), 10))) else: return tuple((encoding, float('inf'))) charset = await self.common_charset_choices() return { v: v for v in [c for c in sorted(charset, key=key_cp) if c not in initial] + initial } @accepts() async def bindip_choices(self): """ List of valid choices for IP addresses to which to bind the SMB service. Addresses assigned by DHCP are excluded from the results. """ choices = {} for i in await self.middleware.call('interface.query'): for alias in i['aliases']: choices[alias['address']] = alias['address'] return choices @accepts() async def domain_choices(self): """ List of domains visible to winbindd. Returns empty list if winbindd is stopped. """ domains = [] wb = await run([SMBCmd.WBINFO.value, '-m'], check=False) if wb.returncode == 0: domains = wb.stdout.decode().splitlines() return domains @private async def common_charset_choices(self): def check_codec(encoding): try: return encoding.upper() if codecs.lookup(encoding) else False except LookupError: return False proc = await Popen(['/usr/bin/iconv', '-l'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output = (await proc.communicate())[0].decode() encodings = set() for line in output.splitlines(): enc = [e for e in line.split() if check_codec(e)] if enc: cp = enc[0] for e in enc: if e in ('UTF-8', 'ASCII', 'GB2312', 'HZ-GB-2312', 'CP1361'): cp = e break encodings.add(cp) return encodings @private async def store_ldap_admin_password(self): """ This is required if the LDAP directory service is enabled. The ldap admin dn and password are stored in private/secrets.tdb file. """ ldap = await self.middleware.call('datastore.config', 'directoryservice.ldap') if not ldap['ldap_enable']: return True set_pass = await run( ['usr/local/bin/smbpasswd', '-w', ldap['ldap_bindpw']], check=False) if set_pass.returncode != 0: self.logger.debug( f"Failed to set set ldap bindpw in secrets.tdb: {set_pass.stdout.decode()}" ) return False return True @private def getparm(self, parm, section): """ Get a parameter from the smb4.conf file. This is more reliable than 'testparm --parameter-name'. testparm will fail in a variety of conditions without returning the parameter's value. """ try: if section.upper() == 'GLOBAL': return param.LoadParm(SMBPath.GLOBALCONF.platform()).get( parm, section) else: return self.middleware.call_sync('sharing.smb.reg_getparm', section, parm) except Exception as e: raise CallError( f'Attempt to query smb4.conf parameter [{parm}] failed with error: {e}' ) @private async def get_smb_ha_mode(self): if await self.middleware.call('cache.has_key', 'SMB_HA_MODE'): return await self.middleware.call('cache.get', 'SMB_HA_MODE') if not await self.middleware.call('system.is_freenas' ) and await self.middleware.call( 'failover.licensed'): system_dataset = await self.middleware.call('systemdataset.config') if system_dataset['pool'] != await self.middleware.call( 'boot.pool_name'): hamode = SMBHAMODE['UNIFIED'].name else: hamode = SMBHAMODE['LEGACY'].name else: hamode = SMBHAMODE['STANDALONE'].name await self.middleware.call('cache.put', 'SMB_HA_MODE', hamode) return hamode @private async def reset_smb_ha_mode(self): await self.middleware.call('cache.pop', 'SMB_HA_MODE') return await self.get_smb_ha_mode() @accepts( Dict( 'smb_update', Str('netbiosname', max_length=15), Str('netbiosname_b', max_length=15), List('netbiosalias', default=[], items=[Str('netbios_alias', max_length=15)]), Str('workgroup'), Str('description'), Bool('enable_smb1'), Str('unixcharset'), Str('loglevel', enum=['NONE', 'MINIMUM', 'NORMAL', 'FULL', 'DEBUG']), Bool('syslog'), Bool('aapl_extensions'), Bool('localmaster'), Str('guest'), Str('admin_group', required=False, default=None, null=True), Str('filemask'), Str('dirmask'), Bool('ntlmv1_auth'), List('bindip', items=[IPAddr('ip')], default=[]), Str('smb_options', max_length=None), update=True, )) async def do_update(self, data): """ Update SMB Service Configuration. `netbiosname` defaults to the original hostname of the system. `workgroup` and `netbiosname` should have different values. `enable_smb1` allows legacy SMB clients to connect to the server when enabled. `localmaster` when set, determines if the system participates in a browser election. `domain_logons` is used to provide netlogin service for older Windows clients if enabled. `guest` attribute is specified to select the account to be used for guest access. It defaults to "nobody". `nullpw` when enabled allows the users to authorize access without a password. `hostlookup` when enabled, allows using hostnames rather then IP addresses in "hostsallow"/"hostsdeny" fields of SMB Shares. """ old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() if data.get('unixcharset') and data[ 'unixcharset'] not in await self.unixcharset_choices(): verrors.add('smb_update.unixcharset', 'Please provide a valid value for unixcharset') for i in ('workgroup', 'netbiosname', 'netbiosname_b', 'netbiosalias'): if i not in data or not data[i]: continue if i == 'netbiosalias': for idx, item in enumerate(data[i]): if not await self.__validate_netbios_name(item): verrors.add(f'smb_update.{i}.{idx}', f'Invalid NetBIOS name: {item}') else: if not await self.__validate_netbios_name(data[i]): verrors.add(f'smb_update.{i}', f'Invalid NetBIOS name: {data[i]}') if new['netbiosname'] and new['netbiosname'].lower( ) == new['workgroup'].lower(): verrors.add('smb_update.netbiosname', 'NetBIOS and Workgroup must be unique') if data.get('bindip'): bindip_choices = list((await self.bindip_choices()).keys()) for idx, item in enumerate(data['bindip']): if item not in bindip_choices: verrors.add( f'smb_update.bindip.{idx}', f'IP address [{item}] is not a configured address for this server' ) for i in ('filemask', 'dirmask'): if i not in data or not data[i]: continue try: if int(data[i], 8) & ~0o11777: raise ValueError('Not an octet') except (ValueError, TypeError): verrors.add(f'smb_update.{i}', 'Not a valid mask') if new['admin_group'] and new['admin_group'] != old['admin_group']: await self.add_admin_group(new['admin_group']) if verrors: raise verrors # TODO: consider using bidict for k, v in LOGLEVEL_MAP.items(): if new['loglevel'] == v: new['loglevel'] = k break await self.compress(new) await self._update_service(old, new) await self.reset_smb_ha_mode() return await self.config() @private async def compress(self, data): data['netbiosalias'] = ' '.join(data['netbiosalias']) data.pop('netbiosname_local', None) return data
class NetworkConfigurationService(ConfigService): class Config: namespace = 'network.configuration' datastore = 'network.globalconfiguration' datastore_prefix = 'gc_' datastore_extend = 'network.configuration.network_config_extend' def network_config_extend(self, data): data['domains'] = data['domains'].split() data['netwait_ip'] = data['netwait_ip'].split() return data async def validate_general_settings(self, data, schema): verrors = ValidationErrors() for key in [key for key in data.keys() if 'nameserver' in key]: nameserver_value = data.get(key) if nameserver_value: try: nameserver_ip = ipaddress.ip_address(nameserver_value) except ValueError as e: verrors.add(f'{schema}.{key}', str(e)) else: if nameserver_ip.is_loopback: verrors.add(f'{schema}.{key}', 'Loopback is not a valid nameserver') elif nameserver_ip.is_unspecified: verrors.add( f'{schema}.{key}', 'Unspecified addresses are not valid as nameservers' ) elif nameserver_ip.version == 4: if nameserver_value == '255.255.255.255': verrors.add( f'{schema}.{key}', 'This is not a valid nameserver address') elif nameserver_value.startswith('169.254'): verrors.add( f'{schema}.{key}', '169.254/16 subnet is not valid for nameserver' ) nameserver_number = int(key[-1]) for i in range(nameserver_number - 1, 0, -1): if f'nameserver{i}' in data.keys( ) and not data[f'nameserver{i}']: verrors.add( f'{schema}.{key}', f'Must fill out namserver{i} before filling out {key}' ) ipv4_gateway_value = data.get('ipv4gateway') if ipv4_gateway_value: if not await self.middleware.call( 'routes.ipv4gw_reachable', ipaddress.ip_address(ipv4_gateway_value).exploded): verrors.add(f'{schema}.ipv4gateway', f'Gateway {ipv4_gateway_value} is unreachable') netwait_ip = data.get('netwait_ip') if netwait_ip: for ip in netwait_ip: try: ipaddress.ip_address(ip) except ValueError as e: verrors.add(f'{schema}.netwait_ip', f'{e.__str__()}') if data.get('domains'): if len(data.get('domains')) > 5: verrors.add(f'{schema}.domains', 'No more than 5 additional domains are allowed') return verrors @accepts( Dict( 'global_configuration', Str('hostname', validators=[Match(r'^[a-zA-Z\.\-\0-9]+$')]), Str('hostname_b', validators=[Match(r'^[a-zA-Z\.\-\0-9]+$')]), Str('hostname_virtual', validators=[Match(r'^[a-zA-Z\.\-\0-9]+$')]), Str('domain', validators=[Match(r'^[a-zA-Z\.\-\0-9]+$')]), List('domains', items=[Str('domains')]), IPAddr('ipv4gateway'), IPAddr('ipv6gateway', allow_zone_index=True), IPAddr('nameserver1'), IPAddr('nameserver2'), IPAddr('nameserver3'), Str('httpproxy'), Bool('netwait_enabled'), List('netwait_ip', items=[Str('netwait_ip')]), Str('hosts'), )) async def do_update(self, data): config = await self.config() new_config = config.copy() if not (not await self.middleware.call('system.is_freenas') and await self.middleware.call('notifier.failover_licensed')): for key in ['hostname_virtual', 'hostname_b']: data.pop(key, None) new_config.update(data) verrors = await self.validate_general_settings( data, 'global_configuration_update') if verrors: raise verrors new_config['domains'] = ' '.join(new_config.get('domains', [])) new_config['netwait_ip'] = ' '.join(new_config.get('netwait_ip', [])) await self.middleware.call('datastore.update', 'network.globalconfiguration', config['id'], new_config, {'prefix': 'gc_'}) new_config['domains'] = new_config['domains'].split() new_config['netwait_ip'] = new_config['netwait_ip'].split() netwait_ip_set = set(new_config.pop('netwait_ip', [])) old_netwait_ip_set = set(config.pop('netwait_ip', [])) data_changed = netwait_ip_set != old_netwait_ip_set if not data_changed: domains_set = set(new_config.pop('domains', [])) old_domains_set = set(config.pop('domains', [])) data_changed = domains_set != old_domains_set if (data_changed or len(set(new_config.items()) ^ set(config.items())) > 0): services_to_reload = ['hostname'] if (new_config['domain'] != config['domain'] or new_config['nameserver1'] != config['nameserver1'] or new_config['nameserver2'] != config['nameserver2'] or new_config['nameserver3'] != config['nameserver3']): services_to_reload.append('resolvconf') if (new_config['ipv4gateway'] != config['ipv4gateway'] or new_config['ipv6gateway'] != config['ipv6gateway']): services_to_reload.append('networkgeneral') await self.middleware.call('routes.sync') if ('hostname_virtual' in new_config.keys() and new_config['hostname_virtual'] != config['hostname_virtual']): srv_service_obj = await self.middleware.call( 'datastore.query', 'service.service', [('srv_service', '=', 'nfs')]) nfs_object = await self.middleware.call( 'datastore.query', 'services.nfs', ) if len(srv_service_obj) > 0 and len(nfs_object) > 0: srv_service_obj = srv_service_obj[0] nfs_object = nfs_object[0] if ((srv_service_obj and srv_service_obj.srv_enable) and (nfs_object and (nfs_object.nfs_srv_v4 and nfs_object.nfs_srv_v4_krb))): await self.middleware.call('service.restart', 'ix-nfsd', {'onetime': False}) services_to_reload.append('mountd') for service_to_reload in services_to_reload: await self.middleware.call('service.reload', service_to_reload, {'onetime': False}) if new_config['httpproxy'] != config['httpproxy']: await self.middleware.call( 'core.event_send', 'network.config', 'CHANGED', {'data': { 'httpproxy': new_config['httpproxy'] }}) return await self.config()
class iSCSIHostService(CRUDService): hosts = {} class Config: namespace = "iscsi.host" datastore = "services.iscsihost" datastore_extend = "iscsi.host.extend" datastore_extend_context = "iscsi.host.extend_context" cli_namespace = "sharing.iscsi.host" @private async def extend_context(self, extra): id_to_iqns = defaultdict(list) for row in await self.middleware.call("datastore.query", "services.iscsihostiqn", [], {"relationships": False}): id_to_iqns[row["host_id"]].append(row["iqn"]) return { "id_to_iqns": id_to_iqns, } @private async def extend(self, row, context): row["iqns"] = context["id_to_iqns"][row["id"]] return row @accepts( Dict( "iscsi_host_create", IPAddr("ip", required=True), Str("description", default=""), List("iqns", items=[Str("iqn", empty=False)], default=[]), Bool("added_automatically", default=False), register=True, )) async def do_create(self, data): """ Creates iSCSI host. `ip` indicates an IP address of the host. `description` is a human-readable name for the host. `iqns` is a list of initiator iSCSI Qualified Names. """ async with LOCK: return await self.create_unlocked(data) @accepts(Ref("iscsi_host_create")) @private async def create_unlocked(self, data): iqns = data.pop("iqns") try: id = await self.middleware.call("datastore.insert", self._config.datastore, data) except IntegrityError: verrors = ValidationErrors() verrors.add("iscsi_host_create.ip", "This IP address already exists", errno.EEXIST) raise verrors await self._set_datastore_iqns(id, iqns) host = await self.get_instance(id) self.hosts[host["ip"]] = host self._set_cache_iqns(id, iqns) return host @accepts(Int("id"), Patch( "iscsi_host_create", "iscsi_host_update", ("attr", { "update": True }), register=True, )) async def do_update(self, id, data): """ Update iSCSI host `id`. """ async with LOCK: return await self.update_unlocked(id, data) @accepts(Int("id"), Ref("iscsi_host_update")) @private async def update_unlocked(self, id, data): old = await self.get_instance(id) new = old.copy() new.update(data) iqns = new.pop("iqns") try: await self.middleware.call("datastore.update", self._config.datastore, id, new) except IntegrityError: verrors = ValidationErrors() verrors.add("iscsi_host_update.ip", "This IP address already exists", errno.EEXIST) raise verrors await self._set_datastore_iqns(id, iqns) host = await self.get_instance(id) self.hosts.pop(old["ip"], None) self.hosts[host["ip"]] = host self._set_cache_iqns(id, iqns) return host @accepts(Int("id")) async def do_delete(self, id): """ Update iSCSI host `id`. """ async with LOCK: return await self.delete_unlocked(id) @private async def delete_unlocked(self, id): host = await self.get_instance(id) await self.middleware.call("datastore.delete", self._config.datastore, id) self.hosts.pop(host["ip"], None) return host async def _set_datastore_iqns(self, id, iqns): await self.middleware.call("datastore.delete", "services.iscsihostiqn", [["iqn", "in", iqns]]) for iqn in iqns: await self.middleware.call("datastore.insert", "services.iscsihostiqn", { "iqn": iqn, "host_id": id, }) def _set_cache_iqns(self, id, iqns): for host in self.hosts.values(): if host["id"] != id: for iqn in iqns: try: host["iqns"].remove(iqn) except ValueError: pass @private async def read_cache(self): self.hosts = {} for host in await self.query(): self.hosts[host["ip"]] = host @accepts( List( "hosts", items=[ Dict( "host", IPAddr("ip", required=True), Str("iqn", required=True), Bool("added_automatically", default=False), ), ], ), ) @private async def batch_update(self, hosts): async with LOCK: try: for host in hosts: if host["ip"] not in self.hosts: await self.create_unlocked({ "ip": host["ip"], "added_automatically": host["added_automatically"], }) db_host = self.hosts[host["ip"]] if host["iqn"] not in db_host["iqns"]: await self.update_unlocked( db_host["id"], {"iqns": db_host["iqns"] + [host["iqn"]]}) except Exception: await self.read_cache() raise
class SystemGeneralService(ConfigService): class Config: namespace = 'system.general' datastore = 'system.settings' datastore_prefix = 'stg_' datastore_extend = 'system.general.general_system_extend' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._language_choices = self._initialize_languages() self._timezone_choices = None self._kbdmap_choices = None self._country_choices = {} @private async def general_system_extend(self, data): for key in list(data.keys()): if key.startswith('gui'): data['ui_' + key[3:]] = data.pop(key) data['sysloglevel'] = data['sysloglevel'].upper() if data['ui_certificate']: data['ui_certificate'] = await self.middleware.call( 'certificate.query', [['id', '=', data['ui_certificate']['id']]], {'get': True}) data['crash_reporting_is_set'] = data['crash_reporting'] is not None if data['crash_reporting'] is None: data['crash_reporting'] = await self.middleware.call( "system.is_freenas") data['usage_collection_is_set'] = data['usage_collection'] is not None if data['usage_collection'] is None: data['usage_collection'] = await self.middleware.call( "system.is_freenas") data.pop('pwenc_check') return data @accepts() def language_choices(self): """ Returns language choices. """ return self._language_choices @private def _initialize_languages(self): languagues = [ ('af', 'Afrikaans'), ('ar', 'Arabic'), ('ast', 'Asturian'), ('az', 'Azerbaijani'), ('bg', 'Bulgarian'), ('be', 'Belarusian'), ('bn', 'Bengali'), ('br', 'Breton'), ('bs', 'Bosnian'), ('ca', 'Catalan'), ('cs', 'Czech'), ('cy', 'Welsh'), ('da', 'Danish'), ('de', 'German'), ('dsb', 'Lower Sorbian'), ('el', 'Greek'), ('en', 'English'), ('en-au', 'Australian English'), ('en-gb', 'British English'), ('eo', 'Esperanto'), ('es', 'Spanish'), ('es-ar', 'Argentinian Spanish'), ('es-co', 'Colombian Spanish'), ('es-mx', 'Mexican Spanish'), ('es-ni', 'Nicaraguan Spanish'), ('es-ve', 'Venezuelan Spanish'), ('et', 'Estonian'), ('eu', 'Basque'), ('fa', 'Persian'), ('fi', 'Finnish'), ('fr', 'French'), ('fy', 'Frisian'), ('ga', 'Irish'), ('gd', 'Scottish Gaelic'), ('gl', 'Galician'), ('he', 'Hebrew'), ('hi', 'Hindi'), ('hr', 'Croatian'), ('hsb', 'Upper Sorbian'), ('hu', 'Hungarian'), ('ia', 'Interlingua'), ('id', 'Indonesian'), ('io', 'Ido'), ('is', 'Icelandic'), ('it', 'Italian'), ('ja', 'Japanese'), ('ka', 'Georgian'), ('kab', 'Kabyle'), ('kk', 'Kazakh'), ('km', 'Khmer'), ('kn', 'Kannada'), ('ko', 'Korean'), ('lb', 'Luxembourgish'), ('lt', 'Lithuanian'), ('lv', 'Latvian'), ('mk', 'Macedonian'), ('ml', 'Malayalam'), ('mn', 'Mongolian'), ('mr', 'Marathi'), ('my', 'Burmese'), ('nb', 'Norwegian Bokmål'), ('ne', 'Nepali'), ('nl', 'Dutch'), ('nn', 'Norwegian Nynorsk'), ('os', 'Ossetic'), ('pa', 'Punjabi'), ('pl', 'Polish'), ('pt', 'Portuguese'), ('pt-br', 'Brazilian Portuguese'), ('ro', 'Romanian'), ('ru', 'Russian'), ('sk', 'Slovak'), ('sl', 'Slovenian'), ('sq', 'Albanian'), ('sr', 'Serbian'), ('sr-latn', 'Serbian Latin'), ('sv', 'Swedish'), ('sw', 'Swahili'), ('ta', 'Tamil'), ('te', 'Telugu'), ('th', 'Thai'), ('tr', 'Turkish'), ('tt', 'Tatar'), ('udm', 'Udmurt'), ('uk', 'Ukrainian'), ('ur', 'Urdu'), ('vi', 'Vietnamese'), ('zh-hans', 'Simplified Chinese'), ('zh-hant', 'Traditional Chinese'), ] return dict(languagues) @private async def _initialize_timezone_choices(self): pipe = await Popen( 'find /usr/share/zoneinfo/ -type f -not -name zone.tab -not -regex \'.*/Etc/GMT.*\'', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) self._timezone_choices = ( await pipe.communicate())[0].decode().strip().split('\n') self._timezone_choices = { x[20:]: x[20:] for x in self._timezone_choices } @accepts() async def timezone_choices(self): """ Returns time zone choices. """ if not self._timezone_choices: await self._initialize_timezone_choices() return self._timezone_choices @accepts() async def country_choices(self): """ Returns country choices. """ if not self._country_choices: await self._initialize_country_choices() return self._country_choices @private async def _initialize_country_choices(self): def _get_index(country_columns, column): index = -1 i = 0 for c in country_columns: if c.lower() == column.lower(): index = i break i += 1 return index country_file = '/etc/iso_3166_2_countries.csv' cni, two_li = None, None with open(country_file, 'r', encoding='utf-8') as csvfile: reader = csv.reader(csvfile) for index, row in enumerate(reader): if index != 0: if row[cni] and row[two_li]: if row[two_li] in self._country_choices: # If two countries in the iso file have the same key, we concatenate their names self._country_choices[ row[two_li]] += f' + {row[cni]}' else: self._country_choices[row[two_li]] = row[cni] else: # ONLY CNI AND TWO_LI ARE BEING CONSIDERED FROM THE CSV cni = _get_index(row, 'Common Name') two_li = _get_index(row, 'ISO 3166-1 2 Letter Code') @private async def _initialize_kbdmap_choices(self): """Populate choices from /usr/share/vt/keymaps/INDEX.keymaps""" index = "/usr/share/vt/keymaps/INDEX.keymaps" if not os.path.exists(index): return [] with open(index, 'rb') as f: d = f.read().decode('utf8', 'ignore') _all = re.findall(r'^(?P<name>[^#\s]+?)\.kbd:en:(?P<desc>.+)$', d, re.M) self._kbdmap_choices = {name: desc for name, desc in _all} @accepts() async def kbdmap_choices(self): """ Returns kbdmap choices. """ if not self._kbdmap_choices: await self._initialize_kbdmap_choices() return self._kbdmap_choices @private async def validate_general_settings(self, data, schema): verrors = ValidationErrors() language = data.get('language') if language: system_languages = self.language_choices() if language not in system_languages.keys(): verrors.add( f'{schema}.language', f'Specified "{language}" language not found, kindly correct it' ) # kbd map needs work timezone = data.get('timezone') if timezone: timezones = await self.timezone_choices() if timezone not in timezones: verrors.add(f'{schema}.timezone', 'Please select a correct timezone') ip_addresses = await self.middleware.call('interface.ip_in_use') ip4_addresses_list = [ alias_dict['address'] for alias_dict in ip_addresses if alias_dict['type'] == 'INET' ] ip6_addresses_list = [ alias_dict['address'] for alias_dict in ip_addresses if alias_dict['type'] == 'INET6' ] ip4_addresses = data.get('ui_address') for ip4_address in ip4_addresses: if (ip4_address and ip4_address != '0.0.0.0' and ip4_address not in ip4_addresses_list): verrors.add( f'{schema}.ui_address', f'{ip4_address} ipv4 address is not associated with this machine' ) ip6_addresses = data.get('ui_v6address') for ip6_address in ip6_addresses: if (ip6_address and ip6_address != '::' and ip6_address not in ip6_addresses_list): verrors.add( f'{schema}.ui_v6address', f'{ip6_address} ipv6 address is not associated with this machine' ) for key, wildcard, ips in [('ui_address', '0.0.0.0', ip4_addresses), ('ui_v6address', '::', ip6_addresses)]: if wildcard in ips and len(ips) > 1: verrors.add( f'{schema}.{key}', f'When "{wildcard}" has been selected, selection of other addresses is not allowed' ) syslog_server = data.get('syslogserver') if syslog_server: match = re.match(r"^[\w\.\-]+(\:\d+)?$", syslog_server) if not match: verrors.add(f'{schema}.syslogserver', 'Invalid syslog server format') elif ':' in syslog_server: port = int(syslog_server.split(':')[-1]) if port < 0 or port > 65535: verrors.add(f'{schema}.syslogserver', 'Port specified should be between 0 - 65535') certificate_id = data.get('ui_certificate') cert = await self.middleware.call('certificate.query', [["id", "=", certificate_id]]) if not cert: verrors.add( f'{schema}.ui_certificate', 'Please specify a valid certificate which exists in the system' ) else: cert = cert[0] verrors.extend(await self.middleware.call( 'certificate.cert_services_validation', certificate_id, f'{schema}.ui_certificate', False)) if cert['fingerprint']: syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_USER) syslog.syslog( syslog.LOG_ERR, 'Fingerprint of the certificate used in UI : ' + cert['fingerprint']) syslog.closelog() return verrors @accepts() async def ui_certificate_choices(self): """ Return choices of certificates which can be used for `ui_certificate`. """ return { i['id']: i['name'] for i in await self.middleware.call('certificate.query', [( 'cert_type_CSR', '=', False)]) } @accepts( Dict( 'general_settings', Int('ui_certificate', null=True), Int('ui_httpsport', validators=[Range(min=1, max=65535)]), Bool('ui_httpsredirect'), Int('ui_port', validators=[Range(min=1, max=65535)]), List('ui_address', items=[IPAddr('addr')], empty=False), List('ui_v6address', items=[IPAddr('addr')], empty=False), Str('kbdmap'), Str('language'), Str('sysloglevel', enum=[ 'F_EMERG', 'F_ALERT', 'F_CRIT', 'F_ERR', 'F_WARNING', 'F_NOTICE', 'F_INFO', 'F_DEBUG', 'F_IS_DEBUG' ]), Str('syslogserver'), Str('timezone'), Bool('crash_reporting', null=True), Bool('usage_collection', null=True), update=True, )) async def do_update(self, data): """ Update System General Service Configuration. `ui_certificate` is used to enable HTTPS access to the system. If `ui_certificate` is not configured on boot, it is automatically created by the system. `ui_httpsredirect` when set, makes sure that all HTTP requests are converted to HTTPS requests to better enhance security. `ui_address` and `ui_v6address` are a list of valid ipv4/ipv6 addresses respectively which the system will listen on. When `syslogserver` is defined, `sysloglevel` makes sure that logs matching the specified level are sent. """ config = await self.config() config['ui_certificate'] = config['ui_certificate']['id'] if config[ 'ui_certificate'] else None if not config.pop('crash_reporting_is_set'): config['crash_reporting'] = None if not config.pop('usage_collection_is_set'): config['usage_collection'] = None new_config = config.copy() new_config.update(data) verrors = await self.validate_general_settings( new_config, 'general_settings_update') if verrors: raise verrors # Converting new_config to map the database table fields new_config['sysloglevel'] = new_config['sysloglevel'].lower() keys = new_config.keys() for key in list(keys): if key.startswith('ui_'): new_config['gui' + key[3:]] = new_config.pop(key) await self.middleware.call('datastore.update', self._config.datastore, config['id'], new_config, {'prefix': 'stg_'}) # case insensitive comparison should be performed for sysloglevel if (config['sysloglevel'].lower() != new_config['sysloglevel'].lower() or config['syslogserver'] != new_config['syslogserver']): await self.middleware.call('service.restart', 'syslogd') if config['kbdmap'] != new_config['kbdmap']: await self.middleware.call('service.restart', 'syscons') if config['timezone'] != new_config['timezone']: await self.middleware.call('zettarepl.update_timezone', new_config['timezone']) await self.middleware.call('service.reload', 'timeservices') await self.middleware.call('service.restart', 'cron') if config['language'] != new_config['language']: await self.middleware.call('system.general.set_language') if config['crash_reporting'] != new_config['crash_reporting']: await self.middleware.call('system.general.set_crash_reporting') await self.middleware.call('service.start', 'ssl') return await self.config() @accepts() async def ui_restart(self): """ Restart HTTP server to use latest UI settings. """ await self.middleware.call('service.restart', 'http') await self.middleware.call('service.restart', 'django') @accepts() async def local_url(self): """ Returns configured local url in the format of protocol://host:port """ config = await self.middleware.call('system.general.config') if config['ui_certificate']: protocol = 'https' port = config['ui_httpsport'] else: protocol = 'http' port = config['ui_port'] if '0.0.0.0' in config['ui_address'] or '127.0.0.1' in config[ 'ui_address']: hosts = ['127.0.0.1'] else: hosts = config['ui_address'] errors = [] for host in hosts: try: reader, writer = await asyncio.wait_for( asyncio.open_connection( host, port=port, ), timeout=5) writer.close() return f'{protocol}://{host}:{port}' except Exception as e: errors.append(f'{host}: {e}') raise CallError( 'Unable to connect to any of the specified UI addresses:\n' + '\n'.join(errors)) @private def set_language(self): language = self.middleware.call_sync( 'system.general.config')['language'] set_language(language) @private def set_crash_reporting(self): CrashReporting.enabled_in_settings = self.middleware.call_sync( 'system.general.config')['crash_reporting']
class NFSService(SystemServiceService): class Config: service = "nfs" service_verb = "restart" datastore_prefix = "nfs_srv_" datastore_extend = 'nfs.nfs_extend' cli_namespace = "service.nfs" @private async def nfs_extend(self, nfs): keytab_has_nfs = await self.middleware.call( "kerberos.keytab.has_nfs_principal") nfs["v4_krb_enabled"] = (nfs["v4_krb"] or keytab_has_nfs) nfs["userd_manage_gids"] = nfs.pop("16") return nfs @private async def nfs_compress(self, nfs): nfs.pop("v4_krb_enabled") nfs["16"] = nfs.pop("userd_manage_gids") return nfs @accepts() async def bindip_choices(self): """ Returns ip choices for NFS service to use """ return { d['address']: d['address'] for d in await self.middleware.call('interface.ip_in_use', {'static': True}) } @private async def bindip(self, config): bindip = [ addr for addr in config['bindip'] if addr not in ['0.0.0.0', '::'] ] if osc.IS_LINUX: bindip = bindip[:1] if bindip: found = False for iface in await self.middleware.call('interface.query'): for alias in iface['state']['aliases']: if alias['address'] in bindip: found = True break if found: break else: found = True if found: await self.middleware.call('alert.oneshot_delete', 'NFSBindAddress', None) return bindip else: if await self.middleware.call('cache.has_key', 'interfaces_are_set_up'): await self.middleware.call('alert.oneshot_create', 'NFSBindAddress', None) return [] @accepts( Dict('nfs_update', Int('servers', validators=[Range(min=1, max=256)]), Bool('udp'), Bool('allow_nonroot'), Bool('v4'), Bool('v4_v3owner'), Bool('v4_krb'), Str('v4_domain'), List('bindip', items=[IPAddr('ip')]), Int('mountd_port', null=True, validators=[Range(min=1, max=65535)]), Int('rpcstatd_port', null=True, validators=[Range(min=1, max=65535)]), Int('rpclockd_port', null=True, validators=[Range(min=1, max=65535)]), Bool('userd_manage_gids'), Bool('mountd_log'), Bool('statd_lockd_log'), update=True)) async def do_update(self, data): """ Update NFS Service Configuration. `servers` represents number of servers to create. When `allow_nonroot` is set, it allows non-root mount requests to be served. `bindip` is a list of IP's on which NFS will listen for requests. When it is unset/empty, NFS listens on all available addresses. `v4` when set means that we switch from NFSv3 to NFSv4. `v4_v3owner` when set means that system will use NFSv3 ownership model for NFSv4. `v4_krb` will force NFS shares to fail if the Kerberos ticket is unavailable. `v4_domain` overrides the default DNS domain name for NFSv4. `mountd_port` specifies the port mountd(8) binds to. `rpcstatd_port` specifies the port rpc.statd(8) binds to. `rpclockd_port` specifies the port rpclockd_port(8) binds to. .. examples(websocket):: Update NFS Service Configuration to listen on 192.168.0.10 and use NFSv4 :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "pool.resilver.update", "params": [{ "bindip": [ "192.168.0.10" ], "v4": true }] } """ if data.get("v4") is False: data.setdefault("v4_v3owner", False) old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() keytab_has_nfs = await self.middleware.call( "kerberos.keytab.has_nfs_principal") new_v4_krb_enabled = new["v4_krb"] or keytab_has_nfs if new["v4"] and new_v4_krb_enabled and await self.middleware.call( "system.is_enterprise"): if await self.middleware.call("failover.licensed"): gc = await self.middleware.call("datastore.config", "network.globalconfiguration") if not gc["gc_hostname_virtual"] or not gc["gc_domain"]: verrors.add( "nfs_update.v4", "Enabling kerberos authentication on TrueNAS HA requires setting the virtual hostname and " "domain") if osc.IS_LINUX: if len(new['bindip']) > 1: verrors.add( 'nfs_update.bindip', 'Listening on more than one address is not supported') bindip_choices = await self.bindip_choices() for i, bindip in enumerate(new['bindip']): if bindip not in bindip_choices: verrors.add(f'nfs_update.bindip.{i}', 'Please provide a valid ip address') if new["v4"] and new_v4_krb_enabled and await self.middleware.call( 'activedirectory.get_state') != "DISABLED": """ In environments with kerberized NFSv4 enabled, we need to tell winbindd to not prefix usernames with the short form of the AD domain. Directly update the db and regenerate the smb.conf to avoid having a service disruption due to restarting the samba server. """ if await self.middleware.call('smb.get_smb_ha_mode') == 'LEGACY': raise ValidationError( 'nfs_update.v4', 'Enabling kerberos authentication on TrueNAS HA requires ' 'the system dataset to be located on a data pool.') ad = await self.middleware.call('activedirectory.config') await self.middleware.call('datastore.update', 'directoryservice.activedirectory', ad['id'], {'ad_use_default_domain': True}) await self.middleware.call('etc.generate', 'smb') await self.middleware.call('service.reload', 'cifs') if not new["v4"] and new["v4_v3owner"]: verrors.add("nfs_update.v4_v3owner", "This option requires enabling NFSv4") if new["v4_v3owner"] and new["userd_manage_gids"]: verrors.add( "nfs_update.userd_manage_gids", "This option is incompatible with NFSv3 ownership model for NFSv4" ) if not new["v4"] and new["v4_domain"]: verrors.add("nfs_update.v4_domain", "This option does not apply to NFSv3") if verrors: raise verrors await self.nfs_compress(new) await self._update_service(old, new) await self.nfs_extend(new) return new
class SMBService(SystemServiceService): class Config: service = 'cifs' service_verb = 'restart' datastore = 'services.cifs' datastore_extend = 'smb.smb_extend' datastore_prefix = 'cifs_srv_' @private async def smb_extend(self, smb): """Extend smb for netbios.""" if not await self.middleware.call('notifier.is_freenas' ) and await self.middleware.call( 'notifier.failover_node') == 'B': smb['netbiosname'] = smb['netbiosname_b'] for i in ('aio_enable', 'aio_rs', 'aio_ws'): smb.pop(i, None) smb['loglevel'] = LOGLEVEL_MAP.get(smb['loglevel']) return smb async def __validate_netbios_name(self, name): return RE_NETBIOSNAME.match(name) async def unixcharset_choices(self): return await self.generate_choices([ 'UTF-8', 'ISO-8859-1', 'ISO-8859-15', 'GB2312', 'EUC-JP', 'ASCII' ]) @private async def generate_choices(self, initial): def key_cp(encoding): cp = re.compile( "(?P<name>CP|GB|ISO-8859-|UTF-)(?P<num>\d+)").match(encoding) if cp: return tuple((cp.group('name'), int(cp.group('num'), 10))) else: return tuple((encoding, float('inf'))) charset = await self.common_charset_choices() return { v: v for v in [c for c in sorted(charset, key=key_cp) if c not in initial] + initial } @private async def common_charset_choices(self): def check_codec(encoding): try: return encoding.upper() if codecs.lookup(encoding) else False except LookupError: return False proc = await Popen(['/usr/bin/iconv', '-l'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output = (await proc.communicate())[0].decode() encodings = set() for line in output.splitlines(): enc = [e for e in line.split() if check_codec(e)] if enc: cp = enc[0] for e in enc: if e in ('UTF-8', 'ASCII', 'GB2312', 'HZ-GB-2312', 'CP1361'): cp = e break encodings.add(cp) return encodings @accepts( Dict( 'smb_update', Str('netbiosname'), Str('netbiosname_b'), Str('netbiosalias'), Str('workgroup'), Str('description'), Bool('allow_smb1'), Str('unixcharset'), Str('loglevel', enum=['NONE', 'MINIMUM', 'NORMAL', 'FULL', 'DEBUG']), Bool('syslog'), Bool('localmaster'), Bool('domain_logons'), Bool('timeserver'), Str('guest'), Str('filemask'), Str('dirmask'), Bool('nullpw'), Bool('unixext'), Bool('zeroconf'), Bool('hostlookup'), Bool('allow_execute_always'), Bool('obey_pam_restrictions'), Bool('ntlmv1_auth'), List('bindip', items=[IPAddr('ip')], default=[]), Str('smb_options'), update=True, )) async def do_update(self, data): old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() if data.get('unixcharset') and data[ 'unixcharset'] not in await self.unixcharset_choices(): verrors.add('smb_update.unixcharset', 'Please provide a valid value for unixcharset') for i in ('workgroup', 'netbiosname', 'netbiosname_b', 'netbiosalias'): if i not in data or not data[i]: continue if not await self.__validate_netbios_name(data[i]): verrors.add(f'smb_update.{i}', 'Invalid NetBIOS name') if new['netbiosname'] and new['netbiosname'].lower( ) == new['workgroup'].lower(): verrors.add('smb_update.netbiosname', 'NetBIOS and Workgroup must be unique') for i in ('filemask', 'dirmask'): if i not in data or not data[i]: continue try: if int(data[i], 8) & ~0o11777: raise ValueError('Not an octet') except (ValueError, TypeError): verrors.add(f'smb_update.{i}', 'Not a valid mask') if verrors: raise verrors # TODO: consider using bidict for k, v in LOGLEVEL_MAP.items(): if new['loglevel'] == v: new['loglevel'] = k break await self._update_service(old, new) return await self.config()
class SharingNFSService(CRUDService): class Config: namespace = "sharing.nfs" datastore = "sharing.nfs_share" datastore_prefix = "nfs_" datastore_extend = "sharing.nfs.extend" @accepts( Dict( "sharingnfs_create", List("paths", items=[Dir("path")]), Str("comment"), List("networks", items=[IPAddr("network", cidr=True)]), List("hosts", items=[Str("host")]), Bool("alldirs"), Bool("ro"), Bool("quiet"), Str("maproot_user", required=False, default=None), Str("maproot_group", required=False, default=None), Str("mapall_user", required=False, default=None), Str("mapall_group", required=False, default=None), List("security", items=[ Str("provider", enum=["SYS", "KRB5", "KRB5I", "KRB5P"]) ]), register=True, )) async def do_create(self, data): verrors = ValidationErrors() await self.validate(data, "sharingnfs_create", verrors) if verrors: raise verrors await self.compress(data) paths = data.pop("paths") data["id"] = await self.middleware.call( "datastore.insert", self._config.datastore, data, {"prefix": self._config.datastore_prefix}, ) for path in paths: await self.middleware.call( "datastore.insert", "sharing.nfs_share_path", { "share_id": data["id"], "path": path, }, ) await self.extend(data) await self.middleware.call("service.reload", "nfs") return data @accepts(Int("id"), Patch("sharingnfs_create", "sharingnfs_update", ("attr", { "update": True }))) async def do_update(self, id, data): verrors = ValidationErrors() old = await self.middleware.call( "datastore.query", self._config.datastore, [("id", "=", id)], { "extend": self._config.datastore_extend, "prefix": self._config.datastore_prefix, "get": True }, ) new = old.copy() new.update(data) await self.validate(new, "sharingnfs_update", verrors, old=old) if verrors: raise verrors await self.compress(new) paths = new.pop("paths") await self.middleware.call("datastore.update", self._config.datastore, id, new, {"prefix": self._config.datastore_prefix}) await self.middleware.call("datastore.delete", "sharing.nfs_share_path", [["share_id", "=", id]]) for path in paths: await self.middleware.call( "datastore.insert", "sharing.nfs_share_path", { "share_id": id, "path": path, }, ) await self.extend(new) new["paths"] = paths await self.middleware.call("service.reload", "nfs") return new @accepts(Int("id")) async def do_delete(self, id): await self.middleware.call("datastore.delete", "sharing.nfs_share_path", [["share_id", "=", id]]) await self.middleware.call("datastore.delete", self._config.datastore, id) @private async def validate(self, data, schema_name, verrors, old=None): if not data["paths"]: verrors.add(f"{schema_name}.paths", "At least one path is required") await self.middleware.run_in_thread(self.validate_paths, data, schema_name, verrors) filters = [] if old: filters.append(["id", "!=", old["id"]]) other_shares = await self.middleware.call("sharing.nfs.query", filters) dns_cache = await self.resolve_hostnames( sum([share["hosts"] for share in other_shares], []) + data["hosts"]) await self.middleware.run_in_thread(self.validate_hosts_and_networks, other_shares, data, schema_name, verrors, dns_cache) for k in ["maproot", "mapall"]: if not data[f"{k}_user"] and not data[f"{k}_group"]: pass elif not data[f"{k}_user"] and data[f"{k}_group"]: verrors.add( f"{schema_name}.{k}_user", "This field is required when map group is specified") else: user = await self.middleware.call( "user.query", [("username", "=", data[f"{k}_user"])]) if not user: verrors.add(f"{schema_name}.{k}_user", "User not found") if data[f"{k}_group"]: group = await self.middleware.call( "group.query", [("group", "=", data[f"{k}_group"])]) if not group: verrors.add(f"{schema_name}.{k}_group", "Group not found") if data["maproot_user"] and data["mapall_user"]: verrors.add(f"{schema_name}.mapall_user", "maproot_user disqualifies mapall_user") if data["security"]: nfs_config = await self.middleware.call("nfs.config") if not nfs_config["v4"]: verrors.add(f"{schema_name}.security", "This is not allowed when NFS v4 is disabled") @private def validate_paths(self, data, schema_name, verrors): dev = None is_mountpoint = False for i, path in enumerate(data["paths"]): stat = os.stat(path) if dev is None: dev = stat.st_dev else: if dev != stat.st_dev: verrors.add( f"{schema_name}.paths.{i}", "Paths for a NFS share must reside within the same filesystem" ) parent = os.path.abspath(os.path.join(path, "..")) if os.stat(parent).st_dev != dev: is_mountpoint = True if any( os.path.abspath(p).startswith(parent + "/") for p in data["paths"] if p != path): verrors.add( f"{schema_name}.paths.{i}", "You cannot share a mount point and subdirectories all at once" ) if not is_mountpoint and data["alldirs"]: verrors.add(f"{schema_name}.alldirs", "This option can only be used for datasets") @private async def resolve_hostnames(self, hostnames): hostnames = list(set(hostnames)) async def resolve(hostname): try: return (await asyncio.wait_for( self.middleware.run_in_thread(socket.getaddrinfo, hostname, None), 5))[0][4][0] except Exception as e: self.logger.warning("Unable to resolve host %r: %r", hostname, e) return None resolved_hostnames = await asyncio_map(resolve, hostnames, 8) return dict(zip(hostnames, resolved_hostnames)) @private def validate_hosts_and_networks(self, other_shares, data, schema_name, verrors, dns_cache): explanation = ( ". This is so because /etc/exports does not act like ACL and it is undefined which rule among " "all overlapping networks will be applied.") dev = os.stat(data["paths"][0]).st_dev used_networks = set() for share in other_shares: try: share_dev = os.stat(share["paths"][0]).st_dev except Exception: self.logger.warning("Failed to stat first path for %r", share, exc_info=True) continue if share_dev == dev: for host in share["hosts"]: host = dns_cache[host] if host is None: continue try: network = ipaddress.ip_network(host) except Exception: self.logger.warning("Got invalid host %r", host) continue else: used_networks.add(network) for network in share["networks"]: try: network = ipaddress.ip_network(network, strict=False) except Exception: self.logger.warning("Got invalid network %r", network) continue else: used_networks.add(network) if not share["hosts"] and not share["networks"]: used_networks.add(ipaddress.ip_network("0.0.0.0/0")) used_networks.add(ipaddress.ip_network("::/0")) if share["alldirs"] and data["alldirs"]: verrors.add( f"{schema_name}.alldirs", "This option is only available once per mountpoint") had_explanation = False for i, host in enumerate(data["hosts"]): host = dns_cache[host] if host is None: continue network = ipaddress.ip_network(host) for another_network in used_networks: if network.overlaps(another_network): verrors.add(f"{schema_name}.hosts.{i}", ( f"You can't share same filesystem with overlapping networks {network} and {another_network}" + ("" if had_explanation else explanation))) had_explanation = True used_networks.add(network) had_explanation = False for i, network in enumerate(data["networks"]): network = ipaddress.ip_network(network, strict=False) for another_network in used_networks: if network.overlaps(another_network): verrors.add(f"{schema_name}.networks.{i}", ( f"You can't share same filesystem with overlapping networks {network} and {another_network}" + ("" if had_explanation else explanation))) had_explanation = True used_networks.add(network) if not data["hosts"] and not data["networks"]: if used_networks: verrors.add( f"{schema_name}.networks", (f"You can't share same filesystem with all hosts twice" + ("" if had_explanation else explanation))) @private async def extend(self, data): data["paths"] = [ path["path"] for path in await self.middleware.call( "datastore.query", "sharing.nfs_share_path", [["share_id", "=", data["id"]]]) ] data["networks"] = data.pop("network").split() data["hosts"] = data["hosts"].split() data["security"] = [s.upper() for s in data["security"]] return data @private async def compress(self, data): data["network"] = " ".join(data.pop("networks")) data["hosts"] = " ".join(data["hosts"]) data["security"] = [s.lower() for s in data["security"]] return data
class SMBService(SystemServiceService): class Config: service = 'cifs' service_verb = 'restart' datastore = 'services.cifs' datastore_extend = 'smb.smb_extend' datastore_prefix = 'cifs_srv_' @private async def smb_extend(self, smb): """Extend smb for netbios.""" if not await self.middleware.call('system.is_freenas') and await self.middleware.call('failover.node') == 'B': smb['netbiosname'] = smb['netbiosname_b'] for i in ('aio_enable', 'aio_rs', 'aio_ws'): smb.pop(i, None) smb['netbiosalias'] = (smb['netbiosalias'] or '').split() smb['loglevel'] = LOGLEVEL_MAP.get(smb['loglevel']) return smb async def __validate_netbios_name(self, name): return RE_NETBIOSNAME.match(name) async def unixcharset_choices(self): return await self.generate_choices( ['UTF-8', 'ISO-8859-1', 'ISO-8859-15', 'GB2312', 'EUC-JP', 'ASCII'] ) @private async def generate_choices(self, initial): def key_cp(encoding): cp = re.compile(r"(?P<name>CP|GB|ISO-8859-|UTF-)(?P<num>\d+)").match(encoding) if cp: return tuple((cp.group('name'), int(cp.group('num'), 10))) else: return tuple((encoding, float('inf'))) charset = await self.common_charset_choices() return { v: v for v in [ c for c in sorted(charset, key=key_cp) if c not in initial ] + initial } @private async def validate_admin_groups(self, sid): """ Check if group mapping already exists because 'net groupmap addmem' will fail if the mapping exists. Remove any entries that should not be present. Extra entries here can pose a significant security risk. The only default entry will have a RID value of "512" (Domain Admins). In LDAP environments, members of S-1-5-32-544 cannot be removed without impacting the entire LDAP environment because this alias exists on the remote LDAP server. """ sid_is_present = False ldap = await self.middleware.call('datastore.config', 'directoryservice.ldap') if ldap['ldap_enable']: self.logger.debug("As a safety precaution, extra alias entries for S-1-5-32-544 cannot be removed while LDAP is enabled. Skipping removal.") return True proc = await Popen( ['/usr/local/bin/net', 'groupmap', 'listmem', 'S-1-5-32-544'], stdout=subprocess.PIPE, stderr=subprocess.PIPE ) member_list = (await proc.communicate())[0].decode() if not member_list: return True for group in member_list.splitlines(): group = group.strip() if group == sid: self.logger.debug(f"SID [{sid}] is already a member of BUILTIN\\administrators") sid_is_present = True if group.rsplit('-', 1)[-1] != "512" and group != sid: self.logger.debug(f"Removing {group} from local admins group.") rem = await Popen( ['/usr/local/bin/net', 'groupmap', 'delmem', 'S-1-5-32-544', group], stdout=subprocess.PIPE, stderr=subprocess.PIPE ) remout = await rem.communicate() if rem.returncode != 0: raise CallError(f'Failed to remove sid [{sid}] from S-1-5-32-544: {remout[1].decode()}') if sid_is_present: return False else: return True @private async def wbinfo_gidtosid(self, gid): verrors = ValidationErrors() proc = await Popen( ['/usr/local/bin/wbinfo', '--gid-to-sid', f"{gid}"], stdout=subprocess.PIPE, stderr=subprocess.PIPE ) output = await proc.communicate() if proc.returncode != 0: if "WBC_ERR_WINBIND_NOT_AVAILABLE" in output[1].decode(): return "WBC_ERR_WINBIND_NOT_AVAILABLE" else: verrors.add('smb_update.admin_group', f"Failed to identify Windows SID for group: {output[1].decode()}") raise verrors return output[0].decode().strip() @private async def add_admin_group(self, admin_group=None, check_deferred=False): """ Add a local or directory service group to BUILTIN\\Administrators (S-1-5-32-544) Members of this group have elevated privileges to the Samba server (ability to take ownership of files, override ACLs, view and modify user quotas, and administer the server via the Computer Management MMC Snap-In. Unfortuntely, group membership must be managed via "net groupmap listmem|addmem|delmem", which requires that winbind be running when the commands are executed. In this situation, net command will fail with WBC_ERR_WINBIND_NOT_AVAILABLE. If this error message is returned, then flag for a deferred command retry when service starts. @param-in (admin_group): This is the group to add to BUILTIN\\Administrators. If unset, then look up the value in the config db. @param-in (check_deferred): If this is True, then only perform the group mapping if this has been flagged as in need of deferred setup (i.e. Samba wasn't running when it was initially called). This is to avoid unecessarily calling during service start. """ verrors = ValidationErrors() if check_deferred: is_deferred = await self.middleware.call('cache.has_key', 'SMB_SET_ADMIN') if not is_deferred: self.logger.debug("No cache entry indicating delayed action to add admin_group was found.") return True else: await self.middleware.call('cache.pop', 'SMB_SET_ADMIN') if not admin_group: smb = await self.middleware.call('smb.config') admin_group = smb['admin_group'] # We must use GIDs because wbinfo --name-to-sid expects a domain prefix "FREENAS\user" group = await self.middleware.call("notifier.get_group_object", admin_group) if not group: verrors.add('smb_update.admin_group', f"Failed to validate group: {admin_group}") raise verrors sid = await self.wbinfo_gidtosid(group[2]) if sid == "WBC_ERR_WINBIND_NOT_AVAILABLE": self.logger.debug("Delaying admin group add until winbind starts") await self.middleware.call('cache.put', 'SMB_SET_ADMIN', True) return True must_add_sid = await self.validate_admin_groups(sid) if not must_add_sid: return True proc = await Popen( ['/usr/local/bin/net', 'groupmap', 'addmem', 'S-1-5-32-544', sid], stdout=subprocess.PIPE, stderr=subprocess.PIPE ) output = await proc.communicate() if proc.returncode != 0: raise CallError(f'net groupmap addmem failed: {output[1].decode()}') self.logger.debug(f"Successfully added {admin_group} to BUILTIN\\Administrators") return True @private async def common_charset_choices(self): def check_codec(encoding): try: return encoding.upper() if codecs.lookup(encoding) else False except LookupError: return False proc = await Popen( ['/usr/bin/iconv', '-l'], stdout=subprocess.PIPE, stderr=subprocess.PIPE ) output = (await proc.communicate())[0].decode() encodings = set() for line in output.splitlines(): enc = [e for e in line.split() if check_codec(e)] if enc: cp = enc[0] for e in enc: if e in ('UTF-8', 'ASCII', 'GB2312', 'HZ-GB-2312', 'CP1361'): cp = e break encodings.add(cp) return encodings @private async def store_ldap_admin_password(self): """ This is required if the LDAP directory service is enabled. The ldap admin dn and password are stored in private/secrets.tdb file. """ ldap = await self.middleware.call('datastore.config', 'directoryservice.ldap') if not ldap['ldap_enable']: return True set_pass = await run(['usr/local/bin/smbpasswd', '-w', ldap['ldap_bindpw']], check=False) if set_pass.returncode != 0: self.logger.debug(f"Failed to set set ldap bindpw in secrets.tdb: {set_pass.stdout.decode()}") return False return True @accepts(Dict( 'smb_update', Str('netbiosname'), Str('netbiosname_b'), List('netbiosalias', default=[]), Str('workgroup'), Str('description'), Bool('enable_smb1'), Str('unixcharset'), Str('loglevel', enum=['NONE', 'MINIMUM', 'NORMAL', 'FULL', 'DEBUG']), Bool('syslog'), Bool('localmaster'), Bool('domain_logons'), Bool('timeserver'), Str('guest'), Str('admin_group'), Str('filemask'), Str('dirmask'), Bool('nullpw'), Bool('unixext'), Bool('zeroconf'), Bool('hostlookup'), Bool('allow_execute_always'), Bool('obey_pam_restrictions'), Bool('ntlmv1_auth'), List('bindip', items=[IPAddr('ip')], default=[]), Str('smb_options'), update=True, )) async def do_update(self, data): """ Update SMB Service Configuration. `netbiosname` defaults to the original hostname of the system. `workgroup` and `netbiosname` should have different values. `enable_smb1` allows legacy SMB clients to connect to the server when enabled. `localmaster` when set, determines if the system participates in a browser election. `domain_logons` is used to provide netlogin service for older Windows clients if enabled. `guest` attribute is specified to select the account to be used for guest access. It defaults to "nobody". `nullpw` when enabled allows the users to authorize access without a password. `zeroconf` should be enabled if macOS Clients will be connecting to the SMB share. `hostlookup` when enabled, allows using hostnames rather then IP addresses in "hostsallow"/"hostsdeny" fields of SMB Shares. """ old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() if data.get('unixcharset') and data['unixcharset'] not in await self.unixcharset_choices(): verrors.add( 'smb_update.unixcharset', 'Please provide a valid value for unixcharset' ) for i in ('workgroup', 'netbiosname', 'netbiosname_b', 'netbiosalias'): if i not in data or not data[i]: continue if i == 'netbiosalias': for idx, item in enumerate(data[i]): if not await self.__validate_netbios_name(item): verrors.add(f'smb_update.{i}.{idx}', f'Invalid NetBIOS name: {item}') else: if not await self.__validate_netbios_name(data[i]): verrors.add(f'smb_update.{i}', f'Invalid NetBIOS name: {data[i]}') if new['netbiosname'] and new['netbiosname'].lower() == new['workgroup'].lower(): verrors.add('smb_update.netbiosname', 'NetBIOS and Workgroup must be unique') for i in ('filemask', 'dirmask'): if i not in data or not data[i]: continue try: if int(data[i], 8) & ~0o11777: raise ValueError('Not an octet') except (ValueError, TypeError): verrors.add(f'smb_update.{i}', 'Not a valid mask') if new['admin_group'] and new['admin_group'] != old['admin_group']: await self.add_admin_group(new['admin_group']) if verrors: raise verrors # TODO: consider using bidict for k, v in LOGLEVEL_MAP.items(): if new['loglevel'] == v: new['loglevel'] = k break await self.compress(new) await self._update_service(old, new) return await self.config() @private async def compress(self, data): data['netbiosalias'] = ' '.join(data['netbiosalias']) return data
class SharingSMBService(CRUDService): class Config: namespace = 'sharing.smb' datastore = 'sharing.cifs_share' datastore_prefix = 'cifs_' datastore_extend = 'sharing.smb.extend' @accepts( Dict('sharingsmb_create', Str('path', required=True), Bool('home', default=False), Str('name'), Str('comment'), Bool('ro', default=False), Bool('browsable', default=True), Bool('recyclebin', default=False), Bool('showhiddenfiles', default=False), Bool('guestok', default=False), Bool('guestonly', default=False), Bool('abe', default=False), List('hostsallow', items=[IPAddr('ip', network=True)], default=[]), List('hostsdeny', items=[IPAddr('ip', network=True)], default=[]), List('vfsobjects', default=['zfs_space', 'zfsacl', 'streams_xattr']), Int('storage_task'), Str('auxsmbconf'), Bool('default_permissions'), register=True)) async def do_create(self, data): verrors = ValidationErrors() path = data['path'] default_perms = data.pop('default_permissions', True) await self.clean(data, 'sharingsmb_create', verrors) await self.validate(data, 'sharingsmb_create', verrors) if verrors: raise verrors if path and not os.path.exists(path): try: os.makedirs(path) except OSError as e: raise CallError(f'Failed to create {path}: {e}') await self.compress(data) await self.set_storage_tasks(data) data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self.extend(data) # We should do this in the insert call ? await self._service_change('cifs', 'reload') await self.apply_default_perms(default_perms, path, data['home']) return data @accepts(Int('id'), Patch('sharingsmb_create', 'sharingsmb_update', ('attr', { 'update': True }))) async def do_update(self, id, data): verrors = ValidationErrors() path = data.get('path') default_perms = data.pop('default_permissions', False) old = await self.middleware.call( 'datastore.query', self._config.datastore, [('id', '=', id)], { 'extend': self._config.datastore_extend, 'prefix': self._config.datastore_prefix, 'get': True }) new = old.copy() new.update(data) await self.clean(new, 'sharingsmb_update', verrors, id=id) await self.validate(new, 'sharingsmb_update', verrors, old=old) if verrors: raise verrors if path and not os.path.exists(path): try: os.makedirs(path) except OSError as e: raise CallError(f'Failed to create {path}: {e}') await self.compress(new) await self.set_storage_tasks(new) await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) await self.extend(new) # same here ? await self._service_change('cifs', 'reload') await self.apply_default_perms(default_perms, path, data['home']) return new @accepts(Int('id')) async def do_delete(self, id): share = await self._get_instance(id) result = await self.middleware.call('datastore.delete', self._config.datastore, id) await self.middleware.call('notifier.sharesec_delete', share['name']) await self._service_change('cifs', 'reload') return result @private async def clean(self, data, schema_name, verrors, id=None): data['name'] = await self.name_exists(data, schema_name, verrors, id) @private async def validate(self, data, schema_name, verrors, old=None): home_result = await self.home_exists(data['home'], schema_name, verrors, old) if home_result: verrors.add(f'{schema_name}.home', 'Only one share is allowed to be a home share.') elif not home_result and not data['path']: verrors.add(f'{schema_name}.path', 'This field is required.') if data['path']: await check_path_resides_within_volume(verrors, self.middleware, f"{schema_name}.path", data['path']) if data.get('name') and data['name'] == 'global': verrors.add( f'{schema_name}.name', 'Global is a reserved section name, please select another one') @private async def home_exists(self, home, schema_name, verrors, old=None): home_filters = [('home', '=', True)] home_result = None if home: if old and old['id'] is not None: id = old['id'] if not old['home']: home_filters.append(('id', '!=', id)) # The user already had this set as the home share home_result = await self.middleware.call( 'datastore.query', self._config.datastore, home_filters, {'prefix': self._config.datastore_prefix}) return home_result @private async def name_exists(self, data, schema_name, verrors, id=None): name = data['name'] path = data['path'] name_filters = [('name', '=', name)] path_filters = [('path', '=', path)] if path and not name: name = path.rsplit('/', 1)[-1] if id is not None: name_filters.append(('id', '!=', id)) path_filters.append(('id', '!=', id)) name_result = await self.middleware.call( 'datastore.query', self._config.datastore, name_filters, {'prefix': self._config.datastore_prefix}) path_result = await self.middleware.call( 'datastore.query', self._config.datastore, path_filters, {'prefix': self._config.datastore_prefix}) if name_result: verrors.add(f'{schema_name}.name', 'A share with this name already exists.') if path_result: verrors.add(f'{schema_name}.path', 'A share with this path already exists.') return name @private async def extend(self, data): data['hostsallow'] = data['hostsallow'].split() data['hostsdeny'] = data['hostsdeny'].split() return data @private async def compress(self, data): data['hostsallow'] = ' '.join(data['hostsallow']) data['hostsdeny'] = ' '.join(data['hostsdeny']) return data @private async def apply_default_perms(self, default_perms, path, is_home): if default_perms: try: (owner, group) = await self.middleware.call('notifier.mp_get_owner', path) except Exception: (owner, group) = ('root', 'wheel') await self.middleware.call('notifier.winacl_reset', path, owner, group, None, not is_home) @accepts(Str('path', required=True)) async def get_storage_tasks(self, path): zfs_datasets = await self.middleware.call( 'zfs.dataset.query', [('type', '=', 'FILESYSTEM')]) task_list = [] task_dict = {} for ds in zfs_datasets: tasks = [] name = ds['name'] mountpoint = ds['properties']['mountpoint']['parsed'] if path == mountpoint: tasks = await self.middleware.call( 'datastore.query', 'storage.task', [['task_filesystem', '=', name]]) elif path.startswith(f'{mountpoint}/'): tasks = await self.middleware.call( 'datastore.query', 'storage.task', [['task_filesystem', '=', name], ['task_recursive', '=', 'True']]) task_list.extend(tasks) for task in task_list: task_id = task['id'] fs = task['task_filesystem'] retcount = task['task_ret_count'] retunit = task['task_ret_unit'] _interval = task['task_interval'] interval = dict(await self.middleware.call('notifier.choices', 'TASK_INTERVAL'))[_interval] msg = f'{fs} - every {interval} - {retcount}{retunit}' task_dict[task_id] = msg return task_dict @private async def set_storage_tasks(self, data): task = data.get('storage_task', None) home = data['home'] path = data['path'] task_list = [] if not task: if path: task_list = await self.get_storage_tasks(path=path) elif home: task_list = await self.get_storage_tasks(home=home) if task_list: data['storage_task'] = list(task_list.keys())[0] return data @accepts() def vfsobjects_choices(self): vfs_modules_path = '/usr/local/lib/shared-modules/vfs' vfs_modules = [] vfs_exclude = {'shadow_copy2', 'recycle', 'aio_pthread'} if os.path.exists(vfs_modules_path): vfs_modules.extend( filter( lambda m: m not in vfs_exclude, map(lambda f: f.rpartition('.')[0], os.listdir(vfs_modules_path)))) else: vfs_modules.extend(['streams_xattr']) return vfs_modules
class CtdbPublicIpService(CRUDService): class Config: namespace = 'ctdb.public.ips' cli_private = True @filterable def query(self, filters, options): """ Retrieve information about configured public IP addresses for the ctdb cluster. This call raise a CallError with errno set to ENXIO if this node is not in a state where it can provide accurate information about cluster. Examples problematic states are: - ctdb or glusterd are not running on this node - ctdb shared volume is not mounted """ if not self.middleware.call_sync('service.started', 'ctdb'): raise CallError( "ctdb is not running. Unable to gather public address info", errno.ENXIO) ctdb_ips = self.middleware.call_sync('ctdb.general.ips') try: shared_vol = Path(CTDBConfig.CTDB_LOCAL_MOUNT.value) mounted = shared_vol.is_mount() except Exception: # can happen when mounted but glusterd service # is stopped/crashed etc mounted = False if not mounted: raise CallError("CTDB shared volume is in unhealthy state.", errno.ENXIO) nodes = {} for entry in self.middleware.call_sync('ctdb.general.listnodes'): """ Skip disabled nodes since they cannot hold public addresses. If a node does not have a public_addresses file, we should still return an entry for it (but without any configured_addresses). This allows us to accurately report cases where perhaps due to user intervention, public address file was removed but ctdb IPs have not been reloaded. """ if not entry['enabled']: continue pnn = entry['pnn'] nodes[pnn] = { 'id': pnn, 'pnn': pnn, 'configured_ips': {}, 'active_ips': {} } with contextlib.suppress(FileNotFoundError): with open(f'{shared_vol}/public_addresses_{pnn}') as f: for i in f.read().splitlines(): if not i.startswith('#'): enabled = True public_ip = i.split('/')[0] else: enabled = False public_ip = i.split('#')[1].split('/')[0] nodes[pnn]['configured_ips'].update({ public_ip: { 'enabled': enabled, 'public_ip': public_ip, 'interface_name': i.split()[-1] } }) for entry in ctdb_ips: if not nodes.get(entry['pnn']): """ Most likely case here is that we're transitioning IP and it's current pnn is -1. Generate log message for now, and we can determine in future whether more action is required. """ self.logger.warning( "%s: active ip address does not exist in public_addresses file", entry['public_ip']) continue nodes[entry['pnn']]['active_ips'].update( {entry['public_ip']: entry['interfaces']}) return filter_list(list(nodes.values()), filters, options) @private async def reload(self): """ Reload the public addresses configuration file on the ctdb nodes. When it completes the public addresses will be reconfigured and reassigned across the cluster as necessary. """ if await self.middleware.call('service.started', 'ctdb'): re = await run(['ctdb', 'reloadips'], encoding='utf8', errors='ignore', check=False) if re.returncode: # this isn't fatal it just means the newly added public ip won't show # up until the ctdb service has been restarted so just log a message self.logger.warning('Failed to reload public ip addresses %r', re.stderr) @accepts( Dict( 'public_create', Int('pnn', required=True), IPAddr('ip', required=True), Int('netmask', required=True), Str('interface', required=True), )) @job(lock=PUB_LOCK) async def do_create(self, job, data): """ Add a ctdb public address to the cluster `pnn` node number of record to adjust `ip` string representing an IP v4/v6 address `netmask` integer representing a cidr notated netmask (i.e. 16/24/48/64 etc) `interface` string representing a network interface to apply the `ip` """ schema_name = 'public_create' verrors = ValidationErrors() await self.middleware.call('ctdb.ips.common_validation', data, schema_name, verrors) await self.middleware.call('ctdb.ips.update_file', data, schema_name) await self.middleware.call('ctdb.public.ips.reload') return await self.middleware.call('ctdb.public.ips.query', [('id', '=', data['pnn'])]) @accepts(Int('pnn', required=True), Dict( 'public_update', Str('ip', validators=[IpAddress()], required=True), Bool('enable', required=True), )) @job(lock=PUB_LOCK) async def do_update(self, job, id, option): """ Update Public IP address in the ctdb cluster. `pnn` - cluster node number `ip` string representing the public ip address `enable` boolean. When True, enable the node else disable the node. """ schema_name = 'public_update' verrors = ValidationErrors() data = await self.get_instance(id) data['enable'] = option['enable'] await self.middleware.call('ctdb.ips.common_validation', data, schema_name, verrors) await self.middleware.call('ctdb.ips.update_file', data, schema_name) await self.middleware.call('ctdb.public.ips.reload') return await self.get_instance(id)
class OpenVPNServerService(SystemServiceService): class Config: namespace = 'openvpn.server' service = 'openvpn_server' service_model = 'openvpnserver' service_verb = 'restart' datastore_extend = 'openvpn.server.server_extend' @private async def server_extend(self, data): data.update({ 'server_certificate': None if not data['server_certificate'] else data['server_certificate']['id'], 'root_ca': None if not data['root_ca'] else data['root_ca']['id'], 'tls_crypt_auth_enabled': bool(data['tls_crypt_auth']), 'interface': 'openvpn-server', }) return data @private async def config_valid(self): config = await self.config() if not config['root_ca']: raise CallError('Please configure root_ca first.') else: if not await self.middleware.call( 'certificateauthority.query', [['id', '=', config['root_ca']], ['revoked', '=', False]]): raise CallError( 'Root CA has been revoked. Please select another Root CA.') if not config['server_certificate']: raise CallError('Please configure server certificate first.') else: if not await self.middleware.call('certificate.query', [[ 'id', '=', config['server_certificate'] ], ['revoked', '=', False]]): raise CallError( 'Server certificate has been revoked. Please select another Server certificate.' ) if not await self.validate_nobind(config): raise CallError( 'Please enable "nobind" on OpenVPN Client to concurrently run OpenVPN Server/Client ' 'on the same local port without any issues.') @accepts() async def authentication_algorithm_choices(self): """ Returns a dictionary of valid authentication algorithms which can be used with OpenVPN server. """ return OpenVPN.digests() @accepts() async def cipher_choices(self): """ Returns a dictionary of valid ciphers which can be used with OpenVPN server. """ return OpenVPN.ciphers() @private async def validate(self, data, schema_name): verrors, data = await OpenVPN.common_validation( self.middleware, data, schema_name, 'server') if not await self.validate_nobind(data): verrors.add( f'{schema_name}.nobind', 'Please enable "nobind" on OpenVPN Client to concurrently run OpenVPN Server/Client ' 'on the same local port without any issues.') if ipaddress.ip_address( data['server']).version == 4 and data['netmask'] > 32: verrors.add( f'{schema_name}.netmask', 'For IPv4 server addresses please provide a netmask value from 0-32.' ) verrors.check() return data @private async def validate_nobind(self, config): client_config = await self.middleware.call('openvpn.client.config') if (await self.middleware.call('service.started', 'openvpn_client') and config['port'] == client_config['port'] and not client_config['nobind']): return False else: return True @private async def generate_static_key(self): keyfile = tempfile.NamedTemporaryFile(mode='w+', dir='/tmp/') await run(['openvpn', '--genkey', '--secret', keyfile.name]) keyfile.seek(0) key = keyfile.read() keyfile.close() return key.strip() @accepts() async def renew_static_key(self): """ Reset OpenVPN server's TLS static key which will be used to encrypt/authenticate control channel packets. """ return await self.update({ 'tls_crypt_auth': (await self.generate_static_key()), 'tls_crypt_auth_enabled': True }) @accepts(Int('client_certificate_id'), Str('server_address', null=True)) async def client_configuration_generation(self, client_certificate_id, server_address=None): """ Returns a configuration for OpenVPN client which can be used with any client to connect to FN/TN OpenVPN server. `client_certificate_id` should be a valid certificate issued for use with OpenVPN client service. `server_address` if specified auto-fills the remote directive in the OpenVPN configuration enabling the end user to use the file without making any edits to connect to OpenVPN server. """ await self.config_valid() config = await self.config() root_ca = await self.middleware.call('certificateauthority.query', [['id', '=', config['root_ca']]], {'get': True}) client_cert = await self.middleware.call( 'certificate.query', [['id', '=', client_certificate_id], ['revoked', '=', False]]) if not client_cert: raise CallError( 'Please provide a client certificate id for a certificate which exists on ' 'the system and hasn\'t been marked as revoked.') else: client_cert = client_cert[0] if (await OpenVPN.common_validation( self.middleware, { **config, 'client_certificate': client_certificate_id }, '', 'client'))[0]: raise CallError( 'Please ensure provided client certificate exists in Root CA chain ' 'and has necessary extensions set.') client_config = [ 'client', f'dev {config["device_type"].lower()}', f'proto {config["protocol"].lower()}', f'port {config["port"]}', f'remote "{server_address or "PLEASE FILL OUT SERVER DOMAIN/IP HERE"}"', 'user nobody', 'group nobody', 'persist-key', 'persist-tun', '<ca>', f'{root_ca["certificate"]}', '</ca>', '<cert>', client_cert['certificate'], '</cert>', '<key>', client_cert['privatekey'], '</key>', 'verb 3', 'remote-cert-tls server', f'compress {config["compression"].lower()}' if config['compression'] else None, f'auth {config["authentication_algorithm"]}' if config['authentication_algorithm'] else None, f'cipher {config["cipher"]}' if config['cipher'] else None, ] if config['tls_crypt_auth_enabled']: client_config.extend( ['<tls-crypt>', config['tls_crypt_auth'], '</tls-crypt>']) return '\n'.join(filter(bool, client_config)).strip() @accepts( Dict('openvpn_server_update', Bool('tls_crypt_auth_enabled'), Int('netmask', validators=[Range(min=0, max=128)]), Int('server_certificate', null=True), Int('port', validators=[Port()]), Int('root_ca', null=True), IPAddr('server'), Str('additional_parameters'), Str('authentication_algorithm', null=True), Str('cipher', null=True), Str('compression', null=True, enum=['LZO', 'LZ4']), Str('device_type', enum=['TUN', 'TAP']), Str('protocol', enum=PROTOCOLS), Str('tls_crypt_auth', null=True), Str('topology', null=True, enum=['NET30', 'P2P', 'SUBNET']), update=True)) async def do_update(self, data): """ Update OpenVPN Server configuration. When `tls_crypt_auth_enabled` is enabled and `tls_crypt_auth` not provided, a static key is automatically generated to be used with OpenVPN server. """ old_config = await self.config() old_config.pop('interface') config = old_config.copy() config.update(data) # If tls_crypt_auth_enabled is set and we don't have a tls_crypt_auth key, # let's generate one please if config['tls_crypt_auth_enabled'] and not config['tls_crypt_auth']: config['tls_crypt_auth'] = await self.generate_static_key() config = await self.validate(config, 'openvpn_server_update') await self._update_service(old_config, config) return await self.config()
class SMBService(SystemServiceService): class Config: service = 'cifs' service_verb = 'restart' datastore = 'services.cifs' datastore_extend = 'smb.smb_extend' datastore_prefix = 'cifs_srv_' @private async def smb_extend(self, smb): """Extend smb for netbios.""" if not await self.middleware.call('notifier.is_freenas' ) and await self.middleware.call( 'notifier.failover_node') == 'B': smb['netbiosname'] = smb['netbiosname_b'] for i in ('aio_enable', 'aio_rs', 'aio_ws'): smb.pop(i, None) smb['loglevel'] = LOGLEVEL_MAP.get(smb['loglevel']) return smb async def __validate_netbios_name(self, name): return RE_NETBIOSNAME.match(name) @accepts( Dict( 'smb_update', Str('netbiosname'), Str('netbiosname_b'), Str('netbiosalias'), Str('workgroup'), Str('description'), Str('doscharset', enum=[ 'CP437', 'CP850', 'CP852', 'CP866', 'CP932', 'CP949', 'CP950', 'CP1026', 'CP1251', 'ASCII', ]), Str('unixcharset', enum=[ 'UTF-8', 'ISO-8859-1', 'ISO-8859-15', 'GB2312', 'EUC-JP', 'ASCII', ]), Str('loglevel', enum=['NONE', 'MINIMUM', 'NORMAL', 'FULL', 'DEBUG']), Bool('syslog'), Bool('localmaster'), Bool('domain_logons'), Bool('timeserver'), Str('guest'), Str('filemask'), Str('dirmask'), Bool('nullpw'), Bool('unixext'), Bool('zeroconf'), Bool('hostlookup'), Bool('allow_execute_always'), Bool('obey_pam_restrictions'), Bool('ntlmv1_auth'), List('bindip', items=[IPAddr('ip')]), Str('smb_options'), update=True, )) async def do_update(self, data): old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() for i in ('workgroup', 'netbiosname', 'netbiosname_b', 'netbiosalias'): if i not in data or not data[i]: continue if not await self.__validate_netbios_name(data[i]): verrors.add(f'smb_update.{i}', 'Invalid NetBIOS name') if new['netbiosname'] and new['netbiosname'].lower( ) == new['workgroup'].lower(): verrors.add('smb_update.netbiosname', 'NetBIOS and Workgroup must be unique') for i in ('filemask', 'dirmask'): if i not in data or not data[i]: continue try: if int(data[i], 8) & ~0o11777: raise ValueError('Not an octet') except (ValueError, TypeError): verrors.add(f'smb_update.{i}', 'Not a valid mask') if verrors: raise verrors # TODO: consider using bidict for k, v in LOGLEVEL_MAP.items(): if new['loglevel'] == v: new['loglevel'] = k break await self._update_service(old, new) return await self.config()