def __rquery_to_start_end(self, query): unit = query.get('unit') if unit: verrors = ValidationErrors() for i in ('start', 'end'): if i in query: verrors.add( f'reporting_query.{i}', f'{i!r} should only be used if "unit" attribute is not provided.', ) verrors.check() else: if 'start' not in query: unit = 'HOURLY' else: starttime = query['start'] endtime = query.get('end') or 'now' if unit: unit = unit[0].lower() page = query['page'] starttime = f'end-{page + 1}{unit}' if not page: endtime = 'now' else: endtime = f'now-{page}{unit}' return starttime, endtime
def add_jwt_secret(self, data): """ Add a `secret` key used to encode/decode JWT messages for sending/receiving gluster events. `secret` String representing the key to be used to encode/decode JWT messages `force` Boolean if set to True, will forcefully wipe any existing jwt key for this peer. Note, if forcefully adding a new key, the other peers in the TSP will also need to be sent this key. Note: this secret is only used for messages that are destined for the api endpoint at http://*:6000/_clusterevents for each peer in the trusted storage pool. """ if not data['force'] and self.JWT_SECRET is not None: verrors = ValidationErrors() verrors.add( 'localevent_add_jwt_secret.{data["secret"]}', 'An existing secret key already exists. Use force to ignore this error' ) verrors.check() self.JWT_SECRET = data['secret'] with open(SECRETS_FILE, 'w+') as f: f.write(data['secret'])
def update_zfs_object_props(self, properties, zfs_object): verrors = ValidationErrors() for k, v in properties.items(): # If prop already exists we just update it, # otherwise create a user property prop = zfs_object.properties.get(k) if v.get('source') == 'INHERIT': if not prop: verrors.add( f'properties.{k}', 'Property does not exist and cannot be inherited') else: if not any(i in v for i in ('parsed', 'value')): verrors.add( f'properties.{k}', '"value" or "parsed" must be specified when setting a property' ) if not prop and ':' not in k: verrors.add(f'properties.{k}', 'User property needs a colon (:) in its name') verrors.check() try: zfs_object.update_properties(properties) except libzfs.ZFSException as e: raise CallError(f'Failed to update properties: {e!r}')
async def do_update(self, data): """ `appdefaults_aux` add parameters to "appdefaults" section of the krb5.conf file. `libdefaults_aux` add parameters to "libdefaults" section of the krb5.conf file. """ verrors = ValidationErrors() old = await self.config() new = old.copy() new.update(data) verrors.add_child( 'kerberos_settings_update', await self._validate_appdefaults(new['appdefaults_aux']) ) verrors.add_child( 'kerberos_settings_update', await self._validate_libdefaults(new['libdefaults_aux']) ) verrors.check() await super().do_update(data) await self.middleware.call('etc.generate', 'kerberos') return await self.config()
def do_create(self, job, data): """ Add `url` webhook that will be called with a POST request that will include the event that was triggered along with the relevant data. `url` is a http address (i.e. http://192.168.1.50/endpoint) `bearer_token` is a bearer token `secret` secret to add JWT bearer token """ verrors = ValidationErrors() # there doesn't seem to be an upper limit on the amount # of webhook endpoints that can be added to the daemon # so place an arbitrary limit of 5 for now if len(self.middleware.call_sync('gluster.eventsd.webhooks')['webhooks']) >= 5: verrors.add( f'webhook_create.{data["url"]}', 'Maximum number of webhooks has been met. ' 'Delete one or more and try again.' ) verrors.check() cmd = self.format_cmd(data) return self.run_cmd(cmd)
async def get_cred(self, data): ''' Get kerberos cred from directory services config to use for `do_kinit`. ''' conf = data.get('conf', {}) if conf.get('kerberos_principal'): return {'kerberos_principal': conf['kerberos_principal']} verrors = ValidationErrors() dstype = DSType[data['dstype']] if dstype is DSType.DS_TYPE_ACTIVEDIRECTORY: for k in ['bindname', 'bindpw', 'domainname']: if not conf.get(k): verrors.add(f'conf.{k}', 'Parameter is required.') verrors.check() return { 'username': f'{conf["bindname"]}@{conf["domainname"].upper()}', 'password': conf['bindpw'] } for k in ['binddn', 'bindpw', 'kerberos_realm']: if not conf.get(k): verrors.add(f'conf.{k}', 'Parameter is required.') verrors.check() krb_realm = await self.middleware.call( 'kerberos.realm.query', [('id', '=', conf['kerberos_realm'])], {'get': True}) bind_cn = (conf['binddn'].split(','))[0].split("=") return { 'username': f'{bind_cn[1]}@{krb_realm["realm"]}', 'password': conf['bindpw'] }
async def get_versions(self, release, options): current_chart = release['chart_metadata'] chart = current_chart['name'] item_details = await self.middleware.call('catalog.get_item_details', chart, { 'catalog': release['catalog'], 'train': release['catalog_train'], }) new_version = options['item_version'] if new_version == 'latest': new_version = await self.middleware.call( 'chart.release.get_latest_version_from_item_versions', item_details['versions'] ) if new_version not in item_details['versions']: raise CallError(f'Unable to locate specified {new_version!r} item version.') verrors = ValidationErrors() if parse_version(new_version) <= parse_version(current_chart['version']): verrors.add( 'upgrade_options.item_version', f'Upgrade version must be greater than {current_chart["version"]!r} current version.' ) verrors.check() return { 'specified_version': item_details['versions'][new_version], 'versions': item_details['versions'], 'latest_version': item_details['versions'][await self.middleware.call( 'chart.release.get_latest_version_from_item_versions', item_details['versions'] )] }
async def do_update(self, pk, data): """ Update attributes of an existing group. """ group = await self._get_instance(pk) add_groupmap = False verrors = ValidationErrors() await self.__common_validation(verrors, data, 'group_update', pk=pk) verrors.check() old_smb = group['smb'] group.update(data) group.pop('users', None) new_smb = group['smb'] if 'name' in data and data['name'] != group['group']: if g := (await self.middleware.call('smb.groupmap_list')).get(group['group']): await self.middleware.call( 'smb.groupmap_delete', {"sid": g['SID']} ) group['group'] = group.pop('name') if new_smb: add_groupmap = True
async def started(self): """ Returns False if disabled, True if healthy, raises exception if faulted. """ verrors = ValidationErrors() ldap = await self.config() if not ldap['enable']: return False await self.common_validate(ldap, ldap, verrors) try: verrors.check() except Exception: await self.middleware.call( 'datastore.update', 'directoryservice.ldap', ldap['id'], {'ldap_enable': False} ) raise CallError('Automatically disabling LDAP service due to invalid configuration.', errno.EINVAL) try: await asyncio.wait_for(self.middleware.call('ldap.get_root_DSE', ldap), timeout=ldap['timeout']) except asyncio.TimeoutError: raise CallError(f'LDAP status check timed out after {ldap["timeout"]} seconds.', errno.ETIMEDOUT) except Exception as e: raise CallError(e) return True
async def do_update(self, id, data): """ Update AFP share `id`. """ verrors = ValidationErrors() old = await self.middleware.call( 'datastore.query', self._config.datastore, [('id', '=', id)], {'extend': self._config.datastore_extend, 'prefix': self._config.datastore_prefix, 'get': True}) path = data.get('path') new = old.copy() new.update(data) await self.clean(new, 'sharingafp_update', verrors, id=id) await self.validate(new, 'sharingafp_update', verrors, old=old) verrors.check() if path and not os.path.exists(path): try: os.makedirs(path) except OSError as e: raise CallError(f'Failed to create {path}: {e}') await self.compress(new) await self.middleware.call( 'datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) await self._service_change('afp', 'reload') return await self.get_instance(id)
async def do_create(self, data): """ Create a new group. If `gid` is not provided it is automatically filled with the next one available. `allow_duplicate_gid` allows distinct group names to share the same gid. `users` is a list of user ids (`id` attribute from `user.query`). """ verrors = ValidationErrors() await self.__common_validation(verrors, data, 'group_create') verrors.check() if not data.get('gid'): data['gid'] = await self.get_next_gid() group = data.copy() group['group'] = group.pop('name') users = group.pop('users', []) pk = await self.middleware.call('datastore.insert', 'account.bsdgroups', group, {'prefix': 'bsdgrp_'}) for user in users: await self.middleware.call('datastore.insert', 'account.bsdgroupmembership', {'bsdgrpmember_group': pk, 'bsdgrpmember_user': user}) await self.middleware.call('service.reload', 'user') await self.middleware.call('smb.groupmap_add', data['name']) return pk
async def do_update(self, data): """ Update AFP service settings. `bindip` is a list of IPs to bind AFP to. Leave blank (empty list) to bind to all available IPs. `map_acls` defines how to map the effective permissions of authenticated users. RIGHTS - Unix-style permissions MODE - ACLs NONE - Do not map `chmod_request` defines advanced permission control that deals with ACLs. PRESERVE - Preserve ZFS ACEs for named users and groups or POSIX ACL group mask SIMPLE - Change permission as requested without any extra steps IGNORE - Permission change requests are ignored """ old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() if new['dbpath']: await check_path_resides_within_volume( verrors, self.middleware, 'afp_update.dbpath', new['dbpath'], ) verrors.check() new = await self.compress(new) await self._update_service(old, new) return await self.config()
def __rquery_to_start_end(self, query): unit = query.get('unit') if unit: verrors = ValidationErrors() for i in ('start', 'end'): if i in query: verrors.add( f'reporting_query.{i}', f'{i!r} should only be used if "unit" attribute is not provided.', ) verrors.check() else: if 'start' not in query: unit = 'HOURLY' else: starttime = query['start'] endtime = query.get('end') or 'now' if unit: unit = unit[0].lower() page = query['page'] starttime = f'end-{page + 1}{unit}' if not page: endtime = 'now' else: endtime = f'now-{page}{unit}' return starttime, endtime
async def do_create(self, data): """ Create a new group. If `gid` is not provided it is automatically filled with the next one available. `allow_duplicate_gid` allows distinct group names to share the same gid. `users` is a list of user ids (`id` attribute from `user.query`). """ verrors = ValidationErrors() await self.__common_validation(verrors, data, 'group_create') verrors.check() if not data.get('gid'): data['gid'] = await self.get_next_gid() group = data.copy() group['group'] = group.pop('name') users = group.pop('users', []) group = await self.group_compress(group) pk = await self.middleware.call('datastore.insert', 'account.bsdgroups', group, {'prefix': 'bsdgrp_'}) for user in users: await self.middleware.call('datastore.insert', 'account.bsdgroupmembership', {'bsdgrpmember_group': pk, 'bsdgrpmember_user': user}) await self.middleware.call('service.reload', 'user') await self.middleware.call('smb.groupmap_add', data['name']) return pk
async def validate_data(self, data, schema): verrors = ValidationErrors() if data['pool'] and not await self.middleware.call( 'pool.query', [['name', '=', data['pool']]]): verrors.add( f'{schema}.pool', 'Please provide a valid pool configured in the system.') if ipaddress.ip_address( data['cluster_dns_ip']) not in ipaddress.ip_network( data['service_cidr']): verrors.add(f'{schema}.cluster_dns_ip', 'Must be in range of "service_cidr".') if data['node_ip'] not in await self.bindip_choices(): verrors.add(f'{schema}.node_ip', 'Please provide a valid IP address.') for k, _ in await self.validate_interfaces(data): verrors.add(f'{schema}.{k}', 'Please specify a valid interface.') for k in ('route_v4', 'route_v6'): gateway = data[f'{k}_gateway'] interface = data[f'{k}_interface'] if (not gateway and not interface) or (gateway and interface): continue for k2 in ('gateway', 'interface'): verrors.add( f'{schema}.{k}_{k2}', f'{k}_gateway and {k}_interface must be specified together.' ) verrors.check()
async def resolve_hostnames(self, hostnames): """ Takes a list of hostnames to be asynchronously resolved to their respective IP address. If IP addresses are given, then it will simply return the IP address """ hostnames = list(set(hostnames)) loop = asyncio.get_event_loop() timeout = 5 verrors = ValidationErrors() results = await asyncio.gather(*[ self._resolve_hostname(loop, host, timeout) for host in hostnames ], return_exceptions=True) ips = [] for host, result in zip(hostnames, results): if isinstance(result, (type(None), asyncio.TimeoutError)): verrors.add(f'resolve_hostname.{host}', 'Failed to resolve hostname') else: ips.append(result) # if any hostnames failed to be resolved # it will be raised here verrors.check() return list(set(ips))
async def do_update(self, id, data): """ update filesystem ACL template with `id`. """ old = await self.get_instance(id) new = old.copy() new.update(data) verrors = ValidationErrors() if old['builtin']: verrors.add("filesystem_acltemplate_update.builtin", "built-in ACL templates may not be changed") if new['name'] != old['name']: name_exists = bool(await self.query([('name', '=', new['name'])])) if name_exists: verrors.add("filesystem_acltemplate_update.name", f"{data['name']}: name is not unique") if len(new['acl']) == 0: verrors.add("filesystem_acltemplate_update.acl", "At least one ACL entry must be specified.") await self.validate_acl(new, "filesystem_acltemplate_update.acl", verrors) verrors.check() await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) return await self.get_instance(id)
async def do_create(self, data): """ Create a new boot environment using `name`. If a new boot environment is desired which is a clone of another boot environment, `source` can be passed. Then, a new boot environment of `name` is created using boot environment `source` by cloning it. Ensure that `name` and `source` are valid boot environment names. """ verrors = ValidationErrors() await self._clean_be_name(verrors, 'bootenv_create', data['name']) verrors.check() args = [self.BE_TOOL, 'create'] source = data.get('source') if source: args += [ '-e', os.path.join(await self.middleware.call('boot.pool_name'), 'ROOT', source) if osc.IS_LINUX else source ] args.append(data['name']) try: await run(args, encoding='utf8', check=True) except subprocess.CalledProcessError as cpe: raise CallError(f'Failed to create boot environment: {cpe.stdout}') return data['name']
async def do_update(self, data): """ Update AFP service settings. `bindip` is a list of IPs to bind AFP to. Leave blank (empty list) to bind to all available IPs. `map_acls` defines how to map the effective permissions of authenticated users. RIGHTS - Unix-style permissions MODE - ACLs NONE - Do not map `chmod_request` defines advanced permission control that deals with ACLs. PRESERVE - Preserve ZFS ACEs for named users and groups or POSIX ACL group mask SIMPLE - Change permission as requested without any extra steps IGNORE - Permission change requests are ignored """ old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() if new['dbpath']: await check_path_resides_within_volume( verrors, self.middleware, 'afp_update.dbpath', new['dbpath'], ) verrors.check() new = await self.compress(new) await self._update_service(old, new) return await self.config()
async def do_create(self, data): """ Create AFP share. `allow`, `deny`, `ro`, and `rw` are lists of users and groups. Groups are designated by an @ prefix. `hostsallow` and `hostsdeny` are lists of hosts and/or networks. """ verrors = ValidationErrors() path = data['path'] await self.clean(data, 'sharingafp_create', verrors) await self.validate(data, 'sharingafp_create', verrors) await check_path_resides_within_volume( verrors, self.middleware, 'sharingafp_create.path', path) verrors.check() if path and not os.path.exists(path): try: os.makedirs(path) except OSError as e: raise CallError(f'Failed to create {path}: {e}') await self.compress(data) data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self.extend(data) await self._service_change('afp', 'reload') return data
async def validate_credentials(middleware, data): # We would like to validate the following bits: # 1) script exists and is executable # 2) user exists # 3) User can access the script in question verrors = ValidationErrors() try: await middleware.call('user.get_user_obj', {'username': data['user']}) except KeyError: verrors.add('user', f'Unable to locate {data["user"]!r} user') await check_path_resides_within_volume(verrors, middleware, 'script', data['script']) try: can_access = await middleware.call( 'filesystem.can_access_as_user', data['user'], data['script'], {'execute': True} ) except CallError as e: verrors.add('script', f'Unable to validate script: {e}') else: if not can_access: verrors.add('user', f'{data["user"]!r} user does not has permission to execute the script') verrors.check() return data
async def do_create(self, data): """ Create AFP share. `allow`, `deny`, `ro`, and `rw` are lists of users and groups. Groups are designated by an @ prefix. `hostsallow` and `hostsdeny` are lists of hosts and/or networks. """ verrors = ValidationErrors() path = data['path'] await self.clean(data, 'sharingafp_create', verrors) await self.validate(data, 'sharingafp_create', verrors) verrors.check() if path and not os.path.exists(path): try: os.makedirs(path) except OSError as e: raise CallError(f'Failed to create {path}: {e}') await self.compress(data) data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self._service_change('afp', 'reload') return await self.get_instance(data['id'])
def check_errors(self, func, *args, **kwargs): verrors = ValidationErrors() try: func(*args, **kwargs) except CatalogValidationErrors as e: verrors.extend(e) verrors.check()
async def do_update(self, data): """ Update SMB Service Configuration. `netbiosname` defaults to the original hostname of the system. `workgroup` and `netbiosname` should have different values. `enable_smb1` allows legacy SMB clients to connect to the server when enabled. `localmaster` when set, determines if the system participates in a browser election. `domain_logons` is used to provide netlogin service for older Windows clients if enabled. `guest` attribute is specified to select the account to be used for guest access. It defaults to "nobody". `nullpw` when enabled allows the users to authorize access without a password. `hostlookup` when enabled, allows using hostnames rather then IP addresses in "hostsallow"/"hostsdeny" fields of SMB Shares. """ old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() ad_enabled = (await self.middleware.call('activedirectory.get_state') != "DISABLED") if ad_enabled: for i in ('workgroup', 'netbiosname', 'netbiosname_b', 'netbiosalias'): if old[i] != new[i]: verrors.add( f'smb_update.{i}', 'This parameter may not be changed after joining Active Directory (AD). ' 'If it must be changed, the proper procedure is to leave the AD domain ' 'and then alter the parameter before re-joining the domain.' ) await self.validate_smb(new, verrors) verrors.check() if new['admin_group'] and new['admin_group'] != old['admin_group']: await self.middleware.call('smb.add_admin_group', new['admin_group']) # TODO: consider using bidict for k, v in LOGLEVEL_MAP.items(): if new['loglevel'] == v: new['loglevel'] = k break await self.compress(new) await self._update_service(old, new) await self.reset_smb_ha_mode() return await self.config()
async def do_update(self, job, data): """ Update System Dataset Service Configuration. `pool` is the name of a valid pool configured in the system which will be used to host the system dataset. `pool_exclude` can be specified to make sure that we don't place the system dataset on that pool if `pool` is not provided. """ config = await self.config() new = config.copy() new.update(data) verrors = ValidationErrors() if new['pool'] and new['pool'] != 'freenas-boot': pool = await self.middleware.call('pool.query', [['name', '=', new['pool']]]) if not pool: verrors.add('sysdataset_update.pool', f'Pool "{new["pool"]}" not found', errno.ENOENT) elif pool[0]['encrypt'] == 2: # This will cover two cases - passphrase being set for a pool and that it might be locked as well verrors.add( 'sysdataset_update.pool', f'Pool "{new["pool"]}" has an encryption passphrase set. ' 'The system dataset cannot be placed on this pool.') elif not new['pool']: for pool in await self.middleware.call('pool.query', [['encrypt', '!=', 2]]): if data.get('pool_exclude') == pool['name']: continue new['pool'] = pool['name'] break else: new['pool'] = 'freenas-boot' verrors.check() new['syslog_usedataset'] = new['syslog'] update_dict = new.copy() for key in ('is_decrypted', 'basename', 'uuid_a', 'syslog', 'path', 'pool_exclude'): update_dict.pop(key, None) await self.middleware.call('datastore.update', 'system.systemdataset', config['id'], update_dict, {'prefix': 'sys_'}) if config['pool'] != new['pool']: await self.migrate(config['pool'], new['pool']) await self.setup() if config['syslog'] != new['syslog']: await self.middleware.call('service.restart', 'syslogd') return await self.config()
def common_validation(self, hostname=None): verrors = ValidationErrors() if hostname: self.middleware.call_sync('gluster.peer.resolve_host_or_ip', hostname, verrors) verrors.check()
async def manual_test(self, disks): """ Run manual SMART tests for `disks`. `type` indicates what type of SMART test will be ran and must be specified. """ verrors = ValidationErrors() if not disks: verrors.add( 'disks', 'Please specify at least one disk.' ) else: test_disks_list = [] disks_data = await self.middleware.call('disk.query') devices = await self.middleware.call('device.get_storage_devices_topology') for index, disk in enumerate(disks): for d in disks_data: if disk['identifier'] == d['identifier']: current_disk = d test_disks_list.append({ 'disk': current_disk['name'], **disk }) break else: verrors.add( f'disks.{index}.identifier', f'{disk["identifier"]} is not valid. Please provide a valid disk identifier.' ) continue if current_disk['name'] is None: verrors.add( f'disks.{index}.identifier', f'Test cannot be performed for {disk["identifier"]} disk. Failed to retrieve name.' ) if current_disk['name'].startswith('nvd'): verrors.add( f'disks.{index}.identifier', f'Test cannot be performed for {disk["identifier"]} disk. NVMe devices cannot be mapped yet.' ) device = devices.get(current_disk['name']) if not device: verrors.add( f'disks.{index}.identifier', f'Test cannot be performed for {disk["identifier"]}. Unable to retrieve disk details.' ) verrors.check() return list( await asyncio_map(functools.partial(self.__manual_test, devices), test_disks_list, 16) )
async def cert_services_validation(self, id, schema_name, raise_verrors=True): # General method to check certificate health wrt usage in services cert = await self.middleware.call('certificate.query', [['id', '=', id]]) verrors = ValidationErrors() valid_key_size = {'EC': 28, 'RSA': 2048} if cert: cert = cert[0] if cert['cert_type'] != 'CERTIFICATE' or cert['cert_type_CSR']: verrors.add( schema_name, 'Selected certificate id is not a valid certificate') elif not cert.get('fingerprint'): verrors.add(schema_name, f'{cert["name"]} certificate is malformed') if not cert['privatekey']: verrors.add( schema_name, 'Selected certificate does not have a private key') elif not cert['key_length']: verrors.add(schema_name, 'Failed to parse certificate\'s private key') elif cert['key_length'] < valid_key_size[cert['key_type']]: verrors.add( schema_name, f'{cert["name"]}\'s private key size is less then {valid_key_size[cert["key_type"]]} bits' ) if cert['until'] and datetime.datetime.strptime( cert['until'], '%a %b %d %H:%M:%S %Y') < datetime.datetime.now(): verrors.add( schema_name, f'{cert["name"]!r} has expired (it was valid until {cert["until"]!r})' ) if cert['digest_algorithm'] in ['MD5', 'SHA1']: verrors.add( schema_name, 'Please use a certificate whose digest algorithm has at least 112 security bits' ) if cert['revoked']: verrors.add(schema_name, 'This certificate is revoked') else: verrors.add(schema_name, f'No Certificate found with the provided id: {id}') if raise_verrors: verrors.check() else: return verrors
async def do_update(self, pk, data): """ Update attributes of an existing group. """ group = await self._get_instance(pk) verrors = ValidationErrors() await self.__common_validation(verrors, data, 'group_update', pk=pk) verrors.check() group.update(data) delete_groupmap = False group.pop('users', None) if 'name' in data and data['name'] != group['group']: delete_groupmap = group['group'] group['group'] = group.pop('name') else: group.pop('name', None) group = await self.group_compress(group) await self.middleware.call('datastore.update', 'account.bsdgroups', pk, group, {'prefix': 'bsdgrp_'}) if 'users' in data: existing = { i['bsdgrpmember_user']['id']: i for i in await self.middleware.call( 'datastore.query', 'account.bsdgroupmembership', [( 'bsdgrpmember_group', '=', pk)]) } to_remove = set(existing.keys()) - set(data['users']) for i in to_remove: await self.middleware.call('datastore.delete', 'account.bsdgroupmembership', existing[i]['id']) to_add = set(data['users']) - set(existing.keys()) for i in to_add: await self.middleware.call('datastore.insert', 'account.bsdgroupmembership', { 'bsdgrpmember_group': pk, 'bsdgrpmember_user': i }) if delete_groupmap: await self.middleware.call('notifier.groupmap_delete', delete_groupmap) await self.middleware.call('service.reload', 'user') await self.middleware.call('smb.groupmap_add', group['group']) return pk
async def common_validation(self, data, schema_name): verrors = ValidationErrors() create_request = schema_name == 'glustervolume_create' if data['name'] == CTDB_VOL_NAME and create_request: verrors.add( f'{schema_name}.{data["name"]}', f'"{data["name"]}" is a reserved name. Choose a different volume name.' ) verrors.check()
async def by_path(self, data): """ Retrieve list of available ACL templates for a given `path`. Supports `query-filters` and `query-options`. `format-options` gives additional options to alter the results of the template query: `canonicalize` - place ACL entries for NFSv4 ACLs in Microsoft canonical order. `ensure_builtins` - ensure all results contain entries for `builtin_users` and `builtin_administrators` groups. `resolve_names` - convert ids in ACL entries into names. """ verrors = ValidationErrors() filters = data.get('query-filters') if data['path']: path = await self.middleware.call( "filesystem.resolve_cluster_path", data['path']) acltype = await self.middleware.call('filesystem.path_get_acltype', path) if acltype == ACLType.DISABLED.name: return [] if acltype == ACLType.POSIX1E.name and data['format-options'][ 'canonicalize']: verrors.add( "filesystem.acltemplate_by_path.format-options.canonicalize", "POSIX1E ACLs may not be sorted into Windows canonical order." ) filters.append(("acltype", "=", acltype)) if not data['path'] and data['format-options']['resolve_names']: verrors.add( "filesystem.acltemplate_by_path.format-options.canonicalize", "ACL entry ids may not be resolved into names unless path is provided." ) verrors.check() templates = await self.query(filters, data['query-options']) for t in templates: if data['format-options']['ensure_builtins']: await self.append_builtins(t) if data['format-options']['resolve_names']: st = await self.middleware.run_in_thread(os.stat(path)) await self.resolve_names(st.st_uid, st.st_gid, t) if data['format-options']['canonicalize'] and t[ 'acltype'] == ACLType.NFS4.name: canonicalized = ACLType[t['acltype']].canonicalize(t['acl']) t['acl'] = canonicalized return templates
async def do_create(self, data): """ Create a new group. If `gid` is not provided it is automatically filled with the next one available. `allow_duplicate_gid` allows distinct group names to share the same gid. `users` is a list of user ids (`id` attribute from `user.query`). `smb` specifies whether the group should be mapped into an NT group. """ allow_duplicate_gid = data['allow_duplicate_gid'] verrors = ValidationErrors() await self.__common_validation(verrors, data, 'group_create') verrors.check() if not data.get('gid'): data['gid'] = await self.get_next_gid() group = data.copy() group['group'] = group.pop('name') users = group.pop('users', []) group = await self.group_compress(group) pk = await self.middleware.call('datastore.insert', 'account.bsdgroups', group, {'prefix': 'bsdgrp_'}) for user in users: await self.middleware.call('datastore.insert', 'account.bsdgroupmembership', { 'bsdgrpmember_group': pk, 'bsdgrpmember_user': user }) await self.middleware.call('service.reload', 'user') if data['smb']: try: await self.middleware.call('smb.groupmap_add', data['name']) except Exception: """ Samba's group mapping database does not allow duplicate gids. Unfortunately, we don't get a useful error message at -d 0. """ if not allow_duplicate_gid: raise else: self.logger.debug( 'Refusing to generate duplicate gid mapping in group_mapping.tdb: %s -> %s', data['name'], data['gid']) return pk
async def common_validation(self, data, schema_name): name = data.get('name', None) all_vols = data.get('all', None) verrors = ValidationErrors() if not name and not all_vols: verrors.add( f'{schema_name}', 'A gluster volume name is rquired or the "all" key must be set to True' ) verrors.check()
async def do_update(self, id, data): """ Update a domain by id. """ old = await self._get_instance(id) new = old.copy() new.update(data) tmp = data.copy() verrors = ValidationErrors() if old['name'] in [x.name for x in DSType] and old['name'] != new['name']: verrors.add( 'idmap_domain_update.name', f'Changing name of default domain {old["name"]} is not permitted' ) if new['options'].get( 'sssd_compat') and not old['options'].get('sssd_compat'): if await self.middleware.call('activedirectory.get_state' ) != 'HEALTHY': verrors.add( 'idmap_domain_update.options', 'AD service must be enabled and started to ' 'generate an SSSD-compatible id range') verrors.check() new['range_low'] = await self.get_sssd_low_range(new['name']) new['range_high'] = new['range_low'] + 100000000 await self.validate('idmap_domain_update', new, verrors) await self.validate_options('idmap_domain_update', new, verrors, ['MISSING']) tmp['idmap_backend'] = new['idmap_backend'] if data.get('options'): await self.validate_options('idmap_domain_update', tmp, verrors, ['EXTRA']) if data.get('certificate_id') and not data['options'].get('ssl'): verrors.add( 'idmap_domain_update.certificate_id', f'The {new["idmap_backend"]} idmap backend does not ' 'generate LDAP traffic. Certificates do not apply.') verrors.check() await self.prune_keys(new) final_options = IdmapBackend[data['idmap_backend']].defaults() final_options.update(new['options']) new['options'] = final_options await self.idmap_compress(new) await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) await self.middleware.call('idmap.clear_idmap_cache') return await self._get_instance(id)
async def do_update(self, pk, data): """ Update attributes of an existing group. """ group = await self._get_instance(pk) verrors = ValidationErrors() await self.__common_validation(verrors, data, 'group_update', pk=pk) verrors.check() group.update(data) delete_groupmap = False group.pop('users', None) if 'name' in data and data['name'] != group['group']: delete_groupmap = group['group'] group['group'] = group.pop('name') else: group.pop('name', None) await self.middleware.call('datastore.update', 'account.bsdgroups', pk, group, {'prefix': 'bsdgrp_'}) if 'users' in data: existing = {i['bsdgrpmember_user']['id']: i for i in await self.middleware.call('datastore.query', 'account.bsdgroupmembership', [('bsdgrpmember_group', '=', pk)])} to_remove = set(existing.keys()) - set(data['users']) for i in to_remove: await self.middleware.call('datastore.delete', 'account.bsdgroupmembership', existing[i]['id']) to_add = set(data['users']) - set(existing.keys()) for i in to_add: await self.middleware.call('datastore.insert', 'account.bsdgroupmembership', {'bsdgrpmember_group': pk, 'bsdgrpmember_user': i}) if delete_groupmap: await self.middleware.call('notifier.groupmap_delete', delete_groupmap) await self.middleware.call('service.reload', 'user') await self.middleware.call('smb.groupmap_add', group['group']) return pk
def validate_data_keys(self, data, model, schema, prefix): verrors = ValidationErrors() fields = list( map( lambda f: f.name.replace(prefix or '', '', 1), chain(model._meta.fields, model._meta.many_to_many) ) ) # _id is a special condition in filter where the key in question can be a related descriptor in django # i.e share_id - so we remove _id and check if the field is present in `fields` list for key in filter( lambda v: all(c not in fields for c in (v, v if not v.endswith('_id') else v[:-3])), data ): verrors.add( f'{schema}.{key}', f'{key} field not recognized' ) verrors.check()
async def common_validation(self, data, schema_name): verrors = ValidationErrors() await check_path_resides_within_volume(verrors, self.middleware, f'{schema_name}.path', data.get('path')) for entity in ('user', 'group'): value = data.get(entity) if value not in map( lambda e: e[entity if entity == 'group' else 'username'], await self.middleware.call(f'{entity}.query') ): verrors.add( f'{schema_name}.{entity}', f'Please specify a valid {entity}' ) verrors.check() data['hostsallow'] = ' '.join(data['hostsallow']) data['hostsdeny'] = ' '.join(data['hostsdeny']) data['mode'] = data['mode'].lower() return data
async def validate_attrs(self, data): verrors = ValidationErrors() additional_params = data.get('additional_params') if additional_params: # Let's be very generic here and introduce very basic validation # Expected format is as following # [ipv6.icmpneighbor] # history = 86400 # enabled = yes # # While we are here, we will also introduce basic formatting to the file to ensure # that we can make it as compliable as possible param_str = '' for i in additional_params.split('\n'): i = i.strip() if not i: continue if i.startswith('#'): # Let's not validate this if i.replace('#', '').startswith('['): param_str += f'\n\n{i}' else: param_str += f'\n\t{i}' continue if i.startswith('[') and not i.endswith(']'): verrors.add( 'netdata_update.additional_params', f'Please correct format for {i}. i.e [system.intr]' ) elif not i.startswith('[') and '=' not in i: verrors.add( 'netdata_update.additional_params', f'Please correct format for {i}. i.e enabled = yes' ) if i.startswith('['): param_str += f'\n\n{i}' else: param_str += f'\n\t{i}' data['additional_params'] = param_str + '\n' bind_to_ips = data.get('bind') if bind_to_ips: valid_ips = [ip['address'] for ip in await self.middleware.call('interface.ip_in_use')] valid_ips.extend(['127.0.0.1', '::1', '0.0.0.0', '::']) for bind_ip in bind_to_ips: if bind_ip not in valid_ips: verrors.add( 'netdata_update.bind', f'Invalid {bind_ip} bind IP' ) else: verrors.add( 'netdata_update.bind', 'This field is required' ) update_alarms = data.pop('update_alarms', {}) valid_alarms = self._alarms if update_alarms: for alarm in update_alarms: if alarm not in valid_alarms: verrors.add( 'netdata_update.alarms', f'{alarm} not a valid alarm' ) verrors.extend( validate_attributes( [Dict(key, Bool('enabled', required=True)) for key in update_alarms], {'attributes': update_alarms} ) ) # Validating streaming metrics now stream_mode = data.get('stream_mode') if stream_mode == 'SLAVE': for key in ('api_key', 'destination'): if not data.get(key): verrors.add( f'netdata_update.{key}', f'{key} is required with stream mode as SLAVE' ) destinations = data.get('destination') if destinations: ip_addr = IpAddress() port = Port() for dest in destinations: ip = dest.split(':')[0] try: ip_addr(ip) except ValueError as e: verrors.add( 'netdata_update.destination', str(e) ) else: if ':' in dest: try: port(int(dest.split(':')[1])) except ValueError as e: verrors.add( 'netdata_update.destination', f'Not a valid port: {e}' ) elif stream_mode == 'MASTER': for key in ('allow_from', 'api_key'): if not data.get(key): verrors.add( f'netdata_update.{key}', f'{key} is required with stream mode as MASTER' ) verrors.check() data['alarms'].update(update_alarms) return data
async def do_update(self, job, data): """ Update System Dataset Service Configuration. `pool` is the name of a valid pool configured in the system which will be used to host the system dataset. `pool_exclude` can be specified to make sure that we don't place the system dataset on that pool if `pool` is not provided. """ config = await self.config() new = config.copy() new.update(data) verrors = ValidationErrors() if new['pool'] and new['pool'] != 'freenas-boot': pool = await self.middleware.call('pool.query', [['name', '=', new['pool']]]) if not pool: verrors.add( 'sysdataset_update.pool', f'Pool "{new["pool"]}" not found', errno.ENOENT ) elif pool[0]['encrypt'] == 2: # This will cover two cases - passphrase being set for a pool and that it might be locked as well verrors.add( 'sysdataset_update.pool', f'Pool "{new["pool"]}" has an encryption passphrase set. ' 'The system dataset cannot be placed on this pool.' ) elif not new['pool']: for pool in await self.middleware.call( 'pool.query', [ ['encrypt', '!=', 2] ] ): if data.get('pool_exclude') == pool['name']: continue new['pool'] = pool['name'] break else: # If a data pool could not be found, reset it to blank # Which will eventually mean its back to freenas-boot (temporarily) new['pool'] = '' verrors.check() new['syslog_usedataset'] = new['syslog'] update_dict = new.copy() for key in ('is_decrypted', 'basename', 'uuid_a', 'syslog', 'path', 'pool_exclude'): update_dict.pop(key, None) await self.middleware.call( 'datastore.update', 'system.systemdataset', config['id'], update_dict, {'prefix': 'sys_'} ) new = await self.config() if config['pool'] != new['pool']: await self.migrate(config['pool'], new['pool']) await self.setup(True, data.get('pool_exclude')) if config['syslog'] != new['syslog']: await self.middleware.call('service.restart', 'syslogd') return await self.config()
async def do_create(self, data): """ Create a new user. If `uid` is not provided it is automatically filled with the next one available. `group` is required if `group_create` is false. `password` is required if `password_disabled` is false. Available choices for `shell` can be retrieved with `user.shell_choices`. `attributes` is a general-purpose object for storing arbitrary user information. """ verrors = ValidationErrors() if ( not data.get('group') and not data.get('group_create') ) or ( data.get('group') is not None and data.get('group_create') ): verrors.add( 'user_create.group', f'Enter either a group name or create a new group to ' 'continue.', errno.EINVAL ) await self.__common_validation(verrors, data, 'user_create') if data.get('sshpubkey') and not data['home'].startswith('/mnt'): verrors.add( 'user_create.sshpubkey', 'The home directory is not writable. Leave this field blank.' ) verrors.check() groups = data.pop('groups') create = data.pop('group_create') if create: group = await self.middleware.call('group.query', [('group', '=', data['username'])]) if group: group = group[0] else: group = await self.middleware.call('group.create', {'name': data['username']}) group = (await self.middleware.call('group.query', [('id', '=', group)]))[0] data['group'] = group['id'] else: group = await self.middleware.call('group.query', [('id', '=', data['group'])]) if not group: raise CallError(f'Group {data["group"]} not found') group = group[0] # Is this a new directory or not? Let's not nuke existing directories, # e.g. /, /root, /mnt/tank/my-dataset, etc ;). new_homedir = False home_mode = data.pop('home_mode') if data['home'] and data['home'] != '/nonexistent': try: try: os.makedirs(data['home'], mode=int(home_mode, 8)) new_homedir = True os.chown(data['home'], data['uid'], group['gid']) except FileExistsError: if not os.path.isdir(data['home']): raise CallError( 'Path for home directory already ' 'exists and is not a directory', errno.EEXIST ) # If it exists, ensure the user is owner os.chown(data['home'], data['uid'], group['gid']) except OSError as oe: raise CallError( 'Failed to create the home directory ' f'({data["home"]}) for user: {oe}' ) if os.stat(data['home']).st_dev == os.stat('/mnt').st_dev: raise CallError( f'The path for the home directory "({data["home"]})" ' 'must include a volume or dataset.' ) except Exception: if new_homedir: shutil.rmtree(data['home']) raise if not data.get('uid'): data['uid'] = await self.get_next_uid() pk = None # Make sure pk exists to rollback in case of an error try: await self.__set_password(data) sshpubkey = data.pop('sshpubkey', None) # datastore does not have sshpubkey pk = await self.middleware.call('datastore.insert', 'account.bsdusers', data, {'prefix': 'bsdusr_'}) await self.__set_groups(pk, groups) except Exception: if pk is not None: await self.middleware.call('datastore.delete', 'account.bsdusers', pk) if new_homedir: # Be as atomic as possible when creating the user if # commands failed to execute cleanly. shutil.rmtree(data['home']) raise await self.middleware.call('service.reload', 'user') await self.__set_smbpasswd(data['username']) if os.path.exists(data['home']): for f in os.listdir(SKEL_PATH): if f.startswith('dot'): dest_file = os.path.join(data['home'], f[3:]) else: dest_file = os.path.join(data['home'], f) if not os.path.exists(dest_file): shutil.copyfile(os.path.join(SKEL_PATH, f), dest_file) os.chown(dest_file, data['uid'], group['gid']) data['sshpubkey'] = sshpubkey try: await self.__update_sshpubkey(data['home'], data, group['group']) except PermissionError as e: self.logger.warn('Failed to update authorized keys', exc_info=True) raise CallError(f'Failed to update authorized keys: {e}') return pk
async def do_update(self, pk, data): """ Update attributes of an existing user. """ user = await self._get_instance(pk) verrors = ValidationErrors() if 'group' in data: group = await self.middleware.call('datastore.query', 'account.bsdgroups', [ ('id', '=', data['group']) ]) if not group: verrors.add('user_update.group', f'Group {data["group"]} not found', errno.ENOENT) group = group[0] else: group = user['group'] user['group'] = group['id'] await self.__common_validation(verrors, data, 'user_update', pk=pk) home = data.get('home') or user['home'] # root user (uid 0) is an exception to the rule if data.get('sshpubkey') and not home.startswith('/mnt') and user['uid'] != 0: verrors.add('user_update.sshpubkey', 'Home directory is not writable, leave this blank"') # Do not allow attributes to be changed for builtin user if user['builtin']: for i in ('group', 'home', 'home_mode', 'uid', 'username'): if i in data: verrors.add(f'user_update.{i}', 'This attribute cannot be changed') verrors.check() # Copy the home directory if it changed if ( 'home' in data and data['home'] not in (user['home'], '/nonexistent') and not data['home'].startswith(f'{user["home"]}/') ): home_copy = True home_old = user['home'] else: home_copy = False # After this point user dict has values from data user.update(data) if home_copy and not os.path.isdir(user['home']): try: os.makedirs(user['home']) os.chown(user['home'], user['uid'], group['bsdgrp_gid']) except OSError: self.logger.warn('Failed to chown homedir', exc_info=True) if not os.path.isdir(user['home']): raise CallError(f'{user["home"]} is not a directory') home_mode = user.pop('home_mode', None) if user['builtin']: home_mode = None def set_home_mode(): if home_mode is not None: try: os.chmod(user['home'], int(home_mode, 8)) except OSError: self.logger.warn('Failed to set homedir mode', exc_info=True) try: await self.__update_sshpubkey( home_old if home_copy else user['home'], user, group['bsdgrp_group'], ) except PermissionError as e: self.logger.warn('Failed to update authorized keys', exc_info=True) raise CallError(f'Failed to update authorized keys: {e}') if home_copy: def do_home_copy(): try: subprocess.run(f"/usr/bin/su - {user['username']} -c '/bin/cp -a {home_old}/ {user['home']}/'", shell=True, check=True) except subprocess.CalledProcessError as e: self.logger.warn(f"Failed to copy homedir: {e}") set_home_mode() asyncio.ensure_future(self.middleware.run_in_thread(do_home_copy)) else: set_home_mode() user.pop('sshpubkey', None) await self.__set_password(user) if 'groups' in user: groups = user.pop('groups') await self.__set_groups(pk, groups) await self.middleware.call('datastore.update', 'account.bsdusers', pk, user, {'prefix': 'bsdusr_'}) await self.middleware.call('service.reload', 'user') await self.__set_smbpasswd(user['username']) return pk