class DiskService(CRUDService): @filterable async def query(self, filters=None, options=None): if filters is None: filters = [] if options is None: options = {} options['prefix'] = 'disk_' filters.append(('expiretime', '=', None)) options['extend'] = 'disk.disk_extend' return await self.middleware.call('datastore.query', 'storage.disk', filters, options) @private def disk_extend(self, disk): disk.pop('enabled', None) return disk @private def get_name(self, disk): if disk["multipath_name"]: return f"multipath/{disk['multipath_name']}" else: return disk["name"] @accepts(Bool("join_partitions")) async def get_unused(self, join_partitions=False): """ Helper method to get all disks that are not in use, either by the boot pool or the user pools. """ disks = await self.query([('name', 'nin', await self.get_reserved())]) if join_partitions: for disk in disks: disk["partitions"] = await self.__get_partitions(disk) return disks @accepts(Dict( 'options', Bool('unused', default=False), )) def get_encrypted(self, options): """ Get all geli providers It might be an entire disk or a partition of type freebsd-zfs """ providers = [] disks_blacklist = [] if options['unused']: disks_blacklist += self.middleware.call_sync('disk.get_reserved') geom.scan() klass_part = geom.class_by_name('PART') klass_label = geom.class_by_name('LABEL') if not klass_part: return providers for g in klass_part.geoms: for p in g.providers: if p.config['type'] != 'freebsd-zfs': continue disk = p.geom.consumer.provider.name if disk in disks_blacklist: continue try: subprocess.run( ['geli', 'dump', p.name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True, ) except subprocess.CalledProcessError: continue dev = None if klass_label: for g in klass_label.geoms: if g.name == p.name: dev = g.provider.name break if dev is None: dev = p.name providers.append({'name': p.name, 'dev': dev, 'disk': disk}) return providers @accepts( List('devices', items=[Str('device')]), Str('passphrase', private=True), ) @job(pipes=['input']) def decrypt(self, job, devices, passphrase=None): """ Decrypt `devices` using uploaded encryption key """ with tempfile.NamedTemporaryFile(dir='/tmp/') as f: f.write(job.pipes.input.r.read()) f.flush() if passphrase: passf = tempfile.NamedTemporaryFile(mode='w+', dir='/tmp/') os.chmod(passf.name, 0o600) passf.write(passphrase) passf.flush() passphrase = passf.name failed = [] for dev in devices: try: self.middleware.call_sync( 'notifier.geli_attach_single', dev, f.name, passphrase, ) except Exception: failed.append(dev) if passphrase: passf.close() if failed: raise CallError( f'The following devices failed to attach: {", ".join(failed)}' ) return True @private async def get_reserved(self): reserved = [ i async for i in await self.middleware.call('boot.get_disks') ] reserved += [ i async for i in await self.middleware.call('pool.get_disks') ] reserved += [i async for i in self.__get_iscsi_targets()] return reserved async def __get_iscsi_targets(self): iscsi_target_extent_paths = [ extent["iscsi_target_extent_path"] for extent in await self.middleware.call( 'datastore.query', 'services.iscsitargetextent', [( 'iscsi_target_extent_type', '=', 'Disk')]) ] for disk in await self.middleware.call( 'datastore.query', 'storage.disk', [('disk_identifier', 'in', iscsi_target_extent_paths)]): yield disk["disk_name"] async def __get_partitions(self, disk): partitions = [] name = await self.middleware.call("disk.get_name", disk) for path in glob.glob(f"/dev/%s[a-fps]*" % name) or [f"/dev/{name}"]: info = (await run("/usr/sbin/diskinfo", path)).stdout.decode("utf-8").split("\t") if len(info) > 3: partitions.append({ "path": path, "capacity": int(info[2]), }) return partitions async def __camcontrol_list(self): """ Parse camcontrol devlist -v output to gather controller id, channel no and driver from a device Returns: dict(devname) = dict(drv, controller, channel) """ """ Hacky workaround It is known that at least some HPT controller have a bug in the camcontrol devlist output with multiple controllers, all controllers will be presented with the same driver with index 0 e.g. two hpt27xx0 instead of hpt27xx0 and hpt27xx1 What we do here is increase the controller id by its order of appearance in the camcontrol output """ hptctlr = defaultdict(int) re_drv_cid = re.compile(r'.* on (?P<drv>.*?)(?P<cid>[0-9]+) bus', re.S | re.M) re_tgt = re.compile( r'target (?P<tgt>[0-9]+) .*?lun (?P<lun>[0-9]+) .*\((?P<dv1>[a-z]+[0-9]+),(?P<dv2>[a-z]+[0-9]+)\)', re.S | re.M) drv, cid, tgt, lun, dev, devtmp = (None, ) * 6 camcontrol = {} proc = await Popen(['camcontrol', 'devlist', '-v'], stdout=subprocess.PIPE) for line in (await proc.communicate())[0].splitlines(): line = line.decode() if not line.startswith('<'): reg = re_drv_cid.search(line) if not reg: continue drv = reg.group('drv') if drv.startswith('hpt'): cid = hptctlr[drv] hptctlr[drv] += 1 else: cid = reg.group('cid') else: reg = re_tgt.search(line) if not reg: continue tgt = reg.group('tgt') lun = reg.group('lun') dev = reg.group('dv1') devtmp = reg.group('dv2') if dev.startswith('pass'): dev = devtmp camcontrol[dev] = { 'drv': drv, 'controller': int(cid), 'channel': int(tgt), 'lun': int(lun) } return camcontrol async def __get_twcli(self, controller): re_port = re.compile(r'^p(?P<port>\d+).*?\bu(?P<unit>\d+)\b', re.S | re.M) proc = await Popen( ['/usr/local/sbin/tw_cli', f'/c{controller}', 'show'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output = (await proc.communicate())[0].decode() units = {} for port, unit in re_port.findall(output): units[int(unit)] = int(port) return units async def __get_smartctl_args(self, devname): args = [f'/dev/{devname}'] camcontrol = await self.__camcontrol_list() info = camcontrol.get(devname) if info is not None: if info.get('drv') == 'rr274x_3x': channel = info['channel'] + 1 if channel > 16: channel -= 16 elif channel > 8: channel -= 8 args = [ '/dev/%s' % info['drv'], '-d', 'hpt,%d/%d' % (info['controller'] + 1, channel) ] elif info.get('drv').startswith('arcmsr'): args = [ '/dev/%s%d' % (info['drv'], info['controller']), '-d', 'areca,%d' % (info['lun'] + 1 + (info['channel'] * 8), ) ] elif info.get('drv').startswith('hpt'): args = [ '/dev/%s' % info['drv'], '-d', 'hpt,%d/%d' % (info['controller'] + 1, info['channel'] + 1) ] elif info.get('drv') == 'ciss': args = [ '/dev/%s%d' % (info['drv'], info['controller']), '-d', 'cciss,%d' % (info['channel'], ) ] elif info.get('drv') == 'twa': twcli = await self.__get_twcli(info['controller']) args = [ '/dev/%s%d' % (info['drv'], info['controller']), '-d', '3ware,%d' % (twcli.get(info['channel'], -1), ) ] return args @private async def toggle_smart_off(self, devname): args = await self.__get_smartctl_args(devname) await run('/usr/local/sbin/smartctl', '--smart=off', *args, check=False) @private async def toggle_smart_on(self, devname): args = await self.__get_smartctl_args(devname) await run('/usr/local/sbin/smartctl', '--smart=on', *args, check=False) @private async def serial_from_device(self, name): args = await self.__get_smartctl_args(name) p1 = await Popen(['smartctl', '-i'] + args, stdout=subprocess.PIPE) output = (await p1.communicate())[0].decode() search = re.search(r'Serial Number:\s+(?P<serial>.+)', output, re.I) if search: return search.group('serial') return None @private @accepts(Str('name')) async def device_to_identifier(self, name): """ Given a device `name` (e.g. da0) returns an unique identifier string for this device. This identifier is in the form of {type}string, "type" can be one of the following: - serial_lunid - for disk serial concatenated with the lunid - serial - disk serial - uuid - uuid of a ZFS GPT partition - label - label name from geom label - devicename - name of the device if any other could not be used/found Returns: str - identifier """ await self.middleware.run_in_thread(geom.scan) g = geom.geom_by_name('DISK', name) if g and g.provider.config.get('ident'): serial = g.provider.config['ident'] lunid = g.provider.config.get('lunid') if lunid: return f'{{serial_lunid}}{serial}_{lunid}' return f'{{serial}}{serial}' serial = await self.serial_from_device(name) if serial: return f'{{serial}}{serial}' klass = geom.class_by_name('PART') if klass: for g in klass.geoms: for p in g.providers: if p.name == name: # freebsd-zfs partition if p.config[ 'rawtype'] == '516e7cba-6ecf-11d6-8ff8-00022d09712b': return f'{{uuid}}{p.config["rawuuid"]}' g = geom.geom_by_name('LABEL', name) if g: return f'{{label}}{g.provider.name}' g = geom.geom_by_name('DEV', name) if g: return f'{{devicename}}{name}' return '' @private @accepts(Str('name')) async def sync(self, name): """ Syncs a disk `name` with the database cache. """ # Skip sync disks on backup node if (not await self.middleware.call('system.is_freenas') and await self.middleware.call('notifier.failover_licensed') and await self.middleware.call('notifier.failover_status') == 'BACKUP'): return # Do not sync geom classes like multipath/hast/etc if name.find("/") != -1: return disks = list((await self.middleware.call('device.get_info', 'DISK')).keys()) # Abort if the disk is not recognized as an available disk if name not in disks: return ident = await self.device_to_identifier(name) qs = await self.middleware.call('datastore.query', 'storage.disk', [('disk_identifier', '=', ident)], {'order_by': ['disk_expiretime']}) if ident and qs: disk = qs[0] new = False else: new = True qs = await self.middleware.call('datastore.query', 'storage.disk', [('disk_name', '=', name)]) for i in qs: i['disk_expiretime'] = datetime.utcnow() + timedelta( days=DISK_EXPIRECACHE_DAYS) await self.middleware.call('datastore.update', 'storage.disk', i['disk_identifier'], i) disk = {'disk_identifier': ident} disk.update({'disk_name': name, 'disk_expiretime': None}) await self.middleware.run_in_thread(geom.scan) g = geom.geom_by_name('DISK', name) if g: if g.provider.config['ident']: disk['disk_serial'] = g.provider.config['ident'] if g.provider.mediasize: disk['disk_size'] = g.provider.mediasize if not disk.get('disk_serial'): disk['disk_serial'] = await self.serial_from_device(name) or '' reg = RE_DSKNAME.search(name) if reg: disk['disk_subsystem'] = reg.group(1) disk['disk_number'] = int(reg.group(2)) if not new: await self.middleware.call('datastore.update', 'storage.disk', disk['disk_identifier'], disk) else: disk['disk_identifier'] = await self.middleware.call( 'datastore.insert', 'storage.disk', disk) # FIXME: use a truenas middleware plugin await self.middleware.call('notifier.sync_disk_extra', disk['disk_identifier'], False) @private @accepts() async def sync_all(self): """ Synchronyze all disks with the cache in database. """ # Skip sync disks on backup node if (not await self.middleware.call('system.is_freenas') and await self.middleware.call('notifier.failover_licensed') and await self.middleware.call('notifier.failover_status') == 'BACKUP'): return sys_disks = list((await self.middleware.call('device.get_info', 'DISK')).keys()) seen_disks = {} serials = [] await self.middleware.run_in_thread(geom.scan) for disk in (await self.middleware.call('datastore.query', 'storage.disk', [], {'order_by': ['disk_expiretime']})): name = await self.middleware.call('notifier.identifier_to_device', disk['disk_identifier']) if not name or name in seen_disks: # If we cant translate the indentifier to a device, give up # If name has already been seen once then we are probably # dealing with with multipath here if not disk['disk_expiretime']: disk['disk_expiretime'] = datetime.utcnow() + timedelta( days=DISK_EXPIRECACHE_DAYS) await self.middleware.call('datastore.update', 'storage.disk', disk['disk_identifier'], disk) elif disk['disk_expiretime'] < datetime.utcnow(): # Disk expire time has surpassed, go ahead and remove it await self.middleware.call('datastore.delete', 'storage.disk', disk['disk_identifier']) continue else: disk['disk_expiretime'] = None disk['disk_name'] = name reg = RE_DSKNAME.search(name) if reg: disk['disk_subsystem'] = reg.group(1) disk['disk_number'] = int(reg.group(2)) serial = '' g = geom.geom_by_name('DISK', name) if g: if g.provider.config['ident']: serial = disk['disk_serial'] = g.provider.config['ident'] serial += g.provider.config.get('lunid') or '' if g.provider.mediasize: disk['disk_size'] = g.provider.mediasize if not disk.get('disk_serial'): serial = disk['disk_serial'] = await self.serial_from_device( name) or '' if serial: serials.append(serial) # If for some reason disk is not identified as a system disk # mark it to expire. if name not in sys_disks and not disk['disk_expiretime']: disk['disk_expiretime'] = datetime.utcnow() + timedelta( days=DISK_EXPIRECACHE_DAYS) await self.middleware.call('datastore.update', 'storage.disk', disk['disk_identifier'], disk) # FIXME: use a truenas middleware plugin await self.middleware.call('notifier.sync_disk_extra', disk['disk_identifier'], False) seen_disks[name] = disk for name in sys_disks: if name not in seen_disks: disk_identifier = await self.device_to_identifier(name) qs = await self.middleware.call( 'datastore.query', 'storage.disk', [('disk_identifier', '=', disk_identifier)]) if qs: new = False disk = qs[0] else: new = True disk = {'disk_identifier': disk_identifier} disk['disk_name'] = name serial = '' g = geom.geom_by_name('DISK', name) if g: if g.provider.config['ident']: serial = disk['disk_serial'] = g.provider.config[ 'ident'] serial += g.provider.config.get('lunid') or '' if g.provider.mediasize: disk['disk_size'] = g.provider.mediasize if not disk.get('disk_serial'): serial = disk[ 'disk_serial'] = await self.serial_from_device(name ) or '' if serial: if serial in serials: # Probably dealing with multipath here, do not add another continue else: serials.append(serial) reg = RE_DSKNAME.search(name) if reg: disk['disk_subsystem'] = reg.group(1) disk['disk_number'] = int(reg.group(2)) if not new: await self.middleware.call('datastore.update', 'storage.disk', disk['disk_identifier'], disk) else: disk['disk_identifier'] = await self.middleware.call( 'datastore.insert', 'storage.disk', disk) # FIXME: use a truenas middleware plugin await self.middleware.call('notifier.sync_disk_extra', disk['disk_identifier'], True) async def __multipath_create(self, name, consumers, mode=None): """ Create an Active/Passive GEOM_MULTIPATH provider with name ``name`` using ``consumers`` as the consumers for it Modes: A - Active/Active R - Active/Read None - Active/Passive Returns: True in case the label succeeded and False otherwise """ cmd = ["/sbin/gmultipath", "label", name] + consumers if mode: cmd.insert(2, f'-{mode}') p1 = await Popen(cmd, stdout=subprocess.PIPE) if (await p1.wait()) != 0: return False return True async def __multipath_next(self): """ Find out the next available name for a multipath named diskX where X is a crescenting value starting from 1 Returns: The string of the multipath name to be created """ await self.middleware.run_in_thread(geom.scan) numbers = sorted([ int(RE_MPATH_NAME.search(g.name).group(1)) for g in geom.class_by_name('MULTIPATH').geoms if RE_MPATH_NAME.match(g.name) ]) if not numbers: numbers = [0] for number in range(1, numbers[-1] + 2): if number not in numbers: break else: raise ValueError('Could not find multipaths') return f'disk{number}' @private @accepts() async def multipath_sync(self): """ Synchronize multipath disks Every distinct GEOM_DISK that shares an ident (aka disk serial) with conjunction of the lunid is considered a multipath and will be handled by GEOM_MULTIPATH. If the disk is not currently in use by some Volume or iSCSI Disk Extent then a gmultipath is automatically created and will be available for use. """ await self.middleware.run_in_thread(geom.scan) mp_disks = [] for g in geom.class_by_name('MULTIPATH').geoms: for c in g.consumers: p_geom = c.provider.geom # For now just DISK is allowed if p_geom.clazz.name != 'DISK': self.logger.warn( "A consumer that is not a disk (%s) is part of a " "MULTIPATH, currently unsupported by middleware", p_geom.clazz.name) continue mp_disks.append(p_geom.name) reserved = await self.get_reserved() is_freenas = await self.middleware.call('system.is_freenas') serials = defaultdict(list) active_active = [] for g in geom.class_by_name('DISK').geoms: if not RE_DA.match( g.name) or g.name in reserved or g.name in mp_disks: continue if not is_freenas: descr = g.provider.config.get('descr') or '' if (descr == 'STEC ZeusRAM' or descr.startswith('VIOLIN') or descr.startswith('3PAR')): active_active.append(g.name) serial = '' v = g.provider.config.get('ident') if v: serial = v v = g.provider.config.get('lunid') if v: serial += v if not serial: continue size = g.provider.mediasize serials[(serial, size)].append(g.name) serials[(serial, size)].sort(key=lambda x: int(x[2:])) disks_pairs = [disks for disks in list(serials.values())] disks_pairs.sort(key=lambda x: int(x[0][2:])) # Mode is Active/Passive for FreeNAS mode = None if is_freenas else 'R' for disks in disks_pairs: if not len(disks) > 1: continue name = await self.__multipath_next() await self.__multipath_create( name, disks, 'A' if disks[0] in active_active else mode) # Scan again to take new multipaths into account await self.middleware.run_in_thread(geom.scan) mp_ids = [] for g in geom.class_by_name('MULTIPATH').geoms: _disks = [] for c in g.consumers: p_geom = c.provider.geom # For now just DISK is allowed if p_geom.clazz.name != 'DISK': continue _disks.append(p_geom.name) qs = await self.middleware.call( 'datastore.query', 'storage.disk', [ [ 'OR', [ ['disk_name', 'in', _disks], ['disk_multipath_member', 'in', _disks], ] ], ]) if qs: diskobj = qs[0] mp_ids.append(diskobj['disk_identifier']) update = False # Make sure to not update if nothing changed if diskobj['disk_multipath_name'] != g.name: update = True diskobj['disk_multipath_name'] = g.name if diskobj['disk_name'] in _disks: _disks.remove(diskobj['disk_name']) if _disks and diskobj['disk_multipath_member'] != _disks[-1]: update = True diskobj['disk_multipath_member'] = _disks.pop() if update: await self.middleware.call('datastore.update', 'storage.disk', diskobj['disk_identifier'], diskobj) # Update all disks which were not identified as MULTIPATH, resetting attributes for disk in (await self.middleware.call( 'datastore.query', 'storage.disk', [('disk_identifier', 'nin', mp_ids)])): if disk['disk_multipath_name'] or disk['disk_multipath_member']: disk['disk_multipath_name'] = '' disk['disk_multipath_member'] = '' await self.middleware.call('datastore.update', 'storage.disk', disk['disk_identifier'], disk) @private async def swaps_configure(self): """ Configures swap partitions in the system. We try to mirror all available swap partitions to avoid a system crash in case one of them dies. """ await self.middleware.run_in_thread(geom.scan) used_partitions = set() swap_devices = [] klass = geom.class_by_name('MIRROR') if klass: for g in klass.geoms: # Skip gmirror that is not swap* if not g.name.startswith('swap') or g.name.endswith('.sync'): continue consumers = list(g.consumers) # If the mirror is degraded lets remove it and make a new pair if len(consumers) == 1: c = consumers[0] await self.swaps_remove_disks([c.provider.geom.name]) else: swap_devices.append(f'mirror/{g.name}') for c in consumers: # Add all partitions used in swap, removing .eli used_partitions.add(c.provider.name.strip('.eli')) klass = geom.class_by_name('PART') if not klass: return # Get all partitions of swap type, indexed by size swap_partitions_by_size = defaultdict(list) for g in klass.geoms: for p in g.providers: # if swap partition if p.config[ 'rawtype'] == '516e7cb5-6ecf-11d6-8ff8-00022d09712b': if p.name not in used_partitions: # Try to save a core dump from that. # Only try savecore if the partition is not already in use # to avoid errors in the console (#27516) await run('savecore', '-z', '-m', '5', '/data/crash/', f'/dev/{p.name}', check=False) swap_partitions_by_size[p.mediasize].append(p.name) dumpdev = False unused_partitions = [] for size, partitions in swap_partitions_by_size.items(): # If we have only one partition add it to unused_partitions list if len(partitions) == 1: unused_partitions += partitions continue for i in range(int(len(partitions) / 2)): if len(swap_devices) > MIRROR_MAX: break part_a, part_b = partitions[0:2] partitions = partitions[2:] if not dumpdev: dumpdev = await dempdev_configure(part_a) try: name = new_swap_name() if name is None: # Which means maximum has been reached and we can stop break await run('gmirror', 'create', '-b', 'prefer', name, part_a, part_b) except Exception: self.logger.warn(f'Failed to create gmirror {name}', exc_info=True) continue swap_devices.append(f'mirror/{name}') # Add remaining partitions to unused list unused_partitions += partitions # If we could not make even a single swap mirror, add the first unused # partition as a swap device if not swap_devices and unused_partitions: if not dumpdev: dumpdev = await dempdev_configure(unused_partitions[0]) swap_devices.append(unused_partitions[0]) for name in swap_devices: if not os.path.exists(f'/dev/{name}.eli'): await run('geli', 'onetime', name) await run('swapon', f'/dev/{name}.eli', check=False) return swap_devices @private async def swaps_remove_disks(self, disks): """ Remove a given disk (e.g. ["da0", "da1"]) from swap. it will offline if from swap, remove it from the gmirror (if exists) and detach the geli. """ await self.middleware.run_in_thread(geom.scan) providers = {} for disk in disks: partgeom = geom.geom_by_name('PART', disk) if not partgeom: continue for p in partgeom.providers: if p.config[ 'rawtype'] == '516e7cb5-6ecf-11d6-8ff8-00022d09712b': providers[p.id] = p break if not providers: return klass = geom.class_by_name('MIRROR') if not klass: return mirrors = set() for g in klass.geoms: for c in g.consumers: if c.provider.id in providers: mirrors.add(g.name) del providers[c.provider.id] for name in mirrors: await run('swapoff', f'/dev/mirror/{name}.eli') if os.path.exists(f'/dev/mirror/{name}.eli'): await run('geli', 'detach', f'mirror/{name}.eli') await run('gmirror', 'destroy', name) for p in providers.values(): await run('swapoff', f'/dev/{p.name}.eli') @private async def wipe_quick(self, dev, size=None): """ Perform a quick wipe of a disk `dev` by the first few and last few megabytes """ # If the size is too small, lets just skip it for now. # In the future we can adjust dd size if size and size < 33554432: return await run('dd', 'if=/dev/zero', f'of=/dev/{dev}', 'bs=1m', 'count=32') try: cp = await run('diskinfo', dev) size = int( int(re.sub(r'\s+', ' ', cp.stdout.decode()).split()[2]) / (1024)) except subprocess.CalledProcessError: self.logger.error(f'Unable to determine size of {dev}') else: # This will fail when EOL is reached await run('dd', 'if=/dev/zero', f'of=/dev/{dev}', 'bs=1m', f'oseek={int(size / 1024) - 32}', check=False) @accepts(Str('dev'), Str('mode', enum=['QUICK', 'FULL', 'FULL_RANDOM'])) @job(lock=lambda args: args[0]) async def wipe(self, job, dev, mode): """ Performs a wipe of a disk `dev`. It can be of the following modes: - QUICK: clean the first few and last megabytes of every partition and disk - FULL: write whole disk with zero's - FULL_RANDOM: write whole disk with random bytes """ await self.swaps_remove_disks([dev]) # First do a quick wipe of every partition to clean things like zfs labels if mode == 'QUICK': await self.middleware.run_in_thread(geom.scan) klass = geom.class_by_name('PART') for g in klass.xml.findall(f'./geom[name=\'{dev}\']'): for p in g.findall('./provider'): size = p.find('./mediasize') if size is not None: try: size = int(size.text) except ValueError: size = None name = p.find('./name') await self.wipe_quick(name.text, size=size) await run('gpart', 'destroy', '-F', f'/dev/{dev}', check=False) # Wipe out the partition table by doing an additional iterate of create/destroy await run('gpart', 'create', '-s', 'gpt', f'/dev/{dev}') await run('gpart', 'destroy', '-F', f'/dev/{dev}') if mode == 'QUICK': await self.wipe_quick(dev) else: cp = await run('diskinfo', dev) size = int(re.sub(r'\s+', ' ', cp.stdout.decode()).split()[2]) proc = await Popen([ 'dd', 'if=/dev/{}'.format('zero' if mode == 'FULL' else 'random'), f'of=/dev/{dev}', 'bs=1m', ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) async def dd_wait(): while True: if proc.returncode is not None: break os.kill(proc.pid, signal.SIGINFO) await asyncio.sleep(1) asyncio.ensure_future(dd_wait()) while True: line = await proc.stderr.readline() if line == b'': break line = line.decode() reg = RE_DD.search(line) if reg: job.set_progress(int(reg.group(1)) / size, extra={'speed': int(reg.group(2))}) await self.sync(dev)
class FilesystemService(Service): @accepts(Str('path', required=True), Ref('query-filters'), Ref('query-options')) def listdir(self, path, filters=None, options=None): """ Get the contents of a directory. Each entry of the list consists of: name(str): name of the file path(str): absolute path of the entry realpath(str): absolute real path of the entry (if SYMLINK) type(str): DIRECTORY | FILESYSTEM | SYMLINK | OTHER size(int): size of the entry mode(int): file mode/permission uid(int): user id of entry owner gid(int): group id of entry onwer acl(bool): extended ACL is present on file """ if not os.path.exists(path): raise CallError(f'Directory {path} does not exist', errno.ENOENT) if not os.path.isdir(path): raise CallError(f'Path {path} is not a directory', errno.ENOTDIR) rv = [] for entry in os.scandir(path): if entry.is_symlink(): etype = 'SYMLINK' elif entry.is_dir(): etype = 'DIRECTORY' elif entry.is_file(): etype = 'FILE' else: etype = 'OTHER' data = { 'name': entry.name, 'path': entry.path, 'realpath': os.path.realpath(entry.path) if etype == 'SYMLINK' else entry.path, 'type': etype, } try: stat = entry.stat() data.update({ 'size': stat.st_size, 'mode': stat.st_mode, 'acl': False if self.acl_is_trivial(data["realpath"]) else True, 'uid': stat.st_uid, 'gid': stat.st_gid, }) except FileNotFoundError: data.update({ 'size': None, 'mode': None, 'acl': None, 'uid': None, 'gid': None }) rv.append(data) return filter_list(rv, filters=filters or [], options=options or {}) @accepts(Str('path')) def stat(self, path): """ Return the filesystem stat(2) for a given `path`. """ try: stat = os.stat(path, follow_symlinks=False) except FileNotFoundError: raise CallError(f'Path {path} not found', errno.ENOENT) stat = { 'size': stat.st_size, 'mode': stat.st_mode, 'uid': stat.st_uid, 'gid': stat.st_gid, 'atime': stat.st_atime, 'mtime': stat.st_mtime, 'ctime': stat.st_ctime, 'dev': stat.st_dev, 'inode': stat.st_ino, 'nlink': stat.st_nlink, } try: stat['user'] = pwd.getpwuid(stat['uid']).pw_name except KeyError: stat['user'] = None try: stat['group'] = grp.getgrgid(stat['gid']).gr_name except KeyError: stat['group'] = None stat['acl'] = False if self.middleware.call_sync( 'filesystem.acl_is_trivial', path) else True return stat @private @accepts( Str('path'), Str('content', max_length=2048000), Dict( 'options', Bool('append', default=False), Int('mode'), ), ) def file_receive(self, path, content, options=None): """ Simplified file receiving method for small files. `content` must be a base 64 encoded file content. """ options = options or {} dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname) if options.get('append'): openmode = 'ab' else: openmode = 'wb+' with open(path, openmode) as f: f.write(binascii.a2b_base64(content)) mode = options.get('mode') if mode: os.chmod(path, mode) return True @private @accepts( Str('path'), Dict( 'options', Int('offset'), Int('maxlen'), ), ) def file_get_contents(self, path, options=None): """ Get contents of a file `path` in base64 encode. DISCLAIMER: DO NOT USE THIS FOR BIG FILES (> 500KB). """ options = options or {} if not os.path.exists(path): return None with open(path, 'rb') as f: if options.get('offset'): f.seek(options['offset']) data = binascii.b2a_base64(f.read( options.get('maxlen'))).decode().strip() return data @accepts(Str('path')) @job(pipes=["output"]) async def get(self, job, path): """ Job to get contents of `path`. """ if not os.path.isfile(path): raise CallError(f'{path} is not a file') with open(path, 'rb') as f: await self.middleware.run_in_thread(shutil.copyfileobj, f, job.pipes.output.w) @accepts( Str('path'), Dict( 'options', Bool('append', default=False), Int('mode'), ), ) @job(pipes=["input"]) async def put(self, job, path, options=None): """ Job to put contents to `path`. """ options = options or {} dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname) if options.get('append'): openmode = 'ab' else: openmode = 'wb+' with open(path, openmode) as f: await self.middleware.run_in_thread(shutil.copyfileobj, job.pipes.input.r, f) mode = options.get('mode') if mode: os.chmod(path, mode) return True @accepts(Str('path')) def statfs(self, path): """ Return stats from the filesystem of a given path. Raises: CallError(ENOENT) - Path not found """ try: statfs = bsd.statfs(path) except FileNotFoundError: raise CallError('Path not found.', errno.ENOENT) return { **statfs.__getstate__(), 'total_bytes': statfs.total_blocks * statfs.blocksize, 'free_bytes': statfs.free_blocks * statfs.blocksize, 'avail_bytes': statfs.avail_blocks * statfs.blocksize, } def __convert_to_basic_permset(self, permset): """ Convert "advanced" ACL permset format to basic format using bitwise operation and constants defined in py-bsd/bsd/acl.pyx, py-bsd/defs.pxd and acl.h. If the advanced ACL can't be converted without losing information, we return 'OTHER'. Reverse process converts the constant's value to a dictionary using a bitwise operation. """ perm = 0 for k, v, in permset.items(): if v: perm |= acl.NFS4Perm[k] try: SimplePerm = (acl.NFS4BasicPermset(perm)).name except Exception: SimplePerm = 'OTHER' return SimplePerm def __convert_to_basic_flagset(self, flagset): flags = 0 for k, v, in flagset.items(): if k == "INHERITED": continue if v: flags |= acl.NFS4Flag[k] try: SimpleFlag = (acl.NFS4BasicFlagset(flags)).name except Exception: SimpleFlag = 'OTHER' return SimpleFlag def __convert_to_adv_permset(self, basic_perm): permset = {} perm_mask = acl.NFS4BasicPermset[basic_perm].value for name, member in acl.NFS4Perm.__members__.items(): if perm_mask & member.value: permset.update({name: True}) else: permset.update({name: False}) return permset def __convert_to_adv_flagset(self, basic_flag): flagset = {} flag_mask = acl.NFS4BasicFlagset[basic_flag].value for name, member in acl.NFS4Flag.__members__.items(): if flag_mask & member.value: flagset.update({name: True}) else: flagset.update({name: False}) return flagset def _winacl(self, path, action, uid, gid, options): chroot_dir = os.path.dirname(path) target = os.path.basename(path) winacl = subprocess.run([ '/usr/local/bin/winacl', '-a', action, '-O', str(uid), '-G', str(gid), '-rx' if options['traverse'] else '-r', '-c', chroot_dir, '-p', target ], check=False, capture_output=True) if winacl.returncode != 0: CallError( f"Winacl {action} on path {path} failed with error: [{winacl.stderr.decode().strip()}]" ) def _common_perm_path_validate(self, path): if not os.path.exists(path): raise CallError(f"Path not found: {path}", errno.ENOENT) if not os.path.realpath(path).startswith('/mnt/'): raise CallError( f"Changing permissions on paths outside of /mnt is not permitted: {path}", errno.EPERM) if os.path.realpath(path) in [ x['path'] for x in self.middleware.call_sync('pool.query') ]: raise CallError( f"Changing permissions of root level dataset is not permitted: {path}", errno.EPERM) @accepts(Str('path')) def acl_is_trivial(self, path): """ Returns True if the ACL can be fully expressed as a file mode without losing any access rules, or if the path does not support NFSv4 ACLs (for example a path on a tmpfs filesystem). """ if not os.path.exists(path): raise CallError('Path not found.', errno.ENOENT) has_nfs4_acl_support = os.pathconf(path, 64) if not has_nfs4_acl_support: return True return acl.ACL(file=path).is_trivial @accepts( Dict( 'filesystem_ownership', Str('path', required=True), Int('uid', null=True, default=None), Int('gid', null=True, default=None), Dict('options', Bool('recursive', default=False), Bool('traverse', default=False)))) @job(lock="perm_change") def chown(self, job, data): """ Change owner or group of file at `path`. `uid` and `gid` specify new owner of the file. If either key is absent or None, then existing value on the file is not changed. `recursive` performs action recursively, but does not traverse filesystem mount points. If `traverse` and `recursive` are specified, then the chown operation will traverse filesystem mount points. """ job.set_progress(0, 'Preparing to change owner.') self._common_perm_path_validate(data['path']) uid = -1 if data['uid'] is None else data['uid'] gid = -1 if data['gid'] is None else data['gid'] options = data['options'] if not options['recursive']: job.set_progress(100, 'Finished changing owner.') os.chown(data['path'], uid, gid) else: job.set_progress(10, f'Recursively changing owner of {data["path"]}.') self._winacl(data['path'], 'chown', uid, gid, options) job.set_progress(100, 'Finished changing owner.') @accepts( Dict( 'filesystem_permission', Str('path', required=True), UnixPerm('mode', null=True), Int('uid', null=True, default=None), Int('gid', null=True, default=None), Dict( 'options', Bool('stripacl', default=False), Bool('recursive', default=False), Bool('traverse', default=False), ))) @job(lock="perm_change") def setperm(self, job, data): """ Remove extended ACL from specified path. If `mode` is specified then the mode will be applied to the path and files and subdirectories depending on which `options` are selected. Mode should be formatted as string representation of octal permissions bits. `uid` the desired UID of the file user. If set to None (the default), then user is not changed. `gid` the desired GID of the file group. If set to None (the default), then group is not changed. `stripacl` setperm will fail if an extended ACL is present on `path`, unless `stripacl` is set to True. `recursive` remove ACLs recursively, but do not traverse dataset boundaries. `traverse` remove ACLs from child datasets. If no `mode` is set, and `stripacl` is True, then non-trivial ACLs will be converted to trivial ACLs. An ACL is trivial if it can be expressed as a file mode without losing any access rules. """ job.set_progress(0, 'Preparing to set permissions.') options = data['options'] mode = data.get('mode', None) uid = -1 if data['uid'] is None else data['uid'] gid = -1 if data['gid'] is None else data['gid'] self._common_perm_path_validate(data['path']) acl_is_trivial = self.middleware.call_sync('filesystem.acl_is_trivial', data['path']) if not acl_is_trivial and not options['stripacl']: raise CallError( f'Non-trivial ACL present on [{data["path"]}]. Option "stripacl" required to change permission.', errno.EINVAL) if mode is not None: mode = int(mode, 8) a = acl.ACL(file=data['path']) a.strip() a.apply(data['path']) if mode: os.chmod(data['path'], mode) if uid or gid: os.chown(data['path'], uid, gid) if not options['recursive']: job.set_progress(100, 'Finished setting permissions.') return action = 'clone' if mode else 'strip' job.set_progress( 10, f'Recursively setting permissions on {data["path"]}.') self._winacl(data['path'], action, uid, gid, options) job.set_progress(100, 'Finished setting permissions.') @accepts() async def default_acl_choices(self): """ Get list of default ACL types. """ acl_choices = [] for x in ACLDefault: if x.value['visible']: acl_choices.append(x.name) return acl_choices @accepts( Str('acl_type', default='OPEN', enum=[x.name for x in ACLDefault]), Str('share_type', default='NONE', enum=['NONE', 'AFP', 'SMB', 'NFS']), ) async def get_default_acl(self, acl_type, share_type): """ Returns a default ACL depending on the usage specified by `acl_type`. If an admin group is defined, then an entry granting it full control will be placed at the top of the ACL. Optionally may pass `share_type` to argument to get share-specific template ACL. """ acl = [] admin_group = (await self.middleware.call('smb.config'))['admin_group'] if acl_type == 'HOME' and (await self.middleware.call( 'activedirectory.get_state')) == 'HEALTHY': acl_type = 'DOMAIN_HOME' if admin_group: acl.append({ 'tag': 'GROUP', 'id': (await self.middleware.call('dscache.get_uncached_group', admin_group))['gr_gid'], 'perms': { 'BASIC': 'FULL_CONTROL' }, 'flags': { 'BASIC': 'INHERIT' }, 'type': 'ALLOW' }) if share_type == 'SMB': acl.append({ 'tag': 'GROUP', 'id': int(SMBBuiltin['USERS'].value[1][9:]), 'perms': { 'BASIC': 'MODIFY' }, 'flags': { 'BASIC': 'INHERIT' }, 'type': 'ALLOW' }) acl.extend((ACLDefault[acl_type].value)['acl']) return acl def _is_inheritable(self, flags): """ Takes ACE flags and return True if any inheritance bits are set. """ inheritance_flags = [ 'FILE_INHERIT', 'DIRECTORY_INHERIT', 'NO_PROPAGATE_INHERIT', 'INHERIT_ONLY' ] for i in inheritance_flags: if flags.get(i): return True return False @private def canonicalize_acl_order(self, acl): """ Convert flags to advanced, then separate the ACL into two lists. One for ACEs that have been inherited, one for aces that have not been inherited. Non-inherited ACEs take precedence and so they are placed first in finalized combined list. Within each list, the ACEs are orderd according to the following: 1) Deny ACEs that apply to the object itself (NOINHERIT) 2) Deny ACEs that apply to a subobject of the object (INHERIT) 3) Allow ACEs that apply to the object itself (NOINHERIT) 4) Allow ACEs that apply to a subobject of the object (INHERIT) See http://docs.microsoft.com/en-us/windows/desktop/secauthz/order-of-aces-in-a-dacl The "INHERITED" bit is stripped in filesystem.getacl when generating a BASIC flag type. It is best practice to use a non-simplified ACL for canonicalization. """ inherited_aces = [] final_acl = [] non_inherited_aces = [] for entry in acl: entry['flags'] = self.__convert_to_adv_flagset( entry['flags'] ['BASIC']) if 'BASIC' in entry['flags'] else entry['flags'] if entry['flags'].get('INHERITED'): inherited_aces.append(entry) else: non_inherited_aces.append(entry) if inherited_aces: inherited_aces = sorted( inherited_aces, key=lambda x: (x['type'] == 'ALLOW', self._is_inheritable(x['flags'])), ) if non_inherited_aces: non_inherited_aces = sorted( non_inherited_aces, key=lambda x: (x['type'] == 'ALLOW', self._is_inheritable(x['flags'])), ) final_acl = non_inherited_aces + inherited_aces return final_acl @accepts( Str('path'), Bool('simplified', default=True), ) def getacl(self, path, simplified=True): """ Return ACL of a given path. Simplified returns a shortened form of the ACL permset and flags `TRAVERSE` sufficient rights to traverse a directory, but not read contents. `READ` sufficient rights to traverse a directory, and read file contents. `MODIFIY` sufficient rights to traverse, read, write, and modify a file. Equivalent to modify_set. `FULL_CONTROL` all permissions. If the permisssions do not fit within one of the pre-defined simplified permissions types, then the full ACL entry will be returned. In all cases we replace USER_OBJ, GROUP_OBJ, and EVERYONE with owner@, group@, everyone@ for consistency with getfacl and setfacl. If one of aforementioned special tags is used, 'id' must be set to None. An inheriting empty everyone@ ACE is appended to non-trivial ACLs in order to enforce Windows expectations regarding permissions inheritance. This entry is removed from NT ACL returned to SMB clients when 'ixnas' samba VFS module is enabled. We also remove it here to avoid confusion. """ if not os.path.exists(path): raise CallError('Path not found.', errno.ENOENT) stat = os.stat(path) a = acl.ACL(file=path) fs_acl = a.__getstate__() if not simplified: advanced_acl = [] for entry in fs_acl: ace = { 'tag': (acl.ACLWho[entry['tag']]).value, 'id': entry['id'], 'type': entry['type'], 'perms': entry['perms'], 'flags': entry['flags'], } if ace['tag'] == 'everyone@' and self.__convert_to_basic_permset( ace['perms']) == 'NOPERMS': continue advanced_acl.append(ace) return { 'uid': stat.st_uid, 'gid': stat.st_gid, 'acl': advanced_acl } if simplified: simple_acl = [] for entry in fs_acl: ace = { 'tag': (acl.ACLWho[entry['tag']]).value, 'id': entry['id'], 'type': entry['type'], 'perms': { 'BASIC': self.__convert_to_basic_permset(entry['perms']) }, 'flags': { 'BASIC': self.__convert_to_basic_flagset(entry['flags']) }, } if ace['tag'] == 'everyone@' and ace['perms'][ 'BASIC'] == 'NOPERMS': continue for key in ['perms', 'flags']: if ace[key]['BASIC'] == 'OTHER': ace[key] = entry[key] simple_acl.append(ace) return {'uid': stat.st_uid, 'gid': stat.st_gid, 'acl': simple_acl} @accepts( Dict( 'filesystem_acl', Str('path', required=True), Int('uid', null=True, default=None), Int('gid', null=True, default=None), List('dacl', items=[ Dict( 'aclentry', Str('tag', enum=[ 'owner@', 'group@', 'everyone@', 'USER', 'GROUP' ]), Int('id', null=True), Str('type', enum=['ALLOW', 'DENY']), Dict( 'perms', Bool('READ_DATA'), Bool('WRITE_DATA'), Bool('APPEND_DATA'), Bool('READ_NAMED_ATTRS'), Bool('WRITE_NAMED_ATTRS'), Bool('EXECUTE'), Bool('DELETE_CHILD'), Bool('READ_ATTRIBUTES'), Bool('WRITE_ATTRIBUTES'), Bool('DELETE'), Bool('READ_ACL'), Bool('WRITE_ACL'), Bool('WRITE_OWNER'), Bool('SYNCHRONIZE'), Str('BASIC', enum=[ 'FULL_CONTROL', 'MODIFY', 'READ', 'TRAVERSE' ]), ), Dict( 'flags', Bool('FILE_INHERIT'), Bool('DIRECTORY_INHERIT'), Bool('NO_PROPAGATE_INHERIT'), Bool('INHERIT_ONLY'), Bool('INHERITED'), Str('BASIC', enum=['INHERIT', 'NOINHERIT']), ), ) ], default=[]), Dict('options', Bool('stripacl', default=False), Bool('recursive', default=False), Bool('traverse', default=False), Bool('canonicalize', default=True)))) @job(lock="perm_change") def setacl(self, job, data): """ Set ACL of a given path. Takes the following parameters: `path` full path to directory or file. `dacl` "simplified" ACL here or a full ACL. `uid` the desired UID of the file user. If set to None (the default), then user is not changed. `gid` the desired GID of the file group. If set to None (the default), then group is not changed. `recursive` apply the ACL recursively `traverse` traverse filestem boundaries (ZFS datasets) `strip` convert ACL to trivial. ACL is trivial if it can be expressed as a file mode without losing any access rules. `canonicalize` reorder ACL entries so that they are in concanical form as described in the Microsoft documentation MS-DTYP 2.4.5 (ACL) In all cases we replace USER_OBJ, GROUP_OBJ, and EVERYONE with owner@, group@, everyone@ for consistency with getfacl and setfacl. If one of aforementioned special tags is used, 'id' must be set to None. An inheriting empty everyone@ ACE is appended to non-trivial ACLs in order to enforce Windows expectations regarding permissions inheritance. This entry is removed from NT ACL returned to SMB clients when 'ixnas' samba VFS module is enabled. """ job.set_progress(0, 'Preparing to set acl.') options = data['options'] dacl = data.get('dacl', []) self._common_perm_path_validate(data['path']) if dacl and options['stripacl']: raise CallError( 'Setting ACL and stripping ACL are not permitted simultaneously.', errno.EINVAL) uid = -1 if data.get('uid', None) is None else data['uid'] gid = -1 if data.get('gid', None) is None else data['gid'] if options['stripacl']: a = acl.ACL(file=data['path']) a.strip() a.apply(data['path']) else: cleaned_acl = [] lockace_is_present = False for entry in dacl: ace = { 'tag': (acl.ACLWho(entry['tag'])).name, 'id': entry['id'], 'type': entry['type'], 'perms': self.__convert_to_adv_permset(entry['perms']['BASIC']) if 'BASIC' in entry['perms'] else entry['perms'], 'flags': self.__convert_to_adv_flagset(entry['flags']['BASIC']) if 'BASIC' in entry['flags'] else entry['flags'], } if ace['flags'].get('INHERIT_ONLY') and not ace['flags'].get( 'DIRECTORY_INHERIT', False) and not ace['flags'].get( 'FILE_INHERIT', False): raise CallError( 'Invalid flag combination. DIRECTORY_INHERIT or FILE_INHERIT must be set if INHERIT_ONLY is set.', errno.EINVAL) if ace['tag'] == 'EVERYONE' and self.__convert_to_basic_permset( ace['perms']) == 'NOPERMS': lockace_is_present = True cleaned_acl.append(ace) if options['canonicalize']: cleaned_acl = self.canonicalize_acl_order(cleaned_acl) if not lockace_is_present: locking_ace = { 'tag': 'EVERYONE', 'id': None, 'type': 'ALLOW', 'perms': self.__convert_to_adv_permset('NOPERMS'), 'flags': self.__convert_to_adv_flagset('INHERIT') } cleaned_acl.append(locking_ace) a = acl.ACL() a.__setstate__(cleaned_acl) a.apply(data['path']) if not options['recursive']: os.chown(data['path'], uid, gid) job.set_progress(100, 'Finished setting ACL.') return job.set_progress(10, f'Recursively setting ACL on {data["path"]}.') self._winacl(data['path'], 'clone', uid, gid, options) job.set_progress(100, 'Finished setting ACL.')
class SharingAFPService(CRUDService): class Config: namespace = 'sharing.afp' datastore = 'sharing.afp_share' datastore_prefix = 'afp_' datastore_extend = 'sharing.afp.extend' @accepts( Dict('sharingafp_create', Str('path', required=True), Bool('home', default=False), Str('name'), Str('comment'), List('allow', default=[]), List('deny', default=[]), List('ro', default=[]), List('rw', default=[]), Bool('timemachine', default=False), Int('timemachine_quota', default=0), Bool('nodev', default=False), Bool('nostat', default=False), Bool('upriv', default=True), UnixPerm('fperm', default='644'), UnixPerm('dperm', default='755'), UnixPerm('umask', default='000'), List('hostsallow', items=[], default=[]), List('hostsdeny', items=[], default=[]), Str('vuid', null=True, default=''), Str('auxparams', max_length=None), Bool('enabled', default=True), register=True)) async def do_create(self, data): """ Create AFP share. `allow`, `deny`, `ro`, and `rw` are lists of users and groups. Groups are designated by an @ prefix. `hostsallow` and `hostsdeny` are lists of hosts and/or networks. """ verrors = ValidationErrors() path = data['path'] await self.clean(data, 'sharingafp_create', verrors) await self.validate(data, 'sharingafp_create', verrors) await check_path_resides_within_volume(verrors, self.middleware, 'sharingafp_create.path', path) verrors.check() if path and not os.path.exists(path): try: os.makedirs(path) except OSError as e: raise CallError(f'Failed to create {path}: {e}') await self.compress(data) data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self.extend(data) await self._service_change('afp', 'reload') return data @accepts(Int('id'), Patch('sharingafp_create', 'sharingafp_update', ('attr', { 'update': True }))) async def do_update(self, id, data): """ Update AFP share `id`. """ verrors = ValidationErrors() old = await self.middleware.call( 'datastore.query', self._config.datastore, [('id', '=', id)], { 'extend': self._config.datastore_extend, 'prefix': self._config.datastore_prefix, 'get': True }) path = data.get('path') new = old.copy() new.update(data) await self.clean(new, 'sharingafp_update', verrors, id=id) await self.validate(new, 'sharingafp_update', verrors, old=old) if path: await check_path_resides_within_volume(verrors, self.middleware, 'sharingafp_create.path', path) if verrors: raise verrors if path and not os.path.exists(path): try: os.makedirs(path) except OSError as e: raise CallError(f'Failed to create {path}: {e}') await self.compress(new) await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) await self.extend(new) await self._service_change('afp', 'reload') return new @accepts(Int('id')) async def do_delete(self, id): """ Delete AFP share `id`. """ result = await self.middleware.call('datastore.delete', self._config.datastore, id) await self._service_change('afp', 'reload') return result @private async def clean(self, data, schema_name, verrors, id=None): data['name'] = await self.name_exists(data, schema_name, verrors, id) @private async def validate(self, data, schema_name, verrors, old=None): await self.home_exists(data['home'], schema_name, verrors, old) if data['vuid']: try: uuid.UUID(data['vuid'], version=4) except ValueError: verrors.add(f'{schema_name}.vuid', 'vuid must be a valid UUID.') @private async def home_exists(self, home, schema_name, verrors, old=None): home_filters = [('home', '=', True)] home_result = None if home: if old and old['id'] is not None: id = old['id'] if not old['home']: home_filters.append(('id', '!=', id)) # The user already had this set as the home share home_result = await self.middleware.call( 'datastore.query', self._config.datastore, home_filters, {'prefix': self._config.datastore_prefix}) if home_result: verrors.add(f'{schema_name}.home', 'Only one share is allowed to be a home share.') @private async def name_exists(self, data, schema_name, verrors, id=None): name = data['name'] path = data['path'] home = data['home'] name_filters = [('name', '=', name)] path_filters = [('path', '=', path)] if not name: if home: name = 'Homes' else: name = path.rsplit('/', 1)[-1] if id is not None: name_filters.append(('id', '!=', id)) path_filters.append(('id', '!=', id)) name_result = await self.middleware.call( 'datastore.query', self._config.datastore, name_filters, {'prefix': self._config.datastore_prefix}) path_result = await self.middleware.call( 'datastore.query', self._config.datastore, path_filters, {'prefix': self._config.datastore_prefix}) if name_result: verrors.add(f'{schema_name}.name', 'A share with this name already exists.') if path_result: verrors.add(f'{schema_name}.path', 'A share with this path already exists.') return name @private async def extend(self, data): data['allow'] = data['allow'].split() data['deny'] = data['deny'].split() data['ro'] = data['ro'].split() data['rw'] = data['rw'].split() data['hostsallow'] = data['hostsallow'].split() data['hostsdeny'] = data['hostsdeny'].split() return data @private async def compress(self, data): data['allow'] = ' '.join(data['allow']) data['deny'] = ' '.join(data['deny']) data['ro'] = ' '.join(data['ro']) data['rw'] = ' '.join(data['rw']) data['hostsallow'] = ' '.join(data['hostsallow']) data['hostsdeny'] = ' '.join(data['hostsdeny']) if not data['vuid'] and data['timemachine']: data['vuid'] = str(uuid.uuid4()) return data
class SharingSMBService(CRUDService): class Config: namespace = 'sharing.smb' datastore = 'sharing.cifs_share' datastore_prefix = 'cifs_' datastore_extend = 'sharing.smb.extend' @accepts( Dict('sharingsmb_create', Str('path', required=True), Bool('home', default=False), Str('name'), Str('comment'), Bool('ro', default=False), Bool('browsable', default=True), Bool('recyclebin', default=False), Bool('showhiddenfiles', default=False), Bool('guestok', default=False), Bool('guestonly', default=False), Bool('abe', default=False), List('hostsallow', items=[IPAddr('ip', network=True)], default=[]), List('hostsdeny', items=[IPAddr('ip', network=True)], default=[]), List('vfsobjects', default=['zfs_space', 'zfsacl', 'streams_xattr']), Int('storage_task'), Str('auxsmbconf'), Bool('default_permissions'), register=True)) async def do_create(self, data): verrors = ValidationErrors() path = data['path'] default_perms = data.pop('default_permissions', True) await self.clean(data, 'sharingsmb_create', verrors) await self.validate(data, 'sharingsmb_create', verrors) if verrors: raise verrors if path and not os.path.exists(path): try: os.makedirs(path) except OSError as e: raise CallError(f'Failed to create {path}: {e}') await self.compress(data) await self.set_storage_tasks(data) data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self.extend(data) # We should do this in the insert call ? await self._service_change('cifs', 'reload') await self.apply_default_perms(default_perms, path, data['home']) return data @accepts(Int('id'), Patch('sharingsmb_create', 'sharingsmb_update', ('attr', { 'update': True }))) async def do_update(self, id, data): verrors = ValidationErrors() path = data.get('path') default_perms = data.pop('default_permissions', False) old = await self.middleware.call( 'datastore.query', self._config.datastore, [('id', '=', id)], { 'extend': self._config.datastore_extend, 'prefix': self._config.datastore_prefix, 'get': True }) new = old.copy() new.update(data) await self.clean(new, 'sharingsmb_update', verrors, id=id) await self.validate(new, 'sharingsmb_update', verrors, old=old) if verrors: raise verrors if path and not os.path.exists(path): try: os.makedirs(path) except OSError as e: raise CallError(f'Failed to create {path}: {e}') await self.compress(new) await self.set_storage_tasks(new) await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) await self.extend(new) # same here ? await self._service_change('cifs', 'reload') await self.apply_default_perms(default_perms, path, data['home']) return new @accepts(Int('id')) async def do_delete(self, id): share = await self._get_instance(id) result = await self.middleware.call('datastore.delete', self._config.datastore, id) await self.middleware.call('notifier.sharesec_delete', share['name']) await self._service_change('cifs', 'reload') return result @private async def clean(self, data, schema_name, verrors, id=None): data['name'] = await self.name_exists(data, schema_name, verrors, id) @private async def validate(self, data, schema_name, verrors, old=None): home_result = await self.home_exists(data['home'], schema_name, verrors, old) if home_result: verrors.add(f'{schema_name}.home', 'Only one share is allowed to be a home share.') elif not home_result and not data['path']: verrors.add(f'{schema_name}.path', 'This field is required.') if data['path']: await check_path_resides_within_volume(verrors, self.middleware, f"{schema_name}.path", data['path']) if data.get('name') and data['name'] == 'global': verrors.add( f'{schema_name}.name', 'Global is a reserved section name, please select another one') @private async def home_exists(self, home, schema_name, verrors, old=None): home_filters = [('home', '=', True)] home_result = None if home: if old and old['id'] is not None: id = old['id'] if not old['home']: home_filters.append(('id', '!=', id)) # The user already had this set as the home share home_result = await self.middleware.call( 'datastore.query', self._config.datastore, home_filters, {'prefix': self._config.datastore_prefix}) return home_result @private async def name_exists(self, data, schema_name, verrors, id=None): name = data['name'] path = data['path'] name_filters = [('name', '=', name)] path_filters = [('path', '=', path)] if path and not name: name = path.rsplit('/', 1)[-1] if id is not None: name_filters.append(('id', '!=', id)) path_filters.append(('id', '!=', id)) name_result = await self.middleware.call( 'datastore.query', self._config.datastore, name_filters, {'prefix': self._config.datastore_prefix}) path_result = await self.middleware.call( 'datastore.query', self._config.datastore, path_filters, {'prefix': self._config.datastore_prefix}) if name_result: verrors.add(f'{schema_name}.name', 'A share with this name already exists.') if path_result: verrors.add(f'{schema_name}.path', 'A share with this path already exists.') return name @private async def extend(self, data): data['hostsallow'] = data['hostsallow'].split() data['hostsdeny'] = data['hostsdeny'].split() return data @private async def compress(self, data): data['hostsallow'] = ' '.join(data['hostsallow']) data['hostsdeny'] = ' '.join(data['hostsdeny']) return data @private async def apply_default_perms(self, default_perms, path, is_home): if default_perms: try: (owner, group) = await self.middleware.call('notifier.mp_get_owner', path) except Exception: (owner, group) = ('root', 'wheel') await self.middleware.call('notifier.winacl_reset', path, owner, group, None, not is_home) @accepts(Str('path', required=True)) async def get_storage_tasks(self, path): zfs_datasets = await self.middleware.call( 'zfs.dataset.query', [('type', '=', 'FILESYSTEM')]) task_list = [] task_dict = {} for ds in zfs_datasets: tasks = [] name = ds['name'] mountpoint = ds['properties']['mountpoint']['parsed'] if path == mountpoint: tasks = await self.middleware.call( 'datastore.query', 'storage.task', [['task_filesystem', '=', name]]) elif path.startswith(f'{mountpoint}/'): tasks = await self.middleware.call( 'datastore.query', 'storage.task', [['task_filesystem', '=', name], ['task_recursive', '=', 'True']]) task_list.extend(tasks) for task in task_list: task_id = task['id'] fs = task['task_filesystem'] retcount = task['task_ret_count'] retunit = task['task_ret_unit'] _interval = task['task_interval'] interval = dict(await self.middleware.call('notifier.choices', 'TASK_INTERVAL'))[_interval] msg = f'{fs} - every {interval} - {retcount}{retunit}' task_dict[task_id] = msg return task_dict @private async def set_storage_tasks(self, data): task = data.get('storage_task', None) home = data['home'] path = data['path'] task_list = [] if not task: if path: task_list = await self.get_storage_tasks(path=path) elif home: task_list = await self.get_storage_tasks(home=home) if task_list: data['storage_task'] = list(task_list.keys())[0] return data @accepts() def vfsobjects_choices(self): vfs_modules_path = '/usr/local/lib/shared-modules/vfs' vfs_modules = [] vfs_exclude = {'shadow_copy2', 'recycle', 'aio_pthread'} if os.path.exists(vfs_modules_path): vfs_modules.extend( filter( lambda m: m not in vfs_exclude, map(lambda f: f.rpartition('.')[0], os.listdir(vfs_modules_path)))) else: vfs_modules.extend(['streams_xattr']) return vfs_modules
class UPSService(SystemServiceService): try: DRIVERS_AVAILABLE = set(os.listdir(DRIVER_BIN_DIR)) except FileNotFoundError: DRIVERS_AVAILABLE = set() class Config: datastore = 'services.ups' datastore_prefix = 'ups_' datastore_extend = 'ups.ups_config_extend' service = 'ups' service_verb = 'restart' @private async def ups_config_extend(self, data): data['mode'] = data['mode'].upper() data['shutdown'] = data['shutdown'].upper() data['toemail'] = [v for v in data['toemail'].split(';') if v] host = 'localhost' if data['mode'] == 'MASTER' else data['remotehost'] data['complete_identifier'] = f'{data["identifier"]}@{host}:{data["remoteport"]}' return data @accepts() async def port_choices(self): ports = [x for x in glob.glob('/dev/cua*') if x.find('.') == -1] ports.extend(glob.glob('/dev/ugen*')) ports.extend(glob.glob('/dev/uhid*')) ports.append('auto') return ports @accepts() def driver_choices(self): """ Returns choices of UPS drivers supported by the system. """ ups_choices = {} if osc.IS_LINUX: driver_list = '/usr/share/nut/driver.list' else: driver_list = '/conf/base/etc/local/nut/driver.list' if os.path.exists(driver_list): with open(driver_list, 'rb') as f: d = f.read().decode('utf8', 'ignore') r = io.StringIO() for line in re.sub(r'[ \t]+', ' ', d, flags=re.M).split('\n'): r.write(line.strip() + '\n') r.seek(0) reader = csv.reader(r, delimiter=' ', quotechar='"') for row in reader: if len(row) == 0 or row[0].startswith('#'): continue if row[-2] == '#': last = -3 else: last = -1 driver_str = row[last] driver_annotation = '' m = re.match(r'(.+) \((.+)\)', driver_str) # "blazer_usb (USB ID 0665:5161)" if m: driver_str, driver_annotation = m.group(1), m.group(2) for driver in driver_str.split(' or '): # can be "blazer_ser or blazer_usb" driver = driver.strip() if driver not in self.DRIVERS_AVAILABLE: continue for i, field in enumerate(list(row)): row[i] = field ups_choices['$'.join([driver, row[3]])] = '%s (%s)' % ( ' '.join(filter(None, row[0:last])), ', '.join(filter(None, [driver, driver_annotation])) ) return ups_choices @private async def validate_data(self, data, schema): verrors = ValidationErrors() driver = data.get('driver') if driver: if driver not in (await self.middleware.call('ups.driver_choices')).keys(): verrors.add( f'{schema}.driver', 'Driver selected does not match local machine\'s driver list' ) identifier = data['identifier'] if identifier: if not re.search(r'^[a-z0-9\.\-_]+$', identifier, re.I): verrors.add( f'{schema}.identifier', 'Use alphanumeric characters, ".", "-" and "_"' ) for field in ['monpwd', 'monuser']: if not data.get(field): verrors.add(f'{schema}.{field}', 'This field is required.') elif re.search(r'[ #]', data[field], re.I): verrors.add(f'{schema}.{field}', 'Spaces or number signs are not allowed.') mode = data.get('mode') if mode == 'MASTER': for field in filter( lambda f: not data[f], ['port', 'driver'] ): verrors.add( f'{schema}.{field}', 'This field is required' ) else: if not data.get('remotehost'): verrors.add( f'{schema}.remotehost', 'This field is required' ) to_emails = data.get('toemail') if to_emails: data['toemail'] = ';'.join(to_emails) else: data['toemail'] = '' data['mode'] = data['mode'].lower() data['shutdown'] = data['shutdown'].lower() return verrors, data @accepts( Dict( 'ups_update', Bool('emailnotify'), Bool('powerdown'), Bool('rmonitor'), Int('nocommwarntime', null=True), Int('remoteport', validators=[Port()]), Int('shutdowntimer'), Int('hostsync', validators=[Range(min=0)]), Str('description'), Str('driver'), Str('extrausers', max_length=None), Str('identifier', empty=False), Str('mode', enum=['MASTER', 'SLAVE']), Str('monpwd', empty=False), Str('monuser', empty=False), Str('options', max_length=None), Str('optionsupsd', max_length=None), Str('port'), Str('remotehost'), Str('shutdown', enum=['LOWBATT', 'BATT']), Str('shutdowncmd', null=True), Str('subject'), List('toemail', items=[Str('email', validators=[Email()])]), update=True ) ) async def do_update(self, data): """ Update UPS Service Configuration. `emailnotify` when enabled, sends out notifications of different UPS events via email. `powerdown` when enabled, sets UPS to power off after shutting down the system. `nocommwarntime` is a value in seconds which makes UPS Service wait the specified seconds before alerting that the Service cannot reach configured UPS. `shutdowntimer` is a value in seconds which tells the Service to wait specified seconds for the UPS before initiating a shutdown. This only applies when `shutdown` is set to "BATT". `shutdowncmd` is the command which is executed to initiate a shutdown. It defaults to "poweroff". `toemail` is a list of valid email id's on which notification emails are sent. """ config = await self.config() config.pop('complete_identifier') old_config = config.copy() config.update(data) verros, config = await self.validate_data(config, 'ups_update') if verros: raise verros old_config['mode'] = old_config['mode'].lower() old_config['shutdown'] = old_config['shutdown'].lower() old_config['toemail'] = ';'.join(old_config['toemail']) if old_config['toemail'] else '' if len(set(old_config.items()) ^ set(config.items())) > 0: if config['identifier'] != old_config['identifier']: await self.dismiss_alerts() await self._update_service(old_config, config) return await self.config() @private async def alerts_mapping(self): return { 'LOWBATT': 'UPSBatteryLow', 'COMMBAD': 'UPSCommbad', 'COMMOK': 'UPSCommok', 'ONBATT': 'UPSOnBattery', 'ONLINE': 'UPSOnline', 'REPLBATT': 'UPSReplbatt' } @private async def dismiss_alerts(self): config = await self.config() for alert in (await self.alerts_mapping()).values(): await self.middleware.call('alert.oneshot_delete', alert, {'ups': config['identifier']}) @private @accepts( Str('notify_type') ) async def upssched_event(self, notify_type): config = await self.config() upsc_identifier = config['complete_identifier'] cp = await run('upsc', upsc_identifier, check=False) if cp.returncode: stats_output = '' self.logger.error('Failed to retrieve ups information: %s', cp.stderr.decode()) else: stats_output = cp.stdout.decode() if RE_TEST_IN_PROGRESS.search(stats_output): self.logger.debug('Self test is in progress and %r notify event should be ignored', notify_type) return if notify_type.lower() == 'shutdown': # Before we start FSD with upsmon, lets ensure that ups is not ONLINE (OL). # There are cases where battery/charger issues can result in ups.status being "OL LB" at the # same time. This will ensure that we don't initiate a shutdown if ups is OL. ups_status = RE_UPS_STATUS.findall(stats_output) if ups_status and 'ol' in ups_status[0].lower(): self.middleware.logger.debug( f'Shutdown not initiated as ups.status ({ups_status[0]}) indicates ' f'{config["identifier"]} is ONLINE (OL).' ) else: syslog.syslog(syslog.LOG_NOTICE, 'upssched-cmd "issuing shutdown"') await run('upsmon', '-c', 'fsd', check=False) elif 'notify' in notify_type.lower(): # notify_type is expected to be of the following format # NOTIFY-EVENT i.e NOTIFY-LOWBATT notify_type = notify_type.split('-')[-1] # We would like to send alerts for the following events alert_mapping = await self.alerts_mapping() await self.dismiss_alerts() if notify_type in alert_mapping: await self.middleware.call( 'alert.oneshot_create', alert_mapping[notify_type], {'ups': config['identifier']} ) if config['emailnotify']: # Email user with the notification event and details # We send the email in the following format ( inclusive line breaks ) # NOTIFICATION: 'LOWBATT' # UPS: 'ups' # # Statistics recovered: # # 1) Battery charge (percent) # battery.charge: 5 # # 2) Remaining battery level when UPS switches to LB (percent) # battery.charge.low: 10 # # 3) Battery runtime (seconds) # battery.runtime: 1860 # # 4) Remaining battery runtime when UPS switches to LB (seconds) # battery.runtime.low: 900 ups_name = config['identifier'] hostname = socket.gethostname() current_time = datetime.datetime.now(tz=dateutil.tz.tzlocal()).strftime('%a %b %d %H:%M:%S %Z %Y') ups_subject = config['subject'].replace('%d', current_time).replace('%h', hostname) body = f'NOTIFICATION: {notify_type!r}\n\nUPS: {ups_name!r}\n\n' # Let's gather following stats data_points = { 'battery.charge': 'Battery charge (percent)', 'battery.charge.low': 'Battery level remaining (percent) when UPS switches to Low Battery (LB)', 'battery.charge.status': 'Battery charge status', 'battery.runtime': 'Battery runtime (seconds)', 'battery.runtime.low': 'Battery runtime remaining (seconds) when UPS switches to Low Battery (LB)', 'battery.runtime.restart': 'Minimum battery runtime (seconds) to allow UPS restart after power-off', } stats_output = ( await run('upsc', upsc_identifier, check=False) ).stdout recovered_stats = re.findall( fr'({"|".join(data_points)}): (.*)', '' if not stats_output else stats_output.decode() ) if recovered_stats: body += 'Statistics recovered:\n\n' # recovered_stats is expected to be a list in this format # [('battery.charge', '5'), ('battery.charge.low', '10'), ('battery.runtime', '1860')] for index, stat in enumerate(recovered_stats): body += f'{index + 1}) {data_points[stat[0]]}\n {stat[0]}: {stat[1]}\n\n' else: body += 'Statistics could not be recovered\n' # Subject and body defined, send email job = await self.middleware.call( 'mail.send', { 'subject': ups_subject, 'text': body, 'to': config['toemail'] } ) await job.wait() if job.error: self.middleware.logger.debug(f'Failed to send UPS status email: {job.error}') else: self.middleware.logger.debug(f'Unrecognized UPS notification event: {notify_type}')
class RsyncTaskService(TaskPathService): share_task_type = 'Rsync' class Config: datastore = 'tasks.rsync' datastore_prefix = 'rsync_' datastore_extend = 'rsynctask.rsync_task_extend' datastore_extend_context = 'rsynctask.rsync_task_extend_context' @private async def rsync_task_extend(self, data, context): data['extra'] = shlex.split(data['extra'].replace('"', r'"\"').replace( "'", r'"\"')) for field in ('mode', 'direction'): data[field] = data[field].upper() Cron.convert_db_format_to_schedule(data) data['job'] = context['jobs'].get(data['id']) return data @private async def rsync_task_extend_context(self, extra): jobs = {} for j in await self.middleware.call("core.get_jobs", [("method", "=", "rsynctask.run")], {"order_by": ["id"]}): try: task_id = int(j["arguments"][0]) except (IndexError, ValueError): continue if task_id in jobs and jobs[task_id]["state"] == "RUNNING": continue jobs[task_id] = j return { "jobs": jobs, } @private async def validate_rsync_task(self, data, schema): verrors = ValidationErrors() # Windows users can have spaces in their usernames # http://www.freebsd.org/cgi/query-pr.cgi?pr=164808 username = data.get('user') if ' ' in username: verrors.add(f'{schema}.user', 'User names cannot have spaces') raise verrors user = None with contextlib.suppress(KeyError): user = await self.middleware.call('dscache.get_uncached_user', username) if not user: verrors.add(f'{schema}.user', f'Provided user "{username}" does not exist') raise verrors await self.validate_path_field(data, schema, verrors) remote_host = data.get('remotehost') if not remote_host: verrors.add(f'{schema}.remotehost', 'Please specify a remote host') if data.get('extra'): data['extra'] = ' '.join(data['extra']) else: data['extra'] = '' mode = data.get('mode') if not mode: verrors.add(f'{schema}.mode', 'This field is required') remote_module = data.get('remotemodule') if mode == 'MODULE' and not remote_module: verrors.add(f'{schema}.remotemodule', 'This field is required') if mode == 'SSH': remote_port = data.get('remoteport') if not remote_port: verrors.add(f'{schema}.remoteport', 'This field is required') remote_path = data.get('remotepath') if not remote_path: verrors.add(f'{schema}.remotepath', 'This field is required') search = os.path.join(user['pw_dir'], '.ssh', 'id_[edr]*') exclude_from_search = os.path.join(user['pw_dir'], '.ssh', 'id_[edr]*pub') key_files = set(glob.glob(search)) - set( glob.glob(exclude_from_search)) if not key_files: verrors.add( f'{schema}.user', 'In order to use rsync over SSH you need a user' ' with a private key (DSA/ECDSA/RSA) set up in home dir.') else: for file in glob.glob(search): if '.pub' not in file: # file holds a private key and it's permissions should be 600 if os.stat(file).st_mode & 0o077 != 0: verrors.add( f'{schema}.user', f'Permissions {oct(os.stat(file).st_mode & 0o777)} for {file} are too open. Please ' f'correct them by running chmod 600 {file}') if (data['enabled'] and data['validate_rpath'] and remote_path and remote_host and remote_port): if '@' in remote_host: remote_username, remote_host = remote_host.rsplit('@', 1) else: remote_username = username try: async with await asyncio.wait_for( asyncssh.connect(remote_host, port=remote_port, username=remote_username, client_keys=key_files, known_hosts=None), timeout=5, ) as conn: await conn.run(f'test -d {shlex.quote(remote_path)}', check=True) except asyncio.TimeoutError: verrors.add( f'{schema}.remotehost', 'SSH timeout occurred. Remote path cannot be validated.' ) except OSError as e: if e.errno == 113: verrors.add( f'{schema}.remotehost', f'Connection to the remote host {remote_host} on port {remote_port} failed.' ) else: verrors.add(f'{schema}.remotehost', e.__str__()) except asyncssh.DisconnectError as e: verrors.add( f'{schema}.remotehost', f'Disconnect Error[ error code {e.code} ] was generated when trying to ' f'communicate with remote host {remote_host} and remote user {remote_username}.' ) except asyncssh.ProcessError as e: if e.code == '1': verrors.add( f'{schema}.remotepath', 'The Remote Path you specified does not exist or is not a directory.' 'Either create one yourself on the remote machine or uncheck the ' 'validate_rpath field') else: verrors.add( f'{schema}.remotepath', f'Connection to Remote Host was successful but failed to verify ' f'Remote Path. {e.__str__()}') except asyncssh.Error as e: if e.__class__.__name__ in e.__str__(): exception_reason = e.__str__() else: exception_reason = e.__class__.__name__ + ' ' + e.__str__( ) verrors.add( f'{schema}.remotepath', f'Remote Path could not be validated. An exception was raised. {exception_reason}' ) elif data['enabled'] and data['validate_rpath']: verrors.add( f'{schema}.remotepath', 'Remote path could not be validated because of missing fields' ) data.pop('validate_rpath', None) # Keeping compatibility with legacy UI for field in ('mode', 'direction'): data[field] = data[field].lower() return verrors, data @accepts( Dict( 'rsync_task_create', Str('path', required=True, max_length=RSYNC_PATH_LIMIT), Str('user', required=True), Str('remotehost'), Int('remoteport'), Str('mode', enum=['MODULE', 'SSH'], default='MODULE'), Str('remotemodule'), Str('remotepath'), Bool('validate_rpath', default=True), Str('direction', enum=['PULL', 'PUSH'], default='PUSH'), Str('desc'), Cron( 'schedule', defaults={'minute': '00'}, ), Bool('recursive'), Bool('times'), Bool('compress'), Bool('archive'), Bool('delete'), Bool('quiet'), Bool('preserveperm'), Bool('preserveattr'), Bool('delayupdates'), List('extra', items=[Str('extra')]), Bool('enabled', default=True), register=True, )) async def do_create(self, data): """ Create a Rsync Task. See the comment in Rsyncmod about `path` length limits. `remotehost` is ip address or hostname of the remote system. If username differs on the remote host, "username@remote_host" format should be used. `mode` represents different operating mechanisms for Rsync i.e Rsync Module mode / Rsync SSH mode. `remotemodule` is the name of remote module, this attribute should be specified when `mode` is set to MODULE. `remotepath` specifies the path on the remote system. `validate_rpath` is a boolean which when sets validates the existence of the remote path. `direction` specifies if data should be PULLED or PUSHED from the remote system. `compress` when set reduces the size of the data which is to be transmitted. `archive` when set makes rsync run recursively, preserving symlinks, permissions, modification times, group, and special files. `delete` when set deletes files in the destination directory which do not exist in the source directory. `preserveperm` when set preserves original file permissions. .. examples(websocket):: Create a Rsync Task which pulls data from a remote system every 5 minutes. :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "rsynctask.create", "params": [{ "enabled": true, "schedule": { "minute": "5", "hour": "*", "dom": "*", "month": "*", "dow": "*" }, "desc": "Test rsync task", "user": "******", "mode": "MODULE", "remotehost": "[email protected]", "compress": true, "archive": true, "direction": "PULL", "path": "/mnt/vol1/rsync_dataset", "remotemodule": "remote_module1" }] } """ verrors, data = await self.validate_rsync_task(data, 'rsync_task_create') if verrors: raise verrors Cron.convert_schedule_to_db_format(data) data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self.middleware.call('service.restart', 'cron') return await self.get_instance(data['id']) @accepts(Int('id', validators=[Range(min=1)]), Patch('rsync_task_create', 'rsync_task_update', ('attr', { 'update': True }))) async def do_update(self, id, data): """ Update Rsync Task of `id`. """ old = await self.query(filters=[('id', '=', id)], options={'get': True}) old.pop('job') new = old.copy() new.update(data) verrors, data = await self.validate_rsync_task(new, 'rsync_task_update') if verrors: raise verrors Cron.convert_schedule_to_db_format(new) new.pop(self.locked_field) await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) await self.middleware.call('service.restart', 'cron') return await self.get_instance(id) @accepts(Int('id')) async def do_delete(self, id): """ Delete Rsync Task of `id`. """ res = await self.middleware.call('datastore.delete', self._config.datastore, id) await self.middleware.call('service.restart', 'cron') return res @private async def commandline(self, id): """ Helper method to generate the rsync command avoiding code duplication. """ rsync = await self._get_instance(id) path = shlex.quote(rsync['path']) line = ['/usr/local/bin/rsync'] for name, flag in ( ('archive', '-a'), ('compress', '-z'), ('delayupdates', '--delay-updates'), ('delete', '--delete-delay'), ('preserveattr', '-X'), ('preserveperm', '-p'), ('recursive', '-r'), ('times', '-t'), ): if rsync[name]: line.append(flag) if rsync['extra']: line.append(' '.join(rsync['extra'])) # Do not use username if one is specified in host field # See #5096 for more details if '@' in rsync['remotehost']: remote = rsync['remotehost'] else: remote = f'"{rsync["user"]}"@{rsync["remotehost"]}' if rsync['mode'] == 'MODULE': module_args = [path, f'{remote}::"{rsync["remotemodule"]}"'] if rsync['direction'] != 'PUSH': module_args.reverse() line += module_args else: line += [ '-e', f'"ssh -p {rsync["remoteport"]} -o BatchMode=yes -o StrictHostKeyChecking=yes"' ] path_args = [ path, f'{remote}:"{shlex.quote(rsync["remotepath"])}"' ] if rsync['direction'] != 'PUSH': path_args.reverse() line += path_args if rsync['quiet']: line += ['>', '/dev/null', '2>&1'] return ' '.join(line) @item_method @accepts(Int('id')) @job(lock=lambda args: args[-1], logs=True) def run(self, job, id): """ Job to run rsync task of `id`. Output is saved to job log excerpt (not syslog). """ rsync = self.middleware.call_sync('rsynctask.get_instance', id) if rsync['locked']: self.middleware.call_sync('rsynctask.generate_locked_alert', id) return commandline = self.middleware.call_sync('rsynctask.commandline', id) cp = run_command_with_user_context(commandline, rsync['user'], lambda v: job.logs_fd.write(v)) for klass in ('RsyncSuccess', 'RsyncFailed') if not rsync['quiet'] else (): self.middleware.call_sync('alert.oneshot_delete', klass, rsync['id']) if cp.returncode not in RsyncReturnCode.nonfatals(): if not rsync['quiet']: self.middleware.call_sync( 'alert.oneshot_create', 'RsyncFailed', { 'id': rsync['id'], 'direction': rsync['direction'], 'path': rsync['path'], }) raise CallError( f'rsync command returned {cp.returncode}. Check logs for further information.' ) elif not rsync['quiet']: self.middleware.call_sync( 'alert.oneshot_create', 'RsyncSuccess', { 'id': rsync['id'], 'direction': rsync['direction'], 'path': rsync['path'], })
class TrueNASService(Service): class Config: cli_namespace = "system.truenas" @accepts() @cli_private async def get_chassis_hardware(self): """ Returns what type of hardware this is, detected from dmidecode. TRUENAS-X10-HA-D TRUENAS-X10-S TRUENAS-X20-HA-D TRUENAS-X20-S TRUENAS-M40-HA TRUENAS-M40-S TRUENAS-M50-HA TRUENAS-M50-S TRUENAS-M60-HA TRUENAS-M60-S TRUENAS-Z20-S TRUENAS-Z20-HA-D TRUENAS-Z30-HA-D TRUENAS-Z30-S TRUENAS-Z35-HA-D TRUENAS-Z35-S TRUENAS-Z50-HA-D TRUENAS-Z50-S Nothing in dmidecode but a M, X or Z class machine: (Note this means production didn't burn the hardware model into SMBIOS. We can detect this case by looking at the motherboard) TRUENAS-M TRUENAS-X TRUENAS-Z Really NFI about hardware at this point. TrueNAS on a Dell? TRUENAS-UNKNOWN """ data = await self.middleware.call('system.dmidecode_info') chassis = data['system-product-name'] if chassis.startswith(('TRUENAS-M', 'TRUENAS-X', 'TRUENAS-Z')): return chassis # We don't match a burned in name for a M, X or Z series. Let's catch # the case where we are a M, X or Z. (shame on you production!) motherboard = data['baseboard-manufacturer'] motherboard_model = data['baseboard-product-name'] if motherboard_model == 'X11DPi-NT' or motherboard_model == 'X11SPi-TF': return 'TRUENAS-M' if motherboard_model == 'iXsystems TrueNAS X10': return 'TRUENAS-X' if motherboard == 'GIGABYTE': return 'TRUENAS-Z' # Give up return 'TRUENAS-UNKNOWN' @accepts() @cli_private def get_eula(self): """ Returns the TrueNAS End-User License Agreement (EULA). """ if not os.path.exists(EULA_FILE): return with open(EULA_FILE, 'r', encoding='utf8') as f: return f.read() @accepts() @cli_private async def is_eula_accepted(self): """ Returns whether the EULA is accepted or not. """ return not os.path.exists(EULA_PENDING_PATH) @accepts() async def accept_eula(self): """ Accept TrueNAS EULA. """ try: os.unlink(EULA_PENDING_PATH) except OSError as e: if e.errno != errno.ENOENT: raise @private async def unaccept_eula(self): with open(EULA_PENDING_PATH, "w"): pass @accepts() async def get_customer_information(self): """ Returns stored customer information. """ result = await self.__fetch_customer_information() return result @accepts(Dict( 'customer_information_update', Str('company'), Dict('administrative_user', *user_attrs), Dict('technical_user', *user_attrs), Dict( 'reseller', Str('company'), Str('first_name'), Str('last_name'), Str('title'), Str('office_phone'), Str('mobile_phone'), ), Dict( 'physical_location', Str('address'), Str('city'), Str('state'), Str('zip'), Str('country'), Str('contact_name'), Str('contact_phone_number'), Str('contact_email'), ), Str('primary_use_case'), Str('other_primary_use_case'), )) async def update_customer_information(self, data): """ Updates customer information. """ customer_information = await self.__fetch_customer_information() await self.middleware.call('datastore.update', 'truenas.customerinformation', customer_information["id"], { "data": json.dumps(data), "updated_at": datetime.utcnow(), }) return customer_information async def __fetch_customer_information(self): result = await self.middleware.call('datastore.config', 'truenas.customerinformation') result["immutable_data"] = await self.__fetch_customer_information_immutable_data() result["data"] = json.loads(result["data"]) result["needs_update"] = datetime.utcnow() - result["updated_at"] > timedelta(days=365) return result async def __fetch_customer_information_immutable_data(self): license = (await self.middleware.call('system.info'))['license'] if license is None: return None return { "serial_number": license['system_serial'], "serial_number_ha": license['system_serial_ha'], "support_level": license['contract_type'].title(), "support_start_date": license['contract_start'].isoformat(), "support_end_date": license['contract_end'].isoformat(), } @accepts() async def is_production(self): """ Returns if system is marked as production. """ return await self.middleware.call('keyvalue.get', 'truenas:production', False) @accepts(Bool('production'), Bool('attach_debug', default=False)) @job() async def set_production(self, job, production, attach_debug): """ Sets system production state and optionally sends initial debug. """ was_production = await self.is_production() await self.middleware.call('keyvalue.set', 'truenas:production', production) if not was_production and production: serial = (await self.middleware.call('system.info'))["system_serial"] return await job.wrap(await self.middleware.call('support.new_ticket', { "title": f"System has been just put into production ({serial})", "body": "This system has been just put into production", "attach_debug": attach_debug, "category": "Installation/Setup", "criticality": "Inquiry", "environment": "Production", "name": "Automatic Alert", "email": "*****@*****.**", "phone": "-", }))
class KubernetesService(ConfigService): class Config: datastore = 'services.kubernetes' datastore_extend = 'kubernetes.k8s_extend' @private async def k8s_extend(self, data): data['dataset'] = os.path.join( data['pool'], 'ix-applications') if data['pool'] else None data.pop('cni_config') return data @private async def validate_data(self, data, schema): verrors = ValidationErrors() if data['pool'] and not await self.middleware.call( 'pool.query', [['name', '=', data['pool']]]): verrors.add( f'{schema}.pool', 'Please provide a valid pool configured in the system.') if ipaddress.ip_address( data['cluster_dns_ip']) not in ipaddress.ip_network( data['service_cidr']): verrors.add(f'{schema}.cluster_dns_ip', 'Must be in range of "service_cidr".') if data['node_ip'] not in await self.bindip_choices(): verrors.add(f'{schema}.node_ip', 'Please provide a valid IP address.') for k, _ in await self.validate_interfaces(data): verrors.add(f'{schema}.{k}', 'Please specify a valid interface.') for k in ('route_v4', 'route_v6'): gateway = data[f'{k}_gateway'] interface = data[f'{k}_interface'] if (not gateway and not interface) or (gateway and interface): continue for k2 in ('gateway', 'interface'): verrors.add( f'{schema}.{k}_{k2}', f'{k}_gateway and {k}_interface must be specified together.' ) verrors.check() @private async def validate_interfaces(self, data): errors = [] interfaces = { i['name']: i for i in await self.middleware.call('interface.query') } for k in filter(lambda k: data[k] and data[k] not in interfaces, ('route_v4_interface', 'route_v6_interface')): errors.append((k, data[k])) return errors @accepts( Dict( 'kubernetes_update', Str('pool', empty=False, null=True), IPAddr('cluster_cidr', cidr=True), IPAddr('service_cidr', cidr=True), IPAddr('cluster_dns_ip'), IPAddr('node_ip'), Str('route_v4_interface', null=True), IPAddr('route_v4_gateway', null=True, v6=False), Str('route_v6_interface', null=True), IPAddr('route_v6_gateway', null=True, v4=False), update=True, )) @job(lock='kubernetes_update') async def do_update(self, job, data): """ `pool` must be a valid ZFS pool configured in the system. Kubernetes service will initialise the pool by creating datasets under `pool_name/ix-applications`. `cluster_cidr` is the CIDR to be used for default NAT network between workloads. `service_cidr` is the CIDR to be used for kubernetes services which are an abstraction and refer to a logically set of kubernetes pods. `cluster_dns_ip` is the IP of the DNS server running for the kubernetes cluster. It must be in the range of `service_cidr`. Specifying values for `cluster_cidr`, `service_cidr` and `cluster_dns_ip` are permanent and a subsequent change requires re-initialisation of the applications. To clarify, system will destroy old `ix-applications` dataset and any data within it when any of the values for the above configuration change. `node_ip` is the IP address which the kubernetes cluster will assign to the TrueNAS node. It defaults to 0.0.0.0 and the cluster in this case will automatically manage which IP address to use for managing traffic for default NAT network. By default kubernetes pods will be using default gateway of the system for outward traffic. This might not be desirable for certain users who want to separate NAT traffic over a specific interface / route. System will create a L3 network which will be routing the traffic towards default gateway for NAT. If users want to restrict traffic over a certain gateway / interface, they can specify a default route for the NAT traffic. `route_v4_interface` and `route_v4_gateway` will set a default route for the kubernetes cluster IPv4 traffic. Similarly `route_v6_interface` and 'route_v6_gateway` can be used to specify default route for IPv6 traffic. """ old_config = await self.config() old_config.pop('dataset') config = old_config.copy() config.update(data) await self.validate_data(config, 'kubernetes_update') if len(set(old_config.items()) ^ set(config.items())) > 0: config['cni_config'] = {} await self.middleware.call('datastore.update', self._config.datastore, old_config['id'], config) await self.middleware.call('kubernetes.status_change') if config['pool'] != old_config['pool']: await self.middleware.call('catalog.sync_all') return await self.config() @accepts() async def bindip_choices(self): """ Returns ip choices for Kubernetes service to use. """ return { d['address']: d['address'] for d in await self.middleware.call('interface.ip_in_use', { 'static': True, 'any': True }) } @private async def validate_k8s_setup(self): k8s_config = await self.middleware.call('kubernetes.config') if not k8s_config['dataset']: raise CallError('Please configure kubernetes pool.') if not await self.middleware.call('service.started', 'kubernetes'): raise CallError('Kubernetes service is not running.')
class ReportingService(ConfigService): class Config: datastore = 'system.reporting' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__rrds = {} for name, klass in RRD_PLUGINS.items(): self.__rrds[name] = klass(self.middleware) @accepts( Dict( 'reporting_update', Bool('cpu_in_percentage'), Str('graphite'), Int('graph_age', validators=[Range(min=1)]), Int('graph_points', validators=[Range(min=1)]), Bool('confirm_rrd_destroy'), update=True ) ) async def do_update(self, data): """ Configure Reporting Database settings. If `cpu_in_percentage` is `true`, collectd reports CPU usage in percentage instead of "jiffies". `graphite` specifies a destination hostname or IP for collectd data sent by the Graphite plugin.. `graph_age` specifies the maximum age of stored graphs in months. `graph_points` is the number of points for each hourly, daily, weekly, etc. graph. Changing these requires destroying the current reporting database, so when these fields are changed, an additional `confirm_rrd_destroy: true` flag must be present. .. examples(websocket):: Update reporting settings :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "reporting.update", "params": [{ "cpu_in_percentage": false, "graphite": "", }] } Recreate reporting database with new settings :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "reporting.update", "params": [{ "graph_age": 12, "graph_points": 1200, "confirm_rrd_destroy": true, }] } """ confirm_rrd_destroy = data.pop('confirm_rrd_destroy', False) old = await self.config() new = copy.deepcopy(old) new.update(data) verrors = ValidationErrors() destroy_database = False for k in ['graph_age', 'graph_points']: if old[k] != new[k]: destroy_database = True if not confirm_rrd_destroy: verrors.add( f'reporting_update.{k}', _('Changing this option requires destroying the reporting database. This action must be ' 'confirmed by setting confirm_rrd_destroy flag'), ) if verrors: raise verrors await self.middleware.call( 'datastore.update', self._config.datastore, old['id'], new, {'prefix': self._config.datastore_prefix} ) if destroy_database: await self.middleware.call('service.stop', 'collectd') await self.middleware.call('service.stop', 'rrdcached') await run('sh', '-c', 'rm -rfx /var/db/collectd/rrd/*', check=False) await self.middleware.call('reporting.setup') await self.middleware.call('service.start', 'rrdcached') await self.middleware.call('service.restart', 'collectd') return await self.config() @private def setup(self): systemdatasetconfig = self.middleware.call_sync('systemdataset.config') if not systemdatasetconfig['path']: self.middleware.logger.error(f'System dataset is not mounted') return False rrd_mount = f'{systemdatasetconfig["path"]}/rrd-{systemdatasetconfig["uuid"]}' if not os.path.exists(rrd_mount): self.middleware.logger.error(f'{rrd_mount} does not exist or is not a directory') return False # Ensure that collectd working path is a symlink to system dataset pwd = '/var/db/collectd/rrd' if os.path.exists(pwd) and (not os.path.isdir(pwd) or not os.path.islink(pwd)): shutil.move(pwd, f'{pwd}.{time.strftime("%Y%m%d%H%M%S")}') if not os.path.exists(pwd): os.symlink(rrd_mount, pwd) # Migrate legacy RAMDisk persist_file = '/data/rrd_dir.tar.bz2' if os.path.isfile(persist_file): with tarfile.open(persist_file) as tar: if 'collectd/rrd' in tar.getnames(): tar.extractall(pwd, get_members(tar, 'collectd/rrd/')) os.unlink('/data/rrd_dir.tar.bz2') hostname = self.middleware.call_sync('system.info')['hostname'] if not hostname: hostname = self.middleware.call_sync('network.configuration.config')['hostname_local'] # Migrate from old version, where `hostname` was a real directory and `localhost` was a symlink. # Skip the case where `hostname` is "localhost", so symlink was not (and is not) needed. if ( hostname != 'localhost' and os.path.isdir(os.path.join(pwd, hostname)) and not os.path.islink(os.path.join(pwd, hostname)) ): if os.path.exists(os.path.join(pwd, 'localhost')): if os.path.islink(os.path.join(pwd, 'localhost')): os.unlink(os.path.join(pwd, 'localhost')) else: # This should not happen, but just in case shutil.move( os.path.join(pwd, 'localhost'), os.path.join(pwd, f'localhost.bak.{time.strftime("%Y%m%d%H%M%S")}') ) shutil.move(os.path.join(pwd, hostname), os.path.join(pwd, 'localhost')) # Remove all directories except "localhost" and its backups (that may be erroneously created by # running collectd before this script) to_remove_dirs = [ os.path.join(pwd, d) for d in os.listdir(pwd) if not d.startswith('localhost') and os.path.isdir(os.path.join(pwd, d)) ] for r_dir in to_remove_dirs: subprocess.run(['rm', '-rfx', r_dir]) # Remove all symlinks (that are stale if hostname was changed). to_remove_symlinks = [ os.path.join(pwd, l) for l in os.listdir(pwd) if os.path.islink(os.path.join(pwd, l)) ] for r_symlink in to_remove_symlinks: os.unlink(r_symlink) # Create "localhost" directory if it does not exist if not os.path.exists(os.path.join(pwd, 'localhost')): os.makedirs(os.path.join(pwd, 'localhost')) # Create "${hostname}" -> "localhost" symlink if necessary if hostname != 'localhost': os.symlink(os.path.join(pwd, 'localhost'), os.path.join(pwd, hostname)) # Let's return a positive value to indicate that necessary collectd operations were performed successfully return True @filterable def graphs(self, filters, options): return filter_list([i.__getstate__() for i in self.__rrds.values()], filters, options) def __rquery_to_start_end(self, query): unit = query.get('unit') if unit: verrors = ValidationErrors() for i in ('start', 'end'): if i in query: verrors.add( f'reporting_query.{i}', f'{i!r} should only be used if "unit" attribute is not provided.', ) verrors.check() else: if 'start' not in query: unit = 'HOURLY' else: starttime = query['start'] endtime = query.get('end') or 'now' if unit: unit = unit[0].lower() page = query['page'] starttime = f'end-{page + 1}{unit}' if not page: endtime = 'now' else: endtime = f'now-{page}{unit}' return starttime, endtime @accepts( List('graphs', items=[ Dict( 'graph', Str('name', required=True), Str('identifier', default=None, null=True), ), ], empty=False), Dict( 'reporting_query', Str('unit', enum=['HOUR', 'DAY', 'WEEK', 'MONTH', 'YEAR']), Int('page', default=0), Str('start', empty=False), Str('end', empty=False), Bool('aggregate', default=True), register=True, ) ) def get_data(self, graphs, query): """ Get reporting data for given graphs. List of possible graphs can be retrieved using `reporting.graphs` call. For the time period of the graph either `unit` and `page` OR `start` and `end` should be used, not both. `aggregate` will return aggregate available data for each graph (e.g. min, max, mean). .. examples(websocket):: Get graph data of "nfsstat" from the last hour. :::javascript { "id": "6841f242-840a-11e6-a437-00e04d680384", "msg": "method", "method": "reporting.get_data", "params": [ [{"name": "nfsstat"}], {"unit": "HOURLY"}, ] } """ starttime, endtime = self.__rquery_to_start_end(query) rv = [] for i in graphs: try: rrd = self.__rrds[i['name']] except KeyError: raise CallError(f'Graph {i["name"]!r} not found.', errno.ENOENT) rv.append( rrd.export(i['identifier'], starttime, endtime, aggregate=query['aggregate']) ) return rv @private @accepts(Ref('reporting_query')) def get_all(self, query): starttime, endtime = self.__rquery_to_start_end(query) rv = [] for rrd in self.__rrds.values(): idents = rrd.get_identifiers() if idents is None: idents = [None] for ident in idents: rv.append(rrd.export(ident, starttime, endtime, aggregate=query['aggregate'])) return rv @private def get_plugin_and_rrd_types(self, name_idents): rv = [] for name, identifier in name_idents: rrd = self.__rrds[name] rv.append(((name, identifier), rrd.plugin, rrd.get_rrd_types(identifier))) return rv
class InterfaceService(Service): delegates = [] class Config: namespace_alias = "interfaces" @private def register_listen_delegate(self, delegate): self.delegates.append(delegate) @accepts() @returns( List('services_to_be_restarted', items=[ Dict( 'service_restart', Str('type', required=True), Str('service', required=True), List('ips', required=True, items=[Str('ip')]), ) ])) async def services_restarted_on_sync(self): """ Returns which services will be set to listen on 0.0.0.0 (and, thus, restarted) on sync. Example result: [ // Samba service will be set ot listen on 0.0.0.0 and restarted because it was set up to listen on // 192.168.0.1 which is being removed. {"type": "SYSTEM_SERVICE", "service": "cifs", "ips": ["192.168.0.1"]}, ] """ return [ dict(await pd.delegate.repr(pd.state), ips=pd.addresses) for pd in await self.listen_delegates_prepare() ] @private async def listen_delegates_prepare(self): original_datastores = await self.middleware.call( "interface.get_original_datastores") if not original_datastores: return [] datastores = await self.middleware.call("interface.get_datastores") old_addresses = self._collect_addresses(original_datastores) addresses = self._collect_addresses(datastores) gone_addresses = old_addresses - addresses result = [] for delegate in self.delegates: state = await delegate.get_listen_state(gone_addresses) delegate_addresses = [ address for address in gone_addresses if await delegate.listens_on(state, address) ] if delegate_addresses: result.append( PreparedDelegate(delegate, state, delegate_addresses)) return result def _collect_addresses(self, datastores): addresses = set() for iface, alias in zip_longest(datastores["interfaces"], datastores["alias"], fillvalue={}): addresses.add(iface.get("int_address", "")) addresses.add(iface.get("int_address_b", "")) addresses.add(iface.get("int_vip", "")) addresses.add(alias.get("alias_address", "")) addresses.add(alias.get("alias_address_b", "")) addresses.add(alias.get("alias_vip", "")) addresses.discard("") return addresses
class PoolDatasetService(Service): class Config: namespace = 'pool.dataset' event_send = False @accepts(Str('dataset')) async def unlock_services_restart_choices(self, dataset): """ Get a mapping of services identifiers and labels that can be restart on dataset unlock. """ await self.middleware.call('pool.dataset.get_instance', dataset) services = { 'afp': 'AFP', 'cifs': 'SMB', 'ftp': 'FTP', 'iscsitarget': 'iSCSI', 'nfs': 'NFS', 'webdav': 'WebDAV', } result = {} for k, v in services.items(): service = await self.middleware.call('service.query', [['service', '=', k]], {'get': True}) if service['enable'] or service['state'] == 'RUNNING': result[k] = v check_services = {'kubernetes': 'Applications', 's3': 'S3', **services} result.update({ k: check_services[k] for k in map( lambda a: a['service'], await self.middleware.call( 'pool.dataset.attachments', dataset)) if k in check_services }) if osc.IS_FREEBSD: try: activated_pool = await self.middleware.call( 'jail.get_activated_pool') except Exception: activated_pool = None # If iocage is not activated yet, there is a chance that this pool might have it activated there if activated_pool is None: result['jails'] = 'Jails/Plugins' if await self.unlock_restarted_vms(dataset): result['vms'] = 'Virtual Machines' return result @private async def unlock_restarted_vms(self, dataset_name): result = [] for vm in await self.middleware.call('vm.query', [('autostart', '=', True)]): for device in vm['devices']: if device['dtype'] not in ('DISK', 'RAW'): continue path = device['attributes'].get('path') if not path: continue if path.startswith( f'/dev/zvol/{dataset_name}/') or path.startswith( f'/mnt/{dataset_name}/'): result.append(vm) break return result @private async def restart_vms_after_unlock(self, dataset_name): for vm in await self.unlock_restarted_vms(dataset_name): if await self.middleware.call('vm.status', vm['id'])['state'] == 'RUNNING': stop_job = await self.middleware.call('vm.stop', vm['id']) await stop_job.wait() if stop_job.error: self.logger.error('Failed to stop %r VM: %s', vm['name'], stop_job.error) try: self.middleware.call_sync('vm.start', vm['id']) except Exception: self.logger.error('Failed to start %r VM after %r unlock', vm['name'], dataset_name, exc_info=True) @private async def restart_services_after_unlock(self, dataset_name, services_to_restart): try: to_restart = [[i] for i in set(services_to_restart) - {'jails', 'vms'}] if not to_restart: return restart_job = await self.middleware.call('core.bulk', 'service.restart', to_restart) statuses = await restart_job.wait() for idx, srv_status in enumerate(statuses): if srv_status['error']: self.logger.error( 'Failed to restart %r service after %r unlock: %s', to_restart[idx], dataset_name, srv_status['error']) if 'jails' in services_to_restart: await self.middleware.call('jail.rc_action', ['RESTART']) if 'vms' in services_to_restart: await self.middleware.call( 'pool.dataset.restart_vms_after_unlock', dataset_name) except Exception: self.logger.error( 'Failed to restart %r services after %r unlock', ', '.join(services_to_restart), id, exc_info=True, )
class AuthService(Service): def __init__(self, *args, **kwargs): super(AuthService, self).__init__(*args, **kwargs) self.authtokens = AuthTokens() @accepts(Str('username'), Str('password')) def check_user(self, username, password): """ Verify username and password """ if username != 'root': return False try: user = self.middleware.call('datastore.query', 'account.bsdusers', [('bsdusr_username', '=', username)], {'get': True}) except IndexError: return False if user['bsdusr_unixhash'] in ('x', '*'): return False return crypt.crypt(password, user['bsdusr_unixhash']) == user['bsdusr_unixhash'] @accepts(Int('ttl', required=False)) def generate_token(self, ttl=None): """Generate a token to be used for authentication.""" if ttl is None: ttl = 600 return self.authtokens.new(ttl)['id'] @no_auth_required @accepts(Str('username'), Str('password')) @pass_app def login(self, app, username, password): """Authenticate session using username and password. Currently only root user is allowed. """ valid = self.check_user(username, password) if valid: app.authenticated = True return valid @no_auth_required @accepts(Str('token')) @pass_app def token(self, app, token): """Authenticate using a given `token` id.""" def update_token(app, message): """ On every new message from the registered connection make sure the token is still valid, updating last time or removing authentication """ token = self.authtokens.get_token_by_sessionid(app.sessionid) if token is None: return if int(time.time()) - token['ttl'] < token['last']: token['last'] = int(time.time()) else: self.authtokens.pop_token(token['id']) app.authenticated = False def remove_session(app): """ On connection close, remove session id from token """ self.authtokens.remove_session(app.sessionid) token = self.authtokens.get_token(token) if token is None: return False """ If token exists and is still valid (TTL) do the following: - authenticate the connection - add the session id to token - register connection callbacks to update/remove token """ if int(time.time()) - token['ttl'] < token['last']: token['last'] = int(time.time()) self.authtokens.add_session(app.sessionid, token) app.register_callback('on_message', update_token) app.register_callback('on_close', remove_session) app.authenticated = True return True else: self.authtokens.pop_token(token['id']) return False
class ReplicationService(Service): @private def ssh_keyscan(self, host, port): proc = Popen([ "/usr/bin/ssh-keyscan", "-p", str(port), "-T", "2", str(host), ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) key, errmsg = proc.communicate() if proc.returncode != 0 or not key: if not errmsg: errmsg = 'ssh key scan failed for unknown reason' raise ValueError(errmsg) return key @private @accepts(Dict( 'replication-pair-data', Str('hostname', required=True), Str('public-key', required=True), Str('user'), )) def pair(self, data): """ Receives public key, storing it to accept SSH connection and return pertinent SSH data of this machine. """ service = self.middleware.call('datastore.query', 'services.services', [('srv_service', '=', 'ssh')], {'get': True}) ssh = self.middleware.call('datastore.query', 'services.ssh', None, {'get': True}) try: user = self.middleware.call('datastore.query', 'account.bsdusers', [('bsdusr_username', '=', data.get('user') or 'root')], {'get': True}) except IndexError: raise ValueError('User "{}" does not exist'.format(data.get('user'))) # Make sure SSH is enabled if not service['srv_enable']: self.middleware.call('datastore.update', 'services.services', service['id'], {'srv_enable': True}) self.middleware.call('notifier.start', 'ssh') # This might be the first time of the service being enabled # which will then result in new host keys we need to grab ssh = self.middleware.call('datastore.query', 'services.ssh', None, {'get': True}) if not os.path.exists(user['bsdusr_home'].encode('utf8')): raise ValueError('Homedir {} does not exist'.format(user['bsdusr_home'])) # Write public key in user authorized_keys for SSH authorized_keys_file = '{}/.ssh/authorized_keys'.format(user['bsdusr_home'].encode('utf8')) with open(authorized_keys_file, 'a+') as f: f.seek(0) if data['public-key'] not in f.read(): f.write('\n' + data['public-key']) ssh_hostkey = '{0} {1}\n{0} {2}\n{0} {3}\n'.format( data['hostname'], base64.b64decode(ssh['ssh_host_rsa_key_pub']), base64.b64decode(ssh['ssh_host_ecdsa_key_pub']), base64.b64decode(ssh['ssh_host_ed25519_key_pub']), ) return { 'ssh_port': ssh['ssh_tcpport'], 'ssh_hostkey': ssh_hostkey, }
class ConsulService(Service): INFLUXDB_API = [ 'host', 'username', 'password', 'database', 'series-name', 'enabled' ] SLACK_API = [ 'cluster-name', 'url', 'channel', 'username', 'icon-url', 'detailed', 'enabled' ] MATTERMOST_API = [ 'cluster', 'url', 'username', 'password', 'team', 'channel', 'enabled' ] PAGERDUTY_API = ['service-key', 'client-name', 'enabled'] HIPCHAT_API = [ 'from', 'cluster-name', 'base-url', 'room-id', 'auth-token', 'enabled' ] OPSGENIE_API = ['cluster-name', 'api-key', 'enabled'] AWSSNS_API = ['reigion', 'topic-arn', 'enabled'] VICTOROPS_API = ['api-key', 'routing-key', 'enabled'] @accepts(Str('key'), Any('value')) def set_kv(self, key, value): """ Sets `key` with `value` in Consul KV. Returns: bool: True if it added successful the value or otherwise False. """ c = consul.Consul() return c.kv.put(str(key), str(value)) @accepts(Str('key')) def get_kv(self, key): """ Gets value of `key` in Consul KV. Returns: str: Return the value or an empty string. """ c = consul.Consul() index = None index, data = c.kv.get(key, index=index) if data is not None: return data['Value'].decode("utf-8") else: return "" @accepts(Str('key')) def delete_kv(self, key): """ Delete a `key` in Consul KV. Returns: bool: True if it could delete the data or otherwise False. """ c = consul.Consul() return c.kv.delete(str(key)) def _convert_keys(self, data): """ Transforms key values that contains "_" to values with "-" Returns: dict: With the values on keys using "-". """ for key in list(data.keys()): new_key = key.replace("_", "-") if new_key != key: data[new_key] = data[key] del data[key] return data def _api_keywords(self, api_list, data): """ Helper to convert the API list into a dict. Returns: dict: With the API_LIST. """ new_dict = {k: data.get(k, None) for k in api_list} return new_dict def _insert_keys(self, prefix, data, api_keywords): """ Helper to insert keys into consul. Note: because 'from' is a reserved word in Python, we can't use it directly and instead we use hfrom and convert it later. """ new_dict = self._api_keywords(api_keywords, data) for k, v in list(new_dict.items()): if k == 'hfrom': k = 'from' self.set_kv(prefix + k, v) def _delete_keys(self, prefix, data, api_keywords): """ Helper to delete keys into consul. Note: The same applies for 'from' like explained on _insert_keys(). """ new_dict = self._api_keywords(api_keywords, data) for k in list(new_dict.keys()): if k == 'hfrom': k = 'from' self.delete_kv(prefix + k) def do_create(self, data): """ Helper to insert keys into consul based on the service API. """ consul_prefix = 'consul-alerts/config/notifiers/' cdata = self._convert_keys(data) alert_service = data.pop('consulalert-type') consul_prefix = consul_prefix + alert_service.lower() + '/' if alert_service == 'InfluxDB': self._insert_keys(consul_prefix, cdata, self.INFLUXDB_API) elif alert_service == 'Slack': self._insert_keys(consul_prefix, cdata, self.SLACK_API) elif alert_service == 'Mattermost': self._insert_keys(consul_prefix, cdata, self.MATTERMOST_API) elif alert_service == 'PagerDuty': self._insert_keys(consul_prefix, cdata, self.PAGERDUTY_API) elif alert_service == 'HipChat': self._insert_keys(consul_prefix, cdata, self.HIPCHAT_API) elif alert_service == 'OpsGenie': self._insert_keys(consul_prefix, cdata, self.OPSGENIE_API) elif alert_service == 'AWS-SNS': self._insert_keys(consul_prefix, cdata, self.AWSSNS_API) elif alert_service == 'VictorOps': self._insert_keys(consul_prefix, cdata, self.VICTOROPS_API) def do_delete(self, alert_service, data): """ Helper to delete the keys from consul based on the service API. """ consul_prefix = 'consul-alerts/config/notifiers/' + alert_service.lower( ) + '/' cdata = self._convert_keys(data) if alert_service == 'InfluxDB': self._delete_keys(consul_prefix, cdata, self.INFLUXDB_API) elif alert_service == 'Slack': self._delete_keys(consul_prefix, cdata, self.SLACK_API) elif alert_service == 'Mattermost': self._delete_keys(consul_prefix, cdata, self.MATTERMOST_API) elif alert_service == 'PagerDuty': self._delete_keys(consul_prefix, cdata, self.PAGERDUTY_API) elif alert_service == 'HipChat': self._delete_keys(consul_prefix, cdata, self.HIPCHAT_API) elif alert_service == 'OpsGenie': self._delete_keys(consul_prefix, cdata, self.OPSGENIE_API) elif alert_service == 'AWS-SNS': self._delete_keys(consul_prefix, cdata, self.AWSSNS_API) elif alert_service == 'VictorOps': self._delete_keys(consul_prefix, cdata, self.VICTOROPS_API)
class DiskService(Service): @private async def destroy_partitions(self, disk): if osc.IS_LINUX: await run(['sgdisk', '-Z', os.path.join('/dev', disk)]) else: await run('gpart', 'destroy', '-F', f'/dev/{disk}', check=False) # Wipe out the partition table by doing an additional iterate of create/destroy await run('gpart', 'create', '-s', 'gpt', f'/dev/{disk}') await run('gpart', 'destroy', '-F', f'/dev/{disk}') @private async def wipe_quick(self, dev, size=None): # If the size is too small, lets just skip it for now. # In the future we can adjust dd size if size and size < 33554432: return await run('dd', 'if=/dev/zero', f'of=/dev/{dev}', 'bs=1M', 'count=32') size = await self.middleware.call('disk.get_dev_size', dev) if not size: self.logger.error(f'Unable to determine size of {dev}') else: # This will fail when EOL is reached await run( 'dd', 'if=/dev/zero', f'of=/dev/{dev}', 'bs=1M', f'oseek={int(size / (1024*1024)) - 32}', check=False ) @accepts( Str('dev'), Str('mode', enum=['QUICK', 'FULL', 'FULL_RANDOM'], required=True), Bool('synccache', default=True), Ref('swap_removal_options'), ) @job(lock=lambda args: args[0]) async def wipe(self, job, dev, mode, sync, options=None): """ Performs a wipe of a disk `dev`. It can be of the following modes: - QUICK: clean the first few and last megabytes of every partition and disk - FULL: write whole disk with zero's - FULL_RANDOM: write whole disk with random bytes """ await self.middleware.call('disk.swaps_remove_disks', [dev], options) if osc.IS_FREEBSD: await self.middleware.call('disk.remove_disk_from_graid', dev) # First do a quick wipe of every partition to clean things like zfs labels if mode == 'QUICK': for part in await self.middleware.call('disk.list_partitions', dev): await self.wipe_quick(part['name'], part['size']) await self.middleware.call('disk.destroy_partitions', dev) if mode == 'QUICK': await self.wipe_quick(dev) else: size = await self.middleware.call('disk.get_dev_size', dev) or 1 proc = await Popen([ 'dd', 'if=/dev/{}'.format('zero' if mode == 'FULL' else 'random'), f'of=/dev/{dev}', 'bs=1M', ], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) async def dd_wait(): while True: if proc.returncode is not None: break os.kill(proc.pid, signal.SIGUSR1 if osc.IS_LINUX else signal.SIGINFO) await asyncio.sleep(1) asyncio.ensure_future(dd_wait()) while True: line = await proc.stderr.readline() if line == b'': break line = line.decode() reg = RE_DD.search(line) if reg: speed = float(reg.group(2)) if osc.IS_LINUX else int(reg.group(2)) if osc.IS_LINUX: mapping = {'gb': 1024 * 1024 * 1024, 'mb': 1024 * 1024, 'kb': 1024, 'b': 1} speed = int(speed * mapping[reg.group(3).lower()]) job.set_progress((int(reg.group(1)) / size) * 100, extra={'speed': speed}) if sync: await self.middleware.call('disk.sync', dev)
class IdmapService(Service): class Config: private = False namespace = 'idmap' @accepts(Str('domain')) async def get_or_create_idmap_by_domain(self, domain): """ Returns idmap settings based on pre-windows 2000 domain name (workgroup) If mapping exists, but there's no corresponding entry in the specified idmap table, then we generate a new one with the next available block of ids and return it. """ my_domain = await self.middleware.call('idmap.domaintobackend.query', [('domain', '=', domain)]) if not my_domain: raise CallError( f'No domain to idmap backend exists for domain [{domain}]', errno.ENOENT) backend_entry = await self.middleware.call( f'idmap.{my_domain[0]["idmap_backend"].lower()}.query', [('domain.idmap_domain_name', '=', domain)]) if backend_entry: return backend_entry[0] next_idmap_range = await self.get_next_idmap_range() new_idmap = await self.middleware.call( f'idmap.{my_domain[0]["idmap_backend"].lower()}.create', { 'domain': { 'id': my_domain[0]['id'] }, 'range_low': next_idmap_range[0], 'range_high': next_idmap_range[1] }) return new_idmap @private async def idmap_domain_choices(self): choices = [] domains = await self.middleware.call('idmap.domain.query') for domain in domains: choices.append(domain['name']) return choices @private async def get_idmap_legacy(self, obj_type, idmap_type): """ This is compatibility shim for legacy idmap code utils.get_idmap() obj_type is dstype. idmap_type is the idmap backend If the we don't have a corresponding entry in the idmap backend table, automatically generate one. """ idmap_type = idmap_type.lower() if idmap_type in ['adex', 'hash']: raise CallError(f'idmap backend {idmap_type} has been deprecated') ds_type = dstype(int(obj_type)).name if ds_type not in [ 'DS_TYPE_ACTIVEDIRECTORY', 'DS_TYPE_LDAP', 'DS_TYPE_DEFAULT_DOMAIN' ]: raise CallError(f'idmap backends are not supported for {ds_type}') res = await self.middleware.call( f'idmap.{idmap_type}.query', [('domain.idmap_domain_name', '=', ds_type)]) if res: return { 'idmap_id': res[0]['id'], 'idmap_type': idmap_type, 'idmap_name': idmap_type } next_idmap_range = await self.get_next_idmap_range() new_idmap = await self.middleware.call( f'idmap.{idmap_type}.create', { 'domain': { 'id': obj_type }, 'range_low': next_idmap_range[0], 'range_high': next_idmap_range[1] }) return { 'idmap_id': new_idmap['id'], 'idmap_type': idmap_type, 'idmap_name': idmap_type } @private async def common_backend_extend(self, data): for key in ['ldap_server', 'schema_mode', 'ssl']: if key in data and data[key] is not None: data[key] = data[key].upper() return data @private async def common_backend_compress(self, data): for key in ['ldap_server', 'schema_mode', 'ssl']: if key in data and data[key] is not None: data[key] = data[key].lower() if 'id' in data['domain'] and data['domain']['id']: data['domain'] = data['domain']['id'] elif 'idmap_domain_name' in data['domain'] and data['domain'][ 'idmap_domain_name']: domain_info = await self.middleware.call( 'idmap.domain.query', [('domain', '=', data['domain']['idmap_domain_name'])]) data['domain'] = domain_info[0]['id'] else: domain_info = await self.middleware.call( 'idmap.domain.query', [('domain', '=', data['domain']['idmap_domain_dns_domain_name'].upper())]) data['domain'] = domain_info[0]['id'] return data @private async def _common_validate(self, data): """ Common validation checks for all idmap backends. """ verrors = ValidationErrors() if data['range_high'] < data['range_low']: verrors.add( f'idmap_range', 'Idmap high range must be greater than idmap low range') return verrors configured_domains = await self.get_configured_idmap_domains() new_range = range(data['range_low'], data['range_high']) for i in configured_domains: if i['domain']['id'] == data['domain']: continue existing_range = range(i['backend_data']['range_low'], i['backend_data']['range_high']) if range(max(existing_range[0], new_range[0]), min(existing_range[-1], new_range[-1]) + 1): verrors.add( f'idmap_range', f'new idmap range conflicts with existing range for domain [{i["domain"]["idmap_domain_name"]}]' ) return verrors @accepts() async def get_configured_idmap_domains(self): """ returns list of all configured idmap domains. A configured domain is one that exists in the domaintobackend table and has a corresponding backend configured in the respective idmap_{backend} table. List is sorted based in ascending order based on the id range. """ domains = await self.middleware.call('idmap.domaintobackend.query') configured_domains = [] for domain in domains: b = await self.middleware.call( f'idmap.{domain["idmap_backend"].lower()}.query', [('domain.idmap_domain_name', '=', domain['domain']['idmap_domain_name'])]) for entry in b: entry.pop('domain') entry.pop('id') domain.update({'backend_data': entry}) configured_domains.append(domain) return sorted(configured_domains, key=lambda domain: domain['backend_data']['range_high']) @private async def get_next_idmap_range(self): """ Increment next high range by 100,000,000 ids. This number has to accomodate the highest available rid value for a domain. Configured idmap ranges _must_ not overlap. """ sorted_idmaps = await self.get_configured_idmap_domains() low_range = sorted_idmaps[-1]['backend_data']['range_high'] + 1 high_range = sorted_idmaps[-1]['backend_data']['range_high'] + 100000000 return (low_range, high_range) @accepts() async def clear_idmap_cache(self): """ Stop samba, remove the winbindd_cache.tdb file, start samba, flush samba's cache. This should be performed after finalizing idmap changes. """ await self.middleware.call('service.stop', 'smb') try: os.remove('/var/db/system/samba4/winbindd_cache.tdb') except Exception as e: self.logger.debug("Failed to remove winbindd_cache.tdb: %s" % e) await self.middleware.call('service.start', 'cifs') gencache_flush = await run(['net', 'cache', 'flush'], check=False) if gencache_flush.returncode != 0: raise CallError( f'Attempt to flush gencache failed with error: {gencache_flush.stderr.decode().strip()}' ) @private async def autodiscover_trusted_domains(self): smb = await self.middleware.call('smb.config') wbinfo = await run(['/usr/local/bin/wbinfo', '-m', '--verbose'], check=False) if wbinfo.returncode != 0: raise CallError( f'wbinfo -m failed with error: {wbinfo.stderr.decode().strip()}' ) for entry in wbinfo.stdout.decode().splitlines(): c = entry.split() if len(c) == 6 and c[0] != smb['workgroup']: await self.middleware.call('idmap.domain.create', { 'name': c[0], 'dns_domain_name': c[1] })
class RsyncModService(SharingService): share_task_type = 'Rsync Module' class Config: datastore = 'services.rsyncmod' datastore_prefix = 'rsyncmod_' datastore_extend = 'rsyncmod.rsync_mod_extend' @private async def rsync_mod_extend(self, data): data['hostsallow'] = data['hostsallow'].split() data['hostsdeny'] = data['hostsdeny'].split() data['mode'] = data['mode'].upper() return data @private async def common_validation(self, data, schema_name): verrors = ValidationErrors() await self.validate_path_field(data, schema_name, verrors) for entity in ('user', 'group'): value = data.get(entity) try: await self.middleware.call(f'{entity}.get_{entity}_obj', {f'{entity}name': value}) except Exception: verrors.add(f'{schema_name}.{entity}', f'Please specify a valid {entity}') verrors.check() data['hostsallow'] = ' '.join(data['hostsallow']) data['hostsdeny'] = ' '.join(data['hostsdeny']) data['mode'] = data['mode'].lower() return data @accepts( Dict( 'rsyncmod_create', Bool('enabled', default=True), Str('name', validators=[Match(r'[^/\]]')]), Str('comment'), Str('path', required=True, max_length=RSYNC_PATH_LIMIT), Str('mode', enum=['RO', 'RW', 'WO']), Int('maxconn'), Str('user', default='nobody'), Str('group', default='nobody'), List('hostsallow', items=[Str('hostsallow')], default=[]), List('hostsdeny', items=[Str('hostdeny')], default=[]), Str('auxiliary', max_length=None), register=True, )) async def do_create(self, data): """ Create a Rsyncmod module. `path` represents the path to a dataset. Path length is limited to 1023 characters maximum as per the limit enforced by FreeBSD. It is possible that we reach this max length recursively while transferring data. In that case, the user must ensure the maximum path will not be too long or modify the recursed path to shorter than the limit. `maxconn` is an integer value representing the maximum number of simultaneous connections. Zero represents unlimited. `hostsallow` is a list of patterns to match hostname/ip address of a connecting client. If list is empty, all hosts are allowed. `hostsdeny` is a list of patterns to match hostname/ip address of a connecting client. If the pattern is matched, access is denied to the client. If no client should be denied, this should be left empty. `auxiliary` attribute can be used to pass on any additional parameters from rsyncd.conf(5). """ data = await self.common_validation(data, 'rsyncmod_create') data['id'] = await self.middleware.call( 'datastore.insert', self._config.datastore, data, {'prefix': self._config.datastore_prefix}) await self._service_change('rsync', 'reload') return await self._get_instance(data['id']) @accepts(Int('id'), Patch('rsyncmod_create', 'rsyncmod_update', ('attr', { 'update': True }))) async def do_update(self, id, data): """ Update Rsyncmod module of `id`. """ module = await self.get_instance(id) module.update(data) module = await self.common_validation(module, 'rsyncmod_update') module.pop(self.locked_field) await self.middleware.call('datastore.update', self._config.datastore, id, module, {'prefix': self._config.datastore_prefix}) await self._service_change('rsync', 'reload') return await self.get_instance(id) @accepts(Int('id')) async def do_delete(self, id): """ Delete Rsyncmod module of `id`. """ return await self.middleware.call('datastore.delete', self._config.datastore, id)
class IdmapDomainService(CRUDService): class Config: datastore = 'directoryservice.idmap_domain' datastore_prefix = 'idmap_domain_' namespace = 'idmap.domain' @accepts( Dict('idmap_domain_create', Str('name', required=True), Str('DNS_domain_name'), register=True)) async def do_create(self, data): """ Create a new IDMAP domain. These domains must be unique. This table will be automatically populated after joining an Active Directory domain if "allow trusted domains" is set to True in the AD service configuration. There are three default system domains: DS_TYPE_ACTIVEDIRECTORY, DS_TYPE_LDAP, DS_TYPE_DEFAULT_DOMAIN. The system domains correspond with the idmap settings under Active Directory, LDAP, and SMB respectively. `name` the pre-windows 2000 domain name. `DNS_domain_name` DNS name of the domain. """ data["id"] = await self.middleware.call( "datastore.insert", self._config.datastore, data, {"prefix": self._config.datastore_prefix}, ) return await self._get_instance(data['id']) @accepts(Int('id', required=True), Patch("idmap_domain_create", "idmap_domain_update", ("attr", { "update": True }))) async def do_update(self, id, data): """ Update a domain by id. """ old = await self._get_instance(id) new = old.copy() new.update(data) verrors = ValidationErrors() verrors.add_child('idmap_domain_update', await self._validate(new)) if verrors: raise verrors await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) return await self._get_instance(id) @accepts(Int('id')) async def do_delete(self, id): """ Delete a domain by id. Deletion of default system domains is not permitted. """ if id <= 5: entry = await self._get_instance(id) raise CallError( f'Deleting system idmap domain [{entry["name"]}] is not permitted.', errno.EPERM) await self.middleware.call("datastore.delete", self._config.datastore, id) @private async def _validate(self, data): verrors = ValidationErrors() if data['id'] <= dstype['DS_TYPE_DEFAULT_DOMAIN'].value: verrors.add( 'id', f'Modifying system idmap domain [{data["name"]}] is not permitted.' ) return verrors
# without the express permission of iXsystems. from datetime import datetime, timedelta import errno import json import os from middlewared.schema import accepts, Bool, Dict, Str from middlewared.service import cli_private, job, private, Service import middlewared.sqlalchemy as sa EULA_FILE = '/usr/local/share/truenas/eula.html' EULA_PENDING_PATH = "/data/truenas-eula-pending" user_attrs = [ Str('first_name'), Str('last_name'), Str('title'), Str('office_phone'), Str('mobile_phone'), Str('primary_email'), Str('secondary_email'), Str('address'), Str('city'), Str('state'), Str('zip'), Str('country'), ] class TruenasCustomerInformationModel(sa.Model):
class IdmapDomainBackendService(CRUDService): class Config: datastore = 'directoryservice.idmap_domaintobackend' datastore_prefix = 'idmap_dtb_' namespace = 'idmap.domaintobackend' @accepts( Dict('idmap_domaintobackend_create', Dict( 'domain', Int('id'), Str('idmap_domain_name'), Str('idmap_domain_dns_domain_name'), ), Str('idmap_backend', enum=[ 'AD', 'AUTORID', 'FRUIT', 'LDAP', 'NSS', 'RFC2307', 'RID', 'SCRIPT', 'TDB' ]), register=True)) async def do_create(self, data): """ Set an idmap backend for a domain. `domain` dictionary containing domain information. Has one-to-one relationship with idmap_domain entries. `idmap_backed` type of idmap backend to use for the domain. Create entry for domain in the respective idmap backend table if one does not exist. """ data = await self.middleware.call('idmap.common_backend_compress', data) verrors = ValidationErrors() if data['domain'] in [ dstype.DS_TYPE_LDAP.value, dstype.DS_TYPE_DEFAULT_DOMAIN.value ]: if data['idmap_backend'] not in ['ldap', 'tdb']: verrors.add( 'domaintobackend_create.idmap_backend', f'idmap backend [{data["idmap_backend"]}] is not appropriate for the system domain type {dstype[data["domain"]]}' ) if verrors: raise verrors backend_entry_is_present = False idmap_data = await self.middleware.call( f'idmap.{data["idmap_backend"]}.query') for i in idmap_data: if not i['domain']: continue if i['domain']['idmap_domain_name'] == data['domain'][ 'idmap_domain_name']: backend_entry_is_present = True break if not backend_entry_is_present: next_idmap_range = await self.middleware.call( 'idmap.get_next_idmap_range') await self.middleware.call( f'idmap.{data["idmap_backend"]}.create', { 'domain': { 'id': data['domain']['id'] }, 'range_low': next_idmap_range[0], 'range_high': next_idmap_range[1] }) data["id"] = await self.middleware.call( "datastore.insert", self._config.datastore, data, {"prefix": self._config.datastore_prefix}, ) return await self._get_instance(data['id']) @accepts(Int('id', required=True), Patch("idmap_domaintobackend_create", "idmap_domaintobackend_update", ("attr", { "update": True }))) async def do_update(self, id, data): """ Update idmap to backend mapping by id. """ old = await self._get_instance(id) new = old.copy() new.update(data) new = await self.middleware.call('idmap.common_backend_compress', new) verrors = ValidationErrors() if new['domain'] in [ dstype.DS_TYPE_LDAP.value, dstype.DS_TYPE_DEFAULT_DOMAIN.value ]: if new['idmap_backend'] not in ['ldap', 'tdb']: verrors.add( 'domaintobackend_create.idmap_backend', f'idmap backend [{new["idmap_backend"]}] is not appropriate for the system domain type {dstype[new["domain"]]}' ) if verrors: raise verrors await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) updated_entry = await self._get_instance(id) try: await self.middleware.call('idmap.get_or_create_idmap_by_domain', updated_entry['domain']['domain_name']) except Exception as e: self.logger.debug('Failed to generate new idmap backend: %s', e) return updated_entry @accepts(Int('id')) async def do_delete(self, id): """ Delete idmap to backend mapping by id """ entry = await self._get_instance(id) if entry['domain']['id'] <= dstype['DS_TYPE_DEFAULT_DOMAIN'].value: raise CallError( f'Deleting mapping for [{entry["domain"]["idmap_domain_name"]}] is not permitted.', errno.EPERM) await self.middleware.call("datastore.delete", self._config.datastore, id)
class DockerImagesService(CRUDService): class Config: namespace = 'docker.images' @filterable async def query(self, filters=None, options=None): results = [] if not await self.middleware.call('service.started', 'docker'): return results async with aiodocker.Docker() as docker: for image in await docker.images.list(): repo_tags = image['RepoTags'] or [] results.append({ 'id': image['Id'], 'labels': image['Labels'], 'repo_tags': repo_tags, 'size': image['Size'], 'created': datetime.fromtimestamp(int(image['Created'])), 'dangling': len(repo_tags) == 1 and repo_tags[0] == '<none>:<none>', }) return filter_list(results, filters, options) @accepts( Dict( 'image_pull', Dict( 'docker_authentication', Str('username', required=True), Str('password', required=True), default=None, null=True, ), Str('from_image', required=True), Str('tag', default=None, null=True), )) async def pull(self, data): """ `from_image` is the name of the image to pull. Format for the name is "registry/repo/image" where registry may be omitted and it will default to docker registry in this case. `tag` specifies tag of the image and defaults to `null`. In case of `null` it will retrieve all the tags of the image. `docker_authentication` should be specified if image to be retrieved is under a private repository. """ await self.docker_checks() async with aiodocker.Docker() as docker: try: response = await docker.images.pull( from_image=data['from_image'], tag=data['tag'], auth=data['docker_authentication']) except aiodocker.DockerError as e: raise CallError(f'Failed to pull image: {e.message}') return response @accepts(Str('id'), Dict( 'options', Bool('force', default=False), )) async def do_delete(self, id, options): """ `options.force` should be used to force delete an image even if it's in use by a stopped container. """ await self.docker_checks() async with aiodocker.Docker() as docker: await docker.images.delete(name=id, force=options['force']) @private async def load_images_from_file(self, path): await self.docker_checks() if not os.path.exists(path): raise CallError(f'"{path}" path does not exist.', errno=errno.ENOENT) resp = [] async with aiodocker.Docker() as client: with open(path, 'rb') as f: async for i in client.images.import_image(data=f, stream=True): if 'error' in i: raise CallError( f'Unable to load images from file: {i["error"]}') else: resp.append(i) return resp @private async def load_default_images(self): await self.load_images_from_file(DEFAULT_DOCKER_IMAGES_PATH) @private async def docker_checks(self): if not await self.middleware.call('service.started', 'docker'): raise CallError('Docker service is not running')
class IdmapADService(CRUDService): class Config: datastore = 'directoryservice.idmap_ad' datastore_prefix = 'idmap_ad_' datastore_extend = 'idmap.common_backend_extend' namespace = 'idmap.ad' @accepts( Dict('idmap_ad_create', Dict( 'domain', Int('id'), Str('idmap_domain_name'), Str('idmap_domain_dns_domain_name'), ), Int('range_low', required=True, validators=[Range(min=1000, max=2147483647)]), Int('range_high', required=True, validators=[Range(min=1000, max=2147483647)]), Str('schema_mode', default='RFC2307', enum=['RFC2307', 'SFU', 'SFU20']), Bool('unix_primary_group', default=False), Bool('unix_nss_info', default=False), register=True)) async def do_create(self, data): """ Create an entry in the idmap backend table. `unix_primary_group` If True, the primary group membership is fetched from the LDAP attributes (gidNumber). If False, the primary group membership is calculated via the "primaryGroupID" LDAP attribute. `unix_nss_info` if True winbind will retrieve the login shell and home directory from the LDAP attributes. If False or if the AD LDAP entry lacks the SFU attributes the smb4.conf parameters `template shell` and `template homedir` are used. `schema_mode` Defines the schema that idmap_ad should use when querying Active Directory regarding user and group information. This can be either the RFC2307 schema support included in Windows 2003 R2 or the Service for Unix (SFU) schema. For SFU 3.0 or 3.5 please choose "SFU", for SFU 2.0 please choose "SFU20". The behavior of primary group membership is controlled by the unix_primary_group option. """ verrors = ValidationErrors() data = await self.middleware.call('idmap.common_backend_compress', data) verrors.add_child( 'idmap_ad_create', await self.middleware.call('idmap._common_validate', data)) if verrors: raise verrors data["id"] = await self.middleware.call( "datastore.insert", self._config.datastore, data, {"prefix": self._config.datastore_prefix}, ) return await self._get_instance(data['id']) @accepts(Int('id', required=True), Patch("idmap_ad_create", "idmap_ad_update", ("attr", { "update": True }))) async def do_update(self, id, data): """ Update an entry in the idmap backend table by id. """ old = await self._get_instance(id) new = old.copy() new.update(data) verrors = ValidationErrors() verrors.add_child( 'idmap_ad_update', await self.middleware.call('idmap._common_validate', new)) if verrors: raise verrors new = await self.middleware.call('idmap.common_backend_compress', new) await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) return await self._get_instance(id) @accepts(Int('id')) async def do_delete(self, id): """ Delete idmap to backend mapping by id """ await self.middleware.call("datastore.delete", self._config.datastore, id)
class SMBService(SystemServiceService): class Config: service = 'cifs' service_verb = 'restart' datastore = 'services.cifs' datastore_extend = 'smb.smb_extend' datastore_prefix = 'cifs_srv_' @private async def smb_extend(self, smb): """Extend smb for netbios.""" if not await self.middleware.call('notifier.is_freenas' ) and await self.middleware.call( 'notifier.failover_node') == 'B': smb['netbiosname'] = smb['netbiosname_b'] for i in ('aio_enable', 'aio_rs', 'aio_ws'): smb.pop(i, None) smb['loglevel'] = LOGLEVEL_MAP.get(smb['loglevel']) return smb async def __validate_netbios_name(self, name): return RE_NETBIOSNAME.match(name) async def unixcharset_choices(self): return await self.generate_choices([ 'UTF-8', 'ISO-8859-1', 'ISO-8859-15', 'GB2312', 'EUC-JP', 'ASCII' ]) @private async def generate_choices(self, initial): def key_cp(encoding): cp = re.compile( "(?P<name>CP|GB|ISO-8859-|UTF-)(?P<num>\d+)").match(encoding) if cp: return tuple((cp.group('name'), int(cp.group('num'), 10))) else: return tuple((encoding, float('inf'))) charset = await self.common_charset_choices() return { v: v for v in [c for c in sorted(charset, key=key_cp) if c not in initial] + initial } @private async def common_charset_choices(self): def check_codec(encoding): try: return encoding.upper() if codecs.lookup(encoding) else False except LookupError: return False proc = await Popen(['/usr/bin/iconv', '-l'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output = (await proc.communicate())[0].decode() encodings = set() for line in output.splitlines(): enc = [e for e in line.split() if check_codec(e)] if enc: cp = enc[0] for e in enc: if e in ('UTF-8', 'ASCII', 'GB2312', 'HZ-GB-2312', 'CP1361'): cp = e break encodings.add(cp) return encodings @accepts( Dict( 'smb_update', Str('netbiosname'), Str('netbiosname_b'), Str('netbiosalias'), Str('workgroup'), Str('description'), Str('unixcharset'), Str('loglevel', enum=['NONE', 'MINIMUM', 'NORMAL', 'FULL', 'DEBUG']), Bool('syslog'), Bool('localmaster'), Bool('domain_logons'), Bool('timeserver'), Str('guest'), Str('filemask'), Str('dirmask'), Bool('nullpw'), Bool('unixext'), Bool('zeroconf'), Bool('hostlookup'), Bool('allow_execute_always'), Bool('obey_pam_restrictions'), Bool('ntlmv1_auth'), List('bindip', items=[IPAddr('ip')], default=[]), Str('smb_options'), update=True, )) async def do_update(self, data): old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() if data.get('unixcharset') and data[ 'unixcharset'] not in await self.unixcharset_choices(): verrors.add('smb_update.unixcharset', 'Please provide a valid value for unixcharset') for i in ('workgroup', 'netbiosname', 'netbiosname_b', 'netbiosalias'): if i not in data or not data[i]: continue if not await self.__validate_netbios_name(data[i]): verrors.add(f'smb_update.{i}', 'Invalid NetBIOS name') if new['netbiosname'] and new['netbiosname'].lower( ) == new['workgroup'].lower(): verrors.add('smb_update.netbiosname', 'NetBIOS and Workgroup must be unique') for i in ('filemask', 'dirmask'): if i not in data or not data[i]: continue try: if int(data[i], 8) & ~0o11777: raise ValueError('Not an octet') except (ValueError, TypeError): verrors.add(f'smb_update.{i}', 'Not a valid mask') if verrors: raise verrors # TODO: consider using bidict for k, v in LOGLEVEL_MAP.items(): if new['loglevel'] == v: new['loglevel'] = k break await self._update_service(old, new) return await self.config()
class IdmapLDAPService(CRUDService): class Config: datastore = 'directoryservice.idmap_ldap' datastore_prefix = 'idmap_ldap_' datastore_extend = 'idmap.common_backend_extend' namespace = 'idmap.ldap' @accepts( Dict('idmap_ldap_create', Dict( 'domain', Int('id'), Str('idmap_domain_name'), Str('idmap_domain_dns_domain_name'), ), Int('range_low', required=True, validators=[Range(min=1000, max=2147483647)]), Int('range_high', required=True, validators=[Range(min=1000, max=2147483647)]), Str('base_dn'), Str('user_dn'), Str('url'), Str('ssl', default='OFF', enum=['OFF', 'ON', 'START_TLS']), Int('certificate'), register=True)) async def do_create(self, data): """ Create an entry in the idmap backend table. """ verrors = ValidationErrors() verrors.add_child( 'idmap_ldap_create', await self.middleware.call('idmap._common_validate', data)) if verrors: raise verrors data = await self.middleware.call('idmap.common_backend_compress', data) data["id"] = await self.middleware.call( "datastore.insert", self._config.datastore, data, {"prefix": self._config.datastore_prefix}, ) return await self._get_instance(data['id']) @accepts(Int('id', required=True), Patch("idmap_ldap_create", "idmap_ldap_update", ("attr", { "update": True }))) async def do_update(self, id, data): """ Update an entry in the idmap backend table by id. """ old = await self._get_instance(id) new = old.copy() new.update(data) verrors = ValidationErrors() verrors.add_child( 'idmap_ldap_update', await self.middleware.call('idmap._common_validate', new)) if verrors: raise verrors new = await self.middleware.call('idmap.common_backend_compress', new) await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) return await self._get_instance(id) @accepts(Int('id')) async def do_delete(self, id): """ Delete idmap to backend mapping by id """ await self.middleware.call("datastore.delete", self._config.datastore, id)
class DatastoreService(Service): def _filters_to_queryset(self, filters, field_suffix=None): opmap = { '=': 'exact', '!=': 'exact', '>': 'gt', '>=': 'gte', '<': 'lt', '<=': 'lte', '~': 'regex', } rv = [] for f in filters: if not isinstance(f, (list, tuple)): raise ValueError('Filter must be a list: {0}'.format(f)) if len(f) == 3: name, op, value = f if field_suffix: name = field_suffix + name if op not in opmap: raise Exception("Invalid operation: {0}".format(op)) q = Q(**{'{0}__{1}'.format(name, opmap[op]): value}) if op == '!=': q.negate() rv.append(q) elif len(f) == 2: op, value = f if op == 'OR': or_value = None for value in self._filters_to_queryset( value, field_suffix=field_suffix): if or_value is None: or_value = value else: or_value |= value rv.append(or_value) else: raise ValueError('Invalid operation: {0}'.format(op)) else: raise Exception("Invalid filter {0}".format(f)) return rv def __get_model(self, name): """Helper method to get Model for given name e.g. network.interfaces -> Interfaces """ app, model = name.split('.', 1) return apps.get_model(app, model) def __queryset_serialize(self, qs, extend=None, field_suffix=None): for i in qs: yield django_modelobj_serialize(self.middleware, i, extend=extend, field_suffix=field_suffix) @accepts( Str('name'), List('query-filters', register=True), Dict( 'query-options', Str('extend'), Dict('extra', additional_attrs=True), List('order_by'), Bool('count'), Bool('get'), Str('suffix'), register=True, ), ) def query(self, name, filters=None, options=None): """Query for items in a given collection `name`. `filters` is a list which each entry can be in one of the following formats: entry: simple_filter | conjuntion simple_filter: '[' attribute_name, OPERATOR, value ']' conjunction: '[' CONJUNTION, '[' simple_filter (',' simple_filter)* ']]' OPERATOR: ('=' | '!=' | '>' | '>=' | '<' | '<=' | '~' ) CONJUNCTION: 'OR' e.g. `['OR', [ ['username', '=', 'root' ], ['uid', '=', 0] ] ]` `[ ['username', '=', 'root' ] ]` .. examples(websocket):: Querying for username "root" and returning a single item: :::javascript { "id": "d51da71b-bb48-4b8b-a8f7-6046fcc892b4", "msg": "method", "method": "datastore.query", "params": ["account.bsdusers", [ ["username", "=", "root" ] ], {"get": true}] } """ model = self.__get_model(name) if options is None: options = {} else: # We do not want to make changes to original options # which might happen with "suffix" options = options.copy() qs = model.objects.all() extra = options.get('extra') if extra: qs = qs.extra(**extra) suffix = options.get('suffix') if filters: qs = qs.filter(*self._filters_to_queryset(filters, suffix)) order_by = options.get('order_by') if order_by: if suffix: # Do not change original order_by order_by = order_by[:] for i, order in enumerate(order_by): if order.startswith('-'): order_by[i] = '-' + suffix + order[1:] else: order_by[i] = suffix + order qs = qs.order_by(*order_by) if options.get('count') is True: return qs.count() result = list( self.__queryset_serialize(qs, extend=options.get('extend'), field_suffix=options.get('suffix'))) if options.get('get') is True: return result[0] return result @accepts(Str('name'), Ref('query-options')) def config(self, name, options=None): """ Get configuration settings object for a given `name`. This is a shortcut for `query(name, {"get": true})`. """ if options is None: options = {} options['get'] = True return self.query(name, None, options) @accepts(Str('name'), Dict('data', additional_attrs=True)) def insert(self, name, data): """ Insert a new entry to `name`. """ model = self.__get_model(name) for field in model._meta.fields: if field.name not in data: continue if isinstance(field, ForeignKey): data[field.name] = field.rel.to.objects.get( pk=data[field.name]) obj = model(**data) obj.save() return obj.pk @accepts(Str('name'), Int('id'), Dict('data', additional_attrs=True)) def update(self, name, id, data): """ Update an entry `id` in `name`. """ model = self.__get_model(name) obj = model.objects.get(pk=id) for field in model._meta.fields: if field.name not in data: continue if isinstance(field, ForeignKey): data[field.name] = field.rel.to.objects.get( pk=data[field.name]) for k, v in list(data.items()): setattr(obj, k, v) obj.save() return obj.pk @accepts(Str('name'), Int('id')) def delete(self, name, id): """ Delete an entry `id` in `name`. """ model = self.__get_model(name) model.objects.get(pk=id).delete() return True @private def sql(self, query, params=None): cursor = connection.cursor() rv = None try: if params is None: cursor.executelocal(query) else: cursor.executelocal(query, params) rv = cursor.fetchall() finally: cursor.close() return rv
class IdmapRFC2307Service(CRUDService): """ In the rfc2307 backend range acts as a filter. Anything falling outside of it is ignored. If no user_dn is specified, then an anonymous bind is performed. ldap_url is only required when using a standalone server. """ class Config: datastore = 'directoryservice.idmap_rfc2307' datastore_prefix = 'idmap_rfc2307_' datastore_extend = 'idmap.common_backend_extend' namespace = 'idmap.rfc2307' @accepts( Dict('idmap_rfc2307_create', Dict( 'domain', Int('id'), Str('idmap_domain_name'), Str('idmap_domain_dns_domain_name'), ), Int('range_low', required=True, validators=[Range(min=1000, max=2147483647)]), Int('range_high', required=True, validators=[Range(min=1000, max=2147483647)]), Str('ldap_server', default='AD', enum=['AD', 'STAND-ALONE']), Str('bind_path_user'), Str('bind_path_group'), Bool('user_cn', default=False), Bool('cn_realm', default=False), Str('ldap_domain'), Str('ldap_url'), Str('ldap_user_dn'), Str('ldap_user_dn_password'), Str('ldap_realm'), Str('ssl', default='OFF', enum=['OFF', 'ON', 'START_TLS']), Int('certificate'), register=True)) async def do_create(self, data): """ Create an entry in the idmap_rfc2307 backend table. `ldap_server` defines the type of LDAP server to use. This can either be an LDAP server provided by the Active Directory Domain (ad) or a stand-alone LDAP server. `bind_path_user` specfies the search base where user objects can be found in the LDAP server. `bind_path_group` specifies the search base where group objects can be found in the LDAP server. `user_cn` query cn attribute instead of uid attribute for the user name in LDAP. `realm` append @realm to cn for groups (and users if user_cn is set) in LDAP queries. `ldmap_domain` when using the LDAP server in the Active Directory server, this allows one to specify the domain where to access the Active Directory server. This allows using trust relationships while keeping all RFC 2307 records in one place. This parameter is optional, the default is to access the AD server in the current domain to query LDAP records. `ldap_url` when using a stand-alone LDAP server, this parameter specifies the LDAP URL for accessing the LDAP server. `ldap_user_dn` defines the user DN to be used for authentication. `realm` defines the realm to use in the user and group names. This is only required when using cn_realm together with a stand-alone ldap server. """ verrors = ValidationErrors() verrors.add_child( 'idmap_rfc2307_create', await self.middleware.call('idmap._common_validate', data)) if verrors: raise verrors data = await self.middleware.call('idmap.common_backend_compress', data) data["id"] = await self.middleware.call( "datastore.insert", self._config.datastore, data, {"prefix": self._config.datastore_prefix}, ) return await self._get_instance(data['id']) @accepts(Int('id', required=True), Patch("idmap_nss_create", "idmap_nss_update", ("attr", { "update": True }))) async def do_update(self, id, data): """ Update an entry in the idmap backend table by id. """ old = await self._get_instance(id) new = old.copy() new.update(data) verrors = ValidationErrors() verrors.add_child( 'idmap_rfc2307_update', await self.middleware.call('idmap._common_validate', new)) if verrors: raise verrors new = await self.middleware.call('idmap.common_backend_compress', new) await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) return await self._get_instance(id) @accepts(Int('id')) async def do_delete(self, id): """ Delete idmap to backend mapping by id """ await self.middleware.call("datastore.delete", self._config.datastore, id)
class ChartReleaseService(Service): class Config: namespace = 'chart.release' @private async def scale_down_workloads_before_snapshot(self, job, release): resources = [] pod_mapping = await self.middleware.call('chart.release.get_workload_to_pod_mapping', release['namespace']) pods_to_watch_for = [] for resource in SCALEABLE_RESOURCES: for workload in await self.middleware.call( f'k8s.{resource.name.lower()}.query', [ [f'metadata.annotations.{SCALE_DOWN_ANNOTATION["key"]}', 'in', SCALE_DOWN_ANNOTATION['value']], ['metadata.namespace', '=', release['namespace']], ] ): resources.append({ 'replica_count': 0, 'type': resource.name, 'name': workload['metadata']['name'], }) pods_to_watch_for.extend(pod_mapping[workload['metadata']['uid']]) if not resources: return job.set_progress(35, f'Scaling down {", ".join([r["name"] for r in resources])} workload(s)') await self.middleware.call('chart.release.scale_workloads', release['id'], resources) await self.middleware.call( 'chart.release.wait_for_pods_to_terminate', release['namespace'], [ ['metadata.name', 'in', pods_to_watch_for], ] ) job.set_progress(40, 'Successfully scaled down workload(s)') @accepts( Str('release_name'), Dict( 'upgrade_options', Dict('values', additional_attrs=True), Str('item_version', default='latest'), ) ) @returns(Ref('chart_release_entry')) @job(lock=lambda args: f'chart_release_upgrade_{args[0]}') async def upgrade(self, job, release_name, options): """ Upgrade `release_name` chart release. `upgrade_options.item_version` specifies to which item version chart release should be upgraded to. System will update container images being used by `release_name` chart release as a chart release upgrade is not considered complete until the images in use have also been updated to latest versions. During upgrade, `upgrade_options.values` can be specified to apply configuration changes for configuration changes for the chart release in question. When chart version is upgraded, system will automatically take a snapshot of `ix_volumes` in question which can be used to rollback later on. """ await self.middleware.call('kubernetes.validate_k8s_setup') release = await self.middleware.call('chart.release.get_instance', release_name) if not release['update_available'] and not release['container_images_update_available']: raise CallError('No update is available for chart release') # We need to update container images before upgrading chart version as it's possible that the chart version # in question needs newer image hashes. job.set_progress(10, 'Updating container images') await ( await self.middleware.call('chart.release.pull_container_images', release_name, {'redeploy': False}) ).wait(raise_error=True) job.set_progress(30, 'Updated container images') await self.scale_down_workloads_before_snapshot(job, release) # If a snapshot of the volumes already exist with the same name in case of a failed upgrade, we will remove # it as we want the current point in time being reflected in the snapshot # TODO: Remove volumes/ix_volumes check in next release as we are going to do a recursive snapshot # from parent volumes ds moving on for filesystem in ('volumes', 'volumes/ix_volumes'): volumes_ds = os.path.join(release['dataset'], filesystem) snap_name = f'{volumes_ds}@{release["version"]}' if await self.middleware.call('zfs.snapshot.query', [['id', '=', snap_name]]): await self.middleware.call('zfs.snapshot.delete', snap_name, {'recursive': True}) await self.middleware.call( 'zfs.snapshot.create', { 'dataset': os.path.join(release['dataset'], 'volumes'), 'name': release['version'], 'recursive': True } ) job.set_progress(50, 'Created snapshot for upgrade') if release['update_available']: await self.upgrade_chart_release(job, release, options) else: await (await self.middleware.call('chart.release.redeploy', release_name)).wait(raise_error=True) chart_release = await self.middleware.call('chart.release.get_instance', release_name) self.middleware.send_event('chart.release.query', 'CHANGED', id=release_name, fields=chart_release) await self.chart_releases_update_checks_internal([['id', '=', release_name]]) job.set_progress(100, 'Upgrade complete for chart release') return chart_release @accepts( Str('release_name'), Dict( 'options', Str('item_version', default='latest', empty=False) ) ) @returns(Dict( Bool('image_update_available', required=True), Bool('item_update_available', required=True), Dict( 'container_images_to_update', additional_attrs=True, description='Dictionary of container image(s) which have an update available against the same tag', ), Str('latest_version'), Str('latest_human_version'), Str('upgrade_version'), Str('upgrade_human_version'), Str('changelog', max_length=None, null=True), List('available_versions_for_upgrade', items=[Dict( 'version_info', Str('version', required=True), Str('human_version', required=True), )]) )) async def upgrade_summary(self, release_name, options): """ Retrieve upgrade summary for `release_name` which will include which container images will be updated and changelog for `options.item_version` chart version specified if applicable. If only container images need to be updated, changelog will be `null`. If chart release `release_name` does not require an upgrade, an error will be raised. """ release = await self.middleware.call( 'chart.release.query', [['id', '=', release_name]], {'extra': {'retrieve_resources': True}, 'get': True} ) if not release['update_available'] and not release['container_images_update_available']: raise CallError('No update is available for chart release', errno=errno.ENOENT) version_info = { 'latest_version': release['chart_metadata']['version'], 'upgrade_version': release['chart_metadata']['version'], 'latest_human_version': release['human_version'], 'upgrade_human_version': release['human_version'], } changelog = None all_newer_versions = [] if release['update_available']: available_items = await self.get_versions(release, options) latest_item = available_items['latest_version'] upgrade_version = available_items['specified_version'] version_info.update({ 'latest_version': latest_item['version'], 'latest_human_version': latest_item['human_version'], 'upgrade_version': upgrade_version['version'], 'upgrade_human_version': upgrade_version['human_version'], }) changelog = upgrade_version['changelog'] all_newer_versions = [ { 'version': v['version'], 'human_version': v['human_version'], } for v in available_items['versions'].values() if parse_version(v['version']) > parse_version(release['chart_metadata']['version']) ] return { 'container_images_to_update': { k: v for k, v in release['resources']['container_images'].items() if v['update_available'] }, 'changelog': changelog, 'available_versions_for_upgrade': all_newer_versions, 'item_update_available': release['update_available'], 'image_update_available': release['container_images_update_available'], **version_info, } @private async def get_version(self, release, options): return (await self.get_versions(release, options))['specified_version'] @private async def get_versions(self, release, options): current_chart = release['chart_metadata'] chart = current_chart['name'] item_details = await self.middleware.call('catalog.get_item_details', chart, { 'catalog': release['catalog'], 'train': release['catalog_train'], }) new_version = options['item_version'] if new_version == 'latest': new_version = await self.middleware.call( 'chart.release.get_latest_version_from_item_versions', item_details['versions'] ) if new_version not in item_details['versions']: raise CallError(f'Unable to locate specified {new_version!r} item version.') verrors = ValidationErrors() if parse_version(new_version) <= parse_version(current_chart['version']): verrors.add( 'upgrade_options.item_version', f'Upgrade version must be greater than {current_chart["version"]!r} current version.' ) verrors.check() return { 'specified_version': item_details['versions'][new_version], 'versions': item_details['versions'], 'latest_version': item_details['versions'][await self.middleware.call( 'chart.release.get_latest_version_from_item_versions', item_details['versions'] )] } @private async def upgrade_chart_release(self, job, release, options): release_orig = copy.deepcopy(release) release_name = release['name'] catalog_item = await self.get_version(release, options) await self.middleware.call('catalog.version_supported_error_check', catalog_item) config = await self.middleware.call('chart.release.upgrade_values', release, catalog_item['location']) release_orig['config'] = config # We will be performing validation for values specified. Why we want to allow user to specify values here # is because the upgraded catalog item version might have different schema which potentially means that # upgrade won't work or even if new k8s are resources are created/deployed, they won't necessarily function # as they should because of changed params or expecting new params # One tricky bit which we need to account for first is removing any key from current configured values # which the upgraded release will potentially not support. So we can safely remove those as otherwise # validation will fail as new schema does not expect those keys. config = clean_values_for_upgrade(config, catalog_item['schema']['questions']) config.update(options['values']) config, context = await self.middleware.call( 'chart.release.normalise_and_validate_values', catalog_item, config, False, release['dataset'], release_orig, ) job.set_progress(50, 'Initial validation complete for upgrading chart version') # We have validated configuration now chart_path = os.path.join(release['path'], 'charts', catalog_item['version']) await self.middleware.run_in_thread(shutil.rmtree, chart_path, ignore_errors=True) await self.middleware.run_in_thread(shutil.copytree, catalog_item['location'], chart_path) await self.middleware.call('chart.release.perform_actions', context) # Let's update context options to reflect that an upgrade is taking place and from which version to which # version it's happening. # Helm considers simple config change as an upgrade as well, and we have no way of determining the old/new # chart versions during helm upgrade in the helm template, hence the requirement for a context object. config[CONTEXT_KEY_NAME].update({ **get_action_context(release_name), 'operation': 'UPGRADE', 'isUpgrade': True, 'upgradeMetadata': { 'oldChartVersion': release['chart_metadata']['version'], 'newChartVersion': catalog_item['version'], 'preUpgradeRevision': release['version'], } }) job.set_progress(60, 'Upgrading chart release version') await self.middleware.call('chart.release.helm_action', release_name, chart_path, config, 'upgrade') await self.middleware.call('chart.release.refresh_events_state', release_name) @private def upgrade_values(self, release, new_version_path): config = copy.deepcopy(release['config']) chart_version = release['chart_metadata']['version'] migration_path = os.path.join(new_version_path, 'migrations') migration_files = [os.path.join(migration_path, k) for k in (f'migrate_from_{chart_version}', 'migrate')] if not os.path.exists(migration_path) or all(not os.access(p, os.X_OK) for p in migration_files): return config # This is guaranteed to exist based on above check file_path = next(f for f in migration_files if os.access(f, os.X_OK)) with tempfile.NamedTemporaryFile(mode='w+') as f: f.write(json.dumps(config)) f.flush() cp = subprocess.Popen([file_path, f.name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = cp.communicate() if cp.returncode: raise CallError(f'Failed to apply migration: {stderr.decode()}') if stdout: # We add this as a safety net in case something went wrong with the migration and we get a null response # or the chart dev mishandled something - although we don't suppress any exceptions which might be raised config = json.loads(stdout.decode()) return config @periodic(interval=86400) @private async def periodic_chart_releases_update_checks(self): sync_job = await self.middleware.call('catalog.sync_all') await sync_job.wait() if not await self.middleware.call('service.started', 'kubernetes'): return await self.chart_releases_update_checks_internal() @private async def chart_releases_update_checks_internal(self, chart_releases_filters=None): chart_releases_filters = chart_releases_filters or [] # Chart release wrt alerts will be considered valid for upgrade/update if either there's a newer # catalog item version available or any of the images it's using is outdated catalog_items = { f'{c["id"]}_{train}_{item}': c['trains'][train][item] for c in await self.middleware.call('catalog.query', [], {'extra': {'item_details': True}}) for train in c['trains'] for item in c['trains'][train] } for application in await self.middleware.call('chart.release.query', chart_releases_filters): if application['container_images_update_available']: await self.middleware.call('alert.oneshot_create', 'ChartReleaseUpdate', application) continue app_id = f'{application["catalog"]}_{application["catalog_train"]}_{application["chart_metadata"]["name"]}' catalog_item = catalog_items.get(app_id) if not catalog_item: continue await self.chart_release_update_check(catalog_item, application) container_config = await self.middleware.call('container.config') if container_config['enable_image_updates']: asyncio.ensure_future(self.middleware.call('container.image.check_update')) @private async def chart_release_update_check(self, catalog_item, application): latest_version = catalog_item['latest_version'] if not latest_version: return if parse_version(latest_version) > parse_version(application['chart_metadata']['version']): await self.middleware.call('alert.oneshot_create', 'ChartReleaseUpdate', application) else: await self.middleware.call('alert.oneshot_delete', 'ChartReleaseUpdate', application['id']) @accepts( Str('release_name'), Dict( 'pull_container_images_options', Bool('redeploy', default=True), ) ) @returns(Dict( 'container_images', additional_attrs=True, description='Dictionary of container image(s) with container image tag as key and update status as value', example={ 'plexinc/pms-docker:1.23.2.4656-85f0adf5b': 'Updated image', } )) @job(lock=lambda args: f'pull_container_images{args[0]}') async def pull_container_images(self, job, release_name, options): """ Update container images being used by `release_name` chart release. `redeploy` when set will redeploy pods which will result in chart release using newer updated versions of the container images. """ await self.middleware.call('kubernetes.validate_k8s_setup') images = [ {'orig_tag': tag, 'from_image': tag.rsplit(':', 1)[0], 'tag': tag.rsplit(':', 1)[-1]} for tag in (await self.middleware.call( 'chart.release.query', [['id', '=', release_name]], {'extra': {'retrieve_resources': True}, 'get': True} ))['resources']['container_images'] ] results = {} bulk_job = await self.middleware.call( 'core.bulk', 'container.image.pull', [ [{'from_image': image['from_image'], 'tag': image['tag']}] for image in images ] ) await bulk_job.wait() if bulk_job.error: raise CallError(f'Failed to update container images for {release_name!r} chart release: {bulk_job.error}') for tag, status in zip(images, bulk_job.result): if status['error']: results[tag['orig_tag']] = f'Failed to pull image: {status["error"]}' else: results[tag['orig_tag']] = 'Updated image' if options['redeploy']: await job.wrap(await self.middleware.call('chart.release.redeploy', release_name)) return results @private async def clear_update_alerts_for_all_chart_releases(self): for chart_release in await self.middleware.call('chart.release.query'): await self.middleware.call('alert.oneshot_delete', 'ChartReleaseUpdate', chart_release['id'])
class IdmapScriptService(CRUDService): class Config: datastore = 'directoryservice.idmap_script' datastore_prefix = 'idmap_script_' datastore_extend = 'idmap.common_backend_extend' namespace = 'idmap.script' @accepts( Dict('idmap_script_create', Dict( 'domain', Int('id'), Str('idmap_domain_name'), Str('idmap_domain_dns_domain_name'), ), Int('range_low', required=True, validators=[Range(min=1000, max=2147483647)]), Int('range_high', required=True, validators=[Range(min=1000, max=2147483647)]), Str('script'), register=True)) async def do_create(self, data): """ Create an entry in the idmap backend table. `script` full path to the script or program that generates the mappings. """ verrors = ValidationErrors() verrors.add_child( 'idmap_script_create', await self.middleware.call('idmap._common_validate', data)) if verrors: raise verrors data = await self.middleware.call('idmap.common_backend_compress', data) data["id"] = await self.middleware.call( "datastore.insert", self._config.datastore, data, {"prefix": self._config.datastore_prefix}, ) return await self._get_instance(data['id']) @accepts(Int('id', required=True), Patch("idmap_script_create", "idmap_script_update", ("attr", { "update": True }))) async def do_update(self, id, data): """ Update an entry in the idmap backend table by id. """ old = await self._get_instance(id) new = old.copy() new.update(data) verrors = ValidationErrors() verrors.add_child( 'idmap_script_update', await self.middleware.call('idmap._common_validate', new)) if verrors: raise verrors new = await self.middleware.call('idmap.common_backend_compress', new) await self.middleware.call('datastore.update', self._config.datastore, id, new, {'prefix': self._config.datastore_prefix}) return await self._get_instance(id) @accepts(Int('id')) async def do_delete(self, id): """ Delete idmap to backend mapping by id """ await self.middleware.call("datastore.delete", self._config.datastore, id)
class AFPService(SystemServiceService): class Config: service = 'afp' datastore_extend = 'afp.extend' datastore_prefix = 'afp_srv_' @private async def extend(self, afp): for i in ('map_acls', 'chmod_request'): afp[i] = afp[i].upper() return afp @private async def compress(self, afp): for i in ('map_acls', 'chmod_request'): value = afp.get(i) if value: afp[i] = value.lower() return afp @accepts( Dict('afp_update', Bool('guest'), Str('guest_user'), List('bindip', items=[Str('ip', validators=[IpAddress()])]), Int('connections_limit', validators=[Range(min=1, max=65535)]), Dir('dbpath'), Str('global_aux', max_length=None), Str('map_acls', enum=['RIGHTS', 'MODE', 'NONE']), Str('chmod_request', enum=['PRESERVE', 'SIMPLE', 'IGNORE']), Str('loglevel', enum=[x.name for x in AFPLogLevel]), update=True)) async def do_update(self, data): """ Update AFP service settings. `bindip` is a list of IPs to bind AFP to. Leave blank (empty list) to bind to all available IPs. `map_acls` defines how to map the effective permissions of authenticated users. RIGHTS - Unix-style permissions MODE - ACLs NONE - Do not map `chmod_request` defines advanced permission control that deals with ACLs. PRESERVE - Preserve ZFS ACEs for named users and groups or POSIX ACL group mask SIMPLE - Change permission as requested without any extra steps IGNORE - Permission change requests are ignored """ old = await self.config() new = old.copy() new.update(data) verrors = ValidationErrors() if new['dbpath']: await check_path_resides_within_volume( verrors, self.middleware, 'afp_update.dbpath', new['dbpath'], ) verrors.check() new = await self.compress(new) await self._update_service(old, new) return await self.config() @accepts() async def bindip_choices(self): """ List of valid choices for IP addresses to which to bind the AFP service. """ return { d['address']: d['address'] for d in await self.middleware.call('interface.ip_in_use') }
class NISService(ConfigService): class Config: service = "nis" datastore = 'directoryservice.nis' datastore_extend = "nis.nis_extend" datastore_prefix = "nis_" @private async def nis_extend(self, nis): nis['servers'] = nis['servers'].split(',') if nis['servers'] else [] return nis @private async def nis_compress(self, nis): nis['servers'] = ','.join(nis['servers']) return nis @accepts( Dict('nis_update', Str('domain'), List('servers'), Bool('secure_mode'), Bool('manycast'), Bool('enable'), update=True)) async def do_update(self, data): """ Update NIS Service Configuration. `domain` is the name of NIS domain. `servers` is a list of hostnames/IP addresses. `secure_mode` when enabled sets ypbind(8) to refuse binding to any NIS server not running as root on a TCP port over 1024. `manycast` when enabled sets ypbind(8) to bind to the server that responds the fastest. `enable` when true disables the configuration of the NIS service. """ must_reload = False old = await self.config() new = old.copy() new.update(data) if old != new: must_reload = True await self.nis_compress(new) await self.middleware.call('datastore.update', 'directoryservice.nis', old['id'], new, {'prefix': 'nis_'}) if must_reload: if new['enable']: await self.middleware.call('nis.start') else: await self.middleware.call('nis.stop') return await self.config() @private async def set_state(self, state): return await self.middleware.call('directoryservices.set_state', {'nis': state.name}) @accepts() async def get_state(self): """ Wrapper function for 'directoryservices.get_state'. Returns only the state of the NIS service. """ return (await self.middleware.call('directoryservices.get_state'))['nis'] @private async def start(self): """ Refuse to start service if the service is alreading in process of starting or stopping. If state is 'HEALTHY' or 'FAULTED', then stop the service first before restarting it to ensure that the service begins in a clean state. """ state = await self.get_state() nis = await self.config() if state in ['FAULTED', 'HEALTHY']: await self.stop() if state in ['EXITING', 'JOINING']: raise CallError( f'Current state of NIS service is: [{state}]. Wait until operation completes.', errno.EBUSY) await self.set_state(DSStatus['JOINING']) await self.middleware.call('datastore.update', 'directoryservice.nis', nis['id'], {'nis_enable': True}) await self.middleware.call('etc.generate', 'rc') await self.middleware.call('etc.generate', 'pam') await self.middleware.call('etc.generate', 'hostname') await self.middleware.call('etc.generate', 'nss') setnisdomain = await run(['/bin/domainname', nis['domain']], check=False) if setnisdomain.returncode != 0: await self.set_state(DSStatus['FAULTED']) raise CallError( f'Failed to set NIS Domain to [{nis["domain"]}]: {setnisdomain.stderr.decode()}' ) ypbind = await run(['/usr/sbin/service', 'ypbind', 'onestart'], check=False) if ypbind.returncode != 0: await self.set_state(DSStatus['FAULTED']) raise CallError(f'ypbind failed: {ypbind.stderr.decode()}') await self.set_state(DSStatus['HEALTHY']) self.logger.debug( f'NIS service successfully started. Setting state to HEALTHY.') await self.middleware.call('nis.fill_cache') return True @private async def __ypwhich(self): """ The return code from ypwhich is not a reliable health indicator. For example, RPC failure will return 0. There are edge cases where ypwhich can hang when NIS is misconfigured. """ ypwhich = await run(['/usr/bin/ypwhich'], check=False) if ypwhich.stderr: raise CallError( f'NIS status check returned [{ypwhich.stderr.decode().strip()}]. Setting state to FAULTED.' ) return True @private async def started(self): ret = False if not (await self.config())['enable']: return ret try: ret = await asyncio.wait_for(self.__ypwhich(), timeout=5.0) except asyncio.TimeoutError: raise CallError('nis.started check timed out after 5 seconds.') return ret @private async def stop(self, force=False): """ Remove NIS_state entry entirely after stopping ypbind. This is so that the 'enable' checkbox becomes the sole source of truth regarding a service's state when it is disabled. """ state = await self.get_state() nis = await self.config() if not force: if state in ['LEAVING', 'JOINING']: raise CallError( f'Current state of NIS service is: [{state}]. Wait until operation completes.', errno.EBUSY) await self.__set_state(DSStatus['LEAVING']) await self.middleware.call('datastore.update', 'directoryservice.nis', nis['id'], {'nis_enable': False}) ypbind = await run(['/usr/sbin/service', 'ypbind', 'onestop'], check=False) if ypbind.returncode != 0: await self.__set_state(DSStatus['FAULTED']) errmsg = ypbind.stderr.decode().strip() if 'ypbind not running' not in errmsg: raise CallError( f'ypbind failed to stop: [{ypbind.stderr.decode().strip()}]' ) await self.middleware.call('cache.pop', 'NIS_State') await self.middleware.call('etc.generate', 'rc') await self.middleware.call('etc.generate', 'pam') await self.middleware.call('etc.generate', 'hostname') await self.middleware.call('etc.generate', 'nss') await self.__set_state(DSStatus['DISABLED']) self.logger.debug( f'NIS service successfully stopped. Setting state to DISABLED.') return True @private @job(lock=lambda args: 'fill_nis_cache') def fill_cache(self, job, force=False): user_next_index = group_next_index = 200000000 if self.middleware.call_sync('cache.has_key', 'NIS_cache') and not force: raise CallError( 'NIS cache already exists. Refusing to generate cache.') self.middleware.call_sync('cache.pop', 'NIS_cache') pwd_list = pwd.getpwall() grp_list = grp.getgrall() local_uid_list = list(u['uid'] for u in self.middleware.call_sync('user.query')) local_gid_list = list( g['gid'] for g in self.middleware.call_sync('group.query')) cache_data = {'users': [], 'groups': []} for u in pwd_list: is_local_user = True if u.pw_uid in local_uid_list else False if is_local_user: continue cache_data['users'].append({ u.pw_name: { 'id': user_next_index, 'uid': u.pw_uid, 'username': u.pw_name, 'unixhash': None, 'smbhash': None, 'group': {}, 'home': '', 'shell': '', 'full_name': u.pw_gecos, 'builtin': False, 'email': '', 'password_disabled': False, 'locked': False, 'sudo': False, 'microsoft_account': False, 'attributes': {}, 'groups': [], 'sshpubkey': None, 'local': False } }) user_next_index += 1 for g in grp_list: is_local_user = True if g.gr_gid in local_gid_list else False if is_local_user: continue cache_data['groups'].append({ g.gr_name: { 'id': group_next_index, 'gid': g.gr_gid, 'group': g.gr_name, 'builtin': False, 'sudo': False, 'users': [], 'local': False } }) group_next_index += 1 self.middleware.call_sync('cache.put', 'NIS_cache', cache_data) self.middleware.call_sync('dscache.backup') @private async def get_cache(self): if not await self.middleware.call('cache.has_key', 'NIS_cache'): await self.middleware.call('nis.fill_cache') self.logger.debug('cache fill is in progress.') return {'users': [], 'groups': []} return await self.middleware.call('cache.get', 'nis_cache')