Beispiel #1
0
    def create(self, size, sparse=False, block_size=None, mkparent=False):
        """Creates storage volume.

        volume = Volume('dpool/tmp/test0')
        volume.create()

        """
        # TODO Check size to make sure it's decent.

        try:
            # -V volume, -s sparse, -b blocksize, -o prop=val
            # -p works like mkdir -p, creates non-existing parent datasets.
            opts = ['create']
            if sparse:
                opts.append('-s')
            if block_size:
                opts.extend(['-b', block_size])
            if mkparent:
                opts.append('-p')
            opts.extend(['-V', size, self.name])

            sh.zfs(*opts)
        except sh.ErrorReturnCode_1:
            # I'm not sure about this; the problem is if it creates the
            # dataset, but fails to mount it for some reason, we're left with
            # the pieces and a simple 1 retval...
            #if self.exists():
            #    self.destroy()
            raise
        # TODO Force scan of this in bg
        return True
Beispiel #2
0
def recover(chroot):
    """ Import pool and mount filesystem in prep for recovery efforts """
    sh.zpool.export('-a', _fg=True)
    sh.zpool('import', '-Nf', '-R', paths.zroot, config.pool_name, _fg=True)
    sh.zfs('load-key', '-a', _fg=True)
    sh.zfs.mount(config.os_root_ds, _fg=True)
    sh.zfs.mount('-a', _fg=True)

    other_mounts()

    if chroot:
        sh.chroot(paths.zroot, '/bin/bash', '--login', _fg=True)
Beispiel #3
0
    def exists(self):
        """Checks if filesystem exists.

        filesystem = Filesystem('dpool/tmp/test0')
        filesystem.exists()

        """
        try:
            sh.zfs('list', '-t', 'filesystem', self.name)
        except sh.ErrorReturnCode_1:
            return False
        return True
Beispiel #4
0
    def exists(self):
        """Checks if volume exists.

        volume = Volume('dpool/tmp/test0')
        volume.exists()

        """
        try:
            sh.zfs('list', '-t', 'volume', self.name)
        except sh.ErrorReturnCode_1:
            return False
        # TODO Force scan of this in bg
        return True
Beispiel #5
0
    def exists(self):
        """Checks if snapshot exists.

        snapshot = Snapshot('dpool/tmp/test0@whoa-snap-0')
        snapshot.exists()

        """
        try:
            sh.zfs('list', '-t', 'snapshot', self.name)
        except sh.ErrorReturnCode_1:
            return False
        # TODO Force scan of this in bg
        return True
Beispiel #6
0
    def destroy(self, confirm=False, recursive=False):
        """Destroys storage filesystem.

        filesystem = Filesystem('dpool/tmp/test0')
        filesystem.destroy()

        """
        if confirm is not True:
            raise ZfsError('Destroy of storage filesystem requires confirm=True')
        opts = ['destroy']
        if recursive:
            opts.append('-r')
        opts.append(self.name)
        sh.zfs(*opts)
        # TODO Force delete of this in bg (with '-d')
        return True
Beispiel #7
0
    def destroy(self, confirm=False, recursive=False):
        """Destroys storage snapshot.

        snapshot = Snapshot('dpool/tmp/test0@whoa-snap-0')
        snapshot.destroy()

        """
        if confirm is not True:
            raise ZfsError('Destroy of storage snapshot requires confirm=True')
        opts = ['destroy']
        if recursive:
            opts.append('-r')
        opts.append(self.name)
        sh.zfs(*opts)
        # TODO Force delete of this in bg
        return True
Beispiel #8
0
    def rename(self, new):
        """Renames storage filesystem.

        filesystem = Filesystem('dpool/r0')
        filesystem.rename('dpool/r1')

        """
        old = self.name
        logging.info("Renaming dataset '%s' to '%s'", old, new)
        try:
            sh.zfs('rename', old, new)
            self.name = new
        except:
            self.name = old
            raise
        finally:
            self.save()
Beispiel #9
0
    def rename(self, new):
        """Renames storage volume.

        volume = Volume('dpool/r0')
        volume.rename('dpool/r1')

        """
        old = self.name
        logger.info("Renaming dataset '%s' to '%s'", old, new)
        try:
            sh.zfs('rename', old, new)
            self.name = new
        except:
            self.name = old
            raise
        finally:
            self.save()
Beispiel #10
0
    def destroy(self, confirm=False, recursive=False):
        """Destroys storage volume.

        volume = Volume('dpool/tmp/test0')
        volume.destroy()

        """
        if confirm is not True:
            raise ZfsError("It is required that you specify confirm=True for such an action of destruction.")

        opts = ['destroy']
        if recursive:
            opts.append('-r')
        opts.append(self.name)
        sh.zfs(*opts)
        # TODO Force delete of this in bg
        return True
Beispiel #11
0
def status():
    print(f'$ config values --------------------\n')
    print(config)

    print(f'\n$ sgdisk --print {config.disk_dev} --------------------\n')
    sh.sgdisk('--print', config.disk_dev, _out=sys.stdout, _ok_code=[0,2])

    print(f'\n$ blkid --------------------------\n')
    sh.blkid(_out=sys.stdout,)

    print('\n$ zpool list ---------------------------\n')
    sh.zpool('list', _out=sys.stdout)

    print('\n$ zfs list ---------------------------\n')
    sh.zfs('list', _out=sys.stdout)

    print('\n$ zfs mount ---------------------------\n')
    sh.zfs('mount', _out=sys.stdout)
Beispiel #12
0
    def _set(self, k, v, ignore=False):
        """Sets Dataset property.

        dataset = Dataset('dpool/carp')
        dataset.properties._set('readonly', 'on')

        """
        if ignore:
            return

        prop = None
        if isinstance(v, DatasetProperty):
            prop = v
            v = prop.value

        sh.zfs('set', '%s=%s' % (k, v), self._parent.name)

        if prop:
            prop.value = v
Beispiel #13
0
    def create(self):
        """Creates storage filesystem.

        filesystem = Filesystem('dpool/tmp/test0')
        filesystem.create()

        """
        try:
            sh.zfs('create', self.name)
        except sh.ErrorReturnCode_1:
            # I'm not sure about this; the problem is if it creates the
            # dataset, but fails to mount it for some reason, we're left with
            # the pieces and a simple 1 retval...
            #if self.exists():
            #    self.destroy()
            raise

        # TODO Force scan of this in bg
        return True
Beispiel #14
0
    def create(self, size):
        """Creates storage snapshot.

        snapshot = Snapshot('dpool/tmp/test0@whoa-snap-0')
        snapshot.create()

        """
        # TODO Check size to make sure it's decent.

        try:
            sh.zfs('snapshot', self.name)
        except sh.ErrorReturnCode_1:
            # I'm not sure about this; the problem is if it creates the
            # dataset, but fails to mount it for some reason, we're left with
            # the pieces and a simple 1 retval...
            #if self.exists():
            #    self.destroy()
            raise
        # TODO Force scan of this in bg
        return True
def main():
    args = parse_args()
    config = load_config()
    config_env = config[args.config_env]["ldap"]

    # Setup logging
    setup_logging(debug=args.debug, noop=False)

    logger.debug4("OPTIONS: %s" % vars(args))
    logger.debug4("CONFIG: %s" % config_env)

    _ldap_url = config_env.get("url")
    _use_tls = config_env.get("tls")
    _bind_dn = config_env.get("bind_dn", None)
    _bind_pass = config_env.get("bind_pass", None)

    local_ldap = LocalLdap(url=_ldap_url[0], use_tls=_use_tls, bind_dn=_bind_dn, bind_pass=_bind_pass, log_level=None)
    ldap_users = local_ldap.paged_search(base=search_base, sfilter=search_filter, attrlist=search_return_attribs, scope=search_scope)

    users_over_quota = []
    users_over_ldap_quota = []
    users_over_zfs_quota = []
    users_ldap_quota_mismatch = []
    zfs_set_cmds = []

    for user in ldap_users:
        _user_data = {}
        _user = LdapUser()
        _user.setattrs(data=user, listvals=["mail"])
        _username = _user.uid
        _uid = _user.uidNumber
        _shell = _user.loginShell
        _quota = _user.quota
        if hasattr(_user, "mail"):
            _mail = ",".join(_user.mail)
        else:
            _mail = ""

        if active_only and _shell != "/bin/bash":
            continue

        mount, softlimit, hardlimit, softinode, hardinode = re.findall(r"^(.*):([0-9]+),([0-9]+),([0-9]+),([0-9]+)$", _quota)[0]
        _ldap_quota = int(hardlimit) * 1024
        zfs_fs = "tank%s" % mount

        # Get current ZFS quota
        userquota_args = ["get", "-H", "-p", "-o", "value", "userquota@%s" % _username, zfs_fs]
        logger.debug("Executing: zfs %s", " ".join(userquota_args))
        userquota_output = zfs(userquota_args)
        _userquota = userquota_output.strip()
        if _userquota != "-":
            current_quota = int(_userquota)
        else:
            current_quota = 0

        # Get current used space
        userused_args = ["get", "-H", "-p", "-o", "value", "userused@%s" % _username, zfs_fs]
        logger.debug("Executing: zfs %s", " ".join(userused_args))
        userused_output = zfs(userused_args)
        _userused = userused_output.strip()
        if _userused != "-":
            current_used = int(_userused)
        else:
            current_used = 0

        _user_data["username"] = _username
        _user_data["uid"] = _uid
        _user_data["mail"] = _mail
        _user_data["zfs_fs"] = zfs_fs
        _user_data["ldap_quota"] = _ldap_quota
        _user_data["zfs_quota"] = current_quota
        _user_data["zfs_used"] = current_used

        if current_used >= _ldap_quota and current_used >= current_quota:
            users_over_quota.append(_user_data)
        elif current_used and current_used >= _ldap_quota:
            users_over_ldap_quota.append(_user_data)
        elif current_used and current_used >= current_quota:
            users_over_zfs_quota.append(_user_data)

        if _ldap_quota != current_quota:
            users_ldap_quota_mismatch.append(_user_data)
            zfs_set_cmd = [
                "set", "userquota@%s=%s" % (_username, _ldap_quota), zfs_fs
            ]
            zfs_set_cmds.append(zfs_set_cmd)

    for user in users_over_quota:
        print_data("WARNING: over quota", user)
    print "---------"

    for user in users_over_ldap_quota:
        print_data("WARNING: over LDAP quota", user)
    print "---------"

    for user in users_over_zfs_quota:
        print_data("WARNING: over ZFS quota", user)
    print "---------"

    for user in users_ldap_quota_mismatch:
        print_data("WARNING: quota does not match LDAP", user)
    print "---------"

    for zfs_set_cmd in zfs_set_cmds:
        logger.debug("Executing: zfs %s", " ".join(zfs_set_cmd))
        if args.noop:
            pass
        else:
            try:
                zfs_set_output = zfs(zfs_set_cmd)
            except ErrorReturnCode:
                logger.error("FAILED to execute zfs set: %s", zfs_set_output)
Beispiel #16
0
def install_os(wipe_first):
    """ Install the OS into the presumably mounted datasets """
    if wipe_first:
        zfs_create(wipe_first)

    username = input('Admin user\'s username?: ')
    password = getpass.getpass('Admin user\'s password?: ')
    password2 = getpass.getpass('Confirm password: '******'Passwords did not match'

    db_tarball_fpath = config.cache_dpath / 'debootstrap.tar'
    if not db_tarball_fpath.exists():
        sh.debootstrap('--make-tarball', db_tarball_fpath, config.release_codename, '/tmp/not-there', _fg=True)

    if not paths.zroot.joinpath('bin').exists():
        sh.zfs('set', 'devices=on', config.os_root_ds)
        sh.debootstrap('--unpack-tarball', db_tarball_fpath, config.release_codename, paths.zroot, _fg=True)
        sh.zfs('set', 'devices=off', config.os_root_ds)

    boot_dpath = paths.zroot / 'boot'
    boot_dpath.joinpath('refind_linux.conf').write_text(boot_refind_conf_tpl.format(zfs_os_root_ds=config.os_root_ds))

    etc_fpath = paths.zroot / 'etc'
    etc_fpath.joinpath('apt', 'sources.list').write_text(apt_sources_list.format(codename=config.release_codename))
    etc_fpath.joinpath('fstab').write_text(etc_fstab_tpl)
    etc_fpath.joinpath('hostname').write_text(config.hostname)
    etc_fpath.joinpath('hosts').write_text(etc_hosts_tpl.format(hostname=config.hostname))

    other_mounts()

    # Customize OS in chroot
    # ----------------------
    chroot = sh.chroot.bake(paths.zroot)

    chroot('apt', 'update')

    chroot('locale-gen', '--purge', 'en_US.UTF-8', _fg=True)
    chroot('update-locale', LANG="en_US.UTF-8", LANGUAGE="en_US:en", _fg=True)

    chroot('ln', '-fs', '/usr/share/zoneinfo/US/Eastern', '/etc/localtime', _fg=True)
    chroot('dpkg-reconfigure', '-f', 'noninteractive', 'tzdata', _fg=True)

    # Have to install the kernel and zfs-initramfs so that ZFS is installed and creating the user's
    # dataset below works.
    chroot('apt', 'install', '--yes', '--no-install-recommends', 'linux-image-generic', _fg=True)
    chroot('apt', 'install', '--yes', 'zfs-initramfs', _fg=True)

    # `update-initramfs -uk all` doesn't work, see:
    # https://bugs.launchpad.net/ubuntu/+source/initramfs-tools/+bug/1829805
    for kernel_version in kernels_in_boot():
        chroot('update-initramfs', '-uk', kernel_version, _fg=True)

    # Create user
    # Todo should each user have their own dataset for their home directory?
    # chroot.zfs.create(f'{config.pool_name}/home/{username}')
    chroot.adduser('--disabled-login', '--gecos=,', username)
    chroot.chpasswd(_in=f'{username}:{password}')

    chroot.addgroup('--system', 'lpadmin')
    chroot.addgroup('--system', 'sambashare')
    chroot.addgroup('--system', 'netdev')
    chroot.usermod('-a', '-G', 'adm,cdrom,dip,lpadmin,netdev,plugdev,sambashare,sudo', username)

    # Full OS Install
    chroot.apt('dist-upgrade', '--yes', _fg=True)
    chroot.apt('install', '--yes', 'xubuntu-desktop', _fg=True)
Beispiel #17
0
def install_os(wipe_first):
    """ Install the OS into the presumably mounted datasets """
    if wipe_first:
        zfs_create(wipe_first)
        print('Sleeping 5 seconds to give zfs datasets time to mount...')
        time.sleep(5)

    db_tarball_fpath = config.cache_dpath / 'debootstrap.tar'
    if not db_tarball_fpath.exists():
        sh.debootstrap('--make-tarball',
                       db_tarball_fpath,
                       config.release_codename,
                       '/tmp/not-there',
                       _fg=True)

    if not paths.zroot.joinpath('bin').exists():
        sh.zfs('set', 'devices=on', config.os_root_ds)
        sh.debootstrap('--unpack-tarball',
                       db_tarball_fpath,
                       config.release_codename,
                       paths.zroot,
                       _fg=True)
        sh.zfs('set', 'devices=off', config.os_root_ds)

    other_mounts()

    boot_dpath = paths.zroot / 'boot'
    refind_conf_content = boot_refind_conf_tpl.format(
        zfs_os_root_ds=config.os_root_ds)
    boot_dpath.joinpath('refind_linux.conf').write_text(refind_conf_content)

    etc_fpath = paths.zroot / 'etc'

    sources_list_content = apt_sources_list.format(
        codename=config.release_codename)
    etc_fpath.joinpath('apt', 'sources.list').write_text(sources_list_content)

    fstab_content = etc_fstab_tpl.format(config=config)
    etc_fpath.joinpath('fstab').write_text(fstab_content)

    etc_fpath.joinpath('hostname').write_text(config.hostname)

    etc_hosts_content = etc_hosts_tpl.format(hostname=config.hostname)
    etc_fpath.joinpath('hosts').write_text(etc_hosts_content)

    etc_fpath.joinpath('apt', 'apt.conf.d', '01-proxy').write_text(
        'Acquire::http { Proxy "http://server.lan:3142"; };')

    # Customize OS in chroot
    # ----------------------
    chroot = sh.chroot.bake(paths.zroot)

    chroot('apt', 'update')

    chroot('locale-gen', '--purge', 'en_US.UTF-8', _fg=True)
    chroot('update-locale', LANG="en_US.UTF-8", LANGUAGE="en_US:en", _fg=True)

    chroot('ln',
           '-fs',
           '/usr/share/zoneinfo/US/Eastern',
           '/etc/localtime',
           _fg=True)
    chroot('dpkg-reconfigure', '-f', 'noninteractive', 'tzdata', _fg=True)

    # Have to install the kernel and zfs-initramfs so that ZFS is installed and creating the user's
    # dataset below works.
    chroot('apt',
           'install',
           '--yes',
           '--no-install-recommends',
           'linux-image-generic',
           _fg=True)
    chroot('apt', 'install', '--yes', 'zfs-initramfs', _fg=True)

    # `update-initramfs -uk all` doesn't work, see:
    # https://bugs.launchpad.net/ubuntu/+source/initramfs-tools/+bug/1829805
    for kernel_version in kernels_in_boot():
        chroot('update-initramfs', '-uk', kernel_version, _fg=True)
Beispiel #18
0
def local_size():
	return int(sh.zfs('list',o='refer',H=True,p='xenfs/vms/%s/local' % sys.argv[1]))