def create(self, size, sparse=False, block_size=None, mkparent=False):
"""Creates storage volume.
volume = Volume('dpool/tmp/test0')
volume.create()
"""
# TODO Check size to make sure it's decent.
try:
# -V volume, -s sparse, -b blocksize, -o prop=val
# -p works like mkdir -p, creates non-existing parent datasets.
opts = ['create']
if sparse:
opts.append('-s')
if block_size:
opts.extend(['-b', block_size])
if mkparent:
opts.append('-p')
opts.extend(['-V', size, self.name])
sh.zfs(*opts)
except sh.ErrorReturnCode_1:
# I'm not sure about this; the problem is if it creates the
# dataset, but fails to mount it for some reason, we're left with
# the pieces and a simple 1 retval...
#if self.exists():
# self.destroy()
raise
# TODO Force scan of this in bg
return True
def recover(chroot):
""" Import pool and mount filesystem in prep for recovery efforts """
sh.zpool.export('-a', _fg=True)
sh.zpool('import', '-Nf', '-R', paths.zroot, config.pool_name, _fg=True)
sh.zfs('load-key', '-a', _fg=True)
sh.zfs.mount(config.os_root_ds, _fg=True)
sh.zfs.mount('-a', _fg=True)
other_mounts()
if chroot:
sh.chroot(paths.zroot, '/bin/bash', '--login', _fg=True)
def exists(self):
"""Checks if filesystem exists.
filesystem = Filesystem('dpool/tmp/test0')
filesystem.exists()
"""
try:
sh.zfs('list', '-t', 'filesystem', self.name)
except sh.ErrorReturnCode_1:
return False
return True
def exists(self):
"""Checks if volume exists.
volume = Volume('dpool/tmp/test0')
volume.exists()
"""
try:
sh.zfs('list', '-t', 'volume', self.name)
except sh.ErrorReturnCode_1:
return False
# TODO Force scan of this in bg
return True
def exists(self):
"""Checks if snapshot exists.
snapshot = Snapshot('dpool/tmp/test0@whoa-snap-0')
snapshot.exists()
"""
try:
sh.zfs('list', '-t', 'snapshot', self.name)
except sh.ErrorReturnCode_1:
return False
# TODO Force scan of this in bg
return True
def destroy(self, confirm=False, recursive=False):
"""Destroys storage filesystem.
filesystem = Filesystem('dpool/tmp/test0')
filesystem.destroy()
"""
if confirm is not True:
raise ZfsError('Destroy of storage filesystem requires confirm=True')
opts = ['destroy']
if recursive:
opts.append('-r')
opts.append(self.name)
sh.zfs(*opts)
# TODO Force delete of this in bg (with '-d')
return True
def destroy(self, confirm=False, recursive=False):
"""Destroys storage snapshot.
snapshot = Snapshot('dpool/tmp/test0@whoa-snap-0')
snapshot.destroy()
"""
if confirm is not True:
raise ZfsError('Destroy of storage snapshot requires confirm=True')
opts = ['destroy']
if recursive:
opts.append('-r')
opts.append(self.name)
sh.zfs(*opts)
# TODO Force delete of this in bg
return True
def rename(self, new):
"""Renames storage filesystem.
filesystem = Filesystem('dpool/r0')
filesystem.rename('dpool/r1')
"""
old = self.name
logging.info("Renaming dataset '%s' to '%s'", old, new)
try:
sh.zfs('rename', old, new)
self.name = new
except:
self.name = old
raise
finally:
self.save()
def rename(self, new):
"""Renames storage volume.
volume = Volume('dpool/r0')
volume.rename('dpool/r1')
"""
old = self.name
logger.info("Renaming dataset '%s' to '%s'", old, new)
try:
sh.zfs('rename', old, new)
self.name = new
except:
self.name = old
raise
finally:
self.save()
def destroy(self, confirm=False, recursive=False):
"""Destroys storage volume.
volume = Volume('dpool/tmp/test0')
volume.destroy()
"""
if confirm is not True:
raise ZfsError("It is required that you specify confirm=True for such an action of destruction.")
opts = ['destroy']
if recursive:
opts.append('-r')
opts.append(self.name)
sh.zfs(*opts)
# TODO Force delete of this in bg
return True
def status():
print(f'$ config values --------------------\n')
print(config)
print(f'\n$ sgdisk --print {config.disk_dev} --------------------\n')
sh.sgdisk('--print', config.disk_dev, _out=sys.stdout, _ok_code=[0,2])
print(f'\n$ blkid --------------------------\n')
sh.blkid(_out=sys.stdout,)
print('\n$ zpool list ---------------------------\n')
sh.zpool('list', _out=sys.stdout)
print('\n$ zfs list ---------------------------\n')
sh.zfs('list', _out=sys.stdout)
print('\n$ zfs mount ---------------------------\n')
sh.zfs('mount', _out=sys.stdout)
def _set(self, k, v, ignore=False):
"""Sets Dataset property.
dataset = Dataset('dpool/carp')
dataset.properties._set('readonly', 'on')
"""
if ignore:
return
prop = None
if isinstance(v, DatasetProperty):
prop = v
v = prop.value
sh.zfs('set', '%s=%s' % (k, v), self._parent.name)
if prop:
prop.value = v
def create(self):
"""Creates storage filesystem.
filesystem = Filesystem('dpool/tmp/test0')
filesystem.create()
"""
try:
sh.zfs('create', self.name)
except sh.ErrorReturnCode_1:
# I'm not sure about this; the problem is if it creates the
# dataset, but fails to mount it for some reason, we're left with
# the pieces and a simple 1 retval...
#if self.exists():
# self.destroy()
raise
# TODO Force scan of this in bg
return True
def create(self, size):
"""Creates storage snapshot.
snapshot = Snapshot('dpool/tmp/test0@whoa-snap-0')
snapshot.create()
"""
# TODO Check size to make sure it's decent.
try:
sh.zfs('snapshot', self.name)
except sh.ErrorReturnCode_1:
# I'm not sure about this; the problem is if it creates the
# dataset, but fails to mount it for some reason, we're left with
# the pieces and a simple 1 retval...
#if self.exists():
# self.destroy()
raise
# TODO Force scan of this in bg
return True
def main():
args = parse_args()
config = load_config()
config_env = config[args.config_env]["ldap"]
# Setup logging
setup_logging(debug=args.debug, noop=False)
logger.debug4("OPTIONS: %s" % vars(args))
logger.debug4("CONFIG: %s" % config_env)
_ldap_url = config_env.get("url")
_use_tls = config_env.get("tls")
_bind_dn = config_env.get("bind_dn", None)
_bind_pass = config_env.get("bind_pass", None)
local_ldap = LocalLdap(url=_ldap_url[0], use_tls=_use_tls, bind_dn=_bind_dn, bind_pass=_bind_pass, log_level=None)
ldap_users = local_ldap.paged_search(base=search_base, sfilter=search_filter, attrlist=search_return_attribs, scope=search_scope)
users_over_quota = []
users_over_ldap_quota = []
users_over_zfs_quota = []
users_ldap_quota_mismatch = []
zfs_set_cmds = []
for user in ldap_users:
_user_data = {}
_user = LdapUser()
_user.setattrs(data=user, listvals=["mail"])
_username = _user.uid
_uid = _user.uidNumber
_shell = _user.loginShell
_quota = _user.quota
if hasattr(_user, "mail"):
_mail = ",".join(_user.mail)
else:
_mail = ""
if active_only and _shell != "/bin/bash":
continue
mount, softlimit, hardlimit, softinode, hardinode = re.findall(r"^(.*):([0-9]+),([0-9]+),([0-9]+),([0-9]+)$", _quota)[0]
_ldap_quota = int(hardlimit) * 1024
zfs_fs = "tank%s" % mount
# Get current ZFS quota
userquota_args = ["get", "-H", "-p", "-o", "value", "userquota@%s" % _username, zfs_fs]
logger.debug("Executing: zfs %s", " ".join(userquota_args))
userquota_output = zfs(userquota_args)
_userquota = userquota_output.strip()
if _userquota != "-":
current_quota = int(_userquota)
else:
current_quota = 0
# Get current used space
userused_args = ["get", "-H", "-p", "-o", "value", "userused@%s" % _username, zfs_fs]
logger.debug("Executing: zfs %s", " ".join(userused_args))
userused_output = zfs(userused_args)
_userused = userused_output.strip()
if _userused != "-":
current_used = int(_userused)
else:
current_used = 0
_user_data["username"] = _username
_user_data["uid"] = _uid
_user_data["mail"] = _mail
_user_data["zfs_fs"] = zfs_fs
_user_data["ldap_quota"] = _ldap_quota
_user_data["zfs_quota"] = current_quota
_user_data["zfs_used"] = current_used
if current_used >= _ldap_quota and current_used >= current_quota:
users_over_quota.append(_user_data)
elif current_used and current_used >= _ldap_quota:
users_over_ldap_quota.append(_user_data)
elif current_used and current_used >= current_quota:
users_over_zfs_quota.append(_user_data)
if _ldap_quota != current_quota:
users_ldap_quota_mismatch.append(_user_data)
zfs_set_cmd = [
"set", "userquota@%s=%s" % (_username, _ldap_quota), zfs_fs
]
zfs_set_cmds.append(zfs_set_cmd)
for user in users_over_quota:
print_data("WARNING: over quota", user)
print "---------"
for user in users_over_ldap_quota:
print_data("WARNING: over LDAP quota", user)
print "---------"
for user in users_over_zfs_quota:
print_data("WARNING: over ZFS quota", user)
print "---------"
for user in users_ldap_quota_mismatch:
print_data("WARNING: quota does not match LDAP", user)
print "---------"
for zfs_set_cmd in zfs_set_cmds:
logger.debug("Executing: zfs %s", " ".join(zfs_set_cmd))
if args.noop:
pass
else:
try:
zfs_set_output = zfs(zfs_set_cmd)
except ErrorReturnCode:
logger.error("FAILED to execute zfs set: %s", zfs_set_output)
def install_os(wipe_first):
""" Install the OS into the presumably mounted datasets """
if wipe_first:
zfs_create(wipe_first)
username = input('Admin user\'s username?: ')
password = getpass.getpass('Admin user\'s password?: ')
password2 = getpass.getpass('Confirm password: '******'Passwords did not match'
db_tarball_fpath = config.cache_dpath / 'debootstrap.tar'
if not db_tarball_fpath.exists():
sh.debootstrap('--make-tarball', db_tarball_fpath, config.release_codename, '/tmp/not-there', _fg=True)
if not paths.zroot.joinpath('bin').exists():
sh.zfs('set', 'devices=on', config.os_root_ds)
sh.debootstrap('--unpack-tarball', db_tarball_fpath, config.release_codename, paths.zroot, _fg=True)
sh.zfs('set', 'devices=off', config.os_root_ds)
boot_dpath = paths.zroot / 'boot'
boot_dpath.joinpath('refind_linux.conf').write_text(boot_refind_conf_tpl.format(zfs_os_root_ds=config.os_root_ds))
etc_fpath = paths.zroot / 'etc'
etc_fpath.joinpath('apt', 'sources.list').write_text(apt_sources_list.format(codename=config.release_codename))
etc_fpath.joinpath('fstab').write_text(etc_fstab_tpl)
etc_fpath.joinpath('hostname').write_text(config.hostname)
etc_fpath.joinpath('hosts').write_text(etc_hosts_tpl.format(hostname=config.hostname))
other_mounts()
# Customize OS in chroot
# ----------------------
chroot = sh.chroot.bake(paths.zroot)
chroot('apt', 'update')
chroot('locale-gen', '--purge', 'en_US.UTF-8', _fg=True)
chroot('update-locale', LANG="en_US.UTF-8", LANGUAGE="en_US:en", _fg=True)
chroot('ln', '-fs', '/usr/share/zoneinfo/US/Eastern', '/etc/localtime', _fg=True)
chroot('dpkg-reconfigure', '-f', 'noninteractive', 'tzdata', _fg=True)
# Have to install the kernel and zfs-initramfs so that ZFS is installed and creating the user's
# dataset below works.
chroot('apt', 'install', '--yes', '--no-install-recommends', 'linux-image-generic', _fg=True)
chroot('apt', 'install', '--yes', 'zfs-initramfs', _fg=True)
# `update-initramfs -uk all` doesn't work, see:
# https://bugs.launchpad.net/ubuntu/+source/initramfs-tools/+bug/1829805
for kernel_version in kernels_in_boot():
chroot('update-initramfs', '-uk', kernel_version, _fg=True)
# Create user
# Todo should each user have their own dataset for their home directory?
# chroot.zfs.create(f'{config.pool_name}/home/{username}')
chroot.adduser('--disabled-login', '--gecos=,', username)
chroot.chpasswd(_in=f'{username}:{password}')
chroot.addgroup('--system', 'lpadmin')
chroot.addgroup('--system', 'sambashare')
chroot.addgroup('--system', 'netdev')
chroot.usermod('-a', '-G', 'adm,cdrom,dip,lpadmin,netdev,plugdev,sambashare,sudo', username)
# Full OS Install
chroot.apt('dist-upgrade', '--yes', _fg=True)
chroot.apt('install', '--yes', 'xubuntu-desktop', _fg=True)
def install_os(wipe_first):
""" Install the OS into the presumably mounted datasets """
if wipe_first:
zfs_create(wipe_first)
print('Sleeping 5 seconds to give zfs datasets time to mount...')
time.sleep(5)
db_tarball_fpath = config.cache_dpath / 'debootstrap.tar'
if not db_tarball_fpath.exists():
sh.debootstrap('--make-tarball',
db_tarball_fpath,
config.release_codename,
'/tmp/not-there',
_fg=True)
if not paths.zroot.joinpath('bin').exists():
sh.zfs('set', 'devices=on', config.os_root_ds)
sh.debootstrap('--unpack-tarball',
db_tarball_fpath,
config.release_codename,
paths.zroot,
_fg=True)
sh.zfs('set', 'devices=off', config.os_root_ds)
other_mounts()
boot_dpath = paths.zroot / 'boot'
refind_conf_content = boot_refind_conf_tpl.format(
zfs_os_root_ds=config.os_root_ds)
boot_dpath.joinpath('refind_linux.conf').write_text(refind_conf_content)
etc_fpath = paths.zroot / 'etc'
sources_list_content = apt_sources_list.format(
codename=config.release_codename)
etc_fpath.joinpath('apt', 'sources.list').write_text(sources_list_content)
fstab_content = etc_fstab_tpl.format(config=config)
etc_fpath.joinpath('fstab').write_text(fstab_content)
etc_fpath.joinpath('hostname').write_text(config.hostname)
etc_hosts_content = etc_hosts_tpl.format(hostname=config.hostname)
etc_fpath.joinpath('hosts').write_text(etc_hosts_content)
etc_fpath.joinpath('apt', 'apt.conf.d', '01-proxy').write_text(
'Acquire::http { Proxy "http://server.lan:3142"; };')
# Customize OS in chroot
# ----------------------
chroot = sh.chroot.bake(paths.zroot)
chroot('apt', 'update')
chroot('locale-gen', '--purge', 'en_US.UTF-8', _fg=True)
chroot('update-locale', LANG="en_US.UTF-8", LANGUAGE="en_US:en", _fg=True)
chroot('ln',
'-fs',
'/usr/share/zoneinfo/US/Eastern',
'/etc/localtime',
_fg=True)
chroot('dpkg-reconfigure', '-f', 'noninteractive', 'tzdata', _fg=True)
# Have to install the kernel and zfs-initramfs so that ZFS is installed and creating the user's
# dataset below works.
chroot('apt',
'install',
'--yes',
'--no-install-recommends',
'linux-image-generic',
_fg=True)
chroot('apt', 'install', '--yes', 'zfs-initramfs', _fg=True)
# `update-initramfs -uk all` doesn't work, see:
# https://bugs.launchpad.net/ubuntu/+source/initramfs-tools/+bug/1829805
for kernel_version in kernels_in_boot():
chroot('update-initramfs', '-uk', kernel_version, _fg=True)
def local_size():
return int(sh.zfs('list',o='refer',H=True,p='xenfs/vms/%s/local' % sys.argv[1]))