def test_loop_creation_no_truncate(self, path_exists, create_loopback,
loopbacks):
"""It does not create a new sparse image for loopback if one exists"""
loopbacks.return_value = {}
path_exists.return_value = True
with patch('subprocess.check_call') as check_call:
loopback.ensure_loopback_device('/tmp/foo.img', '15G')
self.assertFalse(check_call.called)
def test_ensure_loopback_creation(self, path_exists, create_loopback,
loopbacks):
"""It creates a new sparse image for loopback if one does not exists"""
loopbacks.return_value = {}
path_exists.return_value = False
create_loopback.return_value = '/dev/loop0'
with patch(STORAGE_LINUX_LOOPBACK + '.check_call') as check_call:
loopback.ensure_loopback_device('/tmp/foo.img', '15G')
check_call.assert_called_with(
['truncate', '--size', '15G', '/tmp/foo.img'])
def ensure_block_device(block_device):
'''
Confirm block_device, create as loopback if necessary.
:param block_device: str: Full path of block device to ensure.
:returns: str: Full path of ensured block device.
'''
_none = ['None', 'none', None]
if (block_device in _none):
error_out('prepare_storage(): Missing required input: block_device=%s.'
% block_device)
if block_device.startswith('/dev/'):
bdev = block_device
elif block_device.startswith('/'):
_bd = block_device.split('|')
if len(_bd) == 2:
bdev, size = _bd
else:
bdev = block_device
size = DEFAULT_LOOPBACK_SIZE
bdev = ensure_loopback_device(bdev, size)
else:
bdev = '/dev/%s' % block_device
if not is_block_device(bdev):
error_out('Failed to locate valid block device at %s' % bdev)
return bdev
def test_loopback_create_already_exists(self, loopbacks, check_call,
create):
"""It finds existing loopback device for requested file"""
loopbacks.return_value = {'/dev/loop1': '/tmp/bar.img'}
res = loopback.ensure_loopback_device('/tmp/bar.img', '5G')
self.assertEquals(res, '/dev/loop1')
self.assertFalse(create.called)
self.assertFalse(check_call.called)
def configure_lxd_block():
'''Configure a block device for use by LXD for containers'''
log('Configuring LXD container storage if not configured')
# determine if the lxd block has already been configured
if has_storage(LXD_POOL):
log("LXD storage pool {} already configured".format(LXD_POOL))
return
elif filesystem_mounted('/var/lib/lxd'):
log('/var/lib/lxd already configured, skipping')
return
lxd_block_devices = get_block_devices()
if len(lxd_block_devices) < 1:
log('block devices not provided - skipping')
return
if len(lxd_block_devices) > 1:
log("More than one block device is not supported yet, only"
" using the first")
lxd_block_device = lxd_block_devices[0]
dev = None
if lxd_block_device.startswith('/dev/'):
dev = lxd_block_device
elif lxd_block_device.startswith('/'):
log('Configuring loopback device for use with LXD')
_bd = lxd_block_device.split('|')
if len(_bd) == 2:
dev, size = _bd
else:
dev = lxd_block_device
size = DEFAULT_LOOPBACK_SIZE
dev = ensure_loopback_device(dev, size)
if not dev or not is_block_device(dev):
log('Invalid block device provided: %s' % lxd_block_device)
return
# NOTE: check overwrite and ensure its only execute once.
db = kv()
if config('overwrite') and not db.get('scrubbed', False):
clean_storage(dev)
db.set('scrubbed', True)
db.flush()
if not os.path.exists('/var/lib/lxd'):
mkdir('/var/lib/lxd')
if config('storage-type') == 'btrfs':
config_btrfs(dev)
elif config('storage-type') == 'lvm':
config_lvm(dev)
elif config('storage-type') == 'zfs':
config_zfs(dev)
def configure_lxd_block():
'''Configure a block device for use by LXD for containers'''
log('Configuring LXD container storage if not configured')
# determine if the lxd block has already been configured
if has_storage(LXD_POOL):
log("LXD storage pool {} already configured".format(LXD_POOL))
return
elif filesystem_mounted('/var/lib/lxd'):
log('/var/lib/lxd already configured, skipping')
return
lxd_block_devices = get_block_devices()
if len(lxd_block_devices) < 1:
log('block devices not provided - skipping')
return
if len(lxd_block_devices) > 1:
log("More than one block device is not supported yet, only"
" using the first")
lxd_block_device = lxd_block_devices[0]
dev = None
if lxd_block_device.startswith('/dev/'):
dev = lxd_block_device
elif lxd_block_device.startswith('/'):
log('Configuring loopback device for use with LXD')
_bd = lxd_block_device.split('|')
if len(_bd) == 2:
dev, size = _bd
else:
dev = lxd_block_device
size = DEFAULT_LOOPBACK_SIZE
dev = ensure_loopback_device(dev, size)
if not dev or not is_block_device(dev):
log('Invalid block device provided: %s' % lxd_block_device)
return
# NOTE: check overwrite and ensure its only execute once.
db = kv()
if config('overwrite') and not db.get('scrubbed', False):
clean_storage(dev)
db.set('scrubbed', True)
db.flush()
if not os.path.exists('/var/lib/lxd'):
mkdir('/var/lib/lxd')
if config('storage-type') == 'btrfs':
config_btrfs(dev)
elif config('storage-type') == 'lvm':
config_lvm(dev)
elif config('storage-type') == 'zfs':
config_zfs(dev)
def ensure_block_device(block_device):
'''
Confirm block_device, create as loopback if necessary.
:param block_device: str: Full path of block device to ensure.
:returns: str: Full path of ensured block device.
'''
if block_device.startswith("/"):
# Resolve non-relative link(s) if device is a symlink to a real device.
# This fixes/prevents bug #1577408.
real_device = os.path.realpath(block_device)
if real_device != block_device:
log('Block device "{}" is a symlink to "{}". Using "{}".'.format(
block_device, real_device, real_device),
level=INFO)
block_device = real_device
_none = ['None', 'none', None]
if (block_device in _none):
log('prepare_storage(): Missing required input: '
'block_device=%s.' % block_device,
level=ERROR)
raise
if block_device.startswith('/dev/'):
bdev = block_device
elif block_device.startswith('/'):
_bd = block_device.split('|')
if len(_bd) == 2:
bdev, size = _bd
else:
bdev = block_device
size = DEFAULT_LOOPBACK_SIZE
bdev = ensure_loopback_device(bdev, size)
else:
bdev = '/dev/%s' % block_device
if not is_block_device(bdev):
log('Failed to locate valid block device at %s' % bdev, level=ERROR)
# ignore missing block devices
return
return bdev
def ensure_block_device(block_device):
'''
Confirm block_device, create as loopback if necessary.
:param block_device: str: Full path of block device to ensure.
:returns: str: Full path of ensured block device.
'''
if block_device.startswith("/"):
# Resolve non-relative link(s) if device is a symlink to a real device.
# This fixes/prevents bug #1577408.
real_device = os.path.realpath(block_device)
if real_device != block_device:
log('Block device "{}" is a symlink to "{}". Using "{}".'.format(
block_device, real_device, real_device), level=INFO)
block_device = real_device
_none = ['None', 'none', None]
if (block_device in _none):
log('prepare_storage(): Missing required input: '
'block_device=%s.' % block_device, level=ERROR)
raise
if block_device.startswith('/dev/'):
bdev = block_device
elif block_device.startswith('/'):
_bd = block_device.split('|')
if len(_bd) == 2:
bdev, size = _bd
else:
bdev = block_device
size = DEFAULT_LOOPBACK_SIZE
bdev = ensure_loopback_device(bdev, size)
else:
bdev = '/dev/%s' % block_device
if not is_block_device(bdev):
log('Failed to locate valid block device at %s' % bdev, level=ERROR)
# ignore missing block devices
return
return bdev
def configure_lvm_storage(block_devices,
volume_group,
overwrite=False,
remove_missing=False,
remove_missing_force=False):
''' Configure LVM storage on the list of block devices provided
:param block_devices: list: List of whitelisted block devices to detect
and use if found
:param overwrite: bool: Scrub any existing block data if block device is
not already in-use
:param remove_missing: bool: Remove missing physical volumes from volume
group if logical volume not allocated on them
:param remove_missing_force: bool: Remove missing physical volumes from
volume group even if logical volumes are allocated
on them. Overrides 'remove_missing' if set.
'''
log_lvm_info()
devices = []
for block_device in block_devices:
(block_device, size) = _parse_block_device(block_device)
if not is_device_mounted(block_device):
if size == 0 and is_block_device(block_device):
devices.append(block_device)
elif size > 0:
devices.append(ensure_loopback_device(block_device, str(size)))
# NOTE(jamespage)
# might need todo an initial one-time scrub on install if need be
vg_found = False
new_devices = []
for device in devices:
if not is_lvm_physical_volume(device):
# Unused device
if overwrite is True or not has_partition_table(device):
prepare_volume(device)
new_devices.append(device)
elif (is_lvm_physical_volume(device)
and list_lvm_volume_group(device) != volume_group):
# Existing LVM but not part of required VG or new device
if overwrite is True:
prepare_volume(device)
new_devices.append(device)
elif (is_lvm_physical_volume(device)
and list_lvm_volume_group(device) == volume_group):
# Mark vg as found
vg_found = True
log_lvm_info()
if vg_found is False and len(new_devices) > 0:
if overwrite:
ensure_lvm_volume_group_non_existent(volume_group)
# Create new volume group from first device
create_lvm_volume_group(volume_group, new_devices[0])
new_devices.remove(new_devices[0])
# Remove missing physical volumes from volume group
try:
if remove_missing_force:
reduce_lvm_volume_group_missing(volume_group,
extra_args=['--force'])
elif remove_missing:
reduce_lvm_volume_group_missing(volume_group)
except subprocess.CalledProcessError as e:
juju_log(
"reduce_lvm_volume_group_missing() didn't complete."
" LVM may not be fully configured yet. Error was: '{}'.".format(
str(e)))
if len(new_devices) > 0:
# Extend the volume group as required
for new_device in new_devices:
extend_lvm_volume_group(volume_group, new_device)
thin_pools = list_thin_logical_volume_pools(path_mode=True)
if len(thin_pools) == 0:
juju_log("No thin pools found")
elif len(thin_pools) == 1:
juju_log("Thin pool {} found, extending with {}".format(
thin_pools[0], new_device))
extend_logical_volume_by_device(thin_pools[0], new_device)
else:
juju_log("Multiple thin pools ({}) found, "
"skipping auto extending with {}".format(
','.join(thin_pools), new_device))
log_lvm_info()
def configure_lxd_block():
'''Configure a block device for use by LXD for containers'''
log('Configuring LXD container storage')
if filesystem_mounted('/var/lib/lxd'):
log('/var/lib/lxd already configured, skipping')
return
lxd_block_devices = get_block_devices()
if len(lxd_block_devices) < 1:
log('block devices not provided - skipping')
return
if len(lxd_block_devices) > 1:
raise NotImplementedError('Multiple block devices are not supported.')
lxd_block_device = lxd_block_devices[0]
dev = None
if lxd_block_device.startswith('/dev/'):
dev = lxd_block_device
elif lxd_block_device.startswith('/'):
log('Configuring loopback device for use with LXD')
_bd = lxd_block_device.split('|')
if len(_bd) == 2:
dev, size = _bd
else:
dev = lxd_block_device
size = DEFAULT_LOOPBACK_SIZE
dev = ensure_loopback_device(dev, size)
if not dev or not is_block_device(dev):
log('Invalid block device provided: %s' % lxd_block_device)
return
# NOTE: check overwrite and ensure its only execute once.
db = kv()
if config('overwrite') and not db.get('scrubbed', False):
clean_storage(dev)
db.set('scrubbed', True)
db.flush()
if not os.path.exists('/var/lib/lxd'):
mkdir('/var/lib/lxd')
if config('storage-type') == 'btrfs':
status_set('maintenance',
'Configuring btrfs container storage')
lxd_stop()
cmd = ['mkfs.btrfs', '-f', dev]
check_call(cmd)
mount(dev,
'/var/lib/lxd',
options='user_subvol_rm_allowed',
persist=True,
filesystem='btrfs')
cmd = ['btrfs', 'quota', 'enable', '/var/lib/lxd']
check_call(cmd)
lxd_start()
elif config('storage-type') == 'lvm':
if (is_lvm_physical_volume(dev) and
list_lvm_volume_group(dev) == 'lxd_vg'):
log('Device already configured for LVM/LXD, skipping')
return
status_set('maintenance',
'Configuring LVM container storage')
# Enable and startup lvm2-lvmetad to avoid extra output
# in lvm2 commands, which confused lxd.
cmd = ['systemctl', 'enable', 'lvm2-lvmetad']
check_call(cmd)
cmd = ['systemctl', 'start', 'lvm2-lvmetad']
check_call(cmd)
create_lvm_physical_volume(dev)
create_lvm_volume_group('lxd_vg', dev)
cmd = ['lxc', 'config', 'set', 'storage.lvm_vg_name', 'lxd_vg']
check_call(cmd)
# The LVM thinpool logical volume is lazily created, either on
# image import or container creation. This will force LV creation.
create_and_import_busybox_image()
elif config('storage-type') == 'zfs':
status_set('maintenance',
'Configuring zfs container storage')
if ZFS_POOL_NAME in zpools():
log('ZFS pool already exist; skipping zfs configuration')
return
if config('overwrite'):
cmd = ['zpool', 'create', '-f', ZFS_POOL_NAME, dev]
else:
cmd = ['zpool', 'create', ZFS_POOL_NAME, dev]
check_call(cmd)
cmd = ['lxc', 'config', 'set', 'storage.zfs_pool_name',
ZFS_POOL_NAME]
check_call(cmd)
def configure_lvm_storage(block_devices, volume_group, overwrite=False,
remove_missing=False, remove_missing_force=False):
''' Configure LVM storage on the list of block devices provided
:param block_devices: list: List of whitelisted block devices to detect
and use if found
:param overwrite: bool: Scrub any existing block data if block device is
not already in-use
:param remove_missing: bool: Remove missing physical volumes from volume
group if logical volume not allocated on them
:param remove_missing_force: bool: Remove missing physical volumes from
volume group even if logical volumes are allocated
on them. Overrides 'remove_missing' if set.
'''
log_lvm_info()
devices = []
for block_device in block_devices:
(block_device, size) = _parse_block_device(block_device)
if not is_device_mounted(block_device):
if size == 0 and is_block_device(block_device):
devices.append(block_device)
elif size > 0:
devices.append(ensure_loopback_device(block_device, size))
# NOTE(jamespage)
# might need todo an initial one-time scrub on install if need be
vg_found = False
new_devices = []
for device in devices:
if not is_lvm_physical_volume(device):
# Unused device
if overwrite is True or not has_partition_table(device):
prepare_volume(device)
new_devices.append(device)
elif (is_lvm_physical_volume(device) and
list_lvm_volume_group(device) != volume_group):
# Existing LVM but not part of required VG or new device
if overwrite is True:
prepare_volume(device)
new_devices.append(device)
elif (is_lvm_physical_volume(device) and
list_lvm_volume_group(device) == volume_group):
# Mark vg as found
vg_found = True
log_lvm_info()
if vg_found is False and len(new_devices) > 0:
if overwrite:
ensure_lvm_volume_group_non_existent(volume_group)
# Create new volume group from first device
create_lvm_volume_group(volume_group, new_devices[0])
new_devices.remove(new_devices[0])
# Remove missing physical volumes from volume group
if remove_missing_force:
reduce_lvm_volume_group_missing(volume_group, extra_args=['--force'])
elif remove_missing:
reduce_lvm_volume_group_missing(volume_group)
if len(new_devices) > 0:
# Extend the volume group as required
for new_device in new_devices:
extend_lvm_volume_group(volume_group, new_device)
log_lvm_info()
def configure_lvm_storage(block_devices,
volume_group,
overwrite=False,
remove_missing=False,
remove_missing_force=False):
''' Configure LVM storage on the list of block devices provided
:param block_devices: list: List of whitelisted block devices to detect
and use if found
:param overwrite: bool: Scrub any existing block data if block device is
not already in-use
:param remove_missing: bool: Remove missing physical volumes from volume
group if logical volume not allocated on them
:param remove_missing_force: bool: Remove missing physical volumes from
volume group even if logical volumes are allocated
on them. Overrides 'remove_missing' if set.
'''
log_lvm_info()
devices = []
for block_device in block_devices:
(block_device, size) = _parse_block_device(block_device)
if not is_device_mounted(block_device):
if size == 0 and is_block_device(block_device):
devices.append(block_device)
elif size > 0:
devices.append(ensure_loopback_device(block_device, size))
# NOTE(jamespage)
# might need todo an initial one-time scrub on install if need be
vg_found = False
new_devices = []
for device in devices:
if not is_lvm_physical_volume(device):
# Unused device
if overwrite is True or not has_partition_table(device):
prepare_volume(device)
new_devices.append(device)
elif (is_lvm_physical_volume(device)
and list_lvm_volume_group(device) != volume_group):
# Existing LVM but not part of required VG or new device
if overwrite is True:
prepare_volume(device)
new_devices.append(device)
elif (is_lvm_physical_volume(device)
and list_lvm_volume_group(device) == volume_group):
# Mark vg as found
vg_found = True
log_lvm_info()
if vg_found is False and len(new_devices) > 0:
if overwrite:
ensure_lvm_volume_group_non_existent(volume_group)
# Create new volume group from first device
create_lvm_volume_group(volume_group, new_devices[0])
new_devices.remove(new_devices[0])
# Remove missing physical volumes from volume group
if remove_missing_force:
reduce_lvm_volume_group_missing(volume_group, extra_args=['--force'])
elif remove_missing:
reduce_lvm_volume_group_missing(volume_group)
if len(new_devices) > 0:
# Extend the volume group as required
for new_device in new_devices:
extend_lvm_volume_group(volume_group, new_device)
log_lvm_info()
def configure_lxd_block():
'''Configure a block device for use by LXD for containers'''
log('Configuring LXD container storage')
if filesystem_mounted('/var/lib/lxd'):
log('/var/lib/lxd already configured, skipping')
return
lxd_block_devices = get_block_devices()
if len(lxd_block_devices) < 1:
log('block devices not provided - skipping')
return
if len(lxd_block_devices) > 1:
log("More than one block device is not supported yet, only"
" using the first")
lxd_block_device = lxd_block_devices[0]
dev = None
if lxd_block_device.startswith('/dev/'):
dev = lxd_block_device
elif lxd_block_device.startswith('/'):
log('Configuring loopback device for use with LXD')
_bd = lxd_block_device.split('|')
if len(_bd) == 2:
dev, size = _bd
else:
dev = lxd_block_device
size = DEFAULT_LOOPBACK_SIZE
dev = ensure_loopback_device(dev, size)
if not dev or not is_block_device(dev):
log('Invalid block device provided: %s' % lxd_block_device)
return
# NOTE: check overwrite and ensure its only execute once.
db = kv()
if config('overwrite') and not db.get('scrubbed', False):
clean_storage(dev)
db.set('scrubbed', True)
db.flush()
if not os.path.exists('/var/lib/lxd'):
mkdir('/var/lib/lxd')
if config('storage-type') == 'btrfs':
status_set('maintenance', 'Configuring btrfs container storage')
lxd_stop()
cmd = ['mkfs.btrfs', '-f', dev]
check_call(cmd)
mount(dev,
'/var/lib/lxd',
options='user_subvol_rm_allowed',
persist=True,
filesystem='btrfs')
cmd = ['btrfs', 'quota', 'enable', '/var/lib/lxd']
check_call(cmd)
lxd_start()
elif config('storage-type') == 'lvm':
if (is_lvm_physical_volume(dev)
and list_lvm_volume_group(dev) == 'lxd_vg'):
log('Device already configured for LVM/LXD, skipping')
return
status_set('maintenance', 'Configuring LVM container storage')
# Enable and startup lvm2-lvmetad to avoid extra output
# in lvm2 commands, which confused lxd.
cmd = ['systemctl', 'enable', 'lvm2-lvmetad']
check_call(cmd)
cmd = ['systemctl', 'start', 'lvm2-lvmetad']
check_call(cmd)
create_lvm_physical_volume(dev)
create_lvm_volume_group('lxd_vg', dev)
cmd = ['lxc', 'config', 'set', 'storage.lvm_vg_name', 'lxd_vg']
check_call(cmd)
# The LVM thinpool logical volume is lazily created, either on
# image import or container creation. This will force LV creation.
create_and_import_busybox_image()
elif config('storage-type') == 'zfs':
status_set('maintenance', 'Configuring zfs container storage')
if ZFS_POOL_NAME in zpools():
log('ZFS pool already exist; skipping zfs configuration')
return
if config('overwrite'):
cmd = ['zpool', 'create', '-f', ZFS_POOL_NAME, dev]
else:
cmd = ['zpool', 'create', ZFS_POOL_NAME, dev]
check_call(cmd)
cmd = ['lxc', 'config', 'set', 'storage.zfs_pool_name', ZFS_POOL_NAME]
check_call(cmd)
