def test_loop_creation_no_truncate(self, path_exists, create_loopback, loopbacks): """It does not create a new sparse image for loopback if one exists""" loopbacks.return_value = {} path_exists.return_value = True with patch('subprocess.check_call') as check_call: loopback.ensure_loopback_device('/tmp/foo.img', '15G') self.assertFalse(check_call.called)
def test_ensure_loopback_creation(self, path_exists, create_loopback, loopbacks): """It creates a new sparse image for loopback if one does not exists""" loopbacks.return_value = {} path_exists.return_value = False create_loopback.return_value = '/dev/loop0' with patch(STORAGE_LINUX_LOOPBACK + '.check_call') as check_call: loopback.ensure_loopback_device('/tmp/foo.img', '15G') check_call.assert_called_with( ['truncate', '--size', '15G', '/tmp/foo.img'])
def ensure_block_device(block_device): ''' Confirm block_device, create as loopback if necessary. :param block_device: str: Full path of block device to ensure. :returns: str: Full path of ensured block device. ''' _none = ['None', 'none', None] if (block_device in _none): error_out('prepare_storage(): Missing required input: block_device=%s.' % block_device) if block_device.startswith('/dev/'): bdev = block_device elif block_device.startswith('/'): _bd = block_device.split('|') if len(_bd) == 2: bdev, size = _bd else: bdev = block_device size = DEFAULT_LOOPBACK_SIZE bdev = ensure_loopback_device(bdev, size) else: bdev = '/dev/%s' % block_device if not is_block_device(bdev): error_out('Failed to locate valid block device at %s' % bdev) return bdev
def test_loopback_create_already_exists(self, loopbacks, check_call, create): """It finds existing loopback device for requested file""" loopbacks.return_value = {'/dev/loop1': '/tmp/bar.img'} res = loopback.ensure_loopback_device('/tmp/bar.img', '5G') self.assertEquals(res, '/dev/loop1') self.assertFalse(create.called) self.assertFalse(check_call.called)
def configure_lxd_block(): '''Configure a block device for use by LXD for containers''' log('Configuring LXD container storage if not configured') # determine if the lxd block has already been configured if has_storage(LXD_POOL): log("LXD storage pool {} already configured".format(LXD_POOL)) return elif filesystem_mounted('/var/lib/lxd'): log('/var/lib/lxd already configured, skipping') return lxd_block_devices = get_block_devices() if len(lxd_block_devices) < 1: log('block devices not provided - skipping') return if len(lxd_block_devices) > 1: log("More than one block device is not supported yet, only" " using the first") lxd_block_device = lxd_block_devices[0] dev = None if lxd_block_device.startswith('/dev/'): dev = lxd_block_device elif lxd_block_device.startswith('/'): log('Configuring loopback device for use with LXD') _bd = lxd_block_device.split('|') if len(_bd) == 2: dev, size = _bd else: dev = lxd_block_device size = DEFAULT_LOOPBACK_SIZE dev = ensure_loopback_device(dev, size) if not dev or not is_block_device(dev): log('Invalid block device provided: %s' % lxd_block_device) return # NOTE: check overwrite and ensure its only execute once. db = kv() if config('overwrite') and not db.get('scrubbed', False): clean_storage(dev) db.set('scrubbed', True) db.flush() if not os.path.exists('/var/lib/lxd'): mkdir('/var/lib/lxd') if config('storage-type') == 'btrfs': config_btrfs(dev) elif config('storage-type') == 'lvm': config_lvm(dev) elif config('storage-type') == 'zfs': config_zfs(dev)
def configure_lxd_block(): '''Configure a block device for use by LXD for containers''' log('Configuring LXD container storage if not configured') # determine if the lxd block has already been configured if has_storage(LXD_POOL): log("LXD storage pool {} already configured".format(LXD_POOL)) return elif filesystem_mounted('/var/lib/lxd'): log('/var/lib/lxd already configured, skipping') return lxd_block_devices = get_block_devices() if len(lxd_block_devices) < 1: log('block devices not provided - skipping') return if len(lxd_block_devices) > 1: log("More than one block device is not supported yet, only" " using the first") lxd_block_device = lxd_block_devices[0] dev = None if lxd_block_device.startswith('/dev/'): dev = lxd_block_device elif lxd_block_device.startswith('/'): log('Configuring loopback device for use with LXD') _bd = lxd_block_device.split('|') if len(_bd) == 2: dev, size = _bd else: dev = lxd_block_device size = DEFAULT_LOOPBACK_SIZE dev = ensure_loopback_device(dev, size) if not dev or not is_block_device(dev): log('Invalid block device provided: %s' % lxd_block_device) return # NOTE: check overwrite and ensure its only execute once. db = kv() if config('overwrite') and not db.get('scrubbed', False): clean_storage(dev) db.set('scrubbed', True) db.flush() if not os.path.exists('/var/lib/lxd'): mkdir('/var/lib/lxd') if config('storage-type') == 'btrfs': config_btrfs(dev) elif config('storage-type') == 'lvm': config_lvm(dev) elif config('storage-type') == 'zfs': config_zfs(dev)
def ensure_block_device(block_device): ''' Confirm block_device, create as loopback if necessary. :param block_device: str: Full path of block device to ensure. :returns: str: Full path of ensured block device. ''' if block_device.startswith("/"): # Resolve non-relative link(s) if device is a symlink to a real device. # This fixes/prevents bug #1577408. real_device = os.path.realpath(block_device) if real_device != block_device: log('Block device "{}" is a symlink to "{}". Using "{}".'.format( block_device, real_device, real_device), level=INFO) block_device = real_device _none = ['None', 'none', None] if (block_device in _none): log('prepare_storage(): Missing required input: ' 'block_device=%s.' % block_device, level=ERROR) raise if block_device.startswith('/dev/'): bdev = block_device elif block_device.startswith('/'): _bd = block_device.split('|') if len(_bd) == 2: bdev, size = _bd else: bdev = block_device size = DEFAULT_LOOPBACK_SIZE bdev = ensure_loopback_device(bdev, size) else: bdev = '/dev/%s' % block_device if not is_block_device(bdev): log('Failed to locate valid block device at %s' % bdev, level=ERROR) # ignore missing block devices return return bdev
def ensure_block_device(block_device): ''' Confirm block_device, create as loopback if necessary. :param block_device: str: Full path of block device to ensure. :returns: str: Full path of ensured block device. ''' if block_device.startswith("/"): # Resolve non-relative link(s) if device is a symlink to a real device. # This fixes/prevents bug #1577408. real_device = os.path.realpath(block_device) if real_device != block_device: log('Block device "{}" is a symlink to "{}". Using "{}".'.format( block_device, real_device, real_device), level=INFO) block_device = real_device _none = ['None', 'none', None] if (block_device in _none): log('prepare_storage(): Missing required input: ' 'block_device=%s.' % block_device, level=ERROR) raise if block_device.startswith('/dev/'): bdev = block_device elif block_device.startswith('/'): _bd = block_device.split('|') if len(_bd) == 2: bdev, size = _bd else: bdev = block_device size = DEFAULT_LOOPBACK_SIZE bdev = ensure_loopback_device(bdev, size) else: bdev = '/dev/%s' % block_device if not is_block_device(bdev): log('Failed to locate valid block device at %s' % bdev, level=ERROR) # ignore missing block devices return return bdev
def configure_lvm_storage(block_devices, volume_group, overwrite=False, remove_missing=False, remove_missing_force=False): ''' Configure LVM storage on the list of block devices provided :param block_devices: list: List of whitelisted block devices to detect and use if found :param overwrite: bool: Scrub any existing block data if block device is not already in-use :param remove_missing: bool: Remove missing physical volumes from volume group if logical volume not allocated on them :param remove_missing_force: bool: Remove missing physical volumes from volume group even if logical volumes are allocated on them. Overrides 'remove_missing' if set. ''' log_lvm_info() devices = [] for block_device in block_devices: (block_device, size) = _parse_block_device(block_device) if not is_device_mounted(block_device): if size == 0 and is_block_device(block_device): devices.append(block_device) elif size > 0: devices.append(ensure_loopback_device(block_device, str(size))) # NOTE(jamespage) # might need todo an initial one-time scrub on install if need be vg_found = False new_devices = [] for device in devices: if not is_lvm_physical_volume(device): # Unused device if overwrite is True or not has_partition_table(device): prepare_volume(device) new_devices.append(device) elif (is_lvm_physical_volume(device) and list_lvm_volume_group(device) != volume_group): # Existing LVM but not part of required VG or new device if overwrite is True: prepare_volume(device) new_devices.append(device) elif (is_lvm_physical_volume(device) and list_lvm_volume_group(device) == volume_group): # Mark vg as found vg_found = True log_lvm_info() if vg_found is False and len(new_devices) > 0: if overwrite: ensure_lvm_volume_group_non_existent(volume_group) # Create new volume group from first device create_lvm_volume_group(volume_group, new_devices[0]) new_devices.remove(new_devices[0]) # Remove missing physical volumes from volume group try: if remove_missing_force: reduce_lvm_volume_group_missing(volume_group, extra_args=['--force']) elif remove_missing: reduce_lvm_volume_group_missing(volume_group) except subprocess.CalledProcessError as e: juju_log( "reduce_lvm_volume_group_missing() didn't complete." " LVM may not be fully configured yet. Error was: '{}'.".format( str(e))) if len(new_devices) > 0: # Extend the volume group as required for new_device in new_devices: extend_lvm_volume_group(volume_group, new_device) thin_pools = list_thin_logical_volume_pools(path_mode=True) if len(thin_pools) == 0: juju_log("No thin pools found") elif len(thin_pools) == 1: juju_log("Thin pool {} found, extending with {}".format( thin_pools[0], new_device)) extend_logical_volume_by_device(thin_pools[0], new_device) else: juju_log("Multiple thin pools ({}) found, " "skipping auto extending with {}".format( ','.join(thin_pools), new_device)) log_lvm_info()
def configure_lxd_block(): '''Configure a block device for use by LXD for containers''' log('Configuring LXD container storage') if filesystem_mounted('/var/lib/lxd'): log('/var/lib/lxd already configured, skipping') return lxd_block_devices = get_block_devices() if len(lxd_block_devices) < 1: log('block devices not provided - skipping') return if len(lxd_block_devices) > 1: raise NotImplementedError('Multiple block devices are not supported.') lxd_block_device = lxd_block_devices[0] dev = None if lxd_block_device.startswith('/dev/'): dev = lxd_block_device elif lxd_block_device.startswith('/'): log('Configuring loopback device for use with LXD') _bd = lxd_block_device.split('|') if len(_bd) == 2: dev, size = _bd else: dev = lxd_block_device size = DEFAULT_LOOPBACK_SIZE dev = ensure_loopback_device(dev, size) if not dev or not is_block_device(dev): log('Invalid block device provided: %s' % lxd_block_device) return # NOTE: check overwrite and ensure its only execute once. db = kv() if config('overwrite') and not db.get('scrubbed', False): clean_storage(dev) db.set('scrubbed', True) db.flush() if not os.path.exists('/var/lib/lxd'): mkdir('/var/lib/lxd') if config('storage-type') == 'btrfs': status_set('maintenance', 'Configuring btrfs container storage') lxd_stop() cmd = ['mkfs.btrfs', '-f', dev] check_call(cmd) mount(dev, '/var/lib/lxd', options='user_subvol_rm_allowed', persist=True, filesystem='btrfs') cmd = ['btrfs', 'quota', 'enable', '/var/lib/lxd'] check_call(cmd) lxd_start() elif config('storage-type') == 'lvm': if (is_lvm_physical_volume(dev) and list_lvm_volume_group(dev) == 'lxd_vg'): log('Device already configured for LVM/LXD, skipping') return status_set('maintenance', 'Configuring LVM container storage') # Enable and startup lvm2-lvmetad to avoid extra output # in lvm2 commands, which confused lxd. cmd = ['systemctl', 'enable', 'lvm2-lvmetad'] check_call(cmd) cmd = ['systemctl', 'start', 'lvm2-lvmetad'] check_call(cmd) create_lvm_physical_volume(dev) create_lvm_volume_group('lxd_vg', dev) cmd = ['lxc', 'config', 'set', 'storage.lvm_vg_name', 'lxd_vg'] check_call(cmd) # The LVM thinpool logical volume is lazily created, either on # image import or container creation. This will force LV creation. create_and_import_busybox_image() elif config('storage-type') == 'zfs': status_set('maintenance', 'Configuring zfs container storage') if ZFS_POOL_NAME in zpools(): log('ZFS pool already exist; skipping zfs configuration') return if config('overwrite'): cmd = ['zpool', 'create', '-f', ZFS_POOL_NAME, dev] else: cmd = ['zpool', 'create', ZFS_POOL_NAME, dev] check_call(cmd) cmd = ['lxc', 'config', 'set', 'storage.zfs_pool_name', ZFS_POOL_NAME] check_call(cmd)
def configure_lvm_storage(block_devices, volume_group, overwrite=False, remove_missing=False, remove_missing_force=False): ''' Configure LVM storage on the list of block devices provided :param block_devices: list: List of whitelisted block devices to detect and use if found :param overwrite: bool: Scrub any existing block data if block device is not already in-use :param remove_missing: bool: Remove missing physical volumes from volume group if logical volume not allocated on them :param remove_missing_force: bool: Remove missing physical volumes from volume group even if logical volumes are allocated on them. Overrides 'remove_missing' if set. ''' log_lvm_info() devices = [] for block_device in block_devices: (block_device, size) = _parse_block_device(block_device) if not is_device_mounted(block_device): if size == 0 and is_block_device(block_device): devices.append(block_device) elif size > 0: devices.append(ensure_loopback_device(block_device, size)) # NOTE(jamespage) # might need todo an initial one-time scrub on install if need be vg_found = False new_devices = [] for device in devices: if not is_lvm_physical_volume(device): # Unused device if overwrite is True or not has_partition_table(device): prepare_volume(device) new_devices.append(device) elif (is_lvm_physical_volume(device) and list_lvm_volume_group(device) != volume_group): # Existing LVM but not part of required VG or new device if overwrite is True: prepare_volume(device) new_devices.append(device) elif (is_lvm_physical_volume(device) and list_lvm_volume_group(device) == volume_group): # Mark vg as found vg_found = True log_lvm_info() if vg_found is False and len(new_devices) > 0: if overwrite: ensure_lvm_volume_group_non_existent(volume_group) # Create new volume group from first device create_lvm_volume_group(volume_group, new_devices[0]) new_devices.remove(new_devices[0]) # Remove missing physical volumes from volume group if remove_missing_force: reduce_lvm_volume_group_missing(volume_group, extra_args=['--force']) elif remove_missing: reduce_lvm_volume_group_missing(volume_group) if len(new_devices) > 0: # Extend the volume group as required for new_device in new_devices: extend_lvm_volume_group(volume_group, new_device) log_lvm_info()
def configure_lvm_storage(block_devices, volume_group, overwrite=False, remove_missing=False, remove_missing_force=False): ''' Configure LVM storage on the list of block devices provided :param block_devices: list: List of whitelisted block devices to detect and use if found :param overwrite: bool: Scrub any existing block data if block device is not already in-use :param remove_missing: bool: Remove missing physical volumes from volume group if logical volume not allocated on them :param remove_missing_force: bool: Remove missing physical volumes from volume group even if logical volumes are allocated on them. Overrides 'remove_missing' if set. ''' log_lvm_info() devices = [] for block_device in block_devices: (block_device, size) = _parse_block_device(block_device) if not is_device_mounted(block_device): if size == 0 and is_block_device(block_device): devices.append(block_device) elif size > 0: devices.append(ensure_loopback_device(block_device, size)) # NOTE(jamespage) # might need todo an initial one-time scrub on install if need be vg_found = False new_devices = [] for device in devices: if not is_lvm_physical_volume(device): # Unused device if overwrite is True or not has_partition_table(device): prepare_volume(device) new_devices.append(device) elif (is_lvm_physical_volume(device) and list_lvm_volume_group(device) != volume_group): # Existing LVM but not part of required VG or new device if overwrite is True: prepare_volume(device) new_devices.append(device) elif (is_lvm_physical_volume(device) and list_lvm_volume_group(device) == volume_group): # Mark vg as found vg_found = True log_lvm_info() if vg_found is False and len(new_devices) > 0: if overwrite: ensure_lvm_volume_group_non_existent(volume_group) # Create new volume group from first device create_lvm_volume_group(volume_group, new_devices[0]) new_devices.remove(new_devices[0]) # Remove missing physical volumes from volume group if remove_missing_force: reduce_lvm_volume_group_missing(volume_group, extra_args=['--force']) elif remove_missing: reduce_lvm_volume_group_missing(volume_group) if len(new_devices) > 0: # Extend the volume group as required for new_device in new_devices: extend_lvm_volume_group(volume_group, new_device) log_lvm_info()
def configure_lxd_block(): '''Configure a block device for use by LXD for containers''' log('Configuring LXD container storage') if filesystem_mounted('/var/lib/lxd'): log('/var/lib/lxd already configured, skipping') return lxd_block_devices = get_block_devices() if len(lxd_block_devices) < 1: log('block devices not provided - skipping') return if len(lxd_block_devices) > 1: log("More than one block device is not supported yet, only" " using the first") lxd_block_device = lxd_block_devices[0] dev = None if lxd_block_device.startswith('/dev/'): dev = lxd_block_device elif lxd_block_device.startswith('/'): log('Configuring loopback device for use with LXD') _bd = lxd_block_device.split('|') if len(_bd) == 2: dev, size = _bd else: dev = lxd_block_device size = DEFAULT_LOOPBACK_SIZE dev = ensure_loopback_device(dev, size) if not dev or not is_block_device(dev): log('Invalid block device provided: %s' % lxd_block_device) return # NOTE: check overwrite and ensure its only execute once. db = kv() if config('overwrite') and not db.get('scrubbed', False): clean_storage(dev) db.set('scrubbed', True) db.flush() if not os.path.exists('/var/lib/lxd'): mkdir('/var/lib/lxd') if config('storage-type') == 'btrfs': status_set('maintenance', 'Configuring btrfs container storage') lxd_stop() cmd = ['mkfs.btrfs', '-f', dev] check_call(cmd) mount(dev, '/var/lib/lxd', options='user_subvol_rm_allowed', persist=True, filesystem='btrfs') cmd = ['btrfs', 'quota', 'enable', '/var/lib/lxd'] check_call(cmd) lxd_start() elif config('storage-type') == 'lvm': if (is_lvm_physical_volume(dev) and list_lvm_volume_group(dev) == 'lxd_vg'): log('Device already configured for LVM/LXD, skipping') return status_set('maintenance', 'Configuring LVM container storage') # Enable and startup lvm2-lvmetad to avoid extra output # in lvm2 commands, which confused lxd. cmd = ['systemctl', 'enable', 'lvm2-lvmetad'] check_call(cmd) cmd = ['systemctl', 'start', 'lvm2-lvmetad'] check_call(cmd) create_lvm_physical_volume(dev) create_lvm_volume_group('lxd_vg', dev) cmd = ['lxc', 'config', 'set', 'storage.lvm_vg_name', 'lxd_vg'] check_call(cmd) # The LVM thinpool logical volume is lazily created, either on # image import or container creation. This will force LV creation. create_and_import_busybox_image() elif config('storage-type') == 'zfs': status_set('maintenance', 'Configuring zfs container storage') if ZFS_POOL_NAME in zpools(): log('ZFS pool already exist; skipping zfs configuration') return if config('overwrite'): cmd = ['zpool', 'create', '-f', ZFS_POOL_NAME, dev] else: cmd = ['zpool', 'create', ZFS_POOL_NAME, dev] check_call(cmd) cmd = ['lxc', 'config', 'set', 'storage.zfs_pool_name', ZFS_POOL_NAME] check_call(cmd)