def guess_block_devices():
bdevs = find_block_devices(include_mounted=True)
gdevs = []
for dev in bdevs:
if is_device_mounted(dev):
mnt_point = get_mount_point(dev)
if mnt_point and mnt_point.startswith('/srv/node'):
gdevs.append(dev)
else:
gdevs.append(dev)
return gdevs
def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False,
ignore_errors=False, encrypt=False):
if not os.path.exists(dev):
log('Path {} does not exist - bailing'.format(dev))
return
if not is_block_device(dev):
log('Path {} is not a block device - bailing'.format(dev))
return
if (is_osd_disk(dev) and not reformat_osd):
log('Looks like {} is already an'
' OSD data or journal, skipping.'.format(dev))
return
if is_device_mounted(dev):
log('Looks like {} is in use, skipping.'.format(dev))
return
status_set('maintenance', 'Initializing device {}'.format(dev))
cmd = ['ceph-disk', 'prepare']
# Later versions of ceph support more options
if cmp_pkgrevno('ceph', '0.60') >= 0:
if encrypt:
cmd.append('--dmcrypt')
if cmp_pkgrevno('ceph', '0.48.3') >= 0:
if osd_format:
cmd.append('--fs-type')
cmd.append(osd_format)
if reformat_osd:
cmd.append('--zap-disk')
cmd.append(dev)
if osd_journal:
least_used = find_least_used_journal(osd_journal)
cmd.append(least_used)
else:
# Just provide the device - no other options
# for older versions of ceph
cmd.append(dev)
if reformat_osd:
zap_disk(dev)
try:
log("osdize cmd: {}".format(cmd))
subprocess.check_call(cmd)
except subprocess.CalledProcessError as e:
if ignore_errors:
log('Unable to initialize device: {}'.format(dev), WARNING)
else:
log('Unable to initialize device: {}'.format(dev), ERROR)
raise e
def zap():
if not hookenv.action_get('i-really-mean-it'):
hookenv.action_fail('i-really-mean-it is a required parameter')
return
failed_devices = []
not_block_devices = []
devices = get_devices()
for device in devices:
if not is_block_device(device):
not_block_devices.append(device)
if (is_device_mounted(device) or
is_active_bluestore_device(device) or
is_mapped_luks_device(device)):
failed_devices.append(device)
if failed_devices or not_block_devices:
message = ""
if failed_devices:
message = "{} devices are mounted: {}".format(
len(failed_devices),
", ".join(failed_devices))
if not_block_devices:
if message is not '':
message += "\n\n"
message += "{} devices are not block devices: {}".format(
len(not_block_devices),
", ".join(not_block_devices))
hookenv.action_fail(message)
return
db = kv()
used_devices = db.get('osd-devices', [])
for device in devices:
zap_disk(device)
if device in used_devices:
used_devices.remove(device)
db.set('osd-devices', used_devices)
db.flush()
hookenv.action_set({
'message': "{} disk(s) have been zapped, to use them as OSDs, run: \n"
"juju run-action {} add-disk osd-devices=\"{}\"".format(
len(devices),
hookenv.local_unit(),
" ".join(devices))
})
def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False,
ignore_errors=False):
if not os.path.exists(dev):
log('Path {} does not exist - bailing'.format(dev))
return
if not is_block_device(dev):
log('Path {} is not a block device - bailing'.format(dev))
return
if (is_osd_disk(dev) and not reformat_osd):
log('Looks like {} is already an OSD, skipping.'.format(dev))
return
if is_device_mounted(dev):
log('Looks like {} is in use, skipping.'.format(dev))
return
cmd = ['ceph-disk-prepare']
# Later versions of ceph support more options
if cmp_pkgrevno('ceph', '0.48.3') >= 0:
if osd_format:
cmd.append('--fs-type')
cmd.append(osd_format)
if reformat_osd:
cmd.append('--zap-disk')
cmd.append(dev)
if osd_journal and os.path.exists(osd_journal):
cmd.append(osd_journal)
else:
# Just provide the device - no other options
# for older versions of ceph
cmd.append(dev)
if reformat_osd:
zap_disk(dev)
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError as e:
if ignore_errors:
log('Unable to initialize device: {}'.format(dev), WARNING)
else:
log('Unable to initialize device: {}'.format(dev), ERROR)
raise e
def prepare_disks_and_activate():
# NOTE: vault/vaultlocker preflight check
vault_kv = vaultlocker.VaultKVContext(vaultlocker.VAULTLOCKER_BACKEND)
context = vault_kv()
if use_vaultlocker() and not vault_kv.complete:
log('Deferring OSD preparation as vault not ready',
level=DEBUG)
return
elif use_vaultlocker() and vault_kv.complete:
log('Vault ready, writing vaultlocker configuration',
level=DEBUG)
vaultlocker.write_vaultlocker_conf(context)
osd_journal = get_journal_devices()
if not osd_journal.isdisjoint(set(get_devices())):
raise ValueError('`osd-journal` and `osd-devices` options must not'
'overlap.')
log("got journal devs: {}".format(osd_journal), level=DEBUG)
# pre-flight check of eligible device pristinity
devices = get_devices()
# if a device has been previously touched we need to consider it as
# non-pristine. If it needs to be re-processed it has to be zapped
# via the respective action which also clears the unitdata entry.
db = kv()
touched_devices = db.get('osd-devices', [])
devices = [dev for dev in devices if dev not in touched_devices]
log('Skipping osd devices previously processed by this unit: {}'
.format(touched_devices))
# filter osd-devices that are file system paths
devices = [dev for dev in devices if dev.startswith('/dev')]
# filter osd-devices that does not exist on this unit
devices = [dev for dev in devices if os.path.exists(dev)]
# filter osd-devices that are already mounted
devices = [dev for dev in devices if not is_device_mounted(dev)]
# filter osd-devices that are active bluestore devices
devices = [dev for dev in devices
if not ceph.is_active_bluestore_device(dev)]
log('Checking for pristine devices: "{}"'.format(devices), level=DEBUG)
if not all(ceph.is_pristine_disk(dev) for dev in devices):
status_set('blocked',
'Non-pristine devices detected, consult '
'`list-disks`, `zap-disk` and `blacklist-*` actions.')
return
if ceph.is_bootstrapped():
log('ceph bootstrapped, rescanning disks')
emit_cephconf()
bluestore = use_bluestore()
ceph.udevadm_settle()
for dev in get_devices():
ceph.osdize(dev, config('osd-format'),
osd_journal,
config('ignore-device-errors'),
config('osd-encrypt'),
bluestore,
config('osd-encrypt-keymanager'))
# Make it fast!
if config('autotune'):
ceph.tune_dev(dev)
ceph.start_osds(get_devices())
# Notify MON cluster as to how many OSD's this unit bootstrapped
# into the cluster
for r_id in relation_ids('mon'):
relation_set(
relation_id=r_id,
relation_settings={
'bootstrapped-osds': len(db.get('osd-devices', []))
}
)
def configure_lvm_storage(block_devices, volume_group, overwrite=False,
remove_missing=False, remove_missing_force=False):
''' Configure LVM storage on the list of block devices provided
:param block_devices: list: List of whitelisted block devices to detect
and use if found
:param overwrite: bool: Scrub any existing block data if block device is
not already in-use
:param remove_missing: bool: Remove missing physical volumes from volume
group if logical volume not allocated on them
:param remove_missing_force: bool: Remove missing physical volumes from
volume group even if logical volumes are allocated
on them. Overrides 'remove_missing' if set.
'''
log_lvm_info()
devices = []
for block_device in block_devices:
(block_device, size) = _parse_block_device(block_device)
if not is_device_mounted(block_device):
if size == 0 and is_block_device(block_device):
devices.append(block_device)
elif size > 0:
devices.append(ensure_loopback_device(block_device, size))
# NOTE(jamespage)
# might need todo an initial one-time scrub on install if need be
vg_found = False
new_devices = []
for device in devices:
if not is_lvm_physical_volume(device):
# Unused device
if overwrite is True or not has_partition_table(device):
prepare_volume(device)
new_devices.append(device)
elif (is_lvm_physical_volume(device) and
list_lvm_volume_group(device) != volume_group):
# Existing LVM but not part of required VG or new device
if overwrite is True:
prepare_volume(device)
new_devices.append(device)
elif (is_lvm_physical_volume(device) and
list_lvm_volume_group(device) == volume_group):
# Mark vg as found
vg_found = True
log_lvm_info()
if vg_found is False and len(new_devices) > 0:
if overwrite:
ensure_lvm_volume_group_non_existent(volume_group)
# Create new volume group from first device
create_lvm_volume_group(volume_group, new_devices[0])
new_devices.remove(new_devices[0])
# Remove missing physical volumes from volume group
if remove_missing_force:
reduce_lvm_volume_group_missing(volume_group, extra_args=['--force'])
elif remove_missing:
reduce_lvm_volume_group_missing(volume_group)
if len(new_devices) > 0:
# Extend the volume group as required
for new_device in new_devices:
extend_lvm_volume_group(volume_group, new_device)
log_lvm_info()
def _is_storage_ready(partition):
"""
A small helper to determine if a given device is suitabe to be used as
a storage device.
"""
return is_block_device(partition) and not is_device_mounted(partition)
def configure_lvm_storage(block_devices, volume_group, overwrite=False,
remove_missing=False, remove_missing_force=False):
''' Configure LVM storage on the list of block devices provided
:param block_devices: list: List of whitelisted block devices to detect
and use if found
:param overwrite: bool: Scrub any existing block data if block device is
not already in-use
:param remove_missing: bool: Remove missing physical volumes from volume
group if logical volume not allocated on them
:param remove_missing_force: bool: Remove missing physical volumes from
volume group even if logical volumes are allocated
on them. Overrides 'remove_missing' if set.
'''
log_lvm_info()
devices = []
for block_device in block_devices:
(block_device, size) = _parse_block_device(block_device)
if not is_device_mounted(block_device):
if size == 0 and is_block_device(block_device):
devices.append(block_device)
elif size > 0:
devices.append(ensure_loopback_device(block_device, size))
# NOTE(jamespage)
# might need todo an initial one-time scrub on install if need be
vg_found = False
new_devices = []
for device in devices:
if not is_lvm_physical_volume(device):
# Unused device
if overwrite is True or not has_partition_table(device):
prepare_volume(device)
new_devices.append(device)
elif (is_lvm_physical_volume(device) and
list_lvm_volume_group(device) != volume_group):
# Existing LVM but not part of required VG or new device
if overwrite is True:
prepare_volume(device)
new_devices.append(device)
elif (is_lvm_physical_volume(device) and
list_lvm_volume_group(device) == volume_group):
# Mark vg as found
vg_found = True
log_lvm_info()
if vg_found is False and len(new_devices) > 0:
if overwrite:
ensure_lvm_volume_group_non_existent(volume_group)
# Create new volume group from first device
create_lvm_volume_group(volume_group, new_devices[0])
new_devices.remove(new_devices[0])
# Remove missing physical volumes from volume group
if remove_missing_force:
reduce_lvm_volume_group_missing(volume_group, extra_args=['--force'])
elif remove_missing:
reduce_lvm_volume_group_missing(volume_group)
if len(new_devices) > 0:
# Extend the volume group as required
for new_device in new_devices:
extend_lvm_volume_group(volume_group, new_device)
log_lvm_info()
def prepare_disks_and_activate():
# NOTE: vault/vaultlocker preflight check
vault_kv = vaultlocker.VaultKVContext(vaultlocker.VAULTLOCKER_BACKEND)
context = vault_kv()
if use_vaultlocker() and not vault_kv.complete:
log('Deferring OSD preparation as vault not ready',
level=DEBUG)
return
elif use_vaultlocker() and vault_kv.complete:
log('Vault ready, writing vaultlocker configuration',
level=DEBUG)
vaultlocker.write_vaultlocker_conf(context)
osd_journal = get_journal_devices()
if not osd_journal.isdisjoint(set(get_devices())):
raise ValueError('`osd-journal` and `osd-devices` options must not'
'overlap.')
log("got journal devs: {}".format(osd_journal), level=DEBUG)
# pre-flight check of eligible device pristinity
devices = get_devices()
# if a device has been previously touched we need to consider it as
# non-pristine. If it needs to be re-processed it has to be zapped
# via the respective action which also clears the unitdata entry.
db = kv()
touched_devices = db.get('osd-devices', [])
devices = [dev for dev in devices if dev not in touched_devices]
log('Skipping osd devices previously processed by this unit: {}'
.format(touched_devices))
# filter osd-devices that are file system paths
devices = [dev for dev in devices if dev.startswith('/dev')]
# filter osd-devices that does not exist on this unit
devices = [dev for dev in devices if os.path.exists(dev)]
# filter osd-devices that are already mounted
devices = [dev for dev in devices if not is_device_mounted(dev)]
# filter osd-devices that are active bluestore devices
devices = [dev for dev in devices
if not ceph.is_active_bluestore_device(dev)]
log('Checking for pristine devices: "{}"'.format(devices), level=DEBUG)
if not all(ceph.is_pristine_disk(dev) for dev in devices):
status_set('blocked',
'Non-pristine devices detected, consult '
'`list-disks`, `zap-disk` and `blacklist-*` actions.')
return
if ceph.is_bootstrapped():
log('ceph bootstrapped, rescanning disks')
emit_cephconf()
bluestore = use_bluestore()
for dev in get_devices():
ceph.osdize(dev, config('osd-format'),
osd_journal,
config('ignore-device-errors'),
config('osd-encrypt'),
bluestore,
config('osd-encrypt-keymanager'))
# Make it fast!
if config('autotune'):
ceph.tune_dev(dev)
ceph.start_osds(get_devices())
def setup_storage(encrypt=False):
# Preflight check vault relation if encryption is enabled
if encrypt:
if not vaultlocker_installed():
log("Encryption requested but vaultlocker not yet installed",
level=DEBUG)
return
vault_kv = vaultlocker.VaultKVContext(
secret_backend=vaultlocker.VAULTLOCKER_BACKEND
)
context = vault_kv()
if vault_kv.complete:
# NOTE: only write vaultlocker configuration once relation is
# complete otherwise we run the chance of an empty
# configuration file being installed on a machine with other
# vaultlocker based services
vaultlocker.write_vaultlocker_conf(context, priority=90)
else:
log("Encryption requested but vault relation not complete",
level=DEBUG)
return
# Ensure /srv/node exists just in case no disks
# are detected and used.
mkdir(os.path.join('/srv', 'node'),
owner='swift', group='swift',
perms=0o755)
reformat = str(config('overwrite')).lower() == "true"
db = kv()
prepared_devices = db.get('prepared-devices', [])
for dev in determine_block_devices():
if dev in prepared_devices:
log('Device {} already processed by charm,'
' skipping'.format(dev))
continue
if is_device_in_ring(os.path.basename(dev)):
log("Device '%s' already in the ring - ignoring" % (dev))
# NOTE: record existing use of device dealing with
# upgrades from older versions of charms without
# this feature
prepared_devices.append(dev)
db.set('prepared-devices', prepared_devices)
db.flush()
continue
# NOTE: this deals with a dm-crypt'ed block device already in
# use
if is_device_mounted(dev):
log("Device '{}' is already mounted, ignoring".format(dev))
continue
if reformat:
clean_storage(dev)
loopback_device = is_mapped_loopback_device(dev)
options = None
if encrypt and not loopback_device:
dev_uuid = str(uuid.uuid4())
check_call(['vaultlocker', 'encrypt',
'--uuid', dev_uuid,
dev])
dev = '/dev/mapper/crypt-{}'.format(dev_uuid)
options = ','.join([
"defaults",
"nofail",
("x-systemd.requires="
"vaultlocker-decrypt@{uuid}.service".format(uuid=dev_uuid)),
"comment=vaultlocker",
])
try:
# If not cleaned and in use, mkfs should fail.
mkfs_xfs(dev, force=reformat, inode_size=config('xfs-inode-size'))
except subprocess.CalledProcessError as exc:
# This is expected is a formatted device is provided and we are
# forcing the format.
log("Format device '%s' failed (%s) - continuing to next device" %
(dev, exc), level=WARNING)
continue
basename = os.path.basename(dev)
_mp = os.path.join('/srv', 'node', basename)
mkdir(_mp, owner='swift', group='swift')
mountpoint = '/srv/node/%s' % basename
if loopback_device:
# If an exiting fstab entry exists using the image file as the
# source then preserve it, otherwise use the loopback device
# directly to avoid a secound implicit loopback device being
# created on mount. Bug #1762390
fstab = charmhelpers.core.fstab.Fstab()
fstab_entry = fstab.get_entry_by_attr('mountpoint', mountpoint)
if fstab_entry and loopback_device == fstab_entry.device:
dev = loopback_device
options = "loop,nofail,defaults"
dev_blkid = "UUID=" + get_device_blkid(dev)
filesystem = "xfs"
mount(dev, mountpoint, filesystem=filesystem)
fstab_add(dev_blkid, mountpoint, filesystem, options=options)
check_call(['chown', '-R', 'swift:swift', mountpoint])
check_call(['chmod', '-R', '0755', mountpoint])
# NOTE: record preparation of device - this will be used when
# providing block device configuration for ring builders.
prepared_devices.append(dev)
db.set('prepared-devices', prepared_devices)
db.flush()
def _is_storage_ready(partition):
"""
A small helper to determine if a given device is suitabe to be used as
a storage device.
"""
return is_block_device(partition) and not is_device_mounted(partition)
def configure_local_ephemeral_storage():
"""Configure local block device for use as ephemeral instance storage"""
# Preflight check vault relation if encryption is enabled
encrypt = config('encrypt')
if encrypt:
if not vaultlocker_installed():
log("Encryption requested but vaultlocker not yet installed",
level=DEBUG)
return
vault_kv = vaultlocker.VaultKVContext(
secret_backend=vaultlocker.VAULTLOCKER_BACKEND)
context = vault_kv()
if vault_kv.complete:
# NOTE: only write vaultlocker configuration once relation is
# complete otherwise we run the chance of an empty
# configuration file being installed on a machine with other
# vaultlocker based services
vaultlocker.write_vaultlocker_conf(context, priority=80)
else:
log("Encryption requested but vault relation not complete",
level=DEBUG)
return
db = kv()
storage_configured = db.get('storage-configured', False)
if storage_configured:
log("Ephemeral storage already configured, skipping", level=DEBUG)
return
dev = determine_block_device()
if not dev:
log('No block device configuration found, skipping', level=DEBUG)
return
if not is_block_device(dev):
log("Device '{}' is not a block device, "
"unable to configure storage".format(dev),
level=DEBUG)
return
# NOTE: this deals with a dm-crypt'ed block device already in
# use
if is_device_mounted(dev):
log("Device '{}' is already mounted, "
"unable to configure storage".format(dev),
level=DEBUG)
return
options = None
if encrypt:
dev_uuid = str(uuid.uuid4())
check_call(['vaultlocker', 'encrypt', '--uuid', dev_uuid, dev])
dev = '/dev/mapper/crypt-{}'.format(dev_uuid)
options = ','.join([
"defaults",
"nofail",
("x-systemd.requires="
"vaultlocker-decrypt@{uuid}.service".format(uuid=dev_uuid)),
"comment=vaultlocker",
])
# If not cleaned and in use, mkfs should fail.
mkfs_xfs(dev, force=True)
mountpoint = '/var/lib/nova/instances'
filesystem = "xfs"
mount(dev, mountpoint, filesystem=filesystem)
fstab_add(dev, mountpoint, filesystem, options=options)
check_call(['chown', '-R', 'nova:nova', mountpoint])
check_call(['chmod', '-R', '0755', mountpoint])
# NOTE: record preparation of device - this ensures that ephemeral
# storage is never reconfigured by mistake, losing instance disks
db.set('storage-configured', True)
db.flush()