def zap(self, devices=None):
devices = devices or self.args.devices
for device in devices:
mlogger.info("Zapping: %s", device.abspath)
if device.is_mapper:
terminal.error(
"Refusing to zap the mapper device: {}".format(device))
raise SystemExit(1)
if device.is_lvm_member:
self.zap_lvm_member(device)
if device.is_lv:
self.zap_lv(device)
if device.is_partition:
self.zap_partition(device)
if device.is_device:
self.zap_raw_device(device)
if self.args.devices:
terminal.success("Zapping successful for: %s" %
", ".join([str(d) for d in self.args.devices]))
else:
terminal.success(
"Zapping successful for OSD: %s" % self.args.osd_id
or self.args.osd_fsid)
def make_new_volume(self, osd_id, osd_fsid, devices, target_lv):
osd_path = get_osd_path(osd_id, osd_fsid)
mlogger.info(
'Making new volume at {} for OSD: {} ({})'.format(
target_lv.lv_path, osd_id, osd_path))
tag_tracker = VolumeTagTracker(devices, target_lv)
try:
tag_tracker.update_tags_when_lv_create(self.create_type)
stdout, stderr, exit_code = process.call([
'ceph-bluestore-tool',
'--path',
osd_path,
'--dev-target',
target_lv.lv_path,
'--command',
'bluefs-bdev-new-{}'.format(self.create_type)
])
if exit_code != 0:
mlogger.error(
'failed to attach new volume, error code:{}'.format(
exit_code))
raise SystemExit(
"Failed to attach new volume: {}".format(
self.args.target))
else:
system.chown(os.path.join(osd_path, "block.{}".format(
self.create_type)))
terminal.success('New volume attached.')
except:
tag_tracker.undo()
raise
return
def activate(self, args):
lvs = api.Volumes()
# filter them down for the OSD ID and FSID we need to activate
if args.osd_id and args.osd_fsid:
lvs.filter(lv_tags={'ceph.osd_id': args.osd_id, 'ceph.osd_fsid': args.osd_fsid})
elif args.osd_fsid and not args.osd_id:
lvs.filter(lv_tags={'ceph.osd_fsid': args.osd_fsid})
if not lvs:
raise RuntimeError('could not find osd.%s with fsid %s' % (args.osd_id, args.osd_fsid))
# This argument is only available when passed in directly or via
# systemd, not when ``create`` is being used
if getattr(args, 'auto_detect_objectstore', False):
logger.info('auto detecting objectstore')
# may get multiple lvs, so can't do lvs.get() calls here
for lv in lvs:
has_journal = lv.tags.get('ceph.journal_uuid')
if has_journal:
logger.info('found a journal associated with the OSD, assuming filestore')
return activate_filestore(lvs)
logger.info('unable to find a journal associated with the OSD, assuming bluestore')
return activate_bluestore(lvs)
if args.bluestore:
activate_bluestore(lvs)
elif args.filestore:
activate_filestore(lvs)
terminal.success("ceph-volume lvm activate successful for osd ID: %s" % args.osd_id)
def create(self, args):
if not args.osd_fsid:
args.osd_fsid = system.generate_uuid()
prepare_step = Prepare([])
prepare_step.safe_prepare(args)
osd_id = prepare_step.osd_id
try:
# we try this for activate only when 'creating' an OSD,
# because a rollback should not happen when doing normal
# activation. For example when starting an OSD, systemd
# will call activate, which would never need to be rolled
# back.
a = Activate([])
a.args = self.args
a.activate([args.data],
tmpfs=not args.no_tmpfs,
systemd=not args.no_systemd)
except Exception:
logger.exception(
'raw activate was unable to complete, while creating the OSD')
logger.info('will rollback OSD ID creation')
rollback_osd(args, osd_id)
raise
terminal.success("ceph-volume raw create successful for: %s" %
args.data)
def safe_prepare(self, args=None):
"""
An intermediate step between `main()` and `prepare()` so that we can
capture the `self.osd_id` in case we need to rollback
:param args: Injected args, usually from `lvm create` which compounds
both `prepare` and `create`
"""
if args is not None:
self.args = args
try:
vgname, lvname = self.args.data.split('/')
lv = api.get_single_lv(filters={
'lv_name': lvname,
'vg_name': vgname
})
except ValueError:
lv = None
if api.is_ceph_device(lv):
logger.info("device {} is already used".format(self.args.data))
raise RuntimeError("skipping {}, it is already prepared".format(
self.args.data))
try:
self.prepare()
except Exception:
logger.exception('lvm prepare was unable to complete')
logger.info('will rollback OSD ID creation')
rollback_osd(self.args, self.osd_id)
raise
terminal.success("ceph-volume lvm prepare successful for: %s" %
self.args.data)
def zap(self, args):
device = args.device
lv = api.get_lv_from_argument(device)
if lv:
# we are zapping a logical volume
path = lv.lv_path
else:
# we are zapping a partition
#TODO: ensure device is a partition
path = device
mlogger.info("Zapping: %s", path)
# check if there was a pv created with the
# name of device
pv = api.get_pv(pv_name=device)
if pv:
vg_name = pv.vg_name
lv = api.get_lv(vg_name=vg_name)
dmcrypt = False
dmcrypt_uuid = None
if lv:
osd_path = "/var/lib/ceph/osd/{}-{}".format(lv.tags['ceph.cluster_name'], lv.tags['ceph.osd_id'])
dmcrypt_uuid = lv.lv_uuid
dmcrypt = lv.encrypted
if system.path_is_mounted(osd_path):
mlogger.info("Unmounting %s", osd_path)
system.unmount(osd_path)
else:
# we're most likely dealing with a partition here, check to
# see if it was encrypted
partuuid = disk.get_partuuid(device)
if encryption.status("/dev/mapper/{}".format(partuuid)):
dmcrypt_uuid = partuuid
dmcrypt = True
if dmcrypt and dmcrypt_uuid:
dmcrypt_path = "/dev/mapper/{}".format(dmcrypt_uuid)
mlogger.info("Closing encrypted path %s", dmcrypt_path)
encryption.dmcrypt_close(dmcrypt_path)
if args.destroy and pv:
logger.info("Found a physical volume created from %s, will destroy all it's vgs and lvs", device)
vg_name = pv.vg_name
mlogger.info("Destroying volume group %s because --destroy was given", vg_name)
api.remove_vg(vg_name)
mlogger.info("Destroying physical volume %s because --destroy was given", device)
api.remove_pv(device)
elif args.destroy and not pv:
mlogger.info("Skipping --destroy because no associated physical volumes are found for %s", device)
wipefs(path)
zap_data(path)
if lv and not pv:
# remove all lvm metadata
lv.clear_tags()
terminal.success("Zapping successful for: %s" % path)
def zap(self, devices=None):
devices = devices or self.args.devices
for device in devices:
mlogger.info("Zapping: %s", device.abspath)
if device.is_mapper:
terminal.error("Refusing to zap the mapper device: {}".format(device))
raise SystemExit(1)
if device.is_lvm_member:
self.zap_lvm_member(device)
if device.is_lv:
self.zap_lv(device)
if device.is_partition:
self.zap_partition(device)
if device.is_device:
self.zap_raw_device(device)
if self.args.devices:
terminal.success(
"Zapping successful for: %s" % ", ".join([str(d) for d in self.args.devices])
)
else:
terminal.success(
"Zapping successful for OSD: %s" % self.args.osd_id or self.args.osd_fsid
)
def activate(self, args):
lvs = api.Volumes()
# filter them down for the OSD ID and FSID we need to activate
if args.osd_id and args.osd_fsid:
lvs.filter(lv_tags={'ceph.osd_id': args.osd_id, 'ceph.osd_fsid': args.osd_fsid})
elif args.osd_fsid and not args.osd_id:
lvs.filter(lv_tags={'ceph.osd_fsid': args.osd_fsid})
if not lvs:
raise RuntimeError('could not find osd.%s with fsid %s' % (args.osd_id, args.osd_fsid))
# This argument is only available when passed in directly or via
# systemd, not when ``create`` is being used
if getattr(args, 'auto_detect_objectstore', False):
logger.info('auto detecting objectstore')
# may get multiple lvs, so can't do lvs.get() calls here
for lv in lvs:
has_journal = lv.tags.get('ceph.journal_uuid')
if has_journal:
logger.info('found a journal associated with the OSD, assuming filestore')
return activate_filestore(lvs)
logger.info('unable to find a journal associated with the OSD, assuming bluestore')
return activate_bluestore(lvs)
if args.bluestore:
activate_bluestore(lvs)
elif args.filestore:
activate_filestore(lvs)
terminal.success("ceph-volume lvm activate successful for osd ID: %s" % args.osd_id)
def activate_filestore(lvs):
# find the osd
osd_lv = lvs.get(lv_tags={'ceph.type': 'data'})
if not osd_lv:
raise RuntimeError('Unable to find a data LV for filestore activation')
is_encrypted = osd_lv.tags.get('ceph.encrypted', '0') == '1'
osd_id = osd_lv.tags['ceph.osd_id']
conf.cluster = osd_lv.tags['ceph.cluster_name']
# it may have a volume with a journal
osd_journal_lv = lvs.get(lv_tags={'ceph.type': 'journal'})
# TODO: add sensible error reporting if this is ever the case
# blow up with a KeyError if this doesn't exist
osd_fsid = osd_lv.tags['ceph.osd_fsid']
if not osd_journal_lv:
# must be a disk partition, by quering blkid by the uuid we are ensuring that the
# device path is always correct
journal_uuid = osd_lv.tags['ceph.journal_uuid']
osd_journal = disk.get_device_from_partuuid(journal_uuid)
else:
journal_uuid = osd_journal_lv.lv_uuid
osd_journal = osd_lv.tags['ceph.journal_device']
if not osd_journal:
raise RuntimeError('unable to detect an lv or device journal for OSD %s' % osd_id)
# this is done here, so that previous checks that ensure path availability
# and correctness can still be enforced, and report if any issues are found
if is_encrypted:
lockbox_secret = osd_lv.tags['ceph.cephx_lockbox_secret']
# this keyring writing is idempotent
encryption_utils.write_lockbox_keyring(osd_id, osd_fsid, lockbox_secret)
dmcrypt_secret = encryption_utils.get_dmcrypt_key(osd_id, osd_fsid)
encryption_utils.luks_open(dmcrypt_secret, osd_lv.lv_path, osd_lv.lv_uuid)
encryption_utils.luks_open(dmcrypt_secret, osd_journal, journal_uuid)
osd_journal = '/dev/mapper/%s' % journal_uuid
source = '/dev/mapper/%s' % osd_lv.lv_uuid
else:
source = osd_lv.lv_path
# mount the osd
destination = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id)
if not system.device_is_mounted(source, destination=destination):
process.run(['mount', '-v', source, destination])
# always re-do the symlink regardless if it exists, so that the journal
# device path that may have changed can be mapped correctly every time
destination = '/var/lib/ceph/osd/%s-%s/journal' % (conf.cluster, osd_id)
process.run(['ln', '-snf', osd_journal, destination])
# make sure that the journal has proper permissions
system.chown(osd_journal)
# enable the ceph-volume unit for this OSD
systemctl.enable_volume(osd_id, osd_fsid, 'lvm')
# start the OSD
systemctl.start_osd(osd_id)
terminal.success("ceph-volume lvm activate successful for osd ID: %s" % osd_id)
def activate_bluestore(lvs):
# find the osd
osd_lv = lvs.get(lv_tags={'ceph.type': 'block'})
is_encrypted = osd_lv.tags.get('ceph.encrypted', '0') == '1'
dmcrypt_secret = None
osd_id = osd_lv.tags['ceph.osd_id']
conf.cluster = osd_lv.tags['ceph.cluster_name']
osd_fsid = osd_lv.tags['ceph.osd_fsid']
# mount on tmpfs the osd directory
osd_path = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id)
if not system.path_is_mounted(osd_path):
# mkdir -p and mount as tmpfs
prepare_utils.create_osd_path(osd_id, tmpfs=True)
# XXX This needs to be removed once ceph-bluestore-tool can deal with
# symlinks that exist in the osd dir
for link_name in ['block', 'block.db', 'block.wal']:
link_path = os.path.join(osd_path, link_name)
if os.path.exists(link_path):
os.unlink(os.path.join(osd_path, link_name))
# encryption is handled here, before priming the OSD dir
if is_encrypted:
osd_lv_path = '/dev/mapper/%s' % osd_lv.lv_uuid
lockbox_secret = osd_lv.tags['ceph.cephx_lockbox_secret']
encryption_utils.write_lockbox_keyring(osd_id, osd_fsid, lockbox_secret)
dmcrypt_secret = encryption_utils.get_dmcrypt_key(osd_id, osd_fsid)
encryption_utils.luks_open(dmcrypt_secret, osd_lv.lv_path, osd_lv.lv_uuid)
else:
osd_lv_path = osd_lv.lv_path
db_device_path = get_osd_device_path(osd_lv, lvs, 'db', dmcrypt_secret=dmcrypt_secret)
wal_device_path = get_osd_device_path(osd_lv, lvs, 'wal', dmcrypt_secret=dmcrypt_secret)
# Once symlinks are removed, the osd dir can be 'primed again.
process.run([
'ceph-bluestore-tool', '--cluster=%s' % conf.cluster,
'prime-osd-dir', '--dev', osd_lv_path,
'--path', osd_path])
# always re-do the symlink regardless if it exists, so that the block,
# block.wal, and block.db devices that may have changed can be mapped
# correctly every time
process.run(['ln', '-snf', osd_lv_path, os.path.join(osd_path, 'block')])
system.chown(os.path.join(osd_path, 'block'))
system.chown(osd_path)
if db_device_path:
destination = os.path.join(osd_path, 'block.db')
process.run(['ln', '-snf', db_device_path, destination])
system.chown(db_device_path)
if wal_device_path:
destination = os.path.join(osd_path, 'block.wal')
process.run(['ln', '-snf', wal_device_path, destination])
system.chown(wal_device_path)
# enable the ceph-volume unit for this OSD
systemctl.enable_volume(osd_id, osd_fsid, 'lvm')
# start the OSD
systemctl.start_osd(osd_id)
terminal.success("ceph-volume lvm activate successful for osd ID: %s" % osd_id)
def activate_filestore(lvs):
# find the osd
osd_lv = lvs.get(lv_tags={'ceph.type': 'data'})
if not osd_lv:
raise RuntimeError('Unable to find a data LV for filestore activation')
is_encrypted = osd_lv.tags.get('ceph.encrypted', '0') == '1'
osd_id = osd_lv.tags['ceph.osd_id']
conf.cluster = osd_lv.tags['ceph.cluster_name']
# it may have a volume with a journal
osd_journal_lv = lvs.get(lv_tags={'ceph.type': 'journal'})
# TODO: add sensible error reporting if this is ever the case
# blow up with a KeyError if this doesn't exist
osd_fsid = osd_lv.tags['ceph.osd_fsid']
if not osd_journal_lv:
# must be a disk partition, by quering blkid by the uuid we are ensuring that the
# device path is always correct
journal_uuid = osd_lv.tags['ceph.journal_uuid']
osd_journal = disk.get_device_from_partuuid(journal_uuid)
else:
journal_uuid = osd_journal_lv.lv_uuid
osd_journal = osd_lv.tags['ceph.journal_device']
if not osd_journal:
raise RuntimeError('unable to detect an lv or device journal for OSD %s' % osd_id)
# this is done here, so that previous checks that ensure path availability
# and correctness can still be enforced, and report if any issues are found
if is_encrypted:
lockbox_secret = osd_lv.tags['ceph.cephx_lockbox_secret']
# this keyring writing is idempotent
encryption_utils.write_lockbox_keyring(osd_id, osd_fsid, lockbox_secret)
dmcrypt_secret = encryption_utils.get_dmcrypt_key(osd_id, osd_fsid)
encryption_utils.luks_open(dmcrypt_secret, osd_lv.lv_path, osd_lv.lv_uuid)
encryption_utils.luks_open(dmcrypt_secret, osd_journal, journal_uuid)
osd_journal = '/dev/mapper/%s' % journal_uuid
source = '/dev/mapper/%s' % osd_lv.lv_uuid
else:
source = osd_lv.lv_path
# mount the osd
destination = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id)
if not system.device_is_mounted(source, destination=destination):
process.run(['mount', '-v', source, destination])
# always re-do the symlink regardless if it exists, so that the journal
# device path that may have changed can be mapped correctly every time
destination = '/var/lib/ceph/osd/%s-%s/journal' % (conf.cluster, osd_id)
process.run(['ln', '-snf', osd_journal, destination])
# make sure that the journal has proper permissions
system.chown(osd_journal)
# enable the ceph-volume unit for this OSD
systemctl.enable_volume(osd_id, osd_fsid, 'lvm')
# start the OSD
systemctl.start_osd(osd_id)
terminal.success("ceph-volume lvm activate successful for osd ID: %s" % osd_id)
def activate_bluestore(lvs):
# find the osd
osd_lv = lvs.get(lv_tags={'ceph.type': 'block'})
is_encrypted = osd_lv.tags.get('ceph.encrypted', '0') == '1'
dmcrypt_secret = None
osd_id = osd_lv.tags['ceph.osd_id']
conf.cluster = osd_lv.tags['ceph.cluster_name']
osd_fsid = osd_lv.tags['ceph.osd_fsid']
# mount on tmpfs the osd directory
osd_path = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id)
if not system.path_is_mounted(osd_path):
# mkdir -p and mount as tmpfs
prepare_utils.create_osd_path(osd_id, tmpfs=True)
# XXX This needs to be removed once ceph-bluestore-tool can deal with
# symlinks that exist in the osd dir
for link_name in ['block', 'block.db', 'block.wal']:
link_path = os.path.join(osd_path, link_name)
if os.path.exists(link_path):
os.unlink(os.path.join(osd_path, link_name))
# encryption is handled here, before priming the OSD dir
if is_encrypted:
osd_lv_path = '/dev/mapper/%s' % osd_lv.lv_uuid
lockbox_secret = osd_lv.tags['ceph.cephx_lockbox_secret']
encryption_utils.write_lockbox_keyring(osd_id, osd_fsid, lockbox_secret)
dmcrypt_secret = encryption_utils.get_dmcrypt_key(osd_id, osd_fsid)
encryption_utils.luks_open(dmcrypt_secret, osd_lv.lv_path, osd_lv.lv_uuid)
else:
osd_lv_path = osd_lv.lv_path
db_device_path = get_osd_device_path(osd_lv, lvs, 'db', dmcrypt_secret=dmcrypt_secret)
wal_device_path = get_osd_device_path(osd_lv, lvs, 'wal', dmcrypt_secret=dmcrypt_secret)
# Once symlinks are removed, the osd dir can be 'primed again.
process.run([
'ceph-bluestore-tool', '--cluster=%s' % conf.cluster,
'prime-osd-dir', '--dev', osd_lv_path,
'--path', osd_path])
# always re-do the symlink regardless if it exists, so that the block,
# block.wal, and block.db devices that may have changed can be mapped
# correctly every time
process.run(['ln', '-snf', osd_lv_path, os.path.join(osd_path, 'block')])
system.chown(os.path.join(osd_path, 'block'))
system.chown(osd_path)
if db_device_path:
destination = os.path.join(osd_path, 'block.db')
process.run(['ln', '-snf', db_device_path, destination])
system.chown(db_device_path)
if wal_device_path:
destination = os.path.join(osd_path, 'block.wal')
process.run(['ln', '-snf', wal_device_path, destination])
system.chown(wal_device_path)
# enable the ceph-volume unit for this OSD
systemctl.enable_volume(osd_id, osd_fsid, 'lvm')
# start the OSD
systemctl.start_osd(osd_id)
terminal.success("ceph-volume lvm activate successful for osd ID: %s" % osd_id)
def zap(self, args):
for device in args.devices:
if disk.is_mapper_device(device):
terminal.error(
"Refusing to zap the mapper device: {}".format(device))
raise SystemExit(1)
lv = api.get_lv_from_argument(device)
if lv:
# we are zapping a logical volume
path = lv.lv_path
self.unmount_lv(lv)
else:
# we are zapping a partition
#TODO: ensure device is a partition
path = device
# check to if it is encrypted to close
partuuid = disk.get_partuuid(device)
if encryption.status("/dev/mapper/{}".format(partuuid)):
dmcrypt_uuid = partuuid
self.dmcrypt_close(dmcrypt_uuid)
mlogger.info("Zapping: %s", path)
# check if there was a pv created with the
# name of device
pvs = api.PVolumes()
pvs.filter(pv_name=device)
vgs = set([pv.vg_name for pv in pvs])
for pv in pvs:
vg_name = pv.vg_name
lv = None
if pv.lv_uuid:
lv = api.get_lv(vg_name=vg_name, lv_uuid=pv.lv_uuid)
if lv:
self.unmount_lv(lv)
if args.destroy:
for vg_name in vgs:
mlogger.info(
"Destroying volume group %s because --destroy was given",
vg_name)
api.remove_vg(vg_name)
mlogger.info(
"Destroying physical volume %s because --destroy was given",
device)
api.remove_pv(device)
wipefs(path)
zap_data(path)
if lv and not pvs:
# remove all lvm metadata
lv.clear_tags()
terminal.success("Zapping successful for: %s" %
", ".join(args.devices))
def zap(self, args):
device = args.device
lv = api.get_lv_from_argument(device)
if lv:
# we are zapping a logical volume
path = lv.lv_path
else:
# we are zapping a partition
#TODO: ensure device is a partition
path = device
logger.info("Zapping: %s", path)
terminal.write("Zapping: %s" % path)
if args.destroy and not lv:
# check if there was a pv created with the
# name of device
pv = api.PVolumes().get(pv_name=device)
if pv:
logger.info(
"Found a physical volume created from %s, will destroy all it's vgs and lvs",
device)
vg_name = pv.vg_name
logger.info(
"Destroying volume group %s because --destroy was given",
vg_name)
terminal.write(
"Destroying volume group %s because --destroy was given" %
vg_name)
api.remove_vg(vg_name)
logger.info(
"Destroying physical volume %s because --destroy was given",
device)
terminal.write(
"Destroying physical volume %s because --destroy was given"
% device)
api.remove_pv(device)
else:
logger.info(
"Skipping --destroy because no associated physical volumes are found for %s",
device)
terminal.write(
"Skipping --destroy because no associated physical volumes are found for %s"
% device)
wipefs(path)
zap_data(path)
if lv:
# remove all lvm metadata
lv.clear_tags()
terminal.success("Zapping successful for: %s" % path)
def zap(self, args):
device = args.device
lv = api.get_lv_from_argument(device)
if lv:
# we are zapping a logical volume
path = lv.lv_path
else:
# we are zapping a partition
#TODO: ensure device is a partition
path = device
mlogger.info("Zapping: %s", path)
# check if there was a pv created with the
# name of device
pv = api.get_pv(pv_name=device)
if pv:
vg_name = pv.vg_name
lv = api.get_lv(vg_name=vg_name)
if lv:
osd_path = "/var/lib/ceph/osd/{}-{}".format(
lv.tags['ceph.cluster_name'], lv.tags['ceph.osd_id'])
if system.path_is_mounted(osd_path):
mlogger.info("Unmounting %s", osd_path)
system.unmount(osd_path)
if args.destroy and pv:
logger.info(
"Found a physical volume created from %s, will destroy all it's vgs and lvs",
device)
vg_name = pv.vg_name
mlogger.info(
"Destroying volume group %s because --destroy was given",
vg_name)
api.remove_vg(vg_name)
mlogger.info(
"Destroying physical volume %s because --destroy was given",
device)
api.remove_pv(device)
elif args.destroy and not pv:
mlogger.info(
"Skipping --destroy because no associated physical volumes are found for %s",
device)
wipefs(path)
zap_data(path)
if lv and not pv:
# remove all lvm metadata
lv.clear_tags()
terminal.success("Zapping successful for: %s" % path)
def zap(self, args):
for device in args.devices:
if disk.is_mapper_device(device):
terminal.error("Refusing to zap the mapper device: {}".format(device))
raise SystemExit(1)
lv = api.get_lv_from_argument(device)
if lv:
# we are zapping a logical volume
path = lv.lv_path
self.unmount_lv(lv)
else:
# we are zapping a partition
#TODO: ensure device is a partition
path = device
# check to if it is encrypted to close
partuuid = disk.get_partuuid(device)
if encryption.status("/dev/mapper/{}".format(partuuid)):
dmcrypt_uuid = partuuid
self.dmcrypt_close(dmcrypt_uuid)
mlogger.info("Zapping: %s", path)
# check if there was a pv created with the
# name of device
pvs = api.PVolumes()
pvs.filter(pv_name=device)
vgs = set([pv.vg_name for pv in pvs])
for pv in pvs:
vg_name = pv.vg_name
lv = None
if pv.lv_uuid:
lv = api.get_lv(vg_name=vg_name, lv_uuid=pv.lv_uuid)
if lv:
self.unmount_lv(lv)
if args.destroy:
for vg_name in vgs:
mlogger.info("Destroying volume group %s because --destroy was given", vg_name)
api.remove_vg(vg_name)
if not lv:
mlogger.info("Destroying physical volume %s because --destroy was given", device)
api.remove_pv(device)
wipefs(path)
zap_data(path)
if lv and not pvs:
# remove all lvm metadata
lv.clear_tags()
terminal.success("Zapping successful for: %s" % ", ".join(args.devices))
def safe_prepare(self, args):
"""
An intermediate step between `main()` and `prepare()` so that we can
capture the `self.osd_id` in case we need to rollback
"""
try:
self.prepare(args)
except Exception:
logger.error('lvm prepare was unable to complete')
logger.info('will rollback OSD ID creation')
rollback_osd(args, self.osd_id)
raise
terminal.success("ceph-volume lvm prepare successful for: %s" % args.data)
def safe_prepare(self, args):
"""
An intermediate step between `main()` and `prepare()` so that we can
capture the `self.osd_id` in case we need to rollback
"""
try:
self.prepare(args)
except Exception:
logger.error('lvm prepare was unable to complete')
logger.info('will rollback OSD ID creation')
rollback_osd(args, self.osd_id)
raise
terminal.success("ceph-volume lvm prepare successful for: %s" % args.data)
def migrate_to_new(self, osd_id, osd_fsid, devices, target_lv):
source_devices = self.get_source_devices(devices)
target_type = self.get_target_type_by_source(source_devices)
if not target_type:
mlogger.error(
"Unable to determine new volume type,"
" please use new-db or new-wal command before.")
raise SystemExit(
"Unable to migrate to : {}".format(self.args.target))
target_path = target_lv.lv_path
try:
tag_tracker = VolumeTagTracker(devices, target_lv)
# we need to update lvm tags for all the remaining volumes
# and clear for ones which to be removed
# ceph-bluestore-tool removes source volume(s) other than block one
# and attaches target one after successful migration
tag_tracker.replace_lvs(source_devices, target_type)
osd_path = get_osd_path(osd_id, osd_fsid)
source_args = self.get_source_args(osd_path, source_devices)
mlogger.info("Migrate to new, Source: {} Target: {}".format(
source_args, target_path))
stdout, stderr, exit_code = process.call([
'ceph-bluestore-tool',
'--path',
osd_path,
'--dev-target',
target_path,
'--command',
'bluefs-bdev-migrate'] +
source_args)
if exit_code != 0:
mlogger.error(
'Failed to migrate device, error code:{}'.format(exit_code))
raise SystemExit(
'Failed to migrate to : {}'.format(self.args.target))
else:
system.chown(os.path.join(osd_path, "block.{}".format(
target_type)))
terminal.success('Migration successful.')
except:
tag_tracker.undo()
raise
return
def activate_bluestore(meta, tmpfs, systemd):
# find the osd
osd_id = meta['osd_id']
osd_uuid = meta['osd_uuid']
# mount on tmpfs the osd directory
osd_path = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id)
if not system.path_is_mounted(osd_path):
# mkdir -p and mount as tmpfs
prepare_utils.create_osd_path(osd_id, tmpfs=tmpfs)
# XXX This needs to be removed once ceph-bluestore-tool can deal with
# symlinks that exist in the osd dir
for link_name in ['block', 'block.db', 'block.wal']:
link_path = os.path.join(osd_path, link_name)
if os.path.exists(link_path):
os.unlink(os.path.join(osd_path, link_name))
# Once symlinks are removed, the osd dir can be 'primed again. chown first,
# regardless of what currently exists so that ``prime-osd-dir`` can succeed
# even if permissions are somehow messed up
system.chown(osd_path)
prime_command = [
'ceph-bluestore-tool',
'prime-osd-dir',
'--path',
osd_path,
'--no-mon-config',
'--dev',
meta['device'],
]
process.run(prime_command)
# always re-do the symlink regardless if it exists, so that the block,
# block.wal, and block.db devices that may have changed can be mapped
# correctly every time
prepare_utils.link_block(meta['device'], osd_id)
if 'device_db' in meta:
prepare_utils.link_db(meta['device_db'], osd_id, osd_uuid)
if 'device_wal' in meta:
prepare_utils.link_wal(meta['device_wal'], osd_id, osd_uuid)
system.chown(osd_path)
terminal.success("ceph-volume raw activate successful for osd ID: %s" %
osd_id)
def migrate_to_existing(self, osd_id, osd_fsid, devices, target_lv):
target_type = target_lv.tags["ceph.type"]
if target_type == "wal":
mlogger.error("Migrate to WAL is not supported")
raise SystemExit(
"Unable to migrate to : {}".format(self.args.target))
target_filename = self.get_filename_by_type(target_type)
if (target_filename == ""):
mlogger.error(
"Target Logical Volume doesn't have proper volume type "
"(ceph.type LVM tag): {}".format(target_type))
raise SystemExit(
"Unable to migrate to : {}".format(self.args.target))
osd_path = get_osd_path(osd_id, osd_fsid)
source_devices = self.get_source_devices(devices, target_type)
target_path = os.path.join(osd_path, target_filename)
tag_tracker = VolumeTagTracker(devices, target_lv)
try:
# ceph-bluestore-tool removes source volume(s) other than
# block and target ones after successful migration
tag_tracker.remove_lvs(source_devices, target_type)
source_args = self.get_source_args(osd_path, source_devices)
mlogger.info("Migrate to existing, Source: {} Target: {}".format(
source_args, target_path))
stdout, stderr, exit_code = process.call([
'ceph-bluestore-tool',
'--path',
osd_path,
'--dev-target',
target_path,
'--command',
'bluefs-bdev-migrate'] +
source_args)
if exit_code != 0:
mlogger.error(
'Failed to migrate device, error code:{}'.format(exit_code))
raise SystemExit(
'Failed to migrate to : {}'.format(self.args.target))
else:
terminal.success('Migration successful.')
except:
tag_tracker.undo()
raise
return
def create(self, args):
if not args.osd_fsid:
args.osd_fsid = system.generate_uuid()
prepare_step = Prepare([])
prepare_step.safe_prepare(args)
osd_id = prepare_step.osd_id
try:
# we try this for activate only when 'creating' an OSD, because a rollback should not
# happen when doing normal activation. For example when starting an OSD, systemd will call
# activate, which would never need to be rolled back.
Activate([]).activate(args)
except Exception:
logger.exception('lvm activate was unable to complete, while creating the OSD')
logger.info('will rollback OSD ID creation')
rollback_osd(args, osd_id)
raise
terminal.success("ceph-volume lvm create successful for: %s" % args.data)
def activate_filestore(lvs):
# find the osd
osd_lv = lvs.get(lv_tags={'ceph.type': 'data'})
if not osd_lv:
raise RuntimeError('Unable to find a data LV for filestore activation')
osd_id = osd_lv.tags['ceph.osd_id']
conf.cluster = osd_lv.tags['ceph.cluster_name']
# it may have a volume with a journal
osd_journal_lv = lvs.get(lv_tags={'ceph.type': 'journal'})
# TODO: add sensible error reporting if this is ever the case
# blow up with a KeyError if this doesn't exist
osd_fsid = osd_lv.tags['ceph.osd_fsid']
if not osd_journal_lv:
# must be a disk partition, by quering blkid by the uuid we are ensuring that the
# device path is always correct
osd_journal = disk.get_device_from_partuuid(
osd_lv.tags['ceph.journal_uuid'])
else:
osd_journal = osd_lv.tags['ceph.journal_device']
if not osd_journal:
raise RuntimeError(
'unable to detect an lv or device journal for OSD %s' % osd_id)
# mount the osd
source = osd_lv.lv_path
destination = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id)
if not system.device_is_mounted(source, destination=destination):
process.run(['mount', '-v', source, destination])
# always re-do the symlink regardless if it exists, so that the journal
# device path that may have changed can be mapped correctly every time
destination = '/var/lib/ceph/osd/%s-%s/journal' % (conf.cluster, osd_id)
process.run(['ln', '-snf', osd_journal, destination])
# make sure that the journal has proper permissions
system.chown(osd_journal)
# enable the ceph-volume unit for this OSD
systemctl.enable_volume(osd_id, osd_fsid, 'lvm')
# start the OSD
systemctl.start_osd(osd_id)
terminal.success("ceph-volume lvm activate successful for osd ID: %s" %
osd_id)
def safe_prepare(self, args=None):
"""
An intermediate step between `main()` and `prepare()` so that we can
capture the `self.osd_id` in case we need to rollback
:param args: Injected args, usually from `lvm create` which compounds
both `prepare` and `create`
"""
if args is not None:
self.args = args
try:
self.prepare()
except Exception:
logger.exception('lvm prepare was unable to complete')
logger.info('will rollback OSD ID creation')
rollback_osd(self.args, self.osd_id)
raise
terminal.success("ceph-volume lvm prepare successful for: %s" % self.args.data)
def safe_prepare(self, args=None):
"""
An intermediate step between `main()` and `prepare()` so that we can
capture the `self.osd_id` in case we need to rollback
:param args: Injected args, usually from `raw create` which compounds
both `prepare` and `create`
"""
if args is not None:
self.args = args
try:
self.prepare()
except Exception:
logger.exception('raw prepare was unable to complete')
logger.info('will rollback OSD ID creation')
rollback_osd(self.args, self.osd_id)
raise
terminal.success("ceph-volume raw prepare successful for: %s" % self.args.data)
def zap(self, args):
device = args.device
lv = api.get_lv_from_argument(device)
if lv:
# we are zapping a logical volume
path = lv.lv_path
else:
# we are zapping a partition
#TODO: ensure device is a partition
path = device
mlogger.info("Zapping: %s", path)
# check if there was a pv created with the
# name of device
pv = api.get_pv(pv_name=device)
if pv:
vg_name = pv.vg_name
lv = api.get_lv(vg_name=vg_name)
if lv:
osd_path = "/var/lib/ceph/osd/{}-{}".format(lv.tags['ceph.cluster_name'], lv.tags['ceph.osd_id'])
if system.path_is_mounted(osd_path):
mlogger.info("Unmounting %s", osd_path)
system.unmount(osd_path)
if args.destroy and pv:
logger.info("Found a physical volume created from %s, will destroy all it's vgs and lvs", device)
vg_name = pv.vg_name
mlogger.info("Destroying volume group %s because --destroy was given", vg_name)
api.remove_vg(vg_name)
mlogger.info("Destroying physical volume %s because --destroy was given", device)
api.remove_pv(device)
elif args.destroy and not pv:
mlogger.info("Skipping --destroy because no associated physical volumes are found for %s", device)
wipefs(path)
zap_data(path)
if lv and not pv:
# remove all lvm metadata
lv.clear_tags()
terminal.success("Zapping successful for: %s" % path)
def create(self, args):
if not args.osd_fsid:
args.osd_fsid = system.generate_uuid()
prepare_step = Prepare([])
prepare_step.safe_prepare(args)
osd_id = prepare_step.osd_id
try:
# we try this for activate only when 'creating' an OSD, because a rollback should not
# happen when doing normal activation. For example when starting an OSD, systemd will call
# activate, which would never need to be rolled back.
Activate([]).activate(args)
except Exception:
logger.error(
'lvm activate was unable to complete, while creating the OSD')
logger.info('will rollback OSD ID creation')
rollback_osd(args, osd_id)
raise
terminal.success("ceph-volume lvm create successful for: %s" %
args.data)
def activate_filestore(lvs):
# find the osd
osd_lv = lvs.get(lv_tags={'ceph.type': 'data'})
if not osd_lv:
raise RuntimeError('Unable to find a data LV for filestore activation')
osd_id = osd_lv.tags['ceph.osd_id']
conf.cluster = osd_lv.tags['ceph.cluster_name']
# it may have a volume with a journal
osd_journal_lv = lvs.get(lv_tags={'ceph.type': 'journal'})
# TODO: add sensible error reporting if this is ever the case
# blow up with a KeyError if this doesn't exist
osd_fsid = osd_lv.tags['ceph.osd_fsid']
if not osd_journal_lv:
# must be a disk partition, by quering blkid by the uuid we are ensuring that the
# device path is always correct
osd_journal = disk.get_device_from_partuuid(osd_lv.tags['ceph.journal_uuid'])
else:
osd_journal = osd_lv.tags['ceph.journal_device']
if not osd_journal:
raise RuntimeError('unable to detect an lv or device journal for OSD %s' % osd_id)
# mount the osd
source = osd_lv.lv_path
destination = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id)
if not system.device_is_mounted(source, destination=destination):
process.run(['mount', '-v', source, destination])
# always re-do the symlink regardless if it exists, so that the journal
# device path that may have changed can be mapped correctly every time
destination = '/var/lib/ceph/osd/%s-%s/journal' % (conf.cluster, osd_id)
process.run(['ln', '-snf', osd_journal, destination])
# make sure that the journal has proper permissions
system.chown(osd_journal)
# enable the ceph-volume unit for this OSD
systemctl.enable_volume(osd_id, osd_fsid, 'lvm')
# start the OSD
systemctl.start_osd(osd_id)
terminal.success("ceph-volume lvm activate successful for osd ID: %s" % osd_id)
def scan(self, args):
osd_metadata = {'cluster_name': conf.cluster}
device_mounts = system.get_mounts(devices=True)
osd_path = None
logger.info('detecting if argument is a device or a directory: %s', args.osd_path)
if os.path.isdir(args.osd_path):
logger.info('will scan directly, path is a directory')
osd_path = args.osd_path
else:
# assume this is a device, check if it is mounted and use that path
logger.info('path is not a directory, will check if mounted')
if system.device_is_mounted(args.osd_path):
logger.info('argument is a device, which is mounted')
mounted_osd_paths = device_mounts.get(args.osd_path)
osd_path = mounted_osd_paths[0] if len(mounted_osd_paths) else None
# argument is not a directory, and it is not a device that is mounted
# somewhere so temporarily mount it to poke inside, otherwise, scan
# directly
if not osd_path:
logger.info('device is not mounted, will mount it temporarily to scan')
with system.tmp_mount(args.osd_path) as osd_path:
osd_metadata = self.scan_directory(osd_path)
else:
logger.info('will scan OSD directory at path: %s', osd_path)
osd_metadata = self.scan_directory(osd_path)
osd_id = osd_metadata['whoami']
osd_fsid = osd_metadata['fsid']
filename = '%s-%s.json' % (osd_id, osd_fsid)
json_path = os.path.join(self.etc_path, filename)
if os.path.exists(json_path) and not args.stdout:
if not args.force:
raise RuntimeError(
'--force was not used and OSD metadata file exists: %s' % json_path
)
if args.stdout:
print(json.dumps(osd_metadata, indent=4, sort_keys=True, ensure_ascii=False))
else:
with open(json_path, 'w') as fp:
json.dump(osd_metadata, fp, indent=4, sort_keys=True, ensure_ascii=False)
terminal.success(
'OSD %s got scanned and metadata persisted to file: %s' % (
osd_id,
json_path
)
)
terminal.success(
'To take over managment of this scanned OSD, and disable ceph-disk and udev, run:'
)
terminal.success(' ceph-volume simple activate %s %s' % (osd_id, osd_fsid))
if not osd_metadata.get('data'):
msg = 'Unable to determine device mounted on %s' % args.osd_path
logger.warning(msg)
terminal.warning(msg)
terminal.warning('OSD will not be able to start without this information:')
terminal.warning(' "data": "/path/to/device",')
logger.warning('Unable to determine device mounted on %s' % args.osd_path)
def zap(self, args):
device = args.device
lv = api.get_lv_from_argument(device)
if lv:
# we are zapping a logical volume
path = lv.lv_path
else:
# we are zapping a partition
#TODO: ensure device is a partition
path = device
logger.info("Zapping: %s", path)
terminal.write("Zapping: %s" % path)
wipefs(path)
zap_data(path)
if lv:
# remove all lvm metadata
lv.clear_tags()
terminal.success("Zapping successful for: %s" % path)
def zap(self, args):
device = args.device
lv = api.get_lv_from_argument(device)
if lv:
# we are zapping a logical volume
path = lv.lv_path
else:
# we are zapping a partition
#TODO: ensure device is a partition
path = device
logger.info("Zapping: %s", path)
terminal.write("Zapping: %s" % path)
wipefs(path)
zap_data(path)
if lv:
# remove all lvm metadata
lv.clear_tags()
terminal.success("Zapping successful for: %s" % path)
def zap(self, args):
device = args.device
if disk.is_mapper_device(device):
terminal.error(
"Refusing to zap the mapper device: {}".format(device))
raise SystemExit(1)
lv = api.get_lv_from_argument(device)
if lv:
# we are zapping a logical volume
path = lv.lv_path
else:
# we are zapping a partition
#TODO: ensure device is a partition
path = device
mlogger.info("Zapping: %s", path)
# check if there was a pv created with the
# name of device
pv = api.get_pv(pv_name=device)
if pv:
vg_name = pv.vg_name
lv = api.get_lv(vg_name=vg_name)
dmcrypt = False
dmcrypt_uuid = None
if lv:
if lv.tags.get('ceph.cluster_name') and lv.tags.get('ceph.osd_id'):
lv_path = "/var/lib/ceph/osd/{}-{}".format(
lv.tags['ceph.cluster_name'], lv.tags['ceph.osd_id'])
else:
lv_path = lv.path
dmcrypt_uuid = lv.lv_uuid
dmcrypt = lv.encrypted
if system.path_is_mounted(lv_path):
mlogger.info("Unmounting %s", lv_path)
system.unmount(lv_path)
else:
# we're most likely dealing with a partition here, check to
# see if it was encrypted
partuuid = disk.get_partuuid(device)
if encryption.status("/dev/mapper/{}".format(partuuid)):
dmcrypt_uuid = partuuid
dmcrypt = True
if dmcrypt and dmcrypt_uuid:
dmcrypt_path = "/dev/mapper/{}".format(dmcrypt_uuid)
mlogger.info("Closing encrypted path %s", dmcrypt_path)
encryption.dmcrypt_close(dmcrypt_path)
if args.destroy and pv:
logger.info(
"Found a physical volume created from %s, will destroy all it's vgs and lvs",
device)
vg_name = pv.vg_name
mlogger.info(
"Destroying volume group %s because --destroy was given",
vg_name)
api.remove_vg(vg_name)
mlogger.info(
"Destroying physical volume %s because --destroy was given",
device)
api.remove_pv(device)
elif args.destroy and not pv:
mlogger.info(
"Skipping --destroy because no associated physical volumes are found for %s",
device)
wipefs(path)
zap_data(path)
if lv and not pv:
# remove all lvm metadata
lv.clear_tags()
terminal.success("Zapping successful for: %s" % path)
def activate(self, args):
with open(args.json_config, 'r') as fp:
osd_metadata = json.load(fp)
# Make sure that required devices are configured
self.validate_devices(osd_metadata)
osd_id = osd_metadata.get('whoami', args.osd_id)
osd_fsid = osd_metadata.get('fsid', args.osd_fsid)
data_uuid = osd_metadata.get('data', {}).get('uuid')
conf.cluster = osd_metadata.get('cluster_name', 'ceph')
if not data_uuid:
raise RuntimeError(
'Unable to activate OSD %s - no "uuid" key found for data' %
args.osd_id)
# Encryption detection, and capturing of the keys to decrypt
self.is_encrypted = osd_metadata.get('encrypted', False)
self.encryption_type = osd_metadata.get('encryption_type')
if self.is_encrypted:
lockbox_secret = osd_metadata.get('lockbox.keyring')
# write the keyring always so that we can unlock
encryption_utils.write_lockbox_keyring(osd_id, osd_fsid,
lockbox_secret)
# Store the secret around so that the decrypt method can reuse
raw_dmcrypt_secret = encryption_utils.get_dmcrypt_key(
osd_id, osd_fsid)
# Note how both these calls need b64decode. For some reason, the
# way ceph-disk creates these keys, it stores them in the monitor
# *undecoded*, requiring this decode call again. The lvm side of
# encryption doesn't need it, so we are assuming here that anything
# that `simple` scans, will come from ceph-disk and will need this
# extra decode call here
self.dmcrypt_secret = base64.b64decode(raw_dmcrypt_secret)
cluster_name = osd_metadata.get('cluster_name', 'ceph')
osd_dir = '/var/lib/ceph/osd/%s-%s' % (cluster_name, osd_id)
# XXX there is no support for LVM here
data_device = self.get_device(data_uuid)
if not data_device:
raise RuntimeError("osd fsid {} doesn't exist, this file will "
"be skipped, consider cleaning legacy "
"json file {}".format(osd_metadata['fsid'],
args.json_config))
journal_device = self.get_device(
osd_metadata.get('journal', {}).get('uuid'))
block_device = self.get_device(
osd_metadata.get('block', {}).get('uuid'))
block_db_device = self.get_device(
osd_metadata.get('block.db', {}).get('uuid'))
block_wal_device = self.get_device(
osd_metadata.get('block.wal', {}).get('uuid'))
if not system.device_is_mounted(data_device, destination=osd_dir):
if osd_metadata.get('type') == 'filestore':
prepare_utils.mount_osd(data_device, osd_id)
else:
process.run(['mount', '-v', data_device, osd_dir])
device_map = {
'journal': journal_device,
'block': block_device,
'block.db': block_db_device,
'block.wal': block_wal_device
}
for name, device in device_map.items():
if not device:
continue
# always re-do the symlink regardless if it exists, so that the journal
# device path that may have changed can be mapped correctly every time
destination = os.path.join(osd_dir, name)
process.run(['ln', '-snf', device, destination])
# make sure that the journal has proper permissions
system.chown(device)
self.enable_systemd_units(osd_id, osd_fsid)
terminal.success('Successfully activated OSD %s with FSID %s' %
(osd_id, osd_fsid))
def activate(self, args):
with open(args.json_config, 'r') as fp:
osd_metadata = json.load(fp)
osd_id = osd_metadata.get('whoami', args.osd_id)
osd_fsid = osd_metadata.get('fsid', args.osd_fsid)
cluster_name = osd_metadata.get('cluster_name', 'ceph')
osd_dir = '/var/lib/ceph/osd/%s-%s' % (cluster_name, osd_id)
data_uuid = osd_metadata.get('data', {}).get('uuid')
if not data_uuid:
raise RuntimeError(
'Unable to activate OSD %s - no "uuid" key found for data' % args.osd_id
)
data_device = disk.get_device_from_partuuid(data_uuid)
journal_device = disk.get_device_from_partuuid(osd_metadata.get('journal', {}).get('uuid'))
block_device = disk.get_device_from_partuuid(osd_metadata.get('block', {}).get('uuid'))
block_db_device = disk.get_device_from_partuuid(osd_metadata.get('block.db', {}).get('uuid'))
block_wal_device = disk.get_device_from_partuuid(
osd_metadata.get('block.wal', {}).get('uuid')
)
if not system.device_is_mounted(data_device, destination=osd_dir):
process.run(['mount', '-v', data_device, osd_dir])
device_map = {
'journal': journal_device,
'block': block_device,
'block.db': block_db_device,
'block.wal': block_wal_device
}
for name, device in device_map.items():
if not device:
continue
# always re-do the symlink regardless if it exists, so that the journal
# device path that may have changed can be mapped correctly every time
destination = os.path.join(osd_dir, name)
process.run(['ln', '-snf', device, destination])
# make sure that the journal has proper permissions
system.chown(device)
if not self.systemd:
# enable the ceph-volume unit for this OSD
systemctl.enable_volume(osd_id, osd_fsid, 'simple')
# disable any/all ceph-disk units
systemctl.mask_ceph_disk()
# enable the OSD
systemctl.enable_osd(osd_id)
# start the OSD
systemctl.start_osd(osd_id)
terminal.success('Successfully activated OSD %s with FSID %s' % (osd_id, osd_fsid))
terminal.warning(
('All ceph-disk systemd units have been disabled to '
'prevent OSDs getting triggered by UDEV events')
)
def activate(self, args):
with open(args.json_config, 'r') as fp:
osd_metadata = json.load(fp)
osd_id = osd_metadata.get('whoami', args.osd_id)
osd_fsid = osd_metadata.get('fsid', args.osd_fsid)
cluster_name = osd_metadata.get('cluster_name', 'ceph')
osd_dir = '/var/lib/ceph/osd/%s-%s' % (cluster_name, osd_id)
data_uuid = osd_metadata.get('data', {}).get('uuid')
if not data_uuid:
raise RuntimeError(
'Unable to activate OSD %s - no "uuid" key found for data' % args.osd_id
)
data_device = disk.get_device_from_partuuid(data_uuid)
journal_device = disk.get_device_from_partuuid(osd_metadata.get('journal', {}).get('uuid'))
block_device = disk.get_device_from_partuuid(osd_metadata.get('block', {}).get('uuid'))
block_db_device = disk.get_device_from_partuuid(osd_metadata.get('block.db', {}).get('uuid'))
block_wal_device = disk.get_device_from_partuuid(
osd_metadata.get('block.wal', {}).get('uuid')
)
if not system.device_is_mounted(data_device, destination=osd_dir):
process.run(['sudo', 'mount', '-v', data_device, osd_dir])
device_map = {
'journal': journal_device,
'block': block_device,
'block.db': block_db_device,
'block.wal': block_wal_device
}
for name, device in device_map.items():
if not device:
continue
# always re-do the symlink regardless if it exists, so that the journal
# device path that may have changed can be mapped correctly every time
destination = os.path.join(osd_dir, name)
process.run(['sudo', 'ln', '-snf', device, destination])
# make sure that the journal has proper permissions
system.chown(device)
if not self.systemd:
# enable the ceph-volume unit for this OSD
systemctl.enable_volume(osd_id, osd_fsid, 'simple')
# disable any/all ceph-disk units
systemctl.mask_ceph_disk()
# enable the OSD
systemctl.enable_osd(osd_id)
# start the OSD
systemctl.start_osd(osd_id)
if not self.systemd:
terminal.success('Successfully activated OSD %s with FSID %s' % (osd_id, osd_fsid))
terminal.warning(
('All ceph-disk systemd units have been disabled to '
'prevent OSDs getting triggered by UDEV events')
)
def scan(self, args):
osd_metadata = {'cluster_name': conf.cluster}
osd_path = None
logger.info('detecting if argument is a device or a directory: %s', args.osd_path)
if os.path.isdir(args.osd_path):
logger.info('will scan directly, path is a directory')
osd_path = args.osd_path
else:
# assume this is a device, check if it is mounted and use that path
logger.info('path is not a directory, will check if mounted')
if system.device_is_mounted(args.osd_path):
logger.info('argument is a device, which is mounted')
mounted_osd_paths = self.device_mounts.get(args.osd_path)
osd_path = mounted_osd_paths[0] if len(mounted_osd_paths) else None
# argument is not a directory, and it is not a device that is mounted
# somewhere so temporarily mount it to poke inside, otherwise, scan
# directly
if not osd_path:
# check if we have an encrypted device first, so that we can poke at
# the lockbox instead
if self.is_encrypted:
if not self.encryption_metadata.get('lockbox'):
raise RuntimeError(
'Lockbox partition was not found for device: %s' % args.osd_path
)
osd_metadata = self.scan_encrypted()
else:
logger.info('device is not mounted, will mount it temporarily to scan')
with system.tmp_mount(args.osd_path) as osd_path:
osd_metadata = self.scan_directory(osd_path)
else:
if self.is_encrypted:
logger.info('will scan encrypted OSD directory at path: %s', osd_path)
osd_metadata = self.scan_encrypted(osd_path)
else:
logger.info('will scan OSD directory at path: %s', osd_path)
osd_metadata = self.scan_directory(osd_path)
osd_id = osd_metadata['whoami']
osd_fsid = osd_metadata['fsid']
filename = '%s-%s.json' % (osd_id, osd_fsid)
json_path = os.path.join(self.etc_path, filename)
if os.path.exists(json_path) and not args.stdout:
if not args.force:
raise RuntimeError(
'--force was not used and OSD metadata file exists: %s' % json_path
)
if args.stdout:
print(json.dumps(osd_metadata, indent=4, sort_keys=True, ensure_ascii=False))
else:
with open(json_path, 'w') as fp:
json.dump(osd_metadata, fp, indent=4, sort_keys=True, ensure_ascii=False)
fp.write(os.linesep)
terminal.success(
'OSD %s got scanned and metadata persisted to file: %s' % (
osd_id,
json_path
)
)
terminal.success(
'To take over management of this scanned OSD, and disable ceph-disk and udev, run:'
)
terminal.success(' ceph-volume simple activate %s %s' % (osd_id, osd_fsid))
if not osd_metadata.get('data'):
msg = 'Unable to determine device mounted on %s' % args.osd_path
logger.warning(msg)
terminal.warning(msg)
terminal.warning('OSD will not be able to start without this information:')
terminal.warning(' "data": "/path/to/device",')
logger.warning('Unable to determine device mounted on %s' % args.osd_path)
def activate_bluestore(osd_lvs, no_systemd=False, no_tmpfs=False):
for lv in osd_lvs:
if lv.tags.get('ceph.type') == 'block':
osd_block_lv = lv
break
else:
raise RuntimeError('could not find a bluestore OSD to activate')
is_encrypted = osd_block_lv.tags.get('ceph.encrypted', '0') == '1'
dmcrypt_secret = None
osd_id = osd_block_lv.tags['ceph.osd_id']
conf.cluster = osd_block_lv.tags['ceph.cluster_name']
osd_fsid = osd_block_lv.tags['ceph.osd_fsid']
# mount on tmpfs the osd directory
osd_path = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id)
if not system.path_is_mounted(osd_path):
# mkdir -p and mount as tmpfs
prepare_utils.create_osd_path(osd_id, tmpfs=not no_tmpfs)
# XXX This needs to be removed once ceph-bluestore-tool can deal with
# symlinks that exist in the osd dir
for link_name in ['block', 'block.db', 'block.wal']:
link_path = os.path.join(osd_path, link_name)
if os.path.exists(link_path):
os.unlink(os.path.join(osd_path, link_name))
# encryption is handled here, before priming the OSD dir
if is_encrypted:
osd_lv_path = '/dev/mapper/%s' % osd_block_lv.lv_uuid
lockbox_secret = osd_block_lv.tags['ceph.cephx_lockbox_secret']
encryption_utils.write_lockbox_keyring(osd_id, osd_fsid,
lockbox_secret)
dmcrypt_secret = encryption_utils.get_dmcrypt_key(osd_id, osd_fsid)
encryption_utils.luks_open(dmcrypt_secret, osd_block_lv.lv_path,
osd_block_lv.lv_uuid)
else:
osd_lv_path = osd_block_lv.lv_path
db_device_path = get_osd_device_path(osd_lvs,
'db',
dmcrypt_secret=dmcrypt_secret)
wal_device_path = get_osd_device_path(osd_lvs,
'wal',
dmcrypt_secret=dmcrypt_secret)
# Once symlinks are removed, the osd dir can be 'primed again. chown first,
# regardless of what currently exists so that ``prime-osd-dir`` can succeed
# even if permissions are somehow messed up
system.chown(osd_path)
prime_command = [
'ceph-bluestore-tool',
'--cluster=%s' % conf.cluster, 'prime-osd-dir', '--dev', osd_lv_path,
'--path', osd_path, '--no-mon-config'
]
process.run(prime_command)
# always re-do the symlink regardless if it exists, so that the block,
# block.wal, and block.db devices that may have changed can be mapped
# correctly every time
process.run(['ln', '-snf', osd_lv_path, os.path.join(osd_path, 'block')])
system.chown(os.path.join(osd_path, 'block'))
system.chown(osd_path)
if db_device_path:
destination = os.path.join(osd_path, 'block.db')
process.run(['ln', '-snf', db_device_path, destination])
system.chown(db_device_path)
system.chown(destination)
if wal_device_path:
destination = os.path.join(osd_path, 'block.wal')
process.run(['ln', '-snf', wal_device_path, destination])
system.chown(wal_device_path)
system.chown(destination)
if no_systemd is False:
# enable the ceph-volume unit for this OSD
systemctl.enable_volume(osd_id, osd_fsid, 'lvm')
# enable the OSD
systemctl.enable_osd(osd_id)
# start the OSD
systemctl.start_osd(osd_id)
terminal.success("ceph-volume lvm activate successful for osd ID: %s" %
osd_id)
def activate(self, args):
with open(args.json_config, 'r') as fp:
osd_metadata = json.load(fp)
# Make sure that required devices are configured
self.validate_devices(osd_metadata)
osd_id = osd_metadata.get('whoami', args.osd_id)
osd_fsid = osd_metadata.get('fsid', args.osd_fsid)
data_uuid = osd_metadata.get('data', {}).get('uuid')
conf.cluster = osd_metadata.get('cluster_name', 'ceph')
if not data_uuid:
raise RuntimeError(
'Unable to activate OSD %s - no "uuid" key found for data' % args.osd_id
)
# Encryption detection, and capturing of the keys to decrypt
self.is_encrypted = osd_metadata.get('encrypted', False)
self.encryption_type = osd_metadata.get('encryption_type')
if self.is_encrypted:
lockbox_secret = osd_metadata.get('lockbox.keyring')
# write the keyring always so that we can unlock
encryption_utils.write_lockbox_keyring(osd_id, osd_fsid, lockbox_secret)
# Store the secret around so that the decrypt method can reuse
raw_dmcrypt_secret = encryption_utils.get_dmcrypt_key(osd_id, osd_fsid)
# Note how both these calls need b64decode. For some reason, the
# way ceph-disk creates these keys, it stores them in the monitor
# *undecoded*, requiring this decode call again. The lvm side of
# encryption doesn't need it, so we are assuming here that anything
# that `simple` scans, will come from ceph-disk and will need this
# extra decode call here
self.dmcrypt_secret = base64.b64decode(raw_dmcrypt_secret)
cluster_name = osd_metadata.get('cluster_name', 'ceph')
osd_dir = '/var/lib/ceph/osd/%s-%s' % (cluster_name, osd_id)
# XXX there is no support for LVM here
data_device = self.get_device(data_uuid)
journal_device = self.get_device(osd_metadata.get('journal', {}).get('uuid'))
block_device = self.get_device(osd_metadata.get('block', {}).get('uuid'))
block_db_device = self.get_device(osd_metadata.get('block.db', {}).get('uuid'))
block_wal_device = self.get_device(osd_metadata.get('block.wal', {}).get('uuid'))
if not system.device_is_mounted(data_device, destination=osd_dir):
process.run(['mount', '-v', data_device, osd_dir])
device_map = {
'journal': journal_device,
'block': block_device,
'block.db': block_db_device,
'block.wal': block_wal_device
}
for name, device in device_map.items():
if not device:
continue
# always re-do the symlink regardless if it exists, so that the journal
# device path that may have changed can be mapped correctly every time
destination = os.path.join(osd_dir, name)
process.run(['ln', '-snf', device, destination])
# make sure that the journal has proper permissions
system.chown(device)
self.enable_systemd_units(osd_id, osd_fsid)
terminal.success('Successfully activated OSD %s with FSID %s' % (osd_id, osd_fsid))
