def prepare_filestore(device, journal, secrets, id_=None, fsid=None): """ :param device: The name of the logical volume to work with :param journal: similar to device but can also be a regular/plain disk :param secrets: A dict with the secrets needed to create the osd (e.g. cephx) :param id_: The OSD id :param fsid: The OSD fsid, also known as the OSD UUID """ cephx_secret = secrets.get('cephx_secret', prepare_utils.create_key()) json_secrets = json.dumps(secrets) # allow re-using an existing fsid, in case prepare failed fsid = fsid or system.generate_uuid() # allow re-using an id, in case a prepare failed osd_id = id_ or prepare_utils.create_id(fsid, json_secrets) # create the directory prepare_utils.create_osd_path(osd_id) # format the device prepare_utils.format_device(device) # mount the data device prepare_utils.mount_osd(device, osd_id) # symlink the journal prepare_utils.link_journal(journal, osd_id) # get the latest monmap prepare_utils.get_monmap(osd_id) # prepare the osd filesystem prepare_utils.osd_mkfs_filestore(osd_id, fsid) # write the OSD keyring if it doesn't exist already prepare_utils.write_keyring(osd_id, cephx_secret)
def prepare_filestore(device, journal, secrets, id_=None, fsid=None): """ :param device: The name of the logical volume to work with :param journal: similar to device but can also be a regular/plain disk :param secrets: A dict with the secrets needed to create the osd (e.g. cephx) :param id_: The OSD id :param fsid: The OSD fsid, also known as the OSD UUID """ cephx_secret = secrets.get('cephx_secret', prepare_utils.create_key()) json_secrets = json.dumps(secrets) # allow re-using an existing fsid, in case prepare failed fsid = fsid or system.generate_uuid() # allow re-using an id, in case a prepare failed osd_id = id_ or prepare_utils.create_id(fsid, json_secrets) # create the directory prepare_utils.create_osd_path(osd_id) # format the device prepare_utils.format_device(device) # mount the data device prepare_utils.mount_osd(device, osd_id) # symlink the journal prepare_utils.link_journal(journal, osd_id) # get the latest monmap prepare_utils.get_monmap(osd_id) # prepare the osd filesystem prepare_utils.osd_mkfs_filestore(osd_id, fsid) # write the OSD keyring if it doesn't exist already prepare_utils.write_keyring(osd_id, cephx_secret)
def test_default_options(self, conf_ceph_stub, fake_run): conf_ceph_stub(dedent("""[global] fsid = 1234lkjh1234""")) conf.cluster = 'ceph' prepare.mount_osd('/dev/sda1', 1) expected = [ 'mount', '-t', 'xfs', '-o', 'rw,noatime,inode64', # default flags '/dev/sda1', '/var/lib/ceph/osd/ceph-1'] assert expected == fake_run.calls[0]['args'][0]
def test_default_options(self, conf_ceph_stub, fake_run): conf_ceph_stub(dedent("""[global] fsid = 1234lkjh1234""")) conf.cluster = 'ceph' prepare.mount_osd('/dev/sda1', 1) expected = [ 'mount', '-t', 'xfs', '-o', 'rw', 'noatime', 'inode64', # default flags '/dev/sda1', '/var/lib/ceph/osd/ceph-1'] assert expected == fake_run.calls[0]['args'][0]
def test_underscore_mount_options_are_used(self, conf_ceph_stub, fake_run): conf_ceph_stub(dedent("""[global] fsid = 1234lkjh1234 [osd] osd mount options xfs = rw""")) conf.cluster = 'ceph' prepare.mount_osd('/dev/sda1', 1) expected = [ 'mount', '-t', 'xfs', '-o', 'rw', '/dev/sda1', '/var/lib/ceph/osd/ceph-1'] assert expected == fake_run.calls[0]['args'][0]
def test_underscore_mount_options_are_used(self, conf_ceph_stub, fake_run): conf_ceph_stub(dedent("""[global] fsid = 1234lkjh1234 [osd] osd mount options xfs = rw""")) conf.cluster = 'ceph' prepare.mount_osd('/dev/sda1', 1) expected = [ 'mount', '-t', 'xfs', '-o', 'rw', '/dev/sda1', '/var/lib/ceph/osd/ceph-1'] assert expected == fake_run.calls[0]['args'][0]
def prepare_filestore(device, journal, secrets, tags, osd_id, fsid): """ :param device: The name of the logical volume to work with :param journal: similar to device but can also be a regular/plain disk :param secrets: A dict with the secrets needed to create the osd (e.g. cephx) :param id_: The OSD id :param fsid: The OSD fsid, also known as the OSD UUID """ cephx_secret = secrets.get('cephx_secret', prepare_utils.create_key()) # encryption-only operations if secrets.get('dmcrypt_key'): # format and open ('decrypt' devices) and re-assign the device and journal # variables so that the rest of the process can use the mapper paths key = secrets['dmcrypt_key'] device = prepare_dmcrypt(key, device, 'data', tags) journal = prepare_dmcrypt(key, journal, 'journal', tags) # vdo detection is_vdo = api.is_vdo(device) # create the directory prepare_utils.create_osd_path(osd_id) # format the device prepare_utils.format_device(device) # mount the data device prepare_utils.mount_osd(device, osd_id, is_vdo=is_vdo) # symlink the journal prepare_utils.link_journal(journal, osd_id) # get the latest monmap prepare_utils.get_monmap(osd_id) # prepare the osd filesystem prepare_utils.osd_mkfs_filestore(osd_id, fsid, cephx_secret) # write the OSD keyring if it doesn't exist already prepare_utils.write_keyring(osd_id, cephx_secret) if secrets.get('dmcrypt_key'): # if the device is going to get activated right away, this can be done # here, otherwise it will be recreated encryption_utils.write_lockbox_keyring( osd_id, fsid, tags['ceph.cephx_lockbox_secret'] )
def prepare_filestore(device, journal, secrets, tags, osd_id, fsid): """ :param device: The name of the logical volume to work with :param journal: similar to device but can also be a regular/plain disk :param secrets: A dict with the secrets needed to create the osd (e.g. cephx) :param id_: The OSD id :param fsid: The OSD fsid, also known as the OSD UUID """ cephx_secret = secrets.get('cephx_secret', prepare_utils.create_key()) # encryption-only operations if secrets.get('dmcrypt_key'): # format and open ('decrypt' devices) and re-assign the device and journal # variables so that the rest of the process can use the mapper paths key = secrets['dmcrypt_key'] device = prepare_dmcrypt(key, device, 'data', tags) journal = prepare_dmcrypt(key, journal, 'journal', tags) # vdo detection is_vdo = api.is_vdo(device) # create the directory prepare_utils.create_osd_path(osd_id) # format the device prepare_utils.format_device(device) # mount the data device prepare_utils.mount_osd(device, osd_id, is_vdo=is_vdo) # symlink the journal prepare_utils.link_journal(journal, osd_id) # get the latest monmap prepare_utils.get_monmap(osd_id) # prepare the osd filesystem prepare_utils.osd_mkfs_filestore(osd_id, fsid, cephx_secret) # write the OSD keyring if it doesn't exist already prepare_utils.write_keyring(osd_id, cephx_secret) if secrets.get('dmcrypt_key'): # if the device is going to get activated right away, this can be done # here, otherwise it will be recreated encryption_utils.write_lockbox_keyring( osd_id, fsid, tags['ceph.cephx_lockbox_secret'] )
def activate(self, args): with open(args.json_config, 'r') as fp: osd_metadata = json.load(fp) # Make sure that required devices are configured self.validate_devices(osd_metadata) osd_id = osd_metadata.get('whoami', args.osd_id) osd_fsid = osd_metadata.get('fsid', args.osd_fsid) data_uuid = osd_metadata.get('data', {}).get('uuid') conf.cluster = osd_metadata.get('cluster_name', 'ceph') if not data_uuid: raise RuntimeError( 'Unable to activate OSD %s - no "uuid" key found for data' % args.osd_id) # Encryption detection, and capturing of the keys to decrypt self.is_encrypted = osd_metadata.get('encrypted', False) self.encryption_type = osd_metadata.get('encryption_type') if self.is_encrypted: lockbox_secret = osd_metadata.get('lockbox.keyring') # write the keyring always so that we can unlock encryption_utils.write_lockbox_keyring(osd_id, osd_fsid, lockbox_secret) # Store the secret around so that the decrypt method can reuse raw_dmcrypt_secret = encryption_utils.get_dmcrypt_key( osd_id, osd_fsid) # Note how both these calls need b64decode. For some reason, the # way ceph-disk creates these keys, it stores them in the monitor # *undecoded*, requiring this decode call again. The lvm side of # encryption doesn't need it, so we are assuming here that anything # that `simple` scans, will come from ceph-disk and will need this # extra decode call here self.dmcrypt_secret = base64.b64decode(raw_dmcrypt_secret) cluster_name = osd_metadata.get('cluster_name', 'ceph') osd_dir = '/var/lib/ceph/osd/%s-%s' % (cluster_name, osd_id) # XXX there is no support for LVM here data_device = self.get_device(data_uuid) if not data_device: raise RuntimeError("osd fsid {} doesn't exist, this file will " "be skipped, consider cleaning legacy " "json file {}".format(osd_metadata['fsid'], args.json_config)) journal_device = self.get_device( osd_metadata.get('journal', {}).get('uuid')) block_device = self.get_device( osd_metadata.get('block', {}).get('uuid')) block_db_device = self.get_device( osd_metadata.get('block.db', {}).get('uuid')) block_wal_device = self.get_device( osd_metadata.get('block.wal', {}).get('uuid')) if not system.device_is_mounted(data_device, destination=osd_dir): if osd_metadata.get('type') == 'filestore': prepare_utils.mount_osd(data_device, osd_id) else: process.run(['mount', '-v', data_device, osd_dir]) device_map = { 'journal': journal_device, 'block': block_device, 'block.db': block_db_device, 'block.wal': block_wal_device } for name, device in device_map.items(): if not device: continue # always re-do the symlink regardless if it exists, so that the journal # device path that may have changed can be mapped correctly every time destination = os.path.join(osd_dir, name) process.run(['ln', '-snf', device, destination]) # make sure that the journal has proper permissions system.chown(device) self.enable_systemd_units(osd_id, osd_fsid) terminal.success('Successfully activated OSD %s with FSID %s' % (osd_id, osd_fsid))
def activate_filestore(lvs, no_systemd=False): # find the osd osd_lv = lvs.get(lv_tags={'ceph.type': 'data'}) if not osd_lv: raise RuntimeError('Unable to find a data LV for filestore activation') is_encrypted = osd_lv.tags.get('ceph.encrypted', '0') == '1' is_vdo = osd_lv.tags.get('ceph.vdo', '0') osd_id = osd_lv.tags['ceph.osd_id'] conf.cluster = osd_lv.tags['ceph.cluster_name'] # it may have a volume with a journal osd_journal_lv = lvs.get(lv_tags={'ceph.type': 'journal'}) # TODO: add sensible error reporting if this is ever the case # blow up with a KeyError if this doesn't exist osd_fsid = osd_lv.tags['ceph.osd_fsid'] if not osd_journal_lv: # must be a disk partition, by quering blkid by the uuid we are ensuring that the # device path is always correct journal_uuid = osd_lv.tags['ceph.journal_uuid'] osd_journal = disk.get_device_from_partuuid(journal_uuid) else: journal_uuid = osd_journal_lv.lv_uuid osd_journal = osd_lv.tags['ceph.journal_device'] if not osd_journal: raise RuntimeError( 'unable to detect an lv or device journal for OSD %s' % osd_id) # this is done here, so that previous checks that ensure path availability # and correctness can still be enforced, and report if any issues are found if is_encrypted: lockbox_secret = osd_lv.tags['ceph.cephx_lockbox_secret'] # this keyring writing is idempotent encryption_utils.write_lockbox_keyring(osd_id, osd_fsid, lockbox_secret) dmcrypt_secret = encryption_utils.get_dmcrypt_key(osd_id, osd_fsid) encryption_utils.luks_open(dmcrypt_secret, osd_lv.lv_path, osd_lv.lv_uuid) encryption_utils.luks_open(dmcrypt_secret, osd_journal, journal_uuid) osd_journal = '/dev/mapper/%s' % journal_uuid source = '/dev/mapper/%s' % osd_lv.lv_uuid else: source = osd_lv.lv_path # mount the osd destination = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id) if not system.device_is_mounted(source, destination=destination): prepare_utils.mount_osd(source, osd_id, is_vdo=is_vdo) # always re-do the symlink regardless if it exists, so that the journal # device path that may have changed can be mapped correctly every time destination = '/var/lib/ceph/osd/%s-%s/journal' % (conf.cluster, osd_id) process.run(['ln', '-snf', osd_journal, destination]) # make sure that the journal has proper permissions system.chown(osd_journal) if no_systemd is False: # enable the ceph-volume unit for this OSD systemctl.enable_volume(osd_id, osd_fsid, 'lvm') # enable the OSD systemctl.enable_osd(osd_id) # start the OSD systemctl.start_osd(osd_id) terminal.success("ceph-volume lvm activate successful for osd ID: %s" % osd_id)
def activate_filestore(lvs, no_systemd=False): # find the osd osd_lv = lvs.get(lv_tags={'ceph.type': 'data'}) if not osd_lv: raise RuntimeError('Unable to find a data LV for filestore activation') is_encrypted = osd_lv.tags.get('ceph.encrypted', '0') == '1' is_vdo = osd_lv.tags.get('ceph.vdo', '0') osd_id = osd_lv.tags['ceph.osd_id'] conf.cluster = osd_lv.tags['ceph.cluster_name'] # it may have a volume with a journal osd_journal_lv = lvs.get(lv_tags={'ceph.type': 'journal'}) # TODO: add sensible error reporting if this is ever the case # blow up with a KeyError if this doesn't exist osd_fsid = osd_lv.tags['ceph.osd_fsid'] if not osd_journal_lv: # must be a disk partition, by quering blkid by the uuid we are ensuring that the # device path is always correct journal_uuid = osd_lv.tags['ceph.journal_uuid'] osd_journal = disk.get_device_from_partuuid(journal_uuid) else: journal_uuid = osd_journal_lv.lv_uuid osd_journal = osd_lv.tags['ceph.journal_device'] if not osd_journal: raise RuntimeError('unable to detect an lv or device journal for OSD %s' % osd_id) # this is done here, so that previous checks that ensure path availability # and correctness can still be enforced, and report if any issues are found if is_encrypted: lockbox_secret = osd_lv.tags['ceph.cephx_lockbox_secret'] # this keyring writing is idempotent encryption_utils.write_lockbox_keyring(osd_id, osd_fsid, lockbox_secret) dmcrypt_secret = encryption_utils.get_dmcrypt_key(osd_id, osd_fsid) encryption_utils.luks_open(dmcrypt_secret, osd_lv.lv_path, osd_lv.lv_uuid) encryption_utils.luks_open(dmcrypt_secret, osd_journal, journal_uuid) osd_journal = '/dev/mapper/%s' % journal_uuid source = '/dev/mapper/%s' % osd_lv.lv_uuid else: source = osd_lv.lv_path # mount the osd destination = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id) if not system.device_is_mounted(source, destination=destination): prepare_utils.mount_osd(source, osd_id, is_vdo=is_vdo) # always re-do the symlink regardless if it exists, so that the journal # device path that may have changed can be mapped correctly every time destination = '/var/lib/ceph/osd/%s-%s/journal' % (conf.cluster, osd_id) process.run(['ln', '-snf', osd_journal, destination]) # make sure that the journal has proper permissions system.chown(osd_journal) if no_systemd is False: # enable the ceph-volume unit for this OSD systemctl.enable_volume(osd_id, osd_fsid, 'lvm') # enable the OSD systemctl.enable_osd(osd_id) # start the OSD systemctl.start_osd(osd_id) terminal.success("ceph-volume lvm activate successful for osd ID: %s" % osd_id)