def rbd_lock_cleanup(logger, local_ips, rbd_image):
"""
cleanup locks left if this node crashed and was not able to release them
:param logger: logger object to print to
:param local_ips: list of local ip addresses.
:rbd_image: rbd image to clean up locking for
:raise CephiSCSIError.
"""
lock_info = rbd_image.list_lockers()
if not lock_info:
return
lockers = lock_info.get("lockers")
for holder in lockers:
for ip in local_ips:
if ip in holder[2]:
logger.info(
"Cleaning up stale local lock for {} {}".format(
holder[0], holder[1]))
try:
rbd_image.break_lock(holder[0], holder[1])
except Exception as err:
raise CephiSCSIError("Could not break lock for {}. "
"Error {}".format(rbd_image, err))
def add_dev_to_lio(self, in_wwn=None):
"""
Add an rbd device to the LIO configuration
:param in_wwn: optional wwn identifying the rbd image to clients
(must match across gateways)
:return: LIO LUN object
"""
self.logger.info("(LUN.add_dev_to_lio) Adding image "
"'{}' to LIO backstore {}".format(
self.config_key, self.backstore))
new_lun = None
if self.backstore == USER_RBD:
new_lun = self._add_dev_to_lio_user_rbd(in_wwn)
else:
raise CephiSCSIError("Error adding device to lio - "
"Unsupported backstore {}".format(
self.backstore))
if new_lun:
self.logger.info("(LUN.add_dev_to_lio) Successfully added {}"
" to LIO".format(self.config_key))
return new_lun
def define_clients(logger, config):
"""
define the clients (nodeACLs) to the gateway definition
:param logger: logger object to print to
:param config: configuration dict from the rados pool
:raises CephiSCSIError.
"""
# Client configurations (NodeACL's)
for client_iqn in config.config['clients']:
client_metadata = config.config['clients'][client_iqn]
client_chap = CHAP(client_metadata['auth']['chap'])
image_list = client_metadata['luns'].keys()
chap_str = client_chap.chap_str
if client_chap.error:
raise CephiSCSIError("Unable to decode password for {}. "
"CHAP error: {}".format(client_iqn,
client_chap.error_msg))
client = GWClient(logger,
client_iqn,
image_list,
chap_str)
client.manage('present') # ensure the client exists
def deactivate(self):
so = self.lio_stg_object()
if not so:
# Could be due to a restart after failure. Just log and ignore.
self.logger.warning("LUN {} already deactivated".format(self.image))
return
for alun in so.attached_luns:
for mlun in alun.mapped_luns:
node_acl = mlun.parent_nodeacl
if node_acl.session and \
node_acl.session.get('state', '').upper() == 'LOGGED_IN':
raise CephiSCSIError("LUN {} in-use".format(self.image))
self.remove_dev_from_lio()
if self.error:
raise CephiSCSIError("LUN deactivate failure - {}".format(self.error_msg))
def delete_target(self, target_iqn):
target = GWTarget(self.logger, target_iqn, {})
if target.error:
raise CephiSCSIError("Could not initialize target: {}".
format(target.error_msg))
target.load_config()
if target.error:
self.logger.debug("Could not find target {}: {}".
format(target_iqn, target.error_msg))
# Target might not be setup on this node. Ignore.
return
try:
target.delete(self.config)
except RTSLibError as err:
err_msg = "Could not remove target {}: {}".format(target_iqn, err)
raise CephiSCSIError(err_msg)
def map_lun(self, gateway, owner, disk):
target_config = self.config.config['targets'][gateway.iqn]
disk_metadata = self.config.config['disks'][disk]
disk_metadata['owner'] = owner
self.config.update_item("disks", disk, disk_metadata)
target_config['disks'].append(disk)
self.config.update_item("targets", gateway.iqn, target_config)
gateway_dict = self.config.config['gateways'][owner]
gateway_dict['active_luns'] += 1
self.config.update_item('gateways', owner, gateway_dict)
so = self.allocate()
if self.error:
raise CephiSCSIError(self.error_msg)
gateway.map_lun(self.config, so)
if gateway.error:
raise CephiSCSIError(gateway.error_msg)
def define_clients(logger, config, target_iqn):
"""
define the clients (nodeACLs) to the gateway definition
:param logger: logger object to print to
:param config: configuration dict from the rados pool
:raises CephiSCSIError.
"""
# Client configurations (NodeACL's)
target_config = config.config['targets'][target_iqn]
for client_iqn in target_config['clients']:
client_metadata = target_config['clients'][client_iqn]
client_chap = CHAP(client_metadata['auth']['username'],
client_metadata['auth']['password'],
client_metadata['auth']['password_encryption_enabled'])
client_chap_mutual = CHAP(client_metadata['auth']['mutual_username'],
client_metadata['auth']['mutual_password'],
client_metadata['auth'][
'mutual_password_encryption_enabled'])
image_list = list(client_metadata['luns'].keys())
if client_chap.error:
raise CephiSCSIError("Unable to decode password for {}. "
"CHAP error: {}".format(client_iqn,
client_chap.error_msg))
if client_chap_mutual.error:
raise CephiSCSIError("Unable to decode password for {}. "
"CHAP_MUTUAL error: {}".format(client_iqn,
client_chap.error_msg))
client = GWClient(logger,
client_iqn,
image_list,
client_chap.user,
client_chap.password,
client_chap_mutual.user,
client_chap_mutual.password,
target_iqn)
client.manage('present') # ensure the client exists
def __init__(self, cfg_type, cfg_type_key, logger, control_settings):
self.cfg_type = cfg_type
self.cfg_type_key = cfg_type_key
self.logger = logger
self.config = Config(self.logger)
if self.config.error:
raise CephiSCSIError(self.config.error_msg)
# Copy of controls that will not be written until commit is called.
# To update the kernel call the child object's update function.
self.controls = self._get_config_controls().copy()
self._add_properies(control_settings)
def remove_from_config(self, target_iqn):
has_changed = False
target_config = self.config.config["targets"].get(target_iqn)
if target_config:
local_gw = target_config['portals'].get(self.hostname)
if local_gw:
local_gw_ips = local_gw['portal_ip_addresses']
target_config['portals'].pop(self.hostname)
ip_list = target_config['ip_list']
for local_gw_ip in local_gw_ips:
ip_list.remove(local_gw_ip)
for _, remote_gw_config in target_config['portals'].items():
for local_gw_ip in local_gw_ips:
remote_gw_config["gateway_ip_list"].remove(local_gw_ip)
remote_gw_config["inactive_portal_ips"].remove(
local_gw_ip)
tpg_count = remote_gw_config["tpgs"]
remote_gw_config["tpgs"] = tpg_count - 1
if not target_config['portals']:
# Last gw for the target so delete everything that lives
# under the tpg in LIO since we can't create it
target_config['disks'] = {}
target_config['clients'] = {}
target_config['controls'] = {}
target_config['groups'] = {}
has_changed = True
self.config.update_item('targets', target_iqn, target_config)
remove_gateway = True
for _, target in self.config.config["targets"].items():
if self.hostname in target['portals']:
remove_gateway = False
break
if remove_gateway:
# gateway is no longer used, so delete it
has_changed = True
self.config.del_item('gateways', self.hostname)
LUN.reassign_owners(self.logger, self.config)
if has_changed:
self.config.commit("retain")
if self.config.error:
raise CephiSCSIError(self.config.error_msg)
def map_lun(self, gateway, owner, disk):
target_config = self.config.config['targets'][gateway.iqn]
disk_metadata = self.config.config['disks'][disk]
disk_metadata['owner'] = owner
self.config.update_item("disks", disk, disk_metadata)
target_disk_config = target_config['disks'].get(disk)
if not target_disk_config:
lun_id = self._get_next_lun_id(target_config['disks'])
target_config['disks'][disk] = {'lun_id': lun_id}
self.config.update_item("targets", gateway.iqn, target_config)
gateway_dict = self.config.config['gateways'][owner]
gateway_dict['active_luns'] += 1
self.config.update_item('gateways', owner, gateway_dict)
so = self.allocate()
if self.error:
raise CephiSCSIError(self.error_msg)
gateway.map_lun(self.config, so, target_config['disks'][disk])
if gateway.error:
raise CephiSCSIError(gateway.error_msg)
def commit_controls(self):
committed_controls = self._get_config_controls()
if self.controls != committed_controls:
# update our config
self._set_config_controls(self.config, self.controls)
updated_obj = self.config.config[self.cfg_type][self.cfg_type_key]
self.config.update_item(self.cfg_type, self.cfg_type_key,
updated_obj)
self.config.commit()
if self.config.error:
raise CephiSCSIError(self.config.error_msg)
def reassign_owners(logger, config):
"""
Reassign disks across gateways after a gw deletion.
:param logger: logger object to print to
:param config: configuration dict from the rados pool
:raises CephiSCSIError.
"""
updated = False
gateways = config.config['gateways']
for disk, disk_config in config.config['disks'].items():
owner = disk_config.get('owner', None)
if owner is None:
continue
gw = gateways.get(owner, None)
if gw is None:
target = LUN.find_first_mapped_target(config, disk)
if not gateways or target is None:
disk_config.pop('owner')
else:
target_config = config.config['targets'][target]
new_owner = LUN.get_owner(gateways,
target_config['portals'])
logger.info("Changing {}'s owner from {} to {}".format(
disk, owner, new_owner))
disk_config['owner'] = new_owner
gw_config = config.config['gateways'][new_owner]
active_cnt = gw_config['active_luns']
gw_config['active_luns'] = active_cnt + 1
config.update_item("gateways", new_owner, gw_config)
config.update_item("disks", disk, disk_config)
updated = True
if updated:
config.commit("retain")
if config.error:
raise CephiSCSIError("Could not update LUN owners: {}".format(
config.error_msg))
def get_remote_gateways(config, logger, local_gw_required=True):
'''
Return the list of remote gws.
:param: config: Config object with gws setup.
:param: logger: Logger object
:param: local_gw_required: Check if local_gw is defined within gateways configuration
:return: A list of gw names, or CephiSCSIError if not run on a gw in the
config
'''
local_gw = this_host()
logger.debug("this host is {}".format(local_gw))
gateways = [key for key in config if isinstance(config[key], dict)]
logger.debug("all gateways - {}".format(gateways))
if local_gw_required and local_gw not in gateways:
raise CephiSCSIError("{} cannot be used to perform this operation "
"because it is not defined within the gateways "
"configuration".format(local_gw))
if local_gw in gateways:
gateways.remove(local_gw)
logger.debug("remote gateways: {}".format(gateways))
return gateways
def delete_targets(self):
err_msg = None
if self.hostname not in self.config.config['gateways']:
return
# Clear the current config, based on the config objects settings.
# This will fail incoming IO, but wait on outstanding IO to
# complete normally. We rely on the initiator multipath layer
# to handle retries like a normal path failure.
self.logger.info("Removing iSCSI target from LIO")
for target_iqn, target_config in self.config.config['targets'].items():
try:
self.delete_target(target_iqn)
except CephiSCSIError as err:
if err_msg is None:
err_msg = err
continue
if err_msg:
raise CephiSCSIError(err_msg)
def lookup_storage_object(name, backstore):
if backstore == USER_RBD:
return UserBackedStorageObject(name=name)
else:
raise CephiSCSIError("Could not lookup storage object - "
"Unsupported backstore {}".format(backstore))
def define_target(self, target_iqn, gw_ip_list, target_only=False):
"""
define the iSCSI target and tpgs
:param target_iqn: (str) target iqn
:param gw_ip_list: (list) gateway ip list
:param target_only: (bool) if True only setup target
:return: (object) GWTarget object
"""
# GWTarget Definition : Handle the creation of the Target/TPG(s) and
# Portals. Although we create the tpgs, we flick the enable_portal flag
# off so the enabled tpg will not have an outside IP address. This
# prevents clients from logging in too early, failing and giving up
# because the nodeACL hasn't been defined yet (yes Windows I'm looking
# at you!)
# first check if there are tpgs already in LIO (True) - this would
# indicate a restart or reload call has been made. If the tpg count is
# 0, this is a boot time request
self.logger.info("Setting up {}".format(target_iqn))
target = GWTarget(self.logger, target_iqn, gw_ip_list,
enable_portal=self.portals_active(target_iqn))
if target.error:
raise CephiSCSIError("Error initializing iSCSI target: "
"{}".format(target.error_msg))
target.manage('target')
if target.error:
raise CephiSCSIError("Error creating the iSCSI target (target, "
"TPGs, Portals): {}".format(target.error_msg))
if not target_only:
self.logger.info("Processing LUN configuration")
try:
LUN.define_luns(self.logger, self.config, target)
except CephiSCSIError as err:
self.logger.error("{} - Could not define LUNs: "
"{}".format(target.iqn, err))
raise
self.logger.info("{} - Processing client configuration".
format(target.iqn))
try:
GWClient.define_clients(self.logger, self.config, target.iqn)
except CephiSCSIError as err:
self.logger.error("Could not define clients: {}".format(err))
raise
if not target.enable_portal:
# The tpgs, luns and clients are all defined, but the active tpg
# doesn't have an IP bound to it yet (due to the
# enable_portals=False setting above)
self.logger.info("{} - Adding the IP to the enabled tpg, "
"allowing iSCSI logins".format(target.iqn))
target.enable_active_tpg(self.config)
if target.error:
raise CephiSCSIError("{} - Error enabling the IP with the "
"active TPG: {}".format(target.iqn,
target.error_msg))
return target
def activate(self):
disk = self.config.config['disks'].get(self.config_key, None)
if not disk:
raise CephiSCSIError("Image {} not found.".format(self.image))
wwn = disk.get('wwn', None)
if not wwn:
raise CephiSCSIError("LUN {} missing wwn".format(self.image))
# re-add backend storage object
so = self.lio_stg_object()
if not so:
self.add_dev_to_lio(wwn)
if self.error:
raise CephiSCSIError("LUN activate failure - {}".format(
self.error_msg))
# re-add LUN to target
local_gw = this_host()
targets_items = [
item for item in self.config.config['targets'].items()
if self.config_key in item[1]['disks']
and local_gw in item[1]['portals']
]
for target_iqn, target in targets_items:
ip_list = target['ip_list']
# Add the mapping for the lun to ensure the block device is
# present on all TPG's
gateway = GWTarget(self.logger, target_iqn, ip_list)
gateway.manage('map')
if gateway.error:
raise CephiSCSIError("LUN mapping failed - {}".format(
gateway.error_msg))
# re-map LUN to hosts
client_err = ''
for client_iqn in target['clients']:
client_metadata = target['clients'][client_iqn]
if client_metadata.get('group_name', ''):
continue
image_list = list(client_metadata['luns'].keys())
if self.config_key not in image_list:
continue
client_chap = CHAP(client_metadata['auth']['chap'])
chap_str = client_chap.chap_str
if client_chap.error:
raise CephiSCSIError("Password decode issue : "
"{}".format(client_chap.error_msg))
client_chap_mutual = CHAP(
client_metadata['auth']['chap_mutual'])
chap_mutual_str = client_chap_mutual.chap_str
if client_chap_mutual.error:
raise CephiSCSIError("Password decode issue : "
"{}".format(
client_chap_mutual.error_msg))
client = GWClient(self.logger, client_iqn, image_list,
chap_str, chap_mutual_str, target_iqn)
client.manage('present')
if client.error:
client_err = "LUN mapping failed {} - {}".format(
client_iqn, client.error_msg)
# re-map LUN to host groups
for group_name in target['groups']:
host_group = target['groups'][group_name]
members = host_group.get('members')
disks = host_group.get('disks').keys()
if self.config_key not in disks:
continue
group = Group(self.logger, target_iqn, group_name, members,
disks)
group.apply()
if group.error:
client_err = "LUN mapping failed {} - {}".format(
group_name, group.error_msg)
if client_err:
raise CephiSCSIError(client_err)
def define_luns(logger, config, gateway):
"""
define the disks in the config to LIO
:param logger: logger object to print to
:param config: configuration dict from the rados pool
:param gateway: (object) gateway object - used for mapping
:raises CephiSCSIError.
"""
local_gw = this_host()
# sort the disks dict keys, so the disks are registered in a specific
# sequence
disks = config.config['disks']
srtd_disks = sorted(disks)
pools = {disks[disk_key]['pool'] for disk_key in srtd_disks}
if pools is None:
logger.info("No LUNs to export")
return True
ips = ip_addresses()
with rados.Rados(conffile=settings.config.cephconf) as cluster:
for pool in pools:
logger.debug("Processing rbd's in '{}' pool".format(pool))
with cluster.open_ioctx(pool) as ioctx:
pool_disks = [
disk_key for disk_key in srtd_disks
if disk_key.startswith(pool + '.')
]
for disk_key in pool_disks:
is_lun_mapped = False
for _, target_config in config.config['targets'].items(
):
if local_gw in target_config['portals'] \
and disk_key in target_config['disks']:
is_lun_mapped = True
break
if is_lun_mapped:
pool, image_name = disk_key.split('.')
try:
with rbd.Image(ioctx, image_name) as rbd_image:
RBDDev.rbd_lock_cleanup(
logger, ips, rbd_image)
backstore = config.config['disks'][
disk_key]['backstore']
lun = LUN(logger, pool, image_name,
rbd_image.size(), local_gw,
backstore)
if lun.error:
raise CephiSCSIError(
"Error defining rbd "
"image {}".format(disk_key))
lun.allocate()
if lun.error:
raise CephiSCSIError(
"Error unable to "
"register {} with "
"LIO - {}".format(
disk_key, lun.error_msg))
except rbd.ImageNotFound:
raise CephiSCSIError(
"Disk '{}' defined to the "
"config, but image '{}' can "
"not be found in '{}' "
"pool".format(disk_key, image_name, pool))
if gateway:
# Gateway Mapping : Map the LUN's registered to all tpg's within the
# LIO target
gateway.manage('map')
if gateway.error:
raise CephiSCSIError(
"Error mapping the LUNs to the tpg's within "
"the iscsi Target")
def define_luns(logger, config, target):
"""
define the disks in the config to LIO and map to a LUN
:param logger: logger object to print to
:param config: configuration dict from the rados pool
:param target: (object) gateway object - used for mapping
:raises CephiSCSIError.
"""
ips = ip_addresses()
local_gw = this_host()
target_disks = config.config["targets"][target.iqn]['disks']
if not target_disks:
logger.info("No LUNs to export")
return
disks = {}
for disk in target_disks:
disks[disk] = config.config['disks'][disk]
# sort the disks dict keys, so the disks are registered in a specific
# sequence
srtd_disks = sorted(disks)
pools = {disks[disk_key]['pool'] for disk_key in srtd_disks}
ips = ip_addresses()
with rados.Rados(conffile=settings.config.cephconf,
name=settings.config.cluster_client_name) as cluster:
for pool in pools:
logger.debug("Processing rbd's in '{}' pool".format(pool))
with cluster.open_ioctx(pool) as ioctx:
pool_disks = [
disk_key for disk_key in srtd_disks
if disk_key.startswith(pool + '/')
]
for disk_key in pool_disks:
pool, image_name = disk_key.split('/')
with rbd.Image(ioctx, image_name) as rbd_image:
disk_config = config.config['disks'][disk_key]
backstore = disk_config['backstore']
backstore_object_name = disk_config[
'backstore_object_name']
lun = LUN(logger, pool, image_name,
rbd_image.size(), local_gw, backstore,
backstore_object_name)
if lun.error:
raise CephiSCSIError(
"Error defining rbd image {}".format(
disk_key))
so = lun.allocate()
if lun.error:
raise CephiSCSIError("Unable to register {} "
"with LIO: {}".format(
disk_key,
lun.error_msg))
# If not in use by another target on this gw
# clean up stale locks.
if so.status != 'activated':
RBDDev.rbd_lock_cleanup(logger, ips, rbd_image)
target._map_lun(config, so)
if target.error:
raise CephiSCSIError(
"Mapping for {} failed: {}".format(
disk_key, target.error_msg))
def commit_controls(self):
self.update_controls()
self.config.commit()
if self.config.error:
raise CephiSCSIError(self.config.error_msg)
