class Group(object):
def __init__(self, logger, group_name, members=[], disks=[]):
"""
Manage a host group definition. The input for the group object is the
desired state of the group where the logic enforced produces an
idempotent group definition across API/CLI and more importantly Ansible
:param logger: (logging object) used for centralised logging
:param group_name: (str) group name
:param members: (list) iscsi IQN's of the clients
:param disks: (list) disk names of the format pool.image
"""
self.logger = logger
self.error = False
self.error_msg = ''
self.num_changes = 0
self.config = Config(logger)
if self.config.error:
self.error = self.config.error
self.error_msg = self.config.error_msg
return
# check that the config object has a group section
Group._check_config(self.logger, self.config)
self.group_name = group_name
self.group_members = members
self.disks = disks
self.logger.debug("Group : name={}".format(self.group_name))
self.logger.debug("Group : members={}".format(self.group_members))
self.logger.debug("Group : disks={}".format(self.disks))
@classmethod
def _check_config(cls, logger, config_object):
"""
look at the current config object to determine whether it needs the
group section seeding
:param logger: (logging object) destination for log messages
:param config_object: Instance of the Config class
:return: Null
"""
if 'groups' in config_object.config:
logger.debug("Config object contains a 'groups' section - config "
"object upgrade is not required")
return
else:
# Need to upgrade the config object to include the new
# 'groups' section
logger.info("Adding 'groups' section to config object")
config_object.add_item("groups",
element_name=None,
initial_value={})
config_object.update_item("version",
element_name=None,
element_value=3)
config_object.commit()
def __str__(self):
return ("Group: {}\n- Members: {}\n- "
"Disks: {}".format(self.group_name, self.group_members,
self.disks))
def _set_error(self, error_msg):
self.error = True
self.error_msg = error_msg
self.logger.debug("Error: {}".format(self.error_msg))
def _valid_client(self, action, client_iqn):
"""
validate the addition of a specific client
:param action: (str) add or remove request
:param client_iqn: (str) iqn of the client to add tot he group
:return: (bool) true/false whether the client should be accepted
"""
config = self.config.config # use a local var for readability
self.logger.debug("checking '{}'".format(client_iqn))
# to validate the request, pass through a 'negative' filter
if action == 'add':
client = config['clients'].get(client_iqn, {})
if not client:
self._set_error("client '{}' doesn't exist".format(client_iqn))
return False
elif client.get('luns'):
self._set_error("Client '{}' already has luns. "
"Only clients without prior lun maps "
"can be added to a group".format(client_iqn))
return False
elif client.get('group_name'):
self._set_error("Client already assigned to {} - a client "
"can only belong to one host "
"group".format(client.get('group_name')))
return False
else:
# client_iqn must exist in the group
if client_iqn not in config['groups'][self.group_name].get(
'members'):
self._set_error("client '{}' is not a member of "
"{}".format(client_iqn, self.group_name))
return False
# to reach here the request is considered valid
self.logger.debug("'{}' client '{}' for group '{}'"
" is valid".format(action, client_iqn,
self.group_name))
return True
def _valid_disk(self, action, disk):
self.logger.debug("checking disk '{}'".format(disk))
if action == 'add':
if disk not in self.config.config['disks']:
self._set_error("disk '{}' doesn't exist".format(disk))
return False
else:
if disk not in self.config.config['groups'][
self.group_name]['disks']:
self._set_error("disk '{}' is not in the group".format(disk))
return False
return True
def _next_lun(self):
"""
Look at the disk list for the group and return the 1st available free
LUN id used for adding disks to the group
:return: (int) lun Id
"""
lun_range = list(range(0, 256, 1)) # 0->255
group = self.config.config['groups'][self.group_name]
group_disks = group.get('disks')
for d in group_disks:
lun_range.remove(group_disks[d].get('lun_id'))
return lun_range[0]
def apply(self):
"""
setup/manage the group definition
:return: NULL
"""
group_seed = {"members": [], "disks": {}}
new_group = False
config_dict = self.config.config
if self.group_name not in config_dict['groups']:
# New Group definition
self.logger.debug("Processing request for new group "
"'{}'".format(self.group_name))
if len(set(self.group_members)) != len(self.group_members):
self._set_error("Member must contain unique clients - no "
"duplication")
return
# update the config object to include the new group definition
self.logger.debug("New group definition required")
new_group = True
config_dict['groups'][self.group_name] = group_seed
# Now the group definition is at least seeded, so let's look at the
# member and disk information passed
# validate the members exist
# validate the disks exist
this_group = config_dict['groups'][self.group_name]
members = ListComparison(this_group.get('members'), self.group_members)
disks = ListComparison(this_group.get('disks').keys(), self.disks)
if set(self.disks) != set(this_group.get('disks')) or \
set(self.group_members) != set(this_group.get('members')):
group_changed = True
else:
group_changed = False
if not group_changed and not new_group:
# no changes required
self.logger.info("Current group definition matches request "
"- no changes needed")
return
self.logger.info("Validating client membership")
for mbr in members.added:
if not self._valid_client('add', mbr):
self.logger.error("'{}' failed checks".format(mbr))
return
for mbr in members.removed:
if not self._valid_client('remove', mbr):
self.logger.error("'{}' failed checks".format(mbr))
return
self.logger.debug("Client membership checks passed")
self.logger.debug("clients added are : {}".format(members.added))
self.logger.debug("clients removed are: {}".format(members.removed))
# client membership is valid, check disks
self.logger.info("Validating disk membership")
for disk_name in disks.added:
if not self._valid_disk('add', disk_name):
self.logger.error("'{}' failed checks".format(disk_name))
return
for disk_name in disks.removed:
if not self._valid_disk('remove', disk_name):
self.logger.error("'{}' failed checks".format(disk_name))
return
self.logger.info("Disk membership checks passed")
self.logger.debug("disks added are : {}".format(disks.added))
self.logger.debug("disks removed are: {}".format(disks.removed))
# client(s) and disk(s) passed validation
# handle disk membership updates first, then clients
group_disks = this_group.get('disks', {})
if disks.added:
# update the groups disk list
for disk in disks.added:
lun_seq = self._next_lun()
group_disks[disk] = {"lun_id": lun_seq}
self.logger.debug("- adding '{}' to group '{}' @ "
"lun id {}".format(disk, self.group_name,
lun_seq))
if disks.removed:
# remove disk from the group definition
for disk in disks.removed:
del group_disks[disk]
self.logger.debug("- removed '{}' from group "
"{}".format(disk, self.group_name))
# sort the groups disks by 'lun_id'
# FIXME dict sort is python2 specific
image_list = sorted(group_disks.iteritems(),
key=lambda (k, v): v['lun_id'])
# handle client membership
if members.changed:
for client_iqn in members.added:
self.add_client(client_iqn)
self.update_client(client_iqn, image_list)
for client_iqn in members.removed:
self.remove_client(client_iqn)
# handle disk changes
if disks.changed:
# process all *existing* clients in the group with the
# updated image list
for client_iqn in self.group_members:
if client_iqn not in members.added:
self.update_client(client_iqn, image_list)
if self.error:
return
this_group['members'] = self.group_members
this_group['disks'] = group_disks
self.logger.debug("Group updated to {}".format(json.dumps(this_group)))
if new_group:
self.config.add_item("groups", self.group_name, this_group)
else:
self.config.update_item("groups", self.group_name, this_group)
self.config.commit()
def add_client(self, client_iqn):
client_metadata = self.config.config['clients'][client_iqn]
client_metadata['group_name'] = self.group_name
self.config.update_item("clients", client_iqn, client_metadata)
self.logger.info("Added {} to group {}".format(client_iqn,
self.group_name))
def update_client(self, client_iqn, image_list):
client = GWClient(self.logger, client_iqn, image_list, '')
client.define_client() # set up clients ACL
# grab the metadata from the current definition
client.metadata = self.config.config['clients'][client_iqn]
client.setup_luns()
if client.error:
self._set_error(client.error_msg)
return
else:
self.logger.info("Updating config object for "
"client '{}'".format(client_iqn))
client.metadata['group_name'] = self.group_name
self.config.update_item("clients", client_iqn, client.metadata)
def remove_client(self, client_iqn):
client_md = self.config.config["clients"][client_iqn]
# remove the group_name setting from each client
client_md['group_name'] = ''
self.config.update_item("clients", client_iqn, client_md)
self.logger.info("Removed {} from group {}".format(
client_iqn, self.group_name))
def purge(self):
# act on the group name
# get the members from the current definition
groups = self.config.config['groups']
if self.group_name in groups:
for mbr in groups[self.group_name]["members"]:
self.remove_client(mbr)
# issue a del_item to the config object for this group_name
self.config.del_item("groups", self.group_name)
self.config.commit()
self.logger.info("Group {} removed".format(self.group_name))
else:
self._set_error("Group name requested does not exist")
return
def manage(self, mode):
"""
Manage the definition of the gateway, given a mode of 'target', 'map',
'init' or 'clearconfig'. In 'target' mode the LIO TPG is defined,
whereas in map mode, the required LUNs are added to the existing TPG
:param mode: run mode - target, map, init or clearconfig (str)
:return: None - but sets the objects error flags to be checked by
the caller
"""
config = Config(self.logger)
if config.error:
self.error = True
self.error_msg = config.error_msg
return
local_gw = this_host()
if mode == 'target':
if self.exists():
self.load_config()
self.check_tpgs()
else:
self.create_target()
if self.error:
# return to caller, with error state set
return
gateway_group = config.config["gateways"].keys()
# this action could be carried out by multiple nodes concurrently,
# but since the value is the same (i.e all gateway nodes use the
# same iqn) it's not worth worrying about!
if "iqn" not in gateway_group:
self.config_updated = True
config.add_item("gateways", "iqn", initial_value=self.iqn)
if "ip_list" not in gateway_group:
self.config_updated = True
config.add_item("gateways",
"ip_list",
initial_value=self.gateway_ip_list)
if self.controls != config.config.get('controls', {}):
config.set_item('controls', '', self.controls.copy())
self.config_updated = True
if local_gw not in gateway_group:
inactive_portal_ip = list(self.gateway_ip_list)
inactive_portal_ip.remove(self.active_portal_ip)
gateway_metadata = {
"portal_ip_address": self.active_portal_ip,
"iqn": self.iqn,
"active_luns": 0,
"tpgs": len(self.tpg_list),
"inactive_portal_ips": inactive_portal_ip,
"gateway_ip_list": self.gateway_ip_list
}
config.add_item("gateways", local_gw)
config.update_item("gateways", local_gw, gateway_metadata)
config.update_item("gateways", "ip_list", self.gateway_ip_list)
self.config_updated = True
else:
# gateway already defined, so check that the IP list it has
# matches the current request
gw_details = config.config['gateways'][local_gw]
if cmp(gw_details['gateway_ip_list'],
self.gateway_ip_list) != 0:
inactive_portal_ip = list(self.gateway_ip_list)
inactive_portal_ip.remove(self.active_portal_ip)
gw_details['tpgs'] = len(self.tpg_list)
gw_details['gateway_ip_list'] = self.gateway_ip_list
gw_details['inactive_portal_ips'] = inactive_portal_ip
config.update_item('gateways', local_gw, gw_details)
self.config_updated = True
if self.config_updated:
config.commit()
elif mode == 'map':
if self.exists():
self.load_config()
self.map_luns(config)
else:
self.error = True
self.error_msg = ("Attempted to map to a gateway '{}' that "
"hasn't been defined yet...out of order "
"steps?".format(self.iqn))
elif mode == 'init':
# init mode just creates the iscsi target definition and updates
# the config object. It is used by the CLI only
if self.exists():
self.logger.info("GWTarget init request skipped - target "
"already exists")
else:
# create the target
self.create_target()
current_iqn = config.config['gateways'].get('iqn', '')
# First gateway asked to create the target will update the
# config object
if not current_iqn:
config.add_item("gateways", "iqn", initial_value=self.iqn)
config.commit()
elif mode == 'reconfigure':
if self.controls != config.config.get('controls', {}):
config.set_item('controls', '', self.controls.copy())
config.commit()
elif mode == 'clearconfig':
# Called by API from CLI clearconfig command
if self.exists():
self.load_config()
else:
self.error = True
self.error_msg = "IQN provided does not exist"
self.clear_config()
if not self.error:
gw_ip = config.config['gateways'][local_gw][
'portal_ip_address']
config.del_item('gateways', local_gw)
ip_list = config.config['gateways']['ip_list']
ip_list.remove(gw_ip)
if len(ip_list) > 0:
config.update_item('gateways', 'ip_list', ip_list)
else:
# no more gateways in the list, so delete remaining items
config.del_item('gateways', 'ip_list')
config.del_item('gateways', 'iqn')
config.del_item('gateways', 'created')
config.commit()
class LUN(object):
def __init__(self, logger, pool, image, size, allocating_host):
self.logger = logger
self.image = image
self.pool = pool
self.pool_id = 0
self.size = size
self.config_key = '{}.{}'.format(self.pool, self.image)
# the allocating host could be fqdn or shortname - but the config
# only uses shortname so it needs to be converted to shortname format
self.allocating_host = allocating_host.split('.')[0]
self.owner = '' # gateway host that owns the preferred path for this LUN
self.error = False
self.error_msg = ''
self.num_changes = 0
self.dm_device = '' # e.g. /dev/mapper/0-58f8b515f007c
self.config = Config(logger)
if self.config.error:
self.error = self.config.error
self.error_msg = self.config.error_msg
return
self._validate_request()
def _validate_request(self):
# Before we start make sure that the target host is actually defined to the config
if self.allocating_host not in self.config.config['gateways'].keys():
self.logger.critical(
"Owning host is not valid, please provide a valid gateway name for this rbd image"
)
self.error = True
self.error_msg = (
"host name given for {} is not a valid gateway name, "
"listed in the config".format(self.image))
elif not rados_pool(pool=self.pool):
# Could create the pool, but a fat finger moment in the config file would mean rbd images
# get created and mapped, and then need correcting. Better to exit if the pool doesn't exist
self.error = True
self.error_msg = "Pool '{}' does not exist. Unable to continue".format(
self.pool)
@staticmethod
def remove_dm_device(dm_path):
dm_name = os.path.basename(dm_path)
resp = shellcommand('multipath -f {}'.format(dm_name))
return False if resp else True
def remove_lun(self):
this_host = gethostname().split('.')[0]
self.logger.info("LUN deletion request received, rbd removal to be "
"performed by {}".format(self.allocating_host))
# First ensure the LUN is not allocated to a client
clients = self.config.config['clients']
lun_in_use = False
for iqn in clients:
client_luns = clients[iqn]['luns'].keys()
if self.config_key in client_luns:
lun_in_use = True
break
if lun_in_use:
# this will fail the ansible task for this lun/host
self.error = True
self.error_msg = "Unable to delete {} - allocated to {}".format(
self.config_key, iqn)
self.logger.warning(self.error_msg)
return
# Check that the LUN is in LIO - if not there is nothing to do for this request
lun = self.lun_in_lio()
if not lun:
return
# Now we know the request is for a LUN in LIO, and it's not masked to a client
self.remove_dev_from_lio()
if self.error:
return
rbd_image = RBDDev(self.image, '0G', self.pool)
rbd_image.get_rbd_map()
# Check that we have an rbd_map entry - if not the map command failed
if not rbd_image.rbd_map:
self.error = True
self.error_msg = "Unable to get the map device for {}/{}".format(
rbd_image.pool, rbd_image.image)
self.logger.error(self.error_msg)
return
dm_path = LUN.dm_device_name_from_rbd_map(rbd_image.rbd_map)
if LUN.remove_dm_device(dm_path):
rbd_image.unmap_rbd()
if rbd_image.error:
self.error = True
self.error_msg = "Unable to unmap {} from host".format(
self.config_key)
self.logger.error(self.error_msg)
return
self.num_changes += 1
if this_host == self.allocating_host:
# by using the allocating host we ensure the delete is not
# issue by several hosts when initiated through ansible
rbd_image.delete_rbd()
if rbd_image.error:
self.error = True
self.error_msg = "Unable to delete the underlying rbd image {}".format(
self.config_key)
return
# remove the definition from the config object
self.config.del_item('disks', self.config_key)
self.config.commit()
else:
self.error = True
self.error_msg = "Unable to remove dm device for {}".format(
self.config_key)
self.logger.error(self.error_msg)
return
def manage(self, desired_state):
self.logger.debug("lun.manage request for {}, desired state {}".format(
self.image, desired_state))
if desired_state == 'present':
self.allocate()
elif desired_state == 'absent':
self.remove_lun()
def allocate(self):
self.logger.debug(
"LUN.allocate starting, getting a list of rbd devices")
disk_list = RBDDev.rbd_list(pool=self.pool)
self.logger.debug("rados pool '{}' contains the following - {}".format(
self.pool, disk_list))
this_host = gethostname().split('.')[0]
self.logger.debug("Hostname Check - this host is {}, target host for "
"allocations is {}".format(this_host,
self.allocating_host))
rbd_image = RBDDev(self.image, self.size, self.pool)
self.pool_id = rbd_image.pool_id
# if the image required isn't defined, create it!
if self.image not in disk_list:
# create the requested disk if this is the 'owning' host
if this_host == self.allocating_host: # is_this_host(target_host):
rbd_image.create()
if not rbd_image.error:
self.config.add_item('disks', self.config_key)
self.logger.info(
"(LUN.allocate) created {}/{} successfully".format(
self.pool, self.image))
self.num_changes += 1
else:
self.error = True
self.error_msg = rbd_image.error_msg
return
else:
# the image isn't there, and this isn't the 'owning' host
# so wait until the disk arrives
waiting = 0
while self.image not in disk_list:
sleep(settings.config.loop_delay)
disk_list = RBDDev.rbd_list(pool=self.pool)
waiting += settings.config.loop_delay
if waiting >= settings.config.time_out:
self.error = True
self.error_msg = "(LUN.allocate) timed out waiting for rbd to show up"
return
else:
# requested image is defined to ceph, so ensure it's in the config
if self.config_key not in self.config.config['disks']:
self.config.add_item('disks', self.config_key)
# make sure the rbd is mapped. this will also ensure the
# RBDDEV object will hold a valid dm_device attribute
rbd_image.get_rbd_map()
if rbd_image.map_needed:
self.num_changes += 1
self.logger.debug("Check the rbd image size matches the request")
# if updates_made is not set, the disk pre-exists so on the owning host see if it needs to be resized
if self.num_changes == 0 and this_host == self.allocating_host: # is_this_host(target_host):
# check the size, and update if needed
rbd_image.rbd_size()
if rbd_image.error:
self.logger.critical(rbd_image.error_msg)
self.error = True
self.error_msg = rbd_image.error_msg
return
if rbd_image.changed:
self.logger.info("rbd image {} resized to {}".format(
self.image, self.size))
self.num_changes += 1
else:
self.logger.debug(
"rbd image {} size matches the configuration file request".
format(self.image))
# for LIO mapping purposes, we use the device mapper device not the raw /dev/rbdX device
# Using the dm device ensures that any connectivity issue doesn't result in stale device
# structures in the kernel, since device-mapper will tidy those up
self.dm_get_device(rbd_image.rbd_map)
if self.dm_device is None:
self.logger.critical(
"Could not find dm multipath device for {}. Make sure the multipathd"
" service is enabled, and confirm entry is in /dev/mapper/".
format(self.image))
self.error = True
self.error_msg = "Could not find dm multipath device for {}".format(
self.image)
return
# ensure the dm device size matches the request size
if not self.dm_size_ok(rbd_image):
self.error = True
self.error_msg = "Unable to sync the dm device to the parent rbd size - {}".format(
self.image)
self.logger.critical(self.error_msg)
return
self.logger.debug("Begin processing LIO mapping requirement")
self.logger.debug("(LUN.allocate) {} is mapped to {}.".format(
self.image, self.dm_device))
# check this rbd image is in the /etc/ceph/rbdmap file
if rbd_image.rbdmap_entry():
self.logger.debug(
'(LUN.allocate) Entry added to /etc/ceph/rbdmap for {}/{}'.
format(self.pool, self.image))
self.num_changes += 1
# now see if we need to add this rbd image to LIO
lun = self.lun_in_lio()
if not lun:
# this image has not been defined to this hosts LIO, so check the config for the details and
# if it's missing define the wwn/alua_state and update the config
if this_host == self.allocating_host:
# first check to see if the device needs adding
try:
wwn = self.config.config['disks'][self.config_key]['wwn']
except KeyError:
wwn = ''
if wwn == '':
# disk hasn't been defined to LIO yet, it' not been defined to the config yet
# and this is the allocating host
lun = self.add_dev_to_lio()
if self.error:
return
# lun is now in LIO, time for some housekeeping :P
wwn = lun._get_wwn()
self.owner = LUN.set_owner(self.config.config['gateways'])
self.logger.debug("Owner for {} will be {}".format(
self.image, self.owner))
disk_attr = {
"wwn": wwn,
"image": self.image,
"owner": self.owner,
"pool": self.pool,
"pool_id": rbd_image.pool_id,
"dm_device": self.dm_device
}
self.config.update_item('disks', self.config_key,
disk_attr)
gateway_dict = self.config.config['gateways'][self.owner]
gateway_dict['active_luns'] += 1
self.config.update_item('gateways', self.owner,
gateway_dict)
self.logger.debug(
"(LUN.allocate) registered '{}' with wwn '{}' with the"
" config object".format(self.image, wwn))
self.logger.info(
"(LUN.allocate) added '{}/{}' to LIO and config object"
.format(self.pool, self.image))
else:
# config object already had wwn for this rbd image
lun = self.add_dev_to_lio(wwn)
if self.error:
return
self.logger.debug(
"(LUN.allocate) registered '{}' to LIO with wwn '{}' from "
"the config object".format(self.image, wwn))
self.num_changes += 1
else:
# lun is not already in LIO, but this is not the owning node that defines the wwn
# we need the wwn from the config (placed by the allocating host), so we wait!
waiting = 0
while waiting < settings.config.time_out:
self.config.refresh()
if self.config_key in self.config.config['disks']:
if 'wwn' in self.config.config['disks'][
self.config_key]:
if self.config.config['disks'][
self.config_key]['wwn']:
wwn = self.config.config['disks'][
self.config_key]['wwn']
break
sleep(settings.config.loop_delay)
waiting += settings.config.loop_delay
self.logger.debug(
"(LUN.allocate) waiting for config object to show {}"
" with it's wwn".format(self.image))
if waiting >= settings.config.time_out:
self.error = True
self.error_msg = (
"(LUN.allocate) waited too long for the wwn information "
"on image {} to arrive".format(self.image))
return
# At this point we have a wwn from the config for this rbd image, so just add to LIO
lun = self.add_dev_to_lio(wwn)
if self.error:
return
self.logger.info(
"(LUN.allocate) added {} to LIO using wwn '{}'"
" defined by {}".format(self.image, wwn,
self.allocating_host))
self.num_changes += 1
self.logger.debug("config meta data for this disk is {}".format(
self.config.config['disks'][self.config_key]))
# the owning host for an image is the only host that commits to the config
if this_host == self.allocating_host and self.config.changed:
self.logger.debug(
"(LUN.allocate) Committing change(s) to the config object in pool {}"
.format(self.pool))
self.config.commit()
self.error = self.config.error
self.error_msg = self.config.error_msg
def dm_get_device(self, map_device):
"""
set the dm_device attribute based on the rbd map device entry
:param map_device: /dev/rbdX
:return: None
"""
self.dm_device = LUN.dm_device_name_from_rbd_map(map_device)
if self.dm_device is None:
return
if not LUN.dm_wait_for_device(self.dm_device):
self.dm_device = None
def dm_size_ok(self, rbd_object):
"""
Check that the dm device matches the request. if the size request is lower than
current size, just return since resizing down is not support and problematic
for client filesystems anyway
:return boolean indicating whether the size matches
"""
target_bytes = convert_2_bytes(self.size)
if rbd_object.size_bytes > target_bytes:
return True
tmr = 0
size_ok = False
rbd_size_ok = False
dm_path_found = False
# we have to wait for the rbd size to match, since the rbd could have been
# resized on another gateway host when this is called from Ansible
while tmr < settings.config.time_out:
if rbd_object.size_bytes == target_bytes:
rbd_size_ok = True
break
sleep(settings.config.loop_delay)
tmr += settings.config.loop_delay
# since the size matches underneath device mapper, now we ensure the size
# matches with device mapper - if not issue a resize map request
if rbd_size_ok:
# find the dm-X device
dm_devices = glob.glob('/sys/class/block/dm-*/')
# convert the full dm_device path to just the name (last component of path
dm_name = os.path.basename(self.dm_device)
for dm_dev in dm_devices:
if fread(os.path.join(dm_dev, 'dm/name')) == dm_name:
dm_path_found = True
break
if dm_path_found:
# size is in sectors, so read it and * 512 = bytes
dm_size_bytes = int(fread(os.path.join(dm_dev, 'size'))) * 512
if dm_size_bytes != target_bytes:
self.logger.info(
"Issuing a resize map for {}".format(dm_name))
response = shellcommand(
'multipathd resize map {}'.format(dm_name))
self.logger.debug("resize result : {}".format(response))
dm_size_bytes = int(fread(os.path.join(dm_dev,
'size'))) * 512
if response.lower().startswith(
'ok') and dm_size_bytes == target_bytes:
size_ok = True
else:
self.logger.critical(
"multipathd resize map for {} failed".format(
dm_name))
else:
# size matches
size_ok = True
else:
self.logger.critical(
"Unable to locate a dm-X device for this rbd image - {}".
format(self.image))
return size_ok
def lun_in_lio(self):
found_it = False
rtsroot = root.RTSRoot()
for stg_object in rtsroot.storage_objects:
# First match on name, but then check the pool incase the same name exists in multiple pools
if stg_object.name == self.config_key:
found_it = True
break
return stg_object if found_it else None
def add_dev_to_lio(self, in_wwn=None):
"""
Add an rbd device to the LIO configuration
:param in_wwn: optional wwn identifying the rbd image to clients - must match across gateways
:return: LIO LUN object
"""
self.logger.info(
"(LUN.add_dev_to_lio) Adding image '{}' with path {} to LIO".
format(self.image, self.dm_device))
new_lun = None
try:
new_lun = BlockStorageObject(name=self.config_key,
dev=self.dm_device,
wwn=in_wwn)
except RTSLibError as err:
self.error = True
self.error_msg = "failed to add {} to LIO - error({})".format(
self.image, str(err))
return new_lun
def remove_dev_from_lio(self):
lio_root = root.RTSRoot()
# remove the device from all tpgs
for t in lio_root.tpgs:
for lun in t.luns:
if lun.storage_object.name == self.config_key:
try:
lun.delete()
except RTSLibError as e:
self.error = True
self.error_msg = "Delete from LIO/TPG failed - {}".format(
e)
return
else:
break # continue to the next tpg
for stg_object in lio_root.storage_objects:
if stg_object.name == self.config_key:
alua_dir = os.path.join(stg_object.path, "alua")
# remove the alua directories (future versions will handle this
# natively within rtslib_fb
for dirname in next(os.walk(alua_dir))[1]:
if dirname != "default_tg_pt_gp":
try:
alua_tpg = ALUATargetPortGroup(stg_object, dirname)
alua_tpg.delete()
except (RTSLibError, RTSLibNotInCFS) as err:
self.error = True
self.error_msg = "Delete of ALUA directories failed - {}".format(
err)
return
try:
stg_object.delete()
except RTSLibError as e:
self.error = True
self.error_msg = "Delete from LIO/backstores failed - {}".format(
e)
return
break
@staticmethod
def set_owner(gateways):
"""
Determine the gateway in the configuration with the lowest number of active LUNs. This
gateway is then selected as the owner for the primary path of the current LUN being
processed
:param gateways: gateway dict returned from the RADOS configuration object
:return: specific gateway hostname (str) that should provide the active path for the next LUN
"""
# Gateways contains simple attributes and dicts. The dicts define the gateways settings, so
# first we extract only the dicts within the main gateways dict
gw_nodes = {
key: gateways[key]
for key in gateways if isinstance(gateways[key], dict)
}
gw_items = gw_nodes.items()
# first entry is the lowest number of active_luns
gw_items.sort(key=lambda x: (x[1]['active_luns']))
# 1st tuple is gw with lowest active_luns, so return the 1st
# element which is the hostname
return gw_items[0][0]
@staticmethod
def dm_device_name_from_rbd_map(map_device):
"""
take a mapped device name /dev/rbdX to determine the /dev/mapper/X
equivalent by reading the devices attribute files in sysfs
:param map_device: device path of the form /dev/rbdX
:return: device mapper name for the rbd device /dev/mapper/<pool>-<image-id>
"""
rbd_bus_id = map_device[8:]
dm_uid = None
# TODO - could fread encounter an IOerror?
rbd_path = os.path.join('/sys/bus/rbd/devices', rbd_bus_id)
if os.path.exists(rbd_path):
pool_id = fread(os.path.join(rbd_path, "pool_id"))
image_id = fread(os.path.join(rbd_path, "image_id"))
current_snap = fread(os.path.join(rbd_path, "current_snap"))
dm_uid = "/dev/mapper/{}-{}".format(pool_id, image_id)
if current_snap != "-":
dm_uid += "-{}".format(fread(os.path.join(rbd_path,
"snap_id")))
return dm_uid
@staticmethod
def dm_wait_for_device(dm_device):
"""
multipath may take a few seconds for the device to appear, so we
need to wait until we see it - but use a timeout to abort if necessary
:param dm_device: dm device name /dev/mapper/<pool>-<image_id>
:return boolean representing when the device has been found
"""
waiting = 0
# wait for multipathd and udev to setup /dev node
# /dev/mapper/<pool_id>-<rbd_image_id>
# e.g. /dev/mapper/0-519d42ae8944a
while os.path.exists(dm_device) is False:
sleep(settings.config.loop_delay)
waiting += settings.config.loop_delay
if waiting >= settings.config.time_out:
break
return os.path.exists(dm_device)
def manage(self, mode):
"""
Manage the definition of the gateway, given a mode of 'target', 'map',
'init' or 'clearconfig'. In 'target' mode the LIO TPG is defined,
whereas in map mode, the required LUNs are added to the existing TPG
:param mode: run mode - target, map, init or clearconfig (str)
:return: None - but sets the objects error flags to be checked by
the caller
"""
config = Config(self.logger)
if config.error:
self.error = True
self.error_msg = config.error_msg
return
local_gw = this_host()
if mode == 'target':
if self.exists():
self.load_config()
self.check_tpgs()
else:
self.create_target()
if self.error:
# return to caller, with error state set
return
target_config = config.config["targets"][self.iqn]
self.update_acl(target_config['acl_enabled'])
discovery_auth_config = config.config['discovery_auth']
Discovery.set_discovery_auth_lio(
discovery_auth_config['username'],
discovery_auth_config['password'],
discovery_auth_config['password_encryption_enabled'],
discovery_auth_config['mutual_username'],
discovery_auth_config['mutual_password'],
discovery_auth_config['mutual_password_encryption_enabled'])
gateway_group = config.config["gateways"].keys()
if "ip_list" not in target_config:
target_config['ip_list'] = self.gateway_ip_list
config.update_item("targets", self.iqn, target_config)
self.config_updated = True
if self.controls != target_config.get('controls', {}):
target_config['controls'] = self.controls.copy()
config.update_item("targets", self.iqn, target_config)
self.config_updated = True
if local_gw not in gateway_group:
gateway_metadata = {"active_luns": 0}
config.add_item("gateways", local_gw)
config.update_item("gateways", local_gw, gateway_metadata)
self.config_updated = True
if local_gw not in target_config['portals']:
# Update existing gws with the new gw
for remote_gw, remote_gw_config in target_config[
'portals'].items():
if remote_gw_config[
'gateway_ip_list'] == self.gateway_ip_list:
continue
inactive_portal_ip = list(self.gateway_ip_list)
for portal_ip_address in remote_gw_config[
"portal_ip_addresses"]:
inactive_portal_ip.remove(portal_ip_address)
remote_gw_config['gateway_ip_list'] = self.gateway_ip_list
remote_gw_config['tpgs'] = len(self.tpg_list)
remote_gw_config[
'inactive_portal_ips'] = inactive_portal_ip
target_config['portals'][remote_gw] = remote_gw_config
# Add the new gw
inactive_portal_ip = list(self.gateway_ip_list)
for active_portal_ip in self.active_portal_ips:
inactive_portal_ip.remove(active_portal_ip)
portal_metadata = {
"tpgs": len(self.tpg_list),
"gateway_ip_list": self.gateway_ip_list,
"portal_ip_addresses": self.active_portal_ips,
"inactive_portal_ips": inactive_portal_ip
}
target_config['portals'][local_gw] = portal_metadata
target_config['ip_list'] = self.gateway_ip_list
config.update_item("targets", self.iqn, target_config)
self.config_updated = True
if self.config_updated:
config.commit()
elif mode == 'map':
if self.exists():
self.load_config()
self.map_luns(config)
target_config = config.config["targets"][self.iqn]
self.update_acl(target_config['acl_enabled'])
else:
self.error = True
self.error_msg = ("Attempted to map to a gateway '{}' that "
"hasn't been defined yet...out of order "
"steps?".format(self.iqn))
elif mode == 'init':
# init mode just creates the iscsi target definition and updates
# the config object. It is used by the CLI only
if self.exists():
self.logger.info("GWTarget init request skipped - target "
"already exists")
else:
# create the target
self.create_target()
seed_target = {
'disks': {},
'clients': {},
'acl_enabled': True,
'auth': {
'username': '',
'password': '',
'password_encryption_enabled': False,
'mutual_username': '',
'mutual_password': '',
'mutual_password_encryption_enabled': False
},
'portals': {},
'groups': {},
'controls': {}
}
config.add_item("targets", self.iqn, seed_target)
config.commit()
discovery_auth_config = config.config['discovery_auth']
Discovery.set_discovery_auth_lio(
discovery_auth_config['username'],
discovery_auth_config['password'],
discovery_auth_config['password_encryption_enabled'],
discovery_auth_config['mutual_username'],
discovery_auth_config['mutual_password'],
discovery_auth_config['mutual_password_encryption_enabled']
)
elif mode == 'clearconfig':
# Called by API from CLI clearconfig command
if self.exists():
self.load_config()
self.clear_config(config)
if self.error:
return
target_config = config.config["targets"][self.iqn]
if len(target_config['portals']) == 0:
config.del_item('targets', self.iqn)
else:
gw_ips = target_config['portals'][local_gw][
'portal_ip_addresses']
target_config['portals'].pop(local_gw)
ip_list = target_config['ip_list']
for gw_ip in gw_ips:
ip_list.remove(gw_ip)
if len(ip_list) > 0 and len(
target_config['portals'].keys()) > 0:
config.update_item('targets', self.iqn, target_config)
else:
# no more portals in the list, so delete the target
config.del_item('targets', self.iqn)
remove_gateway = True
for _, target in config.config["targets"].items():
if local_gw in target['portals']:
remove_gateway = False
break
if remove_gateway:
# gateway is no longer used, so delete it
config.del_item('gateways', local_gw)
config.commit()
def manage(self, rqst_type, committer=None):
"""
Manage the allocation or removal of this client
:param rqst_type is either 'present' (try and create the nodeACL), or
'absent' - delete the nodeACL
:param committer is the host responsible for any commits to the
configuration - this is not needed for Ansible management, but is used
by the CLI->API->GWClient interaction
"""
# Build a local object representing the rados configuration object
config_object = Config(self.logger)
if config_object.error:
self.error = True
self.error_msg = config_object.error_msg
return
# use current config to hold a copy of the current rados config
# object (dict)
self.current_config = config_object.config
running_under_ansible = ansible_control()
self.logger.debug("(GWClient.manage) running under ansible?"
" {}".format(running_under_ansible))
if running_under_ansible:
update_host = GWClient.get_update_host(self.current_config)
else:
update_host = committer
self.logger.debug("GWClient.manage) update host to handle any config "
"update is {}".format(update_host))
if rqst_type == "present":
###################################################################
# Ensure the client exists in LIO #
###################################################################
# first look at the request to see if it matches the settings
# already in the config object - if so this is just a rerun, or a
# reboot so config object updates are not needed when we change
# the LIO environment
if self.iqn in self.current_config['clients'].keys():
self.metadata = self.current_config['clients'][self.iqn]
config_image_list = sorted(self.metadata['luns'].keys())
#
# Does the request match the current config?
# extract the chap_str from the config object entry
config_chap = CHAP(self.metadata['auth']['chap'])
chap_str = config_chap.chap_str
if config_chap.error:
self.error = True
self.error_msg = config_chap.error_msg
return
if self.chap == chap_str and \
config_image_list == sorted(self.requested_images):
self.commit_enabled = False
else:
# requested iqn is not in the config object
if running_under_ansible:
if update_host == gethostname().split('.')[0]:
self.seed_config(config_object)
else:
# not ansible, so just run the command
self.seed_config(config_object)
self.metadata = GWClient.seed_metadata
self.logger.debug("(manage) config updates to be applied from "
"this host: {}".format(self.commit_enabled))
self.define_client()
if self.error:
# unable to define the client!
return
# if a group name has been set on the client, we need to bypass
# lun setup
if not self.metadata["group_name"]:
# no group_name, so the client is managed individually
bad_images = self.validate_images()
if not bad_images:
self.setup_luns()
if self.error:
return
else:
# request for images to map to this client that haven't
# been added to LIO yet!
self.error = True
self.error_msg = ("Non-existent images {} requested "
"for {}".format(bad_images, self.iqn))
return
# if '/' in self.chap:
if self.chap == '':
self.logger.warning("(main) client '{}' configured without"
" security".format(self.iqn))
self.configure_auth('chap', self.chap)
if self.error:
return
# check the client object's change count, and update the config
# object if this is the updating host
if self.change_count > 0:
if self.commit_enabled:
if update_host == gethostname().split('.')[0]:
# update the config object with this clients settings
self.logger.debug("Updating config object metadata "
"for '{}'".format(self.iqn))
config_object.update_item("clients", self.iqn,
self.metadata)
# persist the config update
config_object.commit()
else:
###################################################################
# Remove the requested client from the config object and LIO #
###################################################################
if self.exists():
self.define_client() # grab the client and parent tpg objects
self.delete() # deletes from the local LIO instance
if self.error:
return
else:
# remove this client from the config
if update_host == gethostname().split('.')[0]:
self.logger.debug("Removing {} from the config "
"object".format(self.iqn))
config_object.del_item("clients", self.iqn)
config_object.commit()
else:
# desired state is absent, but the client does not exist
# in LIO - Nothing to do!
self.logger.info("(main) client {} removal request, but it's"
"not in LIO...skipping".format(self.iqn))
class Group(object):
def __init__(self, logger, target_iqn, group_name, members=[], disks=[]):
"""
Manage a host group definition. The input for the group object is the
desired state of the group where the logic enforced produces an
idempotent group definition across API/CLI and more importantly Ansible
:param logger: (logging object) used for centralised logging
:param target_iqn: (str) target iqn
:param group_name: (str) group name
:param members: (list) iscsi IQN's of the clients
:param disks: (list) disk names of the format pool/image
"""
self.logger = logger
self.error = False
self.error_msg = ''
self.num_changes = 0
self.config = Config(logger)
if self.config.error:
self.error = self.config.error
self.error_msg = self.config.error_msg
return
self.target_iqn = target_iqn
self.group_name = group_name
self.group_members = members
self.disks = disks
target_config = self.config.config['targets'][self.target_iqn]
if group_name in target_config['groups']:
self.new_group = False
else:
self.new_group = True
self.logger.debug("Group : name={}".format(self.group_name))
self.logger.debug("Group : members={}".format(self.group_members))
self.logger.debug("Group : disks={}".format(self.disks))
def __str__(self):
return ("Group: {}\n- Members: {}\n- "
"Disks: {}".format(self.group_name,
self.group_members,
self.disks))
def _set_error(self, error_msg):
self.error = True
self.error_msg = error_msg
self.logger.debug("Error: {}".format(self.error_msg))
def _valid_client(self, action, client_iqn):
"""
validate the addition of a specific client
:param action: (str) add or remove request
:param client_iqn: (str) iqn of the client to add tot he group
:return: (bool) true/false whether the client should be accepted
"""
target_config = self.config.config['targets'][self.target_iqn]
self.logger.debug("checking '{}'".format(client_iqn))
# to validate the request, pass through a 'negative' filter
if action == 'add':
client = target_config['clients'].get(client_iqn, {})
if not client:
self._set_error("client '{}' doesn't exist".format(client_iqn))
return False
elif client.get('luns'):
self._set_error("Client '{}' already has luns. "
"Only clients without prior lun maps "
"can be added to a group".format(client_iqn))
return False
elif client.get('group_name'):
self._set_error("Client already assigned to {} - a client "
"can only belong to one host "
"group".format(client.get('group_name')))
return False
else:
# client_iqn must exist in the group
if client_iqn not in target_config['groups'][self.group_name].get('members'):
self._set_error("client '{}' is not a member of "
"{}".format(client_iqn,
self.group_name))
return False
# to reach here the request is considered valid
self.logger.debug("'{}' client '{}' for group '{}'"
" is valid".format(action,
client_iqn,
self.group_name))
return True
def _valid_disk(self, action, disk):
self.logger.debug("checking disk '{}'".format(disk))
target_config = self.config.config['targets'][self.target_iqn]
if action == 'add':
if disk not in target_config['disks']:
self._set_error("disk '{}' doesn't exist".format(disk))
return False
else:
if disk not in target_config['groups'][self.group_name]['disks']:
self._set_error("disk '{}' is not in the group".format(disk))
return False
return True
def _next_lun(self):
"""
Look at the disk list for the group and return the 1st available free
LUN id used for adding disks to the group
:return: (int) lun Id
"""
lun_range = list(range(0, 256, 1)) # 0->255
target_config = self.config.config['targets'][self.target_iqn]
group = target_config['groups'][self.group_name]
group_disks = group.get('disks')
for d in group_disks:
lun_range.remove(group_disks[d].get('lun_id'))
return lun_range[0]
def apply(self):
"""
setup/manage the group definition
:return: NULL
"""
group_seed = {
"members": [],
"disks": {}
}
target_config = self.config.config['targets'][self.target_iqn]
if self.new_group:
# New Group definition, so seed it
self.logger.debug("Processing request for new group "
"'{}'".format(self.group_name))
if len(set(self.group_members)) != len(self.group_members):
self._set_error("Member must contain unique clients - no "
"duplication")
return
self.logger.debug("New group definition required")
# new_group = True
target_config['groups'][self.group_name] = group_seed
# Now the group definition is at least seeded, so let's look at the
# member and disk information passed
this_group = target_config['groups'][self.group_name]
members = ListComparison(this_group.get('members'),
self.group_members)
disks = ListComparison(this_group.get('disks').keys(),
self.disks)
if set(self.disks) != set(this_group.get('disks')) or \
set(self.group_members) != set(this_group.get('members')):
group_changed = True
else:
group_changed = False
if group_changed or self.new_group:
if self.valid_request(members, disks):
self.update_metadata(members, disks)
else:
self._set_error("Group request failed validation")
return
else:
# no changes required
self.logger.info("Current group definition matches request")
self.enforce_policy()
def valid_request(self, members, disks):
self.logger.info("Validating client membership")
for mbr in members.added:
if not self._valid_client('add', mbr):
self.logger.error("'{}' failed checks".format(mbr))
return False
for mbr in members.removed:
if not self._valid_client('remove', mbr):
self.logger.error("'{}' failed checks".format(mbr))
return False
self.logger.debug("Client membership checks passed")
self.logger.debug("clients to add : {}".format(members.added))
self.logger.debug("clients to remove : {}".format(members.removed))
# client membership is valid, check disks
self.logger.info("Validating disk membership")
for disk_name in disks.added:
if not self._valid_disk('add', disk_name):
self.logger.error("'{}' failed checks".format(disk_name))
return False
for disk_name in disks.removed:
if not self._valid_disk('remove', disk_name):
self.logger.error("'{}' failed checks".format(disk_name))
return False
self.logger.info("Disk membership checks passed")
self.logger.debug("disks to add : {}".format(disks.added))
self.logger.debug("disks to remove : {}".format(disks.removed))
return True
def update_metadata(self, members, disks):
target_config = self.config.config['targets'][self.target_iqn]
this_group = target_config['groups'].get(self.group_name, {})
group_disks = this_group.get('disks', {})
if disks.added:
# update the groups disk list
for disk in disks.added:
lun_seq = self._next_lun()
group_disks[disk] = {"lun_id": lun_seq}
self.logger.debug("- adding '{}' to group '{}' @ "
"lun id {}".format(disk,
self.group_name,
lun_seq))
if disks.removed:
# remove disk from the group definition
for disk in disks.removed:
del group_disks[disk]
self.logger.debug("- removed '{}' from group "
"{}".format(disk,
self.group_name))
if disks.added or disks.removed:
# update each clients meta data
self.logger.debug("updating clients LUN masking with "
"{}".format(json.dumps(group_disks)))
for client_iqn in self.group_members:
self.update_disk_md(client_iqn, group_disks)
# handle client membership
if members.changed:
for client_iqn in members.added:
self.add_client(client_iqn)
self.update_disk_md(client_iqn, group_disks)
for client_iqn in members.removed:
self.remove_client(client_iqn)
this_group['members'] = self.group_members
this_group['disks'] = group_disks
self.logger.debug("Group '{}' updated to "
"{}".format(self.group_name,
json.dumps(this_group)))
target_config['groups'][self.group_name] = this_group
self.config.update_item('targets', self.target_iqn, target_config)
self.config.commit()
def enforce_policy(self):
target_config = self.config.config['targets'][self.target_iqn]
this_group = target_config['groups'][self.group_name]
group_disks = this_group.get('disks')
host_group = this_group.get('members')
image_list = sorted(group_disks.items(),
key=lambda v: v[1]['lun_id'])
for client_iqn in host_group:
self.update_client(client_iqn, image_list)
if self.error:
# Applying the policy failed, so report and abort
self.logger.error("Unable to apply policy to {} "
": {}".format(client_iqn,
self.error_msg))
return
def add_client(self, client_iqn):
target_config = self.config.config['targets'][self.target_iqn]
client_metadata = target_config['clients'][client_iqn]
client_metadata['group_name'] = self.group_name
self.config.update_item('targets', self.target_iqn, target_config)
self.logger.info("Added {} to group {}".format(client_iqn,
self.group_name))
def update_disk_md(self, client_iqn, group_disks):
target_config = self.config.config['targets'][self.target_iqn]
md = target_config['clients'].get(client_iqn)
md['luns'] = group_disks
self.config.update_item('targets', self.target_iqn, target_config)
self.logger.info("updated {} disk map to "
"{}".format(client_iqn,
json.dumps(group_disks)))
def update_client(self, client_iqn, image_list):
client = GWClient(self.logger, client_iqn, image_list, '', '', '', '', self.target_iqn)
client.manage('reconfigure')
# grab the client's metadata from the config (needed by setup_luns)
target_config = self.config.config['targets'][self.target_iqn]
client.metadata = target_config['clients'][client_iqn]
client.setup_luns(self.config.config['disks'])
if client.error:
self._set_error(client.error_msg)
def remove_client(self, client_iqn):
target_config = self.config.config['targets'][self.target_iqn]
client_md = target_config["clients"][client_iqn]
# remove the group_name setting from the client
client_md['group_name'] = ''
self.config.update_item('targets', self.target_iqn, target_config)
self.logger.info("Removed {} from group {}".format(client_iqn,
self.group_name))
def purge(self):
# act on the group name
# get the members from the current definition
target_config = self.config.config['targets'][self.target_iqn]
groups = target_config['groups']
if self.group_name in groups:
for mbr in groups[self.group_name]["members"]:
self.remove_client(mbr)
# issue a del_item to the config object for this group_name
groups.pop(self.group_name)
self.config.update_item('targets', self.target_iqn, target_config)
self.config.commit()
self.logger.info("Group {} removed".format(self.group_name))
else:
self._set_error("Group name requested does not exist")
return
def manage(self, rqst_type, committer=None):
"""
Manage the allocation or removal of this client
:param rqst_type is either 'present' (try and create the nodeACL), or
'absent' - delete the nodeACL
:param committer is the host responsible for any commits to the
configuration - this is not needed for Ansible management, but is used
by the CLI->API->GWClient interaction
"""
# Build a local object representing the rados configuration object
config_object = Config(self.logger)
if config_object.error:
self.error = True
self.error_msg = config_object.error_msg
return
# use current config to hold a copy of the current rados config
# object (dict)
self.current_config = config_object.config
target_config = self.current_config['targets'][self.target_iqn]
update_host = committer
self.logger.debug("GWClient.manage) update host to handle any config "
"update is {}".format(update_host))
if rqst_type == "present":
###################################################################
# Ensure the client exists in LIO #
###################################################################
# first look at the request to see if it matches the settings
# already in the config object - if so this is just a rerun, or a
# reboot so config object updates are not needed when we change
# the LIO environment
if self.iqn in target_config['clients'].keys():
self.metadata = target_config['clients'][self.iqn]
config_image_list = sorted(self.metadata['luns'].keys())
#
# Does the request match the current config?
auth_config = self.metadata['auth']
config_chap = CHAP(auth_config['username'],
auth_config['password'],
auth_config['password_encryption_enabled'])
if config_chap.error:
self.error = True
self.error_msg = config_chap.error_msg
return
# extract the chap_mutual_str from the config object entry
config_chap_mutual = CHAP(
auth_config['mutual_username'],
auth_config['mutual_password'],
auth_config['mutual_password_encryption_enabled'])
if config_chap_mutual.error:
self.error = True
self.error_msg = config_chap_mutual.error_msg
return
if self.username == config_chap.user and \
self.password == config_chap.password and \
self.mutual_username == config_chap_mutual.user and \
self.mutual_password == config_chap_mutual.password and \
config_image_list == sorted(self.requested_images):
self.commit_enabled = False
else:
# requested iqn is not in the config object
self.seed_config(config_object)
self.metadata = GWClient.seed_metadata
self.logger.debug("(manage) config updates to be applied from "
"this host: {}".format(self.commit_enabled))
client_exists = self.exists()
self.define_client()
if self.error:
# unable to define the client!
return
if client_exists and self.metadata["group_name"]:
# bypass setup_luns for existing clients that have an
# associated host group
pass
else:
# either the client didn't exist (new or boot time), or the
# group_name is not defined so run setup_luns for this client
disks_config = self.current_config['disks']
bad_images = self.validate_images(disks_config)
if not bad_images:
self.setup_luns(disks_config)
if self.error:
return
else:
# request for images to map to this client that haven't
# been added to LIO yet!
self.error = True
self.error_msg = ("Non-existent images {} requested "
"for {}".format(bad_images, self.iqn))
return
if not self.username and not self.password and \
not self.mutual_username and not self.mutual_password:
self.logger.warning("(main) client '{}' configured without"
" security".format(self.iqn))
self.configure_auth(self.username, self.password,
self.mutual_username, self.mutual_password,
target_config)
if self.error:
return
# check the client object's change count, and update the config
# object if this is the updating host
if self.change_count > 0:
if self.commit_enabled:
if update_host == this_host():
# update the config object with this clients settings
self.logger.debug("Updating config object metadata "
"for '{}'".format(self.iqn))
target_config['clients'][self.iqn] = self.metadata
config_object.update_item("targets", self.target_iqn,
target_config)
# persist the config update
config_object.commit()
elif rqst_type == 'reconfigure':
self.define_client()
else:
###################################################################
# Remove the requested client from the config object and LIO #
###################################################################
if self.exists():
self.define_client() # grab the client and parent tpg objects
self.delete() # deletes from the local LIO instance
if self.error:
return
else:
# remove this client from the config
if update_host == this_host():
self.logger.debug("Removing {} from the config "
"object".format(self.iqn))
target_config['clients'].pop(self.iqn)
config_object.update_item("targets", self.target_iqn,
target_config)
config_object.commit()
else:
# desired state is absent, but the client does not exist
# in LIO - Nothing to do!
self.logger.info("(main) client {} removal request, but it's"
"not in LIO...skipping".format(self.iqn))
def manage(self, mode):
"""
Manage the definition of the gateway, given a mode of 'target', 'map',
'init' or 'clearconfig'. In 'target' mode the LIO TPG is defined,
whereas in map mode, the required LUNs are added to the existing TPG
:param mode: run mode - target, map, init or clearconfig (str)
:return: None - but sets the objects error flags to be checked by
the caller
"""
config = Config(self.logger)
if config.error:
self.error = True
self.error_msg = config.error_msg
return
local_gw = this_host()
if mode == 'target':
if self.exists():
self.load_config()
self.check_tpgs()
else:
self.create_target()
if self.error:
# return to caller, with error state set
return
Discovery.set_discovery_auth_lio(
config.config['discovery_auth']['chap'],
config.config['discovery_auth']['chap_mutual'])
target_config = config.config["targets"][self.iqn]
gateway_group = config.config["gateways"].keys()
if "ip_list" not in target_config:
target_config['ip_list'] = self.gateway_ip_list
config.update_item("targets", self.iqn, target_config)
self.config_updated = True
if self.controls != target_config.get('controls', {}):
target_config['controls'] = self.controls.copy()
config.update_item("targets", self.iqn, target_config)
self.config_updated = True
if local_gw not in gateway_group:
gateway_metadata = {"active_luns": 0}
config.add_item("gateways", local_gw)
config.update_item("gateways", local_gw, gateway_metadata)
self.config_updated = True
if local_gw not in target_config['portals']:
inactive_portal_ip = list(self.gateway_ip_list)
inactive_portal_ip.remove(self.active_portal_ip)
portal_metadata = {
"tpgs": len(self.tpg_list),
"gateway_ip_list": self.gateway_ip_list,
"portal_ip_address": self.active_portal_ip,
"inactive_portal_ips": inactive_portal_ip
}
target_config['portals'][local_gw] = portal_metadata
target_config['ip_list'] = self.gateway_ip_list
config.update_item("targets", self.iqn, target_config)
self.config_updated = True
else:
# gateway already defined, so check that the IP list it has
# matches the current request
portal_details = target_config['portals'][local_gw]
if portal_details['gateway_ip_list'] != self.gateway_ip_list:
inactive_portal_ip = list(self.gateway_ip_list)
inactive_portal_ip.remove(self.active_portal_ip)
portal_details['gateway_ip_list'] = self.gateway_ip_list
portal_details['tpgs'] = len(self.tpg_list)
portal_details['inactive_portal_ips'] = inactive_portal_ip
target_config['portals'][local_gw] = portal_details
config.update_item("targets", self.iqn, target_config)
self.config_updated = True
if self.config_updated:
config.commit()
elif mode == 'map':
if self.exists():
self.load_config()
self.map_luns(config)
else:
self.error = True
self.error_msg = ("Attempted to map to a gateway '{}' that "
"hasn't been defined yet...out of order "
"steps?".format(self.iqn))
elif mode == 'init':
# init mode just creates the iscsi target definition and updates
# the config object. It is used by the CLI only
if self.exists():
self.logger.info("GWTarget init request skipped - target "
"already exists")
else:
# create the target
self.create_target()
seed_target = {
'disks': [],
'clients': {},
'portals': {},
'groups': {},
'controls': {}
}
config.add_item("targets", self.iqn, seed_target)
config.commit()
Discovery.set_discovery_auth_lio(
config.config['discovery_auth']['chap'],
config.config['discovery_auth']['chap_mutual'])
elif mode == 'clearconfig':
# Called by API from CLI clearconfig command
if self.exists():
self.load_config()
else:
self.error = True
self.error_msg = "Target {} does not exist on {}".format(
self.iqn, local_gw)
return
target_config = config.config["targets"][self.iqn]
self.clear_config()
if not self.error:
if len(target_config['portals']) == 0:
config.del_item('targets', self.iqn)
else:
gw_ip = target_config['portals'][local_gw][
'portal_ip_address']
target_config['portals'].pop(local_gw)
ip_list = target_config['ip_list']
ip_list.remove(gw_ip)
if len(ip_list) > 0 and len(
target_config['portals'].keys()) > 0:
config.update_item('targets', self.iqn, target_config)
else:
# no more portals in the list, so delete the target
config.del_item('targets', self.iqn)
remove_gateway = True
for _, target in config.config["targets"].items():
if local_gw in target['portals']:
remove_gateway = False
break
if remove_gateway:
# gateway is no longer used, so delete it
config.del_item('gateways', local_gw)
config.commit()
class LUN(object):
def __init__(self, logger, pool, image, size, allocating_host):
self.logger = logger
self.image = image
self.pool = pool
self.pool_id = 0
self.size_bytes = convert_2_bytes(size)
self.config_key = '{}.{}'.format(self.pool, self.image)
self.controls = {}
# the allocating host could be fqdn or shortname - but the config
# only uses shortname so it needs to be converted to shortname format
self.allocating_host = allocating_host.split('.')[0]
self.owner = '' # gateway that owns the preferred path for this LUN
self.error = False
self.error_msg = ''
self.num_changes = 0
self.config = Config(logger)
if self.config.error:
self.error = self.config.error
self.error_msg = self.config.error_msg
return
self._validate_request()
if self.config_key in self.config.config['disks']:
self.controls = self.config.config['disks'][self.config_key].get(
'controls', {}).copy()
def _get_max_data_area_mb(self):
max_data_area_mb = self.controls.get('max_data_area_mb', None)
if max_data_area_mb is None:
return settings.config.max_data_area_mb
return max_data_area_mb
def _set_max_data_area_mb(self, value):
if value is None or str(value) == str(
settings.config.max_data_area_mb):
self.controls.pop('max_data_area_mb', None)
else:
self.controls['max_data_area_mb'] = value
max_data_area_mb = property(_get_max_data_area_mb,
_set_max_data_area_mb,
doc="get/set kernel data area (MiB)")
def _validate_request(self):
# Before we start make sure that the target host is actually
# defined to the config
if self.allocating_host not in self.config.config['gateways'].keys():
self.logger.critical("Owning host is not valid, please provide a "
"valid gateway name for this rbd image")
self.error = True
self.error_msg = ("host name given for {} is not a valid gateway"
" name, listed in the config".format(self.image))
elif not rados_pool(pool=self.pool):
# Could create the pool, but a fat finger moment in the config
# file would mean rbd images get created and mapped, and then need
# correcting. Better to exit if the pool doesn't exist
self.error = True
self.error_msg = ("Pool '{}' does not exist. Unable to "
"continue".format(self.pool))
def remove_lun(self):
this_host = gethostname().split('.')[0]
self.logger.info("LUN deletion request received, rbd removal to be "
"performed by {}".format(self.allocating_host))
# First ensure the LUN is not allocated to a client
clients = self.config.config['clients']
lun_in_use = False
for iqn in clients:
client_luns = clients[iqn]['luns'].keys()
if self.config_key in client_luns:
lun_in_use = True
break
if lun_in_use:
# this will fail the ansible task for this lun/host
self.error = True
self.error_msg = ("Unable to delete {} - allocated to "
"{}".format(self.config_key, iqn))
self.logger.warning(self.error_msg)
return
# Check that the LUN is in LIO - if not there is nothing to do for
# this request
lun = self.lio_stg_object()
if not lun:
return
# Now we know the request is for a LUN in LIO, and it's not masked
# to a client
self.remove_dev_from_lio()
if self.error:
return
rbd_image = RBDDev(self.image, '0G', self.pool)
if this_host == self.allocating_host:
# by using the allocating host we ensure the delete is not
# issue by several hosts when initiated through ansible
rbd_image.delete()
if rbd_image.error:
self.error = True
self.error_msg = ("Unable to delete the underlying rbd "
"image {}".format(self.config_key))
return
# determine which host was the path owner
disk_owner = self.config.config['disks'][self.config_key]['owner']
#
# remove the definition from the config object
self.config.del_item('disks', self.config_key)
# update the active_luns count for gateway that owned this
# lun
gw_metadata = self.config.config['gateways'][disk_owner]
if gw_metadata['active_luns'] > 0:
gw_metadata['active_luns'] -= 1
self.config.update_item('gateways', disk_owner, gw_metadata)
self.config.commit()
def manage(self, desired_state):
self.logger.debug("LUN.manage request for {}, desired state "
"{}".format(self.image, desired_state))
if desired_state == 'present':
self.allocate()
elif desired_state == 'absent':
self.remove_lun()
def update_config(self, commit=False):
self.logger.debug("LUN.update_config starting")
# delete/update on-disk attributes
disk_attr = self.config.config['disks'][self.config_key]
if self.controls != disk_attr.get('controls', {}):
disk_attr['controls'] = self.controls
self.config.update_item('disks', self.config_key, disk_attr)
if commit and self.config.changed:
self.config.commit()
self.error = self.config.error
self.error_msg = self.config.error_msg
def allocate(self):
self.logger.debug("LUN.allocate starting, listing rbd devices")
disk_list = RBDDev.rbd_list(pool=self.pool)
self.logger.debug("rados pool '{}' contains the following - "
"{}".format(self.pool, disk_list))
this_host = gethostname().split('.')[0]
self.logger.debug("Hostname Check - this host is {}, target host for "
"allocations is {}".format(this_host,
self.allocating_host))
rbd_image = RBDDev(self.image, self.size_bytes, self.pool)
self.pool_id = rbd_image.pool_id
# if the image required isn't defined, create it!
if self.image not in disk_list:
# create the requested disk if this is the 'owning' host
if this_host == self.allocating_host:
rbd_image.create()
if not rbd_image.error:
self.config.add_item('disks', self.config_key)
self.logger.info("(LUN.allocate) created {}/{} "
"successfully".format(
self.pool, self.image))
self.num_changes += 1
else:
self.error = True
self.error_msg = rbd_image.error_msg
return
else:
# the image isn't there, and this isn't the 'owning' host
# so wait until the disk arrives
waiting = 0
while self.image not in disk_list:
sleep(settings.config.loop_delay)
disk_list = RBDDev.rbd_list(pool=self.pool)
waiting += settings.config.loop_delay
if waiting >= settings.config.time_out:
self.error = True
self.error_msg = ("(LUN.allocate) timed out waiting "
"for rbd to show up")
return
else:
# requested image is already defined to ceph
if rbd_image.valid:
# rbd image is OK to use, so ensure it's in the config
# object
if self.config_key not in self.config.config['disks']:
self.config.add_item('disks', self.config_key)
else:
# rbd image is not valid for export, so abort
self.error = True
self.error_msg = ("(LUN.allocate) rbd '{}' is not compatible "
"with LIO\nOnly image features {} are"
" supported".format(
self.image,
','.join(RBDDev.rbd_feature_list)))
self.logger.error(self.error_msg)
return
self.logger.debug("Check the rbd image size matches the request")
# if updates_made is not set, the disk pre-exists so on the owning
# host see if it needs to be resized
if self.num_changes == 0 and this_host == self.allocating_host:
# check the size, and update if needed
rbd_image.rbd_size()
if rbd_image.error:
self.logger.critical(rbd_image.error_msg)
self.error = True
self.error_msg = rbd_image.error_msg
return
if rbd_image.changed:
self.logger.info("rbd image {} resized "
"to {}".format(self.config_key,
self.size_bytes))
self.num_changes += 1
else:
self.logger.debug("rbd image {} size matches the configuration"
" file request".format(self.config_key))
self.logger.debug("Begin processing LIO mapping")
# now see if we need to add this rbd image to LIO
so = self.lio_stg_object()
if not so:
# this image has not been defined to this hosts LIO, so check the
# config for the details and if it's missing define the
# wwn/alua_state and update the config
if this_host == self.allocating_host:
# first check to see if the device needs adding
try:
wwn = self.config.config['disks'][self.config_key]['wwn']
except KeyError:
wwn = ''
if wwn == '':
# disk hasn't been defined to LIO yet, it' not been defined
# to the config yet and this is the allocating host
lun = self.add_dev_to_lio()
if self.error:
return
# lun is now in LIO, time for some housekeeping :P
wwn = lun._get_wwn()
self.owner = LUN.set_owner(self.config.config['gateways'])
self.logger.debug("{} owner will be {}".format(
self.image, self.owner))
disk_attr = {
"wwn": wwn,
"image": self.image,
"owner": self.owner,
"pool": self.pool,
"pool_id": rbd_image.pool_id,
"controls": self.controls
}
self.config.update_item('disks', self.config_key,
disk_attr)
gateway_dict = self.config.config['gateways'][self.owner]
gateway_dict['active_luns'] += 1
self.config.update_item('gateways', self.owner,
gateway_dict)
self.logger.debug("(LUN.allocate) registered '{}' with "
"wwn '{}' with the config "
"object".format(self.image, wwn))
self.logger.info("(LUN.allocate) added '{}/{}' to LIO and"
" config object".format(
self.pool, self.image))
else:
# config object already had wwn for this rbd image
self.add_dev_to_lio(wwn)
if self.error:
return
self.update_config()
self.logger.debug("(LUN.allocate) registered '{}' to LIO "
"with wwn '{}' from the config "
"object".format(self.image, wwn))
self.num_changes += 1
else:
# lun is not already in LIO, but this is not the owning node
# that defines the wwn we need the wwn from the config
# (placed by the allocating host), so we wait!
waiting = 0
while waiting < settings.config.time_out:
self.config.refresh()
if self.config_key in self.config.config['disks']:
if 'wwn' in self.config.config['disks'][
self.config_key]:
if self.config.config['disks'][
self.config_key]['wwn']:
wwn = self.config.config['disks'][
self.config_key]['wwn']
break
sleep(settings.config.loop_delay)
waiting += settings.config.loop_delay
self.logger.debug(
"(LUN.allocate) waiting for config object"
" to show {} with it's wwn".format(self.image))
if waiting >= settings.config.time_out:
self.error = True
self.error_msg = ("(LUN.allocate) waited too long for the "
"wwn information on image {} to "
"arrive".format(self.image))
return
# At this point we have a wwn from the config for this rbd
# image, so just add to LIO
self.add_dev_to_lio(wwn)
if self.error:
return
self.logger.info("(LUN.allocate) added {} to LIO using wwn "
"'{}' defined by {}".format(
self.image, wwn, self.allocating_host))
self.num_changes += 1
else:
# lun exists in LIO, check the size is correct
if not self.lio_size_ok(rbd_image, so):
self.error = True
self.error_msg = "Unable to sync the rbd device size with LIO"
self.logger.critical(self.error_msg)
return
self.logger.debug("config meta data for this disk is "
"{}".format(
self.config.config['disks'][self.config_key]))
# the owning host for an image is the only host that commits to the
# config
if this_host == self.allocating_host and self.config.changed:
self.logger.debug("(LUN.allocate) Committing change(s) to the "
"config object in pool {}".format(self.pool))
self.config.commit()
self.error = self.config.error
self.error_msg = self.config.error_msg
def lio_size_ok(self, rbd_object, stg_object):
"""
Check that the SO in LIO matches the current size of the rbd. if the
size requested < current size, just return. Downsizing an rbd is not
supported by this code and problematic for client filesystems anyway!
:return boolean indicating whether the size matches
"""
tmr = 0
size_ok = False
rbd_size_ok = False
# dm_path_found = False
# We have to wait for the rbd size to match, since the rbd could have
# been resized on another gateway host
while tmr < settings.config.time_out:
if self.size_bytes <= rbd_object.current_size:
rbd_size_ok = True
break
sleep(settings.config.loop_delay)
tmr += settings.config.loop_delay
# we have the right size for the rbd - check that LIO dev size matches
if rbd_size_ok:
# If the LIO size is not right, poke it with the new value
if stg_object.size < self.size_bytes:
self.logger.info("Resizing {} in LIO "
"to {}".format(self.config_key,
self.size_bytes))
stg_object.set_attribute("dev_size", self.size_bytes)
size_ok = stg_object.size == self.size_bytes
else:
size_ok = True
return size_ok
def lio_stg_object(self):
found_it = False
rtsroot = root.RTSRoot()
for stg_object in rtsroot.storage_objects:
# First match on name, but then check the pool incase the same
# name exists in multiple pools
if stg_object.name == self.config_key:
found_it = True
break
return stg_object if found_it else None
def add_dev_to_lio(self, in_wwn=None):
"""
Add an rbd device to the LIO configuration
:param in_wwn: optional wwn identifying the rbd image to clients
(must match across gateways)
:return: LIO LUN object
"""
self.logger.info("(LUN.add_dev_to_lio) Adding image "
"'{}' to LIO".format(self.config_key))
# extract control parameter overrides (if any) or use default
controls = self.controls.copy()
for k in ['max_data_area_mb']:
if controls.get(k, None) is None:
controls[k] = getattr(settings.config, k, None)
control_string = gen_control_string(controls)
if control_string:
self.logger.debug("control=\"{}\"".format(control_string))
new_lun = None
try:
# config string = rbd identifier / config_key (pool/image) /
# optional osd timeout
cfgstring = "rbd/{}/{};osd_op_timeout={}".format(
self.pool, self.image, settings.config.osd_op_timeout)
if (settings.config.cephconf != '/etc/ceph/ceph.conf'):
cfgstring += ";conf={}".format(settings.config.cephconf)
new_lun = UserBackedStorageObject(name=self.config_key,
config=cfgstring,
size=self.size_bytes,
wwn=in_wwn,
control=control_string)
except RTSLibError as err:
self.error = True
self.error_msg = ("failed to add {} to LIO - "
"error({})".format(self.config_key, str(err)))
self.logger.error(self.error_msg)
return None
try:
new_lun.set_attribute("cmd_time_out", 0)
new_lun.set_attribute("qfull_time_out",
settings.config.qfull_timeout)
except RTSLibError as err:
self.error = True
self.error_msg = ("Could not set LIO device attribute "
"cmd_time_out/qfull_time_out for device: {}. "
"Kernel not supported. - "
"error({})".format(self.config_key, str(err)))
self.logger.error(self.error_msg)
new_lun.delete()
return None
self.logger.info("(LUN.add_dev_to_lio) Successfully added {}"
" to LIO".format(self.config_key))
return new_lun
def remove_dev_from_lio(self):
lio_root = root.RTSRoot()
# remove the device from all tpgs
for t in lio_root.tpgs:
for lun in t.luns:
if lun.storage_object.name == self.config_key:
try:
lun.delete()
except Exception as e:
self.error = True
self.error_msg = ("Delete from LIO/TPG failed - "
"{}".format(e))
return
else:
break # continue to the next tpg
for stg_object in lio_root.storage_objects:
if stg_object.name == self.config_key:
try:
stg_object.delete()
except Exception as e:
self.error = True
self.error_msg = ("Delete from LIO/backstores failed - "
"{}".format(e))
return
break
@staticmethod
def set_owner(gateways):
"""
Determine the gateway in the configuration with the lowest number of
active LUNs. This gateway is then selected as the owner for the
primary path of the current LUN being processed
:param gateways: gateway dict returned from the RADOS configuration
object
:return: specific gateway hostname (str) that should provide the
active path for the next LUN
"""
# Gateways contains simple attributes and dicts. The dicts define the
# gateways settings, so first we extract only the dicts within the
# main gateways dict
gw_nodes = {
key: gateways[key]
for key in gateways if isinstance(gateways[key], dict)
}
gw_items = gw_nodes.items()
# first entry is the lowest number of active_luns
gw_items.sort(key=lambda x: (x[1]['active_luns']))
# 1st tuple is gw with lowest active_luns, so return the 1st
# element which is the hostname
return gw_items[0][0]
@staticmethod
def define_luns(logger, config, gateway):
"""
define the disks in the config to LIO
:param logger: logger object to print to
:param config: configuration dict from the rados pool
:param gateway: (object) gateway object - used for mapping
:raises CephiSCSIError.
"""
local_gw = this_host()
# sort the disks dict keys, so the disks are registered in a specific
# sequence
disks = config.config['disks']
srtd_disks = sorted(disks)
pools = {disks[disk_key]['pool'] for disk_key in srtd_disks}
if pools is None:
logger.info("No LUNs to export")
return True
ips = ipv4_addresses()
with rados.Rados(conffile=settings.config.cephconf) as cluster:
for pool in pools:
logger.debug("Processing rbd's in '{}' pool".format(pool))
with cluster.open_ioctx(pool) as ioctx:
pool_disks = [
disk_key for disk_key in srtd_disks
if disk_key.startswith(pool)
]
for disk_key in pool_disks:
pool, image_name = disk_key.split('.')
try:
with rbd.Image(ioctx, image_name) as rbd_image:
RBDDev.rbd_lock_cleanup(logger, ips, rbd_image)
lun = LUN(logger, pool, image_name,
rbd_image.size(), local_gw)
if lun.error:
raise CephiSCSIError(
"Error defining rbd "
"image {}".format(disk_key))
lun.allocate()
if lun.error:
raise CephiSCSIError("Error unable to "
"register {} with "
"LIO - {}".format(
disk_key,
lun.error_msg))
except rbd.ImageNotFound:
raise CephiSCSIError("Disk '{}' defined to the "
"config, but image '{}' can "
"not be found in '{}' "
"pool".format(
disk_key, image_name,
pool))
# Gateway Mapping : Map the LUN's registered to all tpg's within the
# LIO target
gateway.manage('map')
if gateway.error:
raise CephiSCSIError("Error mapping the LUNs to the tpg's within "
"the iscsi Target")
class LUN(object):
def __init__(self, logger, pool, image, size, allocating_host):
self.logger = logger
self.image = image
self.pool = pool
self.pool_id = 0
self.size = size
self.size_bytes = convert_2_bytes(size)
self.config_key = '{}.{}'.format(self.pool, self.image)
# the allocating host could be fqdn or shortname - but the config
# only uses shortname so it needs to be converted to shortname format
self.allocating_host = allocating_host.split('.')[0]
self.owner = '' # gateway that owns the preferred path for this LUN
self.error = False
self.error_msg = ''
self.num_changes = 0
self.config = Config(logger)
if self.config.error:
self.error = self.config.error
self.error_msg = self.config.error_msg
return
self._validate_request()
def _validate_request(self):
# Before we start make sure that the target host is actually
# defined to the config
if self.allocating_host not in self.config.config['gateways'].keys():
self.logger.critical("Owning host is not valid, please provide a "
"valid gateway name for this rbd image")
self.error = True
self.error_msg = ("host name given for {} is not a valid gateway"
" name, listed in the config".format(self.image))
elif not rados_pool(pool=self.pool):
# Could create the pool, but a fat finger moment in the config
# file would mean rbd images get created and mapped, and then need
# correcting. Better to exit if the pool doesn't exist
self.error = True
self.error_msg = ("Pool '{}' does not exist. Unable to "
"continue".format(self.pool))
def remove_lun(self):
this_host = gethostname().split('.')[0]
self.logger.info("LUN deletion request received, rbd removal to be "
"performed by {}".format(self.allocating_host))
# First ensure the LUN is not allocated to a client
clients = self.config.config['clients']
lun_in_use = False
for iqn in clients:
client_luns = clients[iqn]['luns'].keys()
if self.config_key in client_luns:
lun_in_use = True
break
if lun_in_use:
# this will fail the ansible task for this lun/host
self.error = True
self.error_msg = ("Unable to delete {} - allocated to "
"{}".format(self.config_key, iqn))
self.logger.warning(self.error_msg)
return
# Check that the LUN is in LIO - if not there is nothing to do for
# this request
lun = self.lio_stg_object()
if not lun:
return
# Now we know the request is for a LUN in LIO, and it's not masked
# to a client
self.remove_dev_from_lio()
if self.error:
return
rbd_image = RBDDev(self.image, '0G', self.pool)
if this_host == self.allocating_host:
# by using the allocating host we ensure the delete is not
# issue by several hosts when initiated through ansible
rbd_image.delete()
if rbd_image.error:
self.error = True
self.error_msg = ("Unable to delete the underlying rbd "
"image {}".format(self.config_key))
return
# determine which host was the path owner
disk_owner = self.config.config['disks'][self.config_key]['owner']
#
# remove the definition from the config object
self.config.del_item('disks', self.config_key)
# update the active_luns count for gateway that owned this
# lun
gw_metadata = self.config.config['gateways'][disk_owner]
if gw_metadata['active_luns'] > 0:
gw_metadata['active_luns'] -= 1
self.config.update_item('gateways', disk_owner, gw_metadata)
self.config.commit()
def manage(self, desired_state):
self.logger.debug("LUN.manage request for {}, desired state "
"{}".format(self.image, desired_state))
if desired_state == 'present':
self.allocate()
elif desired_state == 'absent':
self.remove_lun()
def allocate(self):
self.logger.debug("LUN.allocate starting, listing rbd devices")
disk_list = RBDDev.rbd_list(pool=self.pool)
self.logger.debug("rados pool '{}' contains the following - "
"{}".format(self.pool, disk_list))
this_host = gethostname().split('.')[0]
self.logger.debug("Hostname Check - this host is {}, target host for "
"allocations is {}".format(this_host,
self.allocating_host))
rbd_image = RBDDev(self.image, self.size, self.pool)
self.pool_id = rbd_image.pool_id
# if the image required isn't defined, create it!
if self.image not in disk_list:
# create the requested disk if this is the 'owning' host
if this_host == self.allocating_host:
rbd_image.create()
if not rbd_image.error:
self.config.add_item('disks', self.config_key)
self.logger.info("(LUN.allocate) created {}/{} "
"successfully".format(
self.pool, self.image))
self.num_changes += 1
else:
self.error = True
self.error_msg = rbd_image.error_msg
return
else:
# the image isn't there, and this isn't the 'owning' host
# so wait until the disk arrives
waiting = 0
while self.image not in disk_list:
sleep(settings.config.loop_delay)
disk_list = RBDDev.rbd_list(pool=self.pool)
waiting += settings.config.loop_delay
if waiting >= settings.config.time_out:
self.error = True
self.error_msg = ("(LUN.allocate) timed out waiting "
"for rbd to show up")
return
else:
# requested image is already defined to ceph
if rbd_image.valid:
self.size_bytes = rbd_image._get_size_bytes()
# rbd image is OK to use, so ensure it's in the config
# object
if self.config_key not in self.config.config['disks']:
self.config.add_item('disks', self.config_key)
else:
# rbd image is not valid for export, so abort
self.error = True
self.error_msg = ("(LUN.allocate) rbd '{}' is not compatible "
"with LIO\nOnly image features {} are"
" supported".format(
self.image,
','.join(RBDDev.rbd_feature_list)))
self.logger.error(self.error_msg)
return
self.logger.debug("Check the rbd image size matches the request")
# if updates_made is not set, the disk pre-exists so on the owning
# host see if it needs to be resized
if self.num_changes == 0 and this_host == self.allocating_host:
# check the size, and update if needed
rbd_image.rbd_size()
if rbd_image.error:
self.logger.critical(rbd_image.error_msg)
self.error = True
self.error_msg = rbd_image.error_msg
return
if rbd_image.changed:
self.logger.info("rbd image {} resized "
"to {}".format(self.config_key, self.size))
self.num_changes += 1
else:
self.logger.debug("rbd image {} size matches the configuration"
" file request".format(self.config_key))
self.logger.debug("Begin processing LIO mapping")
# now see if we need to add this rbd image to LIO
so = self.lio_stg_object()
if not so:
# this image has not been defined to this hosts LIO, so check the
# config for the details and if it's missing define the
# wwn/alua_state and update the config
if this_host == self.allocating_host:
# first check to see if the device needs adding
try:
wwn = self.config.config['disks'][self.config_key]['wwn']
except KeyError:
wwn = ''
if wwn == '':
# disk hasn't been defined to LIO yet, it' not been defined
# to the config yet and this is the allocating host
lun = self.add_dev_to_lio()
if self.error:
return
# lun is now in LIO, time for some housekeeping :P
wwn = lun._get_wwn()
self.owner = LUN.set_owner(self.config.config['gateways'])
self.logger.debug("{} owner will be {}".format(
self.image, self.owner))
disk_attr = {
"wwn": wwn,
"image": self.image,
"owner": self.owner,
"pool": self.pool,
"pool_id": rbd_image.pool_id
}
self.config.update_item('disks', self.config_key,
disk_attr)
gateway_dict = self.config.config['gateways'][self.owner]
gateway_dict['active_luns'] += 1
self.config.update_item('gateways', self.owner,
gateway_dict)
self.logger.debug("(LUN.allocate) registered '{}' with "
"wwn '{}' with the config "
"object".format(self.image, wwn))
self.logger.info("(LUN.allocate) added '{}/{}' to LIO and"
" config object".format(
self.pool, self.image))
else:
# config object already had wwn for this rbd image
self.add_dev_to_lio(wwn)
if self.error:
return
self.logger.debug("(LUN.allocate) registered '{}' to LIO "
"with wwn '{}' from the config "
"object".format(self.image, wwn))
self.num_changes += 1
else:
# lun is not already in LIO, but this is not the owning node
# that defines the wwn we need the wwn from the config
# (placed by the allocating host), so we wait!
waiting = 0
while waiting < settings.config.time_out:
self.config.refresh()
if self.config_key in self.config.config['disks']:
if 'wwn' in self.config.config['disks'][
self.config_key]:
if self.config.config['disks'][
self.config_key]['wwn']:
wwn = self.config.config['disks'][
self.config_key]['wwn']
break
sleep(settings.config.loop_delay)
waiting += settings.config.loop_delay
self.logger.debug(
"(LUN.allocate) waiting for config object"
" to show {} with it's wwn".format(self.image))
if waiting >= settings.config.time_out:
self.error = True
self.error_msg = ("(LUN.allocate) waited too long for the "
"wwn information on image {} to "
"arrive".format(self.image))
return
# At this point we have a wwn from the config for this rbd
# image, so just add to LIO
self.add_dev_to_lio(wwn)
if self.error:
return
self.logger.info("(LUN.allocate) added {} to LIO using wwn "
"'{}' defined by {}".format(
self.image, wwn, self.allocating_host))
self.num_changes += 1
else:
# lun exists in LIO, check the size is correct
if not self.lio_size_ok(rbd_image, so):
self.error = True
self.error_msg = "Unable to sync the rbd device size with LIO"
self.logger.critical(self.error_msg)
return
self.logger.debug("config meta data for this disk is "
"{}".format(
self.config.config['disks'][self.config_key]))
# the owning host for an image is the only host that commits to the
# config
if this_host == self.allocating_host and self.config.changed:
self.logger.debug("(LUN.allocate) Committing change(s) to the "
"config object in pool {}".format(self.pool))
self.config.commit()
self.error = self.config.error
self.error_msg = self.config.error_msg
def lio_size_ok(self, rbd_object, stg_object):
"""
Check that the SO in LIO matches the current size of the rbd. if the
size requested < current size, just return. Downsizing an rbd is not
supported by this code and problematic for client filesystems anyway!
:return boolean indicating whether the size matches
"""
# Ignore a target size that is less than the current rbd size
if self.size_bytes < rbd_object.size_bytes:
return True
tmr = 0
size_ok = False
rbd_size_ok = False
# dm_path_found = False
# We have to wait for the rbd size to match, since the rbd could have
# been resized on another gateway host
while tmr < settings.config.time_out:
if rbd_object.size_bytes == self.size_bytes:
rbd_size_ok = True
break
sleep(settings.config.loop_delay)
tmr += settings.config.loop_delay
# we have the right size for the rbd - check that LIO dev size matches
if rbd_size_ok:
# If the LIO size is not right, poke it with the new value
if stg_object.size < self.size_bytes:
self.logger.info("Resizing {} in LIO "
"to {}".format(self.config_key,
self.size_bytes))
stg_object._control("dev_size={}".format(self.size_bytes))
size_ok = stg_object.size == self.size_bytes
else:
size_ok = True
return size_ok
def lio_stg_object(self):
found_it = False
rtsroot = root.RTSRoot()
for stg_object in rtsroot.storage_objects:
# First match on name, but then check the pool incase the same
# name exists in multiple pools
if stg_object.name == self.config_key:
found_it = True
break
return stg_object if found_it else None
def add_dev_to_lio(self, in_wwn=None):
"""
Add an rbd device to the LIO configuration
:param in_wwn: optional wwn identifying the rbd image to clients
(must match across gateways)
:return: LIO LUN object
"""
self.logger.info("(LUN.add_dev_to_lio) Adding image "
"'{}' to LIO".format(self.config_key))
new_lun = None
try:
# config string = rbd identifier / config_key (pool/image) /
# optional osd timeout
cfgstring = "rbd/{}/{};osd_op_timeout={}".format(
self.pool, self.image, settings.config.osd_op_timeout)
new_lun = UserBackedStorageObject(name=self.config_key,
config=cfgstring,
size=self.size_bytes,
wwn=in_wwn)
except RTSLibError as err:
self.error = True
self.error_msg = ("failed to add {} to LIO - "
"error({})".format(self.config_key, str(err)))
self.logger.error(self.error_msg)
return None
try:
new_lun.set_attribute("cmd_time_out", 0)
new_lun.set_attribute("qfull_time_out",
settings.config.qfull_timeout)
except RTSLibError as err:
self.error = True
self.error_msg = ("Could not set LIO device attribute "
"cmd_time_out/qfull_time_out for device: {}. "
"Kernel not supported. - "
"error({})".format(self.config_key, str(err)))
self.logger.error(self.error_msg)
new_lun.delete()
return None
self.logger.info("(LUN.add_dev_to_lio) Successfully added {}"
" to LIO".format(self.config_key))
return new_lun
def remove_dev_from_lio(self):
lio_root = root.RTSRoot()
# remove the device from all tpgs
for t in lio_root.tpgs:
for lun in t.luns:
if lun.storage_object.name == self.config_key:
try:
lun.delete()
except RTSLibError as e:
self.error = True
self.error_msg = ("Delete from LIO/TPG failed - "
"{}".format(e))
return
else:
break # continue to the next tpg
for stg_object in lio_root.storage_objects:
if stg_object.name == self.config_key:
# alua_dir = os.path.join(stg_object.path, "alua")
# # remove the alua directories (future versions will handle this
# # natively within rtslib_fb
# for dirname in next(os.walk(alua_dir))[1]:
# if dirname != "default_tg_pt_gp":
# try:
# alua_tpg = ALUATargetPortGroup(stg_object, dirname)
# alua_tpg.delete()
# except (RTSLibError, RTSLibNotInCFS) as err:
# self.error = True
# self.error_msg = ("Delete of ALUA dirs failed - "
# "{}".format(err))
# return
try:
stg_object.delete()
except RTSLibError as e:
self.error = True
self.error_msg = ("Delete from LIO/backstores failed - "
"{}".format(e))
return
break
@staticmethod
def set_owner(gateways):
"""
Determine the gateway in the configuration with the lowest number of
active LUNs. This gateway is then selected as the owner for the
primary path of the current LUN being processed
:param gateways: gateway dict returned from the RADOS configuration
object
:return: specific gateway hostname (str) that should provide the
active path for the next LUN
"""
# Gateways contains simple attributes and dicts. The dicts define the
# gateways settings, so first we extract only the dicts within the
# main gateways dict
gw_nodes = {
key: gateways[key]
for key in gateways if isinstance(gateways[key], dict)
}
gw_items = gw_nodes.items()
# first entry is the lowest number of active_luns
gw_items.sort(key=lambda x: (x[1]['active_luns']))
# 1st tuple is gw with lowest active_luns, so return the 1st
# element which is the hostname
return gw_items[0][0]
