def rbd_lock_cleanup(logger, local_ips, rbd_image): """ cleanup locks left if this node crashed and was not able to release them :param logger: logger object to print to :param local_ips: list of local ip addresses. :rbd_image: rbd image to clean up locking for :raise CephiSCSIError. """ lock_info = rbd_image.list_lockers() if not lock_info: return lockers = lock_info.get("lockers") for holder in lockers: for ip in local_ips: if ip in holder[2]: logger.info( "Cleaning up stale local lock for {} {}".format( holder[0], holder[1])) try: rbd_image.break_lock(holder[0], holder[1]) except Exception as err: raise CephiSCSIError("Could not break lock for {}. " "Error {}".format(rbd_image, err))
def add_dev_to_lio(self, in_wwn=None): """ Add an rbd device to the LIO configuration :param in_wwn: optional wwn identifying the rbd image to clients (must match across gateways) :return: LIO LUN object """ self.logger.info("(LUN.add_dev_to_lio) Adding image " "'{}' to LIO backstore {}".format( self.config_key, self.backstore)) new_lun = None if self.backstore == USER_RBD: new_lun = self._add_dev_to_lio_user_rbd(in_wwn) else: raise CephiSCSIError("Error adding device to lio - " "Unsupported backstore {}".format( self.backstore)) if new_lun: self.logger.info("(LUN.add_dev_to_lio) Successfully added {}" " to LIO".format(self.config_key)) return new_lun
def define_clients(logger, config): """ define the clients (nodeACLs) to the gateway definition :param logger: logger object to print to :param config: configuration dict from the rados pool :raises CephiSCSIError. """ # Client configurations (NodeACL's) for client_iqn in config.config['clients']: client_metadata = config.config['clients'][client_iqn] client_chap = CHAP(client_metadata['auth']['chap']) image_list = client_metadata['luns'].keys() chap_str = client_chap.chap_str if client_chap.error: raise CephiSCSIError("Unable to decode password for {}. " "CHAP error: {}".format(client_iqn, client_chap.error_msg)) client = GWClient(logger, client_iqn, image_list, chap_str) client.manage('present') # ensure the client exists
def deactivate(self): so = self.lio_stg_object() if not so: # Could be due to a restart after failure. Just log and ignore. self.logger.warning("LUN {} already deactivated".format(self.image)) return for alun in so.attached_luns: for mlun in alun.mapped_luns: node_acl = mlun.parent_nodeacl if node_acl.session and \ node_acl.session.get('state', '').upper() == 'LOGGED_IN': raise CephiSCSIError("LUN {} in-use".format(self.image)) self.remove_dev_from_lio() if self.error: raise CephiSCSIError("LUN deactivate failure - {}".format(self.error_msg))
def delete_target(self, target_iqn): target = GWTarget(self.logger, target_iqn, {}) if target.error: raise CephiSCSIError("Could not initialize target: {}". format(target.error_msg)) target.load_config() if target.error: self.logger.debug("Could not find target {}: {}". format(target_iqn, target.error_msg)) # Target might not be setup on this node. Ignore. return try: target.delete(self.config) except RTSLibError as err: err_msg = "Could not remove target {}: {}".format(target_iqn, err) raise CephiSCSIError(err_msg)
def map_lun(self, gateway, owner, disk): target_config = self.config.config['targets'][gateway.iqn] disk_metadata = self.config.config['disks'][disk] disk_metadata['owner'] = owner self.config.update_item("disks", disk, disk_metadata) target_config['disks'].append(disk) self.config.update_item("targets", gateway.iqn, target_config) gateway_dict = self.config.config['gateways'][owner] gateway_dict['active_luns'] += 1 self.config.update_item('gateways', owner, gateway_dict) so = self.allocate() if self.error: raise CephiSCSIError(self.error_msg) gateway.map_lun(self.config, so) if gateway.error: raise CephiSCSIError(gateway.error_msg)
def define_clients(logger, config, target_iqn): """ define the clients (nodeACLs) to the gateway definition :param logger: logger object to print to :param config: configuration dict from the rados pool :raises CephiSCSIError. """ # Client configurations (NodeACL's) target_config = config.config['targets'][target_iqn] for client_iqn in target_config['clients']: client_metadata = target_config['clients'][client_iqn] client_chap = CHAP(client_metadata['auth']['username'], client_metadata['auth']['password'], client_metadata['auth']['password_encryption_enabled']) client_chap_mutual = CHAP(client_metadata['auth']['mutual_username'], client_metadata['auth']['mutual_password'], client_metadata['auth'][ 'mutual_password_encryption_enabled']) image_list = list(client_metadata['luns'].keys()) if client_chap.error: raise CephiSCSIError("Unable to decode password for {}. " "CHAP error: {}".format(client_iqn, client_chap.error_msg)) if client_chap_mutual.error: raise CephiSCSIError("Unable to decode password for {}. " "CHAP_MUTUAL error: {}".format(client_iqn, client_chap.error_msg)) client = GWClient(logger, client_iqn, image_list, client_chap.user, client_chap.password, client_chap_mutual.user, client_chap_mutual.password, target_iqn) client.manage('present') # ensure the client exists
def __init__(self, cfg_type, cfg_type_key, logger, control_settings): self.cfg_type = cfg_type self.cfg_type_key = cfg_type_key self.logger = logger self.config = Config(self.logger) if self.config.error: raise CephiSCSIError(self.config.error_msg) # Copy of controls that will not be written until commit is called. # To update the kernel call the child object's update function. self.controls = self._get_config_controls().copy() self._add_properies(control_settings)
def remove_from_config(self, target_iqn): has_changed = False target_config = self.config.config["targets"].get(target_iqn) if target_config: local_gw = target_config['portals'].get(self.hostname) if local_gw: local_gw_ips = local_gw['portal_ip_addresses'] target_config['portals'].pop(self.hostname) ip_list = target_config['ip_list'] for local_gw_ip in local_gw_ips: ip_list.remove(local_gw_ip) for _, remote_gw_config in target_config['portals'].items(): for local_gw_ip in local_gw_ips: remote_gw_config["gateway_ip_list"].remove(local_gw_ip) remote_gw_config["inactive_portal_ips"].remove( local_gw_ip) tpg_count = remote_gw_config["tpgs"] remote_gw_config["tpgs"] = tpg_count - 1 if not target_config['portals']: # Last gw for the target so delete everything that lives # under the tpg in LIO since we can't create it target_config['disks'] = {} target_config['clients'] = {} target_config['controls'] = {} target_config['groups'] = {} has_changed = True self.config.update_item('targets', target_iqn, target_config) remove_gateway = True for _, target in self.config.config["targets"].items(): if self.hostname in target['portals']: remove_gateway = False break if remove_gateway: # gateway is no longer used, so delete it has_changed = True self.config.del_item('gateways', self.hostname) LUN.reassign_owners(self.logger, self.config) if has_changed: self.config.commit("retain") if self.config.error: raise CephiSCSIError(self.config.error_msg)
def map_lun(self, gateway, owner, disk): target_config = self.config.config['targets'][gateway.iqn] disk_metadata = self.config.config['disks'][disk] disk_metadata['owner'] = owner self.config.update_item("disks", disk, disk_metadata) target_disk_config = target_config['disks'].get(disk) if not target_disk_config: lun_id = self._get_next_lun_id(target_config['disks']) target_config['disks'][disk] = {'lun_id': lun_id} self.config.update_item("targets", gateway.iqn, target_config) gateway_dict = self.config.config['gateways'][owner] gateway_dict['active_luns'] += 1 self.config.update_item('gateways', owner, gateway_dict) so = self.allocate() if self.error: raise CephiSCSIError(self.error_msg) gateway.map_lun(self.config, so, target_config['disks'][disk]) if gateway.error: raise CephiSCSIError(gateway.error_msg)
def commit_controls(self): committed_controls = self._get_config_controls() if self.controls != committed_controls: # update our config self._set_config_controls(self.config, self.controls) updated_obj = self.config.config[self.cfg_type][self.cfg_type_key] self.config.update_item(self.cfg_type, self.cfg_type_key, updated_obj) self.config.commit() if self.config.error: raise CephiSCSIError(self.config.error_msg)
def reassign_owners(logger, config): """ Reassign disks across gateways after a gw deletion. :param logger: logger object to print to :param config: configuration dict from the rados pool :raises CephiSCSIError. """ updated = False gateways = config.config['gateways'] for disk, disk_config in config.config['disks'].items(): owner = disk_config.get('owner', None) if owner is None: continue gw = gateways.get(owner, None) if gw is None: target = LUN.find_first_mapped_target(config, disk) if not gateways or target is None: disk_config.pop('owner') else: target_config = config.config['targets'][target] new_owner = LUN.get_owner(gateways, target_config['portals']) logger.info("Changing {}'s owner from {} to {}".format( disk, owner, new_owner)) disk_config['owner'] = new_owner gw_config = config.config['gateways'][new_owner] active_cnt = gw_config['active_luns'] gw_config['active_luns'] = active_cnt + 1 config.update_item("gateways", new_owner, gw_config) config.update_item("disks", disk, disk_config) updated = True if updated: config.commit("retain") if config.error: raise CephiSCSIError("Could not update LUN owners: {}".format( config.error_msg))
def get_remote_gateways(config, logger, local_gw_required=True): ''' Return the list of remote gws. :param: config: Config object with gws setup. :param: logger: Logger object :param: local_gw_required: Check if local_gw is defined within gateways configuration :return: A list of gw names, or CephiSCSIError if not run on a gw in the config ''' local_gw = this_host() logger.debug("this host is {}".format(local_gw)) gateways = [key for key in config if isinstance(config[key], dict)] logger.debug("all gateways - {}".format(gateways)) if local_gw_required and local_gw not in gateways: raise CephiSCSIError("{} cannot be used to perform this operation " "because it is not defined within the gateways " "configuration".format(local_gw)) if local_gw in gateways: gateways.remove(local_gw) logger.debug("remote gateways: {}".format(gateways)) return gateways
def delete_targets(self): err_msg = None if self.hostname not in self.config.config['gateways']: return # Clear the current config, based on the config objects settings. # This will fail incoming IO, but wait on outstanding IO to # complete normally. We rely on the initiator multipath layer # to handle retries like a normal path failure. self.logger.info("Removing iSCSI target from LIO") for target_iqn, target_config in self.config.config['targets'].items(): try: self.delete_target(target_iqn) except CephiSCSIError as err: if err_msg is None: err_msg = err continue if err_msg: raise CephiSCSIError(err_msg)
def lookup_storage_object(name, backstore): if backstore == USER_RBD: return UserBackedStorageObject(name=name) else: raise CephiSCSIError("Could not lookup storage object - " "Unsupported backstore {}".format(backstore))
def define_target(self, target_iqn, gw_ip_list, target_only=False): """ define the iSCSI target and tpgs :param target_iqn: (str) target iqn :param gw_ip_list: (list) gateway ip list :param target_only: (bool) if True only setup target :return: (object) GWTarget object """ # GWTarget Definition : Handle the creation of the Target/TPG(s) and # Portals. Although we create the tpgs, we flick the enable_portal flag # off so the enabled tpg will not have an outside IP address. This # prevents clients from logging in too early, failing and giving up # because the nodeACL hasn't been defined yet (yes Windows I'm looking # at you!) # first check if there are tpgs already in LIO (True) - this would # indicate a restart or reload call has been made. If the tpg count is # 0, this is a boot time request self.logger.info("Setting up {}".format(target_iqn)) target = GWTarget(self.logger, target_iqn, gw_ip_list, enable_portal=self.portals_active(target_iqn)) if target.error: raise CephiSCSIError("Error initializing iSCSI target: " "{}".format(target.error_msg)) target.manage('target') if target.error: raise CephiSCSIError("Error creating the iSCSI target (target, " "TPGs, Portals): {}".format(target.error_msg)) if not target_only: self.logger.info("Processing LUN configuration") try: LUN.define_luns(self.logger, self.config, target) except CephiSCSIError as err: self.logger.error("{} - Could not define LUNs: " "{}".format(target.iqn, err)) raise self.logger.info("{} - Processing client configuration". format(target.iqn)) try: GWClient.define_clients(self.logger, self.config, target.iqn) except CephiSCSIError as err: self.logger.error("Could not define clients: {}".format(err)) raise if not target.enable_portal: # The tpgs, luns and clients are all defined, but the active tpg # doesn't have an IP bound to it yet (due to the # enable_portals=False setting above) self.logger.info("{} - Adding the IP to the enabled tpg, " "allowing iSCSI logins".format(target.iqn)) target.enable_active_tpg(self.config) if target.error: raise CephiSCSIError("{} - Error enabling the IP with the " "active TPG: {}".format(target.iqn, target.error_msg)) return target
def activate(self): disk = self.config.config['disks'].get(self.config_key, None) if not disk: raise CephiSCSIError("Image {} not found.".format(self.image)) wwn = disk.get('wwn', None) if not wwn: raise CephiSCSIError("LUN {} missing wwn".format(self.image)) # re-add backend storage object so = self.lio_stg_object() if not so: self.add_dev_to_lio(wwn) if self.error: raise CephiSCSIError("LUN activate failure - {}".format( self.error_msg)) # re-add LUN to target local_gw = this_host() targets_items = [ item for item in self.config.config['targets'].items() if self.config_key in item[1]['disks'] and local_gw in item[1]['portals'] ] for target_iqn, target in targets_items: ip_list = target['ip_list'] # Add the mapping for the lun to ensure the block device is # present on all TPG's gateway = GWTarget(self.logger, target_iqn, ip_list) gateway.manage('map') if gateway.error: raise CephiSCSIError("LUN mapping failed - {}".format( gateway.error_msg)) # re-map LUN to hosts client_err = '' for client_iqn in target['clients']: client_metadata = target['clients'][client_iqn] if client_metadata.get('group_name', ''): continue image_list = list(client_metadata['luns'].keys()) if self.config_key not in image_list: continue client_chap = CHAP(client_metadata['auth']['chap']) chap_str = client_chap.chap_str if client_chap.error: raise CephiSCSIError("Password decode issue : " "{}".format(client_chap.error_msg)) client_chap_mutual = CHAP( client_metadata['auth']['chap_mutual']) chap_mutual_str = client_chap_mutual.chap_str if client_chap_mutual.error: raise CephiSCSIError("Password decode issue : " "{}".format( client_chap_mutual.error_msg)) client = GWClient(self.logger, client_iqn, image_list, chap_str, chap_mutual_str, target_iqn) client.manage('present') if client.error: client_err = "LUN mapping failed {} - {}".format( client_iqn, client.error_msg) # re-map LUN to host groups for group_name in target['groups']: host_group = target['groups'][group_name] members = host_group.get('members') disks = host_group.get('disks').keys() if self.config_key not in disks: continue group = Group(self.logger, target_iqn, group_name, members, disks) group.apply() if group.error: client_err = "LUN mapping failed {} - {}".format( group_name, group.error_msg) if client_err: raise CephiSCSIError(client_err)
def define_luns(logger, config, gateway): """ define the disks in the config to LIO :param logger: logger object to print to :param config: configuration dict from the rados pool :param gateway: (object) gateway object - used for mapping :raises CephiSCSIError. """ local_gw = this_host() # sort the disks dict keys, so the disks are registered in a specific # sequence disks = config.config['disks'] srtd_disks = sorted(disks) pools = {disks[disk_key]['pool'] for disk_key in srtd_disks} if pools is None: logger.info("No LUNs to export") return True ips = ip_addresses() with rados.Rados(conffile=settings.config.cephconf) as cluster: for pool in pools: logger.debug("Processing rbd's in '{}' pool".format(pool)) with cluster.open_ioctx(pool) as ioctx: pool_disks = [ disk_key for disk_key in srtd_disks if disk_key.startswith(pool + '.') ] for disk_key in pool_disks: is_lun_mapped = False for _, target_config in config.config['targets'].items( ): if local_gw in target_config['portals'] \ and disk_key in target_config['disks']: is_lun_mapped = True break if is_lun_mapped: pool, image_name = disk_key.split('.') try: with rbd.Image(ioctx, image_name) as rbd_image: RBDDev.rbd_lock_cleanup( logger, ips, rbd_image) backstore = config.config['disks'][ disk_key]['backstore'] lun = LUN(logger, pool, image_name, rbd_image.size(), local_gw, backstore) if lun.error: raise CephiSCSIError( "Error defining rbd " "image {}".format(disk_key)) lun.allocate() if lun.error: raise CephiSCSIError( "Error unable to " "register {} with " "LIO - {}".format( disk_key, lun.error_msg)) except rbd.ImageNotFound: raise CephiSCSIError( "Disk '{}' defined to the " "config, but image '{}' can " "not be found in '{}' " "pool".format(disk_key, image_name, pool)) if gateway: # Gateway Mapping : Map the LUN's registered to all tpg's within the # LIO target gateway.manage('map') if gateway.error: raise CephiSCSIError( "Error mapping the LUNs to the tpg's within " "the iscsi Target")
def define_luns(logger, config, target): """ define the disks in the config to LIO and map to a LUN :param logger: logger object to print to :param config: configuration dict from the rados pool :param target: (object) gateway object - used for mapping :raises CephiSCSIError. """ ips = ip_addresses() local_gw = this_host() target_disks = config.config["targets"][target.iqn]['disks'] if not target_disks: logger.info("No LUNs to export") return disks = {} for disk in target_disks: disks[disk] = config.config['disks'][disk] # sort the disks dict keys, so the disks are registered in a specific # sequence srtd_disks = sorted(disks) pools = {disks[disk_key]['pool'] for disk_key in srtd_disks} ips = ip_addresses() with rados.Rados(conffile=settings.config.cephconf, name=settings.config.cluster_client_name) as cluster: for pool in pools: logger.debug("Processing rbd's in '{}' pool".format(pool)) with cluster.open_ioctx(pool) as ioctx: pool_disks = [ disk_key for disk_key in srtd_disks if disk_key.startswith(pool + '/') ] for disk_key in pool_disks: pool, image_name = disk_key.split('/') with rbd.Image(ioctx, image_name) as rbd_image: disk_config = config.config['disks'][disk_key] backstore = disk_config['backstore'] backstore_object_name = disk_config[ 'backstore_object_name'] lun = LUN(logger, pool, image_name, rbd_image.size(), local_gw, backstore, backstore_object_name) if lun.error: raise CephiSCSIError( "Error defining rbd image {}".format( disk_key)) so = lun.allocate() if lun.error: raise CephiSCSIError("Unable to register {} " "with LIO: {}".format( disk_key, lun.error_msg)) # If not in use by another target on this gw # clean up stale locks. if so.status != 'activated': RBDDev.rbd_lock_cleanup(logger, ips, rbd_image) target._map_lun(config, so) if target.error: raise CephiSCSIError( "Mapping for {} failed: {}".format( disk_key, target.error_msg))
def commit_controls(self): self.update_controls() self.config.commit() if self.config.error: raise CephiSCSIError(self.config.error_msg)