def cache_flush(self, pool): backing_pool = pool['pool_name'] cache_pool = backing_pool + '-cache' try: # set target_max_objects to a small value to force evacuation of # objects from cache before we use rados cache-flush-evict-all # WARNING: assuming cache_pool will be deleted after flush so # we don't have to save/restore the value of target_max_objects # self.cache_pool_set_param(pool, 'target_max_objects', 1) prev_object_count = None wait_interval = MIN_WAIT while True: response, body = self.service.ceph_api.df(body='json') if not response.ok: LOG.warn( _LW("Failed to retrieve cluster free space stats: " "status_code=%d, reason=%s") % (response.status_code, response.reason)) break stats = None for s in body['output']['pools']: if s['name'] == cache_pool: stats = s['stats'] break if not stats: LOG.warn( _LW("Missing pool free space stats: " "cache_pool=%s") % cache_pool) break object_count = stats['objects'] if object_count < constants.CACHE_FLUSH_OBJECTS_THRESHOLD: break if prev_object_count is not None: delta_objects = object_count - prev_object_count if delta_objects > 0: LOG.warn( _LW("Unexpected increase in number " "of objects in cache pool: " "cache_pool=%s, prev_object_count=%d, " "object_count=%d") % (cache_pool, prev_object_count, object_count)) break if delta_objects == 0: wait_interval *= 2 if wait_interval > MAX_WAIT: LOG.warn( _LW("Cache pool number of objects did not " "decrease: cache_pool=%s, object_count=%d, " "wait_interval=%d") % (cache_pool, object_count, wait_interval)) break else: wait_interval = MIN_WAIT time.sleep(wait_interval) prev_object_count = object_count except exception.CephPoolSetParamFailure as e: LOG.warn(e) finally: self.rados_cache_flush_evict_all(pool)
def _get_health(self): try: # we use text since it has all info response, body = self.service.ceph_api.health(body='text', timeout=30) except IOError as e: LOG.warning(_LW("ceph_api.health failed: %s") % str(e.message)) self.cluster_is_up = False return { 'health': constants.CEPH_HEALTH_DOWN, 'detail': 'Ceph cluster is down.' } if not response.ok: LOG.warning(_LW("CEPH health check failed: %s") % response.reason) health_info = [constants.CEPH_HEALTH_DOWN, response.reason] self.cluster_is_up = False else: health_info = body.split(' ', 1) self.cluster_is_up = True health = health_info[0] if len(health_info) > 1: detail = health_info[1] else: detail = health_info[0] return {'health': health.strip(), 'detail': detail.strip()}
def _get_fsid(self): try: response, fsid = self.service.ceph_api.fsid(body='text', timeout=30) except IOError as e: LOG.warning(_LW("ceph_api.fsid failed: %s") % str(e.message)) self.cluster_is_up = False return None if not response.ok: LOG.warning(_LW("Get fsid failed: %s") % response.reason) self.cluster_is_up = False return None self.cluster_is_up = True return fsid.strip()
def auto_heal(self, health): if (health['health'] == constants.CEPH_HEALTH_WARN and (constants.CEPH_HEALTH_WARN_REQUIRE_JEWEL_OSDS_NOT_SET in health['detail'])): try: upgrade = self.service.get_software_upgrade_status() except Exception as ex: LOG.warn( _LW("Getting software upgrade status failed " "with: %s. Skip auto-heal attempt " "(will retry on next ceph status poll).") % str(ex)) return state = upgrade.get('state') # surpress require_jewel_osds in case upgrade is # in progress but not completed or aborting if (not self.surpress_require_jewel_osds_warning and (upgrade.get('from_version') == constants.TITANIUM_SERVER_VERSION_16_10) and state not in [ None, constants.UPGRADE_COMPLETED, constants.UPGRADE_ABORTING, constants.UPGRADE_ABORT_COMPLETING, constants.UPGRADE_ABORTING_ROLLBACK ]): LOG.info(_LI("Surpress require_jewel_osds health warning")) self.surpress_require_jewel_osds_warning = True # set require_jewel_osds in case upgrade is # not in progress or completed if (state in [None, constants.UPGRADE_COMPLETED]): LOG.warn( _LW("No upgrade in progress or update completed " "and require_jewel_osds health warning raised. " "Set require_jewel_osds flag.")) self.set_flag_require_jewel_osds() health = self._remove_require_jewel_osds_warning(health) LOG.info(_LI("Unsurpress require_jewel_osds health warning")) self.surpress_require_jewel_osds_warning = False # unsurpress require_jewel_osds in case upgrade # is aborting if (self.surpress_require_jewel_osds_warning and state in [ constants.UPGRADE_ABORTING, constants.UPGRADE_ABORT_COMPLETING, constants.UPGRADE_ABORTING_ROLLBACK ]): LOG.info(_LI("Unsurpress require_jewel_osds health warning")) self.surpress_require_jewel_osds_warning = False return health
def _select_ds_for_volume(self, dc_moid, cluster_moid, ds_moid, host_moid=None, folders=None): """Select datastore that can accommodate the given volume's backing. Returns the selected datastore summary along with a compute host and its resource pool and folder where the volume can be created :return: (host, resource_pool, folder, summary) """ dc_ref = utils.get_datacenter_moref(self._content, moid=dc_moid) if not dc_ref: LOG.error(_LE("No valid datacenter is available.")) raise exception.NoValidDatacenter() resource_pool = None host_ref = None cluster_ref = utils.get_child_ref_by_moid(dc_ref.hostFolder, cluster_moid) if not cluster_ref: LOG.warn(_LW("No valid cluster is available.")) host_ref = utils.get_child_ref_by_moid(dc_ref.hostFolder, host_moid) if not host_ref: LOG.error(_LE("No valid host is available.")) raise exception.NoValidHost() else: resource_pool = cluster_ref.resourcePool host_ref = utils.get_ref_from_array_by_moid(cluster_ref.host, host_moid) if not host_ref: LOG.warn(_LW("No valid host is specified.")) ds_ref = utils.get_ref_from_array_by_moid(cluster_ref.datastore, ds_moid) if not ds_ref: LOG.error(_LE("No valid datastore is available.")) raise exception.NoValidDatastore() folder_ref = self._get_volume_group_folder(dc_ref, folders) return (resource_pool, host_ref, ds_ref, folder_ref)
def __init__(self, message=None, **kwargs): self.kwargs = kwargs if not message: try: message = self.message % kwargs except TypeError: LOG.warn(_LW('Exception in string format operation')) for name, value in kwargs.iteritems(): LOG.error("%s: %s" % (name, value)) # at least get the core message out if something happened message = self.message super(CephManagerException, self).__init__(message)
def volumedriver_remove(self, name): """ Remove a Docker volume. :param unicode name: The name of the volume. :return: Result indicating success. """ contents = json.loads(name.content.getvalue()) volname = contents['Name'] # Only 1 node in a multinode cluster can try to remove the volume. # Grab lock for volume name. If lock is inuse, just return with no # error. self._lock_volume(volname, 'Remove') vol = self._etcd.get_vol_byname(volname) if vol is None: # Just log an error, but don't fail the docker rm command msg = (_LE('Volume remove name not found %s'), volname) LOG.error(msg) self._unlock_volume(volname) return json.dumps({u"Err": ''}) try: self.hpeplugin_driver.delete_volume(vol) LOG.info(_LI('volume: %(name)s,' 'was successfully deleted'), {'name': volname}) except Exception as ex: msg = (_LE('Err: Failed to remove volume %s, error is %s'), volname, six.text_type(ex)) LOG.error(msg) self._unlock_volume(volname) raise exception.HPEPluginRemoveException(reason=msg) try: self._etcd.delete_vol(vol) except KeyError: msg = (_LW('Warning: Failed to delete volume key: %s from ' 'etcd due to KeyError'), volname) LOG.warning(msg) pass self._unlock_volume(volname) return json.dumps({u"Err": ''})
class SysinvConductorUpgradeApi(object): def __init__(self): self.sysinv_conductor = None super(SysinvConductorUpgradeApi, self).__init__() def get_software_upgrade_status(self): LOG.info(_LI("Getting software upgrade status from sysinv")) cctxt = self.sysinv_conductor.prepare(timeout=2) upgrade = cctxt.call({}, 'get_software_upgrade_status') LOG.info(_LI("Software upgrade status: %s") % str(upgrade)) return upgrade @retry(wait_fixed=1000, retry_on_exception=lambda exception: LOG.warn( _LW("Getting software upgrade status failed " "with: %s. Retrying... ") % str(exception)) or True) def retry_get_software_upgrade_status(self): return self.get_software_upgrade_status()
def update_cache_target_max_bytes(self): "Dynamically compute target_max_bytes of caching pools" # Only compute if cache tiering is enabled if self.config_applied and self.config_desired: if (not self.config_desired.cache_enabled or not self.config_applied.cache_enabled): LOG.debug("Cache tiering disabled, no need to update " "target_max_bytes.") return LOG.debug("Updating target_max_bytes") # Get available space response, body = self.service.ceph_api.osd_df(body='json', output_method='tree') if not response.ok: LOG.warn( _LW("Failed to retrieve cluster free space stats: " "status_code=%d, reason=%s") % (response.status_code, response.reason)) return storage_tier_size = 0 cache_tier_size = 0 replication = constants.CEPH_REPLICATION_FACTOR for node in body['output']['nodes']: if node['name'] == 'storage-tier': storage_tier_size = node['kb'] * 1024 / replication elif node['name'] == 'cache-tier': cache_tier_size = node['kb'] * 1024 / replication if storage_tier_size == 0 or cache_tier_size == 0: LOG.info("Failed to get cluster size " "(storage_tier_size=%s, cache_tier_size=%s)," "retrying on next cycle" % (storage_tier_size, cache_tier_size)) return # Get available pools response, body = self.service.ceph_api.osd_lspools(body='json') if not response.ok: LOG.warn( _LW("Failed to retrieve available pools: " "status_code=%d, reason=%s") % (response.status_code, response.reason)) return pools = [p['poolname'] for p in body['output']] # Separate backing from caching for easy iteration backing_pools = [] caching_pools = [] for p in pools: if p.endswith('-cache'): caching_pools.append(p) else: backing_pools.append(p) LOG.debug("Pools: caching: %s, backing: %s" % (caching_pools, backing_pools)) if not len(caching_pools): # We do not have caching pools created yet return # Get quota from backing pools that are cached stats = {} for p in caching_pools: backing_name = p.replace('-cache', '') stats[backing_name] = {} try: quota = ceph.osd_pool_get_quota(self.service.ceph_api, backing_name) except exception.CephPoolGetQuotaFailure as e: LOG.warn( _LW("Failed to retrieve quota: " "exception: %s") % str(e)) return stats[backing_name]['quota'] = quota['max_bytes'] stats[backing_name]['quota_pt'] = (quota['max_bytes'] * 100.0 / storage_tier_size) LOG.debug("Quota for pool: %s " "is: %s B representing %s pt" % (backing_name, quota['max_bytes'], stats[backing_name]['quota_pt'])) # target_max_bytes logic: # - For computing target_max_bytes cache_tier_size must be equal than # the sum of target_max_bytes of each caching pool # - target_max_bytes for each caching pool is computed as the # percentage of quota in corresponding backing pool # - the caching tiers has to work at full capacity, so if the sum of # all quotas in the backing tier is different than 100% we need to # normalize # - if the quota is zero for any pool we add CACHE_TIERING_MIN_QUOTA # by default *after* normalization so that we have real minimum # We compute the real percentage that need to be normalized after # ensuring that we have CACHE_TIERING_MIN_QUOTA for each pool with # a quota of 0 real_100pt = 90.0 # we start from max and decrease it for each 0 pool # Note: We must avoid reaching 100% at all costs! and # cache_target_full_ratio, the Ceph parameter that is supposed to # protect the cluster against this does not work in Ceph v0.94.6! # Therefore a value of 90% is better suited for this for p in caching_pools: backing_name = p.replace('-cache', '') if stats[backing_name]['quota_pt'] == 0: real_100pt -= constants.CACHE_TIERING_MIN_QUOTA LOG.debug("Quota before normalization for %s is: %s pt" % (p, stats[backing_name]['quota_pt'])) # Compute total percentage of quotas for all backing pools. # Should be 100% if correctly configured total_quota_pt = 0 for p in caching_pools: backing_name = p.replace('-cache', '') total_quota_pt += stats[backing_name]['quota_pt'] LOG.debug("Total quota pt is: %s" % total_quota_pt) # Normalize quota pt to 100% (or real_100pt) if total_quota_pt != 0: # to avoid divide by zero for p in caching_pools: backing_name = p.replace('-cache', '') stats[backing_name]['quota_pt'] = \ (stats[backing_name]['quota_pt'] * (real_100pt / total_quota_pt)) # Do not allow quota to be 0 for any pool total = 0 for p in caching_pools: backing_name = p.replace('-cache', '') if stats[backing_name]['quota_pt'] == 0: stats[backing_name]['quota_pt'] = \ constants.CACHE_TIERING_MIN_QUOTA total += stats[backing_name]['quota_pt'] LOG.debug("Quota after normalization for %s is: %s:" % (p, stats[backing_name]['quota_pt'])) if total > 100: # Supplementary protection, we really have to avoid going above # 100%. Note that real_100pt is less than 100% but we still got # more than 100! LOG.warn("Total sum of quotas should not go above 100% " "but is: %s, recalculating in next cycle" % total) return LOG.debug("Total sum of quotas is %s pt" % total) # Get current target_max_bytes. We cache it to reduce requests # to ceph-rest-api. We are the ones changing it, so not an issue. for p in caching_pools: if p not in self.target_max_bytes: try: value = ceph.osd_get_pool_param(self.service.ceph_api, p, constants.TARGET_MAX_BYTES) except exception.CephPoolGetParamFailure as e: LOG.warn(e) return self.target_max_bytes[p] = value LOG.debug("Existing target_max_bytes got from " "Ceph: %s" % self.target_max_bytes) # Set TARGET_MAX_BYTES LOG.debug("storage_tier_size: %s " "cache_tier_size: %s" % (storage_tier_size, cache_tier_size)) for p in caching_pools: backing_name = p.replace('-cache', '') s = stats[backing_name] target_max_bytes = math.floor(s['quota_pt'] * cache_tier_size / 100.0) target_max_bytes = int(target_max_bytes) LOG.debug("New Target max bytes of pool: %s is: %s B" % (p, target_max_bytes)) # Set the new target_max_bytes only if it changed if self.target_max_bytes.get(p) == target_max_bytes: LOG.debug("Target max bytes of pool: %s " "is already updated" % p) continue try: ceph.osd_set_pool_param(self.service.ceph_api, p, constants.TARGET_MAX_BYTES, target_max_bytes) self.target_max_bytes[p] = target_max_bytes except exception.CephPoolSetParamFailure as e: LOG.warn(e) continue return
def do_enable_cache(self, new_config, applied_config, lock_ownership): LOG.info( _LI("cache_tiering_enable_cache: " "new_config={}, applied_config={}").format( new_config.to_dict(), applied_config.to_dict())) _unwind_actions = [] with lock_ownership(): success = False _exception = None try: self.config_desired.cache_enabled = True self.update_pools_info() for pool in CEPH_POOLS: if (pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL or pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER): object_pool_name = \ self.service.monitor._get_object_pool_name() pool['pool_name'] = object_pool_name self.cache_pool_create(pool) _unwind_actions.append( functools.partial(self.cache_pool_delete, pool)) for pool in CEPH_POOLS: if (pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL or pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER): object_pool_name = \ self.service.monitor._get_object_pool_name() pool['pool_name'] = object_pool_name self.cache_tier_add(pool) _unwind_actions.append( functools.partial(self.cache_tier_remove, pool)) for pool in CEPH_POOLS: if (pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL or pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER): object_pool_name = \ self.service.monitor._get_object_pool_name() pool['pool_name'] = object_pool_name self.cache_mode_set(pool, 'writeback') self.cache_pool_set_config(pool, new_config) self.cache_overlay_create(pool) success = True except Exception as e: LOG.error( _LE('Failed to enable cache: reason=%s') % traceback.format_exc()) for action in reversed(_unwind_actions): try: action() except Exception: LOG.warn( _LW('Failed cache enable ' 'unwind action: reason=%s') % traceback.format_exc()) success = False _exception = str(e) finally: self.service.monitor.monitor_check_cache_tier(success) if success: self.config_applied.cache_enabled = True self.service.sysinv_conductor.call( {}, 'cache_tiering_enable_cache_complete', success=success, exception=_exception, new_config=new_config.to_dict(), applied_config=applied_config.to_dict()) # Run first update of periodic target_max_bytes self.update_cache_target_max_bytes()
def _consolidate_vmdk_volume(self, instance, vm_ref, device, volume_ref, adapter_type=None, disk_type=None): """Consolidate volume backing VMDK files if needed. The volume's VMDK file attached to an instance can be moved by SDRS if enabled on the cluster. By this the VMDK files can get copied onto another datastore and the copy on this new location will be the latest version of the VMDK file. So at the time of detach, we need to consolidate the current backing VMDK file with the VMDK file in the new location. We need to ensure that the VMDK chain (snapshots) remains intact during the consolidation. SDRS retains the chain when it copies VMDK files over, so for consolidation we relocate the backing with move option as moveAllDiskBackingsAndAllowSharing and then delete the older version of the VMDK file attaching the new version VMDK file. In the case of a volume boot the we need to ensure that the volume is on the datastore of the instance. """ original_device = self._get_vmdk_base_volume_device(volume_ref) original_device_path = original_device.backing.fileName current_device_path = device.backing.fileName if original_device_path == current_device_path: # The volume is not moved from its original location. # No consolidation is required. LOG.debug( "The volume has not been displaced from " "its original location: %s. No consolidation " "needed.", current_device_path) return # The volume has been moved from its original location. # Need to consolidate the VMDK files. LOG.info( _LI("The volume's backing has been relocated to %s. Need to " "consolidate backing disk file."), current_device_path) # Pick the host and resource pool on which the instance resides. # Move the volume to the datastore where the new VMDK file is present. host = vm_ref.runtime.host res_pool = host.parent.resourcePool datastore = device.backing.datastore detached = False LOG.debug( "Relocating volume's backing: %(backing)s to resource " "pool: %(rp)s, datastore: %(ds)s, host: %(host)s.", { 'backing': volume_ref, 'rp': res_pool, 'ds': datastore, 'host': host }) try: self._relocate_vmdk_volume(volume_ref, res_pool, datastore, host) except exception.FileNotFound: # Volume's vmdk was moved; remove the device so that we can # relocate the volume. LOG.warn(_LW("Virtual disk: %s of volume's backing not found."), original_device_path, exc_info=True) LOG.debug("Removing disk device of volume's backing and " "reattempting relocate.") self.detach_disk_from_vm(volume_ref, instance, original_device) detached = True self._relocate_vmdk_volume(volume_ref, res_pool, datastore, host) # Volume's backing is relocated now; detach the old vmdk if not done # already. if not detached: self.detach_disk_from_vm(volume_ref, instance, original_device, destroy_disk=True) # Attach the current volume to the volume_ref self._attach_disk_to_vm(volume_ref, instance, adapter_type, disk_type, vmdk_path=current_device_path)