def volumedriver_get(self, name): """ Return volume information. :param unicode name: The name of the volume. :return: Result indicating success. """ contents = json.loads(name.content.getvalue()) volname = contents['Name'] volinfo = self._etcd.get_vol_byname(volname) err = '' if volinfo is None: msg = (_LE('Volume Get: Volume name not found %s'), volname) LOG.warning(msg) response = json.dumps({u"Err": ""}) return response path_info = self._etcd.get_vol_path_info(volname) if path_info is not None: mountdir = path_info['mount_dir'] else: mountdir = '' volume = {'Name': volname, 'Mountpoint': mountdir, 'Status': {}} response = json.dumps({u"Err": err, u"Volume": volume}) return response
def volumedriver_path(self, name): """ Return the path of a locally mounted volume if possible. :param unicode name: The name of the volume. :return: Result indicating success. """ contents = json.loads(name.content.getvalue()) volname = contents['Name'] volinfo = self._etcd.get_vol_byname(volname) if volinfo is None: msg = (_LE('Volume Path: Volume name not found %s'), volname) LOG.warning(msg) response = json.dumps({u"Err": "No Mount Point", u"Mountpoint": ""}) return response path_name = '' path_info = self._etcd.get_vol_path_info(volname) if path_info is not None: path_name = path_info['mount_dir'] response = json.dumps({u"Err": '', u"Mountpoint": path_name}) return response
def volumedriver_get(self, name): """ Return volume information. :param unicode name: The name of the volume. :return: Result indicating success. """ contents = json.loads(name.content.getvalue()) volname = contents['Name'] volinfo = self._etcd.get_vol_byname(volname) err = '' if volinfo is None: msg = (_LE('Volume Get: Volume name not found %s'), volname) LOG.warning(msg) response = json.dumps({u"Err": ""}) return response path_info = self._etcd.get_vol_path_info(volname) if path_info is not None: mountdir = path_info['mount_dir'] else: mountdir = '' volume = {'Name': volname, 'Mountpoint': mountdir, 'Status': {}} response = json.dumps({u"Err": err, u"Volume": volume}) return response
def _get_tiers_size(self): try: resp, body = self.service.ceph_api.osd_df(body='json', output_method='tree') except IOError: return 0 if not resp.ok: LOG.error( _LE("Getting the cluster usage " "information failed: %(reason)s - " "%(body)s") % { "reason": resp.reason, "body": body }) return {} # A node is a crushmap element: root, chassis, host, osd. Create a # dictionary for the nodes with the key as the id used for efficient # searching through nodes. # # For example: storage-0's node has one child node => OSD 0 # { # "id": -4, # "name": "storage-0", # "type": "host", # "type_id": 1, # "reweight": -1.000000, # "kb": 51354096, # "kb_used": 1510348, # "kb_avail": 49843748, # "utilization": 2.941047, # "var": 1.480470, # "pgs": 0, # "children": [ # 0 # ] # }, search_tree = {} for node in body['output']['nodes']: search_tree[node['id']] = node # Extract the tiers as we will return a dict for the size of each tier tiers = {k: v for k, v in search_tree.items() if v['type'] == 'root'} # For each tier, traverse the heirarchy from the root->chassis->host. # Sum the host sizes to determine the overall size of the tier tier_sizes = {} for tier in tiers.values(): tier_size = 0 for chassis_id in tier['children']: chassis_size = 0 chassis = search_tree[chassis_id] for host_id in chassis['children']: host = search_tree[host_id] if (chassis_size == 0 or chassis_size > host['kb']): chassis_size = host['kb'] tier_size += chassis_size / (1024**2) tier_sizes[tier['name']] = tier_size return tier_sizes
def volumedriver_path(self, name): """ Return the path of a locally mounted volume if possible. :param unicode name: The name of the volume. :return: Result indicating success. """ contents = json.loads(name.content.getvalue()) volname = contents['Name'] volinfo = self._etcd.get_vol_byname(volname) if volinfo is None: msg = (_LE('Volume Path: Volume name not found %s'), volname) LOG.warning(msg) response = json.dumps({ u"Err": "No Mount Point", u"Mountpoint": "" }) return response path_name = '' path_info = self._etcd.get_vol_path_info(volname) if path_info is not None: path_name = path_info['mount_dir'] response = json.dumps({u"Err": '', u"Mountpoint": path_name}) return response
def volumedriver_remove(self, name): """ Remove a Docker volume. :param unicode name: The name of the volume. :return: Result indicating success. """ contents = json.loads(name.content.getvalue()) volname = contents['Name'] # Only 1 node in a multinode cluster can try to remove the volume. # Grab lock for volume name. If lock is inuse, just return with no # error. self._lock_volume(volname, 'Remove') vol = self._etcd.get_vol_byname(volname) if vol is None: # Just log an error, but don't fail the docker rm command msg = (_LE('Volume remove name not found %s'), volname) LOG.error(msg) self._unlock_volume(volname) return json.dumps({u"Err": ''}) try: self.hpeplugin_driver.delete_volume(vol) LOG.info(_LI('volume: %(name)s,' 'was successfully deleted'), {'name': volname}) except Exception as ex: msg = (_LE('Err: Failed to remove volume %s, error is %s'), volname, six.text_type(ex)) LOG.error(msg) self._unlock_volume(volname) raise exception.HPEPluginRemoveException(reason=msg) try: self._etcd.delete_vol(vol) except KeyError: msg = (_LW('Warning: Failed to delete volume key: %s from ' 'etcd due to KeyError'), volname) LOG.warning(msg) pass self._unlock_volume(volname) return json.dumps({u"Err": ''})
def volumedriver_remove(self, name): """ Remove a Docker volume. :param unicode name: The name of the volume. :return: Result indicating success. """ contents = json.loads(name.content.getvalue()) volname = contents['Name'] # Only 1 node in a multinode cluster can try to remove the volume. # Grab lock for volume name. If lock is inuse, just return with no # error. self._lock_volume(volname, 'Remove') vol = self._etcd.get_vol_byname(volname) if vol is None: # Just log an error, but don't fail the docker rm command msg = (_LE('Volume remove name not found %s'), volname) LOG.error(msg) self._unlock_volume(volname) return json.dumps({u"Err": ''}) try: self.hpeplugin_driver.delete_volume(vol) LOG.info(_LI('volume: %(name)s,' 'was successfully deleted'), {'name': volname}) except Exception as ex: msg = (_LE('Err: Failed to remove volume %s, error is %s'), volname, six.text_type(ex)) LOG.error(msg) self._unlock_volume(volname) raise exception.HPEPluginRemoveException(reason=msg) try: self._etcd.delete_vol(vol) except KeyError: msg = (_LW('Warning: Failed to delete volume key: %s from ' 'etcd due to KeyError'), volname) LOG.warning(msg) pass self._unlock_volume(volname) return json.dumps({u"Err": ''})
def _select_ds_for_volume(self, dc_moid, cluster_moid, ds_moid, host_moid=None, folders=None): """Select datastore that can accommodate the given volume's backing. Returns the selected datastore summary along with a compute host and its resource pool and folder where the volume can be created :return: (host, resource_pool, folder, summary) """ dc_ref = utils.get_datacenter_moref(self._content, moid=dc_moid) if not dc_ref: LOG.error(_LE("No valid datacenter is available.")) raise exception.NoValidDatacenter() resource_pool = None host_ref = None cluster_ref = utils.get_child_ref_by_moid(dc_ref.hostFolder, cluster_moid) if not cluster_ref: LOG.warn(_LW("No valid cluster is available.")) host_ref = utils.get_child_ref_by_moid(dc_ref.hostFolder, host_moid) if not host_ref: LOG.error(_LE("No valid host is available.")) raise exception.NoValidHost() else: resource_pool = cluster_ref.resourcePool host_ref = utils.get_ref_from_array_by_moid(cluster_ref.host, host_moid) if not host_ref: LOG.warn(_LW("No valid host is specified.")) ds_ref = utils.get_ref_from_array_by_moid(cluster_ref.datastore, ds_moid) if not ds_ref: LOG.error(_LE("No valid datastore is available.")) raise exception.NoValidDatastore() folder_ref = self._get_volume_group_folder(dc_ref, folders) return (resource_pool, host_ref, ds_ref, folder_ref)
def create_inventory_folder(folder_ref, new_folder_name): try: vmfolder = folder_ref.CreateFolder(new_folder_name) except vim.fault.DuplicateName: LOG.error('Another object in the same folder has the target name.') except vim.fault.InvalidName: LOG.error(_LE('The new folder name (%s) is not a valid entity name.'), new_folder_name) return vmfolder
def __init__(self, message=None, **kwargs): self.kwargs = kwargs self.kwargs['message'] = message if 'code' not in self.kwargs: try: self.kwargs['code'] = self.code except AttributeError: pass for k, v in self.kwargs.items(): if isinstance(v, Exception): self.kwargs[k] = six.text_type(v) if self._should_format(): try: message = self.message % kwargs except Exception: exc_info = sys.exc_info() # kwargs doesn't match a variable in the message # log the issue and the kwargs LOG.exception(_LE('Exception in string format operation')) for name, value in kwargs.items(): LOG.error(_LE("%(name)s: %(value)s"), { 'name': name, 'value': value }) if CONF.fatal_exception_format_errors: six.reraise(*exc_info) # at least get the core message out if something happened message = self.message elif isinstance(message, Exception): message = six.text_type(message) # NOTE(luisg): We put the actual message in 'msg' so that we can access # it, because if we try to access the message via 'message' it will be # overshadowed by the class' message attribute self.msg = message super(PluginException, self).__init__(message)
def __init__(self, message=None, **kwargs): self.kwargs = kwargs self.kwargs['message'] = message if 'code' not in self.kwargs: try: self.kwargs['code'] = self.code except AttributeError: pass for k, v in self.kwargs.items(): if isinstance(v, Exception): self.kwargs[k] = six.text_type(v) if self._should_format(): try: message = self.message % kwargs except Exception: exc_info = sys.exc_info() # kwargs doesn't match a variable in the message # log the issue and the kwargs LOG.exception(_LE('Exception in string format operation')) for name, value in kwargs.items(): LOG.error(_LE("%(name)s: %(value)s"), {'name': name, 'value': value}) if CONF.fatal_exception_format_errors: six.reraise(*exc_info) # at least get the core message out if something happened message = self.message elif isinstance(message, Exception): message = six.text_type(message) # NOTE(luisg): We put the actual message in 'msg' so that we can access # it, because if we try to access the message via 'message' it will be # overshadowed by the class' message attribute self.msg = message super(PluginException, self).__init__(message)
def _get_osd_pool_quota(self, pool_name): try: resp, quota = self.service.ceph_api.osd_get_pool_quota(pool_name, body='json') except IOError: return 0 if not resp.ok: LOG.error( _LE("Getting the quota for " "%(name)s pool failed:%(reason)s)") % { "name": pool_name, "reason": resp.reason }) return 0 else: try: quota_gib = int(quota["output"]["quota_max_bytes"]) / (1024**3) return quota_gib except IOError: return 0
def volumedriver_mount(self, name): """ Mount the volume Mount the volume NOTE: If for any reason the mount request fails, Docker will automatically call uMount. So, just make sure uMount can handle partially completed Mount requests. :param unicode name: The name of the volume. :return: Result that includes the mountpoint. """ LOG.debug('In volumedriver_mount') # TODO: use persistent storage to lookup volume for deletion contents = {} contents = json.loads(name.content.getvalue()) volname = contents['Name'] vol = self._etcd.get_vol_byname(volname) if vol is not None: volid = vol['id'] else: msg = (_LE('Volume mount name not found %s'), volname) LOG.error(msg) raise exception.HPEPluginMountException(reason=msg) vol_mount = DEFAULT_MOUNT_VOLUME if ('Opts' in contents and contents['Opts'] and 'mount-volume' in contents['Opts']): vol_mount = str(contents['Opts']['mount-volume']) # Get connector info from OS Brick # TODO: retrieve use_multipath and enforce_multipath from config file root_helper = 'sudo' connector_info = connector.get_connector_properties( root_helper, self._my_ip, multipath=self.use_multipath, enforce_multipath=self.enforce_multipath) try: # Call driver to initialize the connection self.hpeplugin_driver.create_export(vol, connector_info) connection_info = \ self.hpeplugin_driver.initialize_connection( vol, connector_info) LOG.debug( 'connection_info: %(connection_info)s, ' 'was successfully retrieved', {'connection_info': json.dumps(connection_info)}) except Exception as ex: msg = (_('connection info retrieval failed, error is: %s'), six.text_type(ex)) LOG.error(msg) raise exception.HPEPluginMountException(reason=msg) # Call OS Brick to connect volume try: device_info = self.connector.\ connect_volume(connection_info['data']) except Exception as ex: msg = (_('OS Brick connect volume failed, error is: %s'), six.text_type(ex)) LOG.error(msg) raise exception.HPEPluginMountException(reason=msg) # Make sure the path exists path = FilePath(device_info['path']).realpath() if path.exists is False: msg = (_('path: %s, does not exist'), path) LOG.error(msg) raise exception.HPEPluginMountException(reason=msg) LOG.debug( 'path for volume: %(name)s, was successfully created: ' '%(device)s realpath is: %(realpath)s', { 'name': volname, 'device': device_info['path'], 'realpath': path.path }) # Create filesystem on the new device if fileutil.has_filesystem(path.path) is False: fileutil.create_filesystem(path.path) LOG.debug('filesystem successfully created on : %(path)s', {'path': path.path}) # Determine if we need to mount the volume if vol_mount is DEFAULT_MOUNT_VOLUME: # mkdir for mounting the filesystem mount_dir = fileutil.mkdir_for_mounting(device_info['path']) LOG.debug( 'Directory: %(mount_dir)s, ' 'successfully created to mount: ' '%(mount)s', { 'mount_dir': mount_dir, 'mount': device_info['path'] }) # mount the directory fileutil.mount_dir(path.path, mount_dir) LOG.debug('Device: %(path) successfully mounted on %(mount)s', { 'path': path.path, 'mount': mount_dir }) # TODO: find out how to invoke mkfs so that it creates the # filesystem without the lost+found directory # KLUDGE!!!!! lostfound = mount_dir + '/lost+found' lfdir = FilePath(lostfound) if lfdir.exists and fileutil.remove_dir(lostfound): LOG.debug( 'Successfully removed : ' '%(lost)s from mount: %(mount)s', { 'lost': lostfound, 'mount': mount_dir }) else: mount_dir = '' path_info = {} path_info['name'] = volname path_info['path'] = path.path path_info['device_info'] = device_info path_info['connection_info'] = connection_info path_info['mount_dir'] = mount_dir self._etcd.update_vol(volid, 'path_info', json.dumps(path_info)) response = json.dumps({ u"Err": '', u"Name": volname, u"Mountpoint": mount_dir, u"Devicename": path.path }) return response
def volumedriver_unmount(self, name): """ The Docker container is no longer using the given volume, so unmount it. NOTE: Since Docker will automatically call Unmount if the Mount fails, make sure we properly handle partially completed Mounts. :param unicode name: The name of the volume. :return: Result indicating success. """ LOG.info(_LI('In volumedriver_unmount')) contents = json.loads(name.content.getvalue()) volname = contents['Name'] vol = self._etcd.get_vol_byname(volname) if vol is not None: volid = vol['id'] else: msg = (_LE('Volume unmount name not found %s'), volname) LOG.error(msg) raise exception.HPEPluginUMountException(reason=msg) vol_mount = DEFAULT_MOUNT_VOLUME if ('Opts' in contents and contents['Opts'] and 'mount-volume' in contents['Opts']): vol_mount = str(contents['Opts']['mount-volume']) path_info = self._etcd.get_vol_path_info(volname) if path_info: path_name = path_info['path'] connection_info = path_info['connection_info'] mount_dir = path_info['mount_dir'] else: msg = (_LE('Volume unmount path info not found %s'), volname) LOG.error(msg) raise exception.HPEPluginUMountException(reason=msg) # Get connector info from OS Brick # TODO: retrieve use_multipath and enforce_multipath from config file root_helper = 'sudo' connector_info = connector.get_connector_properties( root_helper, self._my_ip, multipath=self.use_multipath, enforce_multipath=self.enforce_multipath) # Determine if we need to unmount a previously mounted volume if vol_mount is DEFAULT_MOUNT_VOLUME: # unmount directory fileutil.umount_dir(mount_dir) # remove directory fileutil.remove_dir(mount_dir) # We're deferring the execution of the disconnect_volume as it can take # substantial # time (over 2 minutes) to cleanup the iscsi files if connection_info: LOG.info(_LI('call os brick to disconnect volume')) d = threads.deferToThread(self.connector.disconnect_volume, connection_info['data'], None) d.addCallbacks(self.disconnect_volume_callback, self.disconnect_volume_error_callback) try: # Call driver to terminate the connection self.hpeplugin_driver.terminate_connection(vol, connector_info) LOG.info( _LI('connection_info: %(connection_info)s, ' 'was successfully terminated'), {'connection_info': json.dumps(connection_info)}) except Exception as ex: msg = (_LE('connection info termination failed %s'), six.text_type(ex)) LOG.error(msg) # Not much we can do here, so just continue on with unmount # We need to ensure we update etcd path_info so the stale # path does not stay around # raise exception.HPEPluginUMountException(reason=msg) # TODO: Create path_info list as we can mount the volume to multiple # hosts at the same time. self._etcd.update_vol(volid, 'path_info', None) LOG.info( _LI('path for volume: %(name)s, was successfully removed: ' '%(path_name)s'), { 'name': volname, 'path_name': path_name }) response = json.dumps({u"Err": ''}) return response
def do_disable_cache(self, new_config, applied_config, lock_ownership): LOG.info( _LI("cache_tiering_disable_cache: " "new_config={}, applied_config={}").format( new_config, applied_config)) with lock_ownership(): success = False _exception = None try: self.config_desired.cache_enabled = False for pool in CEPH_POOLS: if (pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL or pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER): object_pool_name = \ self.service.monitor._get_object_pool_name() pool['pool_name'] = object_pool_name with self.ignore_ceph_failure(): self.cache_mode_set(pool, 'forward') for pool in CEPH_POOLS: if (pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL or pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER): object_pool_name = \ self.service.monitor._get_object_pool_name() pool['pool_name'] = object_pool_name retries_left = 3 while True: try: self.cache_flush(pool) break except exception.CephCacheFlushFailure: retries_left -= 1 if not retries_left: # give up break else: time.sleep(1) for pool in CEPH_POOLS: if (pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL or pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER): object_pool_name = \ self.service.monitor._get_object_pool_name() pool['pool_name'] = object_pool_name with self.ignore_ceph_failure(): self.cache_overlay_delete(pool) self.cache_tier_remove(pool) for pool in CEPH_POOLS: if (pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL or pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER): object_pool_name = \ self.service.monitor._get_object_pool_name() pool['pool_name'] = object_pool_name with self.ignore_ceph_failure(): self.cache_pool_delete(pool) success = True except Exception as e: LOG.warn( _LE('Failed to disable cache: reason=%s') % traceback.format_exc()) _exception = str(e) finally: self.service.monitor.monitor_check_cache_tier(False) if success: self.config_desired.cache_enabled = False self.config_applied.cache_enabled = False self.service.sysinv_conductor.call( {}, 'cache_tiering_disable_cache_complete', success=success, exception=_exception, new_config=new_config.to_dict(), applied_config=applied_config.to_dict())
def do_enable_cache(self, new_config, applied_config, lock_ownership): LOG.info( _LI("cache_tiering_enable_cache: " "new_config={}, applied_config={}").format( new_config.to_dict(), applied_config.to_dict())) _unwind_actions = [] with lock_ownership(): success = False _exception = None try: self.config_desired.cache_enabled = True self.update_pools_info() for pool in CEPH_POOLS: if (pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL or pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER): object_pool_name = \ self.service.monitor._get_object_pool_name() pool['pool_name'] = object_pool_name self.cache_pool_create(pool) _unwind_actions.append( functools.partial(self.cache_pool_delete, pool)) for pool in CEPH_POOLS: if (pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL or pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER): object_pool_name = \ self.service.monitor._get_object_pool_name() pool['pool_name'] = object_pool_name self.cache_tier_add(pool) _unwind_actions.append( functools.partial(self.cache_tier_remove, pool)) for pool in CEPH_POOLS: if (pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL or pool['pool_name'] == constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER): object_pool_name = \ self.service.monitor._get_object_pool_name() pool['pool_name'] = object_pool_name self.cache_mode_set(pool, 'writeback') self.cache_pool_set_config(pool, new_config) self.cache_overlay_create(pool) success = True except Exception as e: LOG.error( _LE('Failed to enable cache: reason=%s') % traceback.format_exc()) for action in reversed(_unwind_actions): try: action() except Exception: LOG.warn( _LW('Failed cache enable ' 'unwind action: reason=%s') % traceback.format_exc()) success = False _exception = str(e) finally: self.service.monitor.monitor_check_cache_tier(success) if success: self.config_applied.cache_enabled = True self.service.sysinv_conductor.call( {}, 'cache_tiering_enable_cache_complete', success=success, exception=_exception, new_config=new_config.to_dict(), applied_config=applied_config.to_dict()) # Run first update of periodic target_max_bytes self.update_cache_target_max_bytes()
def _report_alarm_osds_health(self): response, osd_tree = self.service.ceph_api.osd_tree(body='json') if not response.ok: LOG.error( _LE("Failed to retrieve Ceph OSD tree: " "status_code: %(status_code)s, reason: %(reason)s") % { "status_code": response.status_code, "reason": response.reason }) return osd_tree = dict([(n['id'], n) for n in osd_tree['output']['nodes']]) alarms = [] self._check_storage_tier(osd_tree, "storage-tier", lambda *args: alarms.append(args)) old_alarms = {} for alarm_id in [ fm_constants.FM_ALARM_ID_STORAGE_CEPH_MAJOR, fm_constants.FM_ALARM_ID_STORAGE_CEPH_CRITICAL ]: alarm_list = self.service.fm_api.get_faults_by_id(alarm_id) if not alarm_list: continue for alarm in alarm_list: if alarm.entity_instance_id not in old_alarms: old_alarms[alarm.entity_instance_id] = [] old_alarms[alarm.entity_instance_id].append( (alarm.alarm_id, alarm.reason_text)) for peer_group, reason, severity in alarms: if self._current_health_alarm_equals(reason, severity): continue alarm_critical_major = fm_constants.FM_ALARM_ID_STORAGE_CEPH_MAJOR if severity == fm_constants.FM_ALARM_SEVERITY_CRITICAL: alarm_critical_major = ( fm_constants.FM_ALARM_ID_STORAGE_CEPH_CRITICAL) entity_instance_id = (self.service.entity_instance_id + '.peergroup=' + peer_group) alarm_already_exists = False if entity_instance_id in old_alarms: for alarm_id, old_reason in old_alarms[entity_instance_id]: if (reason == old_reason and alarm_id == alarm_critical_major): # if the alarm is exactly the same, we don't need # to recreate it old_alarms[entity_instance_id].remove( (alarm_id, old_reason)) alarm_already_exists = True elif (alarm_id == alarm_critical_major): # if we change just the reason, then we just remove the # alarm from the list so we don't remove it at the # end of the function old_alarms[entity_instance_id].remove( (alarm_id, old_reason)) if (len(old_alarms[entity_instance_id]) == 0): del old_alarms[entity_instance_id] # in case the alarm is exactly the same, we skip the alarm set if alarm_already_exists is True: continue major_repair_action = constants.REPAIR_ACTION_MAJOR_CRITICAL_ALARM fault = fm_api.Fault( alarm_id=alarm_critical_major, alarm_type=fm_constants.FM_ALARM_TYPE_4, alarm_state=fm_constants.FM_ALARM_STATE_SET, entity_type_id=fm_constants.FM_ENTITY_TYPE_CLUSTER, entity_instance_id=entity_instance_id, severity=severity, reason_text=reason, probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_15, proposed_repair_action=major_repair_action, service_affecting=constants.SERVICE_AFFECTING['HEALTH_WARN']) alarm_uuid = self.service.fm_api.set_fault(fault) if alarm_uuid: LOG.info( _LI("Created storage alarm %(alarm_uuid)s - " "severity: %(severity)s, reason: %(reason)s, " "service_affecting: %(service_affecting)s") % { "alarm_uuid": str(alarm_uuid), "severity": str(severity), "reason": reason, "service_affecting": str(constants.SERVICE_AFFECTING['HEALTH_WARN']) }) else: LOG.error( _LE("Failed to create storage alarm - " "severity: %(severity)s, reason: %(reason)s, " "service_affecting: %(service_affecting)s") % { "severity": str(severity), "reason": reason, "service_affecting": str(constants.SERVICE_AFFECTING['HEALTH_WARN']) }) for entity_instance_id in old_alarms: for alarm_id, old_reason in old_alarms[entity_instance_id]: self.service.fm_api.clear_fault(alarm_id, entity_instance_id)
def _report_fault(self, health, alarm_id): if alarm_id == fm_constants.FM_ALARM_ID_STORAGE_CEPH: new_severity = constants.SEVERITY[health['health']] new_reason_text = self._parse_reason(health) new_service_affecting = \ constants.SERVICE_AFFECTING[health['health']] # Raise or update alarm if necessary if ((not self.current_health_alarm) or (self.current_health_alarm.__dict__['severity'] != new_severity) or (self.current_health_alarm.__dict__['reason_text'] != new_reason_text) or (self.current_health_alarm.__dict__['service_affecting'] != str(new_service_affecting))): fault = fm_api.Fault( alarm_id=fm_constants.FM_ALARM_ID_STORAGE_CEPH, alarm_type=fm_constants.FM_ALARM_TYPE_4, alarm_state=fm_constants.FM_ALARM_STATE_SET, entity_type_id=fm_constants.FM_ENTITY_TYPE_CLUSTER, entity_instance_id=self.service.entity_instance_id, severity=new_severity, reason_text=new_reason_text, probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_15, proposed_repair_action=constants.REPAIR_ACTION, service_affecting=new_service_affecting) alarm_uuid = self.service.fm_api.set_fault(fault) if alarm_uuid: LOG.info( _LI("Created storage alarm %(alarm_uuid)s - " "severity: %(severity)s, reason: %(reason)s, " "service_affecting: %(service_affecting)s") % { "alarm_uuid": alarm_uuid, "severity": new_severity, "reason": new_reason_text, "service_affecting": new_service_affecting }) else: LOG.error( _LE("Failed to create storage alarm - " "severity: %(severity)s, reason: %(reason)s " "service_affecting: %(service_affecting)s") % { "severity": new_severity, "reason": new_reason_text, "service_affecting": new_service_affecting }) # Log detailed reason for later analysis if (self.current_ceph_health != health['health'] or self.detailed_health_reason != health['detail']): LOG.info( _LI("Ceph status changed: %(health)s " "detailed reason: %(detail)s") % health) self.current_ceph_health = health['health'] self.detailed_health_reason = health['detail'] elif (alarm_id == fm_constants.FM_ALARM_ID_STORAGE_CEPH_FREE_SPACE and not health['tier_eid'] in self.current_quota_alarms): quota_reason_text = ("Quota/Space mismatch for the %s tier. The " "sum of Ceph pool quotas does not match the " "tier size." % health['tier_name']) fault = fm_api.Fault( alarm_id=fm_constants.FM_ALARM_ID_STORAGE_CEPH_FREE_SPACE, alarm_state=fm_constants.FM_ALARM_STATE_SET, entity_type_id=fm_constants.FM_ENTITY_TYPE_CLUSTER, entity_instance_id=health['tier_eid'], severity=fm_constants.FM_ALARM_SEVERITY_MINOR, reason_text=quota_reason_text, alarm_type=fm_constants.FM_ALARM_TYPE_7, probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_75, proposed_repair_action=( "Update ceph storage pool quotas to use all available " "cluster space for the %s tier." % health['tier_name']), service_affecting=False) alarm_uuid = self.service.fm_api.set_fault(fault) if alarm_uuid: LOG.info( _LI("Created storage quota storage alarm %(alarm_uuid)s. " "Reason: %(reason)s") % { "alarm_uuid": alarm_uuid, "reason": quota_reason_text }) else: LOG.error( _LE("Failed to create quota " "storage alarm. Reason: %s") % quota_reason_text)
def volumedriver_mount(self, name): """ Mount the volume Mount the volume NOTE: If for any reason the mount request fails, Docker will automatically call uMount. So, just make sure uMount can handle partially completed Mount requests. :param unicode name: The name of the volume. :return: Result that includes the mountpoint. """ LOG.debug('In volumedriver_mount') # TODO: use persistent storage to lookup volume for deletion contents = {} contents = json.loads(name.content.getvalue()) volname = contents['Name'] vol = self._etcd.get_vol_byname(volname) if vol is not None: volid = vol['id'] else: msg = (_LE('Volume mount name not found %s'), volname) LOG.error(msg) raise exception.HPEPluginMountException(reason=msg) # Get connector info from OS Brick # TODO: retrieve use_multipath and enforce_multipath from config file root_helper = 'sudo' use_multipath = False connector_info = connector.get_connector_properties( root_helper, self._my_ip, use_multipath, enforce_multipath=False) try: # Call driver to initialize the connection self.hpeplugin_driver.create_export(vol, connector_info) connection_info = \ self.hpeplugin_driver.initialize_connection( vol, connector_info) LOG.debug('connection_info: %(connection_info)s, ' 'was successfully retrieved', {'connection_info': json.dumps(connection_info)}) except Exception as ex: msg = (_('connection info retrieval failed, error is: %s'), six.text_type(ex)) LOG.error(msg) raise exception.HPEPluginMountException(reason=msg) # Call OS Brick to connect volume try: device_info = self.connector.\ connect_volume(connection_info['data']) except Exception as ex: msg = (_('OS Brick connect volume failed, error is: %s'), six.text_type(ex)) LOG.error(msg) raise exception.HPEPluginMountException(reason=msg) # Make sure the path exists path = FilePath(device_info['path']).realpath() if path.exists is False: msg = (_('path: %s, does not exist'), path) LOG.error(msg) raise exception.HPEPluginMountException(reason=msg) LOG.debug('path for volume: %(name)s, was successfully created: ' '%(device)s realpath is: %(realpath)s', {'name': volname, 'device': device_info['path'], 'realpath': path.path}) # mkdir for mounting the filesystem mount_dir = fileutil.mkdir_for_mounting(device_info['path']) LOG.debug('Directory: %(mount_dir)s, successfully created to mount: ' '%(mount)s', {'mount_dir': mount_dir, 'mount': device_info['path']}) # Create filesystem on the new device if fileutil.has_filesystem(path.path) is False: fileutil.create_filesystem(path.path) LOG.debug('filesystem successfully created on : %(path)s', {'path': path.path}) # mount the directory fileutil.mount_dir(path.path, mount_dir) LOG.debug('Device: %(path) successfully mounted on %(mount)s', {'path': path.path, 'mount': mount_dir}) # TODO: find out how to invoke mkfs so that it creates the filesystem # without the lost+found directory # KLUDGE!!!!! lostfound = mount_dir + '/lost+found' lfdir = FilePath(lostfound) if lfdir.exists and fileutil.remove_dir(lostfound): LOG.debug('Successfully removed : %(lost)s from mount: %(mount)s', {'lost': lostfound, 'mount': mount_dir}) path_info = {} path_info['name'] = volname path_info['path'] = path.path path_info['device_info'] = device_info path_info['connection_info'] = connection_info path_info['mount_dir'] = mount_dir self._etcd.update_vol(volid, 'path_info', json.dumps(path_info)) response = json.dumps({u"Err": '', u"Mountpoint": mount_dir}) return response
def volumedriver_unmount(self, name): """ The Docker container is no longer using the given volume, so unmount it. NOTE: Since Docker will automatically call Unmount if the Mount fails, make sure we properly handle partially completed Mounts. :param unicode name: The name of the volume. :return: Result indicating success. """ LOG.info(_LI('In volumedriver_unmount')) contents = json.loads(name.content.getvalue()) volname = contents['Name'] vol = self._etcd.get_vol_byname(volname) if vol is not None: volid = vol['id'] else: msg = (_LE('Volume unmount name not found %s'), volname) LOG.error(msg) raise exception.HPEPluginUMountException(reason=msg) path_info = self._etcd.get_vol_path_info(volname) if path_info: path_name = path_info['path'] connection_info = path_info['connection_info'] mount_dir = path_info['mount_dir'] else: msg = (_LE('Volume unmount path info not found %s'), volname) LOG.error(msg) raise exception.HPEPluginUMountException(reason=msg) # Get connector info from OS Brick # TODO: retrieve use_multipath and enforce_multipath from config file root_helper = 'sudo' use_multipath = False connector_info = connector.get_connector_properties( root_helper, self._my_ip, use_multipath, enforce_multipath=False) # unmount directory fileutil.umount_dir(mount_dir) # remove directory fileutil.remove_dir(mount_dir) try: # Call driver to terminate the connection self.hpeplugin_driver.terminate_connection(vol, connector_info) LOG.info(_LI('connection_info: %(connection_info)s, ' 'was successfully terminated'), {'connection_info': json.dumps(connection_info)}) except Exception as ex: msg = (_LE('connection info termination failed %s'), six.text_type(ex)) LOG.error(msg) raise exception.HPEPluginUMountException(reason=msg) # We're deferring the execution of the disconnect_volume as it can take # substantial # time (over 2 minutes) to cleanup the iscsi files if connection_info: LOG.info(_LI('call os brick to disconnect volume')) d = threads.deferToThread(self.connector.disconnect_volume, connection_info['data'], None) d.addCallbacks(self.disconnect_volume_callback, self.disconnect_volume_error_callback) # TODO(leeantho) Without this sleep the volume is sometimes not # removed after the unmount. There must be a different way to fix # the issue? time.sleep(1) # TODO: Create path_info list as we can mount the volume to multiple # hosts at the same time. self._etcd.update_vol(volid, 'path_info', None) LOG.info(_LI('path for volume: %(name)s, was successfully removed: ' '%(path_name)s'), {'name': volname, 'path_name': path_name}) response = json.dumps({u"Err": ''}) return response
def volumedriver_unmount(self, name): """ The Docker container is no longer using the given volume, so unmount it. NOTE: Since Docker will automatically call Unmount if the Mount fails, make sure we properly handle partially completed Mounts. :param unicode name: The name of the volume. :return: Result indicating success. """ LOG.info(_LI('In volumedriver_unmount')) contents = json.loads(name.content.getvalue()) volname = contents['Name'] vol = self._etcd.get_vol_byname(volname) if vol is not None: volid = vol['id'] else: msg = (_LE('Volume unmount name not found %s'), volname) LOG.error(msg) raise exception.HPEPluginUMountException(reason=msg) vol_mount = DEFAULT_MOUNT_VOLUME if ('Opts' in contents and contents['Opts'] and 'mount-volume' in contents['Opts']): vol_mount = str(contents['Opts']['mount-volume']) path_info = self._etcd.get_vol_path_info(volname) if path_info: path_name = path_info['path'] connection_info = path_info['connection_info'] mount_dir = path_info['mount_dir'] else: msg = (_LE('Volume unmount path info not found %s'), volname) LOG.error(msg) raise exception.HPEPluginUMountException(reason=msg) # Get connector info from OS Brick # TODO: retrieve use_multipath and enforce_multipath from config file root_helper = 'sudo' connector_info = connector.get_connector_properties( root_helper, self._my_ip, multipath=self.use_multipath, enforce_multipath=self.enforce_multipath) # Determine if we need to unmount a previously mounted volume if vol_mount is DEFAULT_MOUNT_VOLUME: # unmount directory fileutil.umount_dir(mount_dir) # remove directory fileutil.remove_dir(mount_dir) # We're deferring the execution of the disconnect_volume as it can take # substantial # time (over 2 minutes) to cleanup the iscsi files if connection_info: LOG.info(_LI('call os brick to disconnect volume')) d = threads.deferToThread(self.connector.disconnect_volume, connection_info['data'], None) d.addCallbacks(self.disconnect_volume_callback, self.disconnect_volume_error_callback) try: # Call driver to terminate the connection self.hpeplugin_driver.terminate_connection(vol, connector_info) LOG.info(_LI('connection_info: %(connection_info)s, ' 'was successfully terminated'), {'connection_info': json.dumps(connection_info)}) except Exception as ex: msg = (_LE('connection info termination failed %s'), six.text_type(ex)) LOG.error(msg) # Not much we can do here, so just continue on with unmount # We need to ensure we update etcd path_info so the stale # path does not stay around # raise exception.HPEPluginUMountException(reason=msg) # TODO: Create path_info list as we can mount the volume to multiple # hosts at the same time. self._etcd.update_vol(volid, 'path_info', None) LOG.info(_LI('path for volume: %(name)s, was successfully removed: ' '%(path_name)s'), {'name': volname, 'path_name': path_name}) response = json.dumps({u"Err": ''}) return response