def volumedriver_get(self, name):
"""
Return volume information.
:param unicode name: The name of the volume.
:return: Result indicating success.
"""
contents = json.loads(name.content.getvalue())
volname = contents['Name']
volinfo = self._etcd.get_vol_byname(volname)
err = ''
if volinfo is None:
msg = (_LE('Volume Get: Volume name not found %s'), volname)
LOG.warning(msg)
response = json.dumps({u"Err": ""})
return response
path_info = self._etcd.get_vol_path_info(volname)
if path_info is not None:
mountdir = path_info['mount_dir']
else:
mountdir = ''
volume = {'Name': volname, 'Mountpoint': mountdir, 'Status': {}}
response = json.dumps({u"Err": err, u"Volume": volume})
return response
def volumedriver_path(self, name):
"""
Return the path of a locally mounted volume if possible.
:param unicode name: The name of the volume.
:return: Result indicating success.
"""
contents = json.loads(name.content.getvalue())
volname = contents['Name']
volinfo = self._etcd.get_vol_byname(volname)
if volinfo is None:
msg = (_LE('Volume Path: Volume name not found %s'), volname)
LOG.warning(msg)
response = json.dumps({u"Err": "No Mount Point",
u"Mountpoint": ""})
return response
path_name = ''
path_info = self._etcd.get_vol_path_info(volname)
if path_info is not None:
path_name = path_info['mount_dir']
response = json.dumps({u"Err": '', u"Mountpoint": path_name})
return response
def volumedriver_get(self, name):
"""
Return volume information.
:param unicode name: The name of the volume.
:return: Result indicating success.
"""
contents = json.loads(name.content.getvalue())
volname = contents['Name']
volinfo = self._etcd.get_vol_byname(volname)
err = ''
if volinfo is None:
msg = (_LE('Volume Get: Volume name not found %s'), volname)
LOG.warning(msg)
response = json.dumps({u"Err": ""})
return response
path_info = self._etcd.get_vol_path_info(volname)
if path_info is not None:
mountdir = path_info['mount_dir']
else:
mountdir = ''
volume = {'Name': volname,
'Mountpoint': mountdir,
'Status': {}}
response = json.dumps({u"Err": err, u"Volume": volume})
return response
def _get_tiers_size(self):
try:
resp, body = self.service.ceph_api.osd_df(body='json',
output_method='tree')
except IOError:
return 0
if not resp.ok:
LOG.error(
_LE("Getting the cluster usage "
"information failed: %(reason)s - "
"%(body)s") % {
"reason": resp.reason,
"body": body
})
return {}
# A node is a crushmap element: root, chassis, host, osd. Create a
# dictionary for the nodes with the key as the id used for efficient
# searching through nodes.
#
# For example: storage-0's node has one child node => OSD 0
# {
# "id": -4,
# "name": "storage-0",
# "type": "host",
# "type_id": 1,
# "reweight": -1.000000,
# "kb": 51354096,
# "kb_used": 1510348,
# "kb_avail": 49843748,
# "utilization": 2.941047,
# "var": 1.480470,
# "pgs": 0,
# "children": [
# 0
# ]
# },
search_tree = {}
for node in body['output']['nodes']:
search_tree[node['id']] = node
# Extract the tiers as we will return a dict for the size of each tier
tiers = {k: v for k, v in search_tree.items() if v['type'] == 'root'}
# For each tier, traverse the heirarchy from the root->chassis->host.
# Sum the host sizes to determine the overall size of the tier
tier_sizes = {}
for tier in tiers.values():
tier_size = 0
for chassis_id in tier['children']:
chassis_size = 0
chassis = search_tree[chassis_id]
for host_id in chassis['children']:
host = search_tree[host_id]
if (chassis_size == 0 or chassis_size > host['kb']):
chassis_size = host['kb']
tier_size += chassis_size / (1024**2)
tier_sizes[tier['name']] = tier_size
return tier_sizes
def volumedriver_path(self, name):
"""
Return the path of a locally mounted volume if possible.
:param unicode name: The name of the volume.
:return: Result indicating success.
"""
contents = json.loads(name.content.getvalue())
volname = contents['Name']
volinfo = self._etcd.get_vol_byname(volname)
if volinfo is None:
msg = (_LE('Volume Path: Volume name not found %s'), volname)
LOG.warning(msg)
response = json.dumps({
u"Err": "No Mount Point",
u"Mountpoint": ""
})
return response
path_name = ''
path_info = self._etcd.get_vol_path_info(volname)
if path_info is not None:
path_name = path_info['mount_dir']
response = json.dumps({u"Err": '', u"Mountpoint": path_name})
return response
def volumedriver_remove(self, name):
"""
Remove a Docker volume.
:param unicode name: The name of the volume.
:return: Result indicating success.
"""
contents = json.loads(name.content.getvalue())
volname = contents['Name']
# Only 1 node in a multinode cluster can try to remove the volume.
# Grab lock for volume name. If lock is inuse, just return with no
# error.
self._lock_volume(volname, 'Remove')
vol = self._etcd.get_vol_byname(volname)
if vol is None:
# Just log an error, but don't fail the docker rm command
msg = (_LE('Volume remove name not found %s'), volname)
LOG.error(msg)
self._unlock_volume(volname)
return json.dumps({u"Err": ''})
try:
self.hpeplugin_driver.delete_volume(vol)
LOG.info(_LI('volume: %(name)s,'
'was successfully deleted'), {'name': volname})
except Exception as ex:
msg = (_LE('Err: Failed to remove volume %s, error is %s'),
volname, six.text_type(ex))
LOG.error(msg)
self._unlock_volume(volname)
raise exception.HPEPluginRemoveException(reason=msg)
try:
self._etcd.delete_vol(vol)
except KeyError:
msg = (_LW('Warning: Failed to delete volume key: %s from '
'etcd due to KeyError'), volname)
LOG.warning(msg)
pass
self._unlock_volume(volname)
return json.dumps({u"Err": ''})
def volumedriver_remove(self, name):
"""
Remove a Docker volume.
:param unicode name: The name of the volume.
:return: Result indicating success.
"""
contents = json.loads(name.content.getvalue())
volname = contents['Name']
# Only 1 node in a multinode cluster can try to remove the volume.
# Grab lock for volume name. If lock is inuse, just return with no
# error.
self._lock_volume(volname, 'Remove')
vol = self._etcd.get_vol_byname(volname)
if vol is None:
# Just log an error, but don't fail the docker rm command
msg = (_LE('Volume remove name not found %s'), volname)
LOG.error(msg)
self._unlock_volume(volname)
return json.dumps({u"Err": ''})
try:
self.hpeplugin_driver.delete_volume(vol)
LOG.info(_LI('volume: %(name)s,' 'was successfully deleted'),
{'name': volname})
except Exception as ex:
msg = (_LE('Err: Failed to remove volume %s, error is %s'),
volname, six.text_type(ex))
LOG.error(msg)
self._unlock_volume(volname)
raise exception.HPEPluginRemoveException(reason=msg)
try:
self._etcd.delete_vol(vol)
except KeyError:
msg = (_LW('Warning: Failed to delete volume key: %s from '
'etcd due to KeyError'), volname)
LOG.warning(msg)
pass
self._unlock_volume(volname)
return json.dumps({u"Err": ''})
def _select_ds_for_volume(self,
dc_moid,
cluster_moid,
ds_moid,
host_moid=None,
folders=None):
"""Select datastore that can accommodate the given volume's backing.
Returns the selected datastore summary along with a compute host and
its resource pool and folder where the volume can be created
:return: (host, resource_pool, folder, summary)
"""
dc_ref = utils.get_datacenter_moref(self._content, moid=dc_moid)
if not dc_ref:
LOG.error(_LE("No valid datacenter is available."))
raise exception.NoValidDatacenter()
resource_pool = None
host_ref = None
cluster_ref = utils.get_child_ref_by_moid(dc_ref.hostFolder,
cluster_moid)
if not cluster_ref:
LOG.warn(_LW("No valid cluster is available."))
host_ref = utils.get_child_ref_by_moid(dc_ref.hostFolder,
host_moid)
if not host_ref:
LOG.error(_LE("No valid host is available."))
raise exception.NoValidHost()
else:
resource_pool = cluster_ref.resourcePool
host_ref = utils.get_ref_from_array_by_moid(cluster_ref.host,
host_moid)
if not host_ref:
LOG.warn(_LW("No valid host is specified."))
ds_ref = utils.get_ref_from_array_by_moid(cluster_ref.datastore,
ds_moid)
if not ds_ref:
LOG.error(_LE("No valid datastore is available."))
raise exception.NoValidDatastore()
folder_ref = self._get_volume_group_folder(dc_ref, folders)
return (resource_pool, host_ref, ds_ref, folder_ref)
def create_inventory_folder(folder_ref, new_folder_name):
try:
vmfolder = folder_ref.CreateFolder(new_folder_name)
except vim.fault.DuplicateName:
LOG.error('Another object in the same folder has the target name.')
except vim.fault.InvalidName:
LOG.error(_LE('The new folder name (%s) is not a valid entity name.'),
new_folder_name)
return vmfolder
def __init__(self, message=None, **kwargs):
self.kwargs = kwargs
self.kwargs['message'] = message
if 'code' not in self.kwargs:
try:
self.kwargs['code'] = self.code
except AttributeError:
pass
for k, v in self.kwargs.items():
if isinstance(v, Exception):
self.kwargs[k] = six.text_type(v)
if self._should_format():
try:
message = self.message % kwargs
except Exception:
exc_info = sys.exc_info()
# kwargs doesn't match a variable in the message
# log the issue and the kwargs
LOG.exception(_LE('Exception in string format operation'))
for name, value in kwargs.items():
LOG.error(_LE("%(name)s: %(value)s"), {
'name': name,
'value': value
})
if CONF.fatal_exception_format_errors:
six.reraise(*exc_info)
# at least get the core message out if something happened
message = self.message
elif isinstance(message, Exception):
message = six.text_type(message)
# NOTE(luisg): We put the actual message in 'msg' so that we can access
# it, because if we try to access the message via 'message' it will be
# overshadowed by the class' message attribute
self.msg = message
super(PluginException, self).__init__(message)
def __init__(self, message=None, **kwargs):
self.kwargs = kwargs
self.kwargs['message'] = message
if 'code' not in self.kwargs:
try:
self.kwargs['code'] = self.code
except AttributeError:
pass
for k, v in self.kwargs.items():
if isinstance(v, Exception):
self.kwargs[k] = six.text_type(v)
if self._should_format():
try:
message = self.message % kwargs
except Exception:
exc_info = sys.exc_info()
# kwargs doesn't match a variable in the message
# log the issue and the kwargs
LOG.exception(_LE('Exception in string format operation'))
for name, value in kwargs.items():
LOG.error(_LE("%(name)s: %(value)s"),
{'name': name, 'value': value})
if CONF.fatal_exception_format_errors:
six.reraise(*exc_info)
# at least get the core message out if something happened
message = self.message
elif isinstance(message, Exception):
message = six.text_type(message)
# NOTE(luisg): We put the actual message in 'msg' so that we can access
# it, because if we try to access the message via 'message' it will be
# overshadowed by the class' message attribute
self.msg = message
super(PluginException, self).__init__(message)
def _get_osd_pool_quota(self, pool_name):
try:
resp, quota = self.service.ceph_api.osd_get_pool_quota(pool_name,
body='json')
except IOError:
return 0
if not resp.ok:
LOG.error(
_LE("Getting the quota for "
"%(name)s pool failed:%(reason)s)") % {
"name": pool_name,
"reason": resp.reason
})
return 0
else:
try:
quota_gib = int(quota["output"]["quota_max_bytes"]) / (1024**3)
return quota_gib
except IOError:
return 0
def volumedriver_mount(self, name):
"""
Mount the volume
Mount the volume
NOTE: If for any reason the mount request fails, Docker
will automatically call uMount. So, just make sure uMount
can handle partially completed Mount requests.
:param unicode name: The name of the volume.
:return: Result that includes the mountpoint.
"""
LOG.debug('In volumedriver_mount')
# TODO: use persistent storage to lookup volume for deletion
contents = {}
contents = json.loads(name.content.getvalue())
volname = contents['Name']
vol = self._etcd.get_vol_byname(volname)
if vol is not None:
volid = vol['id']
else:
msg = (_LE('Volume mount name not found %s'), volname)
LOG.error(msg)
raise exception.HPEPluginMountException(reason=msg)
vol_mount = DEFAULT_MOUNT_VOLUME
if ('Opts' in contents and contents['Opts']
and 'mount-volume' in contents['Opts']):
vol_mount = str(contents['Opts']['mount-volume'])
# Get connector info from OS Brick
# TODO: retrieve use_multipath and enforce_multipath from config file
root_helper = 'sudo'
connector_info = connector.get_connector_properties(
root_helper,
self._my_ip,
multipath=self.use_multipath,
enforce_multipath=self.enforce_multipath)
try:
# Call driver to initialize the connection
self.hpeplugin_driver.create_export(vol, connector_info)
connection_info = \
self.hpeplugin_driver.initialize_connection(
vol, connector_info)
LOG.debug(
'connection_info: %(connection_info)s, '
'was successfully retrieved',
{'connection_info': json.dumps(connection_info)})
except Exception as ex:
msg = (_('connection info retrieval failed, error is: %s'),
six.text_type(ex))
LOG.error(msg)
raise exception.HPEPluginMountException(reason=msg)
# Call OS Brick to connect volume
try:
device_info = self.connector.\
connect_volume(connection_info['data'])
except Exception as ex:
msg = (_('OS Brick connect volume failed, error is: %s'),
six.text_type(ex))
LOG.error(msg)
raise exception.HPEPluginMountException(reason=msg)
# Make sure the path exists
path = FilePath(device_info['path']).realpath()
if path.exists is False:
msg = (_('path: %s, does not exist'), path)
LOG.error(msg)
raise exception.HPEPluginMountException(reason=msg)
LOG.debug(
'path for volume: %(name)s, was successfully created: '
'%(device)s realpath is: %(realpath)s', {
'name': volname,
'device': device_info['path'],
'realpath': path.path
})
# Create filesystem on the new device
if fileutil.has_filesystem(path.path) is False:
fileutil.create_filesystem(path.path)
LOG.debug('filesystem successfully created on : %(path)s',
{'path': path.path})
# Determine if we need to mount the volume
if vol_mount is DEFAULT_MOUNT_VOLUME:
# mkdir for mounting the filesystem
mount_dir = fileutil.mkdir_for_mounting(device_info['path'])
LOG.debug(
'Directory: %(mount_dir)s, '
'successfully created to mount: '
'%(mount)s', {
'mount_dir': mount_dir,
'mount': device_info['path']
})
# mount the directory
fileutil.mount_dir(path.path, mount_dir)
LOG.debug('Device: %(path) successfully mounted on %(mount)s', {
'path': path.path,
'mount': mount_dir
})
# TODO: find out how to invoke mkfs so that it creates the
# filesystem without the lost+found directory
# KLUDGE!!!!!
lostfound = mount_dir + '/lost+found'
lfdir = FilePath(lostfound)
if lfdir.exists and fileutil.remove_dir(lostfound):
LOG.debug(
'Successfully removed : '
'%(lost)s from mount: %(mount)s', {
'lost': lostfound,
'mount': mount_dir
})
else:
mount_dir = ''
path_info = {}
path_info['name'] = volname
path_info['path'] = path.path
path_info['device_info'] = device_info
path_info['connection_info'] = connection_info
path_info['mount_dir'] = mount_dir
self._etcd.update_vol(volid, 'path_info', json.dumps(path_info))
response = json.dumps({
u"Err": '',
u"Name": volname,
u"Mountpoint": mount_dir,
u"Devicename": path.path
})
return response
def volumedriver_unmount(self, name):
"""
The Docker container is no longer using the given volume,
so unmount it.
NOTE: Since Docker will automatically call Unmount if the Mount
fails, make sure we properly handle partially completed Mounts.
:param unicode name: The name of the volume.
:return: Result indicating success.
"""
LOG.info(_LI('In volumedriver_unmount'))
contents = json.loads(name.content.getvalue())
volname = contents['Name']
vol = self._etcd.get_vol_byname(volname)
if vol is not None:
volid = vol['id']
else:
msg = (_LE('Volume unmount name not found %s'), volname)
LOG.error(msg)
raise exception.HPEPluginUMountException(reason=msg)
vol_mount = DEFAULT_MOUNT_VOLUME
if ('Opts' in contents and contents['Opts']
and 'mount-volume' in contents['Opts']):
vol_mount = str(contents['Opts']['mount-volume'])
path_info = self._etcd.get_vol_path_info(volname)
if path_info:
path_name = path_info['path']
connection_info = path_info['connection_info']
mount_dir = path_info['mount_dir']
else:
msg = (_LE('Volume unmount path info not found %s'), volname)
LOG.error(msg)
raise exception.HPEPluginUMountException(reason=msg)
# Get connector info from OS Brick
# TODO: retrieve use_multipath and enforce_multipath from config file
root_helper = 'sudo'
connector_info = connector.get_connector_properties(
root_helper,
self._my_ip,
multipath=self.use_multipath,
enforce_multipath=self.enforce_multipath)
# Determine if we need to unmount a previously mounted volume
if vol_mount is DEFAULT_MOUNT_VOLUME:
# unmount directory
fileutil.umount_dir(mount_dir)
# remove directory
fileutil.remove_dir(mount_dir)
# We're deferring the execution of the disconnect_volume as it can take
# substantial
# time (over 2 minutes) to cleanup the iscsi files
if connection_info:
LOG.info(_LI('call os brick to disconnect volume'))
d = threads.deferToThread(self.connector.disconnect_volume,
connection_info['data'], None)
d.addCallbacks(self.disconnect_volume_callback,
self.disconnect_volume_error_callback)
try:
# Call driver to terminate the connection
self.hpeplugin_driver.terminate_connection(vol, connector_info)
LOG.info(
_LI('connection_info: %(connection_info)s, '
'was successfully terminated'),
{'connection_info': json.dumps(connection_info)})
except Exception as ex:
msg = (_LE('connection info termination failed %s'),
six.text_type(ex))
LOG.error(msg)
# Not much we can do here, so just continue on with unmount
# We need to ensure we update etcd path_info so the stale
# path does not stay around
# raise exception.HPEPluginUMountException(reason=msg)
# TODO: Create path_info list as we can mount the volume to multiple
# hosts at the same time.
self._etcd.update_vol(volid, 'path_info', None)
LOG.info(
_LI('path for volume: %(name)s, was successfully removed: '
'%(path_name)s'), {
'name': volname,
'path_name': path_name
})
response = json.dumps({u"Err": ''})
return response
def do_disable_cache(self, new_config, applied_config, lock_ownership):
LOG.info(
_LI("cache_tiering_disable_cache: "
"new_config={}, applied_config={}").format(
new_config, applied_config))
with lock_ownership():
success = False
_exception = None
try:
self.config_desired.cache_enabled = False
for pool in CEPH_POOLS:
if (pool['pool_name']
== constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL
or pool['pool_name']
== constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER):
object_pool_name = \
self.service.monitor._get_object_pool_name()
pool['pool_name'] = object_pool_name
with self.ignore_ceph_failure():
self.cache_mode_set(pool, 'forward')
for pool in CEPH_POOLS:
if (pool['pool_name']
== constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL
or pool['pool_name']
== constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER):
object_pool_name = \
self.service.monitor._get_object_pool_name()
pool['pool_name'] = object_pool_name
retries_left = 3
while True:
try:
self.cache_flush(pool)
break
except exception.CephCacheFlushFailure:
retries_left -= 1
if not retries_left:
# give up
break
else:
time.sleep(1)
for pool in CEPH_POOLS:
if (pool['pool_name']
== constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL
or pool['pool_name']
== constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER):
object_pool_name = \
self.service.monitor._get_object_pool_name()
pool['pool_name'] = object_pool_name
with self.ignore_ceph_failure():
self.cache_overlay_delete(pool)
self.cache_tier_remove(pool)
for pool in CEPH_POOLS:
if (pool['pool_name']
== constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL
or pool['pool_name']
== constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER):
object_pool_name = \
self.service.monitor._get_object_pool_name()
pool['pool_name'] = object_pool_name
with self.ignore_ceph_failure():
self.cache_pool_delete(pool)
success = True
except Exception as e:
LOG.warn(
_LE('Failed to disable cache: reason=%s') %
traceback.format_exc())
_exception = str(e)
finally:
self.service.monitor.monitor_check_cache_tier(False)
if success:
self.config_desired.cache_enabled = False
self.config_applied.cache_enabled = False
self.service.sysinv_conductor.call(
{},
'cache_tiering_disable_cache_complete',
success=success,
exception=_exception,
new_config=new_config.to_dict(),
applied_config=applied_config.to_dict())
def do_enable_cache(self, new_config, applied_config, lock_ownership):
LOG.info(
_LI("cache_tiering_enable_cache: "
"new_config={}, applied_config={}").format(
new_config.to_dict(), applied_config.to_dict()))
_unwind_actions = []
with lock_ownership():
success = False
_exception = None
try:
self.config_desired.cache_enabled = True
self.update_pools_info()
for pool in CEPH_POOLS:
if (pool['pool_name']
== constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL
or pool['pool_name']
== constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER):
object_pool_name = \
self.service.monitor._get_object_pool_name()
pool['pool_name'] = object_pool_name
self.cache_pool_create(pool)
_unwind_actions.append(
functools.partial(self.cache_pool_delete, pool))
for pool in CEPH_POOLS:
if (pool['pool_name']
== constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL
or pool['pool_name']
== constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER):
object_pool_name = \
self.service.monitor._get_object_pool_name()
pool['pool_name'] = object_pool_name
self.cache_tier_add(pool)
_unwind_actions.append(
functools.partial(self.cache_tier_remove, pool))
for pool in CEPH_POOLS:
if (pool['pool_name']
== constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL
or pool['pool_name']
== constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER):
object_pool_name = \
self.service.monitor._get_object_pool_name()
pool['pool_name'] = object_pool_name
self.cache_mode_set(pool, 'writeback')
self.cache_pool_set_config(pool, new_config)
self.cache_overlay_create(pool)
success = True
except Exception as e:
LOG.error(
_LE('Failed to enable cache: reason=%s') %
traceback.format_exc())
for action in reversed(_unwind_actions):
try:
action()
except Exception:
LOG.warn(
_LW('Failed cache enable '
'unwind action: reason=%s') %
traceback.format_exc())
success = False
_exception = str(e)
finally:
self.service.monitor.monitor_check_cache_tier(success)
if success:
self.config_applied.cache_enabled = True
self.service.sysinv_conductor.call(
{},
'cache_tiering_enable_cache_complete',
success=success,
exception=_exception,
new_config=new_config.to_dict(),
applied_config=applied_config.to_dict())
# Run first update of periodic target_max_bytes
self.update_cache_target_max_bytes()
def _report_alarm_osds_health(self):
response, osd_tree = self.service.ceph_api.osd_tree(body='json')
if not response.ok:
LOG.error(
_LE("Failed to retrieve Ceph OSD tree: "
"status_code: %(status_code)s, reason: %(reason)s") % {
"status_code": response.status_code,
"reason": response.reason
})
return
osd_tree = dict([(n['id'], n) for n in osd_tree['output']['nodes']])
alarms = []
self._check_storage_tier(osd_tree, "storage-tier",
lambda *args: alarms.append(args))
old_alarms = {}
for alarm_id in [
fm_constants.FM_ALARM_ID_STORAGE_CEPH_MAJOR,
fm_constants.FM_ALARM_ID_STORAGE_CEPH_CRITICAL
]:
alarm_list = self.service.fm_api.get_faults_by_id(alarm_id)
if not alarm_list:
continue
for alarm in alarm_list:
if alarm.entity_instance_id not in old_alarms:
old_alarms[alarm.entity_instance_id] = []
old_alarms[alarm.entity_instance_id].append(
(alarm.alarm_id, alarm.reason_text))
for peer_group, reason, severity in alarms:
if self._current_health_alarm_equals(reason, severity):
continue
alarm_critical_major = fm_constants.FM_ALARM_ID_STORAGE_CEPH_MAJOR
if severity == fm_constants.FM_ALARM_SEVERITY_CRITICAL:
alarm_critical_major = (
fm_constants.FM_ALARM_ID_STORAGE_CEPH_CRITICAL)
entity_instance_id = (self.service.entity_instance_id +
'.peergroup=' + peer_group)
alarm_already_exists = False
if entity_instance_id in old_alarms:
for alarm_id, old_reason in old_alarms[entity_instance_id]:
if (reason == old_reason
and alarm_id == alarm_critical_major):
# if the alarm is exactly the same, we don't need
# to recreate it
old_alarms[entity_instance_id].remove(
(alarm_id, old_reason))
alarm_already_exists = True
elif (alarm_id == alarm_critical_major):
# if we change just the reason, then we just remove the
# alarm from the list so we don't remove it at the
# end of the function
old_alarms[entity_instance_id].remove(
(alarm_id, old_reason))
if (len(old_alarms[entity_instance_id]) == 0):
del old_alarms[entity_instance_id]
# in case the alarm is exactly the same, we skip the alarm set
if alarm_already_exists is True:
continue
major_repair_action = constants.REPAIR_ACTION_MAJOR_CRITICAL_ALARM
fault = fm_api.Fault(
alarm_id=alarm_critical_major,
alarm_type=fm_constants.FM_ALARM_TYPE_4,
alarm_state=fm_constants.FM_ALARM_STATE_SET,
entity_type_id=fm_constants.FM_ENTITY_TYPE_CLUSTER,
entity_instance_id=entity_instance_id,
severity=severity,
reason_text=reason,
probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_15,
proposed_repair_action=major_repair_action,
service_affecting=constants.SERVICE_AFFECTING['HEALTH_WARN'])
alarm_uuid = self.service.fm_api.set_fault(fault)
if alarm_uuid:
LOG.info(
_LI("Created storage alarm %(alarm_uuid)s - "
"severity: %(severity)s, reason: %(reason)s, "
"service_affecting: %(service_affecting)s") % {
"alarm_uuid":
str(alarm_uuid),
"severity":
str(severity),
"reason":
reason,
"service_affecting":
str(constants.SERVICE_AFFECTING['HEALTH_WARN'])
})
else:
LOG.error(
_LE("Failed to create storage alarm - "
"severity: %(severity)s, reason: %(reason)s, "
"service_affecting: %(service_affecting)s") % {
"severity":
str(severity),
"reason":
reason,
"service_affecting":
str(constants.SERVICE_AFFECTING['HEALTH_WARN'])
})
for entity_instance_id in old_alarms:
for alarm_id, old_reason in old_alarms[entity_instance_id]:
self.service.fm_api.clear_fault(alarm_id, entity_instance_id)
def _report_fault(self, health, alarm_id):
if alarm_id == fm_constants.FM_ALARM_ID_STORAGE_CEPH:
new_severity = constants.SEVERITY[health['health']]
new_reason_text = self._parse_reason(health)
new_service_affecting = \
constants.SERVICE_AFFECTING[health['health']]
# Raise or update alarm if necessary
if ((not self.current_health_alarm)
or (self.current_health_alarm.__dict__['severity'] !=
new_severity)
or (self.current_health_alarm.__dict__['reason_text'] !=
new_reason_text)
or (self.current_health_alarm.__dict__['service_affecting']
!= str(new_service_affecting))):
fault = fm_api.Fault(
alarm_id=fm_constants.FM_ALARM_ID_STORAGE_CEPH,
alarm_type=fm_constants.FM_ALARM_TYPE_4,
alarm_state=fm_constants.FM_ALARM_STATE_SET,
entity_type_id=fm_constants.FM_ENTITY_TYPE_CLUSTER,
entity_instance_id=self.service.entity_instance_id,
severity=new_severity,
reason_text=new_reason_text,
probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_15,
proposed_repair_action=constants.REPAIR_ACTION,
service_affecting=new_service_affecting)
alarm_uuid = self.service.fm_api.set_fault(fault)
if alarm_uuid:
LOG.info(
_LI("Created storage alarm %(alarm_uuid)s - "
"severity: %(severity)s, reason: %(reason)s, "
"service_affecting: %(service_affecting)s") % {
"alarm_uuid": alarm_uuid,
"severity": new_severity,
"reason": new_reason_text,
"service_affecting": new_service_affecting
})
else:
LOG.error(
_LE("Failed to create storage alarm - "
"severity: %(severity)s, reason: %(reason)s "
"service_affecting: %(service_affecting)s") % {
"severity": new_severity,
"reason": new_reason_text,
"service_affecting": new_service_affecting
})
# Log detailed reason for later analysis
if (self.current_ceph_health != health['health']
or self.detailed_health_reason != health['detail']):
LOG.info(
_LI("Ceph status changed: %(health)s "
"detailed reason: %(detail)s") % health)
self.current_ceph_health = health['health']
self.detailed_health_reason = health['detail']
elif (alarm_id == fm_constants.FM_ALARM_ID_STORAGE_CEPH_FREE_SPACE
and not health['tier_eid'] in self.current_quota_alarms):
quota_reason_text = ("Quota/Space mismatch for the %s tier. The "
"sum of Ceph pool quotas does not match the "
"tier size." % health['tier_name'])
fault = fm_api.Fault(
alarm_id=fm_constants.FM_ALARM_ID_STORAGE_CEPH_FREE_SPACE,
alarm_state=fm_constants.FM_ALARM_STATE_SET,
entity_type_id=fm_constants.FM_ENTITY_TYPE_CLUSTER,
entity_instance_id=health['tier_eid'],
severity=fm_constants.FM_ALARM_SEVERITY_MINOR,
reason_text=quota_reason_text,
alarm_type=fm_constants.FM_ALARM_TYPE_7,
probable_cause=fm_constants.ALARM_PROBABLE_CAUSE_75,
proposed_repair_action=(
"Update ceph storage pool quotas to use all available "
"cluster space for the %s tier." % health['tier_name']),
service_affecting=False)
alarm_uuid = self.service.fm_api.set_fault(fault)
if alarm_uuid:
LOG.info(
_LI("Created storage quota storage alarm %(alarm_uuid)s. "
"Reason: %(reason)s") % {
"alarm_uuid": alarm_uuid,
"reason": quota_reason_text
})
else:
LOG.error(
_LE("Failed to create quota "
"storage alarm. Reason: %s") % quota_reason_text)
def volumedriver_mount(self, name):
"""
Mount the volume
Mount the volume
NOTE: If for any reason the mount request fails, Docker
will automatically call uMount. So, just make sure uMount
can handle partially completed Mount requests.
:param unicode name: The name of the volume.
:return: Result that includes the mountpoint.
"""
LOG.debug('In volumedriver_mount')
# TODO: use persistent storage to lookup volume for deletion
contents = {}
contents = json.loads(name.content.getvalue())
volname = contents['Name']
vol = self._etcd.get_vol_byname(volname)
if vol is not None:
volid = vol['id']
else:
msg = (_LE('Volume mount name not found %s'), volname)
LOG.error(msg)
raise exception.HPEPluginMountException(reason=msg)
# Get connector info from OS Brick
# TODO: retrieve use_multipath and enforce_multipath from config file
root_helper = 'sudo'
use_multipath = False
connector_info = connector.get_connector_properties(
root_helper, self._my_ip, use_multipath, enforce_multipath=False)
try:
# Call driver to initialize the connection
self.hpeplugin_driver.create_export(vol, connector_info)
connection_info = \
self.hpeplugin_driver.initialize_connection(
vol, connector_info)
LOG.debug('connection_info: %(connection_info)s, '
'was successfully retrieved',
{'connection_info': json.dumps(connection_info)})
except Exception as ex:
msg = (_('connection info retrieval failed, error is: %s'),
six.text_type(ex))
LOG.error(msg)
raise exception.HPEPluginMountException(reason=msg)
# Call OS Brick to connect volume
try:
device_info = self.connector.\
connect_volume(connection_info['data'])
except Exception as ex:
msg = (_('OS Brick connect volume failed, error is: %s'),
six.text_type(ex))
LOG.error(msg)
raise exception.HPEPluginMountException(reason=msg)
# Make sure the path exists
path = FilePath(device_info['path']).realpath()
if path.exists is False:
msg = (_('path: %s, does not exist'), path)
LOG.error(msg)
raise exception.HPEPluginMountException(reason=msg)
LOG.debug('path for volume: %(name)s, was successfully created: '
'%(device)s realpath is: %(realpath)s',
{'name': volname, 'device': device_info['path'],
'realpath': path.path})
# mkdir for mounting the filesystem
mount_dir = fileutil.mkdir_for_mounting(device_info['path'])
LOG.debug('Directory: %(mount_dir)s, successfully created to mount: '
'%(mount)s',
{'mount_dir': mount_dir, 'mount': device_info['path']})
# Create filesystem on the new device
if fileutil.has_filesystem(path.path) is False:
fileutil.create_filesystem(path.path)
LOG.debug('filesystem successfully created on : %(path)s',
{'path': path.path})
# mount the directory
fileutil.mount_dir(path.path, mount_dir)
LOG.debug('Device: %(path) successfully mounted on %(mount)s',
{'path': path.path, 'mount': mount_dir})
# TODO: find out how to invoke mkfs so that it creates the filesystem
# without the lost+found directory
# KLUDGE!!!!!
lostfound = mount_dir + '/lost+found'
lfdir = FilePath(lostfound)
if lfdir.exists and fileutil.remove_dir(lostfound):
LOG.debug('Successfully removed : %(lost)s from mount: %(mount)s',
{'lost': lostfound, 'mount': mount_dir})
path_info = {}
path_info['name'] = volname
path_info['path'] = path.path
path_info['device_info'] = device_info
path_info['connection_info'] = connection_info
path_info['mount_dir'] = mount_dir
self._etcd.update_vol(volid, 'path_info', json.dumps(path_info))
response = json.dumps({u"Err": '', u"Mountpoint": mount_dir})
return response
def volumedriver_unmount(self, name):
"""
The Docker container is no longer using the given volume,
so unmount it.
NOTE: Since Docker will automatically call Unmount if the Mount
fails, make sure we properly handle partially completed Mounts.
:param unicode name: The name of the volume.
:return: Result indicating success.
"""
LOG.info(_LI('In volumedriver_unmount'))
contents = json.loads(name.content.getvalue())
volname = contents['Name']
vol = self._etcd.get_vol_byname(volname)
if vol is not None:
volid = vol['id']
else:
msg = (_LE('Volume unmount name not found %s'), volname)
LOG.error(msg)
raise exception.HPEPluginUMountException(reason=msg)
path_info = self._etcd.get_vol_path_info(volname)
if path_info:
path_name = path_info['path']
connection_info = path_info['connection_info']
mount_dir = path_info['mount_dir']
else:
msg = (_LE('Volume unmount path info not found %s'), volname)
LOG.error(msg)
raise exception.HPEPluginUMountException(reason=msg)
# Get connector info from OS Brick
# TODO: retrieve use_multipath and enforce_multipath from config file
root_helper = 'sudo'
use_multipath = False
connector_info = connector.get_connector_properties(
root_helper, self._my_ip, use_multipath, enforce_multipath=False)
# unmount directory
fileutil.umount_dir(mount_dir)
# remove directory
fileutil.remove_dir(mount_dir)
try:
# Call driver to terminate the connection
self.hpeplugin_driver.terminate_connection(vol, connector_info)
LOG.info(_LI('connection_info: %(connection_info)s, '
'was successfully terminated'),
{'connection_info': json.dumps(connection_info)})
except Exception as ex:
msg = (_LE('connection info termination failed %s'),
six.text_type(ex))
LOG.error(msg)
raise exception.HPEPluginUMountException(reason=msg)
# We're deferring the execution of the disconnect_volume as it can take
# substantial
# time (over 2 minutes) to cleanup the iscsi files
if connection_info:
LOG.info(_LI('call os brick to disconnect volume'))
d = threads.deferToThread(self.connector.disconnect_volume,
connection_info['data'], None)
d.addCallbacks(self.disconnect_volume_callback,
self.disconnect_volume_error_callback)
# TODO(leeantho) Without this sleep the volume is sometimes not
# removed after the unmount. There must be a different way to fix
# the issue?
time.sleep(1)
# TODO: Create path_info list as we can mount the volume to multiple
# hosts at the same time.
self._etcd.update_vol(volid, 'path_info', None)
LOG.info(_LI('path for volume: %(name)s, was successfully removed: '
'%(path_name)s'), {'name': volname,
'path_name': path_name})
response = json.dumps({u"Err": ''})
return response
def volumedriver_unmount(self, name):
"""
The Docker container is no longer using the given volume,
so unmount it.
NOTE: Since Docker will automatically call Unmount if the Mount
fails, make sure we properly handle partially completed Mounts.
:param unicode name: The name of the volume.
:return: Result indicating success.
"""
LOG.info(_LI('In volumedriver_unmount'))
contents = json.loads(name.content.getvalue())
volname = contents['Name']
vol = self._etcd.get_vol_byname(volname)
if vol is not None:
volid = vol['id']
else:
msg = (_LE('Volume unmount name not found %s'), volname)
LOG.error(msg)
raise exception.HPEPluginUMountException(reason=msg)
vol_mount = DEFAULT_MOUNT_VOLUME
if ('Opts' in contents and contents['Opts'] and
'mount-volume' in contents['Opts']):
vol_mount = str(contents['Opts']['mount-volume'])
path_info = self._etcd.get_vol_path_info(volname)
if path_info:
path_name = path_info['path']
connection_info = path_info['connection_info']
mount_dir = path_info['mount_dir']
else:
msg = (_LE('Volume unmount path info not found %s'), volname)
LOG.error(msg)
raise exception.HPEPluginUMountException(reason=msg)
# Get connector info from OS Brick
# TODO: retrieve use_multipath and enforce_multipath from config file
root_helper = 'sudo'
connector_info = connector.get_connector_properties(
root_helper, self._my_ip, multipath=self.use_multipath,
enforce_multipath=self.enforce_multipath)
# Determine if we need to unmount a previously mounted volume
if vol_mount is DEFAULT_MOUNT_VOLUME:
# unmount directory
fileutil.umount_dir(mount_dir)
# remove directory
fileutil.remove_dir(mount_dir)
# We're deferring the execution of the disconnect_volume as it can take
# substantial
# time (over 2 minutes) to cleanup the iscsi files
if connection_info:
LOG.info(_LI('call os brick to disconnect volume'))
d = threads.deferToThread(self.connector.disconnect_volume,
connection_info['data'], None)
d.addCallbacks(self.disconnect_volume_callback,
self.disconnect_volume_error_callback)
try:
# Call driver to terminate the connection
self.hpeplugin_driver.terminate_connection(vol, connector_info)
LOG.info(_LI('connection_info: %(connection_info)s, '
'was successfully terminated'),
{'connection_info': json.dumps(connection_info)})
except Exception as ex:
msg = (_LE('connection info termination failed %s'),
six.text_type(ex))
LOG.error(msg)
# Not much we can do here, so just continue on with unmount
# We need to ensure we update etcd path_info so the stale
# path does not stay around
# raise exception.HPEPluginUMountException(reason=msg)
# TODO: Create path_info list as we can mount the volume to multiple
# hosts at the same time.
self._etcd.update_vol(volid, 'path_info', None)
LOG.info(_LI('path for volume: %(name)s, was successfully removed: '
'%(path_name)s'), {'name': volname,
'path_name': path_name})
response = json.dumps({u"Err": ''})
return response