def test_paginate_entries_list_with_marker(self):
entries = [{
'reference': {
'name': 'vol03'
},
'size': 1
}, {
'reference': {
'name': 'vol01'
},
'size': 3
}, {
'reference': {
'name': 'vol02'
},
'size': 3
}, {
'reference': {
'name': 'vol04'
},
'size': 2
}, {
'reference': {
'name': 'vol06'
},
'size': 3
}, {
'reference': {
'name': 'vol07'
},
'size': 1
}, {
'reference': {
'name': 'vol05'
},
'size': 1
}]
expected = [{
'reference': {
'name': 'vol04'
},
'size': 2
}, {
'reference': {
'name': 'vol03'
},
'size': 1
}, {
'reference': {
'name': 'vol05'
},
'size': 1
}]
res = volume_utils.paginate_entries_list(entries, {'name': 'vol02'}, 3,
1, ['size', 'reference'],
['desc', 'asc'])
self.assertEqual(expected, res)
def _get_manageable_resource_info(self, cinder_resources, resource_type,
marker, limit, offset, sort_keys,
sort_dirs):
entries = []
cinder_ids = [resource['id'] for resource in cinder_resources]
root_helper = utils.get_root_helper()
try:
out, err = self._execute('zfs',
'list',
'-r',
'-H',
'-p',
'-t',
resource_type,
'-oname,volsize',
self.zpool,
root_helper=root_helper,
run_as_root=True)
except processutils.ProcessExecutionError as exc:
exception_message = (_("Failed to zfs list, "
"error message was: %s") %
six.text_type(exc.stderr))
raise exception.VolumeBackendAPIException(data=exception_message)
for entry in out.splitlines():
name, size = entry.strip().split('\t')
if resource_type == 'volume':
potential_id = volutils.extract_id_from_volume_name(name)
else:
potential_id = volutils.extract_id_from_snapshot_name(name)
info = {
'reference': {
'source-name': name
},
'size': int(math.ceil(float(size) / units.Gi)),
'cinder_id': None,
'extra_info': None
}
if potential_id in cinder_ids:
info['safe_to_manage'] = False
info['reason_not_safe'] = 'already managed'
info['cinder_id'] = potential_id
else:
info['safe_to_manage'] = True
info['reason_not_safe'] = None
if resource_type == 'snapshot':
zpool, zvol, snapshot = name.replace('@', '/').split('/')
info['source_reference'] = {'source-name': zvol}
entries.append(info)
return volutils.paginate_entries_list(entries, marker, limit, offset,
sort_keys, sort_dirs)
def _get_manageable_volumes_2_1(self, cinder_volumes, marker, limit,
offset, sort_keys, sort_dirs):
# Use the first volume to determine the tenant we're working under
if cinder_volumes:
tenant = self._create_tenant(cinder_volumes[0])
else:
tenant = None
LOG.debug("Listing manageable Datera volumes")
app_instances = self._issue_api_request(datc.URL_TEMPLATES['ai'](),
api_version='2.1',
tenant=tenant)['data']
results = []
cinder_volume_ids = [vol['id'] for vol in cinder_volumes]
for ai in app_instances:
ai_name = ai['name']
reference = None
size = None
safe_to_manage = False
reason_not_safe = ""
cinder_id = None
extra_info = None
if re.match(datc.UUID4_RE, ai_name):
cinder_id = ai_name.lstrip(datc.OS_PREFIX)
if (not cinder_id and ai_name.lstrip(datc.OS_PREFIX)
not in cinder_volume_ids):
safe_to_manage, reason_not_safe = self._is_manageable_2_1(ai)
if safe_to_manage:
si = list(ai['storage_instances'].values())[0]
si_name = si['name']
vol = list(si['volumes'].values())[0]
vol_name = vol['name']
size = vol['size']
reference = {
"source-name": "{}:{}:{}".format(ai_name, si_name,
vol_name)
}
results.append({
'reference': reference,
'size': size,
'safe_to_manage': safe_to_manage,
'reason_not_safe': reason_not_safe,
'cinder_id': cinder_id,
'extra_info': extra_info
})
page_results = volutils.paginate_entries_list(results, marker, limit,
offset, sort_keys,
sort_dirs)
return page_results
def _get_manageable_volumes_2_1(self, cinder_volumes, marker, limit,
offset, sort_keys, sort_dirs):
# Use the first volume to determine the tenant we're working under
if cinder_volumes:
tenant = self._create_tenant(cinder_volumes[0])
else:
tenant = None
LOG.debug("Listing manageable Datera volumes")
app_instances = self._issue_api_request(
datc.URL_TEMPLATES['ai'](), api_version='2.1',
tenant=tenant)['data']
results = []
cinder_volume_ids = [vol['id'] for vol in cinder_volumes]
for ai in app_instances:
ai_name = ai['name']
reference = None
size = None
safe_to_manage = False
reason_not_safe = ""
cinder_id = None
extra_info = None
if re.match(datc.UUID4_RE, ai_name):
cinder_id = ai_name.lstrip(datc.OS_PREFIX)
if (not cinder_id and
ai_name.lstrip(datc.OS_PREFIX) not in cinder_volume_ids):
safe_to_manage, reason_not_safe = self._is_manageable_2_1(ai)
if safe_to_manage:
si = list(ai['storage_instances'].values())[0]
si_name = si['name']
vol = list(si['volumes'].values())[0]
vol_name = vol['name']
size = vol['size']
reference = {"source-name": "{}:{}:{}".format(
ai_name, si_name, vol_name)}
results.append({
'reference': reference,
'size': size,
'safe_to_manage': safe_to_manage,
'reason_not_safe': reason_not_safe,
'cinder_id': cinder_id,
'extra_info': extra_info})
page_results = volutils.paginate_entries_list(
results, marker, limit, offset, sort_keys, sort_dirs)
return page_results
def test_paginate_entries_list_without_marker(self):
entries = [{'reference': {'name': 'vol03'}, 'size': 1},
{'reference': {'name': 'vol01'}, 'size': 3},
{'reference': {'name': 'vol02'}, 'size': 3},
{'reference': {'name': 'vol04'}, 'size': 2},
{'reference': {'name': 'vol06'}, 'size': 3},
{'reference': {'name': 'vol07'}, 'size': 1},
{'reference': {'name': 'vol05'}, 'size': 1}]
expected = [{'reference': {'name': 'vol07'}, 'size': 1},
{'reference': {'name': 'vol06'}, 'size': 3},
{'reference': {'name': 'vol05'}, 'size': 1}]
res = volume_utils.paginate_entries_list(entries, None, 3, None,
['reference'], ['desc'])
self.assertEqual(expected, res)
def _get_manageable_resource_info(self, cinder_resources, resource_type,
marker, limit, offset, sort_keys,
sort_dirs):
entries = []
lvs = self.vg.get_volumes()
cinder_ids = [resource['id'] for resource in cinder_resources]
for lv in lvs:
is_snap = self.vg.lv_is_snapshot(lv['name'])
if ((resource_type == 'volume' and is_snap)
or (resource_type == 'snapshot' and not is_snap)):
continue
if resource_type == 'volume':
potential_id = volutils.extract_id_from_volume_name(lv['name'])
else:
unescape = self._unescape_snapshot(lv['name'])
potential_id = volutils.extract_id_from_snapshot_name(unescape)
lv_info = {
'reference': {
'source-name': lv['name']
},
'size': int(math.ceil(float(lv['size']))),
'cinder_id': None,
'extra_info': None
}
if potential_id in cinder_ids:
lv_info['safe_to_manage'] = False
lv_info['reason_not_safe'] = 'already managed'
lv_info['cinder_id'] = potential_id
elif self.vg.lv_is_open(lv['name']):
lv_info['safe_to_manage'] = False
lv_info['reason_not_safe'] = '%s in use' % resource_type
else:
lv_info['safe_to_manage'] = True
lv_info['reason_not_safe'] = None
if resource_type == 'snapshot':
origin = self.vg.lv_get_origin(lv['name'])
lv_info['source_reference'] = {'source-name': origin}
entries.append(lv_info)
return volutils.paginate_entries_list(entries, marker, limit, offset,
sort_keys, sort_dirs)
def test_paginate_entries_list_without_marker(self):
entries = [
{"reference": {"name": "vol03"}, "size": 1},
{"reference": {"name": "vol01"}, "size": 3},
{"reference": {"name": "vol02"}, "size": 3},
{"reference": {"name": "vol04"}, "size": 2},
{"reference": {"name": "vol06"}, "size": 3},
{"reference": {"name": "vol07"}, "size": 1},
{"reference": {"name": "vol05"}, "size": 1},
]
expected = [
{"reference": {"name": "vol07"}, "size": 1},
{"reference": {"name": "vol06"}, "size": 3},
{"reference": {"name": "vol05"}, "size": 1},
]
res = volume_utils.paginate_entries_list(entries, None, 3, None, ["reference"], ["desc"])
self.assertEqual(expected, res)
def _get_manageable_resource_info(self, cinder_resources, resource_type,
marker, limit, offset, sort_keys,
sort_dirs):
entries = []
lvs = self.vg.get_volumes()
cinder_ids = [resource['id'] for resource in cinder_resources]
for lv in lvs:
is_snap = self.vg.lv_is_snapshot(lv['name'])
if ((resource_type == 'volume' and is_snap) or
(resource_type == 'snapshot' and not is_snap)):
continue
if resource_type == 'volume':
potential_id = volutils.extract_id_from_volume_name(lv['name'])
else:
unescape = self._unescape_snapshot(lv['name'])
potential_id = volutils.extract_id_from_snapshot_name(unescape)
lv_info = {'reference': {'source-name': lv['name']},
'size': int(math.ceil(float(lv['size']))),
'cinder_id': None,
'extra_info': None}
if potential_id in cinder_ids:
lv_info['safe_to_manage'] = False
lv_info['reason_not_safe'] = 'already managed'
lv_info['cinder_id'] = potential_id
elif self.vg.lv_is_open(lv['name']):
lv_info['safe_to_manage'] = False
lv_info['reason_not_safe'] = '%s in use' % resource_type
else:
lv_info['safe_to_manage'] = True
lv_info['reason_not_safe'] = None
if resource_type == 'snapshot':
origin = self.vg.lv_get_origin(lv['name'])
lv_info['source_reference'] = {'source-name': origin}
entries.append(lv_info)
return volutils.paginate_entries_list(entries, marker, limit, offset,
sort_keys, sort_dirs)
def _get_manageable_resource_info(
self, cinder_resources, resource_type, marker, limit, offset, sort_keys, sort_dirs
):
entries = []
lvs = self.vg.get_volumes()
cinder_ids = [resource["id"] for resource in cinder_resources]
for lv in lvs:
is_snap = self.vg.lv_is_snapshot(lv["name"])
if (resource_type == "volume" and is_snap) or (resource_type == "snapshot" and not is_snap):
continue
if resource_type == "volume":
potential_id = volutils.extract_id_from_volume_name(lv["name"])
else:
unescape = self._unescape_snapshot(lv["name"])
potential_id = volutils.extract_id_from_snapshot_name(unescape)
lv_info = {
"reference": {"source-name": lv["name"]},
"size": int(math.ceil(float(lv["size"]))),
"cinder_id": None,
"extra_info": None,
}
if potential_id in cinder_ids:
lv_info["safe_to_manage"] = False
lv_info["reason_not_safe"] = "already managed"
lv_info["cinder_id"] = potential_id
elif self.vg.lv_is_open(lv["name"]):
lv_info["safe_to_manage"] = False
lv_info["reason_not_safe"] = "%s in use" % resource_type
else:
lv_info["safe_to_manage"] = True
lv_info["reason_not_safe"] = None
if resource_type == "snapshot":
origin = self.vg.lv_get_origin(lv["name"])
lv_info["source_reference"] = {"source-name": origin}
entries.append(lv_info)
return volutils.paginate_entries_list(entries, marker, limit, offset, sort_keys, sort_dirs)
def _get_manageable_resource_info(self, cinder_resources, resource_type,
marker, limit, offset, sort_keys,
sort_dirs):
"""Gets the resources on the backend available for management by Cinder.
Receives the parameters from "get_manageable_volumes" and
"get_manageable_snapshots" and gets the available resources
:param cinder_resources: A list of resources in this host that Cinder
currently manages
:param resource_type: If it's a volume or a snapshot
:param marker: The last item of the previous page; we return the
next results after this value (after sorting)
:param limit: Maximum number of items to return
:param offset: Number of items to skip after marker
:param sort_keys: List of keys to sort results by (valid keys
are 'identifier' and 'size')
:param sort_dirs: List of directions to sort by, corresponding to
sort_keys (valid directions are 'asc' and 'desc')
:returns: list of dictionaries, each specifying a volume or snapshot
(resource) in the host, with the following keys:
- reference (dictionary): The reference for a resource,
which can be passed to "manage_existing_snapshot".
- size (int): The size of the resource according to the storage
backend, rounded up to the nearest GB.
- safe_to_manage (boolean): Whether or not this resource is
safe to manage according to the storage backend.
- reason_not_safe (string): If safe_to_manage is False,
the reason why.
- cinder_id (string): If already managed, provide the Cinder ID.
- extra_info (string): Any extra information to return to the
user
- source_reference (string): Similar to "reference", but for the
snapshot's source volume.
"""
entries = []
exports = {}
bend_rsrc = {}
cinder_ids = [resource.id for resource in cinder_resources]
for service in self.config['services']:
exp_path = self.config['services'][service]['hdp']
exports[exp_path] = (
self.config['services'][service]['export']['fs'])
for exp in exports.keys():
# bend_rsrc has all the resources in the specified exports
# volumes {u'172.24.54.39:/Export-Cinder':
# ['volume-325e7cdc-8f65-40a8-be9a-6172c12c9394',
# ' snapshot-1bfb6f0d-9497-4c12-a052-5426a76cacdc','']}
bend_rsrc[exp] = self._get_volumes_from_export(exp)
mnt_point = self._get_mount_point_for_share(exp)
for resource in bend_rsrc[exp]:
# Ignoring resources of unwanted types
if ((resource_type == 'volume' and
('.' in resource or 'snapshot' in resource))
or (resource_type == 'snapshot' and '.' not in resource
and 'snapshot' not in resource)):
continue
path = '%s/%s' % (exp, resource)
mnt_path = '%s/%s' % (mnt_point, resource)
size = self._get_file_size(mnt_path)
rsrc_inf = {
'reference': {
'source-name': path
},
'size': size,
'cinder_id': None,
'extra_info': None
}
if resource_type == 'volume':
potential_id = utils.extract_id_from_volume_name(resource)
elif 'snapshot' in resource:
# This is for the snapshot legacy case
potential_id = utils.extract_id_from_snapshot_name(
resource)
else:
potential_id = resource.split('.')[1]
# When a resource is already managed by cinder, it's not
# recommended to manage it again. So we set safe_to_manage =
# False. Otherwise, it is set safe_to_manage = True.
if potential_id in cinder_ids:
rsrc_inf['safe_to_manage'] = False
rsrc_inf['reason_not_safe'] = 'already managed'
rsrc_inf['cinder_id'] = potential_id
else:
rsrc_inf['safe_to_manage'] = True
rsrc_inf['reason_not_safe'] = None
# If it's a snapshot, we try to get its source volume. However,
# this search is not reliable in some cases. So, if it's not
# possible to return a precise result, we return unknown as
# source-reference, throw a warning message and fill the
# extra-info.
if resource_type == 'snapshot':
if 'snapshot' not in resource:
origin = self._get_snapshot_origin_from_name(resource)
if 'unmanage' in origin:
origin = origin[16:]
else:
origin = origin[7:]
rsrc_inf['source_reference'] = {'id': origin}
else:
path = path.split(':')[1]
origin = self._get_snapshot_origin(path, exports[exp])
if not origin:
# if origin is empty, the file is not a clone
continue
elif len(origin) == 1:
origin = origin[0].split('/')[2]
origin = utils.extract_id_from_volume_name(origin)
rsrc_inf['source_reference'] = {'id': origin}
else:
LOG.warning(
"Could not determine the volume "
"that owns the snapshot %(snap)s",
{'snap': resource})
rsrc_inf['source_reference'] = {'id': 'unknown'}
rsrc_inf['extra_info'] = ('Could not determine '
'the volume that owns '
'the snapshot')
entries.append(rsrc_inf)
return utils.paginate_entries_list(entries, marker, limit, offset,
sort_keys, sort_dirs)
def _get_manageable_resource_info(self, cinder_resources, resource_type,
marker, limit, offset, sort_keys,
sort_dirs):
"""Gets the resources on the backend available for management by Cinder.
Receives the parameters from "get_manageable_volumes" and
"get_manageable_snapshots" and gets the available resources
:param cinder_resources: A list of resources in this host that Cinder
currently manages
:param resource_type: If it's a volume or a snapshot
:param marker: The last item of the previous page; we return the
next results after this value (after sorting)
:param limit: Maximum number of items to return
:param offset: Number of items to skip after marker
:param sort_keys: List of keys to sort results by (valid keys
are 'identifier' and 'size')
:param sort_dirs: List of directions to sort by, corresponding to
sort_keys (valid directions are 'asc' and 'desc')
:returns: list of dictionaries, each specifying a volume or snapshot
(resource) in the host, with the following keys:
- reference (dictionary): The reference for a resource,
which can be passed to "manage_existing_snapshot".
- size (int): The size of the resource according to the storage
backend, rounded up to the nearest GB.
- safe_to_manage (boolean): Whether or not this resource is
safe to manage according to the storage backend.
- reason_not_safe (string): If safe_to_manage is False,
the reason why.
- cinder_id (string): If already managed, provide the Cinder ID.
- extra_info (string): Any extra information to return to the
user
- source_reference (string): Similar to "reference", but for the
snapshot's source volume.
"""
entries = []
exports = {}
bend_rsrc = {}
cinder_ids = [resource.id for resource in cinder_resources]
for service in self.config['services']:
exp_path = self.config['services'][service]['hdp']
exports[exp_path] = (
self.config['services'][service]['export']['fs'])
for exp in exports.keys():
# bend_rsrc has all the resources in the specified exports
# volumes {u'172.24.54.39:/Export-Cinder':
# ['volume-325e7cdc-8f65-40a8-be9a-6172c12c9394',
# ' snapshot-1bfb6f0d-9497-4c12-a052-5426a76cacdc','']}
bend_rsrc[exp] = self._get_volumes_from_export(exp)
mnt_point = self._get_mount_point_for_share(exp)
for resource in bend_rsrc[exp]:
# Ignoring resources of unwanted types
if ((resource_type == 'volume' and 'snapshot' in resource) or
(resource_type == 'snapshot' and
'volume' in resource)):
continue
path = '%s/%s' % (exp, resource)
mnt_path = '%s/%s' % (mnt_point, resource)
size = self._get_file_size(mnt_path)
rsrc_inf = {'reference': {'source-name': path},
'size': size, 'cinder_id': None,
'extra_info': None}
if resource_type == 'volume':
potential_id = utils.extract_id_from_volume_name(resource)
else:
potential_id = utils.extract_id_from_snapshot_name(
resource)
# When a resource is already managed by cinder, it's not
# recommended to manage it again. So we set safe_to_manage =
# False. Otherwise, it is set safe_to_manage = True.
if potential_id in cinder_ids:
rsrc_inf['safe_to_manage'] = False
rsrc_inf['reason_not_safe'] = 'already managed'
rsrc_inf['cinder_id'] = potential_id
else:
rsrc_inf['safe_to_manage'] = True
rsrc_inf['reason_not_safe'] = None
# If it's a snapshot, we try to get its source volume. However,
# this search is not reliable in some cases. So, if it's not
# possible to return a precise result, we return unknown as
# source-reference, throw a warning message and fill the
# extra-info.
if resource_type == 'snapshot':
path = path.split(':')[1]
origin = self._get_snapshot_origin(path, exports[exp])
if not origin:
# if origin is empty, the file is not a clone
continue
elif len(origin) == 1:
origin = origin[0].split('/')[2]
origin = utils.extract_id_from_volume_name(origin)
rsrc_inf['source_reference'] = {'id': origin}
else:
LOG.warning(_LW("Could not determine the volume that "
"owns the snapshot %(snap)s"),
{'snap': resource})
rsrc_inf['source_reference'] = {'id': 'unknown'}
rsrc_inf['extra_info'] = ('Could not determine the '
'volume that owns the '
'snapshot')
entries.append(rsrc_inf)
return utils.paginate_entries_list(entries, marker, limit, offset,
sort_keys, sort_dirs)
def get_manageable_volumes(self, cinder_volumes, marker, limit, offset,
sort_keys, sort_dirs):
"""List volumes on the backend available for management by Cinder.
Returns a list of dictionaries, each specifying a volume in the host,
with the following keys:
- reference (dictionary): The reference for a volume, which can be
passed to "manage_existing".
- size (int): The size of the volume according to the storage
backend, rounded up to the nearest GB.
- safe_to_manage (boolean): Whether or not this volume is safe to
manage according to the storage backend. For example, is the volume
in use or invalid for any reason.
- reason_not_safe (string): If safe_to_manage is False, the reason why.
- cinder_id (string): If already managed, provide the Cinder ID.
- extra_info (string): Any extra information to return to the user
:param cinder_volumes: A list of volumes in this host that Cinder
currently manages, used to determine if
a volume is manageable or not.
:param marker: The last item of the previous page; we return the
next results after this value (after sorting)
:param limit: Maximum number of items to return
:param offset: Number of items to skip after marker
:param sort_keys: List of keys to sort results by (valid keys are
'identifier' and 'size')
:param sort_dirs: List of directions to sort by, corresponding to
sort_keys (valid directions are 'asc' and 'desc')
"""
LOG.debug("Listing manageable Datera volumes")
app_instances = self._issue_api_request(URL_TEMPLATES['ai']()).values()
results = []
cinder_volume_ids = [vol['id'] for vol in cinder_volumes]
for ai in app_instances:
ai_name = ai['name']
reference = None
size = None
safe_to_manage = False
reason_not_safe = None
cinder_id = None
extra_info = None
if re.match(UUID4_RE, ai_name):
cinder_id = ai_name.lstrip(OS_PREFIX)
if (not cinder_id and
ai_name.lstrip(OS_PREFIX) not in cinder_volume_ids):
safe_to_manage = self._is_manageable(ai)
if safe_to_manage:
si = list(ai['storage_instances'].values())[0]
si_name = si['name']
vol = list(si['volumes'].values())[0]
vol_name = vol['name']
size = vol['size']
reference = {"source-name": "{}:{}:{}".format(
ai_name, si_name, vol_name)}
results.append({
'reference': reference,
'size': size,
'safe_to_manage': safe_to_manage,
'reason_not_safe': reason_not_safe,
'cinder_id': cinder_id,
'extra_info': extra_info})
page_results = volutils.paginate_entries_list(
results, marker, limit, offset, sort_keys, sort_dirs)
return page_results
