def test_paginate_entries_list_with_marker(self): entries = [{ 'reference': { 'name': 'vol03' }, 'size': 1 }, { 'reference': { 'name': 'vol01' }, 'size': 3 }, { 'reference': { 'name': 'vol02' }, 'size': 3 }, { 'reference': { 'name': 'vol04' }, 'size': 2 }, { 'reference': { 'name': 'vol06' }, 'size': 3 }, { 'reference': { 'name': 'vol07' }, 'size': 1 }, { 'reference': { 'name': 'vol05' }, 'size': 1 }] expected = [{ 'reference': { 'name': 'vol04' }, 'size': 2 }, { 'reference': { 'name': 'vol03' }, 'size': 1 }, { 'reference': { 'name': 'vol05' }, 'size': 1 }] res = volume_utils.paginate_entries_list(entries, {'name': 'vol02'}, 3, 1, ['size', 'reference'], ['desc', 'asc']) self.assertEqual(expected, res)
def _get_manageable_resource_info(self, cinder_resources, resource_type, marker, limit, offset, sort_keys, sort_dirs): entries = [] cinder_ids = [resource['id'] for resource in cinder_resources] root_helper = utils.get_root_helper() try: out, err = self._execute('zfs', 'list', '-r', '-H', '-p', '-t', resource_type, '-oname,volsize', self.zpool, root_helper=root_helper, run_as_root=True) except processutils.ProcessExecutionError as exc: exception_message = (_("Failed to zfs list, " "error message was: %s") % six.text_type(exc.stderr)) raise exception.VolumeBackendAPIException(data=exception_message) for entry in out.splitlines(): name, size = entry.strip().split('\t') if resource_type == 'volume': potential_id = volutils.extract_id_from_volume_name(name) else: potential_id = volutils.extract_id_from_snapshot_name(name) info = { 'reference': { 'source-name': name }, 'size': int(math.ceil(float(size) / units.Gi)), 'cinder_id': None, 'extra_info': None } if potential_id in cinder_ids: info['safe_to_manage'] = False info['reason_not_safe'] = 'already managed' info['cinder_id'] = potential_id else: info['safe_to_manage'] = True info['reason_not_safe'] = None if resource_type == 'snapshot': zpool, zvol, snapshot = name.replace('@', '/').split('/') info['source_reference'] = {'source-name': zvol} entries.append(info) return volutils.paginate_entries_list(entries, marker, limit, offset, sort_keys, sort_dirs)
def _get_manageable_volumes_2_1(self, cinder_volumes, marker, limit, offset, sort_keys, sort_dirs): # Use the first volume to determine the tenant we're working under if cinder_volumes: tenant = self._create_tenant(cinder_volumes[0]) else: tenant = None LOG.debug("Listing manageable Datera volumes") app_instances = self._issue_api_request(datc.URL_TEMPLATES['ai'](), api_version='2.1', tenant=tenant)['data'] results = [] cinder_volume_ids = [vol['id'] for vol in cinder_volumes] for ai in app_instances: ai_name = ai['name'] reference = None size = None safe_to_manage = False reason_not_safe = "" cinder_id = None extra_info = None if re.match(datc.UUID4_RE, ai_name): cinder_id = ai_name.lstrip(datc.OS_PREFIX) if (not cinder_id and ai_name.lstrip(datc.OS_PREFIX) not in cinder_volume_ids): safe_to_manage, reason_not_safe = self._is_manageable_2_1(ai) if safe_to_manage: si = list(ai['storage_instances'].values())[0] si_name = si['name'] vol = list(si['volumes'].values())[0] vol_name = vol['name'] size = vol['size'] reference = { "source-name": "{}:{}:{}".format(ai_name, si_name, vol_name) } results.append({ 'reference': reference, 'size': size, 'safe_to_manage': safe_to_manage, 'reason_not_safe': reason_not_safe, 'cinder_id': cinder_id, 'extra_info': extra_info }) page_results = volutils.paginate_entries_list(results, marker, limit, offset, sort_keys, sort_dirs) return page_results
def _get_manageable_volumes_2_1(self, cinder_volumes, marker, limit, offset, sort_keys, sort_dirs): # Use the first volume to determine the tenant we're working under if cinder_volumes: tenant = self._create_tenant(cinder_volumes[0]) else: tenant = None LOG.debug("Listing manageable Datera volumes") app_instances = self._issue_api_request( datc.URL_TEMPLATES['ai'](), api_version='2.1', tenant=tenant)['data'] results = [] cinder_volume_ids = [vol['id'] for vol in cinder_volumes] for ai in app_instances: ai_name = ai['name'] reference = None size = None safe_to_manage = False reason_not_safe = "" cinder_id = None extra_info = None if re.match(datc.UUID4_RE, ai_name): cinder_id = ai_name.lstrip(datc.OS_PREFIX) if (not cinder_id and ai_name.lstrip(datc.OS_PREFIX) not in cinder_volume_ids): safe_to_manage, reason_not_safe = self._is_manageable_2_1(ai) if safe_to_manage: si = list(ai['storage_instances'].values())[0] si_name = si['name'] vol = list(si['volumes'].values())[0] vol_name = vol['name'] size = vol['size'] reference = {"source-name": "{}:{}:{}".format( ai_name, si_name, vol_name)} results.append({ 'reference': reference, 'size': size, 'safe_to_manage': safe_to_manage, 'reason_not_safe': reason_not_safe, 'cinder_id': cinder_id, 'extra_info': extra_info}) page_results = volutils.paginate_entries_list( results, marker, limit, offset, sort_keys, sort_dirs) return page_results
def test_paginate_entries_list_without_marker(self): entries = [{'reference': {'name': 'vol03'}, 'size': 1}, {'reference': {'name': 'vol01'}, 'size': 3}, {'reference': {'name': 'vol02'}, 'size': 3}, {'reference': {'name': 'vol04'}, 'size': 2}, {'reference': {'name': 'vol06'}, 'size': 3}, {'reference': {'name': 'vol07'}, 'size': 1}, {'reference': {'name': 'vol05'}, 'size': 1}] expected = [{'reference': {'name': 'vol07'}, 'size': 1}, {'reference': {'name': 'vol06'}, 'size': 3}, {'reference': {'name': 'vol05'}, 'size': 1}] res = volume_utils.paginate_entries_list(entries, None, 3, None, ['reference'], ['desc']) self.assertEqual(expected, res)
def _get_manageable_resource_info(self, cinder_resources, resource_type, marker, limit, offset, sort_keys, sort_dirs): entries = [] lvs = self.vg.get_volumes() cinder_ids = [resource['id'] for resource in cinder_resources] for lv in lvs: is_snap = self.vg.lv_is_snapshot(lv['name']) if ((resource_type == 'volume' and is_snap) or (resource_type == 'snapshot' and not is_snap)): continue if resource_type == 'volume': potential_id = volutils.extract_id_from_volume_name(lv['name']) else: unescape = self._unescape_snapshot(lv['name']) potential_id = volutils.extract_id_from_snapshot_name(unescape) lv_info = { 'reference': { 'source-name': lv['name'] }, 'size': int(math.ceil(float(lv['size']))), 'cinder_id': None, 'extra_info': None } if potential_id in cinder_ids: lv_info['safe_to_manage'] = False lv_info['reason_not_safe'] = 'already managed' lv_info['cinder_id'] = potential_id elif self.vg.lv_is_open(lv['name']): lv_info['safe_to_manage'] = False lv_info['reason_not_safe'] = '%s in use' % resource_type else: lv_info['safe_to_manage'] = True lv_info['reason_not_safe'] = None if resource_type == 'snapshot': origin = self.vg.lv_get_origin(lv['name']) lv_info['source_reference'] = {'source-name': origin} entries.append(lv_info) return volutils.paginate_entries_list(entries, marker, limit, offset, sort_keys, sort_dirs)
def test_paginate_entries_list_without_marker(self): entries = [ {"reference": {"name": "vol03"}, "size": 1}, {"reference": {"name": "vol01"}, "size": 3}, {"reference": {"name": "vol02"}, "size": 3}, {"reference": {"name": "vol04"}, "size": 2}, {"reference": {"name": "vol06"}, "size": 3}, {"reference": {"name": "vol07"}, "size": 1}, {"reference": {"name": "vol05"}, "size": 1}, ] expected = [ {"reference": {"name": "vol07"}, "size": 1}, {"reference": {"name": "vol06"}, "size": 3}, {"reference": {"name": "vol05"}, "size": 1}, ] res = volume_utils.paginate_entries_list(entries, None, 3, None, ["reference"], ["desc"]) self.assertEqual(expected, res)
def _get_manageable_resource_info(self, cinder_resources, resource_type, marker, limit, offset, sort_keys, sort_dirs): entries = [] lvs = self.vg.get_volumes() cinder_ids = [resource['id'] for resource in cinder_resources] for lv in lvs: is_snap = self.vg.lv_is_snapshot(lv['name']) if ((resource_type == 'volume' and is_snap) or (resource_type == 'snapshot' and not is_snap)): continue if resource_type == 'volume': potential_id = volutils.extract_id_from_volume_name(lv['name']) else: unescape = self._unescape_snapshot(lv['name']) potential_id = volutils.extract_id_from_snapshot_name(unescape) lv_info = {'reference': {'source-name': lv['name']}, 'size': int(math.ceil(float(lv['size']))), 'cinder_id': None, 'extra_info': None} if potential_id in cinder_ids: lv_info['safe_to_manage'] = False lv_info['reason_not_safe'] = 'already managed' lv_info['cinder_id'] = potential_id elif self.vg.lv_is_open(lv['name']): lv_info['safe_to_manage'] = False lv_info['reason_not_safe'] = '%s in use' % resource_type else: lv_info['safe_to_manage'] = True lv_info['reason_not_safe'] = None if resource_type == 'snapshot': origin = self.vg.lv_get_origin(lv['name']) lv_info['source_reference'] = {'source-name': origin} entries.append(lv_info) return volutils.paginate_entries_list(entries, marker, limit, offset, sort_keys, sort_dirs)
def _get_manageable_resource_info( self, cinder_resources, resource_type, marker, limit, offset, sort_keys, sort_dirs ): entries = [] lvs = self.vg.get_volumes() cinder_ids = [resource["id"] for resource in cinder_resources] for lv in lvs: is_snap = self.vg.lv_is_snapshot(lv["name"]) if (resource_type == "volume" and is_snap) or (resource_type == "snapshot" and not is_snap): continue if resource_type == "volume": potential_id = volutils.extract_id_from_volume_name(lv["name"]) else: unescape = self._unescape_snapshot(lv["name"]) potential_id = volutils.extract_id_from_snapshot_name(unescape) lv_info = { "reference": {"source-name": lv["name"]}, "size": int(math.ceil(float(lv["size"]))), "cinder_id": None, "extra_info": None, } if potential_id in cinder_ids: lv_info["safe_to_manage"] = False lv_info["reason_not_safe"] = "already managed" lv_info["cinder_id"] = potential_id elif self.vg.lv_is_open(lv["name"]): lv_info["safe_to_manage"] = False lv_info["reason_not_safe"] = "%s in use" % resource_type else: lv_info["safe_to_manage"] = True lv_info["reason_not_safe"] = None if resource_type == "snapshot": origin = self.vg.lv_get_origin(lv["name"]) lv_info["source_reference"] = {"source-name": origin} entries.append(lv_info) return volutils.paginate_entries_list(entries, marker, limit, offset, sort_keys, sort_dirs)
def _get_manageable_resource_info(self, cinder_resources, resource_type, marker, limit, offset, sort_keys, sort_dirs): """Gets the resources on the backend available for management by Cinder. Receives the parameters from "get_manageable_volumes" and "get_manageable_snapshots" and gets the available resources :param cinder_resources: A list of resources in this host that Cinder currently manages :param resource_type: If it's a volume or a snapshot :param marker: The last item of the previous page; we return the next results after this value (after sorting) :param limit: Maximum number of items to return :param offset: Number of items to skip after marker :param sort_keys: List of keys to sort results by (valid keys are 'identifier' and 'size') :param sort_dirs: List of directions to sort by, corresponding to sort_keys (valid directions are 'asc' and 'desc') :returns: list of dictionaries, each specifying a volume or snapshot (resource) in the host, with the following keys: - reference (dictionary): The reference for a resource, which can be passed to "manage_existing_snapshot". - size (int): The size of the resource according to the storage backend, rounded up to the nearest GB. - safe_to_manage (boolean): Whether or not this resource is safe to manage according to the storage backend. - reason_not_safe (string): If safe_to_manage is False, the reason why. - cinder_id (string): If already managed, provide the Cinder ID. - extra_info (string): Any extra information to return to the user - source_reference (string): Similar to "reference", but for the snapshot's source volume. """ entries = [] exports = {} bend_rsrc = {} cinder_ids = [resource.id for resource in cinder_resources] for service in self.config['services']: exp_path = self.config['services'][service]['hdp'] exports[exp_path] = ( self.config['services'][service]['export']['fs']) for exp in exports.keys(): # bend_rsrc has all the resources in the specified exports # volumes {u'172.24.54.39:/Export-Cinder': # ['volume-325e7cdc-8f65-40a8-be9a-6172c12c9394', # ' snapshot-1bfb6f0d-9497-4c12-a052-5426a76cacdc','']} bend_rsrc[exp] = self._get_volumes_from_export(exp) mnt_point = self._get_mount_point_for_share(exp) for resource in bend_rsrc[exp]: # Ignoring resources of unwanted types if ((resource_type == 'volume' and ('.' in resource or 'snapshot' in resource)) or (resource_type == 'snapshot' and '.' not in resource and 'snapshot' not in resource)): continue path = '%s/%s' % (exp, resource) mnt_path = '%s/%s' % (mnt_point, resource) size = self._get_file_size(mnt_path) rsrc_inf = { 'reference': { 'source-name': path }, 'size': size, 'cinder_id': None, 'extra_info': None } if resource_type == 'volume': potential_id = utils.extract_id_from_volume_name(resource) elif 'snapshot' in resource: # This is for the snapshot legacy case potential_id = utils.extract_id_from_snapshot_name( resource) else: potential_id = resource.split('.')[1] # When a resource is already managed by cinder, it's not # recommended to manage it again. So we set safe_to_manage = # False. Otherwise, it is set safe_to_manage = True. if potential_id in cinder_ids: rsrc_inf['safe_to_manage'] = False rsrc_inf['reason_not_safe'] = 'already managed' rsrc_inf['cinder_id'] = potential_id else: rsrc_inf['safe_to_manage'] = True rsrc_inf['reason_not_safe'] = None # If it's a snapshot, we try to get its source volume. However, # this search is not reliable in some cases. So, if it's not # possible to return a precise result, we return unknown as # source-reference, throw a warning message and fill the # extra-info. if resource_type == 'snapshot': if 'snapshot' not in resource: origin = self._get_snapshot_origin_from_name(resource) if 'unmanage' in origin: origin = origin[16:] else: origin = origin[7:] rsrc_inf['source_reference'] = {'id': origin} else: path = path.split(':')[1] origin = self._get_snapshot_origin(path, exports[exp]) if not origin: # if origin is empty, the file is not a clone continue elif len(origin) == 1: origin = origin[0].split('/')[2] origin = utils.extract_id_from_volume_name(origin) rsrc_inf['source_reference'] = {'id': origin} else: LOG.warning( "Could not determine the volume " "that owns the snapshot %(snap)s", {'snap': resource}) rsrc_inf['source_reference'] = {'id': 'unknown'} rsrc_inf['extra_info'] = ('Could not determine ' 'the volume that owns ' 'the snapshot') entries.append(rsrc_inf) return utils.paginate_entries_list(entries, marker, limit, offset, sort_keys, sort_dirs)
def _get_manageable_resource_info(self, cinder_resources, resource_type, marker, limit, offset, sort_keys, sort_dirs): """Gets the resources on the backend available for management by Cinder. Receives the parameters from "get_manageable_volumes" and "get_manageable_snapshots" and gets the available resources :param cinder_resources: A list of resources in this host that Cinder currently manages :param resource_type: If it's a volume or a snapshot :param marker: The last item of the previous page; we return the next results after this value (after sorting) :param limit: Maximum number of items to return :param offset: Number of items to skip after marker :param sort_keys: List of keys to sort results by (valid keys are 'identifier' and 'size') :param sort_dirs: List of directions to sort by, corresponding to sort_keys (valid directions are 'asc' and 'desc') :returns: list of dictionaries, each specifying a volume or snapshot (resource) in the host, with the following keys: - reference (dictionary): The reference for a resource, which can be passed to "manage_existing_snapshot". - size (int): The size of the resource according to the storage backend, rounded up to the nearest GB. - safe_to_manage (boolean): Whether or not this resource is safe to manage according to the storage backend. - reason_not_safe (string): If safe_to_manage is False, the reason why. - cinder_id (string): If already managed, provide the Cinder ID. - extra_info (string): Any extra information to return to the user - source_reference (string): Similar to "reference", but for the snapshot's source volume. """ entries = [] exports = {} bend_rsrc = {} cinder_ids = [resource.id for resource in cinder_resources] for service in self.config['services']: exp_path = self.config['services'][service]['hdp'] exports[exp_path] = ( self.config['services'][service]['export']['fs']) for exp in exports.keys(): # bend_rsrc has all the resources in the specified exports # volumes {u'172.24.54.39:/Export-Cinder': # ['volume-325e7cdc-8f65-40a8-be9a-6172c12c9394', # ' snapshot-1bfb6f0d-9497-4c12-a052-5426a76cacdc','']} bend_rsrc[exp] = self._get_volumes_from_export(exp) mnt_point = self._get_mount_point_for_share(exp) for resource in bend_rsrc[exp]: # Ignoring resources of unwanted types if ((resource_type == 'volume' and 'snapshot' in resource) or (resource_type == 'snapshot' and 'volume' in resource)): continue path = '%s/%s' % (exp, resource) mnt_path = '%s/%s' % (mnt_point, resource) size = self._get_file_size(mnt_path) rsrc_inf = {'reference': {'source-name': path}, 'size': size, 'cinder_id': None, 'extra_info': None} if resource_type == 'volume': potential_id = utils.extract_id_from_volume_name(resource) else: potential_id = utils.extract_id_from_snapshot_name( resource) # When a resource is already managed by cinder, it's not # recommended to manage it again. So we set safe_to_manage = # False. Otherwise, it is set safe_to_manage = True. if potential_id in cinder_ids: rsrc_inf['safe_to_manage'] = False rsrc_inf['reason_not_safe'] = 'already managed' rsrc_inf['cinder_id'] = potential_id else: rsrc_inf['safe_to_manage'] = True rsrc_inf['reason_not_safe'] = None # If it's a snapshot, we try to get its source volume. However, # this search is not reliable in some cases. So, if it's not # possible to return a precise result, we return unknown as # source-reference, throw a warning message and fill the # extra-info. if resource_type == 'snapshot': path = path.split(':')[1] origin = self._get_snapshot_origin(path, exports[exp]) if not origin: # if origin is empty, the file is not a clone continue elif len(origin) == 1: origin = origin[0].split('/')[2] origin = utils.extract_id_from_volume_name(origin) rsrc_inf['source_reference'] = {'id': origin} else: LOG.warning(_LW("Could not determine the volume that " "owns the snapshot %(snap)s"), {'snap': resource}) rsrc_inf['source_reference'] = {'id': 'unknown'} rsrc_inf['extra_info'] = ('Could not determine the ' 'volume that owns the ' 'snapshot') entries.append(rsrc_inf) return utils.paginate_entries_list(entries, marker, limit, offset, sort_keys, sort_dirs)
def get_manageable_volumes(self, cinder_volumes, marker, limit, offset, sort_keys, sort_dirs): """List volumes on the backend available for management by Cinder. Returns a list of dictionaries, each specifying a volume in the host, with the following keys: - reference (dictionary): The reference for a volume, which can be passed to "manage_existing". - size (int): The size of the volume according to the storage backend, rounded up to the nearest GB. - safe_to_manage (boolean): Whether or not this volume is safe to manage according to the storage backend. For example, is the volume in use or invalid for any reason. - reason_not_safe (string): If safe_to_manage is False, the reason why. - cinder_id (string): If already managed, provide the Cinder ID. - extra_info (string): Any extra information to return to the user :param cinder_volumes: A list of volumes in this host that Cinder currently manages, used to determine if a volume is manageable or not. :param marker: The last item of the previous page; we return the next results after this value (after sorting) :param limit: Maximum number of items to return :param offset: Number of items to skip after marker :param sort_keys: List of keys to sort results by (valid keys are 'identifier' and 'size') :param sort_dirs: List of directions to sort by, corresponding to sort_keys (valid directions are 'asc' and 'desc') """ LOG.debug("Listing manageable Datera volumes") app_instances = self._issue_api_request(URL_TEMPLATES['ai']()).values() results = [] cinder_volume_ids = [vol['id'] for vol in cinder_volumes] for ai in app_instances: ai_name = ai['name'] reference = None size = None safe_to_manage = False reason_not_safe = None cinder_id = None extra_info = None if re.match(UUID4_RE, ai_name): cinder_id = ai_name.lstrip(OS_PREFIX) if (not cinder_id and ai_name.lstrip(OS_PREFIX) not in cinder_volume_ids): safe_to_manage = self._is_manageable(ai) if safe_to_manage: si = list(ai['storage_instances'].values())[0] si_name = si['name'] vol = list(si['volumes'].values())[0] vol_name = vol['name'] size = vol['size'] reference = {"source-name": "{}:{}:{}".format( ai_name, si_name, vol_name)} results.append({ 'reference': reference, 'size': size, 'safe_to_manage': safe_to_manage, 'reason_not_safe': reason_not_safe, 'cinder_id': cinder_id, 'extra_info': extra_info}) page_results = volutils.paginate_entries_list( results, marker, limit, offset, sort_keys, sort_dirs) return page_results