def filter_values_dict(
cls,
result_list: List[Dict[str, Any]],
white_list: List[str] = None,
ignore_list: List[str] = None) -> List[Dict[str, Any]]:
"""Removes unwanted values from a list of dicts.
Use white_list to only pick the values specified.
Use ignore_list to pick everything but the values specified
Both: white_list itmes overwrite ignore_list times, still getting all items not filterd.
Args:
result_list (List[Dict[str, Any]]): items to be filtered
white_list (List[str], optional): items to be kept. Defaults to None.
ignore_list (List[str], optional): items to be removed. Defaults to None.
Raises:
ValueError: no result list specified
Returns:
List[Dict[str, Any]]: list of filtered dicts
"""
if (result_list is None):
raise ValueError("need valuelist to filter values")
new_result_list: List[Dict[str, Any]] = []
# if single object this is a 1 elem list
for result in result_list:
new_result: Dict[str, Any] = {}
# Only aquire items wanted
if (white_list):
for white_key in white_list:
(key, value) = SppUtils.get_nested_kv(key_name=white_key,
nested_dict=result)
if (key in new_result):
key = white_key
new_result[key] = value
# warn if something is missing
if (len(new_result) != len(white_list)):
ExceptionUtils.error_message(
f"Result has not same lenght as whitelist, probably typing error: {result_list}"
)
# aquire all but few unwanted
if (ignore_list is not None):
# add sub-dicts to dictonary itself, filtering inclusive
full_result = cls.get_with_sub_values(mydict=result,
ignore_list=ignore_list)
new_result.update(full_result)
new_result_list.append(new_result)
return new_result_list
def get_objects(self,
endpoint: str = None, uri: str = None,
array_name: str = None,
white_list: List[str] = None, ignore_list: List[str] = None,
add_time_stamp: bool = False) -> List[Dict[str, Any]]:
"""Querys a response(-list) from a REST-API endpoint or URI.
Specify `array_name` if there are multiple results / list.
Use white_list to pick only the values specified.
Use ignore_list to pick everything but the values specified.
Both: white_list items overwrite ignore_list items, still getting all not filtered.
Note:
Do not specify both endpoint and uri, only uri will be used
Keyword Arguments:
endpoint {str} -- endpoint to be queried. Either use this or uri (default: {None})
uri {str} -- uri to be queried. Either use this or endpoint (default: {None})
array_name {str} -- name of array if there are multiple results wanted (default: {None})
white_list {list} -- list of item to query (default: {None})
ignore_list {list} -- query all but these items(-groups). (default: {None})
page_size {int} -- Size of page, recommendation is 100, depending on size of data (default: {100})
add_time_stamp {bool} -- whether to add the capture timestamp (default: {False})
Raises:
ValueError: Neither a endpoint nor uri is specfied
ValueError: Negative or 0 pagesize
ValueError: array_name is specified but it is only a single object
Returns:
{List[Dict[str, Any]]} -- List of dictonarys as the results
"""
if(not endpoint and not uri):
raise ValueError("neiter endpoint nor uri specified")
if(endpoint and uri):
LOGGER.debug("added both endpoint and uri. This is unneccessary, endpoint is ignored")
# if neither specifed, get everything
if(not white_list and not ignore_list):
ignore_list = []
# create uri out of endpoint
if(not uri):
next_page = self.__srv_url + endpoint
else:
next_page = uri
result_list: List[Dict[str, Any]] = []
# Aborts if no nextPage is found
while(next_page):
LOGGER.debug(f"Collected {len(result_list)} items until now. Next page: {next_page}")
if(self.__verbose):
LOGGER.info(f"Collected {len(result_list)} items until now. Next page: {next_page}")
# Request response
(response, send_time) = self.__query_url(url=next_page)
# find follow page if available and set it
(_, next_page_link) = SppUtils.get_nested_kv(key_name="links.nextPage.href", nested_dict=response)
next_page = next_page_link
# Check if single object or not
if(array_name):
# get results for this page, if empty nothing happens
page_result_list: Optional[List[Dict[str, Any]]] = response.get(array_name, None)
if(page_result_list is None):
raise ValueError("array_name does not exist, this is probably a single object")
else:
page_result_list = [response]
filtered_results = ConnectionUtils.filter_values_dict(
result_list=page_result_list,
white_list=white_list,
ignore_list=ignore_list)
if(add_time_stamp): # direct time add to make the timestamps represent the real capture time
for mydict in filtered_results:
time_key, time_val = SppUtils.get_capture_timestamp_sec()
mydict[time_key] = time_val
result_list.extend(filtered_results)
# adjust pagesize
if(send_time > self.__preferred_time or len(page_result_list) == self.__page_size):
self.__page_size = ConnectionUtils.adjust_page_size(
page_size=len(page_result_list),
min_page_size=self.__min_page_size,
preferred_time=self.__preferred_time,
send_time=send_time)
LOGGER.debug("objectList size %d", len(result_list))
return result_list
def _parse_pool_show_cmd(
ssh_command: SshCommand,
ssh_type: SshTypes) -> Tuple[str, List[Dict[str, Any]]]:
"""Parses the result of the `vsnap --json pool show` command, splitting it into its parts.
Arguments:
ssh_command {SshCommand} -- command with saved result
ssh_type {SshTypes} -- type of the client
Raises:
ValueError: no command given or no result saved
ValueError: no ssh type given
Returns:
Tuple[str, List[Dict[str, Any]]] -- Tuple of the tablename and a insert list
"""
if (not ssh_command or not ssh_command.result):
raise ValueError("no command given or empty result")
if (not ssh_type):
raise ValueError("no sshtype given")
pool_result_list: List[Dict[str, Any]] = []
try:
result: Dict[str, List[Dict[str,
Any]]] = json.loads(ssh_command.result)
except json.decoder.JSONDecodeError: # type: ignore
raise ValueError("cant decode json for pool command",
ssh_command.result, ssh_command, ssh_type)
for pool in result['pools']:
pool_dict: Dict[str, Any] = {}
# acts as white list
insert_list = [
'compression', 'compression_ratio', 'deduplication',
'deduplication_ratio', 'diskgroup_size', 'encryption.enabled',
'health', 'id', 'name', 'pool_type', 'size_before_compression',
'size_before_deduplication', 'size_free', 'size_total',
'size_used', 'status'
]
for item in insert_list:
(key, value) = SppUtils.get_nested_kv(item, pool)
pool_dict[key] = value
# rename
pool_dict['encryption_enabled'] = pool_dict.pop('enabled')
# change unit from bytes to megabytes
try:
sz_b_c = SppUtils.parse_unit(
pool_dict['size_before_compression'])
sz_b_d = SppUtils.parse_unit(
pool_dict['size_before_deduplication'])
sz_fr = SppUtils.parse_unit(pool_dict['size_free'])
sz_t = SppUtils.parse_unit(pool_dict['size_total'])
sz_u = SppUtils.parse_unit(pool_dict['size_used'])
pool_dict['size_before_compression'] = int(
sz_b_c / pow(2, 20)) if sz_b_c else None
pool_dict['size_before_deduplication'] = int(
sz_b_d / pow(2, 20)) if sz_b_d else None
pool_dict['size_free'] = int(sz_fr /
pow(2, 20)) if sz_fr else None
pool_dict['size_total'] = int(sz_t /
pow(2, 20)) if sz_t else None
pool_dict['size_used'] = int(sz_u /
pow(2, 20)) if sz_u else None
except KeyError as error:
ExceptionUtils.exception_info(
error=error,
extra_message=
f"failed to reduce size of vsnap pool size for {pool_dict}"
)
# set default needed fields
pool_dict['hostName'] = ssh_command.host_name
pool_dict['ssh_type'] = ssh_type.name
pool_result_list.append(pool_dict)
return (ssh_command.table_name, pool_result_list)
def get_objects(self,
endpoint: str = None, uri: str = None,
params: Dict[str, Any] = None,
post_data: Dict[str, Any] = None,
request_type: RequestType = RequestType.GET,
array_name: str = None,
allow_list: List[str] = None, ignore_list: List[str] = None,
add_time_stamp: bool = False) -> List[Dict[str, Any]]:
"""Querys a response(-list) from a REST-API endpoint or URI from multiple pages
Specify `array_name` if there are multiple results / list.
Use allow_list to pick only the values specified.
Use ignore_list to pick everything but the values specified.
Both: allow_list items overwrite ignore_list items, still getting all not filtered.
Param pageSize is only guranteed to be valid for the first page if included within params.
Note:
Do not specify both endpoint and uri, only uri will be used
Keyword Arguments:
endpoint {str} -- endpoint to be queried. Either use this or uri (default: {None})
uri {str} -- uri to be queried. Either use this or endpoint (default: {None})
params {Dict[str, Any]} -- Dictionary with all URL-Parameters. pageSize only guranteed to be valid for first page (default: {None})
post_data {Dict[str, Any]} -- Dictionary with Body-Data. Only use on POST-Requests
request_type: {RequestType} -- Either GET or POST
array_name {str} -- name of array if there are multiple results wanted (default: {None})
allow_list {list} -- list of item to query (default: {None})
ignore_list {list} -- query all but these items(-groups). (default: {None})
add_time_stamp {bool} -- whether to add the capture timestamp (default: {False})
Raises:
ValueError: Neither a endpoint nor uri is specfied
ValueError: Negative or 0 pagesize
ValueError: array_name is specified but it is only a single object
Returns:
{List[Dict[str, Any]]} -- List of dictonarys as the results
"""
if(not endpoint and not uri):
raise ValueError("neiter endpoint nor uri specified")
if(endpoint and uri):
LOGGER.debug("added both endpoint and uri. This is unneccessary, uri is ignored")
# if neither specifed, get everything
if(not allow_list and not ignore_list):
ignore_list = []
if(params is None):
params = {}
# create uri out of endpoint
if(endpoint):
next_page = self.get_url(endpoint)
else:
next_page = uri
result_list: List[Dict[str, Any]] = []
# Aborts if no nextPage is found
while(next_page):
LOGGER.debug(f"Collected {len(result_list)} items until now. Next page: {next_page}")
if(self.__verbose):
LOGGER.info(f"Collected {len(result_list)} items until now. Next page: {next_page}")
# Request response
(response, send_time) = self.query_url(next_page, params, request_type, post_data)
# find follow page if available and set it
(_, next_page_link) = SppUtils.get_nested_kv(key_name="links.nextPage.href", nested_dict=response)
next_page: Optional[str] = next_page_link
if(next_page):
# Overwrite params with params from next link
params = ConnectionUtils.get_url_params(next_page)
# remove params from page
next_page = ConnectionUtils.url_set_params(next_page, None)
# Check if single object or not
if(array_name):
# get results for this page, if empty nothing happens
page_result_list: Optional[List[Dict[str, Any]]] = response.get(array_name, None)
if(page_result_list is None):
raise ValueError("array_name does not exist, this is probably a single object")
else:
page_result_list = [response]
filtered_results = ConnectionUtils.filter_values_dict(
result_list=page_result_list,
allow_list=allow_list,
ignore_list=ignore_list)
if(add_time_stamp): # direct time add to make the timestamps represent the real capture time
for mydict in filtered_results:
time_key, time_val = SppUtils.get_capture_timestamp_sec()
mydict[time_key] = time_val
result_list.extend(filtered_results)
# adjust pagesize if either the send time is too high
# or regulary adjust on max-page sizes requests
# dont adjust if page isnt full and therefore too quick
if(send_time > self.__preferred_time or len(page_result_list) == self.__page_size):
LOGGER.debug(f"send_time: {send_time}, len: {len(page_result_list)}, pageSize = {self.__page_size} ")
self.__page_size = ConnectionUtils.adjust_page_size(
page_size=len(page_result_list),
min_page_size=self.__min_page_size,
preferred_time=self.__preferred_time,
send_time=send_time)
LOGGER.debug(f"Changed pageSize from {len(page_result_list)} to {self.__page_size} ")
params["pageSize"] = self.__page_size
LOGGER.debug("objectList size %d", len(result_list))
return result_list
