def filter_values_dict( cls, result_list: List[Dict[str, Any]], white_list: List[str] = None, ignore_list: List[str] = None) -> List[Dict[str, Any]]: """Removes unwanted values from a list of dicts. Use white_list to only pick the values specified. Use ignore_list to pick everything but the values specified Both: white_list itmes overwrite ignore_list times, still getting all items not filterd. Args: result_list (List[Dict[str, Any]]): items to be filtered white_list (List[str], optional): items to be kept. Defaults to None. ignore_list (List[str], optional): items to be removed. Defaults to None. Raises: ValueError: no result list specified Returns: List[Dict[str, Any]]: list of filtered dicts """ if (result_list is None): raise ValueError("need valuelist to filter values") new_result_list: List[Dict[str, Any]] = [] # if single object this is a 1 elem list for result in result_list: new_result: Dict[str, Any] = {} # Only aquire items wanted if (white_list): for white_key in white_list: (key, value) = SppUtils.get_nested_kv(key_name=white_key, nested_dict=result) if (key in new_result): key = white_key new_result[key] = value # warn if something is missing if (len(new_result) != len(white_list)): ExceptionUtils.error_message( f"Result has not same lenght as whitelist, probably typing error: {result_list}" ) # aquire all but few unwanted if (ignore_list is not None): # add sub-dicts to dictonary itself, filtering inclusive full_result = cls.get_with_sub_values(mydict=result, ignore_list=ignore_list) new_result.update(full_result) new_result_list.append(new_result) return new_result_list
def get_objects(self, endpoint: str = None, uri: str = None, array_name: str = None, white_list: List[str] = None, ignore_list: List[str] = None, add_time_stamp: bool = False) -> List[Dict[str, Any]]: """Querys a response(-list) from a REST-API endpoint or URI. Specify `array_name` if there are multiple results / list. Use white_list to pick only the values specified. Use ignore_list to pick everything but the values specified. Both: white_list items overwrite ignore_list items, still getting all not filtered. Note: Do not specify both endpoint and uri, only uri will be used Keyword Arguments: endpoint {str} -- endpoint to be queried. Either use this or uri (default: {None}) uri {str} -- uri to be queried. Either use this or endpoint (default: {None}) array_name {str} -- name of array if there are multiple results wanted (default: {None}) white_list {list} -- list of item to query (default: {None}) ignore_list {list} -- query all but these items(-groups). (default: {None}) page_size {int} -- Size of page, recommendation is 100, depending on size of data (default: {100}) add_time_stamp {bool} -- whether to add the capture timestamp (default: {False}) Raises: ValueError: Neither a endpoint nor uri is specfied ValueError: Negative or 0 pagesize ValueError: array_name is specified but it is only a single object Returns: {List[Dict[str, Any]]} -- List of dictonarys as the results """ if(not endpoint and not uri): raise ValueError("neiter endpoint nor uri specified") if(endpoint and uri): LOGGER.debug("added both endpoint and uri. This is unneccessary, endpoint is ignored") # if neither specifed, get everything if(not white_list and not ignore_list): ignore_list = [] # create uri out of endpoint if(not uri): next_page = self.__srv_url + endpoint else: next_page = uri result_list: List[Dict[str, Any]] = [] # Aborts if no nextPage is found while(next_page): LOGGER.debug(f"Collected {len(result_list)} items until now. Next page: {next_page}") if(self.__verbose): LOGGER.info(f"Collected {len(result_list)} items until now. Next page: {next_page}") # Request response (response, send_time) = self.__query_url(url=next_page) # find follow page if available and set it (_, next_page_link) = SppUtils.get_nested_kv(key_name="links.nextPage.href", nested_dict=response) next_page = next_page_link # Check if single object or not if(array_name): # get results for this page, if empty nothing happens page_result_list: Optional[List[Dict[str, Any]]] = response.get(array_name, None) if(page_result_list is None): raise ValueError("array_name does not exist, this is probably a single object") else: page_result_list = [response] filtered_results = ConnectionUtils.filter_values_dict( result_list=page_result_list, white_list=white_list, ignore_list=ignore_list) if(add_time_stamp): # direct time add to make the timestamps represent the real capture time for mydict in filtered_results: time_key, time_val = SppUtils.get_capture_timestamp_sec() mydict[time_key] = time_val result_list.extend(filtered_results) # adjust pagesize if(send_time > self.__preferred_time or len(page_result_list) == self.__page_size): self.__page_size = ConnectionUtils.adjust_page_size( page_size=len(page_result_list), min_page_size=self.__min_page_size, preferred_time=self.__preferred_time, send_time=send_time) LOGGER.debug("objectList size %d", len(result_list)) return result_list
def _parse_pool_show_cmd( ssh_command: SshCommand, ssh_type: SshTypes) -> Tuple[str, List[Dict[str, Any]]]: """Parses the result of the `vsnap --json pool show` command, splitting it into its parts. Arguments: ssh_command {SshCommand} -- command with saved result ssh_type {SshTypes} -- type of the client Raises: ValueError: no command given or no result saved ValueError: no ssh type given Returns: Tuple[str, List[Dict[str, Any]]] -- Tuple of the tablename and a insert list """ if (not ssh_command or not ssh_command.result): raise ValueError("no command given or empty result") if (not ssh_type): raise ValueError("no sshtype given") pool_result_list: List[Dict[str, Any]] = [] try: result: Dict[str, List[Dict[str, Any]]] = json.loads(ssh_command.result) except json.decoder.JSONDecodeError: # type: ignore raise ValueError("cant decode json for pool command", ssh_command.result, ssh_command, ssh_type) for pool in result['pools']: pool_dict: Dict[str, Any] = {} # acts as white list insert_list = [ 'compression', 'compression_ratio', 'deduplication', 'deduplication_ratio', 'diskgroup_size', 'encryption.enabled', 'health', 'id', 'name', 'pool_type', 'size_before_compression', 'size_before_deduplication', 'size_free', 'size_total', 'size_used', 'status' ] for item in insert_list: (key, value) = SppUtils.get_nested_kv(item, pool) pool_dict[key] = value # rename pool_dict['encryption_enabled'] = pool_dict.pop('enabled') # change unit from bytes to megabytes try: sz_b_c = SppUtils.parse_unit( pool_dict['size_before_compression']) sz_b_d = SppUtils.parse_unit( pool_dict['size_before_deduplication']) sz_fr = SppUtils.parse_unit(pool_dict['size_free']) sz_t = SppUtils.parse_unit(pool_dict['size_total']) sz_u = SppUtils.parse_unit(pool_dict['size_used']) pool_dict['size_before_compression'] = int( sz_b_c / pow(2, 20)) if sz_b_c else None pool_dict['size_before_deduplication'] = int( sz_b_d / pow(2, 20)) if sz_b_d else None pool_dict['size_free'] = int(sz_fr / pow(2, 20)) if sz_fr else None pool_dict['size_total'] = int(sz_t / pow(2, 20)) if sz_t else None pool_dict['size_used'] = int(sz_u / pow(2, 20)) if sz_u else None except KeyError as error: ExceptionUtils.exception_info( error=error, extra_message= f"failed to reduce size of vsnap pool size for {pool_dict}" ) # set default needed fields pool_dict['hostName'] = ssh_command.host_name pool_dict['ssh_type'] = ssh_type.name pool_result_list.append(pool_dict) return (ssh_command.table_name, pool_result_list)
def get_objects(self, endpoint: str = None, uri: str = None, params: Dict[str, Any] = None, post_data: Dict[str, Any] = None, request_type: RequestType = RequestType.GET, array_name: str = None, allow_list: List[str] = None, ignore_list: List[str] = None, add_time_stamp: bool = False) -> List[Dict[str, Any]]: """Querys a response(-list) from a REST-API endpoint or URI from multiple pages Specify `array_name` if there are multiple results / list. Use allow_list to pick only the values specified. Use ignore_list to pick everything but the values specified. Both: allow_list items overwrite ignore_list items, still getting all not filtered. Param pageSize is only guranteed to be valid for the first page if included within params. Note: Do not specify both endpoint and uri, only uri will be used Keyword Arguments: endpoint {str} -- endpoint to be queried. Either use this or uri (default: {None}) uri {str} -- uri to be queried. Either use this or endpoint (default: {None}) params {Dict[str, Any]} -- Dictionary with all URL-Parameters. pageSize only guranteed to be valid for first page (default: {None}) post_data {Dict[str, Any]} -- Dictionary with Body-Data. Only use on POST-Requests request_type: {RequestType} -- Either GET or POST array_name {str} -- name of array if there are multiple results wanted (default: {None}) allow_list {list} -- list of item to query (default: {None}) ignore_list {list} -- query all but these items(-groups). (default: {None}) add_time_stamp {bool} -- whether to add the capture timestamp (default: {False}) Raises: ValueError: Neither a endpoint nor uri is specfied ValueError: Negative or 0 pagesize ValueError: array_name is specified but it is only a single object Returns: {List[Dict[str, Any]]} -- List of dictonarys as the results """ if(not endpoint and not uri): raise ValueError("neiter endpoint nor uri specified") if(endpoint and uri): LOGGER.debug("added both endpoint and uri. This is unneccessary, uri is ignored") # if neither specifed, get everything if(not allow_list and not ignore_list): ignore_list = [] if(params is None): params = {} # create uri out of endpoint if(endpoint): next_page = self.get_url(endpoint) else: next_page = uri result_list: List[Dict[str, Any]] = [] # Aborts if no nextPage is found while(next_page): LOGGER.debug(f"Collected {len(result_list)} items until now. Next page: {next_page}") if(self.__verbose): LOGGER.info(f"Collected {len(result_list)} items until now. Next page: {next_page}") # Request response (response, send_time) = self.query_url(next_page, params, request_type, post_data) # find follow page if available and set it (_, next_page_link) = SppUtils.get_nested_kv(key_name="links.nextPage.href", nested_dict=response) next_page: Optional[str] = next_page_link if(next_page): # Overwrite params with params from next link params = ConnectionUtils.get_url_params(next_page) # remove params from page next_page = ConnectionUtils.url_set_params(next_page, None) # Check if single object or not if(array_name): # get results for this page, if empty nothing happens page_result_list: Optional[List[Dict[str, Any]]] = response.get(array_name, None) if(page_result_list is None): raise ValueError("array_name does not exist, this is probably a single object") else: page_result_list = [response] filtered_results = ConnectionUtils.filter_values_dict( result_list=page_result_list, allow_list=allow_list, ignore_list=ignore_list) if(add_time_stamp): # direct time add to make the timestamps represent the real capture time for mydict in filtered_results: time_key, time_val = SppUtils.get_capture_timestamp_sec() mydict[time_key] = time_val result_list.extend(filtered_results) # adjust pagesize if either the send time is too high # or regulary adjust on max-page sizes requests # dont adjust if page isnt full and therefore too quick if(send_time > self.__preferred_time or len(page_result_list) == self.__page_size): LOGGER.debug(f"send_time: {send_time}, len: {len(page_result_list)}, pageSize = {self.__page_size} ") self.__page_size = ConnectionUtils.adjust_page_size( page_size=len(page_result_list), min_page_size=self.__min_page_size, preferred_time=self.__preferred_time, send_time=send_time) LOGGER.debug(f"Changed pageSize from {len(page_result_list)} to {self.__page_size} ") params["pageSize"] = self.__page_size LOGGER.debug("objectList size %d", len(result_list)) return result_list