def get_vms_per_sla(self) -> List[Dict[str, Any]]:
"""retrieves and calculates all vmware per SLA."""
endpoint = "/ngp/slapolicy"
white_list = ["name", "id"]
array_name = "slapolicies"
sla_policty_list = self.__rest_client.get_objects(
endpoint=endpoint,
white_list=white_list,
array_name=array_name,
add_time_stamp=False)
result_list: List[Dict[str, Any]] = []
for sla_policty in sla_policty_list:
try:
sla_name: str = sla_policty["name"]
except KeyError as error:
ExceptionUtils.exception_info(
error,
extra_message="skipping one sla entry due missing name.")
continue
sla_id: Optional[str] = sla_policty.get("id", None)
result_dict: Dict[str, Any] = {}
## hotadd:
sla_name = urllib.parse.quote_plus(sla_name)
endpoint = "/api/hypervisor/search"
endpoint = ConnectionUtils.url_set_param(url=endpoint,
param_name="resourceType",
param_value="vm")
endpoint = ConnectionUtils.url_set_param(url=endpoint,
param_name="from",
param_value="hlo")
filter_str: str = '[{"property":"storageProfileName","value": "' + sla_name + '", "op":"="}]'
endpoint = ConnectionUtils.url_set_param(url=endpoint,
param_name="filter",
param_value=filter_str)
# note: currently only vmware is queried per sla, not hyperV
# need to check if hypervisortype must be specified
post_data = json.dumps({"name": "*", "hypervisorType": "vmware"})
response_json = self.__rest_client.post_data(endpoint=endpoint,
post_data=post_data)
result_dict["slaName"] = sla_name
result_dict["slaId"] = sla_id
result_dict["vmCountBySLA"] = response_json.get("total")
time_key, time = SppUtils.get_capture_timestamp_sec()
result_dict[time_key] = time
result_list.append(result_dict)
return result_list
def get_all_vms(self) -> List[Dict[str, Any]]:
"""retrieves a list of all vm's with their statistics."""
endpoint = "/api/endeavour/catalog/hypervisor/vm"
white_list = [
"id",
"properties.name",
"properties.host",
"catalogTime",
"properties.vmVersion",
"properties.configInfo.osName",
"properties.hypervisorType",
"properties.isProtected",
"properties.inHLO",
"isEncrypted",
"properties.powerSummary.powerState",
"properties.powerSummary.uptime",
"properties.storageSummary.commited",
"properties.storageSummary.uncommited",
"properties.storageSummary.shared",
"properties.datacenter.name",
"properties.cpu",
"properties.coresPerCpu",
"properties.memory",
]
array_name = "children"
endpoint = ConnectionUtils.url_set_param(
url=endpoint,
param_name="embed",
param_value="(children(properties))")
return self.__rest_client.get_objects(endpoint=endpoint,
array_name=array_name,
white_list=white_list,
add_time_stamp=False)
def get_job_log_details(self, job_logs_type: str,
jobsession_id: int) -> List[Dict[str, Any]]:
"""retrieves jobLogs for a certain jobsession.
Arguments:
job_logs_type {str} -- types of joblogs, given as comma seperated string-array: '["DEBUG"]'
page_size {int} -- size of each response
jobsession_id {int} -- only returns joblogs for this sessionID
Raises:
ValueError: No jobsessionid given
ValueError: No joblogType specified
Returns:
List[Dict[str, Any]] -- List of joblogs for the sessionID of the given types.
"""
if (not jobsession_id):
raise ValueError("no jobsession_id given to query Logs by an Id")
if (not job_logs_type):
raise ValueError(
"need to specify the jobLogType you want to query")
# note: job id is the id of the job ( policy)
# jobsessionid is the unique id of a execution of a job
# note: jobLogs may be cleared by maintenance jobs after X days. The value can be specified in the SPP GUI
LOGGER.debug("retrieving jobLogs for jobsessionId: %d", jobsession_id)
endpoint = "/api/endeavour/log/job"
white_list = [
"jobsessionId", "logTime", "id", "messageId", "message",
"messageParams", "type"
]
array_name = "logs"
api_filter = '[{"property":"jobsessionId","value":' + str(jobsession_id) + ',"op":"="},' \
'{"property":"type","value":'+ job_logs_type +',"op":"IN"}]'
#update the filter parameter to list all types if message types, not only info..
endpoint_to_logs = ConnectionUtils.url_set_param(
url=endpoint, param_name="filter", param_value=api_filter)
log_list = self.__rest_client.get_objects(endpoint=endpoint_to_logs,
white_list=white_list,
array_name=array_name)
return log_list
def logout(self) -> None:
"""Logs out of the REST-API.
Raises:
ValueError: Error when logging out.
ValueError: Wrong status code when logging out.
"""
url = self.get_url("/api/endeavour/session")
try:
response_logout: Response = delete(url, headers=self.__headers, verify=False)
except RequestException as error:
ExceptionUtils.exception_info(error=error)
raise ValueError("error when logging out")
if response_logout.status_code != 204:
raise ConnectionUtils.rest_response_error(response_logout, "Wrong Status code when logging out")
if(self.__verbose):
LOGGER.info("Rest-API logout successfull")
LOGGER.debug("Rest-API logout successfull")
def __query_url(self, url: str) -> Tuple[Dict[str, Any], float]:
"""Sends a request to this endpoint. Repeats if timeout error occured.
Adust the pagesize on timeout.
Arguments:
url {str} -- URL to be queried.
Raises:
ValueError: No URL specified
ValueError: Error when requesting endpoint
ValueError: Wrong status code
ValueError: failed to parse result
ValueError: Timeout when sending result
Returns:
Tuple[Dict[str, Any], float] -- Result of the request with the required send time
"""
if(not url):
raise ValueError("no url specified")
LOGGER.debug(f"endpoint request {url}")
failed_trys: int = 0
response_query: Optional[Response] = None
send_time: float = -1 # prevent unbound var
while(response_query is None):
# read pagesize
actual_page_size = ConnectionUtils.url_get_param_value(url=url, param_name="pageSize")
# Always set Pagesize to avoid different pagesizes by system
if(not actual_page_size):
url = ConnectionUtils.url_set_param(url=url, param_name="pageSize", param_value=self.__page_size)
else:
# read the pagesize
try:
actual_page_size = int(actual_page_size[0])
except (ValueError, KeyError) as error:
ExceptionUtils.exception_info(error, extra_message="invalid page size recorded")
actual_page_size = -1
# adjust pagesize of url
if(actual_page_size != self.__page_size):
LOGGER.debug(f"setting new pageSize from {actual_page_size} to {self.__page_size}")
url = ConnectionUtils.url_set_param(url=url, param_name="pageSize", param_value=self.__page_size)
# send the query
try:
start_time = time.perf_counter()
response_query = requests.get( # type: ignore
url=url, headers=self.__headers, verify=False,
timeout=(self.__initial_connection_timeout, self.__timeout))
end_time = time.perf_counter()
send_time = (end_time - start_time)
except requests.exceptions.ReadTimeout as timeout_error:
# timeout occured, increasing failed trys
failed_trys += 1
# #### Aborting cases ######
if(self.__send_retries < failed_trys):
ExceptionUtils.exception_info(error=timeout_error)
# read start index for debugging
start_index = ConnectionUtils.url_get_param_value(url=url, param_name="pageStartIndex")
# report timeout with full information
raise ValueError("timeout after repeating a maximum ammount of times.",
timeout_error, failed_trys, self.__page_size, start_index)
if(self.__page_size == self.__min_page_size):
ExceptionUtils.exception_info(error=timeout_error)
# read start index for debugging
start_index = ConnectionUtils.url_get_param_value(url=url, param_name="pageStartIndex")
# report timeout with full information
raise ValueError("timeout after using minumum pagesize. repeating the request is of no use.",
timeout_error, failed_trys, self.__page_size, start_index)
# #### continuing cases ######
if(self.__send_retries == failed_trys): # last try
LOGGER.debug(f"Timeout error when requesting, now last try of total {self.__send_retries}. Reducing pagesize to minimum for url: {url}")
if(self.__verbose):
LOGGER.info(f"Timeout error when requesting, now last try of total {self.__send_retries}. Reducing pagesize to minimum for url: {url}")
self.__page_size = self.__min_page_size
# repeat with minimal possible size
else: # (self.__send_retries > failed_trys): # more then 1 try left
LOGGER.debug(f"Timeout error when requesting, now on try {failed_trys} of {self.__send_retries}. Reducing pagesizefor url: {url}")
if(self.__verbose):
LOGGER.info(f"Timeout error when requesting, now on try {failed_trys} of {self.__send_retries}. Reducing pagesize for url: {url}")
self.__page_size = ConnectionUtils.adjust_page_size(
page_size=self.__page_size,
min_page_size=self.__min_page_size,
time_out=True)
# repeat with reduced page size
except requests.exceptions.RequestException as error:
ExceptionUtils.exception_info(error=error)
raise ValueError("error when requesting endpoint", error)
if response_query.status_code != 200:
raise ValueError("Wrong Status code when requesting endpoint data",
response_query.status_code, url, response_query)
try:
response_json: Dict[str, Any] = response_query.json()
except (json.decoder.JSONDecodeError, ValueError) as error: # type: ignore
raise ValueError("failed to parse query in restAPI post request", response_query) # type: ignore
return (response_json, send_time)
def get_objects(self,
endpoint: str = None, uri: str = None,
array_name: str = None,
white_list: List[str] = None, ignore_list: List[str] = None,
add_time_stamp: bool = False) -> List[Dict[str, Any]]:
"""Querys a response(-list) from a REST-API endpoint or URI.
Specify `array_name` if there are multiple results / list.
Use white_list to pick only the values specified.
Use ignore_list to pick everything but the values specified.
Both: white_list items overwrite ignore_list items, still getting all not filtered.
Note:
Do not specify both endpoint and uri, only uri will be used
Keyword Arguments:
endpoint {str} -- endpoint to be queried. Either use this or uri (default: {None})
uri {str} -- uri to be queried. Either use this or endpoint (default: {None})
array_name {str} -- name of array if there are multiple results wanted (default: {None})
white_list {list} -- list of item to query (default: {None})
ignore_list {list} -- query all but these items(-groups). (default: {None})
page_size {int} -- Size of page, recommendation is 100, depending on size of data (default: {100})
add_time_stamp {bool} -- whether to add the capture timestamp (default: {False})
Raises:
ValueError: Neither a endpoint nor uri is specfied
ValueError: Negative or 0 pagesize
ValueError: array_name is specified but it is only a single object
Returns:
{List[Dict[str, Any]]} -- List of dictonarys as the results
"""
if(not endpoint and not uri):
raise ValueError("neiter endpoint nor uri specified")
if(endpoint and uri):
LOGGER.debug("added both endpoint and uri. This is unneccessary, endpoint is ignored")
# if neither specifed, get everything
if(not white_list and not ignore_list):
ignore_list = []
# create uri out of endpoint
if(not uri):
next_page = self.__srv_url + endpoint
else:
next_page = uri
result_list: List[Dict[str, Any]] = []
# Aborts if no nextPage is found
while(next_page):
LOGGER.debug(f"Collected {len(result_list)} items until now. Next page: {next_page}")
if(self.__verbose):
LOGGER.info(f"Collected {len(result_list)} items until now. Next page: {next_page}")
# Request response
(response, send_time) = self.__query_url(url=next_page)
# find follow page if available and set it
(_, next_page_link) = SppUtils.get_nested_kv(key_name="links.nextPage.href", nested_dict=response)
next_page = next_page_link
# Check if single object or not
if(array_name):
# get results for this page, if empty nothing happens
page_result_list: Optional[List[Dict[str, Any]]] = response.get(array_name, None)
if(page_result_list is None):
raise ValueError("array_name does not exist, this is probably a single object")
else:
page_result_list = [response]
filtered_results = ConnectionUtils.filter_values_dict(
result_list=page_result_list,
white_list=white_list,
ignore_list=ignore_list)
if(add_time_stamp): # direct time add to make the timestamps represent the real capture time
for mydict in filtered_results:
time_key, time_val = SppUtils.get_capture_timestamp_sec()
mydict[time_key] = time_val
result_list.extend(filtered_results)
# adjust pagesize
if(send_time > self.__preferred_time or len(page_result_list) == self.__page_size):
self.__page_size = ConnectionUtils.adjust_page_size(
page_size=len(page_result_list),
min_page_size=self.__min_page_size,
preferred_time=self.__preferred_time,
send_time=send_time)
LOGGER.debug("objectList size %d", len(result_list))
return result_list
def query_url(
self,
url: str,
params: Dict[str, Any] = None,
request_type: RequestType = RequestType.GET,
post_data: Dict[str, str] = None,
auth: HTTPBasicAuth = None) -> Tuple[Dict[str, Any], float]:
"""Sends a request to this endpoint. Repeats if timeout error occured. Adust the pagesize on timeout.
Arguments:
url {str} -- URL to be queried. Must contain the server-uri and Endpoint. Does not allow encoded parameters
post_data {str} -- additional data with filters/parameters. Only to be send with a POST-Request (default: {None})
auth {HTTPBasicAuth} -- Basic auth to be used to login into SPP via POST-Request(default: {None})
type {RequestType} -- What kind of Request should be made, defaults to GET
Raises:
ValueError: No URL specified
ValueError: Error when requesting endpoint
ValueError: Wrong status code
ValueError: failed to parse result
ValueError: Timeout when sending result
ValueError: No post-data/auth is allowed in a GET-Request
Returns:
Tuple[Dict[str, Any], float] -- Result of the request with the required send time
"""
if(not url):
raise ValueError("no url specified")
if((post_data or auth) and request_type == RequestType.GET):
raise ValueError("No post-data/auth is allowed in a GET-Request")
LOGGER.debug(f"query url: {url}, type: {type}, post_data: {post_data} auth: {True if auth else False}")
if(not params):
params = {}
failed_tries: int = 0
response_query: Optional[Response] = None
send_time: float = -1 # prevent unbound var
# avoid unset pageSize to not get into SPP defaults
if("pageSize" not in params):
LOGGER.debug(f"setting pageSize to {self.__page_size} from unset value")
params["pageSize"] = self.__page_size
elif(params["pageSize"] is None):
params.pop("pageSize")
while(response_query is None):
# send the query
try:
if(request_type == RequestType.GET):
response_query = get(
url=url, headers=self.__headers, verify=False,
params=params,
timeout=(self.__initial_connection_timeout, self.__timeout))
elif(request_type == RequestType.POST):
response_query = post(
url=url, headers=self.__headers, verify=False,
params=params, json=post_data, auth=auth,
timeout=(self.__initial_connection_timeout, self.__timeout))
send_time = response_query.elapsed.total_seconds()
except ReadTimeout as timeout_error:
# timeout occured, increasing failed trys
failed_tries += 1
url_params = ConnectionUtils.get_url_params(url)
# #### Aborting cases ######
if(failed_tries > self.__max_send_retries):
ExceptionUtils.exception_info(error=timeout_error)
# read start index for debugging
start_index = url_params.get("pageStartIndex", None)
page_size = url_params.get("pageSize", None)
# report timeout with full information
raise ValueError("timeout after repeating a maximum ammount of times.",
timeout_error, failed_tries, page_size, start_index)
if(self.__page_size == self.__min_page_size):
ExceptionUtils.exception_info(error=timeout_error)
# read start index for debugging
start_index = url_params.get("pageStartIndex", None)
page_size = url_params.get("pageSize", None)
# report timeout with full information
raise ValueError("timeout after using minumum pagesize. repeating the request is of no use.",
timeout_error, failed_tries, page_size, start_index)
# #### continuing cases ######
if(failed_tries == self.__max_send_retries): # last try
LOGGER.debug(f"Timeout error when requesting, now last try of total {self.__max_send_retries}. Reducing pagesize to minimum for url: {url}")
if(self.__verbose):
LOGGER.info(f"Timeout error when requesting, now last try of total {self.__max_send_retries}. Reducing pagesize to minimum for url: {url}")
# persist reduced size for further requests
self.__page_size = self.__min_page_size
# repeat with minimal possible size
LOGGER.debug(f"setting pageSize from {params.get('pageSize', None)} to {self.__page_size}")
params["pageSize"] = self.__page_size
else: # (failed_tries < self.__max_send_retries): # more then 1 try left
LOGGER.debug(f"Timeout error when requesting, now on try {failed_tries} of {self.__max_send_retries}. Reducing pagesizefor url: {url}")
if(self.__verbose):
LOGGER.info(f"Timeout error when requesting, now on try {failed_tries} of {self.__max_send_retries}. Reducing pagesize for url: {url}")
# persist reduced size for further requests
self.__page_size = ConnectionUtils.adjust_page_size(
page_size=params["pageSize"],
min_page_size=self.__min_page_size,
timeout=True)
# repeat with reduced page size
LOGGER.debug(f"setting pageSize from {params.get('pageSize', None)} to {self.__page_size}")
params["pageSize"] = self.__page_size
except RequestException as error:
ExceptionUtils.exception_info(error=error)
raise ValueError("error when requesting endpoint", error)
if( not response_query.ok):
raise ConnectionUtils.rest_response_error( response_query,
"Wrong Status code when requesting endpoint data",
url)
try:
response_json: Dict[str, Any] = response_query.json()
except (json.decoder.JSONDecodeError, ValueError) as error:
raise ValueError("failed to parse query in restAPI request", response_query)
return (response_json, send_time)
def get_objects(self,
endpoint: str = None, uri: str = None,
params: Dict[str, Any] = None,
post_data: Dict[str, Any] = None,
request_type: RequestType = RequestType.GET,
array_name: str = None,
allow_list: List[str] = None, ignore_list: List[str] = None,
add_time_stamp: bool = False) -> List[Dict[str, Any]]:
"""Querys a response(-list) from a REST-API endpoint or URI from multiple pages
Specify `array_name` if there are multiple results / list.
Use allow_list to pick only the values specified.
Use ignore_list to pick everything but the values specified.
Both: allow_list items overwrite ignore_list items, still getting all not filtered.
Param pageSize is only guranteed to be valid for the first page if included within params.
Note:
Do not specify both endpoint and uri, only uri will be used
Keyword Arguments:
endpoint {str} -- endpoint to be queried. Either use this or uri (default: {None})
uri {str} -- uri to be queried. Either use this or endpoint (default: {None})
params {Dict[str, Any]} -- Dictionary with all URL-Parameters. pageSize only guranteed to be valid for first page (default: {None})
post_data {Dict[str, Any]} -- Dictionary with Body-Data. Only use on POST-Requests
request_type: {RequestType} -- Either GET or POST
array_name {str} -- name of array if there are multiple results wanted (default: {None})
allow_list {list} -- list of item to query (default: {None})
ignore_list {list} -- query all but these items(-groups). (default: {None})
add_time_stamp {bool} -- whether to add the capture timestamp (default: {False})
Raises:
ValueError: Neither a endpoint nor uri is specfied
ValueError: Negative or 0 pagesize
ValueError: array_name is specified but it is only a single object
Returns:
{List[Dict[str, Any]]} -- List of dictonarys as the results
"""
if(not endpoint and not uri):
raise ValueError("neiter endpoint nor uri specified")
if(endpoint and uri):
LOGGER.debug("added both endpoint and uri. This is unneccessary, uri is ignored")
# if neither specifed, get everything
if(not allow_list and not ignore_list):
ignore_list = []
if(params is None):
params = {}
# create uri out of endpoint
if(endpoint):
next_page = self.get_url(endpoint)
else:
next_page = uri
result_list: List[Dict[str, Any]] = []
# Aborts if no nextPage is found
while(next_page):
LOGGER.debug(f"Collected {len(result_list)} items until now. Next page: {next_page}")
if(self.__verbose):
LOGGER.info(f"Collected {len(result_list)} items until now. Next page: {next_page}")
# Request response
(response, send_time) = self.query_url(next_page, params, request_type, post_data)
# find follow page if available and set it
(_, next_page_link) = SppUtils.get_nested_kv(key_name="links.nextPage.href", nested_dict=response)
next_page: Optional[str] = next_page_link
if(next_page):
# Overwrite params with params from next link
params = ConnectionUtils.get_url_params(next_page)
# remove params from page
next_page = ConnectionUtils.url_set_params(next_page, None)
# Check if single object or not
if(array_name):
# get results for this page, if empty nothing happens
page_result_list: Optional[List[Dict[str, Any]]] = response.get(array_name, None)
if(page_result_list is None):
raise ValueError("array_name does not exist, this is probably a single object")
else:
page_result_list = [response]
filtered_results = ConnectionUtils.filter_values_dict(
result_list=page_result_list,
allow_list=allow_list,
ignore_list=ignore_list)
if(add_time_stamp): # direct time add to make the timestamps represent the real capture time
for mydict in filtered_results:
time_key, time_val = SppUtils.get_capture_timestamp_sec()
mydict[time_key] = time_val
result_list.extend(filtered_results)
# adjust pagesize if either the send time is too high
# or regulary adjust on max-page sizes requests
# dont adjust if page isnt full and therefore too quick
if(send_time > self.__preferred_time or len(page_result_list) == self.__page_size):
LOGGER.debug(f"send_time: {send_time}, len: {len(page_result_list)}, pageSize = {self.__page_size} ")
self.__page_size = ConnectionUtils.adjust_page_size(
page_size=len(page_result_list),
min_page_size=self.__min_page_size,
preferred_time=self.__preferred_time,
send_time=send_time)
LOGGER.debug(f"Changed pageSize from {len(page_result_list)} to {self.__page_size} ")
params["pageSize"] = self.__page_size
LOGGER.debug("objectList size %d", len(result_list))
return result_list