def display_dict(j: dict, level=0) -> None:
"""
Walks the cruise-control Response dict and attempts to display
the given top-level keys to the humans.
:param j:
:return:
"""
warnings.warn(
"This function is deprecated as of 0.3.0, as cruise-control already "
"provides human-readable text when supplied the parameter json=false. "
"Please ensure that your Cruise-Control-Version is 2.0.61 or higher, "
"and use json=false for retrieving human-readable text. "
"This function may be removed entirely in future versions.",
DeprecationWarning,
stacklevel=2)
for key in j.keys():
print_with_indent(f"'{key}':", level)
# Display the contents of this key with its key-specific function, if possible.
key_to_display_function = get_key_to_display_function()
if key in key_to_display_function:
key_to_display_function[key](j[key], level)
# Otherwise, just pretty-print this key's contents.
else:
print_with_indent(pformat(j[key]), level)
print_error()
def retrieve_response(self) -> requests.Response:
"""
Returns a final requests.Response object from cruise-control
where Response.text is JSON-formatted.
:return: requests.Response
"""
# cruise-control's JSON response has a 'progress' key in it so long
# as the response is not final.
#
# Once the response is final, it does not contain the 'progress' key.
#
# Accordingly, keep getting the response from this session and checking
# it for the 'progress' key.
#
# Return the response as JSON once we get a valid JSON response that we
# think is 'final'.
# Alert the humans to long-running poll
if self.print_to_stdout_enabled:
print_error("Starting long-running poll of {}".format(self.url))
# TODO: Session.get and Session.post can return a plethora of exceptions;
# they should be handled here.
response = self.session_http_method(self.url, headers=self.headers)
while 'progress' in response.json().keys():
if self.print_to_stdout_enabled:
display_response(response)
response = self.session_http_method(self.url, headers=self.headers)
# return the requests.response object
return response
def display_response(response: Response) -> None:
"""
Handles setting the display options and extracting the dict from the response,
then beginning the (possibly recursive) display parsing.
:param response:
:return:
"""
# Set pandas display options
# Display floats with a ',' 1000s separator, to two decimal places
set_option('display.float_format', "{:,.2f}".format)
# Display all rows and columns in the dataframe. Don't leave any out
set_option('display.max_rows', int(1E12))
set_option('display.max_columns', int(1E12))
# Display the full column width, even if it's very wide. Don't truncate it.
set_option('display.max_colwidth', 10000)
# Don't wrap the table we show
set_option('display.expand_frame_repr', False)
# Don't 'sparsify' sorted multi-index dataframes, since
# sparsifying makes it harder to do bash-type processing of tabular data
set_option('display.multi_sparse', False)
j: dict = response.json()
try:
display_dict(j)
except Exception as e:
print_error(f"Unhandled exception during display; showing raw JSON", e)
print_error(pformat(j))
def display_anomaly_detector_state(AnomalyDetectorState_json: dict,
level=0) -> None:
"""
Displays the 'AnomalyDetectorState' key in a cruise-control response
:param AnomalyDetectorState_json: the JSON structure referenced by the
'AnomalyDetectorState' key.
:return:
"""
warnings.warn(
"This function is deprecated as of 0.3.0, as cruise-control already "
"provides human-readable text when supplied the parameter json=false. "
"Please ensure that your Cruise-Control-Version is 2.0.61 or higher, "
"and use json=false for retrieving human-readable text. "
"This function may be removed entirely in future versions.",
DeprecationWarning,
stacklevel=2)
for key in AnomalyDetectorState_json:
if key == "recentGoalViolations":
print_with_indent(f"\t'{key}:", level)
for item in AnomalyDetectorState_json[key]:
if 'optimizationResult' in item:
optimization_json = json.loads(item['optimizationResult'])
del item['optimizationResult']
print_with_indent(pformat(item), level + 1)
print_with_indent("'optimizationResult':", level + 2)
display_dict(optimization_json, level + 2)
else:
print_with_indent(pformat(item), level + 1)
else:
print_error(f"\t'{key}: {AnomalyDetectorState_json[key]}")
def display_progress(progress_list: list, level=0) -> None:
"""
Displays the 'progress' key in a (non-final) cruise-control response
:param progress_list: the list structure referenced by the 'progress' key.
:return:
"""
warnings.warn(
"This function is deprecated as of 0.3.0, as cruise-control already "
"provides human-readable text when supplied the parameter json=false. "
"Please ensure that your Cruise-Control-Version is 2.0.61 or higher, "
"and use json=false for retrieving human-readable text. "
"This function may be removed entirely in future versions.",
DeprecationWarning,
stacklevel=2)
for elem in progress_list:
print_error(f"operation: {elem['operation']}")
df = DataFrame(elem['operationProgress'])
try:
df.set_index('step', inplace=True)
# The dataframe we get back may not have 'step' as a key
except KeyError:
pass
df_string = df.to_string()
# Indent the dataframe for better display
df_string = df_string.replace("\n", "\n ")
print_with_indent(df_string, level)
# Print ellipses so the humans know we are waiting :)
print_error(".........")
def print_with_indent(s: str, level=0) -> None:
"""
Prints the string with a level of indentation
:param s:
:param level:
:return:
"""
print_error(textwrap.indent(s, "\t" * level))
def display_goal_summary(goalSummary_json: dict, level=0) -> None:
"""
Displays the 'goalSummary' key in a cruise-control response
:param goalSummary_json: the JSON structure referenced by the 'goalSummary' key.
:return:
"""
# Make and print a dataframe for each of the goals in clusterModelState
for elem in goalSummary_json:
print_with_indent(f"{elem['goal']}: {elem['status']}", level)
df = DataFrame(elem['clusterModelStats']['statistics'])
print_with_indent(df.to_string(), level)
print_error()
def display_goal_readiness(goalReadiness_list: list, level=0) -> None:
"""
Displays the 'goalReadiness' key in a cruise-control response.
Note that this is not a top-level key, but is rather a sub-key of the
AnalyzerState top-level key.
:param goalReadiness_list: the list structure referenced by the
'goalReadiness' key.
:return:
"""
for elem in goalReadiness_list:
print_with_indent(f"{elem['name']}: {elem['status']}", level)
print_with_indent(pformat(elem['modelCompleteRequirement']))
print_error()
def print_with_indent(s: str, level=0) -> None:
"""
Prints the string with a level of indentation
:param s:
:param level:
:return:
"""
warnings.warn(
"This function is deprecated as of 0.3.0, as cruise-control already "
"provides human-readable text when supplied the parameter json=false. "
"Please ensure that your Cruise-Control-Version is 2.0.61 or higher, "
"and use json=false for retrieving human-readable text. "
"This function may be removed entirely in future versions.",
DeprecationWarning,
stacklevel=2)
print_error(textwrap.indent(s, "\t" * level))
def display_dict(j: dict, level=0) -> None:
"""
Walks the cruise-control Response dict and attempts to display
the given top-level keys to the humans.
:param j:
:return:
"""
for key in j.keys():
print_with_indent(f"'{key}':", level)
# Display the contents of this key with its key-specific function, if possible.
key_to_display_function = get_key_to_display_function()
if key in key_to_display_function:
key_to_display_function[key](j[key], level)
# Otherwise, just pretty-print this key's contents.
else:
print_with_indent(pformat(j[key]), level)
print_error()
def display_goal_summary(goalSummary_json: dict, level=0) -> None:
"""
Displays the 'goalSummary' key in a cruise-control response
:param goalSummary_json: the JSON structure referenced by the 'goalSummary' key.
:return:
"""
warnings.warn(
"This function is deprecated as of 0.3.0, as cruise-control already "
"provides human-readable text when supplied the parameter json=false. "
"Please ensure that your Cruise-Control-Version is 2.0.61 or higher, "
"and use json=false for retrieving human-readable text. "
"This function may be removed entirely in future versions.",
DeprecationWarning,
stacklevel=2)
# Make and print a dataframe for each of the goals in clusterModelState
for elem in goalSummary_json:
print_with_indent(f"{elem['goal']}: {elem['status']}", level)
df = DataFrame(elem['clusterModelStats']['statistics'])
print_with_indent(df.to_string(), level)
print_error()
def display_progress(progress_list: list, level=0) -> None:
"""
Displays the 'progress' key in a (non-final) cruise-control response
:param progress_list: the list structure referenced by the 'progress' key.
:return:
"""
for elem in progress_list:
print_error(f"operation: {elem['operation']}")
df = DataFrame(elem['operationProgress'])
try:
df.set_index('step', inplace=True)
# The dataframe we get back may not have 'step' as a key
except KeyError:
pass
df_string = df.to_string()
# Indent the dataframe for better display
df_string = df_string.replace("\n", "\n ")
print_with_indent(df_string, level)
# Print ellipses so the humans know we are waiting :)
print_error(".........")
def display_goal_readiness(goalReadiness_list: list, level=0) -> None:
"""
Displays the 'goalReadiness' key in a cruise-control response.
Note that this is not a top-level key, but is rather a sub-key of the
AnalyzerState top-level key.
:param goalReadiness_list: the list structure referenced by the
'goalReadiness' key.
:return:
"""
warnings.warn(
"This function is deprecated as of 0.3.0, as cruise-control already "
"provides human-readable text when supplied the parameter json=false. "
"Please ensure that your Cruise-Control-Version is 2.0.61 or higher, "
"and use json=false for retrieving human-readable text. "
"This function may be removed entirely in future versions.",
DeprecationWarning,
stacklevel=2)
for elem in goalReadiness_list:
print_with_indent(f"{elem['name']}: {elem['status']}", level)
print_with_indent(pformat(elem['modelCompleteRequirement']))
print_error()
def display_anomaly_detector_state(AnomalyDetectorState_json: dict,
level=0) -> None:
"""
Displays the 'AnomalyDetectorState' key in a cruise-control response
:param AnomalyDetectorState_json: the JSON structure referenced by the
'AnomalyDetectorState' key.
:return:
"""
for key in AnomalyDetectorState_json:
if key == "recentGoalViolations":
print_with_indent(f"\t'{key}:", level)
for item in AnomalyDetectorState_json[key]:
if 'optimizationResult' in item:
optimization_json = json.loads(item['optimizationResult'])
del item['optimizationResult']
print_with_indent(pformat(item), level + 1)
print_with_indent("'optimizationResult':", level + 2)
display_dict(optimization_json, level + 2)
else:
print_with_indent(pformat(item), level + 1)
else:
print_error(f"\t'{key}: {AnomalyDetectorState_json[key]}")
def display_response(response: Response) -> None:
"""
Handles setting the display options and extracting the dict from the response,
then beginning the (possibly recursive) display parsing.
:param response:
:return:
"""
warnings.warn(
"This function is deprecated as of 0.3.0, as cruise-control already "
"provides human-readable text when supplied the parameter json=false. "
"Please ensure that your Cruise-Control-Version is 2.0.61 or higher, "
"and use json=false for retrieving human-readable text. "
"This function may be removed entirely in future versions.",
DeprecationWarning,
stacklevel=2)
# Set pandas display options
# Display floats with a ',' 1000s separator, to two decimal places
set_option('display.float_format', "{:,.2f}".format)
# Display all rows and columns in the dataframe. Don't leave any out
set_option('display.max_rows', int(1E12))
set_option('display.max_columns', int(1E12))
# Display the full column width, even if it's very wide. Don't truncate it.
set_option('display.max_colwidth', 10000)
# Don't wrap the table we show
set_option('display.expand_frame_repr', False)
# Don't 'sparsify' sorted multi-index dataframes, since
# sparsifying makes it harder to do bash-type processing of tabular data
set_option('display.multi_sparse', False)
# Handle non-JSON (presumably plain-text) responses
try:
j: dict = response.json()
except json.decoder.JSONDecodeError:
print_error(response.text)
return
# Handle JSON responses
try:
display_dict(j)
except Exception as e:
print_error(f"Unhandled exception during display; showing raw JSON", e)
print_error(pformat(j))
def retrieve_response(self, method, url, **kwargs) -> requests.Response:
"""
Returns a final requests.Response object from cruise-control
where Response.text is JSON-formatted.
:return: requests.Response
"""
# Alert the humans that long-running request is starting
if 'params' in kwargs:
url_with_params = f"{url}?{urlencode(kwargs['params'])}"
else:
url_with_params = url
print_error(f"Starting long-running poll of {url_with_params}")
for key, value in kwargs.items():
if key == 'params':
continue
else:
print_error(f"{key}: {value}")
# Convenience closure to not have to copy-paste the parameters from
# this current environment.
def inner_request_helper():
return self.request(method, url, **kwargs)
def is_response_final(response: requests.Response):
# Define an inner convenience closure to avoid
# repeating ourselves
def json_or_text_guesser():
try:
# Try to guess whether the JSON response is final
return "progress" not in response.json().keys()
except ValueError:
# We have a non-JSON (probably plain text) response,
# and a non-202 status code.
#
# This response may not be final, but we have no
# way of doing further guessing, so warn the humans
# as best as we can, then presume finality.
cc_version = response.headers.get('Cruise-Control-Version')
if cc_version is not None:
warnings.warn(
f"json=False received from cruise-control version ({cc_version}) "
f"that does not support 202 response codes. "
f"Please upgrade cruise-control to >=2.0.61, or "
f"use json=True with cruise-control-client. "
f"Returning a potentially non-final response.")
# No cc_version in the response headers
else:
# cruise-control won't return version information if
# servlet receives too-large-URI request
if response.status_code == 414:
pass
else:
warnings.warn(
"Unable to determine cruise-control version. "
"Returning a potentially non-final response.")
return True
# We're talking to a version of cruise-control that supports
# 202: accepted, and we know that this response is not final.
if response.status_code == 202:
return False
else:
# Guess about whether this version of cruise-control supports 202
if "Cruise-Control-Version" in response.headers:
integer_semver = lambda x: [
int(elem) for elem in x.split('.')
]
cc_version = integer_semver(
response.headers["Cruise-Control-Version"])
# 202 is supported and was not returned; response final
if cc_version >= [2, 0, 61]:
return True
# 202 is not supported and was not returned; guess further
else:
return json_or_text_guesser()
# Probably we're getting a response (like a 414) before cruise-control
# can decorate the headers with version information
else:
return json_or_text_guesser()
response = inner_request_helper()
final_response = is_response_final(response)
while not final_response:
display_response(response)
response = inner_request_helper()
final_response = is_response_final(response)
# return the requests.response object
return response