def try_send_request(self, _attempts, _tries, request_func, request_label, request_retry_func, request_url):
to_raise_exception = None
to_return_response = None
try:
response = request_func()
if response is None:
raise TuneRequestModuleError(
error_message="Request Retry: No response",
error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE
)
if self.is_return_response(request_label, request_retry_func, request_url, response):
to_return_response = response
else:
self.logger.debug(
"Request Retry: Response: Valid: Retry Candidate",
extra={'request_url': request_url,
'request_label': request_label}
)
except tuple(self.request_retry_excps) as retry_ex:
if not self.is_retry_retry_ex(_tries, request_label, request_url, retry_ex):
to_raise_exception = retry_ex
except TuneRequestBaseError as tmv_ex:
if not self.is_retry_non_retry_ex(_tries, request_label, tmv_ex):
to_raise_exception = tmv_ex
except Exception as ex:
is_retry, raised_exception = self.is_retry_non_tune_ex(_tries, ex, request_label, request_url)
if not is_retry:
to_raise_exception = raised_exception
# A final check, whether we need to raise an exception, is in case the number of retries has exhausted.
if not to_raise_exception and self.is_exhausted_retries(
_tries,
partial(
self.logger.error,
"Request Retry: Exhausted Retries",
extra={
'attempts': _attempts,
'tries': _tries,
'request_url': request_url,
'request_label': request_label
}
)
):
to_raise_exception = TuneRequestModuleError(
error_message=("Request Retry: Exhausted Retries: {}: {}").format(request_label, request_url),
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED
)
return to_raise_exception, to_return_response
def exceptions():
exceptions_dict = dict()
exceptions_dict[RequestRetryException.__name__] = RequestRetryException()
exceptions_dict[TuneRequestBaseError.__name__] = TuneRequestBaseError()
exceptions_dict[Exception.__name__] = Exception()
exceptions_dict[TuneRequestModuleError.__name__] = dict()
exceptions_dict[TuneRequestModuleError.__name__][
TuneRequestErrorCodes.
REQ_ERR_RETRY_EXHAUSTED] = TuneRequestModuleError(
error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED)
exceptions_dict[TuneRequestModuleError.__name__][
TuneRequestErrorCodes.
REQ_ERR_UNEXPECTED_VALUE] = TuneRequestModuleError(
error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE)
return exceptions_dict
def is_retry_non_tune_ex(self, _tries, ex, request_label, request_url):
is_retry = True
raised_exception = None
error_exception = ex
ex_extra = {
'error_exception': base_class_name(error_exception),
'error_details': get_exception_message(error_exception),
'request_url': request_url,
'request_label': request_label
}
if not self.request_retry_excps_func or \
not self.request_retry_excps_func(error_exception, request_label):
self.logger.error(
"Request Retry: Unexpected: {}: Not Retry Candidate".format(base_class_name(error_exception)),
extra=ex_extra
)
is_retry = False
raised_exception = error_exception
if is_retry:
self.logger.warning(
"Request Retry: Unexpected: {}: Retry Candidate".format(base_class_name(error_exception)),
extra=ex_extra
)
if self.is_exhausted_retries(_tries, lambda: None):
is_retry = False
raised_exception = TuneRequestModuleError(
error_message="Unexpected: {}".format(base_class_name(error_exception)),
errors=error_exception,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED
)
return is_retry, raised_exception
def is_retry_non_tune_ex(self, tries, ex, request_url, request_label=None):
"""Is Retry Non-TUNE Exception
:param tries:
:param ex:
:param request_url:
:param request_label:
:return:
"""
_request_label = 'Is Retry Non-TUNE Exception'
request_label = '{}: {}'.format(request_label, _request_label) if request_label is not None else _request_label
is_retry = True
raised_exception = None
error_exception = ex
ex_extra = {
'error_exception': base_class_name(error_exception),
'error_details': get_exception_message(error_exception),
'request_url': request_url,
'request_label': request_label
}
if not self.request_retry_excps_func or \
not self.request_retry_excps_func(error_exception, request_label):
self.logger.error(
'{}: Unexpected: {}: Not Retry Candidate'.format(request_label, base_class_name(error_exception)),
extra=ex_extra
)
is_retry = False
raised_exception = error_exception
if is_retry:
self.logger.warning(
'{}: Unexpected: {}: Retry Candidate'.format(request_label, base_class_name(error_exception)),
extra=ex_extra
)
if self.is_exhausted_retries(tries, lambda: None):
is_retry = False
raised_exception = TuneRequestModuleError(
error_message='{}: Unexpected: {}'.format(request_label, base_class_name(error_exception)),
errors=error_exception,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED
)
return is_retry, raised_exception
def _request_data(
self,
request_method,
request_url,
request_params=None,
request_data=None,
request_json=None,
request_headers=None,
request_auth=None,
cookie_payload=None,
request_label=None,
timeout=60,
build_request_curl=True,
allow_redirects=True,
verify=True,
stream=False
):
"""Request Data from requests.
Args:
request_method: request_method for the new :class:`Request` object.
logger: logging instance
request_url: URL for the new :class:`Request` object.
request_params: (optional) Dictionary or bytes to be sent in the
query string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_json: (optional) json data to send in the body of
the :class:`Request`.
request_headers: (optional) Dictionary of HTTP Headers to send
with the :class:`Request`.
request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
timeout: (optional) How long to wait for the server to send data
before giving up.
allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE
redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
stream: (optional) if ``False``, the response content will be
immediately downloaded.
Returns:
requests.Response
"""
if request_label is None:
request_label = 'Request Data'
if not request_method:
raise TuneRequestValueError(error_message="Parameter 'request_method' not defined")
if not request_url:
raise TuneRequestValueError(error_message="Parameter 'request_url' not defined")
self.built_request_curl = None
self.logger.debug(
'{}: Session: Details'.format(request_label),
extra={'cookie_payload': self.tune_request.session.cookies.get_dict(),
'request_label': request_label}
)
response = None
headers = None
if request_headers:
headers = request_headers
request_method = request_method.upper()
if request_data and isinstance(request_data, str):
if len(request_data) <= 20:
request_data_extra = request_data
else:
request_data_extra = request_data[:20] + ' ...'
else:
request_data_extra = safe_str(request_data)
request_extra = {
'request_method': request_method,
'request_url': request_url,
'timeout': timeout,
'request_params': safe_dict(request_params),
'request_data': request_data_extra,
'request_headers': safe_dict(headers),
'request_label': request_label
}
self.logger.debug('{}: Details'.format(request_label), extra=request_extra)
self.built_request_curl = None
kwargs = {}
if headers:
kwargs.update({'headers': headers})
if request_auth:
kwargs.update({'auth': request_auth})
if timeout and isinstance(timeout, int):
kwargs.update({'timeout': timeout})
if allow_redirects:
kwargs.update({'allow_redirects': allow_redirects})
if stream:
kwargs.update({'stream': stream})
if cookie_payload:
kwargs.update({'cookies': cookie_payload})
kwargs.update({'verify': verify})
try:
if build_request_curl:
# In case no authentication information has been provided,
# use session's cookies information, if exists
if not request_auth and self.session.cookies and len(self.session.cookies) > 0:
request_auth = self.session.cookies
self.built_request_curl = command_line_request_curl(
request_method=request_method,
request_url=request_url,
request_headers=headers,
request_params=request_params,
request_data=request_data,
request_json=request_json,
request_auth=request_auth,
request_timeout=timeout,
request_allow_redirects=allow_redirects
)
self.logger.debug(
'{}: Curl'.format(request_label),
extra={
'request_method': request_method,
'request_label': request_label,
'request_curl': self.built_request_curl
}
)
if hasattr(response, 'url'):
self.logger.debug('{}: {}'.format(request_label, request_method), extra={'response_url': response.url})
if request_params:
kwargs.update({'params': request_params})
if request_data:
kwargs.update({'data': request_data})
if request_json:
kwargs.update({'json': request_json})
if headers:
kwargs.update({'headers': headers})
kwargs.update({'request_method': request_method, 'request_url': request_url})
response = self.tune_request.request(**kwargs)
except Exception as ex:
self.logger.error(
'{}: Request Base: Error'.format(request_label),
extra={
'request_label': request_label,
'error_exception': base_class_name(ex),
'error_details': get_exception_message(ex)
}
)
raise
if response is None:
self.logger.error(
'{}: Response: Failed'.format(request_label),
extra={'request_curl': self.built_request_curl},
)
raise TuneRequestModuleError(
error_message='{}: Response: Failed'.format(request_label),
error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE,
error_request_curl=self.built_request_curl
)
http_status_code = response.status_code
response_headers = json.loads(json.dumps(dict(response.headers)))
http_status_type = \
http_status_code_to_type(http_status_code)
http_status_desc = \
http_status_code_to_desc(http_status_code)
response_extra = {
'http_status_code': http_status_code,
'http_status_type': http_status_type,
'http_status_desc': http_status_desc,
'response_headers': safe_dict(response_headers),
}
self.logger.debug(
'{}: Response: Details'.format(request_label),
extra=response_extra,
)
http_status_successful = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.SUCCESSFUL
)
http_status_redirection = is_http_status_type(
http_status_code=http_status_code, http_status_type=HttpStatusType.REDIRECTION
)
if http_status_successful or http_status_redirection:
if hasattr(response, 'url') and \
response.url and \
len(response.url) > 0:
response_extra.update({'response_url': response.url})
self.logger.debug(
'{}: Cookie Payload'.format(request_label),
extra={'cookie_payload': self.tune_request.session.cookies.get_dict(),
'request_label': request_label}
)
assert response
return response
else:
response_extra.update({'error_request_curl': self.built_request_curl})
self.logger.error('{}: Response: Failed'.format(request_label), extra=response_extra)
json_response_error = \
build_response_error_details(
response=response,
request_label=request_label,
request_url=request_url
)
extra_error = copy.deepcopy(json_response_error)
if self.logger_level == logging.INFO:
error_response_details = \
extra_error.get('response_details', None)
if error_response_details and \
isinstance(error_response_details, str) and \
len(error_response_details) > 100:
extra_error['response_details'] = error_response_details[:100] + ' ...'
if self.built_request_curl and \
'error_request_curl' not in extra_error:
extra_error.update({'error_request_curl': self.built_request_curl})
self.logger.error('{}: Error: Response: Details'.format(request_label), extra=extra_error)
kwargs = {
'error_status': json_response_error.get("response_status", None),
'error_reason': json_response_error.get("response_reason", None),
'error_details': json_response_error.get("response_details", None),
'error_request_curl': self.built_request_curl
}
if http_status_code in [
HttpStatusCode.BAD_REQUEST,
HttpStatusCode.UNAUTHORIZED,
HttpStatusCode.FORBIDDEN,
HttpStatusCode.NOT_FOUND,
HttpStatusCode.METHOD_NOT_ALLOWED,
HttpStatusCode.NOT_ACCEPTABLE,
HttpStatusCode.REQUEST_TIMEOUT,
HttpStatusCode.CONFLICT,
HttpStatusCode.GONE,
HttpStatusCode.UNPROCESSABLE_ENTITY,
HttpStatusCode.TOO_MANY_REQUESTS,
]:
kwargs.update({'error_code': http_status_code})
raise TuneRequestClientError(**kwargs)
if http_status_code in [
HttpStatusCode.INTERNAL_SERVER_ERROR,
HttpStatusCode.NOT_IMPLEMENTED,
HttpStatusCode.BAD_GATEWAY,
HttpStatusCode.SERVICE_UNAVAILABLE,
HttpStatusCode.NETWORK_AUTHENTICATION_REQUIRED,
]:
kwargs.update({'error_code': http_status_code})
raise TuneRequestServiceError(**kwargs)
kwargs.update({'error_code': json_response_error['response_status_code']})
extra_unhandled = copy.deepcopy(kwargs)
extra_unhandled.update({'http_status_code': http_status_code})
self.logger.error('{}: Error: Unhandled'.format(request_label), extra=extra_unhandled)
raise TuneRequestModuleError(**kwargs)
def try_send_request(self, attempts, tries, request_func, request_retry_func, request_url, request_label=None):
"""Try Send Request
:param attempts:
:param tries:
:param request_func:
:param request_retry_func:
:param request_url:
:param request_label:
:return:
"""
_request_label = 'Try Send Request'
request_label = '{}: {}'.format(request_label, _request_label) if request_label is not None else _request_label
to_raise_exception = None
to_return_response = None
try:
response = request_func()
if response is None:
raise TuneRequestModuleError(
error_message='{}: No response'.format(request_label),
error_code=TuneRequestErrorCodes.REQ_ERR_UNEXPECTED_VALUE
)
if self.is_return_response(request_retry_func, request_url, response, request_label=request_label):
to_return_response = response
else:
self.logger.debug(
'{}: Response: Valid: Retry Candidate'.format(request_label),
extra={'request_url': request_url,
'request_label': request_label}
)
except tuple(self.request_retry_excps) as retry_ex:
if not self.is_retry_retry_ex(tries, request_url, retry_ex, request_label=request_label):
to_raise_exception = retry_ex
except TuneRequestBaseError as tmv_ex:
if not self.is_retry_non_retry_ex(tries, tmv_ex, request_label=request_label):
to_raise_exception = tmv_ex
except Exception as ex:
is_retry, raised_exception = self.is_retry_non_tune_ex(tries, ex, request_url, request_label=request_label)
if not is_retry:
to_raise_exception = raised_exception
# A final check, whether we need to raise an exception, is in case the number of retries has exhausted.
if not to_raise_exception and self.is_exhausted_retries(
tries,
partial(
self.logger.error,
'{}: Exhausted Retries'.format(request_label),
extra={
'attempts': attempts,
'tries': tries,
'request_url': request_url,
'request_label': request_label
}
)
):
to_raise_exception = TuneRequestModuleError(
error_message=('{}: Exhausted Retries: {}').format(request_label, request_url),
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED
)
return to_raise_exception, to_return_response
def request(
self,
request_method,
request_url,
request_params=None,
request_data=None,
request_json=None,
request_retry=None,
request_retry_excps=None,
request_retry_http_status_codes=None,
request_retry_func=None,
request_retry_excps_func=None,
request_headers=None,
request_auth=None,
cookie_payload=None,
build_request_curl=True,
allow_redirects=True,
verify=True,
stream=False,
request_label=None
):
"""Request data from remote source with retries.
Args:
request_method: request_method for the new :class:`Request` object.
request_url: URL for the new :class:`Request` object.
request_params: (optional) Dictionary or bytes to be sent in the
query string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_json: (optional) json data to send in the body of
the :class:`Request`.
request_retry: (optional) Retry configuration.
request_retry_func: (optional) Retry function, alternative
to request_retry_excps.
request_retry_excps: An exception or a tuple of exceptions
to catch.
request_headers: (optional) Dictionary of HTTP Headers to
send with the :class:`Request`.
request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE
redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
stream: (optional) if ``False``, the response content will be
immediately downloaded.
request_label:
Returns:
requests.Response: Data result if success or None if error.
Raises:
ServiceGatewayTimeoutError: Upon any timeout condition.
Exception: Upon error within this request_method.
Notes:
* tries: the maximum number of attempts. default: 1.
* delay: initial delay between attempts. default: 1.
* max_delay: the maximum value of delay. default: None (no limit).
* backoff: multiplier applied to delay between attempts.
default: 1 (no backoff).
* jitter: extra seconds added to delay between attempts.
default: 0.
"""
if request_label is None:
request_label = 'Request'
self.logger.debug('{}: Start'.format(request_label))
timeout = None
if not verify:
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
if request_method:
request_method = request_method.upper()
if request_label is None:
request_label = 'Request'
if not request_retry:
request_retry = {}
if 'timeout' not in request_retry:
request_retry['timeout'] = self._REQUEST_CONFIG['timeout']
if 'tries' not in request_retry:
request_retry['tries'] = self._REQUEST_CONFIG['tries']
if 'delay' not in request_retry:
request_retry['delay'] = self._REQUEST_CONFIG['delay']
self._prep_request_retry(request_retry, request_retry_http_status_codes)
if not self.tune_request:
self.tune_request = TuneRequest(
retry_tries=self.retry_tries,
retry_backoff=self.retry_backoff,
retry_codes=self.request_retry_http_status_codes
)
key_user_agent = 'User-Agent'
header_user_agent = {key_user_agent: __USER_AGENT__}
if request_headers:
if key_user_agent not in request_headers:
request_headers.update(header_user_agent)
else:
request_headers = header_user_agent
kwargs = {
'request_method': request_method,
'request_url': request_url,
'request_params': request_params,
'request_data': request_data,
'request_json': request_json,
'request_headers': request_headers,
'request_auth': request_auth,
'cookie_payload': cookie_payload,
'request_label': request_label,
'timeout': timeout,
'build_request_curl': build_request_curl,
'allow_redirects': allow_redirects,
'verify': verify,
'stream': stream
}
time_start_req = dt.datetime.now()
if request_retry_func is None:
request_retry_func = self.request_retry_func
if request_retry_excps_func is None:
request_retry_excps_func = self.request_retry_excps_func
if request_retry_excps_func is None:
request_retry_excps_func = mv_request_retry_excps_func
if request_retry_http_status_codes is not None:
self.request_retry_http_status_codes = request_retry_http_status_codes
if request_retry_excps is not None:
self.request_retry_excps = request_retry_excps
extra_request = copy.copy(kwargs)
if request_retry:
extra_request.update({'request_retry': request_retry})
if request_label:
extra_request.update({'request_label': request_label})
if request_retry:
extra_request.update({'request_retry': request_retry})
if request_retry_func:
extra_request.update({'request_retry_func': request_retry_func})
if request_retry_http_status_codes:
extra_request.update({'request_retry_http_status_codes': request_retry_http_status_codes})
if request_retry_excps:
extra_request.update({'request_retry_excps': request_retry_excps})
if request_retry_excps:
extra_request.update({'request_retry_excps_func': request_retry_excps_func})
extra_request.update(env_usage())
self.logger.debug('{}: Start: Details'.format(request_label), extra=extra_request)
try:
self._prep_request_retry(request_retry, request_retry_http_status_codes)
response = self._request_retry(
call_func=self._request_data,
fargs=None,
fkwargs=kwargs,
timeout=timeout,
request_label=request_label,
request_retry_func=request_retry_func,
request_retry_excps_func=request_retry_excps_func
)
except (
requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, requests.exceptions.Timeout,
) as ex_req_timeout:
raise TuneRequestServiceError(
error_message='{}: Exception: Timeout'.format(request_label),
errors=ex_req_timeout,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.GATEWAY_TIMEOUT
)
except requests.exceptions.HTTPError as ex_req_http:
raise TuneRequestModuleError(
error_message='{}: Exception: HTTP Error'.format(request_label),
errors=ex_req_http,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_HTTP
)
except (
requests.exceptions.ConnectionError, requests.exceptions.ProxyError, requests.exceptions.SSLError,
) as ex_req_connect:
raise TuneRequestModuleError(
error_message='{}: Exception: {}'.format(request_label, base_class_name(ex_req_connect)),
errors=ex_req_connect,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT
)
except (BrokenPipeError, ConnectionError,) as ex_ose_connect:
raise TuneRequestModuleError(
error_message='{}: Exception: OSE {}'.format(request_label, base_class_name(ex_ose_connect)),
errors=ex_ose_connect,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_CONNECT
)
except requests.packages.urllib3.exceptions.ProtocolError as ex_req_urllib3_protocol:
raise TuneRequestModuleError(
error_message='{}: Exception: Urllib3: Protocol Error'.format(request_label),
errors=ex_req_urllib3_protocol,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT
)
except requests.packages.urllib3.exceptions.ReadTimeoutError as ex_req_urllib3_read_timeout:
raise TuneRequestServiceError(
error_message='{}: Exception: Urllib3: Read Timeout Error'.format(request_label),
errors=ex_req_urllib3_read_timeout,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.GATEWAY_TIMEOUT
)
except requests.exceptions.TooManyRedirects as ex_req_redirects:
raise TuneRequestModuleError(
error_message='{}: Exception: Too Many Redirects'.format(request_label),
errors=ex_req_redirects,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_REDIRECTS
)
except requests.exceptions.RetryError as ex_req_adapter_retry:
# Expected structure of RetryError:
# RetryError has list 'args' of len 1: RetryError.args[0] == MaxRetryError (from urlib3)
# Structure of MaxRetryError: MaxRetryError.reason == ResponseError (from urlib3)
if len(ex_req_adapter_retry.args) == 1 and hasattr(ex_req_adapter_retry.args[0], 'reason'):
response_err = ex_req_adapter_retry.args[0].reason
status_codes = [int(s) for s in response_err.args[0].split() if s.isdigit()]
if len(status_codes) == 1:
http_status_code = status_codes[0]
http_status_type = http_status_code_to_type(http_status_code)
error_kwargs = {
'errors': ex_req_adapter_retry,
'error_code': http_status_code,
'error_reason': response_err.args[0],
'error_message': "Request: Exception: HTTPAdapter: Retries exhausted on: '{}'"
.format(request_url),
'error_request_curl': self.built_request_curl,
}
self.logger.error(
'{}: Exception: HTTPAdapter: Max retry error'.format(request_label), extra=error_kwargs
)
if http_status_type == HttpStatusType.CLIENT_ERROR:
raise TuneRequestClientError(**error_kwargs)
elif http_status_type == HttpStatusType.SERVER_ERROR:
raise TuneRequestServiceError(**error_kwargs)
# THIS BLOCK SHOULD NOT BE ACTUALLY ACCESSED. IF IT DOES LOOK INTO IT:
self.logger.error(
'{}: Unexpected RetryError occurred'.format(request_label),
extra={'request_curl': self.built_request_curl}
)
raise TuneRequestModuleError(
error_message='{}: Exception: HTTPAdapter: Unexpected Retry Error'.format(request_label),
errors=ex_req_adapter_retry,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED,
)
except requests.exceptions.RequestException as ex_req_request:
raise TuneRequestModuleError(
error_message='{}: Exception: Request Error'.format(request_label),
errors=ex_req_request,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST
)
except TuneRequestBaseError:
raise
except Exception as ex:
print_traceback(ex)
raise TuneRequestModuleError(
error_message='{}: Exception: Unexpected'.format(request_label),
errors=ex,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE
)
time_end_req = dt.datetime.now()
diff_req = time_end_req - time_start_req
request_time_msecs = int(diff_req.total_seconds() * 1000)
self.logger.info(
'{}: Finished'.format(request_label), extra={
'request_time_msecs': request_time_msecs,
}
)
self.logger.debug('{}: Usage'.format(request_label), extra=env_usage())
return response
def validate_response(
response,
request_curl,
request_label=None,
):
"""Validate response
Args:
response:
request_label:
request_url:
Returns:
"""
response_extra = {}
if request_label:
response_extra.update({'request_label': request_label})
if not response:
log.error("Validate Response: Failed: None", extra=response_extra)
raise TuneRequestModuleError(
error_message="Validate Response: Failed: None",
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE)
else:
log.debug("Validate Response: Defined", extra=response_extra)
response_extra.update({'http_status_code': response.status_code})
if hasattr(response, 'text'):
response_text_length = len(response.text)
response_extra.update({'response_text_length': response_text_length})
if response.headers:
if 'Content-Type' in response.headers:
response_headers_content_type = \
safe_str(response.headers['Content-Type'])
response_extra.update(
{'Content-Type': response_headers_content_type})
if 'Content-Length' in response.headers:
response_headers_content_length = \
safe_int(response.headers['Content-Length'])
response_extra.update({
'Content-Length':
convert_size(response_headers_content_length)
})
if 'Content-Encoding' in response.headers:
response_content_encoding = \
safe_str(response.headers['Content-Encoding'])
response_extra.update(
{'Content-Encoding': response_content_encoding})
if 'Transfer-Encoding' in response.headers:
response_transfer_encoding = \
safe_str(response.headers['Transfer-Encoding'])
response_extra.update(
{'Transfer-Encoding': response_transfer_encoding})
if not is_http_status_successful(http_status_code=response.status_code):
log.error("Validate Response: Failed", extra=response_extra)
raise TuneRequestModuleError(
error_message="Validate Request: Failed",
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE)
else:
log.debug("Validate Response: Success", extra=response_extra)
def handle_json_decode_error(
response_decode_ex,
response,
response_extra=None,
request_label=None,
request_curl=None,
):
"""Handle JSON Decode Error
Args:
response_json_decode_error:
response:
response_extra:
request_label:
Returns:
"""
if response_extra is None:
response_extra = {}
if request_label:
response_extra.update({'request_label': request_label})
if hasattr(response, 'text') and \
response.text and \
len(response.text) > 0:
response_details = response.text
response_details_source = 'text'
response_content_length = len(response_details)
if response_details.startswith('<html'):
response_details_source = 'html'
soup_html = bs4.BeautifulSoup(response_details, "html.parser")
# kill all script and style elements
for script in soup_html(["script", "style"]):
script.extract() # rip it out
text_html = soup_html.get_text()
lines_html = [
line for line in text_html.split('\n') if line.strip() != ''
]
lines_html = [line.strip(' ') for line in lines_html]
response_details = lines_html
elif response_details.startswith('<?xml'):
response_details_source = 'xml'
response_details = json.dumps(xmltodict.parse(response_details))
else:
pprint(response_details)
response_extra.update({
'response_details':
response_details,
'response_details_source':
response_details_source,
'response_content_length':
response_content_length,
'error_exception':
base_class_name(response_decode_ex),
'error_details':
get_exception_message(response_decode_ex)
})
log.error("Validate JSON Response: Failed: Invalid", extra=response_extra)
raise TuneRequestModuleError(
error_message="Validate JSON Response: Failed: Invalid",
errors=response_decode_ex,
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE)
def requests_response_json(
response,
request_curl,
request_label=None,
raise_ex_if_not_json_response=True,
):
"""Get JSON from response from requests
Args:
response:
request_label:
Returns:
"""
json_response = None
response_extra = {}
if request_label:
response_extra.update({'request_label': request_label})
try:
json_response = response.json()
response_details_source = 'json'
response_content_length = len(json_response)
response_extra.update({
'response_details_source':
response_details_source,
'response_content_length':
response_content_length
})
except json.decoder.JSONDecodeError as json_decode_ex:
log.error("Validate JSON Response: Failed: JSONDecodeError",
extra=response_extra)
data = requests_toolbelt.utils.dump.dump_all(response)
pprint(data.decode('utf-8'))
pprint(response.text)
handle_json_decode_error(response_decode_ex=json_decode_ex,
response=response,
response_extra=response_extra,
request_label=request_label,
request_curl=request_curl)
except Exception as ex:
log.error("Validate JSON Response: Failed: Exception",
extra=response_extra)
pprint(response.text)
handle_json_decode_error(response_decode_ex=ex,
response=response,
response_extra=response_extra,
request_label=request_label,
request_curl=request_curl)
if json_response is None:
if raise_ex_if_not_json_response:
log.error("Validate JSON Response: Failed: None",
extra=response_extra)
raise TuneRequestModuleError(
error_message="Validate JSON Response: Failed: None",
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE)
else:
log.warning("Validate JSON Response: None", extra=response_extra)
else:
log.debug("Validate JSON Response: Valid", extra=response_extra)
return json_response
def validate_json_response(response,
request_curl,
request_label=None,
response_content_type_expected='application/json',
raise_ex_if_not_json_response=True):
"""Validate JSON response.
Args:
response:
request_label:
response_content_type_expected:
raise_ex_if_not_json_response:
Returns:
"""
validate_response(response, request_label)
json_response = None
response_extra = {}
if request_label:
response_extra.update({'request_label': request_label})
response_extra.update(
{'Content-Type (Expected)': response_content_type_expected})
if hasattr(response, 'headers'):
response_content_type = response.headers.get('Content-Type', None)
if response_content_type is not None:
is_valid_response_content_type = \
response_content_type == response_content_type_expected or \
response_content_type.startswith(response_content_type_expected)
if is_valid_response_content_type:
json_response = requests_response_json(
response=response,
request_curl=request_curl,
request_label=request_label,
)
elif response_content_type.startswith('text/html'):
try:
response_content_html_lines = \
requests_response_text_html(
response=response
)
except Exception as ex:
raise TuneRequestModuleError(
error_message=request_label,
errors=ex,
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.
REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED)
raise TuneRequestModuleError(
error_message="Unexpected 'Content-Type': '{}', Expected: '{}'"
.format(response_content_type, response_content_type_expected),
errors=response_content_html_lines,
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.
REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED)
else:
raise TuneRequestModuleError(
error_message="Unexpected 'Content-Type': '{}', Expected: '{}'"
.format(response_content_type, response_content_type_expected),
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.
REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED)
else:
raise TuneRequestModuleError(error_message="Undefined 'Content-Type'",
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.
REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED)
response_extra.update({
'http_status_code':
response.status_code,
'raise_ex_if_not_json_response':
raise_ex_if_not_json_response
})
log.debug("Validate JSON Response: Details", extra=response_extra)
return json_response
def request(
self,
request_method,
request_url,
request_params=None,
request_data=None,
request_json=None,
request_retry=None,
request_retry_excps=None,
request_retry_http_status_codes=None,
request_retry_func=None,
request_retry_excps_func=None,
request_headers=None,
request_auth=None,
cookie_payload=None,
build_request_curl=True,
allow_redirects=True,
verify=True,
stream=False,
request_label=None
):
"""Request data from remote source with retries.
Args:
request_method: request_method for the new :class:`Request` object.
request_url: URL for the new :class:`Request` object.
request_params: (optional) Dictionary or bytes to be sent in the
query string for the :class:`Request`.
request_data: (optional) Dictionary, bytes, or file-like object to
send in the body of the :class:`Request`.
request_json: (optional) json data to send in the body of
the :class:`Request`.
request_retry: (optional) Retry configuration.
request_retry_func: (optional) Retry function, alternative
to request_retry_excps.
request_retry_excps: An exception or a tuple of exceptions
to catch.
request_headers: (optional) Dictionary of HTTP Headers to
send with the :class:`Request`.
request_auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE
redirect following is allowed.
verify: (optional) whether the SSL cert will be verified. A
CA_BUNDLE path can also be provided. Defaults to ``True``.
stream: (optional) if ``False``, the response content will be
immediately downloaded.
request_label:
Returns:
requests.Response: Data result if success or None if error.
Raises:
ServiceGatewayTimeoutError: Upon any timeout condition.
Exception: Upon error within this request_method.
Notes:
* tries: the maximum number of attempts. default: 1.
* delay: initial delay between attempts. default: 1.
* max_delay: the maximum value of delay. default: None (no limit).
* backoff: multiplier applied to delay between attempts.
default: 1 (no backoff).
* jitter: extra seconds added to delay between attempts.
default: 0.
"""
self.logger.debug("Request: Start: {}".format(request_label if request_label else ""))
timeout = None
retry_tries = None
retry_delay = None
retry_backoff = 0
retry_jitter = 0
retry_max_delay = None
if not verify:
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
if request_method:
request_method = request_method.upper()
if not request_retry:
request_retry = {}
if 'timeout' not in request_retry:
request_retry['timeout'] = self._REQUEST_CONFIG['timeout']
if 'tries' not in request_retry:
request_retry['tries'] = self._REQUEST_CONFIG['tries']
if 'delay' not in request_retry:
request_retry['delay'] = self._REQUEST_CONFIG['delay']
self._prep_request_retry(request_retry, request_retry_http_status_codes)
if not self.tune_request:
self.tune_request = TuneRequest(
retry_tries=self.retry_tries,
retry_backoff=self.retry_backoff,
retry_codes=self.request_retry_http_status_codes
)
key_user_agent = 'User-Agent'
header_user_agent = {key_user_agent: __USER_AGENT__}
if request_headers:
if key_user_agent not in request_headers:
request_headers.update(header_user_agent)
else:
request_headers = header_user_agent
kwargs = {
'request_method': request_method,
'request_url': request_url,
'request_params': request_params,
'request_data': request_data,
'request_json': request_json,
'request_headers': request_headers,
'request_auth': request_auth,
'cookie_payload': cookie_payload,
'request_label': request_label,
'timeout': timeout,
'build_request_curl': build_request_curl,
'allow_redirects': allow_redirects,
'verify': verify,
'stream': stream
}
time_start_req = dt.datetime.now()
if request_retry_func is None:
request_retry_func = self.request_retry_func
if request_retry_excps_func is None:
request_retry_excps_func = self.request_retry_excps_func
if request_retry_http_status_codes is not None:
self.request_retry_http_status_codes = request_retry_http_status_codes
if request_retry_excps is not None:
self.request_retry_excps = request_retry_excps
extra_request = copy.copy(kwargs)
if request_retry:
extra_request.update({'request_retry': request_retry})
if request_label:
extra_request.update({'request_label': request_label})
if request_retry:
extra_request.update({'request_retry': request_retry})
if request_retry_func:
extra_request.update({'request_retry_func': request_retry_func})
if request_retry_http_status_codes:
extra_request.update({'request_retry_http_status_codes': request_retry_http_status_codes})
if request_retry_excps:
extra_request.update({'request_retry_excps': request_retry_excps})
if request_retry_excps:
extra_request.update({'request_retry_excps_func': request_retry_excps_func})
self.logger.debug("Request: Details", extra=extra_request)
try:
self._prep_request_retry(request_retry, request_retry_http_status_codes)
response = self._request_retry(
call_func=self._request,
fargs=None,
fkwargs=kwargs,
timeout=timeout,
request_label=request_label,
request_retry_func=request_retry_func,
request_retry_excps_func=request_retry_excps_func
)
except (
requests.exceptions.ConnectTimeout,
requests.exceptions.ReadTimeout,
requests.exceptions.Timeout,
) as ex_req_timeout:
raise TuneRequestServiceError(
error_message="Request: Exception: Timeout",
errors=ex_req_timeout,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.GATEWAY_TIMEOUT
)
except requests.exceptions.HTTPError as ex_req_http:
raise TuneRequestModuleError(
error_message="Request: Exception: HTTP Error",
errors=ex_req_http,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_HTTP
)
except requests.exceptions.ConnectionError as ex_req_connect:
raise TuneRequestModuleError(
error_message="Request: Exception: Connection Error",
errors=ex_req_connect,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT
)
except BrokenPipeError as ex_broken_pipe:
raise TuneRequestModuleError(
error_message="Request: Exception: Broken Pipe Error",
errors=ex_broken_pipe,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT
)
except ConnectionError as ex_connect:
raise TuneRequestModuleError(
error_message="Request: Exception: Connection Error",
errors=ex_connect,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT
)
except requests.packages.urllib3.exceptions.ProtocolError as ex_req_urllib3_protocol:
raise TuneRequestModuleError(
error_message="Request: Exception: Protocol Error",
errors=ex_req_urllib3_protocol,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_CONNECT
)
except requests.packages.urllib3.exceptions.ReadTimeoutError as ex_req_urllib3_read_timeout:
raise TuneRequestServiceError(
error_message="Request: Exception: Urllib3: Read Timeout Error",
errors=ex_req_urllib3_read_timeout,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.GATEWAY_TIMEOUT
)
except requests.exceptions.TooManyRedirects as ex_req_redirects:
raise TuneRequestModuleError(
error_message="Request: Exception: Too Many Redirects",
errors=ex_req_redirects,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST_REDIRECTS
)
except requests.exceptions.RequestException as ex_req_request:
raise TuneRequestModuleError(
error_message="Request: Exception: Request Error",
errors=ex_req_request,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_REQUEST
)
except TuneRequestBaseError:
raise
except Exception as ex:
print_traceback(ex)
raise TuneRequestModuleError(
error_message="Request: Exception: Unexpected",
errors=ex,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE
)
time_end_req = dt.datetime.now()
diff_req = time_end_req - time_start_req
request_time_msecs = int(diff_req.total_seconds() * 1000)
self.logger.debug(
"Request: Completed", extra={'request_label': request_label,
'request_time_msecs': request_time_msecs}
)
return response
def _request_retry(
self,
call_func,
fargs=None,
fkwargs=None,
timeout=60,
request_retry_func=None,
request_retry_excps_func=None,
request_label=None,
):
"""Request Retry
Args:
call_func: the function to execute.
request_retry_excps: A tuple of exceptions to catch.
fargs: the positional arguments of the function to execute.
fkwargs: the named arguments of the function to execute.
timeout: (optional) How long to wait for the server to send
data before giving up.
request_retry_func: (optional) Retry alternative to request_retry_excps.
retry_tries: the maximum number of attempts.
default: -1 (infinite).
retry_delay: initial delay between attempts.
default: 0.
retry_max_delay:the maximum value of delay.
default: None (no limit).
retry_backoff: multiplier applied to delay between attempts.
default: 1 (no backoff).
retry_jitter: extra seconds added to delay between attempts.
default: 0.
request_label: Label
Returns:
"""
request_retry_extra = {'timeout': timeout}
request_retry_extra.update({
'request_retry_http_status_codes':
self.request_retry_http_status_codes
})
if self.request_retry_excps is not None:
request_retry_excp_names = [
excp.__name__ for excp in list(self.request_retry_excps)
]
request_retry_extra.update(
{'request_retry_excps': request_retry_excp_names})
if request_retry_func is not None:
request_retry_func_name = request_retry_func.__name__
request_retry_extra.update(
{'request_retry_func': request_retry_func_name})
if request_retry_excps_func is not None:
request_retry_excps_func_name = request_retry_excps_func.__name__
request_retry_extra.update(
{'request_retry_excps_func': request_retry_excps_func_name})
self.logger.debug("Request Retry: Start: {}".format(
request_label if request_label else ""),
extra=request_retry_extra)
args = fargs if fargs else list()
kwargs = fkwargs if fkwargs else dict()
request_url = kwargs[
'request_url'] if kwargs and 'request_url' in kwargs else ""
_attempts = 0
_tries, _delay, _timeout = self.retry_tries, self.retry_delay, self.timeout
while _tries:
_attempts += 1
fkwargs['timeout'] = _timeout
request_func = partial(call_func, *args, **kwargs)
self.logger.debug("Request Retry: Attempt: {}: {}".format(
request_label if request_label else "", _attempts),
extra={
'attempts': _attempts,
'timeout': _timeout,
'tries': _tries,
'delay': _delay,
'request_url': request_url
})
error_exception = None
_tries -= 1
try:
response = request_func()
if response is None:
raise TuneRequestModuleError(
error_message="Request Retry: No response",
error_code=TuneRequestErrorCodes.
REQ_ERR_UNEXPECTED_VALUE)
self.logger.debug("Request Retry: Checking Response",
extra={
'request_url': request_url,
'request_label': request_label
})
if request_retry_func is not None:
if not request_retry_func(response):
self.logger.debug(
"Request Retry: Response: Valid: Not Retry Candidate",
extra={
'request_url': request_url,
'request_label': request_label
})
return response
else:
self.logger.debug("Request Retry: Response: Valid",
extra={
'request_url': request_url,
'request_label': request_label
})
return response
self.logger.debug(
"Request Retry: Response: Valid: Retry Candidate",
extra={
'request_url': request_url,
'request_label': request_label
})
except self.request_retry_excps as retry_ex:
error_exception = retry_ex
self.logger.warning(
"Request Retry: Expected: {}: Retry Candidate".format(
base_class_name(error_exception)),
extra={
'error_details':
get_exception_message(error_exception),
'request_url': request_url,
'request_label': request_label
})
if not _tries:
self.logger.error(
"Request Retry: Expected: {}: Exhausted Retries".
format(base_class_name(error_exception)))
raise
except TuneRequestBaseError as tmv_ex:
error_exception = tmv_ex
tmv_ex_extra = tmv_ex.to_dict()
tmv_ex_extra.update({
'error_exception':
base_class_name(error_exception),
'error_details':
get_exception_message(error_exception),
'request_label':
request_label
})
if not self.request_retry_excps_func or \
not self.request_retry_excps_func(tmv_ex, request_label):
self.logger.error(
"Request Retry: Integration: {}: Not Retry Candidate".
format(base_class_name(error_exception)),
extra=tmv_ex_extra)
raise
self.logger.warning(
"Request Retry: Integration: {}: Retry Candidate".format(
base_class_name(error_exception)),
extra=tmv_ex_extra)
if not _tries:
self.logger.error(
"Request Retry: Integration: {}: Exhausted Retries".
format(base_class_name(error_exception)))
raise
except Exception as ex:
error_exception = ex
ex_extra = {
'error_exception': base_class_name(error_exception),
'error_details': get_exception_message(error_exception),
'request_url': request_url,
'request_label': request_label
}
if not self.request_retry_excps_func or \
not self.request_retry_excps_func(error_exception, request_label):
self.logger.error(
"Request Retry: Unexpected: {}: Not Retry Candidate".
format(base_class_name(error_exception)),
extra=ex_extra)
raise
self.logger.warning(
"Request Retry: Unexpected: {}: Retry Candidate".format(
base_class_name(error_exception)),
extra=ex_extra)
if not _tries:
raise TuneRequestModuleError(
error_message="Unexpected: {}".format(
base_class_name(error_exception)),
errors=error_exception,
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.
REQ_ERR_RETRY_EXHAUSTED)
if not _tries:
self.logger.error("Request Retry: Exhausted Retries",
extra={
'attempts': _attempts,
'tries': _tries,
'request_url': request_url,
'request_label': request_label
})
raise TuneRequestModuleError(
error_message=(
"Request Retry: Exhausted Retries: {}: {}").format(
request_label, request_url),
error_request_curl=self.built_request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_RETRY_EXHAUSTED)
self.logger.info("Request Retry: Performing Retry",
extra={
'tries': _tries,
'delay': _delay,
'timeout': _timeout,
'request_url': request_url,
'request_label': request_label
})
time.sleep(_delay)
if self.retry_backoff and self.retry_backoff > 0:
_delay *= self.retry_backoff
if self.retry_jitter and self.retry_jitter > 0:
_delay += self.retry_jitter
if self.retry_max_delay is not None:
_delay = min(_delay, self.retry_max_delay)
def validate_response(
response,
request_curl,
request_label=None,
):
"""Validate response
Args:
response:
request_curl:
request_label:
Returns:
"""
response_extra = {}
if request_label is None:
request_label = 'Validate Response'
if not response:
error_message = f'{request_label}: Failed: None'
log.error(error_message, extra=response_extra)
raise TuneRequestModuleError(
error_message=error_message,
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE)
log.debug(f'{request_label}: Defined', extra=response_extra)
response_extra.update({'http_status_code': response.status_code})
# Not using hasattr on purpose
# Assuming positive approach will give us speedup since when attribute exists
# hasattr takes double the time.
# Anyway we use text attribute here to logging purpose only
try:
response_extra.update({'response_text_length': len(response.text)})
except AttributeError:
pass
if response.headers:
if 'Content-Type' in response.headers:
response_headers_content_type = \
safe_str(response.headers['Content-Type'])
response_extra.update(
{'Content-Type': response_headers_content_type})
if 'Content-Length' in response.headers:
response_headers_content_length = \
safe_int(response.headers['Content-Length'])
response_extra.update({
'Content-Length':
bytes_to_human(response_headers_content_length)
})
if 'Content-Encoding' in response.headers:
response_content_encoding = \
safe_str(response.headers['Content-Encoding'])
response_extra.update(
{'Content-Encoding': response_content_encoding})
if 'Transfer-Encoding' in response.headers:
response_transfer_encoding = \
safe_str(response.headers['Transfer-Encoding'])
response_extra.update(
{'Transfer-Encoding': response_transfer_encoding})
if not is_http_status_successful(http_status_code=response.status_code):
error_message = f'{request_label}: Failed'
log.error(error_message, extra=response_extra)
raise TuneRequestModuleError(
error_message=error_message,
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.REQ_ERR_SOFTWARE)
log.debug(f'{request_label}: Success', extra=response_extra)
def validate_json_response(response,
request_curl,
request_label=None,
response_content_type_expected='application/json',
raise_ex_if_not_json_response=True):
"""Validate JSON response.
Args:
response:
request_curl
request_label:
response_content_type_expected:
raise_ex_if_not_json_response:
Returns:
"""
validate_response(response, request_curl, request_label)
json_response = None
response_extra = {}
if request_label is None:
request_label = 'Validate JSON Response'
response_extra.update({'Content-Type': response_content_type_expected})
try:
response_content_type = response.headers.get('Content-Type', None)
if response.headers.get('Content-Type', None) is not None:
is_valid_response_content_type = \
response_content_type == response_content_type_expected or \
response_content_type.startswith(response_content_type_expected)
if is_valid_response_content_type:
json_response = requests_response_json(
response=response,
request_curl=request_curl,
request_label=request_label,
)
elif response_content_type.startswith('text/html'):
try:
response_content_html_lines = \
requests_response_text_html(
response=response
)
except Exception as ex:
raise TuneRequestModuleError(
error_message=request_label,
errors=ex,
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.
REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED)
raise TuneRequestModuleError(
error_message=
f"Content-Type: Expected: '{response_content_type_expected}', Actual: '{response_content_type}'",
errors=response_content_html_lines,
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.
REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED)
else:
raise TuneRequestModuleError(
error_message=
f"Content-Type: Expected: '{response_content_type_expected}', Actual: '{response_content_type}'",
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.
REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED)
except AttributeError:
raise TuneRequestModuleError(error_message="Content-Type: Undefined",
error_request_curl=request_curl,
error_code=TuneRequestErrorCodes.
REQ_ERR_UNEXPECTED_CONTENT_TYPE_RETURNED)
response_extra.update({
'http_status_code':
response.status_code,
'raise_ex_if_not_json_response':
raise_ex_if_not_json_response
})
log.debug(
f'{request_label}: Success: JSON',
extra=response_extra,
)
return json_response
